1 /*- 2 * Copyright (c) 2011 The NetBSD Foundation, Inc. 3 * All rights reserved. 4 * 5 * This code is derived from software contributed to The NetBSD Foundation 6 * by Matt Thomas of 3am Software Foundry. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 18 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 19 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 21 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 * POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 32 __KERNEL_RCSID(0, "$NetBSD: pmap_synci.c,v 1.3 2016/07/11 16:06:09 matt Exp $"); 33 34 #define __PMAP_PRIVATE 35 36 #include "opt_multiprocessor.h" 37 38 #include <sys/param.h> 39 #include <sys/systm.h> 40 #include <sys/mutex.h> 41 #include <sys/atomic.h> 42 #include <sys/cpu.h> 43 44 #include <uvm/uvm.h> 45 46 #if defined(MULTIPROCESSOR) 47 u_int pmap_tlb_synci_page_mask; 48 u_int pmap_tlb_synci_map_mask; 49 50 void 51 pmap_tlb_syncicache_ast(struct cpu_info *ci) 52 { 53 struct pmap_tlb_info * const ti = cpu_tlb_info(ci); 54 55 KASSERT(kpreempt_disabled()); 56 57 uint32_t page_bitmap = atomic_swap_32(&ti->ti_synci_page_bitmap, 0); 58 #if 0 59 printf("%s: need to sync %#x\n", __func__, page_bitmap); 60 #endif 61 ti->ti_evcnt_synci_asts.ev_count++; 62 /* 63 * If every bit is set in the bitmap, sync the entire icache. 64 */ 65 if (page_bitmap == pmap_tlb_synci_map_mask) { 66 pmap_md_icache_sync_all(); 67 ti->ti_evcnt_synci_all.ev_count++; 68 ti->ti_evcnt_synci_pages.ev_count += pmap_tlb_synci_page_mask+1; 69 return; 70 } 71 72 /* 73 * Loop through the bitmap clearing each set of indices for each page. 74 */ 75 for (vaddr_t va = 0; 76 page_bitmap != 0; 77 page_bitmap >>= 1, va += PAGE_SIZE) { 78 if (page_bitmap & 1) { 79 /* 80 * Each bit set represents a page index to be synced. 81 */ 82 pmap_md_icache_sync_range_index(va, PAGE_SIZE); 83 ti->ti_evcnt_synci_pages.ev_count++; 84 } 85 } 86 } 87 88 void 89 pmap_tlb_syncicache(vaddr_t va, const kcpuset_t *page_onproc) 90 { 91 KASSERT(kpreempt_disabled()); 92 /* 93 * We don't sync the icache here but let ast do it for us just before 94 * returning to userspace. We do this because we don't really know 95 * on which CPU we will return to userspace and if we synch the icache 96 * now it might not be on the CPU we need it on. In addition, others 97 * threads might sync the icache before we get to return to userland 98 * so there's no reason for us to do it. 99 * 100 * Each TLB/cache keeps a synci sequence number which gets advanced 101 * each time that TLB/cache performs a pmap_md_sync_icache_all. When 102 * we return to userland, we check the pmap's corresponding synci 103 * sequence number for that TLB/cache. If they match, it means that 104 * no one has yet synched the icache so we much do it ourselves. If 105 * they don't match someone has already synced the icache for us. 106 * 107 * There is a small chance that the generation numbers will wrap and 108 * then become equal but that's a one in 4 billion cache and will 109 * just cause an extra sync of the icache. 110 */ 111 struct cpu_info * const ci = curcpu(); 112 kcpuset_t *onproc; 113 kcpuset_create(&onproc, true); 114 const uint32_t page_mask = 115 1L << ((va >> PGSHIFT) & pmap_tlb_synci_page_mask); 116 for (size_t i = 0; i < pmap_ntlbs; i++) { 117 struct pmap_tlb_info * const ti = pmap_tlbs[i]; 118 TLBINFO_LOCK(ti); 119 for (;;) { 120 uint32_t old_page_bitmap = ti->ti_synci_page_bitmap; 121 if (old_page_bitmap & page_mask) { 122 ti->ti_evcnt_synci_duplicate.ev_count++; 123 break; 124 } 125 126 uint32_t orig_page_bitmap = atomic_cas_32( 127 &ti->ti_synci_page_bitmap, old_page_bitmap, 128 old_page_bitmap | page_mask); 129 130 if (orig_page_bitmap == old_page_bitmap) { 131 if (old_page_bitmap == 0) { 132 kcpuset_merge(onproc, ti->ti_kcpuset); 133 } else { 134 ti->ti_evcnt_synci_deferred.ev_count++; 135 } 136 ti->ti_evcnt_synci_desired.ev_count++; 137 break; 138 } 139 } 140 #if 0 141 printf("%s: %s: %x to %x on cpus %#x\n", __func__, 142 ti->ti_name, page_mask, ti->ti_synci_page_bitmap, 143 onproc & page_onproc & ti->ti_cpu_mask); 144 #endif 145 TLBINFO_UNLOCK(ti); 146 } 147 kcpuset_intersect(onproc, page_onproc); 148 if (__predict_false(!kcpuset_iszero(onproc))) { 149 /* 150 * If the cpu need to sync this page, tell the current lwp 151 * to sync the icache before it returns to userspace. 152 */ 153 if (kcpuset_isset(onproc, cpu_index(ci))) { 154 if (ci->ci_flags & CPUF_USERPMAP) { 155 curlwp->l_md.md_astpending = 1; /* force call to ast() */ 156 ci->ci_evcnt_synci_onproc_rqst.ev_count++; 157 } else { 158 ci->ci_evcnt_synci_deferred_rqst.ev_count++; 159 } 160 kcpuset_clear(onproc, cpu_index(ci)); 161 } 162 163 /* 164 * For each cpu that is affect, send an IPI telling 165 * that CPU that the current thread needs to sync its icache. 166 * We might cause some spurious icache syncs but that's not 167 * going to break anything. 168 */ 169 for (cpuid_t n = kcpuset_ffs(onproc); 170 n-- > 0; 171 n = kcpuset_ffs(onproc)) { 172 kcpuset_clear(onproc, n); 173 cpu_send_ipi(cpu_lookup(n), IPI_SYNCICACHE); 174 } 175 } 176 kcpuset_destroy(onproc); 177 } 178 179 void 180 pmap_tlb_syncicache_wanted(struct cpu_info *ci) 181 { 182 struct pmap_tlb_info * const ti = cpu_tlb_info(ci); 183 184 KASSERT(cpu_intr_p()); 185 186 TLBINFO_LOCK(ti); 187 188 /* 189 * We might have been notified because another CPU changed an exec 190 * page and now needs us to sync the icache so tell the current lwp 191 * to do the next time it returns to userland (which should be very 192 * soon). 193 */ 194 if (ti->ti_synci_page_bitmap && (ci->ci_flags & CPUF_USERPMAP)) { 195 curlwp->l_md.md_astpending = 1; /* force call to ast() */ 196 ci->ci_evcnt_synci_ipi_rqst.ev_count++; 197 } 198 199 TLBINFO_UNLOCK(ti); 200 201 } 202 #endif /* MULTIPROCESSOR */ 203