1 /*- 2 * Copyright (c) 2011 The NetBSD Foundation, Inc. 3 * All rights reserved. 4 * 5 * This code is derived from software contributed to The NetBSD Foundation 6 * by Matt Thomas of 3am Software Foundry. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 18 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 19 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 21 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 * POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 32 __KERNEL_RCSID(0, "$NetBSD: pmap_synci.c,v 1.1 2012/10/03 00:51:46 christos Exp $"); 33 34 #define __PMAP_PRIVATE 35 36 #include "opt_multiprocessor.h" 37 38 #include <sys/param.h> 39 #include <sys/systm.h> 40 #include <sys/mutex.h> 41 #include <sys/atomic.h> 42 #include <sys/cpu.h> 43 44 #include <uvm/uvm.h> 45 46 #if defined(MULTIPROCESSOR) 47 void 48 pmap_syncicache_ast(struct cpu_info *ci) 49 { 50 struct pmap_tlb_info * const ti = ci->ci_tlb_info; 51 52 KASSERT(kpreempt_disabled()); 53 54 uint32_t page_bitmap = atomic_swap_32(&ti->ti_synci_page_bitmap, 0); 55 #if 0 56 printf("%s: need to sync %#x\n", __func__, page_bitmap); 57 #endif 58 ti->ti_evcnt_synci_asts.ev_count++; 59 /* 60 * If every bit is set in the bitmap, sync the entire icache. 61 */ 62 if (page_bitmap == pmap_tlb_synci_map_mask) { 63 pmap_md_icache_sync_all(); 64 ti->ti_evcnt_synci_all.ev_count++; 65 ti->ti_evcnt_synci_pages.ev_count += pmap_tlb_synci_page_mask+1; 66 kpreempt_enable(); 67 return; 68 } 69 70 /* 71 * Loop through the bitmap clearing each set of indices for each page. 72 */ 73 for (vaddr_t va = 0; 74 page_bitmap != 0; 75 page_bitmap >>= 1, va += PAGE_SIZE) { 76 if (page_bitmap & 1) { 77 /* 78 * Each bit set represents a page index to be synced. 79 */ 80 pmap_md_icache_sync_range_index(va, PAGE_SIZE); 81 ti->ti_evcnt_synci_pages.ev_count++; 82 } 83 } 84 85 kpreempt_enable(); 86 } 87 88 void 89 pmap_tlb_syncicache(vaddr_t va, uint32_t page_onproc) 90 { 91 KASSERT(kpreempt_disabled()); 92 /* 93 * We don't sync the icache here but let ast do it for us just before 94 * returning to userspace. We do this because we don't really know 95 * on which CPU we will return to userspace and if we synch the icache 96 * now it might not be on the CPU we need it on. In addition, others 97 * threads might sync the icache before we get to return to userland 98 * so there's no reason for us to do it. 99 * 100 * Each TLB/cache keeps a synci sequence number which gets advanced 101 * each time that TLB/cache performs a pmap_md_sync_icache_all. When 102 * we return to userland, we check the pmap's corresponding synci 103 * sequence number for that TLB/cache. If they match, it means that 104 * no one has yet synched the icache so we much do it ourselves. If 105 * they don't match someone has already synced the icache for us. 106 * 107 * There is a small chance that the generation numbers will wrap and 108 * then become equal but that's a one in 4 billion cache and will 109 * just cause an extra sync of the icache. 110 */ 111 const uint32_t cpu_mask = 1L << cpu_index(curcpu()); 112 const uint32_t page_mask = 113 1L << ((va >> PGSHIFT) & pmap_tlb_synci_page_mask); 114 uint32_t onproc = 0; 115 for (size_t i = 0; i < pmap_ntlbs; i++) { 116 struct pmap_tlb_info * const ti = pmap_tlbs[i]; 117 TLBINFO_LOCK(ti); 118 for (;;) { 119 uint32_t old_page_bitmap = ti->ti_synci_page_bitmap; 120 if (old_page_bitmap & page_mask) { 121 ti->ti_evcnt_synci_duplicate.ev_count++; 122 break; 123 } 124 125 uint32_t orig_page_bitmap = atomic_cas_32( 126 &ti->ti_synci_page_bitmap, old_page_bitmap, 127 old_page_bitmap | page_mask); 128 129 if (orig_page_bitmap == old_page_bitmap) { 130 if (old_page_bitmap == 0) { 131 onproc |= ti->ti_cpu_mask; 132 } else { 133 ti->ti_evcnt_synci_deferred.ev_count++; 134 } 135 ti->ti_evcnt_synci_desired.ev_count++; 136 break; 137 } 138 } 139 #if 0 140 printf("%s: %s: %x to %x on cpus %#x\n", __func__, 141 ti->ti_name, page_mask, ti->ti_synci_page_bitmap, 142 onproc & page_onproc & ti->ti_cpu_mask); 143 #endif 144 TLBINFO_UNLOCK(ti); 145 } 146 onproc &= page_onproc; 147 if (__predict_false(onproc != 0)) { 148 /* 149 * If the cpu need to sync this page, tell the current lwp 150 * to sync the icache before it returns to userspace. 151 */ 152 if (onproc & cpu_mask) { 153 if (curcpu()->ci_flags & CPUF_USERPMAP) { 154 curlwp->l_md.md_astpending = 1; /* force call to ast() */ 155 curcpu()->ci_evcnt_synci_onproc_rqst.ev_count++; 156 } else { 157 curcpu()->ci_evcnt_synci_deferred_rqst.ev_count++; 158 } 159 onproc ^= cpu_mask; 160 } 161 162 /* 163 * For each cpu that is affect, send an IPI telling 164 * that CPU that the current thread needs to sync its icache. 165 * We might cause some spurious icache syncs but that's not 166 * going to break anything. 167 */ 168 for (u_int n = ffs(onproc); 169 onproc != 0; 170 onproc >>= n, onproc <<= n, n = ffs(onproc)) { 171 cpu_send_ipi(cpu_lookup(n-1), IPI_SYNCICACHE); 172 } 173 } 174 } 175 176 void 177 pmap_tlb_syncicache_wanted(struct cpu_info *ci) 178 { 179 struct pmap_tlb_info * const ti = ci->ci_tlb_info; 180 181 KASSERT(cpu_intr_p()); 182 183 TLBINFO_LOCK(ti); 184 185 /* 186 * We might have been notified because another CPU changed an exec 187 * page and now needs us to sync the icache so tell the current lwp 188 * to do the next time it returns to userland (which should be very 189 * soon). 190 */ 191 if (ti->ti_synci_page_bitmap && (ci->ci_flags & CPUF_USERPMAP)) { 192 curlwp->l_md.md_astpending = 1; /* force call to ast() */ 193 ci->ci_evcnt_synci_ipi_rqst.ev_count++; 194 } 195 196 TLBINFO_UNLOCK(ti); 197 198 } 199 #endif /* MULTIPROCESSOR */ 200