1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <drm/drmP.h> 25 #include "amdgpu.h" 26 #include "amdgpu_ih.h" 27 #include "amdgpu_amdkfd.h" 28 29 /** 30 * amdgpu_ih_ring_alloc - allocate memory for the IH ring 31 * 32 * @adev: amdgpu_device pointer 33 * 34 * Allocate a ring buffer for the interrupt controller. 35 * Returns 0 for success, errors for failure. 36 */ 37 static int amdgpu_ih_ring_alloc(struct amdgpu_device *adev) 38 { 39 int r; 40 41 /* Allocate ring buffer */ 42 if (adev->irq.ih.ring_obj == NULL) { 43 r = amdgpu_bo_create_kernel(adev, adev->irq.ih.ring_size, 44 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 45 &adev->irq.ih.ring_obj, 46 &adev->irq.ih.gpu_addr, 47 (void **)&adev->irq.ih.ring); 48 if (r) { 49 DRM_ERROR("amdgpu: failed to create ih ring buffer (%d).\n", r); 50 return r; 51 } 52 } 53 return 0; 54 } 55 56 /** 57 * amdgpu_ih_ring_init - initialize the IH state 58 * 59 * @adev: amdgpu_device pointer 60 * 61 * Initializes the IH state and allocates a buffer 62 * for the IH ring buffer. 63 * Returns 0 for success, errors for failure. 64 */ 65 int amdgpu_ih_ring_init(struct amdgpu_device *adev, unsigned ring_size, 66 bool use_bus_addr) 67 { 68 u32 rb_bufsz; 69 int r; 70 struct drm_dmamem *dmah; 71 int flags = 0; 72 73 /* Align ring size */ 74 rb_bufsz = order_base_2(ring_size / 4); 75 ring_size = (1 << rb_bufsz) * 4; 76 adev->irq.ih.ring_size = ring_size; 77 adev->irq.ih.ptr_mask = adev->irq.ih.ring_size - 1; 78 adev->irq.ih.rptr = 0; 79 adev->irq.ih.use_bus_addr = use_bus_addr; 80 81 if (adev->irq.ih.use_bus_addr) { 82 if (!adev->irq.ih.ring) { 83 /* add 8 bytes for the rptr/wptr shadows and 84 * add them to the end of the ring allocation. 85 */ 86 #ifdef __linux__ 87 adev->irq.ih.ring = pci_alloc_consistent(adev->pdev, 88 adev->irq.ih.ring_size + 8, 89 &adev->irq.ih.rb_dma_addr); 90 if (adev->irq.ih.ring == NULL) 91 return -ENOMEM; 92 #else 93 dmah = drm_dmamem_alloc(adev->dmat, 94 adev->irq.ih.ring_size + 8, 95 PAGE_SIZE, 1, 96 adev->irq.ih.ring_size + 8, flags, 0); 97 if (dmah == NULL) 98 return -ENOMEM; 99 adev->irq.ih.dmah = dmah; 100 adev->irq.ih.rb_dma_addr = dmah->map->dm_segs[0].ds_addr; 101 adev->irq.ih.ring = (volatile uint32_t *)dmah->kva; 102 #endif 103 memset((void *)adev->irq.ih.ring, 0, adev->irq.ih.ring_size + 8); 104 adev->irq.ih.wptr_offs = (adev->irq.ih.ring_size / 4) + 0; 105 adev->irq.ih.rptr_offs = (adev->irq.ih.ring_size / 4) + 1; 106 } 107 return 0; 108 } else { 109 r = amdgpu_device_wb_get(adev, &adev->irq.ih.wptr_offs); 110 if (r) { 111 dev_err(adev->dev, "(%d) ih wptr_offs wb alloc failed\n", r); 112 return r; 113 } 114 115 r = amdgpu_device_wb_get(adev, &adev->irq.ih.rptr_offs); 116 if (r) { 117 amdgpu_device_wb_free(adev, adev->irq.ih.wptr_offs); 118 dev_err(adev->dev, "(%d) ih rptr_offs wb alloc failed\n", r); 119 return r; 120 } 121 122 return amdgpu_ih_ring_alloc(adev); 123 } 124 } 125 126 /** 127 * amdgpu_ih_ring_fini - tear down the IH state 128 * 129 * @adev: amdgpu_device pointer 130 * 131 * Tears down the IH state and frees buffer 132 * used for the IH ring buffer. 133 */ 134 void amdgpu_ih_ring_fini(struct amdgpu_device *adev) 135 { 136 if (adev->irq.ih.use_bus_addr) { 137 if (adev->irq.ih.ring) { 138 /* add 8 bytes for the rptr/wptr shadows and 139 * add them to the end of the ring allocation. 140 */ 141 #ifdef __linux__ 142 pci_free_consistent(adev->pdev, adev->irq.ih.ring_size + 8, 143 (void *)adev->irq.ih.ring, 144 adev->irq.ih.rb_dma_addr); 145 #else 146 drm_dmamem_free(adev->dmat, adev->irq.ih.dmah); 147 #endif 148 adev->irq.ih.ring = NULL; 149 } 150 } else { 151 amdgpu_bo_free_kernel(&adev->irq.ih.ring_obj, 152 &adev->irq.ih.gpu_addr, 153 (void **)&adev->irq.ih.ring); 154 amdgpu_device_wb_free(adev, adev->irq.ih.wptr_offs); 155 amdgpu_device_wb_free(adev, adev->irq.ih.rptr_offs); 156 } 157 } 158 159 /** 160 * amdgpu_ih_process - interrupt handler 161 * 162 * @adev: amdgpu_device pointer 163 * 164 * Interrupt hander (VI), walk the IH ring. 165 * Returns irq process return code. 166 */ 167 int amdgpu_ih_process(struct amdgpu_device *adev) 168 { 169 struct amdgpu_iv_entry entry; 170 u32 wptr; 171 172 if (!adev->irq.ih.enabled || adev->shutdown) 173 return IRQ_NONE; 174 175 wptr = amdgpu_ih_get_wptr(adev); 176 177 restart_ih: 178 /* is somebody else already processing irqs? */ 179 if (atomic_xchg(&adev->irq.ih.lock, 1)) 180 return IRQ_NONE; 181 182 DRM_DEBUG("%s: rptr %d, wptr %d\n", __func__, adev->irq.ih.rptr, wptr); 183 184 /* Order reading of wptr vs. reading of IH ring data */ 185 rmb(); 186 187 while (adev->irq.ih.rptr != wptr) { 188 u32 ring_index = adev->irq.ih.rptr >> 2; 189 190 /* Prescreening of high-frequency interrupts */ 191 if (!amdgpu_ih_prescreen_iv(adev)) { 192 adev->irq.ih.rptr &= adev->irq.ih.ptr_mask; 193 continue; 194 } 195 196 /* Before dispatching irq to IP blocks, send it to amdkfd */ 197 amdgpu_amdkfd_interrupt(adev, 198 (const void *) &adev->irq.ih.ring[ring_index]); 199 200 entry.iv_entry = (const uint32_t *) 201 &adev->irq.ih.ring[ring_index]; 202 amdgpu_ih_decode_iv(adev, &entry); 203 adev->irq.ih.rptr &= adev->irq.ih.ptr_mask; 204 205 amdgpu_irq_dispatch(adev, &entry); 206 } 207 amdgpu_ih_set_rptr(adev); 208 atomic_set(&adev->irq.ih.lock, 0); 209 210 /* make sure wptr hasn't changed while processing */ 211 wptr = amdgpu_ih_get_wptr(adev); 212 if (wptr != adev->irq.ih.rptr) 213 goto restart_ih; 214 215 return IRQ_HANDLED; 216 } 217 218 /** 219 * amdgpu_ih_add_fault - Add a page fault record 220 * 221 * @adev: amdgpu device pointer 222 * @key: 64-bit encoding of PASID and address 223 * 224 * This should be called when a retry page fault interrupt is 225 * received. If this is a new page fault, it will be added to a hash 226 * table. The return value indicates whether this is a new fault, or 227 * a fault that was already known and is already being handled. 228 * 229 * If there are too many pending page faults, this will fail. Retry 230 * interrupts should be ignored in this case until there is enough 231 * free space. 232 * 233 * Returns 0 if the fault was added, 1 if the fault was already known, 234 * -ENOSPC if there are too many pending faults. 235 */ 236 int amdgpu_ih_add_fault(struct amdgpu_device *adev, u64 key) 237 { 238 unsigned long flags; 239 int r = -ENOSPC; 240 241 if (WARN_ON_ONCE(!adev->irq.ih.faults)) 242 /* Should be allocated in <IP>_ih_sw_init on GPUs that 243 * support retry faults and require retry filtering. 244 */ 245 return r; 246 247 spin_lock_irqsave(&adev->irq.ih.faults->lock, flags); 248 249 /* Only let the hash table fill up to 50% for best performance */ 250 if (adev->irq.ih.faults->count >= (1 << (AMDGPU_PAGEFAULT_HASH_BITS-1))) 251 goto unlock_out; 252 253 r = chash_table_copy_in(&adev->irq.ih.faults->hash, key, NULL); 254 if (!r) 255 adev->irq.ih.faults->count++; 256 257 /* chash_table_copy_in should never fail unless we're losing count */ 258 WARN_ON_ONCE(r < 0); 259 260 unlock_out: 261 spin_unlock_irqrestore(&adev->irq.ih.faults->lock, flags); 262 return r; 263 } 264 265 /** 266 * amdgpu_ih_clear_fault - Remove a page fault record 267 * 268 * @adev: amdgpu device pointer 269 * @key: 64-bit encoding of PASID and address 270 * 271 * This should be called when a page fault has been handled. Any 272 * future interrupt with this key will be processed as a new 273 * page fault. 274 */ 275 void amdgpu_ih_clear_fault(struct amdgpu_device *adev, u64 key) 276 { 277 unsigned long flags; 278 int r; 279 280 if (!adev->irq.ih.faults) 281 return; 282 283 spin_lock_irqsave(&adev->irq.ih.faults->lock, flags); 284 285 r = chash_table_remove(&adev->irq.ih.faults->hash, key, NULL); 286 if (!WARN_ON_ONCE(r < 0)) { 287 adev->irq.ih.faults->count--; 288 WARN_ON_ONCE(adev->irq.ih.faults->count < 0); 289 } 290 291 spin_unlock_irqrestore(&adev->irq.ih.faults->lock, flags); 292 } 293