1 /* $NetBSD: amdgpu_navi10_ih.c,v 1.2 2021/12/18 23:44:58 riastradh Exp $ */
2
3 /*
4 * Copyright 2019 Advanced Micro Devices, Inc.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 */
25
26 #include <sys/cdefs.h>
27 __KERNEL_RCSID(0, "$NetBSD: amdgpu_navi10_ih.c,v 1.2 2021/12/18 23:44:58 riastradh Exp $");
28
29 #include <linux/pci.h>
30
31 #include "amdgpu.h"
32 #include "amdgpu_ih.h"
33
34 #include "oss/osssys_5_0_0_offset.h"
35 #include "oss/osssys_5_0_0_sh_mask.h"
36
37 #include "soc15_common.h"
38 #include "navi10_ih.h"
39
40
41 static void navi10_ih_set_interrupt_funcs(struct amdgpu_device *adev);
42
43 /**
44 * navi10_ih_enable_interrupts - Enable the interrupt ring buffer
45 *
46 * @adev: amdgpu_device pointer
47 *
48 * Enable the interrupt ring buffer (NAVI10).
49 */
navi10_ih_enable_interrupts(struct amdgpu_device * adev)50 static void navi10_ih_enable_interrupts(struct amdgpu_device *adev)
51 {
52 u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
53
54 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1);
55 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1);
56 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
57 adev->irq.ih.enabled = true;
58 }
59
60 /**
61 * navi10_ih_disable_interrupts - Disable the interrupt ring buffer
62 *
63 * @adev: amdgpu_device pointer
64 *
65 * Disable the interrupt ring buffer (NAVI10).
66 */
navi10_ih_disable_interrupts(struct amdgpu_device * adev)67 static void navi10_ih_disable_interrupts(struct amdgpu_device *adev)
68 {
69 u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
70
71 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);
72 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0);
73 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
74 /* set rptr, wptr to 0 */
75 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
76 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
77 adev->irq.ih.enabled = false;
78 adev->irq.ih.rptr = 0;
79 }
80
navi10_ih_rb_cntl(struct amdgpu_ih_ring * ih,uint32_t ih_rb_cntl)81 static uint32_t navi10_ih_rb_cntl(struct amdgpu_ih_ring *ih, uint32_t ih_rb_cntl)
82 {
83 int rb_bufsz = order_base_2(ih->ring_size / 4);
84
85 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
86 MC_SPACE, ih->use_bus_addr ? 1 : 4);
87 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
88 WPTR_OVERFLOW_CLEAR, 1);
89 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
90 WPTR_OVERFLOW_ENABLE, 1);
91 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz);
92 /* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register
93 * value is written to memory
94 */
95 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
96 WPTR_WRITEBACK_ENABLE, 1);
97 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1);
98 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0);
99 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0);
100
101 return ih_rb_cntl;
102 }
103
104 /**
105 * navi10_ih_irq_init - init and enable the interrupt ring
106 *
107 * @adev: amdgpu_device pointer
108 *
109 * Allocate a ring buffer for the interrupt controller,
110 * enable the RLC, disable interrupts, enable the IH
111 * ring buffer and enable it (NAVI).
112 * Called at device load and reume.
113 * Returns 0 for success, errors for failure.
114 */
navi10_ih_irq_init(struct amdgpu_device * adev)115 static int navi10_ih_irq_init(struct amdgpu_device *adev)
116 {
117 struct amdgpu_ih_ring *ih = &adev->irq.ih;
118 u32 ih_rb_cntl, ih_doorbell_rtpr, ih_chicken;
119 u32 tmp;
120
121 /* disable irqs */
122 navi10_ih_disable_interrupts(adev);
123
124 adev->nbio.funcs->ih_control(adev);
125
126 /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
127 WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, ih->gpu_addr >> 8);
128 WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, (ih->gpu_addr >> 40) & 0xff);
129
130 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
131 ih_rb_cntl = navi10_ih_rb_cntl(ih, ih_rb_cntl);
132 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM,
133 !!adev->irq.msi_enabled);
134
135 if (unlikely(adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT)) {
136 if (ih->use_bus_addr) {
137 ih_chicken = RREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN);
138 ih_chicken = REG_SET_FIELD(ih_chicken,
139 IH_CHICKEN, MC_SPACE_GPA_ENABLE, 1);
140 WREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN, ih_chicken);
141 }
142 }
143
144 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
145
146 /* set the writeback address whether it's enabled or not */
147 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO,
148 lower_32_bits(ih->wptr_addr));
149 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI,
150 upper_32_bits(ih->wptr_addr) & 0xFFFF);
151
152 /* set rptr, wptr to 0 */
153 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
154 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
155
156 ih_doorbell_rtpr = RREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR);
157 if (ih->use_doorbell) {
158 ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
159 IH_DOORBELL_RPTR, OFFSET,
160 ih->doorbell_index);
161 ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
162 IH_DOORBELL_RPTR, ENABLE, 1);
163 } else {
164 ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
165 IH_DOORBELL_RPTR, ENABLE, 0);
166 }
167 WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR, ih_doorbell_rtpr);
168
169 adev->nbio.funcs->ih_doorbell_range(adev, ih->use_doorbell,
170 ih->doorbell_index);
171
172 tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL);
173 tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL,
174 CLIENT18_IS_STORM_CLIENT, 1);
175 WREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL, tmp);
176
177 tmp = RREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL);
178 tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1);
179 WREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL, tmp);
180
181 pci_set_master(adev->pdev);
182
183 /* enable interrupts */
184 navi10_ih_enable_interrupts(adev);
185
186 return 0;
187 }
188
189 /**
190 * navi10_ih_irq_disable - disable interrupts
191 *
192 * @adev: amdgpu_device pointer
193 *
194 * Disable interrupts on the hw (NAVI10).
195 */
navi10_ih_irq_disable(struct amdgpu_device * adev)196 static void navi10_ih_irq_disable(struct amdgpu_device *adev)
197 {
198 navi10_ih_disable_interrupts(adev);
199
200 /* Wait and acknowledge irq */
201 mdelay(1);
202 }
203
204 /**
205 * navi10_ih_get_wptr - get the IH ring buffer wptr
206 *
207 * @adev: amdgpu_device pointer
208 *
209 * Get the IH ring buffer wptr from either the register
210 * or the writeback memory buffer (NAVI10). Also check for
211 * ring buffer overflow and deal with it.
212 * Returns the value of the wptr.
213 */
navi10_ih_get_wptr(struct amdgpu_device * adev,struct amdgpu_ih_ring * ih)214 static u32 navi10_ih_get_wptr(struct amdgpu_device *adev,
215 struct amdgpu_ih_ring *ih)
216 {
217 u32 wptr, reg, tmp;
218
219 wptr = le32_to_cpu(*ih->wptr_cpu);
220
221 if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
222 goto out;
223
224 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR);
225 wptr = RREG32_NO_KIQ(reg);
226 if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
227 goto out;
228 wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0);
229
230 /* When a ring buffer overflow happen start parsing interrupt
231 * from the last not overwritten vector (wptr + 32). Hopefully
232 * this should allow us to catch up.
233 */
234 tmp = (wptr + 32) & ih->ptr_mask;
235 dev_warn(adev->dev, "IH ring buffer overflow "
236 "(0x%08X, 0x%08X, 0x%08X)\n",
237 wptr, ih->rptr, tmp);
238 ih->rptr = tmp;
239
240 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL);
241 tmp = RREG32_NO_KIQ(reg);
242 tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
243 WREG32_NO_KIQ(reg, tmp);
244 out:
245 return (wptr & ih->ptr_mask);
246 }
247
248 /**
249 * navi10_ih_decode_iv - decode an interrupt vector
250 *
251 * @adev: amdgpu_device pointer
252 *
253 * Decodes the interrupt vector at the current rptr
254 * position and also advance the position.
255 */
navi10_ih_decode_iv(struct amdgpu_device * adev,struct amdgpu_ih_ring * ih,struct amdgpu_iv_entry * entry)256 static void navi10_ih_decode_iv(struct amdgpu_device *adev,
257 struct amdgpu_ih_ring *ih,
258 struct amdgpu_iv_entry *entry)
259 {
260 /* wptr/rptr are in bytes! */
261 u32 ring_index = ih->rptr >> 2;
262 uint32_t dw[8];
263
264 dw[0] = le32_to_cpu(ih->ring[ring_index + 0]);
265 dw[1] = le32_to_cpu(ih->ring[ring_index + 1]);
266 dw[2] = le32_to_cpu(ih->ring[ring_index + 2]);
267 dw[3] = le32_to_cpu(ih->ring[ring_index + 3]);
268 dw[4] = le32_to_cpu(ih->ring[ring_index + 4]);
269 dw[5] = le32_to_cpu(ih->ring[ring_index + 5]);
270 dw[6] = le32_to_cpu(ih->ring[ring_index + 6]);
271 dw[7] = le32_to_cpu(ih->ring[ring_index + 7]);
272
273 entry->client_id = dw[0] & 0xff;
274 entry->src_id = (dw[0] >> 8) & 0xff;
275 entry->ring_id = (dw[0] >> 16) & 0xff;
276 entry->vmid = (dw[0] >> 24) & 0xf;
277 entry->vmid_src = (dw[0] >> 31);
278 entry->timestamp = dw[1] | ((u64)(dw[2] & 0xffff) << 32);
279 entry->timestamp_src = dw[2] >> 31;
280 entry->pasid = dw[3] & 0xffff;
281 entry->pasid_src = dw[3] >> 31;
282 entry->src_data[0] = dw[4];
283 entry->src_data[1] = dw[5];
284 entry->src_data[2] = dw[6];
285 entry->src_data[3] = dw[7];
286
287 /* wptr/rptr are in bytes! */
288 ih->rptr += 32;
289 }
290
291 /**
292 * navi10_ih_set_rptr - set the IH ring buffer rptr
293 *
294 * @adev: amdgpu_device pointer
295 *
296 * Set the IH ring buffer rptr.
297 */
navi10_ih_set_rptr(struct amdgpu_device * adev,struct amdgpu_ih_ring * ih)298 static void navi10_ih_set_rptr(struct amdgpu_device *adev,
299 struct amdgpu_ih_ring *ih)
300 {
301 if (ih->use_doorbell) {
302 /* XXX check if swapping is necessary on BE */
303 *ih->rptr_cpu = ih->rptr;
304 WDOORBELL32(ih->doorbell_index, ih->rptr);
305 } else
306 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, ih->rptr);
307 }
308
navi10_ih_early_init(void * handle)309 static int navi10_ih_early_init(void *handle)
310 {
311 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
312
313 navi10_ih_set_interrupt_funcs(adev);
314 return 0;
315 }
316
navi10_ih_sw_init(void * handle)317 static int navi10_ih_sw_init(void *handle)
318 {
319 int r;
320 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
321 bool use_bus_addr;
322
323 /* use gpu virtual address for ih ring
324 * until ih_checken is programmed to allow
325 * use bus address for ih ring by psp bl */
326 use_bus_addr =
327 (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) ? false : true;
328 r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 256 * 1024, use_bus_addr);
329 if (r)
330 return r;
331
332 adev->irq.ih.use_doorbell = true;
333 adev->irq.ih.doorbell_index = adev->doorbell_index.ih << 1;
334
335 r = amdgpu_irq_init(adev);
336
337 return r;
338 }
339
navi10_ih_sw_fini(void * handle)340 static int navi10_ih_sw_fini(void *handle)
341 {
342 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
343
344 amdgpu_irq_fini(adev);
345 amdgpu_ih_ring_fini(adev, &adev->irq.ih);
346
347 return 0;
348 }
349
navi10_ih_hw_init(void * handle)350 static int navi10_ih_hw_init(void *handle)
351 {
352 int r;
353 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
354
355 r = navi10_ih_irq_init(adev);
356 if (r)
357 return r;
358
359 return 0;
360 }
361
navi10_ih_hw_fini(void * handle)362 static int navi10_ih_hw_fini(void *handle)
363 {
364 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
365
366 navi10_ih_irq_disable(adev);
367
368 return 0;
369 }
370
navi10_ih_suspend(void * handle)371 static int navi10_ih_suspend(void *handle)
372 {
373 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
374
375 return navi10_ih_hw_fini(adev);
376 }
377
navi10_ih_resume(void * handle)378 static int navi10_ih_resume(void *handle)
379 {
380 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
381
382 return navi10_ih_hw_init(adev);
383 }
384
navi10_ih_is_idle(void * handle)385 static bool navi10_ih_is_idle(void *handle)
386 {
387 /* todo */
388 return true;
389 }
390
navi10_ih_wait_for_idle(void * handle)391 static int navi10_ih_wait_for_idle(void *handle)
392 {
393 /* todo */
394 return -ETIMEDOUT;
395 }
396
navi10_ih_soft_reset(void * handle)397 static int navi10_ih_soft_reset(void *handle)
398 {
399 /* todo */
400 return 0;
401 }
402
navi10_ih_update_clockgating_state(struct amdgpu_device * adev,bool enable)403 static void navi10_ih_update_clockgating_state(struct amdgpu_device *adev,
404 bool enable)
405 {
406 uint32_t data, def, field_val;
407
408 if (adev->cg_flags & AMD_CG_SUPPORT_IH_CG) {
409 def = data = RREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL);
410 field_val = enable ? 0 : 1;
411 data = REG_SET_FIELD(data, IH_CLK_CTRL,
412 DBUS_MUX_CLK_SOFT_OVERRIDE, field_val);
413 data = REG_SET_FIELD(data, IH_CLK_CTRL,
414 OSSSYS_SHARE_CLK_SOFT_OVERRIDE, field_val);
415 data = REG_SET_FIELD(data, IH_CLK_CTRL,
416 LIMIT_SMN_CLK_SOFT_OVERRIDE, field_val);
417 data = REG_SET_FIELD(data, IH_CLK_CTRL,
418 DYN_CLK_SOFT_OVERRIDE, field_val);
419 data = REG_SET_FIELD(data, IH_CLK_CTRL,
420 REG_CLK_SOFT_OVERRIDE, field_val);
421 if (def != data)
422 WREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL, data);
423 }
424
425 return;
426 }
427
navi10_ih_set_clockgating_state(void * handle,enum amd_clockgating_state state)428 static int navi10_ih_set_clockgating_state(void *handle,
429 enum amd_clockgating_state state)
430 {
431 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
432
433 navi10_ih_update_clockgating_state(adev,
434 state == AMD_CG_STATE_GATE);
435 return 0;
436 }
437
navi10_ih_set_powergating_state(void * handle,enum amd_powergating_state state)438 static int navi10_ih_set_powergating_state(void *handle,
439 enum amd_powergating_state state)
440 {
441 return 0;
442 }
443
navi10_ih_get_clockgating_state(void * handle,u32 * flags)444 static void navi10_ih_get_clockgating_state(void *handle, u32 *flags)
445 {
446 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
447
448 if (!RREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL))
449 *flags |= AMD_CG_SUPPORT_IH_CG;
450
451 return;
452 }
453
454 static const struct amd_ip_funcs navi10_ih_ip_funcs = {
455 .name = "navi10_ih",
456 .early_init = navi10_ih_early_init,
457 .late_init = NULL,
458 .sw_init = navi10_ih_sw_init,
459 .sw_fini = navi10_ih_sw_fini,
460 .hw_init = navi10_ih_hw_init,
461 .hw_fini = navi10_ih_hw_fini,
462 .suspend = navi10_ih_suspend,
463 .resume = navi10_ih_resume,
464 .is_idle = navi10_ih_is_idle,
465 .wait_for_idle = navi10_ih_wait_for_idle,
466 .soft_reset = navi10_ih_soft_reset,
467 .set_clockgating_state = navi10_ih_set_clockgating_state,
468 .set_powergating_state = navi10_ih_set_powergating_state,
469 .get_clockgating_state = navi10_ih_get_clockgating_state,
470 };
471
472 static const struct amdgpu_ih_funcs navi10_ih_funcs = {
473 .get_wptr = navi10_ih_get_wptr,
474 .decode_iv = navi10_ih_decode_iv,
475 .set_rptr = navi10_ih_set_rptr
476 };
477
navi10_ih_set_interrupt_funcs(struct amdgpu_device * adev)478 static void navi10_ih_set_interrupt_funcs(struct amdgpu_device *adev)
479 {
480 if (adev->irq.ih_funcs == NULL)
481 adev->irq.ih_funcs = &navi10_ih_funcs;
482 }
483
484 const struct amdgpu_ip_block_version navi10_ih_ip_block =
485 {
486 .type = AMD_IP_BLOCK_TYPE_IH,
487 .major = 5,
488 .minor = 0,
489 .rev = 0,
490 .funcs = &navi10_ih_ip_funcs,
491 };
492