1 /* $NetBSD: amdgpu_amdkfd_gfx_v7.c,v 1.4 2021/12/18 23:44:58 riastradh Exp $ */
2
3 /*
4 * Copyright 2014 Advanced Micro Devices, Inc.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 */
24
25 #include <sys/cdefs.h>
26 __KERNEL_RCSID(0, "$NetBSD: amdgpu_amdkfd_gfx_v7.c,v 1.4 2021/12/18 23:44:58 riastradh Exp $");
27
28 #include <linux/mmu_context.h>
29
30 #include "amdgpu.h"
31 #include "amdgpu_amdkfd.h"
32 #include "cikd.h"
33 #include "cik_sdma.h"
34 #include "gfx_v7_0.h"
35 #include "gca/gfx_7_2_d.h"
36 #include "gca/gfx_7_2_enum.h"
37 #include "gca/gfx_7_2_sh_mask.h"
38 #include "oss/oss_2_0_d.h"
39 #include "oss/oss_2_0_sh_mask.h"
40 #include "gmc/gmc_7_1_d.h"
41 #include "gmc/gmc_7_1_sh_mask.h"
42 #include "cik_structs.h"
43
44 enum hqd_dequeue_request_type {
45 NO_ACTION = 0,
46 DRAIN_PIPE,
47 RESET_WAVES
48 };
49
50 enum {
51 MAX_TRAPID = 8, /* 3 bits in the bitfield. */
52 MAX_WATCH_ADDRESSES = 4
53 };
54
55 enum {
56 ADDRESS_WATCH_REG_ADDR_HI = 0,
57 ADDRESS_WATCH_REG_ADDR_LO,
58 ADDRESS_WATCH_REG_CNTL,
59 ADDRESS_WATCH_REG_MAX
60 };
61
62 /* not defined in the CI/KV reg file */
63 enum {
64 ADDRESS_WATCH_REG_CNTL_ATC_BIT = 0x10000000UL,
65 ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK = 0x00FFFFFF,
66 ADDRESS_WATCH_REG_ADDLOW_MASK_EXTENSION = 0x03000000,
67 /* extend the mask to 26 bits to match the low address field */
68 ADDRESS_WATCH_REG_ADDLOW_SHIFT = 6,
69 ADDRESS_WATCH_REG_ADDHIGH_MASK = 0xFFFF
70 };
71
72 static const uint32_t watchRegs[MAX_WATCH_ADDRESSES * ADDRESS_WATCH_REG_MAX] = {
73 mmTCP_WATCH0_ADDR_H, mmTCP_WATCH0_ADDR_L, mmTCP_WATCH0_CNTL,
74 mmTCP_WATCH1_ADDR_H, mmTCP_WATCH1_ADDR_L, mmTCP_WATCH1_CNTL,
75 mmTCP_WATCH2_ADDR_H, mmTCP_WATCH2_ADDR_L, mmTCP_WATCH2_CNTL,
76 mmTCP_WATCH3_ADDR_H, mmTCP_WATCH3_ADDR_L, mmTCP_WATCH3_CNTL
77 };
78
79 union TCP_WATCH_CNTL_BITS {
80 struct {
81 uint32_t mask:24;
82 uint32_t vmid:4;
83 uint32_t atc:1;
84 uint32_t mode:2;
85 uint32_t valid:1;
86 } bitfields, bits;
87 uint32_t u32All;
88 signed int i32All;
89 float f32All;
90 };
91
92 /* Because of REG_GET_FIELD() being used, we put this function in the
93 * asic specific file.
94 */
get_tile_config(struct kgd_dev * kgd,struct tile_config * config)95 static int get_tile_config(struct kgd_dev *kgd,
96 struct tile_config *config)
97 {
98 struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
99
100 config->gb_addr_config = adev->gfx.config.gb_addr_config;
101 config->num_banks = REG_GET_FIELD(adev->gfx.config.mc_arb_ramcfg,
102 MC_ARB_RAMCFG, NOOFBANK);
103 config->num_ranks = REG_GET_FIELD(adev->gfx.config.mc_arb_ramcfg,
104 MC_ARB_RAMCFG, NOOFRANKS);
105
106 config->tile_config_ptr = adev->gfx.config.tile_mode_array;
107 config->num_tile_configs =
108 ARRAY_SIZE(adev->gfx.config.tile_mode_array);
109 config->macro_tile_config_ptr =
110 adev->gfx.config.macrotile_mode_array;
111 config->num_macro_tile_configs =
112 ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
113
114 return 0;
115 }
116
get_amdgpu_device(struct kgd_dev * kgd)117 static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd)
118 {
119 return (struct amdgpu_device *)kgd;
120 }
121
lock_srbm(struct kgd_dev * kgd,uint32_t mec,uint32_t pipe,uint32_t queue,uint32_t vmid)122 static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe,
123 uint32_t queue, uint32_t vmid)
124 {
125 struct amdgpu_device *adev = get_amdgpu_device(kgd);
126 uint32_t value = PIPEID(pipe) | MEID(mec) | VMID(vmid) | QUEUEID(queue);
127
128 mutex_lock(&adev->srbm_mutex);
129 WREG32(mmSRBM_GFX_CNTL, value);
130 }
131
unlock_srbm(struct kgd_dev * kgd)132 static void unlock_srbm(struct kgd_dev *kgd)
133 {
134 struct amdgpu_device *adev = get_amdgpu_device(kgd);
135
136 WREG32(mmSRBM_GFX_CNTL, 0);
137 mutex_unlock(&adev->srbm_mutex);
138 }
139
acquire_queue(struct kgd_dev * kgd,uint32_t pipe_id,uint32_t queue_id)140 static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,
141 uint32_t queue_id)
142 {
143 struct amdgpu_device *adev = get_amdgpu_device(kgd);
144
145 uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
146 uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
147
148 lock_srbm(kgd, mec, pipe, queue_id, 0);
149 }
150
release_queue(struct kgd_dev * kgd)151 static void release_queue(struct kgd_dev *kgd)
152 {
153 unlock_srbm(kgd);
154 }
155
kgd_program_sh_mem_settings(struct kgd_dev * kgd,uint32_t vmid,uint32_t sh_mem_config,uint32_t sh_mem_ape1_base,uint32_t sh_mem_ape1_limit,uint32_t sh_mem_bases)156 static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
157 uint32_t sh_mem_config,
158 uint32_t sh_mem_ape1_base,
159 uint32_t sh_mem_ape1_limit,
160 uint32_t sh_mem_bases)
161 {
162 struct amdgpu_device *adev = get_amdgpu_device(kgd);
163
164 lock_srbm(kgd, 0, 0, 0, vmid);
165
166 WREG32(mmSH_MEM_CONFIG, sh_mem_config);
167 WREG32(mmSH_MEM_APE1_BASE, sh_mem_ape1_base);
168 WREG32(mmSH_MEM_APE1_LIMIT, sh_mem_ape1_limit);
169 WREG32(mmSH_MEM_BASES, sh_mem_bases);
170
171 unlock_srbm(kgd);
172 }
173
kgd_set_pasid_vmid_mapping(struct kgd_dev * kgd,unsigned int pasid,unsigned int vmid)174 static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
175 unsigned int vmid)
176 {
177 struct amdgpu_device *adev = get_amdgpu_device(kgd);
178
179 /*
180 * We have to assume that there is no outstanding mapping.
181 * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 because
182 * a mapping is in progress or because a mapping finished and the
183 * SW cleared it. So the protocol is to always wait & clear.
184 */
185 uint32_t pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid |
186 ATC_VMID0_PASID_MAPPING__VALID_MASK;
187
188 WREG32(mmATC_VMID0_PASID_MAPPING + vmid, pasid_mapping);
189
190 while (!(RREG32(mmATC_VMID_PASID_MAPPING_UPDATE_STATUS) & (1U << vmid)))
191 cpu_relax();
192 WREG32(mmATC_VMID_PASID_MAPPING_UPDATE_STATUS, 1U << vmid);
193
194 /* Mapping vmid to pasid also for IH block */
195 WREG32(mmIH_VMID_0_LUT + vmid, pasid_mapping);
196
197 return 0;
198 }
199
kgd_init_interrupts(struct kgd_dev * kgd,uint32_t pipe_id)200 static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id)
201 {
202 struct amdgpu_device *adev = get_amdgpu_device(kgd);
203 uint32_t mec;
204 uint32_t pipe;
205
206 mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
207 pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
208
209 lock_srbm(kgd, mec, pipe, 0, 0);
210
211 WREG32(mmCPC_INT_CNTL, CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK |
212 CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK);
213
214 unlock_srbm(kgd);
215
216 return 0;
217 }
218
get_sdma_rlc_reg_offset(struct cik_sdma_rlc_registers * m)219 static inline uint32_t get_sdma_rlc_reg_offset(struct cik_sdma_rlc_registers *m)
220 {
221 uint32_t retval;
222
223 retval = m->sdma_engine_id * SDMA1_REGISTER_OFFSET +
224 m->sdma_queue_id * KFD_CIK_SDMA_QUEUE_OFFSET;
225
226 pr_debug("RLC register offset for SDMA%d RLC%d: 0x%x\n",
227 m->sdma_engine_id, m->sdma_queue_id, retval);
228
229 return retval;
230 }
231
get_mqd(void * mqd)232 static inline struct cik_mqd *get_mqd(void *mqd)
233 {
234 return (struct cik_mqd *)mqd;
235 }
236
get_sdma_mqd(void * mqd)237 static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
238 {
239 return (struct cik_sdma_rlc_registers *)mqd;
240 }
241
kgd_hqd_load(struct kgd_dev * kgd,void * mqd,uint32_t pipe_id,uint32_t queue_id,uint32_t __user * wptr,uint32_t wptr_shift,uint32_t wptr_mask,struct mm_struct * mm)242 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
243 uint32_t queue_id, uint32_t __user *wptr,
244 uint32_t wptr_shift, uint32_t wptr_mask,
245 struct mm_struct *mm)
246 {
247 struct amdgpu_device *adev = get_amdgpu_device(kgd);
248 struct cik_mqd *m;
249 uint32_t *mqd_hqd;
250 uint32_t reg, wptr_val, data;
251 bool valid_wptr = false;
252
253 m = get_mqd(mqd);
254
255 acquire_queue(kgd, pipe_id, queue_id);
256
257 /* HQD registers extend from CP_MQD_BASE_ADDR to CP_MQD_CONTROL. */
258 mqd_hqd = &m->cp_mqd_base_addr_lo;
259
260 for (reg = mmCP_MQD_BASE_ADDR; reg <= mmCP_MQD_CONTROL; reg++)
261 WREG32(reg, mqd_hqd[reg - mmCP_MQD_BASE_ADDR]);
262
263 /* Copy userspace write pointer value to register.
264 * Activate doorbell logic to monitor subsequent changes.
265 */
266 data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control,
267 CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
268 WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data);
269
270 /* read_user_ptr may take the mm->mmap_sem.
271 * release srbm_mutex to avoid circular dependency between
272 * srbm_mutex->mm_sem->reservation_ww_class_mutex->srbm_mutex.
273 */
274 release_queue(kgd);
275 valid_wptr = read_user_wptr(mm, wptr, wptr_val);
276 acquire_queue(kgd, pipe_id, queue_id);
277 if (valid_wptr)
278 WREG32(mmCP_HQD_PQ_WPTR, (wptr_val << wptr_shift) & wptr_mask);
279
280 data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1);
281 WREG32(mmCP_HQD_ACTIVE, data);
282
283 release_queue(kgd);
284
285 return 0;
286 }
287
kgd_hqd_dump(struct kgd_dev * kgd,uint32_t pipe_id,uint32_t queue_id,uint32_t (** dump)[2],uint32_t * n_regs)288 static int kgd_hqd_dump(struct kgd_dev *kgd,
289 uint32_t pipe_id, uint32_t queue_id,
290 uint32_t (**dump)[2], uint32_t *n_regs)
291 {
292 struct amdgpu_device *adev = get_amdgpu_device(kgd);
293 uint32_t i = 0, reg;
294 #define HQD_N_REGS (35+4)
295 #define DUMP_REG(addr) do { \
296 if (WARN_ON_ONCE(i >= HQD_N_REGS)) \
297 break; \
298 (*dump)[i][0] = (addr) << 2; \
299 (*dump)[i++][1] = RREG32(addr); \
300 } while (0)
301
302 *dump = kmalloc_array(HQD_N_REGS * 2, sizeof(uint32_t), GFP_KERNEL);
303 if (*dump == NULL)
304 return -ENOMEM;
305
306 acquire_queue(kgd, pipe_id, queue_id);
307
308 DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE0);
309 DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE1);
310 DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE2);
311 DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE3);
312
313 for (reg = mmCP_MQD_BASE_ADDR; reg <= mmCP_MQD_CONTROL; reg++)
314 DUMP_REG(reg);
315
316 release_queue(kgd);
317
318 WARN_ON_ONCE(i != HQD_N_REGS);
319 *n_regs = i;
320
321 return 0;
322 }
323
kgd_hqd_sdma_load(struct kgd_dev * kgd,void * mqd,uint32_t __user * wptr,struct mm_struct * mm)324 static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
325 uint32_t __user *wptr, struct mm_struct *mm)
326 {
327 struct amdgpu_device *adev = get_amdgpu_device(kgd);
328 struct cik_sdma_rlc_registers *m;
329 unsigned long end_jiffies;
330 uint32_t sdma_rlc_reg_offset;
331 uint32_t data;
332
333 m = get_sdma_mqd(mqd);
334 sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(m);
335
336 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL,
337 m->sdma_rlc_rb_cntl & (~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK));
338
339 end_jiffies = msecs_to_jiffies(2000) + jiffies;
340 while (true) {
341 data = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_CONTEXT_STATUS);
342 if (data & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK)
343 break;
344 if (time_after(jiffies, end_jiffies)) {
345 pr_err("SDMA RLC not idle in %s\n", __func__);
346 return -ETIME;
347 }
348 usleep_range(500, 1000);
349 }
350
351 data = REG_SET_FIELD(m->sdma_rlc_doorbell, SDMA0_RLC0_DOORBELL,
352 ENABLE, 1);
353 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_DOORBELL, data);
354 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR,
355 m->sdma_rlc_rb_rptr);
356
357 if (read_user_wptr(mm, wptr, data))
358 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_WPTR, data);
359 else
360 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_WPTR,
361 m->sdma_rlc_rb_rptr);
362
363 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_VIRTUAL_ADDR,
364 m->sdma_rlc_virtual_addr);
365 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_BASE, m->sdma_rlc_rb_base);
366 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_BASE_HI,
367 m->sdma_rlc_rb_base_hi);
368 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR_ADDR_LO,
369 m->sdma_rlc_rb_rptr_addr_lo);
370 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR_ADDR_HI,
371 m->sdma_rlc_rb_rptr_addr_hi);
372
373 data = REG_SET_FIELD(m->sdma_rlc_rb_cntl, SDMA0_RLC0_RB_CNTL,
374 RB_ENABLE, 1);
375 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL, data);
376
377 return 0;
378 }
379
kgd_hqd_sdma_dump(struct kgd_dev * kgd,uint32_t engine_id,uint32_t queue_id,uint32_t (** dump)[2],uint32_t * n_regs)380 static int kgd_hqd_sdma_dump(struct kgd_dev *kgd,
381 uint32_t engine_id, uint32_t queue_id,
382 uint32_t (**dump)[2], uint32_t *n_regs)
383 {
384 struct amdgpu_device *adev = get_amdgpu_device(kgd);
385 uint32_t sdma_offset = engine_id * SDMA1_REGISTER_OFFSET +
386 queue_id * KFD_CIK_SDMA_QUEUE_OFFSET;
387 uint32_t i = 0, reg;
388 #undef HQD_N_REGS
389 #define HQD_N_REGS (19+4)
390
391 *dump = kmalloc_array(HQD_N_REGS * 2, sizeof(uint32_t), GFP_KERNEL);
392 if (*dump == NULL)
393 return -ENOMEM;
394
395 for (reg = mmSDMA0_RLC0_RB_CNTL; reg <= mmSDMA0_RLC0_DOORBELL; reg++)
396 DUMP_REG(sdma_offset + reg);
397 for (reg = mmSDMA0_RLC0_VIRTUAL_ADDR; reg <= mmSDMA0_RLC0_WATERMARK;
398 reg++)
399 DUMP_REG(sdma_offset + reg);
400
401 WARN_ON_ONCE(i != HQD_N_REGS);
402 *n_regs = i;
403
404 return 0;
405 }
406
kgd_hqd_is_occupied(struct kgd_dev * kgd,uint64_t queue_address,uint32_t pipe_id,uint32_t queue_id)407 static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
408 uint32_t pipe_id, uint32_t queue_id)
409 {
410 struct amdgpu_device *adev = get_amdgpu_device(kgd);
411 uint32_t act;
412 bool retval = false;
413 uint32_t low, high;
414
415 acquire_queue(kgd, pipe_id, queue_id);
416 act = RREG32(mmCP_HQD_ACTIVE);
417 if (act) {
418 low = lower_32_bits(queue_address >> 8);
419 high = upper_32_bits(queue_address >> 8);
420
421 if (low == RREG32(mmCP_HQD_PQ_BASE) &&
422 high == RREG32(mmCP_HQD_PQ_BASE_HI))
423 retval = true;
424 }
425 release_queue(kgd);
426 return retval;
427 }
428
kgd_hqd_sdma_is_occupied(struct kgd_dev * kgd,void * mqd)429 static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
430 {
431 struct amdgpu_device *adev = get_amdgpu_device(kgd);
432 struct cik_sdma_rlc_registers *m;
433 uint32_t sdma_rlc_reg_offset;
434 uint32_t sdma_rlc_rb_cntl;
435
436 m = get_sdma_mqd(mqd);
437 sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(m);
438
439 sdma_rlc_rb_cntl = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL);
440
441 if (sdma_rlc_rb_cntl & SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK)
442 return true;
443
444 return false;
445 }
446
kgd_hqd_destroy(struct kgd_dev * kgd,void * mqd,enum kfd_preempt_type reset_type,unsigned int utimeout,uint32_t pipe_id,uint32_t queue_id)447 static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
448 enum kfd_preempt_type reset_type,
449 unsigned int utimeout, uint32_t pipe_id,
450 uint32_t queue_id)
451 {
452 struct amdgpu_device *adev = get_amdgpu_device(kgd);
453 uint32_t temp;
454 enum hqd_dequeue_request_type type;
455 unsigned long flags, end_jiffies;
456 int retry;
457
458 if (adev->in_gpu_reset)
459 return -EIO;
460
461 acquire_queue(kgd, pipe_id, queue_id);
462 WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
463
464 switch (reset_type) {
465 case KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN:
466 type = DRAIN_PIPE;
467 break;
468 case KFD_PREEMPT_TYPE_WAVEFRONT_RESET:
469 type = RESET_WAVES;
470 break;
471 default:
472 type = DRAIN_PIPE;
473 break;
474 }
475
476 /* Workaround: If IQ timer is active and the wait time is close to or
477 * equal to 0, dequeueing is not safe. Wait until either the wait time
478 * is larger or timer is cleared. Also, ensure that IQ_REQ_PEND is
479 * cleared before continuing. Also, ensure wait times are set to at
480 * least 0x3.
481 */
482 local_irq_save(flags);
483 preempt_disable();
484 retry = 5000; /* wait for 500 usecs at maximum */
485 while (true) {
486 temp = RREG32(mmCP_HQD_IQ_TIMER);
487 if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, PROCESSING_IQ)) {
488 pr_debug("HW is processing IQ\n");
489 goto loop;
490 }
491 if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, ACTIVE)) {
492 if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, RETRY_TYPE)
493 == 3) /* SEM-rearm is safe */
494 break;
495 /* Wait time 3 is safe for CP, but our MMIO read/write
496 * time is close to 1 microsecond, so check for 10 to
497 * leave more buffer room
498 */
499 if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, WAIT_TIME)
500 >= 10)
501 break;
502 pr_debug("IQ timer is active\n");
503 } else
504 break;
505 loop:
506 if (!retry) {
507 pr_err("CP HQD IQ timer status time out\n");
508 break;
509 }
510 ndelay(100);
511 --retry;
512 }
513 retry = 1000;
514 while (true) {
515 temp = RREG32(mmCP_HQD_DEQUEUE_REQUEST);
516 if (!(temp & CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_MASK))
517 break;
518 pr_debug("Dequeue request is pending\n");
519
520 if (!retry) {
521 pr_err("CP HQD dequeue request time out\n");
522 break;
523 }
524 ndelay(100);
525 --retry;
526 }
527 local_irq_restore(flags);
528 preempt_enable();
529
530 WREG32(mmCP_HQD_DEQUEUE_REQUEST, type);
531
532 end_jiffies = (utimeout * HZ / 1000) + jiffies;
533 while (true) {
534 temp = RREG32(mmCP_HQD_ACTIVE);
535 if (!(temp & CP_HQD_ACTIVE__ACTIVE_MASK))
536 break;
537 if (time_after(jiffies, end_jiffies)) {
538 pr_err("cp queue preemption time out\n");
539 release_queue(kgd);
540 return -ETIME;
541 }
542 usleep_range(500, 1000);
543 }
544
545 release_queue(kgd);
546 return 0;
547 }
548
kgd_hqd_sdma_destroy(struct kgd_dev * kgd,void * mqd,unsigned int utimeout)549 static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
550 unsigned int utimeout)
551 {
552 struct amdgpu_device *adev = get_amdgpu_device(kgd);
553 struct cik_sdma_rlc_registers *m;
554 uint32_t sdma_rlc_reg_offset;
555 uint32_t temp;
556 unsigned long end_jiffies = (utimeout * HZ / 1000) + jiffies;
557
558 m = get_sdma_mqd(mqd);
559 sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(m);
560
561 temp = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL);
562 temp = temp & ~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK;
563 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL, temp);
564
565 while (true) {
566 temp = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_CONTEXT_STATUS);
567 if (temp & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK)
568 break;
569 if (time_after(jiffies, end_jiffies)) {
570 pr_err("SDMA RLC not idle in %s\n", __func__);
571 return -ETIME;
572 }
573 usleep_range(500, 1000);
574 }
575
576 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_DOORBELL, 0);
577 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL,
578 RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL) |
579 SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK);
580
581 m->sdma_rlc_rb_rptr = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR);
582
583 return 0;
584 }
585
kgd_address_watch_disable(struct kgd_dev * kgd)586 static int kgd_address_watch_disable(struct kgd_dev *kgd)
587 {
588 struct amdgpu_device *adev = get_amdgpu_device(kgd);
589 union TCP_WATCH_CNTL_BITS cntl;
590 unsigned int i;
591
592 cntl.u32All = 0;
593
594 cntl.bitfields.valid = 0;
595 cntl.bitfields.mask = ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK;
596 cntl.bitfields.atc = 1;
597
598 /* Turning off this address until we set all the registers */
599 for (i = 0; i < MAX_WATCH_ADDRESSES; i++)
600 WREG32(watchRegs[i * ADDRESS_WATCH_REG_MAX +
601 ADDRESS_WATCH_REG_CNTL], cntl.u32All);
602
603 return 0;
604 }
605
kgd_address_watch_execute(struct kgd_dev * kgd,unsigned int watch_point_id,uint32_t cntl_val,uint32_t addr_hi,uint32_t addr_lo)606 static int kgd_address_watch_execute(struct kgd_dev *kgd,
607 unsigned int watch_point_id,
608 uint32_t cntl_val,
609 uint32_t addr_hi,
610 uint32_t addr_lo)
611 {
612 struct amdgpu_device *adev = get_amdgpu_device(kgd);
613 union TCP_WATCH_CNTL_BITS cntl;
614
615 cntl.u32All = cntl_val;
616
617 /* Turning off this watch point until we set all the registers */
618 cntl.bitfields.valid = 0;
619 WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX +
620 ADDRESS_WATCH_REG_CNTL], cntl.u32All);
621
622 WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX +
623 ADDRESS_WATCH_REG_ADDR_HI], addr_hi);
624
625 WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX +
626 ADDRESS_WATCH_REG_ADDR_LO], addr_lo);
627
628 /* Enable the watch point */
629 cntl.bitfields.valid = 1;
630
631 WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX +
632 ADDRESS_WATCH_REG_CNTL], cntl.u32All);
633
634 return 0;
635 }
636
kgd_wave_control_execute(struct kgd_dev * kgd,uint32_t gfx_index_val,uint32_t sq_cmd)637 static int kgd_wave_control_execute(struct kgd_dev *kgd,
638 uint32_t gfx_index_val,
639 uint32_t sq_cmd)
640 {
641 struct amdgpu_device *adev = get_amdgpu_device(kgd);
642 uint32_t data;
643
644 mutex_lock(&adev->grbm_idx_mutex);
645
646 WREG32(mmGRBM_GFX_INDEX, gfx_index_val);
647 WREG32(mmSQ_CMD, sq_cmd);
648
649 /* Restore the GRBM_GFX_INDEX register */
650
651 data = GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES_MASK |
652 GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK |
653 GRBM_GFX_INDEX__SE_BROADCAST_WRITES_MASK;
654
655 WREG32(mmGRBM_GFX_INDEX, data);
656
657 mutex_unlock(&adev->grbm_idx_mutex);
658
659 return 0;
660 }
661
kgd_address_watch_get_offset(struct kgd_dev * kgd,unsigned int watch_point_id,unsigned int reg_offset)662 static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,
663 unsigned int watch_point_id,
664 unsigned int reg_offset)
665 {
666 return watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + reg_offset];
667 }
668
get_atc_vmid_pasid_mapping_info(struct kgd_dev * kgd,uint8_t vmid,uint16_t * p_pasid)669 static bool get_atc_vmid_pasid_mapping_info(struct kgd_dev *kgd,
670 uint8_t vmid, uint16_t *p_pasid)
671 {
672 uint32_t value;
673 struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
674
675 value = RREG32(mmATC_VMID0_PASID_MAPPING + vmid);
676 *p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK;
677
678 return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK);
679 }
680
set_scratch_backing_va(struct kgd_dev * kgd,uint64_t va,uint32_t vmid)681 static void set_scratch_backing_va(struct kgd_dev *kgd,
682 uint64_t va, uint32_t vmid)
683 {
684 struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
685
686 lock_srbm(kgd, 0, 0, 0, vmid);
687 WREG32(mmSH_HIDDEN_PRIVATE_BASE_VMID, va);
688 unlock_srbm(kgd);
689 }
690
set_vm_context_page_table_base(struct kgd_dev * kgd,uint32_t vmid,uint64_t page_table_base)691 static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
692 uint64_t page_table_base)
693 {
694 struct amdgpu_device *adev = get_amdgpu_device(kgd);
695
696 if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) {
697 pr_err("trying to set page table base for wrong VMID\n");
698 return;
699 }
700 WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vmid - 8,
701 lower_32_bits(page_table_base));
702 }
703
704 /**
705 * read_vmid_from_vmfault_reg - read vmid from register
706 *
707 * adev: amdgpu_device pointer
708 * @vmid: vmid pointer
709 * read vmid from register (CIK).
710 */
read_vmid_from_vmfault_reg(struct kgd_dev * kgd)711 static uint32_t read_vmid_from_vmfault_reg(struct kgd_dev *kgd)
712 {
713 struct amdgpu_device *adev = get_amdgpu_device(kgd);
714
715 uint32_t status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS);
716
717 return REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID);
718 }
719
720 const struct kfd2kgd_calls gfx_v7_kfd2kgd = {
721 .program_sh_mem_settings = kgd_program_sh_mem_settings,
722 .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
723 .init_interrupts = kgd_init_interrupts,
724 .hqd_load = kgd_hqd_load,
725 .hqd_sdma_load = kgd_hqd_sdma_load,
726 .hqd_dump = kgd_hqd_dump,
727 .hqd_sdma_dump = kgd_hqd_sdma_dump,
728 .hqd_is_occupied = kgd_hqd_is_occupied,
729 .hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied,
730 .hqd_destroy = kgd_hqd_destroy,
731 .hqd_sdma_destroy = kgd_hqd_sdma_destroy,
732 .address_watch_disable = kgd_address_watch_disable,
733 .address_watch_execute = kgd_address_watch_execute,
734 .wave_control_execute = kgd_wave_control_execute,
735 .address_watch_get_offset = kgd_address_watch_get_offset,
736 .get_atc_vmid_pasid_mapping_info = get_atc_vmid_pasid_mapping_info,
737 .set_scratch_backing_va = set_scratch_backing_va,
738 .get_tile_config = get_tile_config,
739 .set_vm_context_page_table_base = set_vm_context_page_table_base,
740 .read_vmid_from_vmfault_reg = read_vmid_from_vmfault_reg,
741 };
742