1 /* $NetBSD: kfd_mqd_manager_cik.c,v 1.3 2021/12/18 23:44:59 riastradh Exp $ */
2
3 /*
4 * Copyright 2014 Advanced Micro Devices, Inc.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 */
25
26 #include <sys/cdefs.h>
27 __KERNEL_RCSID(0, "$NetBSD: kfd_mqd_manager_cik.c,v 1.3 2021/12/18 23:44:59 riastradh Exp $");
28
29 #include <linux/printk.h>
30 #include <linux/slab.h>
31 #include <linux/mm_types.h>
32
33 #include "kfd_priv.h"
34 #include "kfd_mqd_manager.h"
35 #include "cik_regs.h"
36 #include "cik_structs.h"
37 #include "oss/oss_2_4_sh_mask.h"
38
get_mqd(void * mqd)39 static inline struct cik_mqd *get_mqd(void *mqd)
40 {
41 return (struct cik_mqd *)mqd;
42 }
43
get_sdma_mqd(void * mqd)44 static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
45 {
46 return (struct cik_sdma_rlc_registers *)mqd;
47 }
48
update_cu_mask(struct mqd_manager * mm,void * mqd,struct queue_properties * q)49 static void update_cu_mask(struct mqd_manager *mm, void *mqd,
50 struct queue_properties *q)
51 {
52 struct cik_mqd *m;
53 uint32_t se_mask[4] = {0}; /* 4 is the max # of SEs */
54
55 if (q->cu_mask_count == 0)
56 return;
57
58 mqd_symmetrically_map_cu_mask(mm,
59 q->cu_mask, q->cu_mask_count, se_mask);
60
61 m = get_mqd(mqd);
62 m->compute_static_thread_mgmt_se0 = se_mask[0];
63 m->compute_static_thread_mgmt_se1 = se_mask[1];
64 m->compute_static_thread_mgmt_se2 = se_mask[2];
65 m->compute_static_thread_mgmt_se3 = se_mask[3];
66
67 pr_debug("Update cu mask to %#x %#x %#x %#x\n",
68 m->compute_static_thread_mgmt_se0,
69 m->compute_static_thread_mgmt_se1,
70 m->compute_static_thread_mgmt_se2,
71 m->compute_static_thread_mgmt_se3);
72 }
73
set_priority(struct cik_mqd * m,struct queue_properties * q)74 static void set_priority(struct cik_mqd *m, struct queue_properties *q)
75 {
76 m->cp_hqd_pipe_priority = pipe_priority_map[q->priority];
77 m->cp_hqd_queue_priority = q->priority;
78 }
79
allocate_mqd(struct kfd_dev * kfd,struct queue_properties * q)80 static struct kfd_mem_obj *allocate_mqd(struct kfd_dev *kfd,
81 struct queue_properties *q)
82 {
83 struct kfd_mem_obj *mqd_mem_obj;
84
85 if (kfd_gtt_sa_allocate(kfd, sizeof(struct cik_mqd),
86 &mqd_mem_obj))
87 return NULL;
88
89 return mqd_mem_obj;
90 }
91
init_mqd(struct mqd_manager * mm,void ** mqd,struct kfd_mem_obj * mqd_mem_obj,uint64_t * gart_addr,struct queue_properties * q)92 static void init_mqd(struct mqd_manager *mm, void **mqd,
93 struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
94 struct queue_properties *q)
95 {
96 uint64_t addr;
97 struct cik_mqd *m;
98
99 m = (struct cik_mqd *) mqd_mem_obj->cpu_ptr;
100 addr = mqd_mem_obj->gpu_addr;
101
102 memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256));
103
104 m->header = 0xC0310800;
105 m->compute_pipelinestat_enable = 1;
106 m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF;
107 m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF;
108 m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF;
109 m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF;
110
111 /*
112 * Make sure to use the last queue state saved on mqd when the cp
113 * reassigns the queue, so when queue is switched on/off (e.g over
114 * subscription or quantum timeout) the context will be consistent
115 */
116 m->cp_hqd_persistent_state =
117 DEFAULT_CP_HQD_PERSISTENT_STATE | PRELOAD_REQ;
118
119 m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN;
120 m->cp_mqd_base_addr_lo = lower_32_bits(addr);
121 m->cp_mqd_base_addr_hi = upper_32_bits(addr);
122
123 m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS |
124 QUANTUM_DURATION(10);
125
126 /*
127 * Pipe Priority
128 * Identifies the pipe relative priority when this queue is connected
129 * to the pipeline. The pipe priority is against the GFX pipe and HP3D.
130 * In KFD we are using a fixed pipe priority set to CS_MEDIUM.
131 * 0 = CS_LOW (typically below GFX)
132 * 1 = CS_MEDIUM (typically between HP3D and GFX
133 * 2 = CS_HIGH (typically above HP3D)
134 */
135 set_priority(m, q);
136
137 if (q->format == KFD_QUEUE_FORMAT_AQL)
138 m->cp_hqd_iq_rptr = AQL_ENABLE;
139
140 *mqd = m;
141 if (gart_addr)
142 *gart_addr = addr;
143 mm->update_mqd(mm, m, q);
144 }
145
init_mqd_sdma(struct mqd_manager * mm,void ** mqd,struct kfd_mem_obj * mqd_mem_obj,uint64_t * gart_addr,struct queue_properties * q)146 static void init_mqd_sdma(struct mqd_manager *mm, void **mqd,
147 struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
148 struct queue_properties *q)
149 {
150 struct cik_sdma_rlc_registers *m;
151
152 m = (struct cik_sdma_rlc_registers *) mqd_mem_obj->cpu_ptr;
153
154 memset(m, 0, sizeof(struct cik_sdma_rlc_registers));
155
156 *mqd = m;
157 if (gart_addr)
158 *gart_addr = mqd_mem_obj->gpu_addr;
159
160 mm->update_mqd(mm, m, q);
161 }
162
free_mqd(struct mqd_manager * mm,void * mqd,struct kfd_mem_obj * mqd_mem_obj)163 static void free_mqd(struct mqd_manager *mm, void *mqd,
164 struct kfd_mem_obj *mqd_mem_obj)
165 {
166 kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
167 }
168
169
load_mqd(struct mqd_manager * mm,void * mqd,uint32_t pipe_id,uint32_t queue_id,struct queue_properties * p,struct mm_struct * mms)170 static int load_mqd(struct mqd_manager *mm, void *mqd, uint32_t pipe_id,
171 uint32_t queue_id, struct queue_properties *p,
172 struct mm_struct *mms)
173 {
174 /* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */
175 uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0);
176 uint32_t wptr_mask = (uint32_t)((p->queue_size / 4) - 1);
177
178 return mm->dev->kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id,
179 (uint32_t __user *)p->write_ptr,
180 wptr_shift, wptr_mask, mms);
181 }
182
load_mqd_sdma(struct mqd_manager * mm,void * mqd,uint32_t pipe_id,uint32_t queue_id,struct queue_properties * p,struct mm_struct * mms)183 static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
184 uint32_t pipe_id, uint32_t queue_id,
185 struct queue_properties *p, struct mm_struct *mms)
186 {
187 return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd,
188 (uint32_t __user *)p->write_ptr,
189 mms);
190 }
191
__update_mqd(struct mqd_manager * mm,void * mqd,struct queue_properties * q,unsigned int atc_bit)192 static void __update_mqd(struct mqd_manager *mm, void *mqd,
193 struct queue_properties *q, unsigned int atc_bit)
194 {
195 struct cik_mqd *m;
196
197 m = get_mqd(mqd);
198 m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE |
199 DEFAULT_MIN_AVAIL_SIZE;
200 m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE;
201 if (atc_bit) {
202 m->cp_hqd_pq_control |= PQ_ATC_EN;
203 m->cp_hqd_ib_control |= IB_ATC_EN;
204 }
205
206 /*
207 * Calculating queue size which is log base 2 of actual queue size -1
208 * dwords and another -1 for ffs
209 */
210 m->cp_hqd_pq_control |= order_base_2(q->queue_size / 4) - 1;
211 m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
212 m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
213 m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
214 m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
215 m->cp_hqd_pq_doorbell_control = DOORBELL_OFFSET(q->doorbell_off);
216
217 m->cp_hqd_vmid = q->vmid;
218
219 if (q->format == KFD_QUEUE_FORMAT_AQL)
220 m->cp_hqd_pq_control |= NO_UPDATE_RPTR;
221
222 update_cu_mask(mm, mqd, q);
223 set_priority(m, q);
224
225 q->is_active = QUEUE_IS_ACTIVE(*q);
226 }
227
update_mqd(struct mqd_manager * mm,void * mqd,struct queue_properties * q)228 static void update_mqd(struct mqd_manager *mm, void *mqd,
229 struct queue_properties *q)
230 {
231 __update_mqd(mm, mqd, q, 1);
232 }
233
update_mqd_hawaii(struct mqd_manager * mm,void * mqd,struct queue_properties * q)234 static void update_mqd_hawaii(struct mqd_manager *mm, void *mqd,
235 struct queue_properties *q)
236 {
237 __update_mqd(mm, mqd, q, 0);
238 }
239
update_mqd_sdma(struct mqd_manager * mm,void * mqd,struct queue_properties * q)240 static void update_mqd_sdma(struct mqd_manager *mm, void *mqd,
241 struct queue_properties *q)
242 {
243 struct cik_sdma_rlc_registers *m;
244
245 m = get_sdma_mqd(mqd);
246 m->sdma_rlc_rb_cntl = order_base_2(q->queue_size / 4)
247 << SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT |
248 q->vmid << SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT |
249 1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
250 6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT;
251
252 m->sdma_rlc_rb_base = lower_32_bits(q->queue_address >> 8);
253 m->sdma_rlc_rb_base_hi = upper_32_bits(q->queue_address >> 8);
254 m->sdma_rlc_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
255 m->sdma_rlc_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
256 m->sdma_rlc_doorbell =
257 q->doorbell_off << SDMA0_RLC0_DOORBELL__OFFSET__SHIFT;
258
259 m->sdma_rlc_virtual_addr = q->sdma_vm_addr;
260
261 m->sdma_engine_id = q->sdma_engine_id;
262 m->sdma_queue_id = q->sdma_queue_id;
263
264 q->is_active = QUEUE_IS_ACTIVE(*q);
265 }
266
destroy_mqd(struct mqd_manager * mm,void * mqd,enum kfd_preempt_type type,unsigned int timeout,uint32_t pipe_id,uint32_t queue_id)267 static int destroy_mqd(struct mqd_manager *mm, void *mqd,
268 enum kfd_preempt_type type,
269 unsigned int timeout, uint32_t pipe_id,
270 uint32_t queue_id)
271 {
272 return mm->dev->kfd2kgd->hqd_destroy(mm->dev->kgd, mqd, type, timeout,
273 pipe_id, queue_id);
274 }
275
276 /*
277 * preempt type here is ignored because there is only one way
278 * to preempt sdma queue
279 */
destroy_mqd_sdma(struct mqd_manager * mm,void * mqd,enum kfd_preempt_type type,unsigned int timeout,uint32_t pipe_id,uint32_t queue_id)280 static int destroy_mqd_sdma(struct mqd_manager *mm, void *mqd,
281 enum kfd_preempt_type type,
282 unsigned int timeout, uint32_t pipe_id,
283 uint32_t queue_id)
284 {
285 return mm->dev->kfd2kgd->hqd_sdma_destroy(mm->dev->kgd, mqd, timeout);
286 }
287
is_occupied(struct mqd_manager * mm,void * mqd,uint64_t queue_address,uint32_t pipe_id,uint32_t queue_id)288 static bool is_occupied(struct mqd_manager *mm, void *mqd,
289 uint64_t queue_address, uint32_t pipe_id,
290 uint32_t queue_id)
291 {
292
293 return mm->dev->kfd2kgd->hqd_is_occupied(mm->dev->kgd, queue_address,
294 pipe_id, queue_id);
295
296 }
297
is_occupied_sdma(struct mqd_manager * mm,void * mqd,uint64_t queue_address,uint32_t pipe_id,uint32_t queue_id)298 static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
299 uint64_t queue_address, uint32_t pipe_id,
300 uint32_t queue_id)
301 {
302 return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd);
303 }
304
305 /*
306 * HIQ MQD Implementation, concrete implementation for HIQ MQD implementation.
307 * The HIQ queue in Kaveri is using the same MQD structure as all the user mode
308 * queues but with different initial values.
309 */
310
init_mqd_hiq(struct mqd_manager * mm,void ** mqd,struct kfd_mem_obj * mqd_mem_obj,uint64_t * gart_addr,struct queue_properties * q)311 static void init_mqd_hiq(struct mqd_manager *mm, void **mqd,
312 struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
313 struct queue_properties *q)
314 {
315 init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q);
316 }
317
update_mqd_hiq(struct mqd_manager * mm,void * mqd,struct queue_properties * q)318 static void update_mqd_hiq(struct mqd_manager *mm, void *mqd,
319 struct queue_properties *q)
320 {
321 struct cik_mqd *m;
322
323 m = get_mqd(mqd);
324 m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE |
325 DEFAULT_MIN_AVAIL_SIZE |
326 PRIV_STATE |
327 KMD_QUEUE;
328
329 /*
330 * Calculating queue size which is log base 2 of actual queue
331 * size -1 dwords
332 */
333 m->cp_hqd_pq_control |= order_base_2(q->queue_size / 4) - 1;
334 m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
335 m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
336 m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
337 m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
338 m->cp_hqd_pq_doorbell_control = DOORBELL_OFFSET(q->doorbell_off);
339
340 m->cp_hqd_vmid = q->vmid;
341
342 q->is_active = QUEUE_IS_ACTIVE(*q);
343
344 set_priority(m, q);
345 }
346
347 #if defined(CONFIG_DEBUG_FS)
348
debugfs_show_mqd(struct seq_file * m,void * data)349 static int debugfs_show_mqd(struct seq_file *m, void *data)
350 {
351 seq_hex_dump(m, " ", DUMP_PREFIX_OFFSET, 32, 4,
352 data, sizeof(struct cik_mqd), false);
353 return 0;
354 }
355
debugfs_show_mqd_sdma(struct seq_file * m,void * data)356 static int debugfs_show_mqd_sdma(struct seq_file *m, void *data)
357 {
358 seq_hex_dump(m, " ", DUMP_PREFIX_OFFSET, 32, 4,
359 data, sizeof(struct cik_sdma_rlc_registers), false);
360 return 0;
361 }
362
363 #endif
364
365
mqd_manager_init_cik(enum KFD_MQD_TYPE type,struct kfd_dev * dev)366 struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
367 struct kfd_dev *dev)
368 {
369 struct mqd_manager *mqd;
370
371 if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
372 return NULL;
373
374 mqd = kzalloc(sizeof(*mqd), GFP_KERNEL);
375 if (!mqd)
376 return NULL;
377
378 mqd->dev = dev;
379
380 switch (type) {
381 case KFD_MQD_TYPE_CP:
382 mqd->allocate_mqd = allocate_mqd;
383 mqd->init_mqd = init_mqd;
384 mqd->free_mqd = free_mqd;
385 mqd->load_mqd = load_mqd;
386 mqd->update_mqd = update_mqd;
387 mqd->destroy_mqd = destroy_mqd;
388 mqd->is_occupied = is_occupied;
389 mqd->mqd_size = sizeof(struct cik_mqd);
390 #if defined(CONFIG_DEBUG_FS)
391 mqd->debugfs_show_mqd = debugfs_show_mqd;
392 #endif
393 break;
394 case KFD_MQD_TYPE_HIQ:
395 mqd->allocate_mqd = allocate_hiq_mqd;
396 mqd->init_mqd = init_mqd_hiq;
397 mqd->free_mqd = free_mqd_hiq_sdma;
398 mqd->load_mqd = load_mqd;
399 mqd->update_mqd = update_mqd_hiq;
400 mqd->destroy_mqd = destroy_mqd;
401 mqd->is_occupied = is_occupied;
402 mqd->mqd_size = sizeof(struct cik_mqd);
403 #if defined(CONFIG_DEBUG_FS)
404 mqd->debugfs_show_mqd = debugfs_show_mqd;
405 #endif
406 break;
407 case KFD_MQD_TYPE_DIQ:
408 mqd->allocate_mqd = allocate_mqd;
409 mqd->init_mqd = init_mqd_hiq;
410 mqd->free_mqd = free_mqd;
411 mqd->load_mqd = load_mqd;
412 mqd->update_mqd = update_mqd_hiq;
413 mqd->destroy_mqd = destroy_mqd;
414 mqd->is_occupied = is_occupied;
415 mqd->mqd_size = sizeof(struct cik_mqd);
416 #if defined(CONFIG_DEBUG_FS)
417 mqd->debugfs_show_mqd = debugfs_show_mqd;
418 #endif
419 break;
420 case KFD_MQD_TYPE_SDMA:
421 mqd->allocate_mqd = allocate_sdma_mqd;
422 mqd->init_mqd = init_mqd_sdma;
423 mqd->free_mqd = free_mqd_hiq_sdma;
424 mqd->load_mqd = load_mqd_sdma;
425 mqd->update_mqd = update_mqd_sdma;
426 mqd->destroy_mqd = destroy_mqd_sdma;
427 mqd->is_occupied = is_occupied_sdma;
428 mqd->mqd_size = sizeof(struct cik_sdma_rlc_registers);
429 #if defined(CONFIG_DEBUG_FS)
430 mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
431 #endif
432 break;
433 default:
434 kfree(mqd);
435 return NULL;
436 }
437
438 return mqd;
439 }
440
mqd_manager_init_cik_hawaii(enum KFD_MQD_TYPE type,struct kfd_dev * dev)441 struct mqd_manager *mqd_manager_init_cik_hawaii(enum KFD_MQD_TYPE type,
442 struct kfd_dev *dev)
443 {
444 struct mqd_manager *mqd;
445
446 mqd = mqd_manager_init_cik(type, dev);
447 if (!mqd)
448 return NULL;
449 if (type == KFD_MQD_TYPE_CP)
450 mqd->update_mqd = update_mqd_hawaii;
451 return mqd;
452 }
453