xref: /netbsd-src/sys/external/bsd/drm2/dist/drm/amd/amdkfd/kfd_mqd_manager_cik.c (revision 41ec02673d281bbb3d38e6c78504ce6e30c228c1)
1 /*	$NetBSD: kfd_mqd_manager_cik.c,v 1.3 2021/12/18 23:44:59 riastradh Exp $	*/
2 
3 /*
4  * Copyright 2014 Advanced Micro Devices, Inc.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  */
25 
26 #include <sys/cdefs.h>
27 __KERNEL_RCSID(0, "$NetBSD: kfd_mqd_manager_cik.c,v 1.3 2021/12/18 23:44:59 riastradh Exp $");
28 
29 #include <linux/printk.h>
30 #include <linux/slab.h>
31 #include <linux/mm_types.h>
32 
33 #include "kfd_priv.h"
34 #include "kfd_mqd_manager.h"
35 #include "cik_regs.h"
36 #include "cik_structs.h"
37 #include "oss/oss_2_4_sh_mask.h"
38 
get_mqd(void * mqd)39 static inline struct cik_mqd *get_mqd(void *mqd)
40 {
41 	return (struct cik_mqd *)mqd;
42 }
43 
get_sdma_mqd(void * mqd)44 static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
45 {
46 	return (struct cik_sdma_rlc_registers *)mqd;
47 }
48 
update_cu_mask(struct mqd_manager * mm,void * mqd,struct queue_properties * q)49 static void update_cu_mask(struct mqd_manager *mm, void *mqd,
50 			struct queue_properties *q)
51 {
52 	struct cik_mqd *m;
53 	uint32_t se_mask[4] = {0}; /* 4 is the max # of SEs */
54 
55 	if (q->cu_mask_count == 0)
56 		return;
57 
58 	mqd_symmetrically_map_cu_mask(mm,
59 		q->cu_mask, q->cu_mask_count, se_mask);
60 
61 	m = get_mqd(mqd);
62 	m->compute_static_thread_mgmt_se0 = se_mask[0];
63 	m->compute_static_thread_mgmt_se1 = se_mask[1];
64 	m->compute_static_thread_mgmt_se2 = se_mask[2];
65 	m->compute_static_thread_mgmt_se3 = se_mask[3];
66 
67 	pr_debug("Update cu mask to %#x %#x %#x %#x\n",
68 		m->compute_static_thread_mgmt_se0,
69 		m->compute_static_thread_mgmt_se1,
70 		m->compute_static_thread_mgmt_se2,
71 		m->compute_static_thread_mgmt_se3);
72 }
73 
set_priority(struct cik_mqd * m,struct queue_properties * q)74 static void set_priority(struct cik_mqd *m, struct queue_properties *q)
75 {
76 	m->cp_hqd_pipe_priority = pipe_priority_map[q->priority];
77 	m->cp_hqd_queue_priority = q->priority;
78 }
79 
allocate_mqd(struct kfd_dev * kfd,struct queue_properties * q)80 static struct kfd_mem_obj *allocate_mqd(struct kfd_dev *kfd,
81 					struct queue_properties *q)
82 {
83 	struct kfd_mem_obj *mqd_mem_obj;
84 
85 	if (kfd_gtt_sa_allocate(kfd, sizeof(struct cik_mqd),
86 			&mqd_mem_obj))
87 		return NULL;
88 
89 	return mqd_mem_obj;
90 }
91 
init_mqd(struct mqd_manager * mm,void ** mqd,struct kfd_mem_obj * mqd_mem_obj,uint64_t * gart_addr,struct queue_properties * q)92 static void init_mqd(struct mqd_manager *mm, void **mqd,
93 		struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
94 		struct queue_properties *q)
95 {
96 	uint64_t addr;
97 	struct cik_mqd *m;
98 
99 	m = (struct cik_mqd *) mqd_mem_obj->cpu_ptr;
100 	addr = mqd_mem_obj->gpu_addr;
101 
102 	memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256));
103 
104 	m->header = 0xC0310800;
105 	m->compute_pipelinestat_enable = 1;
106 	m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF;
107 	m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF;
108 	m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF;
109 	m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF;
110 
111 	/*
112 	 * Make sure to use the last queue state saved on mqd when the cp
113 	 * reassigns the queue, so when queue is switched on/off (e.g over
114 	 * subscription or quantum timeout) the context will be consistent
115 	 */
116 	m->cp_hqd_persistent_state =
117 				DEFAULT_CP_HQD_PERSISTENT_STATE | PRELOAD_REQ;
118 
119 	m->cp_mqd_control             = MQD_CONTROL_PRIV_STATE_EN;
120 	m->cp_mqd_base_addr_lo        = lower_32_bits(addr);
121 	m->cp_mqd_base_addr_hi        = upper_32_bits(addr);
122 
123 	m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS |
124 				QUANTUM_DURATION(10);
125 
126 	/*
127 	 * Pipe Priority
128 	 * Identifies the pipe relative priority when this queue is connected
129 	 * to the pipeline. The pipe priority is against the GFX pipe and HP3D.
130 	 * In KFD we are using a fixed pipe priority set to CS_MEDIUM.
131 	 * 0 = CS_LOW (typically below GFX)
132 	 * 1 = CS_MEDIUM (typically between HP3D and GFX
133 	 * 2 = CS_HIGH (typically above HP3D)
134 	 */
135 	set_priority(m, q);
136 
137 	if (q->format == KFD_QUEUE_FORMAT_AQL)
138 		m->cp_hqd_iq_rptr = AQL_ENABLE;
139 
140 	*mqd = m;
141 	if (gart_addr)
142 		*gart_addr = addr;
143 	mm->update_mqd(mm, m, q);
144 }
145 
init_mqd_sdma(struct mqd_manager * mm,void ** mqd,struct kfd_mem_obj * mqd_mem_obj,uint64_t * gart_addr,struct queue_properties * q)146 static void init_mqd_sdma(struct mqd_manager *mm, void **mqd,
147 			struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
148 			struct queue_properties *q)
149 {
150 	struct cik_sdma_rlc_registers *m;
151 
152 	m = (struct cik_sdma_rlc_registers *) mqd_mem_obj->cpu_ptr;
153 
154 	memset(m, 0, sizeof(struct cik_sdma_rlc_registers));
155 
156 	*mqd = m;
157 	if (gart_addr)
158 		*gart_addr = mqd_mem_obj->gpu_addr;
159 
160 	mm->update_mqd(mm, m, q);
161 }
162 
free_mqd(struct mqd_manager * mm,void * mqd,struct kfd_mem_obj * mqd_mem_obj)163 static void free_mqd(struct mqd_manager *mm, void *mqd,
164 			struct kfd_mem_obj *mqd_mem_obj)
165 {
166 	kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
167 }
168 
169 
load_mqd(struct mqd_manager * mm,void * mqd,uint32_t pipe_id,uint32_t queue_id,struct queue_properties * p,struct mm_struct * mms)170 static int load_mqd(struct mqd_manager *mm, void *mqd, uint32_t pipe_id,
171 		    uint32_t queue_id, struct queue_properties *p,
172 		    struct mm_struct *mms)
173 {
174 	/* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */
175 	uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0);
176 	uint32_t wptr_mask = (uint32_t)((p->queue_size / 4) - 1);
177 
178 	return mm->dev->kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id,
179 					  (uint32_t __user *)p->write_ptr,
180 					  wptr_shift, wptr_mask, mms);
181 }
182 
load_mqd_sdma(struct mqd_manager * mm,void * mqd,uint32_t pipe_id,uint32_t queue_id,struct queue_properties * p,struct mm_struct * mms)183 static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
184 			 uint32_t pipe_id, uint32_t queue_id,
185 			 struct queue_properties *p, struct mm_struct *mms)
186 {
187 	return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd,
188 					       (uint32_t __user *)p->write_ptr,
189 					       mms);
190 }
191 
__update_mqd(struct mqd_manager * mm,void * mqd,struct queue_properties * q,unsigned int atc_bit)192 static void __update_mqd(struct mqd_manager *mm, void *mqd,
193 			struct queue_properties *q, unsigned int atc_bit)
194 {
195 	struct cik_mqd *m;
196 
197 	m = get_mqd(mqd);
198 	m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE |
199 				DEFAULT_MIN_AVAIL_SIZE;
200 	m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE;
201 	if (atc_bit) {
202 		m->cp_hqd_pq_control |= PQ_ATC_EN;
203 		m->cp_hqd_ib_control |= IB_ATC_EN;
204 	}
205 
206 	/*
207 	 * Calculating queue size which is log base 2 of actual queue size -1
208 	 * dwords and another -1 for ffs
209 	 */
210 	m->cp_hqd_pq_control |= order_base_2(q->queue_size / 4) - 1;
211 	m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
212 	m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
213 	m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
214 	m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
215 	m->cp_hqd_pq_doorbell_control = DOORBELL_OFFSET(q->doorbell_off);
216 
217 	m->cp_hqd_vmid = q->vmid;
218 
219 	if (q->format == KFD_QUEUE_FORMAT_AQL)
220 		m->cp_hqd_pq_control |= NO_UPDATE_RPTR;
221 
222 	update_cu_mask(mm, mqd, q);
223 	set_priority(m, q);
224 
225 	q->is_active = QUEUE_IS_ACTIVE(*q);
226 }
227 
update_mqd(struct mqd_manager * mm,void * mqd,struct queue_properties * q)228 static void update_mqd(struct mqd_manager *mm, void *mqd,
229 			struct queue_properties *q)
230 {
231 	__update_mqd(mm, mqd, q, 1);
232 }
233 
update_mqd_hawaii(struct mqd_manager * mm,void * mqd,struct queue_properties * q)234 static void update_mqd_hawaii(struct mqd_manager *mm, void *mqd,
235 			struct queue_properties *q)
236 {
237 	__update_mqd(mm, mqd, q, 0);
238 }
239 
update_mqd_sdma(struct mqd_manager * mm,void * mqd,struct queue_properties * q)240 static void update_mqd_sdma(struct mqd_manager *mm, void *mqd,
241 				struct queue_properties *q)
242 {
243 	struct cik_sdma_rlc_registers *m;
244 
245 	m = get_sdma_mqd(mqd);
246 	m->sdma_rlc_rb_cntl = order_base_2(q->queue_size / 4)
247 			<< SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT |
248 			q->vmid << SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT |
249 			1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
250 			6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT;
251 
252 	m->sdma_rlc_rb_base = lower_32_bits(q->queue_address >> 8);
253 	m->sdma_rlc_rb_base_hi = upper_32_bits(q->queue_address >> 8);
254 	m->sdma_rlc_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
255 	m->sdma_rlc_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
256 	m->sdma_rlc_doorbell =
257 		q->doorbell_off << SDMA0_RLC0_DOORBELL__OFFSET__SHIFT;
258 
259 	m->sdma_rlc_virtual_addr = q->sdma_vm_addr;
260 
261 	m->sdma_engine_id = q->sdma_engine_id;
262 	m->sdma_queue_id = q->sdma_queue_id;
263 
264 	q->is_active = QUEUE_IS_ACTIVE(*q);
265 }
266 
destroy_mqd(struct mqd_manager * mm,void * mqd,enum kfd_preempt_type type,unsigned int timeout,uint32_t pipe_id,uint32_t queue_id)267 static int destroy_mqd(struct mqd_manager *mm, void *mqd,
268 			enum kfd_preempt_type type,
269 			unsigned int timeout, uint32_t pipe_id,
270 			uint32_t queue_id)
271 {
272 	return mm->dev->kfd2kgd->hqd_destroy(mm->dev->kgd, mqd, type, timeout,
273 					pipe_id, queue_id);
274 }
275 
276 /*
277  * preempt type here is ignored because there is only one way
278  * to preempt sdma queue
279  */
destroy_mqd_sdma(struct mqd_manager * mm,void * mqd,enum kfd_preempt_type type,unsigned int timeout,uint32_t pipe_id,uint32_t queue_id)280 static int destroy_mqd_sdma(struct mqd_manager *mm, void *mqd,
281 				enum kfd_preempt_type type,
282 				unsigned int timeout, uint32_t pipe_id,
283 				uint32_t queue_id)
284 {
285 	return mm->dev->kfd2kgd->hqd_sdma_destroy(mm->dev->kgd, mqd, timeout);
286 }
287 
is_occupied(struct mqd_manager * mm,void * mqd,uint64_t queue_address,uint32_t pipe_id,uint32_t queue_id)288 static bool is_occupied(struct mqd_manager *mm, void *mqd,
289 			uint64_t queue_address,	uint32_t pipe_id,
290 			uint32_t queue_id)
291 {
292 
293 	return mm->dev->kfd2kgd->hqd_is_occupied(mm->dev->kgd, queue_address,
294 					pipe_id, queue_id);
295 
296 }
297 
is_occupied_sdma(struct mqd_manager * mm,void * mqd,uint64_t queue_address,uint32_t pipe_id,uint32_t queue_id)298 static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
299 			uint64_t queue_address,	uint32_t pipe_id,
300 			uint32_t queue_id)
301 {
302 	return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd);
303 }
304 
305 /*
306  * HIQ MQD Implementation, concrete implementation for HIQ MQD implementation.
307  * The HIQ queue in Kaveri is using the same MQD structure as all the user mode
308  * queues but with different initial values.
309  */
310 
init_mqd_hiq(struct mqd_manager * mm,void ** mqd,struct kfd_mem_obj * mqd_mem_obj,uint64_t * gart_addr,struct queue_properties * q)311 static void init_mqd_hiq(struct mqd_manager *mm, void **mqd,
312 		struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
313 		struct queue_properties *q)
314 {
315 	init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q);
316 }
317 
update_mqd_hiq(struct mqd_manager * mm,void * mqd,struct queue_properties * q)318 static void update_mqd_hiq(struct mqd_manager *mm, void *mqd,
319 				struct queue_properties *q)
320 {
321 	struct cik_mqd *m;
322 
323 	m = get_mqd(mqd);
324 	m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE |
325 				DEFAULT_MIN_AVAIL_SIZE |
326 				PRIV_STATE |
327 				KMD_QUEUE;
328 
329 	/*
330 	 * Calculating queue size which is log base 2 of actual queue
331 	 * size -1 dwords
332 	 */
333 	m->cp_hqd_pq_control |= order_base_2(q->queue_size / 4) - 1;
334 	m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
335 	m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
336 	m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
337 	m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
338 	m->cp_hqd_pq_doorbell_control = DOORBELL_OFFSET(q->doorbell_off);
339 
340 	m->cp_hqd_vmid = q->vmid;
341 
342 	q->is_active = QUEUE_IS_ACTIVE(*q);
343 
344 	set_priority(m, q);
345 }
346 
347 #if defined(CONFIG_DEBUG_FS)
348 
debugfs_show_mqd(struct seq_file * m,void * data)349 static int debugfs_show_mqd(struct seq_file *m, void *data)
350 {
351 	seq_hex_dump(m, "    ", DUMP_PREFIX_OFFSET, 32, 4,
352 		     data, sizeof(struct cik_mqd), false);
353 	return 0;
354 }
355 
debugfs_show_mqd_sdma(struct seq_file * m,void * data)356 static int debugfs_show_mqd_sdma(struct seq_file *m, void *data)
357 {
358 	seq_hex_dump(m, "    ", DUMP_PREFIX_OFFSET, 32, 4,
359 		     data, sizeof(struct cik_sdma_rlc_registers), false);
360 	return 0;
361 }
362 
363 #endif
364 
365 
mqd_manager_init_cik(enum KFD_MQD_TYPE type,struct kfd_dev * dev)366 struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
367 		struct kfd_dev *dev)
368 {
369 	struct mqd_manager *mqd;
370 
371 	if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
372 		return NULL;
373 
374 	mqd = kzalloc(sizeof(*mqd), GFP_KERNEL);
375 	if (!mqd)
376 		return NULL;
377 
378 	mqd->dev = dev;
379 
380 	switch (type) {
381 	case KFD_MQD_TYPE_CP:
382 		mqd->allocate_mqd = allocate_mqd;
383 		mqd->init_mqd = init_mqd;
384 		mqd->free_mqd = free_mqd;
385 		mqd->load_mqd = load_mqd;
386 		mqd->update_mqd = update_mqd;
387 		mqd->destroy_mqd = destroy_mqd;
388 		mqd->is_occupied = is_occupied;
389 		mqd->mqd_size = sizeof(struct cik_mqd);
390 #if defined(CONFIG_DEBUG_FS)
391 		mqd->debugfs_show_mqd = debugfs_show_mqd;
392 #endif
393 		break;
394 	case KFD_MQD_TYPE_HIQ:
395 		mqd->allocate_mqd = allocate_hiq_mqd;
396 		mqd->init_mqd = init_mqd_hiq;
397 		mqd->free_mqd = free_mqd_hiq_sdma;
398 		mqd->load_mqd = load_mqd;
399 		mqd->update_mqd = update_mqd_hiq;
400 		mqd->destroy_mqd = destroy_mqd;
401 		mqd->is_occupied = is_occupied;
402 		mqd->mqd_size = sizeof(struct cik_mqd);
403 #if defined(CONFIG_DEBUG_FS)
404 		mqd->debugfs_show_mqd = debugfs_show_mqd;
405 #endif
406 		break;
407 	case KFD_MQD_TYPE_DIQ:
408 		mqd->allocate_mqd = allocate_mqd;
409 		mqd->init_mqd = init_mqd_hiq;
410 		mqd->free_mqd = free_mqd;
411 		mqd->load_mqd = load_mqd;
412 		mqd->update_mqd = update_mqd_hiq;
413 		mqd->destroy_mqd = destroy_mqd;
414 		mqd->is_occupied = is_occupied;
415 		mqd->mqd_size = sizeof(struct cik_mqd);
416 #if defined(CONFIG_DEBUG_FS)
417 		mqd->debugfs_show_mqd = debugfs_show_mqd;
418 #endif
419 		break;
420 	case KFD_MQD_TYPE_SDMA:
421 		mqd->allocate_mqd = allocate_sdma_mqd;
422 		mqd->init_mqd = init_mqd_sdma;
423 		mqd->free_mqd = free_mqd_hiq_sdma;
424 		mqd->load_mqd = load_mqd_sdma;
425 		mqd->update_mqd = update_mqd_sdma;
426 		mqd->destroy_mqd = destroy_mqd_sdma;
427 		mqd->is_occupied = is_occupied_sdma;
428 		mqd->mqd_size = sizeof(struct cik_sdma_rlc_registers);
429 #if defined(CONFIG_DEBUG_FS)
430 		mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
431 #endif
432 		break;
433 	default:
434 		kfree(mqd);
435 		return NULL;
436 	}
437 
438 	return mqd;
439 }
440 
mqd_manager_init_cik_hawaii(enum KFD_MQD_TYPE type,struct kfd_dev * dev)441 struct mqd_manager *mqd_manager_init_cik_hawaii(enum KFD_MQD_TYPE type,
442 			struct kfd_dev *dev)
443 {
444 	struct mqd_manager *mqd;
445 
446 	mqd = mqd_manager_init_cik(type, dev);
447 	if (!mqd)
448 		return NULL;
449 	if (type == KFD_MQD_TYPE_CP)
450 		mqd->update_mqd = update_mqd_hawaii;
451 	return mqd;
452 }
453