xref: /netbsd-src/sys/external/bsd/drm2/dist/drm/amd/amdgpu/amdgpu_ring.c (revision 7330f729ccf0bd976a06f95fad452fe774fc7fd1)
1 /*	$NetBSD: amdgpu_ring.c,v 1.3 2018/08/27 14:04:50 riastradh Exp $	*/
2 
3 /*
4  * Copyright 2008 Advanced Micro Devices, Inc.
5  * Copyright 2008 Red Hat Inc.
6  * Copyright 2009 Jerome Glisse.
7  *
8  * Permission is hereby granted, free of charge, to any person obtaining a
9  * copy of this software and associated documentation files (the "Software"),
10  * to deal in the Software without restriction, including without limitation
11  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
12  * and/or sell copies of the Software, and to permit persons to whom the
13  * Software is furnished to do so, subject to the following conditions:
14  *
15  * The above copyright notice and this permission notice shall be included in
16  * all copies or substantial portions of the Software.
17  *
18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
21  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
22  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
23  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
24  * OTHER DEALINGS IN THE SOFTWARE.
25  *
26  * Authors: Dave Airlie
27  *          Alex Deucher
28  *          Jerome Glisse
29  *          Christian König
30  */
31 #include <sys/cdefs.h>
32 __KERNEL_RCSID(0, "$NetBSD: amdgpu_ring.c,v 1.3 2018/08/27 14:04:50 riastradh Exp $");
33 
34 #include <asm/byteorder.h>
35 #include <linux/log2.h>
36 #include <linux/seq_file.h>
37 #include <linux/slab.h>
38 #include <drm/drmP.h>
39 #include <drm/amdgpu_drm.h>
40 #include "amdgpu.h"
41 #include "atom.h"
42 
43 /*
44  * Rings
45  * Most engines on the GPU are fed via ring buffers.  Ring
46  * buffers are areas of GPU accessible memory that the host
47  * writes commands into and the GPU reads commands out of.
48  * There is a rptr (read pointer) that determines where the
49  * GPU is currently reading, and a wptr (write pointer)
50  * which determines where the host has written.  When the
51  * pointers are equal, the ring is idle.  When the host
52  * writes commands to the ring buffer, it increments the
53  * wptr.  The GPU then starts fetching commands and executes
54  * them until the pointers are equal again.
55  */
56 static int amdgpu_debugfs_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring);
57 
58 /**
59  * amdgpu_ring_free_size - update the free size
60  *
61  * @adev: amdgpu_device pointer
62  * @ring: amdgpu_ring structure holding ring information
63  *
64  * Update the free dw slots in the ring buffer (all asics).
65  */
66 void amdgpu_ring_free_size(struct amdgpu_ring *ring)
67 {
68 	uint32_t rptr = amdgpu_ring_get_rptr(ring);
69 
70 	/* This works because ring_size is a power of 2 */
71 	ring->ring_free_dw = rptr + (ring->ring_size / 4);
72 	ring->ring_free_dw -= ring->wptr;
73 	ring->ring_free_dw &= ring->ptr_mask;
74 	if (!ring->ring_free_dw) {
75 		/* this is an empty ring */
76 		ring->ring_free_dw = ring->ring_size / 4;
77 	}
78 }
79 
80 /**
81  * amdgpu_ring_alloc - allocate space on the ring buffer
82  *
83  * @adev: amdgpu_device pointer
84  * @ring: amdgpu_ring structure holding ring information
85  * @ndw: number of dwords to allocate in the ring buffer
86  *
87  * Allocate @ndw dwords in the ring buffer (all asics).
88  * Returns 0 on success, error on failure.
89  */
90 int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned ndw)
91 {
92 	int r;
93 
94 	/* make sure we aren't trying to allocate more space than there is on the ring */
95 	if (ndw > (ring->ring_size / 4))
96 		return -ENOMEM;
97 	/* Align requested size with padding so unlock_commit can
98 	 * pad safely */
99 	amdgpu_ring_free_size(ring);
100 	ndw = (ndw + ring->align_mask) & ~ring->align_mask;
101 	while (ndw > (ring->ring_free_dw - 1)) {
102 		amdgpu_ring_free_size(ring);
103 		if (ndw < ring->ring_free_dw) {
104 			break;
105 		}
106 		r = amdgpu_fence_wait_next(ring);
107 		if (r)
108 			return r;
109 	}
110 	ring->count_dw = ndw;
111 	ring->wptr_old = ring->wptr;
112 	return 0;
113 }
114 
115 /**
116  * amdgpu_ring_lock - lock the ring and allocate space on it
117  *
118  * @adev: amdgpu_device pointer
119  * @ring: amdgpu_ring structure holding ring information
120  * @ndw: number of dwords to allocate in the ring buffer
121  *
122  * Lock the ring and allocate @ndw dwords in the ring buffer
123  * (all asics).
124  * Returns 0 on success, error on failure.
125  */
126 int amdgpu_ring_lock(struct amdgpu_ring *ring, unsigned ndw)
127 {
128 	int r;
129 
130 	mutex_lock(ring->ring_lock);
131 	r = amdgpu_ring_alloc(ring, ndw);
132 	if (r) {
133 		mutex_unlock(ring->ring_lock);
134 		return r;
135 	}
136 	return 0;
137 }
138 
139 /** amdgpu_ring_insert_nop - insert NOP packets
140  *
141  * @ring: amdgpu_ring structure holding ring information
142  * @count: the number of NOP packets to insert
143  *
144  * This is the generic insert_nop function for rings except SDMA
145  */
146 void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
147 {
148 	int i;
149 
150 	for (i = 0; i < count; i++)
151 		amdgpu_ring_write(ring, ring->nop);
152 }
153 
154 /**
155  * amdgpu_ring_commit - tell the GPU to execute the new
156  * commands on the ring buffer
157  *
158  * @adev: amdgpu_device pointer
159  * @ring: amdgpu_ring structure holding ring information
160  *
161  * Update the wptr (write pointer) to tell the GPU to
162  * execute new commands on the ring buffer (all asics).
163  */
164 void amdgpu_ring_commit(struct amdgpu_ring *ring)
165 {
166 	uint32_t count;
167 
168 	/* We pad to match fetch size */
169 	count = ring->align_mask + 1 - (ring->wptr & ring->align_mask);
170 	count %= ring->align_mask + 1;
171 	ring->funcs->insert_nop(ring, count);
172 
173 	mb();
174 	amdgpu_ring_set_wptr(ring);
175 }
176 
177 /**
178  * amdgpu_ring_unlock_commit - tell the GPU to execute the new
179  * commands on the ring buffer and unlock it
180  *
181  * @ring: amdgpu_ring structure holding ring information
182  *
183  * Call amdgpu_ring_commit() then unlock the ring (all asics).
184  */
185 void amdgpu_ring_unlock_commit(struct amdgpu_ring *ring)
186 {
187 	amdgpu_ring_commit(ring);
188 	mutex_unlock(ring->ring_lock);
189 }
190 
191 /**
192  * amdgpu_ring_undo - reset the wptr
193  *
194  * @ring: amdgpu_ring structure holding ring information
195  *
196  * Reset the driver's copy of the wptr (all asics).
197  */
198 void amdgpu_ring_undo(struct amdgpu_ring *ring)
199 {
200 	ring->wptr = ring->wptr_old;
201 }
202 
203 /**
204  * amdgpu_ring_unlock_undo - reset the wptr and unlock the ring
205  *
206  * @ring: amdgpu_ring structure holding ring information
207  *
208  * Call amdgpu_ring_undo() then unlock the ring (all asics).
209  */
210 void amdgpu_ring_unlock_undo(struct amdgpu_ring *ring)
211 {
212 	amdgpu_ring_undo(ring);
213 	mutex_unlock(ring->ring_lock);
214 }
215 
216 /**
217  * amdgpu_ring_backup - Back up the content of a ring
218  *
219  * @ring: the ring we want to back up
220  *
221  * Saves all unprocessed commits from a ring, returns the number of dwords saved.
222  */
223 unsigned amdgpu_ring_backup(struct amdgpu_ring *ring,
224 			    uint32_t **data)
225 {
226 	unsigned size, ptr, i;
227 
228 	/* just in case lock the ring */
229 	mutex_lock(ring->ring_lock);
230 	*data = NULL;
231 
232 	if (ring->ring_obj == NULL) {
233 		mutex_unlock(ring->ring_lock);
234 		return 0;
235 	}
236 
237 	/* it doesn't make sense to save anything if all fences are signaled */
238 	if (!amdgpu_fence_count_emitted(ring)) {
239 		mutex_unlock(ring->ring_lock);
240 		return 0;
241 	}
242 
243 	ptr = le32_to_cpu(*ring->next_rptr_cpu_addr);
244 
245 	size = ring->wptr + (ring->ring_size / 4);
246 	size -= ptr;
247 	size &= ring->ptr_mask;
248 	if (size == 0) {
249 		mutex_unlock(ring->ring_lock);
250 		return 0;
251 	}
252 
253 	/* and then save the content of the ring */
254 	*data = kmalloc_array(size, sizeof(uint32_t), GFP_KERNEL);
255 	if (!*data) {
256 		mutex_unlock(ring->ring_lock);
257 		return 0;
258 	}
259 	for (i = 0; i < size; ++i) {
260 		(*data)[i] = ring->ring[ptr++];
261 		ptr &= ring->ptr_mask;
262 	}
263 
264 	mutex_unlock(ring->ring_lock);
265 	return size;
266 }
267 
268 /**
269  * amdgpu_ring_restore - append saved commands to the ring again
270  *
271  * @ring: ring to append commands to
272  * @size: number of dwords we want to write
273  * @data: saved commands
274  *
275  * Allocates space on the ring and restore the previously saved commands.
276  */
277 int amdgpu_ring_restore(struct amdgpu_ring *ring,
278 			unsigned size, uint32_t *data)
279 {
280 	int i, r;
281 
282 	if (!size || !data)
283 		return 0;
284 
285 	/* restore the saved ring content */
286 	r = amdgpu_ring_lock(ring, size);
287 	if (r)
288 		return r;
289 
290 	for (i = 0; i < size; ++i) {
291 		amdgpu_ring_write(ring, data[i]);
292 	}
293 
294 	amdgpu_ring_unlock_commit(ring);
295 	kfree(data);
296 	return 0;
297 }
298 
299 /**
300  * amdgpu_ring_init - init driver ring struct.
301  *
302  * @adev: amdgpu_device pointer
303  * @ring: amdgpu_ring structure holding ring information
304  * @ring_size: size of the ring
305  * @nop: nop packet for this ring
306  *
307  * Initialize the driver information for the selected ring (all asics).
308  * Returns 0 on success, error on failure.
309  */
310 int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
311 		     unsigned ring_size, u32 nop, u32 align_mask,
312 		     struct amdgpu_irq_src *irq_src, unsigned irq_type,
313 		     enum amdgpu_ring_type ring_type)
314 {
315 	u32 rb_bufsz;
316 	int r;
317 
318 	if (ring->adev == NULL) {
319 		if (adev->num_rings >= AMDGPU_MAX_RINGS)
320 			return -EINVAL;
321 
322 		ring->adev = adev;
323 		ring->idx = adev->num_rings++;
324 		adev->rings[ring->idx] = ring;
325 		r = amdgpu_fence_driver_init_ring(ring);
326 		if (r)
327 			return r;
328 	}
329 
330 	r = amdgpu_wb_get(adev, &ring->rptr_offs);
331 	if (r) {
332 		dev_err(adev->dev, "(%d) ring rptr_offs wb alloc failed\n", r);
333 		return r;
334 	}
335 
336 	r = amdgpu_wb_get(adev, &ring->wptr_offs);
337 	if (r) {
338 		dev_err(adev->dev, "(%d) ring wptr_offs wb alloc failed\n", r);
339 		return r;
340 	}
341 
342 	r = amdgpu_wb_get(adev, &ring->fence_offs);
343 	if (r) {
344 		dev_err(adev->dev, "(%d) ring fence_offs wb alloc failed\n", r);
345 		return r;
346 	}
347 
348 	r = amdgpu_wb_get(adev, &ring->next_rptr_offs);
349 	if (r) {
350 		dev_err(adev->dev, "(%d) ring next_rptr wb alloc failed\n", r);
351 		return r;
352 	}
353 	ring->next_rptr_gpu_addr = adev->wb.gpu_addr + (ring->next_rptr_offs * 4);
354 	ring->next_rptr_cpu_addr = &adev->wb.wb[ring->next_rptr_offs];
355 	spin_lock_init(&ring->fence_lock);
356 	r = amdgpu_fence_driver_start_ring(ring, irq_src, irq_type);
357 	if (r) {
358 		dev_err(adev->dev, "failed initializing fences (%d).\n", r);
359 		return r;
360 	}
361 
362 	ring->ring_lock = &adev->ring_lock;
363 	/* Align ring size */
364 	rb_bufsz = order_base_2(ring_size / 8);
365 	ring_size = (1 << (rb_bufsz + 1)) * 4;
366 	ring->ring_size = ring_size;
367 	ring->align_mask = align_mask;
368 	ring->nop = nop;
369 	ring->type = ring_type;
370 
371 	/* Allocate ring buffer */
372 	if (ring->ring_obj == NULL) {
373 		r = amdgpu_bo_create(adev, ring->ring_size, PAGE_SIZE, true,
374 				     AMDGPU_GEM_DOMAIN_GTT, 0,
375 				     NULL, NULL, &ring->ring_obj);
376 		if (r) {
377 			dev_err(adev->dev, "(%d) ring create failed\n", r);
378 			return r;
379 		}
380 		r = amdgpu_bo_reserve(ring->ring_obj, false);
381 		if (unlikely(r != 0))
382 			return r;
383 		r = amdgpu_bo_pin(ring->ring_obj, AMDGPU_GEM_DOMAIN_GTT,
384 					&ring->gpu_addr);
385 		if (r) {
386 			amdgpu_bo_unreserve(ring->ring_obj);
387 			dev_err(adev->dev, "(%d) ring pin failed\n", r);
388 			return r;
389 		}
390 		r = amdgpu_bo_kmap(ring->ring_obj,
391 				       (void **)__UNVOLATILE(&ring->ring));
392 		amdgpu_bo_unreserve(ring->ring_obj);
393 		if (r) {
394 			dev_err(adev->dev, "(%d) ring map failed\n", r);
395 			return r;
396 		}
397 	}
398 	ring->ptr_mask = (ring->ring_size / 4) - 1;
399 	ring->ring_free_dw = ring->ring_size / 4;
400 
401 	if (amdgpu_debugfs_ring_init(adev, ring)) {
402 		DRM_ERROR("Failed to register debugfs file for rings !\n");
403 	}
404 	return 0;
405 }
406 
407 /**
408  * amdgpu_ring_fini - tear down the driver ring struct.
409  *
410  * @adev: amdgpu_device pointer
411  * @ring: amdgpu_ring structure holding ring information
412  *
413  * Tear down the driver information for the selected ring (all asics).
414  */
415 void amdgpu_ring_fini(struct amdgpu_ring *ring)
416 {
417 	int r;
418 	struct amdgpu_bo *ring_obj;
419 
420 	if (ring->ring_lock == NULL)
421 		return;
422 
423 	mutex_lock(ring->ring_lock);
424 	ring_obj = ring->ring_obj;
425 	ring->ready = false;
426 	ring->ring = NULL;
427 	ring->ring_obj = NULL;
428 	mutex_unlock(ring->ring_lock);
429 
430 	amdgpu_wb_free(ring->adev, ring->fence_offs);
431 	amdgpu_wb_free(ring->adev, ring->rptr_offs);
432 	amdgpu_wb_free(ring->adev, ring->wptr_offs);
433 	amdgpu_wb_free(ring->adev, ring->next_rptr_offs);
434 
435 	if (ring_obj) {
436 		r = amdgpu_bo_reserve(ring_obj, false);
437 		if (likely(r == 0)) {
438 			amdgpu_bo_kunmap(ring_obj);
439 			amdgpu_bo_unpin(ring_obj);
440 			amdgpu_bo_unreserve(ring_obj);
441 		}
442 		amdgpu_bo_unref(&ring_obj);
443 	}
444 }
445 
446 /**
447  * amdgpu_ring_from_fence - get ring from fence
448  *
449  * @f: fence structure
450  *
451  * Extract the ring a fence belongs to. Handles both scheduler as
452  * well as hardware fences.
453  */
454 struct amdgpu_ring *amdgpu_ring_from_fence(struct fence *f)
455 {
456 	struct amdgpu_fence *a_fence;
457 	struct amd_sched_fence *s_fence;
458 
459 	s_fence = to_amd_sched_fence(f);
460 	if (s_fence)
461 		return container_of(s_fence->sched, struct amdgpu_ring, sched);
462 
463 	a_fence = to_amdgpu_fence(f);
464 	if (a_fence)
465 		return a_fence->ring;
466 
467 	return NULL;
468 }
469 
470 /*
471  * Debugfs info
472  */
473 #if defined(CONFIG_DEBUG_FS)
474 
475 static int amdgpu_debugfs_ring_info(struct seq_file *m, void *data)
476 {
477 	struct drm_info_node *node = (struct drm_info_node *) m->private;
478 	struct drm_device *dev = node->minor->dev;
479 	struct amdgpu_device *adev = dev->dev_private;
480 	int roffset = *(int*)node->info_ent->data;
481 	struct amdgpu_ring *ring = (void *)(((uint8_t*)adev) + roffset);
482 
483 	uint32_t rptr, wptr, rptr_next;
484 	unsigned count, i, j;
485 
486 	amdgpu_ring_free_size(ring);
487 	count = (ring->ring_size / 4) - ring->ring_free_dw;
488 
489 	wptr = amdgpu_ring_get_wptr(ring);
490 	seq_printf(m, "wptr: 0x%08x [%5d]\n",
491 		   wptr, wptr);
492 
493 	rptr = amdgpu_ring_get_rptr(ring);
494 	seq_printf(m, "rptr: 0x%08x [%5d]\n",
495 		   rptr, rptr);
496 
497 	rptr_next = ~0;
498 
499 	seq_printf(m, "driver's copy of the wptr: 0x%08x [%5d]\n",
500 		   ring->wptr, ring->wptr);
501 	seq_printf(m, "last semaphore signal addr : 0x%016llx\n",
502 		   ring->last_semaphore_signal_addr);
503 	seq_printf(m, "last semaphore wait addr   : 0x%016llx\n",
504 		   ring->last_semaphore_wait_addr);
505 	seq_printf(m, "%u free dwords in ring\n", ring->ring_free_dw);
506 	seq_printf(m, "%u dwords in ring\n", count);
507 
508 	if (!ring->ready)
509 		return 0;
510 
511 	/* print 8 dw before current rptr as often it's the last executed
512 	 * packet that is the root issue
513 	 */
514 	i = (rptr + ring->ptr_mask + 1 - 32) & ring->ptr_mask;
515 	for (j = 0; j <= (count + 32); j++) {
516 		seq_printf(m, "r[%5d]=0x%08x", i, ring->ring[i]);
517 		if (rptr == i)
518 			seq_puts(m, " *");
519 		if (rptr_next == i)
520 			seq_puts(m, " #");
521 		seq_puts(m, "\n");
522 		i = (i + 1) & ring->ptr_mask;
523 	}
524 	return 0;
525 }
526 
527 /* TODO: clean this up !*/
528 static int amdgpu_gfx_index = offsetof(struct amdgpu_device, gfx.gfx_ring[0]);
529 static int cayman_cp1_index = offsetof(struct amdgpu_device, gfx.compute_ring[0]);
530 static int cayman_cp2_index = offsetof(struct amdgpu_device, gfx.compute_ring[1]);
531 static int amdgpu_dma1_index = offsetof(struct amdgpu_device, sdma.instance[0].ring);
532 static int amdgpu_dma2_index = offsetof(struct amdgpu_device, sdma.instance[1].ring);
533 static int r600_uvd_index = offsetof(struct amdgpu_device, uvd.ring);
534 static int si_vce1_index = offsetof(struct amdgpu_device, vce.ring[0]);
535 static int si_vce2_index = offsetof(struct amdgpu_device, vce.ring[1]);
536 
537 static struct drm_info_list amdgpu_debugfs_ring_info_list[] = {
538 	{"amdgpu_ring_gfx", amdgpu_debugfs_ring_info, 0, &amdgpu_gfx_index},
539 	{"amdgpu_ring_cp1", amdgpu_debugfs_ring_info, 0, &cayman_cp1_index},
540 	{"amdgpu_ring_cp2", amdgpu_debugfs_ring_info, 0, &cayman_cp2_index},
541 	{"amdgpu_ring_dma1", amdgpu_debugfs_ring_info, 0, &amdgpu_dma1_index},
542 	{"amdgpu_ring_dma2", amdgpu_debugfs_ring_info, 0, &amdgpu_dma2_index},
543 	{"amdgpu_ring_uvd", amdgpu_debugfs_ring_info, 0, &r600_uvd_index},
544 	{"amdgpu_ring_vce1", amdgpu_debugfs_ring_info, 0, &si_vce1_index},
545 	{"amdgpu_ring_vce2", amdgpu_debugfs_ring_info, 0, &si_vce2_index},
546 };
547 
548 #endif
549 
550 static int amdgpu_debugfs_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring)
551 {
552 #if defined(CONFIG_DEBUG_FS)
553 	unsigned i;
554 	for (i = 0; i < ARRAY_SIZE(amdgpu_debugfs_ring_info_list); ++i) {
555 		struct drm_info_list *info = &amdgpu_debugfs_ring_info_list[i];
556 		int roffset = *(int*)amdgpu_debugfs_ring_info_list[i].data;
557 		struct amdgpu_ring *other = (void *)(((uint8_t*)adev) + roffset);
558 		unsigned r;
559 
560 		if (other != ring)
561 			continue;
562 
563 		r = amdgpu_debugfs_add_files(adev, info, 1);
564 		if (r)
565 			return r;
566 	}
567 #endif
568 	return 0;
569 }
570