1 /* $NetBSD: amdgpu_ring.c,v 1.3 2018/08/27 14:04:50 riastradh Exp $ */ 2 3 /* 4 * Copyright 2008 Advanced Micro Devices, Inc. 5 * Copyright 2008 Red Hat Inc. 6 * Copyright 2009 Jerome Glisse. 7 * 8 * Permission is hereby granted, free of charge, to any person obtaining a 9 * copy of this software and associated documentation files (the "Software"), 10 * to deal in the Software without restriction, including without limitation 11 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 12 * and/or sell copies of the Software, and to permit persons to whom the 13 * Software is furnished to do so, subject to the following conditions: 14 * 15 * The above copyright notice and this permission notice shall be included in 16 * all copies or substantial portions of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 22 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 23 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 24 * OTHER DEALINGS IN THE SOFTWARE. 25 * 26 * Authors: Dave Airlie 27 * Alex Deucher 28 * Jerome Glisse 29 * Christian König 30 */ 31 #include <sys/cdefs.h> 32 __KERNEL_RCSID(0, "$NetBSD: amdgpu_ring.c,v 1.3 2018/08/27 14:04:50 riastradh Exp $"); 33 34 #include <asm/byteorder.h> 35 #include <linux/log2.h> 36 #include <linux/seq_file.h> 37 #include <linux/slab.h> 38 #include <drm/drmP.h> 39 #include <drm/amdgpu_drm.h> 40 #include "amdgpu.h" 41 #include "atom.h" 42 43 /* 44 * Rings 45 * Most engines on the GPU are fed via ring buffers. Ring 46 * buffers are areas of GPU accessible memory that the host 47 * writes commands into and the GPU reads commands out of. 48 * There is a rptr (read pointer) that determines where the 49 * GPU is currently reading, and a wptr (write pointer) 50 * which determines where the host has written. When the 51 * pointers are equal, the ring is idle. When the host 52 * writes commands to the ring buffer, it increments the 53 * wptr. The GPU then starts fetching commands and executes 54 * them until the pointers are equal again. 55 */ 56 static int amdgpu_debugfs_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring); 57 58 /** 59 * amdgpu_ring_free_size - update the free size 60 * 61 * @adev: amdgpu_device pointer 62 * @ring: amdgpu_ring structure holding ring information 63 * 64 * Update the free dw slots in the ring buffer (all asics). 65 */ 66 void amdgpu_ring_free_size(struct amdgpu_ring *ring) 67 { 68 uint32_t rptr = amdgpu_ring_get_rptr(ring); 69 70 /* This works because ring_size is a power of 2 */ 71 ring->ring_free_dw = rptr + (ring->ring_size / 4); 72 ring->ring_free_dw -= ring->wptr; 73 ring->ring_free_dw &= ring->ptr_mask; 74 if (!ring->ring_free_dw) { 75 /* this is an empty ring */ 76 ring->ring_free_dw = ring->ring_size / 4; 77 } 78 } 79 80 /** 81 * amdgpu_ring_alloc - allocate space on the ring buffer 82 * 83 * @adev: amdgpu_device pointer 84 * @ring: amdgpu_ring structure holding ring information 85 * @ndw: number of dwords to allocate in the ring buffer 86 * 87 * Allocate @ndw dwords in the ring buffer (all asics). 88 * Returns 0 on success, error on failure. 89 */ 90 int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned ndw) 91 { 92 int r; 93 94 /* make sure we aren't trying to allocate more space than there is on the ring */ 95 if (ndw > (ring->ring_size / 4)) 96 return -ENOMEM; 97 /* Align requested size with padding so unlock_commit can 98 * pad safely */ 99 amdgpu_ring_free_size(ring); 100 ndw = (ndw + ring->align_mask) & ~ring->align_mask; 101 while (ndw > (ring->ring_free_dw - 1)) { 102 amdgpu_ring_free_size(ring); 103 if (ndw < ring->ring_free_dw) { 104 break; 105 } 106 r = amdgpu_fence_wait_next(ring); 107 if (r) 108 return r; 109 } 110 ring->count_dw = ndw; 111 ring->wptr_old = ring->wptr; 112 return 0; 113 } 114 115 /** 116 * amdgpu_ring_lock - lock the ring and allocate space on it 117 * 118 * @adev: amdgpu_device pointer 119 * @ring: amdgpu_ring structure holding ring information 120 * @ndw: number of dwords to allocate in the ring buffer 121 * 122 * Lock the ring and allocate @ndw dwords in the ring buffer 123 * (all asics). 124 * Returns 0 on success, error on failure. 125 */ 126 int amdgpu_ring_lock(struct amdgpu_ring *ring, unsigned ndw) 127 { 128 int r; 129 130 mutex_lock(ring->ring_lock); 131 r = amdgpu_ring_alloc(ring, ndw); 132 if (r) { 133 mutex_unlock(ring->ring_lock); 134 return r; 135 } 136 return 0; 137 } 138 139 /** amdgpu_ring_insert_nop - insert NOP packets 140 * 141 * @ring: amdgpu_ring structure holding ring information 142 * @count: the number of NOP packets to insert 143 * 144 * This is the generic insert_nop function for rings except SDMA 145 */ 146 void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) 147 { 148 int i; 149 150 for (i = 0; i < count; i++) 151 amdgpu_ring_write(ring, ring->nop); 152 } 153 154 /** 155 * amdgpu_ring_commit - tell the GPU to execute the new 156 * commands on the ring buffer 157 * 158 * @adev: amdgpu_device pointer 159 * @ring: amdgpu_ring structure holding ring information 160 * 161 * Update the wptr (write pointer) to tell the GPU to 162 * execute new commands on the ring buffer (all asics). 163 */ 164 void amdgpu_ring_commit(struct amdgpu_ring *ring) 165 { 166 uint32_t count; 167 168 /* We pad to match fetch size */ 169 count = ring->align_mask + 1 - (ring->wptr & ring->align_mask); 170 count %= ring->align_mask + 1; 171 ring->funcs->insert_nop(ring, count); 172 173 mb(); 174 amdgpu_ring_set_wptr(ring); 175 } 176 177 /** 178 * amdgpu_ring_unlock_commit - tell the GPU to execute the new 179 * commands on the ring buffer and unlock it 180 * 181 * @ring: amdgpu_ring structure holding ring information 182 * 183 * Call amdgpu_ring_commit() then unlock the ring (all asics). 184 */ 185 void amdgpu_ring_unlock_commit(struct amdgpu_ring *ring) 186 { 187 amdgpu_ring_commit(ring); 188 mutex_unlock(ring->ring_lock); 189 } 190 191 /** 192 * amdgpu_ring_undo - reset the wptr 193 * 194 * @ring: amdgpu_ring structure holding ring information 195 * 196 * Reset the driver's copy of the wptr (all asics). 197 */ 198 void amdgpu_ring_undo(struct amdgpu_ring *ring) 199 { 200 ring->wptr = ring->wptr_old; 201 } 202 203 /** 204 * amdgpu_ring_unlock_undo - reset the wptr and unlock the ring 205 * 206 * @ring: amdgpu_ring structure holding ring information 207 * 208 * Call amdgpu_ring_undo() then unlock the ring (all asics). 209 */ 210 void amdgpu_ring_unlock_undo(struct amdgpu_ring *ring) 211 { 212 amdgpu_ring_undo(ring); 213 mutex_unlock(ring->ring_lock); 214 } 215 216 /** 217 * amdgpu_ring_backup - Back up the content of a ring 218 * 219 * @ring: the ring we want to back up 220 * 221 * Saves all unprocessed commits from a ring, returns the number of dwords saved. 222 */ 223 unsigned amdgpu_ring_backup(struct amdgpu_ring *ring, 224 uint32_t **data) 225 { 226 unsigned size, ptr, i; 227 228 /* just in case lock the ring */ 229 mutex_lock(ring->ring_lock); 230 *data = NULL; 231 232 if (ring->ring_obj == NULL) { 233 mutex_unlock(ring->ring_lock); 234 return 0; 235 } 236 237 /* it doesn't make sense to save anything if all fences are signaled */ 238 if (!amdgpu_fence_count_emitted(ring)) { 239 mutex_unlock(ring->ring_lock); 240 return 0; 241 } 242 243 ptr = le32_to_cpu(*ring->next_rptr_cpu_addr); 244 245 size = ring->wptr + (ring->ring_size / 4); 246 size -= ptr; 247 size &= ring->ptr_mask; 248 if (size == 0) { 249 mutex_unlock(ring->ring_lock); 250 return 0; 251 } 252 253 /* and then save the content of the ring */ 254 *data = kmalloc_array(size, sizeof(uint32_t), GFP_KERNEL); 255 if (!*data) { 256 mutex_unlock(ring->ring_lock); 257 return 0; 258 } 259 for (i = 0; i < size; ++i) { 260 (*data)[i] = ring->ring[ptr++]; 261 ptr &= ring->ptr_mask; 262 } 263 264 mutex_unlock(ring->ring_lock); 265 return size; 266 } 267 268 /** 269 * amdgpu_ring_restore - append saved commands to the ring again 270 * 271 * @ring: ring to append commands to 272 * @size: number of dwords we want to write 273 * @data: saved commands 274 * 275 * Allocates space on the ring and restore the previously saved commands. 276 */ 277 int amdgpu_ring_restore(struct amdgpu_ring *ring, 278 unsigned size, uint32_t *data) 279 { 280 int i, r; 281 282 if (!size || !data) 283 return 0; 284 285 /* restore the saved ring content */ 286 r = amdgpu_ring_lock(ring, size); 287 if (r) 288 return r; 289 290 for (i = 0; i < size; ++i) { 291 amdgpu_ring_write(ring, data[i]); 292 } 293 294 amdgpu_ring_unlock_commit(ring); 295 kfree(data); 296 return 0; 297 } 298 299 /** 300 * amdgpu_ring_init - init driver ring struct. 301 * 302 * @adev: amdgpu_device pointer 303 * @ring: amdgpu_ring structure holding ring information 304 * @ring_size: size of the ring 305 * @nop: nop packet for this ring 306 * 307 * Initialize the driver information for the selected ring (all asics). 308 * Returns 0 on success, error on failure. 309 */ 310 int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring, 311 unsigned ring_size, u32 nop, u32 align_mask, 312 struct amdgpu_irq_src *irq_src, unsigned irq_type, 313 enum amdgpu_ring_type ring_type) 314 { 315 u32 rb_bufsz; 316 int r; 317 318 if (ring->adev == NULL) { 319 if (adev->num_rings >= AMDGPU_MAX_RINGS) 320 return -EINVAL; 321 322 ring->adev = adev; 323 ring->idx = adev->num_rings++; 324 adev->rings[ring->idx] = ring; 325 r = amdgpu_fence_driver_init_ring(ring); 326 if (r) 327 return r; 328 } 329 330 r = amdgpu_wb_get(adev, &ring->rptr_offs); 331 if (r) { 332 dev_err(adev->dev, "(%d) ring rptr_offs wb alloc failed\n", r); 333 return r; 334 } 335 336 r = amdgpu_wb_get(adev, &ring->wptr_offs); 337 if (r) { 338 dev_err(adev->dev, "(%d) ring wptr_offs wb alloc failed\n", r); 339 return r; 340 } 341 342 r = amdgpu_wb_get(adev, &ring->fence_offs); 343 if (r) { 344 dev_err(adev->dev, "(%d) ring fence_offs wb alloc failed\n", r); 345 return r; 346 } 347 348 r = amdgpu_wb_get(adev, &ring->next_rptr_offs); 349 if (r) { 350 dev_err(adev->dev, "(%d) ring next_rptr wb alloc failed\n", r); 351 return r; 352 } 353 ring->next_rptr_gpu_addr = adev->wb.gpu_addr + (ring->next_rptr_offs * 4); 354 ring->next_rptr_cpu_addr = &adev->wb.wb[ring->next_rptr_offs]; 355 spin_lock_init(&ring->fence_lock); 356 r = amdgpu_fence_driver_start_ring(ring, irq_src, irq_type); 357 if (r) { 358 dev_err(adev->dev, "failed initializing fences (%d).\n", r); 359 return r; 360 } 361 362 ring->ring_lock = &adev->ring_lock; 363 /* Align ring size */ 364 rb_bufsz = order_base_2(ring_size / 8); 365 ring_size = (1 << (rb_bufsz + 1)) * 4; 366 ring->ring_size = ring_size; 367 ring->align_mask = align_mask; 368 ring->nop = nop; 369 ring->type = ring_type; 370 371 /* Allocate ring buffer */ 372 if (ring->ring_obj == NULL) { 373 r = amdgpu_bo_create(adev, ring->ring_size, PAGE_SIZE, true, 374 AMDGPU_GEM_DOMAIN_GTT, 0, 375 NULL, NULL, &ring->ring_obj); 376 if (r) { 377 dev_err(adev->dev, "(%d) ring create failed\n", r); 378 return r; 379 } 380 r = amdgpu_bo_reserve(ring->ring_obj, false); 381 if (unlikely(r != 0)) 382 return r; 383 r = amdgpu_bo_pin(ring->ring_obj, AMDGPU_GEM_DOMAIN_GTT, 384 &ring->gpu_addr); 385 if (r) { 386 amdgpu_bo_unreserve(ring->ring_obj); 387 dev_err(adev->dev, "(%d) ring pin failed\n", r); 388 return r; 389 } 390 r = amdgpu_bo_kmap(ring->ring_obj, 391 (void **)__UNVOLATILE(&ring->ring)); 392 amdgpu_bo_unreserve(ring->ring_obj); 393 if (r) { 394 dev_err(adev->dev, "(%d) ring map failed\n", r); 395 return r; 396 } 397 } 398 ring->ptr_mask = (ring->ring_size / 4) - 1; 399 ring->ring_free_dw = ring->ring_size / 4; 400 401 if (amdgpu_debugfs_ring_init(adev, ring)) { 402 DRM_ERROR("Failed to register debugfs file for rings !\n"); 403 } 404 return 0; 405 } 406 407 /** 408 * amdgpu_ring_fini - tear down the driver ring struct. 409 * 410 * @adev: amdgpu_device pointer 411 * @ring: amdgpu_ring structure holding ring information 412 * 413 * Tear down the driver information for the selected ring (all asics). 414 */ 415 void amdgpu_ring_fini(struct amdgpu_ring *ring) 416 { 417 int r; 418 struct amdgpu_bo *ring_obj; 419 420 if (ring->ring_lock == NULL) 421 return; 422 423 mutex_lock(ring->ring_lock); 424 ring_obj = ring->ring_obj; 425 ring->ready = false; 426 ring->ring = NULL; 427 ring->ring_obj = NULL; 428 mutex_unlock(ring->ring_lock); 429 430 amdgpu_wb_free(ring->adev, ring->fence_offs); 431 amdgpu_wb_free(ring->adev, ring->rptr_offs); 432 amdgpu_wb_free(ring->adev, ring->wptr_offs); 433 amdgpu_wb_free(ring->adev, ring->next_rptr_offs); 434 435 if (ring_obj) { 436 r = amdgpu_bo_reserve(ring_obj, false); 437 if (likely(r == 0)) { 438 amdgpu_bo_kunmap(ring_obj); 439 amdgpu_bo_unpin(ring_obj); 440 amdgpu_bo_unreserve(ring_obj); 441 } 442 amdgpu_bo_unref(&ring_obj); 443 } 444 } 445 446 /** 447 * amdgpu_ring_from_fence - get ring from fence 448 * 449 * @f: fence structure 450 * 451 * Extract the ring a fence belongs to. Handles both scheduler as 452 * well as hardware fences. 453 */ 454 struct amdgpu_ring *amdgpu_ring_from_fence(struct fence *f) 455 { 456 struct amdgpu_fence *a_fence; 457 struct amd_sched_fence *s_fence; 458 459 s_fence = to_amd_sched_fence(f); 460 if (s_fence) 461 return container_of(s_fence->sched, struct amdgpu_ring, sched); 462 463 a_fence = to_amdgpu_fence(f); 464 if (a_fence) 465 return a_fence->ring; 466 467 return NULL; 468 } 469 470 /* 471 * Debugfs info 472 */ 473 #if defined(CONFIG_DEBUG_FS) 474 475 static int amdgpu_debugfs_ring_info(struct seq_file *m, void *data) 476 { 477 struct drm_info_node *node = (struct drm_info_node *) m->private; 478 struct drm_device *dev = node->minor->dev; 479 struct amdgpu_device *adev = dev->dev_private; 480 int roffset = *(int*)node->info_ent->data; 481 struct amdgpu_ring *ring = (void *)(((uint8_t*)adev) + roffset); 482 483 uint32_t rptr, wptr, rptr_next; 484 unsigned count, i, j; 485 486 amdgpu_ring_free_size(ring); 487 count = (ring->ring_size / 4) - ring->ring_free_dw; 488 489 wptr = amdgpu_ring_get_wptr(ring); 490 seq_printf(m, "wptr: 0x%08x [%5d]\n", 491 wptr, wptr); 492 493 rptr = amdgpu_ring_get_rptr(ring); 494 seq_printf(m, "rptr: 0x%08x [%5d]\n", 495 rptr, rptr); 496 497 rptr_next = ~0; 498 499 seq_printf(m, "driver's copy of the wptr: 0x%08x [%5d]\n", 500 ring->wptr, ring->wptr); 501 seq_printf(m, "last semaphore signal addr : 0x%016llx\n", 502 ring->last_semaphore_signal_addr); 503 seq_printf(m, "last semaphore wait addr : 0x%016llx\n", 504 ring->last_semaphore_wait_addr); 505 seq_printf(m, "%u free dwords in ring\n", ring->ring_free_dw); 506 seq_printf(m, "%u dwords in ring\n", count); 507 508 if (!ring->ready) 509 return 0; 510 511 /* print 8 dw before current rptr as often it's the last executed 512 * packet that is the root issue 513 */ 514 i = (rptr + ring->ptr_mask + 1 - 32) & ring->ptr_mask; 515 for (j = 0; j <= (count + 32); j++) { 516 seq_printf(m, "r[%5d]=0x%08x", i, ring->ring[i]); 517 if (rptr == i) 518 seq_puts(m, " *"); 519 if (rptr_next == i) 520 seq_puts(m, " #"); 521 seq_puts(m, "\n"); 522 i = (i + 1) & ring->ptr_mask; 523 } 524 return 0; 525 } 526 527 /* TODO: clean this up !*/ 528 static int amdgpu_gfx_index = offsetof(struct amdgpu_device, gfx.gfx_ring[0]); 529 static int cayman_cp1_index = offsetof(struct amdgpu_device, gfx.compute_ring[0]); 530 static int cayman_cp2_index = offsetof(struct amdgpu_device, gfx.compute_ring[1]); 531 static int amdgpu_dma1_index = offsetof(struct amdgpu_device, sdma.instance[0].ring); 532 static int amdgpu_dma2_index = offsetof(struct amdgpu_device, sdma.instance[1].ring); 533 static int r600_uvd_index = offsetof(struct amdgpu_device, uvd.ring); 534 static int si_vce1_index = offsetof(struct amdgpu_device, vce.ring[0]); 535 static int si_vce2_index = offsetof(struct amdgpu_device, vce.ring[1]); 536 537 static struct drm_info_list amdgpu_debugfs_ring_info_list[] = { 538 {"amdgpu_ring_gfx", amdgpu_debugfs_ring_info, 0, &amdgpu_gfx_index}, 539 {"amdgpu_ring_cp1", amdgpu_debugfs_ring_info, 0, &cayman_cp1_index}, 540 {"amdgpu_ring_cp2", amdgpu_debugfs_ring_info, 0, &cayman_cp2_index}, 541 {"amdgpu_ring_dma1", amdgpu_debugfs_ring_info, 0, &amdgpu_dma1_index}, 542 {"amdgpu_ring_dma2", amdgpu_debugfs_ring_info, 0, &amdgpu_dma2_index}, 543 {"amdgpu_ring_uvd", amdgpu_debugfs_ring_info, 0, &r600_uvd_index}, 544 {"amdgpu_ring_vce1", amdgpu_debugfs_ring_info, 0, &si_vce1_index}, 545 {"amdgpu_ring_vce2", amdgpu_debugfs_ring_info, 0, &si_vce2_index}, 546 }; 547 548 #endif 549 550 static int amdgpu_debugfs_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring) 551 { 552 #if defined(CONFIG_DEBUG_FS) 553 unsigned i; 554 for (i = 0; i < ARRAY_SIZE(amdgpu_debugfs_ring_info_list); ++i) { 555 struct drm_info_list *info = &amdgpu_debugfs_ring_info_list[i]; 556 int roffset = *(int*)amdgpu_debugfs_ring_info_list[i].data; 557 struct amdgpu_ring *other = (void *)(((uint8_t*)adev) + roffset); 558 unsigned r; 559 560 if (other != ring) 561 continue; 562 563 r = amdgpu_debugfs_add_files(adev, info, 1); 564 if (r) 565 return r; 566 } 567 #endif 568 return 0; 569 } 570