1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 */ 25 26 #include "amdgpu.h" 27 #include "amdgpu_gfx.h" 28 #include "amdgpu_rlc.h" 29 #include "amdgpu_ras.h" 30 31 /* delay 0.1 second to enable gfx off feature */ 32 #define GFX_OFF_DELAY_ENABLE msecs_to_jiffies(100) 33 34 /* 35 * GPU GFX IP block helpers function. 36 */ 37 38 int amdgpu_gfx_mec_queue_to_bit(struct amdgpu_device *adev, int mec, 39 int pipe, int queue) 40 { 41 int bit = 0; 42 43 bit += mec * adev->gfx.mec.num_pipe_per_mec 44 * adev->gfx.mec.num_queue_per_pipe; 45 bit += pipe * adev->gfx.mec.num_queue_per_pipe; 46 bit += queue; 47 48 return bit; 49 } 50 51 void amdgpu_queue_mask_bit_to_mec_queue(struct amdgpu_device *adev, int bit, 52 int *mec, int *pipe, int *queue) 53 { 54 *queue = bit % adev->gfx.mec.num_queue_per_pipe; 55 *pipe = (bit / adev->gfx.mec.num_queue_per_pipe) 56 % adev->gfx.mec.num_pipe_per_mec; 57 *mec = (bit / adev->gfx.mec.num_queue_per_pipe) 58 / adev->gfx.mec.num_pipe_per_mec; 59 60 } 61 62 bool amdgpu_gfx_is_mec_queue_enabled(struct amdgpu_device *adev, 63 int mec, int pipe, int queue) 64 { 65 return test_bit(amdgpu_gfx_mec_queue_to_bit(adev, mec, pipe, queue), 66 adev->gfx.mec.queue_bitmap); 67 } 68 69 int amdgpu_gfx_me_queue_to_bit(struct amdgpu_device *adev, 70 int me, int pipe, int queue) 71 { 72 int bit = 0; 73 74 bit += me * adev->gfx.me.num_pipe_per_me 75 * adev->gfx.me.num_queue_per_pipe; 76 bit += pipe * adev->gfx.me.num_queue_per_pipe; 77 bit += queue; 78 79 return bit; 80 } 81 82 void amdgpu_gfx_bit_to_me_queue(struct amdgpu_device *adev, int bit, 83 int *me, int *pipe, int *queue) 84 { 85 *queue = bit % adev->gfx.me.num_queue_per_pipe; 86 *pipe = (bit / adev->gfx.me.num_queue_per_pipe) 87 % adev->gfx.me.num_pipe_per_me; 88 *me = (bit / adev->gfx.me.num_queue_per_pipe) 89 / adev->gfx.me.num_pipe_per_me; 90 } 91 92 bool amdgpu_gfx_is_me_queue_enabled(struct amdgpu_device *adev, 93 int me, int pipe, int queue) 94 { 95 return test_bit(amdgpu_gfx_me_queue_to_bit(adev, me, pipe, queue), 96 adev->gfx.me.queue_bitmap); 97 } 98 99 /** 100 * amdgpu_gfx_scratch_get - Allocate a scratch register 101 * 102 * @adev: amdgpu_device pointer 103 * @reg: scratch register mmio offset 104 * 105 * Allocate a CP scratch register for use by the driver (all asics). 106 * Returns 0 on success or -EINVAL on failure. 107 */ 108 int amdgpu_gfx_scratch_get(struct amdgpu_device *adev, uint32_t *reg) 109 { 110 int i; 111 112 i = ffs(adev->gfx.scratch.free_mask); 113 if (i != 0 && i <= adev->gfx.scratch.num_reg) { 114 i--; 115 adev->gfx.scratch.free_mask &= ~(1u << i); 116 *reg = adev->gfx.scratch.reg_base + i; 117 return 0; 118 } 119 return -EINVAL; 120 } 121 122 /** 123 * amdgpu_gfx_scratch_free - Free a scratch register 124 * 125 * @adev: amdgpu_device pointer 126 * @reg: scratch register mmio offset 127 * 128 * Free a CP scratch register allocated for use by the driver (all asics) 129 */ 130 void amdgpu_gfx_scratch_free(struct amdgpu_device *adev, uint32_t reg) 131 { 132 adev->gfx.scratch.free_mask |= 1u << (reg - adev->gfx.scratch.reg_base); 133 } 134 135 /** 136 * amdgpu_gfx_parse_disable_cu - Parse the disable_cu module parameter 137 * 138 * @mask: array in which the per-shader array disable masks will be stored 139 * @max_se: number of SEs 140 * @max_sh: number of SHs 141 * 142 * The bitmask of CUs to be disabled in the shader array determined by se and 143 * sh is stored in mask[se * max_sh + sh]. 144 */ 145 void amdgpu_gfx_parse_disable_cu(unsigned *mask, unsigned max_se, unsigned max_sh) 146 { 147 unsigned se, sh, cu; 148 const char *p; 149 150 memset(mask, 0, sizeof(*mask) * max_se * max_sh); 151 152 if (!amdgpu_disable_cu || !*amdgpu_disable_cu) 153 return; 154 155 #ifdef notyet 156 p = amdgpu_disable_cu; 157 for (;;) { 158 char *next; 159 int ret = sscanf(p, "%u.%u.%u", &se, &sh, &cu); 160 if (ret < 3) { 161 DRM_ERROR("amdgpu: could not parse disable_cu\n"); 162 return; 163 } 164 165 if (se < max_se && sh < max_sh && cu < 16) { 166 DRM_INFO("amdgpu: disabling CU %u.%u.%u\n", se, sh, cu); 167 mask[se * max_sh + sh] |= 1u << cu; 168 } else { 169 DRM_ERROR("amdgpu: disable_cu %u.%u.%u is out of range\n", 170 se, sh, cu); 171 } 172 173 next = strchr(p, ','); 174 if (!next) 175 break; 176 p = next + 1; 177 } 178 #endif 179 } 180 181 static bool amdgpu_gfx_is_multipipe_capable(struct amdgpu_device *adev) 182 { 183 if (amdgpu_compute_multipipe != -1) { 184 DRM_INFO("amdgpu: forcing compute pipe policy %d\n", 185 amdgpu_compute_multipipe); 186 return amdgpu_compute_multipipe == 1; 187 } 188 189 /* FIXME: spreading the queues across pipes causes perf regressions 190 * on POLARIS11 compute workloads */ 191 if (adev->asic_type == CHIP_POLARIS11) 192 return false; 193 194 return adev->gfx.mec.num_mec > 1; 195 } 196 197 bool amdgpu_gfx_is_high_priority_compute_queue(struct amdgpu_device *adev, 198 int pipe, int queue) 199 { 200 bool multipipe_policy = amdgpu_gfx_is_multipipe_capable(adev); 201 int cond; 202 /* Policy: alternate between normal and high priority */ 203 cond = multipipe_policy ? pipe : queue; 204 205 return ((cond % 2) != 0); 206 207 } 208 209 void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev) 210 { 211 int i, queue, pipe; 212 bool multipipe_policy = amdgpu_gfx_is_multipipe_capable(adev); 213 int max_queues_per_mec = min(adev->gfx.mec.num_pipe_per_mec * 214 adev->gfx.mec.num_queue_per_pipe, 215 adev->gfx.num_compute_rings); 216 217 if (multipipe_policy) { 218 /* policy: make queues evenly cross all pipes on MEC1 only */ 219 for (i = 0; i < max_queues_per_mec; i++) { 220 pipe = i % adev->gfx.mec.num_pipe_per_mec; 221 queue = (i / adev->gfx.mec.num_pipe_per_mec) % 222 adev->gfx.mec.num_queue_per_pipe; 223 224 set_bit(pipe * adev->gfx.mec.num_queue_per_pipe + queue, 225 adev->gfx.mec.queue_bitmap); 226 } 227 } else { 228 /* policy: amdgpu owns all queues in the given pipe */ 229 for (i = 0; i < max_queues_per_mec; ++i) 230 set_bit(i, adev->gfx.mec.queue_bitmap); 231 } 232 233 dev_dbg(adev->dev, "mec queue bitmap weight=%d\n", bitmap_weight(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES)); 234 } 235 236 void amdgpu_gfx_graphics_queue_acquire(struct amdgpu_device *adev) 237 { 238 int i, queue, me; 239 240 for (i = 0; i < AMDGPU_MAX_GFX_QUEUES; ++i) { 241 queue = i % adev->gfx.me.num_queue_per_pipe; 242 me = (i / adev->gfx.me.num_queue_per_pipe) 243 / adev->gfx.me.num_pipe_per_me; 244 245 if (me >= adev->gfx.me.num_me) 246 break; 247 /* policy: amdgpu owns the first queue per pipe at this stage 248 * will extend to mulitple queues per pipe later */ 249 if (me == 0 && queue < 1) 250 set_bit(i, adev->gfx.me.queue_bitmap); 251 } 252 253 /* update the number of active graphics rings */ 254 adev->gfx.num_gfx_rings = 255 bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES); 256 } 257 258 static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev, 259 struct amdgpu_ring *ring) 260 { 261 int queue_bit; 262 int mec, pipe, queue; 263 264 queue_bit = adev->gfx.mec.num_mec 265 * adev->gfx.mec.num_pipe_per_mec 266 * adev->gfx.mec.num_queue_per_pipe; 267 268 while (queue_bit-- >= 0) { 269 if (test_bit(queue_bit, adev->gfx.mec.queue_bitmap)) 270 continue; 271 272 amdgpu_queue_mask_bit_to_mec_queue(adev, queue_bit, &mec, &pipe, &queue); 273 274 /* 275 * 1. Using pipes 2/3 from MEC 2 seems cause problems. 276 * 2. It must use queue id 0, because CGPG_IDLE/SAVE/LOAD/RUN 277 * only can be issued on queue 0. 278 */ 279 if ((mec == 1 && pipe > 1) || queue != 0) 280 continue; 281 282 ring->me = mec + 1; 283 ring->pipe = pipe; 284 ring->queue = queue; 285 286 return 0; 287 } 288 289 dev_err(adev->dev, "Failed to find a queue for KIQ\n"); 290 return -EINVAL; 291 } 292 293 int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev, 294 struct amdgpu_ring *ring, 295 struct amdgpu_irq_src *irq) 296 { 297 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 298 int r = 0; 299 300 mtx_init(&kiq->ring_lock, IPL_TTY); 301 302 ring->adev = NULL; 303 ring->ring_obj = NULL; 304 ring->use_doorbell = true; 305 ring->doorbell_index = adev->doorbell_index.kiq; 306 307 r = amdgpu_gfx_kiq_acquire(adev, ring); 308 if (r) 309 return r; 310 311 ring->eop_gpu_addr = kiq->eop_gpu_addr; 312 ring->no_scheduler = true; 313 snprintf(ring->name, sizeof(ring->name), "kiq_%d.%d.%d", ring->me, ring->pipe, ring->queue); 314 r = amdgpu_ring_init(adev, ring, 1024, 315 irq, AMDGPU_CP_KIQ_IRQ_DRIVER0, 316 AMDGPU_RING_PRIO_DEFAULT); 317 if (r) 318 dev_warn(adev->dev, "(%d) failed to init kiq ring\n", r); 319 320 return r; 321 } 322 323 void amdgpu_gfx_kiq_free_ring(struct amdgpu_ring *ring) 324 { 325 amdgpu_ring_fini(ring); 326 } 327 328 void amdgpu_gfx_kiq_fini(struct amdgpu_device *adev) 329 { 330 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 331 332 amdgpu_bo_free_kernel(&kiq->eop_obj, &kiq->eop_gpu_addr, NULL); 333 } 334 335 int amdgpu_gfx_kiq_init(struct amdgpu_device *adev, 336 unsigned hpd_size) 337 { 338 int r; 339 u32 *hpd; 340 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 341 342 r = amdgpu_bo_create_kernel(adev, hpd_size, PAGE_SIZE, 343 AMDGPU_GEM_DOMAIN_GTT, &kiq->eop_obj, 344 &kiq->eop_gpu_addr, (void **)&hpd); 345 if (r) { 346 dev_warn(adev->dev, "failed to create KIQ bo (%d).\n", r); 347 return r; 348 } 349 350 memset(hpd, 0, hpd_size); 351 352 r = amdgpu_bo_reserve(kiq->eop_obj, true); 353 if (unlikely(r != 0)) 354 dev_warn(adev->dev, "(%d) reserve kiq eop bo failed\n", r); 355 amdgpu_bo_kunmap(kiq->eop_obj); 356 amdgpu_bo_unreserve(kiq->eop_obj); 357 358 return 0; 359 } 360 361 /* create MQD for each compute/gfx queue */ 362 int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev, 363 unsigned mqd_size) 364 { 365 struct amdgpu_ring *ring = NULL; 366 int r, i; 367 368 /* create MQD for KIQ */ 369 ring = &adev->gfx.kiq.ring; 370 if (!ring->mqd_obj) { 371 /* originaly the KIQ MQD is put in GTT domain, but for SRIOV VRAM domain is a must 372 * otherwise hypervisor trigger SAVE_VF fail after driver unloaded which mean MQD 373 * deallocated and gart_unbind, to strict diverage we decide to use VRAM domain for 374 * KIQ MQD no matter SRIOV or Bare-metal 375 */ 376 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE, 377 AMDGPU_GEM_DOMAIN_VRAM, &ring->mqd_obj, 378 &ring->mqd_gpu_addr, &ring->mqd_ptr); 379 if (r) { 380 dev_warn(adev->dev, "failed to create ring mqd ob (%d)", r); 381 return r; 382 } 383 384 /* prepare MQD backup */ 385 adev->gfx.mec.mqd_backup[AMDGPU_MAX_COMPUTE_RINGS] = kmalloc(mqd_size, GFP_KERNEL); 386 if (!adev->gfx.mec.mqd_backup[AMDGPU_MAX_COMPUTE_RINGS]) 387 dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name); 388 } 389 390 if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) { 391 /* create MQD for each KGQ */ 392 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 393 ring = &adev->gfx.gfx_ring[i]; 394 if (!ring->mqd_obj) { 395 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE, 396 AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj, 397 &ring->mqd_gpu_addr, &ring->mqd_ptr); 398 if (r) { 399 dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r); 400 return r; 401 } 402 403 /* prepare MQD backup */ 404 adev->gfx.me.mqd_backup[i] = kmalloc(mqd_size, GFP_KERNEL); 405 if (!adev->gfx.me.mqd_backup[i]) 406 dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name); 407 } 408 } 409 } 410 411 /* create MQD for each KCQ */ 412 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 413 ring = &adev->gfx.compute_ring[i]; 414 if (!ring->mqd_obj) { 415 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE, 416 AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj, 417 &ring->mqd_gpu_addr, &ring->mqd_ptr); 418 if (r) { 419 dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r); 420 return r; 421 } 422 423 /* prepare MQD backup */ 424 adev->gfx.mec.mqd_backup[i] = kmalloc(mqd_size, GFP_KERNEL); 425 if (!adev->gfx.mec.mqd_backup[i]) 426 dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name); 427 } 428 } 429 430 return 0; 431 } 432 433 void amdgpu_gfx_mqd_sw_fini(struct amdgpu_device *adev) 434 { 435 struct amdgpu_ring *ring = NULL; 436 int i; 437 438 if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) { 439 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 440 ring = &adev->gfx.gfx_ring[i]; 441 kfree(adev->gfx.me.mqd_backup[i]); 442 amdgpu_bo_free_kernel(&ring->mqd_obj, 443 &ring->mqd_gpu_addr, 444 &ring->mqd_ptr); 445 } 446 } 447 448 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 449 ring = &adev->gfx.compute_ring[i]; 450 kfree(adev->gfx.mec.mqd_backup[i]); 451 amdgpu_bo_free_kernel(&ring->mqd_obj, 452 &ring->mqd_gpu_addr, 453 &ring->mqd_ptr); 454 } 455 456 ring = &adev->gfx.kiq.ring; 457 kfree(adev->gfx.mec.mqd_backup[AMDGPU_MAX_COMPUTE_RINGS]); 458 amdgpu_bo_free_kernel(&ring->mqd_obj, 459 &ring->mqd_gpu_addr, 460 &ring->mqd_ptr); 461 } 462 463 int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev) 464 { 465 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 466 struct amdgpu_ring *kiq_ring = &kiq->ring; 467 int i; 468 469 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues) 470 return -EINVAL; 471 472 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size * 473 adev->gfx.num_compute_rings)) 474 return -ENOMEM; 475 476 for (i = 0; i < adev->gfx.num_compute_rings; i++) 477 kiq->pmf->kiq_unmap_queues(kiq_ring, &adev->gfx.compute_ring[i], 478 RESET_QUEUES, 0, 0); 479 480 return amdgpu_ring_test_helper(kiq_ring); 481 } 482 483 int amdgpu_queue_mask_bit_to_set_resource_bit(struct amdgpu_device *adev, 484 int queue_bit) 485 { 486 int mec, pipe, queue; 487 int set_resource_bit = 0; 488 489 amdgpu_queue_mask_bit_to_mec_queue(adev, queue_bit, &mec, &pipe, &queue); 490 491 set_resource_bit = mec * 4 * 8 + pipe * 8 + queue; 492 493 return set_resource_bit; 494 } 495 496 int amdgpu_gfx_enable_kcq(struct amdgpu_device *adev) 497 { 498 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 499 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; 500 uint64_t queue_mask = 0; 501 int r, i; 502 503 if (!kiq->pmf || !kiq->pmf->kiq_map_queues || !kiq->pmf->kiq_set_resources) 504 return -EINVAL; 505 506 for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) { 507 if (!test_bit(i, adev->gfx.mec.queue_bitmap)) 508 continue; 509 510 /* This situation may be hit in the future if a new HW 511 * generation exposes more than 64 queues. If so, the 512 * definition of queue_mask needs updating */ 513 if (WARN_ON(i > (sizeof(queue_mask)*8))) { 514 DRM_ERROR("Invalid KCQ enabled: %d\n", i); 515 break; 516 } 517 518 queue_mask |= (1ull << amdgpu_queue_mask_bit_to_set_resource_bit(adev, i)); 519 } 520 521 DRM_INFO("kiq ring mec %d pipe %d q %d\n", kiq_ring->me, kiq_ring->pipe, 522 kiq_ring->queue); 523 524 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size * 525 adev->gfx.num_compute_rings + 526 kiq->pmf->set_resources_size); 527 if (r) { 528 DRM_ERROR("Failed to lock KIQ (%d).\n", r); 529 return r; 530 } 531 532 kiq->pmf->kiq_set_resources(kiq_ring, queue_mask); 533 for (i = 0; i < adev->gfx.num_compute_rings; i++) 534 kiq->pmf->kiq_map_queues(kiq_ring, &adev->gfx.compute_ring[i]); 535 536 r = amdgpu_ring_test_helper(kiq_ring); 537 if (r) 538 DRM_ERROR("KCQ enable failed\n"); 539 540 return r; 541 } 542 543 /* amdgpu_gfx_off_ctrl - Handle gfx off feature enable/disable 544 * 545 * @adev: amdgpu_device pointer 546 * @bool enable true: enable gfx off feature, false: disable gfx off feature 547 * 548 * 1. gfx off feature will be enabled by gfx ip after gfx cg gp enabled. 549 * 2. other client can send request to disable gfx off feature, the request should be honored. 550 * 3. other client can cancel their request of disable gfx off feature 551 * 4. other client should not send request to enable gfx off feature before disable gfx off feature. 552 */ 553 554 void amdgpu_gfx_off_ctrl(struct amdgpu_device *adev, bool enable) 555 { 556 if (!(adev->pm.pp_feature & PP_GFXOFF_MASK)) 557 return; 558 559 mutex_lock(&adev->gfx.gfx_off_mutex); 560 561 if (enable) { 562 /* If the count is already 0, it means there's an imbalance bug somewhere. 563 * Note that the bug may be in a different caller than the one which triggers the 564 * WARN_ON_ONCE. 565 */ 566 if (WARN_ON_ONCE(adev->gfx.gfx_off_req_count == 0)) 567 goto unlock; 568 569 adev->gfx.gfx_off_req_count--; 570 571 if (adev->gfx.gfx_off_req_count == 0 && !adev->gfx.gfx_off_state) 572 schedule_delayed_work(&adev->gfx.gfx_off_delay_work, GFX_OFF_DELAY_ENABLE); 573 } else { 574 if (adev->gfx.gfx_off_req_count == 0) { 575 cancel_delayed_work_sync(&adev->gfx.gfx_off_delay_work); 576 577 if (adev->gfx.gfx_off_state && 578 !amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, false)) { 579 adev->gfx.gfx_off_state = false; 580 581 if (adev->gfx.funcs->init_spm_golden) { 582 dev_dbg(adev->dev, 583 "GFXOFF is disabled, re-init SPM golden settings\n"); 584 amdgpu_gfx_init_spm_golden(adev); 585 } 586 } 587 } 588 589 adev->gfx.gfx_off_req_count++; 590 } 591 592 unlock: 593 mutex_unlock(&adev->gfx.gfx_off_mutex); 594 } 595 596 int amdgpu_get_gfx_off_status(struct amdgpu_device *adev, uint32_t *value) 597 { 598 599 int r = 0; 600 601 mutex_lock(&adev->gfx.gfx_off_mutex); 602 603 r = smu_get_status_gfxoff(adev, value); 604 605 mutex_unlock(&adev->gfx.gfx_off_mutex); 606 607 return r; 608 } 609 610 int amdgpu_gfx_ras_late_init(struct amdgpu_device *adev) 611 { 612 int r; 613 struct ras_fs_if fs_info = { 614 .sysfs_name = "gfx_err_count", 615 }; 616 struct ras_ih_if ih_info = { 617 .cb = amdgpu_gfx_process_ras_data_cb, 618 }; 619 620 if (!adev->gfx.ras_if) { 621 adev->gfx.ras_if = kmalloc(sizeof(struct ras_common_if), GFP_KERNEL); 622 if (!adev->gfx.ras_if) 623 return -ENOMEM; 624 adev->gfx.ras_if->block = AMDGPU_RAS_BLOCK__GFX; 625 adev->gfx.ras_if->type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; 626 adev->gfx.ras_if->sub_block_index = 0; 627 strlcpy(adev->gfx.ras_if->name, "gfx", sizeof(adev->gfx.ras_if->name)); 628 } 629 fs_info.head = ih_info.head = *adev->gfx.ras_if; 630 631 r = amdgpu_ras_late_init(adev, adev->gfx.ras_if, 632 &fs_info, &ih_info); 633 if (r) 634 goto free; 635 636 if (amdgpu_ras_is_supported(adev, adev->gfx.ras_if->block)) { 637 r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0); 638 if (r) 639 goto late_fini; 640 } else { 641 /* free gfx ras_if if ras is not supported */ 642 r = 0; 643 goto free; 644 } 645 646 return 0; 647 late_fini: 648 amdgpu_ras_late_fini(adev, adev->gfx.ras_if, &ih_info); 649 free: 650 kfree(adev->gfx.ras_if); 651 adev->gfx.ras_if = NULL; 652 return r; 653 } 654 655 void amdgpu_gfx_ras_fini(struct amdgpu_device *adev) 656 { 657 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX) && 658 adev->gfx.ras_if) { 659 struct ras_common_if *ras_if = adev->gfx.ras_if; 660 struct ras_ih_if ih_info = { 661 .head = *ras_if, 662 .cb = amdgpu_gfx_process_ras_data_cb, 663 }; 664 665 amdgpu_ras_late_fini(adev, ras_if, &ih_info); 666 kfree(ras_if); 667 } 668 } 669 670 int amdgpu_gfx_process_ras_data_cb(struct amdgpu_device *adev, 671 void *err_data, 672 struct amdgpu_iv_entry *entry) 673 { 674 /* TODO ue will trigger an interrupt. 675 * 676 * When “Full RAS” is enabled, the per-IP interrupt sources should 677 * be disabled and the driver should only look for the aggregated 678 * interrupt via sync flood 679 */ 680 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) { 681 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev); 682 if (adev->gfx.funcs->query_ras_error_count) 683 adev->gfx.funcs->query_ras_error_count(adev, err_data); 684 amdgpu_ras_reset_gpu(adev); 685 } 686 return AMDGPU_RAS_SUCCESS; 687 } 688 689 int amdgpu_gfx_cp_ecc_error_irq(struct amdgpu_device *adev, 690 struct amdgpu_irq_src *source, 691 struct amdgpu_iv_entry *entry) 692 { 693 struct ras_common_if *ras_if = adev->gfx.ras_if; 694 struct ras_dispatch_if ih_data = { 695 .entry = entry, 696 }; 697 698 if (!ras_if) 699 return 0; 700 701 ih_data.head = *ras_if; 702 703 DRM_ERROR("CP ECC ERROR IRQ\n"); 704 amdgpu_ras_interrupt_dispatch(adev, &ih_data); 705 return 0; 706 } 707 708 uint32_t amdgpu_kiq_rreg(struct amdgpu_device *adev, uint32_t reg) 709 { 710 signed long r, cnt = 0; 711 unsigned long flags; 712 uint32_t seq, reg_val_offs = 0, value = 0; 713 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 714 struct amdgpu_ring *ring = &kiq->ring; 715 716 if (adev->in_pci_err_recovery) 717 return 0; 718 719 BUG_ON(!ring->funcs->emit_rreg); 720 721 spin_lock_irqsave(&kiq->ring_lock, flags); 722 if (amdgpu_device_wb_get(adev, ®_val_offs)) { 723 pr_err("critical bug! too many kiq readers\n"); 724 goto failed_unlock; 725 } 726 amdgpu_ring_alloc(ring, 32); 727 amdgpu_ring_emit_rreg(ring, reg, reg_val_offs); 728 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 729 if (r) 730 goto failed_undo; 731 732 amdgpu_ring_commit(ring); 733 spin_unlock_irqrestore(&kiq->ring_lock, flags); 734 735 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 736 737 /* don't wait anymore for gpu reset case because this way may 738 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg 739 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will 740 * never return if we keep waiting in virt_kiq_rreg, which cause 741 * gpu_recover() hang there. 742 * 743 * also don't wait anymore for IRQ context 744 * */ 745 if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt())) 746 goto failed_kiq_read; 747 748 might_sleep(); 749 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 750 drm_msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 751 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 752 } 753 754 if (cnt > MAX_KIQ_REG_TRY) 755 goto failed_kiq_read; 756 757 mb(); 758 value = adev->wb.wb[reg_val_offs]; 759 amdgpu_device_wb_free(adev, reg_val_offs); 760 return value; 761 762 failed_undo: 763 amdgpu_ring_undo(ring); 764 failed_unlock: 765 spin_unlock_irqrestore(&kiq->ring_lock, flags); 766 failed_kiq_read: 767 if (reg_val_offs) 768 amdgpu_device_wb_free(adev, reg_val_offs); 769 dev_err(adev->dev, "failed to read reg:%x\n", reg); 770 return ~0; 771 } 772 773 void amdgpu_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) 774 { 775 signed long r, cnt = 0; 776 unsigned long flags; 777 uint32_t seq; 778 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 779 struct amdgpu_ring *ring = &kiq->ring; 780 781 BUG_ON(!ring->funcs->emit_wreg); 782 783 if (adev->in_pci_err_recovery) 784 return; 785 786 spin_lock_irqsave(&kiq->ring_lock, flags); 787 amdgpu_ring_alloc(ring, 32); 788 amdgpu_ring_emit_wreg(ring, reg, v); 789 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 790 if (r) 791 goto failed_undo; 792 793 amdgpu_ring_commit(ring); 794 spin_unlock_irqrestore(&kiq->ring_lock, flags); 795 796 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 797 798 /* don't wait anymore for gpu reset case because this way may 799 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg 800 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will 801 * never return if we keep waiting in virt_kiq_rreg, which cause 802 * gpu_recover() hang there. 803 * 804 * also don't wait anymore for IRQ context 805 * */ 806 if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt())) 807 goto failed_kiq_write; 808 809 might_sleep(); 810 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 811 812 drm_msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 813 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 814 } 815 816 if (cnt > MAX_KIQ_REG_TRY) 817 goto failed_kiq_write; 818 819 return; 820 821 failed_undo: 822 amdgpu_ring_undo(ring); 823 spin_unlock_irqrestore(&kiq->ring_lock, flags); 824 failed_kiq_write: 825 dev_err(adev->dev, "failed to write reg:%x\n", reg); 826 } 827