1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 23 #include "amdgpu_amdkfd.h" 24 #include "amd_shared.h" 25 #include <drm/drmP.h> 26 #include "amdgpu.h" 27 #include "amdgpu_gfx.h" 28 #include <linux/module.h> 29 30 const struct kgd2kfd_calls *kgd2kfd; 31 bool (*kgd2kfd_init_p)(unsigned int, const struct kgd2kfd_calls**); 32 33 static const unsigned int compute_vmid_bitmap = 0xFF00; 34 35 int amdgpu_amdkfd_init(void) 36 { 37 int ret; 38 39 #if defined(CONFIG_HSA_AMD_MODULE) 40 int (*kgd2kfd_init_p)(unsigned int, const struct kgd2kfd_calls**); 41 42 kgd2kfd_init_p = symbol_request(kgd2kfd_init); 43 44 if (kgd2kfd_init_p == NULL) 45 return -ENOENT; 46 47 ret = kgd2kfd_init_p(KFD_INTERFACE_VERSION, &kgd2kfd); 48 if (ret) { 49 symbol_put(kgd2kfd_init); 50 kgd2kfd = NULL; 51 } 52 53 54 #elif defined(CONFIG_HSA_AMD) 55 56 ret = kgd2kfd_init(KFD_INTERFACE_VERSION, &kgd2kfd); 57 if (ret) 58 kgd2kfd = NULL; 59 60 #else 61 kgd2kfd = NULL; 62 ret = -ENOENT; 63 #endif 64 65 #if defined(CONFIG_HSA_AMD_MODULE) || defined(CONFIG_HSA_AMD) 66 amdgpu_amdkfd_gpuvm_init_mem_limits(); 67 #endif 68 69 return ret; 70 } 71 72 void amdgpu_amdkfd_fini(void) 73 { 74 if (kgd2kfd) { 75 kgd2kfd->exit(); 76 symbol_put(kgd2kfd_init); 77 } 78 } 79 80 void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev) 81 { 82 const struct kfd2kgd_calls *kfd2kgd; 83 84 if (!kgd2kfd) 85 return; 86 87 switch (adev->asic_type) { 88 #ifdef CONFIG_DRM_AMDGPU_CIK 89 case CHIP_KAVERI: 90 case CHIP_HAWAII: 91 kfd2kgd = amdgpu_amdkfd_gfx_7_get_functions(); 92 break; 93 #endif 94 case CHIP_CARRIZO: 95 case CHIP_TONGA: 96 case CHIP_FIJI: 97 case CHIP_POLARIS10: 98 case CHIP_POLARIS11: 99 kfd2kgd = amdgpu_amdkfd_gfx_8_0_get_functions(); 100 break; 101 case CHIP_VEGA10: 102 case CHIP_RAVEN: 103 kfd2kgd = amdgpu_amdkfd_gfx_9_0_get_functions(); 104 break; 105 default: 106 dev_info(adev->dev, "kfd not supported on this ASIC\n"); 107 return; 108 } 109 110 adev->kfd = kgd2kfd->probe((struct kgd_dev *)adev, 111 adev->pdev, kfd2kgd); 112 } 113 114 /** 115 * amdgpu_doorbell_get_kfd_info - Report doorbell configuration required to 116 * setup amdkfd 117 * 118 * @adev: amdgpu_device pointer 119 * @aperture_base: output returning doorbell aperture base physical address 120 * @aperture_size: output returning doorbell aperture size in bytes 121 * @start_offset: output returning # of doorbell bytes reserved for amdgpu. 122 * 123 * amdgpu and amdkfd share the doorbell aperture. amdgpu sets it up, 124 * takes doorbells required for its own rings and reports the setup to amdkfd. 125 * amdgpu reserved doorbells are at the start of the doorbell aperture. 126 */ 127 static void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev, 128 phys_addr_t *aperture_base, 129 size_t *aperture_size, 130 size_t *start_offset) 131 { 132 /* 133 * The first num_doorbells are used by amdgpu. 134 * amdkfd takes whatever's left in the aperture. 135 */ 136 if (adev->doorbell.size > adev->doorbell.num_doorbells * sizeof(u32)) { 137 *aperture_base = adev->doorbell.base; 138 *aperture_size = adev->doorbell.size; 139 *start_offset = adev->doorbell.num_doorbells * sizeof(u32); 140 } else { 141 *aperture_base = 0; 142 *aperture_size = 0; 143 *start_offset = 0; 144 } 145 } 146 147 void amdgpu_amdkfd_device_init(struct amdgpu_device *adev) 148 { 149 int i; 150 int last_valid_bit; 151 if (adev->kfd) { 152 struct kgd2kfd_shared_resources gpu_resources = { 153 .compute_vmid_bitmap = compute_vmid_bitmap, 154 .num_pipe_per_mec = adev->gfx.mec.num_pipe_per_mec, 155 .num_queue_per_pipe = adev->gfx.mec.num_queue_per_pipe, 156 .gpuvm_size = min(adev->vm_manager.max_pfn 157 << AMDGPU_GPU_PAGE_SHIFT, 158 AMDGPU_VA_HOLE_START), 159 #ifdef notyet 160 .drm_render_minor = adev->ddev->render->index 161 #endif 162 }; 163 164 /* this is going to have a few of the MSBs set that we need to 165 * clear */ 166 bitmap_complement(gpu_resources.queue_bitmap, 167 adev->gfx.mec.queue_bitmap, 168 KGD_MAX_QUEUES); 169 170 /* remove the KIQ bit as well */ 171 if (adev->gfx.kiq.ring.ready) 172 clear_bit(amdgpu_gfx_queue_to_bit(adev, 173 adev->gfx.kiq.ring.me - 1, 174 adev->gfx.kiq.ring.pipe, 175 adev->gfx.kiq.ring.queue), 176 gpu_resources.queue_bitmap); 177 178 /* According to linux/bitmap.h we shouldn't use bitmap_clear if 179 * nbits is not compile time constant */ 180 last_valid_bit = 1 /* only first MEC can have compute queues */ 181 * adev->gfx.mec.num_pipe_per_mec 182 * adev->gfx.mec.num_queue_per_pipe; 183 for (i = last_valid_bit; i < KGD_MAX_QUEUES; ++i) 184 clear_bit(i, gpu_resources.queue_bitmap); 185 186 amdgpu_doorbell_get_kfd_info(adev, 187 &gpu_resources.doorbell_physical_address, 188 &gpu_resources.doorbell_aperture_size, 189 &gpu_resources.doorbell_start_offset); 190 if (adev->asic_type >= CHIP_VEGA10) { 191 /* On SOC15 the BIF is involved in routing 192 * doorbells using the low 12 bits of the 193 * address. Communicate the assignments to 194 * KFD. KFD uses two doorbell pages per 195 * process in case of 64-bit doorbells so we 196 * can use each doorbell assignment twice. 197 */ 198 gpu_resources.sdma_doorbell[0][0] = 199 AMDGPU_DOORBELL64_sDMA_ENGINE0; 200 gpu_resources.sdma_doorbell[0][1] = 201 AMDGPU_DOORBELL64_sDMA_ENGINE0 + 0x200; 202 gpu_resources.sdma_doorbell[1][0] = 203 AMDGPU_DOORBELL64_sDMA_ENGINE1; 204 gpu_resources.sdma_doorbell[1][1] = 205 AMDGPU_DOORBELL64_sDMA_ENGINE1 + 0x200; 206 /* Doorbells 0x0f0-0ff and 0x2f0-2ff are reserved for 207 * SDMA, IH and VCN. So don't use them for the CP. 208 */ 209 gpu_resources.reserved_doorbell_mask = 0x1f0; 210 gpu_resources.reserved_doorbell_val = 0x0f0; 211 } 212 213 kgd2kfd->device_init(adev->kfd, &gpu_resources); 214 } 215 } 216 217 void amdgpu_amdkfd_device_fini(struct amdgpu_device *adev) 218 { 219 if (adev->kfd) { 220 kgd2kfd->device_exit(adev->kfd); 221 adev->kfd = NULL; 222 } 223 } 224 225 void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev, 226 const void *ih_ring_entry) 227 { 228 if (adev->kfd) 229 kgd2kfd->interrupt(adev->kfd, ih_ring_entry); 230 } 231 232 void amdgpu_amdkfd_suspend(struct amdgpu_device *adev) 233 { 234 if (adev->kfd) 235 kgd2kfd->suspend(adev->kfd); 236 } 237 238 int amdgpu_amdkfd_resume(struct amdgpu_device *adev) 239 { 240 int r = 0; 241 242 if (adev->kfd) 243 r = kgd2kfd->resume(adev->kfd); 244 245 return r; 246 } 247 248 int amdgpu_amdkfd_pre_reset(struct amdgpu_device *adev) 249 { 250 int r = 0; 251 252 if (adev->kfd) 253 r = kgd2kfd->pre_reset(adev->kfd); 254 255 return r; 256 } 257 258 int amdgpu_amdkfd_post_reset(struct amdgpu_device *adev) 259 { 260 int r = 0; 261 262 if (adev->kfd) 263 r = kgd2kfd->post_reset(adev->kfd); 264 265 return r; 266 } 267 268 void amdgpu_amdkfd_gpu_reset(struct kgd_dev *kgd) 269 { 270 struct amdgpu_device *adev = (struct amdgpu_device *)kgd; 271 272 amdgpu_device_gpu_recover(adev, NULL, false); 273 } 274 275 int alloc_gtt_mem(struct kgd_dev *kgd, size_t size, 276 void **mem_obj, uint64_t *gpu_addr, 277 void **cpu_ptr, bool mqd_gfx9) 278 { 279 struct amdgpu_device *adev = (struct amdgpu_device *)kgd; 280 struct amdgpu_bo *bo = NULL; 281 struct amdgpu_bo_param bp; 282 int r; 283 void *cpu_ptr_tmp = NULL; 284 285 memset(&bp, 0, sizeof(bp)); 286 bp.size = size; 287 bp.byte_align = PAGE_SIZE; 288 bp.domain = AMDGPU_GEM_DOMAIN_GTT; 289 bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC; 290 bp.type = ttm_bo_type_kernel; 291 bp.resv = NULL; 292 293 if (mqd_gfx9) 294 bp.flags |= AMDGPU_GEM_CREATE_MQD_GFX9; 295 296 r = amdgpu_bo_create(adev, &bp, &bo); 297 if (r) { 298 dev_err(adev->dev, 299 "failed to allocate BO for amdkfd (%d)\n", r); 300 return r; 301 } 302 303 /* map the buffer */ 304 r = amdgpu_bo_reserve(bo, true); 305 if (r) { 306 dev_err(adev->dev, "(%d) failed to reserve bo for amdkfd\n", r); 307 goto allocate_mem_reserve_bo_failed; 308 } 309 310 r = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT); 311 if (r) { 312 dev_err(adev->dev, "(%d) failed to pin bo for amdkfd\n", r); 313 goto allocate_mem_pin_bo_failed; 314 } 315 316 r = amdgpu_ttm_alloc_gart(&bo->tbo); 317 if (r) { 318 dev_err(adev->dev, "%p bind failed\n", bo); 319 goto allocate_mem_kmap_bo_failed; 320 } 321 322 r = amdgpu_bo_kmap(bo, &cpu_ptr_tmp); 323 if (r) { 324 dev_err(adev->dev, 325 "(%d) failed to map bo to kernel for amdkfd\n", r); 326 goto allocate_mem_kmap_bo_failed; 327 } 328 329 *mem_obj = bo; 330 *gpu_addr = amdgpu_bo_gpu_offset(bo); 331 *cpu_ptr = cpu_ptr_tmp; 332 333 amdgpu_bo_unreserve(bo); 334 335 return 0; 336 337 allocate_mem_kmap_bo_failed: 338 amdgpu_bo_unpin(bo); 339 allocate_mem_pin_bo_failed: 340 amdgpu_bo_unreserve(bo); 341 allocate_mem_reserve_bo_failed: 342 amdgpu_bo_unref(&bo); 343 344 return r; 345 } 346 347 void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj) 348 { 349 struct amdgpu_bo *bo = (struct amdgpu_bo *) mem_obj; 350 351 amdgpu_bo_reserve(bo, true); 352 amdgpu_bo_kunmap(bo); 353 amdgpu_bo_unpin(bo); 354 amdgpu_bo_unreserve(bo); 355 amdgpu_bo_unref(&(bo)); 356 } 357 358 void get_local_mem_info(struct kgd_dev *kgd, 359 struct kfd_local_mem_info *mem_info) 360 { 361 STUB(); 362 #if 0 363 struct amdgpu_device *adev = (struct amdgpu_device *)kgd; 364 uint64_t address_mask = adev->dev->dma_mask ? ~*adev->dev->dma_mask : 365 ~((1ULL << 32) - 1); 366 resource_size_t aper_limit = adev->gmc.aper_base + adev->gmc.aper_size; 367 368 memset(mem_info, 0, sizeof(*mem_info)); 369 if (!(adev->gmc.aper_base & address_mask || aper_limit & address_mask)) { 370 mem_info->local_mem_size_public = adev->gmc.visible_vram_size; 371 mem_info->local_mem_size_private = adev->gmc.real_vram_size - 372 adev->gmc.visible_vram_size; 373 } else { 374 mem_info->local_mem_size_public = 0; 375 mem_info->local_mem_size_private = adev->gmc.real_vram_size; 376 } 377 mem_info->vram_width = adev->gmc.vram_width; 378 379 pr_debug("Address base: %pap limit %pap public 0x%llx private 0x%llx\n", 380 &adev->gmc.aper_base, &aper_limit, 381 mem_info->local_mem_size_public, 382 mem_info->local_mem_size_private); 383 384 if (amdgpu_sriov_vf(adev)) 385 mem_info->mem_clk_max = adev->clock.default_mclk / 100; 386 else if (adev->powerplay.pp_funcs) 387 mem_info->mem_clk_max = amdgpu_dpm_get_mclk(adev, false) / 100; 388 else 389 mem_info->mem_clk_max = 100; 390 #endif 391 } 392 393 uint64_t get_gpu_clock_counter(struct kgd_dev *kgd) 394 { 395 struct amdgpu_device *adev = (struct amdgpu_device *)kgd; 396 397 if (adev->gfx.funcs->get_gpu_clock_counter) 398 return adev->gfx.funcs->get_gpu_clock_counter(adev); 399 return 0; 400 } 401 402 uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd) 403 { 404 struct amdgpu_device *adev = (struct amdgpu_device *)kgd; 405 406 /* the sclk is in quantas of 10kHz */ 407 if (amdgpu_sriov_vf(adev)) 408 return adev->clock.default_sclk / 100; 409 else if (adev->powerplay.pp_funcs) 410 return amdgpu_dpm_get_sclk(adev, false) / 100; 411 else 412 return 100; 413 } 414 415 void get_cu_info(struct kgd_dev *kgd, struct kfd_cu_info *cu_info) 416 { 417 struct amdgpu_device *adev = (struct amdgpu_device *)kgd; 418 struct amdgpu_cu_info acu_info = adev->gfx.cu_info; 419 420 memset(cu_info, 0, sizeof(*cu_info)); 421 if (sizeof(cu_info->cu_bitmap) != sizeof(acu_info.bitmap)) 422 return; 423 424 cu_info->cu_active_number = acu_info.number; 425 cu_info->cu_ao_mask = acu_info.ao_cu_mask; 426 memcpy(&cu_info->cu_bitmap[0], &acu_info.bitmap[0], 427 sizeof(acu_info.bitmap)); 428 cu_info->num_shader_engines = adev->gfx.config.max_shader_engines; 429 cu_info->num_shader_arrays_per_engine = adev->gfx.config.max_sh_per_se; 430 cu_info->num_cu_per_sh = adev->gfx.config.max_cu_per_sh; 431 cu_info->simd_per_cu = acu_info.simd_per_cu; 432 cu_info->max_waves_per_simd = acu_info.max_waves_per_simd; 433 cu_info->wave_front_size = acu_info.wave_front_size; 434 cu_info->max_scratch_slots_per_cu = acu_info.max_scratch_slots_per_cu; 435 cu_info->lds_size = acu_info.lds_size; 436 } 437 438 uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd) 439 { 440 struct amdgpu_device *adev = (struct amdgpu_device *)kgd; 441 442 return amdgpu_vram_mgr_usage(&adev->mman.bdev.man[TTM_PL_VRAM]); 443 } 444 445 int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine, 446 uint32_t vmid, uint64_t gpu_addr, 447 uint32_t *ib_cmd, uint32_t ib_len) 448 { 449 struct amdgpu_device *adev = (struct amdgpu_device *)kgd; 450 struct amdgpu_job *job; 451 struct amdgpu_ib *ib; 452 struct amdgpu_ring *ring; 453 struct dma_fence *f = NULL; 454 int ret; 455 456 switch (engine) { 457 case KGD_ENGINE_MEC1: 458 ring = &adev->gfx.compute_ring[0]; 459 break; 460 case KGD_ENGINE_SDMA1: 461 ring = &adev->sdma.instance[0].ring; 462 break; 463 case KGD_ENGINE_SDMA2: 464 ring = &adev->sdma.instance[1].ring; 465 break; 466 default: 467 pr_err("Invalid engine in IB submission: %d\n", engine); 468 ret = -EINVAL; 469 goto err; 470 } 471 472 ret = amdgpu_job_alloc(adev, 1, &job, NULL); 473 if (ret) 474 goto err; 475 476 ib = &job->ibs[0]; 477 memset(ib, 0, sizeof(struct amdgpu_ib)); 478 479 ib->gpu_addr = gpu_addr; 480 ib->ptr = ib_cmd; 481 ib->length_dw = ib_len; 482 /* This works for NO_HWS. TODO: need to handle without knowing VMID */ 483 job->vmid = vmid; 484 485 ret = amdgpu_ib_schedule(ring, 1, ib, job, &f); 486 if (ret) { 487 DRM_ERROR("amdgpu: failed to schedule IB.\n"); 488 goto err_ib_sched; 489 } 490 491 ret = dma_fence_wait(f, false); 492 493 err_ib_sched: 494 dma_fence_put(f); 495 amdgpu_job_free(job); 496 err: 497 return ret; 498 } 499 500 void amdgpu_amdkfd_set_compute_idle(struct kgd_dev *kgd, bool idle) 501 { 502 struct amdgpu_device *adev = (struct amdgpu_device *)kgd; 503 504 if (adev->powerplay.pp_funcs && 505 adev->powerplay.pp_funcs->switch_power_profile) 506 amdgpu_dpm_switch_power_profile(adev, 507 PP_SMC_POWER_PROFILE_COMPUTE, 508 !idle); 509 } 510 511 bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid) 512 { 513 if (adev->kfd) { 514 if ((1 << vmid) & compute_vmid_bitmap) 515 return true; 516 } 517 518 return false; 519 } 520 521 #if !defined(CONFIG_HSA_AMD_MODULE) && !defined(CONFIG_HSA_AMD) 522 bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm) 523 { 524 return false; 525 } 526 527 void amdgpu_amdkfd_unreserve_system_memory_limit(struct amdgpu_bo *bo) 528 { 529 } 530 531 void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev, 532 struct amdgpu_vm *vm) 533 { 534 } 535 536 struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f) 537 { 538 return NULL; 539 } 540 541 int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm) 542 { 543 return 0; 544 } 545 546 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void) 547 { 548 return NULL; 549 } 550 551 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void) 552 { 553 return NULL; 554 } 555 556 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_9_0_get_functions(void) 557 { 558 return NULL; 559 } 560 #endif 561