1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2020 Intel Corporation 4 */ 5 6 #include <linux/slab.h> /* fault-inject.h is not standalone! */ 7 8 #include <linux/fault-inject.h> 9 #include <asm/set_memory.h> 10 11 #include "i915_trace.h" 12 #include "intel_gt.h" 13 #include "intel_gtt.h" 14 15 void stash_init(struct pagestash *stash) 16 { 17 pagevec_init(&stash->pvec); 18 mtx_init(&stash->lock, IPL_NONE); 19 } 20 21 static struct vm_page *stash_pop_page(struct pagestash *stash) 22 { 23 struct vm_page *page = NULL; 24 25 spin_lock(&stash->lock); 26 if (likely(stash->pvec.nr)) 27 page = stash->pvec.pages[--stash->pvec.nr]; 28 spin_unlock(&stash->lock); 29 30 return page; 31 } 32 33 static void stash_push_pagevec(struct pagestash *stash, struct pagevec *pvec) 34 { 35 unsigned int nr; 36 37 spin_lock_nested(&stash->lock, SINGLE_DEPTH_NESTING); 38 39 nr = min_t(typeof(nr), pvec->nr, pagevec_space(&stash->pvec)); 40 memcpy(stash->pvec.pages + stash->pvec.nr, 41 pvec->pages + pvec->nr - nr, 42 sizeof(pvec->pages[0]) * nr); 43 stash->pvec.nr += nr; 44 45 spin_unlock(&stash->lock); 46 47 pvec->nr -= nr; 48 } 49 50 static struct vm_page *vm_alloc_page(struct i915_address_space *vm, gfp_t gfp) 51 { 52 struct pagevec stack; 53 struct vm_page *page; 54 55 if (I915_SELFTEST_ONLY(should_fail(&vm->fault_attr, 1))) 56 i915_gem_shrink_all(vm->i915); 57 58 page = stash_pop_page(&vm->free_pages); 59 if (page) 60 return page; 61 62 if (!vm->pt_kmap_wc) 63 return alloc_page(gfp); 64 65 /* Look in our global stash of WC pages... */ 66 page = stash_pop_page(&vm->i915->mm.wc_stash); 67 if (page) 68 return page; 69 70 /* 71 * Otherwise batch allocate pages to amortize cost of set_pages_wc. 72 * 73 * We have to be careful as page allocation may trigger the shrinker 74 * (via direct reclaim) which will fill up the WC stash underneath us. 75 * So we add our WB pages into a temporary pvec on the stack and merge 76 * them into the WC stash after all the allocations are complete. 77 */ 78 pagevec_init(&stack); 79 do { 80 struct vm_page *page; 81 82 page = alloc_page(gfp); 83 if (unlikely(!page)) 84 break; 85 86 stack.pages[stack.nr++] = page; 87 } while (pagevec_space(&stack)); 88 89 if (stack.nr && !set_pages_array_wc(stack.pages, stack.nr)) { 90 page = stack.pages[--stack.nr]; 91 92 /* Merge spare WC pages to the global stash */ 93 if (stack.nr) 94 stash_push_pagevec(&vm->i915->mm.wc_stash, &stack); 95 96 /* Push any surplus WC pages onto the local VM stash */ 97 if (stack.nr) 98 stash_push_pagevec(&vm->free_pages, &stack); 99 } 100 101 /* Return unwanted leftovers */ 102 if (unlikely(stack.nr)) { 103 WARN_ON_ONCE(set_pages_array_wb(stack.pages, stack.nr)); 104 __pagevec_release(&stack); 105 } 106 107 return page; 108 } 109 110 static void vm_free_pages_release(struct i915_address_space *vm, 111 bool immediate) 112 { 113 struct pagevec *pvec = &vm->free_pages.pvec; 114 struct pagevec stack; 115 116 lockdep_assert_held(&vm->free_pages.lock); 117 GEM_BUG_ON(!pagevec_count(pvec)); 118 119 if (vm->pt_kmap_wc) { 120 /* 121 * When we use WC, first fill up the global stash and then 122 * only if full immediately free the overflow. 123 */ 124 stash_push_pagevec(&vm->i915->mm.wc_stash, pvec); 125 126 /* 127 * As we have made some room in the VM's free_pages, 128 * we can wait for it to fill again. Unless we are 129 * inside i915_address_space_fini() and must 130 * immediately release the pages! 131 */ 132 if (pvec->nr <= (immediate ? 0 : PAGEVEC_SIZE - 1)) 133 return; 134 135 /* 136 * We have to drop the lock to allow ourselves to sleep, 137 * so take a copy of the pvec and clear the stash for 138 * others to use it as we sleep. 139 */ 140 stack = *pvec; 141 pagevec_reinit(pvec); 142 spin_unlock(&vm->free_pages.lock); 143 144 pvec = &stack; 145 set_pages_array_wb(pvec->pages, pvec->nr); 146 147 spin_lock(&vm->free_pages.lock); 148 } 149 150 __pagevec_release(pvec); 151 } 152 153 static void vm_free_page(struct i915_address_space *vm, struct vm_page *page) 154 { 155 /* 156 * On !llc, we need to change the pages back to WB. We only do so 157 * in bulk, so we rarely need to change the page attributes here, 158 * but doing so requires a stop_machine() from deep inside arch/x86/mm. 159 * To make detection of the possible sleep more likely, use an 160 * unconditional might_sleep() for everybody. 161 */ 162 might_sleep(); 163 spin_lock(&vm->free_pages.lock); 164 while (!pagevec_space(&vm->free_pages.pvec)) 165 vm_free_pages_release(vm, false); 166 GEM_BUG_ON(pagevec_count(&vm->free_pages.pvec) >= PAGEVEC_SIZE); 167 pagevec_add(&vm->free_pages.pvec, page); 168 spin_unlock(&vm->free_pages.lock); 169 } 170 171 void __i915_vm_close(struct i915_address_space *vm) 172 { 173 struct i915_vma *vma, *vn; 174 175 if (!atomic_dec_and_mutex_lock(&vm->open, &vm->mutex)) 176 return; 177 178 list_for_each_entry_safe(vma, vn, &vm->bound_list, vm_link) { 179 struct drm_i915_gem_object *obj = vma->obj; 180 181 /* Keep the obj (and hence the vma) alive as _we_ destroy it */ 182 if (!kref_get_unless_zero(&obj->base.refcount)) 183 continue; 184 185 atomic_and(~I915_VMA_PIN_MASK, &vma->flags); 186 WARN_ON(__i915_vma_unbind(vma)); 187 __i915_vma_put(vma); 188 189 i915_gem_object_put(obj); 190 } 191 GEM_BUG_ON(!list_empty(&vm->bound_list)); 192 193 mutex_unlock(&vm->mutex); 194 } 195 196 void i915_address_space_fini(struct i915_address_space *vm) 197 { 198 spin_lock(&vm->free_pages.lock); 199 if (pagevec_count(&vm->free_pages.pvec)) 200 vm_free_pages_release(vm, true); 201 GEM_BUG_ON(pagevec_count(&vm->free_pages.pvec)); 202 spin_unlock(&vm->free_pages.lock); 203 204 drm_mm_takedown(&vm->mm); 205 206 mutex_destroy(&vm->mutex); 207 } 208 209 static void __i915_vm_release(struct work_struct *work) 210 { 211 struct i915_address_space *vm = 212 container_of(work, struct i915_address_space, rcu.work); 213 214 vm->cleanup(vm); 215 i915_address_space_fini(vm); 216 217 kfree(vm); 218 } 219 220 void i915_vm_release(struct kref *kref) 221 { 222 struct i915_address_space *vm = 223 container_of(kref, struct i915_address_space, ref); 224 225 GEM_BUG_ON(i915_is_ggtt(vm)); 226 trace_i915_ppgtt_release(vm); 227 228 queue_rcu_work(vm->i915->wq, &vm->rcu); 229 } 230 231 void i915_address_space_init(struct i915_address_space *vm, int subclass) 232 { 233 kref_init(&vm->ref); 234 INIT_RCU_WORK(&vm->rcu, __i915_vm_release); 235 atomic_set(&vm->open, 1); 236 237 /* 238 * The vm->mutex must be reclaim safe (for use in the shrinker). 239 * Do a dummy acquire now under fs_reclaim so that any allocation 240 * attempt holding the lock is immediately reported by lockdep. 241 */ 242 rw_init(&vm->mutex, "vmlk"); 243 lockdep_set_subclass(&vm->mutex, subclass); 244 i915_gem_shrinker_taints_mutex(vm->i915, &vm->mutex); 245 246 GEM_BUG_ON(!vm->total); 247 drm_mm_init(&vm->mm, 0, vm->total); 248 vm->mm.head_node.color = I915_COLOR_UNEVICTABLE; 249 250 stash_init(&vm->free_pages); 251 252 INIT_LIST_HEAD(&vm->bound_list); 253 } 254 255 void clear_pages(struct i915_vma *vma) 256 { 257 GEM_BUG_ON(!vma->pages); 258 259 if (vma->pages != vma->obj->mm.pages) { 260 sg_free_table(vma->pages); 261 kfree(vma->pages); 262 } 263 vma->pages = NULL; 264 265 memset(&vma->page_sizes, 0, sizeof(vma->page_sizes)); 266 } 267 268 #ifdef __linux__ 269 static int __setup_page_dma(struct i915_address_space *vm, 270 struct i915_page_dma *p, 271 gfp_t gfp) 272 { 273 p->page = vm_alloc_page(vm, gfp | I915_GFP_ALLOW_FAIL); 274 if (unlikely(!p->page)) 275 return -ENOMEM; 276 277 p->daddr = dma_map_page_attrs(vm->dma, 278 p->page, 0, PAGE_SIZE, 279 PCI_DMA_BIDIRECTIONAL, 280 DMA_ATTR_SKIP_CPU_SYNC | 281 DMA_ATTR_NO_WARN); 282 if (unlikely(dma_mapping_error(vm->dma, p->daddr))) { 283 vm_free_page(vm, p->page); 284 return -ENOMEM; 285 } 286 287 return 0; 288 } 289 #else 290 static int __setup_page_dma(struct i915_address_space *vm, 291 struct i915_page_dma *p, 292 gfp_t gfp) 293 { 294 p->page = vm_alloc_page(vm, gfp | I915_GFP_ALLOW_FAIL); 295 if (unlikely(!p->page)) 296 return -ENOMEM; 297 298 p->daddr = VM_PAGE_TO_PHYS(p->page); 299 300 return 0; 301 } 302 #endif 303 304 int setup_page_dma(struct i915_address_space *vm, struct i915_page_dma *p) 305 { 306 return __setup_page_dma(vm, p, __GFP_HIGHMEM); 307 } 308 309 void cleanup_page_dma(struct i915_address_space *vm, struct i915_page_dma *p) 310 { 311 #ifdef __linux__ 312 dma_unmap_page(vm->dma, p->daddr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 313 #endif 314 vm_free_page(vm, p->page); 315 } 316 317 void 318 fill_page_dma(const struct i915_page_dma *p, const u64 val, unsigned int count) 319 { 320 kunmap_atomic(memset64(kmap_atomic(p->page), val, count)); 321 } 322 323 static void poison_scratch_page(struct vm_page *page, unsigned long size) 324 { 325 if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) 326 return; 327 328 GEM_BUG_ON(!IS_ALIGNED(size, PAGE_SIZE)); 329 330 do { 331 void *vaddr; 332 333 vaddr = kmap(page); 334 #ifdef __linux__ 335 memset(vaddr, POISON_FREE, PAGE_SIZE); 336 kunmap(page); 337 #else 338 poison_mem(vaddr, PAGE_SIZE); 339 kunmap_va(vaddr); 340 #endif 341 342 page = pfn_to_page(page_to_pfn(page) + 1); 343 size -= PAGE_SIZE; 344 } while (size); 345 } 346 347 int setup_scratch_page(struct i915_address_space *vm, gfp_t gfp) 348 { 349 unsigned long size; 350 351 /* 352 * In order to utilize 64K pages for an object with a size < 2M, we will 353 * need to support a 64K scratch page, given that every 16th entry for a 354 * page-table operating in 64K mode must point to a properly aligned 64K 355 * region, including any PTEs which happen to point to scratch. 356 * 357 * This is only relevant for the 48b PPGTT where we support 358 * huge-gtt-pages, see also i915_vma_insert(). However, as we share the 359 * scratch (read-only) between all vm, we create one 64k scratch page 360 * for all. 361 */ 362 size = I915_GTT_PAGE_SIZE_4K; 363 if (i915_vm_is_4lvl(vm) && 364 HAS_PAGE_SIZES(vm->i915, I915_GTT_PAGE_SIZE_64K)) { 365 size = I915_GTT_PAGE_SIZE_64K; 366 gfp |= __GFP_NOWARN; 367 } 368 gfp |= __GFP_ZERO | __GFP_RETRY_MAYFAIL; 369 370 do { 371 unsigned int order = get_order(size); 372 struct vm_page *page; 373 dma_addr_t addr; 374 375 page = alloc_pages(gfp, order); 376 if (unlikely(!page)) 377 goto skip; 378 379 /* 380 * Use a non-zero scratch page for debugging. 381 * 382 * We want a value that should be reasonably obvious 383 * to spot in the error state, while also causing a GPU hang 384 * if executed. We prefer using a clear page in production, so 385 * should it ever be accidentally used, the effect should be 386 * fairly benign. 387 */ 388 poison_scratch_page(page, size); 389 390 #ifdef __linux__ 391 addr = dma_map_page_attrs(vm->dma, 392 page, 0, size, 393 PCI_DMA_BIDIRECTIONAL, 394 DMA_ATTR_SKIP_CPU_SYNC | 395 DMA_ATTR_NO_WARN); 396 if (unlikely(dma_mapping_error(vm->dma, addr))) 397 goto free_page; 398 #else 399 addr = VM_PAGE_TO_PHYS(page); 400 #endif 401 402 if (unlikely(!IS_ALIGNED(addr, size))) 403 goto unmap_page; 404 405 vm->scratch[0].base.page = page; 406 vm->scratch[0].base.daddr = addr; 407 vm->scratch_order = order; 408 return 0; 409 410 unmap_page: 411 #ifdef __linux__ 412 dma_unmap_page(vm->dma, addr, size, PCI_DMA_BIDIRECTIONAL); 413 free_page: 414 #endif 415 __free_pages(page, order); 416 skip: 417 if (size == I915_GTT_PAGE_SIZE_4K) 418 return -ENOMEM; 419 420 size = I915_GTT_PAGE_SIZE_4K; 421 gfp &= ~__GFP_NOWARN; 422 } while (1); 423 } 424 425 void cleanup_scratch_page(struct i915_address_space *vm) 426 { 427 struct i915_page_dma *p = px_base(&vm->scratch[0]); 428 unsigned int order = vm->scratch_order; 429 430 #ifdef __linux__ 431 dma_unmap_page(vm->dma, p->daddr, BIT(order) << PAGE_SHIFT, 432 PCI_DMA_BIDIRECTIONAL); 433 #endif 434 __free_pages(p->page, order); 435 } 436 437 void free_scratch(struct i915_address_space *vm) 438 { 439 int i; 440 441 if (!px_dma(&vm->scratch[0])) /* set to 0 on clones */ 442 return; 443 444 for (i = 1; i <= vm->top; i++) { 445 if (!px_dma(&vm->scratch[i])) 446 break; 447 cleanup_page_dma(vm, px_base(&vm->scratch[i])); 448 } 449 450 cleanup_scratch_page(vm); 451 } 452 453 void gtt_write_workarounds(struct intel_gt *gt) 454 { 455 struct drm_i915_private *i915 = gt->i915; 456 struct intel_uncore *uncore = gt->uncore; 457 458 /* 459 * This function is for gtt related workarounds. This function is 460 * called on driver load and after a GPU reset, so you can place 461 * workarounds here even if they get overwritten by GPU reset. 462 */ 463 /* WaIncreaseDefaultTLBEntries:chv,bdw,skl,bxt,kbl,glk,cfl,cnl,icl */ 464 if (IS_BROADWELL(i915)) 465 intel_uncore_write(uncore, 466 GEN8_L3_LRA_1_GPGPU, 467 GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_BDW); 468 else if (IS_CHERRYVIEW(i915)) 469 intel_uncore_write(uncore, 470 GEN8_L3_LRA_1_GPGPU, 471 GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_CHV); 472 else if (IS_GEN9_LP(i915)) 473 intel_uncore_write(uncore, 474 GEN8_L3_LRA_1_GPGPU, 475 GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT); 476 else if (INTEL_GEN(i915) >= 9 && INTEL_GEN(i915) <= 11) 477 intel_uncore_write(uncore, 478 GEN8_L3_LRA_1_GPGPU, 479 GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_SKL); 480 481 /* 482 * To support 64K PTEs we need to first enable the use of the 483 * Intermediate-Page-Size(IPS) bit of the PDE field via some magical 484 * mmio, otherwise the page-walker will simply ignore the IPS bit. This 485 * shouldn't be needed after GEN10. 486 * 487 * 64K pages were first introduced from BDW+, although technically they 488 * only *work* from gen9+. For pre-BDW we instead have the option for 489 * 32K pages, but we don't currently have any support for it in our 490 * driver. 491 */ 492 if (HAS_PAGE_SIZES(i915, I915_GTT_PAGE_SIZE_64K) && 493 INTEL_GEN(i915) <= 10) 494 intel_uncore_rmw(uncore, 495 GEN8_GAMW_ECO_DEV_RW_IA, 496 0, 497 GAMW_ECO_ENABLE_64K_IPS_FIELD); 498 499 if (IS_GEN_RANGE(i915, 8, 11)) { 500 bool can_use_gtt_cache = true; 501 502 /* 503 * According to the BSpec if we use 2M/1G pages then we also 504 * need to disable the GTT cache. At least on BDW we can see 505 * visual corruption when using 2M pages, and not disabling the 506 * GTT cache. 507 */ 508 if (HAS_PAGE_SIZES(i915, I915_GTT_PAGE_SIZE_2M)) 509 can_use_gtt_cache = false; 510 511 /* WaGttCachingOffByDefault */ 512 intel_uncore_write(uncore, 513 HSW_GTT_CACHE_EN, 514 can_use_gtt_cache ? GTT_CACHE_EN_ALL : 0); 515 drm_WARN_ON_ONCE(&i915->drm, can_use_gtt_cache && 516 intel_uncore_read(uncore, 517 HSW_GTT_CACHE_EN) == 0); 518 } 519 } 520 521 static void tgl_setup_private_ppat(struct intel_uncore *uncore) 522 { 523 /* TGL doesn't support LLC or AGE settings */ 524 intel_uncore_write(uncore, GEN12_PAT_INDEX(0), GEN8_PPAT_WB); 525 intel_uncore_write(uncore, GEN12_PAT_INDEX(1), GEN8_PPAT_WC); 526 intel_uncore_write(uncore, GEN12_PAT_INDEX(2), GEN8_PPAT_WT); 527 intel_uncore_write(uncore, GEN12_PAT_INDEX(3), GEN8_PPAT_UC); 528 intel_uncore_write(uncore, GEN12_PAT_INDEX(4), GEN8_PPAT_WB); 529 intel_uncore_write(uncore, GEN12_PAT_INDEX(5), GEN8_PPAT_WB); 530 intel_uncore_write(uncore, GEN12_PAT_INDEX(6), GEN8_PPAT_WB); 531 intel_uncore_write(uncore, GEN12_PAT_INDEX(7), GEN8_PPAT_WB); 532 } 533 534 static void cnl_setup_private_ppat(struct intel_uncore *uncore) 535 { 536 intel_uncore_write(uncore, 537 GEN10_PAT_INDEX(0), 538 GEN8_PPAT_WB | GEN8_PPAT_LLC); 539 intel_uncore_write(uncore, 540 GEN10_PAT_INDEX(1), 541 GEN8_PPAT_WC | GEN8_PPAT_LLCELLC); 542 intel_uncore_write(uncore, 543 GEN10_PAT_INDEX(2), 544 GEN8_PPAT_WT | GEN8_PPAT_LLCELLC); 545 intel_uncore_write(uncore, 546 GEN10_PAT_INDEX(3), 547 GEN8_PPAT_UC); 548 intel_uncore_write(uncore, 549 GEN10_PAT_INDEX(4), 550 GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)); 551 intel_uncore_write(uncore, 552 GEN10_PAT_INDEX(5), 553 GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)); 554 intel_uncore_write(uncore, 555 GEN10_PAT_INDEX(6), 556 GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)); 557 intel_uncore_write(uncore, 558 GEN10_PAT_INDEX(7), 559 GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3)); 560 } 561 562 /* 563 * The GGTT and PPGTT need a private PPAT setup in order to handle cacheability 564 * bits. When using advanced contexts each context stores its own PAT, but 565 * writing this data shouldn't be harmful even in those cases. 566 */ 567 static void bdw_setup_private_ppat(struct intel_uncore *uncore) 568 { 569 u64 pat; 570 571 pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC) | /* for normal objects, no eLLC */ 572 GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */ 573 GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */ 574 GEN8_PPAT(3, GEN8_PPAT_UC) | /* Uncached objects, mostly for scanout */ 575 GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) | 576 GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) | 577 GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) | 578 GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3)); 579 580 intel_uncore_write(uncore, GEN8_PRIVATE_PAT_LO, lower_32_bits(pat)); 581 intel_uncore_write(uncore, GEN8_PRIVATE_PAT_HI, upper_32_bits(pat)); 582 } 583 584 static void chv_setup_private_ppat(struct intel_uncore *uncore) 585 { 586 u64 pat; 587 588 /* 589 * Map WB on BDW to snooped on CHV. 590 * 591 * Only the snoop bit has meaning for CHV, the rest is 592 * ignored. 593 * 594 * The hardware will never snoop for certain types of accesses: 595 * - CPU GTT (GMADR->GGTT->no snoop->memory) 596 * - PPGTT page tables 597 * - some other special cycles 598 * 599 * As with BDW, we also need to consider the following for GT accesses: 600 * "For GGTT, there is NO pat_sel[2:0] from the entry, 601 * so RTL will always use the value corresponding to 602 * pat_sel = 000". 603 * Which means we must set the snoop bit in PAT entry 0 604 * in order to keep the global status page working. 605 */ 606 607 pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) | 608 GEN8_PPAT(1, 0) | 609 GEN8_PPAT(2, 0) | 610 GEN8_PPAT(3, 0) | 611 GEN8_PPAT(4, CHV_PPAT_SNOOP) | 612 GEN8_PPAT(5, CHV_PPAT_SNOOP) | 613 GEN8_PPAT(6, CHV_PPAT_SNOOP) | 614 GEN8_PPAT(7, CHV_PPAT_SNOOP); 615 616 intel_uncore_write(uncore, GEN8_PRIVATE_PAT_LO, lower_32_bits(pat)); 617 intel_uncore_write(uncore, GEN8_PRIVATE_PAT_HI, upper_32_bits(pat)); 618 } 619 620 void setup_private_pat(struct intel_uncore *uncore) 621 { 622 struct drm_i915_private *i915 = uncore->i915; 623 624 GEM_BUG_ON(INTEL_GEN(i915) < 8); 625 626 if (INTEL_GEN(i915) >= 12) 627 tgl_setup_private_ppat(uncore); 628 else if (INTEL_GEN(i915) >= 10) 629 cnl_setup_private_ppat(uncore); 630 else if (IS_CHERRYVIEW(i915) || IS_GEN9_LP(i915)) 631 chv_setup_private_ppat(uncore); 632 else 633 bdw_setup_private_ppat(uncore); 634 } 635 636 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 637 #include "selftests/mock_gtt.c" 638 #endif 639