Lines Matching refs:engine

51 	int (*init_legacy)(struct intel_engine_cs *engine);
52 int (*init_execlists)(struct intel_engine_cs *engine);
201 struct intel_engine_cs *engine; in intel_engine_setup() local
206 GEM_BUG_ON(dev_priv->engine[id]); in intel_engine_setup()
207 engine = kzalloc(sizeof(*engine), GFP_KERNEL); in intel_engine_setup()
208 if (!engine) in intel_engine_setup()
211 engine->id = id; in intel_engine_setup()
212 engine->i915 = dev_priv; in intel_engine_setup()
213 WARN_ON(snprintf(engine->name, sizeof(engine->name), "%s%u", in intel_engine_setup()
215 sizeof(engine->name)); in intel_engine_setup()
216 engine->uabi_id = info->uabi_id; in intel_engine_setup()
217 engine->hw_id = engine->guc_id = info->hw_id; in intel_engine_setup()
218 engine->mmio_base = info->mmio_base; in intel_engine_setup()
219 engine->irq_shift = info->irq_shift; in intel_engine_setup()
220 engine->class = info->class; in intel_engine_setup()
221 engine->instance = info->instance; in intel_engine_setup()
223 engine->context_size = __intel_engine_context_size(dev_priv, in intel_engine_setup()
224 engine->class); in intel_engine_setup()
225 if (WARN_ON(engine->context_size > BIT(20))) in intel_engine_setup()
226 engine->context_size = 0; in intel_engine_setup()
229 engine->schedule = NULL; in intel_engine_setup()
231 ATOMIC_INIT_NOTIFIER_HEAD(&engine->context_status_notifier); in intel_engine_setup()
233 dev_priv->engine[id] = engine; in intel_engine_setup()
247 struct intel_engine_cs *engine; in intel_engines_init_mmio() local
287 for_each_engine(engine, dev_priv, id) in intel_engines_init_mmio()
288 kfree(engine); in intel_engines_init_mmio()
300 struct intel_engine_cs *engine; in intel_engines_init() local
304 for_each_engine(engine, dev_priv, id) { in intel_engines_init()
306 &intel_engine_classes[engine->class]; in intel_engines_init()
307 int (*init)(struct intel_engine_cs *engine); in intel_engines_init()
320 err = init(engine); in intel_engines_init()
324 GEM_BUG_ON(!engine->submit_request); in intel_engines_init()
330 for_each_engine(engine, dev_priv, id) { in intel_engines_init()
332 kfree(engine); in intel_engines_init()
333 dev_priv->engine[id] = NULL; in intel_engines_init()
335 dev_priv->gt.cleanup_engine(engine); in intel_engines_init()
341 void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno) in intel_engine_init_global_seqno() argument
343 struct drm_i915_private *dev_priv = engine->i915; in intel_engine_init_global_seqno()
354 I915_WRITE(RING_SYNC_0(engine->mmio_base), 0); in intel_engine_init_global_seqno()
355 I915_WRITE(RING_SYNC_1(engine->mmio_base), 0); in intel_engine_init_global_seqno()
357 I915_WRITE(RING_SYNC_2(engine->mmio_base), 0); in intel_engine_init_global_seqno()
365 memset(semaphores + GEN8_SEMAPHORE_OFFSET(engine->id, 0), in intel_engine_init_global_seqno()
367 drm_clflush_virt_range(semaphores + GEN8_SEMAPHORE_OFFSET(engine->id, 0), in intel_engine_init_global_seqno()
372 intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno); in intel_engine_init_global_seqno()
373 clear_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted); in intel_engine_init_global_seqno()
378 intel_engine_wakeup(engine); in intel_engine_init_global_seqno()
380 GEM_BUG_ON(intel_engine_get_seqno(engine) != seqno); in intel_engine_init_global_seqno()
383 static void intel_engine_init_timeline(struct intel_engine_cs *engine) in intel_engine_init_timeline() argument
385 engine->timeline = &engine->i915->gt.global_timeline.engine[engine->id]; in intel_engine_init_timeline()
405 static void intel_engine_init_execlist(struct intel_engine_cs *engine) in intel_engine_init_execlist() argument
407 struct intel_engine_execlists * const execlists = &engine->execlists; in intel_engine_init_execlist()
409 execlists->csb_use_mmio = csb_force_mmio(engine->i915); in intel_engine_init_execlist()
428 void intel_engine_setup_common(struct intel_engine_cs *engine) in intel_engine_setup_common() argument
430 intel_engine_init_execlist(engine); in intel_engine_setup_common()
432 intel_engine_init_timeline(engine); in intel_engine_setup_common()
433 intel_engine_init_hangcheck(engine); in intel_engine_setup_common()
434 i915_gem_batch_pool_init(engine, &engine->batch_pool); in intel_engine_setup_common()
436 intel_engine_init_cmd_parser(engine); in intel_engine_setup_common()
439 int intel_engine_create_scratch(struct intel_engine_cs *engine, int size) in intel_engine_create_scratch() argument
445 WARN_ON(engine->scratch); in intel_engine_create_scratch()
447 obj = i915_gem_object_create_stolen(engine->i915, size); in intel_engine_create_scratch()
449 obj = i915_gem_object_create_internal(engine->i915, size); in intel_engine_create_scratch()
455 vma = i915_vma_instance(obj, &engine->i915->ggtt.base, NULL); in intel_engine_create_scratch()
465 engine->scratch = vma; in intel_engine_create_scratch()
467 engine->name, i915_ggtt_offset(vma)); in intel_engine_create_scratch()
475 static void intel_engine_cleanup_scratch(struct intel_engine_cs *engine) in intel_engine_cleanup_scratch() argument
477 i915_vma_unpin_and_release(&engine->scratch); in intel_engine_cleanup_scratch()
480 static void cleanup_phys_status_page(struct intel_engine_cs *engine) in cleanup_phys_status_page() argument
482 struct drm_i915_private *dev_priv = engine->i915; in cleanup_phys_status_page()
488 engine->status_page.page_addr = NULL; in cleanup_phys_status_page()
491 static void cleanup_status_page(struct intel_engine_cs *engine) in cleanup_status_page() argument
496 vma = fetch_and_zero(&engine->status_page.vma); in cleanup_status_page()
509 static int init_status_page(struct intel_engine_cs *engine) in init_status_page() argument
517 obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE); in init_status_page()
527 vma = i915_vma_instance(obj, &engine->i915->ggtt.base, NULL); in init_status_page()
534 if (!HAS_LLC(engine->i915)) in init_status_page()
558 engine->status_page.vma = vma; in init_status_page()
559 engine->status_page.ggtt_offset = i915_ggtt_offset(vma); in init_status_page()
560 engine->status_page.page_addr = memset(vaddr, 0, PAGE_SIZE); in init_status_page()
563 engine->name, i915_ggtt_offset(vma)); in init_status_page()
573 static int init_phys_status_page(struct intel_engine_cs *engine) in init_phys_status_page() argument
575 struct drm_i915_private *dev_priv = engine->i915; in init_phys_status_page()
577 GEM_BUG_ON(engine->id != RCS); in init_phys_status_page()
584 engine->status_page.page_addr = dev_priv->status_page_dmah->vaddr; in init_phys_status_page()
585 memset(engine->status_page.page_addr, 0, PAGE_SIZE); in init_phys_status_page()
601 int intel_engine_init_common(struct intel_engine_cs *engine) in intel_engine_init_common() argument
606 engine->set_default_submission(engine); in intel_engine_init_common()
615 ring = engine->context_pin(engine, engine->i915->kernel_context); in intel_engine_init_common()
623 if (INTEL_INFO(engine->i915)->has_logical_ring_preemption) { in intel_engine_init_common()
624 ring = engine->context_pin(engine, in intel_engine_init_common()
625 engine->i915->preempt_context); in intel_engine_init_common()
632 ret = intel_engine_init_breadcrumbs(engine); in intel_engine_init_common()
636 ret = i915_gem_render_state_init(engine); in intel_engine_init_common()
640 if (HWS_NEEDS_PHYSICAL(engine->i915)) in intel_engine_init_common()
641 ret = init_phys_status_page(engine); in intel_engine_init_common()
643 ret = init_status_page(engine); in intel_engine_init_common()
650 i915_gem_render_state_fini(engine); in intel_engine_init_common()
652 intel_engine_fini_breadcrumbs(engine); in intel_engine_init_common()
654 if (INTEL_INFO(engine->i915)->has_logical_ring_preemption) in intel_engine_init_common()
655 engine->context_unpin(engine, engine->i915->preempt_context); in intel_engine_init_common()
657 engine->context_unpin(engine, engine->i915->kernel_context); in intel_engine_init_common()
668 void intel_engine_cleanup_common(struct intel_engine_cs *engine) in intel_engine_cleanup_common() argument
670 intel_engine_cleanup_scratch(engine); in intel_engine_cleanup_common()
672 if (HWS_NEEDS_PHYSICAL(engine->i915)) in intel_engine_cleanup_common()
673 cleanup_phys_status_page(engine); in intel_engine_cleanup_common()
675 cleanup_status_page(engine); in intel_engine_cleanup_common()
677 i915_gem_render_state_fini(engine); in intel_engine_cleanup_common()
678 intel_engine_fini_breadcrumbs(engine); in intel_engine_cleanup_common()
679 intel_engine_cleanup_cmd_parser(engine); in intel_engine_cleanup_common()
680 i915_gem_batch_pool_fini(&engine->batch_pool); in intel_engine_cleanup_common()
682 if (INTEL_INFO(engine->i915)->has_logical_ring_preemption) in intel_engine_cleanup_common()
683 engine->context_unpin(engine, engine->i915->preempt_context); in intel_engine_cleanup_common()
684 engine->context_unpin(engine, engine->i915->kernel_context); in intel_engine_cleanup_common()
687 u64 intel_engine_get_active_head(struct intel_engine_cs *engine) in intel_engine_get_active_head() argument
689 struct drm_i915_private *dev_priv = engine->i915; in intel_engine_get_active_head()
693 acthd = I915_READ64_2x32(RING_ACTHD(engine->mmio_base), in intel_engine_get_active_head()
694 RING_ACTHD_UDW(engine->mmio_base)); in intel_engine_get_active_head()
696 acthd = I915_READ(RING_ACTHD(engine->mmio_base)); in intel_engine_get_active_head()
703 u64 intel_engine_get_last_batch_head(struct intel_engine_cs *engine) in intel_engine_get_last_batch_head() argument
705 struct drm_i915_private *dev_priv = engine->i915; in intel_engine_get_last_batch_head()
709 bbaddr = I915_READ64_2x32(RING_BBADDR(engine->mmio_base), in intel_engine_get_last_batch_head()
710 RING_BBADDR_UDW(engine->mmio_base)); in intel_engine_get_last_batch_head()
712 bbaddr = I915_READ(RING_BBADDR(engine->mmio_base)); in intel_engine_get_last_batch_head()
767 void intel_engine_get_instdone(struct intel_engine_cs *engine, in intel_engine_get_instdone() argument
770 struct drm_i915_private *dev_priv = engine->i915; in intel_engine_get_instdone()
771 u32 mmio_base = engine->mmio_base; in intel_engine_get_instdone()
781 if (engine->id != RCS) in intel_engine_get_instdone()
797 if (engine->id != RCS) in intel_engine_get_instdone()
810 if (engine->id == RCS) in intel_engine_get_instdone()
854 static int wa_ring_whitelist_reg(struct intel_engine_cs *engine, in wa_ring_whitelist_reg() argument
857 struct drm_i915_private *dev_priv = engine->i915; in wa_ring_whitelist_reg()
859 const uint32_t index = wa->hw_whitelist_count[engine->id]; in wa_ring_whitelist_reg()
864 I915_WRITE(RING_FORCE_TO_NONPRIV(engine->mmio_base, index), in wa_ring_whitelist_reg()
866 wa->hw_whitelist_count[engine->id]++; in wa_ring_whitelist_reg()
871 static int gen8_init_workarounds(struct intel_engine_cs *engine) in gen8_init_workarounds() argument
873 struct drm_i915_private *dev_priv = engine->i915; in gen8_init_workarounds()
922 static int bdw_init_workarounds(struct intel_engine_cs *engine) in bdw_init_workarounds() argument
924 struct drm_i915_private *dev_priv = engine->i915; in bdw_init_workarounds()
927 ret = gen8_init_workarounds(engine); in bdw_init_workarounds()
954 static int chv_init_workarounds(struct intel_engine_cs *engine) in chv_init_workarounds() argument
956 struct drm_i915_private *dev_priv = engine->i915; in chv_init_workarounds()
959 ret = gen8_init_workarounds(engine); in chv_init_workarounds()
972 static int gen9_init_workarounds(struct intel_engine_cs *engine) in gen9_init_workarounds() argument
974 struct drm_i915_private *dev_priv = engine->i915; in gen9_init_workarounds()
1113 ret = wa_ring_whitelist_reg(engine, GEN9_CTX_PREEMPT_REG); in gen9_init_workarounds()
1120 ret = wa_ring_whitelist_reg(engine, GEN8_CS_CHICKEN1); in gen9_init_workarounds()
1125 ret = wa_ring_whitelist_reg(engine, GEN8_HDC_CHICKEN1); in gen9_init_workarounds()
1132 static int skl_tune_iz_hashing(struct intel_engine_cs *engine) in skl_tune_iz_hashing() argument
1134 struct drm_i915_private *dev_priv = engine->i915; in skl_tune_iz_hashing()
1173 static int skl_init_workarounds(struct intel_engine_cs *engine) in skl_init_workarounds() argument
1175 struct drm_i915_private *dev_priv = engine->i915; in skl_init_workarounds()
1178 ret = gen9_init_workarounds(engine); in skl_init_workarounds()
1197 ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4); in skl_init_workarounds()
1201 return skl_tune_iz_hashing(engine); in skl_init_workarounds()
1204 static int bxt_init_workarounds(struct intel_engine_cs *engine) in bxt_init_workarounds() argument
1206 struct drm_i915_private *dev_priv = engine->i915; in bxt_init_workarounds()
1209 ret = gen9_init_workarounds(engine); in bxt_init_workarounds()
1246 ret = wa_ring_whitelist_reg(engine, GEN9_CS_DEBUG_MODE1); in bxt_init_workarounds()
1250 ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4); in bxt_init_workarounds()
1277 static int cnl_init_workarounds(struct intel_engine_cs *engine) in cnl_init_workarounds() argument
1279 struct drm_i915_private *dev_priv = engine->i915; in cnl_init_workarounds()
1326 ret= wa_ring_whitelist_reg(engine, GEN8_CS_CHICKEN1); in cnl_init_workarounds()
1333 static int kbl_init_workarounds(struct intel_engine_cs *engine) in kbl_init_workarounds() argument
1335 struct drm_i915_private *dev_priv = engine->i915; in kbl_init_workarounds()
1338 ret = gen9_init_workarounds(engine); in kbl_init_workarounds()
1377 ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4); in kbl_init_workarounds()
1384 static int glk_init_workarounds(struct intel_engine_cs *engine) in glk_init_workarounds() argument
1386 struct drm_i915_private *dev_priv = engine->i915; in glk_init_workarounds()
1389 ret = gen9_init_workarounds(engine); in glk_init_workarounds()
1394 ret = wa_ring_whitelist_reg(engine, GEN9_SLICE_COMMON_ECO_CHICKEN1); in glk_init_workarounds()
1405 static int cfl_init_workarounds(struct intel_engine_cs *engine) in cfl_init_workarounds() argument
1407 struct drm_i915_private *dev_priv = engine->i915; in cfl_init_workarounds()
1410 ret = gen9_init_workarounds(engine); in cfl_init_workarounds()
1439 int init_workarounds_ring(struct intel_engine_cs *engine) in init_workarounds_ring() argument
1441 struct drm_i915_private *dev_priv = engine->i915; in init_workarounds_ring()
1444 WARN_ON(engine->id != RCS); in init_workarounds_ring()
1447 dev_priv->workarounds.hw_whitelist_count[engine->id] = 0; in init_workarounds_ring()
1450 err = bdw_init_workarounds(engine); in init_workarounds_ring()
1452 err = chv_init_workarounds(engine); in init_workarounds_ring()
1454 err = skl_init_workarounds(engine); in init_workarounds_ring()
1456 err = bxt_init_workarounds(engine); in init_workarounds_ring()
1458 err = kbl_init_workarounds(engine); in init_workarounds_ring()
1460 err = glk_init_workarounds(engine); in init_workarounds_ring()
1462 err = cfl_init_workarounds(engine); in init_workarounds_ring()
1464 err = cnl_init_workarounds(engine); in init_workarounds_ring()
1471 engine->name, dev_priv->workarounds.count); in init_workarounds_ring()
1484 ret = req->engine->emit_flush(req, EMIT_BARRIER); in intel_ring_workarounds_emit()
1501 ret = req->engine->emit_flush(req, EMIT_BARRIER); in intel_ring_workarounds_emit()
1508 static bool ring_is_idle(struct intel_engine_cs *engine) in ring_is_idle() argument
1510 struct drm_i915_private *dev_priv = engine->i915; in ring_is_idle()
1516 if ((I915_READ_HEAD(engine) & HEAD_ADDR) != in ring_is_idle()
1517 (I915_READ_TAIL(engine) & TAIL_ADDR)) in ring_is_idle()
1521 if (INTEL_GEN(dev_priv) > 2 && !(I915_READ_MODE(engine) & MODE_IDLE)) in ring_is_idle()
1536 bool intel_engine_is_idle(struct intel_engine_cs *engine) in intel_engine_is_idle() argument
1538 struct drm_i915_private *dev_priv = engine->i915; in intel_engine_is_idle()
1545 if (!i915_seqno_passed(intel_engine_get_seqno(engine), in intel_engine_is_idle()
1546 intel_engine_last_submit(engine))) in intel_engine_is_idle()
1549 if (I915_SELFTEST_ONLY(engine->breadcrumbs.mock)) in intel_engine_is_idle()
1553 if (test_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted)) in intel_engine_is_idle()
1557 if (READ_ONCE(engine->execlists.active)) in intel_engine_is_idle()
1561 if (READ_ONCE(engine->execlists.first)) in intel_engine_is_idle()
1565 if (!ring_is_idle(engine)) in intel_engine_is_idle()
1573 struct intel_engine_cs *engine; in intel_engines_are_idle() local
1585 for_each_engine(engine, dev_priv, id) { in intel_engines_are_idle()
1586 if (!intel_engine_is_idle(engine)) in intel_engines_are_idle()
1595 struct intel_engine_cs *engine; in intel_engines_reset_default_submission() local
1598 for_each_engine(engine, i915, id) in intel_engines_reset_default_submission()
1599 engine->set_default_submission(engine); in intel_engines_reset_default_submission()
1604 struct intel_engine_cs *engine; in intel_engines_mark_idle() local
1607 for_each_engine(engine, i915, id) { in intel_engines_mark_idle()
1608 intel_engine_disarm_breadcrumbs(engine); in intel_engines_mark_idle()
1609 i915_gem_batch_pool_fini(&engine->batch_pool); in intel_engines_mark_idle()
1610 tasklet_kill(&engine->execlists.irq_tasklet); in intel_engines_mark_idle()
1611 engine->execlists.no_priolist = false; in intel_engines_mark_idle()
1615 bool intel_engine_can_store_dword(struct intel_engine_cs *engine) in intel_engine_can_store_dword() argument
1617 switch (INTEL_GEN(engine->i915)) { in intel_engine_can_store_dword()
1622 return !(IS_I915G(engine->i915) || IS_I915GM(engine->i915)); in intel_engine_can_store_dword()
1624 return engine->class != VIDEO_DECODE_CLASS; /* b0rked */ in intel_engine_can_store_dword()
1643 void intel_engine_dump(struct intel_engine_cs *engine, struct drm_printer *m) in intel_engine_dump() argument
1645 struct intel_breadcrumbs * const b = &engine->breadcrumbs; in intel_engine_dump()
1646 const struct intel_engine_execlists * const execlists = &engine->execlists; in intel_engine_dump()
1647 struct i915_gpu_error * const error = &engine->i915->gpu_error; in intel_engine_dump()
1648 struct drm_i915_private *dev_priv = engine->i915; in intel_engine_dump()
1653 drm_printf(m, "%s\n", engine->name); in intel_engine_dump()
1655 intel_engine_get_seqno(engine), in intel_engine_dump()
1656 intel_engine_last_submit(engine), in intel_engine_dump()
1657 engine->hangcheck.seqno, in intel_engine_dump()
1658 jiffies_to_msecs(jiffies - engine->hangcheck.action_timestamp), in intel_engine_dump()
1659 engine->timeline->inflight_seqnos); in intel_engine_dump()
1661 i915_reset_engine_count(error, engine)); in intel_engine_dump()
1667 rq = list_first_entry(&engine->timeline->requests, in intel_engine_dump()
1669 if (&rq->link != &engine->timeline->requests) in intel_engine_dump()
1672 rq = list_last_entry(&engine->timeline->requests, in intel_engine_dump()
1674 if (&rq->link != &engine->timeline->requests) in intel_engine_dump()
1677 rq = i915_gem_find_active_request(engine); in intel_engine_dump()
1688 I915_READ(RING_START(engine->mmio_base)), in intel_engine_dump()
1691 I915_READ(RING_HEAD(engine->mmio_base)) & HEAD_ADDR, in intel_engine_dump()
1694 I915_READ(RING_TAIL(engine->mmio_base)) & TAIL_ADDR, in intel_engine_dump()
1697 I915_READ(RING_CTL(engine->mmio_base)), in intel_engine_dump()
1698 I915_READ(RING_CTL(engine->mmio_base)) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? "waiting" : ""); in intel_engine_dump()
1702 addr = intel_engine_get_active_head(engine); in intel_engine_dump()
1705 addr = intel_engine_get_last_batch_head(engine); in intel_engine_dump()
1710 const u32 *hws = &engine->status_page.page_addr[I915_HWS_CSB_BUF0_INDEX]; in intel_engine_dump()
1715 I915_READ(RING_EXECLIST_STATUS_LO(engine)), in intel_engine_dump()
1716 I915_READ(RING_EXECLIST_STATUS_HI(engine))); in intel_engine_dump()
1718 ptr = I915_READ(RING_CONTEXT_STATUS_PTR(engine)); in intel_engine_dump()
1724 intel_read_status_page(engine, intel_hws_csb_write_index(engine->i915)), in intel_engine_dump()
1726 &engine->irq_posted))); in intel_engine_dump()
1737 I915_READ(RING_CONTEXT_STATUS_BUF_LO(engine, idx)), in intel_engine_dump()
1739 I915_READ(RING_CONTEXT_STATUS_BUF_HI(engine, idx)), in intel_engine_dump()
1761 I915_READ(RING_PP_DIR_BASE(engine))); in intel_engine_dump()
1763 I915_READ(RING_PP_DIR_BASE_READ(engine))); in intel_engine_dump()
1765 I915_READ(RING_PP_DIR_DCLV(engine))); in intel_engine_dump()
1768 spin_lock_irq(&engine->timeline->lock); in intel_engine_dump()
1769 list_for_each_entry(rq, &engine->timeline->requests, link) in intel_engine_dump()
1778 spin_unlock_irq(&engine->timeline->lock); in intel_engine_dump()