1 /* 2 * Copyright © 2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/kthread.h> 26 #include <trace/events/dma_fence.h> 27 #ifdef notyet 28 #include <uapi/linux/sched/types.h> 29 #endif 30 31 #include "i915_drv.h" 32 #include "i915_trace.h" 33 #include "intel_breadcrumbs.h" 34 #include "intel_context.h" 35 #include "intel_engine_pm.h" 36 #include "intel_gt_pm.h" 37 #include "intel_gt_requests.h" 38 39 static bool irq_enable(struct intel_engine_cs *engine) 40 { 41 if (!engine->irq_enable) 42 return false; 43 44 /* Caller disables interrupts */ 45 spin_lock(&engine->gt->irq_lock); 46 engine->irq_enable(engine); 47 spin_unlock(&engine->gt->irq_lock); 48 49 return true; 50 } 51 52 static void irq_disable(struct intel_engine_cs *engine) 53 { 54 if (!engine->irq_disable) 55 return; 56 57 /* Caller disables interrupts */ 58 spin_lock(&engine->gt->irq_lock); 59 engine->irq_disable(engine); 60 spin_unlock(&engine->gt->irq_lock); 61 } 62 63 static void __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b) 64 { 65 /* 66 * Since we are waiting on a request, the GPU should be busy 67 * and should have its own rpm reference. 68 */ 69 if (GEM_WARN_ON(!intel_gt_pm_get_if_awake(b->irq_engine->gt))) 70 return; 71 72 /* 73 * The breadcrumb irq will be disarmed on the interrupt after the 74 * waiters are signaled. This gives us a single interrupt window in 75 * which we can add a new waiter and avoid the cost of re-enabling 76 * the irq. 77 */ 78 WRITE_ONCE(b->irq_armed, true); 79 80 /* Requests may have completed before we could enable the interrupt. */ 81 if (!b->irq_enabled++ && irq_enable(b->irq_engine)) 82 irq_work_queue(&b->irq_work); 83 } 84 85 static void intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b) 86 { 87 if (!b->irq_engine) 88 return; 89 90 spin_lock(&b->irq_lock); 91 if (!b->irq_armed) 92 __intel_breadcrumbs_arm_irq(b); 93 spin_unlock(&b->irq_lock); 94 } 95 96 static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b) 97 { 98 GEM_BUG_ON(!b->irq_enabled); 99 if (!--b->irq_enabled) 100 irq_disable(b->irq_engine); 101 102 WRITE_ONCE(b->irq_armed, false); 103 intel_gt_pm_put_async(b->irq_engine->gt); 104 } 105 106 static void intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b) 107 { 108 spin_lock(&b->irq_lock); 109 if (b->irq_armed) 110 __intel_breadcrumbs_disarm_irq(b); 111 spin_unlock(&b->irq_lock); 112 } 113 114 static void add_signaling_context(struct intel_breadcrumbs *b, 115 struct intel_context *ce) 116 { 117 lockdep_assert_held(&ce->signal_lock); 118 119 spin_lock(&b->signalers_lock); 120 list_add_rcu(&ce->signal_link, &b->signalers); 121 spin_unlock(&b->signalers_lock); 122 } 123 124 static bool remove_signaling_context(struct intel_breadcrumbs *b, 125 struct intel_context *ce) 126 { 127 lockdep_assert_held(&ce->signal_lock); 128 129 if (!list_empty(&ce->signals)) 130 return false; 131 132 spin_lock(&b->signalers_lock); 133 list_del_rcu(&ce->signal_link); 134 spin_unlock(&b->signalers_lock); 135 136 return true; 137 } 138 139 __maybe_unused static bool 140 check_signal_order(struct intel_context *ce, struct i915_request *rq) 141 { 142 if (rq->context != ce) 143 return false; 144 145 if (!list_is_last(&rq->signal_link, &ce->signals) && 146 i915_seqno_passed(rq->fence.seqno, 147 list_next_entry(rq, signal_link)->fence.seqno)) 148 return false; 149 150 if (!list_is_first(&rq->signal_link, &ce->signals) && 151 i915_seqno_passed(list_prev_entry(rq, signal_link)->fence.seqno, 152 rq->fence.seqno)) 153 return false; 154 155 return true; 156 } 157 158 static bool 159 __dma_fence_signal(struct dma_fence *fence) 160 { 161 return !test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags); 162 } 163 164 static void 165 __dma_fence_signal__timestamp(struct dma_fence *fence, ktime_t timestamp) 166 { 167 fence->timestamp = timestamp; 168 set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags); 169 trace_dma_fence_signaled(fence); 170 } 171 172 static void 173 __dma_fence_signal__notify(struct dma_fence *fence, 174 const struct list_head *list) 175 { 176 struct dma_fence_cb *cur, *tmp; 177 178 lockdep_assert_held(fence->lock); 179 180 list_for_each_entry_safe(cur, tmp, list, node) { 181 INIT_LIST_HEAD(&cur->node); 182 cur->func(fence, cur); 183 } 184 } 185 186 static void add_retire(struct intel_breadcrumbs *b, struct intel_timeline *tl) 187 { 188 if (b->irq_engine) 189 intel_engine_add_retire(b->irq_engine, tl); 190 } 191 192 static bool __signal_request(struct i915_request *rq) 193 { 194 GEM_BUG_ON(test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)); 195 196 if (!__dma_fence_signal(&rq->fence)) { 197 i915_request_put(rq); 198 return false; 199 } 200 201 return true; 202 } 203 204 static struct llist_node * 205 slist_add(struct llist_node *node, struct llist_node *head) 206 { 207 node->next = head; 208 return node; 209 } 210 211 static void signal_irq_work(struct irq_work *work) 212 { 213 struct intel_breadcrumbs *b = container_of(work, typeof(*b), irq_work); 214 const ktime_t timestamp = ktime_get(); 215 struct llist_node *signal, *sn; 216 struct intel_context *ce; 217 218 signal = NULL; 219 if (unlikely(!llist_empty(&b->signaled_requests))) 220 signal = llist_del_all(&b->signaled_requests); 221 222 /* 223 * Keep the irq armed until the interrupt after all listeners are gone. 224 * 225 * Enabling/disabling the interrupt is rather costly, roughly a couple 226 * of hundred microseconds. If we are proactive and enable/disable 227 * the interrupt around every request that wants a breadcrumb, we 228 * quickly drown in the extra orders of magnitude of latency imposed 229 * on request submission. 230 * 231 * So we try to be lazy, and keep the interrupts enabled until no 232 * more listeners appear within a breadcrumb interrupt interval (that 233 * is until a request completes that no one cares about). The 234 * observation is that listeners come in batches, and will often 235 * listen to a bunch of requests in succession. Though note on icl+, 236 * interrupts are always enabled due to concerns with rc6 being 237 * dysfunctional with per-engine interrupt masking. 238 * 239 * We also try to avoid raising too many interrupts, as they may 240 * be generated by userspace batches and it is unfortunately rather 241 * too easy to drown the CPU under a flood of GPU interrupts. Thus 242 * whenever no one appears to be listening, we turn off the interrupts. 243 * Fewer interrupts should conserve power -- at the very least, fewer 244 * interrupt draw less ire from other users of the system and tools 245 * like powertop. 246 */ 247 if (!signal && READ_ONCE(b->irq_armed) && list_empty(&b->signalers)) 248 intel_breadcrumbs_disarm_irq(b); 249 250 rcu_read_lock(); 251 list_for_each_entry_rcu(ce, &b->signalers, signal_link) { 252 struct i915_request *rq; 253 254 list_for_each_entry_rcu(rq, &ce->signals, signal_link) { 255 bool release; 256 257 if (!__i915_request_is_complete(rq)) 258 break; 259 260 if (!test_and_clear_bit(I915_FENCE_FLAG_SIGNAL, 261 &rq->fence.flags)) 262 break; 263 264 /* 265 * Queue for execution after dropping the signaling 266 * spinlock as the callback chain may end up adding 267 * more signalers to the same context or engine. 268 */ 269 spin_lock(&ce->signal_lock); 270 list_del_rcu(&rq->signal_link); 271 release = remove_signaling_context(b, ce); 272 spin_unlock(&ce->signal_lock); 273 274 if (__signal_request(rq)) 275 /* We own signal_node now, xfer to local list */ 276 signal = slist_add(&rq->signal_node, signal); 277 278 if (release) { 279 add_retire(b, ce->timeline); 280 intel_context_put(ce); 281 } 282 } 283 } 284 rcu_read_unlock(); 285 286 llist_for_each_safe(signal, sn, signal) { 287 struct i915_request *rq = 288 llist_entry(signal, typeof(*rq), signal_node); 289 struct list_head cb_list; 290 291 spin_lock(&rq->lock); 292 list_replace(&rq->fence.cb_list, &cb_list); 293 __dma_fence_signal__timestamp(&rq->fence, timestamp); 294 __dma_fence_signal__notify(&rq->fence, &cb_list); 295 spin_unlock(&rq->lock); 296 297 i915_request_put(rq); 298 } 299 300 if (!READ_ONCE(b->irq_armed) && !list_empty(&b->signalers)) 301 intel_breadcrumbs_arm_irq(b); 302 } 303 304 struct intel_breadcrumbs * 305 intel_breadcrumbs_create(struct intel_engine_cs *irq_engine) 306 { 307 struct intel_breadcrumbs *b; 308 309 b = kzalloc(sizeof(*b), GFP_KERNEL); 310 if (!b) 311 return NULL; 312 313 b->irq_engine = irq_engine; 314 315 mtx_init(&b->signalers_lock, IPL_NONE); 316 INIT_LIST_HEAD(&b->signalers); 317 init_llist_head(&b->signaled_requests); 318 319 mtx_init(&b->irq_lock, IPL_TTY); 320 init_irq_work(&b->irq_work, signal_irq_work); 321 322 return b; 323 } 324 325 void intel_breadcrumbs_reset(struct intel_breadcrumbs *b) 326 { 327 unsigned long flags; 328 329 if (!b->irq_engine) 330 return; 331 332 spin_lock_irqsave(&b->irq_lock, flags); 333 334 if (b->irq_enabled) 335 irq_enable(b->irq_engine); 336 else 337 irq_disable(b->irq_engine); 338 339 spin_unlock_irqrestore(&b->irq_lock, flags); 340 } 341 342 void intel_breadcrumbs_park(struct intel_breadcrumbs *b) 343 { 344 /* Kick the work once more to drain the signalers */ 345 irq_work_sync(&b->irq_work); 346 while (unlikely(READ_ONCE(b->irq_armed))) { 347 local_irq_disable(); 348 signal_irq_work(&b->irq_work); 349 local_irq_enable(); 350 cond_resched(); 351 } 352 GEM_BUG_ON(!list_empty(&b->signalers)); 353 } 354 355 void intel_breadcrumbs_free(struct intel_breadcrumbs *b) 356 { 357 irq_work_sync(&b->irq_work); 358 GEM_BUG_ON(!list_empty(&b->signalers)); 359 GEM_BUG_ON(b->irq_armed); 360 kfree(b); 361 } 362 363 static void insert_breadcrumb(struct i915_request *rq) 364 { 365 struct intel_breadcrumbs *b = READ_ONCE(rq->engine)->breadcrumbs; 366 struct intel_context *ce = rq->context; 367 struct list_head *pos; 368 369 if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) 370 return; 371 372 i915_request_get(rq); 373 374 /* 375 * If the request is already completed, we can transfer it 376 * straight onto a signaled list, and queue the irq worker for 377 * its signal completion. 378 */ 379 if (__i915_request_is_complete(rq)) { 380 if (__signal_request(rq) && 381 llist_add(&rq->signal_node, &b->signaled_requests)) 382 irq_work_queue(&b->irq_work); 383 return; 384 } 385 386 if (list_empty(&ce->signals)) { 387 intel_context_get(ce); 388 add_signaling_context(b, ce); 389 pos = &ce->signals; 390 } else { 391 /* 392 * We keep the seqno in retirement order, so we can break 393 * inside intel_engine_signal_breadcrumbs as soon as we've 394 * passed the last completed request (or seen a request that 395 * hasn't event started). We could walk the timeline->requests, 396 * but keeping a separate signalers_list has the advantage of 397 * hopefully being much smaller than the full list and so 398 * provides faster iteration and detection when there are no 399 * more interrupts required for this context. 400 * 401 * We typically expect to add new signalers in order, so we 402 * start looking for our insertion point from the tail of 403 * the list. 404 */ 405 list_for_each_prev(pos, &ce->signals) { 406 struct i915_request *it = 407 list_entry(pos, typeof(*it), signal_link); 408 409 if (i915_seqno_passed(rq->fence.seqno, it->fence.seqno)) 410 break; 411 } 412 } 413 list_add_rcu(&rq->signal_link, pos); 414 GEM_BUG_ON(!check_signal_order(ce, rq)); 415 GEM_BUG_ON(test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags)); 416 set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); 417 418 /* 419 * Defer enabling the interrupt to after HW submission and recheck 420 * the request as it may have completed and raised the interrupt as 421 * we were attaching it into the lists. 422 */ 423 irq_work_queue(&b->irq_work); 424 } 425 426 bool i915_request_enable_breadcrumb(struct i915_request *rq) 427 { 428 struct intel_context *ce = rq->context; 429 430 /* Serialises with i915_request_retire() using rq->lock */ 431 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags)) 432 return true; 433 434 /* 435 * Peek at i915_request_submit()/i915_request_unsubmit() status. 436 * 437 * If the request is not yet active (and not signaled), we will 438 * attach the breadcrumb later. 439 */ 440 if (!test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) 441 return true; 442 443 spin_lock(&ce->signal_lock); 444 if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) 445 insert_breadcrumb(rq); 446 spin_unlock(&ce->signal_lock); 447 448 return true; 449 } 450 451 void i915_request_cancel_breadcrumb(struct i915_request *rq) 452 { 453 struct intel_context *ce = rq->context; 454 bool release; 455 456 spin_lock(&ce->signal_lock); 457 if (!test_and_clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) { 458 spin_unlock(&ce->signal_lock); 459 return; 460 } 461 462 list_del_rcu(&rq->signal_link); 463 release = remove_signaling_context(rq->engine->breadcrumbs, ce); 464 spin_unlock(&ce->signal_lock); 465 if (release) 466 intel_context_put(ce); 467 468 i915_request_put(rq); 469 } 470 471 static void print_signals(struct intel_breadcrumbs *b, struct drm_printer *p) 472 { 473 struct intel_context *ce; 474 struct i915_request *rq; 475 476 drm_printf(p, "Signals:\n"); 477 478 rcu_read_lock(); 479 list_for_each_entry_rcu(ce, &b->signalers, signal_link) { 480 list_for_each_entry_rcu(rq, &ce->signals, signal_link) 481 drm_printf(p, "\t[%llx:%llx%s] @ %dms\n", 482 rq->fence.context, rq->fence.seqno, 483 i915_request_completed(rq) ? "!" : 484 i915_request_started(rq) ? "*" : 485 "", 486 jiffies_to_msecs(jiffies - rq->emitted_jiffies)); 487 } 488 rcu_read_unlock(); 489 } 490 491 void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine, 492 struct drm_printer *p) 493 { 494 struct intel_breadcrumbs *b; 495 496 b = engine->breadcrumbs; 497 if (!b) 498 return; 499 500 drm_printf(p, "IRQ: %s\n", enableddisabled(b->irq_armed)); 501 if (!list_empty(&b->signalers)) 502 print_signals(b, p); 503 } 504