1 /* $NetBSD: kfd_interrupt.c,v 1.3 2021/12/18 23:44:59 riastradh Exp $ */
2
3 /*
4 * Copyright 2014 Advanced Micro Devices, Inc.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 */
24
25 /*
26 * KFD Interrupts.
27 *
28 * AMD GPUs deliver interrupts by pushing an interrupt description onto the
29 * interrupt ring and then sending an interrupt. KGD receives the interrupt
30 * in ISR and sends us a pointer to each new entry on the interrupt ring.
31 *
32 * We generally can't process interrupt-signaled events from ISR, so we call
33 * out to each interrupt client module (currently only the scheduler) to ask if
34 * each interrupt is interesting. If they return true, then it requires further
35 * processing so we copy it to an internal interrupt ring and call each
36 * interrupt client again from a work-queue.
37 *
38 * There's no acknowledgment for the interrupts we use. The hardware simply
39 * queues a new interrupt each time without waiting.
40 *
41 * The fixed-size internal queue means that it's possible for us to lose
42 * interrupts because we have no back-pressure to the hardware.
43 */
44
45 #include <sys/cdefs.h>
46 __KERNEL_RCSID(0, "$NetBSD: kfd_interrupt.c,v 1.3 2021/12/18 23:44:59 riastradh Exp $");
47
48 #include <linux/slab.h>
49 #include <linux/device.h>
50 #include <linux/kfifo.h>
51 #include "kfd_priv.h"
52
53 #define KFD_IH_NUM_ENTRIES 8192
54
55 static void interrupt_wq(struct work_struct *);
56
kfd_interrupt_init(struct kfd_dev * kfd)57 int kfd_interrupt_init(struct kfd_dev *kfd)
58 {
59 int r;
60
61 r = kfifo_alloc(&kfd->ih_fifo,
62 KFD_IH_NUM_ENTRIES * kfd->device_info->ih_ring_entry_size,
63 GFP_KERNEL);
64 if (r) {
65 dev_err(kfd_chardev(), "Failed to allocate IH fifo\n");
66 return r;
67 }
68
69 kfd->ih_wq = alloc_workqueue("KFD IH", WQ_HIGHPRI, 1);
70 if (unlikely(!kfd->ih_wq)) {
71 kfifo_free(&kfd->ih_fifo);
72 dev_err(kfd_chardev(), "Failed to allocate KFD IH workqueue\n");
73 return -ENOMEM;
74 }
75 spin_lock_init(&kfd->interrupt_lock);
76
77 INIT_WORK(&kfd->interrupt_work, interrupt_wq);
78
79 kfd->interrupts_active = true;
80
81 /*
82 * After this function returns, the interrupt will be enabled. This
83 * barrier ensures that the interrupt running on a different processor
84 * sees all the above writes.
85 */
86 smp_wmb();
87
88 return 0;
89 }
90
kfd_interrupt_exit(struct kfd_dev * kfd)91 void kfd_interrupt_exit(struct kfd_dev *kfd)
92 {
93 /*
94 * Stop the interrupt handler from writing to the ring and scheduling
95 * workqueue items. The spinlock ensures that any interrupt running
96 * after we have unlocked sees interrupts_active = false.
97 */
98 unsigned long flags;
99
100 spin_lock_irqsave(&kfd->interrupt_lock, flags);
101 kfd->interrupts_active = false;
102 spin_unlock_irqrestore(&kfd->interrupt_lock, flags);
103
104 /*
105 * flush_work ensures that there are no outstanding
106 * work-queue items that will access interrupt_ring. New work items
107 * can't be created because we stopped interrupt handling above.
108 */
109 flush_workqueue(kfd->ih_wq);
110
111 kfifo_free(&kfd->ih_fifo);
112 }
113
114 /*
115 * Assumption: single reader/writer. This function is not re-entrant
116 */
enqueue_ih_ring_entry(struct kfd_dev * kfd,const void * ih_ring_entry)117 bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry)
118 {
119 int count;
120
121 count = kfifo_in(&kfd->ih_fifo, ih_ring_entry,
122 kfd->device_info->ih_ring_entry_size);
123 if (count != kfd->device_info->ih_ring_entry_size) {
124 dev_err_ratelimited(kfd_chardev(),
125 "Interrupt ring overflow, dropping interrupt %d\n",
126 count);
127 return false;
128 }
129
130 return true;
131 }
132
133 /*
134 * Assumption: single reader/writer. This function is not re-entrant
135 */
dequeue_ih_ring_entry(struct kfd_dev * kfd,void * ih_ring_entry)136 static bool dequeue_ih_ring_entry(struct kfd_dev *kfd, void *ih_ring_entry)
137 {
138 int count;
139
140 count = kfifo_out(&kfd->ih_fifo, ih_ring_entry,
141 kfd->device_info->ih_ring_entry_size);
142
143 WARN_ON(count && count != kfd->device_info->ih_ring_entry_size);
144
145 return count == kfd->device_info->ih_ring_entry_size;
146 }
147
interrupt_wq(struct work_struct * work)148 static void interrupt_wq(struct work_struct *work)
149 {
150 struct kfd_dev *dev = container_of(work, struct kfd_dev,
151 interrupt_work);
152 uint32_t ih_ring_entry[KFD_MAX_RING_ENTRY_SIZE];
153
154 if (dev->device_info->ih_ring_entry_size > sizeof(ih_ring_entry)) {
155 dev_err_once(kfd_chardev(), "Ring entry too small\n");
156 return;
157 }
158
159 while (dequeue_ih_ring_entry(dev, ih_ring_entry))
160 dev->device_info->event_interrupt_class->interrupt_wq(dev,
161 ih_ring_entry);
162 }
163
interrupt_is_wanted(struct kfd_dev * dev,const uint32_t * ih_ring_entry,uint32_t * patched_ihre,bool * flag)164 bool interrupt_is_wanted(struct kfd_dev *dev,
165 const uint32_t *ih_ring_entry,
166 uint32_t *patched_ihre, bool *flag)
167 {
168 /* integer and bitwise OR so there is no boolean short-circuiting */
169 unsigned int wanted = 0;
170
171 wanted |= dev->device_info->event_interrupt_class->interrupt_isr(dev,
172 ih_ring_entry, patched_ihre, flag);
173
174 return wanted != 0;
175 }
176