1 /* $NetBSD: i915_gem_client_blt.c,v 1.4 2021/12/19 11:27:12 riastradh Exp $ */
2
3 // SPDX-License-Identifier: MIT
4 /*
5 * Copyright © 2019 Intel Corporation
6 */
7
8 #include <sys/cdefs.h>
9 __KERNEL_RCSID(0, "$NetBSD: i915_gem_client_blt.c,v 1.4 2021/12/19 11:27:12 riastradh Exp $");
10
11 #include "i915_drv.h"
12 #include "gt/intel_context.h"
13 #include "gt/intel_engine_pm.h"
14 #include "gt/intel_engine_pool.h"
15 #include "i915_gem_client_blt.h"
16 #include "i915_gem_object_blt.h"
17
18 struct i915_sleeve {
19 struct i915_vma *vma;
20 struct drm_i915_gem_object *obj;
21 struct sg_table *pages;
22 struct i915_page_sizes page_sizes;
23 };
24
vma_set_pages(struct i915_vma * vma)25 static int vma_set_pages(struct i915_vma *vma)
26 {
27 struct i915_sleeve *sleeve = vma->private;
28
29 vma->pages = sleeve->pages;
30 vma->page_sizes = sleeve->page_sizes;
31
32 return 0;
33 }
34
vma_clear_pages(struct i915_vma * vma)35 static void vma_clear_pages(struct i915_vma *vma)
36 {
37 GEM_BUG_ON(!vma->pages);
38 vma->pages = NULL;
39 }
40
vma_bind(struct i915_vma * vma,enum i915_cache_level cache_level,u32 flags)41 static int vma_bind(struct i915_vma *vma,
42 enum i915_cache_level cache_level,
43 u32 flags)
44 {
45 return vma->vm->vma_ops.bind_vma(vma, cache_level, flags);
46 }
47
vma_unbind(struct i915_vma * vma)48 static void vma_unbind(struct i915_vma *vma)
49 {
50 vma->vm->vma_ops.unbind_vma(vma);
51 }
52
53 static const struct i915_vma_ops proxy_vma_ops = {
54 .set_pages = vma_set_pages,
55 .clear_pages = vma_clear_pages,
56 .bind_vma = vma_bind,
57 .unbind_vma = vma_unbind,
58 };
59
create_sleeve(struct i915_address_space * vm,struct drm_i915_gem_object * obj,struct sg_table * pages,struct i915_page_sizes * page_sizes)60 static struct i915_sleeve *create_sleeve(struct i915_address_space *vm,
61 struct drm_i915_gem_object *obj,
62 struct sg_table *pages,
63 struct i915_page_sizes *page_sizes)
64 {
65 struct i915_sleeve *sleeve;
66 struct i915_vma *vma;
67 int err;
68
69 sleeve = kzalloc(sizeof(*sleeve), GFP_KERNEL);
70 if (!sleeve)
71 return ERR_PTR(-ENOMEM);
72
73 vma = i915_vma_instance(obj, vm, NULL);
74 if (IS_ERR(vma)) {
75 err = PTR_ERR(vma);
76 goto err_free;
77 }
78
79 vma->private = sleeve;
80 vma->ops = &proxy_vma_ops;
81
82 sleeve->vma = vma;
83 sleeve->pages = pages;
84 sleeve->page_sizes = *page_sizes;
85
86 return sleeve;
87
88 err_free:
89 kfree(sleeve);
90 return ERR_PTR(err);
91 }
92
destroy_sleeve(struct i915_sleeve * sleeve)93 static void destroy_sleeve(struct i915_sleeve *sleeve)
94 {
95 kfree(sleeve);
96 }
97
98 struct clear_pages_work {
99 struct dma_fence dma;
100 struct dma_fence_cb cb;
101 struct i915_sw_fence wait;
102 struct work_struct work;
103 struct irq_work irq_work;
104 struct i915_sleeve *sleeve;
105 struct intel_context *ce;
106 u32 value;
107 };
108
clear_pages_work_driver_name(struct dma_fence * fence)109 static const char *clear_pages_work_driver_name(struct dma_fence *fence)
110 {
111 return DRIVER_NAME;
112 }
113
clear_pages_work_timeline_name(struct dma_fence * fence)114 static const char *clear_pages_work_timeline_name(struct dma_fence *fence)
115 {
116 return "clear";
117 }
118
clear_pages_work_release(struct dma_fence * fence)119 static void clear_pages_work_release(struct dma_fence *fence)
120 {
121 struct clear_pages_work *w = container_of(fence, typeof(*w), dma);
122
123 destroy_sleeve(w->sleeve);
124
125 i915_sw_fence_fini(&w->wait);
126
127 BUILD_BUG_ON(offsetof(typeof(*w), dma));
128 dma_fence_free(&w->dma);
129 }
130
131 static const struct dma_fence_ops clear_pages_work_ops = {
132 .get_driver_name = clear_pages_work_driver_name,
133 .get_timeline_name = clear_pages_work_timeline_name,
134 .release = clear_pages_work_release,
135 };
136
clear_pages_signal_irq_worker(struct irq_work * work)137 static void clear_pages_signal_irq_worker(struct irq_work *work)
138 {
139 struct clear_pages_work *w = container_of(work, typeof(*w), irq_work);
140
141 dma_fence_signal(&w->dma);
142 dma_fence_put(&w->dma);
143 }
144
clear_pages_dma_fence_cb(struct dma_fence * fence,struct dma_fence_cb * cb)145 static void clear_pages_dma_fence_cb(struct dma_fence *fence,
146 struct dma_fence_cb *cb)
147 {
148 struct clear_pages_work *w = container_of(cb, typeof(*w), cb);
149
150 if (fence->error)
151 dma_fence_set_error(&w->dma, fence->error);
152
153 /*
154 * Push the signalling of the fence into yet another worker to avoid
155 * the nightmare locking around the fence spinlock.
156 */
157 irq_work_queue(&w->irq_work);
158 }
159
clear_pages_worker(struct work_struct * work)160 static void clear_pages_worker(struct work_struct *work)
161 {
162 struct clear_pages_work *w = container_of(work, typeof(*w), work);
163 struct drm_i915_gem_object *obj = w->sleeve->vma->obj;
164 struct i915_vma *vma = w->sleeve->vma;
165 struct i915_request *rq;
166 struct i915_vma *batch;
167 int err = w->dma.error;
168
169 if (unlikely(err))
170 goto out_signal;
171
172 if (obj->cache_dirty) {
173 if (i915_gem_object_has_struct_page(obj))
174 drm_clflush_sg(w->sleeve->pages);
175 obj->cache_dirty = false;
176 }
177 obj->read_domains = I915_GEM_GPU_DOMAINS;
178 obj->write_domain = 0;
179
180 err = i915_vma_pin(vma, 0, 0, PIN_USER);
181 if (unlikely(err))
182 goto out_signal;
183
184 batch = intel_emit_vma_fill_blt(w->ce, vma, w->value);
185 if (IS_ERR(batch)) {
186 err = PTR_ERR(batch);
187 goto out_unpin;
188 }
189
190 rq = intel_context_create_request(w->ce);
191 if (IS_ERR(rq)) {
192 err = PTR_ERR(rq);
193 goto out_batch;
194 }
195
196 /* There's no way the fence has signalled */
197 if (dma_fence_add_callback(&rq->fence, &w->cb,
198 clear_pages_dma_fence_cb))
199 GEM_BUG_ON(1);
200
201 err = intel_emit_vma_mark_active(batch, rq);
202 if (unlikely(err))
203 goto out_request;
204
205 if (w->ce->engine->emit_init_breadcrumb) {
206 err = w->ce->engine->emit_init_breadcrumb(rq);
207 if (unlikely(err))
208 goto out_request;
209 }
210
211 /*
212 * w->dma is already exported via (vma|obj)->resv we need only
213 * keep track of the GPU activity within this vma/request, and
214 * propagate the signal from the request to w->dma.
215 */
216 err = __i915_vma_move_to_active(vma, rq);
217 if (err)
218 goto out_request;
219
220 err = w->ce->engine->emit_bb_start(rq,
221 batch->node.start, batch->node.size,
222 0);
223 out_request:
224 if (unlikely(err)) {
225 i915_request_skip(rq, err);
226 err = 0;
227 }
228
229 i915_request_add(rq);
230 out_batch:
231 intel_emit_vma_release(w->ce, batch);
232 out_unpin:
233 i915_vma_unpin(vma);
234 out_signal:
235 if (unlikely(err)) {
236 dma_fence_set_error(&w->dma, err);
237 dma_fence_signal(&w->dma);
238 dma_fence_put(&w->dma);
239 }
240 }
241
242 static int __i915_sw_fence_call
clear_pages_work_notify(struct i915_sw_fence * fence,enum i915_sw_fence_notify state)243 clear_pages_work_notify(struct i915_sw_fence *fence,
244 enum i915_sw_fence_notify state)
245 {
246 struct clear_pages_work *w = container_of(fence, typeof(*w), wait);
247
248 switch (state) {
249 case FENCE_COMPLETE:
250 schedule_work(&w->work);
251 break;
252
253 case FENCE_FREE:
254 dma_fence_put(&w->dma);
255 break;
256 }
257
258 return NOTIFY_DONE;
259 }
260
261 static DEFINE_SPINLOCK(fence_lock);
262
263 /* XXX: better name please */
i915_gem_schedule_fill_pages_blt(struct drm_i915_gem_object * obj,struct intel_context * ce,struct sg_table * pages,struct i915_page_sizes * page_sizes,u32 value)264 int i915_gem_schedule_fill_pages_blt(struct drm_i915_gem_object *obj,
265 struct intel_context *ce,
266 struct sg_table *pages,
267 struct i915_page_sizes *page_sizes,
268 u32 value)
269 {
270 struct clear_pages_work *work;
271 struct i915_sleeve *sleeve;
272 int err;
273
274 sleeve = create_sleeve(ce->vm, obj, pages, page_sizes);
275 if (IS_ERR(sleeve))
276 return PTR_ERR(sleeve);
277
278 work = kmalloc(sizeof(*work), GFP_KERNEL);
279 if (!work) {
280 destroy_sleeve(sleeve);
281 return -ENOMEM;
282 }
283
284 work->value = value;
285 work->sleeve = sleeve;
286 work->ce = ce;
287
288 INIT_WORK(&work->work, clear_pages_worker);
289
290 init_irq_work(&work->irq_work, clear_pages_signal_irq_worker);
291
292 dma_fence_init(&work->dma, &clear_pages_work_ops, &fence_lock, 0, 0);
293 i915_sw_fence_init(&work->wait, clear_pages_work_notify);
294
295 i915_gem_object_lock(obj);
296 err = i915_sw_fence_await_reservation(&work->wait,
297 obj->base.resv, NULL,
298 true, I915_FENCE_TIMEOUT,
299 I915_FENCE_GFP);
300 if (err < 0) {
301 dma_fence_set_error(&work->dma, err);
302 } else {
303 dma_resv_add_excl_fence(obj->base.resv, &work->dma);
304 err = 0;
305 }
306 i915_gem_object_unlock(obj);
307
308 dma_fence_get(&work->dma);
309 i915_sw_fence_commit(&work->wait);
310
311 return err;
312 }
313
314 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
315 #include "selftests/i915_gem_client_blt.c"
316 #endif
317