1 /* $NetBSD: nouveau_svm.c,v 1.3 2021/12/19 11:34:44 riastradh Exp $ */
2
3 /*
4 * Copyright 2018 Red Hat Inc.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 */
24 #include <sys/cdefs.h>
25 __KERNEL_RCSID(0, "$NetBSD: nouveau_svm.c,v 1.3 2021/12/19 11:34:44 riastradh Exp $");
26
27 #include "nouveau_svm.h"
28 #include "nouveau_drv.h"
29 #include "nouveau_chan.h"
30 #include "nouveau_dmem.h"
31
32 #include <nvif/notify.h>
33 #include <nvif/object.h>
34 #include <nvif/vmm.h>
35
36 #include <nvif/class.h>
37 #include <nvif/clb069.h>
38 #include <nvif/ifc00d.h>
39
40 #include <linux/sched/mm.h>
41 #include <linux/sort.h>
42 #include <linux/hmm.h>
43
44 struct nouveau_svm {
45 struct nouveau_drm *drm;
46 struct mutex mutex;
47 struct list_head inst;
48
49 struct nouveau_svm_fault_buffer {
50 int id;
51 struct nvif_object object;
52 u32 entries;
53 u32 getaddr;
54 u32 putaddr;
55 u32 get;
56 u32 put;
57 struct nvif_notify notify;
58
59 struct nouveau_svm_fault {
60 u64 inst;
61 u64 addr;
62 u64 time;
63 u32 engine;
64 u8 gpc;
65 u8 hub;
66 u8 access;
67 u8 client;
68 u8 fault;
69 struct nouveau_svmm *svmm;
70 } **fault;
71 int fault_nr;
72 } buffer[1];
73 };
74
75 #define SVM_DBG(s,f,a...) NV_DEBUG((s)->drm, "svm: "f"\n", ##a)
76 #define SVM_ERR(s,f,a...) NV_WARN((s)->drm, "svm: "f"\n", ##a)
77
78 struct nouveau_ivmm {
79 struct nouveau_svmm *svmm;
80 u64 inst;
81 struct list_head head;
82 };
83
84 static struct nouveau_ivmm *
nouveau_ivmm_find(struct nouveau_svm * svm,u64 inst)85 nouveau_ivmm_find(struct nouveau_svm *svm, u64 inst)
86 {
87 struct nouveau_ivmm *ivmm;
88 list_for_each_entry(ivmm, &svm->inst, head) {
89 if (ivmm->inst == inst)
90 return ivmm;
91 }
92 return NULL;
93 }
94
95 struct nouveau_svmm {
96 struct mmu_notifier notifier;
97 struct nouveau_vmm *vmm;
98 struct {
99 unsigned long start;
100 unsigned long limit;
101 } unmanaged;
102
103 struct mutex mutex;
104 };
105
106 #define SVMM_DBG(s,f,a...) \
107 NV_DEBUG((s)->vmm->cli->drm, "svm-%p: "f"\n", (s), ##a)
108 #define SVMM_ERR(s,f,a...) \
109 NV_WARN((s)->vmm->cli->drm, "svm-%p: "f"\n", (s), ##a)
110
111 int
nouveau_svmm_bind(struct drm_device * dev,void * data,struct drm_file * file_priv)112 nouveau_svmm_bind(struct drm_device *dev, void *data,
113 struct drm_file *file_priv)
114 {
115 struct nouveau_cli *cli = nouveau_cli(file_priv);
116 struct drm_nouveau_svm_bind *args = data;
117 unsigned target, cmd, priority;
118 unsigned long addr, end, size;
119 struct mm_struct *mm;
120
121 args->va_start &= PAGE_MASK;
122 args->va_end &= PAGE_MASK;
123
124 /* Sanity check arguments */
125 if (args->reserved0 || args->reserved1)
126 return -EINVAL;
127 if (args->header & (~NOUVEAU_SVM_BIND_VALID_MASK))
128 return -EINVAL;
129 if (args->va_start >= args->va_end)
130 return -EINVAL;
131 if (!args->npages)
132 return -EINVAL;
133
134 cmd = args->header >> NOUVEAU_SVM_BIND_COMMAND_SHIFT;
135 cmd &= NOUVEAU_SVM_BIND_COMMAND_MASK;
136 switch (cmd) {
137 case NOUVEAU_SVM_BIND_COMMAND__MIGRATE:
138 break;
139 default:
140 return -EINVAL;
141 }
142
143 priority = args->header >> NOUVEAU_SVM_BIND_PRIORITY_SHIFT;
144 priority &= NOUVEAU_SVM_BIND_PRIORITY_MASK;
145
146 /* FIXME support CPU target ie all target value < GPU_VRAM */
147 target = args->header >> NOUVEAU_SVM_BIND_TARGET_SHIFT;
148 target &= NOUVEAU_SVM_BIND_TARGET_MASK;
149 switch (target) {
150 case NOUVEAU_SVM_BIND_TARGET__GPU_VRAM:
151 break;
152 default:
153 return -EINVAL;
154 }
155
156 /*
157 * FIXME: For now refuse non 0 stride, we need to change the migrate
158 * kernel function to handle stride to avoid to create a mess within
159 * each device driver.
160 */
161 if (args->stride)
162 return -EINVAL;
163
164 size = ((unsigned long)args->npages) << PAGE_SHIFT;
165 if ((args->va_start + size) <= args->va_start)
166 return -EINVAL;
167 if ((args->va_start + size) > args->va_end)
168 return -EINVAL;
169
170 /*
171 * Ok we are ask to do something sane, for now we only support migrate
172 * commands but we will add things like memory policy (what to do on
173 * page fault) and maybe some other commands.
174 */
175
176 mm = get_task_mm(current);
177 down_read(&mm->mmap_sem);
178
179 for (addr = args->va_start, end = args->va_start + size; addr < end;) {
180 struct vm_area_struct *vma;
181 unsigned long next;
182
183 vma = find_vma_intersection(mm, addr, end);
184 if (!vma)
185 break;
186
187 next = min(vma->vm_end, end);
188 /* This is a best effort so we ignore errors */
189 nouveau_dmem_migrate_vma(cli->drm, vma, addr, next);
190 addr = next;
191 }
192
193 /*
194 * FIXME Return the number of page we have migrated, again we need to
195 * update the migrate API to return that information so that we can
196 * report it to user space.
197 */
198 args->result = 0;
199
200 up_read(&mm->mmap_sem);
201 mmput(mm);
202
203 return 0;
204 }
205
206 /* Unlink channel instance from SVMM. */
207 void
nouveau_svmm_part(struct nouveau_svmm * svmm,u64 inst)208 nouveau_svmm_part(struct nouveau_svmm *svmm, u64 inst)
209 {
210 struct nouveau_ivmm *ivmm;
211 if (svmm) {
212 mutex_lock(&svmm->vmm->cli->drm->svm->mutex);
213 ivmm = nouveau_ivmm_find(svmm->vmm->cli->drm->svm, inst);
214 if (ivmm) {
215 list_del(&ivmm->head);
216 kfree(ivmm);
217 }
218 mutex_unlock(&svmm->vmm->cli->drm->svm->mutex);
219 }
220 }
221
222 /* Link channel instance to SVMM. */
223 int
nouveau_svmm_join(struct nouveau_svmm * svmm,u64 inst)224 nouveau_svmm_join(struct nouveau_svmm *svmm, u64 inst)
225 {
226 struct nouveau_ivmm *ivmm;
227 if (svmm) {
228 if (!(ivmm = kmalloc(sizeof(*ivmm), GFP_KERNEL)))
229 return -ENOMEM;
230 ivmm->svmm = svmm;
231 ivmm->inst = inst;
232
233 mutex_lock(&svmm->vmm->cli->drm->svm->mutex);
234 list_add(&ivmm->head, &svmm->vmm->cli->drm->svm->inst);
235 mutex_unlock(&svmm->vmm->cli->drm->svm->mutex);
236 }
237 return 0;
238 }
239
240 /* Invalidate SVMM address-range on GPU. */
241 static void
nouveau_svmm_invalidate(struct nouveau_svmm * svmm,u64 start,u64 limit)242 nouveau_svmm_invalidate(struct nouveau_svmm *svmm, u64 start, u64 limit)
243 {
244 if (limit > start) {
245 bool super = svmm->vmm->vmm.object.client->super;
246 svmm->vmm->vmm.object.client->super = true;
247 nvif_object_mthd(&svmm->vmm->vmm.object, NVIF_VMM_V0_PFNCLR,
248 &(struct nvif_vmm_pfnclr_v0) {
249 .addr = start,
250 .size = limit - start,
251 }, sizeof(struct nvif_vmm_pfnclr_v0));
252 svmm->vmm->vmm.object.client->super = super;
253 }
254 }
255
256 static int
nouveau_svmm_invalidate_range_start(struct mmu_notifier * mn,const struct mmu_notifier_range * update)257 nouveau_svmm_invalidate_range_start(struct mmu_notifier *mn,
258 const struct mmu_notifier_range *update)
259 {
260 struct nouveau_svmm *svmm =
261 container_of(mn, struct nouveau_svmm, notifier);
262 unsigned long start = update->start;
263 unsigned long limit = update->end;
264
265 if (!mmu_notifier_range_blockable(update))
266 return -EAGAIN;
267
268 SVMM_DBG(svmm, "invalidate %016lx-%016lx", start, limit);
269
270 mutex_lock(&svmm->mutex);
271 if (unlikely(!svmm->vmm))
272 goto out;
273
274 if (limit > svmm->unmanaged.start && start < svmm->unmanaged.limit) {
275 if (start < svmm->unmanaged.start) {
276 nouveau_svmm_invalidate(svmm, start,
277 svmm->unmanaged.limit);
278 }
279 start = svmm->unmanaged.limit;
280 }
281
282 nouveau_svmm_invalidate(svmm, start, limit);
283
284 out:
285 mutex_unlock(&svmm->mutex);
286 return 0;
287 }
288
nouveau_svmm_free_notifier(struct mmu_notifier * mn)289 static void nouveau_svmm_free_notifier(struct mmu_notifier *mn)
290 {
291 kfree(container_of(mn, struct nouveau_svmm, notifier));
292 }
293
294 static const struct mmu_notifier_ops nouveau_mn_ops = {
295 .invalidate_range_start = nouveau_svmm_invalidate_range_start,
296 .free_notifier = nouveau_svmm_free_notifier,
297 };
298
299 void
nouveau_svmm_fini(struct nouveau_svmm ** psvmm)300 nouveau_svmm_fini(struct nouveau_svmm **psvmm)
301 {
302 struct nouveau_svmm *svmm = *psvmm;
303 if (svmm) {
304 mutex_lock(&svmm->mutex);
305 svmm->vmm = NULL;
306 mutex_unlock(&svmm->mutex);
307 mmu_notifier_put(&svmm->notifier);
308 mutex_destroy(&svmm->mutex);
309 *psvmm = NULL;
310 }
311 }
312
313 int
nouveau_svmm_init(struct drm_device * dev,void * data,struct drm_file * file_priv)314 nouveau_svmm_init(struct drm_device *dev, void *data,
315 struct drm_file *file_priv)
316 {
317 struct nouveau_cli *cli = nouveau_cli(file_priv);
318 struct nouveau_svmm *svmm;
319 struct drm_nouveau_svm_init *args = data;
320 int ret;
321
322 /* Allocate tracking for SVM-enabled VMM. */
323 if (!(svmm = kzalloc(sizeof(*svmm), GFP_KERNEL)))
324 return -ENOMEM;
325 svmm->vmm = &cli->svm;
326 svmm->unmanaged.start = args->unmanaged_addr;
327 svmm->unmanaged.limit = args->unmanaged_addr + args->unmanaged_size;
328 mutex_init(&svmm->mutex);
329
330 /* Check that SVM isn't already enabled for the client. */
331 mutex_lock(&cli->mutex);
332 if (cli->svm.cli) {
333 ret = -EBUSY;
334 goto out_free;
335 }
336
337 /* Allocate a new GPU VMM that can support SVM (managed by the
338 * client, with replayable faults enabled).
339 *
340 * All future channel/memory allocations will make use of this
341 * VMM instead of the standard one.
342 */
343 ret = nvif_vmm_init(&cli->mmu, cli->vmm.vmm.object.oclass, true,
344 args->unmanaged_addr, args->unmanaged_size,
345 &(struct gp100_vmm_v0) {
346 .fault_replay = true,
347 }, sizeof(struct gp100_vmm_v0), &cli->svm.vmm);
348 if (ret)
349 goto out_free;
350
351 down_write(¤t->mm->mmap_sem);
352 svmm->notifier.ops = &nouveau_mn_ops;
353 ret = __mmu_notifier_register(&svmm->notifier, current->mm);
354 if (ret)
355 goto out_mm_unlock;
356 /* Note, ownership of svmm transfers to mmu_notifier */
357
358 cli->svm.svmm = svmm;
359 cli->svm.cli = cli;
360 up_write(¤t->mm->mmap_sem);
361 mutex_unlock(&cli->mutex);
362 return 0;
363
364 out_mm_unlock:
365 up_write(¤t->mm->mmap_sem);
366 out_free:
367 mutex_unlock(&cli->mutex);
368 mutex_destroy(&svmm->mutex);
369 kfree(svmm);
370 return ret;
371 }
372
373 static const u64
374 nouveau_svm_pfn_flags[HMM_PFN_FLAG_MAX] = {
375 [HMM_PFN_VALID ] = NVIF_VMM_PFNMAP_V0_V,
376 [HMM_PFN_WRITE ] = NVIF_VMM_PFNMAP_V0_W,
377 [HMM_PFN_DEVICE_PRIVATE] = NVIF_VMM_PFNMAP_V0_VRAM,
378 };
379
380 static const u64
381 nouveau_svm_pfn_values[HMM_PFN_VALUE_MAX] = {
382 [HMM_PFN_ERROR ] = ~NVIF_VMM_PFNMAP_V0_V,
383 [HMM_PFN_NONE ] = NVIF_VMM_PFNMAP_V0_NONE,
384 [HMM_PFN_SPECIAL] = ~NVIF_VMM_PFNMAP_V0_V,
385 };
386
387 /* Issue fault replay for GPU to retry accesses that faulted previously. */
388 static void
nouveau_svm_fault_replay(struct nouveau_svm * svm)389 nouveau_svm_fault_replay(struct nouveau_svm *svm)
390 {
391 SVM_DBG(svm, "replay");
392 WARN_ON(nvif_object_mthd(&svm->drm->client.vmm.vmm.object,
393 GP100_VMM_VN_FAULT_REPLAY,
394 &(struct gp100_vmm_fault_replay_vn) {},
395 sizeof(struct gp100_vmm_fault_replay_vn)));
396 }
397
398 /* Cancel a replayable fault that could not be handled.
399 *
400 * Cancelling the fault will trigger recovery to reset the engine
401 * and kill the offending channel (ie. GPU SIGSEGV).
402 */
403 static void
nouveau_svm_fault_cancel(struct nouveau_svm * svm,u64 inst,u8 hub,u8 gpc,u8 client)404 nouveau_svm_fault_cancel(struct nouveau_svm *svm,
405 u64 inst, u8 hub, u8 gpc, u8 client)
406 {
407 SVM_DBG(svm, "cancel %016llx %d %02x %02x", inst, hub, gpc, client);
408 WARN_ON(nvif_object_mthd(&svm->drm->client.vmm.vmm.object,
409 GP100_VMM_VN_FAULT_CANCEL,
410 &(struct gp100_vmm_fault_cancel_v0) {
411 .hub = hub,
412 .gpc = gpc,
413 .client = client,
414 .inst = inst,
415 }, sizeof(struct gp100_vmm_fault_cancel_v0)));
416 }
417
418 static void
nouveau_svm_fault_cancel_fault(struct nouveau_svm * svm,struct nouveau_svm_fault * fault)419 nouveau_svm_fault_cancel_fault(struct nouveau_svm *svm,
420 struct nouveau_svm_fault *fault)
421 {
422 nouveau_svm_fault_cancel(svm, fault->inst,
423 fault->hub,
424 fault->gpc,
425 fault->client);
426 }
427
428 static int
nouveau_svm_fault_cmp(const void * a,const void * b)429 nouveau_svm_fault_cmp(const void *a, const void *b)
430 {
431 const struct nouveau_svm_fault *fa = *(struct nouveau_svm_fault **)a;
432 const struct nouveau_svm_fault *fb = *(struct nouveau_svm_fault **)b;
433 int ret;
434 if ((ret = (s64)fa->inst - fb->inst))
435 return ret;
436 if ((ret = (s64)fa->addr - fb->addr))
437 return ret;
438 /*XXX: atomic? */
439 return (fa->access == 0 || fa->access == 3) -
440 (fb->access == 0 || fb->access == 3);
441 }
442
443 static void
nouveau_svm_fault_cache(struct nouveau_svm * svm,struct nouveau_svm_fault_buffer * buffer,u32 offset)444 nouveau_svm_fault_cache(struct nouveau_svm *svm,
445 struct nouveau_svm_fault_buffer *buffer, u32 offset)
446 {
447 struct nvif_object *memory = &buffer->object;
448 const u32 instlo = nvif_rd32(memory, offset + 0x00);
449 const u32 insthi = nvif_rd32(memory, offset + 0x04);
450 const u32 addrlo = nvif_rd32(memory, offset + 0x08);
451 const u32 addrhi = nvif_rd32(memory, offset + 0x0c);
452 const u32 timelo = nvif_rd32(memory, offset + 0x10);
453 const u32 timehi = nvif_rd32(memory, offset + 0x14);
454 const u32 engine = nvif_rd32(memory, offset + 0x18);
455 const u32 info = nvif_rd32(memory, offset + 0x1c);
456 const u64 inst = (u64)insthi << 32 | instlo;
457 const u8 gpc = (info & 0x1f000000) >> 24;
458 const u8 hub = (info & 0x00100000) >> 20;
459 const u8 client = (info & 0x00007f00) >> 8;
460 struct nouveau_svm_fault *fault;
461
462 //XXX: i think we're supposed to spin waiting */
463 if (WARN_ON(!(info & 0x80000000)))
464 return;
465
466 nvif_mask(memory, offset + 0x1c, 0x80000000, 0x00000000);
467
468 if (!buffer->fault[buffer->fault_nr]) {
469 fault = kmalloc(sizeof(*fault), GFP_KERNEL);
470 if (WARN_ON(!fault)) {
471 nouveau_svm_fault_cancel(svm, inst, hub, gpc, client);
472 return;
473 }
474 buffer->fault[buffer->fault_nr] = fault;
475 }
476
477 fault = buffer->fault[buffer->fault_nr++];
478 fault->inst = inst;
479 fault->addr = (u64)addrhi << 32 | addrlo;
480 fault->time = (u64)timehi << 32 | timelo;
481 fault->engine = engine;
482 fault->gpc = gpc;
483 fault->hub = hub;
484 fault->access = (info & 0x000f0000) >> 16;
485 fault->client = client;
486 fault->fault = (info & 0x0000001f);
487
488 SVM_DBG(svm, "fault %016llx %016llx %02x",
489 fault->inst, fault->addr, fault->access);
490 }
491
492 struct svm_notifier {
493 struct mmu_interval_notifier notifier;
494 struct nouveau_svmm *svmm;
495 };
496
nouveau_svm_range_invalidate(struct mmu_interval_notifier * mni,const struct mmu_notifier_range * range,unsigned long cur_seq)497 static bool nouveau_svm_range_invalidate(struct mmu_interval_notifier *mni,
498 const struct mmu_notifier_range *range,
499 unsigned long cur_seq)
500 {
501 struct svm_notifier *sn =
502 container_of(mni, struct svm_notifier, notifier);
503
504 /*
505 * serializes the update to mni->invalidate_seq done by caller and
506 * prevents invalidation of the PTE from progressing while HW is being
507 * programmed. This is very hacky and only works because the normal
508 * notifier that does invalidation is always called after the range
509 * notifier.
510 */
511 if (mmu_notifier_range_blockable(range))
512 mutex_lock(&sn->svmm->mutex);
513 else if (!mutex_trylock(&sn->svmm->mutex))
514 return false;
515 mmu_interval_set_seq(mni, cur_seq);
516 mutex_unlock(&sn->svmm->mutex);
517 return true;
518 }
519
520 static const struct mmu_interval_notifier_ops nouveau_svm_mni_ops = {
521 .invalidate = nouveau_svm_range_invalidate,
522 };
523
nouveau_range_fault(struct nouveau_svmm * svmm,struct nouveau_drm * drm,void * data,u32 size,u64 * pfns,struct svm_notifier * notifier)524 static int nouveau_range_fault(struct nouveau_svmm *svmm,
525 struct nouveau_drm *drm, void *data, u32 size,
526 u64 *pfns, struct svm_notifier *notifier)
527 {
528 unsigned long timeout =
529 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
530 /* Have HMM fault pages within the fault window to the GPU. */
531 struct hmm_range range = {
532 .notifier = ¬ifier->notifier,
533 .start = notifier->notifier.interval_tree.start,
534 .end = notifier->notifier.interval_tree.last + 1,
535 .pfns = pfns,
536 .flags = nouveau_svm_pfn_flags,
537 .values = nouveau_svm_pfn_values,
538 .pfn_shift = NVIF_VMM_PFNMAP_V0_ADDR_SHIFT,
539 };
540 struct mm_struct *mm = notifier->notifier.mm;
541 long ret;
542
543 while (true) {
544 if (time_after(jiffies, timeout))
545 return -EBUSY;
546
547 range.notifier_seq = mmu_interval_read_begin(range.notifier);
548 range.default_flags = 0;
549 range.pfn_flags_mask = -1UL;
550 down_read(&mm->mmap_sem);
551 ret = hmm_range_fault(&range, 0);
552 up_read(&mm->mmap_sem);
553 if (ret <= 0) {
554 if (ret == 0 || ret == -EBUSY)
555 continue;
556 return ret;
557 }
558
559 mutex_lock(&svmm->mutex);
560 if (mmu_interval_read_retry(range.notifier,
561 range.notifier_seq)) {
562 mutex_unlock(&svmm->mutex);
563 continue;
564 }
565 break;
566 }
567
568 nouveau_dmem_convert_pfn(drm, &range);
569
570 svmm->vmm->vmm.object.client->super = true;
571 ret = nvif_object_ioctl(&svmm->vmm->vmm.object, data, size, NULL);
572 svmm->vmm->vmm.object.client->super = false;
573 mutex_unlock(&svmm->mutex);
574
575 return ret;
576 }
577
578 static int
nouveau_svm_fault(struct nvif_notify * notify)579 nouveau_svm_fault(struct nvif_notify *notify)
580 {
581 struct nouveau_svm_fault_buffer *buffer =
582 container_of(notify, typeof(*buffer), notify);
583 struct nouveau_svm *svm =
584 container_of(buffer, typeof(*svm), buffer[buffer->id]);
585 struct nvif_object *device = &svm->drm->client.device.object;
586 struct nouveau_svmm *svmm;
587 struct {
588 struct {
589 struct nvif_ioctl_v0 i;
590 struct nvif_ioctl_mthd_v0 m;
591 struct nvif_vmm_pfnmap_v0 p;
592 } i;
593 u64 phys[16];
594 } args;
595 struct vm_area_struct *vma;
596 u64 inst, start, limit;
597 int fi, fn, pi, fill;
598 int replay = 0, ret;
599
600 /* Parse available fault buffer entries into a cache, and update
601 * the GET pointer so HW can reuse the entries.
602 */
603 SVM_DBG(svm, "fault handler");
604 if (buffer->get == buffer->put) {
605 buffer->put = nvif_rd32(device, buffer->putaddr);
606 buffer->get = nvif_rd32(device, buffer->getaddr);
607 if (buffer->get == buffer->put)
608 return NVIF_NOTIFY_KEEP;
609 }
610 buffer->fault_nr = 0;
611
612 SVM_DBG(svm, "get %08x put %08x", buffer->get, buffer->put);
613 while (buffer->get != buffer->put) {
614 nouveau_svm_fault_cache(svm, buffer, buffer->get * 0x20);
615 if (++buffer->get == buffer->entries)
616 buffer->get = 0;
617 }
618 nvif_wr32(device, buffer->getaddr, buffer->get);
619 SVM_DBG(svm, "%d fault(s) pending", buffer->fault_nr);
620
621 /* Sort parsed faults by instance pointer to prevent unnecessary
622 * instance to SVMM translations, followed by address and access
623 * type to reduce the amount of work when handling the faults.
624 */
625 sort(buffer->fault, buffer->fault_nr, sizeof(*buffer->fault),
626 nouveau_svm_fault_cmp, NULL);
627
628 /* Lookup SVMM structure for each unique instance pointer. */
629 mutex_lock(&svm->mutex);
630 for (fi = 0, svmm = NULL; fi < buffer->fault_nr; fi++) {
631 if (!svmm || buffer->fault[fi]->inst != inst) {
632 struct nouveau_ivmm *ivmm =
633 nouveau_ivmm_find(svm, buffer->fault[fi]->inst);
634 svmm = ivmm ? ivmm->svmm : NULL;
635 inst = buffer->fault[fi]->inst;
636 SVM_DBG(svm, "inst %016llx -> svm-%p", inst, svmm);
637 }
638 buffer->fault[fi]->svmm = svmm;
639 }
640 mutex_unlock(&svm->mutex);
641
642 /* Process list of faults. */
643 args.i.i.version = 0;
644 args.i.i.type = NVIF_IOCTL_V0_MTHD;
645 args.i.m.version = 0;
646 args.i.m.method = NVIF_VMM_V0_PFNMAP;
647 args.i.p.version = 0;
648
649 for (fi = 0; fn = fi + 1, fi < buffer->fault_nr; fi = fn) {
650 struct svm_notifier notifier;
651 struct mm_struct *mm;
652
653 /* Cancel any faults from non-SVM channels. */
654 if (!(svmm = buffer->fault[fi]->svmm)) {
655 nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]);
656 continue;
657 }
658 SVMM_DBG(svmm, "addr %016llx", buffer->fault[fi]->addr);
659
660 /* We try and group handling of faults within a small
661 * window into a single update.
662 */
663 start = buffer->fault[fi]->addr;
664 limit = start + (ARRAY_SIZE(args.phys) << PAGE_SHIFT);
665 if (start < svmm->unmanaged.limit)
666 limit = min_t(u64, limit, svmm->unmanaged.start);
667 else
668 if (limit > svmm->unmanaged.start)
669 start = max_t(u64, start, svmm->unmanaged.limit);
670 SVMM_DBG(svmm, "wndw %016llx-%016llx", start, limit);
671
672 mm = svmm->notifier.mm;
673 if (!mmget_not_zero(mm)) {
674 nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]);
675 continue;
676 }
677
678 /* Intersect fault window with the CPU VMA, cancelling
679 * the fault if the address is invalid.
680 */
681 down_read(&mm->mmap_sem);
682 vma = find_vma_intersection(mm, start, limit);
683 if (!vma) {
684 SVMM_ERR(svmm, "wndw %016llx-%016llx", start, limit);
685 up_read(&mm->mmap_sem);
686 mmput(mm);
687 nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]);
688 continue;
689 }
690 start = max_t(u64, start, vma->vm_start);
691 limit = min_t(u64, limit, vma->vm_end);
692 up_read(&mm->mmap_sem);
693 SVMM_DBG(svmm, "wndw %016llx-%016llx", start, limit);
694
695 if (buffer->fault[fi]->addr != start) {
696 SVMM_ERR(svmm, "addr %016llx", buffer->fault[fi]->addr);
697 mmput(mm);
698 nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]);
699 continue;
700 }
701
702 /* Prepare the GPU-side update of all pages within the
703 * fault window, determining required pages and access
704 * permissions based on pending faults.
705 */
706 args.i.p.page = PAGE_SHIFT;
707 args.i.p.addr = start;
708 for (fn = fi, pi = 0;;) {
709 /* Determine required permissions based on GPU fault
710 * access flags.
711 *XXX: atomic?
712 */
713 if (buffer->fault[fn]->access != 0 /* READ. */ &&
714 buffer->fault[fn]->access != 3 /* PREFETCH. */) {
715 args.phys[pi++] = NVIF_VMM_PFNMAP_V0_V |
716 NVIF_VMM_PFNMAP_V0_W;
717 } else {
718 args.phys[pi++] = NVIF_VMM_PFNMAP_V0_V;
719 }
720 args.i.p.size = pi << PAGE_SHIFT;
721
722 /* It's okay to skip over duplicate addresses from the
723 * same SVMM as faults are ordered by access type such
724 * that only the first one needs to be handled.
725 *
726 * ie. WRITE faults appear first, thus any handling of
727 * pending READ faults will already be satisfied.
728 */
729 while (++fn < buffer->fault_nr &&
730 buffer->fault[fn]->svmm == svmm &&
731 buffer->fault[fn ]->addr ==
732 buffer->fault[fn - 1]->addr);
733
734 /* If the next fault is outside the window, or all GPU
735 * faults have been dealt with, we're done here.
736 */
737 if (fn >= buffer->fault_nr ||
738 buffer->fault[fn]->svmm != svmm ||
739 buffer->fault[fn]->addr >= limit)
740 break;
741
742 /* Fill in the gap between this fault and the next. */
743 fill = (buffer->fault[fn ]->addr -
744 buffer->fault[fn - 1]->addr) >> PAGE_SHIFT;
745 while (--fill)
746 args.phys[pi++] = NVIF_VMM_PFNMAP_V0_NONE;
747 }
748
749 SVMM_DBG(svmm, "wndw %016llx-%016llx covering %d fault(s)",
750 args.i.p.addr,
751 args.i.p.addr + args.i.p.size, fn - fi);
752
753 notifier.svmm = svmm;
754 ret = mmu_interval_notifier_insert(¬ifier.notifier,
755 svmm->notifier.mm,
756 args.i.p.addr, args.i.p.size,
757 &nouveau_svm_mni_ops);
758 if (!ret) {
759 ret = nouveau_range_fault(
760 svmm, svm->drm, &args,
761 sizeof(args.i) + pi * sizeof(args.phys[0]),
762 args.phys, ¬ifier);
763 mmu_interval_notifier_remove(¬ifier.notifier);
764 }
765 mmput(mm);
766
767 /* Cancel any faults in the window whose pages didn't manage
768 * to keep their valid bit, or stay writeable when required.
769 *
770 * If handling failed completely, cancel all faults.
771 */
772 while (fi < fn) {
773 struct nouveau_svm_fault *fault = buffer->fault[fi++];
774 pi = (fault->addr - args.i.p.addr) >> PAGE_SHIFT;
775 if (ret ||
776 !(args.phys[pi] & NVIF_VMM_PFNMAP_V0_V) ||
777 (!(args.phys[pi] & NVIF_VMM_PFNMAP_V0_W) &&
778 fault->access != 0 && fault->access != 3)) {
779 nouveau_svm_fault_cancel_fault(svm, fault);
780 continue;
781 }
782 replay++;
783 }
784 }
785
786 /* Issue fault replay to the GPU. */
787 if (replay)
788 nouveau_svm_fault_replay(svm);
789 return NVIF_NOTIFY_KEEP;
790 }
791
792 static void
nouveau_svm_fault_buffer_fini(struct nouveau_svm * svm,int id)793 nouveau_svm_fault_buffer_fini(struct nouveau_svm *svm, int id)
794 {
795 struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id];
796 nvif_notify_put(&buffer->notify);
797 }
798
799 static int
nouveau_svm_fault_buffer_init(struct nouveau_svm * svm,int id)800 nouveau_svm_fault_buffer_init(struct nouveau_svm *svm, int id)
801 {
802 struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id];
803 struct nvif_object *device = &svm->drm->client.device.object;
804 buffer->get = nvif_rd32(device, buffer->getaddr);
805 buffer->put = nvif_rd32(device, buffer->putaddr);
806 SVM_DBG(svm, "get %08x put %08x (init)", buffer->get, buffer->put);
807 return nvif_notify_get(&buffer->notify);
808 }
809
810 static void
nouveau_svm_fault_buffer_dtor(struct nouveau_svm * svm,int id)811 nouveau_svm_fault_buffer_dtor(struct nouveau_svm *svm, int id)
812 {
813 struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id];
814 int i;
815
816 if (buffer->fault) {
817 for (i = 0; buffer->fault[i] && i < buffer->entries; i++)
818 kfree(buffer->fault[i]);
819 kvfree(buffer->fault);
820 }
821
822 nouveau_svm_fault_buffer_fini(svm, id);
823
824 nvif_notify_fini(&buffer->notify);
825 nvif_object_fini(&buffer->object);
826 }
827
828 static int
nouveau_svm_fault_buffer_ctor(struct nouveau_svm * svm,s32 oclass,int id)829 nouveau_svm_fault_buffer_ctor(struct nouveau_svm *svm, s32 oclass, int id)
830 {
831 struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id];
832 struct nouveau_drm *drm = svm->drm;
833 struct nvif_object *device = &drm->client.device.object;
834 struct nvif_clb069_v0 args = {};
835 int ret;
836
837 buffer->id = id;
838
839 ret = nvif_object_init(device, 0, oclass, &args, sizeof(args),
840 &buffer->object);
841 if (ret < 0) {
842 SVM_ERR(svm, "Fault buffer allocation failed: %d", ret);
843 return ret;
844 }
845
846 nvif_object_map(&buffer->object, NULL, 0);
847 buffer->entries = args.entries;
848 buffer->getaddr = args.get;
849 buffer->putaddr = args.put;
850
851 ret = nvif_notify_init(&buffer->object, nouveau_svm_fault, true,
852 NVB069_V0_NTFY_FAULT, NULL, 0, 0,
853 &buffer->notify);
854 if (ret)
855 return ret;
856
857 buffer->fault = kvzalloc(sizeof(*buffer->fault) * buffer->entries, GFP_KERNEL);
858 if (!buffer->fault)
859 return -ENOMEM;
860
861 return nouveau_svm_fault_buffer_init(svm, id);
862 }
863
864 void
nouveau_svm_resume(struct nouveau_drm * drm)865 nouveau_svm_resume(struct nouveau_drm *drm)
866 {
867 struct nouveau_svm *svm = drm->svm;
868 if (svm)
869 nouveau_svm_fault_buffer_init(svm, 0);
870 }
871
872 void
nouveau_svm_suspend(struct nouveau_drm * drm)873 nouveau_svm_suspend(struct nouveau_drm *drm)
874 {
875 struct nouveau_svm *svm = drm->svm;
876 if (svm)
877 nouveau_svm_fault_buffer_fini(svm, 0);
878 }
879
880 void
nouveau_svm_fini(struct nouveau_drm * drm)881 nouveau_svm_fini(struct nouveau_drm *drm)
882 {
883 struct nouveau_svm *svm = drm->svm;
884 if (svm) {
885 nouveau_svm_fault_buffer_dtor(svm, 0);
886 kfree(drm->svm);
887 drm->svm = NULL;
888 }
889 }
890
891 void
nouveau_svm_init(struct nouveau_drm * drm)892 nouveau_svm_init(struct nouveau_drm *drm)
893 {
894 static const struct nvif_mclass buffers[] = {
895 { VOLTA_FAULT_BUFFER_A, 0 },
896 { MAXWELL_FAULT_BUFFER_A, 0 },
897 {}
898 };
899 struct nouveau_svm *svm;
900 int ret;
901
902 /* Disable on Volta and newer until channel recovery is fixed,
903 * otherwise clients will have a trivial way to trash the GPU
904 * for everyone.
905 */
906 if (drm->client.device.info.family > NV_DEVICE_INFO_V0_PASCAL)
907 return;
908
909 if (!(drm->svm = svm = kzalloc(sizeof(*drm->svm), GFP_KERNEL)))
910 return;
911
912 drm->svm->drm = drm;
913 mutex_init(&drm->svm->mutex);
914 INIT_LIST_HEAD(&drm->svm->inst);
915
916 ret = nvif_mclass(&drm->client.device.object, buffers);
917 if (ret < 0) {
918 SVM_DBG(svm, "No supported fault buffer class");
919 nouveau_svm_fini(drm);
920 return;
921 }
922
923 ret = nouveau_svm_fault_buffer_ctor(svm, buffers[ret].oclass, 0);
924 if (ret) {
925 nouveau_svm_fini(drm);
926 return;
927 }
928
929 SVM_DBG(svm, "Initialised");
930 }
931