1 /* $NetBSD: nouveau_dmem.c,v 1.3 2021/12/19 11:34:44 riastradh Exp $ */
2
3 /*
4 * Copyright 2018 Red Hat Inc.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 */
24 #include <sys/cdefs.h>
25 __KERNEL_RCSID(0, "$NetBSD: nouveau_dmem.c,v 1.3 2021/12/19 11:34:44 riastradh Exp $");
26
27 #include "nouveau_dmem.h"
28 #include "nouveau_drv.h"
29 #include "nouveau_chan.h"
30 #include "nouveau_dma.h"
31 #include "nouveau_mem.h"
32 #include "nouveau_bo.h"
33
34 #include <nvif/class.h>
35 #include <nvif/object.h>
36 #include <nvif/if500b.h>
37 #include <nvif/if900b.h>
38
39 #include <linux/sched/mm.h>
40 #include <linux/hmm.h>
41
42 /*
43 * FIXME: this is ugly right now we are using TTM to allocate vram and we pin
44 * it in vram while in use. We likely want to overhaul memory management for
45 * nouveau to be more page like (not necessarily with system page size but a
46 * bigger page size) at lowest level and have some shim layer on top that would
47 * provide the same functionality as TTM.
48 */
49 #define DMEM_CHUNK_SIZE (2UL << 20)
50 #define DMEM_CHUNK_NPAGES (DMEM_CHUNK_SIZE >> PAGE_SHIFT)
51
52 enum nouveau_aper {
53 NOUVEAU_APER_VIRT,
54 NOUVEAU_APER_VRAM,
55 NOUVEAU_APER_HOST,
56 };
57
58 typedef int (*nouveau_migrate_copy_t)(struct nouveau_drm *drm, u64 npages,
59 enum nouveau_aper, u64 dst_addr,
60 enum nouveau_aper, u64 src_addr);
61
62 struct nouveau_dmem_chunk {
63 struct list_head list;
64 struct nouveau_bo *bo;
65 struct nouveau_drm *drm;
66 unsigned long pfn_first;
67 unsigned long callocated;
68 unsigned long bitmap[BITS_TO_LONGS(DMEM_CHUNK_NPAGES)];
69 spinlock_t lock;
70 };
71
72 struct nouveau_dmem_migrate {
73 nouveau_migrate_copy_t copy_func;
74 struct nouveau_channel *chan;
75 };
76
77 struct nouveau_dmem {
78 struct nouveau_drm *drm;
79 struct dev_pagemap pagemap;
80 struct nouveau_dmem_migrate migrate;
81 struct list_head chunk_free;
82 struct list_head chunk_full;
83 struct list_head chunk_empty;
84 struct mutex mutex;
85 };
86
page_to_dmem(struct page * page)87 static inline struct nouveau_dmem *page_to_dmem(struct page *page)
88 {
89 return container_of(page->pgmap, struct nouveau_dmem, pagemap);
90 }
91
nouveau_dmem_page_addr(struct page * page)92 static unsigned long nouveau_dmem_page_addr(struct page *page)
93 {
94 struct nouveau_dmem_chunk *chunk = page->zone_device_data;
95 unsigned long idx = page_to_pfn(page) - chunk->pfn_first;
96
97 return (idx << PAGE_SHIFT) + chunk->bo->bo.offset;
98 }
99
nouveau_dmem_page_free(struct page * page)100 static void nouveau_dmem_page_free(struct page *page)
101 {
102 struct nouveau_dmem_chunk *chunk = page->zone_device_data;
103 unsigned long idx = page_to_pfn(page) - chunk->pfn_first;
104
105 /*
106 * FIXME:
107 *
108 * This is really a bad example, we need to overhaul nouveau memory
109 * management to be more page focus and allow lighter locking scheme
110 * to be use in the process.
111 */
112 spin_lock(&chunk->lock);
113 clear_bit(idx, chunk->bitmap);
114 WARN_ON(!chunk->callocated);
115 chunk->callocated--;
116 /*
117 * FIXME when chunk->callocated reach 0 we should add the chunk to
118 * a reclaim list so that it can be freed in case of memory pressure.
119 */
120 spin_unlock(&chunk->lock);
121 }
122
nouveau_dmem_fence_done(struct nouveau_fence ** fence)123 static void nouveau_dmem_fence_done(struct nouveau_fence **fence)
124 {
125 if (fence) {
126 nouveau_fence_wait(*fence, true, false);
127 nouveau_fence_unref(fence);
128 } else {
129 /*
130 * FIXME wait for channel to be IDLE before calling finalizing
131 * the hmem object.
132 */
133 }
134 }
135
nouveau_dmem_fault_copy_one(struct nouveau_drm * drm,struct vm_fault * vmf,struct migrate_vma * args,dma_addr_t * dma_addr)136 static vm_fault_t nouveau_dmem_fault_copy_one(struct nouveau_drm *drm,
137 struct vm_fault *vmf, struct migrate_vma *args,
138 dma_addr_t *dma_addr)
139 {
140 struct device *dev = drm->dev->dev;
141 struct page *dpage, *spage;
142
143 spage = migrate_pfn_to_page(args->src[0]);
144 if (!spage || !(args->src[0] & MIGRATE_PFN_MIGRATE))
145 return 0;
146
147 dpage = alloc_page_vma(GFP_HIGHUSER, vmf->vma, vmf->address);
148 if (!dpage)
149 return VM_FAULT_SIGBUS;
150 lock_page(dpage);
151
152 *dma_addr = dma_map_page(dev, dpage, 0, PAGE_SIZE, DMA_BIDIRECTIONAL);
153 if (dma_mapping_error(dev, *dma_addr))
154 goto error_free_page;
155
156 if (drm->dmem->migrate.copy_func(drm, 1, NOUVEAU_APER_HOST, *dma_addr,
157 NOUVEAU_APER_VRAM, nouveau_dmem_page_addr(spage)))
158 goto error_dma_unmap;
159
160 args->dst[0] = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED;
161 return 0;
162
163 error_dma_unmap:
164 dma_unmap_page(dev, *dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
165 error_free_page:
166 __free_page(dpage);
167 return VM_FAULT_SIGBUS;
168 }
169
nouveau_dmem_migrate_to_ram(struct vm_fault * vmf)170 static vm_fault_t nouveau_dmem_migrate_to_ram(struct vm_fault *vmf)
171 {
172 struct nouveau_dmem *dmem = page_to_dmem(vmf->page);
173 struct nouveau_drm *drm = dmem->drm;
174 struct nouveau_fence *fence;
175 unsigned long src = 0, dst = 0;
176 dma_addr_t dma_addr = 0;
177 vm_fault_t ret;
178 struct migrate_vma args = {
179 .vma = vmf->vma,
180 .start = vmf->address,
181 .end = vmf->address + PAGE_SIZE,
182 .src = &src,
183 .dst = &dst,
184 };
185
186 /*
187 * FIXME what we really want is to find some heuristic to migrate more
188 * than just one page on CPU fault. When such fault happens it is very
189 * likely that more surrounding page will CPU fault too.
190 */
191 if (migrate_vma_setup(&args) < 0)
192 return VM_FAULT_SIGBUS;
193 if (!args.cpages)
194 return 0;
195
196 ret = nouveau_dmem_fault_copy_one(drm, vmf, &args, &dma_addr);
197 if (ret || dst == 0)
198 goto done;
199
200 nouveau_fence_new(dmem->migrate.chan, false, &fence);
201 migrate_vma_pages(&args);
202 nouveau_dmem_fence_done(&fence);
203 dma_unmap_page(drm->dev->dev, dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
204 done:
205 migrate_vma_finalize(&args);
206 return ret;
207 }
208
209 static const struct dev_pagemap_ops nouveau_dmem_pagemap_ops = {
210 .page_free = nouveau_dmem_page_free,
211 .migrate_to_ram = nouveau_dmem_migrate_to_ram,
212 };
213
214 static int
nouveau_dmem_chunk_alloc(struct nouveau_drm * drm)215 nouveau_dmem_chunk_alloc(struct nouveau_drm *drm)
216 {
217 struct nouveau_dmem_chunk *chunk;
218 int ret;
219
220 if (drm->dmem == NULL)
221 return -EINVAL;
222
223 mutex_lock(&drm->dmem->mutex);
224 chunk = list_first_entry_or_null(&drm->dmem->chunk_empty,
225 struct nouveau_dmem_chunk,
226 list);
227 if (chunk == NULL) {
228 mutex_unlock(&drm->dmem->mutex);
229 return -ENOMEM;
230 }
231
232 list_del(&chunk->list);
233 mutex_unlock(&drm->dmem->mutex);
234
235 ret = nouveau_bo_new(&drm->client, DMEM_CHUNK_SIZE, 0,
236 TTM_PL_FLAG_VRAM, 0, 0, NULL, NULL,
237 &chunk->bo);
238 if (ret)
239 goto out;
240
241 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
242 if (ret) {
243 nouveau_bo_ref(NULL, &chunk->bo);
244 goto out;
245 }
246
247 bitmap_zero(chunk->bitmap, DMEM_CHUNK_NPAGES);
248 spin_lock_init(&chunk->lock);
249
250 out:
251 mutex_lock(&drm->dmem->mutex);
252 if (chunk->bo)
253 list_add(&chunk->list, &drm->dmem->chunk_empty);
254 else
255 list_add_tail(&chunk->list, &drm->dmem->chunk_empty);
256 mutex_unlock(&drm->dmem->mutex);
257
258 return ret;
259 }
260
261 static struct nouveau_dmem_chunk *
nouveau_dmem_chunk_first_free_locked(struct nouveau_drm * drm)262 nouveau_dmem_chunk_first_free_locked(struct nouveau_drm *drm)
263 {
264 struct nouveau_dmem_chunk *chunk;
265
266 chunk = list_first_entry_or_null(&drm->dmem->chunk_free,
267 struct nouveau_dmem_chunk,
268 list);
269 if (chunk)
270 return chunk;
271
272 chunk = list_first_entry_or_null(&drm->dmem->chunk_empty,
273 struct nouveau_dmem_chunk,
274 list);
275 if (chunk->bo)
276 return chunk;
277
278 return NULL;
279 }
280
281 static int
nouveau_dmem_pages_alloc(struct nouveau_drm * drm,unsigned long npages,unsigned long * pages)282 nouveau_dmem_pages_alloc(struct nouveau_drm *drm,
283 unsigned long npages,
284 unsigned long *pages)
285 {
286 struct nouveau_dmem_chunk *chunk;
287 unsigned long c;
288 int ret;
289
290 memset(pages, 0xff, npages * sizeof(*pages));
291
292 mutex_lock(&drm->dmem->mutex);
293 for (c = 0; c < npages;) {
294 unsigned long i;
295
296 chunk = nouveau_dmem_chunk_first_free_locked(drm);
297 if (chunk == NULL) {
298 mutex_unlock(&drm->dmem->mutex);
299 ret = nouveau_dmem_chunk_alloc(drm);
300 if (ret) {
301 if (c)
302 return 0;
303 return ret;
304 }
305 mutex_lock(&drm->dmem->mutex);
306 continue;
307 }
308
309 spin_lock(&chunk->lock);
310 i = find_first_zero_bit(chunk->bitmap, DMEM_CHUNK_NPAGES);
311 while (i < DMEM_CHUNK_NPAGES && c < npages) {
312 pages[c] = chunk->pfn_first + i;
313 set_bit(i, chunk->bitmap);
314 chunk->callocated++;
315 c++;
316
317 i = find_next_zero_bit(chunk->bitmap,
318 DMEM_CHUNK_NPAGES, i);
319 }
320 spin_unlock(&chunk->lock);
321 }
322 mutex_unlock(&drm->dmem->mutex);
323
324 return 0;
325 }
326
327 static struct page *
nouveau_dmem_page_alloc_locked(struct nouveau_drm * drm)328 nouveau_dmem_page_alloc_locked(struct nouveau_drm *drm)
329 {
330 unsigned long pfns[1];
331 struct page *page;
332 int ret;
333
334 /* FIXME stop all the miss-match API ... */
335 ret = nouveau_dmem_pages_alloc(drm, 1, pfns);
336 if (ret)
337 return NULL;
338
339 page = pfn_to_page(pfns[0]);
340 get_page(page);
341 lock_page(page);
342 return page;
343 }
344
345 static void
nouveau_dmem_page_free_locked(struct nouveau_drm * drm,struct page * page)346 nouveau_dmem_page_free_locked(struct nouveau_drm *drm, struct page *page)
347 {
348 unlock_page(page);
349 put_page(page);
350 }
351
352 void
nouveau_dmem_resume(struct nouveau_drm * drm)353 nouveau_dmem_resume(struct nouveau_drm *drm)
354 {
355 struct nouveau_dmem_chunk *chunk;
356 int ret;
357
358 if (drm->dmem == NULL)
359 return;
360
361 mutex_lock(&drm->dmem->mutex);
362 list_for_each_entry (chunk, &drm->dmem->chunk_free, list) {
363 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
364 /* FIXME handle pin failure */
365 WARN_ON(ret);
366 }
367 list_for_each_entry (chunk, &drm->dmem->chunk_full, list) {
368 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
369 /* FIXME handle pin failure */
370 WARN_ON(ret);
371 }
372 mutex_unlock(&drm->dmem->mutex);
373 }
374
375 void
nouveau_dmem_suspend(struct nouveau_drm * drm)376 nouveau_dmem_suspend(struct nouveau_drm *drm)
377 {
378 struct nouveau_dmem_chunk *chunk;
379
380 if (drm->dmem == NULL)
381 return;
382
383 mutex_lock(&drm->dmem->mutex);
384 list_for_each_entry (chunk, &drm->dmem->chunk_free, list) {
385 nouveau_bo_unpin(chunk->bo);
386 }
387 list_for_each_entry (chunk, &drm->dmem->chunk_full, list) {
388 nouveau_bo_unpin(chunk->bo);
389 }
390 mutex_unlock(&drm->dmem->mutex);
391 }
392
393 void
nouveau_dmem_fini(struct nouveau_drm * drm)394 nouveau_dmem_fini(struct nouveau_drm *drm)
395 {
396 struct nouveau_dmem_chunk *chunk, *tmp;
397
398 if (drm->dmem == NULL)
399 return;
400
401 mutex_lock(&drm->dmem->mutex);
402
403 WARN_ON(!list_empty(&drm->dmem->chunk_free));
404 WARN_ON(!list_empty(&drm->dmem->chunk_full));
405
406 list_for_each_entry_safe (chunk, tmp, &drm->dmem->chunk_empty, list) {
407 if (chunk->bo) {
408 nouveau_bo_unpin(chunk->bo);
409 nouveau_bo_ref(NULL, &chunk->bo);
410 }
411 list_del(&chunk->list);
412 spin_lock_destroy(&chunk->lock);
413 kfree(chunk);
414 }
415
416 mutex_unlock(&drm->dmem->mutex);
417 }
418
419 static int
nvc0b5_migrate_copy(struct nouveau_drm * drm,u64 npages,enum nouveau_aper dst_aper,u64 dst_addr,enum nouveau_aper src_aper,u64 src_addr)420 nvc0b5_migrate_copy(struct nouveau_drm *drm, u64 npages,
421 enum nouveau_aper dst_aper, u64 dst_addr,
422 enum nouveau_aper src_aper, u64 src_addr)
423 {
424 struct nouveau_channel *chan = drm->dmem->migrate.chan;
425 u32 launch_dma = (1 << 9) /* MULTI_LINE_ENABLE. */ |
426 (1 << 8) /* DST_MEMORY_LAYOUT_PITCH. */ |
427 (1 << 7) /* SRC_MEMORY_LAYOUT_PITCH. */ |
428 (1 << 2) /* FLUSH_ENABLE_TRUE. */ |
429 (2 << 0) /* DATA_TRANSFER_TYPE_NON_PIPELINED. */;
430 int ret;
431
432 ret = RING_SPACE(chan, 13);
433 if (ret)
434 return ret;
435
436 if (src_aper != NOUVEAU_APER_VIRT) {
437 switch (src_aper) {
438 case NOUVEAU_APER_VRAM:
439 BEGIN_IMC0(chan, NvSubCopy, 0x0260, 0);
440 break;
441 case NOUVEAU_APER_HOST:
442 BEGIN_IMC0(chan, NvSubCopy, 0x0260, 1);
443 break;
444 default:
445 return -EINVAL;
446 }
447 launch_dma |= 0x00001000; /* SRC_TYPE_PHYSICAL. */
448 }
449
450 if (dst_aper != NOUVEAU_APER_VIRT) {
451 switch (dst_aper) {
452 case NOUVEAU_APER_VRAM:
453 BEGIN_IMC0(chan, NvSubCopy, 0x0264, 0);
454 break;
455 case NOUVEAU_APER_HOST:
456 BEGIN_IMC0(chan, NvSubCopy, 0x0264, 1);
457 break;
458 default:
459 return -EINVAL;
460 }
461 launch_dma |= 0x00002000; /* DST_TYPE_PHYSICAL. */
462 }
463
464 BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8);
465 OUT_RING (chan, upper_32_bits(src_addr));
466 OUT_RING (chan, lower_32_bits(src_addr));
467 OUT_RING (chan, upper_32_bits(dst_addr));
468 OUT_RING (chan, lower_32_bits(dst_addr));
469 OUT_RING (chan, PAGE_SIZE);
470 OUT_RING (chan, PAGE_SIZE);
471 OUT_RING (chan, PAGE_SIZE);
472 OUT_RING (chan, npages);
473 BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1);
474 OUT_RING (chan, launch_dma);
475 return 0;
476 }
477
478 static int
nouveau_dmem_migrate_init(struct nouveau_drm * drm)479 nouveau_dmem_migrate_init(struct nouveau_drm *drm)
480 {
481 switch (drm->ttm.copy.oclass) {
482 case PASCAL_DMA_COPY_A:
483 case PASCAL_DMA_COPY_B:
484 case VOLTA_DMA_COPY_A:
485 case TURING_DMA_COPY_A:
486 drm->dmem->migrate.copy_func = nvc0b5_migrate_copy;
487 drm->dmem->migrate.chan = drm->ttm.chan;
488 return 0;
489 default:
490 break;
491 }
492 return -ENODEV;
493 }
494
495 void
nouveau_dmem_init(struct nouveau_drm * drm)496 nouveau_dmem_init(struct nouveau_drm *drm)
497 {
498 struct device *device = drm->dev->dev;
499 struct resource *res;
500 unsigned long i, size, pfn_first;
501 int ret;
502
503 /* This only make sense on PASCAL or newer */
504 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_PASCAL)
505 return;
506
507 if (!(drm->dmem = kzalloc(sizeof(*drm->dmem), GFP_KERNEL)))
508 return;
509
510 drm->dmem->drm = drm;
511 mutex_init(&drm->dmem->mutex);
512 INIT_LIST_HEAD(&drm->dmem->chunk_free);
513 INIT_LIST_HEAD(&drm->dmem->chunk_full);
514 INIT_LIST_HEAD(&drm->dmem->chunk_empty);
515
516 size = ALIGN(drm->client.device.info.ram_user, DMEM_CHUNK_SIZE);
517
518 /* Initialize migration dma helpers before registering memory */
519 ret = nouveau_dmem_migrate_init(drm);
520 if (ret)
521 goto out_free;
522
523 /*
524 * FIXME we need some kind of policy to decide how much VRAM we
525 * want to register with HMM. For now just register everything
526 * and latter if we want to do thing like over commit then we
527 * could revisit this.
528 */
529 res = devm_request_free_mem_region(device, &iomem_resource, size);
530 if (IS_ERR(res))
531 goto out_free;
532 drm->dmem->pagemap.type = MEMORY_DEVICE_PRIVATE;
533 drm->dmem->pagemap.res = *res;
534 drm->dmem->pagemap.ops = &nouveau_dmem_pagemap_ops;
535 if (IS_ERR(devm_memremap_pages(device, &drm->dmem->pagemap)))
536 goto out_free;
537
538 pfn_first = res->start >> PAGE_SHIFT;
539 for (i = 0; i < (size / DMEM_CHUNK_SIZE); ++i) {
540 struct nouveau_dmem_chunk *chunk;
541 struct page *page;
542 unsigned long j;
543
544 chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
545 if (chunk == NULL) {
546 nouveau_dmem_fini(drm);
547 return;
548 }
549
550 chunk->drm = drm;
551 chunk->pfn_first = pfn_first + (i * DMEM_CHUNK_NPAGES);
552 list_add_tail(&chunk->list, &drm->dmem->chunk_empty);
553
554 page = pfn_to_page(chunk->pfn_first);
555 for (j = 0; j < DMEM_CHUNK_NPAGES; ++j, ++page)
556 page->zone_device_data = chunk;
557 }
558
559 NV_INFO(drm, "DMEM: registered %ldMB of device memory\n", size >> 20);
560 return;
561 out_free:
562 mutex_destroy(&drm->dmem->mutex);
563 kfree(drm->dmem);
564 drm->dmem = NULL;
565 }
566
nouveau_dmem_migrate_copy_one(struct nouveau_drm * drm,unsigned long src,dma_addr_t * dma_addr)567 static unsigned long nouveau_dmem_migrate_copy_one(struct nouveau_drm *drm,
568 unsigned long src, dma_addr_t *dma_addr)
569 {
570 struct device *dev = drm->dev->dev;
571 struct page *dpage, *spage;
572
573 spage = migrate_pfn_to_page(src);
574 if (!spage || !(src & MIGRATE_PFN_MIGRATE))
575 goto out;
576
577 dpage = nouveau_dmem_page_alloc_locked(drm);
578 if (!dpage)
579 return 0;
580
581 *dma_addr = dma_map_page(dev, spage, 0, PAGE_SIZE, DMA_BIDIRECTIONAL);
582 if (dma_mapping_error(dev, *dma_addr))
583 goto out_free_page;
584
585 if (drm->dmem->migrate.copy_func(drm, 1, NOUVEAU_APER_VRAM,
586 nouveau_dmem_page_addr(dpage), NOUVEAU_APER_HOST,
587 *dma_addr))
588 goto out_dma_unmap;
589
590 return migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED;
591
592 out_dma_unmap:
593 dma_unmap_page(dev, *dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
594 out_free_page:
595 nouveau_dmem_page_free_locked(drm, dpage);
596 out:
597 return 0;
598 }
599
nouveau_dmem_migrate_chunk(struct nouveau_drm * drm,struct migrate_vma * args,dma_addr_t * dma_addrs)600 static void nouveau_dmem_migrate_chunk(struct nouveau_drm *drm,
601 struct migrate_vma *args, dma_addr_t *dma_addrs)
602 {
603 struct nouveau_fence *fence;
604 unsigned long addr = args->start, nr_dma = 0, i;
605
606 for (i = 0; addr < args->end; i++) {
607 args->dst[i] = nouveau_dmem_migrate_copy_one(drm, args->src[i],
608 dma_addrs + nr_dma);
609 if (args->dst[i])
610 nr_dma++;
611 addr += PAGE_SIZE;
612 }
613
614 nouveau_fence_new(drm->dmem->migrate.chan, false, &fence);
615 migrate_vma_pages(args);
616 nouveau_dmem_fence_done(&fence);
617
618 while (nr_dma--) {
619 dma_unmap_page(drm->dev->dev, dma_addrs[nr_dma], PAGE_SIZE,
620 DMA_BIDIRECTIONAL);
621 }
622 /*
623 * FIXME optimization: update GPU page table to point to newly migrated
624 * memory.
625 */
626 migrate_vma_finalize(args);
627 }
628
629 int
nouveau_dmem_migrate_vma(struct nouveau_drm * drm,struct vm_area_struct * vma,unsigned long start,unsigned long end)630 nouveau_dmem_migrate_vma(struct nouveau_drm *drm,
631 struct vm_area_struct *vma,
632 unsigned long start,
633 unsigned long end)
634 {
635 unsigned long npages = (end - start) >> PAGE_SHIFT;
636 unsigned long max = min(SG_MAX_SINGLE_ALLOC, npages);
637 dma_addr_t *dma_addrs;
638 struct migrate_vma args = {
639 .vma = vma,
640 .start = start,
641 };
642 unsigned long c, i;
643 int ret = -ENOMEM;
644
645 args.src = kcalloc(max, sizeof(*args.src), GFP_KERNEL);
646 if (!args.src)
647 goto out;
648 args.dst = kcalloc(max, sizeof(*args.dst), GFP_KERNEL);
649 if (!args.dst)
650 goto out_free_src;
651
652 dma_addrs = kmalloc_array(max, sizeof(*dma_addrs), GFP_KERNEL);
653 if (!dma_addrs)
654 goto out_free_dst;
655
656 for (i = 0; i < npages; i += c) {
657 c = min(SG_MAX_SINGLE_ALLOC, npages);
658 args.end = start + (c << PAGE_SHIFT);
659 ret = migrate_vma_setup(&args);
660 if (ret)
661 goto out_free_dma;
662
663 if (args.cpages)
664 nouveau_dmem_migrate_chunk(drm, &args, dma_addrs);
665 args.start = args.end;
666 }
667
668 ret = 0;
669 out_free_dma:
670 kfree(dma_addrs);
671 out_free_dst:
672 kfree(args.dst);
673 out_free_src:
674 kfree(args.src);
675 out:
676 return ret;
677 }
678
679 static inline bool
nouveau_dmem_page(struct nouveau_drm * drm,struct page * page)680 nouveau_dmem_page(struct nouveau_drm *drm, struct page *page)
681 {
682 return is_device_private_page(page) && drm->dmem == page_to_dmem(page);
683 }
684
685 void
nouveau_dmem_convert_pfn(struct nouveau_drm * drm,struct hmm_range * range)686 nouveau_dmem_convert_pfn(struct nouveau_drm *drm,
687 struct hmm_range *range)
688 {
689 unsigned long i, npages;
690
691 npages = (range->end - range->start) >> PAGE_SHIFT;
692 for (i = 0; i < npages; ++i) {
693 struct page *page;
694 uint64_t addr;
695
696 page = hmm_device_entry_to_page(range, range->pfns[i]);
697 if (page == NULL)
698 continue;
699
700 if (!(range->pfns[i] & range->flags[HMM_PFN_DEVICE_PRIVATE])) {
701 continue;
702 }
703
704 if (!nouveau_dmem_page(drm, page)) {
705 WARN(1, "Some unknown device memory !\n");
706 range->pfns[i] = 0;
707 continue;
708 }
709
710 addr = nouveau_dmem_page_addr(page);
711 range->pfns[i] &= ((1UL << range->pfn_shift) - 1);
712 range->pfns[i] |= (addr >> PAGE_SHIFT) << range->pfn_shift;
713 }
714 }
715