1 /* $NetBSD: nouveau_nvkm_subdev_mmu_base.c,v 1.8 2021/12/19 10:51:58 riastradh Exp $ */
2
3 /*
4 * Copyright 2010 Red Hat Inc.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Ben Skeggs
25 */
26 #include <sys/cdefs.h>
27 __KERNEL_RCSID(0, "$NetBSD: nouveau_nvkm_subdev_mmu_base.c,v 1.8 2021/12/19 10:51:58 riastradh Exp $");
28
29 #include "ummu.h"
30 #include "vmm.h"
31
32 #include <subdev/bar.h>
33 #include <subdev/fb.h>
34
35 #include <nvif/if500d.h>
36 #include <nvif/if900d.h>
37
38 #include <linux/nbsd-namespace.h>
39
40 struct nvkm_mmu_ptp {
41 struct nvkm_mmu_pt *pt;
42 struct list_head head;
43 u8 shift;
44 u16 mask;
45 u16 free;
46 };
47
48 static void
nvkm_mmu_ptp_put(struct nvkm_mmu * mmu,bool force,struct nvkm_mmu_pt * pt)49 nvkm_mmu_ptp_put(struct nvkm_mmu *mmu, bool force, struct nvkm_mmu_pt *pt)
50 {
51 const int slot = pt->base >> pt->ptp->shift;
52 struct nvkm_mmu_ptp *ptp = pt->ptp;
53
54 /* If there were no free slots in the parent allocation before,
55 * there will be now, so return PTP to the cache.
56 */
57 if (!ptp->free)
58 list_add(&ptp->head, &mmu->ptp.list);
59 ptp->free |= BIT(slot);
60
61 /* If there's no more sub-allocations, destroy PTP. */
62 if (ptp->free == ptp->mask) {
63 nvkm_mmu_ptc_put(mmu, force, &ptp->pt);
64 list_del(&ptp->head);
65 kfree(ptp);
66 }
67
68 kfree(pt);
69 }
70
71 static struct nvkm_mmu_pt *
nvkm_mmu_ptp_get(struct nvkm_mmu * mmu,u32 size,bool zero)72 nvkm_mmu_ptp_get(struct nvkm_mmu *mmu, u32 size, bool zero)
73 {
74 struct nvkm_mmu_pt *pt;
75 struct nvkm_mmu_ptp *ptp;
76 int slot;
77
78 if (!(pt = kzalloc(sizeof(*pt), GFP_KERNEL)))
79 return NULL;
80
81 ptp = list_first_entry_or_null(&mmu->ptp.list, typeof(*ptp), head);
82 if (!ptp) {
83 /* Need to allocate a new parent to sub-allocate from. */
84 if (!(ptp = kmalloc(sizeof(*ptp), GFP_KERNEL))) {
85 kfree(pt);
86 return NULL;
87 }
88
89 ptp->pt = nvkm_mmu_ptc_get(mmu, 0x1000, 0x1000, false);
90 if (!ptp->pt) {
91 kfree(ptp);
92 kfree(pt);
93 return NULL;
94 }
95
96 ptp->shift = order_base_2(size);
97 slot = nvkm_memory_size(ptp->pt->memory) >> ptp->shift;
98 ptp->mask = (1 << slot) - 1;
99 ptp->free = ptp->mask;
100 list_add(&ptp->head, &mmu->ptp.list);
101 }
102 pt->ptp = ptp;
103 pt->sub = true;
104
105 /* Sub-allocate from parent object, removing PTP from cache
106 * if there's no more free slots left.
107 */
108 slot = __ffs(ptp->free);
109 ptp->free &= ~BIT(slot);
110 if (!ptp->free)
111 list_del(&ptp->head);
112
113 pt->memory = pt->ptp->pt->memory;
114 pt->base = slot << ptp->shift;
115 pt->addr = pt->ptp->pt->addr + pt->base;
116 return pt;
117 }
118
119 struct nvkm_mmu_ptc {
120 struct list_head head;
121 struct list_head item;
122 u32 size;
123 u32 refs;
124 };
125
126 static inline struct nvkm_mmu_ptc *
nvkm_mmu_ptc_find(struct nvkm_mmu * mmu,u32 size)127 nvkm_mmu_ptc_find(struct nvkm_mmu *mmu, u32 size)
128 {
129 struct nvkm_mmu_ptc *ptc;
130
131 list_for_each_entry(ptc, &mmu->ptc.list, head) {
132 if (ptc->size == size)
133 return ptc;
134 }
135
136 ptc = kmalloc(sizeof(*ptc), GFP_KERNEL);
137 if (ptc) {
138 INIT_LIST_HEAD(&ptc->item);
139 ptc->size = size;
140 ptc->refs = 0;
141 list_add(&ptc->head, &mmu->ptc.list);
142 }
143
144 return ptc;
145 }
146
147 void
nvkm_mmu_ptc_put(struct nvkm_mmu * mmu,bool force,struct nvkm_mmu_pt ** ppt)148 nvkm_mmu_ptc_put(struct nvkm_mmu *mmu, bool force, struct nvkm_mmu_pt **ppt)
149 {
150 struct nvkm_mmu_pt *pt = *ppt;
151 if (pt) {
152 /* Handle sub-allocated page tables. */
153 if (pt->sub) {
154 mutex_lock(&mmu->ptp.mutex);
155 nvkm_mmu_ptp_put(mmu, force, pt);
156 mutex_unlock(&mmu->ptp.mutex);
157 return;
158 }
159
160 /* Either cache or free the object. */
161 mutex_lock(&mmu->ptc.mutex);
162 if (pt->ptc->refs < 8 /* Heuristic. */ && !force) {
163 list_add_tail(&pt->head, &pt->ptc->item);
164 pt->ptc->refs++;
165 } else {
166 nvkm_memory_unref(&pt->memory);
167 kfree(pt);
168 }
169 mutex_unlock(&mmu->ptc.mutex);
170 }
171 }
172
173 struct nvkm_mmu_pt *
nvkm_mmu_ptc_get(struct nvkm_mmu * mmu,u32 size,u32 align,bool zero)174 nvkm_mmu_ptc_get(struct nvkm_mmu *mmu, u32 size, u32 align, bool zero)
175 {
176 struct nvkm_mmu_ptc *ptc;
177 struct nvkm_mmu_pt *pt;
178 int ret;
179
180 /* Sub-allocated page table (ie. GP100 LPT). */
181 if (align < 0x1000) {
182 mutex_lock(&mmu->ptp.mutex);
183 pt = nvkm_mmu_ptp_get(mmu, align, zero);
184 mutex_unlock(&mmu->ptp.mutex);
185 return pt;
186 }
187
188 /* Lookup cache for this page table size. */
189 mutex_lock(&mmu->ptc.mutex);
190 ptc = nvkm_mmu_ptc_find(mmu, size);
191 if (!ptc) {
192 mutex_unlock(&mmu->ptc.mutex);
193 return NULL;
194 }
195
196 /* If there's a free PT in the cache, reuse it. */
197 pt = list_first_entry_or_null(&ptc->item, typeof(*pt), head);
198 if (pt) {
199 if (zero)
200 nvkm_fo64(pt->memory, 0, 0, size >> 3);
201 list_del(&pt->head);
202 ptc->refs--;
203 mutex_unlock(&mmu->ptc.mutex);
204 return pt;
205 }
206 mutex_unlock(&mmu->ptc.mutex);
207
208 /* No such luck, we need to allocate. */
209 if (!(pt = kmalloc(sizeof(*pt), GFP_KERNEL)))
210 return NULL;
211 pt->ptc = ptc;
212 pt->sub = false;
213
214 ret = nvkm_memory_new(mmu->subdev.device, NVKM_MEM_TARGET_INST,
215 size, align, zero, &pt->memory);
216 if (ret) {
217 kfree(pt);
218 return NULL;
219 }
220
221 pt->base = 0;
222 pt->addr = nvkm_memory_addr(pt->memory);
223 return pt;
224 }
225
226 void
nvkm_mmu_ptc_dump(struct nvkm_mmu * mmu)227 nvkm_mmu_ptc_dump(struct nvkm_mmu *mmu)
228 {
229 struct nvkm_mmu_ptc *ptc;
230 list_for_each_entry(ptc, &mmu->ptc.list, head) {
231 struct nvkm_mmu_pt *pt, *tt;
232 list_for_each_entry_safe(pt, tt, &ptc->item, head) {
233 nvkm_memory_unref(&pt->memory);
234 list_del(&pt->head);
235 kfree(pt);
236 }
237 }
238 }
239
240 static void
nvkm_mmu_ptc_fini(struct nvkm_mmu * mmu)241 nvkm_mmu_ptc_fini(struct nvkm_mmu *mmu)
242 {
243 struct nvkm_mmu_ptc *ptc, *ptct;
244
245 list_for_each_entry_safe(ptc, ptct, &mmu->ptc.list, head) {
246 WARN_ON(!list_empty(&ptc->item));
247 list_del(&ptc->head);
248 kfree(ptc);
249 }
250
251 mutex_destroy(&mmu->ptp.mutex);
252 mutex_destroy(&mmu->ptc.mutex);
253 }
254
255 static void
nvkm_mmu_ptc_init(struct nvkm_mmu * mmu)256 nvkm_mmu_ptc_init(struct nvkm_mmu *mmu)
257 {
258 mutex_init(&mmu->ptc.mutex);
259 INIT_LIST_HEAD(&mmu->ptc.list);
260 mutex_init(&mmu->ptp.mutex);
261 INIT_LIST_HEAD(&mmu->ptp.list);
262 }
263
264 static void
nvkm_mmu_type(struct nvkm_mmu * mmu,int heap,u8 type)265 nvkm_mmu_type(struct nvkm_mmu *mmu, int heap, u8 type)
266 {
267 if (heap >= 0 && !WARN_ON(mmu->type_nr == ARRAY_SIZE(mmu->type))) {
268 mmu->type[mmu->type_nr].type = type | mmu->heap[heap].type;
269 mmu->type[mmu->type_nr].heap = heap;
270 mmu->type_nr++;
271 }
272 }
273
274 static int
nvkm_mmu_heap(struct nvkm_mmu * mmu,u8 type,u64 size)275 nvkm_mmu_heap(struct nvkm_mmu *mmu, u8 type, u64 size)
276 {
277 if (size) {
278 if (!WARN_ON(mmu->heap_nr == ARRAY_SIZE(mmu->heap))) {
279 mmu->heap[mmu->heap_nr].type = type;
280 mmu->heap[mmu->heap_nr].size = size;
281 return mmu->heap_nr++;
282 }
283 }
284 return -EINVAL;
285 }
286
287 static void
nvkm_mmu_host(struct nvkm_mmu * mmu)288 nvkm_mmu_host(struct nvkm_mmu *mmu)
289 {
290 struct nvkm_device *device = mmu->subdev.device;
291 u8 type = NVKM_MEM_KIND * !!mmu->func->kind_sys;
292 int heap;
293
294 /* Non-mappable system memory. */
295 heap = nvkm_mmu_heap(mmu, NVKM_MEM_HOST, ~0ULL);
296 nvkm_mmu_type(mmu, heap, type);
297
298 /* Non-coherent, cached, system memory.
299 *
300 * Block-linear mappings of system memory must be done through
301 * BAR1, and cannot be supported on systems where we're unable
302 * to map BAR1 with write-combining.
303 */
304 type |= NVKM_MEM_MAPPABLE;
305 if (!device->bar || device->bar->iomap_uncached)
306 nvkm_mmu_type(mmu, heap, type & ~NVKM_MEM_KIND);
307 else
308 nvkm_mmu_type(mmu, heap, type);
309
310 /* Coherent, cached, system memory.
311 *
312 * Unsupported on systems that aren't able to support snooped
313 * mappings, and also for block-linear mappings which must be
314 * done through BAR1.
315 */
316 type |= NVKM_MEM_COHERENT;
317 if (device->func->cpu_coherent)
318 nvkm_mmu_type(mmu, heap, type & ~NVKM_MEM_KIND);
319
320 /* Uncached system memory. */
321 nvkm_mmu_type(mmu, heap, type |= NVKM_MEM_UNCACHED);
322 }
323
324 static void
nvkm_mmu_vram(struct nvkm_mmu * mmu)325 nvkm_mmu_vram(struct nvkm_mmu *mmu)
326 {
327 struct nvkm_device *device = mmu->subdev.device;
328 struct nvkm_mm *mm = &device->fb->ram->vram;
329 const u32 sizeN = nvkm_mm_heap_size(mm, NVKM_RAM_MM_NORMAL);
330 const u32 sizeU = nvkm_mm_heap_size(mm, NVKM_RAM_MM_NOMAP);
331 const u32 sizeM = nvkm_mm_heap_size(mm, NVKM_RAM_MM_MIXED);
332 u8 type = NVKM_MEM_KIND * !!mmu->func->kind;
333 u8 heap = NVKM_MEM_VRAM;
334 int heapM, heapN, heapU;
335
336 /* Mixed-memory doesn't support compression or display. */
337 heapM = nvkm_mmu_heap(mmu, heap, sizeM << NVKM_RAM_MM_SHIFT);
338
339 heap |= NVKM_MEM_COMP;
340 heap |= NVKM_MEM_DISP;
341 heapN = nvkm_mmu_heap(mmu, heap, sizeN << NVKM_RAM_MM_SHIFT);
342 heapU = nvkm_mmu_heap(mmu, heap, sizeU << NVKM_RAM_MM_SHIFT);
343
344 /* Add non-mappable VRAM types first so that they're preferred
345 * over anything else. Mixed-memory will be slower than other
346 * heaps, it's prioritised last.
347 */
348 nvkm_mmu_type(mmu, heapU, type);
349 nvkm_mmu_type(mmu, heapN, type);
350 nvkm_mmu_type(mmu, heapM, type);
351
352 /* Add host memory types next, under the assumption that users
353 * wanting mappable memory want to use them as staging buffers
354 * or the like.
355 */
356 nvkm_mmu_host(mmu);
357
358 /* Mappable VRAM types go last, as they're basically the worst
359 * possible type to ask for unless there's no other choice.
360 */
361 if (device->bar) {
362 /* Write-combined BAR1 access. */
363 type |= NVKM_MEM_MAPPABLE;
364 if (!device->bar->iomap_uncached) {
365 nvkm_mmu_type(mmu, heapN, type);
366 nvkm_mmu_type(mmu, heapM, type);
367 }
368
369 /* Uncached BAR1 access. */
370 type |= NVKM_MEM_COHERENT;
371 type |= NVKM_MEM_UNCACHED;
372 nvkm_mmu_type(mmu, heapN, type);
373 nvkm_mmu_type(mmu, heapM, type);
374 }
375 }
376
377 static int
nvkm_mmu_oneinit(struct nvkm_subdev * subdev)378 nvkm_mmu_oneinit(struct nvkm_subdev *subdev)
379 {
380 struct nvkm_mmu *mmu = nvkm_mmu(subdev);
381
382 /* Determine available memory types. */
383 if (mmu->subdev.device->fb && mmu->subdev.device->fb->ram)
384 nvkm_mmu_vram(mmu);
385 else
386 nvkm_mmu_host(mmu);
387
388 if (mmu->func->vmm.global) {
389 int ret = nvkm_vmm_new(subdev->device, 0, 0, NULL, 0, NULL,
390 "gart", &mmu->vmm);
391 if (ret)
392 return ret;
393 }
394
395 return 0;
396 }
397
398 static int
nvkm_mmu_init(struct nvkm_subdev * subdev)399 nvkm_mmu_init(struct nvkm_subdev *subdev)
400 {
401 struct nvkm_mmu *mmu = nvkm_mmu(subdev);
402 if (mmu->func->init)
403 mmu->func->init(mmu);
404 return 0;
405 }
406
407 static void *
nvkm_mmu_dtor(struct nvkm_subdev * subdev)408 nvkm_mmu_dtor(struct nvkm_subdev *subdev)
409 {
410 struct nvkm_mmu *mmu = nvkm_mmu(subdev);
411
412 nvkm_vmm_unref(&mmu->vmm);
413
414 nvkm_mmu_ptc_fini(mmu);
415 return mmu;
416 }
417
418 static const struct nvkm_subdev_func
419 nvkm_mmu = {
420 .dtor = nvkm_mmu_dtor,
421 .oneinit = nvkm_mmu_oneinit,
422 .init = nvkm_mmu_init,
423 };
424
425 void
nvkm_mmu_ctor(const struct nvkm_mmu_func * func,struct nvkm_device * device,int index,struct nvkm_mmu * mmu)426 nvkm_mmu_ctor(const struct nvkm_mmu_func *func, struct nvkm_device *device,
427 int index, struct nvkm_mmu *mmu)
428 {
429 nvkm_subdev_ctor(&nvkm_mmu, device, index, &mmu->subdev);
430 mmu->func = func;
431 mmu->dma_bits = func->dma_bits;
432 nvkm_mmu_ptc_init(mmu);
433 mmu->user.ctor = nvkm_ummu_new;
434 mmu->user.base = func->mmu.user;
435 }
436
437 int
nvkm_mmu_new_(const struct nvkm_mmu_func * func,struct nvkm_device * device,int index,struct nvkm_mmu ** pmmu)438 nvkm_mmu_new_(const struct nvkm_mmu_func *func, struct nvkm_device *device,
439 int index, struct nvkm_mmu **pmmu)
440 {
441 if (!(*pmmu = kzalloc(sizeof(**pmmu), GFP_KERNEL)))
442 return -ENOMEM;
443 nvkm_mmu_ctor(func, device, index, *pmmu);
444 return 0;
445 }
446