1 /* $NetBSD: nouveau_nvkm_subdev_mmu_vmmnv44.c,v 1.4 2021/12/19 11:26:26 riastradh Exp $ */
2
3 /*
4 * Copyright 2017 Red Hat Inc.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 */
24 #include <sys/cdefs.h>
25 __KERNEL_RCSID(0, "$NetBSD: nouveau_nvkm_subdev_mmu_vmmnv44.c,v 1.4 2021/12/19 11:26:26 riastradh Exp $");
26
27 #include "vmm.h"
28
29 #include <subdev/timer.h>
30
31 static void
nv44_vmm_pgt_fill(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,dma_addr_t * list,u32 ptei,u32 ptes)32 nv44_vmm_pgt_fill(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
33 dma_addr_t *list, u32 ptei, u32 ptes)
34 {
35 u32 pteo = (ptei << 2) & ~0x0000000f;
36 u32 tmp[4];
37
38 tmp[0] = nvkm_ro32(pt->memory, pteo + 0x0);
39 tmp[1] = nvkm_ro32(pt->memory, pteo + 0x4);
40 tmp[2] = nvkm_ro32(pt->memory, pteo + 0x8);
41 tmp[3] = nvkm_ro32(pt->memory, pteo + 0xc);
42
43 while (ptes--) {
44 u32 addr = (list ? *list++ : vmm->null) >> 12;
45 switch (ptei++ & 0x3) {
46 case 0:
47 tmp[0] &= ~0x07ffffff;
48 tmp[0] |= addr;
49 break;
50 case 1:
51 tmp[0] &= ~0xf8000000;
52 tmp[0] |= addr << 27;
53 tmp[1] &= ~0x003fffff;
54 tmp[1] |= addr >> 5;
55 break;
56 case 2:
57 tmp[1] &= ~0xffc00000;
58 tmp[1] |= addr << 22;
59 tmp[2] &= ~0x0001ffff;
60 tmp[2] |= addr >> 10;
61 break;
62 case 3:
63 tmp[2] &= ~0xfffe0000;
64 tmp[2] |= addr << 17;
65 tmp[3] &= ~0x00000fff;
66 tmp[3] |= addr >> 15;
67 break;
68 }
69 }
70
71 VMM_WO032(pt, vmm, pteo + 0x0, tmp[0]);
72 VMM_WO032(pt, vmm, pteo + 0x4, tmp[1]);
73 VMM_WO032(pt, vmm, pteo + 0x8, tmp[2]);
74 VMM_WO032(pt, vmm, pteo + 0xc, tmp[3] | 0x40000000);
75 }
76
77 static void __unused
nv44_vmm_pgt_pte(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes,struct nvkm_vmm_map * map,u64 addr)78 nv44_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
79 u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr)
80 {
81 dma_addr_t tmp[4], i;
82
83 if (ptei & 3) {
84 const u32 pten = min(ptes, 4 - (ptei & 3));
85 for (i = 0; i < pten; i++, addr += 0x1000)
86 tmp[i] = addr;
87 nv44_vmm_pgt_fill(vmm, pt, tmp, ptei, pten);
88 ptei += pten;
89 ptes -= pten;
90 }
91
92 while (ptes >= 4) {
93 for (i = 0; i < 4; i++, addr += 0x1000)
94 tmp[i] = addr >> 12;
95 VMM_WO032(pt, vmm, ptei++ * 4, tmp[0] >> 0 | tmp[1] << 27);
96 VMM_WO032(pt, vmm, ptei++ * 4, tmp[1] >> 5 | tmp[2] << 22);
97 VMM_WO032(pt, vmm, ptei++ * 4, tmp[2] >> 10 | tmp[3] << 17);
98 VMM_WO032(pt, vmm, ptei++ * 4, tmp[3] >> 15 | 0x40000000);
99 ptes -= 4;
100 }
101
102 if (ptes) {
103 for (i = 0; i < ptes; i++, addr += 0x1000)
104 tmp[i] = addr;
105 nv44_vmm_pgt_fill(vmm, pt, tmp, ptei, ptes);
106 }
107 }
108
109 #ifndef __NetBSD__
110 static void
nv44_vmm_pgt_sgl(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes,struct nvkm_vmm_map * map)111 nv44_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
112 u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
113 {
114 VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, nv44_vmm_pgt_pte);
115 }
116 #endif
117
118 static void
nv44_vmm_pgt_dma(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes,struct nvkm_vmm_map * map)119 nv44_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
120 u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
121 {
122 #if PAGE_SHIFT == 12
123 nvkm_kmap(pt->memory);
124 if (ptei & 3) {
125 const u32 pten = min(ptes, 4 - (ptei & 3));
126 nv44_vmm_pgt_fill(vmm, pt, map->dma, ptei, pten);
127 ptei += pten;
128 ptes -= pten;
129 map->dma += pten;
130 }
131
132 while (ptes >= 4) {
133 u32 tmp[4], i;
134 for (i = 0; i < 4; i++)
135 tmp[i] = *map->dma++ >> 12;
136 VMM_WO032(pt, vmm, ptei++ * 4, tmp[0] >> 0 | tmp[1] << 27);
137 VMM_WO032(pt, vmm, ptei++ * 4, tmp[1] >> 5 | tmp[2] << 22);
138 VMM_WO032(pt, vmm, ptei++ * 4, tmp[2] >> 10 | tmp[3] << 17);
139 VMM_WO032(pt, vmm, ptei++ * 4, tmp[3] >> 15 | 0x40000000);
140 ptes -= 4;
141 }
142
143 if (ptes) {
144 nv44_vmm_pgt_fill(vmm, pt, map->dma, ptei, ptes);
145 map->dma += ptes;
146 }
147 nvkm_done(pt->memory);
148 #else
149 VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, nv44_vmm_pgt_pte);
150 #endif
151 }
152
153 static void
nv44_vmm_pgt_unmap(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes)154 nv44_vmm_pgt_unmap(struct nvkm_vmm *vmm,
155 struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
156 {
157 nvkm_kmap(pt->memory);
158 if (ptei & 3) {
159 const u32 pten = min(ptes, 4 - (ptei & 3));
160 nv44_vmm_pgt_fill(vmm, pt, NULL, ptei, pten);
161 ptei += pten;
162 ptes -= pten;
163 }
164
165 while (ptes > 4) {
166 VMM_WO032(pt, vmm, ptei++ * 4, 0x00000000);
167 VMM_WO032(pt, vmm, ptei++ * 4, 0x00000000);
168 VMM_WO032(pt, vmm, ptei++ * 4, 0x00000000);
169 VMM_WO032(pt, vmm, ptei++ * 4, 0x00000000);
170 ptes -= 4;
171 }
172
173 if (ptes)
174 nv44_vmm_pgt_fill(vmm, pt, NULL, ptei, ptes);
175 nvkm_done(pt->memory);
176 }
177
178 static const struct nvkm_vmm_desc_func
179 nv44_vmm_desc_pgt = {
180 .unmap = nv44_vmm_pgt_unmap,
181 .dma = nv44_vmm_pgt_dma,
182 #ifndef __NetBSD__
183 .sgl = nv44_vmm_pgt_sgl,
184 #endif
185 };
186
187 static const struct nvkm_vmm_desc
188 nv44_vmm_desc_12[] = {
189 { PGT, 17, 4, 0x80000, &nv44_vmm_desc_pgt },
190 {}
191 };
192
193 static void
nv44_vmm_flush(struct nvkm_vmm * vmm,int level)194 nv44_vmm_flush(struct nvkm_vmm *vmm, int level)
195 {
196 struct nvkm_device *device = vmm->mmu->subdev.device;
197 nvkm_wr32(device, 0x100814, vmm->limit - 4096);
198 nvkm_wr32(device, 0x100808, 0x000000020);
199 nvkm_msec(device, 2000,
200 if (nvkm_rd32(device, 0x100808) & 0x00000001)
201 break;
202 );
203 nvkm_wr32(device, 0x100808, 0x00000000);
204 }
205
206 static const struct nvkm_vmm_func
207 nv44_vmm = {
208 .valid = nv04_vmm_valid,
209 .flush = nv44_vmm_flush,
210 .page = {
211 { 12, &nv44_vmm_desc_12[0], NVKM_VMM_PAGE_HOST },
212 {}
213 }
214 };
215
216 int
nv44_vmm_new(struct nvkm_mmu * mmu,bool managed,u64 addr,u64 size,void * argv,u32 argc,struct lock_class_key * key,const char * name,struct nvkm_vmm ** pvmm)217 nv44_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size,
218 void *argv, u32 argc, struct lock_class_key *key, const char *name,
219 struct nvkm_vmm **pvmm)
220 {
221 struct nvkm_subdev *subdev = &mmu->subdev;
222 struct nvkm_vmm *vmm;
223 int ret;
224
225 ret = nv04_vmm_new_(&nv44_vmm, mmu, 0, managed, addr, size,
226 argv, argc, key, name, &vmm);
227 *pvmm = vmm;
228 if (ret)
229 return ret;
230
231 #ifdef __NetBSD__
232 do {
233 const bus_dma_tag_t dmat =
234 subdev->device->func->dma_tag(subdev->device);
235 const unsigned nullsz = 16 * 1024;
236 int nsegs;
237
238 /* XXX errno NetBSD->Linux */
239 ret = -bus_dmamem_alloc(dmat, nullsz, PAGE_SIZE, 0,
240 &vmm->nullseg, 1, &nsegs, BUS_DMA_WAITOK);
241 if (ret) {
242 fail0: vmm->nullp = NULL;
243 break;
244 }
245 KASSERT(nsegs == 1);
246
247 /* XXX errno NetBSD->Linux */
248 ret = -bus_dmamap_create(dmat, nullsz /* size */, 1 /* maxnseg */,
249 nullsz /* maxsegsz */, 0, BUS_DMA_WAITOK, &vmm->nullmap);
250 if (ret) {
251 fail1: bus_dmamem_free(dmat, &vmm->nullseg, 1);
252 goto fail0;
253 }
254
255 /* XXX errno NetBSD->Linux */
256 ret = -bus_dmamem_map(dmat, &vmm->nullseg, 1, nullsz,
257 &vmm->nullp, BUS_DMA_WAITOK|BUS_DMA_COHERENT);
258 if (ret) {
259 fail2: bus_dmamap_destroy(dmat, vmm->nullmap);
260 goto fail1;
261 }
262
263 /* XXX errno NetBSD->Linux */
264 ret = -bus_dmamap_load(dmat, vmm->nullmap, vmm->nullp, nullsz,
265 NULL, BUS_DMA_WAITOK);
266 if (ret) {
267 fail3: __unused bus_dmamem_unmap(dmat, vmm->nullp, nullsz);
268 goto fail2;
269 }
270 vmm->null = vmm->nullmap->dm_segs[0].ds_addr;
271 } while (0);
272 #else
273 vmm->nullp = dma_alloc_coherent(subdev->device->dev, 16 * 1024,
274 &vmm->null, GFP_KERNEL);
275 #endif
276 if (!vmm->nullp) {
277 nvkm_warn(subdev, "unable to allocate dummy pages\n");
278 vmm->null = 0;
279 }
280
281 return 0;
282 }
283