xref: /netbsd-src/sys/external/bsd/drm2/dist/drm/nouveau/nvkm/subdev/mmu/vmm.h (revision 7649e88fcfe6a7c92de68bd5e592dec3e35224fb)
1 /*	$NetBSD: vmm.h,v 1.3 2021/12/19 10:51:58 riastradh Exp $	*/
2 
3 #ifndef __NVKM_VMM_H__
4 #define __NVKM_VMM_H__
5 #include "priv.h"
6 #include <core/memory.h>
7 enum nvkm_memory_target;
8 
9 struct nvkm_vmm_pt {
10 	/* Some GPUs have a mapping level with a dual page tables to
11 	 * support large and small pages in the same address-range.
12 	 *
13 	 * We track the state of both page tables in one place, which
14 	 * is why there's multiple PT pointers/refcounts here.
15 	 */
16 	struct nvkm_mmu_pt *pt[2];
17 	u32 refs[2];
18 
19 	/* Page size handled by this PT.
20 	 *
21 	 * Tesla backend needs to know this when writinge PDEs,
22 	 * otherwise unnecessary.
23 	 */
24 	u8 page;
25 
26 	/* Entire page table sparse.
27 	 *
28 	 * Used to propagate sparseness to child page tables.
29 	 */
30 	bool sparse:1;
31 
32 	/* Tracking for page directories.
33 	 *
34 	 * The array is indexed by PDE, and will either point to the
35 	 * child page table, or indicate the PDE is marked as sparse.
36 	 **/
37 #define NVKM_VMM_PDE_INVALID(pde) IS_ERR_OR_NULL(pde)
38 #define NVKM_VMM_PDE_SPARSED(pde) IS_ERR(pde)
39 #define NVKM_VMM_PDE_SPARSE       ERR_PTR(-EBUSY)
40 	struct nvkm_vmm_pt **pde;
41 
42 	/* Tracking for dual page tables.
43 	 *
44 	 * There's one entry for each LPTE, keeping track of whether
45 	 * there are valid SPTEs in the same address-range.
46 	 *
47 	 * This information is used to manage LPTE state transitions.
48 	 */
49 #define NVKM_VMM_PTE_SPARSE 0x80
50 #define NVKM_VMM_PTE_VALID  0x40
51 #define NVKM_VMM_PTE_SPTES  0x3f
52 	u8 pte[];
53 };
54 
55 typedef void (*nvkm_vmm_pxe_func)(struct nvkm_vmm *,
56 				  struct nvkm_mmu_pt *, u32 ptei, u32 ptes);
57 typedef void (*nvkm_vmm_pde_func)(struct nvkm_vmm *,
58 				  struct nvkm_vmm_pt *, u32 pdei);
59 typedef void (*nvkm_vmm_pte_func)(struct nvkm_vmm *, struct nvkm_mmu_pt *,
60 				  u32 ptei, u32 ptes, struct nvkm_vmm_map *);
61 
62 struct nvkm_vmm_desc_func {
63 	nvkm_vmm_pxe_func invalid;
64 	nvkm_vmm_pxe_func unmap;
65 	nvkm_vmm_pxe_func sparse;
66 
67 	nvkm_vmm_pde_func pde;
68 
69 	nvkm_vmm_pte_func mem;
70 	nvkm_vmm_pte_func dma;
71 	nvkm_vmm_pte_func sgl;
72 
73 	nvkm_vmm_pte_func pfn;
74 	bool (*pfn_clear)(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32 ptei, u32 ptes);
75 	nvkm_vmm_pxe_func pfn_unmap;
76 };
77 
78 extern const struct nvkm_vmm_desc_func gf100_vmm_pgd;
79 void gf100_vmm_pgd_pde(struct nvkm_vmm *, struct nvkm_vmm_pt *, u32);
80 extern const struct nvkm_vmm_desc_func gf100_vmm_pgt;
81 void gf100_vmm_pgt_unmap(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32);
82 void gf100_vmm_pgt_mem(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32,
83 		       struct nvkm_vmm_map *);
84 void gf100_vmm_pgt_dma(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32,
85 		       struct nvkm_vmm_map *);
86 void gf100_vmm_pgt_sgl(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32,
87 		       struct nvkm_vmm_map *);
88 
89 void gk104_vmm_lpt_invalid(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32);
90 
91 struct nvkm_vmm_desc {
92 	enum {
93 		PGD,
94 		PGT,
95 		SPT,
96 		LPT,
97 	} type;
98 	u8 bits;	/* VMA bits covered by PT. */
99 	u8 size;	/* Bytes-per-PTE. */
100 	u32 align;	/* PT address alignment. */
101 	const struct nvkm_vmm_desc_func *func;
102 };
103 
104 extern const struct nvkm_vmm_desc nv50_vmm_desc_12[];
105 extern const struct nvkm_vmm_desc nv50_vmm_desc_16[];
106 
107 extern const struct nvkm_vmm_desc gk104_vmm_desc_16_12[];
108 extern const struct nvkm_vmm_desc gk104_vmm_desc_16_16[];
109 extern const struct nvkm_vmm_desc gk104_vmm_desc_17_12[];
110 extern const struct nvkm_vmm_desc gk104_vmm_desc_17_17[];
111 
112 extern const struct nvkm_vmm_desc gm200_vmm_desc_16_12[];
113 extern const struct nvkm_vmm_desc gm200_vmm_desc_16_16[];
114 extern const struct nvkm_vmm_desc gm200_vmm_desc_17_12[];
115 extern const struct nvkm_vmm_desc gm200_vmm_desc_17_17[];
116 
117 extern const struct nvkm_vmm_desc gp100_vmm_desc_12[];
118 extern const struct nvkm_vmm_desc gp100_vmm_desc_16[];
119 
120 struct nvkm_vmm_page {
121 	u8 shift;
122 	const struct nvkm_vmm_desc *desc;
123 #define NVKM_VMM_PAGE_SPARSE                                               0x01
124 #define NVKM_VMM_PAGE_VRAM                                                 0x02
125 #define NVKM_VMM_PAGE_HOST                                                 0x04
126 #define NVKM_VMM_PAGE_COMP                                                 0x08
127 #define NVKM_VMM_PAGE_Sxxx                                (NVKM_VMM_PAGE_SPARSE)
128 #define NVKM_VMM_PAGE_xVxx                                  (NVKM_VMM_PAGE_VRAM)
129 #define NVKM_VMM_PAGE_SVxx             (NVKM_VMM_PAGE_Sxxx | NVKM_VMM_PAGE_VRAM)
130 #define NVKM_VMM_PAGE_xxHx                                  (NVKM_VMM_PAGE_HOST)
131 #define NVKM_VMM_PAGE_SxHx             (NVKM_VMM_PAGE_Sxxx | NVKM_VMM_PAGE_HOST)
132 #define NVKM_VMM_PAGE_xVHx             (NVKM_VMM_PAGE_xVxx | NVKM_VMM_PAGE_HOST)
133 #define NVKM_VMM_PAGE_SVHx             (NVKM_VMM_PAGE_SVxx | NVKM_VMM_PAGE_HOST)
134 #define NVKM_VMM_PAGE_xVxC             (NVKM_VMM_PAGE_xVxx | NVKM_VMM_PAGE_COMP)
135 #define NVKM_VMM_PAGE_SVxC             (NVKM_VMM_PAGE_SVxx | NVKM_VMM_PAGE_COMP)
136 #define NVKM_VMM_PAGE_xxHC             (NVKM_VMM_PAGE_xxHx | NVKM_VMM_PAGE_COMP)
137 #define NVKM_VMM_PAGE_SxHC             (NVKM_VMM_PAGE_SxHx | NVKM_VMM_PAGE_COMP)
138 	u8 type;
139 };
140 
141 struct nvkm_vmm_func {
142 	int (*join)(struct nvkm_vmm *, struct nvkm_memory *inst);
143 	void (*part)(struct nvkm_vmm *, struct nvkm_memory *inst);
144 
145 	int (*aper)(enum nvkm_memory_target);
146 	int (*valid)(struct nvkm_vmm *, void *argv, u32 argc,
147 		     struct nvkm_vmm_map *);
148 	void (*flush)(struct nvkm_vmm *, int depth);
149 
150 	int (*mthd)(struct nvkm_vmm *, struct nvkm_client *,
151 		    u32 mthd, void *argv, u32 argc);
152 
153 	void (*invalidate_pdb)(struct nvkm_vmm *, u64 addr);
154 
155 	u64 page_block;
156 	const struct nvkm_vmm_page page[];
157 };
158 
159 struct nvkm_vmm_join {
160 	struct nvkm_memory *inst;
161 	struct list_head head;
162 };
163 
164 int nvkm_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *,
165 		  u32 pd_header, bool managed, u64 addr, u64 size,
166 		  struct lock_class_key *, const char *name,
167 		  struct nvkm_vmm **);
168 int nvkm_vmm_ctor(const struct nvkm_vmm_func *, struct nvkm_mmu *,
169 		  u32 pd_header, bool managed, u64 addr, u64 size,
170 		  struct lock_class_key *, const char *name, struct nvkm_vmm *);
171 struct nvkm_vma *nvkm_vmm_node_search(struct nvkm_vmm *, u64 addr);
172 struct nvkm_vma *nvkm_vmm_node_split(struct nvkm_vmm *, struct nvkm_vma *,
173 				     u64 addr, u64 size);
174 int nvkm_vmm_get_locked(struct nvkm_vmm *, bool getref, bool mapref,
175 			bool sparse, u8 page, u8 align, u64 size,
176 			struct nvkm_vma **pvma);
177 void nvkm_vmm_put_locked(struct nvkm_vmm *, struct nvkm_vma *);
178 void nvkm_vmm_unmap_locked(struct nvkm_vmm *, struct nvkm_vma *, bool pfn);
179 void nvkm_vmm_unmap_region(struct nvkm_vmm *, struct nvkm_vma *);
180 
181 #define NVKM_VMM_PFN_ADDR                                 0xfffffffffffff000ULL
182 #define NVKM_VMM_PFN_ADDR_SHIFT                                              12
183 #define NVKM_VMM_PFN_APER                                 0x00000000000000f0ULL
184 #define NVKM_VMM_PFN_HOST                                 0x0000000000000000ULL
185 #define NVKM_VMM_PFN_VRAM                                 0x0000000000000010ULL
186 #define NVKM_VMM_PFN_W                                    0x0000000000000002ULL
187 #define NVKM_VMM_PFN_V                                    0x0000000000000001ULL
188 #define NVKM_VMM_PFN_NONE                                 0x0000000000000000ULL
189 
190 int nvkm_vmm_pfn_map(struct nvkm_vmm *, u8 page, u64 addr, u64 size, u64 *pfn);
191 int nvkm_vmm_pfn_unmap(struct nvkm_vmm *, u64 addr, u64 size);
192 
193 struct nvkm_vma *nvkm_vma_tail(struct nvkm_vma *, u64 tail);
194 
195 int nv04_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, u32,
196 		  bool, u64, u64, void *, u32, struct lock_class_key *,
197 		  const char *, struct nvkm_vmm **);
198 int nv04_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *);
199 
200 int nv50_vmm_join(struct nvkm_vmm *, struct nvkm_memory *);
201 void nv50_vmm_part(struct nvkm_vmm *, struct nvkm_memory *);
202 int nv50_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *);
203 void nv50_vmm_flush(struct nvkm_vmm *, int);
204 
205 int gf100_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *,
206 		   struct nvkm_mmu *, bool, u64, u64, void *, u32,
207 		   struct lock_class_key *, const char *, struct nvkm_vmm **);
208 int gf100_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base);
209 int gf100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *);
210 void gf100_vmm_part(struct nvkm_vmm *, struct nvkm_memory *);
211 int gf100_vmm_aper(enum nvkm_memory_target);
212 int gf100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *);
213 void gf100_vmm_flush(struct nvkm_vmm *, int);
214 void gf100_vmm_invalidate(struct nvkm_vmm *, u32 type);
215 void gf100_vmm_invalidate_pdb(struct nvkm_vmm *, u64 addr);
216 
217 int gk20a_vmm_aper(enum nvkm_memory_target);
218 
219 int gm200_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *,
220 		   struct nvkm_mmu *, bool, u64, u64, void *, u32,
221 		   struct lock_class_key *, const char *, struct nvkm_vmm **);
222 int gm200_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base);
223 int gm200_vmm_join(struct nvkm_vmm *, struct nvkm_memory *);
224 
225 int gp100_vmm_new_(const struct nvkm_vmm_func *,
226 		   struct nvkm_mmu *, bool, u64, u64, void *, u32,
227 		   struct lock_class_key *, const char *, struct nvkm_vmm **);
228 int gp100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *);
229 int gp100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *);
230 void gp100_vmm_flush(struct nvkm_vmm *, int);
231 int gp100_vmm_mthd(struct nvkm_vmm *, struct nvkm_client *, u32, void *, u32);
232 void gp100_vmm_invalidate_pdb(struct nvkm_vmm *, u64 addr);
233 
234 int gv100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *);
235 
236 int nv04_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
237 		 struct lock_class_key *, const char *, struct nvkm_vmm **);
238 int nv41_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
239 		 struct lock_class_key *, const char *, struct nvkm_vmm **);
240 int nv44_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
241 		 struct lock_class_key *, const char *, struct nvkm_vmm **);
242 int nv50_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
243 		 struct lock_class_key *, const char *, struct nvkm_vmm **);
244 int mcp77_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
245 		  struct lock_class_key *, const char *, struct nvkm_vmm **);
246 int g84_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
247 		struct lock_class_key *, const char *, struct nvkm_vmm **);
248 int gf100_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
249 		  struct lock_class_key *, const char *, struct nvkm_vmm **);
250 int gk104_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
251 		  struct lock_class_key *, const char *, struct nvkm_vmm **);
252 int gk20a_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
253 		  struct lock_class_key *, const char *, struct nvkm_vmm **);
254 int gm200_vmm_new_fixed(struct nvkm_mmu *, bool, u64, u64, void *, u32,
255 			struct lock_class_key *, const char *,
256 			struct nvkm_vmm **);
257 int gm200_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
258 		  struct lock_class_key *, const char *,
259 		  struct nvkm_vmm **);
260 int gm20b_vmm_new_fixed(struct nvkm_mmu *, bool, u64, u64, void *, u32,
261 			struct lock_class_key *, const char *,
262 			struct nvkm_vmm **);
263 int gm20b_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
264 		  struct lock_class_key *, const char *,
265 		  struct nvkm_vmm **);
266 int gp100_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
267 		  struct lock_class_key *, const char *,
268 		  struct nvkm_vmm **);
269 int gp10b_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
270 		  struct lock_class_key *, const char *,
271 		  struct nvkm_vmm **);
272 int gv100_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
273 		  struct lock_class_key *, const char *,
274 		  struct nvkm_vmm **);
275 int tu102_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
276 		  struct lock_class_key *, const char *,
277 		  struct nvkm_vmm **);
278 
279 #define VMM_PRINT(l,v,p,f,a...) do {                                           \
280 	struct nvkm_vmm *_vmm = (v);                                           \
281 	if (CONFIG_NOUVEAU_DEBUG >= (l) && _vmm->debug >= (l)) {               \
282 		nvkm_printk_(&_vmm->mmu->subdev, 0, p, "%s: "f"\n",            \
283 			     _vmm->name, ##a);                                 \
284 	}                                                                      \
285 } while(0)
286 #define VMM_DEBUG(v,f,a...) VMM_PRINT(NV_DBG_DEBUG, (v), info, f, ##a)
287 #define VMM_TRACE(v,f,a...) VMM_PRINT(NV_DBG_TRACE, (v), info, f, ##a)
288 #define VMM_SPAM(v,f,a...)  VMM_PRINT(NV_DBG_SPAM , (v),  dbg, f, ##a)
289 
290 #define VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL,BASE,SIZE,NEXT) do {            \
291 	nvkm_kmap((PT)->memory);                                               \
292 	while (PTEN) {                                                         \
293 		u64 _ptes = ((SIZE) - MAP->off) >> MAP->page->shift;           \
294 		u64 _addr = ((BASE) + MAP->off);                               \
295                                                                                \
296 		if (_ptes > PTEN) {                                            \
297 			MAP->off += PTEN << MAP->page->shift;                  \
298 			_ptes = PTEN;                                          \
299 		} else {                                                       \
300 			MAP->off = 0;                                          \
301 			NEXT;                                                  \
302 		}                                                              \
303                                                                                \
304 		VMM_SPAM(VMM, "ITER %08x %08x PTE(s)", PTEI, (u32)_ptes);      \
305                                                                                \
306 		FILL(VMM, PT, PTEI, _ptes, MAP, _addr);                        \
307 		PTEI += _ptes;                                                 \
308 		PTEN -= _ptes;                                                 \
309 	};                                                                     \
310 	nvkm_done((PT)->memory);                                               \
311 } while(0)
312 
313 #define VMM_MAP_ITER_MEM(VMM,PT,PTEI,PTEN,MAP,FILL)                            \
314 	VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL,                                \
315 		     ((u64)MAP->mem->offset << NVKM_RAM_MM_SHIFT),             \
316 		     ((u64)MAP->mem->length << NVKM_RAM_MM_SHIFT),             \
317 		     (MAP->mem = MAP->mem->next))
318 #define VMM_MAP_ITER_DMA(VMM,PT,PTEI,PTEN,MAP,FILL)                            \
319 	VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL,                                \
320 		     *MAP->dma, PAGE_SIZE, MAP->dma++)
321 #define VMM_MAP_ITER_SGL(VMM,PT,PTEI,PTEN,MAP,FILL)                            \
322 	VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL,                                \
323 		     sg_dma_address(MAP->sgl), sg_dma_len(MAP->sgl),           \
324 		     (MAP->sgl = sg_next(MAP->sgl)))
325 
326 #define VMM_FO(m,o,d,c,b) nvkm_fo##b((m)->memory, (o), (d), (c))
327 #define VMM_WO(m,o,d,c,b) nvkm_wo##b((m)->memory, (o), (d))
328 #define VMM_XO(m,v,o,d,c,b,fn,f,a...) do {                                     \
329 	const u32 _pteo = (o); u##b _data = (d);                               \
330 	VMM_SPAM((v), "   %010"PRIx64" "f, (u64)(m)->addr + _pteo, _data, ##a);          \
331 	VMM_##fn((m), (m)->base + _pteo, _data, (c), b);                       \
332 } while(0)
333 
334 #define VMM_WO032(m,v,o,d) VMM_XO((m),(v),(o),(d),  1, 32, WO, "%08x")
335 #define VMM_FO032(m,v,o,d,c)                                                   \
336 	VMM_XO((m),(v),(o),(d),(c), 32, FO, "%08x %08x", (c))
337 
338 #define VMM_WO064(m,v,o,d) VMM_XO((m),(v),(o),(d),  1, 64, WO, "%016"PRIx64"")
339 #define VMM_FO064(m,v,o,d,c)                                                   \
340 	VMM_XO((m),(v),(o),(d),(c), 64, FO, "%016"PRIx64" %08x", (c))
341 
342 #define VMM_XO128(m,v,o,lo,hi,c,f,a...) do {                                   \
343 	u32 _pteo = (o), _ptes = (c);                                          \
344 	const u64 _addr = (m)->addr + _pteo;                                   \
345 	VMM_SPAM((v), "   %010"PRIx64" %016"PRIx64"%016"PRIx64""f, _addr, (u64)(hi), (u64)(lo), ##a);   \
346 	while (_ptes--) {                                                      \
347 		nvkm_wo64((m)->memory, (m)->base + _pteo + 0, (lo));           \
348 		nvkm_wo64((m)->memory, (m)->base + _pteo + 8, (hi));           \
349 		_pteo += 0x10;                                                 \
350 	}                                                                      \
351 } while(0)
352 
353 #define VMM_WO128(m,v,o,lo,hi) VMM_XO128((m),(v),(o),(lo),(hi), 1, "")
354 #define VMM_FO128(m,v,o,lo,hi,c) do {                                          \
355 	nvkm_kmap((m)->memory);                                                \
356 	VMM_XO128((m),(v),(o),(lo),(hi),(c), " %08x", (c));                    \
357 	nvkm_done((m)->memory);                                                \
358 } while(0)
359 #endif
360