1 /* $NetBSD: nouveau_nvkm_engine_fifo_gpfifogv100.c,v 1.3 2021/12/19 10:51:57 riastradh Exp $ */
2
3 /*
4 * Copyright 2018 Red Hat Inc.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 */
24 #include <sys/cdefs.h>
25 __KERNEL_RCSID(0, "$NetBSD: nouveau_nvkm_engine_fifo_gpfifogv100.c,v 1.3 2021/12/19 10:51:57 riastradh Exp $");
26
27 #include "changk104.h"
28 #include "cgrp.h"
29
30 #include <core/client.h>
31 #include <core/gpuobj.h>
32
33 #include <nvif/clc36f.h>
34 #include <nvif/unpack.h>
35
36 static u32
gv100_fifo_gpfifo_submit_token(struct nvkm_fifo_chan * chan)37 gv100_fifo_gpfifo_submit_token(struct nvkm_fifo_chan *chan)
38 {
39 return chan->chid;
40 }
41
42 static int
gv100_fifo_gpfifo_engine_valid(struct gk104_fifo_chan * chan,bool ce,bool valid)43 gv100_fifo_gpfifo_engine_valid(struct gk104_fifo_chan *chan, bool ce, bool valid)
44 {
45 struct nvkm_subdev *subdev = &chan->base.fifo->engine.subdev;
46 struct nvkm_device *device = subdev->device;
47 const u32 mask = ce ? 0x00020000 : 0x00010000;
48 const u32 data = valid ? mask : 0x00000000;
49 int ret;
50
51 /* Block runlist to prevent the channel from being rescheduled. */
52 mutex_lock(&subdev->mutex);
53 nvkm_mask(device, 0x002630, BIT(chan->runl), BIT(chan->runl));
54
55 /* Preempt the channel. */
56 ret = gk104_fifo_gpfifo_kick_locked(chan);
57 if (ret == 0) {
58 /* Update engine context validity. */
59 nvkm_kmap(chan->base.inst);
60 nvkm_mo32(chan->base.inst, 0x0ac, mask, data);
61 nvkm_done(chan->base.inst);
62 }
63
64 /* Resume runlist. */
65 nvkm_mask(device, 0x002630, BIT(chan->runl), 0);
66 mutex_unlock(&subdev->mutex);
67 return ret;
68 }
69
70 int
gv100_fifo_gpfifo_engine_fini(struct nvkm_fifo_chan * base,struct nvkm_engine * engine,bool suspend)71 gv100_fifo_gpfifo_engine_fini(struct nvkm_fifo_chan *base,
72 struct nvkm_engine *engine, bool suspend)
73 {
74 struct gk104_fifo_chan *chan = gk104_fifo_chan(base);
75 struct nvkm_gpuobj *inst = chan->base.inst;
76 int ret;
77
78 if (engine->subdev.index >= NVKM_ENGINE_CE0 &&
79 engine->subdev.index <= NVKM_ENGINE_CE_LAST)
80 return gk104_fifo_gpfifo_kick(chan);
81
82 ret = gv100_fifo_gpfifo_engine_valid(chan, false, false);
83 if (ret && suspend)
84 return ret;
85
86 nvkm_kmap(inst);
87 nvkm_wo32(inst, 0x0210, 0x00000000);
88 nvkm_wo32(inst, 0x0214, 0x00000000);
89 nvkm_done(inst);
90 return ret;
91 }
92
93 int
gv100_fifo_gpfifo_engine_init(struct nvkm_fifo_chan * base,struct nvkm_engine * engine)94 gv100_fifo_gpfifo_engine_init(struct nvkm_fifo_chan *base,
95 struct nvkm_engine *engine)
96 {
97 struct gk104_fifo_chan *chan = gk104_fifo_chan(base);
98 struct nvkm_gpuobj *inst = chan->base.inst;
99 u64 addr;
100
101 if (engine->subdev.index >= NVKM_ENGINE_CE0 &&
102 engine->subdev.index <= NVKM_ENGINE_CE_LAST)
103 return 0;
104
105 addr = chan->engn[engine->subdev.index].vma->addr;
106 nvkm_kmap(inst);
107 nvkm_wo32(inst, 0x210, lower_32_bits(addr) | 0x00000004);
108 nvkm_wo32(inst, 0x214, upper_32_bits(addr));
109 nvkm_done(inst);
110
111 return gv100_fifo_gpfifo_engine_valid(chan, false, true);
112 }
113
114 static const struct nvkm_fifo_chan_func
115 gv100_fifo_gpfifo = {
116 .dtor = gk104_fifo_gpfifo_dtor,
117 .init = gk104_fifo_gpfifo_init,
118 .fini = gk104_fifo_gpfifo_fini,
119 .ntfy = gf100_fifo_chan_ntfy,
120 .engine_ctor = gk104_fifo_gpfifo_engine_ctor,
121 .engine_dtor = gk104_fifo_gpfifo_engine_dtor,
122 .engine_init = gv100_fifo_gpfifo_engine_init,
123 .engine_fini = gv100_fifo_gpfifo_engine_fini,
124 .submit_token = gv100_fifo_gpfifo_submit_token,
125 };
126
127 int
gv100_fifo_gpfifo_new_(const struct nvkm_fifo_chan_func * func,struct gk104_fifo * fifo,u64 * runlists,u16 * chid,u64 vmm,u64 ioffset,u64 ilength,u64 * inst,bool priv,u32 * token,const struct nvkm_oclass * oclass,struct nvkm_object ** pobject)128 gv100_fifo_gpfifo_new_(const struct nvkm_fifo_chan_func *func,
129 struct gk104_fifo *fifo, u64 *runlists, u16 *chid,
130 u64 vmm, u64 ioffset, u64 ilength, u64 *inst, bool priv,
131 u32 *token, const struct nvkm_oclass *oclass,
132 struct nvkm_object **pobject)
133 {
134 struct nvkm_device *device = fifo->base.engine.subdev.device;
135 struct gk104_fifo_chan *chan;
136 int runlist = ffs(*runlists) -1, ret, i;
137 unsigned long engm;
138 u64 subdevs = 0;
139 u64 usermem, mthd;
140 u32 size;
141
142 if (!vmm || runlist < 0 || runlist >= fifo->runlist_nr)
143 return -EINVAL;
144 *runlists = BIT_ULL(runlist);
145
146 engm = fifo->runlist[runlist].engm;
147 for_each_set_bit(i, &engm, fifo->engine_nr) {
148 if (fifo->engine[i].engine)
149 subdevs |= BIT_ULL(fifo->engine[i].engine->subdev.index);
150 }
151
152 /* Allocate the channel. */
153 if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL)))
154 return -ENOMEM;
155 *pobject = &chan->base.object;
156 chan->fifo = fifo;
157 chan->runl = runlist;
158 INIT_LIST_HEAD(&chan->head);
159
160 ret = nvkm_fifo_chan_ctor(func, &fifo->base, 0x1000, 0x1000, true, vmm,
161 0, subdevs, 1, fifo->user.bar->addr, 0x200,
162 oclass, &chan->base);
163 if (ret)
164 return ret;
165
166 *chid = chan->base.chid;
167 *inst = chan->base.inst->addr;
168 *token = chan->base.func->submit_token(&chan->base);
169
170 /* Hack to support GPUs where even individual channels should be
171 * part of a channel group.
172 */
173 if (fifo->func->cgrp_force) {
174 if (!(chan->cgrp = kmalloc(sizeof(*chan->cgrp), GFP_KERNEL)))
175 return -ENOMEM;
176 chan->cgrp->id = chan->base.chid;
177 INIT_LIST_HEAD(&chan->cgrp->head);
178 INIT_LIST_HEAD(&chan->cgrp->chan);
179 chan->cgrp->chan_nr = 0;
180 }
181
182 /* Clear channel control registers. */
183 usermem = chan->base.chid * 0x200;
184 ilength = order_base_2(ilength / 8);
185
186 nvkm_kmap(fifo->user.mem);
187 for (i = 0; i < 0x200; i += 4)
188 nvkm_wo32(fifo->user.mem, usermem + i, 0x00000000);
189 nvkm_done(fifo->user.mem);
190 usermem = nvkm_memory_addr(fifo->user.mem) + usermem;
191
192 /* Allocate fault method buffer (magics come from nvgpu). */
193 size = nvkm_rd32(device, 0x104028); /* NV_PCE_PCE_MAP */
194 size = 27 * 5 * (((9 + 1 + 3) * hweight32(size)) + 2);
195 size = roundup(size, PAGE_SIZE);
196
197 ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, size, 0x1000, true,
198 &chan->mthd);
199 if (ret)
200 return ret;
201
202 mthd = nvkm_memory_bar2(chan->mthd);
203 if (mthd == ~0ULL)
204 return -EFAULT;
205
206 /* RAMFC */
207 nvkm_kmap(chan->base.inst);
208 nvkm_wo32(chan->base.inst, 0x008, lower_32_bits(usermem));
209 nvkm_wo32(chan->base.inst, 0x00c, upper_32_bits(usermem));
210 nvkm_wo32(chan->base.inst, 0x010, 0x0000face);
211 nvkm_wo32(chan->base.inst, 0x030, 0x7ffff902);
212 nvkm_wo32(chan->base.inst, 0x048, lower_32_bits(ioffset));
213 nvkm_wo32(chan->base.inst, 0x04c, upper_32_bits(ioffset) |
214 (ilength << 16));
215 nvkm_wo32(chan->base.inst, 0x084, 0x20400000);
216 nvkm_wo32(chan->base.inst, 0x094, 0x30000001);
217 nvkm_wo32(chan->base.inst, 0x0e4, priv ? 0x00000020 : 0x00000000);
218 nvkm_wo32(chan->base.inst, 0x0e8, chan->base.chid);
219 nvkm_wo32(chan->base.inst, 0x0f4, 0x00001000);
220 nvkm_wo32(chan->base.inst, 0x0f8, 0x10003080);
221 nvkm_mo32(chan->base.inst, 0x218, 0x00000000, 0x00000000);
222 nvkm_wo32(chan->base.inst, 0x220, lower_32_bits(mthd));
223 nvkm_wo32(chan->base.inst, 0x224, upper_32_bits(mthd));
224 nvkm_done(chan->base.inst);
225 return gv100_fifo_gpfifo_engine_valid(chan, true, true);
226 }
227
228 int
gv100_fifo_gpfifo_new(struct gk104_fifo * fifo,const struct nvkm_oclass * oclass,void * data,u32 size,struct nvkm_object ** pobject)229 gv100_fifo_gpfifo_new(struct gk104_fifo *fifo, const struct nvkm_oclass *oclass,
230 void *data, u32 size, struct nvkm_object **pobject)
231 {
232 struct nvkm_object *parent = oclass->parent;
233 union {
234 struct volta_channel_gpfifo_a_v0 v0;
235 } *args = data;
236 int ret = -ENOSYS;
237
238 nvif_ioctl(parent, "create channel gpfifo size %d\n", size);
239 if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
240 nvif_ioctl(parent, "create channel gpfifo vers %d vmm %"PRIx64" "
241 "ioffset %016"PRIx64" ilength %08x "
242 "runlist %016"PRIx64" priv %d\n",
243 args->v0.version, args->v0.vmm, args->v0.ioffset,
244 args->v0.ilength, args->v0.runlist, args->v0.priv);
245 if (args->v0.priv && !oclass->client->super)
246 return -EINVAL;
247 return gv100_fifo_gpfifo_new_(&gv100_fifo_gpfifo, fifo,
248 &args->v0.runlist,
249 &args->v0.chid,
250 args->v0.vmm,
251 args->v0.ioffset,
252 args->v0.ilength,
253 &args->v0.inst,
254 args->v0.priv,
255 &args->v0.token,
256 oclass, pobject);
257 }
258
259 return ret;
260 }
261