xref: /netbsd-src/sys/arch/arm/apple/apple_nvme.c (revision e24a9df7237c4ed793f8823d70fe5b3f33a21742)
1 /*	$NetBSD: apple_nvme.c,v 1.1 2022/05/07 08:20:03 skrll Exp $	*/
2 /*	$OpenBSD: aplns.c,v 1.5 2021/08/29 11:23:29 kettenis Exp $ */
3 
4 /*-
5  * Copyright (c) 2022 The NetBSD Foundation, Inc.
6  * All rights reserved.
7  *
8  * This code is derived from software contributed to The NetBSD Foundation
9  * by Nick Hudson
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions
13  * are met:
14  * 1. Redistributions of source code must retain the above copyright
15  *    notice, this list of conditions and the following disclaimer.
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in the
18  *    documentation and/or other materials provided with the distribution.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30  * POSSIBILITY OF SUCH DAMAGE.
31  */
32 
33 /*
34  * Copyright (c) 2014, 2021 David Gwynne <dlg@openbsd.org>
35  *
36  * Permission to use, copy, modify, and distribute this software for any
37  * purpose with or without fee is hereby granted, provided that the above
38  * copyright notice and this permission notice appear in all copies.
39  *
40  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
41  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
42  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
43  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
44  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
45  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
46  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
47  */
48 
49 #include <sys/cdefs.h>
50 __KERNEL_RCSID(0, "$NetBSD: apple_nvme.c,v 1.1 2022/05/07 08:20:03 skrll Exp $");
51 
52 #include <sys/param.h>
53 
54 #include <sys/bus.h>
55 #include <sys/device.h>
56 #include <sys/kmem.h>
57 #include <sys/intr.h>
58 
59 #include <dev/ic/nvmereg.h>
60 #include <dev/ic/nvmevar.h>
61 
62 #include <dev/fdt/fdtvar.h>
63 
64 #include <arm/apple/apple_rtkit.h>
65 
66 int apple_nvme_mpsafe = 1;
67 
68 #define NVME_IO_Q 1
69 
70 #define ANS_CPU_CTRL		0x0044
71 #define ANS_CPU_CTRL_RUN		__BIT(4)
72 
73 #define ANS_MAX_PEND_CMDS_CTRL	0x01210
74 #define  ANS_MAX_QUEUE_DEPTH		64
75 #define ANS_BOOT_STATUS		0x01300
76 #define  ANS_BOOT_STATUS_OK		0xde71ce55
77 
78 #define ANS_MODESEL_REG		0x01304
79 #define ANS_UNKNOWN_CTRL	0x24008
80 #define  ANS_PRP_NULL_CHECK		__BIT(11)
81 #define ANS_LINEAR_SQ_CTRL	0x24908
82 #define  ANS_LINEAR_SQ_CTRL_EN		__BIT(0)
83 #define ANS_LINEAR_ASQ_DB	0x2490c
84 #define ANS_LINEAR_IOSQ_DB	0x24910
85 
86 #define ANS_NVMMU_NUM		0x28100
87 #define ANS_NVMMU_BASE_ASQ	0x28108
88 #define ANS_NVMMU_BASE_IOSQ	0x28110
89 #define ANS_NVMMU_TCB_INVAL	0x28118
90 #define ANS_NVMMU_TCB_STAT	0x28120
91 
92 #define ANS_NVMMU_TCB_SIZE	0x4000
93 #define ANS_NVMMU_TCB_PITCH	0x80
94 
95 struct ans_nvmmu_tcb {
96 	uint8_t		tcb_opcode;
97 	uint8_t		tcb_flags;
98 #define ANS_NVMMU_TCB_WRITE		__BIT(0)
99 #define ANS_NVMMU_TCB_READ		__BIT(1)
100 	uint8_t		tcb_cid;
101 	uint8_t		tcb_pad0[1];
102 
103 	uint32_t	tcb_prpl_len;
104 	uint8_t		tcb_pad1[16];
105 
106 	uint64_t	tcb_prp[2];
107 };
108 
109 void	nvme_ans_enable(struct nvme_softc *);
110 
111 int	nvme_ans_q_alloc(struct nvme_softc *, struct nvme_queue *);
112 void	nvme_ans_q_free(struct nvme_softc *, struct nvme_queue *);
113 
114 uint32_t
115 	nvme_ans_sq_enter(struct nvme_softc *, struct nvme_queue *,
116 			  struct nvme_ccb *);
117 void	nvme_ans_sq_leave(struct nvme_softc *,
118 			  struct nvme_queue *, struct nvme_ccb *);
119 
120 void	nvme_ans_cq_done(struct nvme_softc *,
121 			 struct nvme_queue *, struct nvme_ccb *);
122 
123 static const struct nvme_ops nvme_ans_ops = {
124 	.op_enable		= nvme_ans_enable,
125 
126 	.op_q_alloc		= nvme_ans_q_alloc,
127 	.op_q_free		= nvme_ans_q_free,
128 
129 	.op_sq_enter		= nvme_ans_sq_enter,
130 	.op_sq_leave		= nvme_ans_sq_leave,
131 	.op_sq_enter_locked	= nvme_ans_sq_enter,
132 	.op_sq_leave_locked	= nvme_ans_sq_leave,
133 
134 	.op_cq_done		= nvme_ans_cq_done,
135 };
136 
137 static const struct device_compatible_entry compat_data[] = {
138 	{ .compat = "apple,nvme-m1" },
139 	{ .compat = "apple,nvme-ans2" },
140 	DEVICE_COMPAT_EOL
141 };
142 
143 struct apple_nvme_softc {
144 	struct nvme_softc asc_nvme;
145 	int asc_phandle;
146 
147 	bus_space_tag_t	asc_iot;
148 	bus_space_handle_t asc_ioh;
149 	bus_size_t asc_size;
150 
151 	struct rtkit_state *asc_rtkit;
152 
153 	size_t asc_nintrs;
154 	void *asc_ihs;
155 };
156 
157 void
nvme_ans_enable(struct nvme_softc * sc)158 nvme_ans_enable(struct nvme_softc *sc)
159 {
160 	nvme_write4(sc, ANS_NVMMU_NUM,
161 	    (ANS_NVMMU_TCB_SIZE / ANS_NVMMU_TCB_PITCH) - 1);
162 	nvme_write4(sc, ANS_MODESEL_REG, 0);
163 }
164 
165 
166 int
nvme_ans_q_alloc(struct nvme_softc * sc,struct nvme_queue * q)167 nvme_ans_q_alloc(struct nvme_softc *sc, struct nvme_queue *q)
168 {
169 	bus_size_t db, base;
170 
171 	KASSERT(q->q_entries <= (ANS_NVMMU_TCB_SIZE / ANS_NVMMU_TCB_PITCH));
172 
173 	q->q_nvmmu_dmamem = nvme_dmamem_alloc(sc, ANS_NVMMU_TCB_SIZE);
174 	if (q->q_nvmmu_dmamem == NULL)
175 		return -1;
176 
177 	memset(NVME_DMA_KVA(q->q_nvmmu_dmamem), 0,
178 	    NVME_DMA_LEN(q->q_nvmmu_dmamem));
179 
180 	switch (q->q_id) {
181 	case NVME_IO_Q:
182 		db = ANS_LINEAR_IOSQ_DB;
183 		base = ANS_NVMMU_BASE_IOSQ;
184 		break;
185 	case NVME_ADMIN_Q:
186 		db = ANS_LINEAR_ASQ_DB;
187 		base = ANS_NVMMU_BASE_ASQ;
188 		break;
189 	default:
190 		panic("unsupported queue id %u", q->q_id);
191 		/* NOTREACHED */
192 	}
193 
194 	q->q_sqtdbl = db;
195 
196 	nvme_dmamem_sync(sc, q->q_nvmmu_dmamem, BUS_DMASYNC_PREWRITE);
197 	nvme_write8(sc, base, NVME_DMA_DVA(q->q_nvmmu_dmamem));
198 
199 	return 0;
200 }
201 
202 void
nvme_ans_q_free(struct nvme_softc * sc,struct nvme_queue * q)203 nvme_ans_q_free(struct nvme_softc *sc,
204     struct nvme_queue *q)
205 {
206 	nvme_dmamem_sync(sc, q->q_nvmmu_dmamem, BUS_DMASYNC_POSTWRITE);
207 	nvme_dmamem_free(sc, q->q_nvmmu_dmamem);
208 }
209 
210 uint32_t
nvme_ans_sq_enter(struct nvme_softc * sc,struct nvme_queue * q,struct nvme_ccb * ccb)211 nvme_ans_sq_enter(struct nvme_softc *sc,
212     struct nvme_queue *q, struct nvme_ccb *ccb)
213 {
214 	return ccb->ccb_id;
215 }
216 
217 static inline struct ans_nvmmu_tcb *
nvme_ans_tcb(struct nvme_queue * q,unsigned int qid)218 nvme_ans_tcb(struct nvme_queue *q, unsigned int qid)
219 {
220 	uint8_t *ptr = NVME_DMA_KVA(q->q_nvmmu_dmamem);
221 	ptr += qid * ANS_NVMMU_TCB_PITCH;
222 
223 	return (struct ans_nvmmu_tcb *)ptr;
224 }
225 
226 void
nvme_ans_sq_leave(struct nvme_softc * sc,struct nvme_queue * q,struct nvme_ccb * ccb)227 nvme_ans_sq_leave(struct nvme_softc *sc,
228     struct nvme_queue *q, struct nvme_ccb *ccb)
229 {
230 	unsigned int id = ccb->ccb_id;
231 	struct ans_nvmmu_tcb *tcb = nvme_ans_tcb(q, id);
232 	struct nvme_sqe_io *sqe = NVME_DMA_KVA(q->q_sq_dmamem);
233 	sqe += id;
234 
235 	bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_nvmmu_dmamem),
236 	    ANS_NVMMU_TCB_PITCH * id, sizeof(*tcb), BUS_DMASYNC_POSTWRITE);
237 
238 	memset(tcb, 0, sizeof(*tcb));
239 	tcb->tcb_opcode = sqe->opcode;
240 	tcb->tcb_flags = ANS_NVMMU_TCB_WRITE | ANS_NVMMU_TCB_READ;
241 	tcb->tcb_cid = id;
242 	tcb->tcb_prpl_len = sqe->nlb;
243 	tcb->tcb_prp[0] = sqe->entry.prp[0];
244 	tcb->tcb_prp[1] = sqe->entry.prp[1];
245 
246 	bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_nvmmu_dmamem),
247 	    ANS_NVMMU_TCB_PITCH * id, sizeof(*tcb), BUS_DMASYNC_PREWRITE);
248 	nvme_write4(sc, q->q_sqtdbl, id);
249 }
250 
251 void
nvme_ans_cq_done(struct nvme_softc * sc,struct nvme_queue * q,struct nvme_ccb * ccb)252 nvme_ans_cq_done(struct nvme_softc *sc,
253     struct nvme_queue *q, struct nvme_ccb *ccb)
254 {
255 	unsigned int id = ccb->ccb_id;
256 	struct ans_nvmmu_tcb *tcb = nvme_ans_tcb(q, id);
257 
258 	bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_nvmmu_dmamem),
259 	    ANS_NVMMU_TCB_PITCH * id, sizeof(*tcb), BUS_DMASYNC_POSTWRITE);
260 	memset(tcb, 0, sizeof(*tcb));
261 	bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_nvmmu_dmamem),
262 	    ANS_NVMMU_TCB_PITCH * id, sizeof(*tcb), BUS_DMASYNC_PREWRITE);
263 
264 	nvme_write4(sc, ANS_NVMMU_TCB_INVAL, id);
265 	uint32_t stat = nvme_read4(sc, ANS_NVMMU_TCB_STAT);
266 	if (stat != 0) {
267 		printf("%s: nvmmu tcp stat is non-zero: 0x%08x\n",
268 		    device_xname(sc->sc_dev), stat);
269 	}
270 }
271 
272 
273 static int
apple_nvme_intr_establish(struct nvme_softc * sc,uint16_t qid,struct nvme_queue * q)274 apple_nvme_intr_establish(struct nvme_softc *sc, uint16_t qid,
275     struct nvme_queue *q)
276 {
277 	struct apple_nvme_softc * const asc =
278 	    container_of(sc, struct apple_nvme_softc, asc_nvme);
279 	const int phandle = asc->asc_phandle;
280 	char intr_xname[INTRDEVNAMEBUF];
281 	char intrstr[128];
282 	const device_t self = sc->sc_dev;
283 
284 	KASSERT(sc->sc_use_mq || qid == NVME_ADMIN_Q);
285 	KASSERT(sc->sc_ih[qid] == NULL);
286 
287 	if (!fdtbus_intr_str(phandle, 0, intrstr, sizeof(intrstr))) {
288 		aprint_error(": couldn't decode interrupt\n");
289 		return 1;
290 	}
291 	sc->sc_ih[qid] = fdtbus_intr_establish_xname(phandle, 0, IPL_BIO,
292 	    FDT_INTR_MPSAFE, nvme_intr, sc, device_xname(sc->sc_dev));
293 	if (sc->sc_ih[qid] == NULL) {
294 		aprint_error_dev(self, "couldn't establish interrupt on %s\n",
295 		    intrstr);
296 		return 1;
297 	}
298 
299 	/* establish also the software interrupt */
300 	sc->sc_softih[qid] = softint_establish(
301 	    SOFTINT_BIO|(apple_nvme_mpsafe ? SOFTINT_MPSAFE : 0),
302 	    nvme_softintr_intx, q);
303 	if (sc->sc_softih[qid] == NULL) {
304 		fdtbus_intr_disestablish(phandle, sc->sc_ih[qid]);
305 		sc->sc_ih[qid] = NULL;
306 
307 		aprint_error_dev(sc->sc_dev,
308 		    "unable to establish %s soft interrupt\n",
309 		    intr_xname);
310 		return 1;
311 	}
312 
313 	if (!sc->sc_use_mq) {
314 		aprint_normal_dev(sc->sc_dev, "interrupting on %s\n", intrstr);
315 	} else if (qid == NVME_ADMIN_Q) {
316 		aprint_normal_dev(sc->sc_dev,
317 		    "for admin queue interrupting on %s\n", intrstr);
318 	} else {
319 		aprint_normal_dev(sc->sc_dev,
320 		    "for io queue %d interrupting on %s\n", qid, intrstr);
321 	}
322 	return 0;
323 }
324 
325 static int
apple_nvme_intr_disestablish(struct nvme_softc * sc,uint16_t qid)326 apple_nvme_intr_disestablish(struct nvme_softc *sc, uint16_t qid)
327 {
328 	struct apple_nvme_softc * const asc =
329 	   container_of(sc, struct apple_nvme_softc, asc_nvme);
330 
331 	KASSERT(sc->sc_use_mq || qid == NVME_ADMIN_Q);
332 	KASSERT(sc->sc_ih[qid] != NULL);
333 
334 	if (sc->sc_softih) {
335 		softint_disestablish(sc->sc_softih[qid]);
336 		sc->sc_softih[qid] = NULL;
337 	}
338 
339 	fdtbus_intr_disestablish(asc->asc_phandle, sc->sc_ih[qid]);
340 	sc->sc_ih[qid] = NULL;
341 
342 	return 0;
343 }
344 
345 
346 static int
apple_nvme_setup_intr(struct fdt_attach_args * const faa,struct apple_nvme_softc * const asc)347 apple_nvme_setup_intr(struct fdt_attach_args * const faa,
348     struct apple_nvme_softc * const asc)
349 {
350 	struct nvme_softc * const sc = &asc->asc_nvme;
351 
352 	asc->asc_nintrs = 1;
353 	asc->asc_phandle = faa->faa_phandle;
354 
355 	sc->sc_use_mq = asc->asc_nintrs > 1;
356 	sc->sc_nq = 1;	/* sc_use_mq */
357 
358 	return 0;
359 }
360 
361 
362 static int
apple_nvme_match(device_t parent,cfdata_t cf,void * aux)363 apple_nvme_match(device_t parent, cfdata_t cf, void *aux)
364 {
365 	struct fdt_attach_args * const faa = aux;
366 
367 	return of_compatible_match(faa->faa_phandle, compat_data);
368 }
369 
370 static void
apple_nvme_attach(device_t parent,device_t self,void * aux)371 apple_nvme_attach(device_t parent, device_t self, void *aux)
372 {
373 	struct apple_nvme_softc * const asc = device_private(self);
374 	struct nvme_softc *sc = &asc->asc_nvme;
375 	struct fdt_attach_args * const faa = aux;
376 	const int phandle = faa->faa_phandle;
377 	bus_addr_t addr, ans_addr;
378 	bus_size_t size, ans_size;
379 	uint32_t ctrl, status;
380 
381 
382 	if (fdtbus_get_reg(phandle, 0, &addr, &size) != 0) {
383 		aprint_error(": couldn't get NVME registers\n");
384 		return;
385 	}
386 
387 	if (fdtbus_get_reg(phandle, 1, &ans_addr, &ans_size) != 0) {
388 		aprint_error(": couldn't get ANS registers\n");
389 		return;
390 	}
391 
392 	sc->sc_dev = self;
393 	sc->sc_iot = asc->asc_iot = faa->faa_bst;
394 	sc->sc_ios = size;
395 
396 	if (bus_space_map(sc->sc_iot, addr, size, 0, &sc->sc_ioh) != 0) {
397 		aprint_error(": couldn't map NVME registers\n");
398 		return;
399 	}
400 
401 	if (bus_space_map(asc->asc_iot, ans_addr, ans_size, 0, &asc->asc_ioh) != 0) {
402 		aprint_error(": couldn't map ANS registers\n");
403 		goto fail_ansmap;
404 	}
405 
406 	sc->sc_dmat = faa->faa_dmat;
407 	sc->sc_ops = &nvme_ans_ops;
408 	aprint_naive("\n");
409 	aprint_normal(": Apple NVME\n");
410 
411 	apple_nvme_setup_intr(faa, asc);
412 
413 	sc->sc_intr_establish = apple_nvme_intr_establish;
414 	sc->sc_intr_disestablish = apple_nvme_intr_disestablish;
415 
416 	sc->sc_ih = kmem_zalloc(sizeof(*sc->sc_ih) * asc->asc_nintrs, KM_SLEEP);
417 	sc->sc_softih = kmem_zalloc(sizeof(*sc->sc_softih) * asc->asc_nintrs,
418 	    KM_SLEEP);
419 
420 	asc->asc_rtkit = rtkit_init(phandle, NULL);
421 	if (asc->asc_rtkit == NULL) {
422 		aprint_error("can't map mailbox channel\n");
423 		goto fail_rtkit;
424 	}
425 
426 	ctrl = bus_space_read_4(asc->asc_iot, asc->asc_ioh, ANS_CPU_CTRL);
427 	bus_space_write_4(asc->asc_iot, asc->asc_ioh, ANS_CPU_CTRL,
428 	    ctrl | ANS_CPU_CTRL_RUN);
429 
430 	status = bus_space_read_4(sc->sc_iot, sc->sc_ioh, ANS_BOOT_STATUS);
431 	if (status != ANS_BOOT_STATUS_OK)
432 		rtkit_boot(asc->asc_rtkit);
433 
434 	status = bus_space_read_4(sc->sc_iot, sc->sc_ioh, ANS_BOOT_STATUS);
435 	if (status != ANS_BOOT_STATUS_OK) {
436 		aprint_error("firmware not ready\n");
437 		goto fail_ansnotready;
438 	}
439 
440 	bus_space_write_4(sc->sc_iot, sc->sc_ioh, ANS_LINEAR_SQ_CTRL,
441 	    ANS_LINEAR_SQ_CTRL_EN);
442 	bus_space_write_4(sc->sc_iot, sc->sc_ioh, ANS_MAX_PEND_CMDS_CTRL,
443 	    (ANS_MAX_QUEUE_DEPTH << 16) | ANS_MAX_QUEUE_DEPTH);
444 
445 	ctrl = bus_space_read_4(sc->sc_iot, sc->sc_ioh, ANS_UNKNOWN_CTRL);
446 	bus_space_write_4(sc->sc_iot, sc->sc_ioh, ANS_UNKNOWN_CTRL,
447 	    ctrl & ~ANS_PRP_NULL_CHECK);
448 
449 	if (nvme_attach(sc) != 0) {
450 		/* error printed by nvme_attach() */
451 		return;
452 	}
453 
454 	SET(sc->sc_flags, NVME_F_ATTACHED);
455 	return;
456 fail_ansnotready:
457 
458 fail_rtkit:
459 
460 fail_ansmap:
461 	bus_space_unmap(sc->sc_iot, sc->sc_ioh, size);
462 }
463 
464 CFATTACH_DECL_NEW(apple_nvme, sizeof(struct apple_nvme_softc),
465 	apple_nvme_match, apple_nvme_attach, NULL, NULL);
466