1 /* $NetBSD: bus_space_alignstride_chipdep.c,v 1.32 2022/09/29 07:00:46 skrll Exp $ */
2
3 /*-
4 * Copyright (c) 1998, 2000, 2001 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
9 * NASA Ames Research Center.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE.
31 */
32
33 /*
34 * Copyright (c) 1995, 1996 Carnegie-Mellon University.
35 * All rights reserved.
36 *
37 * Author: Chris G. Demetriou
38 *
39 * Permission to use, copy, modify and distribute this software and
40 * its documentation is hereby granted, provided that both the copyright
41 * notice and this permission notice appear in all copies of the
42 * software, derivative works or modified versions, and any portions
43 * thereof, and that both notices appear in supporting documentation.
44 *
45 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
46 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
47 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
48 *
49 * Carnegie Mellon requests users of this software to return to
50 *
51 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
52 * School of Computer Science
53 * Carnegie Mellon University
54 * Pittsburgh PA 15213-3890
55 *
56 * any improvements or extensions that they make and grant Carnegie the
57 * rights to redistribute these changes.
58 */
59
60 /*
61 * Common Chipset "bus I/O" functions.
62 *
63 * uses:
64 * CHIP name of the 'chip' it's being compiled for.
65 * CHIP_BASE memory or I/O space base to use.
66 * CHIP_EX_STORE
67 * If defined, device-provided static storage area
68 * for the memory or I/O space extent. If this is
69 * defined, CHIP_EX_STORE_SIZE must also be defined.
70 * If this is not defined, a static area will be
71 * declared.
72 * CHIP_EX_STORE_SIZE
73 * Size of the device-provided static storage area
74 * for the memory or I/O memory space extent.
75 * CHIP_LITTLE_ENDIAN | CHIP_BIG_ENDIAN
76 * For endian-specific busses, like PCI (little).
77 * CHIP_WRONG_ENDIAN
78 * For things like PCI bridges with endian conversion that
79 * can't be turned off, so we need to switch address bits
80 * for 8 and 16bit accesses.
81 * Example: MACE PCI bridge in SGI O2
82 * CHIP_ACCESS_SIZE
83 * Size (in bytes) of minimum bus access, e.g. 4
84 * to indicate all bus cycles are 32-bits. Defaults
85 * to 1, indicating any access size is valid.
86 */
87
88 #include <sys/cdefs.h>
89 __KERNEL_RCSID(0, "$NetBSD: bus_space_alignstride_chipdep.c,v 1.32 2022/09/29 07:00:46 skrll Exp $");
90
91 #ifdef CHIP_EXTENT
92 #include <sys/extent.h>
93 #endif
94
95 #include <mips/locore.h>
96
97 #include <uvm/uvm_extern.h>
98
99 #if defined(__mips_o32) && defined(MIPS3)
100 #define NEED_64BIT_ASM
101 #endif
102
103 #define __C(A,B) __CONCAT(A,B)
104 #define __S(S) __STRING(S)
105
106 #ifdef CHIP_IO
107 #define __BS(A) __C(__C(CHIP,_bus_io_),A)
108 #endif
109 #ifdef CHIP_MEM
110 #define __BS(A) __C(__C(CHIP,_bus_mem_),A)
111 #endif
112
113 #if defined(CHIP_LITTLE_ENDIAN)
114 #define CHIP_SWAP16(x) le16toh(x)
115 #define CHIP_SWAP32(x) le32toh(x)
116 #define CHIP_SWAP64(x) le64toh(x)
117 #define CHIP_NEED_STREAM 1
118 #elif defined(CHIP_BIG_ENDIAN)
119 #define CHIP_SWAP16(x) be16toh(x)
120 #define CHIP_SWAP32(x) be32toh(x)
121 #define CHIP_SWAP64(x) be64toh(x)
122 #define CHIP_NEED_STREAM 1
123 #else
124 #define CHIP_SWAP16(x) (x)
125 #define CHIP_SWAP32(x) (x)
126 #define CHIP_SWAP64(x) (x)
127 #endif
128
129 #ifndef CHIP_ACCESS_SIZE
130 #define CHIP_ACCESS_SIZE 1
131 #endif
132
133 #if CHIP_ACCESS_SIZE==1
134 # define CHIP_SWAP_ACCESS(x) (x)
135 #elif CHIP_ACCESS_SIZE==2
136 # define CHIP_SWAP_ACCESS(x) CHIP_SWAP16(x)
137 #elif CHIP_ACCESS_SIZE==4
138 # define CHIP_SWAP_ACCESS(x) CHIP_SWAP32(x)
139 #elif CHIP_ACCESS_SIZE==8
140 # ifndef MIPS3_PLUS
141 # error 8 byte access size not available
142 # endif
143 # define CHIP_SWAP_ACCESS(x) CHIP_SWAP64(x)
144 #else
145 # error your access size not implemented
146 #endif
147
148 /*
149 * The logic here determines a few macros to support requirements for
150 * whole-word accesses:
151 *
152 * CHIP_TYPE is a uintXX_t that represents the native access type for the bus.
153 *
154 * CHIP_SHIFTXX is the number of bits to shift a big-endian value to convert
155 * convert between the CHIP_TYPE and uintXX_t.
156 *
157 * The idea is that if we want to do a 16bit load from a bus that only
158 * supports 32-bit accesses, we will access the first 16 bits of the
159 * addressed 32-bit word.
160 *
161 * Obviously (hopefully) this method is inadequate to support addressing the
162 * second half of a 16-bit word, or the upper 3/4 of a 32-bit value, etc.
163 * In other words, the drivers should probably not be relying on this!
164 *
165 * We should probably come back in here some day and handle offsets properly.
166 * to do that, we need to mask off the low order bits of the address, and
167 * then figure out which bits they correspond to.
168 *
169 * If we have fixed access size limitations, we need to make sure that
170 * handle shifting required for big-endian storage. The reality is
171 * that if the bus only supports size "n", then drivers should
172 * probably only access it using "n" sized (or bigger) accesses.
173 */
174
175 #if CHIP_ACCESS_SIZE == 1
176 #define CHIP_TYPE uint8_t
177 #endif
178
179 #if CHIP_ACCESS_SIZE == 2
180 #define CHIP_TYPE uint16_t
181 #endif
182
183 #if CHIP_ACCESS_SIZE == 4
184 #define CHIP_TYPE uint32_t
185 #endif
186
187 #if CHIP_ACCESS_SIZE == 8
188 #define CHIP_TYPE uint64_t
189 #endif
190
191 #ifndef CHIP_TYPE
192 #error "Invalid chip access size!"
193 #endif
194
195 #ifdef CHIP_EXTENT
196 #ifndef CHIP_EX_STORE
197 static long
198 __BS(ex_storage)[EXTENT_FIXED_STORAGE_SIZE(8) / sizeof(long)];
199 #define CHIP_EX_STORE(v) (__BS(ex_storage))
200 #define CHIP_EX_STORE_SIZE(v) (sizeof __BS(ex_storage))
201 #endif
202 #endif /* CHIP_EXTENT */
203
204 #ifndef CHIP_ALIGN_STRIDE
205 #define CHIP_ALIGN_STRIDE 0
206 #endif
207
208 #ifdef CHIP_WRONG_ENDIAN
209 #define CHIP_OFF8(o) ((o) ^ 3)
210 #else
211 #if CHIP_ALIGN_STRIDE > 0
212 #define CHIP_OFF8(o) ((o) << (CHIP_ALIGN_STRIDE))
213 #else
214 #define CHIP_OFF8(o) (o)
215 #endif
216 #endif
217
218 #ifdef CHIP_WRONG_ENDIAN
219 #define CHIP_OFF16(o) ((o) ^ 2)
220 #else
221 #if CHIP_ALIGN_STRIDE > 1
222 #define CHIP_OFF16(o) ((o) << (CHIP_ALIGN_STRIDE - 1))
223 #else
224 #define CHIP_OFF16(o) (o)
225 #endif
226 #endif
227
228 #if CHIP_ALIGN_STRIDE > 2
229 #define CHIP_OFF32(o) ((o) << (CHIP_ALIGN_STRIDE - 2))
230 #else
231 #define CHIP_OFF32(o) (o)
232 #endif
233
234 #if CHIP_ALIGN_STRIDE > 3
235 #define CHIP_OFF64(o) ((o) << (CHIP_ALIGN_STRIDE - 3))
236 #else
237 #define CHIP_OFF64(o) (o)
238 #endif
239
240
241 static int
__BS(get_window)242 __BS(get_window)(void *v, int window, struct mips_bus_space_translation *mbst)
243 {
244
245 switch (window) {
246 #ifdef CHIP_W1_BUS_START
247 case 0:
248 mbst->mbst_bus_start = CHIP_W1_BUS_START(v);
249 mbst->mbst_bus_end = CHIP_W1_BUS_END(v);
250 mbst->mbst_sys_start = CHIP_W1_SYS_START(v);
251 mbst->mbst_sys_end = CHIP_W1_SYS_END(v);
252 mbst->mbst_align_stride = CHIP_ALIGN_STRIDE;
253 mbst->mbst_flags = 0;
254 break;
255 #endif
256
257 #ifdef CHIP_W2_BUS_START
258 case 1:
259 mbst->mbst_bus_start = CHIP_W2_BUS_START(v);
260 mbst->mbst_bus_end = CHIP_W2_BUS_END(v);
261 mbst->mbst_sys_start = CHIP_W2_SYS_START(v);
262 mbst->mbst_sys_end = CHIP_W2_SYS_END(v);
263 mbst->mbst_align_stride = CHIP_ALIGN_STRIDE;
264 mbst->mbst_flags = 0;
265 break;
266 #endif
267
268 #ifdef CHIP_W3_BUS_START
269 case 2:
270 mbst->mbst_bus_start = CHIP_W3_BUS_START(v);
271 mbst->mbst_bus_end = CHIP_W3_BUS_END(v);
272 mbst->mbst_sys_start = CHIP_W3_SYS_START(v);
273 mbst->mbst_sys_end = CHIP_W3_SYS_END(v);
274 mbst->mbst_align_stride = CHIP_ALIGN_STRIDE;
275 mbst->mbst_flags = 0;
276 break;
277 #endif
278
279 default:
280 panic(__S(__BS(get_window)) ": invalid window %d",
281 window);
282 }
283
284 return (0);
285 }
286
287 static int
__BS(translate)288 __BS(translate)(void *v, bus_addr_t addr, bus_size_t len, int flags,
289 struct mips_bus_space_translation *mbst)
290 {
291 bus_addr_t end = addr + (len - 1);
292 #if CHIP_ALIGN_STRIDE != 0
293 int linear = flags & BUS_SPACE_MAP_LINEAR;
294
295 /*
296 * Can't map xxx space linearly.
297 */
298 if (linear)
299 return (EOPNOTSUPP);
300 #endif
301
302 #ifdef CHIP_W1_BUS_START
303 if (addr >= CHIP_W1_BUS_START(v) && end <= CHIP_W1_BUS_END(v))
304 return (__BS(get_window)(v, 0, mbst));
305 #endif
306
307 #ifdef CHIP_W2_BUS_START
308 if (addr >= CHIP_W2_BUS_START(v) && end <= CHIP_W2_BUS_END(v))
309 return (__BS(get_window)(v, 1, mbst));
310 #endif
311
312 #ifdef CHIP_W3_BUS_START
313 if (addr >= CHIP_W3_BUS_START(v) && end <= CHIP_W3_BUS_END(v))
314 return (__BS(get_window)(v, 2, mbst));
315 #endif
316
317 #ifdef EXTENT_DEBUG
318 printf("\n");
319 #ifdef CHIP_W1_BUS_START
320 printf("%s: window[1]=0x%lx-0x%lx\n", __S(__BS(map)),
321 (u_long)CHIP_W1_BUS_START(v), (u_long)CHIP_W1_BUS_END(v));
322 #endif
323 #ifdef CHIP_W2_BUS_START
324 printf("%s: window[2]=0x%lx-0x%lx\n", __S(__BS(map)),
325 (u_long)CHIP_W2_BUS_START(v), (u_long)CHIP_W2_BUS_END(v));
326 #endif
327 #ifdef CHIP_W3_BUS_START
328 printf("%s: window[3]=0x%lx-0x%lx\n", __S(__BS(map)),
329 (u_long)CHIP_W3_BUS_START(v), (u_long)CHIP_W3_BUS_END(v));
330 #endif
331 #endif /* EXTENT_DEBUG */
332 /* No translation. */
333 return (EINVAL);
334 }
335
336 static int
__BS(map)337 __BS(map)(void *v, bus_addr_t addr, bus_size_t size, int flags,
338 bus_space_handle_t *hp, int acct)
339 {
340 struct mips_bus_space_translation mbst;
341 int error;
342
343 /*
344 * Get the translation for this address.
345 */
346 error = __BS(translate)(v, addr, size, flags, &mbst);
347 if (error)
348 return (error);
349
350 #ifdef CHIP_EXTENT
351 if (acct == 0)
352 goto mapit;
353
354 #ifdef EXTENT_DEBUG
355 printf("%s: allocating %#"PRIxBUSADDR" to %#"PRIxBUSADDR"\n",
356 __S(__BS(map)), addr, addr + size - 1);
357 #endif
358 error = extent_alloc_region(CHIP_EXTENT(v), addr, size,
359 EX_NOWAIT | (CHIP_EX_MALLOC_SAFE(v) ? EX_MALLOCOK : 0));
360 if (error) {
361 #ifdef EXTENT_DEBUG
362 printf("%s: allocation failed (%d)\n", __S(__BS(map)), error);
363 extent_print(CHIP_EXTENT(v));
364 #endif
365 return (error);
366 }
367
368 mapit:
369 #endif /* CHIP_EXTENT */
370
371 addr = mbst.mbst_sys_start + (addr - mbst.mbst_bus_start);
372
373 #if defined(__mips_n32) || defined(_LP64)
374 if (flags & BUS_SPACE_MAP_CACHEABLE) {
375 #ifdef __mips_n32
376 if (((addr + size) & ~MIPS_PHYS_MASK) == 0)
377 *hp = (intptr_t)MIPS_PHYS_TO_KSEG0(addr);
378 else
379 #endif
380 *hp = MIPS_PHYS_TO_XKPHYS_CACHED(addr);
381 } else if (flags & BUS_SPACE_MAP_PREFETCHABLE) {
382 *hp = MIPS_PHYS_TO_XKPHYS_ACC(addr);
383 } else {
384 #ifdef __mips_n32
385 if (((addr + size) & ~MIPS_PHYS_MASK) == 0)
386 *hp = (intptr_t)MIPS_PHYS_TO_KSEG1(addr);
387 else
388 #endif
389 *hp = MIPS_PHYS_TO_XKPHYS_UNCACHED(addr);
390 }
391 #else
392 if (((addr + size) & ~MIPS_PHYS_MASK) != 0) {
393 vaddr_t va;
394 paddr_t pa;
395 int s;
396
397 size = round_page((addr % PAGE_SIZE) + size);
398 va = uvm_km_alloc(kernel_map, size, PAGE_SIZE,
399 UVM_KMF_VAONLY | UVM_KMF_NOWAIT);
400 if (va == 0)
401 return ENOMEM;
402
403 /* check use of handle_is_km in BS(unmap) */
404 KASSERT(!(MIPS_KSEG0_P(va) || MIPS_KSEG1_P(va)));
405
406 *hp = va + (addr & PAGE_MASK);
407 pa = trunc_page(addr);
408
409 s = splhigh();
410 while (size != 0) {
411 pmap_kenter_pa(va, pa, VM_PROT_READ | VM_PROT_WRITE, 0);
412 pa += PAGE_SIZE;
413 va += PAGE_SIZE;
414 size -= PAGE_SIZE;
415 }
416 pmap_update(pmap_kernel());
417 splx(s);
418 } else {
419 if (flags & BUS_SPACE_MAP_CACHEABLE)
420 *hp = (intptr_t)MIPS_PHYS_TO_KSEG0(addr);
421 else
422 *hp = (intptr_t)MIPS_PHYS_TO_KSEG1(addr);
423 }
424 #endif
425
426 return (0);
427 }
428
429 static void
__BS(unmap)430 __BS(unmap)(void *v, bus_space_handle_t h, bus_size_t size, int acct)
431 {
432 #if !defined(_LP64) || defined(CHIP_EXTENT)
433 bus_addr_t addr = 0; /* initialize to appease gcc */
434 #endif
435 #ifndef _LP64
436 bool handle_is_km;
437
438 /* determine if h is addr obtained from uvm_km_alloc */
439 handle_is_km = !(MIPS_KSEG0_P(h) || MIPS_KSEG1_P(h));
440 #ifdef __mips_n32
441 if (handle_is_km == true)
442 handle_is_km = !MIPS_XKPHYS_P(h);
443 #endif
444 if (handle_is_km == true) {
445 paddr_t pa;
446 vaddr_t va = (vaddr_t)trunc_page(h);
447 vsize_t sz = (vsize_t)round_page((h % PAGE_SIZE) + size);
448 int s;
449
450 s = splhigh();
451
452 if (pmap_extract(pmap_kernel(), (vaddr_t)h, &pa) == false)
453 panic("%s: pmap_extract failed", __func__);
454 addr = (bus_addr_t)pa;
455 #if 0
456 printf("%s:%d: addr %#"PRIxBUSADDR", sz %#"PRIxVSIZE"\n",
457 __func__, __LINE__, addr, sz);
458 #endif
459 /* sanity check: this is why we couldn't map w/ kseg[0,1] */
460 KASSERT (((addr + sz) & ~MIPS_PHYS_MASK) != 0);
461
462 pmap_kremove(va, sz);
463 pmap_update(pmap_kernel());
464 uvm_km_free(kernel_map, va, sz, UVM_KMF_VAONLY);
465
466 splx(s);
467 }
468 #endif /* _LP64 */
469
470 #ifdef CHIP_EXTENT
471
472 if (acct == 0)
473 return;
474
475 #ifdef EXTENT_DEBUG
476 printf("%s: freeing handle %#"PRIxBSH" for %#"PRIxBUSSIZE"\n",
477 __S(__BS(unmap)), h, size);
478 #endif
479
480 #ifdef _LP64
481 KASSERT(MIPS_XKPHYS_P(h));
482 addr = MIPS_XKPHYS_TO_PHYS(h);
483 #else
484 if (handle_is_km == false) {
485 if (MIPS_KSEG0_P(h))
486 addr = MIPS_KSEG0_TO_PHYS(h);
487 #ifdef __mips_n32
488 else if (MIPS_XKPHYS_P(h))
489 addr = MIPS_XKPHYS_TO_PHYS(h);
490 #endif
491 else
492 addr = MIPS_KSEG1_TO_PHYS(h);
493 }
494 #endif
495
496 #ifdef CHIP_W1_BUS_START
497 if (addr >= CHIP_W1_SYS_START(v) && addr <= CHIP_W1_SYS_END(v)) {
498 addr = CHIP_W1_BUS_START(v) + (addr - CHIP_W1_SYS_START(v));
499 } else
500 #endif
501 #ifdef CHIP_W2_BUS_START
502 if (addr >= CHIP_W2_SYS_START(v) && addr <= CHIP_W2_SYS_END(v)) {
503 addr = CHIP_W2_BUS_START(v) + (addr - CHIP_W2_SYS_START(v));
504 } else
505 #endif
506 #ifdef CHIP_W3_BUS_START
507 if (addr >= CHIP_W3_SYS_START(v) && addr <= CHIP_W3_SYS_END(v)) {
508 addr = CHIP_W3_BUS_START(v) + (addr - CHIP_W3_SYS_START(v));
509 } else
510 #endif
511 {
512 printf("\n");
513 #ifdef CHIP_W1_BUS_START
514 printf("%s: sys window[1]=0x%lx-0x%lx\n",
515 __S(__BS(unmap)), (u_long)CHIP_W1_SYS_START(v),
516 (u_long)CHIP_W1_SYS_END(v));
517 #endif
518 #ifdef CHIP_W2_BUS_START
519 printf("%s: sys window[2]=0x%lx-0x%lx\n",
520 __S(__BS(unmap)), (u_long)CHIP_W2_SYS_START(v),
521 (u_long)CHIP_W2_SYS_END(v));
522 #endif
523 #ifdef CHIP_W3_BUS_START
524 printf("%s: sys window[3]=0x%lx-0x%lx\n",
525 __S(__BS(unmap)), (u_long)CHIP_W3_SYS_START(v),
526 (u_long)CHIP_W3_SYS_END(v));
527 #endif
528 panic("%s: don't know how to unmap %#"PRIxBSH, __S(__BS(unmap)), h);
529 }
530
531 #ifdef EXTENT_DEBUG
532 printf("%s: freeing %#"PRIxBUSADDR" to %#"PRIxBUSADDR"\n",
533 __S(__BS(unmap)), addr, addr + size - 1);
534 #endif
535 int error = extent_free(CHIP_EXTENT(v), addr, size,
536 EX_NOWAIT | (CHIP_EX_MALLOC_SAFE(v) ? EX_MALLOCOK : 0));
537 if (error) {
538 printf("%s: WARNING: could not unmap"
539 " %#"PRIxBUSADDR"-%#"PRIxBUSADDR" (error %d)\n",
540 __S(__BS(unmap)), addr, addr + size - 1, error);
541 #ifdef EXTENT_DEBUG
542 extent_print(CHIP_EXTENT(v));
543 #endif
544 }
545 #endif /* CHIP_EXTENT */
546 #if !defined(_LP64) || defined(CHIP_EXTENT)
547 __USE(addr);
548 #endif
549 }
550
551 static int
__BS(subregion)552 __BS(subregion)(void *v, bus_space_handle_t h, bus_size_t offset,
553 bus_size_t size, bus_space_handle_t *nh)
554 {
555
556 *nh = h + (offset << CHIP_ALIGN_STRIDE);
557 return (0);
558 }
559
560 static int
__BS(alloc)561 __BS(alloc)(void *v, bus_addr_t rstart, bus_addr_t rend, bus_size_t size,
562 bus_size_t align, bus_size_t boundary, int flags, bus_addr_t *addrp,
563 bus_space_handle_t *bshp)
564 {
565 #ifdef CHIP_EXTENT
566 struct mips_bus_space_translation mbst;
567 u_long addr; /* bogus but makes extent happy */
568 int error;
569 #if CHIP_ALIGN_STRIDE != 0
570 int linear = flags & BUS_SPACE_MAP_LINEAR;
571
572 /*
573 * Can't map xxx space linearly.
574 */
575 if (linear)
576 return (EOPNOTSUPP);
577 #endif
578
579 /*
580 * Do the requested allocation.
581 */
582 #ifdef EXTENT_DEBUG
583 printf("%s: allocating from %#"PRIxBUSADDR" to %#"PRIxBUSADDR"\n",
584 __S(__BS(alloc)), rstart, rend);
585 #endif
586 error = extent_alloc_subregion(CHIP_EXTENT(v), rstart, rend, size,
587 align, boundary,
588 EX_FAST | EX_NOWAIT | (CHIP_EX_MALLOC_SAFE(v) ? EX_MALLOCOK : 0),
589 &addr);
590 if (error) {
591 #ifdef EXTENT_DEBUG
592 printf("%s: allocation failed (%d)\n", __S(__BS(alloc)), error);
593 extent_print(CHIP_EXTENT(v));
594 #endif
595 return (error);
596 }
597
598 #ifdef EXTENT_DEBUG
599 printf("%s: allocated 0x%lx to %#"PRIxBUSSIZE"\n",
600 __S(__BS(alloc)), addr, addr + size - 1);
601 #endif
602
603 error = __BS(translate)(v, addr, size, flags, &mbst);
604 if (error) {
605 (void) extent_free(CHIP_EXTENT(v), addr, size,
606 EX_NOWAIT | (CHIP_EX_MALLOC_SAFE(v) ? EX_MALLOCOK : 0));
607 return (error);
608 }
609
610 *addrp = addr;
611 #if !defined(__mips_o32)
612 if (flags & BUS_SPACE_MAP_CACHEABLE) {
613 *bshp = MIPS_PHYS_TO_XKPHYS_CACHED(mbst.mbst_sys_start +
614 (addr - mbst.mbst_bus_start));
615 } else {
616 *bshp = MIPS_PHYS_TO_XKPHYS_UNCACHED(mbst.mbst_sys_start +
617 (addr - mbst.mbst_bus_start));
618 }
619 #else
620 if (flags & BUS_SPACE_MAP_CACHEABLE) {
621 *bshp = MIPS_PHYS_TO_KSEG0(mbst.mbst_sys_start +
622 (addr - mbst.mbst_bus_start));
623 } else
624 *bshp = MIPS_PHYS_TO_KSEG1(mbst.mbst_sys_start +
625 (addr - mbst.mbst_bus_start));
626 #endif
627
628 return (0);
629 #else /* ! CHIP_EXTENT */
630 return (EOPNOTSUPP);
631 #endif /* CHIP_EXTENT */
632 }
633
634 static void
__BS(free)635 __BS(free)(void *v, bus_space_handle_t bsh, bus_size_t size)
636 {
637
638 /* Unmap does all we need to do. */
639 __BS(unmap)(v, bsh, size, 1);
640 }
641
642 static void *
__BS(vaddr)643 __BS(vaddr)(void *v, bus_space_handle_t bsh)
644 {
645
646 #if (CHIP_ALIGN_STRIDE != 0)
647 /* Linear mappings not possible. */
648 return (NULL);
649 #elif defined(__mips_n32)
650 if (MIPS_KSEG0_P(bsh) || MIPS_KSEG1_P(bsh) || MIPS_KSEG2_P(bsh))
651 return ((void *)(intptr_t)bsh);
652 return NULL;
653 #else
654 return ((void *)bsh);
655 #endif
656 }
657
658 static paddr_t
__BS(mmap)659 __BS(mmap)(void *v, bus_addr_t addr, off_t off, int prot, int flags)
660 {
661 #ifdef CHIP_IO
662
663 /* Not supported for I/O space. */
664 return (-1);
665 #elif defined(CHIP_MEM)
666 paddr_t ret;
667 struct mips_bus_space_translation mbst;
668 int error;
669
670 /*
671 * Get the translation for this address.
672 */
673 error = __BS(translate)(v, addr, off + PAGE_SIZE, flags,
674 &mbst);
675 if (error)
676 return (-1);
677 ret = mbst.mbst_sys_start + (addr - mbst.mbst_bus_start) + off;
678 #if defined(_MIPS_PADDR_T_64BIT) || defined(_LP64)
679 if (flags & BUS_SPACE_MAP_PREFETCHABLE) {
680 ret |= PGC_PREFETCH;
681 }
682 #endif
683
684 return (mips_btop(ret));
685 #else
686 # error must define one of CHIP_IO or CHIP_MEM
687 #endif
688 }
689
690 static void
__BS(barrier)691 __BS(barrier)(void *v, bus_space_handle_t h, bus_size_t o, bus_size_t l, int f)
692 {
693
694 /* XXX XXX XXX */
695 if ((f & BUS_SPACE_BARRIER_WRITE) != 0)
696 wbflush();
697 }
698
699 static uint8_t
__BS(read_1)700 __BS(read_1)(void *v, bus_space_handle_t h, bus_size_t off)
701 {
702 h += CHIP_OFF8(off);
703
704 const int shift = (h & (CHIP_ACCESS_SIZE - 1)) * 8;
705 h &= ~((bus_space_handle_t)(CHIP_ACCESS_SIZE - 1));
706 #if CHIP_ACCESS_SIZE == 8
707 const CHIP_TYPE val = mips3_ld(h);
708 #elif CHIP_ACCESS_SIZE == 4
709 const CHIP_TYPE val = mips_lwu(h);
710 #elif CHIP_ACCESS_SIZE == 2
711 const CHIP_TYPE val = mips_lhu(h);
712 #else
713 const uint8_t val = mips_lbu(h);
714 #endif
715 const uint8_t r = (uint8_t)(CHIP_SWAP_ACCESS(val) >> shift);
716
717 return r;
718 }
719
720 static uint16_t
__BS(read_2)721 __BS(read_2)(void *v, bus_space_handle_t h, bus_size_t off)
722 {
723 KASSERT((off & 1) == 0);
724 h += CHIP_OFF16(off);
725
726 const int shift = (h & (CHIP_ACCESS_SIZE - 1)) * 8;
727 h &= ~((bus_space_handle_t)(CHIP_ACCESS_SIZE - 1));
728 #if CHIP_ACCESS_SIZE == 8
729 const CHIP_TYPE val = mips3_ld(h);
730 #elif CHIP_ACCESS_SIZE == 4
731 const CHIP_TYPE val = mips_lwu(h);
732 #else
733 const uint16_t val = mips_lhu(h);
734 #endif
735 const uint16_t r = (uint16_t)CHIP_SWAP16(val >> shift);
736
737 return r;
738 }
739
740 static uint32_t
__BS(read_4)741 __BS(read_4)(void *v, bus_space_handle_t h, bus_size_t off)
742 {
743 KASSERT((off & 3) == 0);
744
745 h += CHIP_OFF32(off);
746 const int shift = (h & (CHIP_ACCESS_SIZE - 1)) * 8;
747 h &= ~((bus_space_handle_t)(CHIP_ACCESS_SIZE - 1));
748 #if CHIP_ACCESS_SIZE > 4
749 const CHIP_TYPE val = mips3_ld(h);
750 #else /* CHIP_ACCESS_SIZE > 4 */
751 const uint32_t val = mips_lwu(h);
752 #endif
753 const uint32_t r = (uint32_t)CHIP_SWAP32(val >> shift);
754
755 return r;
756 }
757
758 static uint64_t
__BS(read_8)759 __BS(read_8)(void *v, bus_space_handle_t h, bus_size_t off)
760 {
761 #ifdef MIPS3_64BIT
762 KASSERT((off & 7) == 0);
763 h += CHIP_OFF64(off);
764 const int shift = (h & (CHIP_ACCESS_SIZE - 1)) * 8;
765 h &= ~((bus_space_handle_t)(CHIP_ACCESS_SIZE - 1));
766 const uint64_t r = CHIP_SWAP64(mips3_ld(h) >> shift);
767
768 return r;
769 #else
770 panic("%s: not implemented!", __func__);
771 #endif
772 }
773
774
775 #define CHIP_read_multi_N(BYTES,TYPE) \
776 static void \
777 __C(__BS(read_multi_),BYTES)(void *v, bus_space_handle_t h, \
778 bus_size_t o, TYPE *a, bus_size_t c) \
779 { \
780 \
781 while (c-- > 0) { \
782 __BS(barrier)(v, h, o, sizeof *a, \
783 BUS_SPACE_BARRIER_READ); \
784 *a++ = __C(__BS(read_),BYTES)(v, h, o); \
785 } \
786 }
787 CHIP_read_multi_N(1,uint8_t)
788 CHIP_read_multi_N(2,uint16_t)
789 CHIP_read_multi_N(4,uint32_t)
790 CHIP_read_multi_N(8,uint64_t)
791
792 #define CHIP_read_region_N(BYTES,TYPE) \
793 static void \
794 __C(__BS(read_region_),BYTES)(void *v, bus_space_handle_t h, \
795 bus_size_t o, TYPE *a, bus_size_t c) \
796 { \
797 \
798 while (c-- > 0) { \
799 *a++ = __C(__BS(read_),BYTES)(v, h, o); \
800 o += sizeof *a; \
801 } \
802 }
803 CHIP_read_region_N(1,uint8_t)
804 CHIP_read_region_N(2,uint16_t)
805 CHIP_read_region_N(4,uint32_t)
806 CHIP_read_region_N(8,uint64_t)
807
808
809 static void
__BS(write_1)810 __BS(write_1)(void *v, bus_space_handle_t h, bus_size_t off, uint8_t val)
811 {
812 h += CHIP_OFF8(off);
813
814 #if CHIP_ACCESS_SIZE == 1
815 mips_sb(h, val);
816 #else
817 const int shift = (h & (CHIP_ACCESS_SIZE - 1)) * 8;
818 h &= ~((bus_space_handle_t)(CHIP_ACCESS_SIZE - 1));
819 CHIP_TYPE cval = CHIP_SWAP_ACCESS(((CHIP_TYPE)val) << shift);
820 # if CHIP_ACCESS_SIZE == 8
821 mips3_sd(h, cval);
822 # elif CHIP_ACCESS_SIZE == 4
823 mips_sw(h, cval);
824 # else
825 mips_sh(h, cval);
826 # endif
827 #endif
828 }
829
830 static void
__BS(write_2)831 __BS(write_2)(void *v, bus_space_handle_t h, bus_size_t off, uint16_t val)
832 {
833 KASSERT((h & 1) == 0);
834 KASSERT((off & 1) == 0);
835
836 h += CHIP_OFF16(off);
837 #if CHIP_ACCESS_SIZE <= 2
838 mips_sh(h, CHIP_SWAP16(val));
839 #else
840 const int shift = (h & (CHIP_ACCESS_SIZE - 1)) * 8;
841 h &= ~((bus_space_handle_t)(CHIP_ACCESS_SIZE - 1));
842 CHIP_TYPE cval = ((CHIP_TYPE)CHIP_SWAP16(val)) << shift;
843 # if CHIP_ACCESS_SIZE == 8
844 mips3_sd(h, cval);
845 # else
846 mips_sw(h, cval);
847 # endif
848 #endif
849 }
850
851 static void
__BS(write_4)852 __BS(write_4)(void *v, bus_space_handle_t h, bus_size_t off, uint32_t val)
853 {
854 KASSERT((h & 3) == 0);
855 KASSERT((off & 3) == 0);
856
857 h += CHIP_OFF32(off);
858 #if CHIP_ACCESS_SIZE <= 4
859 mips_sw(h, CHIP_SWAP32(val));
860 #else
861 const int shift = (h & (CHIP_ACCESS_SIZE - 1)) * 8;
862 h &= ~((bus_space_handle_t)(CHIP_ACCESS_SIZE - 1));
863 mips3_sd(h, ((CHIP_TYPE)CHIP_SWAP32(val)) << shift);
864 #endif
865 }
866
867 static void
__BS(write_8)868 __BS(write_8)(void *v, bus_space_handle_t h, bus_size_t off, uint64_t val)
869 {
870 #ifdef MIPS3_64BIT
871 KASSERT((h & 7) == 0);
872 KASSERT((off & 7) == 0);
873
874 h += CHIP_OFF64(off);
875 mips3_sd(h, CHIP_SWAP64(val));
876 #else
877 panic("%s: not implemented!", __func__);
878 #endif
879 }
880
881 #define CHIP_write_multi_N(BYTES,TYPE) \
882 static void \
883 __C(__BS(write_multi_),BYTES)(void *v, bus_space_handle_t h, \
884 bus_size_t o, const TYPE *a, bus_size_t c) \
885 { \
886 \
887 while (c-- > 0) { \
888 __C(__BS(write_),BYTES)(v, h, o, *a++); \
889 __BS(barrier)(v, h, o, sizeof *a, \
890 BUS_SPACE_BARRIER_WRITE); \
891 } \
892 }
893 CHIP_write_multi_N(1,uint8_t)
894 CHIP_write_multi_N(2,uint16_t)
895 CHIP_write_multi_N(4,uint32_t)
896 CHIP_write_multi_N(8,uint64_t)
897
898 #define CHIP_write_region_N(BYTES,TYPE) \
899 static void \
900 __C(__BS(write_region_),BYTES)(void *v, bus_space_handle_t h, \
901 bus_size_t o, const TYPE *a, bus_size_t c) \
902 { \
903 \
904 while (c-- > 0) { \
905 __C(__BS(write_),BYTES)(v, h, o, *a++); \
906 o += sizeof *a; \
907 } \
908 }
909 CHIP_write_region_N(1,uint8_t)
910 CHIP_write_region_N(2,uint16_t)
911 CHIP_write_region_N(4,uint32_t)
912 CHIP_write_region_N(8,uint64_t)
913
914 #define CHIP_set_multi_N(BYTES,TYPE) \
915 static void \
916 __C(__BS(set_multi_),BYTES)(void *v, bus_space_handle_t h, \
917 bus_size_t o, TYPE val, bus_size_t c) \
918 { \
919 \
920 while (c-- > 0) { \
921 __C(__BS(write_),BYTES)(v, h, o, val); \
922 __BS(barrier)(v, h, o, sizeof val, \
923 BUS_SPACE_BARRIER_WRITE); \
924 } \
925 }
926 CHIP_set_multi_N(1,uint8_t)
927 CHIP_set_multi_N(2,uint16_t)
928 CHIP_set_multi_N(4,uint32_t)
929 CHIP_set_multi_N(8,uint64_t)
930
931 #define CHIP_set_region_N(BYTES,TYPE) \
932 static void \
933 __C(__BS(set_region_),BYTES)(void *v, bus_space_handle_t h, \
934 bus_size_t o, TYPE val, bus_size_t c) \
935 { \
936 \
937 while (c-- > 0) { \
938 __C(__BS(write_),BYTES)(v, h, o, val); \
939 o += sizeof val; \
940 } \
941 }
942 CHIP_set_region_N(1,uint8_t)
943 CHIP_set_region_N(2,uint16_t)
944 CHIP_set_region_N(4,uint32_t)
945 CHIP_set_region_N(8,uint64_t)
946
947 #define CHIP_copy_region_N(BYTES) \
948 static void \
949 __C(__BS(copy_region_),BYTES)(void *v, bus_space_handle_t h1, \
950 bus_size_t o1, bus_space_handle_t h2, bus_size_t o2, bus_size_t c) \
951 { \
952 bus_size_t o; \
953 \
954 if ((h1 + o1) >= (h2 + o2)) { \
955 /* src after dest: copy forward */ \
956 for (o = 0; c != 0; c--, o += BYTES) \
957 __C(__BS(write_),BYTES)(v, h2, o2 + o, \
958 __C(__BS(read_),BYTES)(v, h1, o1 + o)); \
959 } else { \
960 /* dest after src: copy backwards */ \
961 for (o = (c - 1) * BYTES; c != 0; c--, o -= BYTES) \
962 __C(__BS(write_),BYTES)(v, h2, o2 + o, \
963 __C(__BS(read_),BYTES)(v, h1, o1 + o)); \
964 } \
965 }
966 CHIP_copy_region_N(1)
967 CHIP_copy_region_N(2)
968 CHIP_copy_region_N(4)
969 CHIP_copy_region_N(8)
970
971 #ifdef CHIP_NEED_STREAM
972
973 static uint8_t
__BS(read_stream_1)974 __BS(read_stream_1)(void *v, bus_space_handle_t h, bus_size_t off)
975 {
976 h += CHIP_OFF8(off);
977
978 #if CHIP_ACCESS_SIZE == 8
979 return (uint8_t)mips3_ld(h);
980 #elif CHIP_ACCESS_SIZE == 4
981 return (uint8_t)mips_lwu(h);
982 #elif CHIP_ACCESS_SIZE == 2
983 return (uint8_t)mips_lhu(h);
984 #else
985 return mips_lbu(h);
986 #endif
987 }
988
989 static uint16_t
__BS(read_stream_2)990 __BS(read_stream_2)(void *v, bus_space_handle_t h, bus_size_t off)
991 {
992 h += CHIP_OFF16(off);
993 #if CHIP_ACCESS_SIZE == 8
994 return (uint16_t)mips3_ld(h);
995 #elif CHIP_ACCESS_SIZE == 4
996 return (uint16_t)mips_lwu(h);
997 #else
998 return (uint16_t)mips_lbu(h);
999 #endif
1000 }
1001
1002 static uint32_t
__BS(read_stream_4)1003 __BS(read_stream_4)(void *v, bus_space_handle_t h, bus_size_t off)
1004 {
1005 h += CHIP_OFF32(off);
1006 #if CHIP_ACCESS_SIZE == 8
1007 return (uint32_t)mips3_ld(h);
1008 #else
1009 return (uint32_t)mips_lwu(h);
1010 #endif
1011 }
1012
1013 static uint64_t
__BS(read_stream_8)1014 __BS(read_stream_8)(void *v, bus_space_handle_t h, bus_size_t off)
1015 {
1016 #ifdef MIPS3_64BIT
1017 h += CHIP_OFF64(off);
1018 return mips3_ld(h);
1019 #else
1020 panic("%s: not implemented!", __func__);
1021 #endif
1022 }
1023
1024 #define CHIP_read_multi_stream_N(BYTES,TYPE) \
1025 static void \
1026 __C(__BS(read_multi_stream_),BYTES)(void *v, bus_space_handle_t h, \
1027 bus_size_t o, TYPE *a, bus_size_t c) \
1028 { \
1029 \
1030 while (c-- > 0) { \
1031 __BS(barrier)(v, h, o, sizeof *a, \
1032 BUS_SPACE_BARRIER_READ); \
1033 *a++ = __C(__BS(read_stream_),BYTES)(v, h, o); \
1034 } \
1035 }
1036 CHIP_read_multi_stream_N(1,uint8_t)
1037 CHIP_read_multi_stream_N(2,uint16_t)
1038 CHIP_read_multi_stream_N(4,uint32_t)
1039 CHIP_read_multi_stream_N(8,uint64_t)
1040
1041 #define CHIP_read_region_stream_N(BYTES,TYPE) \
1042 static void \
1043 __C(__BS(read_region_stream_),BYTES)(void *v, bus_space_handle_t h, \
1044 bus_size_t o, TYPE *a, bus_size_t c) \
1045 { \
1046 \
1047 while (c-- > 0) { \
1048 *a++ = __C(__BS(read_stream_),BYTES)(v, h, o); \
1049 o += sizeof *a; \
1050 } \
1051 }
1052 CHIP_read_region_stream_N(1,uint8_t)
1053 CHIP_read_region_stream_N(2,uint16_t)
1054 CHIP_read_region_stream_N(4,uint32_t)
1055 CHIP_read_region_stream_N(8,uint64_t)
1056
1057 static void
__BS(write_stream_1)1058 __BS(write_stream_1)(void *v, bus_space_handle_t h, bus_size_t off,
1059 uint8_t val)
1060 {
1061 #if CHIP_ACCESS_SIZE == 8
1062 mips3_sd(h, val);
1063 #elif CHIP_ACCESS_SIZE == 4
1064 mips_sw(h, val);
1065 #elif CHIP_ACCESS_SIZE == 2
1066 mips_sh(h, val);
1067 #else
1068 mips_sb(h, val);
1069 #endif
1070 }
1071
1072 static void
__BS(write_stream_2)1073 __BS(write_stream_2)(void *v, bus_space_handle_t h, bus_size_t off,
1074 uint16_t val)
1075 {
1076 #if CHIP_ACCESS_SIZE == 8
1077 mips3_sd(h, val);
1078 #elif CHIP_ACCESS_SIZE == 4
1079 mips_sw(h, val);
1080 #else
1081 mips_sh(h, val);
1082 #endif
1083 }
1084
1085 static void
__BS(write_stream_4)1086 __BS(write_stream_4)(void *v, bus_space_handle_t h, bus_size_t off,
1087 uint32_t val)
1088 {
1089 h += CHIP_OFF32(off);
1090 #if CHIP_ACCESS_SIZE == 8
1091 mips3_sd(h, val);
1092 #else
1093 mips_sw(h, val);
1094 #endif
1095 }
1096
1097 static void
__BS(write_stream_8)1098 __BS(write_stream_8)(void *v, bus_space_handle_t h, bus_size_t off,
1099 uint64_t val)
1100 {
1101 #ifdef MIPS3_64BIT
1102 h += CHIP_OFF64(off);
1103 mips3_sd(h, val);
1104 #else
1105 panic("%s: not implemented!", __func__);
1106 #endif
1107 }
1108
1109 #define CHIP_write_multi_stream_N(BYTES,TYPE) \
1110 static void \
1111 __C(__BS(write_multi_stream_),BYTES)(void *v, bus_space_handle_t h, \
1112 bus_size_t o, const TYPE *a, bus_size_t c) \
1113 { \
1114 \
1115 while (c-- > 0) { \
1116 __C(__BS(write_stream_),BYTES)(v, h, o, *a++); \
1117 __BS(barrier)(v, h, o, sizeof *a, \
1118 BUS_SPACE_BARRIER_WRITE); \
1119 } \
1120 }
1121 CHIP_write_multi_stream_N(1,uint8_t)
1122 CHIP_write_multi_stream_N(2,uint16_t)
1123 CHIP_write_multi_stream_N(4,uint32_t)
1124 CHIP_write_multi_stream_N(8,uint64_t)
1125
1126 #define CHIP_write_region_stream_N(BYTES,TYPE) \
1127 static void \
1128 __C(__BS(write_region_stream_),BYTES)(void *v, bus_space_handle_t h, \
1129 bus_size_t o, const TYPE *a, bus_size_t c) \
1130 { \
1131 \
1132 while (c-- > 0) { \
1133 __C(__BS(write_stream_),BYTES)(v, h, o, *a++); \
1134 o += sizeof *a; \
1135 } \
1136 }
1137 CHIP_write_region_stream_N(1,uint8_t)
1138 CHIP_write_region_stream_N(2,uint16_t)
1139 CHIP_write_region_stream_N(4,uint32_t)
1140 CHIP_write_region_stream_N(8,uint64_t)
1141
1142 #endif /* CHIP_NEED_STREAM */
1143
1144 void
__BS(init)1145 __BS(init)(bus_space_tag_t t, void *v)
1146 {
1147 #ifdef CHIP_EXTENT
1148 struct extent *ex;
1149 #endif
1150
1151 /*
1152 * Initialize the bus space tag.
1153 */
1154
1155 /* cookie */
1156 t->bs_cookie = v;
1157
1158 /* mapping/unmapping */
1159 t->bs_map = __BS(map);
1160 t->bs_unmap = __BS(unmap);
1161 t->bs_subregion = __BS(subregion);
1162
1163 t->bs_translate = __BS(translate);
1164 t->bs_get_window = __BS(get_window);
1165
1166 /* allocation/deallocation */
1167 t->bs_alloc = __BS(alloc);
1168 t->bs_free = __BS(free);
1169
1170 /* get kernel virtual address */
1171 t->bs_vaddr = __BS(vaddr);
1172
1173 /* mmap for user */
1174 t->bs_mmap = __BS(mmap);
1175
1176 /* barrier */
1177 t->bs_barrier = __BS(barrier);
1178
1179 /* read (single) */
1180 t->bs_r_1 = __BS(read_1);
1181 t->bs_r_2 = __BS(read_2);
1182 t->bs_r_4 = __BS(read_4);
1183 t->bs_r_8 = __BS(read_8);
1184
1185 /* read multiple */
1186 t->bs_rm_1 = __BS(read_multi_1);
1187 t->bs_rm_2 = __BS(read_multi_2);
1188 t->bs_rm_4 = __BS(read_multi_4);
1189 t->bs_rm_8 = __BS(read_multi_8);
1190
1191 /* read region */
1192 t->bs_rr_1 = __BS(read_region_1);
1193 t->bs_rr_2 = __BS(read_region_2);
1194 t->bs_rr_4 = __BS(read_region_4);
1195 t->bs_rr_8 = __BS(read_region_8);
1196
1197 /* write (single) */
1198 t->bs_w_1 = __BS(write_1);
1199 t->bs_w_2 = __BS(write_2);
1200 t->bs_w_4 = __BS(write_4);
1201 t->bs_w_8 = __BS(write_8);
1202
1203 /* write multiple */
1204 t->bs_wm_1 = __BS(write_multi_1);
1205 t->bs_wm_2 = __BS(write_multi_2);
1206 t->bs_wm_4 = __BS(write_multi_4);
1207 t->bs_wm_8 = __BS(write_multi_8);
1208
1209 /* write region */
1210 t->bs_wr_1 = __BS(write_region_1);
1211 t->bs_wr_2 = __BS(write_region_2);
1212 t->bs_wr_4 = __BS(write_region_4);
1213 t->bs_wr_8 = __BS(write_region_8);
1214
1215 /* set multiple */
1216 t->bs_sm_1 = __BS(set_multi_1);
1217 t->bs_sm_2 = __BS(set_multi_2);
1218 t->bs_sm_4 = __BS(set_multi_4);
1219 t->bs_sm_8 = __BS(set_multi_8);
1220
1221 /* set region */
1222 t->bs_sr_1 = __BS(set_region_1);
1223 t->bs_sr_2 = __BS(set_region_2);
1224 t->bs_sr_4 = __BS(set_region_4);
1225 t->bs_sr_8 = __BS(set_region_8);
1226
1227 /* copy */
1228 t->bs_c_1 = __BS(copy_region_1);
1229 t->bs_c_2 = __BS(copy_region_2);
1230 t->bs_c_4 = __BS(copy_region_4);
1231 t->bs_c_8 = __BS(copy_region_8);
1232
1233 #ifdef CHIP_NEED_STREAM
1234 /* read (single), stream */
1235 t->bs_rs_1 = __BS(read_stream_1);
1236 t->bs_rs_2 = __BS(read_stream_2);
1237 t->bs_rs_4 = __BS(read_stream_4);
1238 t->bs_rs_8 = __BS(read_stream_8);
1239
1240 /* read multiple, stream */
1241 t->bs_rms_1 = __BS(read_multi_stream_1);
1242 t->bs_rms_2 = __BS(read_multi_stream_2);
1243 t->bs_rms_4 = __BS(read_multi_stream_4);
1244 t->bs_rms_8 = __BS(read_multi_stream_8);
1245
1246 /* read region, stream */
1247 t->bs_rrs_1 = __BS(read_region_stream_1);
1248 t->bs_rrs_2 = __BS(read_region_stream_2);
1249 t->bs_rrs_4 = __BS(read_region_stream_4);
1250 t->bs_rrs_8 = __BS(read_region_stream_8);
1251
1252 /* write (single), stream */
1253 t->bs_ws_1 = __BS(write_stream_1);
1254 t->bs_ws_2 = __BS(write_stream_2);
1255 t->bs_ws_4 = __BS(write_stream_4);
1256 t->bs_ws_8 = __BS(write_stream_8);
1257
1258 /* write multiple, stream */
1259 t->bs_wms_1 = __BS(write_multi_stream_1);
1260 t->bs_wms_2 = __BS(write_multi_stream_2);
1261 t->bs_wms_4 = __BS(write_multi_stream_4);
1262 t->bs_wms_8 = __BS(write_multi_stream_8);
1263
1264 /* write region, stream */
1265 t->bs_wrs_1 = __BS(write_region_stream_1);
1266 t->bs_wrs_2 = __BS(write_region_stream_2);
1267 t->bs_wrs_4 = __BS(write_region_stream_4);
1268 t->bs_wrs_8 = __BS(write_region_stream_8);
1269
1270 #else /* CHIP_NEED_STREAM */
1271
1272 /* read (single), stream */
1273 t->bs_rs_1 = __BS(read_1);
1274 t->bs_rs_2 = __BS(read_2);
1275 t->bs_rs_4 = __BS(read_4);
1276 t->bs_rs_8 = __BS(read_8);
1277
1278 /* read multiple, stream */
1279 t->bs_rms_1 = __BS(read_multi_1);
1280 t->bs_rms_2 = __BS(read_multi_2);
1281 t->bs_rms_4 = __BS(read_multi_4);
1282 t->bs_rms_8 = __BS(read_multi_8);
1283
1284 /* read region, stream */
1285 t->bs_rrs_1 = __BS(read_region_1);
1286 t->bs_rrs_2 = __BS(read_region_2);
1287 t->bs_rrs_4 = __BS(read_region_4);
1288 t->bs_rrs_8 = __BS(read_region_8);
1289
1290 /* write (single), stream */
1291 t->bs_ws_1 = __BS(write_1);
1292 t->bs_ws_2 = __BS(write_2);
1293 t->bs_ws_4 = __BS(write_4);
1294 t->bs_ws_8 = __BS(write_8);
1295
1296 /* write multiple, stream */
1297 t->bs_wms_1 = __BS(write_multi_1);
1298 t->bs_wms_2 = __BS(write_multi_2);
1299 t->bs_wms_4 = __BS(write_multi_4);
1300 t->bs_wms_8 = __BS(write_multi_8);
1301
1302 /* write region, stream */
1303 t->bs_wrs_1 = __BS(write_region_1);
1304 t->bs_wrs_2 = __BS(write_region_2);
1305 t->bs_wrs_4 = __BS(write_region_4);
1306 t->bs_wrs_8 = __BS(write_region_8);
1307 #endif /* CHIP_NEED_STREAM */
1308
1309 #ifdef CHIP_EXTENT
1310 /* XXX WE WANT EXTENT_NOCOALESCE, BUT WE CAN'T USE IT. XXX */
1311 ex = extent_create(__S(__BS(bus)), 0x0UL, ~0UL,
1312 (void *)CHIP_EX_STORE(v), CHIP_EX_STORE_SIZE(v), EX_NOWAIT);
1313 extent_alloc_region(ex, 0, ~0UL, EX_NOWAIT);
1314
1315 #ifdef CHIP_W1_BUS_START
1316 /*
1317 * The window may be disabled. We notice this by seeing
1318 * -1 as the bus base address.
1319 */
1320 if (CHIP_W1_BUS_START(v) == (bus_addr_t) -1) {
1321 #ifdef EXTENT_DEBUG
1322 printf("%s: this space is disabled\n", __S(__BS(init)));
1323 #endif
1324 return;
1325 }
1326
1327 #ifdef EXTENT_DEBUG
1328 printf("%s: freeing from %#"PRIxBUSADDR" to %#"PRIxBUSADDR"\n",
1329 __S(__BS(init)), (bus_addr_t)CHIP_W1_BUS_START(v),
1330 (bus_addr_t)CHIP_W1_BUS_END(v));
1331 #endif
1332 extent_free(ex, CHIP_W1_BUS_START(v),
1333 CHIP_W1_BUS_END(v) - CHIP_W1_BUS_START(v) + 1, EX_NOWAIT);
1334 #endif
1335 #ifdef CHIP_W2_BUS_START
1336 if (CHIP_W2_BUS_START(v) != CHIP_W1_BUS_START(v)) {
1337 #ifdef EXTENT_DEBUG
1338 printf("xxx: freeing from 0x%lx to 0x%lx\n",
1339 (u_long)CHIP_W2_BUS_START(v), (u_long)CHIP_W2_BUS_END(v));
1340 #endif
1341 extent_free(ex, CHIP_W2_BUS_START(v),
1342 CHIP_W2_BUS_END(v) - CHIP_W2_BUS_START(v) + 1, EX_NOWAIT);
1343 } else {
1344 #ifdef EXTENT_DEBUG
1345 printf("xxx: window 2 (0x%lx to 0x%lx) overlaps window 1\n",
1346 (u_long)CHIP_W2_BUS_START(v), (u_long)CHIP_W2_BUS_END(v));
1347 #endif
1348 }
1349 #endif
1350 #ifdef CHIP_W3_BUS_START
1351 if (CHIP_W3_BUS_START(v) != CHIP_W1_BUS_START(v) &&
1352 CHIP_W3_BUS_START(v) != CHIP_W2_BUS_START(v)) {
1353 #ifdef EXTENT_DEBUG
1354 printf("xxx: freeing from 0x%lx to 0x%lx\n",
1355 (u_long)CHIP_W3_BUS_START(v), (u_long)CHIP_W3_BUS_END(v));
1356 #endif
1357 extent_free(ex, CHIP_W3_BUS_START(v),
1358 CHIP_W3_BUS_END(v) - CHIP_W3_BUS_START(v) + 1, EX_NOWAIT);
1359 } else {
1360 #ifdef EXTENT_DEBUG
1361 printf("xxx: window 2 (0x%lx to 0x%lx) overlaps window 1\n",
1362 (u_long)CHIP_W2_BUS_START(v), (u_long)CHIP_W2_BUS_END(v));
1363 #endif
1364 }
1365 #endif
1366
1367 #ifdef EXTENT_DEBUG
1368 extent_print(ex);
1369 #endif
1370 CHIP_EXTENT(v) = ex;
1371 #endif /* CHIP_EXTENT */
1372 }
1373
1374