1 /* $NetBSD: usb_mem.c,v 1.68 2016/04/30 14:31:39 skrll Exp $ */ 2 3 /* 4 * Copyright (c) 1998 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Lennart Augustsson (lennart@augustsson.net) at 9 * Carlstedt Research & Technology. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * USB DMA memory allocation. 35 * We need to allocate a lot of small (many 8 byte, some larger) 36 * memory blocks that can be used for DMA. Using the bus_dma 37 * routines directly would incur large overheads in space and time. 38 */ 39 40 #include <sys/cdefs.h> 41 __KERNEL_RCSID(0, "$NetBSD: usb_mem.c,v 1.68 2016/04/30 14:31:39 skrll Exp $"); 42 43 #ifdef _KERNEL_OPT 44 #include "opt_usb.h" 45 #endif 46 47 #include <sys/param.h> 48 #include <sys/systm.h> 49 #include <sys/kernel.h> 50 #include <sys/kmem.h> 51 #include <sys/queue.h> 52 #include <sys/device.h> /* for usbdivar.h */ 53 #include <sys/bus.h> 54 #include <sys/cpu.h> 55 #include <sys/once.h> 56 57 #ifdef DIAGNOSTIC 58 #include <sys/proc.h> 59 #endif 60 61 #include <dev/usb/usb.h> 62 #include <dev/usb/usbdi.h> 63 #include <dev/usb/usbdivar.h> /* just for usb_dma_t */ 64 #include <dev/usb/usb_mem.h> 65 #include <dev/usb/usbhist.h> 66 67 #define DPRINTF(FMT,A,B,C,D) USBHIST_LOG(usbdebug,FMT,A,B,C,D) 68 #define DPRINTFN(N,FMT,A,B,C,D) USBHIST_LOGN(usbdebug,N,FMT,A,B,C,D) 69 70 #define USB_MEM_SMALL roundup(64, CACHE_LINE_SIZE) 71 #define USB_MEM_CHUNKS 64 72 #define USB_MEM_BLOCK (USB_MEM_SMALL * USB_MEM_CHUNKS) 73 74 /* This struct is overlayed on free fragments. */ 75 struct usb_frag_dma { 76 usb_dma_block_t *ufd_block; 77 u_int ufd_offs; 78 LIST_ENTRY(usb_frag_dma) ufd_next; 79 }; 80 81 Static usbd_status usb_block_allocmem(bus_dma_tag_t, size_t, size_t, 82 usb_dma_block_t **, bool); 83 Static void usb_block_freemem(usb_dma_block_t *); 84 85 LIST_HEAD(usb_dma_block_qh, usb_dma_block); 86 Static struct usb_dma_block_qh usb_blk_freelist = 87 LIST_HEAD_INITIALIZER(usb_blk_freelist); 88 kmutex_t usb_blk_lock; 89 90 #ifdef DEBUG 91 Static struct usb_dma_block_qh usb_blk_fraglist = 92 LIST_HEAD_INITIALIZER(usb_blk_fraglist); 93 Static struct usb_dma_block_qh usb_blk_fulllist = 94 LIST_HEAD_INITIALIZER(usb_blk_fulllist); 95 #endif 96 Static u_int usb_blk_nfree = 0; 97 /* XXX should have different free list for different tags (for speed) */ 98 Static LIST_HEAD(, usb_frag_dma) usb_frag_freelist = 99 LIST_HEAD_INITIALIZER(usb_frag_freelist); 100 101 Static int usb_mem_init(void); 102 103 Static int 104 usb_mem_init(void) 105 { 106 107 mutex_init(&usb_blk_lock, MUTEX_DEFAULT, IPL_NONE); 108 return 0; 109 } 110 111 Static usbd_status 112 usb_block_allocmem(bus_dma_tag_t tag, size_t size, size_t align, 113 usb_dma_block_t **dmap, bool multiseg) 114 { 115 usb_dma_block_t *b; 116 int error; 117 118 USBHIST_FUNC(); USBHIST_CALLED(usbdebug); 119 DPRINTFN(5, "size=%zu align=%zu", size, align, 0, 0); 120 121 ASSERT_SLEEPABLE(); 122 KASSERT(size != 0); 123 KASSERT(mutex_owned(&usb_blk_lock)); 124 125 /* First check the free list. */ 126 LIST_FOREACH(b, &usb_blk_freelist, next) { 127 /* Don't allocate multiple segments to unwilling callers */ 128 if (b->nsegs != 1 && !multiseg) 129 continue; 130 if (b->tag == tag && b->size >= size && b->align >= align) { 131 LIST_REMOVE(b, next); 132 usb_blk_nfree--; 133 *dmap = b; 134 DPRINTFN(6, "free list size=%zu", b->size, 0, 0, 0); 135 return USBD_NORMAL_COMPLETION; 136 } 137 } 138 139 DPRINTFN(6, "no free", 0, 0, 0, 0); 140 mutex_exit(&usb_blk_lock); 141 142 b = kmem_zalloc(sizeof(*b), KM_SLEEP); 143 if (b == NULL) { 144 goto fail; 145 } 146 147 b->tag = tag; 148 b->size = size; 149 b->align = align; 150 151 if (!multiseg) 152 /* Caller wants one segment */ 153 b->nsegs = 1; 154 else 155 b->nsegs = (size + (PAGE_SIZE-1)) / PAGE_SIZE; 156 157 b->segs = kmem_alloc(b->nsegs * sizeof(*b->segs), KM_SLEEP); 158 if (b->segs == NULL) { 159 kmem_free(b, sizeof(*b)); 160 goto fail; 161 } 162 b->nsegs_alloc = b->nsegs; 163 164 error = bus_dmamem_alloc(tag, b->size, align, 0, 165 b->segs, b->nsegs, 166 &b->nsegs, BUS_DMA_WAITOK); 167 if (error) 168 goto free0; 169 170 error = bus_dmamem_map(tag, b->segs, b->nsegs, b->size, 171 &b->kaddr, BUS_DMA_WAITOK|BUS_DMA_COHERENT); 172 if (error) 173 goto free1; 174 175 error = bus_dmamap_create(tag, b->size, b->nsegs, b->size, 176 0, BUS_DMA_WAITOK, &b->map); 177 if (error) 178 goto unmap; 179 180 error = bus_dmamap_load(tag, b->map, b->kaddr, b->size, NULL, 181 BUS_DMA_WAITOK); 182 if (error) 183 goto destroy; 184 185 *dmap = b; 186 #ifdef USB_FRAG_DMA_WORKAROUND 187 memset(b->kaddr, 0, b->size); 188 #endif 189 mutex_enter(&usb_blk_lock); 190 191 return USBD_NORMAL_COMPLETION; 192 193 destroy: 194 bus_dmamap_destroy(tag, b->map); 195 unmap: 196 bus_dmamem_unmap(tag, b->kaddr, b->size); 197 free1: 198 bus_dmamem_free(tag, b->segs, b->nsegs); 199 free0: 200 kmem_free(b->segs, b->nsegs_alloc * sizeof(*b->segs)); 201 kmem_free(b, sizeof(*b)); 202 fail: 203 mutex_enter(&usb_blk_lock); 204 205 return USBD_NOMEM; 206 } 207 208 #if 0 209 void 210 usb_block_real_freemem(usb_dma_block_t *b) 211 { 212 #ifdef DIAGNOSTIC 213 if (cpu_softintr_p() || cpu_intr_p()) { 214 printf("usb_block_real_freemem: in interrupt context\n"); 215 return; 216 } 217 #endif 218 bus_dmamap_unload(b->tag, b->map); 219 bus_dmamap_destroy(b->tag, b->map); 220 bus_dmamem_unmap(b->tag, b->kaddr, b->size); 221 bus_dmamem_free(b->tag, b->segs, b->nsegs); 222 kmem_free(b->segs, b->nsegs_alloc * sizeof(*b->segs)); 223 kmem_free(b, sizeof(*b)); 224 } 225 #endif 226 227 #ifdef DEBUG 228 static bool 229 usb_valid_block_p(usb_dma_block_t *b, struct usb_dma_block_qh *qh) 230 { 231 usb_dma_block_t *xb; 232 LIST_FOREACH(xb, qh, next) { 233 if (xb == b) 234 return true; 235 } 236 return false; 237 } 238 #endif 239 240 /* 241 * Do not free the memory unconditionally since we might be called 242 * from an interrupt context and that is BAD. 243 * XXX when should we really free? 244 */ 245 Static void 246 usb_block_freemem(usb_dma_block_t *b) 247 { 248 249 KASSERT(mutex_owned(&usb_blk_lock)); 250 251 USBHIST_FUNC(); USBHIST_CALLED(usbdebug); 252 DPRINTFN(6, "size=%zu", b->size, 0, 0, 0); 253 #ifdef DEBUG 254 LIST_REMOVE(b, next); 255 #endif 256 LIST_INSERT_HEAD(&usb_blk_freelist, b, next); 257 usb_blk_nfree++; 258 } 259 260 usbd_status 261 usb_allocmem(struct usbd_bus *bus, size_t size, size_t align, usb_dma_t *p) 262 { 263 264 return usb_allocmem_flags(bus, size, align, p, 0); 265 } 266 267 usbd_status 268 usb_allocmem_flags(struct usbd_bus *bus, size_t size, size_t align, usb_dma_t *p, 269 int flags) 270 { 271 bus_dma_tag_t tag = bus->ub_dmatag; 272 usbd_status err; 273 struct usb_frag_dma *f; 274 usb_dma_block_t *b; 275 int i; 276 static ONCE_DECL(init_control); 277 bool frag; 278 279 USBHIST_FUNC(); USBHIST_CALLED(usbdebug); 280 281 ASSERT_SLEEPABLE(); 282 283 RUN_ONCE(&init_control, usb_mem_init); 284 285 frag = (flags & USBMALLOC_MULTISEG); 286 287 /* If the request is large then just use a full block. */ 288 if (size > USB_MEM_SMALL || align > USB_MEM_SMALL) { 289 DPRINTFN(1, "large alloc %d", size, 0, 0, 0); 290 size = (size + USB_MEM_BLOCK - 1) & ~(USB_MEM_BLOCK - 1); 291 mutex_enter(&usb_blk_lock); 292 err = usb_block_allocmem(tag, size, align, &p->udma_block, frag); 293 if (!err) { 294 #ifdef DEBUG 295 LIST_INSERT_HEAD(&usb_blk_fulllist, p->udma_block, next); 296 #endif 297 p->udma_block->flags = USB_DMA_FULLBLOCK; 298 p->udma_offs = 0; 299 } 300 mutex_exit(&usb_blk_lock); 301 return err; 302 } 303 304 mutex_enter(&usb_blk_lock); 305 /* Check for free fragments. */ 306 LIST_FOREACH(f, &usb_frag_freelist, ufd_next) { 307 KDASSERTMSG(usb_valid_block_p(f->ufd_block, &usb_blk_fraglist), 308 "%s: usb frag %p: unknown block pointer %p", 309 __func__, f, f->ufd_block); 310 if (f->ufd_block->tag == tag) 311 break; 312 } 313 if (f == NULL) { 314 DPRINTFN(1, "adding fragments", 0, 0, 0, 0); 315 err = usb_block_allocmem(tag, USB_MEM_BLOCK, USB_MEM_SMALL, &b, 316 false); 317 if (err) { 318 mutex_exit(&usb_blk_lock); 319 return err; 320 } 321 #ifdef DEBUG 322 LIST_INSERT_HEAD(&usb_blk_fraglist, b, next); 323 #endif 324 b->flags = 0; 325 for (i = 0; i < USB_MEM_BLOCK; i += USB_MEM_SMALL) { 326 f = (struct usb_frag_dma *)((char *)b->kaddr + i); 327 f->ufd_block = b; 328 f->ufd_offs = i; 329 LIST_INSERT_HEAD(&usb_frag_freelist, f, ufd_next); 330 #ifdef USB_FRAG_DMA_WORKAROUND 331 i += 1 * USB_MEM_SMALL; 332 #endif 333 } 334 f = LIST_FIRST(&usb_frag_freelist); 335 } 336 p->udma_block = f->ufd_block; 337 p->udma_offs = f->ufd_offs; 338 #ifdef USB_FRAG_DMA_WORKAROUND 339 p->udma_offs += USB_MEM_SMALL; 340 #endif 341 LIST_REMOVE(f, ufd_next); 342 mutex_exit(&usb_blk_lock); 343 DPRINTFN(5, "use frag=%p size=%d", f, size, 0, 0); 344 345 return USBD_NORMAL_COMPLETION; 346 } 347 348 void 349 usb_freemem(struct usbd_bus *bus, usb_dma_t *p) 350 { 351 struct usb_frag_dma *f; 352 353 USBHIST_FUNC(); USBHIST_CALLED(usbdebug); 354 355 mutex_enter(&usb_blk_lock); 356 if (p->udma_block->flags & USB_DMA_FULLBLOCK) { 357 KDASSERTMSG(usb_valid_block_p(p->udma_block, &usb_blk_fulllist), 358 "%s: dma %p: invalid block pointer %p", 359 __func__, p, p->udma_block); 360 DPRINTFN(1, "large free", 0, 0, 0, 0); 361 usb_block_freemem(p->udma_block); 362 mutex_exit(&usb_blk_lock); 363 return; 364 } 365 KDASSERTMSG(usb_valid_block_p(p->udma_block, &usb_blk_fraglist), 366 "%s: dma %p: invalid block pointer %p", 367 __func__, p, p->udma_block); 368 //usb_syncmem(p, 0, USB_MEM_SMALL, BUS_DMASYNC_POSTREAD); 369 f = KERNADDR(p, 0); 370 #ifdef USB_FRAG_DMA_WORKAROUND 371 f = (void *)((uintptr_t)f - USB_MEM_SMALL); 372 #endif 373 f->ufd_block = p->udma_block; 374 f->ufd_offs = p->udma_offs; 375 #ifdef USB_FRAG_DMA_WORKAROUND 376 f->ufd_offs -= USB_MEM_SMALL; 377 #endif 378 LIST_INSERT_HEAD(&usb_frag_freelist, f, ufd_next); 379 mutex_exit(&usb_blk_lock); 380 DPRINTFN(5, "frag=%p", f, 0, 0, 0); 381 } 382 383 bus_addr_t 384 usb_dmaaddr(usb_dma_t *dma, unsigned int offset) 385 { 386 unsigned int i; 387 bus_size_t seg_offs; 388 389 offset += dma->udma_offs; 390 391 KASSERTMSG(offset < dma->udma_block->size, "offset %d vs %zu", offset, 392 dma->udma_block->size); 393 394 if (dma->udma_block->nsegs == 1) { 395 KASSERT(dma->udma_block->map->dm_segs[0].ds_len > offset); 396 return dma->udma_block->map->dm_segs[0].ds_addr + offset; 397 } 398 399 /* 400 * Search for a bus_segment_t corresponding to this offset. With no 401 * record of the offset in the map to a particular dma_segment_t, we 402 * have to iterate from the start of the list each time. Could be 403 * improved 404 */ 405 seg_offs = 0; 406 for (i = 0; i < dma->udma_block->nsegs; i++) { 407 if (seg_offs + dma->udma_block->map->dm_segs[i].ds_len > offset) 408 break; 409 410 seg_offs += dma->udma_block->map->dm_segs[i].ds_len; 411 } 412 413 KASSERT(i != dma->udma_block->nsegs); 414 offset -= seg_offs; 415 return dma->udma_block->map->dm_segs[i].ds_addr + offset; 416 } 417 418 void 419 usb_syncmem(usb_dma_t *p, bus_addr_t offset, bus_size_t len, int ops) 420 { 421 422 bus_dmamap_sync(p->udma_block->tag, p->udma_block->map, p->udma_offs + offset, 423 len, ops); 424 } 425