1 /* 2 * Copyright (C) 2012-2013 Matteo Landi, Luigi Rizzo, Giuseppe Lettieri. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 /* __FBSDID("$FreeBSD: head/sys/dev/netmap/netmap.c 241723 2012-10-19 09:41:45Z glebius $"); */ 27 28 #include <sys/types.h> 29 #include <sys/malloc.h> 30 #include <sys/proc.h> 31 #include <sys/socket.h> /* sockaddrs */ 32 #include <sys/sysctl.h> 33 #include <sys/bus.h> /* bus_dmamap_* */ 34 35 #include <vm/vm.h> /* vtophys */ 36 #include <vm/pmap.h> /* vtophys */ 37 38 #include <net/if.h> 39 #include <net/if_var.h> 40 #include <net/netmap.h> 41 42 #include "netmap_kern.h" 43 #include "netmap_mem2.h" 44 45 #define NMA_LOCK_INIT(n) lockinit(&(n)->nm_mtx, "netmap memory allocator lock", 0, LK_CANRECURSE) 46 #define NMA_LOCK_DESTROY(n) lockuninit(&(n)->nm_mtx) 47 #define NMA_LOCK(n) lockmgr(&(n)->nm_mtx, LK_EXCLUSIVE) 48 #define NMA_UNLOCK(n) lockmgr(&(n)->nm_mtx, LK_RELEASE) 49 50 struct netmap_obj_params netmap_params[NETMAP_POOLS_NR] = { 51 [NETMAP_IF_POOL] = { 52 .size = 1024, 53 .num = 100, 54 }, 55 [NETMAP_RING_POOL] = { 56 .size = 9*PAGE_SIZE, 57 .num = 200, 58 }, 59 [NETMAP_BUF_POOL] = { 60 .size = 2048, 61 .num = NETMAP_BUF_MAX_NUM, 62 }, 63 }; 64 65 66 /* 67 * nm_mem is the memory allocator used for all physical interfaces 68 * running in netmap mode. 69 * Virtual (VALE) ports will have each its own allocator. 70 */ 71 static int netmap_mem_global_config(struct netmap_mem_d *nmd); 72 static int netmap_mem_global_finalize(struct netmap_mem_d *nmd); 73 static void netmap_mem_global_deref(struct netmap_mem_d *nmd); 74 struct netmap_mem_d nm_mem = { /* Our memory allocator. */ 75 .pools = { 76 [NETMAP_IF_POOL] = { 77 .name = "netmap_if", 78 .objminsize = sizeof(struct netmap_if), 79 .objmaxsize = 4096, 80 .nummin = 10, /* don't be stingy */ 81 .nummax = 10000, /* XXX very large */ 82 }, 83 [NETMAP_RING_POOL] = { 84 .name = "netmap_ring", 85 .objminsize = sizeof(struct netmap_ring), 86 .objmaxsize = 32*PAGE_SIZE, 87 .nummin = 2, 88 .nummax = 1024, 89 }, 90 [NETMAP_BUF_POOL] = { 91 .name = "netmap_buf", 92 .objminsize = 64, 93 .objmaxsize = 65536, 94 .nummin = 4, 95 .nummax = 1000000, /* one million! */ 96 }, 97 }, 98 .config = netmap_mem_global_config, 99 .finalize = netmap_mem_global_finalize, 100 .deref = netmap_mem_global_deref, 101 }; 102 103 104 // XXX logically belongs to nm_mem 105 struct lut_entry *netmap_buffer_lut; /* exported */ 106 107 /* blueprint for the private memory allocators */ 108 static int netmap_mem_private_config(struct netmap_mem_d *nmd); 109 static int netmap_mem_private_finalize(struct netmap_mem_d *nmd); 110 static void netmap_mem_private_deref(struct netmap_mem_d *nmd); 111 const struct netmap_mem_d nm_blueprint = { 112 .pools = { 113 [NETMAP_IF_POOL] = { 114 .name = "%s_if", 115 .objminsize = sizeof(struct netmap_if), 116 .objmaxsize = 4096, 117 .nummin = 1, 118 .nummax = 10, 119 }, 120 [NETMAP_RING_POOL] = { 121 .name = "%s_ring", 122 .objminsize = sizeof(struct netmap_ring), 123 .objmaxsize = 32*PAGE_SIZE, 124 .nummin = 2, 125 .nummax = 1024, 126 }, 127 [NETMAP_BUF_POOL] = { 128 .name = "%s_buf", 129 .objminsize = 64, 130 .objmaxsize = 65536, 131 .nummin = 4, 132 .nummax = 1000000, /* one million! */ 133 }, 134 }, 135 .config = netmap_mem_private_config, 136 .finalize = netmap_mem_private_finalize, 137 .deref = netmap_mem_private_deref, 138 139 .flags = NETMAP_MEM_PRIVATE, 140 }; 141 142 /* memory allocator related sysctls */ 143 144 #define STRINGIFY(x) #x 145 146 147 #define DECLARE_SYSCTLS(id, name) \ 148 SYSCTL_INT(_dev_netmap, OID_AUTO, name##_size, \ 149 CTLFLAG_RW, &netmap_params[id].size, 0, "Requested size of netmap " STRINGIFY(name) "s"); \ 150 SYSCTL_INT(_dev_netmap, OID_AUTO, name##_curr_size, \ 151 CTLFLAG_RD, &nm_mem.pools[id]._objsize, 0, "Current size of netmap " STRINGIFY(name) "s"); \ 152 SYSCTL_INT(_dev_netmap, OID_AUTO, name##_num, \ 153 CTLFLAG_RW, &netmap_params[id].num, 0, "Requested number of netmap " STRINGIFY(name) "s"); \ 154 SYSCTL_INT(_dev_netmap, OID_AUTO, name##_curr_num, \ 155 CTLFLAG_RD, &nm_mem.pools[id].objtotal, 0, "Current number of netmap " STRINGIFY(name) "s") 156 157 SYSCTL_DECL(_dev_netmap); 158 DECLARE_SYSCTLS(NETMAP_IF_POOL, if); 159 DECLARE_SYSCTLS(NETMAP_RING_POOL, ring); 160 DECLARE_SYSCTLS(NETMAP_BUF_POOL, buf); 161 162 /* 163 * First, find the allocator that contains the requested offset, 164 * then locate the cluster through a lookup table. 165 */ 166 vm_paddr_t 167 netmap_mem_ofstophys(struct netmap_mem_d* nmd, vm_ooffset_t offset) 168 { 169 int i; 170 vm_ooffset_t o = offset; 171 vm_paddr_t pa; 172 struct netmap_obj_pool *p; 173 174 NMA_LOCK(nmd); 175 p = nmd->pools; 176 177 for (i = 0; i < NETMAP_POOLS_NR; offset -= p[i].memtotal, i++) { 178 if (offset >= p[i].memtotal) 179 continue; 180 // now lookup the cluster's address 181 pa = p[i].lut[offset / p[i]._objsize].paddr + 182 offset % p[i]._objsize; 183 NMA_UNLOCK(nmd); 184 return pa; 185 } 186 /* this is only in case of errors */ 187 D("invalid ofs 0x%x out of 0x%x 0x%x 0x%x", (u_int)o, 188 p[NETMAP_IF_POOL].memtotal, 189 p[NETMAP_IF_POOL].memtotal 190 + p[NETMAP_RING_POOL].memtotal, 191 p[NETMAP_IF_POOL].memtotal 192 + p[NETMAP_RING_POOL].memtotal 193 + p[NETMAP_BUF_POOL].memtotal); 194 NMA_UNLOCK(nmd); 195 return 0; // XXX bad address 196 } 197 198 int 199 netmap_mem_get_info(struct netmap_mem_d* nmd, u_int* size, u_int *memflags) 200 { 201 int error = 0; 202 NMA_LOCK(nmd); 203 error = nmd->config(nmd); 204 if (error) 205 goto out; 206 if (nmd->flags & NETMAP_MEM_FINALIZED) { 207 *size = nmd->nm_totalsize; 208 } else { 209 int i; 210 *size = 0; 211 for (i = 0; i < NETMAP_POOLS_NR; i++) { 212 struct netmap_obj_pool *p = nmd->pools + i; 213 *size += (p->_numclusters * p->_clustsize); 214 } 215 } 216 *memflags = nmd->flags; 217 out: 218 NMA_UNLOCK(nmd); 219 return error; 220 } 221 222 /* 223 * we store objects by kernel address, need to find the offset 224 * within the pool to export the value to userspace. 225 * Algorithm: scan until we find the cluster, then add the 226 * actual offset in the cluster 227 */ 228 static ssize_t 229 netmap_obj_offset(struct netmap_obj_pool *p, const void *vaddr) 230 { 231 int i, k = p->_clustentries, n = p->objtotal; 232 ssize_t ofs = 0; 233 234 for (i = 0; i < n; i += k, ofs += p->_clustsize) { 235 const char *base = p->lut[i].vaddr; 236 ssize_t relofs = (const char *) vaddr - base; 237 238 if (relofs < 0 || relofs >= p->_clustsize) 239 continue; 240 241 ofs = ofs + relofs; 242 ND("%s: return offset %d (cluster %d) for pointer %p", 243 p->name, ofs, i, vaddr); 244 return ofs; 245 } 246 D("address %p is not contained inside any cluster (%s)", 247 vaddr, p->name); 248 return 0; /* An error occurred */ 249 } 250 251 /* Helper functions which convert virtual addresses to offsets */ 252 #define netmap_if_offset(n, v) \ 253 netmap_obj_offset(&(n)->pools[NETMAP_IF_POOL], (v)) 254 255 #define netmap_ring_offset(n, v) \ 256 ((n)->pools[NETMAP_IF_POOL].memtotal + \ 257 netmap_obj_offset(&(n)->pools[NETMAP_RING_POOL], (v))) 258 259 #define netmap_buf_offset(n, v) \ 260 ((n)->pools[NETMAP_IF_POOL].memtotal + \ 261 (n)->pools[NETMAP_RING_POOL].memtotal + \ 262 netmap_obj_offset(&(n)->pools[NETMAP_BUF_POOL], (v))) 263 264 265 ssize_t 266 netmap_mem_if_offset(struct netmap_mem_d *nmd, const void *addr) 267 { 268 ssize_t v; 269 NMA_LOCK(nmd); 270 v = netmap_if_offset(nmd, addr); 271 NMA_UNLOCK(nmd); 272 return v; 273 } 274 275 /* 276 * report the index, and use start position as a hint, 277 * otherwise buffer allocation becomes terribly expensive. 278 */ 279 static void * 280 netmap_obj_malloc(struct netmap_obj_pool *p, u_int len, uint32_t *start, uint32_t *index) 281 { 282 uint32_t i = 0; /* index in the bitmap */ 283 uint32_t mask, j; /* slot counter */ 284 void *vaddr = NULL; 285 286 if (len > p->_objsize) { 287 D("%s request size %d too large", p->name, len); 288 // XXX cannot reduce the size 289 return NULL; 290 } 291 292 if (p->objfree == 0) { 293 D("%s allocator: run out of memory", p->name); 294 return NULL; 295 } 296 if (start) 297 i = *start; 298 299 /* termination is guaranteed by p->free, but better check bounds on i */ 300 while (vaddr == NULL && i < p->bitmap_slots) { 301 uint32_t cur = p->bitmap[i]; 302 if (cur == 0) { /* bitmask is fully used */ 303 i++; 304 continue; 305 } 306 /* locate a slot */ 307 for (j = 0, mask = 1; (cur & mask) == 0; j++, mask <<= 1) 308 ; 309 310 p->bitmap[i] &= ~mask; /* mark object as in use */ 311 p->objfree--; 312 313 vaddr = p->lut[i * 32 + j].vaddr; 314 if (index) 315 *index = i * 32 + j; 316 } 317 ND("%s allocator: allocated object @ [%d][%d]: vaddr %p", i, j, vaddr); 318 319 if (start) 320 *start = i; 321 return vaddr; 322 } 323 324 325 /* 326 * free by index, not by address. This is slow, but is only used 327 * for a small number of objects (rings, nifp) 328 */ 329 static void 330 netmap_obj_free(struct netmap_obj_pool *p, uint32_t j) 331 { 332 if (j >= p->objtotal) { 333 D("invalid index %u, max %u", j, p->objtotal); 334 return; 335 } 336 p->bitmap[j / 32] |= (1 << (j % 32)); 337 p->objfree++; 338 return; 339 } 340 341 static void 342 netmap_obj_free_va(struct netmap_obj_pool *p, void *vaddr) 343 { 344 u_int i, j, n = p->numclusters; 345 346 for (i = 0, j = 0; i < n; i++, j += p->_clustentries) { 347 void *base = p->lut[i * p->_clustentries].vaddr; 348 ssize_t relofs = (ssize_t) vaddr - (ssize_t) base; 349 350 /* Given address, is out of the scope of the current cluster.*/ 351 if (vaddr < base || relofs >= p->_clustsize) 352 continue; 353 354 j = j + relofs / p->_objsize; 355 /* KASSERT(j != 0, ("Cannot free object 0")); */ 356 netmap_obj_free(p, j); 357 return; 358 } 359 D("address %p is not contained inside any cluster (%s)", 360 vaddr, p->name); 361 } 362 363 #define netmap_if_malloc(n, len) netmap_obj_malloc(&(n)->pools[NETMAP_IF_POOL], len, NULL, NULL) 364 #define netmap_if_free(n, v) netmap_obj_free_va(&(n)->pools[NETMAP_IF_POOL], (v)) 365 #define netmap_ring_malloc(n, len) netmap_obj_malloc(&(n)->pools[NETMAP_RING_POOL], len, NULL, NULL) 366 #define netmap_ring_free(n, v) netmap_obj_free_va(&(n)->pools[NETMAP_RING_POOL], (v)) 367 #define netmap_buf_malloc(n, _pos, _index) \ 368 netmap_obj_malloc(&(n)->pools[NETMAP_BUF_POOL], NETMAP_BDG_BUF_SIZE(n), _pos, _index) 369 370 371 /* Return the index associated to the given packet buffer */ 372 #define netmap_buf_index(n, v) \ 373 (netmap_obj_offset(&(n)->pools[NETMAP_BUF_POOL], (v)) / NETMAP_BDG_BUF_SIZE(n)) 374 375 376 /* Return nonzero on error */ 377 static int 378 netmap_new_bufs(struct netmap_mem_d *nmd, struct netmap_slot *slot, u_int n) 379 { 380 struct netmap_obj_pool *p = &nmd->pools[NETMAP_BUF_POOL]; 381 u_int i = 0; /* slot counter */ 382 uint32_t pos = 0; /* slot in p->bitmap */ 383 uint32_t index = 0; /* buffer index */ 384 385 for (i = 0; i < n; i++) { 386 void *vaddr = netmap_buf_malloc(nmd, &pos, &index); 387 if (vaddr == NULL) { 388 D("unable to locate empty packet buffer"); 389 goto cleanup; 390 } 391 slot[i].buf_idx = index; 392 slot[i].len = p->_objsize; 393 /* XXX setting flags=NS_BUF_CHANGED forces a pointer reload 394 * in the NIC ring. This is a hack that hides missing 395 * initializations in the drivers, and should go away. 396 */ 397 // slot[i].flags = NS_BUF_CHANGED; 398 } 399 400 ND("allocated %d buffers, %d available, first at %d", n, p->objfree, pos); 401 return (0); 402 403 cleanup: 404 while (i > 0) { 405 i--; 406 netmap_obj_free(p, slot[i].buf_idx); 407 } 408 bzero(slot, n * sizeof(slot[0])); 409 return (ENOMEM); 410 } 411 412 413 static void 414 netmap_free_buf(struct netmap_mem_d *nmd, uint32_t i) 415 { 416 struct netmap_obj_pool *p = &nmd->pools[NETMAP_BUF_POOL]; 417 418 if (i < 2 || i >= p->objtotal) { 419 D("Cannot free buf#%d: should be in [2, %d[", i, p->objtotal); 420 return; 421 } 422 netmap_obj_free(p, i); 423 } 424 425 static void 426 netmap_reset_obj_allocator(struct netmap_obj_pool *p) 427 { 428 429 if (p == NULL) 430 return; 431 if (p->bitmap) 432 kfree(p->bitmap, M_NETMAP); 433 p->bitmap = NULL; 434 if (p->lut) { 435 u_int i; 436 size_t sz = p->_clustsize; 437 438 for (i = 0; i < p->objtotal; i += p->_clustentries) { 439 if (p->lut[i].vaddr) 440 contigfree(p->lut[i].vaddr, sz, M_NETMAP); 441 } 442 bzero(p->lut, sizeof(struct lut_entry) * p->objtotal); 443 kfree(p->lut, M_NETMAP); 444 } 445 p->lut = NULL; 446 p->objtotal = 0; 447 p->memtotal = 0; 448 p->numclusters = 0; 449 p->objfree = 0; 450 } 451 452 /* 453 * Free all resources related to an allocator. 454 */ 455 static void 456 netmap_destroy_obj_allocator(struct netmap_obj_pool *p) 457 { 458 if (p == NULL) 459 return; 460 netmap_reset_obj_allocator(p); 461 } 462 463 /* 464 * We receive a request for objtotal objects, of size objsize each. 465 * Internally we may round up both numbers, as we allocate objects 466 * in small clusters multiple of the page size. 467 * We need to keep track of objtotal and clustentries, 468 * as they are needed when freeing memory. 469 * 470 * XXX note -- userspace needs the buffers to be contiguous, 471 * so we cannot afford gaps at the end of a cluster. 472 */ 473 474 475 /* call with NMA_LOCK held */ 476 static int 477 netmap_config_obj_allocator(struct netmap_obj_pool *p, u_int objtotal, u_int objsize) 478 { 479 int i; 480 u_int clustsize; /* the cluster size, multiple of page size */ 481 u_int clustentries; /* how many objects per entry */ 482 483 /* we store the current request, so we can 484 * detect configuration changes later */ 485 p->r_objtotal = objtotal; 486 p->r_objsize = objsize; 487 488 #define MAX_CLUSTSIZE (1<<17) 489 #define LINE_ROUND 64 490 if (objsize >= MAX_CLUSTSIZE) { 491 /* we could do it but there is no point */ 492 D("unsupported allocation for %d bytes", objsize); 493 return EINVAL; 494 } 495 /* make sure objsize is a multiple of LINE_ROUND */ 496 i = (objsize & (LINE_ROUND - 1)); 497 if (i) { 498 D("XXX aligning object by %d bytes", LINE_ROUND - i); 499 objsize += LINE_ROUND - i; 500 } 501 if (objsize < p->objminsize || objsize > p->objmaxsize) { 502 D("requested objsize %d out of range [%d, %d]", 503 objsize, p->objminsize, p->objmaxsize); 504 return EINVAL; 505 } 506 if (objtotal < p->nummin || objtotal > p->nummax) { 507 D("requested objtotal %d out of range [%d, %d]", 508 objtotal, p->nummin, p->nummax); 509 return EINVAL; 510 } 511 /* 512 * Compute number of objects using a brute-force approach: 513 * given a max cluster size, 514 * we try to fill it with objects keeping track of the 515 * wasted space to the next page boundary. 516 */ 517 for (clustentries = 0, i = 1;; i++) { 518 u_int delta, used = i * objsize; 519 if (used > MAX_CLUSTSIZE) 520 break; 521 delta = used % PAGE_SIZE; 522 if (delta == 0) { // exact solution 523 clustentries = i; 524 break; 525 } 526 if (delta > ( (clustentries*objsize) % PAGE_SIZE) ) 527 clustentries = i; 528 } 529 // D("XXX --- ouch, delta %d (bad for buffers)", delta); 530 /* compute clustsize and round to the next page */ 531 clustsize = clustentries * objsize; 532 i = (clustsize & (PAGE_SIZE - 1)); 533 if (i) 534 clustsize += PAGE_SIZE - i; 535 if (netmap_verbose) 536 D("objsize %d clustsize %d objects %d", 537 objsize, clustsize, clustentries); 538 539 /* 540 * The number of clusters is n = ceil(objtotal/clustentries) 541 * objtotal' = n * clustentries 542 */ 543 p->_clustentries = clustentries; 544 p->_clustsize = clustsize; 545 p->_numclusters = (objtotal + clustentries - 1) / clustentries; 546 547 /* actual values (may be larger than requested) */ 548 p->_objsize = objsize; 549 p->_objtotal = p->_numclusters * clustentries; 550 551 return 0; 552 } 553 554 555 /* call with NMA_LOCK held */ 556 static int 557 netmap_finalize_obj_allocator(struct netmap_obj_pool *p) 558 { 559 int i; /* must be signed */ 560 size_t n; 561 562 /* optimistically assume we have enough memory */ 563 p->numclusters = p->_numclusters; 564 p->objtotal = p->_objtotal; 565 566 n = sizeof(struct lut_entry) * p->objtotal; 567 p->lut = kmalloc(n, M_NETMAP, M_NOWAIT | M_ZERO); 568 if (p->lut == NULL) { 569 D("Unable to create lookup table (%d bytes) for '%s'", (int)n, p->name); 570 goto clean; 571 } 572 573 /* Allocate the bitmap */ 574 n = (p->objtotal + 31) / 32; 575 p->bitmap = kmalloc(sizeof(uint32_t) * n, M_NETMAP, M_NOWAIT | M_ZERO); 576 if (p->bitmap == NULL) { 577 D("Unable to create bitmap (%d entries) for allocator '%s'", (int)n, 578 p->name); 579 goto clean; 580 } 581 p->bitmap_slots = n; 582 583 /* 584 * Allocate clusters, init pointers and bitmap 585 */ 586 587 n = p->_clustsize; 588 for (i = 0; i < (int)p->objtotal;) { 589 int lim = i + p->_clustentries; 590 char *clust; 591 592 clust = contigmalloc(n, M_NETMAP, M_NOWAIT | M_ZERO, 593 (size_t)0, -1UL, PAGE_SIZE, 0); 594 if (clust == NULL) { 595 /* 596 * If we get here, there is a severe memory shortage, 597 * so halve the allocated memory to reclaim some. 598 */ 599 D("Unable to create cluster at %d for '%s' allocator", 600 i, p->name); 601 if (i < 2) /* nothing to halve */ 602 goto out; 603 lim = i / 2; 604 for (i--; i >= lim; i--) { 605 p->bitmap[ (i>>5) ] &= ~( 1 << (i & 31) ); 606 if (i % p->_clustentries == 0 && p->lut[i].vaddr) 607 contigfree(p->lut[i].vaddr, 608 n, M_NETMAP); 609 } 610 out: 611 p->objtotal = i; 612 /* we may have stopped in the middle of a cluster */ 613 p->numclusters = (i + p->_clustentries - 1) / p->_clustentries; 614 break; 615 } 616 for (; i < lim; i++, clust += p->_objsize) { 617 p->bitmap[ (i>>5) ] |= ( 1 << (i & 31) ); 618 p->lut[i].vaddr = clust; 619 p->lut[i].paddr = vtophys(clust); 620 } 621 } 622 p->objfree = p->objtotal; 623 p->memtotal = p->numclusters * p->_clustsize; 624 if (p->objfree == 0) 625 goto clean; 626 if (netmap_verbose) 627 D("Pre-allocated %d clusters (%d/%dKB) for '%s'", 628 p->numclusters, p->_clustsize >> 10, 629 p->memtotal >> 10, p->name); 630 631 return 0; 632 633 clean: 634 netmap_reset_obj_allocator(p); 635 return ENOMEM; 636 } 637 638 /* call with lock held */ 639 static int 640 netmap_memory_config_changed(struct netmap_mem_d *nmd) 641 { 642 int i; 643 644 for (i = 0; i < NETMAP_POOLS_NR; i++) { 645 if (nmd->pools[i].r_objsize != netmap_params[i].size || 646 nmd->pools[i].r_objtotal != netmap_params[i].num) 647 return 1; 648 } 649 return 0; 650 } 651 652 static void 653 netmap_mem_reset_all(struct netmap_mem_d *nmd) 654 { 655 int i; 656 D("resetting %p", nmd); 657 for (i = 0; i < NETMAP_POOLS_NR; i++) { 658 netmap_reset_obj_allocator(&nmd->pools[i]); 659 } 660 nmd->flags &= ~NETMAP_MEM_FINALIZED; 661 } 662 663 static int 664 netmap_mem_finalize_all(struct netmap_mem_d *nmd) 665 { 666 int i; 667 if (nmd->flags & NETMAP_MEM_FINALIZED) 668 return 0; 669 nmd->lasterr = 0; 670 nmd->nm_totalsize = 0; 671 for (i = 0; i < NETMAP_POOLS_NR; i++) { 672 nmd->lasterr = netmap_finalize_obj_allocator(&nmd->pools[i]); 673 if (nmd->lasterr) 674 goto error; 675 nmd->nm_totalsize += nmd->pools[i].memtotal; 676 } 677 /* buffers 0 and 1 are reserved */ 678 nmd->pools[NETMAP_BUF_POOL].objfree -= 2; 679 nmd->pools[NETMAP_BUF_POOL].bitmap[0] = ~3; 680 nmd->flags |= NETMAP_MEM_FINALIZED; 681 682 D("Have %d KB for interfaces, %d KB for rings and %d MB for buffers", 683 nmd->pools[NETMAP_IF_POOL].memtotal >> 10, 684 nmd->pools[NETMAP_RING_POOL].memtotal >> 10, 685 nmd->pools[NETMAP_BUF_POOL].memtotal >> 20); 686 687 D("Free buffers: %d", nmd->pools[NETMAP_BUF_POOL].objfree); 688 689 690 return 0; 691 error: 692 netmap_mem_reset_all(nmd); 693 return nmd->lasterr; 694 } 695 696 697 698 void 699 netmap_mem_private_delete(struct netmap_mem_d *nmd) 700 { 701 if (nmd == NULL) 702 return; 703 D("deleting %p", nmd); 704 if (nmd->refcount > 0) 705 D("bug: deleting mem allocator with refcount=%d!", nmd->refcount); 706 D("done deleting %p", nmd); 707 NMA_LOCK_DESTROY(nmd); 708 kfree(nmd, M_DEVBUF); 709 } 710 711 static int 712 netmap_mem_private_config(struct netmap_mem_d *nmd) 713 { 714 /* nothing to do, we are configured on creation 715 * and configuration never changes thereafter 716 */ 717 return 0; 718 } 719 720 static int 721 netmap_mem_private_finalize(struct netmap_mem_d *nmd) 722 { 723 int err; 724 NMA_LOCK(nmd); 725 nmd->refcount++; 726 err = netmap_mem_finalize_all(nmd); 727 NMA_UNLOCK(nmd); 728 return err; 729 730 } 731 732 static void 733 netmap_mem_private_deref(struct netmap_mem_d *nmd) 734 { 735 NMA_LOCK(nmd); 736 if (--nmd->refcount <= 0) 737 netmap_mem_reset_all(nmd); 738 NMA_UNLOCK(nmd); 739 } 740 741 struct netmap_mem_d * 742 netmap_mem_private_new(const char *name, u_int txr, u_int txd, u_int rxr, u_int rxd) 743 { 744 struct netmap_mem_d *d = NULL; 745 struct netmap_obj_params p[NETMAP_POOLS_NR]; 746 int i; 747 u_int maxd; 748 749 d = kmalloc(sizeof(struct netmap_mem_d), 750 M_DEVBUF, M_NOWAIT | M_ZERO); 751 if (d == NULL) 752 return NULL; 753 754 *d = nm_blueprint; 755 756 /* XXX the rest of the code assumes the stack rings are alwasy present */ 757 txr++; 758 rxr++; 759 p[NETMAP_IF_POOL].size = sizeof(struct netmap_if) + 760 sizeof(ssize_t) * (txr + rxr); 761 p[NETMAP_IF_POOL].num = 2; 762 maxd = (txd > rxd) ? txd : rxd; 763 p[NETMAP_RING_POOL].size = sizeof(struct netmap_ring) + 764 sizeof(struct netmap_slot) * maxd; 765 p[NETMAP_RING_POOL].num = txr + rxr; 766 p[NETMAP_BUF_POOL].size = 2048; /* XXX find a way to let the user choose this */ 767 p[NETMAP_BUF_POOL].num = rxr * (rxd + 2) + txr * (txd + 2); 768 769 D("req if %d*%d ring %d*%d buf %d*%d", 770 p[NETMAP_IF_POOL].num, 771 p[NETMAP_IF_POOL].size, 772 p[NETMAP_RING_POOL].num, 773 p[NETMAP_RING_POOL].size, 774 p[NETMAP_BUF_POOL].num, 775 p[NETMAP_BUF_POOL].size); 776 777 for (i = 0; i < NETMAP_POOLS_NR; i++) { 778 ksnprintf(d->pools[i].name, NETMAP_POOL_MAX_NAMSZ, 779 nm_blueprint.pools[i].name, 780 name); 781 if (netmap_config_obj_allocator(&d->pools[i], 782 p[i].num, p[i].size)) 783 goto error; 784 } 785 786 d->flags &= ~NETMAP_MEM_FINALIZED; 787 788 NMA_LOCK_INIT(d); 789 790 return d; 791 error: 792 netmap_mem_private_delete(d); 793 return NULL; 794 } 795 796 797 /* call with lock held */ 798 static int 799 netmap_mem_global_config(struct netmap_mem_d *nmd) 800 { 801 int i; 802 803 if (nmd->refcount) 804 /* already in use, we cannot change the configuration */ 805 goto out; 806 807 if (!netmap_memory_config_changed(nmd)) 808 goto out; 809 810 D("reconfiguring"); 811 812 if (nmd->flags & NETMAP_MEM_FINALIZED) { 813 /* reset previous allocation */ 814 for (i = 0; i < NETMAP_POOLS_NR; i++) { 815 netmap_reset_obj_allocator(&nmd->pools[i]); 816 } 817 nmd->flags &= ~NETMAP_MEM_FINALIZED; 818 } 819 820 for (i = 0; i < NETMAP_POOLS_NR; i++) { 821 nmd->lasterr = netmap_config_obj_allocator(&nmd->pools[i], 822 netmap_params[i].num, netmap_params[i].size); 823 if (nmd->lasterr) 824 goto out; 825 } 826 827 out: 828 829 return nmd->lasterr; 830 } 831 832 static int 833 netmap_mem_global_finalize(struct netmap_mem_d *nmd) 834 { 835 int err; 836 837 NMA_LOCK(nmd); 838 839 840 /* update configuration if changed */ 841 if (netmap_mem_global_config(nmd)) 842 goto out; 843 844 nmd->refcount++; 845 846 if (nmd->flags & NETMAP_MEM_FINALIZED) { 847 /* may happen if config is not changed */ 848 ND("nothing to do"); 849 goto out; 850 } 851 852 if (netmap_mem_finalize_all(nmd)) 853 goto out; 854 855 /* backward compatibility */ 856 netmap_buf_size = nmd->pools[NETMAP_BUF_POOL]._objsize; 857 netmap_total_buffers = nmd->pools[NETMAP_BUF_POOL].objtotal; 858 859 netmap_buffer_lut = nmd->pools[NETMAP_BUF_POOL].lut; 860 netmap_buffer_base = nmd->pools[NETMAP_BUF_POOL].lut[0].vaddr; 861 862 nmd->lasterr = 0; 863 864 out: 865 if (nmd->lasterr) 866 nmd->refcount--; 867 err = nmd->lasterr; 868 869 NMA_UNLOCK(nmd); 870 871 return err; 872 873 } 874 875 int 876 netmap_mem_init(void) 877 { 878 NMA_LOCK_INIT(&nm_mem); 879 return (0); 880 } 881 882 void 883 netmap_mem_fini(void) 884 { 885 int i; 886 887 for (i = 0; i < NETMAP_POOLS_NR; i++) { 888 netmap_destroy_obj_allocator(&nm_mem.pools[i]); 889 } 890 NMA_LOCK_DESTROY(&nm_mem); 891 } 892 893 static void 894 netmap_free_rings(struct netmap_adapter *na) 895 { 896 u_int i; 897 if (!na->tx_rings) 898 return; 899 for (i = 0; i < na->num_tx_rings + 1; i++) { 900 if (na->tx_rings[i].ring) { 901 netmap_ring_free(na->nm_mem, na->tx_rings[i].ring); 902 na->tx_rings[i].ring = NULL; 903 } 904 } 905 for (i = 0; i < na->num_rx_rings + 1; i++) { 906 if (na->rx_rings[i].ring) { 907 netmap_ring_free(na->nm_mem, na->rx_rings[i].ring); 908 na->rx_rings[i].ring = NULL; 909 } 910 } 911 } 912 913 /* call with NMA_LOCK held * 914 * 915 * Allocate netmap rings and buffers for this card 916 * The rings are contiguous, but have variable size. 917 */ 918 int 919 netmap_mem_rings_create(struct netmap_adapter *na) 920 { 921 struct netmap_ring *ring; 922 u_int len, ndesc; 923 struct netmap_kring *kring; 924 925 NMA_LOCK(na->nm_mem); 926 927 for (kring = na->tx_rings; kring != na->rx_rings; kring++) { /* Transmit rings */ 928 ndesc = kring->nkr_num_slots; 929 len = sizeof(struct netmap_ring) + 930 ndesc * sizeof(struct netmap_slot); 931 ring = netmap_ring_malloc(na->nm_mem, len); 932 if (ring == NULL) { 933 D("Cannot allocate tx_ring"); 934 goto cleanup; 935 } 936 ND("txring[%d] at %p ofs %d", i, ring); 937 kring->ring = ring; 938 *(uint32_t *)(uintptr_t)&ring->num_slots = ndesc; 939 *(ssize_t *)(uintptr_t)&ring->buf_ofs = 940 (na->nm_mem->pools[NETMAP_IF_POOL].memtotal + 941 na->nm_mem->pools[NETMAP_RING_POOL].memtotal) - 942 netmap_ring_offset(na->nm_mem, ring); 943 944 ring->avail = kring->nr_hwavail; 945 ring->cur = kring->nr_hwcur; 946 *(uint16_t *)(uintptr_t)&ring->nr_buf_size = 947 NETMAP_BDG_BUF_SIZE(na->nm_mem); 948 ND("initializing slots for txring"); 949 if (netmap_new_bufs(na->nm_mem, ring->slot, ndesc)) { 950 D("Cannot allocate buffers for tx_ring"); 951 goto cleanup; 952 } 953 } 954 955 for ( ; kring != na->tailroom; kring++) { /* Receive rings */ 956 ndesc = kring->nkr_num_slots; 957 len = sizeof(struct netmap_ring) + 958 ndesc * sizeof(struct netmap_slot); 959 ring = netmap_ring_malloc(na->nm_mem, len); 960 if (ring == NULL) { 961 D("Cannot allocate rx_ring"); 962 goto cleanup; 963 } 964 ND("rxring at %p ofs %d", ring); 965 966 kring->ring = ring; 967 *(uint32_t *)(uintptr_t)&ring->num_slots = ndesc; 968 *(ssize_t *)(uintptr_t)&ring->buf_ofs = 969 (na->nm_mem->pools[NETMAP_IF_POOL].memtotal + 970 na->nm_mem->pools[NETMAP_RING_POOL].memtotal) - 971 netmap_ring_offset(na->nm_mem, ring); 972 973 ring->cur = kring->nr_hwcur; 974 ring->avail = kring->nr_hwavail; 975 *(int *)(uintptr_t)&ring->nr_buf_size = 976 NETMAP_BDG_BUF_SIZE(na->nm_mem); 977 ND("initializing slots for rxring[%d]", i); 978 if (netmap_new_bufs(na->nm_mem, ring->slot, ndesc)) { 979 D("Cannot allocate buffers for rx_ring"); 980 goto cleanup; 981 } 982 } 983 984 NMA_UNLOCK(na->nm_mem); 985 986 return 0; 987 988 cleanup: 989 netmap_free_rings(na); 990 991 NMA_UNLOCK(na->nm_mem); 992 993 return ENOMEM; 994 } 995 996 void 997 netmap_mem_rings_delete(struct netmap_adapter *na) 998 { 999 /* last instance, release bufs and rings */ 1000 u_int i, lim; 1001 struct netmap_kring *kring; 1002 struct netmap_ring *ring; 1003 1004 NMA_LOCK(na->nm_mem); 1005 1006 for (kring = na->tx_rings; kring != na->tailroom; kring++) { 1007 ring = kring->ring; 1008 if (ring == NULL) 1009 continue; 1010 lim = kring->nkr_num_slots; 1011 for (i = 0; i < lim; i++) 1012 netmap_free_buf(na->nm_mem, ring->slot[i].buf_idx); 1013 } 1014 netmap_free_rings(na); 1015 1016 NMA_UNLOCK(na->nm_mem); 1017 } 1018 1019 1020 /* call with NMA_LOCK held */ 1021 /* 1022 * Allocate the per-fd structure netmap_if. 1023 * 1024 * We assume that the configuration stored in na 1025 * (number of tx/rx rings and descs) does not change while 1026 * the interface is in netmap mode. 1027 */ 1028 struct netmap_if * 1029 netmap_mem_if_new(const char *ifname, struct netmap_adapter *na) 1030 { 1031 struct netmap_if *nifp; 1032 ssize_t base; /* handy for relative offsets between rings and nifp */ 1033 u_int i, len, ntx, nrx; 1034 1035 /* 1036 * verify whether virtual port need the stack ring 1037 */ 1038 ntx = na->num_tx_rings + 1; /* shorthand, include stack ring */ 1039 nrx = na->num_rx_rings + 1; /* shorthand, include stack ring */ 1040 /* 1041 * the descriptor is followed inline by an array of offsets 1042 * to the tx and rx rings in the shared memory region. 1043 * For virtual rx rings we also allocate an array of 1044 * pointers to assign to nkr_leases. 1045 */ 1046 1047 NMA_LOCK(na->nm_mem); 1048 1049 len = sizeof(struct netmap_if) + (nrx + ntx) * sizeof(ssize_t); 1050 nifp = netmap_if_malloc(na->nm_mem, len); 1051 if (nifp == NULL) { 1052 NMA_UNLOCK(na->nm_mem); 1053 return NULL; 1054 } 1055 1056 /* initialize base fields -- override const */ 1057 *(u_int *)(uintptr_t)&nifp->ni_tx_rings = na->num_tx_rings; 1058 *(u_int *)(uintptr_t)&nifp->ni_rx_rings = na->num_rx_rings; 1059 strncpy(nifp->ni_name, ifname, (size_t)IFNAMSIZ); 1060 1061 /* 1062 * fill the slots for the rx and tx rings. They contain the offset 1063 * between the ring and nifp, so the information is usable in 1064 * userspace to reach the ring from the nifp. 1065 */ 1066 base = netmap_if_offset(na->nm_mem, nifp); 1067 for (i = 0; i < ntx; i++) { 1068 *(ssize_t *)(uintptr_t)&nifp->ring_ofs[i] = 1069 netmap_ring_offset(na->nm_mem, na->tx_rings[i].ring) - base; 1070 } 1071 for (i = 0; i < nrx; i++) { 1072 *(ssize_t *)(uintptr_t)&nifp->ring_ofs[i+ntx] = 1073 netmap_ring_offset(na->nm_mem, na->rx_rings[i].ring) - base; 1074 } 1075 1076 NMA_UNLOCK(na->nm_mem); 1077 1078 return (nifp); 1079 } 1080 1081 void 1082 netmap_mem_if_delete(struct netmap_adapter *na, struct netmap_if *nifp) 1083 { 1084 if (nifp == NULL) 1085 /* nothing to do */ 1086 return; 1087 NMA_LOCK(na->nm_mem); 1088 1089 netmap_if_free(na->nm_mem, nifp); 1090 1091 NMA_UNLOCK(na->nm_mem); 1092 } 1093 1094 static void 1095 netmap_mem_global_deref(struct netmap_mem_d *nmd) 1096 { 1097 NMA_LOCK(nmd); 1098 1099 nmd->refcount--; 1100 if (netmap_verbose) 1101 D("refcount = %d", nmd->refcount); 1102 1103 NMA_UNLOCK(nmd); 1104 } 1105 1106 int 1107 netmap_mem_finalize(struct netmap_mem_d *nmd) 1108 { 1109 return nmd->finalize(nmd); 1110 } 1111 1112 void 1113 netmap_mem_deref(struct netmap_mem_d *nmd) 1114 { 1115 return nmd->deref(nmd); 1116 } 1117