Lines Matching defs:rv
174 #define vm_reserv_lockptr(rv) (&(rv)->lock)
175 #define vm_reserv_assert_locked(rv) \
176 mtx_assert(vm_reserv_lockptr(rv), MA_OWNED)
177 #define vm_reserv_lock(rv) mtx_lock(vm_reserv_lockptr(rv))
178 #define vm_reserv_trylock(rv) mtx_trylock(vm_reserv_lockptr(rv))
179 #define vm_reserv_unlock(rv) mtx_unlock(vm_reserv_lockptr(rv))
278 static void vm_reserv_break(vm_reserv_t rv);
279 static void vm_reserv_depopulate(vm_reserv_t rv, int index);
281 static boolean_t vm_reserv_has_pindex(vm_reserv_t rv,
283 static void vm_reserv_populate(vm_reserv_t rv, int index);
284 static void vm_reserv_reclaim(vm_reserv_t rv);
297 vm_reserv_t rv;
305 rv = seg->first_reserv + (paddr >> VM_LEVEL_0_SHIFT) -
308 rv = &vm_reserv_array[paddr >> VM_LEVEL_0_SHIFT];
312 fullpop += rv->popcnt == VM_LEVEL_0_NPAGES;
314 rv++;
327 vm_reserv_t rv;
340 TAILQ_FOREACH(rv, &vm_rvd[domain].partpop, partpopq) {
341 if (rv == &vm_rvd[domain].marker)
344 unused_pages += VM_LEVEL_0_NPAGES - rv->popcnt;
361 vm_reserv_remove(vm_reserv_t rv)
365 vm_reserv_assert_locked(rv);
366 CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d",
367 __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq);
368 KASSERT(rv->object != NULL,
369 ("vm_reserv_remove: reserv %p is free", rv));
370 KASSERT(!rv->inpartpopq,
371 ("vm_reserv_remove: reserv %p's inpartpopq is TRUE", rv));
372 object = rv->object;
374 LIST_REMOVE(rv, objq);
375 rv->object = NULL;
383 vm_reserv_insert(vm_reserv_t rv, vm_object_t object, vm_pindex_t pindex)
386 vm_reserv_assert_locked(rv);
388 "%s: rv %p(%p) object %p new %p popcnt %d",
389 __FUNCTION__, rv, rv->pages, rv->object, object,
390 rv->popcnt);
391 KASSERT(rv->object == NULL,
392 ("vm_reserv_insert: reserv %p isn't free", rv));
393 KASSERT(rv->popcnt == 0,
394 ("vm_reserv_insert: reserv %p's popcnt is corrupted", rv));
395 KASSERT(!rv->inpartpopq,
396 ("vm_reserv_insert: reserv %p's inpartpopq is TRUE", rv));
397 KASSERT(bit_ntest(rv->popmap, 0, VM_LEVEL_0_NPAGES - 1, 0),
398 ("vm_reserv_insert: reserv %p's popmap is corrupted", rv));
400 rv->pindex = pindex;
401 rv->object = object;
402 rv->lasttick = ticks;
403 LIST_INSERT_HEAD(&object->rvq, rv, objq);
409 vm_reserv_is_sublevel_full(vm_reserv_t rv, int index)
417 return (((uint16_t *)rv->popmap)[index / 16] == UINT16_MAX);
420 return (((uint64_t *)rv->popmap)[index] == UINT64_MAX &&
421 ((uint64_t *)rv->popmap)[index + 1] == UINT64_MAX);
435 vm_reserv_depopulate(vm_reserv_t rv, int index)
439 vm_reserv_assert_locked(rv);
440 CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d",
441 __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq);
442 KASSERT(rv->object != NULL,
443 ("vm_reserv_depopulate: reserv %p is free", rv));
444 KASSERT(bit_test(rv->popmap, index),
445 ("vm_reserv_depopulate: reserv %p's popmap[%d] is clear", rv,
447 KASSERT(rv->popcnt > 0,
448 ("vm_reserv_depopulate: reserv %p's popcnt is corrupted", rv));
449 KASSERT(rv->domain < vm_ndomains,
451 rv, rv->domain));
452 if (rv->popcnt == VM_LEVEL_0_NPAGES) {
453 KASSERT(rv->pages->psind == VM_LEVEL_0_PSIND,
455 rv));
456 rv->pages->psind = VM_LEVEL_0_PSIND - 1;
459 if (vm_reserv_is_sublevel_full(rv, index))
460 rv->pages[rounddown2(index, VM_SUBLEVEL_0_NPAGES)].psind = 0;
462 bit_clear(rv->popmap, index);
463 rv->popcnt--;
464 if ((unsigned)(ticks - rv->lasttick) >= PARTPOPSLOP ||
465 rv->popcnt == 0) {
466 vm_reserv_domain_lock(rv->domain);
467 if (rv->inpartpopq) {
468 TAILQ_REMOVE(&vm_rvd[rv->domain].partpop, rv, partpopq);
469 rv->inpartpopq = FALSE;
471 if (rv->popcnt != 0) {
472 rv->inpartpopq = TRUE;
473 TAILQ_INSERT_TAIL(&vm_rvd[rv->domain].partpop, rv,
476 vm_reserv_domain_unlock(rv->domain);
477 rv->lasttick = ticks;
479 vmd = VM_DOMAIN(rv->domain);
480 if (rv->popcnt == 0) {
481 vm_reserv_remove(rv);
483 vm_phys_free_pages(rv->pages, rv->pages->pool, VM_LEVEL_0_ORDER);
514 vm_reserv_t rv;
523 rv = vm_reserv_from_page(mpred);
524 if (rv->object == object && vm_reserv_has_pindex(rv, pindex))
532 rv = vm_reserv_from_page(msucc);
533 if (rv->object == object && vm_reserv_has_pindex(rv, pindex))
536 rv = NULL;
541 return (rv);
549 vm_reserv_has_pindex(vm_reserv_t rv, vm_pindex_t pindex)
552 return (((pindex - rv->pindex) & ~(VM_LEVEL_0_NPAGES - 1)) == 0);
560 vm_reserv_populate(vm_reserv_t rv, int index)
563 vm_reserv_assert_locked(rv);
564 CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d",
565 __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq);
566 KASSERT(rv->object != NULL,
567 ("vm_reserv_populate: reserv %p is free", rv));
568 KASSERT(!bit_test(rv->popmap, index),
569 ("vm_reserv_populate: reserv %p's popmap[%d] is set", rv,
571 KASSERT(rv->popcnt < VM_LEVEL_0_NPAGES,
572 ("vm_reserv_populate: reserv %p is already full", rv));
573 KASSERT(rv->pages->psind >= 0 &&
574 rv->pages->psind < VM_LEVEL_0_PSIND,
575 ("vm_reserv_populate: reserv %p is already promoted", rv));
576 KASSERT(rv->domain < vm_ndomains,
578 rv, rv->domain));
579 bit_set(rv->popmap, index);
581 if (vm_reserv_is_sublevel_full(rv, index))
582 rv->pages[rounddown2(index, VM_SUBLEVEL_0_NPAGES)].psind = 1;
584 rv->popcnt++;
585 if ((unsigned)(ticks - rv->lasttick) < PARTPOPSLOP &&
586 rv->inpartpopq && rv->popcnt != VM_LEVEL_0_NPAGES)
588 rv->lasttick = ticks;
589 vm_reserv_domain_lock(rv->domain);
590 if (rv->inpartpopq) {
591 TAILQ_REMOVE(&vm_rvd[rv->domain].partpop, rv, partpopq);
592 rv->inpartpopq = FALSE;
594 if (rv->popcnt < VM_LEVEL_0_NPAGES) {
595 rv->inpartpopq = TRUE;
596 TAILQ_INSERT_TAIL(&vm_rvd[rv->domain].partpop, rv, partpopq);
598 KASSERT(rv->pages->psind == VM_LEVEL_0_PSIND - 1,
600 rv));
601 rv->pages->psind = VM_LEVEL_0_PSIND;
603 vm_reserv_domain_unlock(rv->domain);
630 vm_reserv_t rv;
662 rv = vm_reserv_from_object(object, pindex, mpred, &msucc);
663 if (rv != NULL) {
664 KASSERT(object != kernel_object || rv->domain == domain,
670 domain = rv->domain;
672 vm_reserv_lock(rv);
674 if (rv->object != object)
676 m = &rv->pages[index];
682 if (!bit_ntest(rv->popmap, index, index + npages - 1, 0))
687 vm_reserv_populate(rv, index + i);
688 vm_reserv_unlock(rv);
691 vm_reserv_unlock(rv);
710 if ((rv = vm_reserv_from_page(mpred))->object != object)
713 leftcap = rv->pindex + VM_LEVEL_0_NPAGES;
720 if ((rv = vm_reserv_from_page(msucc))->object != object)
723 rightcap = rv->pindex;
787 rv = vm_reserv_from_page(m);
788 KASSERT(rv->pages == m,
790 rv));
791 vm_reserv_lock(rv);
792 vm_reserv_insert(rv, object, first);
795 vm_reserv_populate(rv, index + i);
798 m_ret = &rv->pages[index];
801 vm_reserv_unlock(rv);
824 vm_reserv_t rv;
839 rv = vm_reserv_from_object(object, pindex, mpred, &msucc);
840 if (rv != NULL) {
841 KASSERT(object != kernel_object || rv->domain == domain,
843 domain = rv->domain;
846 m = &rv->pages[index];
847 vm_reserv_lock(rv);
849 if (rv->object != object ||
851 bit_test(rv->popmap, index)) {
858 vm_reserv_populate(rv, index);
860 vm_reserv_unlock(rv);
875 if ((rv = vm_reserv_from_page(mpred))->object != object)
878 leftcap = rv->pindex + VM_LEVEL_0_NPAGES;
885 if ((rv = vm_reserv_from_page(msucc))->object != object)
888 rightcap = rv->pindex;
922 rv = vm_reserv_from_page(m);
923 vm_reserv_lock(rv);
924 KASSERT(rv->pages == m,
925 ("vm_reserv_alloc_page: reserv %p's pages is corrupted", rv));
926 vm_reserv_insert(rv, object, first);
928 vm_reserv_populate(rv, index);
929 vm_reserv_unlock(rv);
931 return (&rv->pages[index]);
943 vm_reserv_break(vm_reserv_t rv)
948 vm_reserv_assert_locked(rv);
949 CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d",
950 __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq);
951 vm_reserv_remove(rv);
952 m = rv->pages;
954 for (; m < rv->pages + VM_LEVEL_0_NPAGES; m += VM_SUBLEVEL_0_NPAGES)
957 pos0 = bit_test(rv->popmap, 0) ? -1 : 0;
961 bit_ff_at(rv->popmap, pos + 1, VM_LEVEL_0_NPAGES,
971 vm_domain_free_lock(VM_DOMAIN(rv->domain));
972 vm_phys_enqueue_contig(&rv->pages[pos0], VM_FREEPOOL_DEFAULT,
974 vm_domain_free_unlock(VM_DOMAIN(rv->domain));
976 bit_nclear(rv->popmap, 0, VM_LEVEL_0_NPAGES - 1);
977 rv->popcnt = 0;
987 vm_reserv_t rv;
996 while ((rv = LIST_FIRST(&object->rvq)) != NULL) {
997 vm_reserv_lock(rv);
999 if (rv->object != object) {
1000 vm_reserv_unlock(rv);
1003 vm_reserv_domain_lock(rv->domain);
1004 if (rv->inpartpopq) {
1005 TAILQ_REMOVE(&vm_rvd[rv->domain].partpop, rv, partpopq);
1006 rv->inpartpopq = FALSE;
1008 vm_reserv_domain_unlock(rv->domain);
1009 vm_reserv_break(rv);
1010 vm_reserv_unlock(rv);
1021 vm_reserv_t rv;
1024 rv = vm_reserv_from_page(m);
1025 if (rv->object == NULL)
1027 vm_reserv_lock(rv);
1029 if (rv->object != NULL) {
1030 vm_reserv_depopulate(rv, m - rv->pages);
1034 vm_reserv_unlock(rv);
1050 struct vm_reserv *rv;
1075 rv = seg->first_reserv + (paddr >> VM_LEVEL_0_SHIFT) -
1079 rv->pages = PHYS_TO_VM_PAGE(paddr);
1080 rv->domain = seg->domain;
1081 mtx_init(&rv->lock, "vm reserv", NULL, MTX_DEF);
1083 rv++;
1112 vm_reserv_t rv;
1114 rv = vm_reserv_from_page(m);
1115 if (rv->object == NULL)
1117 return (!bit_test(rv->popmap, m - rv->pages));
1127 vm_reserv_t rv;
1134 rv = vm_reserv_from_page(m);
1135 if (rv->object == NULL)
1137 index = rounddown2(m - rv->pages, npages);
1138 return (bit_ntest(rv->popmap, index, index + npages - 1, 1));
1148 vm_reserv_t rv;
1150 rv = vm_reserv_from_page(m);
1152 return (rv->object != NULL ? 1 : -1);
1154 return (rv->object != NULL ? 0 : -1);
1165 vm_reserv_t rv;
1167 rv = vm_reserv_from_page(m);
1168 if (rv->popcnt == VM_LEVEL_0_NPAGES) {
1171 } else if (rv->pages != NULL &&
1172 vm_reserv_is_sublevel_full(rv, m - rv->pages)) {
1183 vm_reserv_dequeue(vm_reserv_t rv)
1186 vm_reserv_domain_assert_locked(rv->domain);
1187 vm_reserv_assert_locked(rv);
1188 CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d",
1189 __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq);
1190 KASSERT(rv->inpartpopq,
1191 ("vm_reserv_reclaim: reserv %p's inpartpopq is FALSE", rv));
1193 TAILQ_REMOVE(&vm_rvd[rv->domain].partpop, rv, partpopq);
1194 rv->inpartpopq = FALSE;
1202 vm_reserv_reclaim(vm_reserv_t rv)
1205 vm_reserv_assert_locked(rv);
1206 CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d",
1207 __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq);
1208 if (rv->inpartpopq) {
1209 vm_reserv_domain_lock(rv->domain);
1210 vm_reserv_dequeue(rv);
1211 vm_reserv_domain_unlock(rv->domain);
1213 vm_reserv_break(rv);
1225 vm_reserv_t rv;
1228 TAILQ_FOREACH(rv, &vm_rvd[domain].partpop, partpopq) {
1233 if (rv != &vm_rvd[domain].marker && vm_reserv_trylock(rv)) {
1234 vm_reserv_dequeue(rv);
1239 if (rv != NULL) {
1240 vm_reserv_reclaim(rv);
1241 vm_reserv_unlock(rv);
1254 vm_reserv_find_contig(vm_reserv_t rv, int npages, int lo,
1258 vm_reserv_assert_locked(rv);
1269 while (bit_ffc_area_at(rv->popmap, lo, hi, npages, &lo), lo != -1) {
1296 vm_reserv_t marker, rv, rvn;
1327 TAILQ_FOREACH_SAFE(rv, queue, partpopq, rvn) {
1328 pa = VM_PAGE_TO_PHYS(&rv->pages[0]);
1342 if (vm_reserv_trylock(rv) == 0) {
1343 TAILQ_INSERT_AFTER(queue, rv, marker, partpopq);
1345 vm_reserv_lock(rv);
1347 rv) {
1348 vm_reserv_unlock(rv);
1362 posn = vm_reserv_find_contig(rv, (int)npages, lo, hi,
1367 rv->popcnt += npages;
1368 bit_nset(rv->popmap, posn, posn + npages - 1);
1369 vm_reserv_reclaim(rv);
1370 vm_reserv_unlock(rv);
1371 m_ret = &rv->pages[posn];
1380 rvn = TAILQ_NEXT(rv, partpopq);
1381 vm_reserv_unlock(rv);
1397 vm_reserv_t rv;
1400 rv = vm_reserv_from_page(m);
1401 if (rv->object == old_object) {
1402 vm_reserv_lock(rv);
1404 "%s: rv %p object %p new %p popcnt %d inpartpop %d",
1405 __FUNCTION__, rv, rv->object, new_object, rv->popcnt,
1406 rv->inpartpopq);
1407 if (rv->object == old_object) {
1409 rv->object = NULL;
1410 LIST_REMOVE(rv, objq);
1413 rv->object = new_object;
1414 rv->pindex -= old_object_offset;
1415 LIST_INSERT_HEAD(&new_object->rvq, rv, objq);
1418 vm_reserv_unlock(rv);
1506 vm_reserv_t rv;
1509 rv = vm_reserv_from_page(m);
1510 if (rv->object == m->object) {
1511 if (rv->popcnt == VM_LEVEL_0_NPAGES)
1512 return (rv->pages);
1514 if (vm_reserv_is_sublevel_full(rv, m - rv->pages))
1515 return (rv->pages + rounddown2(m - rv->pages,