Lines Matching defs:ph
408 pr_item_bitmap_index(const struct pool *pp, const struct pool_item_header *ph,
415 idx = (cp - (char *)ph->ph_page - ph->ph_off) / pp->pr_size;
426 pr_item_bitmap_put(const struct pool *pp, struct pool_item_header *ph,
429 unsigned int idx = pr_item_bitmap_index(pp, ph, obj);
430 pool_item_bitmap_t *bitmap = ph->ph_bitmap + (idx / BITMAP_SIZE);
441 pr_item_bitmap_get(const struct pool *pp, struct pool_item_header *ph)
443 pool_item_bitmap_t *bitmap = ph->ph_bitmap;
464 return (char *)ph->ph_page + ph->ph_off + idx * pp->pr_size;
468 pr_item_bitmap_init(const struct pool *pp, struct pool_item_header *ph)
470 pool_item_bitmap_t *bitmap = ph->ph_bitmap;
482 pr_item_linkedlist_put(const struct pool *pp, struct pool_item_header *ph,
501 LIST_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list);
505 pr_item_linkedlist_get(struct pool *pp, struct pool_item_header *ph)
510 v = pi = LIST_FIRST(&ph->ph_itemlist);
522 pp->pr_wchan, pi->pi_magic, ph->ph_page, pi);
536 pr_phinpage_check(struct pool *pp, struct pool_item_header *ph, void *page,
539 if (__predict_false((void *)ph->ph_page != page)) {
543 if (__predict_false((char *)object < (char *)page + ph->ph_off)) {
547 if (__predict_false(ph->ph_poolid != pp->pr_poolid)) {
549 pp->pr_wchan, object, ph->ph_poolid, pp->pr_poolid);
556 struct pool_item_header *ph;
562 ph = (struct pool_item_header *)page;
564 pr_phinpage_check(pp, ph, page, object);
591 struct pool_item_header *ph, tmp;
594 ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp);
595 if (ph == NULL) {
596 ph = SPLAY_ROOT(&pp->pr_phtree);
597 if (ph != NULL && phtree_compare(&tmp, ph) >= 0) {
598 ph = SPLAY_NEXT(phtree, &pp->pr_phtree, ph);
600 KASSERT(ph == NULL || phtree_compare(&tmp, ph) < 0);
603 return ph;
612 struct pool_item_header *ph, tmp;
615 ph = pr_find_pagehead_noalign(pp, v);
619 ph = (struct pool_item_header *)page;
620 pr_phinpage_check(pp, ph, page, v);
623 ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp);
627 KASSERT(ph == NULL || ((pp->pr_roflags & PR_PHINPAGE) != 0) ||
628 ((char *)ph->ph_page <= (char *)v &&
629 (char *)v < (char *)ph->ph_page + pp->pr_alloc->pa_pagesz));
630 return ph;
636 struct pool_item_header *ph;
638 while ((ph = LIST_FIRST(pq)) != NULL) {
639 LIST_REMOVE(ph, ph_pagelist);
640 pool_allocator_free(pp, ph->ph_page);
642 pool_put(pp->pr_phpool, ph);
650 pr_rmpage(struct pool *pp, struct pool_item_header *ph,
659 if (ph->ph_nmissing == 0) {
672 LIST_REMOVE(ph, ph_pagelist);
674 if (__predict_false(ph->ph_poolid != pp->pr_poolid)) {
675 panic("%s: [%s] ph %p poolid %u != %u",
676 __func__, pp->pr_wchan, ph, ph->ph_poolid,
680 SPLAY_REMOVE(phtree, &pp->pr_phtree, ph);
682 LIST_INSERT_HEAD(pq, ph, ph_pagelist);
1024 struct pool_item_header *ph;
1058 while ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL)
1059 pr_rmpage(pp, ph, &pq);
1082 struct pool_item_header *ph;
1085 ph = storage;
1087 ph = pool_get(pp->pr_phpool, flags);
1089 return ph;
1098 struct pool_item_header *ph;
1174 if ((ph = pp->pr_curpage) == NULL) {
1215 KASSERTMSG((ph->ph_nmissing < pp->pr_itemsperpage),
1217 v = pr_item_bitmap_get(pp, ph);
1219 v = pr_item_linkedlist_get(pp, ph);
1223 if (ph->ph_nmissing == 0) {
1231 LIST_REMOVE(ph, ph_pagelist);
1232 LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist);
1234 ph->ph_nmissing++;
1235 if (ph->ph_nmissing == pp->pr_itemsperpage) {
1237 LIST_EMPTY(&ph->ph_itemlist)),
1239 pp->pr_wchan, ph->ph_nmissing);
1244 LIST_REMOVE(ph, ph_pagelist);
1245 LIST_INSERT_HEAD(&pp->pr_fullpages, ph, ph_pagelist);
1279 struct pool_item_header *ph;
1290 if (__predict_false((ph = pr_find_pagehead(pp, v)) == NULL)) {
1298 pr_item_bitmap_put(pp, ph, v);
1300 pr_item_linkedlist_put(pp, ph, v);
1302 KDASSERT(ph->ph_nmissing != 0);
1303 ph->ph_nmissing--;
1310 pp->pr_curpage = ph;
1330 if (ph->ph_nmissing == 0) {
1336 pr_rmpage(pp, ph, pq);
1338 LIST_REMOVE(ph, ph_pagelist);
1339 LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist);
1350 ph->ph_time = time_uptime;
1361 else if (ph->ph_nmissing == (pp->pr_itemsperpage - 1)) {
1362 LIST_REMOVE(ph, ph_pagelist);
1363 LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist);
1364 pp->pr_curpage = ph;
1395 struct pool_item_header *ph;
1433 ph = pool_alloc_item_header(pp, storage, flags);
1434 if (__predict_false(ph == NULL)) {
1441 pool_prime_page(pp, storage, ph);
1478 pool_prime_page(struct pool *pp, void *storage, struct pool_item_header *ph)
1493 LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist);
1494 LIST_INIT(&ph->ph_itemlist);
1495 ph->ph_page = storage;
1496 ph->ph_nmissing = 0;
1497 ph->ph_time = time_uptime;
1499 ph->ph_poolid = pp->pr_poolid;
1501 SPLAY_INSERT(phtree, &pp->pr_phtree, ph);
1508 ph->ph_off = pp->pr_itemoffset;
1513 ph->ph_off += pp->pr_curcolor;
1514 cp = (char *)cp + ph->ph_off;
1527 pr_item_bitmap_init(pp, ph);
1535 LIST_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list);
1549 pp->pr_curpage = ph;
1661 struct pool_item_header *ph, *phnext;
1704 for (ph = LIST_FIRST(&pp->pr_emptypages); ph != NULL; ph = phnext) {
1705 phnext = LIST_NEXT(ph, ph_pagelist);
1711 KASSERT(ph->ph_nmissing == 0);
1712 if (curtime - ph->ph_time < pool_inactive_time)
1723 pr_rmpage(pp, ph, &pq);
1852 struct pool_item_header *ph;
1854 LIST_FOREACH(ph, pl, ph_pagelist) {
1856 ph->ph_page, ph->ph_nmissing, ph->ph_time);
1860 LIST_FOREACH(pi, &ph->ph_itemlist, pi_list) {
1874 struct pool_item_header *ph;
1935 if ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL)
1938 if ((ph = LIST_FIRST(&pp->pr_fullpages)) != NULL)
1941 if ((ph = LIST_FIRST(&pp->pr_partpages)) != NULL)
2002 pool_chk_page(struct pool *pp, const char *label, struct pool_item_header *ph)
2009 page = POOL_OBJ_TO_PAGE(pp, ph);
2010 if (page != ph->ph_page &&
2016 pp->pr_wchan, ph->ph_page,
2017 ph, page);
2025 for (pi = LIST_FIRST(&ph->ph_itemlist), n = 0;
2035 pp->pr_wchan, pi->pi_magic, ph->ph_page,
2044 if (page == ph->ph_page)
2051 pp->pr_wchan, ph->ph_page,
2062 struct pool_item_header *ph;
2066 LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) {
2067 r = pool_chk_page(pp, label, ph);
2072 LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) {
2073 r = pool_chk_page(pp, label, ph);
2078 LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) {
2079 r = pool_chk_page(pp, label, ph);
3289 pool_in_page(struct pool *pp, struct pool_item_header *ph, uintptr_t addr)
3292 return (uintptr_t)ph->ph_page <= addr &&
3293 addr < (uintptr_t)ph->ph_page + pp->pr_alloc->pa_pagesz;
3320 pool_allocated(struct pool *pp, struct pool_item_header *ph, uintptr_t addr)
3324 unsigned int idx = pr_item_bitmap_index(pp, ph, (void *)addr);
3326 ph->ph_bitmap + (idx / BITMAP_SIZE);
3333 LIST_FOREACH(pi, &ph->ph_itemlist, pi_list) {
3348 struct pool_item_header *ph;
3357 LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) {
3358 if (pool_in_page(pp, ph, addr)) {
3362 LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) {
3363 if (pool_in_page(pp, ph, addr)) {
3365 pool_allocated(pp, ph, addr);
3369 LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) {
3370 if (pool_in_page(pp, ph, addr)) {
3377 ph = pr_find_pagehead_noalign(pp, (void *)addr);
3378 if (ph == NULL || !pool_in_page(pp, ph, addr)) {
3381 allocated = pool_allocated(pp, ph, addr);
3417 item = (uintptr_t)ph->ph_page + ph->ph_off;