91 #define	UAO_SWHASH_HASH(aobj, idx) \  argument
92     (&(aobj)->u_swhash[(((idx) >> UAO_SWHASH_CLUSTER_SHIFT) \
93     & (aobj)->u_swhashmask)])
100 #define	UAO_USES_SWHASH(aobj) \  argument
101     ((aobj)->u_pages > UAO_SWHASH_THRESHOLD)
105 #define	UAO_SWHASH_BUCKETS(aobj) \  argument
106     (MIN((aobj)->u_pages >> UAO_SWHASH_CLUSTER_SHIFT, UAO_SWHASH_MAXBUCKETS))
204 uao_find_swhash_elt(struct uvm_aobj *aobj, int pageidx, bool create)
210 	swhash = UAO_SWHASH_HASH(aobj, pageidx);
250 	struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;  in uao_find_swslot()  local
259 	if (aobj->u_flags & UAO_FLAG_NOSWAP)  in uao_find_swslot()
266 	if (UAO_USES_SWHASH(aobj)) {  in uao_find_swslot()
267 		elt = uao_find_swhash_elt(aobj, pageidx, false);  in uao_find_swslot()
275 	return aobj->u_swslots[pageidx];  in uao_find_swslot()
290 	struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;  in uao_set_swslot()  local
295 	    (uintptr_t)aobj, pageidx, slot, 0);  in uao_set_swslot()
304 	if (aobj->u_flags & UAO_FLAG_NOSWAP) {  in uao_set_swslot()
313 	if (UAO_USES_SWHASH(aobj)) {  in uao_set_swslot()
321 		elt = uao_find_swhash_elt(aobj, pageidx, slot != 0);  in uao_set_swslot()
348 		oldslot = aobj->u_swslots[pageidx];  in uao_set_swslot()
349 		aobj->u_swslots[pageidx] = slot;  in uao_set_swslot()
367 uao_free(struct uvm_aobj *aobj)  in uao_free()  argument
369 	struct uvm_object *uobj = &aobj->u_obj;  in uao_free()
377 	if (UAO_USES_SWHASH(aobj)) {  in uao_free()
383 		hashdone(aobj->u_swhash, HASH_LIST, aobj->u_swhashmask);  in uao_free()
390 		kmem_free(aobj->u_swslots, aobj->u_pages * sizeof(int));  in uao_free()
399 	kmem_free(aobj, sizeof(struct uvm_aobj));  in uao_free()
422 	struct uvm_aobj *aobj;  in uao_create()  local
431 		aobj = &kernel_object_store;  in uao_create()
432 		aobj->u_pages = pages;  in uao_create()
433 		aobj->u_flags = UAO_FLAG_NOSWAP;  in uao_create()
438 		aobj = &kernel_object_store;  in uao_create()
442 		aobj = kmem_alloc(sizeof(struct uvm_aobj), KM_SLEEP);  in uao_create()
443 		aobj->u_pages = pages;  in uao_create()
444 		aobj->u_flags = 0;  in uao_create()
452 	aobj->u_freelist = VM_NFREELIST;  in uao_create()
466 		if (UAO_USES_SWHASH(aobj)) {  in uao_create()
467 			aobj->u_swhash = hashinit(UAO_SWHASH_BUCKETS(aobj),  in uao_create()
468 			    HASH_LIST, true, &aobj->u_swhashmask);  in uao_create()
470 			aobj->u_swslots = kmem_zalloc(pages * sizeof(int),  in uao_create()
482 			aobj->u_obj.vmobjlock = rw_obj_alloc();  in uao_create()
486 			aobj->u_flags &= ~UAO_FLAG_NOSWAP; /* clear noswap */  in uao_create()
487 			return &aobj->u_obj;  in uao_create()
496 	uvm_obj_init(&aobj->u_obj, &aobj_pager, !kernobj, refs);  in uao_create()
500 		uvm_obj_setlock(&aobj->u_obj, &bootstrap_kernel_object_lock);  in uao_create()
508 	LIST_INSERT_HEAD(&uao_list, aobj, u_list);  in uao_create()
510 	return(&aobj->u_obj);  in uao_create()
523 	struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;  in uao_set_pgfl()  local
529 	aobj->u_freelist = freelist;  in uao_set_pgfl()
539 	struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;  in uao_pagealloc()  local
541 	if (__predict_true(aobj->u_freelist == VM_NFREELIST))  in uao_pagealloc()
545 		    UVM_PGA_STRAT_ONLY, aobj->u_freelist);  in uao_pagealloc()
587 	struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;  in uao_detach()  local
619 	LIST_REMOVE(aobj, u_list);  in uao_detach()
639 		uao_dropswap(&aobj->u_obj, pg->offset >> PAGE_SHIFT);  in uao_detach()
648 	uao_free(aobj);  in uao_detach()
672 	struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;  in uao_put()  local
683 		stop = aobj->u_pages << PAGE_SHIFT;  in uao_put()
687 			stop = aobj->u_pages << PAGE_SHIFT;  in uao_put()
691 		if (stop > (uint64_t)(aobj->u_pages << PAGE_SHIFT)) {  in uao_put()
695 			    (uintmax_t)(aobj->u_pages << PAGE_SHIFT));  in uao_put()
696 			stop = aobj->u_pages << PAGE_SHIFT;  in uao_put()
1106 	struct uvm_aobj *aobj;  in uao_swap_off()  local
1112 	if ((aobj = LIST_FIRST(&uao_list)) == NULL) {  in uao_swap_off()
1116 	uao_reference(&aobj->u_obj);  in uao_swap_off()
1127 		if ((nextaobj = LIST_NEXT(aobj, u_list)) != NULL) {  in uao_swap_off()
1135 		rw_enter(aobj->u_obj.vmobjlock, RW_WRITER);  in uao_swap_off()
1136 		rv = uao_pagein(aobj, startslot, endslot);  in uao_swap_off()
1137 		rw_exit(aobj->u_obj.vmobjlock);  in uao_swap_off()
1140 		uao_detach(&aobj->u_obj);  in uao_swap_off()
1148 		aobj = nextaobj;  in uao_swap_off()
1150 	} while (aobj);  in uao_swap_off()
1163 uao_pagein(struct uvm_aobj *aobj, int startslot, int endslot)  in uao_pagein()  argument
1167 	if (UAO_USES_SWHASH(aobj)) {  in uao_pagein()
1172 		for (buck = aobj->u_swhashmask; buck >= 0; buck--) {  in uao_pagein()
1173 			for (elt = LIST_FIRST(&aobj->u_swhash[buck]);  in uao_pagein()
1197 					rv = uao_pagein_page(aobj,  in uao_pagein()
1209 		for (i = 0; i < aobj->u_pages; i++) {  in uao_pagein()
1210 			int slot = aobj->u_swslots[i];  in uao_pagein()
1224 			rv = uao_pagein_page(aobj, i);  in uao_pagein()
1242 uao_pagein_page(struct uvm_aobj *aobj, int pageidx)  in uao_pagein_page()  argument
1244 	struct uvm_object *uobj = &aobj->u_obj;  in uao_pagein_page()
1283 	uao_dropswap(&aobj->u_obj, pageidx);  in uao_pagein_page()
1310 	struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;  in uao_dropswap_range()  local
1320 	if (UAO_USES_SWHASH(aobj)) {  in uao_dropswap_range()
1321 		int i, hashbuckets = aobj->u_swhashmask + 1;  in uao_dropswap_range()
1331 			for (elt = LIST_FIRST(&aobj->u_swhash[i]);  in uao_dropswap_range()
1360 					KASSERT(uvm_pagelookup(&aobj->u_obj,  in uao_dropswap_range()
1381 		if (aobj->u_pages < end) {  in uao_dropswap_range()
1382 			end = aobj->u_pages;  in uao_dropswap_range()
1385 			int slot = aobj->u_swslots[i];  in uao_dropswap_range()