27  * A simple DMA pool losely based on dmapool.c. It has certain advantages
29  * - Pool collects resently freed pages for reuse (and hooks up to
77  * The pool structure. There are usually six pools:
85  * @type: Type of the pool
87  * used with irqsave/irqrestore variants because pool allocator maybe called
89  * @inuse_list: Pool of pages that are in use. The order is very important and
91  * @free_list: Pool of pages that are free to be used. No order requirements.
96  * @nfrees: Stats when pool is shrinking.
97  * @nrefills: Stats when the pool is grown.
99  * @name: Name of the pool.
136  * Limits for the pool. They are handled without locks because only place where
151  * @dev: The 'struct device' associated with the 'pool'
152  * @pool: The 'struct dma_pool' associated with the 'dev'
157 	struct dma_pool *pool;  member
165  * @options: Limits for the pool.
306 static int ttm_set_pages_caching(struct dma_pool *pool,  in ttm_set_pages_caching()  argument
311 	if (pool->type & IS_UC) {  in ttm_set_pages_caching()
315 			       pool->dev_name, cpages);  in ttm_set_pages_caching()
317 	if (pool->type & IS_WC) {  in ttm_set_pages_caching()
321 			       pool->dev_name, cpages);  in ttm_set_pages_caching()
326 static void __ttm_dma_free_page(struct dma_pool *pool, struct dma_page *d_page)  in __ttm_dma_free_page()  argument
329 	dma_free_coherent(pool->dev, pool->size, d_page->vaddr, dma);  in __ttm_dma_free_page()
334 static struct dma_page *__ttm_dma_alloc_page(struct dma_pool *pool)  in __ttm_dma_alloc_page()  argument
342 	d_page->vaddr = dma_alloc_coherent(pool->dev, pool->size,  in __ttm_dma_alloc_page()
344 					   pool->gfp_flags);  in __ttm_dma_alloc_page()
369 static void ttm_pool_update_free_locked(struct dma_pool *pool,  in ttm_pool_update_free_locked()  argument
372 	pool->npages_free -= freed_pages;  in ttm_pool_update_free_locked()
373 	pool->nfrees += freed_pages;  in ttm_pool_update_free_locked()
378 static void ttm_dma_pages_put(struct dma_pool *pool, struct list_head *d_pages,  in ttm_dma_pages_put()  argument
383 	/* Don't set WB on WB page pool. */  in ttm_dma_pages_put()
384 	if (npages && !(pool->type & IS_CACHED) &&  in ttm_dma_pages_put()
387 		       pool->dev_name, npages);  in ttm_dma_pages_put()
391 		__ttm_dma_free_page(pool, d_page);  in ttm_dma_pages_put()
395 static void ttm_dma_page_put(struct dma_pool *pool, struct dma_page *d_page)  in ttm_dma_page_put()  argument
397 	/* Don't set WB on WB page pool. */  in ttm_dma_page_put()
398 	if (!(pool->type & IS_CACHED) && set_pages_array_wb(&d_page->p, 1))  in ttm_dma_page_put()
400 		       pool->dev_name, 1);  in ttm_dma_page_put()
403 	__ttm_dma_free_page(pool, d_page);  in ttm_dma_page_put()
407  * Free pages from pool.
412  * @pool: to free the pages from
413  * @nr_free: If set to true will free all pages in pool
415 static unsigned ttm_dma_page_pool_free(struct dma_pool *pool, unsigned nr_free)  in ttm_dma_page_pool_free()  argument
429 			 pool->dev_name, pool->name, current->pid,  in ttm_dma_page_pool_free()
437 		pr_err("%s: Failed to allocate memory for pool free operation\n",  in ttm_dma_page_pool_free()
438 		       pool->dev_name);  in ttm_dma_page_pool_free()
443 	spin_lock_irqsave(&pool->lock, irq_flags);  in ttm_dma_page_pool_free()
446 	list_for_each_entry_safe_reverse(dma_p, tmp, &pool->free_list,  in ttm_dma_page_pool_free()
458 			ttm_pool_update_free_locked(pool, freed_pages);  in ttm_dma_page_pool_free()
461 			 * we unlock the pool to prevent stalling.  in ttm_dma_page_pool_free()
463 			spin_unlock_irqrestore(&pool->lock, irq_flags);  in ttm_dma_page_pool_free()
465 			ttm_dma_pages_put(pool, &d_pages, pages_to_free,  in ttm_dma_page_pool_free()
493 	/* remove range of pages from the pool */  in ttm_dma_page_pool_free()
495 		ttm_pool_update_free_locked(pool, freed_pages);  in ttm_dma_page_pool_free()
499 	spin_unlock_irqrestore(&pool->lock, irq_flags);  in ttm_dma_page_pool_free()
502 		ttm_dma_pages_put(pool, &d_pages, pages_to_free, freed_pages);  in ttm_dma_page_pool_free()
511 	struct dma_pool *pool;  in ttm_dma_free_pool()  local
520 		pool = p->pool;  in ttm_dma_free_pool()
521 		if (pool->type != type)  in ttm_dma_free_pool()
529 	list_for_each_entry_reverse(pool, &dev->dma_pools, pools) {  in ttm_dma_free_pool()
530 		if (pool->type != type)  in ttm_dma_free_pool()
533 		ttm_dma_page_pool_free(pool, FREE_ALL_PAGES);  in ttm_dma_free_pool()
534 		WARN_ON(((pool->npages_in_use + pool->npages_free) != 0));  in ttm_dma_free_pool()
539 		list_del(&pool->pools);  in ttm_dma_free_pool()
540 		kfree(pool);  in ttm_dma_free_pool()
548  * Albeit the pool might have already been freed earlier.
552 	struct dma_pool *pool = *(struct dma_pool **)res;  in ttm_dma_pool_release()  local
554 	if (pool)  in ttm_dma_pool_release()
555 		ttm_dma_free_pool(dev, pool->type);  in ttm_dma_pool_release()
569 	struct dma_pool *pool = NULL, **ptr;  in ttm_dma_pool_init()  local
583 	pool = kmalloc_node(sizeof(struct dma_pool), GFP_KERNEL,  in ttm_dma_pool_init()
585 	if (!pool)  in ttm_dma_pool_init()
595 	sec_pool->pool =  pool;  in ttm_dma_pool_init()
597 	INIT_LIST_HEAD(&pool->free_list);  in ttm_dma_pool_init()
598 	INIT_LIST_HEAD(&pool->inuse_list);  in ttm_dma_pool_init()
599 	INIT_LIST_HEAD(&pool->pools);  in ttm_dma_pool_init()
600 	spin_lock_init(&pool->lock);  in ttm_dma_pool_init()
601 	pool->dev = dev;  in ttm_dma_pool_init()
602 	pool->npages_free = pool->npages_in_use = 0;  in ttm_dma_pool_init()
603 	pool->nfrees = 0;  in ttm_dma_pool_init()
604 	pool->gfp_flags = flags;  in ttm_dma_pool_init()
605 	pool->size = PAGE_SIZE;  in ttm_dma_pool_init()
606 	pool->type = type;  in ttm_dma_pool_init()
607 	pool->nrefills = 0;  in ttm_dma_pool_init()
608 	p = pool->name;  in ttm_dma_pool_init()
611 			p += snprintf(p, sizeof(pool->name) - (p - pool->name),  in ttm_dma_pool_init()
618 	snprintf(pool->dev_name, sizeof(pool->dev_name), "%s %s",  in ttm_dma_pool_init()
625 	list_add(&pool->pools, &dev->dma_pools);  in ttm_dma_pool_init()
628 	*ptr = pool;  in ttm_dma_pool_init()
631 	return pool;  in ttm_dma_pool_init()
635 	kfree(pool);  in ttm_dma_pool_init()
642 	struct dma_pool *pool, *tmp, *found = NULL;  in ttm_dma_find_pool()  local
658 	list_for_each_entry_safe(pool, tmp, &dev->dma_pools, pools) {  in ttm_dma_find_pool()
659 		if (pool->type != type)  in ttm_dma_find_pool()
661 		found = pool;  in ttm_dma_find_pool()
670  * pool.
672 static void ttm_dma_handle_caching_state_failure(struct dma_pool *pool,  in ttm_dma_handle_caching_state_failure()  argument
690 		__ttm_dma_free_page(pool, d_page);  in ttm_dma_handle_caching_state_failure()
705 static int ttm_dma_pool_alloc_new_pages(struct dma_pool *pool,  in ttm_dma_pool_alloc_new_pages()  argument
722 		       pool->dev_name);  in ttm_dma_pool_alloc_new_pages()
728 			 pool->dev_name, pool->name, current->pid, count);  in ttm_dma_pool_alloc_new_pages()
732 		dma_p = __ttm_dma_alloc_page(pool);  in ttm_dma_pool_alloc_new_pages()
735 			       pool->dev_name, i);  in ttm_dma_pool_alloc_new_pages()
737 			/* store already allocated pages in the pool after  in ttm_dma_pool_alloc_new_pages()
740 				r = ttm_set_pages_caching(pool, caching_array,  in ttm_dma_pool_alloc_new_pages()
744 						pool, d_pages, caching_array,  in ttm_dma_pool_alloc_new_pages()
761 				r = ttm_set_pages_caching(pool, caching_array,  in ttm_dma_pool_alloc_new_pages()
765 						pool, d_pages, caching_array,  in ttm_dma_pool_alloc_new_pages()
776 		r = ttm_set_pages_caching(pool, caching_array, cpages);  in ttm_dma_pool_alloc_new_pages()
778 			ttm_dma_handle_caching_state_failure(pool, d_pages,  in ttm_dma_pool_alloc_new_pages()
789 static int ttm_dma_page_pool_fill_locked(struct dma_pool *pool,  in ttm_dma_page_pool_fill_locked()  argument
793 	int r = pool->npages_free;  in ttm_dma_page_pool_fill_locked()
795 	if (count > pool->npages_free) {  in ttm_dma_page_pool_fill_locked()
800 		spin_unlock_irqrestore(&pool->lock, *irq_flags);  in ttm_dma_page_pool_fill_locked()
804 		r = ttm_dma_pool_alloc_new_pages(pool, &d_pages, count);  in ttm_dma_page_pool_fill_locked()
806 		spin_lock_irqsave(&pool->lock, *irq_flags);  in ttm_dma_page_pool_fill_locked()
809 			list_splice(&d_pages, &pool->free_list);  in ttm_dma_page_pool_fill_locked()
810 			++pool->nrefills;  in ttm_dma_page_pool_fill_locked()
811 			pool->npages_free += count;  in ttm_dma_page_pool_fill_locked()
817 			pr_err("%s: Failed to fill %s pool (r:%d)!\n",  in ttm_dma_page_pool_fill_locked()
818 			       pool->dev_name, pool->name, r);  in ttm_dma_page_pool_fill_locked()
823 			list_splice_tail(&d_pages, &pool->free_list);  in ttm_dma_page_pool_fill_locked()
824 			pool->npages_free += cpages;  in ttm_dma_page_pool_fill_locked()
836 static int ttm_dma_pool_get_pages(struct dma_pool *pool,  in ttm_dma_pool_get_pages()  argument
845 	spin_lock_irqsave(&pool->lock, irq_flags);  in ttm_dma_pool_get_pages()
846 	count = ttm_dma_page_pool_fill_locked(pool, &irq_flags);  in ttm_dma_pool_get_pages()
848 		d_page = list_first_entry(&pool->free_list, struct dma_page, page_list);  in ttm_dma_pool_get_pages()
853 		pool->npages_in_use += 1;  in ttm_dma_pool_get_pages()
854 		pool->npages_free -= 1;  in ttm_dma_pool_get_pages()
856 	spin_unlock_irqrestore(&pool->lock, irq_flags);  in ttm_dma_pool_get_pages()
868 	struct dma_pool *pool;  in ttm_dma_populate()  local
885 	pool = ttm_dma_find_pool(dev, type);  in ttm_dma_populate()
886 	if (!pool) {  in ttm_dma_populate()
887 		pool = ttm_dma_pool_init(dev, gfp_flags, type);  in ttm_dma_populate()
888 		if (IS_ERR_OR_NULL(pool)) {  in ttm_dma_populate()
895 		ret = ttm_dma_pool_get_pages(pool, ttm_dma, i);  in ttm_dma_populate()
930 		total += p->pool->npages_free;  in ttm_dma_pool_get_num_unused_pages()
935 /* Put all pages in pages list to correct pool to wait for reuse */
939 	struct dma_pool *pool;  in ttm_dma_unpopulate()  local
947 	pool = ttm_dma_find_pool(dev, type);  in ttm_dma_unpopulate()
948 	if (!pool)  in ttm_dma_unpopulate()
951 	is_cached = (ttm_dma_find_pool(pool->dev,  in ttm_dma_unpopulate()
952 		     ttm_to_type(ttm->page_flags, tt_cached)) == pool);  in ttm_dma_unpopulate()
960 	spin_lock_irqsave(&pool->lock, irq_flags);  in ttm_dma_unpopulate()
961 	pool->npages_in_use -= count;  in ttm_dma_unpopulate()
963 		pool->nfrees += count;  in ttm_dma_unpopulate()
965 		pool->npages_free += count;  in ttm_dma_unpopulate()
966 		list_splice(&ttm_dma->pages_list, &pool->free_list);  in ttm_dma_unpopulate()
968 		if (pool->npages_free > _manager->options.max_size) {  in ttm_dma_unpopulate()
969 			npages = pool->npages_free - _manager->options.max_size;  in ttm_dma_unpopulate()
976 	spin_unlock_irqrestore(&pool->lock, irq_flags);  in ttm_dma_unpopulate()
982 			ttm_dma_page_put(pool, d_page);  in ttm_dma_unpopulate()
997 	/* shrink pool if necessary (only on !is_cached pools)*/  in ttm_dma_unpopulate()
999 		ttm_dma_page_pool_free(pool, npages);  in ttm_dma_unpopulate()
1005  * Callback for mm to request pool to reduce number of page held.
1032 		shrink_pages = ttm_dma_page_pool_free(p->pool, nr_free);  in ttm_dma_pool_mm_shrink()
1034 			 p->pool->dev_name, p->pool->name, current->pid,  in ttm_dma_pool_mm_shrink()
1038 	/* return estimated number of unused pages in pool */  in ttm_dma_pool_mm_shrink()
1060 	pr_info("Initializing DMA pool allocator\n");  in ttm_dma_page_alloc_init()
1090 	pr_info("Finalizing DMA pool allocator\n");  in ttm_dma_page_alloc_fini()
1094 		dev_dbg(p->dev, "(%s:%d) Freeing.\n", p->pool->name,  in ttm_dma_page_alloc_fini()
1097 			ttm_dma_pool_match, p->pool));  in ttm_dma_page_alloc_fini()
1098 		ttm_dma_free_pool(p->dev, p->pool->type);  in ttm_dma_page_alloc_fini()
1107 	struct dma_pool *pool = NULL;  in ttm_dma_page_alloc_debugfs()  local
1108 	char *h[] = {"pool", "refills", "pages freed", "inuse", "available",  in ttm_dma_page_alloc_debugfs()
1112 		seq_printf(m, "No pool allocator running.\n");  in ttm_dma_page_alloc_debugfs()
1122 		pool = p->pool;  in ttm_dma_page_alloc_debugfs()
1124 				pool->name, pool->nrefills,  in ttm_dma_page_alloc_debugfs()
1125 				pool->nfrees, pool->npages_in_use,  in ttm_dma_page_alloc_debugfs()
1126 				pool->npages_free,  in ttm_dma_page_alloc_debugfs()
1127 				pool->dev_name);  in ttm_dma_page_alloc_debugfs()