diff options
author | Shrenuj Bansal <shrenujb@codeaurora.org> | 2016-04-29 09:28:02 -0600 |
---|---|---|
committer | Jeevan Shriram <jshriram@codeaurora.org> | 2016-05-11 17:44:06 -0700 |
commit | 8cdea23e0363e5edc074b5031cbd94e772efd9c9 (patch) | |
tree | 4b2f30836ad93a8b5755f7454b830ecccc5f125a | |
parent | 01da6d8db6880db3ffe2e7c7c720c214ba18136f (diff) |
msm: kgsl: Add 1M and 8K pools to the allocator
This change includes the below:
- Add 1M and 8k pools and structure the allocator to use all pools
from the largest page to the smallest
- Reserve a set number of pages for each of these pools at init time
- When allocating, use the reserved pools and then fall back to
allocating from system memory using only 8k and 4k pages
- Remove maximums on the pool sizes
- Zero the memory when we create the pool initially and add pages
back to the pool on free
CRs-Fixed: 995735
Change-Id: I9440bad62d3e13b434902f167c9d23467b1c4235
Signed-off-by: Shrenuj Bansal <shrenujb@codeaurora.org>
-rw-r--r-- | drivers/gpu/msm/kgsl_pool.c | 144 | ||||
-rw-r--r-- | drivers/gpu/msm/kgsl_pool.h | 4 | ||||
-rw-r--r-- | drivers/gpu/msm/kgsl_sharedmem.c | 85 |
3 files changed, 137 insertions, 96 deletions
diff --git a/drivers/gpu/msm/kgsl_pool.c b/drivers/gpu/msm/kgsl_pool.c index 497cfdaae2ec..48b7b0872425 100644 --- a/drivers/gpu/msm/kgsl_pool.c +++ b/drivers/gpu/msm/kgsl_pool.c @@ -21,18 +21,21 @@ #include "kgsl_device.h" #include "kgsl_pool.h" -/* - * Maximum pool size in terms of pages - * = (Number of pools * Max size per pool) +/** + * struct kgsl_page_pool - Structure to hold information for the pool + * @pool_order: Page order describing the size of the page + * @page_count: Number of pages currently present in the pool + * @reserved_pages: Number of pages reserved at init for the pool + * @allocation_allowed: Tells if reserved pool gets exhausted, can we allocate + * from system memory + * @list_lock: Spinlock for page list in the pool + * @page_list: List of pages held/reserved in this pool */ -#define KGSL_POOL_MAX_PAGES (2 * 4096) - -/* Set the max pool size to 8192 pages */ -static unsigned int kgsl_pool_max_pages = KGSL_POOL_MAX_PAGES; - struct kgsl_page_pool { unsigned int pool_order; int page_count; + unsigned int reserved_pages; + bool allocation_allowed; spinlock_t list_lock; struct list_head page_list; }; @@ -40,15 +43,34 @@ struct kgsl_page_pool { static struct kgsl_page_pool kgsl_pools[] = { { .pool_order = 0, + .reserved_pages = 2048, + .allocation_allowed = true, .list_lock = __SPIN_LOCK_UNLOCKED(kgsl_pools[0].list_lock), .page_list = LIST_HEAD_INIT(kgsl_pools[0].page_list), }, #ifndef CONFIG_ALLOC_BUFFERS_IN_4K_CHUNKS { - .pool_order = 4, + .pool_order = 1, + .reserved_pages = 1024, + .allocation_allowed = true, .list_lock = __SPIN_LOCK_UNLOCKED(kgsl_pools[1].list_lock), .page_list = LIST_HEAD_INIT(kgsl_pools[1].page_list), }, + { + .pool_order = 4, + .reserved_pages = 256, + .allocation_allowed = false, + .list_lock = __SPIN_LOCK_UNLOCKED(kgsl_pools[2].list_lock), + .page_list = LIST_HEAD_INIT(kgsl_pools[2].page_list), + }, + { + .pool_order = 8, + .reserved_pages = 32, + .allocation_allowed = false, + .list_lock = __SPIN_LOCK_UNLOCKED(kgsl_pools[3].list_lock), + .page_list = LIST_HEAD_INIT(kgsl_pools[3].page_list), + }, + #endif }; @@ -68,10 +90,28 @@ _kgsl_get_pool_from_order(unsigned int order) return NULL; } +/* Map the page into kernel and zero it out */ +static void +_kgsl_pool_zero_page(struct page *p, unsigned int pool_order) +{ + int i; + + for (i = 0; i < (1 << pool_order); i++) { + struct page *page = nth_page(p, i); + void *addr = kmap_atomic(page); + + memset(addr, 0, PAGE_SIZE); + dmac_flush_range(addr, addr + PAGE_SIZE); + kunmap_atomic(addr); + } +} + /* Add a page to specified pool */ static void _kgsl_pool_add_page(struct kgsl_page_pool *pool, struct page *p) { + _kgsl_pool_zero_page(p, pool->pool_order); + spin_lock(&pool->list_lock); list_add_tail(&p->lru, &pool->page_list); pool->page_count++; @@ -156,14 +196,14 @@ _kgsl_pool_shrink(struct kgsl_page_pool *pool, int num_pages) * (current_pool_size - target_pages) pages from pool * starting from higher order pool. */ -static int +static unsigned long kgsl_pool_reduce(unsigned int target_pages) { int total_pages = 0; int i; int nr_removed; struct kgsl_page_pool *pool; - unsigned int pcount = 0; + unsigned long pcount = 0; total_pages = kgsl_pool_size_total(); @@ -223,6 +263,17 @@ void kgsl_pool_free_sgt(struct sg_table *sgt) } } +static int kgsl_pool_idx_lookup(unsigned int order) +{ + int i; + + for (i = 0; i < KGSL_NUM_POOLS; i++) + if (order == kgsl_pools[i].pool_order) + return i; + + return -ENOMEM; +} + /** * kgsl_pool_alloc_page() - Allocate a page of requested size * @page_size: Size of the page to be allocated @@ -232,35 +283,55 @@ void kgsl_pool_free_sgt(struct sg_table *sgt) * * Return total page count on success and negative value on failure */ -int kgsl_pool_alloc_page(int page_size, struct page **pages, - unsigned int pages_len) +int kgsl_pool_alloc_page(int *page_size, struct page **pages, + unsigned int pages_len, unsigned int *align) { int j; int pcount = 0; struct kgsl_page_pool *pool; struct page *page = NULL; struct page *p = NULL; + int order = get_order(*page_size); + int pool_idx; - if ((pages == NULL) || pages_len < (page_size >> PAGE_SHIFT)) + if ((pages == NULL) || pages_len < (*page_size >> PAGE_SHIFT)) return -EINVAL; - pool = _kgsl_get_pool_from_order(get_order(page_size)); + pool = _kgsl_get_pool_from_order(order); + pool_idx = kgsl_pool_idx_lookup(order); if (pool != NULL) page = _kgsl_pool_get_page(pool); /* Allocate a new page if not allocated from pool */ if (page == NULL) { - gfp_t gfp_mask = kgsl_gfp_mask(get_order(page_size)); + gfp_t gfp_mask = kgsl_gfp_mask(order); + + /* Only allocate non-reserved memory for certain pools */ + if (!pool->allocation_allowed) { + *page_size = PAGE_SIZE << + kgsl_pools[pool_idx-1].pool_order; + *align = ilog2(*page_size); + return -EAGAIN; + } - page = alloc_pages(gfp_mask, - get_order(page_size)); + page = alloc_pages(gfp_mask, order); + + if (!page) { + if (pool_idx > 0) { + /* Retry with lower order pages */ + *page_size = PAGE_SIZE << + kgsl_pools[pool_idx-1].pool_order; + *align = ilog2(*page_size); + return -EAGAIN; + } else + return -ENOMEM; + } - if (!page) - return -ENOMEM; + _kgsl_pool_zero_page(page, order); } - for (j = 0; j < (page_size >> PAGE_SHIFT); j++) { + for (j = 0; j < (*page_size >> PAGE_SHIFT); j++) { p = nth_page(page, j); pages[pcount] = p; pcount++; @@ -279,18 +350,34 @@ void kgsl_pool_free_page(struct page *page) page_order = compound_order(page); - if (kgsl_pool_size_total() < kgsl_pool_max_pages) { - pool = _kgsl_get_pool_from_order(page_order); - if (pool != NULL) { - _kgsl_pool_add_page(pool, page); - return; - } + pool = _kgsl_get_pool_from_order(page_order); + if (pool != NULL) { + _kgsl_pool_add_page(pool, page); + return; } /* Give back to system as not added to pool */ __free_pages(page, page_order); } +static void kgsl_pool_reserve_pages(void) +{ + int i, j; + + for (i = 0; i < KGSL_NUM_POOLS; i++) { + struct page *page; + + for (j = 0; j < kgsl_pools[i].reserved_pages; j++) { + int order = kgsl_pools[i].pool_order; + gfp_t gfp_mask = kgsl_gfp_mask(order); + + page = alloc_pages(gfp_mask, order); + if (page != NULL) + _kgsl_pool_add_page(&kgsl_pools[i], page); + } + } +} + /* Functions for the shrinker */ static unsigned long @@ -326,6 +413,9 @@ static struct shrinker kgsl_pool_shrinker = { void kgsl_init_page_pools(void) { + /* Reserve the appropriate number of pages for each pool */ + kgsl_pool_reserve_pages(); + /* Initialize shrinker */ register_shrinker(&kgsl_pool_shrinker); } diff --git a/drivers/gpu/msm/kgsl_pool.h b/drivers/gpu/msm/kgsl_pool.h index f76c8aadbf30..f2cdda19140b 100644 --- a/drivers/gpu/msm/kgsl_pool.h +++ b/drivers/gpu/msm/kgsl_pool.h @@ -36,8 +36,8 @@ kgsl_gfp_mask(unsigned int page_order) void kgsl_pool_free_sgt(struct sg_table *sgt); void kgsl_init_page_pools(void); void kgsl_exit_page_pools(void); -int kgsl_pool_alloc_page(int page_size, struct page **pages, - unsigned int pages_len); +int kgsl_pool_alloc_page(int *page_size, struct page **pages, + unsigned int pages_len, unsigned int *align); void kgsl_pool_free_page(struct page *p); #endif /* __KGSL_POOL_H */ diff --git a/drivers/gpu/msm/kgsl_sharedmem.c b/drivers/gpu/msm/kgsl_sharedmem.c index 4cb9bc5d1651..50dcd39fac58 100644 --- a/drivers/gpu/msm/kgsl_sharedmem.c +++ b/drivers/gpu/msm/kgsl_sharedmem.c @@ -656,8 +656,14 @@ EXPORT_SYMBOL(kgsl_cache_range_op); #ifndef CONFIG_ALLOC_BUFFERS_IN_4K_CHUNKS static inline int get_page_size(size_t size, unsigned int align) { - return (align >= ilog2(SZ_64K) && size >= SZ_64K) - ? SZ_64K : PAGE_SIZE; + if (align >= ilog2(SZ_1M) && size >= SZ_1M) + return SZ_1M; + else if (align >= ilog2(SZ_64K) && size >= SZ_64K) + return SZ_64K; + else if (align >= ilog2(SZ_8K) && size >= SZ_8K) + return SZ_8K; + else + return PAGE_SIZE; } #else static inline int get_page_size(size_t size, unsigned int align) @@ -666,56 +672,6 @@ static inline int get_page_size(size_t size, unsigned int align) } #endif -static void kgsl_zero_pages(struct page **pages, unsigned int pcount) -{ - unsigned int j; - unsigned int step = ((VMALLOC_END - VMALLOC_START)/8) >> PAGE_SHIFT; - pgprot_t page_prot = pgprot_writecombine(PAGE_KERNEL); - void *ptr; - - /* - * All memory that goes to the user has to be zeroed out before it gets - * exposed to userspace. This means that the memory has to be mapped in - * the kernel, zeroed (memset) and then unmapped. This also means that - * the dcache has to be flushed to ensure coherency between the kernel - * and user pages. We used to pass __GFP_ZERO to alloc_page which mapped - * zeroed and unmaped each individual page, and then we had to turn - * around and call flush_dcache_page() on that page to clear the caches. - * This was killing us for performance. Instead, we found it is much - * faster to allocate the pages without GFP_ZERO, map a chunk of the - * range ('step' pages), memset it, flush it and then unmap - * - this results in a factor of 4 improvement for speed for large - * buffers. There is a small decrease in speed for small buffers, - * but only on the order of a few microseconds at best. The 'step' - * size is based on a guess at the amount of free vmalloc space, but - * will scale down if there's not enough free space. - */ - for (j = 0; j < pcount; j += step) { - step = min(step, pcount - j); - - ptr = vmap(&pages[j], step, VM_IOREMAP, page_prot); - - if (ptr != NULL) { - memset(ptr, 0, step * PAGE_SIZE); - dmac_flush_range(ptr, ptr + step * PAGE_SIZE); - vunmap(ptr); - } else { - int k; - /* Very, very, very slow path */ - - for (k = j; k < j + step; k++) { - ptr = kmap_atomic(pages[k]); - memset(ptr, 0, PAGE_SIZE); - dmac_flush_range(ptr, ptr + PAGE_SIZE); - kunmap_atomic(ptr); - } - /* scale down the step size to avoid this path */ - if (step > 1) - step >>= 1; - } - } -} - static int kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc, struct kgsl_pagetable *pagetable, @@ -741,8 +697,10 @@ kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc, * larger however to accomodate hardware quirks */ - if (align < ilog2(page_size)) + if (align < ilog2(page_size)) { kgsl_memdesc_set_align(memdesc, ilog2(page_size)); + align = ilog2(page_size); + } /* * There needs to be enough room in the page array to be able to @@ -775,18 +733,13 @@ kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc, while (len > 0) { int page_count; - /* don't waste space at the end of the allocation*/ - if (len < page_size) - page_size = PAGE_SIZE; - - page_count = kgsl_pool_alloc_page(page_size, - pages + pcount, len_alloc - pcount); + page_count = kgsl_pool_alloc_page(&page_size, + pages + pcount, len_alloc - pcount, + &align); if (page_count <= 0) { - if (page_size != PAGE_SIZE) { - page_size = PAGE_SIZE; + if (page_count == -EAGAIN) continue; - } /* * Update sglen and memdesc size,as requested allocation @@ -807,6 +760,9 @@ kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc, pcount += page_count; len -= page_size; memdesc->size += page_size; + + /* Get the needed page size for the next iteration */ + page_size = get_page_size(len, align); } ret = sg_alloc_table_from_pages(memdesc->sgt, pages, pcount, 0, @@ -844,11 +800,6 @@ kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc, KGSL_STATS_ADD(memdesc->size, &kgsl_driver.stats.page_alloc, &kgsl_driver.stats.page_alloc_max); - /* - * Zero out the pages. - */ - kgsl_zero_pages(pages, pcount); - done: if (ret) { if (pages) { |