msm: kgsl: Add 1M and 8K pools to the allocator

This change includes the below:
- Add 1M and 8k pools and structure the allocator to use all pools
from the largest page to the smallest
- Reserve a set number of pages for each of these pools at init time
- When allocating, use the reserved pools and then fall back to
allocating from system memory using only 8k and 4k pages
- Remove maximums on the pool sizes
- Zero the memory when we create the pool initially and add pages
back to the pool on free

CRs-Fixed: 995735
Change-Id: I9440bad62d3e13b434902f167c9d23467b1c4235
Signed-off-by: Shrenuj Bansal <shrenujb@codeaurora.org>

3 files changed, 137 insertions, 96 deletions

diff --git a/drivers/gpu/msm/kgsl_pool.c b/drivers/gpu/msm/kgsl_pool.c
index 497cfdaae2ec..48b7b0872425 100644
--- a/drivers/gpu/msm/kgsl_pool.c
+++ b/drivers/gpu/msm/kgsl_pool.c
@@ -21,18 +21,21 @@
 #include "kgsl_device.h"
 #include "kgsl_pool.h"
 
-/*
- * Maximum pool size in terms of pages
- * = (Number of pools * Max size per pool)
+/**
+ * struct kgsl_page_pool - Structure to hold information for the pool
+ * @pool_order: Page order describing the size of the page
+ * @page_count: Number of pages currently present in the pool
+ * @reserved_pages: Number of pages reserved at init for the pool
+ * @allocation_allowed: Tells if reserved pool gets exhausted, can we allocate
+ * from system memory
+ * @list_lock: Spinlock for page list in the pool
+ * @page_list: List of pages held/reserved in this pool
  */
-#define KGSL_POOL_MAX_PAGES (2 * 4096)
-
-/* Set the max pool size to 8192 pages */
-static unsigned int kgsl_pool_max_pages = KGSL_POOL_MAX_PAGES;
-
 struct kgsl_page_pool {
 	unsigned int pool_order;
 	int page_count;
+	unsigned int reserved_pages;
+	bool allocation_allowed;
 	spinlock_t list_lock;
 	struct list_head page_list;
 };
@@ -40,15 +43,34 @@ struct kgsl_page_pool {
 static struct kgsl_page_pool kgsl_pools[] = {
 	{
 		.pool_order = 0,
+		.reserved_pages = 2048,
+		.allocation_allowed = true,
 		.list_lock = __SPIN_LOCK_UNLOCKED(kgsl_pools[0].list_lock),
 		.page_list = LIST_HEAD_INIT(kgsl_pools[0].page_list),
 	},
 #ifndef CONFIG_ALLOC_BUFFERS_IN_4K_CHUNKS
 	{
-		.pool_order = 4,
+		.pool_order = 1,
+		.reserved_pages = 1024,
+		.allocation_allowed = true,
 		.list_lock = __SPIN_LOCK_UNLOCKED(kgsl_pools[1].list_lock),
 		.page_list = LIST_HEAD_INIT(kgsl_pools[1].page_list),
 	},
+	{
+		.pool_order = 4,
+		.reserved_pages = 256,
+		.allocation_allowed = false,
+		.list_lock = __SPIN_LOCK_UNLOCKED(kgsl_pools[2].list_lock),
+		.page_list = LIST_HEAD_INIT(kgsl_pools[2].page_list),
+	},
+	{
+		.pool_order = 8,
+		.reserved_pages = 32,
+		.allocation_allowed = false,
+		.list_lock = __SPIN_LOCK_UNLOCKED(kgsl_pools[3].list_lock),
+		.page_list = LIST_HEAD_INIT(kgsl_pools[3].page_list),
+	},
+
 #endif
 };
 
@@ -68,10 +90,28 @@ _kgsl_get_pool_from_order(unsigned int order)
 	return NULL;
 }
 
+/* Map the page into kernel and zero it out */
+static void
+_kgsl_pool_zero_page(struct page *p, unsigned int pool_order)
+{
+	int i;
+
+	for (i = 0; i < (1 << pool_order); i++) {
+		struct page *page = nth_page(p, i);
+		void *addr = kmap_atomic(page);
+
+		memset(addr, 0, PAGE_SIZE);
+		dmac_flush_range(addr, addr + PAGE_SIZE);
+		kunmap_atomic(addr);
+	}
+}
+
 /* Add a page to specified pool */
 static void
 _kgsl_pool_add_page(struct kgsl_page_pool *pool, struct page *p)
 {
+	_kgsl_pool_zero_page(p, pool->pool_order);
+
 	spin_lock(&pool->list_lock);
 	list_add_tail(&p->lru, &pool->page_list);
 	pool->page_count++;
@@ -156,14 +196,14 @@ _kgsl_pool_shrink(struct kgsl_page_pool *pool, int num_pages)
  * (current_pool_size - target_pages) pages from pool
  * starting from higher order pool.
  */
-static int
+static unsigned long
 kgsl_pool_reduce(unsigned int target_pages)
 {
 	int total_pages = 0;
 	int i;
 	int nr_removed;
 	struct kgsl_page_pool *pool;
-	unsigned int pcount = 0;
+	unsigned long pcount = 0;
 
 	total_pages = kgsl_pool_size_total();
 
@@ -223,6 +263,17 @@ void kgsl_pool_free_sgt(struct sg_table *sgt)
 	}
 }
 
+static int kgsl_pool_idx_lookup(unsigned int order)
+{
+	int i;
+
+	for (i = 0; i < KGSL_NUM_POOLS; i++)
+		if (order == kgsl_pools[i].pool_order)
+			return i;
+
+	return -ENOMEM;
+}
+
 /**
  * kgsl_pool_alloc_page() - Allocate a page of requested size
  * @page_size: Size of the page to be allocated
@@ -232,35 +283,55 @@ void kgsl_pool_free_sgt(struct sg_table *sgt)
  *
  * Return total page count on success and negative value on failure
  */
-int kgsl_pool_alloc_page(int page_size, struct page **pages,
-					unsigned int pages_len)
+int kgsl_pool_alloc_page(int *page_size, struct page **pages,
+			unsigned int pages_len, unsigned int *align)
 {
 	int j;
 	int pcount = 0;
 	struct kgsl_page_pool *pool;
 	struct page *page = NULL;
 	struct page *p = NULL;
+	int order = get_order(*page_size);
+	int pool_idx;
 
-	if ((pages == NULL) || pages_len < (page_size >> PAGE_SHIFT))
+	if ((pages == NULL) || pages_len < (*page_size >> PAGE_SHIFT))
 		return -EINVAL;
 
-	pool = _kgsl_get_pool_from_order(get_order(page_size));
+	pool = _kgsl_get_pool_from_order(order);
+	pool_idx = kgsl_pool_idx_lookup(order);
 
 	if (pool != NULL)
 		page = _kgsl_pool_get_page(pool);
 
 	/* Allocate a new page if not allocated from pool */
 	if (page == NULL) {
-		gfp_t gfp_mask = kgsl_gfp_mask(get_order(page_size));
+		gfp_t gfp_mask = kgsl_gfp_mask(order);
+
+		/* Only allocate non-reserved memory for certain pools */
+		if (!pool->allocation_allowed) {
+			*page_size = PAGE_SIZE <<
+					kgsl_pools[pool_idx-1].pool_order;
+			*align = ilog2(*page_size);
+			return -EAGAIN;
+		}
 
-		page = alloc_pages(gfp_mask,
-					get_order(page_size));
+		page = alloc_pages(gfp_mask, order);
+
+		if (!page) {
+			if (pool_idx > 0) {
+				/* Retry with lower order pages */
+				*page_size = PAGE_SIZE <<
+					kgsl_pools[pool_idx-1].pool_order;
+				*align = ilog2(*page_size);
+				return -EAGAIN;
+			} else
+				return -ENOMEM;
+		}
 
-		if (!page)
-			return -ENOMEM;
+		_kgsl_pool_zero_page(page, order);
 	}
 
-	for (j = 0; j < (page_size >> PAGE_SHIFT); j++) {
+	for (j = 0; j < (*page_size >> PAGE_SHIFT); j++) {
 		p = nth_page(page, j);
 		pages[pcount] = p;
 		pcount++;
@@ -279,18 +350,34 @@ void kgsl_pool_free_page(struct page *page)
 
 	page_order = compound_order(page);
 
-	if (kgsl_pool_size_total() < kgsl_pool_max_pages) {
-		pool = _kgsl_get_pool_from_order(page_order);
-		if (pool != NULL) {
-			_kgsl_pool_add_page(pool, page);
-			return;
-		}
+	pool = _kgsl_get_pool_from_order(page_order);
+	if (pool != NULL) {
+		_kgsl_pool_add_page(pool, page);
+		return;
 	}
 
 	/* Give back to system as not added to pool */
 	__free_pages(page, page_order);
 }
 
+static void kgsl_pool_reserve_pages(void)
+{
+	int i, j;
+
+	for (i = 0; i < KGSL_NUM_POOLS; i++) {
+		struct page *page;
+
+		for (j = 0; j < kgsl_pools[i].reserved_pages; j++) {
+			int order = kgsl_pools[i].pool_order;
+			gfp_t gfp_mask = kgsl_gfp_mask(order);
+
+			page = alloc_pages(gfp_mask, order);
+			if (page != NULL)
+				_kgsl_pool_add_page(&kgsl_pools[i], page);
+		}
+	}
+}
+
 /* Functions for the shrinker */
 
 static unsigned long
@@ -326,6 +413,9 @@ static struct shrinker kgsl_pool_shrinker = {
 
 void kgsl_init_page_pools(void)
 {
+	/* Reserve the appropriate number of pages for each pool */
+	kgsl_pool_reserve_pages();
+
 	/* Initialize shrinker */
 	register_shrinker(&kgsl_pool_shrinker);
 }
diff --git a/drivers/gpu/msm/kgsl_pool.h b/drivers/gpu/msm/kgsl_pool.h
index f76c8aadbf30..f2cdda19140b 100644
--- a/drivers/gpu/msm/kgsl_pool.h
+++ b/drivers/gpu/msm/kgsl_pool.h
@@ -36,8 +36,8 @@ kgsl_gfp_mask(unsigned int page_order)
 void kgsl_pool_free_sgt(struct sg_table *sgt);
 void kgsl_init_page_pools(void);
 void kgsl_exit_page_pools(void);
-int kgsl_pool_alloc_page(int page_size, struct page **pages,
-						unsigned int pages_len);
+int kgsl_pool_alloc_page(int *page_size, struct page **pages,
+			unsigned int pages_len, unsigned int *align);
 void kgsl_pool_free_page(struct page *p);
 #endif /* __KGSL_POOL_H */
 
diff --git a/drivers/gpu/msm/kgsl_sharedmem.c b/drivers/gpu/msm/kgsl_sharedmem.c
index 4cb9bc5d1651..50dcd39fac58 100644
--- a/drivers/gpu/msm/kgsl_sharedmem.c
+++ b/drivers/gpu/msm/kgsl_sharedmem.c
@@ -656,8 +656,14 @@ EXPORT_SYMBOL(kgsl_cache_range_op);
 #ifndef CONFIG_ALLOC_BUFFERS_IN_4K_CHUNKS
 static inline int get_page_size(size_t size, unsigned int align)
 {
-	return (align >= ilog2(SZ_64K) && size >= SZ_64K)
-					? SZ_64K : PAGE_SIZE;
+	if (align >= ilog2(SZ_1M) && size >= SZ_1M)
+		return SZ_1M;
+	else if (align >= ilog2(SZ_64K) && size >= SZ_64K)
+		return SZ_64K;
+	else if (align >= ilog2(SZ_8K) && size >= SZ_8K)
+		return SZ_8K;
+	else
+		return PAGE_SIZE;
 }
 #else
 static inline int get_page_size(size_t size, unsigned int align)
@@ -666,56 +672,6 @@ static inline int get_page_size(size_t size, unsigned int align)
 }
 #endif
 
-static void kgsl_zero_pages(struct page **pages, unsigned int pcount)
-{
-	unsigned int j;
-	unsigned int step = ((VMALLOC_END - VMALLOC_START)/8) >> PAGE_SHIFT;
-	pgprot_t page_prot = pgprot_writecombine(PAGE_KERNEL);
-	void *ptr;
-
-	/*
-	 * All memory that goes to the user has to be zeroed out before it gets
-	 * exposed to userspace. This means that the memory has to be mapped in
-	 * the kernel, zeroed (memset) and then unmapped.  This also means that
-	 * the dcache has to be flushed to ensure coherency between the kernel
-	 * and user pages. We used to pass __GFP_ZERO to alloc_page which mapped
-	 * zeroed and unmaped each individual page, and then we had to turn
-	 * around and call flush_dcache_page() on that page to clear the caches.
-	 * This was killing us for performance. Instead, we found it is much
-	 * faster to allocate the pages without GFP_ZERO, map a chunk of the
-	 * range ('step' pages), memset it, flush it and then unmap
-	 * - this results in a factor of 4 improvement for speed for large
-	 * buffers. There is a small decrease in speed for small buffers,
-	 * but only on the order of a few microseconds at best. The 'step'
-	 * size is based on a guess at the amount of free vmalloc space, but
-	 * will scale down if there's not enough free space.
-	 */
-	for (j = 0; j < pcount; j += step) {
-		step = min(step, pcount - j);
-
-		ptr = vmap(&pages[j], step, VM_IOREMAP, page_prot);
-
-		if (ptr != NULL) {
-			memset(ptr, 0, step * PAGE_SIZE);
-			dmac_flush_range(ptr, ptr + step * PAGE_SIZE);
-			vunmap(ptr);
-		} else {
-			int k;
-			/* Very, very, very slow path */
-
-			for (k = j; k < j + step; k++) {
-				ptr = kmap_atomic(pages[k]);
-				memset(ptr, 0, PAGE_SIZE);
-				dmac_flush_range(ptr, ptr + PAGE_SIZE);
-				kunmap_atomic(ptr);
-			}
-			/* scale down the step size to avoid this path */
-			if (step > 1)
-				step >>= 1;
-		}
-	}
-}
-
 static int
 kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc,
 			struct kgsl_pagetable *pagetable,
@@ -741,8 +697,10 @@ kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc,
 	 * larger however to accomodate hardware quirks
 	 */
 
-	if (align < ilog2(page_size))
+	if (align < ilog2(page_size)) {
 		kgsl_memdesc_set_align(memdesc, ilog2(page_size));
+		align = ilog2(page_size);
+	}
 
 	/*
 	 * There needs to be enough room in the page array to be able to
@@ -775,18 +733,13 @@ kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc,
 	while (len > 0) {
 		int page_count;
 
-		/* don't waste space at the end of the allocation*/
-		if (len < page_size)
-			page_size = PAGE_SIZE;
-
-		page_count = kgsl_pool_alloc_page(page_size,
-					pages + pcount, len_alloc - pcount);
+		page_count = kgsl_pool_alloc_page(&page_size,
+					pages + pcount, len_alloc - pcount,
+					&align);
 
 		if (page_count <= 0) {
-			if (page_size != PAGE_SIZE) {
-				page_size = PAGE_SIZE;
+			if (page_count == -EAGAIN)
 				continue;
-			}
 
 			/*
 			 * Update sglen and memdesc size,as requested allocation
@@ -807,6 +760,9 @@ kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc,
 		pcount += page_count;
 		len -= page_size;
 		memdesc->size += page_size;
+
+		/* Get the needed page size for the next iteration */
+		page_size = get_page_size(len, align);
 	}
 
 	ret = sg_alloc_table_from_pages(memdesc->sgt, pages, pcount, 0,
@@ -844,11 +800,6 @@ kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc,
 	KGSL_STATS_ADD(memdesc->size, &kgsl_driver.stats.page_alloc,
 		&kgsl_driver.stats.page_alloc_max);
 
-	/*
-	 * Zero out the pages.
-	 */
-	kgsl_zero_pages(pages, pcount);
-
 done:
 	if (ret) {
 		if (pages) {