drm/msm: Add performance counter tracking

Adreno GPUs have a certain number of fixed performance counters
most of which can be programmed to a large number of different
items (countables). A centralized database in the kernel is
needed to make the most efficient use of counters across processes.

Add performance counter tracking and APIs to allow applications to
reserve performance counters by requesting a group ID and a countable
(countables differ from block to block). The kernel will check to
see if an active counter is already selected for that countable or
if a new one should be assigned. Different processes can share the
same counter if they both need the same countable.

Counters are reserved with DRM_IOCTL_MSM_COUNTER_GET which returns
a counter ID for the reserved counter and the hi/lo offset of
the counter register. The reserving application can either read the
counter from within a PM4 stream or it can use the group ID and
counter ID and read the value of the counter with
DRM_MSM_COUNTER_READ. After the counter is no longer needed
DRM_IOCTL_MSM_COUNTER_PUT returns it and it can be released for
other countables if no other processes are sharing it.

Reservations are tracked for each process and cleaned up if the
process dies without putting back the counters.

Change-Id: Ic0dedbadc45e85ab0063331b39ca6f3289523038
Signed-off-by: Jordan Crouse <jcrouse@codeaurora.org>

10 files changed, 954 insertions, 1 deletions

diff --git a/drivers/gpu/drm/msm/Makefile b/drivers/gpu/drm/msm/Makefile
index 79ea5a9f90ea..ebf8be80a3d9 100644
--- a/drivers/gpu/drm/msm/Makefile
+++ b/drivers/gpu/drm/msm/Makefile
@@ -59,7 +59,8 @@ msm_drm-y += adreno/adreno_device.o \
 	adreno/a5xx_gpu.o \
 	adreno/a5xx_power.o \
 	adreno/a5xx_preempt.o \
-	adreno/a5xx_snapshot.o
+	adreno/a5xx_snapshot.o \
+	adreno/a5xx_counters.o
 endif
 
 msm_drm-$(CONFIG_DRM_MSM_MDP4) += mdp/mdp4/mdp4_crtc.o \
diff --git a/drivers/gpu/drm/msm/adreno/a5xx_counters.c b/drivers/gpu/drm/msm/adreno/a5xx_counters.c
new file mode 100644
index 000000000000..f1fac5535359
--- /dev/null
+++ b/drivers/gpu/drm/msm/adreno/a5xx_counters.c
@@ -0,0 +1,689 @@
+/* Copyright (c) 2017 The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include "a5xx_gpu.h"
+
+/*
+ * Fixed counters are not selectable, they always count the same thing.
+ * The countable is an index into the group: countable 0 = register 0,
+ * etc and they have no select register
+ */
+static int a5xx_counter_get_fixed(struct msm_gpu *gpu,
+		struct adreno_counter_group *group,
+		u32 countable, u32 *lo, u32 *hi)
+{
+	if (countable >= group->nr_counters)
+		return -EINVAL;
+
+	if (lo)
+		*lo = group->counters[countable].lo;
+	if (hi)
+		*hi = group->counters[countable].hi;
+
+	return countable;
+}
+
+/*
+ * Most counters are selectable in that they can be programmed to count
+ * different events; in most cases there are many more countables than
+ * counters. When a new counter is requested, first walk the list to see if any
+ * other counters in that group are counting the same countable and if so reuse
+ * that counter. If not find the first empty counter in the list and register
+ * that for the desired countable. If we are out of counters too bad so sad.
+ */
+static int a5xx_counter_get(struct msm_gpu *gpu,
+		struct adreno_counter_group *group,
+		u32 countable, u32 *lo, u32 *hi)
+{
+	struct adreno_counter *counter;
+	int i, empty = -1;
+
+	spin_lock(&group->lock);
+
+	for (i = 0; i < group->nr_counters; i++) {
+		counter = &group->counters[i];
+
+		if (counter->refcount) {
+			if (counter->countable == countable) {
+				counter->refcount++;
+
+				if (lo)
+					*lo = counter->lo;
+				if (hi)
+					*hi = counter->hi;
+
+				spin_unlock(&group->lock);
+				return i;
+			}
+		} else
+			empty = (empty == -1) ? i : empty;
+	}
+
+	if (empty == -1) {
+		spin_unlock(&group->lock);
+		return -EBUSY;
+	}
+
+	counter = &group->counters[empty];
+
+	counter->refcount = 1;
+	counter->countable = countable;
+
+	if (lo)
+		*lo = counter->lo;
+	if (hi)
+		*hi = counter->hi;
+
+	spin_unlock(&group->lock);
+
+	if (group->funcs.enable)
+		group->funcs.enable(gpu, group, empty);
+
+	return empty;
+}
+
+/* The majority of the non-fixed counter selects can be programmed by the CPU */
+static void a5xx_counter_enable_cpu(struct msm_gpu *gpu,
+		struct adreno_counter_group *group, int counterid)
+{
+	struct adreno_counter *counter = &group->counters[counterid];
+
+	gpu_write(gpu, counter->sel, counter->countable);
+}
+
+static void a5xx_counter_enable_pm4(struct msm_gpu *gpu,
+		struct adreno_counter_group *group, int counterid)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
+	struct msm_ringbuffer *ring = gpu->rb[MSM_GPU_MAX_RINGS - 1];
+	struct adreno_counter *counter = &group->counters[counterid];
+
+	mutex_lock(&gpu->dev->struct_mutex);
+
+	/* Turn off preemption for the duration of this command */
+	OUT_PKT7(ring, CP_PREEMPT_ENABLE_GLOBAL, 1);
+	OUT_RING(ring, 0x02);
+
+	/* Turn off protected mode to write to special registers */
+	OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
+	OUT_RING(ring, 0);
+
+	/* Set the save preemption record for the ring/command */
+	OUT_PKT4(ring, REG_A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO, 2);
+	OUT_RING(ring, lower_32_bits(a5xx_gpu->preempt_iova[ring->id]));
+	OUT_RING(ring, upper_32_bits(a5xx_gpu->preempt_iova[ring->id]));
+
+	/* Turn back on protected mode */
+	OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
+	OUT_RING(ring, 1);
+
+	/* Idle the GPU */
+	OUT_PKT7(ring, CP_WAIT_FOR_IDLE, 0);
+
+	/* Enable the counter */
+	OUT_PKT4(ring, counter->sel, 1);
+	OUT_RING(ring, counter->countable);
+
+	/* Re-enable preemption */
+	OUT_PKT7(ring, CP_PREEMPT_ENABLE_GLOBAL, 1);
+	OUT_RING(ring, 0x00);
+
+	OUT_PKT7(ring, CP_PREEMPT_ENABLE_LOCAL, 1);
+	OUT_RING(ring, 0x01);
+
+	OUT_PKT7(ring, CP_YIELD_ENABLE, 1);
+	OUT_RING(ring, 0x01);
+
+	/* Yield */
+	OUT_PKT7(ring, CP_CONTEXT_SWITCH_YIELD, 4);
+	OUT_RING(ring, 0x00);
+	OUT_RING(ring, 0x00);
+	OUT_RING(ring, 0x01);
+	OUT_RING(ring, 0x01);
+
+	gpu->funcs->flush(gpu, ring);
+
+	/* Preempt into our ring if we need to */
+	a5xx_preempt_trigger(gpu);
+
+	/* wait for the operation to complete */
+	a5xx_idle(gpu, ring);
+
+	mutex_unlock(&gpu->dev->struct_mutex);
+}
+
+/*
+ * GPMU counters are selectable but the selects are muxed together in two
+ * registers
+ */
+static void a5xx_counter_enable_gpmu(struct msm_gpu *gpu,
+		struct adreno_counter_group *group, int counterid)
+{
+	struct adreno_counter *counter = &group->counters[counterid];
+	u32 reg;
+	int shift;
+
+	/*
+	 * The selects for the GPMU counters are grouped together in two
+	 * registers, a nibble for each counter. Counters 0-3 are located in
+	 * GPMU_POWER_COUNTER_SELECT0 and 4-5 are in GPMU_POWER_COUNTER_SELECT1
+	 */
+	if (counterid <= 3) {
+		shift = counterid << 3;
+		reg = REG_A5XX_GPMU_POWER_COUNTER_SELECT_0;
+	} else {
+		shift = (counterid - 4) << 3;
+		reg = REG_A5XX_GPMU_POWER_COUNTER_SELECT_1;
+	}
+
+	gpu_rmw(gpu, reg, 0xFF << shift, (counter->countable & 0xff) << shift);
+}
+
+/* VBIF counters are selectable but have their own programming process */
+static void a5xx_counter_enable_vbif(struct msm_gpu *gpu,
+		struct adreno_counter_group *group, int counterid)
+{
+	struct adreno_counter *counter = &group->counters[counterid];
+
+	gpu_write(gpu, REG_A5XX_VBIF_PERF_CNT_CLR(counterid), 1);
+	gpu_write(gpu, REG_A5XX_VBIF_PERF_CNT_CLR(counterid), 0);
+	gpu_write(gpu, REG_A5XX_VBIF_PERF_CNT_SEL(counterid),
+		counter->countable);
+	gpu_write(gpu, REG_A5XX_VBIF_PERF_CNT_EN(counterid), 1);
+}
+
+/*
+ * VBIF power counters are not slectable but need to be cleared/enabled before
+ * use
+ */
+static void a5xx_counter_enable_vbif_power(struct msm_gpu *gpu,
+		struct adreno_counter_group *group, int counterid)
+{
+	gpu_write(gpu, REG_A5XX_VBIF_PERF_PWR_CNT_CLR(counterid), 1);
+	gpu_write(gpu, REG_A5XX_VBIF_PERF_PWR_CNT_CLR(counterid), 0);
+	gpu_write(gpu, REG_A5XX_VBIF_PERF_PWR_CNT_EN(counterid), 1);
+}
+
+/* GPMU always on counter needs to be enabled before use */
+static void a5xx_counter_enable_alwayson_power(struct msm_gpu *gpu,
+		struct adreno_counter_group *group, int counterid)
+{
+	gpu_write(gpu, REG_A5XX_GPMU_ALWAYS_ON_COUNTER_RESET, 1);
+}
+
+static u64 a5xx_counter_read(struct msm_gpu *gpu,
+		struct adreno_counter_group *group, int counterid)
+{
+	if (counterid >= group->nr_counters)
+		return 0;
+
+	return gpu_read64(gpu, group->counters[counterid].lo,
+		group->counters[counterid].hi);
+}
+
+/*
+ * Selectable counters that are no longer used reset the countable to 0 to mark
+ * the counter as free
+ */
+static void a5xx_counter_put(struct msm_gpu *gpu,
+		struct adreno_counter_group *group, int counterid)
+{
+	struct adreno_counter *counter;
+
+	if (counterid >= group->nr_counters)
+		return;
+
+	counter = &group->counters[counterid];
+
+	spin_lock(&group->lock);
+	if (counter->refcount > 0)
+		counter->refcount--;
+	spin_unlock(&group->lock);
+}
+
+static struct adreno_counter a5xx_counters_alwayson[1] = {
+	{ REG_A5XX_RBBM_ALWAYSON_COUNTER_LO,
+		REG_A5XX_RBBM_ALWAYSON_COUNTER_HI },
+};
+
+static struct adreno_counter a5xx_counters_ccu[] = {
+	{ REG_A5XX_RBBM_PERFCTR_CCU_0_LO, REG_A5XX_RBBM_PERFCTR_CCU_0_HI,
+		REG_A5XX_RB_PERFCTR_CCU_SEL_0 },
+	{ REG_A5XX_RBBM_PERFCTR_CCU_1_LO, REG_A5XX_RBBM_PERFCTR_CCU_1_HI,
+		REG_A5XX_RB_PERFCTR_CCU_SEL_1 },
+	{ REG_A5XX_RBBM_PERFCTR_CCU_2_LO, REG_A5XX_RBBM_PERFCTR_CCU_2_HI,
+		REG_A5XX_RB_PERFCTR_CCU_SEL_2 },
+	{ REG_A5XX_RBBM_PERFCTR_CCU_3_LO, REG_A5XX_RBBM_PERFCTR_CCU_3_HI,
+		REG_A5XX_RB_PERFCTR_CCU_SEL_3 },
+};
+
+static struct adreno_counter a5xx_counters_cmp[] = {
+	{ REG_A5XX_RBBM_PERFCTR_CMP_0_LO, REG_A5XX_RBBM_PERFCTR_CMP_0_HI,
+		REG_A5XX_RB_PERFCTR_CMP_SEL_0 },
+	{ REG_A5XX_RBBM_PERFCTR_CMP_1_LO, REG_A5XX_RBBM_PERFCTR_CMP_1_HI,
+		REG_A5XX_RB_PERFCTR_CMP_SEL_1 },
+	{ REG_A5XX_RBBM_PERFCTR_CMP_2_LO, REG_A5XX_RBBM_PERFCTR_CMP_2_HI,
+		REG_A5XX_RB_PERFCTR_CMP_SEL_2 },
+	{ REG_A5XX_RBBM_PERFCTR_CMP_3_LO, REG_A5XX_RBBM_PERFCTR_CMP_3_HI,
+		REG_A5XX_RB_PERFCTR_CMP_SEL_3 },
+};
+
+static struct adreno_counter a5xx_counters_cp[] = {
+	{ REG_A5XX_RBBM_PERFCTR_CP_0_LO, REG_A5XX_RBBM_PERFCTR_CP_0_HI,
+		REG_A5XX_CP_PERFCTR_CP_SEL_0 },
+	{ REG_A5XX_RBBM_PERFCTR_CP_1_LO, REG_A5XX_RBBM_PERFCTR_CP_1_HI,
+		REG_A5XX_CP_PERFCTR_CP_SEL_1 },
+	{ REG_A5XX_RBBM_PERFCTR_CP_2_LO, REG_A5XX_RBBM_PERFCTR_CP_2_HI,
+		REG_A5XX_CP_PERFCTR_CP_SEL_2 },
+	{ REG_A5XX_RBBM_PERFCTR_CP_3_LO, REG_A5XX_RBBM_PERFCTR_CP_3_HI,
+		REG_A5XX_CP_PERFCTR_CP_SEL_3 },
+	{ REG_A5XX_RBBM_PERFCTR_CP_4_LO, REG_A5XX_RBBM_PERFCTR_CP_4_HI,
+		REG_A5XX_CP_PERFCTR_CP_SEL_4 },
+	{ REG_A5XX_RBBM_PERFCTR_CP_5_LO, REG_A5XX_RBBM_PERFCTR_CP_5_HI,
+		REG_A5XX_CP_PERFCTR_CP_SEL_5 },
+	{ REG_A5XX_RBBM_PERFCTR_CP_6_LO, REG_A5XX_RBBM_PERFCTR_CP_6_HI,
+		REG_A5XX_CP_PERFCTR_CP_SEL_6 },
+	{ REG_A5XX_RBBM_PERFCTR_CP_7_LO, REG_A5XX_RBBM_PERFCTR_CP_7_HI,
+		REG_A5XX_CP_PERFCTR_CP_SEL_7 },
+};
+
+static struct adreno_counter a5xx_counters_hlsq[] = {
+	{ REG_A5XX_RBBM_PERFCTR_HLSQ_0_LO, REG_A5XX_RBBM_PERFCTR_HLSQ_0_HI,
+		REG_A5XX_HLSQ_PERFCTR_HLSQ_SEL_0 },
+	{ REG_A5XX_RBBM_PERFCTR_HLSQ_1_LO, REG_A5XX_RBBM_PERFCTR_HLSQ_1_HI,
+		REG_A5XX_HLSQ_PERFCTR_HLSQ_SEL_1 },
+	{ REG_A5XX_RBBM_PERFCTR_HLSQ_2_LO, REG_A5XX_RBBM_PERFCTR_HLSQ_2_HI,
+		REG_A5XX_HLSQ_PERFCTR_HLSQ_SEL_2 },
+	{ REG_A5XX_RBBM_PERFCTR_HLSQ_3_LO, REG_A5XX_RBBM_PERFCTR_HLSQ_3_HI,
+		REG_A5XX_HLSQ_PERFCTR_HLSQ_SEL_3 },
+	{ REG_A5XX_RBBM_PERFCTR_HLSQ_4_LO, REG_A5XX_RBBM_PERFCTR_HLSQ_4_HI,
+		REG_A5XX_HLSQ_PERFCTR_HLSQ_SEL_4 },
+	{ REG_A5XX_RBBM_PERFCTR_HLSQ_5_LO, REG_A5XX_RBBM_PERFCTR_HLSQ_5_HI,
+		REG_A5XX_HLSQ_PERFCTR_HLSQ_SEL_5 },
+	{ REG_A5XX_RBBM_PERFCTR_HLSQ_6_LO, REG_A5XX_RBBM_PERFCTR_HLSQ_6_HI,
+		REG_A5XX_HLSQ_PERFCTR_HLSQ_SEL_6 },
+	{ REG_A5XX_RBBM_PERFCTR_HLSQ_7_LO, REG_A5XX_RBBM_PERFCTR_HLSQ_7_HI,
+		REG_A5XX_HLSQ_PERFCTR_HLSQ_SEL_7 },
+};
+
+static struct adreno_counter a5xx_counters_lrz[] = {
+	{ REG_A5XX_RBBM_PERFCTR_LRZ_0_LO, REG_A5XX_RBBM_PERFCTR_LRZ_0_HI,
+		REG_A5XX_GRAS_PERFCTR_LRZ_SEL_0 },
+	{ REG_A5XX_RBBM_PERFCTR_LRZ_1_LO, REG_A5XX_RBBM_PERFCTR_LRZ_1_HI,
+		REG_A5XX_GRAS_PERFCTR_LRZ_SEL_1 },
+	{ REG_A5XX_RBBM_PERFCTR_LRZ_2_LO, REG_A5XX_RBBM_PERFCTR_LRZ_2_HI,
+		REG_A5XX_GRAS_PERFCTR_LRZ_SEL_2 },
+	{ REG_A5XX_RBBM_PERFCTR_LRZ_3_LO, REG_A5XX_RBBM_PERFCTR_LRZ_3_HI,
+		REG_A5XX_GRAS_PERFCTR_LRZ_SEL_3 },
+};
+
+static struct adreno_counter a5xx_counters_pc[] = {
+	{ REG_A5XX_RBBM_PERFCTR_PC_0_LO, REG_A5XX_RBBM_PERFCTR_PC_0_HI,
+		REG_A5XX_PC_PERFCTR_PC_SEL_0 },
+	{ REG_A5XX_RBBM_PERFCTR_PC_1_LO, REG_A5XX_RBBM_PERFCTR_PC_1_HI,
+		REG_A5XX_PC_PERFCTR_PC_SEL_1 },
+	{ REG_A5XX_RBBM_PERFCTR_PC_2_LO, REG_A5XX_RBBM_PERFCTR_PC_2_HI,
+		REG_A5XX_PC_PERFCTR_PC_SEL_2 },
+	{ REG_A5XX_RBBM_PERFCTR_PC_3_LO, REG_A5XX_RBBM_PERFCTR_PC_3_HI,
+		REG_A5XX_PC_PERFCTR_PC_SEL_3 },
+	{ REG_A5XX_RBBM_PERFCTR_PC_4_LO, REG_A5XX_RBBM_PERFCTR_PC_4_HI,
+		REG_A5XX_PC_PERFCTR_PC_SEL_4 },
+	{ REG_A5XX_RBBM_PERFCTR_PC_5_LO, REG_A5XX_RBBM_PERFCTR_PC_5_HI,
+		REG_A5XX_PC_PERFCTR_PC_SEL_5 },
+	{ REG_A5XX_RBBM_PERFCTR_PC_6_LO, REG_A5XX_RBBM_PERFCTR_PC_6_HI,
+		REG_A5XX_PC_PERFCTR_PC_SEL_6 },
+	{ REG_A5XX_RBBM_PERFCTR_PC_7_LO, REG_A5XX_RBBM_PERFCTR_PC_7_HI,
+		REG_A5XX_PC_PERFCTR_PC_SEL_7 },
+};
+
+static struct adreno_counter a5xx_counters_ras[] = {
+	{ REG_A5XX_RBBM_PERFCTR_RAS_0_LO, REG_A5XX_RBBM_PERFCTR_RAS_0_HI,
+		REG_A5XX_GRAS_PERFCTR_RAS_SEL_0 },
+	{ REG_A5XX_RBBM_PERFCTR_RAS_1_LO, REG_A5XX_RBBM_PERFCTR_RAS_1_HI,
+		REG_A5XX_GRAS_PERFCTR_RAS_SEL_1 },
+	{ REG_A5XX_RBBM_PERFCTR_RAS_2_LO, REG_A5XX_RBBM_PERFCTR_RAS_2_HI,
+		REG_A5XX_GRAS_PERFCTR_RAS_SEL_2 },
+	{ REG_A5XX_RBBM_PERFCTR_RAS_3_LO, REG_A5XX_RBBM_PERFCTR_RAS_3_HI,
+		REG_A5XX_GRAS_PERFCTR_RAS_SEL_3 },
+};
+
+static struct adreno_counter a5xx_counters_rb[] = {
+	{ REG_A5XX_RBBM_PERFCTR_RB_0_LO, REG_A5XX_RBBM_PERFCTR_RB_0_HI,
+		REG_A5XX_RB_PERFCTR_RB_SEL_0 },
+	{ REG_A5XX_RBBM_PERFCTR_RB_1_LO, REG_A5XX_RBBM_PERFCTR_RB_1_HI,
+		REG_A5XX_RB_PERFCTR_RB_SEL_1 },
+	{ REG_A5XX_RBBM_PERFCTR_RB_2_LO, REG_A5XX_RBBM_PERFCTR_RB_2_HI,
+		REG_A5XX_RB_PERFCTR_RB_SEL_2 },
+	{ REG_A5XX_RBBM_PERFCTR_RB_3_LO, REG_A5XX_RBBM_PERFCTR_RB_3_HI,
+		REG_A5XX_RB_PERFCTR_RB_SEL_3 },
+	{ REG_A5XX_RBBM_PERFCTR_RB_4_LO, REG_A5XX_RBBM_PERFCTR_RB_4_HI,
+		REG_A5XX_RB_PERFCTR_RB_SEL_4 },
+	{ REG_A5XX_RBBM_PERFCTR_RB_5_LO, REG_A5XX_RBBM_PERFCTR_RB_5_HI,
+		REG_A5XX_RB_PERFCTR_RB_SEL_5 },
+	{ REG_A5XX_RBBM_PERFCTR_RB_6_LO, REG_A5XX_RBBM_PERFCTR_RB_6_HI,
+		REG_A5XX_RB_PERFCTR_RB_SEL_6 },
+	{ REG_A5XX_RBBM_PERFCTR_RB_7_LO, REG_A5XX_RBBM_PERFCTR_RB_7_HI,
+		REG_A5XX_RB_PERFCTR_RB_SEL_7 },
+};
+
+static struct adreno_counter a5xx_counters_rbbm[] = {
+	{ REG_A5XX_RBBM_PERFCTR_RBBM_0_LO, REG_A5XX_RBBM_PERFCTR_RBBM_0_HI,
+		REG_A5XX_RBBM_PERFCTR_RBBM_SEL_0 },
+	{ REG_A5XX_RBBM_PERFCTR_RBBM_1_LO, REG_A5XX_RBBM_PERFCTR_RBBM_1_HI,
+		REG_A5XX_RBBM_PERFCTR_RBBM_SEL_1 },
+	{ REG_A5XX_RBBM_PERFCTR_RBBM_2_LO, REG_A5XX_RBBM_PERFCTR_RBBM_2_HI,
+		REG_A5XX_RBBM_PERFCTR_RBBM_SEL_2 },
+	{ REG_A5XX_RBBM_PERFCTR_RBBM_3_LO, REG_A5XX_RBBM_PERFCTR_RBBM_3_HI,
+		REG_A5XX_RBBM_PERFCTR_RBBM_SEL_3 },
+};
+
+static struct adreno_counter a5xx_counters_sp[] = {
+	{ REG_A5XX_RBBM_PERFCTR_SP_0_LO, REG_A5XX_RBBM_PERFCTR_SP_0_HI,
+		REG_A5XX_SP_PERFCTR_SP_SEL_0 },
+	{ REG_A5XX_RBBM_PERFCTR_SP_1_LO, REG_A5XX_RBBM_PERFCTR_SP_1_HI,
+		REG_A5XX_SP_PERFCTR_SP_SEL_1 },
+	{ REG_A5XX_RBBM_PERFCTR_SP_2_LO, REG_A5XX_RBBM_PERFCTR_SP_2_HI,
+		REG_A5XX_SP_PERFCTR_SP_SEL_2 },
+	{ REG_A5XX_RBBM_PERFCTR_SP_3_LO, REG_A5XX_RBBM_PERFCTR_SP_3_HI,
+		REG_A5XX_SP_PERFCTR_SP_SEL_3 },
+	{ REG_A5XX_RBBM_PERFCTR_SP_4_LO, REG_A5XX_RBBM_PERFCTR_SP_4_HI,
+		REG_A5XX_SP_PERFCTR_SP_SEL_4 },
+	{ REG_A5XX_RBBM_PERFCTR_SP_5_LO, REG_A5XX_RBBM_PERFCTR_SP_5_HI,
+		REG_A5XX_SP_PERFCTR_SP_SEL_5 },
+	{ REG_A5XX_RBBM_PERFCTR_SP_6_LO, REG_A5XX_RBBM_PERFCTR_SP_6_HI,
+		REG_A5XX_SP_PERFCTR_SP_SEL_6 },
+	{ REG_A5XX_RBBM_PERFCTR_SP_7_LO, REG_A5XX_RBBM_PERFCTR_SP_7_HI,
+		REG_A5XX_SP_PERFCTR_SP_SEL_7 },
+	{ REG_A5XX_RBBM_PERFCTR_SP_8_LO, REG_A5XX_RBBM_PERFCTR_SP_8_HI,
+		REG_A5XX_SP_PERFCTR_SP_SEL_8 },
+	{ REG_A5XX_RBBM_PERFCTR_SP_9_LO, REG_A5XX_RBBM_PERFCTR_SP_9_HI,
+		REG_A5XX_SP_PERFCTR_SP_SEL_9 },
+	{ REG_A5XX_RBBM_PERFCTR_SP_10_LO, REG_A5XX_RBBM_PERFCTR_SP_10_HI,
+		REG_A5XX_SP_PERFCTR_SP_SEL_10 },
+	{ REG_A5XX_RBBM_PERFCTR_SP_11_LO, REG_A5XX_RBBM_PERFCTR_SP_11_HI,
+		REG_A5XX_SP_PERFCTR_SP_SEL_11 },
+};
+
+static struct adreno_counter a5xx_counters_tp[] = {
+	{ REG_A5XX_RBBM_PERFCTR_TP_0_LO, REG_A5XX_RBBM_PERFCTR_TP_0_HI,
+		REG_A5XX_TPL1_PERFCTR_TP_SEL_0 },
+	{ REG_A5XX_RBBM_PERFCTR_TP_1_LO, REG_A5XX_RBBM_PERFCTR_TP_1_HI,
+		REG_A5XX_TPL1_PERFCTR_TP_SEL_1 },
+	{ REG_A5XX_RBBM_PERFCTR_TP_2_LO, REG_A5XX_RBBM_PERFCTR_TP_2_HI,
+		REG_A5XX_TPL1_PERFCTR_TP_SEL_2 },
+	{ REG_A5XX_RBBM_PERFCTR_TP_3_LO, REG_A5XX_RBBM_PERFCTR_TP_3_HI,
+		REG_A5XX_TPL1_PERFCTR_TP_SEL_3 },
+	{ REG_A5XX_RBBM_PERFCTR_TP_4_LO, REG_A5XX_RBBM_PERFCTR_TP_4_HI,
+		REG_A5XX_TPL1_PERFCTR_TP_SEL_4 },
+	{ REG_A5XX_RBBM_PERFCTR_TP_5_LO, REG_A5XX_RBBM_PERFCTR_TP_5_HI,
+		REG_A5XX_TPL1_PERFCTR_TP_SEL_5 },
+	{ REG_A5XX_RBBM_PERFCTR_TP_6_LO, REG_A5XX_RBBM_PERFCTR_TP_6_HI,
+		REG_A5XX_TPL1_PERFCTR_TP_SEL_6 },
+	{ REG_A5XX_RBBM_PERFCTR_TP_7_LO, REG_A5XX_RBBM_PERFCTR_TP_7_HI,
+		REG_A5XX_TPL1_PERFCTR_TP_SEL_7 },
+};
+
+static struct adreno_counter a5xx_counters_tse[] = {
+	{ REG_A5XX_RBBM_PERFCTR_TSE_0_LO, REG_A5XX_RBBM_PERFCTR_TSE_0_HI,
+		REG_A5XX_GRAS_PERFCTR_TSE_SEL_0 },
+	{ REG_A5XX_RBBM_PERFCTR_TSE_1_LO, REG_A5XX_RBBM_PERFCTR_TSE_1_HI,
+		REG_A5XX_GRAS_PERFCTR_TSE_SEL_1 },
+	{ REG_A5XX_RBBM_PERFCTR_TSE_2_LO, REG_A5XX_RBBM_PERFCTR_TSE_2_HI,
+		REG_A5XX_GRAS_PERFCTR_TSE_SEL_2 },
+	{ REG_A5XX_RBBM_PERFCTR_TSE_3_LO, REG_A5XX_RBBM_PERFCTR_TSE_3_HI,
+		REG_A5XX_GRAS_PERFCTR_TSE_SEL_3 },
+};
+
+static struct adreno_counter a5xx_counters_uche[] = {
+	{ REG_A5XX_RBBM_PERFCTR_UCHE_0_LO, REG_A5XX_RBBM_PERFCTR_UCHE_0_HI,
+		REG_A5XX_UCHE_PERFCTR_UCHE_SEL_0 },
+	{ REG_A5XX_RBBM_PERFCTR_UCHE_1_LO, REG_A5XX_RBBM_PERFCTR_UCHE_1_HI,
+		REG_A5XX_UCHE_PERFCTR_UCHE_SEL_1 },
+	{ REG_A5XX_RBBM_PERFCTR_UCHE_2_LO, REG_A5XX_RBBM_PERFCTR_UCHE_2_HI,
+		REG_A5XX_UCHE_PERFCTR_UCHE_SEL_2 },
+	{ REG_A5XX_RBBM_PERFCTR_UCHE_3_LO, REG_A5XX_RBBM_PERFCTR_UCHE_3_HI,
+		REG_A5XX_UCHE_PERFCTR_UCHE_SEL_3 },
+	{ REG_A5XX_RBBM_PERFCTR_UCHE_4_LO, REG_A5XX_RBBM_PERFCTR_UCHE_4_HI,
+		REG_A5XX_UCHE_PERFCTR_UCHE_SEL_4 },
+	{ REG_A5XX_RBBM_PERFCTR_UCHE_5_LO, REG_A5XX_RBBM_PERFCTR_UCHE_5_HI,
+		REG_A5XX_UCHE_PERFCTR_UCHE_SEL_5 },
+	{ REG_A5XX_RBBM_PERFCTR_UCHE_6_LO, REG_A5XX_RBBM_PERFCTR_UCHE_6_HI,
+		REG_A5XX_UCHE_PERFCTR_UCHE_SEL_6 },
+	{ REG_A5XX_RBBM_PERFCTR_UCHE_7_LO, REG_A5XX_RBBM_PERFCTR_UCHE_7_HI,
+		REG_A5XX_UCHE_PERFCTR_UCHE_SEL_7 },
+};
+
+static struct adreno_counter a5xx_counters_vfd[] = {
+	{ REG_A5XX_RBBM_PERFCTR_VFD_0_LO, REG_A5XX_RBBM_PERFCTR_VFD_0_HI,
+		REG_A5XX_VFD_PERFCTR_VFD_SEL_0 },
+	{ REG_A5XX_RBBM_PERFCTR_VFD_1_LO, REG_A5XX_RBBM_PERFCTR_VFD_1_HI,
+		REG_A5XX_VFD_PERFCTR_VFD_SEL_1 },
+	{ REG_A5XX_RBBM_PERFCTR_VFD_2_LO, REG_A5XX_RBBM_PERFCTR_VFD_2_HI,
+		REG_A5XX_VFD_PERFCTR_VFD_SEL_2 },
+	{ REG_A5XX_RBBM_PERFCTR_VFD_3_LO, REG_A5XX_RBBM_PERFCTR_VFD_3_HI,
+		REG_A5XX_VFD_PERFCTR_VFD_SEL_3 },
+	{ REG_A5XX_RBBM_PERFCTR_VFD_4_LO, REG_A5XX_RBBM_PERFCTR_VFD_4_HI,
+		REG_A5XX_VFD_PERFCTR_VFD_SEL_4 },
+	{ REG_A5XX_RBBM_PERFCTR_VFD_5_LO, REG_A5XX_RBBM_PERFCTR_VFD_5_HI,
+		REG_A5XX_VFD_PERFCTR_VFD_SEL_5 },
+	{ REG_A5XX_RBBM_PERFCTR_VFD_6_LO, REG_A5XX_RBBM_PERFCTR_VFD_6_HI,
+		REG_A5XX_VFD_PERFCTR_VFD_SEL_6 },
+	{ REG_A5XX_RBBM_PERFCTR_VFD_7_LO, REG_A5XX_RBBM_PERFCTR_VFD_7_HI,
+		REG_A5XX_VFD_PERFCTR_VFD_SEL_7 },
+};
+
+static struct adreno_counter a5xx_counters_vpc[] = {
+	{ REG_A5XX_RBBM_PERFCTR_VPC_0_LO, REG_A5XX_RBBM_PERFCTR_VPC_0_HI,
+		REG_A5XX_VPC_PERFCTR_VPC_SEL_0 },
+	{ REG_A5XX_RBBM_PERFCTR_VPC_1_LO, REG_A5XX_RBBM_PERFCTR_VPC_1_HI,
+		REG_A5XX_VPC_PERFCTR_VPC_SEL_1 },
+	{ REG_A5XX_RBBM_PERFCTR_VPC_2_LO, REG_A5XX_RBBM_PERFCTR_VPC_2_HI,
+		REG_A5XX_VPC_PERFCTR_VPC_SEL_2 },
+	{ REG_A5XX_RBBM_PERFCTR_VPC_3_LO, REG_A5XX_RBBM_PERFCTR_VPC_3_HI,
+		REG_A5XX_VPC_PERFCTR_VPC_SEL_3 },
+};
+
+static struct adreno_counter a5xx_counters_vsc[] = {
+	{ REG_A5XX_RBBM_PERFCTR_VSC_0_LO, REG_A5XX_RBBM_PERFCTR_VSC_0_HI,
+		REG_A5XX_VSC_PERFCTR_VSC_SEL_0 },
+	{ REG_A5XX_RBBM_PERFCTR_VSC_1_LO, REG_A5XX_RBBM_PERFCTR_VSC_1_HI,
+		REG_A5XX_VSC_PERFCTR_VSC_SEL_1 },
+};
+
+static struct adreno_counter a5xx_counters_power_ccu[] = {
+	{ REG_A5XX_CCU_POWER_COUNTER_0_LO, REG_A5XX_CCU_POWER_COUNTER_0_HI,
+		REG_A5XX_RB_POWERCTR_CCU_SEL_0 },
+	{ REG_A5XX_CCU_POWER_COUNTER_1_LO, REG_A5XX_CCU_POWER_COUNTER_1_HI,
+		REG_A5XX_RB_POWERCTR_CCU_SEL_1 },
+};
+
+static struct adreno_counter a5xx_counters_power_cp[] = {
+	{ REG_A5XX_CP_POWER_COUNTER_0_LO, REG_A5XX_CP_POWER_COUNTER_0_HI,
+		REG_A5XX_CP_POWERCTR_CP_SEL_0 },
+	{ REG_A5XX_CP_POWER_COUNTER_1_LO, REG_A5XX_CP_POWER_COUNTER_1_HI,
+		REG_A5XX_CP_POWERCTR_CP_SEL_1 },
+	{ REG_A5XX_CP_POWER_COUNTER_2_LO, REG_A5XX_CP_POWER_COUNTER_2_HI,
+		REG_A5XX_CP_POWERCTR_CP_SEL_2 },
+	{ REG_A5XX_CP_POWER_COUNTER_3_LO, REG_A5XX_CP_POWER_COUNTER_3_HI,
+		REG_A5XX_CP_POWERCTR_CP_SEL_3 },
+};
+
+static struct adreno_counter a5xx_counters_power_rb[] = {
+	{ REG_A5XX_RB_POWER_COUNTER_0_LO, REG_A5XX_RB_POWER_COUNTER_0_HI,
+		REG_A5XX_RB_POWERCTR_RB_SEL_0 },
+	{ REG_A5XX_RB_POWER_COUNTER_1_LO, REG_A5XX_RB_POWER_COUNTER_1_HI,
+		REG_A5XX_RB_POWERCTR_RB_SEL_1 },
+	{ REG_A5XX_RB_POWER_COUNTER_2_LO, REG_A5XX_RB_POWER_COUNTER_2_HI,
+		REG_A5XX_RB_POWERCTR_RB_SEL_2 },
+	{ REG_A5XX_RB_POWER_COUNTER_3_LO, REG_A5XX_RB_POWER_COUNTER_3_HI,
+		REG_A5XX_RB_POWERCTR_RB_SEL_3 },
+};
+
+static struct adreno_counter a5xx_counters_power_sp[] = {
+	{ REG_A5XX_SP_POWER_COUNTER_0_LO, REG_A5XX_SP_POWER_COUNTER_0_HI,
+		REG_A5XX_SP_POWERCTR_SP_SEL_0 },
+	{ REG_A5XX_SP_POWER_COUNTER_1_LO, REG_A5XX_SP_POWER_COUNTER_1_HI,
+		REG_A5XX_SP_POWERCTR_SP_SEL_1 },
+	{ REG_A5XX_SP_POWER_COUNTER_2_LO, REG_A5XX_SP_POWER_COUNTER_2_HI,
+		REG_A5XX_SP_POWERCTR_SP_SEL_2 },
+	{ REG_A5XX_SP_POWER_COUNTER_3_LO, REG_A5XX_SP_POWER_COUNTER_3_HI,
+		REG_A5XX_SP_POWERCTR_SP_SEL_3 },
+};
+
+static struct adreno_counter a5xx_counters_power_tp[] = {
+	{ REG_A5XX_TP_POWER_COUNTER_0_LO, REG_A5XX_TP_POWER_COUNTER_0_HI,
+		REG_A5XX_TPL1_POWERCTR_TP_SEL_0 },
+	{ REG_A5XX_TP_POWER_COUNTER_1_LO, REG_A5XX_TP_POWER_COUNTER_1_HI,
+		REG_A5XX_TPL1_POWERCTR_TP_SEL_1 },
+	{ REG_A5XX_TP_POWER_COUNTER_2_LO, REG_A5XX_TP_POWER_COUNTER_2_HI,
+		REG_A5XX_TPL1_POWERCTR_TP_SEL_2 },
+	{ REG_A5XX_TP_POWER_COUNTER_3_LO, REG_A5XX_TP_POWER_COUNTER_3_HI,
+		REG_A5XX_TPL1_POWERCTR_TP_SEL_3 },
+};
+
+static struct adreno_counter a5xx_counters_power_uche[] = {
+	{ REG_A5XX_UCHE_POWER_COUNTER_0_LO, REG_A5XX_UCHE_POWER_COUNTER_0_HI,
+		REG_A5XX_UCHE_POWERCTR_UCHE_SEL_0 },
+	{ REG_A5XX_UCHE_POWER_COUNTER_1_LO, REG_A5XX_UCHE_POWER_COUNTER_1_HI,
+		REG_A5XX_UCHE_POWERCTR_UCHE_SEL_1 },
+	{ REG_A5XX_UCHE_POWER_COUNTER_2_LO, REG_A5XX_UCHE_POWER_COUNTER_2_HI,
+		REG_A5XX_UCHE_POWERCTR_UCHE_SEL_2 },
+	{ REG_A5XX_UCHE_POWER_COUNTER_3_LO, REG_A5XX_UCHE_POWER_COUNTER_3_HI,
+		REG_A5XX_UCHE_POWERCTR_UCHE_SEL_3 },
+};
+
+static struct adreno_counter a5xx_counters_vbif[] = {
+	{ REG_A5XX_VBIF_PERF_CNT_LOW0, REG_A5XX_VBIF_PERF_CNT_HIGH0 },
+	{ REG_A5XX_VBIF_PERF_CNT_LOW1, REG_A5XX_VBIF_PERF_CNT_HIGH1 },
+	{ REG_A5XX_VBIF_PERF_CNT_LOW2, REG_A5XX_VBIF_PERF_CNT_HIGH2 },
+	{ REG_A5XX_VBIF_PERF_CNT_LOW3, REG_A5XX_VBIF_PERF_CNT_HIGH3 },
+};
+
+static struct adreno_counter a5xx_counters_gpmu[] = {
+	{ REG_A5XX_GPMU_POWER_COUNTER_0_LO, REG_A5XX_GPMU_POWER_COUNTER_0_HI },
+	{ REG_A5XX_GPMU_POWER_COUNTER_1_LO, REG_A5XX_GPMU_POWER_COUNTER_1_HI },
+	{ REG_A5XX_GPMU_POWER_COUNTER_2_LO, REG_A5XX_GPMU_POWER_COUNTER_2_HI },
+	{ REG_A5XX_GPMU_POWER_COUNTER_3_LO, REG_A5XX_GPMU_POWER_COUNTER_3_HI },
+	{ REG_A5XX_GPMU_POWER_COUNTER_4_LO, REG_A5XX_GPMU_POWER_COUNTER_4_HI },
+	{ REG_A5XX_GPMU_POWER_COUNTER_5_LO, REG_A5XX_GPMU_POWER_COUNTER_5_HI },
+};
+
+static struct adreno_counter a5xx_counters_vbif_power[] = {
+	{ REG_A5XX_VBIF_PERF_PWR_CNT_LOW0, REG_A5XX_VBIF_PERF_PWR_CNT_HIGH0 },
+	{ REG_A5XX_VBIF_PERF_PWR_CNT_LOW1, REG_A5XX_VBIF_PERF_PWR_CNT_HIGH1 },
+	{ REG_A5XX_VBIF_PERF_PWR_CNT_LOW2, REG_A5XX_VBIF_PERF_PWR_CNT_HIGH2 },
+};
+
+static struct adreno_counter a5xx_counters_alwayson_power[] = {
+	{ REG_A5XX_GPMU_ALWAYS_ON_COUNTER_LO,
+		REG_A5XX_GPMU_ALWAYS_ON_COUNTER_HI },
+};
+
+#define DEFINE_COUNTER_GROUP(_name, _array, _get, _enable, _put) \
+static struct adreno_counter_group _name = { \
+	.counters = _array, \
+	.nr_counters = ARRAY_SIZE(_array), \
+	.lock = __SPIN_LOCK_UNLOCKED(_name.lock), \
+	.funcs = { \
+		.get = _get, \
+		.enable = _enable, \
+		.read = a5xx_counter_read, \
+		.put = _put, \
+	}, \
+}
+
+#define DEFAULT_COUNTER_GROUP(_name, _array) DEFINE_COUNTER_GROUP(_name, \
+	_array, a5xx_counter_get, a5xx_counter_enable_cpu, a5xx_counter_put)
+
+#define SPTP_COUNTER_GROUP(_name, _array) DEFINE_COUNTER_GROUP(_name, \
+	_array, a5xx_counter_get, a5xx_counter_enable_pm4, a5xx_counter_put)
+
+/* "standard" counters */
+DEFAULT_COUNTER_GROUP(a5xx_counter_group_cp, a5xx_counters_cp);
+DEFAULT_COUNTER_GROUP(a5xx_counter_group_rbbm, a5xx_counters_rbbm);
+DEFAULT_COUNTER_GROUP(a5xx_counter_group_pc, a5xx_counters_pc);
+DEFAULT_COUNTER_GROUP(a5xx_counter_group_vfd, a5xx_counters_vfd);
+DEFAULT_COUNTER_GROUP(a5xx_counter_group_vpc, a5xx_counters_vpc);
+DEFAULT_COUNTER_GROUP(a5xx_counter_group_ccu, a5xx_counters_ccu);
+DEFAULT_COUNTER_GROUP(a5xx_counter_group_cmp, a5xx_counters_cmp);
+DEFAULT_COUNTER_GROUP(a5xx_counter_group_tse, a5xx_counters_tse);
+DEFAULT_COUNTER_GROUP(a5xx_counter_group_ras, a5xx_counters_ras);
+DEFAULT_COUNTER_GROUP(a5xx_counter_group_uche, a5xx_counters_uche);
+DEFAULT_COUNTER_GROUP(a5xx_counter_group_rb, a5xx_counters_rb);
+DEFAULT_COUNTER_GROUP(a5xx_counter_group_vsc, a5xx_counters_vsc);
+DEFAULT_COUNTER_GROUP(a5xx_counter_group_lrz, a5xx_counters_lrz);
+
+/* SP/TP counters */
+SPTP_COUNTER_GROUP(a5xx_counter_group_hlsq, a5xx_counters_hlsq);
+SPTP_COUNTER_GROUP(a5xx_counter_group_tp, a5xx_counters_tp);
+SPTP_COUNTER_GROUP(a5xx_counter_group_sp, a5xx_counters_sp);
+
+/* Power counters */
+DEFAULT_COUNTER_GROUP(a5xx_counter_group_power_ccu, a5xx_counters_power_ccu);
+DEFAULT_COUNTER_GROUP(a5xx_counter_group_power_cp, a5xx_counters_power_cp);
+DEFAULT_COUNTER_GROUP(a5xx_counter_group_power_rb, a5xx_counters_power_rb);
+DEFAULT_COUNTER_GROUP(a5xx_counter_group_power_sp, a5xx_counters_power_sp);
+DEFAULT_COUNTER_GROUP(a5xx_counter_group_power_tp, a5xx_counters_power_tp);
+DEFAULT_COUNTER_GROUP(a5xx_counter_group_power_uche, a5xx_counters_power_uche);
+
+DEFINE_COUNTER_GROUP(a5xx_counter_group_alwayson, a5xx_counters_alwayson,
+	a5xx_counter_get_fixed, NULL, NULL);
+DEFINE_COUNTER_GROUP(a5xx_counter_group_vbif, a5xx_counters_vbif,
+	a5xx_counter_get, a5xx_counter_enable_vbif, a5xx_counter_put);
+DEFINE_COUNTER_GROUP(a5xx_counter_group_gpmu, a5xx_counters_gpmu,
+	a5xx_counter_get, a5xx_counter_enable_gpmu, a5xx_counter_put);
+DEFINE_COUNTER_GROUP(a5xx_counter_group_vbif_power, a5xx_counters_vbif_power,
+	a5xx_counter_get_fixed, a5xx_counter_enable_vbif_power, NULL);
+DEFINE_COUNTER_GROUP(a5xx_counter_group_alwayson_power,
+		a5xx_counters_alwayson_power, a5xx_counter_get_fixed,
+		a5xx_counter_enable_alwayson_power, NULL);
+
+static const struct adreno_counter_group *a5xx_counter_groups[] = {
+	[MSM_COUNTER_GROUP_ALWAYSON] = &a5xx_counter_group_alwayson,
+	[MSM_COUNTER_GROUP_CCU] = &a5xx_counter_group_ccu,
+	[MSM_COUNTER_GROUP_CMP] = &a5xx_counter_group_cmp,
+	[MSM_COUNTER_GROUP_CP] = &a5xx_counter_group_cp,
+	[MSM_COUNTER_GROUP_HLSQ] = &a5xx_counter_group_hlsq,
+	[MSM_COUNTER_GROUP_LRZ] = &a5xx_counter_group_lrz,
+	[MSM_COUNTER_GROUP_PC] = &a5xx_counter_group_pc,
+	[MSM_COUNTER_GROUP_RAS] = &a5xx_counter_group_ras,
+	[MSM_COUNTER_GROUP_RB] = &a5xx_counter_group_rb,
+	[MSM_COUNTER_GROUP_RBBM] = &a5xx_counter_group_rbbm,
+	[MSM_COUNTER_GROUP_SP] = &a5xx_counter_group_sp,
+	[MSM_COUNTER_GROUP_TP] = &a5xx_counter_group_tp,
+	[MSM_COUNTER_GROUP_TSE] = &a5xx_counter_group_tse,
+	[MSM_COUNTER_GROUP_UCHE] = &a5xx_counter_group_uche,
+	[MSM_COUNTER_GROUP_VFD] = &a5xx_counter_group_vfd,
+	[MSM_COUNTER_GROUP_VPC] = &a5xx_counter_group_vpc,
+	[MSM_COUNTER_GROUP_VSC] = &a5xx_counter_group_vsc,
+	[MSM_COUNTER_GROUP_VBIF] = &a5xx_counter_group_vbif,
+	[MSM_COUNTER_GROUP_GPMU_PWR] = &a5xx_counter_group_gpmu,
+	[MSM_COUNTER_GROUP_CCU_PWR] = &a5xx_counter_group_power_ccu,
+	[MSM_COUNTER_GROUP_CP_PWR] = &a5xx_counter_group_power_cp,
+	[MSM_COUNTER_GROUP_RB_PWR] = &a5xx_counter_group_power_rb,
+	[MSM_COUNTER_GROUP_SP_PWR] = &a5xx_counter_group_power_sp,
+	[MSM_COUNTER_GROUP_TP_PWR] = &a5xx_counter_group_power_tp,
+	[MSM_COUNTER_GROUP_UCHE_PWR] = &a5xx_counter_group_power_uche,
+	[MSM_COUNTER_GROUP_VBIF_PWR] = &a5xx_counter_group_vbif_power,
+	[MSM_COUNTER_GROUP_ALWAYSON_PWR] =
+		&a5xx_counter_group_alwayson_power,
+};
+
+int a5xx_counters_init(struct adreno_gpu *adreno_gpu)
+{
+	adreno_gpu->counter_groups = a5xx_counter_groups;
+	adreno_gpu->nr_counter_groups = ARRAY_SIZE(a5xx_counter_groups);
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c
index b5140d2c772a..02c4f2e3155d 100644
--- a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c
@@ -1210,6 +1210,9 @@ static const struct adreno_gpu_funcs funcs = {
 		.show = a5xx_show,
 #endif
 		.snapshot = a5xx_snapshot,
+		.get_counter = adreno_get_counter,
+		.read_counter = adreno_read_counter,
+		.put_counter = adreno_put_counter,
 	},
 	.get_timestamp = a5xx_get_timestamp,
 };
@@ -1333,5 +1336,7 @@ struct msm_gpu *a5xx_gpu_init(struct drm_device *dev)
 	/* Set up the preemption specific bits and pieces for each ringbuffer */
 	a5xx_preempt_init(gpu);
 
+	a5xx_counters_init(adreno_gpu);
+
 	return gpu;
 }
diff --git a/drivers/gpu/drm/msm/adreno/a5xx_gpu.h b/drivers/gpu/drm/msm/adreno/a5xx_gpu.h
index 3de14fe42a1b..8eb3838ffe90 100644
--- a/drivers/gpu/drm/msm/adreno/a5xx_gpu.h
+++ b/drivers/gpu/drm/msm/adreno/a5xx_gpu.h
@@ -184,4 +184,6 @@ static inline bool a5xx_in_preempt(struct a5xx_gpu *a5xx_gpu)
 	return !(atomic_read(&a5xx_gpu->preempt_state) == PREEMPT_NONE);
 }
 
+int a5xx_counters_init(struct adreno_gpu *adreno_gpu);
+
 #endif /* __A5XX_GPU_H__ */
diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
index f1883825354e..969ed810ce9d 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
@@ -709,3 +709,52 @@ void adreno_snapshot(struct msm_gpu *gpu, struct msm_snapshot *snapshot)
 	adreno_snapshot_os(gpu, snapshot);
 	adreno_snapshot_ringbuffers(gpu, snapshot);
 }
+
+/* Return the group struct associated with the counter id */
+
+static struct adreno_counter_group *get_counter_group(struct msm_gpu *gpu,
+		u32 groupid)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+
+	if (!adreno_gpu->counter_groups)
+		return ERR_PTR(-ENODEV);
+
+	if (groupid >= adreno_gpu->nr_counter_groups)
+		return ERR_PTR(-EINVAL);
+
+	return (struct adreno_counter_group *)
+		adreno_gpu->counter_groups[groupid];
+}
+
+int adreno_get_counter(struct msm_gpu *gpu, u32 groupid, u32 countable,
+		u32 *lo, u32 *hi)
+{
+	struct adreno_counter_group *group =
+		get_counter_group(gpu, groupid);
+
+	if (!IS_ERR_OR_NULL(group) && group->funcs.get)
+		return group->funcs.get(gpu, group, countable, lo, hi);
+
+	return -ENODEV;
+}
+
+u64 adreno_read_counter(struct msm_gpu *gpu, u32 groupid, int counterid)
+{
+	struct adreno_counter_group *group =
+		get_counter_group(gpu, groupid);
+
+	if (!IS_ERR(group) && group->funcs.read)
+		return group->funcs.read(gpu, group, counterid);
+
+	return 0;
+}
+
+void adreno_put_counter(struct msm_gpu *gpu, u32 groupid, int counterid)
+{
+	struct adreno_counter_group *group =
+		get_counter_group(gpu, groupid);
+
+	if (!IS_ERR(group) && group->funcs.put)
+		group->funcs.put(gpu, group, counterid);
+}
diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.h b/drivers/gpu/drm/msm/adreno/adreno_gpu.h
index 30461115281c..8e8f3e5182d6 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_gpu.h
+++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.h
@@ -99,6 +99,30 @@ struct adreno_rbmemptrs {
 	volatile unsigned int contextidr[MSM_GPU_MAX_RINGS];
 };
 
+struct adreno_counter {
+	u32 lo;
+	u32 hi;
+	u32 sel;
+	u32 countable;
+	u32 refcount;
+};
+
+struct adreno_counter_group {
+	struct adreno_counter *counters;
+	size_t nr_counters;
+	spinlock_t lock;
+	struct {
+		int (*get)(struct msm_gpu *,
+			struct adreno_counter_group *, u32, u32 *, u32 *);
+		void (*enable)(struct msm_gpu *,
+			struct adreno_counter_group *, int);
+		u64 (*read)(struct msm_gpu *,
+			struct adreno_counter_group *, int);
+		void (*put)(struct msm_gpu *,
+			struct adreno_counter_group *, int);
+	} funcs;
+};
+
 struct adreno_gpu {
 	struct msm_gpu base;
 	struct adreno_rev rev;
@@ -129,6 +153,9 @@ struct adreno_gpu {
 
 	uint32_t quirks;
 	uint32_t speed_bin;
+
+	const struct adreno_counter_group **counter_groups;
+	int nr_counter_groups;
 };
 #define to_adreno_gpu(x) container_of(x, struct adreno_gpu, base)
 
@@ -235,6 +262,11 @@ void adreno_gpu_cleanup(struct adreno_gpu *gpu);
 
 void adreno_snapshot(struct msm_gpu *gpu, struct msm_snapshot *snapshot);
 
+int adreno_get_counter(struct msm_gpu *gpu, u32 groupid, u32 countable,
+		u32 *lo, u32 *hi);
+u64 adreno_read_counter(struct msm_gpu *gpu, u32 groupid, int counterid);
+void adreno_put_counter(struct msm_gpu *gpu, u32 groupid, int counterid);
+
 /* ringbuffer helpers (the parts that are adreno specific) */
 
 static inline void
diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c
index 532ff8677259..276329b7b10c 100644
--- a/drivers/gpu/drm/msm/msm_drv.c
+++ b/drivers/gpu/drm/msm/msm_drv.c
@@ -606,6 +606,8 @@ static int msm_open(struct drm_device *dev, struct drm_file *file)
 	if (IS_ERR(ctx))
 		return PTR_ERR(ctx);
 
+	INIT_LIST_HEAD(&ctx->counters);
+
 	file->driver_priv = ctx;
 
 	kms = priv->kms;
@@ -634,6 +636,9 @@ static void msm_postclose(struct drm_device *dev, struct drm_file *file)
 	if (kms && kms->funcs && kms->funcs->postclose)
 		kms->funcs->postclose(kms, file);
 
+	if (priv->gpu)
+		msm_gpu_cleanup_counters(priv->gpu, ctx);
+
 	mutex_lock(&dev->struct_mutex);
 	if (ctx && ctx->aspace && ctx->aspace != priv->gpu->aspace) {
 		ctx->aspace->mmu->funcs->detach(ctx->aspace->mmu);
@@ -1584,6 +1589,41 @@ void msm_send_crtc_notification(struct drm_crtc *crtc,
 	spin_unlock_irqrestore(&dev->event_lock, flags);
 }
 
+static int msm_ioctl_counter_get(struct drm_device *dev, void *data,
+		struct drm_file *file)
+{
+	struct msm_file_private *ctx = file->driver_priv;
+	struct msm_drm_private *priv = dev->dev_private;
+
+	if (priv->gpu)
+		return msm_gpu_counter_get(priv->gpu, data, ctx);
+
+	return -ENODEV;
+}
+
+static int msm_ioctl_counter_put(struct drm_device *dev, void *data,
+		struct drm_file *file)
+{
+	struct msm_file_private *ctx = file->driver_priv;
+	struct msm_drm_private *priv = dev->dev_private;
+
+	if (priv->gpu)
+		return msm_gpu_counter_put(priv->gpu, data, ctx);
+
+	return -ENODEV;
+}
+
+static int msm_ioctl_counter_read(struct drm_device *dev, void *data,
+		struct drm_file *file)
+{
+	struct msm_drm_private *priv = dev->dev_private;
+
+	if (priv->gpu)
+		return msm_gpu_counter_read(priv->gpu, data);
+
+	return -ENODEV;
+}
+
 int msm_release(struct inode *inode, struct file *filp)
 {
 	struct drm_file *file_priv = filp->private_data;
@@ -1619,6 +1659,12 @@ static const struct drm_ioctl_desc msm_ioctls[] = {
 			  DRM_UNLOCKED|DRM_CONTROL_ALLOW),
 	DRM_IOCTL_DEF_DRV(MSM_DEREGISTER_EVENT,  msm_ioctl_deregister_event,
 			  DRM_UNLOCKED|DRM_CONTROL_ALLOW),
+	DRM_IOCTL_DEF_DRV(MSM_COUNTER_GET, msm_ioctl_counter_get,
+			  DRM_AUTH|DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(MSM_COUNTER_PUT, msm_ioctl_counter_put,
+			  DRM_AUTH|DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(MSM_COUNTER_READ, msm_ioctl_counter_read,
+			  DRM_AUTH|DRM_RENDER_ALLOW),
 };
 
 static const struct vm_operations_struct vm_ops = {
diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h
index d8a4c34e9be0..d2d118cf7e07 100644
--- a/drivers/gpu/drm/msm/msm_drv.h
+++ b/drivers/gpu/drm/msm/msm_drv.h
@@ -76,6 +76,7 @@ struct msm_gem_vma;
 
 struct msm_file_private {
 	struct msm_gem_address_space *aspace;
+	struct list_head counters;
 };
 
 enum msm_mdp_plane_property {
diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c
index 14c0cfc58270..5a505a8bf328 100644
--- a/drivers/gpu/drm/msm/msm_gpu.c
+++ b/drivers/gpu/drm/msm/msm_gpu.c
@@ -587,6 +587,118 @@ int msm_gpu_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)
 	return ret;
 }
 
+struct msm_context_counter {
+	u32 groupid;
+	int counterid;
+	struct list_head node;
+};
+
+int msm_gpu_counter_get(struct msm_gpu *gpu, struct drm_msm_counter *data,
+	struct msm_file_private *ctx)
+{
+	struct msm_context_counter *entry;
+	int counterid;
+	u32 lo = 0, hi = 0;
+
+	if (!ctx || !gpu->funcs->get_counter)
+		return -ENODEV;
+
+	counterid = gpu->funcs->get_counter(gpu, data->groupid, data->countable,
+		&lo, &hi);
+
+	if (counterid < 0)
+		return counterid;
+
+	/*
+	 * Check to see if the counter in question is already held by this
+	 * process. If it does, put it back and return an error.
+	 */
+	list_for_each_entry(entry, &ctx->counters, node) {
+		if (entry->groupid == data->groupid &&
+			entry->counterid == counterid) {
+			gpu->funcs->put_counter(gpu, data->groupid, counterid);
+			return -EBUSY;
+		}
+	}
+
+	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
+	if (!entry) {
+		gpu->funcs->put_counter(gpu, data->groupid, counterid);
+		return -ENOMEM;
+	}
+
+	entry->groupid = data->groupid;
+	entry->counterid = counterid;
+	list_add_tail(&entry->node, &ctx->counters);
+
+	data->counterid = counterid;
+	data->counter_lo = lo;
+	data->counter_hi = hi;
+
+	return 0;
+}
+
+int msm_gpu_counter_put(struct msm_gpu *gpu, struct drm_msm_counter *data,
+	struct msm_file_private *ctx)
+{
+	struct msm_context_counter *entry;
+
+	list_for_each_entry(entry, &ctx->counters, node) {
+		if (entry->groupid == data->groupid &&
+			entry->counterid == data->counterid) {
+			gpu->funcs->put_counter(gpu, data->groupid,
+				data->counterid);
+
+			list_del(&entry->node);
+			kfree(entry);
+
+			return 0;
+		}
+	}
+
+	return -EINVAL;
+}
+
+void msm_gpu_cleanup_counters(struct msm_gpu *gpu,
+	struct msm_file_private *ctx)
+{
+	struct msm_context_counter *entry, *tmp;
+
+	if (!ctx)
+		return;
+
+	list_for_each_entry_safe(entry, tmp, &ctx->counters, node) {
+		gpu->funcs->put_counter(gpu, entry->groupid, entry->counterid);
+		list_del(&entry->node);
+		kfree(entry);
+	}
+}
+
+u64 msm_gpu_counter_read(struct msm_gpu *gpu, struct drm_msm_counter_read *data)
+{
+	int i;
+
+	if (!gpu->funcs->read_counter)
+		return 0;
+
+	for (i = 0; i < data->nr_ops; i++) {
+		struct drm_msm_counter_read_op op;
+		void __user *ptr = (void __user *)(uintptr_t)
+			(data->ops + (i * sizeof(op)));
+
+		if (copy_from_user(&op, ptr, sizeof(op)))
+			return -EFAULT;
+
+		op.value = gpu->funcs->read_counter(gpu, op.groupid,
+			op.counterid);
+
+		if (copy_to_user(ptr, &op, sizeof(op)))
+			return -EFAULT;
+	}
+
+	return 0;
+}
+
 /*
  * Init/Cleanup:
  */
diff --git a/drivers/gpu/drm/msm/msm_gpu.h b/drivers/gpu/drm/msm/msm_gpu.h
index 06dfaabbfcfe..3fac423929c5 100644
--- a/drivers/gpu/drm/msm/msm_gpu.h
+++ b/drivers/gpu/drm/msm/msm_gpu.h
@@ -71,6 +71,10 @@ struct msm_gpu_funcs {
 	void (*show)(struct msm_gpu *gpu, struct seq_file *m);
 #endif
 	int (*snapshot)(struct msm_gpu *gpu, struct msm_snapshot *snapshot);
+	int (*get_counter)(struct msm_gpu *gpu, u32 groupid, u32 countable,
+		u32 *lo, u32 *hi);
+	void (*put_counter)(struct msm_gpu *gpu, u32 groupid, int counterid);
+	u64 (*read_counter)(struct msm_gpu *gpu, u32 groupid, int counterid);
 };
 
 struct msm_gpu {
@@ -258,4 +262,16 @@ struct msm_gpu *adreno_load_gpu(struct drm_device *dev);
 void __init adreno_register(void);
 void __exit adreno_unregister(void);
 
+int msm_gpu_counter_get(struct msm_gpu *gpu, struct drm_msm_counter *data,
+	struct msm_file_private *ctx);
+
+int msm_gpu_counter_put(struct msm_gpu *gpu, struct drm_msm_counter *data,
+	struct msm_file_private *ctx);
+
+void msm_gpu_cleanup_counters(struct msm_gpu *gpu,
+	struct msm_file_private *ctx);
+
+u64 msm_gpu_counter_read(struct msm_gpu *gpu,
+		struct drm_msm_counter_read *data);
+
 #endif /* __MSM_GPU_H__ */