lpm-stats: Optimize stats path

Stats for cpu and cluster exit path query the ktime for every level and
this increases the overall idle exit latency. Get the timestamp only
once and use the same information to populate into different level.

Change-Id: Iece36015910fcb16bd04a25dae1bf0396ab463e2
Signed-off-by: Archana Sathyakumar <asathyak@codeaurora.org>

Conflicts:
	drivers/cpuidle/lpm-levels.c
	drivers/power/qcom/lpm-stats.c

3 files changed, 69 insertions, 48 deletions

diff --git a/drivers/cpuidle/lpm-levels.c b/drivers/cpuidle/lpm-levels.c
index 08753375b4ea..d4d598bcf0d6 100644
--- a/drivers/cpuidle/lpm-levels.c
+++ b/drivers/cpuidle/lpm-levels.c
@@ -92,9 +92,11 @@ static int lpm_cpu_callback(struct notifier_block *cpu_nb,
 				unsigned long action, void *hcpu);
 
 static void cluster_unprepare(struct lpm_cluster *cluster,
-		const struct cpumask *cpu, int child_idx, bool from_idle);
+		const struct cpumask *cpu, int child_idx, bool from_idle,
+		int64_t time);
 static void cluster_prepare(struct lpm_cluster *cluster,
-		const struct cpumask *cpu, int child_idx, bool from_idle);
+		const struct cpumask *cpu, int child_idx, bool from_idle,
+		int64_t time);
 
 static struct notifier_block __refdata lpm_cpu_nblk = {
 	.notifier_call = lpm_cpu_callback,
@@ -173,11 +175,11 @@ static int lpm_cpu_callback(struct notifier_block *cpu_nb,
 	switch (action & ~CPU_TASKS_FROZEN) {
 	case CPU_DYING:
 		cluster_prepare(cluster, get_cpu_mask((unsigned int) cpu),
-					NR_LPM_LEVELS, false);
+					NR_LPM_LEVELS, false, 0);
 		break;
 	case CPU_STARTING:
 		cluster_unprepare(cluster, get_cpu_mask((unsigned int) cpu),
-					NR_LPM_LEVELS, false);
+					NR_LPM_LEVELS, false, 0);
 		break;
 	default:
 		break;
@@ -559,7 +561,8 @@ failed_set_mode:
 }
 
 static void cluster_prepare(struct lpm_cluster *cluster,
-		const struct cpumask *cpu, int child_idx, bool from_idle)
+		const struct cpumask *cpu, int child_idx, bool from_idle,
+		int64_t start_time)
 {
 	int i;
 
@@ -600,15 +603,21 @@ static void cluster_prepare(struct lpm_cluster *cluster,
 	if (cluster_configure(cluster, i, from_idle))
 		goto failed;
 
+	cluster->stats->sleep_time = start_time;
 	cluster_prepare(cluster->parent, &cluster->num_children_in_sync, i,
-			from_idle);
+			from_idle, start_time);
+
+	spin_unlock(&cluster->sync_lock);
+	return;
 failed:
 	spin_unlock(&cluster->sync_lock);
+	cluster->stats->sleep_time = 0;
 	return;
 }
 
 static void cluster_unprepare(struct lpm_cluster *cluster,
-		const struct cpumask *cpu, int child_idx, bool from_idle)
+		const struct cpumask *cpu, int child_idx, bool from_idle,
+		int64_t end_time)
 {
 	struct lpm_cluster_level *level;
 	bool first_cpu;
@@ -638,6 +647,9 @@ static void cluster_unprepare(struct lpm_cluster *cluster,
 	if (!first_cpu || cluster->last_level == cluster->default_level)
 		goto unlock_return;
 
+	if (cluster->stats->sleep_time)
+		cluster->stats->sleep_time = end_time -
+			cluster->stats->sleep_time;
 	lpm_stats_cluster_exit(cluster->stats, cluster->last_level, true);
 
 	level = &cluster->levels[cluster->last_level];
@@ -673,7 +685,7 @@ static void cluster_unprepare(struct lpm_cluster *cluster,
 
 	cluster_notify(cluster, &cluster->levels[last_level], false);
 	cluster_unprepare(cluster->parent, &cluster->child_cpus,
-			last_level, from_idle);
+			last_level, from_idle, end_time);
 unlock_return:
 	spin_unlock(&cluster->sync_lock);
 }
@@ -872,19 +884,21 @@ static int lpm_cpuidle_enter(struct cpuidle_device *dev,
 	pwr_params = &cluster->cpu->levels[idx].pwr;
 
 	cpu_prepare(cluster, idx, true);
-	cluster_prepare(cluster, cpumask, idx, true);
+	cluster_prepare(cluster, cpumask, idx, true, ktime_to_ns(ktime_get()));
 
 	if (need_resched() || (idx < 0))
 		goto exit;
 
 	BUG_ON(!use_psci);
 	trace_cpu_idle_enter(idx);
-	lpm_stats_cpu_enter(idx);
+	lpm_stats_cpu_enter(idx, start_time);
 	success = psci_enter_sleep(cluster, idx, true);
 
 exit:
-	lpm_stats_cpu_exit(idx, success);
-	cluster_unprepare(cluster, cpumask, idx, true);
+	end_time = ktime_to_ns(ktime_get());
+	lpm_stats_cpu_exit(idx, end_time, success);
+
+	cluster_unprepare(cluster, cpumask, idx, true, end_time);
 	cpu_unprepare(cluster, idx, true);
 
 	trace_cpu_idle_exit(idx, success);
@@ -1095,6 +1109,7 @@ static int lpm_suspend_enter(suspend_state_t state)
 	struct lpm_cpu *lpm_cpu = cluster->cpu;
 	const struct cpumask *cpumask = get_cpu_mask(cpu);
 	int idx;
+	int64_t time = ktime_to_ns(ktime_get());
 
 	for (idx = lpm_cpu->nlevels - 1; idx >= 0; idx--) {
 
@@ -1106,7 +1121,7 @@ static int lpm_suspend_enter(suspend_state_t state)
 		return 0;
 	}
 	cpu_prepare(cluster, idx, false);
-	cluster_prepare(cluster, cpumask, idx, false);
+	cluster_prepare(cluster, cpumask, idx, false, time);
 	if (idx > 0)
 		update_debug_pc_event(CPU_ENTER, idx, 0xdeaffeed,
 					0xdeaffeed, false);
@@ -1125,7 +1140,9 @@ static int lpm_suspend_enter(suspend_state_t state)
 	if (idx > 0)
 		update_debug_pc_event(CPU_EXIT, idx, true, 0xdeaffeed,
 					false);
-	cluster_unprepare(cluster, cpumask, idx, false);
+
+	time = ktime_to_ns(ktime_get());
+	cluster_unprepare(cluster, cpumask, idx, false, time);
 	cpu_unprepare(cluster, idx, false);
 	return 0;
 }
diff --git a/drivers/power/qcom/lpm-stats.c b/drivers/power/qcom/lpm-stats.c
index 3b855ea33b50..d3cafc411a77 100644
--- a/drivers/power/qcom/lpm-stats.c
+++ b/drivers/power/qcom/lpm-stats.c
@@ -24,6 +24,7 @@
 #include <linux/suspend.h>
 #include <soc/qcom/spm.h>
 #include <soc/qcom/pm.h>
+#include <soc/qcom/lpm-stats.h>
 
 #define MAX_STR_LEN 256
 #define MAX_TIME_LEN 20
@@ -48,24 +49,6 @@ struct level_stats {
 	uint64_t enter_time;
 };
 
-struct lifo_stats {
-	uint32_t last_in;
-	uint32_t first_out;
-};
-
-struct lpm_stats {
-	char name[MAX_STR_LEN];
-	struct level_stats *time_stats;
-	uint32_t num_levels;
-	struct lifo_stats lifo;
-	struct lpm_stats *parent;
-	struct list_head sibling;
-	struct list_head child;
-	struct cpumask mask;
-	struct dentry *directory;
-	bool is_cpu;
-};
-
 static struct level_stats suspend_time_stats;
 
 static DEFINE_PER_CPU_SHARED_ALIGNED(struct lpm_stats, cpu_stats);
@@ -443,13 +426,9 @@ static inline void update_exit_stats(struct lpm_stats *stats, uint32_t index,
 	uint64_t exit_time = 0;
 
 	/* Update time stats only when exit is preceded by enter */
-	if (stats->time_stats[index].enter_time) {
-		exit_time = sched_clock() -
-				stats->time_stats[index].enter_time;
-		update_level_stats(&stats->time_stats[index], exit_time,
+	exit_time = stats->sleep_time;
+	update_level_stats(&stats->time_stats[index], exit_time,
 					success);
-		stats->time_stats[index].enter_time = 0;
-	}
 }
 
 static int config_level(const char *name, const char **levels,
@@ -792,8 +771,6 @@ void lpm_stats_cluster_enter(struct lpm_stats *stats, uint32_t index)
 	if (IS_ERR_OR_NULL(stats))
 		return;
 
-	stats->time_stats[index].enter_time = sched_clock();
-
 	update_last_in_stats(stats);
 }
 EXPORT_SYMBOL(lpm_stats_cluster_enter);
@@ -830,14 +807,15 @@ EXPORT_SYMBOL(lpm_stats_cluster_exit);
  * Function to communicate the low power mode level that the cpu is
  * prepared to enter.
  */
-void lpm_stats_cpu_enter(uint32_t index)
+void lpm_stats_cpu_enter(uint32_t index, uint64_t time)
 {
 	struct lpm_stats *stats = &(*this_cpu_ptr(&(cpu_stats)));
 
+	stats->sleep_time = time;
+
 	if (!stats->time_stats)
 		return;
 
-	stats->time_stats[index].enter_time = sched_clock();
 }
 EXPORT_SYMBOL(lpm_stats_cpu_enter);
 
@@ -849,13 +827,15 @@ EXPORT_SYMBOL(lpm_stats_cpu_enter);
  *
  * Function to communicate the low power mode level that the cpu exited.
  */
-void lpm_stats_cpu_exit(uint32_t index, bool success)
+void lpm_stats_cpu_exit(uint32_t index, uint64_t time, bool success)
 {
 	struct lpm_stats *stats = &(*this_cpu_ptr(&(cpu_stats)));
 
 	if (!stats->time_stats)
 		return;
 
+	stats->sleep_time = time - stats->sleep_time;
+
 	update_exit_stats(stats, index, success);
 }
 EXPORT_SYMBOL(lpm_stats_cpu_exit);
diff --git a/include/soc/qcom/lpm-stats.h b/include/soc/qcom/lpm-stats.h
index 05f5516f4177..d689d6a8d3db 100644
--- a/include/soc/qcom/lpm-stats.h
+++ b/include/soc/qcom/lpm-stats.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2014-2015, The Linux Foundation. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 and
@@ -17,6 +17,29 @@
 
 struct lpm_stats;
 
+#define MAX_STR_LEN 256
+
+struct lifo_stats {
+	uint32_t last_in;
+	uint32_t first_out;
+};
+
+struct lpm_stats {
+	char name[MAX_STR_LEN];
+	struct level_stats *time_stats;
+	uint32_t num_levels;
+	struct lifo_stats lifo;
+	struct lpm_stats *parent;
+	struct list_head sibling;
+	struct list_head child;
+	struct cpumask mask;
+	struct dentry *directory;
+	int64_t sleep_time;
+	bool is_cpu;
+};
+
+
+
 #ifdef CONFIG_MSM_IDLE_STATS
 struct lpm_stats *lpm_stats_config_level(const char *name,
 	const char **levels, int num_levels, struct lpm_stats *parent,
@@ -24,8 +47,8 @@ struct lpm_stats *lpm_stats_config_level(const char *name,
 void lpm_stats_cluster_enter(struct lpm_stats *stats, uint32_t index);
 void lpm_stats_cluster_exit(struct lpm_stats *stats, uint32_t index,
 				bool success);
-void lpm_stats_cpu_enter(uint32_t index);
-void lpm_stats_cpu_exit(uint32_t index, bool success);
+void lpm_stats_cpu_enter(uint32_t index, uint64_t time);
+void lpm_stats_cpu_exit(uint32_t index, uint64_t time, bool success);
 void lpm_stats_suspend_enter(void);
 void lpm_stats_suspend_exit(void);
 #else
@@ -48,12 +71,13 @@ static inline void lpm_stats_cluster_exit(struct lpm_stats *stats,
 	return;
 }
 
-static inline void lpm_stats_cpu_enter(uint32_t index)
+static inline void lpm_stats_cpu_enter(uint32_t index, uint64_t time)
 {
 	return;
 }
 
-static inline void lpm_stats_cpu_exit(uint32_t index, bool success)
+static inline void lpm_stats_cpu_exit(uint32_t index, bool success,
+							uint64_t time)
 {
 	return;
 }