sched: Notify cpufreq governor early about potential big tasks

Tasks that are on the runqueue continuously for a certain amount of time
have the potential to be big tasks at the end of the window in which they
are runnable. In such scenarios ramping the CPU frequency early can
boost performance rather than waiting till the end of a window for the
governor to query load. Notify the governor early at every tick when a
task has been observed to execute beyond some percentage of the tick
period.

The threshold beyond which a task is eligible for early detection can be
changed via the tunable sched_early_detection_duration. The feature itself
is enabled only when scheduler boost is in effect.

Change-Id: I528b72bbc79a55b4593d1b8ab45450411c6d70f3
Signed-off-by: Syed Rameez Mustafa <rameezmustafa@codeaurora.org>
[joonwoop@codeaurora.org: fixed conflict in scheduler_tick() in
 kernel/sched/core.c.  fixed minor conflicts in include/linux/sched.h,
 include/linux/sched/sysctl.h and kernel/sysctl.c due to
 CONFIG_SCHED_QHMP.]
Signed-off-by: Joonwoo Park <joonwoop@codeaurora.org>

3 files changed, 103 insertions, 5 deletions

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index ae6e90e167b2..5bab9845df71 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -812,6 +812,22 @@ void sched_set_cluster_dstate(const cpumask_t *cluster_cpus, int dstate,
 
 #endif /* CONFIG_SMP */
 
+#ifdef CONFIG_SCHED_HMP
+static inline void clear_ed_task(struct task_struct *p, struct rq *rq)
+{
+	if (p == rq->ed_task)
+		rq->ed_task = NULL;
+}
+
+static inline void set_task_last_wake(struct task_struct *p, u64 wallclock)
+{
+	p->last_wake_ts = wallclock;
+}
+#else
+static inline void clear_ed_task(struct task_struct *p, struct rq *rq) {}
+static inline void set_task_last_wake(struct task_struct *p, u64 wallclock) {}
+#endif
+
 #if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \
 			(defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH)))
 /*
@@ -906,6 +922,9 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
 	if (task_contributes_to_load(p))
 		rq->nr_uninterruptible++;
 
+	if (flags & DEQUEUE_SLEEP)
+		clear_ed_task(p, rq);
+
 	dequeue_task(rq, p, flags);
 }
 
@@ -1218,6 +1237,13 @@ static inline void clear_hmp_request(int cpu) { }
  * IMPORTANT: Initialize both copies to same value!!
  */
 
+/*
+ * Tasks that are runnable continuously for a period greather than
+ * sysctl_early_detection_duration can be flagged early as potential
+ * high load tasks.
+ */
+__read_mostly unsigned int sysctl_early_detection_duration = 9500000;
+
 static __read_mostly unsigned int sched_ravg_hist_size = 5;
 __read_mostly unsigned int sysctl_sched_ravg_hist_size = 5;
 
@@ -2123,7 +2149,7 @@ static inline void mark_task_starting(struct task_struct *p)
 		return;
 	}
 
-	p->ravg.mark_start = wallclock;
+	p->ravg.mark_start = p->last_wake_ts = wallclock;
 }
 
 static inline void set_window_start(struct rq *rq)
@@ -2196,6 +2222,7 @@ void sched_exit(struct task_struct *p)
 	p->ravg.mark_start = wallclock;
 	p->ravg.sum_history[0] = EXITING_TASK_MARKER;
 	enqueue_task(rq, p, 0);
+	clear_ed_task(p, rq);
 	raw_spin_unlock_irqrestore(&rq->lock, flags);
 
 	put_cpu();
@@ -2341,6 +2368,7 @@ void sched_get_cpus_busy(struct sched_load *busy,
 	u64 load[cpus], nload[cpus];
 	unsigned int cur_freq[cpus], max_freq[cpus];
 	int notifier_sent[cpus];
+	int early_detection[cpus];
 	int cpu, i = 0;
 	unsigned int window_size;
 
@@ -2374,6 +2402,7 @@ void sched_get_cpus_busy(struct sched_load *busy,
 		nload[i] = scale_load_to_cpu(nload[i], cpu);
 
 		notifier_sent[i] = rq->notifier_sent;
+		early_detection[i] = (rq->ed_task != NULL);
 		rq->notifier_sent = 0;
 		cur_freq[i] = rq->cur_freq;
 		max_freq[i] = rq->max_freq;
@@ -2388,6 +2417,13 @@ void sched_get_cpus_busy(struct sched_load *busy,
 	for_each_cpu(cpu, query_cpus) {
 		rq = cpu_rq(cpu);
 
+		if (early_detection[i]) {
+			busy[i].prev_load = div64_u64(sched_ravg_window,
+							NSEC_PER_USEC);
+			busy[i].new_task_load = 0;
+			goto exit_early;
+		}
+
 		if (!notifier_sent[i]) {
 			load[i] = scale_load_to_freq(load[i], max_freq[i],
 						     cur_freq[i]);
@@ -2412,8 +2448,9 @@ void sched_get_cpus_busy(struct sched_load *busy,
 		busy[i].prev_load = div64_u64(load[i], NSEC_PER_USEC);
 		busy[i].new_task_load = div64_u64(nload[i], NSEC_PER_USEC);
 
+exit_early:
 		trace_sched_get_busy(cpu, busy[i].prev_load,
-				     busy[i].new_task_load);
+				     busy[i].new_task_load, early_detection[i]);
 		i++;
 	}
 }
@@ -2481,9 +2518,14 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu)
 	bool new_task;
 
 	if (!sched_enable_hmp || !sched_migration_fixup ||
-		 exiting_task(p) || (!p->on_rq && p->state != TASK_WAKING))
+		 (!p->on_rq && p->state != TASK_WAKING))
 			return;
 
+	if (exiting_task(p)) {
+		clear_ed_task(p, src_rq);
+		return;
+	}
+
 	if (p->state == TASK_WAKING)
 		double_rq_lock(src_rq, dest_rq);
 
@@ -2521,6 +2563,12 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu)
 		}
 	}
 
+	if (p == src_rq->ed_task) {
+		src_rq->ed_task = NULL;
+		if (!dest_rq->ed_task)
+			dest_rq->ed_task = p;
+	}
+
 	BUG_ON((s64)src_rq->prev_runnable_sum < 0);
 	BUG_ON((s64)src_rq->curr_runnable_sum < 0);
 	BUG_ON((s64)src_rq->nt_prev_runnable_sum < 0);
@@ -3927,8 +3975,9 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
 		wake_flags |= WF_MIGRATED;
 		set_task_cpu(p, cpu);
 	}
-#endif /* CONFIG_SMP */
 
+	set_task_last_wake(p, wallclock);
+#endif /* CONFIG_SMP */
 	ttwu_queue(p, cpu);
 stat:
 	ttwu_stat(p, cpu, wake_flags);
@@ -4013,6 +4062,7 @@ static void try_to_wake_up_local(struct task_struct *p)
 		update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0);
 		update_task_ravg(p, rq, TASK_WAKE, wallclock, 0);
 		ttwu_activate(rq, p, ENQUEUE_WAKEUP);
+		set_task_last_wake(p, wallclock);
 	}
 
 	ttwu_do_wakeup(rq, p, 0);
@@ -4832,6 +4882,38 @@ unsigned long long task_sched_runtime(struct task_struct *p)
 	return ns;
 }
 
+#ifdef CONFIG_SCHED_HMP
+static bool early_detection_notify(struct rq *rq, u64 wallclock)
+{
+	struct task_struct *p;
+	int loop_max = 10;
+
+	if (!sched_boost() || !rq->cfs.h_nr_running)
+		return 0;
+
+	rq->ed_task = NULL;
+	list_for_each_entry(p, &rq->cfs_tasks, se.group_node) {
+		if (!loop_max)
+			break;
+
+		if (wallclock - p->last_wake_ts >=
+				sysctl_early_detection_duration) {
+			rq->ed_task = p;
+			return 1;
+		}
+
+		loop_max--;
+	}
+
+	return 0;
+}
+#else /* CONFIG_SCHED_HMP */
+static bool early_detection_notify(struct rq *rq, u64 wallclock)
+{
+	return 0;
+}
+#endif /* CONFIG_SCHED_HMP */
+
 /*
  * This function gets called by the timer code, with HZ frequency.
  * We call it with interrupts disabled.
@@ -4841,6 +4923,8 @@ void scheduler_tick(void)
 	int cpu = smp_processor_id();
 	struct rq *rq = cpu_rq(cpu);
 	struct task_struct *curr = rq->curr;
+	u64 wallclock;
+	bool early_notif;
 
 	sched_clock_tick();
 
@@ -4850,9 +4934,15 @@ void scheduler_tick(void)
 	curr->sched_class->task_tick(rq, curr, 0);
 	update_cpu_load_active(rq);
 	calc_global_load_tick(rq);
-	update_task_ravg(rq->curr, rq, TASK_UPDATE, sched_clock(), 0);
+	wallclock = sched_clock();
+	update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0);
+	early_notif = early_detection_notify(rq, wallclock);
 	raw_spin_unlock(&rq->lock);
 
+	if (early_notif)
+		atomic_notifier_call_chain(&load_alert_notifier_head,
+					0, (void *)(long)cpu);
+
 	perf_event_task_tick();
 
 #ifdef CONFIG_SMP
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 183dca8f05d8..2045dec5e80c 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -698,6 +698,7 @@ struct rq {
 	u64 irqload_ts;
 	unsigned int static_cpu_pwr_cost;
 	unsigned int static_cluster_pwr_cost;
+	struct task_struct *ed_task;
 
 #ifdef CONFIG_SCHED_FREQ_INPUT
 	unsigned int old_busy_time;
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index 369120e75efe..509d59837983 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -439,6 +439,13 @@ static struct ctl_table kern_table[] = {
 		.proc_handler   = sched_window_update_handler,
 	},
 	{
+		.procname       = "sched_early_detection_duration",
+		.data           = &sysctl_early_detection_duration,
+		.maxlen         = sizeof(unsigned int),
+		.mode           = 0644,
+		.proc_handler   = proc_dointvec,
+	},
+	{
 		.procname	= "sched_boost",
 		.data		= &sysctl_sched_boost,
 		.maxlen		= sizeof(unsigned int),