From db573f06e88104faa4393687716e5d3c263dff48 Mon Sep 17 00:00:00 2001 From: Srivatsa Vaddagiri Date: Thu, 12 Jun 2014 02:39:32 -0700 Subject: sched: Move around code Move up chunk of code to be defined early. This helps a subsequent patch that needs update_min_max_capacity() Change-Id: I9403c7b4dcc74ba4ef1034327241c81df97b01ea Signed-off-by: Srivatsa Vaddagiri Signed-off-by: Syed Rameez Mustafa [joonwoop@codeaurora.org: fixed trivial conflicts.] Signed-off-by: Joonwoo Park --- kernel/sched/core.c | 418 ++++++++++++++++++++++++++-------------------------- 1 file changed, 207 insertions(+), 211 deletions(-) (limited to 'kernel/sched') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 4567e413fb69..f2b287313525 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -1102,6 +1102,27 @@ unsigned int min_possible_efficiency = 1024; __read_mostly int sysctl_sched_freq_inc_notify_slack_pct; __read_mostly int sysctl_sched_freq_dec_notify_slack_pct = 25; +/* + * Maximum possible frequency across all cpus. Task demand and cpu + * capacity (cpu_power) metrics are scaled in reference to it. + */ +unsigned int max_possible_freq = 1; + +/* + * Minimum possible max_freq across all cpus. This will be same as + * max_possible_freq on homogeneous systems and could be different from + * max_possible_freq on heterogenous systems. min_max_freq is used to derive + * capacity (cpu_power) of cpus. + */ +unsigned int min_max_freq = 1; + +unsigned int max_capacity = 1024; /* max(rq->capacity) */ +unsigned int min_capacity = 1024; /* min(rq->capacity) */ + +static unsigned int sync_cpu; +static u64 sched_init_jiffy; +static u64 sched_clock_at_init_jiffy; + /* Returns how undercommitted a CPU is given its current frequency and * task load (as measured in the previous window). Returns this value * as a percentage of the CPU's maximum frequency. A negative value @@ -1401,10 +1422,6 @@ static inline void mark_task_starting(struct task_struct *p) p->ravg.prev_window = p->ravg.demand; } -static unsigned int sync_cpu; -static u64 sched_init_jiffy; -static u64 sched_clock_at_init_jiffy; - static inline void set_window_start(struct rq *rq) { int cpu = cpu_of(rq); @@ -1513,6 +1530,192 @@ void sched_set_window(u64 window_start, unsigned int window_size) local_irq_restore(flags); } +/* Keep track of max/min capacity possible across CPUs "currently" */ +static void update_min_max_capacity(void) +{ + int i; + int max = 0, min = INT_MAX; + + for_each_possible_cpu(i) { + if (cpu_rq(i)->capacity > max) + max = cpu_rq(i)->capacity; + if (cpu_rq(i)->capacity < min) + min = cpu_rq(i)->capacity; + } + + max_capacity = max; + min_capacity = min; +} + +/* + * Return 'capacity' of a cpu in reference to "least" efficient cpu, such that + * least efficient cpu gets capacity of 1024 + */ +unsigned long capacity_scale_cpu_efficiency(int cpu) +{ + return (1024 * cpu_rq(cpu)->efficiency) / min_possible_efficiency; +} + +/* + * Return 'capacity' of a cpu in reference to cpu with lowest max_freq + * (min_max_freq), such that one with lowest max_freq gets capacity of 1024. + */ +unsigned long capacity_scale_cpu_freq(int cpu) +{ + return (1024 * cpu_rq(cpu)->max_freq) / min_max_freq; +} + +/* + * Return load_scale_factor of a cpu in reference to "most" efficient cpu, so + * that "most" efficient cpu gets a load_scale_factor of 1 + */ +static inline unsigned long load_scale_cpu_efficiency(int cpu) +{ + return (1024 * max_possible_efficiency) / cpu_rq(cpu)->efficiency; +} + +/* + * Return load_scale_factor of a cpu in reference to cpu with best max_freq + * (max_possible_freq), so that one with best max_freq gets a load_scale_factor + * of 1. + */ +static inline unsigned long load_scale_cpu_freq(int cpu) +{ + return (1024 * max_possible_freq) / cpu_rq(cpu)->max_freq; +} + +static int compute_capacity(int cpu) +{ + int capacity = 1024; + + capacity *= capacity_scale_cpu_efficiency(cpu); + capacity >>= 10; + + capacity *= capacity_scale_cpu_freq(cpu); + capacity >>= 10; + + return capacity; +} + +static int compute_load_scale_factor(int cpu) +{ + int load_scale = 1024; + + /* + * load_scale_factor accounts for the fact that task load + * is in reference to "best" performing cpu. Task's load will need to be + * scaled (up) by a factor to determine suitability to be placed on a + * (little) cpu. + */ + load_scale *= load_scale_cpu_efficiency(cpu); + load_scale >>= 10; + + load_scale *= load_scale_cpu_freq(cpu); + load_scale >>= 10; + + return load_scale; +} + +static int cpufreq_notifier_policy(struct notifier_block *nb, + unsigned long val, void *data) +{ + struct cpufreq_policy *policy = (struct cpufreq_policy *)data; + int i; + unsigned int min_max = min_max_freq; + const struct cpumask *cpus = policy->related_cpus; + int orig_min_max_freq = min_max_freq; + + if (val != CPUFREQ_NOTIFY) + return 0; + + for_each_cpu(i, policy->related_cpus) { + cpumask_copy(&cpu_rq(i)->freq_domain_cpumask, + policy->related_cpus); + cpu_rq(i)->min_freq = policy->min; + cpu_rq(i)->max_freq = policy->max; + cpu_rq(i)->max_possible_freq = policy->cpuinfo.max_freq; + } + + max_possible_freq = max(max_possible_freq, policy->cpuinfo.max_freq); + if (min_max_freq == 1) + min_max = UINT_MAX; + min_max_freq = min(min_max, policy->cpuinfo.max_freq); + BUG_ON(!min_max_freq); + BUG_ON(!policy->max); + + if (min_max_freq != orig_min_max_freq) + cpus = cpu_online_mask; + + /* + * Changed load_scale_factor can trigger reclassification of tasks as + * big or small. Make this change "atomic" so that tasks are accounted + * properly due to changed load_scale_factor + */ + pre_big_small_task_count_change(); + for_each_cpu(i, cpus) { + struct rq *rq = cpu_rq(i); + + rq->capacity = compute_capacity(i); + rq->load_scale_factor = compute_load_scale_factor(i); + } + + update_min_max_capacity(); + post_big_small_task_count_change(); + + return 0; +} + +static int cpufreq_notifier_trans(struct notifier_block *nb, + unsigned long val, void *data) +{ + struct cpufreq_freqs *freq = (struct cpufreq_freqs *)data; + unsigned int cpu = freq->cpu, new_freq = freq->new; + struct rq *rq = cpu_rq(cpu); + unsigned long flags; + + if (val != CPUFREQ_POSTCHANGE) + return 0; + + BUG_ON(!new_freq); + + raw_spin_lock_irqsave(&rq->lock, flags); + update_task_ravg(rq->curr, rq, TASK_UPDATE, sched_clock(), NULL); + cpu_rq(cpu)->cur_freq = new_freq; + raw_spin_unlock_irqrestore(&rq->lock, flags); + + return 0; +} + +static struct notifier_block notifier_policy_block = { + .notifier_call = cpufreq_notifier_policy +}; + +static struct notifier_block notifier_trans_block = { + .notifier_call = cpufreq_notifier_trans +}; + +static int register_sched_callback(void) +{ + int ret; + + ret = cpufreq_register_notifier(¬ifier_policy_block, + CPUFREQ_POLICY_NOTIFIER); + + if (!ret) + ret = cpufreq_register_notifier(¬ifier_trans_block, + CPUFREQ_TRANSITION_NOTIFIER); + + return 0; +} + +/* + * cpufreq callbacks can be registered at core_initcall or later time. + * Any registration done prior to that is "forgotten" by cpufreq. See + * initialization of variable init_cpufreq_transition_notifier_list_called + * for further information. + */ +core_initcall(register_sched_callback); + #else /* CONFIG_SCHED_FREQ_INPUT || CONFIG_SCHED_HMP */ static inline void @@ -8005,213 +8208,6 @@ void __init sched_init_smp(void) #endif /* CONFIG_SMP */ -#if defined(CONFIG_SCHED_FREQ_INPUT) || defined(CONFIG_SCHED_HMP) - -/* - * Maximum possible frequency across all cpus. Task demand and cpu - * capacity (cpu_power) metrics are scaled in reference to it. - */ -unsigned int max_possible_freq = 1; - -/* - * Minimum possible max_freq across all cpus. This will be same as - * max_possible_freq on homogeneous systems and could be different from - * max_possible_freq on heterogenous systems. min_max_freq is used to derive - * capacity (cpu_power) of cpus. - */ -unsigned int min_max_freq = 1; - -unsigned int max_capacity = 1024; /* max(rq->capacity) */ -unsigned int min_capacity = 1024; /* min(rq->capacity) */ - -/* Keep track of max/min capacity possible across CPUs "currently" */ -static void update_min_max_capacity(void) -{ - int i; - int max = 0, min = INT_MAX; - - for_each_possible_cpu(i) { - if (cpu_rq(i)->capacity > max) - max = cpu_rq(i)->capacity; - if (cpu_rq(i)->capacity < min) - min = cpu_rq(i)->capacity; - } - - max_capacity = max; - min_capacity = min; -} - -/* - * Return 'capacity' of a cpu in reference to "least" efficient cpu, such that - * least efficient cpu gets capacity of 1024 - */ -unsigned long capacity_scale_cpu_efficiency(int cpu) -{ - return (1024 * cpu_rq(cpu)->efficiency) / min_possible_efficiency; -} - -/* - * Return 'capacity' of a cpu in reference to cpu with lowest max_freq - * (min_max_freq), such that one with lowest max_freq gets capacity of 1024. - */ -unsigned long capacity_scale_cpu_freq(int cpu) -{ - return (1024 * cpu_rq(cpu)->max_freq) / min_max_freq; -} - -/* - * Return load_scale_factor of a cpu in reference to "most" efficient cpu, so - * that "most" efficient cpu gets a load_scale_factor of 1 - */ -static inline unsigned long load_scale_cpu_efficiency(int cpu) -{ - return (1024 * max_possible_efficiency) / cpu_rq(cpu)->efficiency; -} - -/* - * Return load_scale_factor of a cpu in reference to cpu with best max_freq - * (max_possible_freq), so that one with best max_freq gets a load_scale_factor - * of 1. - */ -static inline unsigned long load_scale_cpu_freq(int cpu) -{ - return (1024 * max_possible_freq) / cpu_rq(cpu)->max_freq; -} - -static int compute_capacity(int cpu) -{ - int capacity = 1024; - - capacity *= capacity_scale_cpu_efficiency(cpu); - capacity >>= 10; - - capacity *= capacity_scale_cpu_freq(cpu); - capacity >>= 10; - - return capacity; -} - -static int compute_load_scale_factor(int cpu) -{ - int load_scale = 1024; - - /* - * load_scale_factor accounts for the fact that task load - * is in reference to "best" performing cpu. Task's load will need to be - * scaled (up) by a factor to determine suitability to be placed on a - * (little) cpu. - */ - load_scale *= load_scale_cpu_efficiency(cpu); - load_scale >>= 10; - - load_scale *= load_scale_cpu_freq(cpu); - load_scale >>= 10; - - return load_scale; -} - -static int cpufreq_notifier_policy(struct notifier_block *nb, - unsigned long val, void *data) -{ - struct cpufreq_policy *policy = (struct cpufreq_policy *)data; - int i; - unsigned int min_max = min_max_freq; - const struct cpumask *cpus = policy->related_cpus; - int orig_min_max_freq = min_max_freq; - - if (val != CPUFREQ_NOTIFY) - return 0; - - for_each_cpu(i, policy->related_cpus) { - cpumask_copy(&cpu_rq(i)->freq_domain_cpumask, - policy->related_cpus); - cpu_rq(i)->min_freq = policy->min; - cpu_rq(i)->max_freq = policy->max; - cpu_rq(i)->max_possible_freq = policy->cpuinfo.max_freq; - } - - max_possible_freq = max(max_possible_freq, policy->cpuinfo.max_freq); - if (min_max_freq == 1) - min_max = UINT_MAX; - min_max_freq = min(min_max, policy->cpuinfo.max_freq); - BUG_ON(!min_max_freq); - BUG_ON(!policy->max); - - if (min_max_freq != orig_min_max_freq) - cpus = cpu_online_mask; - - /* - * Changed load_scale_factor can trigger reclassification of tasks as - * big or small. Make this change "atomic" so that tasks are accounted - * properly due to changed load_scale_factor - */ - pre_big_small_task_count_change(); - for_each_cpu(i, cpus) { - struct rq *rq = cpu_rq(i); - - rq->capacity = compute_capacity(i); - rq->load_scale_factor = compute_load_scale_factor(i); - } - - update_min_max_capacity(); - post_big_small_task_count_change(); - - return 0; -} - -static int cpufreq_notifier_trans(struct notifier_block *nb, - unsigned long val, void *data) -{ - struct cpufreq_freqs *freq = (struct cpufreq_freqs *)data; - unsigned int cpu = freq->cpu, new_freq = freq->new; - struct rq *rq = cpu_rq(cpu); - unsigned long flags; - - if (val != CPUFREQ_POSTCHANGE) - return 0; - - BUG_ON(!new_freq); - - raw_spin_lock_irqsave(&rq->lock, flags); - update_task_ravg(rq->curr, rq, TASK_UPDATE, sched_clock(), NULL); - cpu_rq(cpu)->cur_freq = new_freq; - raw_spin_unlock_irqrestore(&rq->lock, flags); - - return 0; -} - -static struct notifier_block notifier_policy_block = { - .notifier_call = cpufreq_notifier_policy -}; - -static struct notifier_block notifier_trans_block = { - .notifier_call = cpufreq_notifier_trans -}; - -static int register_sched_callback(void) -{ - int ret; - - ret = cpufreq_register_notifier(¬ifier_policy_block, - CPUFREQ_POLICY_NOTIFIER); - - if (!ret) - ret = cpufreq_register_notifier(¬ifier_trans_block, - CPUFREQ_TRANSITION_NOTIFIER); - - return 0; -} - -/* - * cpufreq callbacks can be registered at core_initcall or later time. - * Any registration done prior to that is "forgotten" by cpufreq. See - * initialization of variable init_cpufreq_transition_notifier_list_called - * for further information. - */ -core_initcall(register_sched_callback); - -#endif /* CONFIG_SCHED_FREQ_INPUT || CONFIG_SCHED_HMP */ - int in_sched_functions(unsigned long addr) { return in_lock_functions(addr) || -- cgit v1.2.3