5、EAS(Energy-Aware Scheduling)
5.1、smp rebalance
通过搜索关键字“energy_aware()”，来查看EAS对smp负载均衡的影响。
可以看到EAS对负载均衡的策略是这样的：在overutilized的情况下，使用传统的smp/hmp负载均衡方法；在非overutilized的情况下，使用eas的均衡方法。
EAS的负载均衡和原有方法的区别有几部分：
- 1、在EAS使能且没有overutilized的情况下，hmp负载均衡不工作；
- 2、在EAS使能且没有overutilized的情况下，smp负载均衡不工作；
- 3、在EAS使能且没有overutilized的情况下，EAS的主要工作集中在进程唤醒/新建时选择运行cpu上select_task_rq_fair()；
5.1.1、rebalance_domains()
- 1、在EAS使能且没有overutilized的情况下，hmp负载均衡不使能；
static void run_rebalance_domains(struct softirq_action *h)
{
struct rq *this_rq = this_rq();
enum cpu_idle_type idle = this_rq->idle_balance ?
CPU_IDLE : CPU_NOT_IDLE;
int this_cpu = smp_processor_id();
/* bypass load balance of HMP if EAS consideration */
if ((!energy_aware() && sched_feat(SCHED_HMP)) ||
(hybrid_support() && cpu_rq(this_cpu)->rd->overutilized))
hmp_force_up_migration(this_cpu);
/*
* If this cpu has a pending nohz_balance_kick, then do the
* balancing on behalf of the other idle cpus whose ticks are
* stopped. Do nohz_idle_balance *before* rebalance_domains to
* give the idle cpus a chance to load balance. Else we may
* load balance only within the local sched_domain hierarchy
* and abort nohz_idle_balance altogether if we pull some load.
*/
nohz_idle_balance(this_rq, idle);
rebalance_domains(this_rq, idle);
}- 2、在load_balance() -> find_busiest_group()中，如果在EAS使能且没有overutilized的情况下，不进行常规的smp负载均衡；
static struct sched_group *find_busiest_group(struct lb_env *env)
{
if (energy_aware() && !env->dst_rq->rd->overutilized && !same_clus)
goto out_balanced;
out_balanced:
env->imbalance = 0;
return NULL;
}
5.1.2、select_task_rq_fair()
参考4.1.2.3、select_task_rq_fair()这一节的详细描述。
5.2、cpufreq_sched/schedutil governor
sched governor比较传统interactive governor有以下优点：
- 1、负载变化的时间更快。interactive是20ms统计一次负载，而sched governor是在schedule_tick()中更新负载，tick的时间更短；
- 2、interactive的负载计算有问题：历史负载没有老化；历史频率除以现在频率不合理；
interactive governor的主要思想就是综合rt、cfs的负载，判断当前freq的capacity是否满足需求，是否需要调频。
5.2.1、rt request
针对CONFIG_CPU_FREQ_GOV_SCHED，rt有3条关键计算路径：
- 1、rt负载的(rq->rt_avg)的累加：scheduler_tick() -> task_tick_rt() -> update_curr_rt() -> sched_rt_avg_update()
rq->rt_avg = 累加时间分量 * 当前frq分量(最大1024)
static inline void sched_rt_avg_update(struct rq *rq, u64 rt_delta)
{
rq->rt_avg += rt_delta * arch_scale_freq_capacity(NULL, cpu_of(rq));
}- 2、rt负载的老化：scheduler_tick() -> __update_cpu_load() -> __update_cpu_load() -> sched_avg_update()
或者scheduler_tick() -> task_tick_rt() -> sched_rt_update_capacity_req() -> sched_avg_update()
rq->rt_avg的老化比较简单，每个period老化1/2。
void sched_avg_update(struct rq *rq)
{
/* (1) 默认老化周期为1s/2 = 500ms */
s64 period = sched_avg_period();
while ((s64)(rq_clock(rq) - rq->age_stamp) > period) {
/*
* Inline assembly required to prevent the compiler
* optimising this loop into a divmod call.
* See __iter_div_u64_rem() for another example of this.
*/
asm("" : "+rm" (rq->age_stamp));
rq->age_stamp += period;
/* (2) 每个老化周期，负载老化为原来的1/2 */
rq->rt_avg /= 2;
rq->dl_avg /= 2;
}
}
|→
static inline u64 sched_avg_period(void)
{
/* (1.1) 老化周期 = sysctl_sched_time_avg/2 = 500ms */
return (u64)sysctl_sched_time_avg * NSEC_PER_MSEC / 2;
}- 3、rt request的更新：scheduler_tick() -> task_tick_rt() -> sched_rt_update_capacity_req() -> set_rt_cpu_capacity()
rt request的计算有点粗糙: request = rt_avg/(sched_avg_period() + delta)，rt_avg中没有加上delta时间的负载。
static void sched_rt_update_capacity_req(struct rq *rq)
{
u64 total, used, age_stamp, avg;
s64 delta;
if (!sched_freq())
return;
/* (1) 最新的负载进行老化 */
sched_avg_update(rq);
/*
* Since we're reading these variables without serialization make sure
* we read them once before doing sanity checks on them.
*/
age_stamp = READ_ONCE(rq->age_stamp);
/* (2) avg=老化后的负载 */
avg = READ_ONCE(rq->rt_avg);
delta = rq_clock(rq) - age_stamp;
if (unlikely(delta < 0))
delta = 0;
/* (3) total时间=一个老化周期+上次老化剩余时间 */
total = sched_avg_period() + delta;
/* (4) avg/total=request，(最大频率=1024) */
used = div_u64(avg, total);
if (unlikely(used > SCHED_CAPACITY_SCALE))
used = SCHED_CAPACITY_SCALE;
/* (5) update request */
set_rt_cpu_capacity(rq->cpu, true, (unsigned long)(used), SCHE_ONESHOT);
}
|→
static inline void set_rt_cpu_capacity(int cpu, bool request,
unsigned long capacity,
int type)
{
#ifdef CONFIG_CPU_FREQ_SCHED_ASSIST
if (true) {
#else
if (per_cpu(cpu_sched_capacity_reqs, cpu).rt != capacity) {
#endif
/* (5.1) 把RT负载更新到per_cpu(cpu_sched_capacity_reqs, cpu).rt */
per_cpu(cpu_sched_capacity_reqs, cpu).rt = capacity;
update_cpu_capacity_request(cpu, request, type);
}
}5.2.2、cfs request
同样，cfs也有3条关键计算路径：
- 1、cfs负载的(rq->rt_avg)的累加：scheduler_tick() -> task_tick_fair() -> entity_tick() -> update_load_avg()
- 2、cfs负载的老化：scheduler_tick() -> task_tick_fair() -> entity_tick() -> update_load_avg()
- 3、cfs request的更新：scheduler_tick() -> sched_freq_tick() -> set_cfs_cpu_capacity()
static void sched_freq_tick(int cpu)
{
struct sched_capacity_reqs *scr;
unsigned long capacity_orig, capacity_curr;
unsigned long capacity_req;
struct sched_domain *sd = rcu_dereference(per_cpu(sd_ea, cpu));
if (!sched_freq())
return;
capacity_orig = capacity_orig_of(cpu);
capacity_curr = capacity_curr_of(cpu);
/* (1) 如果当前频率已经是最高频率，直接返回
目前只支持频率从低往高调整？
*/
if (capacity_curr == capacity_orig)
return;
/*
* To make free room for a task that is building up its "real"
* utilization and to harm its performance the least, request
* a jump to bigger OPP as soon as the margin of free capacity is
* impacted (specified by capacity_margin).
*/
scr = &per_cpu(cpu_sched_capacity_reqs, cpu);
/* (2) 计算最新的(cfs capacity+ rt capacity) * (1126/1024)
放大一些，等于对capacity的需求request
ooooo这里的计算有问题：cpu_util(cpu)是带capacity分量的，而scr->rt是不带capacity分量的，不能直接相加？
*/
/* capacity_req which includes RT loading & capacity_margin */
capacity_req = sum_capacity_reqs(cpu_util(cpu), scr);
/* (3) 如果capacity request大于当前频率的capacity */
if (capacity_curr <= capacity_req) {
if (sd) {### 5.3.1、WALT的负载计算
const struct sched_group_energy *const sge = sd->groups->sge;
int nr_cap_states = sge->nr_cap_states;
int idx, tmp_idx;
int opp_jump_step;
for (idx = 0; idx < nr_cap_states; idx++) {
if (sge->cap_states[idx].cap > capacity_curr+1)
break;
}
/* (4) 尝试计算一个合理的频率等级来满足capacity request */
if (idx < nr_cap_states/3)
opp_jump_step = 2; /* far step */
else
opp_jump_step = 1; /* near step */
tmp_idx = idx + (opp_jump_step - 1);
idx = tmp_idx > (nr_cap_states - 1) ?
(nr_cap_states - 1) : tmp_idx;
if (idx)
capacity_req = (sge->cap_states[idx].cap +
sge->cap_states[idx-1].cap)/2;
else
/* should not arrive here!*/
capacity_req = sge->cap_states[idx].cap + 2;
}
/* (5) 去掉request中的capacity分量，转化成scale_freq */
/* convert scale-invariant capacity */
capacity_req = capacity_req * SCHED_CAPACITY_SCALE / capacity_orig_of(cpu);
/* (6) update request，
ooooo这里有问题啊：capacity_req计算的时候是按照rt+cfs加起来计算的，怎么有把结果配置给了scr->cfs？
*/
/*
* If free room ~5% impact, jump to 1 more index hihger OPP.
* Whatever it should be better than capacity_max.
*/
set_cfs_cpu_capacity(cpu, true, capacity_req, SCHE_ONESHOT);
}
}
|→
static inline void set_cfs_cpu_capacity(int cpu, bool request,
unsigned long capacity, int type)
{
#ifdef CONFIG_CPU_FREQ_SCHED_ASSIST
if (true) {
#else
if (per_cpu(cpu_sched_capacity_reqs, cpu).cfs != capacity) {
#endif
/* (6.1) 把RT负载更新到per_cpu(cpu_sched_capacity_reqs, cpu).cfs */
per_cpu(cpu_sched_capacity_reqs, cpu).cfs = capacity;
update_cpu_capacity_request(cpu, request, type);
}
}5.2.3、freq target
void update_cpu_capacity_request(int cpu, bool request, int type)
{
unsigned long new_capacity;
struct sched_capacity_reqs *scr;
/* The rq lock serializes access to the CPU's sched_capacity_reqs. */
lockdep_assert_held(&cpu_rq(cpu)->lock);
scr = &per_cpu(cpu_sched_capacity_reqs, cpu);
/* (1) 综合rt、cfs的request */
new_capacity = scr->cfs + scr->rt;
new_capacity = new_capacity * capacity_margin_dvfs
/ SCHED_CAPACITY_SCALE;
new_capacity += scr->dl;
#ifndef CONFIG_CPU_FREQ_SCHED_ASSIST
if (new_capacity == scr->total)
return;
#endif
scr->total = new_capacity;
if (request)
update_fdomain_capacity_request(cpu, type);
}
|→
static void update_fdomain_capacity_request(int cpu, int type)
{
unsigned int freq_new, cpu_tmp;
struct gov_data *gd;
unsigned long capacity = 0;
#ifdef CONFIG_CPU_FREQ_SCHED_ASSIST
int cid = arch_get_cluster_id(cpu);
struct cpumask cls_cpus;
#endif
struct cpufreq_policy *policy = NULL;
/*
* Avoid grabbing the policy if possible. A test is still
* required after locking the CPU's policy to avoid racing
* with the governor changing.
*/
if (!per_cpu(enabled, cpu))
return;
#ifdef CONFIG_CPU_FREQ_SCHED_ASSIST
gd = g_gd[cid];
/* bail early if we are throttled */
if (ktime_before(ktime_get(), gd->throttle))
goto out;
arch_get_cluster_cpus(&cls_cpus, cid);
/* find max capacity requested by cpus in this policy */
for_each_cpu(cpu_tmp, &cls_cpus) {
struct sched_capacity_reqs *scr;
if (!cpu_online(cpu_tmp))
continue;
scr = &per_cpu(cpu_sched_capacity_reqs, cpu_tmp);
capacity = max(capacity, scr->total);
}
freq_new = capacity * arch_scale_get_max_freq(cpu) >> SCHED_CAPACITY_SHIFT;
#else
if (likely(cpu_online(cpu)))
policy = cpufreq_cpu_get(cpu);
if (IS_ERR_OR_NULL(policy))
return;
if (policy->governor != &cpufreq_gov_sched ||
!policy->governor_data)
goto out;
gd = policy->governor_data;
/* bail early if we are throttled */
if (ktime_before(ktime_get(), gd->throttle))
goto out;
/* (1) 选择policy cpus中最大的capacity */
/* find max capacity requested by cpus in this policy */
for_each_cpu(cpu_tmp, policy->cpus) {
struct sched_capacity_reqs *scr;
scr = &per_cpu(cpu_sched_capacity_reqs, cpu_tmp);
capacity = max(capacity, scr->total);
}
/* (2) 把相对capacity转换成绝对freq */
/* Convert the new maximum capacity request into a cpu frequency */
freq_new = capacity * policy->max >> SCHED_CAPACITY_SHIFT;
if (freq_new == gd->requested_freq)
goto out;
#endif /* !CONFIG_CPU_FREQ_SCHED_ASSIST */
gd->requested_freq = freq_new;
gd->target_cpu = cpu;
/* (3) 使用irq_work或者直接配置的方式来配置新的频率
直接在schedule_tick()中配置频率的方式估计不会使用，因为这样会阻塞中断
*/
/*
* Throttling is not yet supported on platforms with fast cpufreq
* drivers.
*/
if (cpufreq_driver_slow)
irq_work_queue_on(&gd->irq_work, cpu);
else
cpufreq_sched_try_driver_target(cpu, policy, freq_new, type);
out:
if (policy)
cpufreq_cpu_put(policy);
}
5.3、WALT(Windows Assisted Load Tracking)
在qualcomm 8898中，使用了WALT作为负载计算方法，也使用了自己的负载均衡算法来使用WALT负载。代码中使用CONFIG_SCHED_HMP来标示qualcomm自己负载均衡方法。
5.3.1、WALT的负载计算
Walt的本质也是时间窗分量，结合freq分量、capacity分量等一起表达的一个负载相对值。我们首先来看看几个分量的计算方法。
- 1、cluster->efficiency计算：从dts中读取，我们可以看到，四个小核的efficiency是1024，四个大核的efficiency是1638；
static struct sched_cluster *alloc_new_cluster(const struct cpumask *cpus)
{
cluster->efficiency = arch_get_cpu_efficiency(cpumask_first(cpus));
if (cluster->efficiency > max_possible_efficiency)
max_possible_efficiency = cluster->efficiency;
if (cluster->efficiency < min_possible_efficiency)
min_possible_efficiency = cluster->efficiency;
}
unsigned long arch_get_cpu_efficiency(int cpu)
{
return per_cpu(cpu_efficiency, cpu);
}
static void __init parse_dt_cpu_power(void)
{
/*
* The CPU efficiency value passed from the device tree
* overrides the value defined in the table_efficiency[]
*/
if (of_property_read_u32(cn, "efficiency", &efficiency) < 0) {
}
per_cpu(cpu_efficiency, cpu) = efficiency;
}
从 arch/arm64/boot/dts/qcom/sdm660.dtsi读到"efficiency"配置：
cpus {
#address-cells = <2>;
#size-cells = <0>;
CPU0: cpu@0 {
efficiency = <1024>;
};
CPU1: cpu@1 {
efficiency = <1024>;
};
CPU2: cpu@2 {
efficiency = <1024>;
};
CPU3: cpu@3 {
efficiency = <1024>;
};
CPU4: cpu@100 {
efficiency = <1638>;
};
CPU5: cpu@101 {
efficiency = <1638>;
};
CPU6: cpu@102 {
efficiency = <1638>;
};
CPU7: cpu@103 {
efficiency = <1638>;
};
cpu-map {
cluster0 {
core0 {
cpu = <&CPU0>;
};
core1 {
cpu = <&CPU1>;
};
core2 {
cpu = <&CPU2>;
};
core3 {
cpu = <&CPU3>;
};
};
cluster1 {
core0 {
cpu = <&CPU4>;
};
core1 {
cpu = <&CPU5>;
};
core2 {
cpu = <&CPU6>;
};
core3 {
cpu = <&CPU7>;
};
};
};
}
- 2、cluster->capacity：计算和最小值的正比：capacity = 1024 * (cluster->efficiency*cluster_max_freq(cluster)) / (min_possible_efficiency*min_max_freq)
- 3、cluster->max_possible_capacity：计算和最小值的正比：capacity = 1024 * (cluster->efficiency*cluster->max_possible_freq) / (min_possible_efficiency*min_max_freq)
- 4、cluster->load_scale_factor：计算和最大值的反比：lsf = 1024 * (max_possible_efficiency*max_possible_freq) / (cluster->efficiency*cluster_max_freq(cluster))
- 5、cluster->exec_scale_factor：计算和最大值的正比：exec_scale_factor = 1024 * cluster->efficiency / max_possible_efficiency
static void update_all_clusters_stats(void)
{
struct sched_cluster *cluster;
u64 highest_mpc = 0, lowest_mpc = U64_MAX;
pre_big_task_count_change(cpu_possible_mask);
for_each_sched_cluster(cluster) {
u64 mpc;
/* (1) 计算cluster->capacity：capacity = efficiency * cluster_max_freq
最小值：min_possible_efficiency*min_max_freq = 1024，
计算和最小值的正比：capacity = 1024 * (cluster->efficiency*cluster_max_freq(cluster)) / (min_possible_efficiency*min_max_freq)
*/
cluster->capacity = compute_capacity(cluster);
/* (2) 计算cluster->max_possible_capacity：capacity = efficiency * cluster_max_freq
最小值：min_possible_efficiency*min_max_freq = 1024，
计算和最小值的正比：capacity = 1024 * (cluster->efficiency*cluster->max_possible_freq) / (min_possible_efficiency*min_max_freq)
*/
mpc = cluster->max_possible_capacity =
compute_max_possible_capacity(cluster);
/* (3) 计算cluster->load_scale_factor： lsf = efficiency * cluster_max_freq
最大值：max_possible_efficiency*max_possible_freq = 1024
计算和最大值的反比：lsf = 1024 * (max_possible_efficiency*max_possible_freq) / (cluster->efficiency*cluster_max_freq(cluster))
*/
cluster->load_scale_factor = compute_load_scale_factor(cluster);
/* (4) 计算cluster->exec_scale_factor：
最大值：max_possible_efficiency = 1024
计算和最大值的正比：exec_scale_factor = 1024 * cluster->efficiency / max_possible_efficiency
*/
cluster->exec_scale_factor =
DIV_ROUND_UP(cluster->efficiency * 1024,
max_possible_efficiency);
if (mpc > highest_mpc)
highest_mpc = mpc;
if (mpc < lowest_mpc)
lowest_mpc = mpc;
}
max_possible_capacity = highest_mpc;
min_max_possible_capacity = lowest_mpc;
__update_min_max_capacity();
sched_update_freq_max_load(cpu_possible_mask);
post_big_task_count_change(cpu_possible_mask);
}
|→
static int compute_capacity(struct sched_cluster *cluster)
{
int capacity = 1024;
capacity *= capacity_scale_cpu_efficiency(cluster);
capacity >>= 10;
capacity *= capacity_scale_cpu_freq(cluster);
capacity >>= 10;
return capacity;
}
||→
/*
* Return 'capacity' of a cpu in reference to "least" efficient cpu, such that
* least efficient cpu gets capacity of 1024
*/
static unsigned long
capacity_scale_cpu_efficiency(struct sched_cluster *cluster)
{
return (1024 * cluster->efficiency) / min_possible_efficiency;
}
||→
/*
* Return 'capacity' of a cpu in reference to cpu with lowest max_freq
* (min_max_freq), such that one with lowest max_freq gets capacity of 1024.
*/
static unsigned long capacity_scale_cpu_freq(struct sched_cluster *cluster)
{
return (1024 * cluster_max_freq(cluster)) / min_max_freq;
}
|→
static int compute_load_scale_factor(struct sched_cluster *cluster)
{
int load_scale = 1024;
/*
* load_scale_factor accounts for the fact that task load
* is in reference to "best" performing cpu. Task's load will need to be
* scaled (up) by a factor to determine suitability to be placed on a
* (little) cpu.
*/
load_scale *= load_scale_cpu_efficiency(cluster);
load_scale >>= 10;
load_scale *= load_scale_cpu_freq(cluster);
load_scale >>= 10;
return load_scale;
}
||→
/*
* Return load_scale_factor of a cpu in reference to "most" efficient cpu, so
* that "most" efficient cpu gets a load_scale_factor of 1
*/
static inline unsigned long
load_scale_cpu_efficiency(struct sched_cluster *cluster)
{
return DIV_ROUND_UP(1024 * max_possible_efficiency,
cluster->efficiency);
}
||→
/*
* Return load_scale_factor of a cpu in reference to cpu with best max_freq
* (max_possible_freq), so that one with best max_freq gets a load_scale_factor
* of 1.
*/
static inline unsigned long load_scale_cpu_freq(struct sched_cluster *cluster)
{
return DIV_ROUND_UP(1024 * max_possible_freq,
cluster_max_freq(cluster));
}
- 6、cluster->max_power_cost：cluster的最大功耗 = voltage^2 * frequence
- 7、cluster->min_power_cost：cluster的最小功耗 = voltage^2 * frequence
static void sort_clusters(void)
{
for_each_sched_cluster(cluster) {
cluster->max_power_cost = power_cost(cluster_first_cpu(cluster),
max_task_load());
cluster->min_power_cost = power_cost(cluster_first_cpu(cluster),
0);
if (cluster->max_power_cost > tmp_max)
tmp_max = cluster->max_power_cost;
}
max_power_cost = tmp_max;
}
|→
unsigned int power_cost(int cpu, u64 demand)
{
int first, mid, last;
struct cpu_pwr_stats *per_cpu_info = get_cpu_pwr_stats();
struct cpu_pstate_pwr *costs;
struct freq_max_load *max_load;
int total_static_pwr_cost = 0;
struct rq *rq = cpu_rq(cpu);
unsigned int pc;
if (!per_cpu_info || !per_cpu_info[cpu].ptable)
/*
* When power aware scheduling is not in use, or CPU
* power data is not available, just use the CPU
* capacity as a rough stand-in for real CPU power
* numbers, assuming bigger CPUs are more power
* hungry.
*/
return cpu_max_possible_capacity(cpu);
rcu_read_lock();
max_load = rcu_dereference(per_cpu(freq_max_load, cpu));
if (!max_load) {
pc = cpu_max_possible_capacity(cpu);
goto unlock;
}
costs = per_cpu_info[cpu].ptable;
if (demand <= max_load->freqs[0].hdemand) {
pc = costs[0].power;
goto unlock;
} else if (demand > max_load->freqs[max_load->length - 1].hdemand) {
pc = costs[max_load->length - 1].power;
goto unlock;
}
first = 0;
last = max_load->length - 1;
mid = (last - first) >> 1;
while (1) {
if (demand <= max_load->freqs[mid].hdemand)
last = mid;
else
first = mid;
if (last - first == 1)
break;
mid = first + ((last - first) >> 1);
}
pc = costs[last].power;
unlock:
rcu_read_unlock();
if (idle_cpu(cpu) && rq->cstate) {
total_static_pwr_cost += rq->static_cpu_pwr_cost;
if (rq->cluster->dstate)
total_static_pwr_cost +=
rq->cluster->static_cluster_pwr_cost;
}
return pc + total_static_pwr_cost;
}
/* qualcom的power的计算公式 = voltage^2 * frequence */
static int msm_get_power_values(int cpu, struct cpu_static_info *sp)
{
int i = 0, j;
int ret = 0;
uint64_t power;
/* Calculate dynamic power spent for every frequency using formula:
* Power = V * V * f
* where V = voltage for frequency
* f = frequency
* */
sp->power = allocate_2d_array_uint32_t(sp->num_of_freqs);
if (IS_ERR_OR_NULL(sp->power))
return PTR_ERR(sp->power);
for (i = 0; i < TEMP_DATA_POINTS; i++) {
for (j = 0; j < sp->num_of_freqs; j++) {
power = sp->voltage[j] *
sp->table[j].frequency;
do_div(power, 1000);
do_div(power, 1000);
power *= sp->voltage[j];
do_div(power, 1000);
sp->power[i][j] = power;
}
}
return ret;
}
5.3.1.1、update_task_ravg()
walt关于进程的负载计算流程如下：
- 1、把时间分成一个个window窗口，累加时间时，需要综合efficiency和freq分量(也就是capacity)：delta = delta_time * (curr_freq/max_possible_freq) * (cluster->efficiency/max_possible_efficiency);
static inline u64 scale_exec_time(u64 delta, struct rq *rq)
{
u32 freq;
/* curr_freq / max_possible_freq */
freq = cpu_cycles_to_freq(rq->cc.cycles, rq->cc.time);
delta = DIV64_U64_ROUNDUP(delta * freq, max_possible_freq);
/* exec_scale_factor = cluster->efficiency / max_possible_efficiency */
delta *= rq->cluster->exec_scale_factor;
delta >>= 10;
return delta;
}- 2、统计runnable状态的时间：account_busy_for_task_demand()屏蔽掉runnable以外的其他状态的时间统计；
static int account_busy_for_task_demand(struct task_struct *p, int event)
{
/*
* No need to bother updating task demand for exiting tasks
* or the idle task.
*/
/* (3.1.1) exit、idle任务不计入统计 */
if (exiting_task(p) || is_idle_task(p))
return 0;
/*
* When a task is waking up it is completing a segment of non-busy
* time. Likewise, if wait time is not treated as busy time, then
* when a task begins to run or is migrated, it is not running and
* is completing a segment of non-busy time.
*/
/* (3.1.2) 任务被wakeup，之前的等待时间不计入统计
SCHED_ACCOUNT_WAIT_TIME用来控制ruannable的等待时间是否计入统计，默认是计入的
*/
if (event == TASK_WAKE || (!SCHED_ACCOUNT_WAIT_TIME &&
(event == PICK_NEXT_TASK || event == TASK_MIGRATE)))
return 0;
return 1;
}- 3、在统计时间时，可能碰到的3种组合情况：
4、如果一个window还没有完成，会逐步累加时间到p->ravg.sum；如果一个window完成，存储最新window负载到p->ravg.sum_history[RAVG_HIST_SIZE_MAX]中，sum_history[]一共有5个槽位；系统根据sched_window_stats_policy选择策略(RECENT、MAX、AVG、MAX_RECENT_AVG)，根据sum_history[]计算选择一个合适的值作为进程负载p->ravg.demand；同时根据sum_history[]的计算进程的负载预测p->ravg.pred_demand；
5、walt的task级别的负载是p->ravg.demand，cpu级别负载是rq->hmp_stats.cumulative_runnable_avg；
6、
具体的update_task_ravg()代码解析如下：
scheduler_tick() -> update_task_ravg()
↓
/* Reflect task activity on its demand and cpu's busy time statistics */
void update_task_ravg(struct task_struct *p, struct rq *rq, int event,
u64 wallclock, u64 irqtime)
{
u64 runtime;
if (!rq->window_start || sched_disable_window_stats ||
p->ravg.mark_start == wallclock)
return;
lockdep_assert_held(&rq->lock);
/* (1) 根据wallclock更新rq->window_start */
update_window_start(rq, wallclock);
if (!p->ravg.mark_start) {
update_task_cpu_cycles(p, cpu_of(rq));
goto done;
}
/* (2) 更新cycle、walltime的差值，用来计算cpu的当前freq */
update_task_rq_cpu_cycles(p, rq, event, wallclock, irqtime);
/* (3) 更新task的负载demand */
runtime = update_task_demand(p, rq, event, wallclock);
if (runtime)
update_task_burst(p, rq, event, runtime);
/* (4) 更新cpu的busy时间 */
update_cpu_busy_time(p, rq, event, wallclock, irqtime);
/* (5) 更新task的负载预测pred_demand */
update_task_pred_demand(rq, p, event);
done:
trace_sched_update_task_ravg(p, rq, event, wallclock, irqtime,
rq->cc.cycles, rq->cc.time,
p->grp ? &rq->grp_time : NULL);
/* (6) 更新task的时间更新点：p->ravg.mark_start */
p->ravg.mark_start = wallclock;
}
|→
static u64 update_task_demand(struct task_struct *p, struct rq *rq,
int event, u64 wallclock)
{
u64 mark_start = p->ravg.mark_start;
u64 delta, window_start = rq->window_start;
int new_window, nr_full_windows;
u32 window_size = sched_ravg_window;
u64 runtime;
new_window = mark_start < window_start;
/* (3.1) 这是一个关键点，非runnable状态的统计需要在这里异常返回 */
if (!account_busy_for_task_demand(p, event)) {
if (new_window)
/*
* If the time accounted isn't being accounted as
* busy time, and a new window started, only the
* previous window need be closed out with the
* pre-existing demand. Multiple windows may have
* elapsed, but since empty windows are dropped,
* it is not necessary to account those.
*/
update_history(rq, p, p->ravg.sum, 1, event);
return 0;
}
/* (3.2) 第一种情况：还在原窗口内，简单继续累加p->ravg.sum */
if (!new_window) {
/*
* The simple case - busy time contained within the existing
* window.
*/
return add_to_task_demand(rq, p, wallclock - mark_start);
}
/* (3.3) 第二、三种情况：原窗口已经填满 */
/*
* Busy time spans at least two windows. Temporarily rewind
* window_start to first window boundary after mark_start.
*/
delta = window_start - mark_start;
nr_full_windows = div64_u64(delta, window_size);
window_start -= (u64)nr_full_windows * (u64)window_size;
/* (3.4.1) 补全第一个窗口 */
/* Process (window_start - mark_start) first */
runtime = add_to_task_demand(rq, p, window_start - mark_start);
/* (3.4.2) 把第一个窗口更新到进程task负载history中,
更新p->ravg.demand、p->ravg.pred_demand
*/
/* Push new sample(s) into task's demand history */
update_history(rq, p, p->ravg.sum, 1, event);
/* (3.5) 如果中间有几个完整窗口，更新负载，更新history */
if (nr_full_windows) {
u64 scaled_window = scale_exec_time(window_size, rq);
update_history(rq, p, scaled_window, nr_full_windows, event);
runtime += nr_full_windows * scaled_window;
}
/* (3.6) 最后一个没有完成的窗口，只是简单累加时间，不更新history */
/*
* Roll window_start back to current to process any remainder
* in current window.
*/
window_start += (u64)nr_full_windows * (u64)window_size;
/* Process (wallclock - window_start) next */
mark_start = window_start;
runtime += add_to_task_demand(rq, p, wallclock - mark_start);
return runtime;
}
||→
static int account_busy_for_task_demand(struct task_struct *p, int event)
{
/*
* No need to bother updating task demand for exiting tasks
* or the idle task.
*/
/* (3.1.1) exit、idle任务不计入统计 */
if (exiting_task(p) || is_idle_task(p))
return 0;
/*
* When a task is waking up it is completing a segment of non-busy
* time. Likewise, if wait time is not treated as busy time, then
* when a task begins to run or is migrated, it is not running and
* is completing a segment of non-busy time.
*/
/* (3.1.2) 任务被wakeup，之前的等待时间不计入统计
SCHED_ACCOUNT_WAIT_TIME用来控制ruannable的等待时间是否计入统计，默认是计入的
*/
if (event == TASK_WAKE || (!SCHED_ACCOUNT_WAIT_TIME &&
(event == PICK_NEXT_TASK || event == TASK_MIGRATE)))
return 0;
return 1;
}
||→
static void add_to_task_demand(struct rq *rq, struct task_struct *p,
u64 delta)
{
/* (3.4.1) 累加窗口的时间值 */
delta = scale_exec_time(delta, rq);
p->ravg.sum += delta;
if (unlikely(p->ravg.sum > walt_ravg_window))
p->ravg.sum = walt_ravg_window;
}
static inline u64 scale_exec_time(u64 delta, struct rq *rq)
{
u32 freq;
/* curr_freq / max_possible_freq */
freq = cpu_cycles_to_freq(rq->cc.cycles, rq->cc.time);
delta = DIV64_U64_ROUNDUP(delta * freq, max_possible_freq);
/* exec_scale_factor = cluster->efficiency / max_possible_efficiency */
delta *= rq->cluster->exec_scale_factor;
delta >>= 10;
return delta;
}
||→
static void update_history(struct rq *rq, struct task_struct *p,
u32 runtime, int samples, int event)
{
u32 *hist = &p->ravg.sum_history[0];
int ridx, widx;
u32 max = 0, avg, demand, pred_demand;
u64 sum = 0;
/* (3.4.2.1) 不活跃的进程不进行更新 */
/* Ignore windows where task had no activity */
if (!runtime || is_idle_task(p) || exiting_task(p) || !samples)
goto done;
/* (3.4.2.2) 把新窗口的runtime推送到history stack中 */
/* Push new 'runtime' value onto stack */
widx = sched_ravg_hist_size - 1;
ridx = widx - samples;
for (; ridx >= 0; --widx, --ridx) {
hist[widx] = hist[ridx];
sum += hist[widx];
if (hist[widx] > max)
max = hist[widx];
}
for (widx = 0; widx < samples && widx < sched_ravg_hist_size; widx++) {
hist[widx] = runtime;
sum += hist[widx];
if (hist[widx] > max)
max = hist[widx];
}
p->ravg.sum = 0;
/* (3.4.2.3) 根据sched_window_stats_policy策略(RECENT、MAX、AVG、MAX_RECENT_AVG)，
从sum_history[]中选择合适的值作为进程负载p->ravg.demand
*/
if (sched_window_stats_policy == WINDOW_STATS_RECENT) {
demand = runtime;
} else if (sched_window_stats_policy == WINDOW_STATS_MAX) {
demand = max;
} else {
avg = div64_u64(sum, sched_ravg_hist_size);
if (sched_window_stats_policy == WINDOW_STATS_AVG)
demand = avg;
else
demand = max(avg, runtime);
}
/* (3.4.2.4) 计算进程的预测负载 */
pred_demand = predict_and_update_buckets(rq, p, runtime);
/*
* A throttled deadline sched class task gets dequeued without
* changing p->on_rq. Since the dequeue decrements hmp stats
* avoid decrementing it here again.
*/
/* (3.4.2.5) 更新进程负载(p->ravg.demand)到cpu负载(rq->hmp_stats.cumulative_runnable_avg)中
cfs中p->sched_class->fixup_hmp_sched_stats对应函数fixup_hmp_sched_stats_fair()
*/
if (task_on_rq_queued(p) && (!task_has_dl_policy(p) ||
!p->dl.dl_throttled))
p->sched_class->fixup_hmp_sched_stats(rq, p, demand,
pred_demand);
p->ravg.demand = demand;
p->ravg.pred_demand = pred_demand;
done:
trace_sched_update_history(rq, p, runtime, samples, event);
}
|||→
static inline u32 predict_and_update_buckets(struct rq *rq,
struct task_struct *p, u32 runtime) {
int bidx;
u32 pred_demand;
/* (3.4.2.4.1) 把window负载转换成bucket index(最大10) */
bidx = busy_to_bucket(runtime);
/* (3.4.2.4.2) 根据index，找到历史曾经达到过的更大值，取历史的值作为预测值 */
pred_demand = get_pred_busy(rq, p, bidx, runtime);
/* (3.4.2.4.3) 对bucket[]中本次index权重进行增加，其他权重减少 */
bucket_increase(p->ravg.busy_buckets, bidx);
return pred_demand;
}
|||→
static void
fixup_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p,
u32 new_task_load, u32 new_pred_demand)
{
/* (3.4.2.5.1) 计算task负载和预测的变化值delta */
s64 task_load_delta = (s64)new_task_load - task_load(p);
s64 pred_demand_delta = PRED_DEMAND_DELTA;
/* (3.4.2.5.2) 将进程级别的delta计入cpu级别的负载统计(rq->hmp_stats)中 */
fixup_cumulative_runnable_avg(&rq->hmp_stats, p, task_load_delta,
pred_demand_delta);
/* (3.4.2.5.3) 更新cpu级别big_task的数量 */
fixup_nr_big_tasks(&rq->hmp_stats, p, task_load_delta);
}
static inline void
fixup_cumulative_runnable_avg(struct hmp_sched_stats *stats,
struct task_struct *p, s64 task_load_delta,
s64 pred_demand_delta)
{
if (sched_disable_window_stats)
return;
stats->cumulative_runnable_avg += task_load_delta;
BUG_ON((s64)stats->cumulative_runnable_avg < 0);
stats->pred_demands_sum += pred_demand_delta;
BUG_ON((s64)stats->pred_demands_sum < 0);
}
void fixup_nr_big_tasks(struct hmp_sched_stats *stats,
struct task_struct *p, s64 delta)
{
u64 new_task_load;
u64 old_task_load;
if (sched_disable_window_stats)
return;
/* task_load按照capacity反比放大，让所有cpu处在同一级别 */
old_task_load = scale_load_to_cpu(task_load(p), task_cpu(p));
new_task_load = scale_load_to_cpu(delta + task_load(p), task_cpu(p));
/* 如果进程负载 > 最大负载 * 80% (sysctl_sched_upmigrate_pct)
该任务为big_task
*/
if (__is_big_task(p, old_task_load) && !__is_big_task(p, new_task_load))
stats->nr_big_tasks--;
else if (!__is_big_task(p, old_task_load) &&
__is_big_task(p, new_task_load))
stats->nr_big_tasks++;
BUG_ON(stats->nr_big_tasks < 0);
}
我们再来详细看看cpu级别的busy time计算：
static void update_cpu_busy_time(struct task_struct *p, struct rq *rq,
int event, u64 wallclock, u64 irqtime)
{
int new_window, full_window = 0;
int p_is_curr_task = (p == rq->curr);
u64 mark_start = p->ravg.mark_start;
u64 window_start = rq->window_start;
u32 window_size = sched_ravg_window;
u64 delta;
u64 *curr_runnable_sum = &rq->curr_runnable_sum;
u64 *prev_runnable_sum = &rq->prev_runnable_sum;
u64 *nt_curr_runnable_sum = &rq->nt_curr_runnable_sum;
u64 *nt_prev_runnable_sum = &rq->nt_prev_runnable_sum;
bool new_task;
struct related_thread_group *grp;
int cpu = rq->cpu;
u32 old_curr_window = p->ravg.curr_window;
new_window = mark_start < window_start;
if (new_window) {
full_window = (window_start - mark_start) >= window_size;
if (p->ravg.active_windows < USHRT_MAX)
p->ravg.active_windows++;
}
new_task = is_new_task(p);
/*
* Handle per-task window rollover. We don't care about the idle
* task or exiting tasks.
*/
/* (1) 如果有新window，滚动进程窗口：p->ravg.prev_window、p->ravg.curr_window */
if (!is_idle_task(p) && !exiting_task(p)) {
if (new_window)
rollover_task_window(p, full_window);
}
/* (2) 如果有新window且进程是rq的当前进程，
cpu级别的窗口滚动：rq->prev_runnable_sum、rq->curr_runnable_sum
cpu级别的进程统计窗口滚动：rq->top_tasks[prev_table]、rq->top_tasks[curr_table]
*/
if (p_is_curr_task && new_window) {
rollover_cpu_window(rq, full_window);
rollover_top_tasks(rq, full_window);
}
/* (3) 判断哪些情况可以统计进cpu time */
if (!account_busy_for_cpu_time(rq, p, irqtime, event))
goto done;
grp = p->grp;
if (grp && sched_freq_aggregate) {
struct group_cpu_time *cpu_time = &rq->grp_time;
curr_runnable_sum = &cpu_time->curr_runnable_sum;
prev_runnable_sum = &cpu_time->prev_runnable_sum;
nt_curr_runnable_sum = &cpu_time->nt_curr_runnable_sum;
nt_prev_runnable_sum = &cpu_time->nt_prev_runnable_sum;
}
/* (4) 如果时间没有达到新window，
在cpu级别的当前负载上累加：rq->curr_runnable_sum
在进程级别的基础上累加：p->ravg.curr_window
*/
if (!new_window) {
/*
* account_busy_for_cpu_time() = 1 so busy time needs
* to be accounted to the current window. No rollover
* since we didn't start a new window. An example of this is
* when a task starts execution and then sleeps within the
* same window.
*/
if (!irqtime || !is_idle_task(p) || cpu_is_waiting_on_io(rq))
delta = wallclock - mark_start;
else
delta = irqtime;
delta = scale_exec_time(delta, rq);
*curr_runnable_sum += delta;
if (new_task)
*nt_curr_runnable_sum += delta;
if (!is_idle_task(p) && !exiting_task(p)) {
p->ravg.curr_window += delta;
p->ravg.curr_window_cpu[cpu] += delta;
}
goto done;
}
/* (5) 如果时间达到新window，但是进程不是rq的当前进程
在进程级别的基础上累加：p->ravg.prev_window、p->ravg.curr_window
在cpu级别的当前负载上累加：rq->prev_runnable_sum、rq->curr_runnable_sum
*/
if (!p_is_curr_task) {
/*
* account_busy_for_cpu_time() = 1 so busy time needs
* to be accounted to the current window. A new window
* has also started, but p is not the current task, so the
* window is not rolled over - just split up and account
* as necessary into curr and prev. The window is only
* rolled over when a new window is processed for the current
* task.
*
* Irqtime can't be accounted by a task that isn't the
* currently running task.
*/
if (!full_window) {
/*
* A full window hasn't elapsed, account partial
* contribution to previous completed window.
*/
delta = scale_exec_time(window_start - mark_start, rq);
if (!exiting_task(p)) {
p->ravg.prev_window += delta;
p->ravg.prev_window_cpu[cpu] += delta;
}
} else {
/*
* Since at least one full window has elapsed,
* the contribution to the previous window is the
* full window (window_size).
*/
delta = scale_exec_time(window_size, rq);
if (!exiting_task(p)) {
p->ravg.prev_window = delta;
p->ravg.prev_window_cpu[cpu] = delta;
}
}
*prev_runnable_sum += delta;
if (new_task)
*nt_prev_runnable_sum += delta;
/* Account piece of busy time in the current window. */
delta = scale_exec_time(wallclock - window_start, rq);
*curr_runnable_sum += delta;
if (new_task)
*nt_curr_runnable_sum += delta;
if (!exiting_task(p)) {
p->ravg.curr_window = delta;
p->ravg.curr_window_cpu[cpu] = delta;
}
goto done;
}
/* (6) 如果时间达到新window，且进程是rq的当前进程
在进程级别的基础上累加：p->ravg.prev_window、p->ravg.curr_window
在cpu级别的当前负载上累加：rq->prev_runnable_sum、rq->curr_runnable_sum
*/
if (!irqtime || !is_idle_task(p) || cpu_is_waiting_on_io(rq)) {
/*
* account_busy_for_cpu_time() = 1 so busy time needs
* to be accounted to the current window. A new window
* has started and p is the current task so rollover is
* needed. If any of these three above conditions are true
* then this busy time can't be accounted as irqtime.
*
* Busy time for the idle task or exiting tasks need not
* be accounted.
*
* An example of this would be a task that starts execution
* and then sleeps once a new window has begun.
*/
if (!full_window) {
/*
* A full window hasn't elapsed, account partial
* contribution to previous completed window.
*/
delta = scale_exec_time(window_start - mark_start, rq);
if (!is_idle_task(p) && !exiting_task(p)) {
p->ravg.prev_window += delta;
p->ravg.prev_window_cpu[cpu] += delta;
}
} else {
/*
* Since at least one full window has elapsed,
* the contribution to the previous window is the
* full window (window_size).
*/
delta = scale_exec_time(window_size, rq);
if (!is_idle_task(p) && !exiting_task(p)) {
p->ravg.prev_window = delta;
p->ravg.prev_window_cpu[cpu] = delta;
}
}
/*
* Rollover is done here by overwriting the values in
* prev_runnable_sum and curr_runnable_sum.
*/
*prev_runnable_sum += delta;
if (new_task)
*nt_prev_runnable_sum += delta;
/* Account piece of busy time in the current window. */
delta = scale_exec_time(wallclock - window_start, rq);
*curr_runnable_sum += delta;
if (new_task)
*nt_curr_runnable_sum += delta;
if (!is_idle_task(p) && !exiting_task(p)) {
p->ravg.curr_window = delta;
p->ravg.curr_window_cpu[cpu] = delta;
}
goto done;
}
if (irqtime) {
/*
* account_busy_for_cpu_time() = 1 so busy time needs
* to be accounted to the current window. A new window
* has started and p is the current task so rollover is
* needed. The current task must be the idle task because
* irqtime is not accounted for any other task.
*
* Irqtime will be accounted each time we process IRQ activity
* after a period of idleness, so we know the IRQ busy time
* started at wallclock - irqtime.
*/
BUG_ON(!is_idle_task(p));
mark_start = wallclock - irqtime;
/*
* Roll window over. If IRQ busy time was just in the current
* window then that is all that need be accounted.
*/
if (mark_start > window_start) {
*curr_runnable_sum = scale_exec_time(irqtime, rq);
return;
}
/*
* The IRQ busy time spanned multiple windows. Process the
* busy time preceding the current window start first.
*/
delta = window_start - mark_start;
if (delta > window_size)
delta = window_size;
delta = scale_exec_time(delta, rq);
*prev_runnable_sum += delta;
/* Process the remaining IRQ busy time in the current window. */
delta = wallclock - window_start;
rq->curr_runnable_sum = scale_exec_time(delta, rq);
return;
}
done:
/* (7) 更新cpu上的top task */
if (!is_idle_task(p) && !exiting_task(p))
update_top_tasks(p, rq, old_curr_window,
new_window, full_window);
}
|→
static void update_top_tasks(struct task_struct *p, struct rq *rq,
u32 old_curr_window, int new_window, bool full_window)
{
u8 curr = rq->curr_table;
u8 prev = 1 - curr;
u8 *curr_table = rq->top_tasks[curr];
u8 *prev_table = rq->top_tasks[prev];
int old_index, new_index, update_index;
u32 curr_window = p->ravg.curr_window;
u32 prev_window = p->ravg.prev_window;
bool zero_index_update;
if (old_curr_window == curr_window && !new_window)
return;
/* (1) 把就进程p的"当前window负载"、"旧的当前window负载"转换成index(NUM_LOAD_INDICES=1000) */
old_index = load_to_index(old_curr_window);
new_index = load_to_index(curr_window);
/* (2) 如果没有新window
更新当前top表rq->top_tasks[curr][]中新旧index的计数
根据index的计数是否为0，更新rq->top_tasks_bitmap[curr] bitmap中对应index的值
*/
if (!new_window) {
zero_index_update = !old_curr_window && curr_window;
if (old_index != new_index || zero_index_update) {
if (old_curr_window)
curr_table[old_index] -= 1;
if (curr_window)
curr_table[new_index] += 1;
if (new_index > rq->curr_top)
rq->curr_top = new_index;
}
if (!curr_table[old_index])
__clear_bit(NUM_LOAD_INDICES - old_index - 1,
rq->top_tasks_bitmap[curr]);
if (curr_table[new_index] == 1)
__set_bit(NUM_LOAD_INDICES - new_index - 1,
rq->top_tasks_bitmap[curr]);
return;
}
/*
* The window has rolled over for this task. By the time we get
* here, curr/prev swaps would has already occurred. So we need
* to use prev_window for the new index.
*/
update_index = load_to_index(prev_window);
if (full_window) {
/*
* Two cases here. Either 'p' ran for the entire window or
* it didn't run at all. In either case there is no entry
* in the prev table. If 'p' ran the entire window, we just
* need to create a new entry in the prev table. In this case
* update_index will be correspond to sched_ravg_window
* so we can unconditionally update the top index.
*/
if (prev_window) {
prev_table[update_index] += 1;
rq->prev_top = update_index;
}
if (prev_table[update_index] == 1)
__set_bit(NUM_LOAD_INDICES - update_index - 1,
rq->top_tasks_bitmap[prev]);
} else {
zero_index_update = !old_curr_window && prev_window;
if (old_index != update_index || zero_index_update) {
if (old_curr_window)
prev_table[old_index] -= 1;
prev_table[update_index] += 1;
if (update_index > rq->prev_top)
rq->prev_top = update_index;
if (!prev_table[old_index])
__clear_bit(NUM_LOAD_INDICES - old_index - 1,
rq->top_tasks_bitmap[prev]);
if (prev_table[update_index] == 1)
__set_bit(NUM_LOAD_INDICES - update_index - 1,
rq->top_tasks_bitmap[prev]);
}
}
if (curr_window) {
curr_table[new_index] += 1;
if (new_index > rq->curr_top)
rq->curr_top = new_index;
if (curr_table[new_index] == 1)
__set_bit(NUM_LOAD_INDICES - new_index - 1,
rq->top_tasks_bitmap[curr]);
}
}
5.3.2、基于WALT的负载均衡
5.3.2.1、load_balance()
其他部分和主干内核算法一致，这里只标识出qualcom的HMP算法特有的部分。在负载均衡部分，walt用来找出cpu；但是在负载迁移时，计算负载还是使用pelt？
- 在find_busiest_queue()中：原本是找出cfs_rq->runnable_load_avg * capacity负载最大的cpu，qualcom HMP改为找出walt runnable负载(rq->hmp_stats.cumulative_runnable_avg)最重的cpu。
run_rebalance_domains() -> rebalance_domains() -> load_balance() -> find_busiest_queue() -> find_busiest_queue_hmp()
↓
static struct rq *find_busiest_queue_hmp(struct lb_env *env,
struct sched_group *group)
{
struct rq *busiest = NULL, *busiest_big = NULL;
u64 max_runnable_avg = 0, max_runnable_avg_big = 0;
int max_nr_big = 0, nr_big;
bool find_big = !!(env->flags & LBF_BIG_TASK_ACTIVE_BALANCE);
int i;
cpumask_t cpus;
cpumask_andnot(&cpus, sched_group_cpus(group), cpu_isolated_mask);
/* (1) 遍历sg中的cpu */
for_each_cpu(i, &cpus) {
struct rq *rq = cpu_rq(i);
u64 cumulative_runnable_avg =
rq->hmp_stats.cumulative_runnable_avg;
if (!cpumask_test_cpu(i, env->cpus))
continue;
/* (2) 考虑big_task，找出big_task最重的cpu */
if (find_big) {
nr_big = nr_big_tasks(rq);
if (nr_big > max_nr_big ||
(nr_big > 0 && nr_big == max_nr_big &&
cumulative_runnable_avg > max_runnable_avg_big)) {
max_runnable_avg_big = cumulative_runnable_avg;
busiest_big = rq;
max_nr_big = nr_big;
continue;
}
}
/* (3) 找出walt runnable负载(rq->hmp_stats.cumulative_runnable_avg)最重的cpu */
if (cumulative_runnable_avg > max_runnable_avg) {
max_runnable_avg = cumulative_runnable_avg;
busiest = rq;
}
}
if (busiest_big)
return busiest_big;
env->flags &= ~LBF_BIG_TASK_ACTIVE_BALANCE;
return busiest;
}
5.3.2.2、nohz_idle_balance()
- _nohz_kick_needed()：
scheduler_tick() -> trigger_load_balance() -> nohz_kick_needed() -> _nohz_kick_needed() -> nohz_kick_needed_hmp()
↓
static inline int _nohz_kick_needed_hmp(struct rq *rq, int cpu, int *type)
{
struct sched_domain *sd;
int i;
if (rq->nr_running < 2)
return 0;
/* (1) 如果是SCHED_BOOST_ON_ALL，返回true */
if (!sysctl_sched_restrict_cluster_spill ||
sched_boost_policy() == SCHED_BOOST_ON_ALL)
return 1;
/* (2) 如果当前cpu是max cpu，返回true */
if (cpu_max_power_cost(cpu) == max_power_cost)
return 1;
rcu_read_lock();
sd = rcu_dereference_check_sched_domain(rq->sd);
if (!sd) {
rcu_read_unlock();
return 0;
}
for_each_cpu(i, sched_domain_span(sd)) {
if (cpu_load(i) < sched_spill_load &&
cpu_rq(i)->nr_running <
sysctl_sched_spill_nr_run) {
/* Change the kick type to limit to CPUs that
* are of equal or lower capacity.
*/
*type = NOHZ_KICK_RESTRICT;
break;
}
}
rcu_read_unlock();
return 1;
}
- find_new_hmp_ilb()：原本是找出nohz.idle_cpus_mask中的第一个cpu作为ilb cpu，qualcom HMP改为尝试在nohz.idle_cpus_mask中找到一个max power小于当前cpu的作为ilb cpu。
scheduler_tick() -> trigger_load_balance() -> nohz_balancer_kick() -> find_new_ilb()
↓
static inline int find_new_hmp_ilb(int type)
{
int call_cpu = raw_smp_processor_id();
struct sched_domain *sd;
int ilb;
rcu_read_lock();
/* Pick an idle cpu "closest" to call_cpu */
for_each_domain(call_cpu, sd) {
for_each_cpu_and(ilb, nohz.idle_cpus_mask,
sched_domain_span(sd)) {
/* (1) 尝试找到一个max power小于当前power的cpu作为ilb cpu */
if (idle_cpu(ilb) && (type != NOHZ_KICK_RESTRICT ||
cpu_max_power_cost(ilb) <=
cpu_max_power_cost(call_cpu))) {
rcu_read_unlock();
reset_balance_interval(ilb);
return ilb;
}
}
}
rcu_read_unlock();
return nr_cpu_ids;
}5.3.2.3、select_task_rq_fair()
- select_task_rq_fair()：使用qualcom自己的算法，综合capacity、power、idle给出一个best cpu。
select_task_rq_fair() -> select_best_cpu()
↓
/* return cheapest cpu that can fit this task */
static int select_best_cpu(struct task_struct *p, int target, int reason,
int sync)
{
struct sched_cluster *cluster, *pref_cluster = NULL;
struct cluster_cpu_stats stats;
struct related_thread_group *grp;
unsigned int sbc_flag = 0;
int cpu = raw_smp_processor_id();
bool special;
struct cpu_select_env env = {
.p = p,
.reason = reason,
.need_idle = wake_to_idle(p),
.need_waker_cluster = 0,
.sync = sync,
.prev_cpu = target,
.rtg = NULL,
.sbc_best_flag = 0,
.sbc_best_cluster_flag = 0,
.pack_task = false,
};
rcu_read_lock();
env.boost_policy = task_sched_boost(p) ?
sched_boost_policy() : SCHED_BOOST_NONE;
bitmap_copy(env.candidate_list, all_cluster_ids, NR_CPUS);
bitmap_zero(env.backup_list, NR_CPUS);
cpumask_and(&env.search_cpus, tsk_cpus_allowed(p), cpu_active_mask);
cpumask_andnot(&env.search_cpus, &env.search_cpus, cpu_isolated_mask);
init_cluster_cpu_stats(&stats);
special = env_has_special_flags(&env);
grp = task_related_thread_group(p);
if (grp && grp->preferred_cluster) {
pref_cluster = grp->preferred_cluster;
if (!cluster_allowed(&env, pref_cluster))
clear_bit(pref_cluster->id, env.candidate_list);
else
env.rtg = grp;
} else if (!special) {
cluster = cpu_rq(cpu)->cluster;
if (wake_to_waker_cluster(&env)) {
if (bias_to_waker_cpu(&env, cpu)) {
target = cpu;
sbc_flag = SBC_FLAG_WAKER_CLUSTER |
SBC_FLAG_WAKER_CPU;
goto out;
} else if (cluster_allowed(&env, cluster)) {
env.need_waker_cluster = 1;
bitmap_zero(env.candidate_list, NR_CPUS);
__set_bit(cluster->id, env.candidate_list);
env.sbc_best_cluster_flag =
SBC_FLAG_WAKER_CLUSTER;
}
} else if (bias_to_prev_cpu(&env, &stats)) {
sbc_flag = SBC_FLAG_PREV_CPU;
goto out;
}
}
if (!special && is_short_burst_task(p)) {
env.pack_task = true;
sbc_flag = SBC_FLAG_PACK_TASK;
}
retry:
/* (1) 从低到高找到一个power最低，且capacity能满足task_load的cluster */
cluster = select_least_power_cluster(&env);
if (!cluster)
goto out;
/*
* 'cluster' now points to the minimum power cluster which can satisfy
* task's perf goals. Walk down the cluster list starting with that
* cluster. For non-small tasks, skip clusters that don't have
* mostly_idle/idle cpus
*/
do {
/* (2) 全方位统计：capacity spare、cost、idle */
find_best_cpu_in_cluster(cluster, &env, &stats);
} while ((cluster = next_best_cluster(cluster, &env, &stats)));
/* (3) 从idle角度给出best cpu */
if (env.need_idle) {
if (stats.best_idle_cpu >= 0) {
target = stats.best_idle_cpu;
sbc_flag |= SBC_FLAG_IDLE_CSTATE;
} else if (stats.least_loaded_cpu >= 0) {
target = stats.least_loaded_cpu;
sbc_flag |= SBC_FLAG_IDLE_LEAST_LOADED;
}
/* (4) 从综合角度给出best cpu */
} else if (stats.best_cpu >= 0) {
if (stats.best_sibling_cpu >= 0 &&
stats.best_cpu != task_cpu(p) &&
stats.min_cost == stats.best_sibling_cpu_cost) {
stats.best_cpu = stats.best_sibling_cpu;
sbc_flag |= SBC_FLAG_BEST_SIBLING;
}
sbc_flag |= env.sbc_best_flag;
target = stats.best_cpu;
} else {
if (env.rtg && env.boost_policy == SCHED_BOOST_NONE) {
env.rtg = NULL;
goto retry;
}
/*
* With boost_policy == SCHED_BOOST_ON_BIG, we reach here with
* backup_list = little cluster, candidate_list = none and
* stats->best_capacity_cpu points the best spare capacity
* CPU among the CPUs in the big cluster.
*/
if (env.boost_policy == SCHED_BOOST_ON_BIG &&
stats.best_capacity_cpu >= 0)
sbc_flag |= SBC_FLAG_BOOST_CLUSTER;
else
find_backup_cluster(&env, &stats);
if (stats.best_capacity_cpu >= 0) {
target = stats.best_capacity_cpu;
sbc_flag |= SBC_FLAG_BEST_CAP_CPU;
}
}
p->last_cpu_selected_ts = sched_ktime_clock();
out:
sbc_flag |= env.sbc_best_cluster_flag;
rcu_read_unlock();
trace_sched_task_load(p, sched_boost_policy() && task_sched_boost(p),
env.reason, env.sync, env.need_idle, sbc_flag, target);
return target;
}5.3.2.4、Interaction Governor & sched_load
qualcom对interactive governor进行了改造，打造成了可以使用sched_load的interactive governor。
- 1、interactive governor注册回调函数，接收sched_load变化事件；
static ssize_t store_use_sched_load(
struct cpufreq_interactive_tunables *tunables,
const char *buf, size_t count)
{
int ret;
unsigned long val;
ret = kstrtoul(buf, 0, &val);
if (ret < 0)
return ret;
if (tunables->use_sched_load == (bool) val)
return count;
tunables->use_sched_load = val;
if (val)
ret = cpufreq_interactive_enable_sched_input(tunables);
else
ret = cpufreq_interactive_disable_sched_input(tunables);
if (ret) {
tunables->use_sched_load = !val;
return ret;
}
return count;
}
|→
static int cpufreq_interactive_enable_sched_input(
struct cpufreq_interactive_tunables *tunables)
{
int rc = 0, j;
struct cpufreq_interactive_tunables *t;
mutex_lock(&sched_lock);
set_window_count++;
if (set_window_count > 1) {
for_each_possible_cpu(j) {
if (!per_cpu(polinfo, j))
continue;
t = per_cpu(polinfo, j)->cached_tunables;
if (t && t->use_sched_load) {
tunables->timer_rate = t->timer_rate;
tunables->io_is_busy = t->io_is_busy;
break;
}
}
} else {
/* (1) 设置walt窗口大小 */
rc = set_window_helper(tunables);
if (rc) {
pr_err("%s: Failed to set sched window\n", __func__);
set_window_count--;
goto out;
}
sched_set_io_is_busy(tunables->io_is_busy);
}
if (!tunables->use_migration_notif)
goto out;
migration_register_count++;
if (migration_register_count > 1)
goto out;
else
/* (2) 注册sched_load变化的回调函数 */
atomic_notifier_chain_register(&load_alert_notifier_head,
&load_notifier_block);
out:
mutex_unlock(&sched_lock);
return rc;
}
||→
static inline int set_window_helper(
struct cpufreq_interactive_tunables *tunables)
{
/* 设置默认窗口size为DEFAULT_TIMER_RATE(20ms) */
return sched_set_window(round_to_nw_start(get_jiffies_64(), tunables),
usecs_to_jiffies(tunables->timer_rate));
}
static struct notifier_block load_notifier_block = {
.notifier_call = load_change_callback,
};
- 2、sched_load的变化通过回调函数通知给Interaction Governor；
check_for_freq_change（） -> load_alert_notifier_head -> load_change_callback()
↓
static int load_change_callback(struct notifier_block *nb, unsigned long val,
void *data)
{
unsigned long cpu = (unsigned long) data;
struct cpufreq_interactive_policyinfo *ppol = per_cpu(polinfo, cpu);
struct cpufreq_interactive_tunables *tunables;
unsigned long flags;
if (!ppol || ppol->reject_notification)
return 0;
if (!down_read_trylock(&ppol->enable_sem))
return 0;
if (!ppol->governor_enabled)
goto exit;
tunables = ppol->policy->governor_data;
if (!tunables->use_sched_load || !tunables->use_migration_notif)
goto exit;
spin_lock_irqsave(&ppol->target_freq_lock, flags);
ppol->notif_pending = true;
ppol->notif_cpu = cpu;
spin_unlock_irqrestore(&ppol->target_freq_lock, flags);
if (!hrtimer_is_queued(&ppol->notif_timer))
hrtimer_start(&ppol->notif_timer, ms_to_ktime(1),
HRTIMER_MODE_REL);
exit:
up_read(&ppol->enable_sem);
return 0;
}
- 3、除了事件通知，interactive governor还会在20ms timer中轮询sched_load的变化来决定是否需要调频。
static void cpufreq_interactive_timer(unsigned long data)
{
s64 now;
unsigned int delta_time;
u64 cputime_speedadj;
int cpu_load;
int pol_load = 0;
struct cpufreq_interactive_policyinfo *ppol = per_cpu(polinfo, data);
struct cpufreq_interactive_tunables *tunables =
ppol->policy->governor_data;
struct sched_load *sl = ppol->sl;
struct cpufreq_interactive_cpuinfo *pcpu;
unsigned int new_freq;
unsigned int prev_laf = 0, t_prevlaf;
unsigned int pred_laf = 0, t_predlaf = 0;
unsigned int prev_chfreq, pred_chfreq, chosen_freq;
unsigned int index;
unsigned long flags;
unsigned long max_cpu;
int cpu, i;
int new_load_pct = 0;
int prev_l, pred_l = 0;
struct cpufreq_govinfo govinfo;
bool skip_hispeed_logic, skip_min_sample_time;
bool jump_to_max_no_ts = false;
bool jump_to_max = false;
bool start_hyst = true;
if (!down_read_trylock(&ppol->enable_sem))
return;
if (!ppol->governor_enabled)
goto exit;
now = ktime_to_us(ktime_get());
spin_lock_irqsave(&ppol->target_freq_lock, flags);
spin_lock(&ppol->load_lock);
skip_hispeed_logic =
tunables->ignore_hispeed_on_notif && ppol->notif_pending;
skip_min_sample_time = tunables->fast_ramp_down && ppol->notif_pending;
ppol->notif_pending = false;
now = ktime_to_us(ktime_get());
ppol->last_evaluated_jiffy = get_jiffies_64();
/* (1) sched_load模式，查询最新的sched_load */
if (tunables->use_sched_load)
sched_get_cpus_busy(sl, ppol->policy->cpus);
max_cpu = cpumask_first(ppol->policy->cpus);
i = 0;
for_each_cpu(cpu, ppol->policy->cpus) {
pcpu = &per_cpu(cpuinfo, cpu);
/* (2) sched_load模式，使用sched_load来计算负载变化 */
if (tunables->use_sched_load) {
/* (2.1) 根据上个窗口负载，获得当前目标值 */
t_prevlaf = sl_busy_to_laf(ppol, sl[i].prev_load);
prev_l = t_prevlaf / ppol->target_freq;
/* (2.2) 根据上个窗口负载预测，获得当前的预测值 */
if (tunables->enable_prediction) {
t_predlaf = sl_busy_to_laf(ppol,
sl[i].predicted_load);
pred_l = t_predlaf / ppol->target_freq;
}
if (sl[i].prev_load)
new_load_pct = sl[i].new_task_load * 100 /
sl[i].prev_load;
else
new_load_pct = 0;
/* (3) 传统模式，使用time*freq的模式来计算负载变化 */
} else {
now = update_load(cpu);
delta_time = (unsigned int)
(now - pcpu->cputime_speedadj_timestamp);
if (WARN_ON_ONCE(!delta_time))
continue;
cputime_speedadj = pcpu->cputime_speedadj;
do_div(cputime_speedadj, delta_time);
t_prevlaf = (unsigned int)cputime_speedadj * 100;
prev_l = t_prevlaf / ppol->target_freq;
}
/* find max of loadadjfreq inside policy */
if (t_prevlaf > prev_laf) {
prev_laf = t_prevlaf;
max_cpu = cpu;
}
pred_laf = max(t_predlaf, pred_laf);
cpu_load = max(prev_l, pred_l);
pol_load = max(pol_load, cpu_load);
trace_cpufreq_interactive_cpuload(cpu, cpu_load, new_load_pct,
prev_l, pred_l);
/* save loadadjfreq for notification */
pcpu->loadadjfreq = max(t_prevlaf, t_predlaf);
/* detect heavy new task and jump to policy->max */
if (prev_l >= tunables->go_hispeed_load &&
new_load_pct >= NEW_TASK_RATIO) {
skip_hispeed_logic = true;
jump_to_max = true;
}
i++;
}
spin_unlock(&ppol->load_lock);
tunables->boosted = tunables->boost_val || now < tunables->boostpulse_endtime;
/* (4) 取目标值和预测值中的较大值，作为调频目标 */
prev_chfreq = choose_freq(ppol, prev_laf);
pred_chfreq = choose_freq(ppol, pred_laf);
chosen_freq = max(prev_chfreq, pred_chfreq);
if (prev_chfreq < ppol->policy->max && pred_chfreq >= ppol->policy->max)
if (!jump_to_max)
jump_to_max_no_ts = true;
if (now - ppol->max_freq_hyst_start_time <
tunables->max_freq_hysteresis &&
pol_load >= tunables->go_hispeed_load &&
ppol->target_freq < ppol->policy->max) {
skip_hispeed_logic = true;
skip_min_sample_time = true;
if (!jump_to_max)
jump_to_max_no_ts = true;
}
new_freq = chosen_freq;
if (jump_to_max_no_ts || jump_to_max) {
new_freq = ppol->policy->cpuinfo.max_freq;
} else if (!skip_hispeed_logic) {
if (pol_load >= tunables->go_hispeed_load ||
tunables->boosted) {
if (ppol->target_freq < tunables->hispeed_freq)
new_freq = tunables->hispeed_freq;
else
new_freq = max(new_freq,
tunables->hispeed_freq);
}
}
if (now - ppol->max_freq_hyst_start_time <
tunables->max_freq_hysteresis) {
if (new_freq < ppol->policy->max &&
ppol->policy->max <= tunables->hispeed_freq)
start_hyst = false;
new_freq = max(tunables->hispeed_freq, new_freq);
}
if (!skip_hispeed_logic &&
ppol->target_freq >= tunables->hispeed_freq &&
new_freq > ppol->target_freq &&
now - ppol->hispeed_validate_time <
freq_to_above_hispeed_delay(tunables, ppol->target_freq)) {
trace_cpufreq_interactive_notyet(
max_cpu, pol_load, ppol->target_freq,
ppol->policy->cur, new_freq);
spin_unlock_irqrestore(&ppol->target_freq_lock, flags);
goto rearm;
}
ppol->hispeed_validate_time = now;
if (cpufreq_frequency_table_target(&ppol->p_nolim, ppol->freq_table,
new_freq, CPUFREQ_RELATION_L,
&index)) {
spin_unlock_irqrestore(&ppol->target_freq_lock, flags);
goto rearm;
}
new_freq = ppol->freq_table[index].frequency;
/*
* Do not scale below floor_freq unless we have been at or above the
* floor frequency for the minimum sample time since last validated.
*/
if (!skip_min_sample_time && new_freq < ppol->floor_freq) {
if (now - ppol->floor_validate_time <
tunables->min_sample_time) {
trace_cpufreq_interactive_notyet(
max_cpu, pol_load, ppol->target_freq,
ppol->policy->cur, new_freq);
spin_unlock_irqrestore(&ppol->target_freq_lock, flags);
goto rearm;
}
}
/*
* Update the timestamp for checking whether speed has been held at
* or above the selected frequency for a minimum of min_sample_time,
* if not boosted to hispeed_freq. If boosted to hispeed_freq then we
* allow the speed to drop as soon as the boostpulse duration expires
* (or the indefinite boost is turned off). If policy->max is restored
* for max_freq_hysteresis, don't extend the timestamp. Otherwise, it
* could incorrectly extended the duration of max_freq_hysteresis by
* min_sample_time.
*/
if ((!tunables->boosted || new_freq > tunables->hispeed_freq)
&& !jump_to_max_no_ts) {
ppol->floor_freq = new_freq;
ppol->floor_validate_time = now;
}
if (start_hyst && new_freq >= ppol->policy->max && !jump_to_max_no_ts)
ppol->max_freq_hyst_start_time = now;
if (ppol->target_freq == new_freq &&
ppol->target_freq <= ppol->policy->cur) {
trace_cpufreq_interactive_already(
max_cpu, pol_load, ppol->target_freq,
ppol->policy->cur, new_freq);
spin_unlock_irqrestore(&ppol->target_freq_lock, flags);
goto rearm;
}
trace_cpufreq_interactive_target(max_cpu, pol_load, ppol->target_freq,
ppol->policy->cur, new_freq);
ppol->target_freq = new_freq;
spin_unlock_irqrestore(&ppol->target_freq_lock, flags);
spin_lock_irqsave(&speedchange_cpumask_lock, flags);
cpumask_set_cpu(max_cpu, &speedchange_cpumask);
spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
wake_up_process_no_notif(speedchange_task);
rearm:
if (!timer_pending(&ppol->policy_timer))
cpufreq_interactive_timer_resched(data, false);
/*
* Send govinfo notification.
* Govinfo notification could potentially wake up another thread
* managed by its clients. Thread wakeups might trigger a load
* change callback that executes this function again. Therefore
* no spinlock could be held when sending the notification.
*/
for_each_cpu(i, ppol->policy->cpus) {
pcpu = &per_cpu(cpuinfo, i);
govinfo.cpu = i;
govinfo.load = pcpu->loadadjfreq / ppol->policy->max;
govinfo.sampling_rate_us = tunables->timer_rate;
atomic_notifier_call_chain(&cpufreq_govinfo_notifier_list,
CPUFREQ_LOAD_CHANGE, &govinfo);
}
exit:
up_read(&ppol->enable_sem);
return;
}
|→
void sched_get_cpus_busy(struct sched_load *busy,
const struct cpumask *query_cpus)
{
unsigned long flags;
struct rq *rq;
const int cpus = cpumask_weight(query_cpus);
u64 load[cpus], group_load[cpus];
u64 nload[cpus], ngload[cpus];
u64 pload[cpus];
unsigned int max_freq[cpus];
int notifier_sent = 0;
int early_detection[cpus];
int cpu, i = 0;
unsigned int window_size;
u64 max_prev_sum = 0;
int max_busy_cpu = cpumask_first(query_cpus);
u64 total_group_load = 0, total_ngload = 0;
bool aggregate_load = false;
struct sched_cluster *cluster = cpu_cluster(cpumask_first(query_cpus));
if (unlikely(cpus == 0))
return;
local_irq_save(flags);
/*
* This function could be called in timer context, and the
* current task may have been executing for a long time. Ensure
* that the window stats are current by doing an update.
*/
for_each_cpu(cpu, query_cpus)
raw_spin_lock_nested(&cpu_rq(cpu)->lock, cpu);
window_size = sched_ravg_window;
/*
* We don't really need the cluster lock for this entire for loop
* block. However, there is no advantage in optimizing this as rq
* locks are held regardless and would prevent migration anyways
*/
raw_spin_lock(&cluster->load_lock);
for_each_cpu(cpu, query_cpus) {
rq = cpu_rq(cpu);
update_task_ravg(rq->curr, rq, TASK_UPDATE, sched_ktime_clock(),
0);
account_load_subtractions(rq);
/* (1) 获取:
cpu上一个窗口的负载：rq->prev_runnable_sum
cpu上一个窗口的的新任务负载：rq->nt_prev_runnable_sum
cpu上一个窗口的负载预测：rq->hmp_stats.pred_demands_sum
*/
load[i] = rq->prev_runnable_sum;
nload[i] = rq->nt_prev_runnable_sum;
pload[i] = rq->hmp_stats.pred_demands_sum;
rq->old_estimated_time = pload[i];
if (load[i] > max_prev_sum) {
max_prev_sum = load[i];
max_busy_cpu = cpu;
}
/*
* sched_get_cpus_busy() is called for all CPUs in a
* frequency domain. So the notifier_sent flag per
* cluster works even when a frequency domain spans
* more than 1 cluster.
*/
if (rq->cluster->notifier_sent) {
notifier_sent = 1;
rq->cluster->notifier_sent = 0;
}
early_detection[i] = (rq->ed_task != NULL);
max_freq[i] = cpu_max_freq(cpu);
i++;
}
raw_spin_unlock(&cluster->load_lock);
group_load_in_freq_domain(
&cpu_rq(max_busy_cpu)->freq_domain_cpumask,
&total_group_load, &total_ngload);
aggregate_load = !!(total_group_load > sched_freq_aggregate_threshold);
i = 0;
for_each_cpu(cpu, query_cpus) {
group_load[i] = 0;
ngload[i] = 0;
if (early_detection[i])
goto skip_early;
rq = cpu_rq(cpu);
if (aggregate_load) {
if (cpu == max_busy_cpu) {
group_load[i] = total_group_load;
ngload[i] = total_ngload;
}
} else {
group_load[i] = rq->grp_time.prev_runnable_sum;
ngload[i] = rq->grp_time.nt_prev_runnable_sum;
}
load[i] += group_load[i];
nload[i] += ngload[i];
load[i] = freq_policy_load(rq, load[i]);
rq->old_busy_time = load[i];
/*
* Scale load in reference to cluster max_possible_freq.
*
* Note that scale_load_to_cpu() scales load in reference to
* the cluster max_freq.
*/
load[i] = scale_load_to_cpu(load[i], cpu);
nload[i] = scale_load_to_cpu(nload[i], cpu);
pload[i] = scale_load_to_cpu(pload[i], cpu);
skip_early:
i++;
}
for_each_cpu(cpu, query_cpus)
raw_spin_unlock(&(cpu_rq(cpu))->lock);
local_irq_restore(flags);
i = 0;
for_each_cpu(cpu, query_cpus) {
rq = cpu_rq(cpu);
if (early_detection[i]) {
busy[i].prev_load = div64_u64(sched_ravg_window,
NSEC_PER_USEC);
busy[i].new_task_load = 0;
busy[i].predicted_load = 0;
goto exit_early;
}
load[i] = scale_load_to_freq(load[i], max_freq[i],
cpu_max_possible_freq(cpu));
nload[i] = scale_load_to_freq(nload[i], max_freq[i],
cpu_max_possible_freq(cpu));
pload[i] = scale_load_to_freq(pload[i], max_freq[i],
rq->cluster->max_possible_freq);
/* (2) 负载经过转换后赋值给busy:
cpu上一个窗口的负载：busy[i].prev_load
cpu上一个窗口的的新任务负载：busy[i].new_task_load
cpu上一个窗口的负载预测：busy[i].predicted_load
*/
busy[i].prev_load = div64_u64(load[i], NSEC_PER_USEC);
busy[i].new_task_load = div64_u64(nload[i], NSEC_PER_USEC);
busy[i].predicted_load = div64_u64(pload[i], NSEC_PER_USEC);
exit_early:
trace_sched_get_busy(cpu, busy[i].prev_load,
busy[i].new_task_load,
busy[i].predicted_load,
early_detection[i],
aggregate_load &&
cpu == max_busy_cpu);
i++;
}
}