kpriv/linux-hardened
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

arm64: capabilities: Batch cpu_enable callbacks

Browse source 
We use a stop_machine call for each available capability to
enable it on all the CPUs available at boot time. Instead
we could batch the cpu_enable callbacks to a single stop_machine()
call to save us some time.

Reviewed-by: Vladimir Murzin <vladimir.murzin@arm.com>
Tested-by: Vladimir Murzin <vladimir.murzin@arm.com>
Signed-off-by: Suzuki K Poulose <suzuki.poulose@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
This commit is contained in:
Suzuki K Poulose 2018-11-30 17:18:06 +00:00 committed by Will Deacon
parent 606f8e7b27
commit 0b587c84e4
2 changed files with 48 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
arch/arm64/include/asm/cpufeature.h
View file

@ -357,6 +357,9 @@ extern DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
extern struct static_key_false cpu_hwcap_keys[ARM64_NCAPS];
extern struct static_key_false arm64_const_caps_ready;
#define for_each_available_cap(cap) \
for_each_set_bit(cap, cpu_hwcaps, ARM64_NCAPS)
bool this_cpu_has_cap(unsigned int cap);
static inline bool cpu_have_feature(unsigned int num)

72
arch/arm64/kernel/cpufeature.c
View file

@ -1531,11 +1531,27 @@ static void update_cpu_capabilities(u16 scope_mask)
}
}
static int __enable_cpu_capability(void *arg)
/*
* Enable all the available capabilities on this CPU. The capabilities
* with BOOT_CPU scope are handled separately and hence skipped here.
*/
static int cpu_enable_non_boot_scope_capabilities(void *__unused)
{
const struct arm64_cpu_capabilities *cap = arg;
int i;
u16 non_boot_scope = SCOPE_ALL & ~SCOPE_BOOT_CPU;
cap->cpu_enable(cap);
for_each_available_cap(i) {
const struct arm64_cpu_capabilities *cap = cpu_hwcaps_ptrs[i];
if (WARN_ON(!cap))
continue;
if (!(cap->type & non_boot_scope))
continue;
if (cap->cpu_enable)
cap->cpu_enable(cap);
}
return 0;
}
@ -1543,21 +1559,29 @@ static int __enable_cpu_capability(void *arg)
* Run through the enabled capabilities and enable() it on all active
* CPUs
*/
static void __init
__enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps,
u16 scope_mask)
static void __init enable_cpu_capabilities(u16 scope_mask)
{
scope_mask &= ARM64_CPUCAP_SCOPE_MASK;
for (; caps->matches; caps++) {
unsigned int num = caps->capability;
int i;
const struct arm64_cpu_capabilities *caps;
bool boot_scope;
if (!(caps->type & scope_mask) || !cpus_have_cap(num))
scope_mask &= ARM64_CPUCAP_SCOPE_MASK;
boot_scope = !!(scope_mask & SCOPE_BOOT_CPU);
for (i = 0; i < ARM64_NCAPS; i++) {
unsigned int num;
caps = cpu_hwcaps_ptrs[i];
if (!caps || !(caps->type & scope_mask))
continue;
num = caps->capability;
if (!cpus_have_cap(num))
continue;
/* Ensure cpus_have_const_cap(num) works */
static_branch_enable(&cpu_hwcap_keys[num]);
if (caps->cpu_enable) {
if (boot_scope && caps->cpu_enable)
/*
* Capabilities with SCOPE_BOOT_CPU scope are finalised
* before any secondary CPU boots. Thus, each secondary
@ -1566,25 +1590,19 @@ __enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps,
* the boot CPU, for which the capability must be
* enabled here. This approach avoids costly
* stop_machine() calls for this case.
*
* Otherwise, use stop_machine() as it schedules the
* work allowing us to modify PSTATE, instead of
* on_each_cpu() which uses an IPI, giving us a PSTATE
* that disappears when we return.
*/
if (scope_mask & SCOPE_BOOT_CPU)
caps->cpu_enable(caps);
else
stop_machine(__enable_cpu_capability,
(void *)caps, cpu_online_mask);
}
caps->cpu_enable(caps);
}
}
static void __init enable_cpu_capabilities(u16 scope_mask)
{
__enable_cpu_capabilities(arm64_errata, scope_mask);
__enable_cpu_capabilities(arm64_features, scope_mask);
/*
* For all non-boot scope capabilities, use stop_machine()
* as it schedules the work allowing us to modify PSTATE,
* instead of on_each_cpu() which uses an IPI, giving us a
* PSTATE that disappears when we return.
*/
if (!boot_scope)
stop_machine(cpu_enable_non_boot_scope_capabilities,
NULL, cpu_online_mask);
}
/*