diff options
Diffstat (limited to 'arch/x86/kernel')
| -rw-r--r-- | arch/x86/kernel/cpu/cacheinfo.c | 59 | ||||
| -rw-r--r-- | arch/x86/kernel/cpu/common.c | 3 | ||||
| -rw-r--r-- | arch/x86/kernel/cpu/mtrr/mtrr.c | 88 | ||||
| -rw-r--r-- | arch/x86/kernel/setup.c | 3 | ||||
| -rw-r--r-- | arch/x86/kernel/smpboot.c | 4 |
5 files changed, 66 insertions, 91 deletions
diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index 063d556d5759..4e155bd2574f 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -15,6 +15,7 @@ #include <linux/capability.h> #include <linux/sysfs.h> #include <linux/pci.h> +#include <linux/stop_machine.h> #include <asm/cpufeature.h> #include <asm/cacheinfo.h> @@ -1121,7 +1122,7 @@ void cache_enable(void) __releases(cache_disable_lock) raw_spin_unlock(&cache_disable_lock); } -void cache_cpu_init(void) +static void cache_cpu_init(void) { unsigned long flags; @@ -1149,3 +1150,59 @@ bool get_cache_aps_delayed_init(void) { return cache_aps_delayed_init; } + +static int cache_rendezvous_handler(void *unused) +{ + if (get_cache_aps_delayed_init() || !cpu_online(smp_processor_id())) + cache_cpu_init(); + + return 0; +} + +void __init cache_bp_init(void) +{ + mtrr_bp_init(); + + if (memory_caching_control) + cache_cpu_init(); +} + +void cache_bp_restore(void) +{ + if (memory_caching_control) + cache_cpu_init(); +} + +void cache_ap_init(void) +{ + if (!memory_caching_control || get_cache_aps_delayed_init()) + return; + + /* + * Ideally we should hold mtrr_mutex here to avoid MTRR entries + * changed, but this routine will be called in CPU boot time, + * holding the lock breaks it. + * + * This routine is called in two cases: + * + * 1. very early time of software resume, when there absolutely + * isn't MTRR entry changes; + * + * 2. CPU hotadd time. We let mtrr_add/del_page hold cpuhotplug + * lock to prevent MTRR entry changes + */ + stop_machine_from_inactive_cpu(cache_rendezvous_handler, NULL, + cpu_callout_mask); +} + +/* + * Delayed cache initialization for all AP's + */ +void cache_aps_init(void) +{ + if (!memory_caching_control || !get_cache_aps_delayed_init()) + return; + + stop_machine(cache_rendezvous_handler, NULL, cpu_online_mask); + set_cache_aps_delayed_init(false); +} diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 3e508f239098..fd058b547f8d 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -52,6 +52,7 @@ #include <asm/cpu.h> #include <asm/mce.h> #include <asm/msr.h> +#include <asm/cacheinfo.h> #include <asm/memtype.h> #include <asm/microcode.h> #include <asm/microcode_intel.h> @@ -1948,7 +1949,7 @@ void identify_secondary_cpu(struct cpuinfo_x86 *c) #ifdef CONFIG_X86_32 enable_sep_cpu(); #endif - mtrr_ap_init(); + cache_ap_init(); validate_apic_and_package_id(c); x86_spec_ctrl_setup_ap(); update_srbds_msr(); diff --git a/arch/x86/kernel/cpu/mtrr/mtrr.c b/arch/x86/kernel/cpu/mtrr/mtrr.c index 15ee6d72fb1f..99b6973a69b4 100644 --- a/arch/x86/kernel/cpu/mtrr/mtrr.c +++ b/arch/x86/kernel/cpu/mtrr/mtrr.c @@ -73,9 +73,6 @@ static const struct mtrr_ops *mtrr_ops[X86_VENDOR_NUM] __ro_after_init; const struct mtrr_ops *mtrr_if; -static void set_mtrr(unsigned int reg, unsigned long base, - unsigned long size, mtrr_type type); - void __init set_mtrr_ops(const struct mtrr_ops *ops) { if (ops->vendor && ops->vendor < X86_VENDOR_NUM) @@ -158,26 +155,8 @@ static int mtrr_rendezvous_handler(void *info) { struct set_mtrr_data *data = info; - /* - * We use this same function to initialize the mtrrs during boot, - * resume, runtime cpu online and on an explicit request to set a - * specific MTRR. - * - * During boot or suspend, the state of the boot cpu's mtrrs has been - * saved, and we want to replicate that across all the cpus that come - * online (either at the end of boot or resume or during a runtime cpu - * online). If we're doing that, @reg is set to something special and on - * all the CPUs we do cache_cpu_init() (On the logical CPU that - * started the boot/resume sequence, this might be a duplicate - * cache_cpu_init()). - */ - if (data->smp_reg != ~0U) { - mtrr_if->set(data->smp_reg, data->smp_base, - data->smp_size, data->smp_type); - } else if (get_cache_aps_delayed_init() || - !cpu_online(smp_processor_id())) { - cache_cpu_init(); - } + mtrr_if->set(data->smp_reg, data->smp_base, + data->smp_size, data->smp_type); return 0; } @@ -247,19 +226,6 @@ static void set_mtrr_cpuslocked(unsigned int reg, unsigned long base, stop_machine_cpuslocked(mtrr_rendezvous_handler, &data, cpu_online_mask); } -static void set_mtrr_from_inactive_cpu(unsigned int reg, unsigned long base, - unsigned long size, mtrr_type type) -{ - struct set_mtrr_data data = { .smp_reg = reg, - .smp_base = base, - .smp_size = size, - .smp_type = type - }; - - stop_machine_from_inactive_cpu(mtrr_rendezvous_handler, &data, - cpu_callout_mask); -} - /** * mtrr_add_page - Add a memory type region * @base: Physical base address of region in pages (in units of 4 kB!) @@ -761,7 +727,6 @@ void __init mtrr_bp_init(void) if (get_mtrr_state()) { memory_caching_control |= CACHE_MTRR | CACHE_PAT; changed_by_mtrr_cleanup = mtrr_cleanup(phys_addr); - cache_cpu_init(); } else { mtrr_if = NULL; } @@ -780,27 +745,6 @@ void __init mtrr_bp_init(void) } } -void mtrr_ap_init(void) -{ - if (!memory_caching_control || get_cache_aps_delayed_init()) - return; - - /* - * Ideally we should hold mtrr_mutex here to avoid mtrr entries - * changed, but this routine will be called in cpu boot time, - * holding the lock breaks it. - * - * This routine is called in two cases: - * - * 1. very early time of software resume, when there absolutely - * isn't mtrr entry changes; - * - * 2. cpu hotadd time. We let mtrr_add/del_page hold cpuhotplug - * lock to prevent mtrr entry changes - */ - set_mtrr_from_inactive_cpu(~0U, 0, 0, 0); -} - /** * mtrr_save_state - Save current fixed-range MTRR state of the first * cpu in cpu_online_mask. @@ -816,34 +760,6 @@ void mtrr_save_state(void) smp_call_function_single(first_cpu, mtrr_save_fixed_ranges, NULL, 1); } -/* - * Delayed MTRR initialization for all AP's - */ -void mtrr_aps_init(void) -{ - if (!memory_caching_control) - return; - - /* - * Check if someone has requested the delay of AP MTRR initialization, - * by doing set_mtrr_aps_delayed_init(), prior to this point. If not, - * then we are done. - */ - if (!get_cache_aps_delayed_init()) - return; - - set_mtrr(~0U, 0, 0, 0); - set_cache_aps_delayed_init(false); -} - -void mtrr_bp_restore(void) -{ - if (!memory_caching_control) - return; - - cache_cpu_init(); -} - static int __init mtrr_init_finialize(void) { if (!mtrr_enabled()) diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index 216fee7144ee..e0e185ee0229 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -34,6 +34,7 @@ #include <asm/numa.h> #include <asm/bios_ebda.h> #include <asm/bugs.h> +#include <asm/cacheinfo.h> #include <asm/cpu.h> #include <asm/efi.h> #include <asm/gart.h> @@ -1075,7 +1076,7 @@ void __init setup_arch(char **cmdline_p) /* update e820 for memory not covered by WB MTRRs */ if (IS_ENABLED(CONFIG_MTRR)) - mtrr_bp_init(); + cache_bp_init(); else pat_disable("PAT support disabled because CONFIG_MTRR is disabled in the kernel."); diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index 13c71ab29d84..1b61a480c966 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -1445,7 +1445,7 @@ void arch_thaw_secondary_cpus_begin(void) void arch_thaw_secondary_cpus_end(void) { - mtrr_aps_init(); + cache_aps_init(); } /* @@ -1488,7 +1488,7 @@ void __init native_smp_cpus_done(unsigned int max_cpus) nmi_selftest(); impress_friends(); - mtrr_aps_init(); + cache_aps_init(); } static int __initdata setup_possible_cpus = -1; |