From 61de9b7036f26448a1916291c456f62dd6bf07ea Mon Sep 17 00:00:00 2001 From: Borislav Petkov Date: Mon, 19 Dec 2022 22:06:55 +0100 Subject: [PATCH 01/94] x86/microcode/AMD: Rename a couple of functions - Rename apply_microcode_early_amd() to early_apply_microcode(): simplify the name so that it is clear what it does and when does it do it. - Rename __load_ucode_amd() to find_blobs_in_containers(): the new name actually explains what it does. Document some. No functional changes. Signed-off-by: Borislav Petkov Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/20221219210656.5140-1-bp@alien8.de --- arch/x86/kernel/cpu/microcode/amd.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c index 56471f750762..339c9666c8bc 100644 --- a/arch/x86/kernel/cpu/microcode/amd.c +++ b/arch/x86/kernel/cpu/microcode/amd.c @@ -414,8 +414,7 @@ static int __apply_microcode_amd(struct microcode_amd *mc) * * Returns true if container found (sets @desc), false otherwise. */ -static bool -apply_microcode_early_amd(u32 cpuid_1_eax, void *ucode, size_t size, bool save_patch) +static bool early_apply_microcode(u32 cpuid_1_eax, void *ucode, size_t size, bool save_patch) { struct cont_desc desc = { 0 }; u8 (*patch)[PATCH_MAX_SIZE]; @@ -481,7 +480,7 @@ static bool get_builtin_microcode(struct cpio_data *cp, unsigned int family) return false; } -static void __load_ucode_amd(unsigned int cpuid_1_eax, struct cpio_data *ret) +static void find_blobs_in_containers(unsigned int cpuid_1_eax, struct cpio_data *ret) { struct ucode_cpu_info *uci; struct cpio_data cp; @@ -511,11 +510,11 @@ void __init load_ucode_amd_bsp(unsigned int cpuid_1_eax) { struct cpio_data cp = { }; - __load_ucode_amd(cpuid_1_eax, &cp); + find_blobs_in_containers(cpuid_1_eax, &cp); if (!(cp.data && cp.size)) return; - apply_microcode_early_amd(cpuid_1_eax, cp.data, cp.size, true); + early_apply_microcode(cpuid_1_eax, cp.data, cp.size, true); } void load_ucode_amd_ap(unsigned int cpuid_1_eax) @@ -546,11 +545,11 @@ void load_ucode_amd_ap(unsigned int cpuid_1_eax) } } - __load_ucode_amd(cpuid_1_eax, &cp); + find_blobs_in_containers(cpuid_1_eax, &cp); if (!(cp.data && cp.size)) return; - apply_microcode_early_amd(cpuid_1_eax, cp.data, cp.size, false); + early_apply_microcode(cpuid_1_eax, cp.data, cp.size, false); } static enum ucode_state @@ -816,6 +815,7 @@ static int verify_and_add_patch(u8 family, u8 *fw, unsigned int leftover, return 0; } +/* Scan the blob in @data and add microcode patches to the cache. */ static enum ucode_state __load_microcode_amd(u8 family, const u8 *data, size_t size) { From ba73e369b706a853cdafa60570854fecec9f9fdd Mon Sep 17 00:00:00 2001 From: Borislav Petkov Date: Mon, 19 Dec 2022 22:06:56 +0100 Subject: [PATCH 02/94] x86/microcode/AMD: Handle multiple glued containers properly It can happen that - especially during testing - the microcode blobs of all families are all glued together in the initrd. The current code doesn't check whether the current container matched a microcode patch and continues to the next one, which leads to save_microcode_in_initrd_amd() to look at the next and thus wrong one: microcode: parse_container: ucode: 0xffff88807e9d9082 microcode: verify_patch: buf: 0xffff88807e9d90ce, buf_size: 26428 microcode: verify_patch: proc_id: 0x8082, patch_fam: 0x17, this family: 0x17 microcode: verify_patch: buf: 0xffff88807e9d9d56, buf_size: 23220 microcode: verify_patch: proc_id: 0x8012, patch_fam: 0x17, this family: 0x17 microcode: parse_container: MATCH: eq_id: 0x8012, patch proc_rev_id: 0x8012 <-- matching patch found microcode: verify_patch: buf: 0xffff88807e9da9de, buf_size: 20012 microcode: verify_patch: proc_id: 0x8310, patch_fam: 0x17, this family: 0x17 microcode: verify_patch: buf: 0xffff88807e9db666, buf_size: 16804 microcode: Invalid type field (0x414d44) in container file section header. microcode: Patch section fail <-- checking chokes on the microcode magic value of the next container. microcode: parse_container: saving container 0xffff88807e9d9082 microcode: save_microcode_in_initrd_amd: scanned containers, data: 0xffff88807e9d9082, size: 9700a and now if there's a next (and last container) it'll use that in save_microcode_in_initrd_amd() and not find a proper patch, ofc. Fix that by moving the out: label up, before the desc->mc check which jots down the pointer of the matching patch and is used to signal to the caller that it has found a matching patch in the current container. Signed-off-by: Borislav Petkov Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/20221219210656.5140-2-bp@alien8.de --- arch/x86/kernel/cpu/microcode/amd.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c index 339c9666c8bc..d144f918a896 100644 --- a/arch/x86/kernel/cpu/microcode/amd.c +++ b/arch/x86/kernel/cpu/microcode/amd.c @@ -330,8 +330,9 @@ static size_t parse_container(u8 *ucode, size_t size, struct cont_desc *desc) ret = verify_patch(x86_family(desc->cpuid_1_eax), buf, size, &patch_size, true); if (ret < 0) { /* - * Patch verification failed, skip to the next - * container, if there's one: + * Patch verification failed, skip to the next container, if + * there is one. Before exit, check whether that container has + * found a patch already. If so, use it. */ goto out; } else if (ret > 0) { @@ -350,6 +351,7 @@ skip: size -= patch_size + SECTION_HDR_SIZE; } +out: /* * If we have found a patch (desc->mc), it means we're looking at the * container which has a patch for this CPU so return 0 to mean, @ucode @@ -364,7 +366,6 @@ skip: return 0; } -out: return orig_size - size; } From 2117654e806c08c343e3d0567bbedf33eab040c8 Mon Sep 17 00:00:00 2001 From: Smita Koralahalli Date: Tue, 6 Dec 2022 11:36:06 -0600 Subject: [PATCH 03/94] x86/mce: Define a function to extract ErrorAddr from MCA_ADDR Move MCA_ADDR[ErrorAddr] extraction into a separate helper function. This will be further refactored to support extended ErrorAddr bits in MCA_ADDR in newer AMD CPUs. [ bp: Massage. ] Signed-off-by: Smita Koralahalli Signed-off-by: Borislav Petkov Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Yazen Ghannam Link: https://lore.kernel.org/all/20220225193342.215780-3-Smita.KoralahalliChannabasappa@amd.com/ --- arch/x86/kernel/cpu/mce/amd.c | 10 +--------- arch/x86/kernel/cpu/mce/core.c | 10 +--------- arch/x86/kernel/cpu/mce/internal.h | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/arch/x86/kernel/cpu/mce/amd.c b/arch/x86/kernel/cpu/mce/amd.c index 10fb5b5c9efa..c6b1fd5a34b8 100644 --- a/arch/x86/kernel/cpu/mce/amd.c +++ b/arch/x86/kernel/cpu/mce/amd.c @@ -736,15 +736,7 @@ static void __log_error(unsigned int bank, u64 status, u64 addr, u64 misc) if (m.status & MCI_STATUS_ADDRV) { m.addr = addr; - /* - * Extract [55:] where lsb is the least significant - * *valid* bit of the address bits. - */ - if (mce_flags.smca) { - u8 lsb = (m.addr >> 56) & 0x3f; - - m.addr &= GENMASK_ULL(55, lsb); - } + smca_extract_err_addr(&m); } if (mce_flags.smca) { diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index 2c8ec5c71712..d0ade77c41f7 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -633,15 +633,7 @@ static noinstr void mce_read_aux(struct mce *m, int i) m->addr <<= shift; } - /* - * Extract [55:] where lsb is the least significant - * *valid* bit of the address bits. - */ - if (mce_flags.smca) { - u8 lsb = (m->addr >> 56) & 0x3f; - - m->addr &= GENMASK_ULL(55, lsb); - } + smca_extract_err_addr(m); } if (mce_flags.smca) { diff --git a/arch/x86/kernel/cpu/mce/internal.h b/arch/x86/kernel/cpu/mce/internal.h index 7e03f5b7f6bd..6dcb94fe0f65 100644 --- a/arch/x86/kernel/cpu/mce/internal.h +++ b/arch/x86/kernel/cpu/mce/internal.h @@ -189,8 +189,23 @@ extern bool filter_mce(struct mce *m); #ifdef CONFIG_X86_MCE_AMD extern bool amd_filter_mce(struct mce *m); + +/* Extract [55:] where lsb is the LS-*valid* bit of the address bits. */ +static __always_inline void smca_extract_err_addr(struct mce *m) +{ + u8 lsb; + + if (!mce_flags.smca) + return; + + lsb = (m->addr >> 56) & 0x3f; + + m->addr &= GENMASK_ULL(55, lsb); +} + #else static inline bool amd_filter_mce(struct mce *m) { return false; } +static inline void smca_extract_err_addr(struct mce *m) { } #endif #ifdef CONFIG_X86_ANCIENT_MCE From fcd343a285cb41894a7bd02dbd675042d394758d Mon Sep 17 00:00:00 2001 From: Smita Koralahalli Date: Tue, 6 Dec 2022 11:36:07 -0600 Subject: [PATCH 04/94] x86/mce: Add support for Extended Physical Address MCA changes Newer AMD CPUs support more physical address bits. That is, the MCA_ADDR registers on Scalable MCA systems contain the ErrorAddr in bits [56:0] instead of [55:0]. Hence, the existing LSB field from bits [61:56] in MCA_ADDR must be moved around to accommodate the larger ErrorAddr size. MCA_CONFIG[McaLsbInStatusSupported] indicates this change. If set, the LSB field will be found in MCA_STATUS rather than MCA_ADDR. Each logical CPU has unique MCA bank in hardware and is not shared with other logical CPUs. Additionally, on SMCA systems, each feature bit may be different for each bank within same logical CPU. Check for MCA_CONFIG[McaLsbInStatusSupported] for each MCA bank and for each CPU. Additionally, all MCA banks do not support maximum ErrorAddr bits in MCA_ADDR. Some banks might support fewer bits but the remaining bits are marked as reserved. [ Yazen: Rebased and fixed up formatting. bp: Massage comments. ] Signed-off-by: Smita Koralahalli Signed-off-by: Yazen Ghannam Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/20221206173607.1185907-5-yazen.ghannam@amd.com --- arch/x86/kernel/cpu/mce/amd.c | 2 ++ arch/x86/kernel/cpu/mce/core.c | 8 +------- arch/x86/kernel/cpu/mce/internal.h | 31 +++++++++++++++++++++++++++++- 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/arch/x86/kernel/cpu/mce/amd.c b/arch/x86/kernel/cpu/mce/amd.c index c6b1fd5a34b8..23c5072fbbb7 100644 --- a/arch/x86/kernel/cpu/mce/amd.c +++ b/arch/x86/kernel/cpu/mce/amd.c @@ -306,6 +306,8 @@ static void smca_configure(unsigned int bank, unsigned int cpu) if ((low & BIT(5)) && !((high >> 5) & 0x3)) high |= BIT(5); + this_cpu_ptr(mce_banks_array)[bank].lsb_in_status = !!(low & BIT(8)); + wrmsr(smca_config, low, high); } diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index d0ade77c41f7..626a1c666252 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -67,13 +67,7 @@ DEFINE_PER_CPU(unsigned, mce_exception_count); DEFINE_PER_CPU_READ_MOSTLY(unsigned int, mce_num_banks); -struct mce_bank { - u64 ctl; /* subevents to enable */ - - __u64 init : 1, /* initialise bank? */ - __reserved_1 : 63; -}; -static DEFINE_PER_CPU_READ_MOSTLY(struct mce_bank[MAX_NR_BANKS], mce_banks_array); +DEFINE_PER_CPU_READ_MOSTLY(struct mce_bank[MAX_NR_BANKS], mce_banks_array); #define ATTR_LEN 16 /* One object for each MCE bank, shared by all CPUs */ diff --git a/arch/x86/kernel/cpu/mce/internal.h b/arch/x86/kernel/cpu/mce/internal.h index 6dcb94fe0f65..91a415553c27 100644 --- a/arch/x86/kernel/cpu/mce/internal.h +++ b/arch/x86/kernel/cpu/mce/internal.h @@ -177,6 +177,24 @@ struct mce_vendor_flags { extern struct mce_vendor_flags mce_flags; +struct mce_bank { + /* subevents to enable */ + u64 ctl; + + /* initialise bank? */ + __u64 init : 1, + + /* + * (AMD) MCA_CONFIG[McaLsbInStatusSupported]: When set, this bit indicates + * the LSB field is found in MCA_STATUS and not in MCA_ADDR. + */ + lsb_in_status : 1, + + __reserved_1 : 62; +}; + +DECLARE_PER_CPU_READ_MOSTLY(struct mce_bank[MAX_NR_BANKS], mce_banks_array); + enum mca_msr { MCA_CTL, MCA_STATUS, @@ -190,7 +208,10 @@ extern bool filter_mce(struct mce *m); #ifdef CONFIG_X86_MCE_AMD extern bool amd_filter_mce(struct mce *m); -/* Extract [55:] where lsb is the LS-*valid* bit of the address bits. */ +/* + * If MCA_CONFIG[McaLsbInStatusSupported] is set, extract ErrAddr in bits + * [56:0] of MCA_STATUS, else in bits [55:0] of MCA_ADDR. + */ static __always_inline void smca_extract_err_addr(struct mce *m) { u8 lsb; @@ -198,6 +219,14 @@ static __always_inline void smca_extract_err_addr(struct mce *m) if (!mce_flags.smca) return; + if (this_cpu_ptr(mce_banks_array)[m->bank].lsb_in_status) { + lsb = (m->status >> 24) & 0x3f; + + m->addr &= GENMASK_ULL(56, lsb); + + return; + } + lsb = (m->addr >> 56) & 0x3f; m->addr &= GENMASK_ULL(55, lsb); From beaa1ffe551c330d8ea23de158432ecaad6c0410 Mon Sep 17 00:00:00 2001 From: Yunying Sun Date: Wed, 16 Nov 2022 16:22:21 +0800 Subject: [PATCH 05/94] clocksource: Print clocksource name when clocksource is tested unstable Some "TSC fall back to HPET" messages appear on systems having more than 2 NUMA nodes: clocksource: timekeeping watchdog on CPU168: hpet read-back delay of 4296200ns, attempt 4, marking unstable The "hpet" here is misleading the clocksource watchdog is really doing repeated reads of "hpet" in order to check for unrelated delays. Therefore, print the name of the clocksource under test, prefixed by "wd-" and suffixed by "-wd", for example, "wd-tsc-wd". Signed-off-by: Yunying Sun Signed-off-by: Paul E. McKenney --- kernel/time/clocksource.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index 9cf32ccda715..4a2c3bb92e2e 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -257,8 +257,8 @@ static enum wd_read_status cs_watchdog_read(struct clocksource *cs, u64 *csnow, goto skip_test; } - pr_warn("timekeeping watchdog on CPU%d: %s read-back delay of %lldns, attempt %d, marking unstable\n", - smp_processor_id(), watchdog->name, wd_delay, nretries); + pr_warn("timekeeping watchdog on CPU%d: wd-%s-wd read-back delay of %lldns, attempt %d, marking unstable\n", + smp_processor_id(), cs->name, wd_delay, nretries); return WD_READ_UNSTABLE; skip_test: From c37e85c135cead4256dc8860073c468d8925c3df Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 6 Dec 2022 19:36:10 -0800 Subject: [PATCH 06/94] clocksource: Loosen clocksource watchdog constraints Currently, MAX_SKEW_USEC is set to 100 microseconds, which has worked reasonably well. However, NTP is willing to tolerate 500 microseconds of skew per second, and a clocksource that is good enough for NTP should be good enough for the clocksource watchdog. The watchdog's skew is controlled by MAX_SKEW_USEC and the CLOCKSOURCE_WATCHDOG_MAX_SKEW_US Kconfig option. However, these values are doubled before being associated with a clocksource's ->uncertainty_margin, and the ->uncertainty_margin values of the pair of clocksource's being compared are summed before checking against the skew. Therefore, set both MAX_SKEW_USEC and the default for the CLOCKSOURCE_WATCHDOG_MAX_SKEW_US Kconfig option to 125 microseconds of skew per second, resulting in 500 microseconds of skew per second in the clocksource watchdog's skew comparison. Suggested-by Rik van Riel Signed-off-by: Paul E. McKenney --- kernel/time/Kconfig | 6 +++++- kernel/time/clocksource.c | 15 +++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig index a41753be1a2b..bae8f11070be 100644 --- a/kernel/time/Kconfig +++ b/kernel/time/Kconfig @@ -200,10 +200,14 @@ config CLOCKSOURCE_WATCHDOG_MAX_SKEW_US int "Clocksource watchdog maximum allowable skew (in μs)" depends on CLOCKSOURCE_WATCHDOG range 50 1000 - default 100 + default 125 help Specify the maximum amount of allowable watchdog skew in microseconds before reporting the clocksource to be unstable. + The default is based on a half-second clocksource watchdog + interval and NTP's maximum frequency drift of 500 parts + per million. If the clocksource is good enough for NTP, + it is good enough for the clocksource watchdog! endmenu endif diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index 4a2c3bb92e2e..a3d19f6660ac 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -95,6 +95,11 @@ static char override_name[CS_NAME_LEN]; static int finished_booting; static u64 suspend_start; +/* + * Interval: 0.5sec. + */ +#define WATCHDOG_INTERVAL (HZ >> 1) + /* * Threshold: 0.0312s, when doubled: 0.0625s. * Also a default for cs->uncertainty_margin when registering clocks. @@ -106,11 +111,14 @@ static u64 suspend_start; * clocksource surrounding a read of the clocksource being validated. * This delay could be due to SMIs, NMIs, or to VCPU preemptions. Used as * a lower bound for cs->uncertainty_margin values when registering clocks. + * + * The default of 500 parts per million is based on NTP's limits. + * If a clocksource is good enough for NTP, it is good enough for us! */ #ifdef CONFIG_CLOCKSOURCE_WATCHDOG_MAX_SKEW_US #define MAX_SKEW_USEC CONFIG_CLOCKSOURCE_WATCHDOG_MAX_SKEW_US #else -#define MAX_SKEW_USEC 100 +#define MAX_SKEW_USEC (125 * WATCHDOG_INTERVAL / HZ) #endif #define WATCHDOG_MAX_SKEW (MAX_SKEW_USEC * NSEC_PER_USEC) @@ -140,11 +148,6 @@ static inline void clocksource_watchdog_unlock(unsigned long *flags) static int clocksource_watchdog_kthread(void *data); static void __clocksource_change_rating(struct clocksource *cs, int rating); -/* - * Interval: 0.5sec. - */ -#define WATCHDOG_INTERVAL (HZ >> 1) - static void clocksource_watchdog_work(struct work_struct *work) { /* From f092eb34b33043152bfb8a4ca01db9a06728261d Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 13 Dec 2022 13:57:28 -0800 Subject: [PATCH 07/94] clocksource: Improve read-back-delay message When cs_watchdog_read() is unable to get a qualifying clocksource read within the limit set by max_cswd_read_retries, it prints a message and marks the clocksource under test as unstable. But that message is unclear to anyone unfamiliar with the code: clocksource: timekeeping watchdog on CPU13: wd-tsc-wd read-back delay 1000614ns, attempt 3, marking unstable Therefore, add some context so that the message appears as follows: clocksource: timekeeping watchdog on CPU13: wd-tsc-wd excessive read-back delay of 1000614ns vs. limit of 125000ns, wd-wd read-back delay only 27ns, attempt 3, marking tsc unstable Signed-off-by: Paul E. McKenney Cc: John Stultz Cc: Thomas Gleixner Cc: Stephen Boyd Cc: Feng Tang --- kernel/time/clocksource.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index a3d19f6660ac..b59914953809 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -260,8 +260,8 @@ static enum wd_read_status cs_watchdog_read(struct clocksource *cs, u64 *csnow, goto skip_test; } - pr_warn("timekeeping watchdog on CPU%d: wd-%s-wd read-back delay of %lldns, attempt %d, marking unstable\n", - smp_processor_id(), cs->name, wd_delay, nretries); + pr_warn("timekeeping watchdog on CPU%d: wd-%s-wd excessive read-back delay of %lldns vs. limit of %ldns, wd-wd read-back delay only %lldns, attempt %d, marking %s unstable\n", + smp_processor_id(), cs->name, wd_delay, WATCHDOG_MAX_SKEW, wd_seq_delay, nretries, cs->name); return WD_READ_UNSTABLE; skip_test: From 5d1dd961e74334a2178264193ea813d44ce5e725 Mon Sep 17 00:00:00 2001 From: "Borislav Petkov (AMD)" Date: Thu, 22 Dec 2022 12:42:24 +0100 Subject: [PATCH 08/94] x86/alternatives: Add alt_instr.flags Add a struct alt_instr.flags field which will contain different flags controlling alternatives patching behavior. The initial idea was to be able to specify it as a separate macro parameter but that would mean touching all possible invocations of the alternatives macros and thus a lot of churn. What is more, as PeterZ suggested, being able to say ALT_NOT(feature) is very readable and explains exactly what is meant. So make the feature field a u32 where the patching flags are the upper u16 part of the dword quantity while the lower u16 word is the feature. The highest feature number currently is 0x26a (i.e., word 19) so there is plenty of space. If that becomes insufficient, the field can be extended to u64 which will then make struct alt_instr of the nice size of 16 bytes (14 bytes currently). There should be no functional changes resulting from this. Suggested-by: Peter Zijlstra (Intel) Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Ingo Molnar Link: https://lore.kernel.org/r/Y6RCoJEtxxZWwotd@zn.tnic --- arch/x86/include/asm/alternative.h | 130 ++++++++++-------- arch/x86/kernel/alternative.c | 14 +- tools/objtool/arch/x86/include/arch/special.h | 6 +- 3 files changed, 84 insertions(+), 66 deletions(-) diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h index 7659217f4d49..e2975a32d443 100644 --- a/arch/x86/include/asm/alternative.h +++ b/arch/x86/include/asm/alternative.h @@ -6,8 +6,10 @@ #include #include -#define ALTINSTR_FLAG_INV (1 << 15) -#define ALT_NOT(feat) ((feat) | ALTINSTR_FLAG_INV) +#define ALT_FLAGS_SHIFT 16 + +#define ALT_FLAG_NOT BIT(0) +#define ALT_NOT(feature) ((ALT_FLAG_NOT << ALT_FLAGS_SHIFT) | (feature)) #ifndef __ASSEMBLY__ @@ -59,10 +61,27 @@ ".long 999b - .\n\t" \ ".popsection\n\t" +/* + * The patching flags are part of the upper bits of the @ft_flags parameter when + * specifying them. The split is currently like this: + * + * [31... flags ...16][15... CPUID feature bit ...0] + * + * but since this is all hidden in the macros argument being split, those fields can be + * extended in the future to fit in a u64 or however the need arises. + */ struct alt_instr { s32 instr_offset; /* original instruction */ s32 repl_offset; /* offset to replacement instruction */ - u16 cpuid; /* cpuid bit set for replacement */ + + union { + struct { + u32 cpuid: 16; /* CPUID bit set for replacement */ + u32 flags: 16; /* patching control flags */ + }; + u32 ft_flags; + }; + u8 instrlen; /* length of original instruction */ u8 replacementlen; /* length of new instruction */ } __packed; @@ -182,10 +201,10 @@ static inline int alternatives_text_reserved(void *start, void *end) " - (" alt_slen ")), 0x90\n" \ alt_end_marker ":\n" -#define ALTINSTR_ENTRY(feature, num) \ +#define ALTINSTR_ENTRY(ft_flags, num) \ " .long 661b - .\n" /* label */ \ " .long " b_replacement(num)"f - .\n" /* new instruction */ \ - " .word " __stringify(feature) "\n" /* feature bit */ \ + " .4byte " __stringify(ft_flags) "\n" /* feature + flags */ \ " .byte " alt_total_slen "\n" /* source len */ \ " .byte " alt_rlen(num) "\n" /* replacement len */ @@ -194,20 +213,20 @@ static inline int alternatives_text_reserved(void *start, void *end) b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n" /* alternative assembly primitive: */ -#define ALTERNATIVE(oldinstr, newinstr, feature) \ +#define ALTERNATIVE(oldinstr, newinstr, ft_flags) \ OLDINSTR(oldinstr, 1) \ ".pushsection .altinstructions,\"a\"\n" \ - ALTINSTR_ENTRY(feature, 1) \ + ALTINSTR_ENTRY(ft_flags, 1) \ ".popsection\n" \ ".pushsection .altinstr_replacement, \"ax\"\n" \ ALTINSTR_REPLACEMENT(newinstr, 1) \ ".popsection\n" -#define ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2)\ +#define ALTERNATIVE_2(oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2) \ OLDINSTR_2(oldinstr, 1, 2) \ ".pushsection .altinstructions,\"a\"\n" \ - ALTINSTR_ENTRY(feature1, 1) \ - ALTINSTR_ENTRY(feature2, 2) \ + ALTINSTR_ENTRY(ft_flags1, 1) \ + ALTINSTR_ENTRY(ft_flags2, 2) \ ".popsection\n" \ ".pushsection .altinstr_replacement, \"ax\"\n" \ ALTINSTR_REPLACEMENT(newinstr1, 1) \ @@ -215,21 +234,22 @@ static inline int alternatives_text_reserved(void *start, void *end) ".popsection\n" /* If @feature is set, patch in @newinstr_yes, otherwise @newinstr_no. */ -#define ALTERNATIVE_TERNARY(oldinstr, feature, newinstr_yes, newinstr_no) \ +#define ALTERNATIVE_TERNARY(oldinstr, ft_flags, newinstr_yes, newinstr_no) \ ALTERNATIVE_2(oldinstr, newinstr_no, X86_FEATURE_ALWAYS, \ - newinstr_yes, feature) + newinstr_yes, ft_flags) -#define ALTERNATIVE_3(oldinsn, newinsn1, feat1, newinsn2, feat2, newinsn3, feat3) \ - OLDINSTR_3(oldinsn, 1, 2, 3) \ - ".pushsection .altinstructions,\"a\"\n" \ - ALTINSTR_ENTRY(feat1, 1) \ - ALTINSTR_ENTRY(feat2, 2) \ - ALTINSTR_ENTRY(feat3, 3) \ - ".popsection\n" \ - ".pushsection .altinstr_replacement, \"ax\"\n" \ - ALTINSTR_REPLACEMENT(newinsn1, 1) \ - ALTINSTR_REPLACEMENT(newinsn2, 2) \ - ALTINSTR_REPLACEMENT(newinsn3, 3) \ +#define ALTERNATIVE_3(oldinsn, newinsn1, ft_flags1, newinsn2, ft_flags2, \ + newinsn3, ft_flags3) \ + OLDINSTR_3(oldinsn, 1, 2, 3) \ + ".pushsection .altinstructions,\"a\"\n" \ + ALTINSTR_ENTRY(ft_flags1, 1) \ + ALTINSTR_ENTRY(ft_flags2, 2) \ + ALTINSTR_ENTRY(ft_flags3, 3) \ + ".popsection\n" \ + ".pushsection .altinstr_replacement, \"ax\"\n" \ + ALTINSTR_REPLACEMENT(newinsn1, 1) \ + ALTINSTR_REPLACEMENT(newinsn2, 2) \ + ALTINSTR_REPLACEMENT(newinsn3, 3) \ ".popsection\n" /* @@ -244,14 +264,14 @@ static inline int alternatives_text_reserved(void *start, void *end) * For non barrier like inlines please define new variants * without volatile and memory clobber. */ -#define alternative(oldinstr, newinstr, feature) \ - asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, feature) : : : "memory") +#define alternative(oldinstr, newinstr, ft_flags) \ + asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, ft_flags) : : : "memory") -#define alternative_2(oldinstr, newinstr1, feature1, newinstr2, feature2) \ - asm_inline volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2) ::: "memory") +#define alternative_2(oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2) \ + asm_inline volatile(ALTERNATIVE_2(oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2) ::: "memory") -#define alternative_ternary(oldinstr, feature, newinstr_yes, newinstr_no) \ - asm_inline volatile(ALTERNATIVE_TERNARY(oldinstr, feature, newinstr_yes, newinstr_no) ::: "memory") +#define alternative_ternary(oldinstr, ft_flags, newinstr_yes, newinstr_no) \ + asm_inline volatile(ALTERNATIVE_TERNARY(oldinstr, ft_flags, newinstr_yes, newinstr_no) ::: "memory") /* * Alternative inline assembly with input. @@ -261,8 +281,8 @@ static inline int alternatives_text_reserved(void *start, void *end) * Argument numbers start with 1. * Leaving an unused argument 0 to keep API compatibility. */ -#define alternative_input(oldinstr, newinstr, feature, input...) \ - asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, feature) \ +#define alternative_input(oldinstr, newinstr, ft_flags, input...) \ + asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, ft_flags) \ : : "i" (0), ## input) /* @@ -273,20 +293,20 @@ static inline int alternatives_text_reserved(void *start, void *end) * Otherwise, if CPU has feature1, newinstr1 is used. * Otherwise, oldinstr is used. */ -#define alternative_input_2(oldinstr, newinstr1, feature1, newinstr2, \ - feature2, input...) \ - asm_inline volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1, \ - newinstr2, feature2) \ +#define alternative_input_2(oldinstr, newinstr1, ft_flags1, newinstr2, \ + ft_flags2, input...) \ + asm_inline volatile(ALTERNATIVE_2(oldinstr, newinstr1, ft_flags1, \ + newinstr2, ft_flags2) \ : : "i" (0), ## input) /* Like alternative_input, but with a single output argument */ -#define alternative_io(oldinstr, newinstr, feature, output, input...) \ - asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, feature) \ +#define alternative_io(oldinstr, newinstr, ft_flags, output, input...) \ + asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, ft_flags) \ : output : "i" (0), ## input) /* Like alternative_io, but for replacing a direct call with another one. */ -#define alternative_call(oldfunc, newfunc, feature, output, input...) \ - asm_inline volatile (ALTERNATIVE("call %P[old]", "call %P[new]", feature) \ +#define alternative_call(oldfunc, newfunc, ft_flags, output, input...) \ + asm_inline volatile (ALTERNATIVE("call %P[old]", "call %P[new]", ft_flags) \ : output : [old] "i" (oldfunc), [new] "i" (newfunc), ## input) /* @@ -295,10 +315,10 @@ static inline int alternatives_text_reserved(void *start, void *end) * Otherwise, if CPU has feature1, function1 is used. * Otherwise, old function is used. */ -#define alternative_call_2(oldfunc, newfunc1, feature1, newfunc2, feature2, \ +#define alternative_call_2(oldfunc, newfunc1, ft_flags1, newfunc2, ft_flags2, \ output, input...) \ - asm_inline volatile (ALTERNATIVE_2("call %P[old]", "call %P[new1]", feature1,\ - "call %P[new2]", feature2) \ + asm_inline volatile (ALTERNATIVE_2("call %P[old]", "call %P[new1]", ft_flags1,\ + "call %P[new2]", ft_flags2) \ : output, ASM_CALL_CONSTRAINT \ : [old] "i" (oldfunc), [new1] "i" (newfunc1), \ [new2] "i" (newfunc2), ## input) @@ -347,10 +367,10 @@ static inline int alternatives_text_reserved(void *start, void *end) * enough information for the alternatives patching code to patch an * instruction. See apply_alternatives(). */ -.macro altinstruction_entry orig alt feature orig_len alt_len +.macro altinstr_entry orig alt ft_flags orig_len alt_len .long \orig - . .long \alt - . - .word \feature + .4byte \ft_flags .byte \orig_len .byte \alt_len .endm @@ -361,7 +381,7 @@ static inline int alternatives_text_reserved(void *start, void *end) * @newinstr. ".skip" directive takes care of proper instruction padding * in case @newinstr is longer than @oldinstr. */ -.macro ALTERNATIVE oldinstr, newinstr, feature +.macro ALTERNATIVE oldinstr, newinstr, ft_flags 140: \oldinstr 141: @@ -369,7 +389,7 @@ static inline int alternatives_text_reserved(void *start, void *end) 142: .pushsection .altinstructions,"a" - altinstruction_entry 140b,143f,\feature,142b-140b,144f-143f + altinstr_entry 140b,143f,\ft_flags,142b-140b,144f-143f .popsection .pushsection .altinstr_replacement,"ax" @@ -399,7 +419,7 @@ static inline int alternatives_text_reserved(void *start, void *end) * has @feature1, it replaces @oldinstr with @newinstr1. If CPU has * @feature2, it replaces @oldinstr with @feature2. */ -.macro ALTERNATIVE_2 oldinstr, newinstr1, feature1, newinstr2, feature2 +.macro ALTERNATIVE_2 oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2 140: \oldinstr 141: @@ -408,8 +428,8 @@ static inline int alternatives_text_reserved(void *start, void *end) 142: .pushsection .altinstructions,"a" - altinstruction_entry 140b,143f,\feature1,142b-140b,144f-143f - altinstruction_entry 140b,144f,\feature2,142b-140b,145f-144f + altinstr_entry 140b,143f,\ft_flags1,142b-140b,144f-143f + altinstr_entry 140b,144f,\ft_flags2,142b-140b,145f-144f .popsection .pushsection .altinstr_replacement,"ax" @@ -421,7 +441,7 @@ static inline int alternatives_text_reserved(void *start, void *end) .popsection .endm -.macro ALTERNATIVE_3 oldinstr, newinstr1, feature1, newinstr2, feature2, newinstr3, feature3 +.macro ALTERNATIVE_3 oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2, newinstr3, ft_flags3 140: \oldinstr 141: @@ -430,9 +450,9 @@ static inline int alternatives_text_reserved(void *start, void *end) 142: .pushsection .altinstructions,"a" - altinstruction_entry 140b,143f,\feature1,142b-140b,144f-143f - altinstruction_entry 140b,144f,\feature2,142b-140b,145f-144f - altinstruction_entry 140b,145f,\feature3,142b-140b,146f-145f + altinstr_entry 140b,143f,\ft_flags1,142b-140b,144f-143f + altinstr_entry 140b,144f,\ft_flags2,142b-140b,145f-144f + altinstr_entry 140b,145f,\ft_flags3,142b-140b,146f-145f .popsection .pushsection .altinstr_replacement,"ax" @@ -447,9 +467,9 @@ static inline int alternatives_text_reserved(void *start, void *end) .endm /* If @feature is set, patch in @newinstr_yes, otherwise @newinstr_no. */ -#define ALTERNATIVE_TERNARY(oldinstr, feature, newinstr_yes, newinstr_no) \ +#define ALTERNATIVE_TERNARY(oldinstr, ft_flags, newinstr_yes, newinstr_no) \ ALTERNATIVE_2 oldinstr, newinstr_no, X86_FEATURE_ALWAYS, \ - newinstr_yes, feature + newinstr_yes, ft_flags #endif /* __ASSEMBLY__ */ diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 7d8c3cbde368..9a45aa1c775c 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -282,27 +282,25 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start, */ for (a = start; a < end; a++) { int insn_buff_sz = 0; - /* Mask away "NOT" flag bit for feature to test. */ - u16 feature = a->cpuid & ~ALTINSTR_FLAG_INV; instr = (u8 *)&a->instr_offset + a->instr_offset; replacement = (u8 *)&a->repl_offset + a->repl_offset; BUG_ON(a->instrlen > sizeof(insn_buff)); - BUG_ON(feature >= (NCAPINTS + NBUGINTS) * 32); + BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32); /* * Patch if either: * - feature is present - * - feature not present but ALTINSTR_FLAG_INV is set to mean, + * - feature not present but ALT_FLAG_NOT is set to mean, * patch if feature is *NOT* present. */ - if (!boot_cpu_has(feature) == !(a->cpuid & ALTINSTR_FLAG_INV)) + if (!boot_cpu_has(a->cpuid) == !(a->flags & ALT_FLAG_NOT)) goto next; DPRINTK("feat: %s%d*32+%d, old: (%pS (%px) len: %d), repl: (%px, len: %d)", - (a->cpuid & ALTINSTR_FLAG_INV) ? "!" : "", - feature >> 5, - feature & 0x1f, + (a->flags & ALT_FLAG_NOT) ? "!" : "", + a->cpuid >> 5, + a->cpuid & 0x1f, instr, instr, a->instrlen, replacement, a->replacementlen); diff --git a/tools/objtool/arch/x86/include/arch/special.h b/tools/objtool/arch/x86/include/arch/special.h index f2918f789a0a..ca8131352994 100644 --- a/tools/objtool/arch/x86/include/arch/special.h +++ b/tools/objtool/arch/x86/include/arch/special.h @@ -11,11 +11,11 @@ #define JUMP_NEW_OFFSET 4 #define JUMP_KEY_OFFSET 8 -#define ALT_ENTRY_SIZE 12 +#define ALT_ENTRY_SIZE 14 #define ALT_ORIG_OFFSET 0 #define ALT_NEW_OFFSET 4 #define ALT_FEATURE_OFFSET 8 -#define ALT_ORIG_LEN_OFFSET 10 -#define ALT_NEW_LEN_OFFSET 11 +#define ALT_ORIG_LEN_OFFSET 12 +#define ALT_NEW_LEN_OFFSET 13 #endif /* _X86_ARCH_SPECIAL_H */ From dd029269947a32047b8ce1f8513b0b3b13f0df32 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 13 Dec 2022 16:42:15 -0800 Subject: [PATCH 09/94] clocksource: Improve "skew is too large" messages When clocksource_watchdog() detects excessive clocksource skew compared to the watchdog clocksource, it marks the clocksource under test as unstable and prints several lines worth of message. But that message is unclear to anyone unfamiliar with the code: clocksource: timekeeping watchdog on CPU2: Marking clocksource 'wdtest-ktime' as unstable because the skew is too large: clocksource: 'kvm-clock' wd_nsec: 400744390 wd_now: 612625c2c wd_last: 5fa7f7c66 mask: ffffffffffffffff clocksource: 'wdtest-ktime' cs_nsec: 600744034 cs_now: 173081397a292d4f cs_last: 17308139565a8ced mask: ffffffffffffffff clocksource: 'kvm-clock' (not 'wdtest-ktime') is current clocksource. Therefore, add the following line near the end of that message: Clocksource 'wdtest-ktime' skewed 199999644 ns (199 ms) over watchdog 'kvm-clock' interval of 400744390 ns (400 ms) This new line clearly indicates the amount of skew between the two clocksources, along with the duration of the time interval over which the skew occurred, both in nanoseconds and milliseconds. Signed-off-by: Paul E. McKenney Cc: John Stultz Cc: Thomas Gleixner Cc: Stephen Boyd Cc: Feng Tang --- kernel/time/clocksource.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index b59914953809..fc486cd97263 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -446,12 +446,20 @@ static void clocksource_watchdog(struct timer_list *unused) /* Check the deviation from the watchdog clocksource. */ md = cs->uncertainty_margin + watchdog->uncertainty_margin; if (abs(cs_nsec - wd_nsec) > md) { + u64 cs_wd_msec; + u64 wd_msec; + u32 wd_rem; + pr_warn("timekeeping watchdog on CPU%d: Marking clocksource '%s' as unstable because the skew is too large:\n", smp_processor_id(), cs->name); pr_warn(" '%s' wd_nsec: %lld wd_now: %llx wd_last: %llx mask: %llx\n", watchdog->name, wd_nsec, wdnow, wdlast, watchdog->mask); pr_warn(" '%s' cs_nsec: %lld cs_now: %llx cs_last: %llx mask: %llx\n", cs->name, cs_nsec, csnow, cslast, cs->mask); + cs_wd_msec = div_u64_rem(cs_nsec - wd_nsec, 1000U * 1000U, &wd_rem); + wd_msec = div_u64_rem(wd_nsec, 1000U * 1000U, &wd_rem); + pr_warn(" Clocksource '%s' skewed %lld ns (%lld ms) over watchdog '%s' interval of %lld ns (%lld ms)\n", + cs->name, cs_nsec - wd_nsec, cs_wd_msec, watchdog->name, wd_nsec, wd_msec); if (curr_clocksource == cs) pr_warn(" '%s' is current clocksource.\n", cs->name); else if (curr_clocksource) From c67ea7d22eeb133363e8227342cf14cbf6eaa9a0 Mon Sep 17 00:00:00 2001 From: Shubhrajyoti Datta Date: Wed, 4 Jan 2023 14:15:11 +0530 Subject: [PATCH 10/94] dt-bindings: edac: Add bindings for Xilinx ZynqMP OCM Add bindings for Xilinx ZynqMP OCM controller. Signed-off-by: Shubhrajyoti Datta Co-developed-by: Sai Krishna Potthuri Signed-off-by: Sai Krishna Potthuri Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20230104084512.1855243-2-sai.krishna.potthuri@amd.com --- .../xlnx,zynqmp-ocmc-1.0.yaml | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 Documentation/devicetree/bindings/memory-controllers/xlnx,zynqmp-ocmc-1.0.yaml diff --git a/Documentation/devicetree/bindings/memory-controllers/xlnx,zynqmp-ocmc-1.0.yaml b/Documentation/devicetree/bindings/memory-controllers/xlnx,zynqmp-ocmc-1.0.yaml new file mode 100644 index 000000000000..ca9fc747bf4f --- /dev/null +++ b/Documentation/devicetree/bindings/memory-controllers/xlnx,zynqmp-ocmc-1.0.yaml @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/memory-controllers/xlnx,zynqmp-ocmc-1.0.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Xilinx Zynqmp OCM(On-Chip Memory) Controller + +maintainers: + - Shubhrajyoti Datta + - Sai Krishna Potthuri + +description: | + The OCM supports 64-bit wide ECC functionality to detect multi-bit errors + and recover from a single-bit memory fault.On a write, if all bytes are + being written, the ECC is generated and written into the ECC RAM along with + the write-data that is written into the data RAM. If one or more bytes are + not written, then the read operation results in an correctable error or + uncorrectable error. + +properties: + compatible: + const: xlnx,zynqmp-ocmc-1.0 + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + +additionalProperties: false + +examples: + - | + #include + memory-controller@ff960000 { + compatible = "xlnx,zynqmp-ocmc-1.0"; + reg = <0xff960000 0x1000>; + interrupts = ; + }; From 3bd2706c910fd328e4ab96ae0aabdcd7c4a90fbf Mon Sep 17 00:00:00 2001 From: Sai Krishna Potthuri Date: Wed, 4 Jan 2023 14:15:12 +0530 Subject: [PATCH 11/94] EDAC/zynqmp: Add EDAC support for Xilinx ZynqMP OCM Add EDAC support for Xilinx ZynqMP OCM Controller, so this driver reports CE and UE errors upon interrupt generation. Also add debugfs files for error injection. On Xilinx ZynqMP platform, both OCM Controller driver(zynqmp_edac) and DDR Memory Controller driver(synopsys_edac) co-exist which means both can be loaded at a time. This scenario is tested on Xilinx ZynqMP platform. Fix following issue reported by the robot: "MAINTAINERS references a file that doesn't exist: Documentation/devicetree/bindings/edac/xlnx,zynqmp-ocmc.yaml" [ bp: - Massage commit message - s/EDAC_ZYNQMP_OCM/EDAC_ZYNQMP/ - Touchups ] Reported-by: kernel test robot Co-developed-by: Shubhrajyoti Datta Signed-off-by: Shubhrajyoti Datta Signed-off-by: Sai Krishna Potthuri Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/20230104084512.1855243-3-sai.krishna.potthuri@amd.com --- MAINTAINERS | 7 + drivers/edac/Kconfig | 8 + drivers/edac/Makefile | 1 + drivers/edac/zynqmp_edac.c | 467 +++++++++++++++++++++++++++++++++++++ 4 files changed, 483 insertions(+) create mode 100644 drivers/edac/zynqmp_edac.c diff --git a/MAINTAINERS b/MAINTAINERS index a36df9ed283d..27cc2a00cc75 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -22908,6 +22908,13 @@ F: Documentation/devicetree/bindings/dma/xilinx/xlnx,zynqmp-dpdma.yaml F: drivers/dma/xilinx/xilinx_dpdma.c F: include/dt-bindings/dma/xlnx-zynqmp-dpdma.h +XILINX ZYNQMP OCM EDAC DRIVER +M: Shubhrajyoti Datta +M: Sai Krishna Potthuri +S: Maintained +F: Documentation/devicetree/bindings/memory-controllers/xlnx,zynqmp-ocmc-1.0.yaml +F: drivers/edac/zynqmp_edac.c + XILINX ZYNQMP PSGTR PHY DRIVER M: Anurag Kumar Vulisha M: Laurent Pinchart diff --git a/drivers/edac/Kconfig b/drivers/edac/Kconfig index 4cfdefbd744d..68f576700911 100644 --- a/drivers/edac/Kconfig +++ b/drivers/edac/Kconfig @@ -542,4 +542,12 @@ config EDAC_DMC520 Support for error detection and correction on the SoCs with ARM DMC-520 DRAM controller. +config EDAC_ZYNQMP + tristate "Xilinx ZynqMP OCM Controller" + depends on ARCH_ZYNQMP || COMPILE_TEST + help + This driver supports error detection and correction for the + Xilinx ZynqMP OCM (On Chip Memory) controller. It can also be + built as a module. In that case it will be called zynqmp_edac. + endif # EDAC diff --git a/drivers/edac/Makefile b/drivers/edac/Makefile index 2d1641a27a28..9b025c5b3061 100644 --- a/drivers/edac/Makefile +++ b/drivers/edac/Makefile @@ -84,3 +84,4 @@ obj-$(CONFIG_EDAC_QCOM) += qcom_edac.o obj-$(CONFIG_EDAC_ASPEED) += aspeed_edac.o obj-$(CONFIG_EDAC_BLUEFIELD) += bluefield_edac.o obj-$(CONFIG_EDAC_DMC520) += dmc520_edac.o +obj-$(CONFIG_EDAC_ZYNQMP) += zynqmp_edac.o diff --git a/drivers/edac/zynqmp_edac.c b/drivers/edac/zynqmp_edac.c new file mode 100644 index 000000000000..ac7d1e0b324c --- /dev/null +++ b/drivers/edac/zynqmp_edac.c @@ -0,0 +1,467 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Xilinx ZynqMP OCM ECC Driver + * + * Copyright (C) 2022 Advanced Micro Devices, Inc. + */ + +#include +#include +#include +#include +#include +#include + +#include "edac_module.h" + +#define ZYNQMP_OCM_EDAC_MSG_SIZE 256 + +#define ZYNQMP_OCM_EDAC_STRING "zynqmp_ocm" + +/* Error/Interrupt registers */ +#define ERR_CTRL_OFST 0x0 +#define OCM_ISR_OFST 0x04 +#define OCM_IMR_OFST 0x08 +#define OCM_IEN_OFST 0x0C +#define OCM_IDS_OFST 0x10 + +/* ECC control register */ +#define ECC_CTRL_OFST 0x14 + +/* Correctable error info registers */ +#define CE_FFA_OFST 0x1C +#define CE_FFD0_OFST 0x20 +#define CE_FFD1_OFST 0x24 +#define CE_FFD2_OFST 0x28 +#define CE_FFD3_OFST 0x2C +#define CE_FFE_OFST 0x30 + +/* Uncorrectable error info registers */ +#define UE_FFA_OFST 0x34 +#define UE_FFD0_OFST 0x38 +#define UE_FFD1_OFST 0x3C +#define UE_FFD2_OFST 0x40 +#define UE_FFD3_OFST 0x44 +#define UE_FFE_OFST 0x48 + +/* ECC control register bit field definitions */ +#define ECC_CTRL_CLR_CE_ERR 0x40 +#define ECC_CTRL_CLR_UE_ERR 0x80 + +/* Fault injection data and count registers */ +#define OCM_FID0_OFST 0x4C +#define OCM_FID1_OFST 0x50 +#define OCM_FID2_OFST 0x54 +#define OCM_FID3_OFST 0x58 +#define OCM_FIC_OFST 0x74 + +#define UE_MAX_BITPOS_LOWER 31 +#define UE_MIN_BITPOS_UPPER 32 +#define UE_MAX_BITPOS_UPPER 63 + +/* Interrupt masks */ +#define OCM_CEINTR_MASK BIT(6) +#define OCM_UEINTR_MASK BIT(7) +#define OCM_ECC_ENABLE_MASK BIT(0) + +#define OCM_FICOUNT_MASK GENMASK(23, 0) +#define OCM_NUM_UE_BITPOS 2 +#define OCM_BASEVAL 0xFFFC0000 +#define EDAC_DEVICE "ZynqMP-OCM" + +/** + * struct ecc_error_info - ECC error log information + * @addr: Fault generated at this address + * @fault_lo: Generated fault data (lower 32-bit) + * @fault_hi: Generated fault data (upper 32-bit) + */ +struct ecc_error_info { + u32 addr; + u32 fault_lo; + u32 fault_hi; +}; + +/** + * struct ecc_status - ECC status information to report + * @ce_cnt: Correctable error count + * @ue_cnt: Uncorrectable error count + * @ceinfo: Correctable error log information + * @ueinfo: Uncorrectable error log information + */ +struct ecc_status { + u32 ce_cnt; + u32 ue_cnt; + struct ecc_error_info ceinfo; + struct ecc_error_info ueinfo; +}; + +/** + * struct edac_priv - OCM private instance data + * @baseaddr: Base address of the OCM + * @message: Buffer for framing the event specific info + * @stat: ECC status information + * @ce_cnt: Correctable Error count + * @ue_cnt: Uncorrectable Error count + * @debugfs_dir: Directory entry for debugfs + * @ce_bitpos: Bit position for Correctable Error + * @ue_bitpos: Array to store UnCorrectable Error bit positions + * @fault_injection_cnt: Fault Injection Counter value + */ +struct edac_priv { + void __iomem *baseaddr; + char message[ZYNQMP_OCM_EDAC_MSG_SIZE]; + struct ecc_status stat; + u32 ce_cnt; + u32 ue_cnt; +#ifdef CONFIG_EDAC_DEBUG + struct dentry *debugfs_dir; + u8 ce_bitpos; + u8 ue_bitpos[OCM_NUM_UE_BITPOS]; + u32 fault_injection_cnt; +#endif +}; + +/** + * get_error_info - Get the current ECC error info + * @base: Pointer to the base address of the OCM + * @p: Pointer to the OCM ECC status structure + * @mask: Status register mask value + * + * Determines there is any ECC error or not + * + */ +static void get_error_info(void __iomem *base, struct ecc_status *p, int mask) +{ + if (mask & OCM_CEINTR_MASK) { + p->ce_cnt++; + p->ceinfo.fault_lo = readl(base + CE_FFD0_OFST); + p->ceinfo.fault_hi = readl(base + CE_FFD1_OFST); + p->ceinfo.addr = (OCM_BASEVAL | readl(base + CE_FFA_OFST)); + writel(ECC_CTRL_CLR_CE_ERR, base + OCM_ISR_OFST); + } else if (mask & OCM_UEINTR_MASK) { + p->ue_cnt++; + p->ueinfo.fault_lo = readl(base + UE_FFD0_OFST); + p->ueinfo.fault_hi = readl(base + UE_FFD1_OFST); + p->ueinfo.addr = (OCM_BASEVAL | readl(base + UE_FFA_OFST)); + writel(ECC_CTRL_CLR_UE_ERR, base + OCM_ISR_OFST); + } +} + +/** + * handle_error - Handle error types CE and UE + * @dci: Pointer to the EDAC device instance + * @p: Pointer to the OCM ECC status structure + * + * Handles correctable and uncorrectable errors. + */ +static void handle_error(struct edac_device_ctl_info *dci, struct ecc_status *p) +{ + struct edac_priv *priv = dci->pvt_info; + struct ecc_error_info *pinf; + + if (p->ce_cnt) { + pinf = &p->ceinfo; + snprintf(priv->message, ZYNQMP_OCM_EDAC_MSG_SIZE, + "\nOCM ECC error type :%s\nAddr: [0x%x]\nFault Data[0x%08x%08x]", + "CE", pinf->addr, pinf->fault_hi, pinf->fault_lo); + edac_device_handle_ce(dci, 0, 0, priv->message); + } + + if (p->ue_cnt) { + pinf = &p->ueinfo; + snprintf(priv->message, ZYNQMP_OCM_EDAC_MSG_SIZE, + "\nOCM ECC error type :%s\nAddr: [0x%x]\nFault Data[0x%08x%08x]", + "UE", pinf->addr, pinf->fault_hi, pinf->fault_lo); + edac_device_handle_ue(dci, 0, 0, priv->message); + } + + memset(p, 0, sizeof(*p)); +} + +/** + * intr_handler - ISR routine + * @irq: irq number + * @dev_id: device id pointer + * + * Return: IRQ_NONE, if CE/UE interrupt not set or IRQ_HANDLED otherwise + */ +static irqreturn_t intr_handler(int irq, void *dev_id) +{ + struct edac_device_ctl_info *dci = dev_id; + struct edac_priv *priv = dci->pvt_info; + int regval; + + regval = readl(priv->baseaddr + OCM_ISR_OFST); + if (!(regval & (OCM_CEINTR_MASK | OCM_UEINTR_MASK))) { + WARN_ONCE(1, "Unhandled IRQ%d, ISR: 0x%x", irq, regval); + return IRQ_NONE; + } + + get_error_info(priv->baseaddr, &priv->stat, regval); + + priv->ce_cnt += priv->stat.ce_cnt; + priv->ue_cnt += priv->stat.ue_cnt; + handle_error(dci, &priv->stat); + + return IRQ_HANDLED; +} + +/** + * get_eccstate - Return the ECC status + * @base: Pointer to the OCM base address + * + * Get the ECC enable/disable status + * + * Return: ECC status 0/1. + */ +static bool get_eccstate(void __iomem *base) +{ + return readl(base + ECC_CTRL_OFST) & OCM_ECC_ENABLE_MASK; +} + +#ifdef CONFIG_EDAC_DEBUG +/** + * write_fault_count - write fault injection count + * @priv: Pointer to the EDAC private struct + * + * Update the fault injection count register, once the counter reaches + * zero, it injects errors + */ +static void write_fault_count(struct edac_priv *priv) +{ + u32 ficount = priv->fault_injection_cnt; + + if (ficount & ~OCM_FICOUNT_MASK) { + ficount &= OCM_FICOUNT_MASK; + edac_printk(KERN_INFO, EDAC_DEVICE, + "Fault injection count value truncated to %d\n", ficount); + } + + writel(ficount, priv->baseaddr + OCM_FIC_OFST); +} + +/* + * To get the Correctable Error injected, the following steps are needed: + * - Setup the optional Fault Injection Count: + * echo > /sys/kernel/debug/edac/ocm/inject_fault_count + * - Write the Correctable Error bit position value: + * echo > /sys/kernel/debug/edac/ocm/inject_ce_bitpos + */ +static ssize_t inject_ce_write(struct file *file, const char __user *data, + size_t count, loff_t *ppos) +{ + struct edac_device_ctl_info *edac_dev = file->private_data; + struct edac_priv *priv = edac_dev->pvt_info; + int ret; + + if (!data) + return -EFAULT; + + ret = kstrtou8_from_user(data, count, 0, &priv->ce_bitpos); + if (ret) + return ret; + + if (priv->ce_bitpos > UE_MAX_BITPOS_UPPER) + return -EINVAL; + + if (priv->ce_bitpos <= UE_MAX_BITPOS_LOWER) { + writel(BIT(priv->ce_bitpos), priv->baseaddr + OCM_FID0_OFST); + writel(0, priv->baseaddr + OCM_FID1_OFST); + } else { + writel(BIT(priv->ce_bitpos - UE_MIN_BITPOS_UPPER), + priv->baseaddr + OCM_FID1_OFST); + writel(0, priv->baseaddr + OCM_FID0_OFST); + } + + write_fault_count(priv); + + return count; +} + +static const struct file_operations inject_ce_fops = { + .open = simple_open, + .write = inject_ce_write, + .llseek = generic_file_llseek, +}; + +/* + * To get the Uncorrectable Error injected, the following steps are needed: + * - Setup the optional Fault Injection Count: + * echo > /sys/kernel/debug/edac/ocm/inject_fault_count + * - Write the Uncorrectable Error bit position values: + * echo , > /sys/kernel/debug/edac/ocm/inject_ue_bitpos + */ +static ssize_t inject_ue_write(struct file *file, const char __user *data, + size_t count, loff_t *ppos) +{ + struct edac_device_ctl_info *edac_dev = file->private_data; + struct edac_priv *priv = edac_dev->pvt_info; + char buf[6], *pbuf, *token[2]; + u64 ue_bitpos; + int i, ret; + u8 len; + + if (!data) + return -EFAULT; + + len = min_t(size_t, count, sizeof(buf)); + if (copy_from_user(buf, data, len)) + return -EFAULT; + + buf[len] = '\0'; + pbuf = &buf[0]; + for (i = 0; i < OCM_NUM_UE_BITPOS; i++) + token[i] = strsep(&pbuf, ","); + + ret = kstrtou8(token[0], 0, &priv->ue_bitpos[0]); + if (ret) + return ret; + + ret = kstrtou8(token[1], 0, &priv->ue_bitpos[1]); + if (ret) + return ret; + + if (priv->ue_bitpos[0] > UE_MAX_BITPOS_UPPER || + priv->ue_bitpos[1] > UE_MAX_BITPOS_UPPER) + return -EINVAL; + + if (priv->ue_bitpos[0] == priv->ue_bitpos[1]) { + edac_printk(KERN_ERR, EDAC_DEVICE, "Bit positions should not be equal\n"); + return -EINVAL; + } + + ue_bitpos = BIT(priv->ue_bitpos[0]) | BIT(priv->ue_bitpos[1]); + + writel((u32)ue_bitpos, priv->baseaddr + OCM_FID0_OFST); + writel((u32)(ue_bitpos >> 32), priv->baseaddr + OCM_FID1_OFST); + + write_fault_count(priv); + + return count; +} + +static const struct file_operations inject_ue_fops = { + .open = simple_open, + .write = inject_ue_write, + .llseek = generic_file_llseek, +}; + +static void setup_debugfs(struct edac_device_ctl_info *edac_dev) +{ + struct edac_priv *priv = edac_dev->pvt_info; + + priv->debugfs_dir = edac_debugfs_create_dir("ocm"); + if (!priv->debugfs_dir) + return; + + edac_debugfs_create_x32("inject_fault_count", 0644, priv->debugfs_dir, + &priv->fault_injection_cnt); + edac_debugfs_create_file("inject_ue_bitpos", 0644, priv->debugfs_dir, + edac_dev, &inject_ue_fops); + edac_debugfs_create_file("inject_ce_bitpos", 0644, priv->debugfs_dir, + edac_dev, &inject_ce_fops); +} +#endif + +static int edac_probe(struct platform_device *pdev) +{ + struct edac_device_ctl_info *dci; + struct edac_priv *priv; + void __iomem *baseaddr; + struct resource *res; + int irq, ret; + + baseaddr = devm_platform_get_and_ioremap_resource(pdev, 0, &res); + if (IS_ERR(baseaddr)) + return PTR_ERR(baseaddr); + + if (!get_eccstate(baseaddr)) { + edac_printk(KERN_INFO, EDAC_DEVICE, "ECC not enabled\n"); + return -ENXIO; + } + + dci = edac_device_alloc_ctl_info(sizeof(*priv), ZYNQMP_OCM_EDAC_STRING, + 1, ZYNQMP_OCM_EDAC_STRING, 1, 0, NULL, 0, + edac_device_alloc_index()); + if (!dci) + return -ENOMEM; + + priv = dci->pvt_info; + platform_set_drvdata(pdev, dci); + dci->dev = &pdev->dev; + priv->baseaddr = baseaddr; + dci->mod_name = pdev->dev.driver->name; + dci->ctl_name = ZYNQMP_OCM_EDAC_STRING; + dci->dev_name = dev_name(&pdev->dev); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) { + ret = irq; + goto free_dev_ctl; + } + + ret = devm_request_irq(&pdev->dev, irq, intr_handler, 0, + dev_name(&pdev->dev), dci); + if (ret) { + edac_printk(KERN_ERR, EDAC_DEVICE, "Failed to request Irq\n"); + goto free_dev_ctl; + } + + /* Enable UE, CE interrupts */ + writel((OCM_CEINTR_MASK | OCM_UEINTR_MASK), priv->baseaddr + OCM_IEN_OFST); + +#ifdef CONFIG_EDAC_DEBUG + setup_debugfs(dci); +#endif + + ret = edac_device_add_device(dci); + if (ret) + goto free_dev_ctl; + + return 0; + +free_dev_ctl: + edac_device_free_ctl_info(dci); + + return ret; +} + +static int edac_remove(struct platform_device *pdev) +{ + struct edac_device_ctl_info *dci = platform_get_drvdata(pdev); + struct edac_priv *priv = dci->pvt_info; + + /* Disable UE, CE interrupts */ + writel((OCM_CEINTR_MASK | OCM_UEINTR_MASK), priv->baseaddr + OCM_IDS_OFST); + +#ifdef CONFIG_EDAC_DEBUG + debugfs_remove_recursive(priv->debugfs_dir); +#endif + + edac_device_del_device(&pdev->dev); + edac_device_free_ctl_info(dci); + + return 0; +} + +static const struct of_device_id zynqmp_ocm_edac_match[] = { + { .compatible = "xlnx,zynqmp-ocmc-1.0"}, + { /* end of table */ } +}; + +MODULE_DEVICE_TABLE(of, zynqmp_ocm_edac_match); + +static struct platform_driver zynqmp_ocm_edac_driver = { + .driver = { + .name = "zynqmp-ocm-edac", + .of_match_table = zynqmp_ocm_edac_match, + }, + .probe = edac_probe, + .remove = edac_remove, +}; + +module_platform_driver(zynqmp_ocm_edac_driver); + +MODULE_AUTHOR("Advanced Micro Devices, Inc"); +MODULE_DESCRIPTION("Xilinx ZynqMP OCM ECC driver"); +MODULE_LICENSE("GPL"); From 8a01ec97dc066009dd89e43bfcf55644f2dd6d19 Mon Sep 17 00:00:00 2001 From: Tony Luck Date: Mon, 9 Jan 2023 07:29:36 -0800 Subject: [PATCH 12/94] x86/mce: Mask out non-address bits from machine check bank Systems that support various memory encryption schemes (MKTME, TDX, SEV) use high order physical address bits to indicate which key should be used for a specific memory location. When a memory error is reported, some systems may report those key bits in the IA32_MCi_ADDR machine check MSR. The Intel SDM has a footnote for the contents of the address register that says: "Useful bits in this field depend on the address methodology in use when the register state is saved." AMD Processor Programming Reference has a more explicit description of the MCA_ADDR register: "For physical addresses, the most significant bit is given by Core::X86::Cpuid::LongModeInfo[PhysAddrSize]." Add a new #define MCI_ADDR_PHYSADDR for the mask of valid physical address bits within the machine check bank address register. Use this mask for recoverable machine check handling and in the EDAC driver to ignore any key bits that may be present. [ Tony: Based on independent fixes proposed by Fan Du and Isaku Yamahata ] Reported-by: Isaku Yamahata Reported-by: Fan Du Signed-off-by: Tony Luck Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Yazen Ghannam Link: https://lore.kernel.org/r/20230109152936.397862-1-tony.luck@intel.com --- arch/x86/include/asm/mce.h | 3 +++ arch/x86/kernel/cpu/mce/core.c | 14 +++++++++----- drivers/edac/skx_common.c | 2 +- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h index 6e986088817d..9646ed6e8c0b 100644 --- a/arch/x86/include/asm/mce.h +++ b/arch/x86/include/asm/mce.h @@ -88,6 +88,9 @@ #define MCI_MISC_ADDR_MEM 3 /* memory address */ #define MCI_MISC_ADDR_GENERIC 7 /* generic */ +/* MCi_ADDR register defines */ +#define MCI_ADDR_PHYSADDR GENMASK_ULL(boot_cpu_data.x86_phys_bits - 1, 0) + /* CTL2 register defines */ #define MCI_CTL2_CMCI_EN BIT_ULL(30) #define MCI_CTL2_CMCI_THRESHOLD_MASK 0x7fffULL diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index 626a1c666252..7832a69d170e 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -573,7 +573,7 @@ static int uc_decode_notifier(struct notifier_block *nb, unsigned long val, mce->severity != MCE_DEFERRED_SEVERITY) return NOTIFY_DONE; - pfn = mce->addr >> PAGE_SHIFT; + pfn = (mce->addr & MCI_ADDR_PHYSADDR) >> PAGE_SHIFT; if (!memory_failure(pfn, 0)) { set_mce_nospec(pfn); mce->kflags |= MCE_HANDLED_UC; @@ -1294,6 +1294,7 @@ static void kill_me_maybe(struct callback_head *cb) { struct task_struct *p = container_of(cb, struct task_struct, mce_kill_me); int flags = MF_ACTION_REQUIRED; + unsigned long pfn; int ret; p->mce_count = 0; @@ -1302,9 +1303,10 @@ static void kill_me_maybe(struct callback_head *cb) if (!p->mce_ripv) flags |= MF_MUST_KILL; - ret = memory_failure(p->mce_addr >> PAGE_SHIFT, flags); + pfn = (p->mce_addr & MCI_ADDR_PHYSADDR) >> PAGE_SHIFT; + ret = memory_failure(pfn, flags); if (!ret) { - set_mce_nospec(p->mce_addr >> PAGE_SHIFT); + set_mce_nospec(pfn); sync_core(); return; } @@ -1326,11 +1328,13 @@ static void kill_me_maybe(struct callback_head *cb) static void kill_me_never(struct callback_head *cb) { struct task_struct *p = container_of(cb, struct task_struct, mce_kill_me); + unsigned long pfn; p->mce_count = 0; pr_err("Kernel accessed poison in user space at %llx\n", p->mce_addr); - if (!memory_failure(p->mce_addr >> PAGE_SHIFT, 0)) - set_mce_nospec(p->mce_addr >> PAGE_SHIFT); + pfn = (p->mce_addr & MCI_ADDR_PHYSADDR) >> PAGE_SHIFT; + if (!memory_failure(pfn, 0)) + set_mce_nospec(pfn); } static void queue_task_work(struct mce *m, char *msg, void (*func)(struct callback_head *)) diff --git a/drivers/edac/skx_common.c b/drivers/edac/skx_common.c index f0f8e98f6efb..806986f03177 100644 --- a/drivers/edac/skx_common.c +++ b/drivers/edac/skx_common.c @@ -657,7 +657,7 @@ int skx_mce_check_error(struct notifier_block *nb, unsigned long val, memset(&res, 0, sizeof(res)); res.mce = mce; - res.addr = mce->addr; + res.addr = mce->addr & MCI_ADDR_PHYSADDR; /* Try driver decoder first */ if (!(driver_decode && driver_decode(&res))) { From b7d1f15b5c274999dfe8dda60a9a516eebfbc3d0 Mon Sep 17 00:00:00 2001 From: Wang Yong Date: Sun, 11 Dec 2022 10:38:49 +0000 Subject: [PATCH 13/94] x86/boot/e820: Fix typo in e820.c comment change "itsmain" to "its main". Fixes: 544a0f47e780 ("x86/boot/e820: Rename e820_table_saved to e820_table_firmware and improve the description") Signed-off-by: Wang Yong Signed-off-by: Ingo Molnar Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/20221211103849.173870-1-yongw.kernel@gmail.com --- arch/x86/kernel/e820.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kernel/e820.c b/arch/x86/kernel/e820.c index 9dac24680ff8..0614a79c4619 100644 --- a/arch/x86/kernel/e820.c +++ b/arch/x86/kernel/e820.c @@ -53,7 +53,7 @@ * * Once the E820 map has been converted to the standard Linux memory layout * information its role stops - modifying it has no effect and does not get - * re-propagated. So itsmain role is a temporary bootstrap storage of firmware + * re-propagated. So its main role is a temporary bootstrap storage of firmware * specific memory layout data during early bootup. */ static struct e820_table e820_table_init __initdata; From 50c66d7b049ddec095644a005bfe0b28e22c4b02 Mon Sep 17 00:00:00 2001 From: Yuntao Wang Date: Wed, 1 Jun 2022 20:29:14 +0800 Subject: [PATCH 14/94] x86/setup: Move duplicate boot_cpu_data definition out of the ifdeffery Both the if and else blocks define an exact same boot_cpu_data variable, move the duplicate variable definition out of the if/else block. In addition, do some other minor cleanups. [ bp: Massage. ] Signed-off-by: Yuntao Wang Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/20220601122914.820890-1-ytcoode@gmail.com --- arch/x86/kernel/e820.c | 4 ++-- arch/x86/kernel/setup.c | 10 ++-------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/arch/x86/kernel/e820.c b/arch/x86/kernel/e820.c index 0614a79c4619..fb8cf953380d 100644 --- a/arch/x86/kernel/e820.c +++ b/arch/x86/kernel/e820.c @@ -395,7 +395,7 @@ int __init e820__update_table(struct e820_table *table) /* Continue building up new map based on this information: */ if (current_type != last_type || e820_nomerge(current_type)) { - if (last_type != 0) { + if (last_type) { new_entries[new_nr_entries].size = change_point[chg_idx]->addr - last_addr; /* Move forward only if the new size was non-zero: */ if (new_entries[new_nr_entries].size != 0) @@ -403,7 +403,7 @@ int __init e820__update_table(struct e820_table *table) if (++new_nr_entries >= max_nr_entries) break; } - if (current_type != 0) { + if (current_type) { new_entries[new_nr_entries].addr = change_point[chg_idx]->addr; new_entries[new_nr_entries].type = current_type; last_addr = change_point[chg_idx]->addr; diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index 88188549647c..16babff771bd 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -114,11 +114,6 @@ static struct resource bss_resource = { #ifdef CONFIG_X86_32 /* CPU data as detected by the assembly code in head_32.S */ struct cpuinfo_x86 new_cpu_data; - -/* Common CPU data for all CPUs */ -struct cpuinfo_x86 boot_cpu_data __read_mostly; -EXPORT_SYMBOL(boot_cpu_data); - unsigned int def_to_bigsmp; struct apm_info apm_info; @@ -132,11 +127,10 @@ EXPORT_SYMBOL(ist_info); struct ist_info ist_info; #endif -#else -struct cpuinfo_x86 boot_cpu_data __read_mostly; -EXPORT_SYMBOL(boot_cpu_data); #endif +struct cpuinfo_x86 boot_cpu_data __read_mostly; +EXPORT_SYMBOL(boot_cpu_data); #if !defined(CONFIG_X86_PAE) || defined(CONFIG_X86_64) __visible unsigned long mmu_cr4_features __ro_after_init; From 9f76d59173d9d146e96c66886b671c1915a5c5e5 Mon Sep 17 00:00:00 2001 From: Jann Horn Date: Thu, 5 Jan 2023 14:44:03 +0100 Subject: [PATCH 15/94] timers: Prevent union confusion from unexpected restart_syscall() The nanosleep syscalls use the restart_block mechanism, with a quirk: The `type` and `rmtp`/`compat_rmtp` fields are set up unconditionally on syscall entry, while the rest of the restart_block is only set up in the unlikely case that the syscall is actually interrupted by a signal (or pseudo-signal) that doesn't have a signal handler. If the restart_block was set up by a previous syscall (futex(..., FUTEX_WAIT, ...) or poll()) and hasn't been invalidated somehow since then, this will clobber some of the union fields used by futex_wait_restart() and do_restart_poll(). If userspace afterwards wrongly calls the restart_syscall syscall, futex_wait_restart()/do_restart_poll() will read struct fields that have been clobbered. This doesn't actually lead to anything particularly interesting because none of the union fields contain trusted kernel data, and futex(..., FUTEX_WAIT, ...) and poll() aren't syscalls where it makes much sense to apply seccomp filters to their arguments. So the current consequences are just of the "if userspace does bad stuff, it can damage itself, and that's not a problem" flavor. But still, it seems like a hazard for future developers, so invalidate the restart_block when partly setting it up in the nanosleep syscalls. Signed-off-by: Jann Horn Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20230105134403.754986-1-jannh@google.com --- kernel/time/hrtimer.c | 2 ++ kernel/time/posix-stubs.c | 2 ++ kernel/time/posix-timers.c | 2 ++ 3 files changed, 6 insertions(+) diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 3ae661ab6260..e4f0e3b0c4f4 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -2126,6 +2126,7 @@ SYSCALL_DEFINE2(nanosleep, struct __kernel_timespec __user *, rqtp, if (!timespec64_valid(&tu)) return -EINVAL; + current->restart_block.fn = do_no_restart_syscall; current->restart_block.nanosleep.type = rmtp ? TT_NATIVE : TT_NONE; current->restart_block.nanosleep.rmtp = rmtp; return hrtimer_nanosleep(timespec64_to_ktime(tu), HRTIMER_MODE_REL, @@ -2147,6 +2148,7 @@ SYSCALL_DEFINE2(nanosleep_time32, struct old_timespec32 __user *, rqtp, if (!timespec64_valid(&tu)) return -EINVAL; + current->restart_block.fn = do_no_restart_syscall; current->restart_block.nanosleep.type = rmtp ? TT_COMPAT : TT_NONE; current->restart_block.nanosleep.compat_rmtp = rmtp; return hrtimer_nanosleep(timespec64_to_ktime(tu), HRTIMER_MODE_REL, diff --git a/kernel/time/posix-stubs.c b/kernel/time/posix-stubs.c index 90ea5f373e50..828aeecbd1e8 100644 --- a/kernel/time/posix-stubs.c +++ b/kernel/time/posix-stubs.c @@ -147,6 +147,7 @@ SYSCALL_DEFINE4(clock_nanosleep, const clockid_t, which_clock, int, flags, return -EINVAL; if (flags & TIMER_ABSTIME) rmtp = NULL; + current->restart_block.fn = do_no_restart_syscall; current->restart_block.nanosleep.type = rmtp ? TT_NATIVE : TT_NONE; current->restart_block.nanosleep.rmtp = rmtp; texp = timespec64_to_ktime(t); @@ -240,6 +241,7 @@ SYSCALL_DEFINE4(clock_nanosleep_time32, clockid_t, which_clock, int, flags, return -EINVAL; if (flags & TIMER_ABSTIME) rmtp = NULL; + current->restart_block.fn = do_no_restart_syscall; current->restart_block.nanosleep.type = rmtp ? TT_COMPAT : TT_NONE; current->restart_block.nanosleep.compat_rmtp = rmtp; texp = timespec64_to_ktime(t); diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c index 5dead89308b7..0c8a87a11b39 100644 --- a/kernel/time/posix-timers.c +++ b/kernel/time/posix-timers.c @@ -1270,6 +1270,7 @@ SYSCALL_DEFINE4(clock_nanosleep, const clockid_t, which_clock, int, flags, return -EINVAL; if (flags & TIMER_ABSTIME) rmtp = NULL; + current->restart_block.fn = do_no_restart_syscall; current->restart_block.nanosleep.type = rmtp ? TT_NATIVE : TT_NONE; current->restart_block.nanosleep.rmtp = rmtp; @@ -1297,6 +1298,7 @@ SYSCALL_DEFINE4(clock_nanosleep_time32, clockid_t, which_clock, int, flags, return -EINVAL; if (flags & TIMER_ABSTIME) rmtp = NULL; + current->restart_block.fn = do_no_restart_syscall; current->restart_block.nanosleep.type = rmtp ? TT_COMPAT : TT_NONE; current->restart_block.nanosleep.compat_rmtp = rmtp; From d55dcb7384b1234d3caf70c2f8199c4184502ff9 Mon Sep 17 00:00:00 2001 From: Juergen Gross Date: Fri, 13 Jan 2023 14:01:26 +0100 Subject: [PATCH 16/94] x86/cpu: Remove misleading comment The comment of the "#endif" after setup_disable_pku() is wrong. As the related #ifdef is only a few lines above, just remove the comment. Signed-off-by: Juergen Gross Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20230113130126.1966-1-jgross@suse.com --- arch/x86/kernel/cpu/common.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 9cfca3d7d0e2..7670397438c6 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -567,7 +567,7 @@ static __init int setup_disable_pku(char *arg) return 1; } __setup("nopku", setup_disable_pku); -#endif /* CONFIG_X86_64 */ +#endif #ifdef CONFIG_X86_KERNEL_IBT From ef6dfc4b238a435ab552207ec09e4a82b6426d31 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sat, 14 Jan 2023 18:33:09 +0100 Subject: [PATCH 17/94] x86/signal: Fix the value returned by strict_sas_size() Functions used with __setup() return 1 when the argument has been successfully parsed. Reverse the returned value so that 1 is returned when kstrtobool() is successful (i.e. returns 0). My understanding of these __setup() functions is that returning 1 or 0 does not change much anyway - so this is more of a cleanup than a functional fix. I spot it and found it spurious while looking at something else. Even if the output is not perfect, you'll get the idea with: $ git grep -B2 -A10 retu.*kstrtobool | grep __setup -B10 Fixes: 3aac3ebea08f ("x86/signal: Implement sigaltstack size validation") Signed-off-by: Christophe JAILLET Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/73882d43ebe420c9d8fb82d0560021722b243000.1673717552.git.christophe.jaillet@wanadoo.fr --- arch/x86/kernel/signal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kernel/signal.c b/arch/x86/kernel/signal.c index 1504eb8d25aa..004cb30b7419 100644 --- a/arch/x86/kernel/signal.c +++ b/arch/x86/kernel/signal.c @@ -360,7 +360,7 @@ static bool strict_sigaltstack_size __ro_after_init = false; static int __init strict_sas_size(char *arg) { - return kstrtobool(arg, &strict_sigaltstack_size); + return kstrtobool(arg, &strict_sigaltstack_size) == 0; } __setup("strict_sas_size", strict_sas_size); From 59047d942bedc4ce88a2121a9f9e656b9532dd30 Mon Sep 17 00:00:00 2001 From: "Guangju Wang[baidu]" Date: Wed, 18 Jan 2023 10:35:54 +0800 Subject: [PATCH 18/94] x86/microcode: Use the DEVICE_ATTR_RO() macro Use DEVICE_ATTR_RO() helper instead of open-coded DEVICE_ATTR(), which makes the code a bit shorter and easier to read. No change in functionality. Signed-off-by: Guangju Wang[baidu] Signed-off-by: Ingo Molnar Acked-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/20230118023554.1898-1-wgj900@163.com --- arch/x86/kernel/cpu/microcode/core.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c index 712aafff96e0..99abb318aa7f 100644 --- a/arch/x86/kernel/cpu/microcode/core.c +++ b/arch/x86/kernel/cpu/microcode/core.c @@ -507,7 +507,7 @@ static ssize_t version_show(struct device *dev, return sprintf(buf, "0x%x\n", uci->cpu_sig.rev); } -static ssize_t pf_show(struct device *dev, +static ssize_t processor_flags_show(struct device *dev, struct device_attribute *attr, char *buf) { struct ucode_cpu_info *uci = ucode_cpu_info + dev->id; @@ -515,8 +515,8 @@ static ssize_t pf_show(struct device *dev, return sprintf(buf, "0x%x\n", uci->cpu_sig.pf); } -static DEVICE_ATTR(version, 0444, version_show, NULL); -static DEVICE_ATTR(processor_flags, 0444, pf_show, NULL); +static DEVICE_ATTR_RO(version); +static DEVICE_ATTR_RO(processor_flags); static struct attribute *mc_default_attrs[] = { &dev_attr_version.attr, From 8d8fcc391f50e2d9686d42b85e9b1db89b1bbb35 Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Wed, 18 Jan 2023 20:38:49 +0530 Subject: [PATCH 19/94] EDAC/qcom: Add platform_device_id table for module autoloading Add a device ID table so that the driver loads automatically when the associated platform_device gets registered. Reported-by: Andrew Halaney Signed-off-by: Manivannan Sadhasivam Signed-off-by: Borislav Petkov (AMD) Tested-by: Steev Klimaszewski # Thinkpad X13s Tested-by: Andrew Halaney # sa8540p-ride Link: https://lore.kernel.org/r/20230118150904.26913-3-manivannan.sadhasivam@linaro.org --- drivers/edac/qcom_edac.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/edac/qcom_edac.c b/drivers/edac/qcom_edac.c index 97a27e42dd61..9e77fa84e84f 100644 --- a/drivers/edac/qcom_edac.c +++ b/drivers/edac/qcom_edac.c @@ -397,12 +397,19 @@ static int qcom_llcc_edac_remove(struct platform_device *pdev) return 0; } +static const struct platform_device_id qcom_llcc_edac_id_table[] = { + { .name = "qcom_llcc_edac" }, + {} +}; +MODULE_DEVICE_TABLE(platform, qcom_llcc_edac_id_table); + static struct platform_driver qcom_llcc_edac_driver = { .probe = qcom_llcc_edac_probe, .remove = qcom_llcc_edac_remove, .driver = { .name = "qcom_llcc_edac", }, + .id_table = qcom_llcc_edac_id_table, }; module_platform_driver(qcom_llcc_edac_driver); From ab31c74455c64e69342ddab21fd9426fcbfefde7 Mon Sep 17 00:00:00 2001 From: Ashok Raj Date: Mon, 9 Jan 2023 07:35:50 -0800 Subject: [PATCH 20/94] x86/microcode: Add a parameter to microcode_check() to store CPU capabilities Add a parameter to store CPU capabilities before performing a microcode update so that CPU capabilities can be compared before and after update. [ bp: Massage. ] Signed-off-by: Ashok Raj Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/20230109153555.4986-2-ashok.raj@intel.com --- arch/x86/include/asm/processor.h | 2 +- arch/x86/kernel/cpu/common.c | 21 +++++++++++++-------- arch/x86/kernel/cpu/microcode/core.c | 3 ++- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 4e35c66edeb7..f256a4ddd25d 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -697,7 +697,7 @@ bool xen_set_default_idle(void); #endif void __noreturn stop_this_cpu(void *dummy); -void microcode_check(void); +void microcode_check(struct cpuinfo_x86 *prev_info); enum l1tf_mitigations { L1TF_MITIGATION_OFF, diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 9cfca3d7d0e2..0f5a173d0871 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -2297,30 +2297,35 @@ void cpu_init_secondary(void) #endif #ifdef CONFIG_MICROCODE_LATE_LOADING -/* +/** + * microcode_check() - Check if any CPU capabilities changed after an update. + * @prev_info: CPU capabilities stored before an update. + * * The microcode loader calls this upon late microcode load to recheck features, * only when microcode has been updated. Caller holds microcode_mutex and CPU * hotplug lock. + * + * Return: None */ -void microcode_check(void) +void microcode_check(struct cpuinfo_x86 *prev_info) { - struct cpuinfo_x86 info; - perf_check_microcode(); /* Reload CPUID max function as it might've changed. */ - info.cpuid_level = cpuid_eax(0); + prev_info->cpuid_level = cpuid_eax(0); /* * Copy all capability leafs to pick up the synthetic ones so that * memcmp() below doesn't fail on that. The ones coming from CPUID will * get overwritten in get_cpu_cap(). */ - memcpy(&info.x86_capability, &boot_cpu_data.x86_capability, sizeof(info.x86_capability)); + memcpy(&prev_info->x86_capability, &boot_cpu_data.x86_capability, + sizeof(prev_info->x86_capability)); - get_cpu_cap(&info); + get_cpu_cap(prev_info); - if (!memcmp(&info.x86_capability, &boot_cpu_data.x86_capability, sizeof(info.x86_capability))) + if (!memcmp(&prev_info->x86_capability, &boot_cpu_data.x86_capability, + sizeof(prev_info->x86_capability))) return; pr_warn("x86/CPU: CPU features have changed after loading microcode, but might not take effect.\n"); diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c index 99abb318aa7f..dc5dfba7e78e 100644 --- a/arch/x86/kernel/cpu/microcode/core.c +++ b/arch/x86/kernel/cpu/microcode/core.c @@ -438,6 +438,7 @@ wait_for_siblings: static int microcode_reload_late(void) { int old = boot_cpu_data.microcode, ret; + struct cpuinfo_x86 prev_info; pr_err("Attempting late microcode loading - it is dangerous and taints the kernel.\n"); pr_err("You should switch to early loading, if possible.\n"); @@ -447,7 +448,7 @@ static int microcode_reload_late(void) ret = stop_machine_cpuslocked(__reload_late, NULL, cpu_online_mask); if (ret == 0) - microcode_check(); + microcode_check(&prev_info); pr_info("Reload completed, microcode revision: 0x%x -> 0x%x\n", old, boot_cpu_data.microcode); From c0dd9245aa9e25a697181f6085692272c9ec61bc Mon Sep 17 00:00:00 2001 From: Ashok Raj Date: Mon, 9 Jan 2023 07:35:51 -0800 Subject: [PATCH 21/94] x86/microcode: Check CPU capabilities after late microcode update correctly The kernel caches each CPU's feature bits at boot in an x86_capability[] structure. However, the capabilities in the BSP's copy can be turned off as a result of certain command line parameters or configuration restrictions, for example the SGX bit. This can cause a mismatch when comparing the values before and after the microcode update. Another example is X86_FEATURE_SRBDS_CTRL which gets added only after microcode update: --- cpuid.before 2023-01-21 14:54:15.652000747 +0100 +++ cpuid.after 2023-01-21 14:54:26.632001024 +0100 @@ -10,7 +10,7 @@ CPU: 0x00000004 0x04: eax=0x00000000 ebx=0x00000000 ecx=0x00000000 edx=0x00000000 0x00000005 0x00: eax=0x00000040 ebx=0x00000040 ecx=0x00000003 edx=0x11142120 0x00000006 0x00: eax=0x000027f7 ebx=0x00000002 ecx=0x00000001 edx=0x00000000 - 0x00000007 0x00: eax=0x00000000 ebx=0x029c6fbf ecx=0x40000000 edx=0xbc002400 + 0x00000007 0x00: eax=0x00000000 ebx=0x029c6fbf ecx=0x40000000 edx=0xbc002e00 ^^^ and which proves for a gazillionth time that late loading is a bad bad idea. microcode_check() is called after an update to report any previously cached CPUID bits which might have changed due to the update. Therefore, store the cached CPU caps before the update and compare them with the CPU caps after the microcode update has succeeded. Thus, the comparison is done between the CPUID *hardware* bits before and after the upgrade instead of using the cached, possibly runtime modified values in BSP's boot_cpu_data copy. As a result, false warnings about CPUID bits changes are avoided. [ bp: - Massage. - Add SRBDS_CTRL example. - Add kernel-doc. - Incorporate forgotten review feedback from dhansen. ] Fixes: 1008c52c09dc ("x86/CPU: Add a microcode loader callback") Signed-off-by: Ashok Raj Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/20230109153555.4986-3-ashok.raj@intel.com --- arch/x86/include/asm/processor.h | 1 + arch/x86/kernel/cpu/common.c | 36 ++++++++++++++++++---------- arch/x86/kernel/cpu/microcode/core.c | 6 +++++ 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index f256a4ddd25d..a77dee6a2bf2 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -698,6 +698,7 @@ bool xen_set_default_idle(void); void __noreturn stop_this_cpu(void *dummy); void microcode_check(struct cpuinfo_x86 *prev_info); +void store_cpu_caps(struct cpuinfo_x86 *info); enum l1tf_mitigations { L1TF_MITIGATION_OFF, diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 0f5a173d0871..5ff73baa5583 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -2297,6 +2297,25 @@ void cpu_init_secondary(void) #endif #ifdef CONFIG_MICROCODE_LATE_LOADING +/** + * store_cpu_caps() - Store a snapshot of CPU capabilities + * @curr_info: Pointer where to store it + * + * Returns: None + */ +void store_cpu_caps(struct cpuinfo_x86 *curr_info) +{ + /* Reload CPUID max function as it might've changed. */ + curr_info->cpuid_level = cpuid_eax(0); + + /* Copy all capability leafs and pick up the synthetic ones. */ + memcpy(&curr_info->x86_capability, &boot_cpu_data.x86_capability, + sizeof(curr_info->x86_capability)); + + /* Get the hardware CPUID leafs */ + get_cpu_cap(curr_info); +} + /** * microcode_check() - Check if any CPU capabilities changed after an update. * @prev_info: CPU capabilities stored before an update. @@ -2309,22 +2328,13 @@ void cpu_init_secondary(void) */ void microcode_check(struct cpuinfo_x86 *prev_info) { + struct cpuinfo_x86 curr_info; + perf_check_microcode(); - /* Reload CPUID max function as it might've changed. */ - prev_info->cpuid_level = cpuid_eax(0); + store_cpu_caps(&curr_info); - /* - * Copy all capability leafs to pick up the synthetic ones so that - * memcmp() below doesn't fail on that. The ones coming from CPUID will - * get overwritten in get_cpu_cap(). - */ - memcpy(&prev_info->x86_capability, &boot_cpu_data.x86_capability, - sizeof(prev_info->x86_capability)); - - get_cpu_cap(prev_info); - - if (!memcmp(&prev_info->x86_capability, &boot_cpu_data.x86_capability, + if (!memcmp(&prev_info->x86_capability, &curr_info.x86_capability, sizeof(prev_info->x86_capability))) return; diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c index dc5dfba7e78e..8ec38c107dc3 100644 --- a/arch/x86/kernel/cpu/microcode/core.c +++ b/arch/x86/kernel/cpu/microcode/core.c @@ -446,6 +446,12 @@ static int microcode_reload_late(void) atomic_set(&late_cpus_in, 0); atomic_set(&late_cpus_out, 0); + /* + * Take a snapshot before the microcode update in order to compare and + * check whether any bits changed after an update. + */ + store_cpu_caps(&prev_info); + ret = stop_machine_cpuslocked(__reload_late, NULL, cpu_online_mask); if (ret == 0) microcode_check(&prev_info); From 6eab3abac7043226e5375e9ead0c7607ced6767b Mon Sep 17 00:00:00 2001 From: Ashok Raj Date: Mon, 9 Jan 2023 07:35:52 -0800 Subject: [PATCH 22/94] x86/microcode: Adjust late loading result reporting message During late microcode loading, the "Reload completed" message is issued unconditionally, regardless of success or failure. Adjust the message to report the result of the update. [ bp: Massage. ] Fixes: 9bd681251b7c ("x86/microcode: Announce reload operation's completion") Suggested-by: Thomas Gleixner Signed-off-by: Ashok Raj Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Tony Luck Link: https://lore.kernel.org/lkml/874judpqqd.ffs@tglx/ --- arch/x86/kernel/cpu/microcode/core.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c index 8ec38c107dc3..61d57d9b93ee 100644 --- a/arch/x86/kernel/cpu/microcode/core.c +++ b/arch/x86/kernel/cpu/microcode/core.c @@ -453,11 +453,14 @@ static int microcode_reload_late(void) store_cpu_caps(&prev_info); ret = stop_machine_cpuslocked(__reload_late, NULL, cpu_online_mask); - if (ret == 0) + if (!ret) { + pr_info("Reload succeeded, microcode revision: 0x%x -> 0x%x\n", + old, boot_cpu_data.microcode); microcode_check(&prev_info); - - pr_info("Reload completed, microcode revision: 0x%x -> 0x%x\n", - old, boot_cpu_data.microcode); + } else { + pr_info("Reload failed, current microcode revision: 0x%x\n", + boot_cpu_data.microcode); + } return ret; } From 174f1b909ab0984e5369a634971fb14a618ca797 Mon Sep 17 00:00:00 2001 From: Ashok Raj Date: Mon, 9 Jan 2023 07:35:53 -0800 Subject: [PATCH 23/94] x86/microcode/intel: Pass the microcode revision to print_ucode_info() directly print_ucode_info() takes a struct ucode_cpu_info pointer as parameter. Its sole purpose is to print the microcode revision. The only available ucode_cpu_info always describes the currently loaded microcode revision. After a microcode update is successful, this is the new revision, or on failure it is the original revision. In preparation for future changes, replace the struct ucode_cpu_info pointer parameter with a plain integer which contains the revision number and adjust the call sites accordingly. No functional change. [ bp: - Fix + cleanup commit message. - Revert arbitrary, unrelated change. ] Signed-off-by: Ashok Raj Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Thomas Gleixner Link: https://lore.kernel.org/r/20230120161923.118882-5-ashok.raj@intel.com --- arch/x86/kernel/cpu/microcode/intel.c | 30 ++++++++------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c index cac2bdb57f0b..9dbd007219a1 100644 --- a/arch/x86/kernel/cpu/microcode/intel.c +++ b/arch/x86/kernel/cpu/microcode/intel.c @@ -308,11 +308,10 @@ static bool load_builtin_intel_microcode(struct cpio_data *cp) /* * Print ucode update info. */ -static void -print_ucode_info(struct ucode_cpu_info *uci, unsigned int date) +static void print_ucode_info(unsigned int new_rev, unsigned int date) { pr_info_once("microcode updated early to revision 0x%x, date = %04x-%02x-%02x\n", - uci->cpu_sig.rev, + new_rev, date & 0xffff, date >> 24, (date >> 16) & 0xff); @@ -332,7 +331,7 @@ void show_ucode_info_early(void) if (delay_ucode_info) { intel_cpu_collect_info(&uci); - print_ucode_info(&uci, current_mc_date); + print_ucode_info(uci.cpu_sig.rev, current_mc_date); delay_ucode_info = 0; } } @@ -341,33 +340,22 @@ void show_ucode_info_early(void) * At this point, we can not call printk() yet. Delay printing microcode info in * show_ucode_info_early() until printk() works. */ -static void print_ucode(struct ucode_cpu_info *uci) +static void print_ucode(int new_rev, int date) { - struct microcode_intel *mc; int *delay_ucode_info_p; int *current_mc_date_p; - mc = uci->mc; - if (!mc) - return; - delay_ucode_info_p = (int *)__pa_nodebug(&delay_ucode_info); current_mc_date_p = (int *)__pa_nodebug(¤t_mc_date); *delay_ucode_info_p = 1; - *current_mc_date_p = mc->hdr.date; + *current_mc_date_p = date; } #else -static inline void print_ucode(struct ucode_cpu_info *uci) +static inline void print_ucode(int new_rev, int date) { - struct microcode_intel *mc; - - mc = uci->mc; - if (!mc) - return; - - print_ucode_info(uci, mc->hdr.date); + print_ucode_info(new_rev, date); } #endif @@ -407,9 +395,9 @@ static int apply_microcode_early(struct ucode_cpu_info *uci, bool early) uci->cpu_sig.rev = rev; if (early) - print_ucode(uci); + print_ucode(uci->cpu_sig.rev, mc->hdr.date); else - print_ucode_info(uci, mc->hdr.date); + print_ucode_info(uci->cpu_sig.rev, mc->hdr.date); return 0; } From a9a5cac225b0830d1879640e25231a37e537f0da Mon Sep 17 00:00:00 2001 From: Ashok Raj Date: Sun, 15 Jan 2023 19:57:27 +0100 Subject: [PATCH 24/94] x86/microcode/intel: Print old and new revision during early boot Make early loading message match late loading message and print both old and new revisions. This is helpful to know what the BIOS loaded revision is before an early update. Cache the early BIOS revision before the microcode update and have print_ucode_info() print both the old and new revision in the same format as microcode_reload_late(). [ bp: Massage, remove useless comment. ] Signed-off-by: Ashok Raj Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Thomas Gleixner Link: https://lore.kernel.org/r/20230120161923.118882-6-ashok.raj@intel.com --- arch/x86/kernel/cpu/microcode/intel.c | 28 +++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c index 9dbd007219a1..467cf37ea90a 100644 --- a/arch/x86/kernel/cpu/microcode/intel.c +++ b/arch/x86/kernel/cpu/microcode/intel.c @@ -305,12 +305,10 @@ static bool load_builtin_intel_microcode(struct cpio_data *cp) return false; } -/* - * Print ucode update info. - */ -static void print_ucode_info(unsigned int new_rev, unsigned int date) +static void print_ucode_info(int old_rev, int new_rev, unsigned int date) { - pr_info_once("microcode updated early to revision 0x%x, date = %04x-%02x-%02x\n", + pr_info_once("updated early: 0x%x -> 0x%x, date = %04x-%02x-%02x\n", + old_rev, new_rev, date & 0xffff, date >> 24, @@ -321,6 +319,7 @@ static void print_ucode_info(unsigned int new_rev, unsigned int date) static int delay_ucode_info; static int current_mc_date; +static int early_old_rev; /* * Print early updated ucode info after printk works. This is delayed info dump. @@ -331,7 +330,7 @@ void show_ucode_info_early(void) if (delay_ucode_info) { intel_cpu_collect_info(&uci); - print_ucode_info(uci.cpu_sig.rev, current_mc_date); + print_ucode_info(early_old_rev, uci.cpu_sig.rev, current_mc_date); delay_ucode_info = 0; } } @@ -340,29 +339,32 @@ void show_ucode_info_early(void) * At this point, we can not call printk() yet. Delay printing microcode info in * show_ucode_info_early() until printk() works. */ -static void print_ucode(int new_rev, int date) +static void print_ucode(int old_rev, int new_rev, int date) { int *delay_ucode_info_p; int *current_mc_date_p; + int *early_old_rev_p; delay_ucode_info_p = (int *)__pa_nodebug(&delay_ucode_info); current_mc_date_p = (int *)__pa_nodebug(¤t_mc_date); + early_old_rev_p = (int *)__pa_nodebug(&early_old_rev); *delay_ucode_info_p = 1; *current_mc_date_p = date; + *early_old_rev_p = old_rev; } #else -static inline void print_ucode(int new_rev, int date) +static inline void print_ucode(int old_rev, int new_rev, int date) { - print_ucode_info(new_rev, date); + print_ucode_info(old_rev, new_rev, date); } #endif static int apply_microcode_early(struct ucode_cpu_info *uci, bool early) { struct microcode_intel *mc; - u32 rev; + u32 rev, old_rev; mc = uci->mc; if (!mc) @@ -379,6 +381,8 @@ static int apply_microcode_early(struct ucode_cpu_info *uci, bool early) return UCODE_OK; } + old_rev = rev; + /* * Writeback and invalidate caches before updating microcode to avoid * internal issues depending on what the microcode is updating. @@ -395,9 +399,9 @@ static int apply_microcode_early(struct ucode_cpu_info *uci, bool early) uci->cpu_sig.rev = rev; if (early) - print_ucode(uci->cpu_sig.rev, mc->hdr.date); + print_ucode(old_rev, uci->cpu_sig.rev, mc->hdr.date); else - print_ucode_info(uci->cpu_sig.rev, mc->hdr.date); + print_ucode_info(old_rev, uci->cpu_sig.rev, mc->hdr.date); return 0; } From fc3b618c87833e4d258b66acfca3557c49c4fe97 Mon Sep 17 00:00:00 2001 From: Babu Moger Date: Fri, 13 Jan 2023 09:20:27 -0600 Subject: [PATCH 25/94] x86/resctrl: Replace smp_call_function_many() with on_each_cpu_mask() on_each_cpu_mask() runs the function on each CPU specified by cpumask, which may include the local processor. Replace smp_call_function_many() with on_each_cpu_mask() to simplify the code. Signed-off-by: Babu Moger Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Reinette Chatre Link: https://lore.kernel.org/r/20230113152039.770054-2-babu.moger@amd.com --- arch/x86/kernel/cpu/resctrl/ctrlmondata.c | 11 +++------ arch/x86/kernel/cpu/resctrl/rdtgroup.c | 29 +++++++---------------- 2 files changed, 11 insertions(+), 29 deletions(-) diff --git a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c index 1df0e3262bca..7eece3d2d0c3 100644 --- a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c +++ b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c @@ -310,7 +310,6 @@ int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid) enum resctrl_conf_type t; cpumask_var_t cpu_mask; struct rdt_domain *d; - int cpu; u32 idx; if (!zalloc_cpumask_var(&cpu_mask, GFP_KERNEL)) @@ -341,13 +340,9 @@ int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid) if (cpumask_empty(cpu_mask)) goto done; - cpu = get_cpu(); - /* Update resource control msr on this CPU if it's in cpu_mask. */ - if (cpumask_test_cpu(cpu, cpu_mask)) - rdt_ctrl_update(&msr_param); - /* Update resource control msr on other CPUs. */ - smp_call_function_many(cpu_mask, rdt_ctrl_update, &msr_param, 1); - put_cpu(); + + /* Update resource control msr on all the CPUs. */ + on_each_cpu_mask(cpu_mask, rdt_ctrl_update, &msr_param, 1); done: free_cpumask_var(cpu_mask); diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c index 5993da21d822..d23fbc45ecd1 100644 --- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c +++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c @@ -325,12 +325,7 @@ static void update_cpu_closid_rmid(void *info) static void update_closid_rmid(const struct cpumask *cpu_mask, struct rdtgroup *r) { - int cpu = get_cpu(); - - if (cpumask_test_cpu(cpu, cpu_mask)) - update_cpu_closid_rmid(r); - smp_call_function_many(cpu_mask, update_cpu_closid_rmid, r, 1); - put_cpu(); + on_each_cpu_mask(cpu_mask, update_cpu_closid_rmid, r, 1); } static int cpus_mon_write(struct rdtgroup *rdtgrp, cpumask_var_t newmask, @@ -1866,13 +1861,9 @@ static int set_cache_qos_cfg(int level, bool enable) /* Pick one CPU from each domain instance to update MSR */ cpumask_set_cpu(cpumask_any(&d->cpu_mask), cpu_mask); } - cpu = get_cpu(); - /* Update QOS_CFG MSR on this cpu if it's in cpu_mask. */ - if (cpumask_test_cpu(cpu, cpu_mask)) - update(&enable); - /* Update QOS_CFG MSR on all other cpus in cpu_mask. */ - smp_call_function_many(cpu_mask, update, &enable, 1); - put_cpu(); + + /* Update QOS_CFG MSR on all the CPUs in cpu_mask */ + on_each_cpu_mask(cpu_mask, update, &enable, 1); free_cpumask_var(cpu_mask); @@ -2349,7 +2340,7 @@ static int reset_all_ctrls(struct rdt_resource *r) struct msr_param msr_param; cpumask_var_t cpu_mask; struct rdt_domain *d; - int i, cpu; + int i; if (!zalloc_cpumask_var(&cpu_mask, GFP_KERNEL)) return -ENOMEM; @@ -2370,13 +2361,9 @@ static int reset_all_ctrls(struct rdt_resource *r) for (i = 0; i < hw_res->num_closid; i++) hw_dom->ctrl_val[i] = r->default_ctrl; } - cpu = get_cpu(); - /* Update CBM on this cpu if it's in cpu_mask. */ - if (cpumask_test_cpu(cpu, cpu_mask)) - rdt_ctrl_update(&msr_param); - /* Update CBM on all other cpus in cpu_mask. */ - smp_call_function_many(cpu_mask, rdt_ctrl_update, &msr_param, 1); - put_cpu(); + + /* Update CBM on all the CPUs in cpu_mask */ + on_each_cpu_mask(cpu_mask, rdt_ctrl_update, &msr_param, 1); free_cpumask_var(cpu_mask); From f334f723a63cfc25789b1cdf70a08ffbaea4bf2e Mon Sep 17 00:00:00 2001 From: Babu Moger Date: Fri, 13 Jan 2023 09:20:28 -0600 Subject: [PATCH 26/94] x86/cpufeatures: Add Slow Memory Bandwidth Allocation feature flag Add the new AMD feature X86_FEATURE_SMBA. With it, the QOS enforcement policies can be applied to external slow memory connected to the host. QOS enforcement is accomplished by assigning a Class Of Service (COS) to a processor and specifying allocations or limits for that COS for each resource to be allocated. This feature is identified by the CPUID function 0x8000_0020_EBX_x0[2]: L3SBE - L3 external slow memory bandwidth enforcement. CXL.memory is the only supported "slow" memory device. With SMBA, the hardware enables bandwidth allocation on the slow memory devices. If there are multiple slow memory devices in the system, then the throttling logic groups all the slow sources together and applies the limit on them as a whole. The presence of the SMBA feature (with CXL.memory) is independent of whether slow memory device is actually present in the system. If there is no slow memory in the system, then setting a SMBA limit will have no impact on the performance of the system. Presence of CXL memory can be identified by the numactl command: $numactl -H available: 2 nodes (0-1) node 0 cpus: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 node 0 size: 63678 MB node 0 free: 59542 MB node 1 cpus: node 1 size: 16122 MB node 1 free: 15627 MB node distances: node 0 1 0: 10 50 1: 50 10 CPU list for CXL memory will be empty. The cpu-cxl node distance is greater than cpu-to-cpu distances. Node 1 has the CXL memory in this case. CXL memory can also be identified using ACPI SRAT table and memory maps. Feature description is available in the specification, "AMD64 Technology Platform Quality of Service Extensions, Revision: 1.03 Publication # 56375 Revision: 1.03 Issue Date: February 2022" at https://bugzilla.kernel.org/attachment.cgi?id=301365 See also https://www.amd.com/en/support/tech-docs/amd64-technology-platform-quality-service-extensions Signed-off-by: Babu Moger Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Reinette Chatre Link: https://lore.kernel.org/r/20230113152039.770054-3-babu.moger@amd.com --- arch/x86/include/asm/cpufeatures.h | 1 + arch/x86/kernel/cpu/scattered.c | 1 + 2 files changed, 2 insertions(+) diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 61012476d66e..3f5f64b8d23e 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -307,6 +307,7 @@ #define X86_FEATURE_SGX_EDECCSSA (11*32+18) /* "" SGX EDECCSSA user leaf function */ #define X86_FEATURE_CALL_DEPTH (11*32+19) /* "" Call depth tracking for RSB stuffing */ #define X86_FEATURE_MSR_TSX_CTRL (11*32+20) /* "" MSR IA32_TSX_CTRL (Intel) implemented */ +#define X86_FEATURE_SMBA (11*32+21) /* "" Slow Memory Bandwidth Allocation */ /* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */ #define X86_FEATURE_AVX_VNNI (12*32+ 4) /* AVX VNNI instructions */ diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c index f53944fb8f7f..d925753084fb 100644 --- a/arch/x86/kernel/cpu/scattered.c +++ b/arch/x86/kernel/cpu/scattered.c @@ -45,6 +45,7 @@ static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_CPB, CPUID_EDX, 9, 0x80000007, 0 }, { X86_FEATURE_PROC_FEEDBACK, CPUID_EDX, 11, 0x80000007, 0 }, { X86_FEATURE_MBA, CPUID_EBX, 6, 0x80000008, 0 }, + { X86_FEATURE_SMBA, CPUID_EBX, 2, 0x80000020, 0 }, { X86_FEATURE_PERFMON_V2, CPUID_EAX, 0, 0x80000022, 0 }, { X86_FEATURE_AMD_LBR_V2, CPUID_EAX, 1, 0x80000022, 0 }, { 0, 0, 0, 0, 0 } From a5b699665580725de8c0c01f2163a15af78b6866 Mon Sep 17 00:00:00 2001 From: Babu Moger Date: Fri, 13 Jan 2023 09:20:29 -0600 Subject: [PATCH 27/94] x86/resctrl: Add a new resource type RDT_RESOURCE_SMBA Add a new resource type RDT_RESOURCE_SMBA to handle the QoS enforcement policies on the external slow memory. Mostly initialization of the essentials. Setting fflags to RFTYPE_RES_MB configures the SMBA resource to have the same resctrl files as the existing MBA resource. The SMBA resource has identical properties to the existing MBA resource. These properties will be enumerated in an upcoming change and exposed via resctrl because of this flag. Signed-off-by: Babu Moger Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Reinette Chatre Link: https://lore.kernel.org/r/20230113152039.770054-4-babu.moger@amd.com --- arch/x86/kernel/cpu/resctrl/core.c | 12 ++++++++++++ arch/x86/kernel/cpu/resctrl/internal.h | 1 + 2 files changed, 13 insertions(+) diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c index c98e52ff5f20..f6af3ac1ef20 100644 --- a/arch/x86/kernel/cpu/resctrl/core.c +++ b/arch/x86/kernel/cpu/resctrl/core.c @@ -100,6 +100,18 @@ struct rdt_hw_resource rdt_resources_all[] = { .fflags = RFTYPE_RES_MB, }, }, + [RDT_RESOURCE_SMBA] = + { + .r_resctrl = { + .rid = RDT_RESOURCE_SMBA, + .name = "SMBA", + .cache_level = 3, + .domains = domain_init(RDT_RESOURCE_SMBA), + .parse_ctrlval = parse_bw, + .format_str = "%d=%*u", + .fflags = RFTYPE_RES_MB, + }, + }, }; /* diff --git a/arch/x86/kernel/cpu/resctrl/internal.h b/arch/x86/kernel/cpu/resctrl/internal.h index 5ebd28e6aa0c..fdbbf66312ec 100644 --- a/arch/x86/kernel/cpu/resctrl/internal.h +++ b/arch/x86/kernel/cpu/resctrl/internal.h @@ -409,6 +409,7 @@ enum resctrl_res_level { RDT_RESOURCE_L3, RDT_RESOURCE_L2, RDT_RESOURCE_MBA, + RDT_RESOURCE_SMBA, /* Must be the last */ RDT_NUM_RESOURCES, From 78335aac6156eadad0025ab34469e2adcc60218b Mon Sep 17 00:00:00 2001 From: Babu Moger Date: Fri, 13 Jan 2023 09:20:30 -0600 Subject: [PATCH 28/94] x86/cpufeatures: Add Bandwidth Monitoring Event Configuration feature flag Newer AMD processors support the new feature Bandwidth Monitoring Event Configuration (BMEC). The feature support is identified via CPUID Fn8000_0020_EBX_x0[3]: EVT_CFG - Bandwidth Monitoring Event Configuration (BMEC) The bandwidth monitoring events mbm_total_bytes and mbm_local_bytes are set to count all the total and local reads/writes, respectively. With the introduction of slow memory, the two counters are not enough to count all the different types of memory events. Therefore, BMEC provides the option to configure mbm_total_bytes and mbm_local_bytes to count the specific type of events. Each BMEC event has a configuration MSR which contains one field for each bandwidth type that can be used to configure the bandwidth event to track any combination of supported bandwidth types. The event will count requests from every bandwidth type bit that is set in the corresponding configuration register. Following are the types of events supported: ==== ======================================================== Bits Description ==== ======================================================== 6 Dirty Victims from the QOS domain to all types of memory 5 Reads to slow memory in the non-local NUMA domain 4 Reads to slow memory in the local NUMA domain 3 Non-temporal writes to non-local NUMA domain 2 Non-temporal writes to local NUMA domain 1 Reads to memory in the non-local NUMA domain 0 Reads to memory in the local NUMA domain ==== ======================================================== By default, the mbm_total_bytes configuration is set to 0x7F to count all the event types and the mbm_local_bytes configuration is set to 0x15 to count all the local memory events. Feature description is available in the specification, "AMD64 Technology Platform Quality of Service Extensions, Revision: 1.03 Publication" at https://bugzilla.kernel.org/attachment.cgi?id=301365 Signed-off-by: Babu Moger Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Reinette Chatre Link: https://lore.kernel.org/r/20230113152039.770054-5-babu.moger@amd.com --- arch/x86/include/asm/cpufeatures.h | 1 + arch/x86/kernel/cpu/cpuid-deps.c | 2 ++ arch/x86/kernel/cpu/scattered.c | 1 + 3 files changed, 4 insertions(+) diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 3f5f64b8d23e..f6a41ec42fd7 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -308,6 +308,7 @@ #define X86_FEATURE_CALL_DEPTH (11*32+19) /* "" Call depth tracking for RSB stuffing */ #define X86_FEATURE_MSR_TSX_CTRL (11*32+20) /* "" MSR IA32_TSX_CTRL (Intel) implemented */ #define X86_FEATURE_SMBA (11*32+21) /* "" Slow Memory Bandwidth Allocation */ +#define X86_FEATURE_BMEC (11*32+22) /* "" Bandwidth Monitoring Event Configuration */ /* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */ #define X86_FEATURE_AVX_VNNI (12*32+ 4) /* AVX VNNI instructions */ diff --git a/arch/x86/kernel/cpu/cpuid-deps.c b/arch/x86/kernel/cpu/cpuid-deps.c index d95221117129..f6748c8bd647 100644 --- a/arch/x86/kernel/cpu/cpuid-deps.c +++ b/arch/x86/kernel/cpu/cpuid-deps.c @@ -68,6 +68,8 @@ static const struct cpuid_dep cpuid_deps[] = { { X86_FEATURE_CQM_OCCUP_LLC, X86_FEATURE_CQM_LLC }, { X86_FEATURE_CQM_MBM_TOTAL, X86_FEATURE_CQM_LLC }, { X86_FEATURE_CQM_MBM_LOCAL, X86_FEATURE_CQM_LLC }, + { X86_FEATURE_BMEC, X86_FEATURE_CQM_MBM_TOTAL }, + { X86_FEATURE_BMEC, X86_FEATURE_CQM_MBM_LOCAL }, { X86_FEATURE_AVX512_BF16, X86_FEATURE_AVX512VL }, { X86_FEATURE_AVX512_FP16, X86_FEATURE_AVX512BW }, { X86_FEATURE_ENQCMD, X86_FEATURE_XSAVES }, diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c index d925753084fb..0dad49a09b7a 100644 --- a/arch/x86/kernel/cpu/scattered.c +++ b/arch/x86/kernel/cpu/scattered.c @@ -46,6 +46,7 @@ static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_PROC_FEEDBACK, CPUID_EDX, 11, 0x80000007, 0 }, { X86_FEATURE_MBA, CPUID_EBX, 6, 0x80000008, 0 }, { X86_FEATURE_SMBA, CPUID_EBX, 2, 0x80000020, 0 }, + { X86_FEATURE_BMEC, CPUID_EBX, 3, 0x80000020, 0 }, { X86_FEATURE_PERFMON_V2, CPUID_EAX, 0, 0x80000022, 0 }, { X86_FEATURE_AMD_LBR_V2, CPUID_EAX, 1, 0x80000022, 0 }, { 0, 0, 0, 0, 0 } From a76f65c89f928b87c49d88b7fd60cb4d7203ffff Mon Sep 17 00:00:00 2001 From: Babu Moger Date: Fri, 13 Jan 2023 09:20:31 -0600 Subject: [PATCH 29/94] x86/resctrl: Include new features in command line options Add the command line options to enable or disable the new resctrl features: smba: Slow Memory Bandwidth Allocation bmec: Bandwidth Monitor Event Configuration. Signed-off-by: Babu Moger Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Reinette Chatre Link: https://lore.kernel.org/r/20230113152039.770054-6-babu.moger@amd.com --- Documentation/admin-guide/kernel-parameters.txt | 2 +- arch/x86/kernel/cpu/resctrl/core.c | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 6cfa6e3996cf..0ee891133d76 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -5221,7 +5221,7 @@ rdt= [HW,X86,RDT] Turn on/off individual RDT features. List is: cmt, mbmtotal, mbmlocal, l3cat, l3cdp, l2cat, l2cdp, - mba. + mba, smba, bmec. E.g. to turn on cmt and turn off mba use: rdt=cmt,!mba diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c index f6af3ac1ef20..10a8c9d96f32 100644 --- a/arch/x86/kernel/cpu/resctrl/core.c +++ b/arch/x86/kernel/cpu/resctrl/core.c @@ -659,6 +659,8 @@ enum { RDT_FLAG_L2_CAT, RDT_FLAG_L2_CDP, RDT_FLAG_MBA, + RDT_FLAG_SMBA, + RDT_FLAG_BMEC, }; #define RDT_OPT(idx, n, f) \ @@ -682,6 +684,8 @@ static struct rdt_options rdt_options[] __initdata = { RDT_OPT(RDT_FLAG_L2_CAT, "l2cat", X86_FEATURE_CAT_L2), RDT_OPT(RDT_FLAG_L2_CDP, "l2cdp", X86_FEATURE_CDP_L2), RDT_OPT(RDT_FLAG_MBA, "mba", X86_FEATURE_MBA), + RDT_OPT(RDT_FLAG_SMBA, "smba", X86_FEATURE_SMBA), + RDT_OPT(RDT_FLAG_BMEC, "bmec", X86_FEATURE_BMEC), }; #define NUM_RDT_OPTIONS ARRAY_SIZE(rdt_options) From 5b6fac3fa44bafee12e0c3d1c5cbae6d058e9c98 Mon Sep 17 00:00:00 2001 From: Babu Moger Date: Fri, 13 Jan 2023 09:20:32 -0600 Subject: [PATCH 30/94] x86/resctrl: Detect and configure Slow Memory Bandwidth Allocation The QoS slow memory configuration details are available via CPUID_Fn80000020_EDX_x02. Detect the available details and initialize the rest to defaults. Signed-off-by: Babu Moger Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Reinette Chatre Link: https://lore.kernel.org/r/20230113152039.770054-7-babu.moger@amd.com --- arch/x86/include/asm/msr-index.h | 1 + arch/x86/kernel/cpu/resctrl/core.c | 36 +++++++++++++++++++++-- arch/x86/kernel/cpu/resctrl/ctrlmondata.c | 2 +- arch/x86/kernel/cpu/resctrl/rdtgroup.c | 8 +++-- 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 37ff47552bcb..e0a40027aa62 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -1061,6 +1061,7 @@ /* - AMD: */ #define MSR_IA32_MBA_BW_BASE 0xc0000200 +#define MSR_IA32_SMBA_BW_BASE 0xc0000280 /* MSR_IA32_VMX_MISC bits */ #define MSR_IA32_VMX_MISC_INTEL_PT (1ULL << 14) diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c index 10a8c9d96f32..b4fc851f6489 100644 --- a/arch/x86/kernel/cpu/resctrl/core.c +++ b/arch/x86/kernel/cpu/resctrl/core.c @@ -162,6 +162,13 @@ bool is_mba_sc(struct rdt_resource *r) if (!r) return rdt_resources_all[RDT_RESOURCE_MBA].r_resctrl.membw.mba_sc; + /* + * The software controller support is only applicable to MBA resource. + * Make sure to check for resource type. + */ + if (r->rid != RDT_RESOURCE_MBA) + return false; + return r->membw.mba_sc; } @@ -225,9 +232,15 @@ static bool __rdt_get_mem_config_amd(struct rdt_resource *r) struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); union cpuid_0x10_3_eax eax; union cpuid_0x10_x_edx edx; - u32 ebx, ecx; + u32 ebx, ecx, subleaf; - cpuid_count(0x80000020, 1, &eax.full, &ebx, &ecx, &edx.full); + /* + * Query CPUID_Fn80000020_EDX_x01 for MBA and + * CPUID_Fn80000020_EDX_x02 for SMBA + */ + subleaf = (r->rid == RDT_RESOURCE_SMBA) ? 2 : 1; + + cpuid_count(0x80000020, subleaf, &eax.full, &ebx, &ecx, &edx.full); hw_res->num_closid = edx.split.cos_max + 1; r->default_ctrl = MAX_MBA_BW_AMD; @@ -750,6 +763,19 @@ static __init bool get_mem_config(void) return false; } +static __init bool get_slow_mem_config(void) +{ + struct rdt_hw_resource *hw_res = &rdt_resources_all[RDT_RESOURCE_SMBA]; + + if (!rdt_cpu_has(X86_FEATURE_SMBA)) + return false; + + if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) + return __rdt_get_mem_config_amd(&hw_res->r_resctrl); + + return false; +} + static __init bool get_rdt_alloc_resources(void) { struct rdt_resource *r; @@ -780,6 +806,9 @@ static __init bool get_rdt_alloc_resources(void) if (get_mem_config()) ret = true; + if (get_slow_mem_config()) + ret = true; + return ret; } @@ -869,6 +898,9 @@ static __init void rdt_init_res_defs_amd(void) } else if (r->rid == RDT_RESOURCE_MBA) { hw_res->msr_base = MSR_IA32_MBA_BW_BASE; hw_res->msr_update = mba_wrmsr_amd; + } else if (r->rid == RDT_RESOURCE_SMBA) { + hw_res->msr_base = MSR_IA32_SMBA_BW_BASE; + hw_res->msr_update = mba_wrmsr_amd; } } } diff --git a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c index 7eece3d2d0c3..eb07d4435391 100644 --- a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c +++ b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c @@ -209,7 +209,7 @@ static int parse_line(char *line, struct resctrl_schema *s, unsigned long dom_id; if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP && - r->rid == RDT_RESOURCE_MBA) { + (r->rid == RDT_RESOURCE_MBA || r->rid == RDT_RESOURCE_SMBA)) { rdt_last_cmd_puts("Cannot pseudo-lock MBA resource\n"); return -EINVAL; } diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c index d23fbc45ecd1..75169680516b 100644 --- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c +++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c @@ -1210,7 +1210,7 @@ static bool rdtgroup_mode_test_exclusive(struct rdtgroup *rdtgrp) list_for_each_entry(s, &resctrl_schema_all, list) { r = s->res; - if (r->rid == RDT_RESOURCE_MBA) + if (r->rid == RDT_RESOURCE_MBA || r->rid == RDT_RESOURCE_SMBA) continue; has_cache = true; list_for_each_entry(d, &r->domains, list) { @@ -1399,7 +1399,8 @@ static int rdtgroup_size_show(struct kernfs_open_file *of, ctrl = resctrl_arch_get_config(r, d, closid, type); - if (r->rid == RDT_RESOURCE_MBA) + if (r->rid == RDT_RESOURCE_MBA || + r->rid == RDT_RESOURCE_SMBA) size = ctrl; else size = rdtgroup_cbm_to_size(r, d, ctrl); @@ -2842,7 +2843,8 @@ static int rdtgroup_init_alloc(struct rdtgroup *rdtgrp) list_for_each_entry(s, &resctrl_schema_all, list) { r = s->res; - if (r->rid == RDT_RESOURCE_MBA) { + if (r->rid == RDT_RESOURCE_MBA || + r->rid == RDT_RESOURCE_SMBA) { rdtgroup_init_mba(r, rdtgrp->closid); if (is_mba_sc(r)) continue; From bd334c86b5d70e5d1c6169991802e62c828d6f38 Mon Sep 17 00:00:00 2001 From: Babu Moger Date: Fri, 13 Jan 2023 09:20:33 -0600 Subject: [PATCH 31/94] x86/resctrl: Add __init attribute to rdt_get_mon_l3_config() In an upcoming change, rdt_get_mon_l3_config() needs to call rdt_cpu_has() to query the monitor related features. It cannot be called right now because rdt_cpu_has() has the __init attribute but rdt_get_mon_l3_config() doesn't. Add the __init attribute to rdt_get_mon_l3_config() that is only called by get_rdt_mon_resources() that already has the __init attribute. Also make rdt_cpu_has() available to by rdt_get_mon_l3_config() via the internal header file. Signed-off-by: Babu Moger Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Reinette Chatre Link: https://lore.kernel.org/r/20230113152039.770054-8-babu.moger@amd.com Signed-off-by: Borislav Petkov (AMD) --- arch/x86/kernel/cpu/resctrl/core.c | 2 +- arch/x86/kernel/cpu/resctrl/internal.h | 1 + arch/x86/kernel/cpu/resctrl/monitor.c | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c index b4fc851f6489..030d3b409768 100644 --- a/arch/x86/kernel/cpu/resctrl/core.c +++ b/arch/x86/kernel/cpu/resctrl/core.c @@ -728,7 +728,7 @@ static int __init set_rdt_options(char *str) } __setup("rdt", set_rdt_options); -static bool __init rdt_cpu_has(int flag) +bool __init rdt_cpu_has(int flag) { bool ret = boot_cpu_has(flag); struct rdt_options *o; diff --git a/arch/x86/kernel/cpu/resctrl/internal.h b/arch/x86/kernel/cpu/resctrl/internal.h index fdbbf66312ec..f16b8bc5448c 100644 --- a/arch/x86/kernel/cpu/resctrl/internal.h +++ b/arch/x86/kernel/cpu/resctrl/internal.h @@ -512,6 +512,7 @@ void closid_free(int closid); int alloc_rmid(void); void free_rmid(u32 rmid); int rdt_get_mon_l3_config(struct rdt_resource *r); +bool __init rdt_cpu_has(int flag); void mon_event_count(void *info); int rdtgroup_mondata_show(struct seq_file *m, void *arg); void mon_event_read(struct rmid_read *rr, struct rdt_resource *r, diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c index 77538abeb72a..5665ab9725ce 100644 --- a/arch/x86/kernel/cpu/resctrl/monitor.c +++ b/arch/x86/kernel/cpu/resctrl/monitor.c @@ -763,7 +763,7 @@ static void l3_mon_evt_init(struct rdt_resource *r) list_add_tail(&mbm_local_event.list, &r->evt_list); } -int rdt_get_mon_l3_config(struct rdt_resource *r) +int __init rdt_get_mon_l3_config(struct rdt_resource *r) { unsigned int mbm_offset = boot_cpu_data.x86_cache_mbm_width_offset; struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); From d507f83ced55eda848a3f397cf87b07ab11384b9 Mon Sep 17 00:00:00 2001 From: Babu Moger Date: Fri, 13 Jan 2023 09:20:34 -0600 Subject: [PATCH 32/94] x86/resctrl: Support monitor configuration Add a new field in struct mon_evt to support Bandwidth Monitoring Event Configuration (BMEC) and also update the "mon_features" display. The resctrl file "mon_features" will display the supported events and files that can be used to configure those events if monitor configuration is supported. Before the change: $ cat /sys/fs/resctrl/info/L3_MON/mon_features llc_occupancy mbm_total_bytes mbm_local_bytes After the change when BMEC is supported: $ cat /sys/fs/resctrl/info/L3_MON/mon_features llc_occupancy mbm_total_bytes mbm_total_bytes_config mbm_local_bytes mbm_local_bytes_config Signed-off-by: Babu Moger Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Reinette Chatre Link: https://lore.kernel.org/r/20230113152039.770054-9-babu.moger@amd.com --- arch/x86/kernel/cpu/resctrl/internal.h | 2 ++ arch/x86/kernel/cpu/resctrl/monitor.c | 7 +++++++ arch/x86/kernel/cpu/resctrl/rdtgroup.c | 5 ++++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/arch/x86/kernel/cpu/resctrl/internal.h b/arch/x86/kernel/cpu/resctrl/internal.h index f16b8bc5448c..0605b04f1b7a 100644 --- a/arch/x86/kernel/cpu/resctrl/internal.h +++ b/arch/x86/kernel/cpu/resctrl/internal.h @@ -52,11 +52,13 @@ DECLARE_STATIC_KEY_FALSE(rdt_mon_enable_key); * struct mon_evt - Entry in the event list of a resource * @evtid: event id * @name: name of the event + * @configurable: true if the event is configurable * @list: entry in &rdt_resource->evt_list */ struct mon_evt { enum resctrl_event_id evtid; char *name; + bool configurable; struct list_head list; }; diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c index 5665ab9725ce..cf7b6e59fa29 100644 --- a/arch/x86/kernel/cpu/resctrl/monitor.c +++ b/arch/x86/kernel/cpu/resctrl/monitor.c @@ -800,6 +800,13 @@ int __init rdt_get_mon_l3_config(struct rdt_resource *r) if (ret) return ret; + if (rdt_cpu_has(X86_FEATURE_BMEC)) { + if (rdt_cpu_has(X86_FEATURE_CQM_MBM_TOTAL)) + mbm_total_event.configurable = true; + if (rdt_cpu_has(X86_FEATURE_CQM_MBM_LOCAL)) + mbm_local_event.configurable = true; + } + l3_mon_evt_init(r); r->mon_capable = true; diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c index 75169680516b..bf0bd4378e13 100644 --- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c +++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c @@ -998,8 +998,11 @@ static int rdt_mon_features_show(struct kernfs_open_file *of, struct rdt_resource *r = of->kn->parent->priv; struct mon_evt *mevt; - list_for_each_entry(mevt, &r->evt_list, list) + list_for_each_entry(mevt, &r->evt_list, list) { seq_printf(seq, "%s\n", mevt->name); + if (mevt->configurable) + seq_printf(seq, "%s_config\n", mevt->name); + } return 0; } From dc2a3e857981f859889933cf66ded117d74edff1 Mon Sep 17 00:00:00 2001 From: Babu Moger Date: Fri, 13 Jan 2023 09:20:35 -0600 Subject: [PATCH 33/94] x86/resctrl: Add interface to read mbm_total_bytes_config The event configuration can be viewed by the user by reading the configuration file /sys/fs/resctrl/info/L3_MON/mbm_total_bytes_config. The event configuration settings are domain specific and will affect all the CPUs in the domain. Following are the types of events supported: ==== =========================================================== Bits Description ==== =========================================================== 6 Dirty Victims from the QOS domain to all types of memory 5 Reads to slow memory in the non-local NUMA domain 4 Reads to slow memory in the local NUMA domain 3 Non-temporal writes to non-local NUMA domain 2 Non-temporal writes to local NUMA domain 1 Reads to memory in the non-local NUMA domain 0 Reads to memory in the local NUMA domain ==== =========================================================== By default, the mbm_total_bytes_config is set to 0x7f to count all the event types. For example: $cat /sys/fs/resctrl/info/L3_MON/mbm_total_bytes_config 0=0x7f;1=0x7f;2=0x7f;3=0x7f In this case, the event mbm_total_bytes is configured with 0x7f on domains 0 to 3. Signed-off-by: Babu Moger Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Reinette Chatre Link: https://lore.kernel.org/r/20230113152039.770054-10-babu.moger@amd.com --- arch/x86/include/asm/msr-index.h | 1 + arch/x86/kernel/cpu/resctrl/internal.h | 24 ++++++ arch/x86/kernel/cpu/resctrl/monitor.c | 4 +- arch/x86/kernel/cpu/resctrl/rdtgroup.c | 102 +++++++++++++++++++++++++ 4 files changed, 130 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index e0a40027aa62..2e5f57c93605 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -1062,6 +1062,7 @@ /* - AMD: */ #define MSR_IA32_MBA_BW_BASE 0xc0000200 #define MSR_IA32_SMBA_BW_BASE 0xc0000280 +#define MSR_IA32_EVT_CFG_BASE 0xc0000400 /* MSR_IA32_VMX_MISC bits */ #define MSR_IA32_VMX_MISC_INTEL_PT (1ULL << 14) diff --git a/arch/x86/kernel/cpu/resctrl/internal.h b/arch/x86/kernel/cpu/resctrl/internal.h index 0605b04f1b7a..8edecc5763d8 100644 --- a/arch/x86/kernel/cpu/resctrl/internal.h +++ b/arch/x86/kernel/cpu/resctrl/internal.h @@ -30,6 +30,29 @@ */ #define MBM_CNTR_WIDTH_OFFSET_MAX (62 - MBM_CNTR_WIDTH_BASE) +/* Reads to Local DRAM Memory */ +#define READS_TO_LOCAL_MEM BIT(0) + +/* Reads to Remote DRAM Memory */ +#define READS_TO_REMOTE_MEM BIT(1) + +/* Non-Temporal Writes to Local Memory */ +#define NON_TEMP_WRITE_TO_LOCAL_MEM BIT(2) + +/* Non-Temporal Writes to Remote Memory */ +#define NON_TEMP_WRITE_TO_REMOTE_MEM BIT(3) + +/* Reads to Local Memory the system identifies as "Slow Memory" */ +#define READS_TO_LOCAL_S_MEM BIT(4) + +/* Reads to Remote Memory the system identifies as "Slow Memory" */ +#define READS_TO_REMOTE_S_MEM BIT(5) + +/* Dirty Victims to All Types of Memory */ +#define DIRTY_VICTIMS_TO_ALL_MEM BIT(6) + +/* Max event bits supported */ +#define MAX_EVT_CONFIG_BITS GENMASK(6, 0) struct rdt_fs_context { struct kernfs_fs_context kfc; @@ -531,5 +554,6 @@ bool has_busy_rmid(struct rdt_resource *r, struct rdt_domain *d); void __check_limbo(struct rdt_domain *d, bool force_free); void rdt_domain_reconfigure_cdp(struct rdt_resource *r); void __init thread_throttle_mode_init(void); +void __init mbm_config_rftype_init(const char *config); #endif /* _ASM_X86_RESCTRL_INTERNAL_H */ diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c index cf7b6e59fa29..0ff45e34d83c 100644 --- a/arch/x86/kernel/cpu/resctrl/monitor.c +++ b/arch/x86/kernel/cpu/resctrl/monitor.c @@ -801,8 +801,10 @@ int __init rdt_get_mon_l3_config(struct rdt_resource *r) return ret; if (rdt_cpu_has(X86_FEATURE_BMEC)) { - if (rdt_cpu_has(X86_FEATURE_CQM_MBM_TOTAL)) + if (rdt_cpu_has(X86_FEATURE_CQM_MBM_TOTAL)) { mbm_total_event.configurable = true; + mbm_config_rftype_init("mbm_total_bytes_config"); + } if (rdt_cpu_has(X86_FEATURE_CQM_MBM_LOCAL)) mbm_local_event.configurable = true; } diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c index bf0bd4378e13..cd4e668e5019 100644 --- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c +++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c @@ -1420,6 +1420,93 @@ out: return ret; } +struct mon_config_info { + u32 evtid; + u32 mon_config; +}; + +#define INVALID_CONFIG_INDEX UINT_MAX + +/** + * mon_event_config_index_get - get the hardware index for the + * configurable event + * @evtid: event id. + * + * Return: 0 for evtid == QOS_L3_MBM_TOTAL_EVENT_ID + * 1 for evtid == QOS_L3_MBM_LOCAL_EVENT_ID + * INVALID_CONFIG_INDEX for invalid evtid + */ +static inline unsigned int mon_event_config_index_get(u32 evtid) +{ + switch (evtid) { + case QOS_L3_MBM_TOTAL_EVENT_ID: + return 0; + case QOS_L3_MBM_LOCAL_EVENT_ID: + return 1; + default: + /* Should never reach here */ + return INVALID_CONFIG_INDEX; + } +} + +static void mon_event_config_read(void *info) +{ + struct mon_config_info *mon_info = info; + unsigned int index; + u32 h; + + index = mon_event_config_index_get(mon_info->evtid); + if (index == INVALID_CONFIG_INDEX) { + pr_warn_once("Invalid event id %d\n", mon_info->evtid); + return; + } + rdmsr(MSR_IA32_EVT_CFG_BASE + index, mon_info->mon_config, h); + + /* Report only the valid event configuration bits */ + mon_info->mon_config &= MAX_EVT_CONFIG_BITS; +} + +static void mondata_config_read(struct rdt_domain *d, struct mon_config_info *mon_info) +{ + smp_call_function_any(&d->cpu_mask, mon_event_config_read, mon_info, 1); +} + +static int mbm_config_show(struct seq_file *s, struct rdt_resource *r, u32 evtid) +{ + struct mon_config_info mon_info = {0}; + struct rdt_domain *dom; + bool sep = false; + + mutex_lock(&rdtgroup_mutex); + + list_for_each_entry(dom, &r->domains, list) { + if (sep) + seq_puts(s, ";"); + + memset(&mon_info, 0, sizeof(struct mon_config_info)); + mon_info.evtid = evtid; + mondata_config_read(dom, &mon_info); + + seq_printf(s, "%d=0x%02x", dom->id, mon_info.mon_config); + sep = true; + } + seq_puts(s, "\n"); + + mutex_unlock(&rdtgroup_mutex); + + return 0; +} + +static int mbm_total_bytes_config_show(struct kernfs_open_file *of, + struct seq_file *seq, void *v) +{ + struct rdt_resource *r = of->kn->parent->priv; + + mbm_config_show(seq, r, QOS_L3_MBM_TOTAL_EVENT_ID); + + return 0; +} + /* rdtgroup information files for one cache resource. */ static struct rftype res_common_files[] = { { @@ -1518,6 +1605,12 @@ static struct rftype res_common_files[] = { .seq_show = max_threshold_occ_show, .fflags = RF_MON_INFO | RFTYPE_RES_CACHE, }, + { + .name = "mbm_total_bytes_config", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = mbm_total_bytes_config_show, + }, { .name = "cpus", .mode = 0644, @@ -1624,6 +1717,15 @@ void __init thread_throttle_mode_init(void) rft->fflags = RF_CTRL_INFO | RFTYPE_RES_MB; } +void __init mbm_config_rftype_init(const char *config) +{ + struct rftype *rft; + + rft = rdtgroup_get_rftype_by_name(config); + if (rft) + rft->fflags = RF_MON_INFO | RFTYPE_RES_CACHE; +} + /** * rdtgroup_kn_mode_restrict - Restrict user access to named resctrl file * @r: The resource group with which the file is associated. From 73afb2d3ce2d7e0f9eee14e3f2b53d2bbfefd9c8 Mon Sep 17 00:00:00 2001 From: Babu Moger Date: Fri, 13 Jan 2023 09:20:36 -0600 Subject: [PATCH 34/94] x86/resctrl: Add interface to read mbm_local_bytes_config The event configuration can be viewed by the user by reading the configuration file /sys/fs/resctrl/info/L3_MON/mbm_local_bytes_config. The event configuration settings are domain specific and will affect all the CPUs in the domain. Following are the types of events supported: ==== =========================================================== Bits Description ==== =========================================================== 6 Dirty Victims from the QOS domain to all types of memory 5 Reads to slow memory in the non-local NUMA domain 4 Reads to slow memory in the local NUMA domain 3 Non-temporal writes to non-local NUMA domain 2 Non-temporal writes to local NUMA domain 1 Reads to memory in the non-local NUMA domain 0 Reads to memory in the local NUMA domain ==== =========================================================== By default, the mbm_local_bytes_config is set to 0x15 to count all the local event types. For example: $cat /sys/fs/resctrl/info/L3_MON/mbm_local_bytes_config 0=0x15;1=0x15;2=0x15;3=0x15 In this case, the event mbm_local_bytes is configured with 0x15 on domains 0 to 3. Signed-off-by: Babu Moger Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Reinette Chatre Link: https://lore.kernel.org/r/20230113152039.770054-11-babu.moger@amd.com --- arch/x86/kernel/cpu/resctrl/monitor.c | 4 +++- arch/x86/kernel/cpu/resctrl/rdtgroup.c | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c index 0ff45e34d83c..28c65008fce1 100644 --- a/arch/x86/kernel/cpu/resctrl/monitor.c +++ b/arch/x86/kernel/cpu/resctrl/monitor.c @@ -805,8 +805,10 @@ int __init rdt_get_mon_l3_config(struct rdt_resource *r) mbm_total_event.configurable = true; mbm_config_rftype_init("mbm_total_bytes_config"); } - if (rdt_cpu_has(X86_FEATURE_CQM_MBM_LOCAL)) + if (rdt_cpu_has(X86_FEATURE_CQM_MBM_LOCAL)) { mbm_local_event.configurable = true; + mbm_config_rftype_init("mbm_local_bytes_config"); + } } l3_mon_evt_init(r); diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c index cd4e668e5019..ad3c7014adf6 100644 --- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c +++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c @@ -1507,6 +1507,16 @@ static int mbm_total_bytes_config_show(struct kernfs_open_file *of, return 0; } +static int mbm_local_bytes_config_show(struct kernfs_open_file *of, + struct seq_file *seq, void *v) +{ + struct rdt_resource *r = of->kn->parent->priv; + + mbm_config_show(seq, r, QOS_L3_MBM_LOCAL_EVENT_ID); + + return 0; +} + /* rdtgroup information files for one cache resource. */ static struct rftype res_common_files[] = { { @@ -1611,6 +1621,12 @@ static struct rftype res_common_files[] = { .kf_ops = &rdtgroup_kf_single_ops, .seq_show = mbm_total_bytes_config_show, }, + { + .name = "mbm_local_bytes_config", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = mbm_local_bytes_config_show, + }, { .name = "cpus", .mode = 0644, From 92bd5a1390335bb3cc76bdf1b4356edbc94d408d Mon Sep 17 00:00:00 2001 From: Babu Moger Date: Fri, 13 Jan 2023 09:20:37 -0600 Subject: [PATCH 35/94] x86/resctrl: Add interface to write mbm_total_bytes_config The event configuration for mbm_total_bytes can be changed by the user by writing to the file /sys/fs/resctrl/info/L3_MON/mbm_total_bytes_config. The event configuration settings are domain specific and affect all the CPUs in the domain. Following are the types of events supported: ==== =========================================================== Bits Description ==== =========================================================== 6 Dirty Victims from the QOS domain to all types of memory 5 Reads to slow memory in the non-local NUMA domain 4 Reads to slow memory in the local NUMA domain 3 Non-temporal writes to non-local NUMA domain 2 Non-temporal writes to local NUMA domain 1 Reads to memory in the non-local NUMA domain 0 Reads to memory in the local NUMA domain ==== =========================================================== For example: To change the mbm_total_bytes to count only reads on domain 0, the bits 0, 1, 4 and 5 needs to be set, which is 110011b (in hex 0x33). Run the command: $echo 0=0x33 > /sys/fs/resctrl/info/L3_MON/mbm_total_bytes_config To change the mbm_total_bytes to count all the slow memory reads on domain 1, the bits 4 and 5 needs to be set which is 110000b (in hex 0x30). Run the command: $echo 1=0x30 > /sys/fs/resctrl/info/L3_MON/mbm_total_bytes_config Signed-off-by: Babu Moger Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Reinette Chatre Link: https://lore.kernel.org/r/20230113152039.770054-12-babu.moger@amd.com --- arch/x86/kernel/cpu/resctrl/monitor.c | 17 ++++ arch/x86/kernel/cpu/resctrl/rdtgroup.c | 124 ++++++++++++++++++++++++- include/linux/resctrl.h | 11 +++ 3 files changed, 151 insertions(+), 1 deletion(-) diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c index 28c65008fce1..7fe51488e136 100644 --- a/arch/x86/kernel/cpu/resctrl/monitor.c +++ b/arch/x86/kernel/cpu/resctrl/monitor.c @@ -204,6 +204,23 @@ void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_domain *d, } } +/* + * Assumes that hardware counters are also reset and thus that there is + * no need to record initial non-zero counts. + */ +void resctrl_arch_reset_rmid_all(struct rdt_resource *r, struct rdt_domain *d) +{ + struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d); + + if (is_mbm_total_enabled()) + memset(hw_dom->arch_mbm_total, 0, + sizeof(*hw_dom->arch_mbm_total) * r->num_rmid); + + if (is_mbm_local_enabled()) + memset(hw_dom->arch_mbm_local, 0, + sizeof(*hw_dom->arch_mbm_local) * r->num_rmid); +} + static u64 mbm_overflow_count(u64 prev_msr, u64 cur_msr, unsigned int width) { u64 shift = 64 - width, chunks; diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c index ad3c7014adf6..03284a61c1a0 100644 --- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c +++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c @@ -1517,6 +1517,127 @@ static int mbm_local_bytes_config_show(struct kernfs_open_file *of, return 0; } +static void mon_event_config_write(void *info) +{ + struct mon_config_info *mon_info = info; + unsigned int index; + + index = mon_event_config_index_get(mon_info->evtid); + if (index == INVALID_CONFIG_INDEX) { + pr_warn_once("Invalid event id %d\n", mon_info->evtid); + return; + } + wrmsr(MSR_IA32_EVT_CFG_BASE + index, mon_info->mon_config, 0); +} + +static int mbm_config_write_domain(struct rdt_resource *r, + struct rdt_domain *d, u32 evtid, u32 val) +{ + struct mon_config_info mon_info = {0}; + int ret = 0; + + /* mon_config cannot be more than the supported set of events */ + if (val > MAX_EVT_CONFIG_BITS) { + rdt_last_cmd_puts("Invalid event configuration\n"); + return -EINVAL; + } + + /* + * Read the current config value first. If both are the same then + * no need to write it again. + */ + mon_info.evtid = evtid; + mondata_config_read(d, &mon_info); + if (mon_info.mon_config == val) + goto out; + + mon_info.mon_config = val; + + /* + * Update MSR_IA32_EVT_CFG_BASE MSR on one of the CPUs in the + * domain. The MSRs offset from MSR MSR_IA32_EVT_CFG_BASE + * are scoped at the domain level. Writing any of these MSRs + * on one CPU is observed by all the CPUs in the domain. + */ + smp_call_function_any(&d->cpu_mask, mon_event_config_write, + &mon_info, 1); + + /* + * When an Event Configuration is changed, the bandwidth counters + * for all RMIDs and Events will be cleared by the hardware. The + * hardware also sets MSR_IA32_QM_CTR.Unavailable (bit 62) for + * every RMID on the next read to any event for every RMID. + * Subsequent reads will have MSR_IA32_QM_CTR.Unavailable (bit 62) + * cleared while it is tracked by the hardware. Clear the + * mbm_local and mbm_total counts for all the RMIDs. + */ + resctrl_arch_reset_rmid_all(r, d); + +out: + return ret; +} + +static int mon_config_write(struct rdt_resource *r, char *tok, u32 evtid) +{ + char *dom_str = NULL, *id_str; + unsigned long dom_id, val; + struct rdt_domain *d; + int ret = 0; + +next: + if (!tok || tok[0] == '\0') + return 0; + + /* Start processing the strings for each domain */ + dom_str = strim(strsep(&tok, ";")); + id_str = strsep(&dom_str, "="); + + if (!id_str || kstrtoul(id_str, 10, &dom_id)) { + rdt_last_cmd_puts("Missing '=' or non-numeric domain id\n"); + return -EINVAL; + } + + if (!dom_str || kstrtoul(dom_str, 16, &val)) { + rdt_last_cmd_puts("Non-numeric event configuration value\n"); + return -EINVAL; + } + + list_for_each_entry(d, &r->domains, list) { + if (d->id == dom_id) { + ret = mbm_config_write_domain(r, d, evtid, val); + if (ret) + return -EINVAL; + goto next; + } + } + + return -EINVAL; +} + +static ssize_t mbm_total_bytes_config_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, + loff_t off) +{ + struct rdt_resource *r = of->kn->parent->priv; + int ret; + + /* Valid input requires a trailing newline */ + if (nbytes == 0 || buf[nbytes - 1] != '\n') + return -EINVAL; + + mutex_lock(&rdtgroup_mutex); + + rdt_last_cmd_clear(); + + buf[nbytes - 1] = '\0'; + + ret = mon_config_write(r, buf, QOS_L3_MBM_TOTAL_EVENT_ID); + + mutex_unlock(&rdtgroup_mutex); + + return ret ?: nbytes; +} + /* rdtgroup information files for one cache resource. */ static struct rftype res_common_files[] = { { @@ -1617,9 +1738,10 @@ static struct rftype res_common_files[] = { }, { .name = "mbm_total_bytes_config", - .mode = 0444, + .mode = 0644, .kf_ops = &rdtgroup_kf_single_ops, .seq_show = mbm_total_bytes_config_show, + .write = mbm_total_bytes_config_write, }, { .name = "mbm_local_bytes_config", diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h index 0cee154abc9f..8334eeacfec5 100644 --- a/include/linux/resctrl.h +++ b/include/linux/resctrl.h @@ -250,6 +250,17 @@ int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain *d, void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_domain *d, u32 rmid, enum resctrl_event_id eventid); +/** + * resctrl_arch_reset_rmid_all() - Reset all private state associated with + * all rmids and eventids. + * @r: The resctrl resource. + * @d: The domain for which all architectural counter state will + * be cleared. + * + * This can be called from any CPU. + */ +void resctrl_arch_reset_rmid_all(struct rdt_resource *r, struct rdt_domain *d); + extern unsigned int resctrl_rmid_realloc_threshold; extern unsigned int resctrl_rmid_realloc_limit; From 4fe61bff5a4100e92f81427dab06b7f3a025f6a2 Mon Sep 17 00:00:00 2001 From: Babu Moger Date: Fri, 13 Jan 2023 09:20:38 -0600 Subject: [PATCH 36/94] x86/resctrl: Add interface to write mbm_local_bytes_config The event configuration for mbm_local_bytes can be changed by the user by writing to the configuration file /sys/fs/resctrl/info/L3_MON/mbm_local_bytes_config. The event configuration settings are domain specific and will affect all the CPUs in the domain. Following are the types of events supported: ==== =========================================================== Bits Description ==== =========================================================== 6 Dirty Victims from the QOS domain to all types of memory 5 Reads to slow memory in the non-local NUMA domain 4 Reads to slow memory in the local NUMA domain 3 Non-temporal writes to non-local NUMA domain 2 Non-temporal writes to local NUMA domain 1 Reads to memory in the non-local NUMA domain 0 Reads to memory in the local NUMA domain ==== =========================================================== For example, to change the mbm_local_bytes_config to count all the non-temporal writes on domain 0, the bits 2 and 3 needs to be set which is 1100b (in hex 0xc). Run the command: $echo 0=0xc > /sys/fs/resctrl/info/L3_MON/mbm_local_bytes_config To change the mbm_local_bytes to count only reads to local NUMA domain 1, the bit 0 needs to be set which 1b (in hex 0x1). Run the command: $echo 1=0x1 > /sys/fs/resctrl/info/L3_MON/mbm_local_bytes_config Signed-off-by: Babu Moger Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Reinette Chatre Link: https://lore.kernel.org/r/20230113152039.770054-13-babu.moger@amd.com --- arch/x86/kernel/cpu/resctrl/rdtgroup.c | 27 +++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c index 03284a61c1a0..5990589f8a21 100644 --- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c +++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c @@ -1638,6 +1638,30 @@ static ssize_t mbm_total_bytes_config_write(struct kernfs_open_file *of, return ret ?: nbytes; } +static ssize_t mbm_local_bytes_config_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, + loff_t off) +{ + struct rdt_resource *r = of->kn->parent->priv; + int ret; + + /* Valid input requires a trailing newline */ + if (nbytes == 0 || buf[nbytes - 1] != '\n') + return -EINVAL; + + mutex_lock(&rdtgroup_mutex); + + rdt_last_cmd_clear(); + + buf[nbytes - 1] = '\0'; + + ret = mon_config_write(r, buf, QOS_L3_MBM_LOCAL_EVENT_ID); + + mutex_unlock(&rdtgroup_mutex); + + return ret ?: nbytes; +} + /* rdtgroup information files for one cache resource. */ static struct rftype res_common_files[] = { { @@ -1745,9 +1769,10 @@ static struct rftype res_common_files[] = { }, { .name = "mbm_local_bytes_config", - .mode = 0444, + .mode = 0644, .kf_ops = &rdtgroup_kf_single_ops, .seq_show = mbm_local_bytes_config_show, + .write = mbm_local_bytes_config_write, }, { .name = "cpus", From 0a363fb23ee2f7beb08437ad7db86d195878d79f Mon Sep 17 00:00:00 2001 From: Babu Moger Date: Fri, 13 Jan 2023 09:20:39 -0600 Subject: [PATCH 37/94] Documentation/x86: Update resctrl.rst for new features Update the documentation for the new features: 1. Slow Memory Bandwidth allocation (SMBA). With this feature, the QOS enforcement policies can be applied to the external slow memory connected to the host. QOS enforcement is accomplished by assigning a Class Of Service (COS) to a processor and specifying allocations or limits for that COS for each resource to be allocated. 2. Bandwidth Monitoring Event Configuration (BMEC). The bandwidth monitoring events mbm_total_bytes and mbm_local_bytes are set to count all the total and local reads/writes respectively. With the introduction of slow memory, the two counters are not enough to count all the different types of memory events. With the feature BMEC, the users have the option to configure mbm_total_bytes and mbm_local_bytes to count the specific type of events. Also add configuration instructions with examples. Signed-off-by: Babu Moger Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Reinette Chatre Link: https://lore.kernel.org/r/20230113152039.770054-14-babu.moger@amd.com --- Documentation/x86/resctrl.rst | 147 +++++++++++++++++++++++++++++++++- 1 file changed, 145 insertions(+), 2 deletions(-) diff --git a/Documentation/x86/resctrl.rst b/Documentation/x86/resctrl.rst index 71a531061e4e..058257dc56c8 100644 --- a/Documentation/x86/resctrl.rst +++ b/Documentation/x86/resctrl.rst @@ -17,14 +17,21 @@ AMD refers to this feature as AMD Platform Quality of Service(AMD QoS). This feature is enabled by the CONFIG_X86_CPU_RESCTRL and the x86 /proc/cpuinfo flag bits: -============================================= ================================ +=============================================== ================================ RDT (Resource Director Technology) Allocation "rdt_a" CAT (Cache Allocation Technology) "cat_l3", "cat_l2" CDP (Code and Data Prioritization) "cdp_l3", "cdp_l2" CQM (Cache QoS Monitoring) "cqm_llc", "cqm_occup_llc" MBM (Memory Bandwidth Monitoring) "cqm_mbm_total", "cqm_mbm_local" MBA (Memory Bandwidth Allocation) "mba" -============================================= ================================ +SMBA (Slow Memory Bandwidth Allocation) "" +BMEC (Bandwidth Monitoring Event Configuration) "" +=============================================== ================================ + +Historically, new features were made visible by default in /proc/cpuinfo. This +resulted in the feature flags becoming hard to parse by humans. Adding a new +flag to /proc/cpuinfo should be avoided if user space can obtain information +about the feature from resctrl's info directory. To use the feature mount the file system:: @@ -161,6 +168,83 @@ with the following files: "mon_features": Lists the monitoring events if monitoring is enabled for the resource. + Example:: + + # cat /sys/fs/resctrl/info/L3_MON/mon_features + llc_occupancy + mbm_total_bytes + mbm_local_bytes + + If the system supports Bandwidth Monitoring Event + Configuration (BMEC), then the bandwidth events will + be configurable. The output will be:: + + # cat /sys/fs/resctrl/info/L3_MON/mon_features + llc_occupancy + mbm_total_bytes + mbm_total_bytes_config + mbm_local_bytes + mbm_local_bytes_config + +"mbm_total_bytes_config", "mbm_local_bytes_config": + Read/write files containing the configuration for the mbm_total_bytes + and mbm_local_bytes events, respectively, when the Bandwidth + Monitoring Event Configuration (BMEC) feature is supported. + The event configuration settings are domain specific and affect + all the CPUs in the domain. When either event configuration is + changed, the bandwidth counters for all RMIDs of both events + (mbm_total_bytes as well as mbm_local_bytes) are cleared for that + domain. The next read for every RMID will report "Unavailable" + and subsequent reads will report the valid value. + + Following are the types of events supported: + + ==== ======================================================== + Bits Description + ==== ======================================================== + 6 Dirty Victims from the QOS domain to all types of memory + 5 Reads to slow memory in the non-local NUMA domain + 4 Reads to slow memory in the local NUMA domain + 3 Non-temporal writes to non-local NUMA domain + 2 Non-temporal writes to local NUMA domain + 1 Reads to memory in the non-local NUMA domain + 0 Reads to memory in the local NUMA domain + ==== ======================================================== + + By default, the mbm_total_bytes configuration is set to 0x7f to count + all the event types and the mbm_local_bytes configuration is set to + 0x15 to count all the local memory events. + + Examples: + + * To view the current configuration:: + :: + + # cat /sys/fs/resctrl/info/L3_MON/mbm_total_bytes_config + 0=0x7f;1=0x7f;2=0x7f;3=0x7f + + # cat /sys/fs/resctrl/info/L3_MON/mbm_local_bytes_config + 0=0x15;1=0x15;3=0x15;4=0x15 + + * To change the mbm_total_bytes to count only reads on domain 0, + the bits 0, 1, 4 and 5 needs to be set, which is 110011b in binary + (in hexadecimal 0x33): + :: + + # echo "0=0x33" > /sys/fs/resctrl/info/L3_MON/mbm_total_bytes_config + + # cat /sys/fs/resctrl/info/L3_MON/mbm_total_bytes_config + 0=0x33;1=0x7f;2=0x7f;3=0x7f + + * To change the mbm_local_bytes to count all the slow memory reads on + domain 0 and 1, the bits 4 and 5 needs to be set, which is 110000b + in binary (in hexadecimal 0x30): + :: + + # echo "0=0x30;1=0x30" > /sys/fs/resctrl/info/L3_MON/mbm_local_bytes_config + + # cat /sys/fs/resctrl/info/L3_MON/mbm_local_bytes_config + 0=0x30;1=0x30;3=0x15;4=0x15 "max_threshold_occupancy": Read/write file provides the largest value (in @@ -464,6 +548,25 @@ Memory bandwidth domain is L3 cache. MB:=bw_MBps0;=bw_MBps1;... +Slow Memory Bandwidth Allocation (SMBA) +--------------------------------------- +AMD hardware supports Slow Memory Bandwidth Allocation (SMBA). +CXL.memory is the only supported "slow" memory device. With the +support of SMBA, the hardware enables bandwidth allocation on +the slow memory devices. If there are multiple such devices in +the system, the throttling logic groups all the slow sources +together and applies the limit on them as a whole. + +The presence of SMBA (with CXL.memory) is independent of slow memory +devices presence. If there are no such devices on the system, then +configuring SMBA will have no impact on the performance of the system. + +The bandwidth domain for slow memory is L3 cache. Its schemata file +is formatted as: +:: + + SMBA:=bandwidth0;=bandwidth1;... + Reading/writing the schemata file --------------------------------- Reading the schemata file will show the state of all resources @@ -479,6 +582,46 @@ which you wish to change. E.g. L3DATA:0=fffff;1=fffff;2=3c0;3=fffff L3CODE:0=fffff;1=fffff;2=fffff;3=fffff +Reading/writing the schemata file (on AMD systems) +-------------------------------------------------- +Reading the schemata file will show the current bandwidth limit on all +domains. The allocated resources are in multiples of one eighth GB/s. +When writing to the file, you need to specify what cache id you wish to +configure the bandwidth limit. + +For example, to allocate 2GB/s limit on the first cache id: + +:: + + # cat schemata + MB:0=2048;1=2048;2=2048;3=2048 + L3:0=ffff;1=ffff;2=ffff;3=ffff + + # echo "MB:1=16" > schemata + # cat schemata + MB:0=2048;1= 16;2=2048;3=2048 + L3:0=ffff;1=ffff;2=ffff;3=ffff + +Reading/writing the schemata file (on AMD systems) with SMBA feature +-------------------------------------------------------------------- +Reading and writing the schemata file is the same as without SMBA in +above section. + +For example, to allocate 8GB/s limit on the first cache id: + +:: + + # cat schemata + SMBA:0=2048;1=2048;2=2048;3=2048 + MB:0=2048;1=2048;2=2048;3=2048 + L3:0=ffff;1=ffff;2=ffff;3=ffff + + # echo "SMBA:1=64" > schemata + # cat schemata + SMBA:0=2048;1= 64;2=2048;3=2048 + MB:0=2048;1=2048;2=2048;3=2048 + L3:0=ffff;1=ffff;2=ffff;3=ffff + Cache Pseudo-Locking ==================== CAT enables a user to specify the amount of cache space that an From b7082cdfc464bf9231300605d03eebf943dda307 Mon Sep 17 00:00:00 2001 From: Feng Tang Date: Tue, 20 Dec 2022 16:25:12 +0800 Subject: [PATCH 38/94] clocksource: Suspend the watchdog temporarily when high read latency detected Bugs have been reported on 8 sockets x86 machines in which the TSC was wrongly disabled when the system is under heavy workload. [ 818.380354] clocksource: timekeeping watchdog on CPU336: hpet wd-wd read-back delay of 1203520ns [ 818.436160] clocksource: wd-tsc-wd read-back delay of 181880ns, clock-skew test skipped! [ 819.402962] clocksource: timekeeping watchdog on CPU338: hpet wd-wd read-back delay of 324000ns [ 819.448036] clocksource: wd-tsc-wd read-back delay of 337240ns, clock-skew test skipped! [ 819.880863] clocksource: timekeeping watchdog on CPU339: hpet read-back delay of 150280ns, attempt 3, marking unstable [ 819.936243] tsc: Marking TSC unstable due to clocksource watchdog [ 820.068173] TSC found unstable after boot, most likely due to broken BIOS. Use 'tsc=unstable'. [ 820.092382] sched_clock: Marking unstable (818769414384, 1195404998) [ 820.643627] clocksource: Checking clocksource tsc synchronization from CPU 267 to CPUs 0,4,25,70,126,430,557,564. [ 821.067990] clocksource: Switched to clocksource hpet This can be reproduced by running memory intensive 'stream' tests, or some of the stress-ng subcases such as 'ioport'. The reason for these issues is the when system is under heavy load, the read latency of the clocksources can be very high. Even lightweight TSC reads can show high latencies, and latencies are much worse for external clocksources such as HPET or the APIC PM timer. These latencies can result in false-positive clocksource-unstable determinations. These issues were initially reported by a customer running on a production system, and this problem was reproduced on several generations of Xeon servers, especially when running the stress-ng test. These Xeon servers were not production systems, but they did have the latest steppings and firmware. Given that the clocksource watchdog is a continual diagnostic check with frequency of twice a second, there is no need to rush it when the system is under heavy load. Therefore, when high clocksource read latencies are detected, suspend the watchdog timer for 5 minutes. Signed-off-by: Feng Tang Acked-by: Waiman Long Cc: John Stultz Cc: Thomas Gleixner Cc: Stephen Boyd Cc: Feng Tang Signed-off-by: Paul E. McKenney --- kernel/time/clocksource.c | 45 ++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index fc486cd97263..91836b727cef 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -387,6 +387,15 @@ void clocksource_verify_percpu(struct clocksource *cs) } EXPORT_SYMBOL_GPL(clocksource_verify_percpu); +static inline void clocksource_reset_watchdog(void) +{ + struct clocksource *cs; + + list_for_each_entry(cs, &watchdog_list, wd_list) + cs->flags &= ~CLOCK_SOURCE_WATCHDOG; +} + + static void clocksource_watchdog(struct timer_list *unused) { u64 csnow, wdnow, cslast, wdlast, delta; @@ -394,6 +403,7 @@ static void clocksource_watchdog(struct timer_list *unused) int64_t wd_nsec, cs_nsec; struct clocksource *cs; enum wd_read_status read_ret; + unsigned long extra_wait = 0; u32 md; spin_lock(&watchdog_lock); @@ -413,13 +423,30 @@ static void clocksource_watchdog(struct timer_list *unused) read_ret = cs_watchdog_read(cs, &csnow, &wdnow); - if (read_ret != WD_READ_SUCCESS) { - if (read_ret == WD_READ_UNSTABLE) - /* Clock readout unreliable, so give it up. */ - __clocksource_unstable(cs); + if (read_ret == WD_READ_UNSTABLE) { + /* Clock readout unreliable, so give it up. */ + __clocksource_unstable(cs); continue; } + /* + * When WD_READ_SKIP is returned, it means the system is likely + * under very heavy load, where the latency of reading + * watchdog/clocksource is very big, and affect the accuracy of + * watchdog check. So give system some space and suspend the + * watchdog check for 5 minutes. + */ + if (read_ret == WD_READ_SKIP) { + /* + * As the watchdog timer will be suspended, and + * cs->last could keep unchanged for 5 minutes, reset + * the counters. + */ + clocksource_reset_watchdog(); + extra_wait = HZ * 300; + break; + } + /* Clocksource initialized ? */ if (!(cs->flags & CLOCK_SOURCE_WATCHDOG) || atomic_read(&watchdog_reset_pending)) { @@ -523,7 +550,7 @@ static void clocksource_watchdog(struct timer_list *unused) * pair clocksource_stop_watchdog() clocksource_start_watchdog(). */ if (!timer_pending(&watchdog_timer)) { - watchdog_timer.expires += WATCHDOG_INTERVAL; + watchdog_timer.expires += WATCHDOG_INTERVAL + extra_wait; add_timer_on(&watchdog_timer, next_cpu); } out: @@ -548,14 +575,6 @@ static inline void clocksource_stop_watchdog(void) watchdog_running = 0; } -static inline void clocksource_reset_watchdog(void) -{ - struct clocksource *cs; - - list_for_each_entry(cs, &watchdog_list, wd_list) - cs->flags &= ~CLOCK_SOURCE_WATCHDOG; -} - static void clocksource_resume_watchdog(void) { atomic_inc(&watchdog_reset_pending); From 54628de6792bfa86a8d73520b6fa0029971f9fc6 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Tue, 24 Jan 2023 10:17:53 -0800 Subject: [PATCH 39/94] x86/Kconfig: Fix spellos & punctuation Fix spelling (reported by codespell) & punctuation in arch/x86/ Kconfig. Signed-off-by: Randy Dunlap Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20230124181753.19309-1-rdunlap@infradead.org --- arch/x86/Kconfig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 3604074a878b..a825bf031f49 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1502,7 +1502,7 @@ config X86_5LEVEL depends on X86_64 help 5-level paging enables access to larger address space: - upto 128 PiB of virtual address space and 4 PiB of + up to 128 PiB of virtual address space and 4 PiB of physical address space. It will be supported by future Intel CPUs. @@ -2609,8 +2609,8 @@ config CALL_THUNKS_DEBUG a noisy dmesg about callthunks generation and call patching for trouble shooting. The debug prints need to be enabled on the kernel command line with 'debug-callthunks'. - Only enable this, when you are debugging call thunks as this - creates a noticable runtime overhead. If unsure say N. + Only enable this when you are debugging call thunks as this + creates a noticeable runtime overhead. If unsure say N. config CPU_IBPB_ENTRY bool "Enable IBPB on kernel entry" From 4c382d723edce1b3c72b55b1b505cf5526a56afc Mon Sep 17 00:00:00 2001 From: Brian Gerst Date: Tue, 24 Jan 2023 13:40:19 -0500 Subject: [PATCH 40/94] x86/vdso: Move VDSO image init to vdso2c generated code Generate an init function for each VDSO image, replacing init_vdso() and sysenter_setup(). Signed-off-by: Brian Gerst Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20230124184019.26850-1-brgerst@gmail.com --- arch/x86/entry/vdso/vdso2c.h | 6 ++++++ arch/x86/entry/vdso/vdso32-setup.c | 9 --------- arch/x86/entry/vdso/vma.c | 19 ++++--------------- arch/x86/include/asm/processor.h | 1 - arch/x86/include/asm/vdso.h | 2 +- arch/x86/kernel/cpu/common.c | 1 - 6 files changed, 11 insertions(+), 27 deletions(-) diff --git a/arch/x86/entry/vdso/vdso2c.h b/arch/x86/entry/vdso/vdso2c.h index 5264daa8859f..67b3e37576a6 100644 --- a/arch/x86/entry/vdso/vdso2c.h +++ b/arch/x86/entry/vdso/vdso2c.h @@ -179,6 +179,7 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len, fprintf(outfile, "/* AUTOMATICALLY GENERATED -- DO NOT EDIT */\n\n"); fprintf(outfile, "#include \n"); + fprintf(outfile, "#include \n"); fprintf(outfile, "#include \n"); fprintf(outfile, "#include \n"); fprintf(outfile, "\n"); @@ -218,5 +219,10 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len, fprintf(outfile, "\t.sym_%s = %" PRIi64 ",\n", required_syms[i].name, (int64_t)syms[i]); } + fprintf(outfile, "};\n\n"); + fprintf(outfile, "static __init int init_%s(void) {\n", image_name); + fprintf(outfile, "\treturn init_vdso_image(&%s);\n", image_name); fprintf(outfile, "};\n"); + fprintf(outfile, "subsys_initcall(init_%s);\n", image_name); + } diff --git a/arch/x86/entry/vdso/vdso32-setup.c b/arch/x86/entry/vdso/vdso32-setup.c index 43842fade8fa..3b300a773c7e 100644 --- a/arch/x86/entry/vdso/vdso32-setup.c +++ b/arch/x86/entry/vdso/vdso32-setup.c @@ -51,17 +51,8 @@ __setup("vdso32=", vdso32_setup); __setup_param("vdso=", vdso_setup, vdso32_setup, 0); #endif -int __init sysenter_setup(void) -{ - init_vdso_image(&vdso_image_32); - - return 0; -} - #ifdef CONFIG_X86_64 -subsys_initcall(sysenter_setup); - #ifdef CONFIG_SYSCTL /* Register vsyscall32 into the ABI table */ #include diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c index b8f3f9b9e53c..2738eb28cb2e 100644 --- a/arch/x86/entry/vdso/vma.c +++ b/arch/x86/entry/vdso/vma.c @@ -44,13 +44,16 @@ unsigned int vclocks_used __read_mostly; unsigned int __read_mostly vdso64_enabled = 1; #endif -void __init init_vdso_image(const struct vdso_image *image) +int __init init_vdso_image(const struct vdso_image *image) { + BUILD_BUG_ON(VDSO_CLOCKMODE_MAX >= 32); BUG_ON(image->size % PAGE_SIZE != 0); apply_alternatives((struct alt_instr *)(image->data + image->alt), (struct alt_instr *)(image->data + image->alt + image->alt_len)); + + return 0; } static const struct vm_special_mapping vvar_mapping; @@ -418,18 +421,4 @@ static __init int vdso_setup(char *s) return 1; } __setup("vdso=", vdso_setup); - -static int __init init_vdso(void) -{ - BUILD_BUG_ON(VDSO_CLOCKMODE_MAX >= 32); - - init_vdso_image(&vdso_image_64); - -#ifdef CONFIG_X86_X32_ABI - init_vdso_image(&vdso_image_x32); -#endif - - return 0; -} -subsys_initcall(init_vdso); #endif /* CONFIG_X86_64 */ diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 4e35c66edeb7..2dd0c8a85e62 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -542,7 +542,6 @@ enum idle_boot_override {IDLE_NO_OVERRIDE=0, IDLE_HALT, IDLE_NOMWAIT, IDLE_POLL}; extern void enable_sep_cpu(void); -extern int sysenter_setup(void); /* Defined in head.S */ diff --git a/arch/x86/include/asm/vdso.h b/arch/x86/include/asm/vdso.h index 2963a2f5dbc4..d7f6592b74a9 100644 --- a/arch/x86/include/asm/vdso.h +++ b/arch/x86/include/asm/vdso.h @@ -45,7 +45,7 @@ extern const struct vdso_image vdso_image_x32; extern const struct vdso_image vdso_image_32; #endif -extern void __init init_vdso_image(const struct vdso_image *image); +extern int __init init_vdso_image(const struct vdso_image *image); extern int map_vdso_once(const struct vdso_image *image, unsigned long addr); diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 9cfca3d7d0e2..3a9043ef3d6b 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1953,7 +1953,6 @@ void __init identify_boot_cpu(void) if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT)) pr_info("CET detected: Indirect Branch Tracking enabled\n"); #ifdef CONFIG_X86_32 - sysenter_setup(); enable_sep_cpu(); #endif cpu_detect_tlb(&boot_cpu_data); From 6e8746cb735166eaf3ceb086b31bb0431f5e3532 Mon Sep 17 00:00:00 2001 From: Qiuxu Zhuo Date: Fri, 13 Jan 2023 11:27:58 +0800 Subject: [PATCH 41/94] EDAC/skx_common: Enable EDAC support for the "near" memory The current {skx,i10nm}_edac miss the EDAC support to decode errors from the 1st level memory (the fast "near" memory as cache) of the 2-level memory system. Introduce a helper function skx_error_in_mem() to check whether errors are from memory at the beginning of skx_mce_check_error(). As long as the errors are from memory (either the 1-level memory system or the 2-level memory system), decode the errors. Reported-and-tested-by: Youquan Song Signed-off-by: Qiuxu Zhuo Signed-off-by: Tony Luck Link: https://lore.kernel.org/all/20230113032802.41752-1-qiuxu.zhuo@intel.com --- drivers/edac/skx_common.c | 18 ++++++++++++------ drivers/edac/skx_common.h | 24 ++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/drivers/edac/skx_common.c b/drivers/edac/skx_common.c index f0f8e98f6efb..5fdcb6d5b96b 100644 --- a/drivers/edac/skx_common.c +++ b/drivers/edac/skx_common.c @@ -632,12 +632,18 @@ static bool skx_error_in_1st_level_mem(const struct mce *m) if (!skx_mem_cfg_2lm) return false; - errcode = GET_BITFIELD(m->status, 0, 15); + errcode = GET_BITFIELD(m->status, 0, 15) & MCACOD_MEM_ERR_MASK; - if ((errcode & 0xef80) != 0x280) - return false; + return errcode == MCACOD_EXT_MEM_ERR; +} - return true; +static bool skx_error_in_mem(const struct mce *m) +{ + u32 errcode; + + errcode = GET_BITFIELD(m->status, 0, 15) & MCACOD_MEM_ERR_MASK; + + return (errcode == MCACOD_MEM_CTL_ERR || errcode == MCACOD_EXT_MEM_ERR); } int skx_mce_check_error(struct notifier_block *nb, unsigned long val, @@ -651,8 +657,8 @@ int skx_mce_check_error(struct notifier_block *nb, unsigned long val, if (mce->kflags & MCE_HANDLED_CEC) return NOTIFY_DONE; - /* ignore unless this is memory related with an address */ - if ((mce->status & 0xefff) >> 7 != 1 || !(mce->status & MCI_STATUS_ADDRV)) + /* Ignore unless this is memory related with an address */ + if (!skx_error_in_mem(mce) || !(mce->status & MCI_STATUS_ADDRV)) return NOTIFY_DONE; memset(&res, 0, sizeof(res)); diff --git a/drivers/edac/skx_common.h b/drivers/edac/skx_common.h index 0cbadd3d2cd3..312032657264 100644 --- a/drivers/edac/skx_common.h +++ b/drivers/edac/skx_common.h @@ -56,6 +56,30 @@ #define MCI_MISC_ECC_MODE(m) (((m) >> 59) & 15) #define MCI_MISC_ECC_DDRT 8 /* read from DDRT */ +/* + * According to Intel Architecture spec vol 3B, + * Table 15-10 "IA32_MCi_Status [15:0] Compound Error Code Encoding" + * memory errors should fit one of these masks: + * 000f 0000 1mmm cccc (binary) + * 000f 0010 1mmm cccc (binary) [RAM used as cache] + * where: + * f = Correction Report Filtering Bit. If 1, subsequent errors + * won't be shown + * mmm = error type + * cccc = channel + */ +#define MCACOD_MEM_ERR_MASK 0xef80 +/* + * Errors from either the memory of the 1-level memory system or the + * 2nd level memory (the slow "far" memory) of the 2-level memory system. + */ +#define MCACOD_MEM_CTL_ERR 0x80 +/* + * Errors from the 1st level memory (the fast "near" memory as cache) + * of the 2-level memory system. + */ +#define MCACOD_EXT_MEM_ERR 0x280 + /* * Each cpu socket contains some pci devices that provide global * information, and also some that are local to each of the two From d2415e2e5330fc11f4c688fa518751bdc90259f5 Mon Sep 17 00:00:00 2001 From: Qiuxu Zhuo Date: Fri, 13 Jan 2023 11:27:59 +0800 Subject: [PATCH 42/94] EDAC/skx_common: Delete duplicated and unreachable code skx_mce_check_error() returns early if the error isn't from memory. So when skx_mce_output_error() is invoked from skx_mce_check_error(), it doesn't need to re-check whether the error is from memory. Delete the duplicated and unreachable code from skx_mce_output_error(). Signed-off-by: Qiuxu Zhuo Signed-off-by: Tony Luck Link: https://lore.kernel.org/all/20230113032802.41752-1-qiuxu.zhuo@intel.com --- drivers/edac/skx_common.c | 58 ++++++++++++++------------------------- 1 file changed, 21 insertions(+), 37 deletions(-) diff --git a/drivers/edac/skx_common.c b/drivers/edac/skx_common.c index 5fdcb6d5b96b..fd8186fe005c 100644 --- a/drivers/edac/skx_common.c +++ b/drivers/edac/skx_common.c @@ -560,44 +560,28 @@ static void skx_mce_output_error(struct mem_ctl_info *mci, tp_event = HW_EVENT_ERR_CORRECTED; } - /* - * According to Intel Architecture spec vol 3B, - * Table 15-10 "IA32_MCi_Status [15:0] Compound Error Code Encoding" - * memory errors should fit one of these masks: - * 000f 0000 1mmm cccc (binary) - * 000f 0010 1mmm cccc (binary) [RAM used as cache] - * where: - * f = Correction Report Filtering Bit. If 1, subsequent errors - * won't be shown - * mmm = error type - * cccc = channel - * If the mask doesn't match, report an error to the parsing logic - */ - if (!((errcode & 0xef80) == 0x80 || (errcode & 0xef80) == 0x280)) { - optype = "Can't parse: it is not a mem"; - } else { - switch (optypenum) { - case 0: - optype = "generic undef request error"; - break; - case 1: - optype = "memory read error"; - break; - case 2: - optype = "memory write error"; - break; - case 3: - optype = "addr/cmd error"; - break; - case 4: - optype = "memory scrubbing error"; - scrub_err = true; - break; - default: - optype = "reserved"; - break; - } + switch (optypenum) { + case 0: + optype = "generic undef request error"; + break; + case 1: + optype = "memory read error"; + break; + case 2: + optype = "memory write error"; + break; + case 3: + optype = "addr/cmd error"; + break; + case 4: + optype = "memory scrubbing error"; + scrub_err = true; + break; + default: + optype = "reserved"; + break; } + if (res->decoded_by_adxl) { len = snprintf(skx_msg, MSG_SIZE, "%s%s err_code:0x%04x:0x%04x %s", overflow ? " OVERFLOW" : "", From e4b2bc6616e21f4a7ce4e7452f716e3db8fe66b6 Mon Sep 17 00:00:00 2001 From: Qiuxu Zhuo Date: Fri, 13 Jan 2023 11:28:00 +0800 Subject: [PATCH 43/94] EDAC/i10nm: Add Intel Emerald Rapids server support The Emerald Rapids CPU model uses similar memory controller registers as Sapphire Rapids server. Add Emerald Rapids CPU model number ID for EDAC support. Tested-by: Li Zhang Signed-off-by: Qiuxu Zhuo Signed-off-by: Tony Luck Link: https://lore.kernel.org/all/20230113032802.41752-1-qiuxu.zhuo@intel.com --- drivers/edac/i10nm_base.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/edac/i10nm_base.c b/drivers/edac/i10nm_base.c index 65aeea53e2df..e11726f7fe36 100644 --- a/drivers/edac/i10nm_base.c +++ b/drivers/edac/i10nm_base.c @@ -637,6 +637,7 @@ static const struct x86_cpu_id i10nm_cpuids[] = { X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(ICELAKE_X, X86_STEPPINGS(0x4, 0xf), &i10nm_cfg1), X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(ICELAKE_D, X86_STEPPINGS(0x0, 0xf), &i10nm_cfg1), X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SAPPHIRERAPIDS_X, X86_STEPPINGS(0x0, 0xf), &spr_cfg), + X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(EMERALDRAPIDS_X, X86_STEPPINGS(0x0, 0xf), &spr_cfg), {} }; MODULE_DEVICE_TABLE(x86cpu, i10nm_cpuids); From dd7814b78539416c6e561eeaa0951b3e88ac799e Mon Sep 17 00:00:00 2001 From: Qiuxu Zhuo Date: Fri, 13 Jan 2023 11:28:01 +0800 Subject: [PATCH 44/94] EDAC/i10nm: Make more configurations CPU model specific The numbers of memory controllers per socket, channels per memory controller, DIMMs per channel and the triples of bus/device/function of PCI devices used in i10nm_edac can be CPU model specific. Add new fields to the structure res_config for above numbers and triples to make them CPU model specific. Signed-off-by: Qiuxu Zhuo Signed-off-by: Tony Luck Link: https://lore.kernel.org/all/20230113032802.41752-1-qiuxu.zhuo@intel.com --- drivers/edac/i10nm_base.c | 131 ++++++++++++++++++++++++++------------ drivers/edac/skx_common.h | 32 +++++++++- 2 files changed, 121 insertions(+), 42 deletions(-) diff --git a/drivers/edac/i10nm_base.c b/drivers/edac/i10nm_base.c index e11726f7fe36..5682d64cceb6 100644 --- a/drivers/edac/i10nm_base.c +++ b/drivers/edac/i10nm_base.c @@ -148,35 +148,47 @@ static void __enable_retry_rd_err_log(struct skx_imc *imc, int chan, bool enable static void enable_retry_rd_err_log(bool enable) { + int i, j, imc_num, chan_num; struct skx_imc *imc; struct skx_dev *d; - int i, j; edac_dbg(2, "\n"); - list_for_each_entry(d, i10nm_edac_list, list) - for (i = 0; i < I10NM_NUM_IMC; i++) { + list_for_each_entry(d, i10nm_edac_list, list) { + imc_num = res_cfg->ddr_imc_num; + chan_num = res_cfg->ddr_chan_num; + + for (i = 0; i < imc_num; i++) { imc = &d->imc[i]; if (!imc->mbase) continue; - for (j = 0; j < I10NM_NUM_CHANNELS; j++) { - if (imc->hbm_mc) { - __enable_retry_rd_err_log(imc, j, enable, - res_cfg->offsets_scrub_hbm0, - res_cfg->offsets_demand_hbm0, - NULL); - __enable_retry_rd_err_log(imc, j, enable, - res_cfg->offsets_scrub_hbm1, - res_cfg->offsets_demand_hbm1, - NULL); - } else { - __enable_retry_rd_err_log(imc, j, enable, - res_cfg->offsets_scrub, - res_cfg->offsets_demand, - res_cfg->offsets_demand2); - } + for (j = 0; j < chan_num; j++) + __enable_retry_rd_err_log(imc, j, enable, + res_cfg->offsets_scrub, + res_cfg->offsets_demand, + res_cfg->offsets_demand2); + } + + imc_num += res_cfg->hbm_imc_num; + chan_num = res_cfg->hbm_chan_num; + + for (; i < imc_num; i++) { + imc = &d->imc[i]; + if (!imc->mbase || !imc->hbm_mc) + continue; + + for (j = 0; j < chan_num; j++) { + __enable_retry_rd_err_log(imc, j, enable, + res_cfg->offsets_scrub_hbm0, + res_cfg->offsets_demand_hbm0, + NULL); + __enable_retry_rd_err_log(imc, j, enable, + res_cfg->offsets_scrub_hbm1, + res_cfg->offsets_demand_hbm1, + NULL); } + } } } @@ -318,9 +330,9 @@ static bool i10nm_check_2lm(struct res_config *cfg) int i; list_for_each_entry(d, i10nm_edac_list, list) { - d->sad_all = pci_get_dev_wrapper(d->seg, d->bus[1], - PCI_SLOT(cfg->sad_all_devfn), - PCI_FUNC(cfg->sad_all_devfn)); + d->sad_all = pci_get_dev_wrapper(d->seg, d->bus[res_cfg->sad_all_bdf.bus], + res_cfg->sad_all_bdf.dev, + res_cfg->sad_all_bdf.fun); if (!d->sad_all) continue; @@ -444,11 +456,15 @@ static int i10nm_get_ddr_munits(void) u64 base; list_for_each_entry(d, i10nm_edac_list, list) { - d->util_all = pci_get_dev_wrapper(d->seg, d->bus[1], 29, 1); + d->util_all = pci_get_dev_wrapper(d->seg, d->bus[res_cfg->util_all_bdf.bus], + res_cfg->util_all_bdf.dev, + res_cfg->util_all_bdf.fun); if (!d->util_all) return -ENODEV; - d->uracu = pci_get_dev_wrapper(d->seg, d->bus[0], 0, 1); + d->uracu = pci_get_dev_wrapper(d->seg, d->bus[res_cfg->uracu_bdf.bus], + res_cfg->uracu_bdf.dev, + res_cfg->uracu_bdf.fun); if (!d->uracu) return -ENODEV; @@ -461,9 +477,10 @@ static int i10nm_get_ddr_munits(void) edac_dbg(2, "socket%d mmio base 0x%llx (reg 0x%x)\n", j++, base, reg); - for (i = 0; i < I10NM_NUM_DDR_IMC; i++) { - mdev = pci_get_dev_wrapper(d->seg, d->bus[0], - 12 + i, 0); + for (i = 0; i < res_cfg->ddr_imc_num; i++) { + mdev = pci_get_dev_wrapper(d->seg, d->bus[res_cfg->ddr_mdev_bdf.bus], + res_cfg->ddr_mdev_bdf.dev + i, + res_cfg->ddr_mdev_bdf.fun); if (i == 0 && !mdev) { i10nm_printk(KERN_ERR, "No IMC found\n"); return -ENODEV; @@ -519,7 +536,9 @@ static int i10nm_get_hbm_munits(void) u64 base; list_for_each_entry(d, i10nm_edac_list, list) { - d->pcu_cr3 = pci_get_dev_wrapper(d->seg, d->bus[1], 30, 3); + d->pcu_cr3 = pci_get_dev_wrapper(d->seg, d->bus[res_cfg->pcu_cr3_bdf.bus], + res_cfg->pcu_cr3_bdf.dev, + res_cfg->pcu_cr3_bdf.fun); if (!d->pcu_cr3) return -ENODEV; @@ -540,11 +559,13 @@ static int i10nm_get_hbm_munits(void) } base += I10NM_GET_HBM_IMC_MMIO_OFFSET(reg); - lmc = I10NM_NUM_DDR_IMC; + lmc = res_cfg->ddr_imc_num; + + for (i = 0; i < res_cfg->hbm_imc_num; i++) { + mdev = pci_get_dev_wrapper(d->seg, d->bus[res_cfg->hbm_mdev_bdf.bus], + res_cfg->hbm_mdev_bdf.dev + i / 4, + res_cfg->hbm_mdev_bdf.fun + i % 4); - for (i = 0; i < I10NM_NUM_HBM_IMC; i++) { - mdev = pci_get_dev_wrapper(d->seg, d->bus[0], - 12 + i / 4, 1 + i % 4); if (i == 0 && !mdev) { i10nm_printk(KERN_ERR, "No hbm mc found\n"); return -ENODEV; @@ -594,8 +615,16 @@ static struct res_config i10nm_cfg0 = { .type = I10NM, .decs_did = 0x3452, .busno_cfg_offset = 0xcc, + .ddr_imc_num = 4, + .ddr_chan_num = 2, + .ddr_dimm_num = 2, .ddr_chan_mmio_sz = 0x4000, - .sad_all_devfn = PCI_DEVFN(29, 0), + .sad_all_bdf = {1, 29, 0}, + .pcu_cr3_bdf = {1, 30, 3}, + .util_all_bdf = {1, 29, 1}, + .uracu_bdf = {0, 0, 1}, + .ddr_mdev_bdf = {0, 12, 0}, + .hbm_mdev_bdf = {0, 12, 1}, .sad_all_offset = 0x108, .offsets_scrub = offsets_scrub_icx, .offsets_demand = offsets_demand_icx, @@ -605,8 +634,16 @@ static struct res_config i10nm_cfg1 = { .type = I10NM, .decs_did = 0x3452, .busno_cfg_offset = 0xd0, + .ddr_imc_num = 4, + .ddr_chan_num = 2, + .ddr_dimm_num = 2, .ddr_chan_mmio_sz = 0x4000, - .sad_all_devfn = PCI_DEVFN(29, 0), + .sad_all_bdf = {1, 29, 0}, + .pcu_cr3_bdf = {1, 30, 3}, + .util_all_bdf = {1, 29, 1}, + .uracu_bdf = {0, 0, 1}, + .ddr_mdev_bdf = {0, 12, 0}, + .hbm_mdev_bdf = {0, 12, 1}, .sad_all_offset = 0x108, .offsets_scrub = offsets_scrub_icx, .offsets_demand = offsets_demand_icx, @@ -616,10 +653,21 @@ static struct res_config spr_cfg = { .type = SPR, .decs_did = 0x3252, .busno_cfg_offset = 0xd0, + .ddr_imc_num = 4, + .ddr_chan_num = 2, + .ddr_dimm_num = 2, + .hbm_imc_num = 16, + .hbm_chan_num = 2, + .hbm_dimm_num = 1, .ddr_chan_mmio_sz = 0x8000, .hbm_chan_mmio_sz = 0x4000, .support_ddr5 = true, - .sad_all_devfn = PCI_DEVFN(10, 0), + .sad_all_bdf = {1, 10, 0}, + .pcu_cr3_bdf = {1, 30, 3}, + .util_all_bdf = {1, 29, 1}, + .uracu_bdf = {0, 0, 1}, + .ddr_mdev_bdf = {0, 12, 0}, + .hbm_mdev_bdf = {0, 12, 1}, .sad_all_offset = 0x300, .offsets_scrub = offsets_scrub_spr, .offsets_scrub_hbm0 = offsets_scrub_spr_hbm0, @@ -753,6 +801,7 @@ static int __init i10nm_init(void) struct skx_dev *d; int rc, i, off[3] = {0xd0, 0xc8, 0xcc}; u64 tolm, tohm; + int imc_num; edac_dbg(2, "\n"); @@ -793,6 +842,8 @@ static int __init i10nm_init(void) if (i10nm_get_hbm_munits() && rc) goto fail; + imc_num = res_cfg->ddr_imc_num + res_cfg->hbm_imc_num; + list_for_each_entry(d, i10nm_edac_list, list) { rc = skx_get_src_id(d, 0xf8, &src_id); if (rc < 0) @@ -803,7 +854,7 @@ static int __init i10nm_init(void) goto fail; edac_dbg(2, "src_id = %d node_id = %d\n", src_id, node_id); - for (i = 0; i < I10NM_NUM_IMC; i++) { + for (i = 0; i < imc_num; i++) { if (!d->imc[i].mdev) continue; @@ -813,12 +864,12 @@ static int __init i10nm_init(void) d->imc[i].node_id = node_id; if (d->imc[i].hbm_mc) { d->imc[i].chan_mmio_sz = cfg->hbm_chan_mmio_sz; - d->imc[i].num_channels = I10NM_NUM_HBM_CHANNELS; - d->imc[i].num_dimms = I10NM_NUM_HBM_DIMMS; + d->imc[i].num_channels = cfg->hbm_chan_num; + d->imc[i].num_dimms = cfg->hbm_dimm_num; } else { d->imc[i].chan_mmio_sz = cfg->ddr_chan_mmio_sz; - d->imc[i].num_channels = I10NM_NUM_DDR_CHANNELS; - d->imc[i].num_dimms = I10NM_NUM_DDR_DIMMS; + d->imc[i].num_channels = cfg->ddr_chan_num; + d->imc[i].num_dimms = cfg->ddr_dimm_num; } rc = skx_register_mci(&d->imc[i], d->imc[i].mdev, diff --git a/drivers/edac/skx_common.h b/drivers/edac/skx_common.h index 312032657264..982e1bcb1edf 100644 --- a/drivers/edac/skx_common.h +++ b/drivers/edac/skx_common.h @@ -173,19 +173,47 @@ struct decoded_addr { bool decoded_by_adxl; }; +struct pci_bdf { + u32 bus : 8; + u32 dev : 5; + u32 fun : 3; +}; + struct res_config { enum type type; /* Configuration agent device ID */ unsigned int decs_did; /* Default bus number configuration register offset */ int busno_cfg_offset; + /* DDR memory controllers per socket */ + int ddr_imc_num; + /* DDR channels per DDR memory controller */ + int ddr_chan_num; + /* DDR DIMMs per DDR memory channel */ + int ddr_dimm_num; /* Per DDR channel memory-mapped I/O size */ int ddr_chan_mmio_sz; + /* HBM memory controllers per socket */ + int hbm_imc_num; + /* HBM channels per HBM memory controller */ + int hbm_chan_num; + /* HBM DIMMs per HBM memory channel */ + int hbm_dimm_num; /* Per HBM channel memory-mapped I/O size */ int hbm_chan_mmio_sz; bool support_ddr5; - /* SAD device number and function number */ - unsigned int sad_all_devfn; + /* SAD device BDF */ + struct pci_bdf sad_all_bdf; + /* PCU device BDF */ + struct pci_bdf pcu_cr3_bdf; + /* UTIL device BDF */ + struct pci_bdf util_all_bdf; + /* URACU device BDF */ + struct pci_bdf uracu_bdf; + /* DDR mdev device BDF */ + struct pci_bdf ddr_mdev_bdf; + /* HBM mdev device BDF */ + struct pci_bdf hbm_mdev_bdf; int sad_all_offset; /* Offsets of retry_rd_err_log registers */ u32 *offsets_scrub; From ba987eaaabf99b462cdfed86274e3455d5126349 Mon Sep 17 00:00:00 2001 From: Qiuxu Zhuo Date: Fri, 13 Jan 2023 11:28:02 +0800 Subject: [PATCH 45/94] EDAC/i10nm: Add Intel Granite Rapids server support The Granite Rapids CPU model uses similar memory controller registers as Sapphire Rapids server but with some different configurations: - Various memory controller numbers for different Granite Rapids CPUs. So detect the number of present memory controllers at run time. - Different MMIO offsets of memory controllers. - Different triples of bus/dev/fun of some PCI devices used in i10nm_edac. Add above configurations and Granite Rapids CPU model ID for EDAC support. [Tony: Fixed 2 typos s/strcture/structure/] Signed-off-by: Qiuxu Zhuo Signed-off-by: Tony Luck Link: https://lore.kernel.org/all/20230113032802.41752-1-qiuxu.zhuo@intel.com --- drivers/edac/i10nm_base.c | 237 ++++++++++++++++++++++++++++++++++---- drivers/edac/skx_common.h | 5 +- 2 files changed, 217 insertions(+), 25 deletions(-) diff --git a/drivers/edac/i10nm_base.c b/drivers/edac/i10nm_base.c index 5682d64cceb6..46034310b78e 100644 --- a/drivers/edac/i10nm_base.c +++ b/drivers/edac/i10nm_base.c @@ -13,7 +13,7 @@ #include "edac_module.h" #include "skx_common.h" -#define I10NM_REVISION "v0.0.5" +#define I10NM_REVISION "v0.0.6" #define EDAC_MOD_STR "i10nm_edac" /* Debug macros */ @@ -22,25 +22,34 @@ #define I10NM_GET_SCK_BAR(d, reg) \ pci_read_config_dword((d)->uracu, 0xd0, &(reg)) -#define I10NM_GET_IMC_BAR(d, i, reg) \ - pci_read_config_dword((d)->uracu, 0xd8 + (i) * 4, &(reg)) +#define I10NM_GET_IMC_BAR(d, i, reg) \ + pci_read_config_dword((d)->uracu, \ + (res_cfg->type == GNR ? 0xd4 : 0xd8) + (i) * 4, &(reg)) #define I10NM_GET_SAD(d, offset, i, reg)\ - pci_read_config_dword((d)->sad_all, (offset) + (i) * 8, &(reg)) + pci_read_config_dword((d)->sad_all, (offset) + (i) * \ + (res_cfg->type == GNR ? 12 : 8), &(reg)) #define I10NM_GET_HBM_IMC_BAR(d, reg) \ pci_read_config_dword((d)->uracu, 0xd4, &(reg)) #define I10NM_GET_CAPID3_CFG(d, reg) \ - pci_read_config_dword((d)->pcu_cr3, 0x90, &(reg)) + pci_read_config_dword((d)->pcu_cr3, \ + res_cfg->type == GNR ? 0x290 : 0x90, &(reg)) +#define I10NM_GET_CAPID5_CFG(d, reg) \ + pci_read_config_dword((d)->pcu_cr3, \ + res_cfg->type == GNR ? 0x298 : 0x98, &(reg)) #define I10NM_GET_DIMMMTR(m, i, j) \ - readl((m)->mbase + ((m)->hbm_mc ? 0x80c : 0x2080c) + \ + readl((m)->mbase + ((m)->hbm_mc ? 0x80c : \ + (res_cfg->type == GNR ? 0xc0c : 0x2080c)) + \ (i) * (m)->chan_mmio_sz + (j) * 4) #define I10NM_GET_MCDDRTCFG(m, i) \ readl((m)->mbase + ((m)->hbm_mc ? 0x970 : 0x20970) + \ (i) * (m)->chan_mmio_sz) #define I10NM_GET_MCMTR(m, i) \ - readl((m)->mbase + ((m)->hbm_mc ? 0xef8 : 0x20ef8) + \ + readl((m)->mbase + ((m)->hbm_mc ? 0xef8 : \ + (res_cfg->type == GNR ? 0xaf8 : 0x20ef8)) + \ (i) * (m)->chan_mmio_sz) #define I10NM_GET_AMAP(m, i) \ - readl((m)->mbase + ((m)->hbm_mc ? 0x814 : 0x20814) + \ + readl((m)->mbase + ((m)->hbm_mc ? 0x814 : \ + (res_cfg->type == GNR ? 0xc14 : 0x20814)) + \ (i) * (m)->chan_mmio_sz) #define I10NM_GET_REG32(m, i, offset) \ readl((m)->mbase + (i) * (m)->chan_mmio_sz + (offset)) @@ -56,7 +65,10 @@ #define I10NM_GET_HBM_IMC_MMIO_OFFSET(reg) \ ((GET_BITFIELD(reg, 0, 10) << 12) + 0x140000) +#define I10NM_GNR_IMC_MMIO_OFFSET 0x24c000 +#define I10NM_GNR_IMC_MMIO_SIZE 0x4000 #define I10NM_HBM_IMC_MMIO_SIZE 0x9000 +#define I10NM_DDR_IMC_CH_CNT(reg) GET_BITFIELD(reg, 21, 24) #define I10NM_IS_HBM_PRESENT(reg) GET_BITFIELD(reg, 27, 30) #define I10NM_IS_HBM_IMC(reg) GET_BITFIELD(reg, 29, 29) @@ -323,6 +335,79 @@ static struct pci_dev *pci_get_dev_wrapper(int dom, unsigned int bus, return pdev; } +/** + * i10nm_get_imc_num() - Get the number of present DDR memory controllers. + * + * @cfg : The pointer to the structure of EDAC resource configurations. + * + * For Granite Rapids CPUs, the number of present DDR memory controllers read + * at runtime overwrites the value statically configured in @cfg->ddr_imc_num. + * For other CPUs, the number of present DDR memory controllers is statically + * configured in @cfg->ddr_imc_num. + * + * RETURNS : 0 on success, < 0 on failure. + */ +static int i10nm_get_imc_num(struct res_config *cfg) +{ + int n, imc_num, chan_num = 0; + struct skx_dev *d; + u32 reg; + + list_for_each_entry(d, i10nm_edac_list, list) { + d->pcu_cr3 = pci_get_dev_wrapper(d->seg, d->bus[res_cfg->pcu_cr3_bdf.bus], + res_cfg->pcu_cr3_bdf.dev, + res_cfg->pcu_cr3_bdf.fun); + if (!d->pcu_cr3) + continue; + + if (I10NM_GET_CAPID5_CFG(d, reg)) + continue; + + n = I10NM_DDR_IMC_CH_CNT(reg); + + if (!chan_num) { + chan_num = n; + edac_dbg(2, "Get DDR CH number: %d\n", chan_num); + } else if (chan_num != n) { + i10nm_printk(KERN_NOTICE, "Get DDR CH numbers: %d, %d\n", chan_num, n); + } + } + + switch (cfg->type) { + case GNR: + /* + * One channel per DDR memory controller for Granite Rapids CPUs. + */ + imc_num = chan_num; + + if (!imc_num) { + i10nm_printk(KERN_ERR, "Invalid DDR MC number\n"); + return -ENODEV; + } + + if (imc_num > I10NM_NUM_DDR_IMC) { + i10nm_printk(KERN_ERR, "Need to make I10NM_NUM_DDR_IMC >= %d\n", imc_num); + return -EINVAL; + } + + if (cfg->ddr_imc_num != imc_num) { + /* + * Store the number of present DDR memory controllers. + */ + cfg->ddr_imc_num = imc_num; + edac_dbg(2, "Set DDR MC number: %d", imc_num); + } + + return 0; + default: + /* + * For other CPUs, the number of present DDR memory controllers + * is statically pre-configured in cfg->ddr_imc_num. + */ + return 0; + } +} + static bool i10nm_check_2lm(struct res_config *cfg) { struct skx_dev *d; @@ -445,6 +530,98 @@ static bool i10nm_mc_decode(struct decoded_addr *res) return true; } +/** + * get_gnr_mdev() - Get the PCI device of the @logical_idx-th DDR memory controller. + * + * @d : The pointer to the structure of CPU socket EDAC device. + * @logical_idx : The logical index of the present memory controller (0 ~ max present MC# - 1). + * @physical_idx : To store the corresponding physical index of @logical_idx. + * + * RETURNS : The PCI device of the @logical_idx-th DDR memory controller, NULL on failure. + */ +static struct pci_dev *get_gnr_mdev(struct skx_dev *d, int logical_idx, int *physical_idx) +{ +#define GNR_MAX_IMC_PCI_CNT 28 + + struct pci_dev *mdev; + int i, logical = 0; + + /* + * Detect present memory controllers from { PCI device: 8-5, function 7-1 } + */ + for (i = 0; i < GNR_MAX_IMC_PCI_CNT; i++) { + mdev = pci_get_dev_wrapper(d->seg, + d->bus[res_cfg->ddr_mdev_bdf.bus], + res_cfg->ddr_mdev_bdf.dev + i / 7, + res_cfg->ddr_mdev_bdf.fun + i % 7); + + if (mdev) { + if (logical == logical_idx) { + *physical_idx = i; + return mdev; + } + + pci_dev_put(mdev); + logical++; + } + } + + return NULL; +} + +/** + * get_ddr_munit() - Get the resource of the i-th DDR memory controller. + * + * @d : The pointer to the structure of CPU socket EDAC device. + * @i : The index of the CPU socket relative DDR memory controller. + * @offset : To store the MMIO offset of the i-th DDR memory controller. + * @size : To store the MMIO size of the i-th DDR memory controller. + * + * RETURNS : The PCI device of the i-th DDR memory controller, NULL on failure. + */ +static struct pci_dev *get_ddr_munit(struct skx_dev *d, int i, u32 *offset, unsigned long *size) +{ + struct pci_dev *mdev; + int physical_idx; + u32 reg; + + switch (res_cfg->type) { + case GNR: + if (I10NM_GET_IMC_BAR(d, 0, reg)) { + i10nm_printk(KERN_ERR, "Failed to get mc0 bar\n"); + return NULL; + } + + mdev = get_gnr_mdev(d, i, &physical_idx); + if (!mdev) + return NULL; + + *offset = I10NM_GET_IMC_MMIO_OFFSET(reg) + + I10NM_GNR_IMC_MMIO_OFFSET + + physical_idx * I10NM_GNR_IMC_MMIO_SIZE; + *size = I10NM_GNR_IMC_MMIO_SIZE; + + break; + default: + if (I10NM_GET_IMC_BAR(d, i, reg)) { + i10nm_printk(KERN_ERR, "Failed to get mc%d bar\n", i); + return NULL; + } + + mdev = pci_get_dev_wrapper(d->seg, + d->bus[res_cfg->ddr_mdev_bdf.bus], + res_cfg->ddr_mdev_bdf.dev + i, + res_cfg->ddr_mdev_bdf.fun); + if (!mdev) + return NULL; + + *offset = I10NM_GET_IMC_MMIO_OFFSET(reg); + *size = I10NM_GET_IMC_MMIO_SIZE(reg); + } + + return mdev; +} + static int i10nm_get_ddr_munits(void) { struct pci_dev *mdev; @@ -478,9 +655,8 @@ static int i10nm_get_ddr_munits(void) j++, base, reg); for (i = 0; i < res_cfg->ddr_imc_num; i++) { - mdev = pci_get_dev_wrapper(d->seg, d->bus[res_cfg->ddr_mdev_bdf.bus], - res_cfg->ddr_mdev_bdf.dev + i, - res_cfg->ddr_mdev_bdf.fun); + mdev = get_ddr_munit(d, i, &off, &size); + if (i == 0 && !mdev) { i10nm_printk(KERN_ERR, "No IMC found\n"); return -ENODEV; @@ -490,13 +666,6 @@ static int i10nm_get_ddr_munits(void) d->imc[i].mdev = mdev; - if (I10NM_GET_IMC_BAR(d, i, reg)) { - i10nm_printk(KERN_ERR, "Failed to get mc bar\n"); - return -ENODEV; - } - - off = I10NM_GET_IMC_MMIO_OFFSET(reg); - size = I10NM_GET_IMC_MMIO_SIZE(reg); edac_dbg(2, "mc%d mmio base 0x%llx size 0x%lx (reg 0x%x)\n", i, base + off, size, reg); @@ -536,9 +705,6 @@ static int i10nm_get_hbm_munits(void) u64 base; list_for_each_entry(d, i10nm_edac_list, list) { - d->pcu_cr3 = pci_get_dev_wrapper(d->seg, d->bus[res_cfg->pcu_cr3_bdf.bus], - res_cfg->pcu_cr3_bdf.dev, - res_cfg->pcu_cr3_bdf.fun); if (!d->pcu_cr3) return -ENODEV; @@ -678,6 +844,23 @@ static struct res_config spr_cfg = { .offsets_demand_hbm1 = offsets_demand_spr_hbm1, }; +static struct res_config gnr_cfg = { + .type = GNR, + .decs_did = 0x3252, + .busno_cfg_offset = 0xd0, + .ddr_imc_num = 12, + .ddr_chan_num = 1, + .ddr_dimm_num = 2, + .ddr_chan_mmio_sz = 0x4000, + .support_ddr5 = true, + .sad_all_bdf = {0, 13, 0}, + .pcu_cr3_bdf = {0, 5, 0}, + .util_all_bdf = {0, 13, 1}, + .uracu_bdf = {0, 0, 1}, + .ddr_mdev_bdf = {0, 5, 1}, + .sad_all_offset = 0x300, +}; + static const struct x86_cpu_id i10nm_cpuids[] = { X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(ATOM_TREMONT_D, X86_STEPPINGS(0x0, 0x3), &i10nm_cfg0), X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(ATOM_TREMONT_D, X86_STEPPINGS(0x4, 0xf), &i10nm_cfg1), @@ -686,6 +869,7 @@ static const struct x86_cpu_id i10nm_cpuids[] = { X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(ICELAKE_D, X86_STEPPINGS(0x0, 0xf), &i10nm_cfg1), X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SAPPHIRERAPIDS_X, X86_STEPPINGS(0x0, 0xf), &spr_cfg), X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(EMERALDRAPIDS_X, X86_STEPPINGS(0x0, 0xf), &spr_cfg), + X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(GRANITERAPIDS_X, X86_STEPPINGS(0x0, 0xf), &gnr_cfg), {} }; MODULE_DEVICE_TABLE(x86cpu, i10nm_cpuids); @@ -705,7 +889,7 @@ static int i10nm_get_dimm_config(struct mem_ctl_info *mci, { struct skx_pvt *pvt = mci->pvt_info; struct skx_imc *imc = pvt->imc; - u32 mtr, amap, mcddrtcfg; + u32 mtr, amap, mcddrtcfg = 0; struct dimm_info *dimm; int i, j, ndimms; @@ -715,7 +899,10 @@ static int i10nm_get_dimm_config(struct mem_ctl_info *mci, ndimms = 0; amap = I10NM_GET_AMAP(imc, i); - mcddrtcfg = I10NM_GET_MCDDRTCFG(imc, i); + + if (res_cfg->type != GNR) + mcddrtcfg = I10NM_GET_MCDDRTCFG(imc, i); + for (j = 0; j < imc->num_dimms; j++) { dimm = edac_get_dimm(mci, i, j, 0); mtr = I10NM_GET_DIMMMTR(imc, i, j); @@ -834,6 +1021,10 @@ static int __init i10nm_init(void) return -ENODEV; } + rc = i10nm_get_imc_num(cfg); + if (rc < 0) + goto fail; + mem_cfg_2lm = i10nm_check_2lm(cfg); skx_set_mem_cfg(mem_cfg_2lm); diff --git a/drivers/edac/skx_common.h b/drivers/edac/skx_common.h index 982e1bcb1edf..b6d3607dffe2 100644 --- a/drivers/edac/skx_common.h +++ b/drivers/edac/skx_common.h @@ -33,7 +33,7 @@ #define SKX_NUM_CHANNELS 3 /* Channels per memory controller */ #define SKX_NUM_DIMMS 2 /* Max DIMMS per channel */ -#define I10NM_NUM_DDR_IMC 4 +#define I10NM_NUM_DDR_IMC 12 #define I10NM_NUM_DDR_CHANNELS 2 #define I10NM_NUM_DDR_DIMMS 2 @@ -129,7 +129,8 @@ struct skx_pvt { enum type { SKX, I10NM, - SPR + SPR, + GNR }; enum { From 793207bad71c5339c614d12ac21d627da7bf771d Mon Sep 17 00:00:00 2001 From: "Borislav Petkov (AMD)" Date: Tue, 24 Jan 2023 19:01:05 +0100 Subject: [PATCH 46/94] x86/resctrl: Fix a silly -Wunused-but-set-variable warning clang correctly complains arch/x86/kernel/cpu/resctrl/rdtgroup.c:1456:6: warning: variable \ 'h' set but not used [-Wunused-but-set-variable] u32 h; ^ but it can't know whether this use is innocuous or really a problem. There's a reason why those warning switches are behind a W=1 and not enabled by default - yes, one needs to do: make W=1 CC=clang HOSTCC=clang arch/x86/kernel/cpu/resctrl/ with clang 14 in order to trigger it. I would normally not take a silly fix like that but this one is simple and doesn't make the code uglier so... Reported-by: kernel test robot Signed-off-by: Borislav Petkov (AMD) Acked-by: Reinette Chatre Tested-by: Babu Moger Link: https://lore.kernel.org/r/202301242015.kbzkVteJ-lkp@intel.com --- arch/x86/kernel/cpu/resctrl/rdtgroup.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c index 5990589f8a21..e2c1599d1b37 100644 --- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c +++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c @@ -1453,17 +1453,17 @@ static void mon_event_config_read(void *info) { struct mon_config_info *mon_info = info; unsigned int index; - u32 h; + u64 msrval; index = mon_event_config_index_get(mon_info->evtid); if (index == INVALID_CONFIG_INDEX) { pr_warn_once("Invalid event id %d\n", mon_info->evtid); return; } - rdmsr(MSR_IA32_EVT_CFG_BASE + index, mon_info->mon_config, h); + rdmsrl(MSR_IA32_EVT_CFG_BASE + index, msrval); /* Report only the valid event configuration bits */ - mon_info->mon_config &= MAX_EVT_CONFIG_BITS; + mon_info->mon_config = msrval & MAX_EVT_CONFIG_BITS; } static void mondata_config_read(struct rdt_domain *d, struct mon_config_info *mon_info) From c14fd3dcacaa480394d3ac0b4a91a7d17a4b5516 Mon Sep 17 00:00:00 2001 From: Davidlohr Bueso Date: Mon, 23 Jan 2023 09:32:05 -0800 Subject: [PATCH 47/94] hrtimer: Rely on rt_task() for DL tasks too Checking dl_task() is redundant as rt_task() returns true for deadline tasks too. Signed-off-by: Davidlohr Bueso Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20230123173206.6764-2-dave@stgolabs.net --- kernel/time/hrtimer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index e4f0e3b0c4f4..667b713bab42 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -2089,7 +2089,7 @@ long hrtimer_nanosleep(ktime_t rqtp, const enum hrtimer_mode mode, u64 slack; slack = current->timer_slack_ns; - if (dl_task(current) || rt_task(current)) + if (rt_task(current)) slack = 0; hrtimer_init_sleeper_on_stack(&t, clockid, mode); From 0c52310f260014d95c1310364379772cb74cf82d Mon Sep 17 00:00:00 2001 From: Davidlohr Bueso Date: Mon, 23 Jan 2023 09:32:06 -0800 Subject: [PATCH 48/94] hrtimer: Ignore slack time for RT tasks in schedule_hrtimeout_range() While in theory the timer can be triggered before expires + delta, for the cases of RT tasks they really have no business giving any lenience for extra slack time, so override any passed value by the user and always use zero for schedule_hrtimeout_range() calls. Furthermore, this is similar to what the nanosleep(2) family already does with current->timer_slack_ns. Signed-off-by: Davidlohr Bueso Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20230123173206.6764-3-dave@stgolabs.net --- kernel/time/hrtimer.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 667b713bab42..e8c08292defc 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -2272,7 +2272,7 @@ void __init hrtimers_init(void) /** * schedule_hrtimeout_range_clock - sleep until timeout * @expires: timeout value (ktime_t) - * @delta: slack in expires timeout (ktime_t) + * @delta: slack in expires timeout (ktime_t) for SCHED_OTHER tasks * @mode: timer mode * @clock_id: timer clock to be used */ @@ -2299,6 +2299,13 @@ schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta, return -EINTR; } + /* + * Override any slack passed by the user if under + * rt contraints. + */ + if (rt_task(current)) + delta = 0; + hrtimer_init_sleeper_on_stack(&t, clock_id, mode); hrtimer_set_expires_range_ns(&t.timer, *expires, delta); hrtimer_sleeper_start_expires(&t, mode); @@ -2318,7 +2325,7 @@ EXPORT_SYMBOL_GPL(schedule_hrtimeout_range_clock); /** * schedule_hrtimeout_range - sleep until timeout * @expires: timeout value (ktime_t) - * @delta: slack in expires timeout (ktime_t) + * @delta: slack in expires timeout (ktime_t) for SCHED_OTHER tasks * @mode: timer mode * * Make the current task sleep until the given expiry time has @@ -2326,7 +2333,8 @@ EXPORT_SYMBOL_GPL(schedule_hrtimeout_range_clock); * the current task state has been set (see set_current_state()). * * The @delta argument gives the kernel the freedom to schedule the - * actual wakeup to a time that is both power and performance friendly. + * actual wakeup to a time that is both power and performance friendly + * for regular (non RT/DL) tasks. * The kernel give the normal best effort behavior for "@expires+@delta", * but may decide to fire the timer earlier, but no earlier than @expires. * From cbdb1f163af2bb90d01be1f0263df1d8d5c9d9d3 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 28 Nov 2022 16:10:03 +0200 Subject: [PATCH 49/94] vdso/bits.h: Add BIT_ULL() for the sake of consistency The minimization done in 3945ff37d2f4 ("linux/bits.h: Extract common header for vDSO") was required to isolate the VDSO build from the larger kernel header impact. The split added some inconsistency since BIT() and BIT_ULL() are now defined in the different files which confuses unprepared reader. Move BIT_ULL() to vdso/bits.h. No functional change. Signed-off-by: Andy Shevchenko Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20221128141003.77929-1-andriy.shevchenko@linux.intel.com --- include/linux/bits.h | 1 - include/vdso/bits.h | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/bits.h b/include/linux/bits.h index 87d112650dfb..7c0cf5031abe 100644 --- a/include/linux/bits.h +++ b/include/linux/bits.h @@ -6,7 +6,6 @@ #include #include -#define BIT_ULL(nr) (ULL(1) << (nr)) #define BIT_MASK(nr) (UL(1) << ((nr) % BITS_PER_LONG)) #define BIT_WORD(nr) ((nr) / BITS_PER_LONG) #define BIT_ULL_MASK(nr) (ULL(1) << ((nr) % BITS_PER_LONG_LONG)) diff --git a/include/vdso/bits.h b/include/vdso/bits.h index 6d005a1f5d94..388b212088ea 100644 --- a/include/vdso/bits.h +++ b/include/vdso/bits.h @@ -5,5 +5,6 @@ #include #define BIT(nr) (UL(1) << (nr)) +#define BIT_ULL(nr) (ULL(1) << (nr)) #endif /* __VDSO_BITS_H */ From db7adcfd1cec4e95155e37bc066fddab302c6340 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Mon, 23 Jan 2023 21:59:16 +0100 Subject: [PATCH 50/94] x86/alternatives: Introduce int3_emulate_jcc() Move the kprobe Jcc emulation into int3_emulate_jcc() so it can be used by more code -- specifically static_call() will need this. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Masami Hiramatsu (Google) Link: https://lore.kernel.org/r/20230123210607.057678245@infradead.org --- arch/x86/include/asm/text-patching.h | 31 +++++++++++++++++++++++ arch/x86/kernel/kprobes/core.c | 38 ++++++---------------------- 2 files changed, 39 insertions(+), 30 deletions(-) diff --git a/arch/x86/include/asm/text-patching.h b/arch/x86/include/asm/text-patching.h index f4b87f08f5c5..29832c338cdc 100644 --- a/arch/x86/include/asm/text-patching.h +++ b/arch/x86/include/asm/text-patching.h @@ -184,6 +184,37 @@ void int3_emulate_ret(struct pt_regs *regs) unsigned long ip = int3_emulate_pop(regs); int3_emulate_jmp(regs, ip); } + +static __always_inline +void int3_emulate_jcc(struct pt_regs *regs, u8 cc, unsigned long ip, unsigned long disp) +{ + static const unsigned long jcc_mask[6] = { + [0] = X86_EFLAGS_OF, + [1] = X86_EFLAGS_CF, + [2] = X86_EFLAGS_ZF, + [3] = X86_EFLAGS_CF | X86_EFLAGS_ZF, + [4] = X86_EFLAGS_SF, + [5] = X86_EFLAGS_PF, + }; + + bool invert = cc & 1; + bool match; + + if (cc < 0xc) { + match = regs->flags & jcc_mask[cc >> 1]; + } else { + match = ((regs->flags & X86_EFLAGS_SF) >> X86_EFLAGS_SF_BIT) ^ + ((regs->flags & X86_EFLAGS_OF) >> X86_EFLAGS_OF_BIT); + if (cc >= 0xe) + match = match || (regs->flags & X86_EFLAGS_ZF); + } + + if ((match && !invert) || (!match && invert)) + ip += disp; + + int3_emulate_jmp(regs, ip); +} + #endif /* !CONFIG_UML_X86 */ #endif /* _ASM_X86_TEXT_PATCHING_H */ diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c index 66299682b6b7..d48638cc71eb 100644 --- a/arch/x86/kernel/kprobes/core.c +++ b/arch/x86/kernel/kprobes/core.c @@ -460,50 +460,26 @@ static void kprobe_emulate_call(struct kprobe *p, struct pt_regs *regs) } NOKPROBE_SYMBOL(kprobe_emulate_call); -static nokprobe_inline -void __kprobe_emulate_jmp(struct kprobe *p, struct pt_regs *regs, bool cond) +static void kprobe_emulate_jmp(struct kprobe *p, struct pt_regs *regs) { unsigned long ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size; - if (cond) - ip += p->ainsn.rel32; + ip += p->ainsn.rel32; int3_emulate_jmp(regs, ip); } - -static void kprobe_emulate_jmp(struct kprobe *p, struct pt_regs *regs) -{ - __kprobe_emulate_jmp(p, regs, true); -} NOKPROBE_SYMBOL(kprobe_emulate_jmp); -static const unsigned long jcc_mask[6] = { - [0] = X86_EFLAGS_OF, - [1] = X86_EFLAGS_CF, - [2] = X86_EFLAGS_ZF, - [3] = X86_EFLAGS_CF | X86_EFLAGS_ZF, - [4] = X86_EFLAGS_SF, - [5] = X86_EFLAGS_PF, -}; - static void kprobe_emulate_jcc(struct kprobe *p, struct pt_regs *regs) { - bool invert = p->ainsn.jcc.type & 1; - bool match; + unsigned long ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size; - if (p->ainsn.jcc.type < 0xc) { - match = regs->flags & jcc_mask[p->ainsn.jcc.type >> 1]; - } else { - match = ((regs->flags & X86_EFLAGS_SF) >> X86_EFLAGS_SF_BIT) ^ - ((regs->flags & X86_EFLAGS_OF) >> X86_EFLAGS_OF_BIT); - if (p->ainsn.jcc.type >= 0xe) - match = match || (regs->flags & X86_EFLAGS_ZF); - } - __kprobe_emulate_jmp(p, regs, (match && !invert) || (!match && invert)); + int3_emulate_jcc(regs, p->ainsn.jcc.type, ip, p->ainsn.rel32); } NOKPROBE_SYMBOL(kprobe_emulate_jcc); static void kprobe_emulate_loop(struct kprobe *p, struct pt_regs *regs) { + unsigned long ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size; bool match; if (p->ainsn.loop.type != 3) { /* LOOP* */ @@ -531,7 +507,9 @@ static void kprobe_emulate_loop(struct kprobe *p, struct pt_regs *regs) else if (p->ainsn.loop.type == 1) /* LOOPE */ match = match && (regs->flags & X86_EFLAGS_ZF); - __kprobe_emulate_jmp(p, regs, match); + if (match) + ip += p->ainsn.rel32; + int3_emulate_jmp(regs, ip); } NOKPROBE_SYMBOL(kprobe_emulate_loop); From ac0ee0a9560c97fa5fe1409e450c2425d4ebd17a Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Mon, 23 Jan 2023 21:59:17 +0100 Subject: [PATCH 51/94] x86/alternatives: Teach text_poke_bp() to patch Jcc.d32 instructions In order to re-write Jcc.d32 instructions text_poke_bp() needs to be taught about them. The biggest hurdle is that the whole machinery is currently made for 5 byte instructions and extending this would grow struct text_poke_loc which is currently a nice 16 bytes and used in an array. However, since text_poke_loc contains a full copy of the (s32) displacement, it is possible to map the Jcc.d32 2 byte opcodes to Jcc.d8 1 byte opcode for the int3 emulation. This then leaves the replacement bytes; fudge that by only storing the last 5 bytes and adding the rule that 'length == 6' instruction will be prefixed with a 0x0f byte. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Masami Hiramatsu (Google) Link: https://lore.kernel.org/r/20230123210607.115718513@infradead.org --- arch/x86/kernel/alternative.c | 62 ++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 15 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 9a45aa1c775c..f615e0cb6d93 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -338,6 +338,12 @@ next: } } +static inline bool is_jcc32(struct insn *insn) +{ + /* Jcc.d32 second opcode byte is in the range: 0x80-0x8f */ + return insn->opcode.bytes[0] == 0x0f && (insn->opcode.bytes[1] & 0xf0) == 0x80; +} + #if defined(CONFIG_RETPOLINE) && defined(CONFIG_OBJTOOL) /* @@ -376,12 +382,6 @@ static int emit_indirect(int op, int reg, u8 *bytes) return i; } -static inline bool is_jcc32(struct insn *insn) -{ - /* Jcc.d32 second opcode byte is in the range: 0x80-0x8f */ - return insn->opcode.bytes[0] == 0x0f && (insn->opcode.bytes[1] & 0xf0) == 0x80; -} - static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 *bytes) { u8 op = insn->opcode.bytes[0]; @@ -1770,6 +1770,11 @@ void text_poke_sync(void) on_each_cpu(do_sync_core, NULL, 1); } +/* + * NOTE: crazy scheme to allow patching Jcc.d32 but not increase the size of + * this thing. When len == 6 everything is prefixed with 0x0f and we map + * opcode to Jcc.d8, using len to distinguish. + */ struct text_poke_loc { /* addr := _stext + rel_addr */ s32 rel_addr; @@ -1891,6 +1896,10 @@ noinstr int poke_int3_handler(struct pt_regs *regs) int3_emulate_jmp(regs, (long)ip + tp->disp); break; + case 0x70 ... 0x7f: /* Jcc */ + int3_emulate_jcc(regs, tp->opcode & 0xf, (long)ip, tp->disp); + break; + default: BUG(); } @@ -1964,16 +1973,26 @@ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries * Second step: update all but the first byte of the patched range. */ for (do_sync = 0, i = 0; i < nr_entries; i++) { - u8 old[POKE_MAX_OPCODE_SIZE] = { tp[i].old, }; + u8 old[POKE_MAX_OPCODE_SIZE+1] = { tp[i].old, }; + u8 _new[POKE_MAX_OPCODE_SIZE+1]; + const u8 *new = tp[i].text; int len = tp[i].len; if (len - INT3_INSN_SIZE > 0) { memcpy(old + INT3_INSN_SIZE, text_poke_addr(&tp[i]) + INT3_INSN_SIZE, len - INT3_INSN_SIZE); + + if (len == 6) { + _new[0] = 0x0f; + memcpy(_new + 1, new, 5); + new = _new; + } + text_poke(text_poke_addr(&tp[i]) + INT3_INSN_SIZE, - (const char *)tp[i].text + INT3_INSN_SIZE, + new + INT3_INSN_SIZE, len - INT3_INSN_SIZE); + do_sync++; } @@ -2001,8 +2020,7 @@ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries * The old instruction is recorded so that the event can be * processed forwards or backwards. */ - perf_event_text_poke(text_poke_addr(&tp[i]), old, len, - tp[i].text, len); + perf_event_text_poke(text_poke_addr(&tp[i]), old, len, new, len); } if (do_sync) { @@ -2019,10 +2037,15 @@ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries * replacing opcode. */ for (do_sync = 0, i = 0; i < nr_entries; i++) { - if (tp[i].text[0] == INT3_INSN_OPCODE) + u8 byte = tp[i].text[0]; + + if (tp[i].len == 6) + byte = 0x0f; + + if (byte == INT3_INSN_OPCODE) continue; - text_poke(text_poke_addr(&tp[i]), tp[i].text, INT3_INSN_SIZE); + text_poke(text_poke_addr(&tp[i]), &byte, INT3_INSN_SIZE); do_sync++; } @@ -2040,9 +2063,11 @@ static void text_poke_loc_init(struct text_poke_loc *tp, void *addr, const void *opcode, size_t len, const void *emulate) { struct insn insn; - int ret, i; + int ret, i = 0; - memcpy((void *)tp->text, opcode, len); + if (len == 6) + i = 1; + memcpy((void *)tp->text, opcode+i, len-i); if (!emulate) emulate = opcode; @@ -2053,6 +2078,13 @@ static void text_poke_loc_init(struct text_poke_loc *tp, void *addr, tp->len = len; tp->opcode = insn.opcode.bytes[0]; + if (is_jcc32(&insn)) { + /* + * Map Jcc.d32 onto Jcc.d8 and use len to distinguish. + */ + tp->opcode = insn.opcode.bytes[1] - 0x10; + } + switch (tp->opcode) { case RET_INSN_OPCODE: case JMP32_INSN_OPCODE: @@ -2069,7 +2101,6 @@ static void text_poke_loc_init(struct text_poke_loc *tp, void *addr, BUG_ON(len != insn.length); } - switch (tp->opcode) { case INT3_INSN_OPCODE: case RET_INSN_OPCODE: @@ -2078,6 +2109,7 @@ static void text_poke_loc_init(struct text_poke_loc *tp, void *addr, case CALL_INSN_OPCODE: case JMP32_INSN_OPCODE: case JMP8_INSN_OPCODE: + case 0x70 ... 0x7f: /* Jcc */ tp->disp = insn.immediate.value; break; From 923510c88d2b7d947c4217835fd9ca6bd65cc56c Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Thu, 26 Jan 2023 16:34:27 +0100 Subject: [PATCH 52/94] x86/static_call: Add support for Jcc tail-calls Clang likes to create conditional tail calls like: 0000000000000350 : 350: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 351: R_X86_64_NONE __fentry__-0x4 355: 48 83 bf 20 01 00 00 00 cmpq $0x0,0x120(%rdi) 35d: 0f 85 00 00 00 00 jne 363 35f: R_X86_64_PLT32 __SCT__amd_pmu_branch_add-0x4 363: e9 00 00 00 00 jmp 368 364: R_X86_64_PLT32 __x86_return_thunk-0x4 Where 0x35d is a static call site that's turned into a conditional tail-call using the Jcc class of instructions. Teach the in-line static call text patching about this. Notably, since there is no conditional-ret, in that case patch the Jcc to point at an empty stub function that does the ret -- or the return thunk when needed. Reported-by: "Erhard F." Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Masami Hiramatsu (Google) Link: https://lore.kernel.org/r/Y9Kdg9QjHkr9G5b5@hirez.programming.kicks-ass.net --- arch/x86/kernel/static_call.c | 50 ++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/static_call.c b/arch/x86/kernel/static_call.c index 2ebc338980bc..b70670a98597 100644 --- a/arch/x86/kernel/static_call.c +++ b/arch/x86/kernel/static_call.c @@ -9,6 +9,7 @@ enum insn_type { NOP = 1, /* site cond-call */ JMP = 2, /* tramp / site tail-call */ RET = 3, /* tramp / site cond-tail-call */ + JCC = 4, }; /* @@ -25,12 +26,40 @@ static const u8 xor5rax[] = { 0x2e, 0x2e, 0x2e, 0x31, 0xc0 }; static const u8 retinsn[] = { RET_INSN_OPCODE, 0xcc, 0xcc, 0xcc, 0xcc }; +static u8 __is_Jcc(u8 *insn) /* Jcc.d32 */ +{ + u8 ret = 0; + + if (insn[0] == 0x0f) { + u8 tmp = insn[1]; + if ((tmp & 0xf0) == 0x80) + ret = tmp; + } + + return ret; +} + +extern void __static_call_return(void); + +asm (".global __static_call_return\n\t" + ".type __static_call_return, @function\n\t" + ASM_FUNC_ALIGN "\n\t" + "__static_call_return:\n\t" + ANNOTATE_NOENDBR + ANNOTATE_RETPOLINE_SAFE + "ret; int3\n\t" + ".size __static_call_return, . - __static_call_return \n\t"); + static void __ref __static_call_transform(void *insn, enum insn_type type, void *func, bool modinit) { const void *emulate = NULL; int size = CALL_INSN_SIZE; const void *code; + u8 op, buf[6]; + + if ((type == JMP || type == RET) && (op = __is_Jcc(insn))) + type = JCC; switch (type) { case CALL: @@ -57,6 +86,20 @@ static void __ref __static_call_transform(void *insn, enum insn_type type, else code = &retinsn; break; + + case JCC: + if (!func) { + func = __static_call_return; + if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) + func = x86_return_thunk; + } + + buf[0] = 0x0f; + __text_gen_insn(buf+1, op, insn+1, func, 5); + code = buf; + size = 6; + + break; } if (memcmp(insn, code, size) == 0) @@ -68,9 +111,9 @@ static void __ref __static_call_transform(void *insn, enum insn_type type, text_poke_bp(insn, code, size, emulate); } -static void __static_call_validate(void *insn, bool tail, bool tramp) +static void __static_call_validate(u8 *insn, bool tail, bool tramp) { - u8 opcode = *(u8 *)insn; + u8 opcode = insn[0]; if (tramp && memcmp(insn+5, tramp_ud, 3)) { pr_err("trampoline signature fail"); @@ -79,7 +122,8 @@ static void __static_call_validate(void *insn, bool tail, bool tramp) if (tail) { if (opcode == JMP32_INSN_OPCODE || - opcode == RET_INSN_OPCODE) + opcode == RET_INSN_OPCODE || + __is_Jcc(insn)) return; } else { if (opcode == CALL_INSN_OPCODE || From 25d0dc4b957cc8674f8554e85f18a00467e876d7 Mon Sep 17 00:00:00 2001 From: Ashok Raj Date: Mon, 30 Jan 2023 13:39:48 -0800 Subject: [PATCH 53/94] x86/microcode: Allow only "1" as a late reload trigger value Microcode gets reloaded late only if "1" is written to the reload file. However, the code silently treats any other unsigned integer as a successful write even though no actions are performed to load microcode. Make the loader more strict to accept only "1" as a trigger value and return an error otherwise. [ bp: Massage commit message. ] Suggested-by: Thomas Gleixner Signed-off-by: Ashok Raj Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/20230130213955.6046-3-ashok.raj@intel.com --- arch/x86/kernel/cpu/microcode/core.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c index 61d57d9b93ee..fdd1e7eb90f9 100644 --- a/arch/x86/kernel/cpu/microcode/core.c +++ b/arch/x86/kernel/cpu/microcode/core.c @@ -475,11 +475,8 @@ static ssize_t reload_store(struct device *dev, ssize_t ret = 0; ret = kstrtoul(buf, 0, &val); - if (ret) - return ret; - - if (val != 1) - return size; + if (ret || val != 1) + return -EINVAL; cpus_read_lock(); From a7ec817d55421ac214cac9d3e5ebb65d848198dc Mon Sep 17 00:00:00 2001 From: Feng Tang Date: Wed, 4 Jan 2023 16:19:38 +0800 Subject: [PATCH 54/94] x86/tsc: Add option to force frequency recalibration with HW timer The kernel assumes that the TSC frequency which is provided by the hardware / firmware via MSRs or CPUID(0x15) is correct after applying a few basic consistency checks. This disables the TSC recalibration against HPET or PM timer. As a result there is no mechanism to validate that frequency in cases where a firmware or hardware defect is suspected. And there was case that some user used atomic clock to measure the TSC frequency and reported an inaccuracy issue, which was later fixed in firmware. Add an option 'recalibrate' for 'tsc' kernel parameter to force the tsc freq recalibration with HPET or PM timer, and warn if the deviation from previous value is more than about 500 PPM, which provides a way to verify the data from hardware / firmware. There is no functional change to existing work flow. Recently there was a real-world case: "The 40ms/s divergence between TSC and HPET was observed on hardware that is quite recent" [1], on that platform the TSC frequence 1896 MHz was got from CPUID(0x15), and the force-reclibration with HPET/PMTIMER both calibrated out value of 1975 MHz, which also matched with check from software 'chronyd', indicating it's a problem of BIOS or firmware. [Thanks tglx for helping improving the commit log] [ paulmck: Wordsmith Kconfig help text. ] [1]. https://lore.kernel.org/lkml/20221117230910.GI4001@paulmck-ThinkPad-P17-Gen-1/ Signed-off-by: Feng Tang Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: "H. Peter Anvin" Cc: Jonathan Corbet Cc: Cc: Signed-off-by: Paul E. McKenney --- .../admin-guide/kernel-parameters.txt | 4 +++ arch/x86/kernel/tsc.c | 34 ++++++++++++++++--- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 6cfa6e3996cf..95f0d104c232 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -6369,6 +6369,10 @@ in situations with strict latency requirements (where interruptions from clocksource watchdog are not acceptable). + [x86] recalibrate: force recalibration against a HW timer + (HPET or PM timer) on systems whose TSC frequency was + obtained from HW or FW using either an MSR or CPUID(0x15). + Warn if the difference is more than 500 ppm. tsc_early_khz= [X86] Skip early TSC calibration and use the given value instead. Useful when the early TSC frequency discovery diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index a78e73da4a74..92bbc4a6b3fc 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -48,6 +48,8 @@ static DEFINE_STATIC_KEY_FALSE(__use_tsc); int tsc_clocksource_reliable; +static int __read_mostly tsc_force_recalibrate; + static u32 art_to_tsc_numerator; static u32 art_to_tsc_denominator; static u64 art_to_tsc_offset; @@ -303,6 +305,8 @@ static int __init tsc_setup(char *str) mark_tsc_unstable("boot parameter"); if (!strcmp(str, "nowatchdog")) no_tsc_watchdog = 1; + if (!strcmp(str, "recalibrate")) + tsc_force_recalibrate = 1; return 1; } @@ -1374,6 +1378,25 @@ restart: else freq = calc_pmtimer_ref(delta, ref_start, ref_stop); + /* Will hit this only if tsc_force_recalibrate has been set */ + if (boot_cpu_has(X86_FEATURE_TSC_KNOWN_FREQ)) { + + /* Warn if the deviation exceeds 500 ppm */ + if (abs(tsc_khz - freq) > (tsc_khz >> 11)) { + pr_warn("Warning: TSC freq calibrated by CPUID/MSR differs from what is calibrated by HW timer, please check with vendor!!\n"); + pr_info("Previous calibrated TSC freq:\t %lu.%03lu MHz\n", + (unsigned long)tsc_khz / 1000, + (unsigned long)tsc_khz % 1000); + } + + pr_info("TSC freq recalibrated by [%s]:\t %lu.%03lu MHz\n", + hpet ? "HPET" : "PM_TIMER", + (unsigned long)freq / 1000, + (unsigned long)freq % 1000); + + return; + } + /* Make sure we're within 1% */ if (abs(tsc_khz - freq) > tsc_khz/100) goto out; @@ -1407,8 +1430,10 @@ static int __init init_tsc_clocksource(void) if (!boot_cpu_has(X86_FEATURE_TSC) || !tsc_khz) return 0; - if (tsc_unstable) - goto unreg; + if (tsc_unstable) { + clocksource_unregister(&clocksource_tsc_early); + return 0; + } if (boot_cpu_has(X86_FEATURE_NONSTOP_TSC_S3)) clocksource_tsc.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP; @@ -1421,9 +1446,10 @@ static int __init init_tsc_clocksource(void) if (boot_cpu_has(X86_FEATURE_ART)) art_related_clocksource = &clocksource_tsc; clocksource_register_khz(&clocksource_tsc, tsc_khz); -unreg: clocksource_unregister(&clocksource_tsc_early); - return 0; + + if (!tsc_force_recalibrate) + return 0; } schedule_delayed_work(&tsc_irqwork, 0); From efc8b329c7fdc30921a7dfc109237523e1e5b1cc Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 21 Dec 2022 16:20:25 -0800 Subject: [PATCH 55/94] clocksource: Verify HPET and PMTMR when TSC unverified On systems with two or fewer sockets, when the boot CPU has CONSTANT_TSC, NONSTOP_TSC, and TSC_ADJUST, clocksource watchdog verification of the TSC is disabled. This works well much of the time, but there is the occasional production-level system that meets all of these criteria, but which still has a TSC that skews significantly from atomic-clock time. This is usually attributed to a firmware or hardware fault. Yes, the various NTP daemons do express their opinions of userspace-to-atomic-clock time skew, but they put them in various places, depending on the daemon and distro in question. It would therefore be good for the kernel to have some clue that there is a problem. The old behavior of marking the TSC unstable is a non-starter because a great many workloads simply cannot tolerate the overheads and latencies of the various non-TSC clocksources. In addition, NTP-corrected systems sometimes can tolerate significant kernel-space time skew as long as the userspace time sources are within epsilon of atomic-clock time. Therefore, when watchdog verification of TSC is disabled, enable it for HPET and PMTMR (AKA ACPI PM timer). This provides the needed in-kernel time-skew diagnostic without degrading the system's performance. Signed-off-by: Paul E. McKenney Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: "H. Peter Anvin" Cc: Daniel Lezcano Cc: Waiman Long Cc: Tested-by: Feng Tang --- arch/x86/include/asm/time.h | 1 + arch/x86/kernel/hpet.c | 2 ++ arch/x86/kernel/tsc.c | 5 +++++ drivers/clocksource/acpi_pm.c | 6 ++++-- 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/time.h b/arch/x86/include/asm/time.h index 8ac563abb567..a53961c64a56 100644 --- a/arch/x86/include/asm/time.h +++ b/arch/x86/include/asm/time.h @@ -8,6 +8,7 @@ extern void hpet_time_init(void); extern void time_init(void); extern bool pit_timer_init(void); +extern bool tsc_clocksource_watchdog_disabled(void); extern struct clock_event_device *global_clock_event; diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c index 71f336425e58..c8eb1ac5125a 100644 --- a/arch/x86/kernel/hpet.c +++ b/arch/x86/kernel/hpet.c @@ -1091,6 +1091,8 @@ int __init hpet_enable(void) if (!hpet_counting()) goto out_nohpet; + if (tsc_clocksource_watchdog_disabled()) + clocksource_hpet.flags |= CLOCK_SOURCE_MUST_VERIFY; clocksource_register_hz(&clocksource_hpet, (u32)hpet_freq); if (id & HPET_ID_LEGSUP) { diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index 92bbc4a6b3fc..a5371c6d4b64 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -1190,6 +1190,11 @@ static void __init tsc_disable_clocksource_watchdog(void) clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY; } +bool tsc_clocksource_watchdog_disabled(void) +{ + return !(clocksource_tsc.flags & CLOCK_SOURCE_MUST_VERIFY); +} + static void __init check_system_tsc_reliable(void) { #if defined(CONFIG_MGEODEGX1) || defined(CONFIG_MGEODE_LX) || defined(CONFIG_X86_GENERIC) diff --git a/drivers/clocksource/acpi_pm.c b/drivers/clocksource/acpi_pm.c index 279ddff81ab4..82338773602c 100644 --- a/drivers/clocksource/acpi_pm.c +++ b/drivers/clocksource/acpi_pm.c @@ -23,6 +23,7 @@ #include #include #include +#include /* * The I/O port the PMTMR resides at. @@ -210,8 +211,9 @@ static int __init init_acpi_pm_clocksource(void) return -ENODEV; } - return clocksource_register_hz(&clocksource_acpi_pm, - PMTMR_TICKS_PER_SEC); + if (tsc_clocksource_watchdog_disabled()) + clocksource_acpi_pm.flags |= CLOCK_SOURCE_MUST_VERIFY; + return clocksource_register_hz(&clocksource_acpi_pm, PMTMR_TICKS_PER_SEC); } /* We use fs_initcall because we want the PCI fixups to have run From 2355370cd941cbb20882cc3f34460f9f2b8f9a18 Mon Sep 17 00:00:00 2001 From: "Borislav Petkov (AMD)" Date: Tue, 17 Jan 2023 23:59:24 +0100 Subject: [PATCH 56/94] x86/microcode/amd: Remove load_microcode_amd()'s bsp parameter It is always the BSP. No functional changes. Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/20230130161709.11615-2-bp@alien8.de --- arch/x86/kernel/cpu/microcode/amd.c | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c index d144f918a896..c2ac6c42ae32 100644 --- a/arch/x86/kernel/cpu/microcode/amd.c +++ b/arch/x86/kernel/cpu/microcode/amd.c @@ -553,8 +553,7 @@ void load_ucode_amd_ap(unsigned int cpuid_1_eax) early_apply_microcode(cpuid_1_eax, cp.data, cp.size, false); } -static enum ucode_state -load_microcode_amd(bool save, u8 family, const u8 *data, size_t size); +static enum ucode_state load_microcode_amd(u8 family, const u8 *data, size_t size); int __init save_microcode_in_initrd_amd(unsigned int cpuid_1_eax) { @@ -572,7 +571,7 @@ int __init save_microcode_in_initrd_amd(unsigned int cpuid_1_eax) if (!desc.mc) return -EINVAL; - ret = load_microcode_amd(true, x86_family(cpuid_1_eax), desc.data, desc.size); + ret = load_microcode_amd(x86_family(cpuid_1_eax), desc.data, desc.size); if (ret > UCODE_UPDATED) return -EINVAL; @@ -851,8 +850,7 @@ static enum ucode_state __load_microcode_amd(u8 family, const u8 *data, return UCODE_OK; } -static enum ucode_state -load_microcode_amd(bool save, u8 family, const u8 *data, size_t size) +static enum ucode_state load_microcode_amd(u8 family, const u8 *data, size_t size) { struct ucode_patch *p; enum ucode_state ret; @@ -876,10 +874,6 @@ load_microcode_amd(bool save, u8 family, const u8 *data, size_t size) ret = UCODE_NEW; } - /* save BSP's matching patch for early load */ - if (!save) - return ret; - memset(amd_ucode_patch, 0, PATCH_MAX_SIZE); memcpy(amd_ucode_patch, p->data, min_t(u32, p->size, PATCH_MAX_SIZE)); @@ -906,14 +900,9 @@ static enum ucode_state request_microcode_amd(int cpu, struct device *device) { char fw_name[36] = "amd-ucode/microcode_amd.bin"; struct cpuinfo_x86 *c = &cpu_data(cpu); - bool bsp = c->cpu_index == boot_cpu_data.cpu_index; enum ucode_state ret = UCODE_NFOUND; const struct firmware *fw; - /* reload ucode container only on the boot cpu */ - if (!bsp) - return UCODE_OK; - if (c->x86 >= 0x15) snprintf(fw_name, sizeof(fw_name), "amd-ucode/microcode_amd_fam%.2xh.bin", c->x86); @@ -926,7 +915,7 @@ static enum ucode_state request_microcode_amd(int cpu, struct device *device) if (!verify_container(fw->data, fw->size, false)) goto fw_release; - ret = load_microcode_amd(bsp, c->x86, fw->data, fw->size); + ret = load_microcode_amd(c->x86, fw->data, fw->size); fw_release: release_firmware(fw); From a5ad92134bd153a9ccdcddf09a95b088f36c3cce Mon Sep 17 00:00:00 2001 From: "Borislav Petkov (AMD)" Date: Thu, 26 Jan 2023 00:08:03 +0100 Subject: [PATCH 57/94] x86/microcode/AMD: Add a @cpu parameter to the reloading functions Will be used in a subsequent change. Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/20230130161709.11615-3-bp@alien8.de --- arch/x86/include/asm/microcode.h | 4 ++-- arch/x86/include/asm/microcode_amd.h | 4 ++-- arch/x86/kernel/cpu/microcode/amd.c | 2 +- arch/x86/kernel/cpu/microcode/core.c | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/x86/include/asm/microcode.h b/arch/x86/include/asm/microcode.h index d5a58bde091c..320566a0443d 100644 --- a/arch/x86/include/asm/microcode.h +++ b/arch/x86/include/asm/microcode.h @@ -125,13 +125,13 @@ static inline unsigned int x86_cpuid_family(void) #ifdef CONFIG_MICROCODE extern void __init load_ucode_bsp(void); extern void load_ucode_ap(void); -void reload_early_microcode(void); +void reload_early_microcode(unsigned int cpu); extern bool initrd_gone; void microcode_bsp_resume(void); #else static inline void __init load_ucode_bsp(void) { } static inline void load_ucode_ap(void) { } -static inline void reload_early_microcode(void) { } +static inline void reload_early_microcode(unsigned int cpu) { } static inline void microcode_bsp_resume(void) { } #endif diff --git a/arch/x86/include/asm/microcode_amd.h b/arch/x86/include/asm/microcode_amd.h index ac31f9140d07..e6662adf3af4 100644 --- a/arch/x86/include/asm/microcode_amd.h +++ b/arch/x86/include/asm/microcode_amd.h @@ -47,12 +47,12 @@ struct microcode_amd { extern void __init load_ucode_amd_bsp(unsigned int family); extern void load_ucode_amd_ap(unsigned int family); extern int __init save_microcode_in_initrd_amd(unsigned int family); -void reload_ucode_amd(void); +void reload_ucode_amd(unsigned int cpu); #else static inline void __init load_ucode_amd_bsp(unsigned int family) {} static inline void load_ucode_amd_ap(unsigned int family) {} static inline int __init save_microcode_in_initrd_amd(unsigned int family) { return -EINVAL; } -static inline void reload_ucode_amd(void) {} +static inline void reload_ucode_amd(unsigned int cpu) {} #endif #endif /* _ASM_X86_MICROCODE_AMD_H */ diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c index c2ac6c42ae32..1023be6a6954 100644 --- a/arch/x86/kernel/cpu/microcode/amd.c +++ b/arch/x86/kernel/cpu/microcode/amd.c @@ -578,7 +578,7 @@ int __init save_microcode_in_initrd_amd(unsigned int cpuid_1_eax) return 0; } -void reload_ucode_amd(void) +void reload_ucode_amd(unsigned int cpu) { struct microcode_amd *mc; u32 rev, dummy __always_unused; diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c index fdd1e7eb90f9..ddc0958428a5 100644 --- a/arch/x86/kernel/cpu/microcode/core.c +++ b/arch/x86/kernel/cpu/microcode/core.c @@ -298,7 +298,7 @@ struct cpio_data find_microcode_in_initrd(const char *path, bool use_pa) #endif } -void reload_early_microcode(void) +void reload_early_microcode(unsigned int cpu) { int vendor, family; @@ -312,7 +312,7 @@ void reload_early_microcode(void) break; case X86_VENDOR_AMD: if (family >= 0x10) - reload_ucode_amd(); + reload_ucode_amd(cpu); break; default: break; @@ -564,7 +564,7 @@ void microcode_bsp_resume(void) if (uci->mc) microcode_ops->apply_microcode(cpu); else - reload_early_microcode(); + reload_early_microcode(cpu); } static struct syscore_ops mc_syscore_ops = { From 7ff6edf4fef38ab404ee7861f257e28eaaeed35f Mon Sep 17 00:00:00 2001 From: "Borislav Petkov (AMD)" Date: Thu, 26 Jan 2023 16:26:17 +0100 Subject: [PATCH 58/94] x86/microcode/AMD: Fix mixed steppings support The AMD side of the loader has always claimed to support mixed steppings. But somewhere along the way, it broke that by assuming that the cached patch blob is a single one instead of it being one per *node*. So turn it into a per-node one so that each node can stash the blob relevant for it. [ NB: Fixes tag is not really the exactly correct one but it is good enough. ] Fixes: fe055896c040 ("x86/microcode: Merge the early microcode loader") Signed-off-by: Borislav Petkov (AMD) Cc: # 2355370cd941 ("x86/microcode/amd: Remove load_microcode_amd()'s bsp parameter") Cc: # a5ad92134bd1 ("x86/microcode/AMD: Add a @cpu parameter to the reloading functions") Link: https://lore.kernel.org/r/20230130161709.11615-4-bp@alien8.de --- arch/x86/kernel/cpu/microcode/amd.c | 34 ++++++++++++++++++----------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c index 1023be6a6954..9eb457b10341 100644 --- a/arch/x86/kernel/cpu/microcode/amd.c +++ b/arch/x86/kernel/cpu/microcode/amd.c @@ -55,7 +55,9 @@ struct cont_desc { }; static u32 ucode_new_rev; -static u8 amd_ucode_patch[PATCH_MAX_SIZE]; + +/* One blob per node. */ +static u8 amd_ucode_patch[MAX_NUMNODES][PATCH_MAX_SIZE]; /* * Microcode patch container file is prepended to the initrd in cpio @@ -428,7 +430,7 @@ static bool early_apply_microcode(u32 cpuid_1_eax, void *ucode, size_t size, boo patch = (u8 (*)[PATCH_MAX_SIZE])__pa_nodebug(&amd_ucode_patch); #else new_rev = &ucode_new_rev; - patch = &amd_ucode_patch; + patch = &amd_ucode_patch[0]; #endif desc.cpuid_1_eax = cpuid_1_eax; @@ -580,10 +582,10 @@ int __init save_microcode_in_initrd_amd(unsigned int cpuid_1_eax) void reload_ucode_amd(unsigned int cpu) { - struct microcode_amd *mc; u32 rev, dummy __always_unused; + struct microcode_amd *mc; - mc = (struct microcode_amd *)amd_ucode_patch; + mc = (struct microcode_amd *)amd_ucode_patch[cpu_to_node(cpu)]; rdmsr(MSR_AMD64_PATCH_LEVEL, rev, dummy); @@ -852,6 +854,8 @@ static enum ucode_state __load_microcode_amd(u8 family, const u8 *data, static enum ucode_state load_microcode_amd(u8 family, const u8 *data, size_t size) { + struct cpuinfo_x86 *c; + unsigned int nid, cpu; struct ucode_patch *p; enum ucode_state ret; @@ -864,18 +868,22 @@ static enum ucode_state load_microcode_amd(u8 family, const u8 *data, size_t siz return ret; } - p = find_patch(0); - if (!p) { - return ret; - } else { - if (boot_cpu_data.microcode >= p->patch_id) - return ret; + for_each_node(nid) { + cpu = cpumask_first(cpumask_of_node(nid)); + c = &cpu_data(cpu); + + p = find_patch(cpu); + if (!p) + continue; + + if (c->microcode >= p->patch_id) + continue; ret = UCODE_NEW; - } - memset(amd_ucode_patch, 0, PATCH_MAX_SIZE); - memcpy(amd_ucode_patch, p->data, min_t(u32, p->size, PATCH_MAX_SIZE)); + memset(&amd_ucode_patch[nid], 0, PATCH_MAX_SIZE); + memcpy(&amd_ucode_patch[nid], p->data, min_t(u32, p->size, PATCH_MAX_SIZE)); + } return ret; } From f33e0c893b22bf94d7985f1f2aa3872237560c74 Mon Sep 17 00:00:00 2001 From: "Borislav Petkov (AMD)" Date: Mon, 30 Jan 2023 13:48:04 +0100 Subject: [PATCH 59/94] x86/microcode/core: Return an error only when necessary Return an error from the late loading function which is run on each CPU only when an error has actually been encountered during the update. Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/20230130161709.11615-5-bp@alien8.de --- arch/x86/kernel/cpu/microcode/core.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c index ddc0958428a5..7a329e561354 100644 --- a/arch/x86/kernel/cpu/microcode/core.c +++ b/arch/x86/kernel/cpu/microcode/core.c @@ -409,10 +409,10 @@ static int __reload_late(void *info) goto wait_for_siblings; if (err >= UCODE_NFOUND) { - if (err == UCODE_ERROR) + if (err == UCODE_ERROR) { pr_warn("Error reloading microcode on CPU %d\n", cpu); - - ret = -1; + ret = -1; + } } wait_for_siblings: From 915d4ad3830aa1a2dafda9b737749fb410cb9790 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Mon, 16 Jan 2023 17:53:37 +0100 Subject: [PATCH 60/94] posix-timers: Use atomic64_try_cmpxchg() in __update_gt_cputime() Use atomic64_try_cmpxchg() instead of atomic64_cmpxchg() in __update_gt_cputime(). The x86 CMPXCHG instruction returns success in ZF flag, so this change saves a compare after cmpxchg() (and related move instruction in front of cmpxchg()). Also, atomic64_try_cmpxchg() implicitly assigns old *ptr value to "old" when cmpxchg() fails. There is no need to re-read the value in the loop. No functional change intended. Signed-off-by: Uros Bizjak Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20230116165337.5810-1-ubizjak@gmail.com --- kernel/time/posix-cpu-timers.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index cb925e8ef9a8..2f5e9b34022c 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -243,13 +243,12 @@ static void proc_sample_cputime_atomic(struct task_cputime_atomic *at, */ static inline void __update_gt_cputime(atomic64_t *cputime, u64 sum_cputime) { - u64 curr_cputime; -retry: - curr_cputime = atomic64_read(cputime); - if (sum_cputime > curr_cputime) { - if (atomic64_cmpxchg(cputime, curr_cputime, sum_cputime) != curr_cputime) - goto retry; - } + u64 curr_cputime = atomic64_read(cputime); + + do { + if (sum_cputime <= curr_cputime) + return; + } while (!atomic64_try_cmpxchg(cputime, &curr_cputime, sum_cputime)); } static void update_gt_cputime(struct task_cputime_atomic *cputime_atomic, From 717cce3bdcf34705417f641bf2fcdf9b038ec36c Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Fri, 25 Nov 2022 10:42:14 +0100 Subject: [PATCH 61/94] x86/cpu: Provide the full setup for getcpu() on x86-32 setup_getcpu() configures two things: - it writes the current CPU & node information into MSR_TSC_AUX - it writes the same information as a GDT entry. By using the "full" setup_getcpu() on i386 it is possible to read the CPU information in userland via RDTSCP() or via LSL from the GDT. Provide an GDT_ENTRY_CPUNODE for x86-32 and make the setup function unconditionally available. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Reviewed-by: Roland Mainz Link: https://lore.kernel.org/r/20221125094216.3663444-2-bigeasy@linutronix.de --- arch/x86/include/asm/segment.h | 8 ++++---- arch/x86/kernel/cpu/common.c | 4 +--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/arch/x86/include/asm/segment.h b/arch/x86/include/asm/segment.h index c390a672d560..794f69625780 100644 --- a/arch/x86/include/asm/segment.h +++ b/arch/x86/include/asm/segment.h @@ -96,7 +96,7 @@ * * 26 - ESPFIX small SS * 27 - per-cpu [ offset to per-cpu data area ] - * 28 - unused + * 28 - VDSO getcpu * 29 - unused * 30 - unused * 31 - TSS for double fault handler @@ -119,6 +119,7 @@ #define GDT_ENTRY_ESPFIX_SS 26 #define GDT_ENTRY_PERCPU 27 +#define GDT_ENTRY_CPUNODE 28 #define GDT_ENTRY_DOUBLEFAULT_TSS 31 @@ -159,6 +160,8 @@ # define __KERNEL_PERCPU 0 #endif +#define __CPUNODE_SEG (GDT_ENTRY_CPUNODE*8 + 3) + #else /* 64-bit: */ #include @@ -226,8 +229,6 @@ #define GDT_ENTRY_TLS_ENTRIES 3 #define TLS_SIZE (GDT_ENTRY_TLS_ENTRIES* 8) -#ifdef CONFIG_X86_64 - /* Bit size and mask of CPU number stored in the per CPU data (and TSC_AUX) */ #define VDSO_CPUNODE_BITS 12 #define VDSO_CPUNODE_MASK 0xfff @@ -265,7 +266,6 @@ static inline void vdso_read_cpunode(unsigned *cpu, unsigned *node) } #endif /* !__ASSEMBLY__ */ -#endif /* CONFIG_X86_64 */ #ifdef __KERNEL__ diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 3a9043ef3d6b..a26ce206f7b0 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -2124,7 +2124,6 @@ static void wait_for_master_cpu(int cpu) #endif } -#ifdef CONFIG_X86_64 static inline void setup_getcpu(int cpu) { unsigned long cpudata = vdso_encode_cpunode(cpu, early_cpu_to_node(cpu)); @@ -2146,6 +2145,7 @@ static inline void setup_getcpu(int cpu) write_gdt_entry(get_cpu_gdt_rw(cpu), GDT_ENTRY_CPUNODE, &d, DESCTYPE_S); } +#ifdef CONFIG_X86_64 static inline void ucode_cpu_init(int cpu) { if (cpu) @@ -2165,8 +2165,6 @@ static inline void tss_setup_ist(struct tss_struct *tss) #else /* CONFIG_X86_64 */ -static inline void setup_getcpu(int cpu) { } - static inline void ucode_cpu_init(int cpu) { show_ucode_info_early(); From 92d33063c081a82d25dd08a9cce03947c8ed9164 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Fri, 25 Nov 2022 10:42:15 +0100 Subject: [PATCH 62/94] x86/vdso: Provide getcpu for x86-32. Wire up __vdso_getcpu() for x86-32. The 64bit version is reused with trivial modifications. Contrary to vclock_gettime.c there is no requirement to fake any defines in the case of 32bit VDSO on a 64bit kernel because the GDT entry from which the CPU and node information is read is always the native one. Adopt vdso_getcpu.c by: - removing the unneeded time* header files which lead to compile errors for 32bit. - adding segment.h which provides vdso_read_cpunode() and the defines required by it. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20221125094216.3663444-3-bigeasy@linutronix.de --- arch/x86/entry/vdso/Makefile | 3 ++- arch/x86/entry/vdso/vdso32/vdso32.lds.S | 1 + arch/x86/entry/vdso/vdso32/vgetcpu.c | 2 ++ arch/x86/entry/vdso/vgetcpu.c | 3 +-- 4 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 arch/x86/entry/vdso/vdso32/vgetcpu.c diff --git a/arch/x86/entry/vdso/Makefile b/arch/x86/entry/vdso/Makefile index 838613ac15b8..1506a22a4fb6 100644 --- a/arch/x86/entry/vdso/Makefile +++ b/arch/x86/entry/vdso/Makefile @@ -29,7 +29,7 @@ VDSO32-$(CONFIG_IA32_EMULATION) := y # files to link into the vdso vobjs-y := vdso-note.o vclock_gettime.o vgetcpu.o vobjs32-y := vdso32/note.o vdso32/system_call.o vdso32/sigreturn.o -vobjs32-y += vdso32/vclock_gettime.o +vobjs32-y += vdso32/vclock_gettime.o vdso32/vgetcpu.o vobjs-$(CONFIG_X86_SGX) += vsgx.o # files to link into kernel @@ -104,6 +104,7 @@ $(vobjs): KBUILD_AFLAGS += -DBUILD_VDSO CFLAGS_REMOVE_vclock_gettime.o = -pg CFLAGS_REMOVE_vdso32/vclock_gettime.o = -pg CFLAGS_REMOVE_vgetcpu.o = -pg +CFLAGS_REMOVE_vdso32/vgetcpu.o = -pg CFLAGS_REMOVE_vsgx.o = -pg # diff --git a/arch/x86/entry/vdso/vdso32/vdso32.lds.S b/arch/x86/entry/vdso/vdso32/vdso32.lds.S index c7720995ab1a..8a3be07006bb 100644 --- a/arch/x86/entry/vdso/vdso32/vdso32.lds.S +++ b/arch/x86/entry/vdso/vdso32/vdso32.lds.S @@ -28,6 +28,7 @@ VERSION __vdso_time; __vdso_clock_getres; __vdso_clock_gettime64; + __vdso_getcpu; }; LINUX_2.5 { diff --git a/arch/x86/entry/vdso/vdso32/vgetcpu.c b/arch/x86/entry/vdso/vdso32/vgetcpu.c new file mode 100644 index 000000000000..b777f84ffae9 --- /dev/null +++ b/arch/x86/entry/vdso/vdso32/vgetcpu.c @@ -0,0 +1,2 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "../vgetcpu.c" diff --git a/arch/x86/entry/vdso/vgetcpu.c b/arch/x86/entry/vdso/vgetcpu.c index b88a82bbc359..0a9007c24056 100644 --- a/arch/x86/entry/vdso/vgetcpu.c +++ b/arch/x86/entry/vdso/vgetcpu.c @@ -7,8 +7,7 @@ #include #include -#include -#include +#include notrace long __vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused) From 5646bbd6684acf5c9b9dedb863b7d2f6f5a330fb Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Fri, 25 Nov 2022 10:42:16 +0100 Subject: [PATCH 63/94] selftests: Emit a warning if getcpu() is missing on 32bit The VDSO implementation for getcpu() has been wired up on 32bit so warn if missing. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Acked-by: Shuah Khan Link: https://lore.kernel.org/r/20221125094216.3663444-4-bigeasy@linutronix.de --- tools/testing/selftests/x86/test_vsyscall.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/x86/test_vsyscall.c b/tools/testing/selftests/x86/test_vsyscall.c index 5b45e6986aea..47cab972807c 100644 --- a/tools/testing/selftests/x86/test_vsyscall.c +++ b/tools/testing/selftests/x86/test_vsyscall.c @@ -92,11 +92,8 @@ static void init_vdso(void) printf("[WARN]\tfailed to find time in vDSO\n"); vdso_getcpu = (getcpu_t)dlsym(vdso, "__vdso_getcpu"); - if (!vdso_getcpu) { - /* getcpu() was never wired up in the 32-bit vDSO. */ - printf("[%s]\tfailed to find getcpu in vDSO\n", - sizeof(long) == 8 ? "WARN" : "NOTE"); - } + if (!vdso_getcpu) + printf("[WARN]\tfailed to find getcpu in vDSO\n"); } static int init_vsys(void) From 0051293c533017e2a860e0a0a33517bc40240fff Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 1 Feb 2023 13:53:07 -0800 Subject: [PATCH 64/94] clocksource: Enable TSC watchdog checking of HPET and PMTMR only when requested Unconditionally enabling TSC watchdog checking of the HPET and PMTMR clocksources can degrade latency and performance. Therefore, provide a new "watchdog" option to the tsc= boot parameter that opts into such checking. Note that tsc=watchdog is overridden by a tsc=nowatchdog regardless of their relative positions in the list of boot parameters. Reported-by: Thomas Gleixner Reported-by: Waiman Long Signed-off-by: Paul E. McKenney Acked-by: Waiman Long --- .../admin-guide/kernel-parameters.txt | 6 ++++++ arch/x86/kernel/tsc.c | 18 ++++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 95f0d104c232..7b4df6d89d3c 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -6373,6 +6373,12 @@ (HPET or PM timer) on systems whose TSC frequency was obtained from HW or FW using either an MSR or CPUID(0x15). Warn if the difference is more than 500 ppm. + [x86] watchdog: Use TSC as the watchdog clocksource with + which to check other HW timers (HPET or PM timer), but + only on systems where TSC has been deemed trustworthy. + This will be suppressed by an earlier tsc=nowatchdog and + can be overridden by a later tsc=nowatchdog. A console + message will flag any such suppression or overriding. tsc_early_khz= [X86] Skip early TSC calibration and use the given value instead. Useful when the early TSC frequency discovery diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index a5371c6d4b64..306c233c98d8 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -294,6 +294,7 @@ __setup("notsc", notsc_setup); static int no_sched_irq_time; static int no_tsc_watchdog; +static int tsc_as_watchdog; static int __init tsc_setup(char *str) { @@ -303,10 +304,22 @@ static int __init tsc_setup(char *str) no_sched_irq_time = 1; if (!strcmp(str, "unstable")) mark_tsc_unstable("boot parameter"); - if (!strcmp(str, "nowatchdog")) + if (!strcmp(str, "nowatchdog")) { no_tsc_watchdog = 1; + if (tsc_as_watchdog) + pr_alert("%s: Overriding earlier tsc=watchdog with tsc=nowatchdog\n", + __func__); + tsc_as_watchdog = 0; + } if (!strcmp(str, "recalibrate")) tsc_force_recalibrate = 1; + if (!strcmp(str, "watchdog")) { + if (no_tsc_watchdog) + pr_alert("%s: tsc=watchdog overridden by earlier tsc=nowatchdog\n", + __func__); + else + tsc_as_watchdog = 1; + } return 1; } @@ -1192,7 +1205,8 @@ static void __init tsc_disable_clocksource_watchdog(void) bool tsc_clocksource_watchdog_disabled(void) { - return !(clocksource_tsc.flags & CLOCK_SOURCE_MUST_VERIFY); + return !(clocksource_tsc.flags & CLOCK_SOURCE_MUST_VERIFY) && + tsc_as_watchdog && !no_tsc_watchdog; } static void __init check_system_tsc_reliable(void) From 877cff5296faa6ef312f8002216bbf73bea006c7 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Tue, 7 Feb 2023 11:46:33 +0100 Subject: [PATCH 65/94] x86/vdso: Fake 32bit VDSO build on 64bit compile for vgetcpu The 64bit register constrains in __arch_hweight64() cannot be fulfilled in a 32-bit build. The function is only declared but not used within vclock_gettime.c and gcc does not care. LLVM complains and aborts. Reportedly because it validates extended asm even if latter would get compiled out, see https://lore.kernel.org/r/Y%2BJ%2BUQ1vAKr6RHuH@dev-arch.thelio-3990X i.e., a long standing design difference between gcc and LLVM. Move the "fake a 32 bit kernel configuration" bits from vclock_gettime.c into a common header file. Use this from vclock_gettime.c and vgetcpu.c. [ bp: Add background info from Nathan. ] Fixes: 92d33063c081a ("x86/vdso: Provide getcpu for x86-32.") Reported-by: kernel test robot Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/Y+IsCWQdXEr8d9Vy@linutronix.de --- arch/x86/entry/vdso/vdso32/fake_32bit_build.h | 25 +++++++++++++++++ arch/x86/entry/vdso/vdso32/vclock_gettime.c | 27 +------------------ arch/x86/entry/vdso/vdso32/vgetcpu.c | 1 + 3 files changed, 27 insertions(+), 26 deletions(-) create mode 100644 arch/x86/entry/vdso/vdso32/fake_32bit_build.h diff --git a/arch/x86/entry/vdso/vdso32/fake_32bit_build.h b/arch/x86/entry/vdso/vdso32/fake_32bit_build.h new file mode 100644 index 000000000000..db1b15f686e3 --- /dev/null +++ b/arch/x86/entry/vdso/vdso32/fake_32bit_build.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifdef CONFIG_X86_64 + +/* + * in case of a 32 bit VDSO for a 64 bit kernel fake a 32 bit kernel + * configuration + */ +#undef CONFIG_64BIT +#undef CONFIG_X86_64 +#undef CONFIG_COMPAT +#undef CONFIG_PGTABLE_LEVELS +#undef CONFIG_ILLEGAL_POINTER_VALUE +#undef CONFIG_SPARSEMEM_VMEMMAP +#undef CONFIG_NR_CPUS +#undef CONFIG_PARAVIRT_XXL + +#define CONFIG_X86_32 1 +#define CONFIG_PGTABLE_LEVELS 2 +#define CONFIG_PAGE_OFFSET 0 +#define CONFIG_ILLEGAL_POINTER_VALUE 0 +#define CONFIG_NR_CPUS 1 + +#define BUILD_VDSO32_64 + +#endif diff --git a/arch/x86/entry/vdso/vdso32/vclock_gettime.c b/arch/x86/entry/vdso/vdso32/vclock_gettime.c index 283ed9d00426..86981decfea8 100644 --- a/arch/x86/entry/vdso/vdso32/vclock_gettime.c +++ b/arch/x86/entry/vdso/vdso32/vclock_gettime.c @@ -1,29 +1,4 @@ // SPDX-License-Identifier: GPL-2.0 #define BUILD_VDSO32 - -#ifdef CONFIG_X86_64 - -/* - * in case of a 32 bit VDSO for a 64 bit kernel fake a 32 bit kernel - * configuration - */ -#undef CONFIG_64BIT -#undef CONFIG_X86_64 -#undef CONFIG_COMPAT -#undef CONFIG_PGTABLE_LEVELS -#undef CONFIG_ILLEGAL_POINTER_VALUE -#undef CONFIG_SPARSEMEM_VMEMMAP -#undef CONFIG_NR_CPUS -#undef CONFIG_PARAVIRT_XXL - -#define CONFIG_X86_32 1 -#define CONFIG_PGTABLE_LEVELS 2 -#define CONFIG_PAGE_OFFSET 0 -#define CONFIG_ILLEGAL_POINTER_VALUE 0 -#define CONFIG_NR_CPUS 1 - -#define BUILD_VDSO32_64 - -#endif - +#include "fake_32bit_build.h" #include "../vclock_gettime.c" diff --git a/arch/x86/entry/vdso/vdso32/vgetcpu.c b/arch/x86/entry/vdso/vdso32/vgetcpu.c index b777f84ffae9..3a9791f5e998 100644 --- a/arch/x86/entry/vdso/vdso32/vgetcpu.c +++ b/arch/x86/entry/vdso/vdso32/vgetcpu.c @@ -1,2 +1,3 @@ // SPDX-License-Identifier: GPL-2.0 +#include "fake_32bit_build.h" #include "../vgetcpu.c" From dac0da428f8dfb23666ab4ea54a7768c152b857a Mon Sep 17 00:00:00 2001 From: "Borislav Petkov (AMD)" Date: Tue, 7 Feb 2023 16:13:24 +0100 Subject: [PATCH 66/94] x86/vdso: Fix -Wmissing-prototypes warnings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix those: In file included from arch/x86/entry/vdso/vdso32/vclock_gettime.c:4: arch/x86/entry/vdso/vdso32/../vclock_gettime.c:70:5: warning: no previous prototype for ‘__vdso_clock_gettime64’ [-Wmissing-prototypes] 70 | int __vdso_clock_gettime64(clockid_t clock, struct __kernel_timespec *ts) | In file included from arch/x86/entry/vdso/vdso32/vgetcpu.c:3: arch/x86/entry/vdso/vdso32/../vgetcpu.c:13:1: warning: no previous prototype for ‘__vdso_getcpu’ [-Wmissing-prototypes] 13 | __vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused) | ^~~~~~~~~~~~~ Reported-by: kernel test robot Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/202302070742.iYcnoJwk-lkp@intel.com --- arch/x86/include/asm/vdso/gettimeofday.h | 2 ++ arch/x86/include/asm/vdso/processor.h | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/arch/x86/include/asm/vdso/gettimeofday.h b/arch/x86/include/asm/vdso/gettimeofday.h index 1936f21ed8cd..4cf6794f9d68 100644 --- a/arch/x86/include/asm/vdso/gettimeofday.h +++ b/arch/x86/include/asm/vdso/gettimeofday.h @@ -318,6 +318,8 @@ u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult) } #define vdso_calc_delta vdso_calc_delta +int __vdso_clock_gettime64(clockid_t clock, struct __kernel_timespec *ts); + #endif /* !__ASSEMBLY__ */ #endif /* __ASM_VDSO_GETTIMEOFDAY_H */ diff --git a/arch/x86/include/asm/vdso/processor.h b/arch/x86/include/asm/vdso/processor.h index 57b1a7034c64..2cbce97d29ea 100644 --- a/arch/x86/include/asm/vdso/processor.h +++ b/arch/x86/include/asm/vdso/processor.h @@ -18,6 +18,10 @@ static __always_inline void cpu_relax(void) rep_nop(); } +struct getcpu_cache; + +notrace long __vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused); + #endif /* __ASSEMBLY__ */ #endif /* __ASM_VDSO_PROCESSOR_H */ From 221aa03fb4e0740b1f4d9ecbf3d9af9fd9f7f85e Mon Sep 17 00:00:00 2001 From: Youquan Song Date: Mon, 6 Feb 2023 09:14:23 +0800 Subject: [PATCH 67/94] EDAC/i10nm: Add driver decoder for Sapphire Rapids server Intel SDM (December 2022) vol3B 17.13.2 contains IMC MC error codes for Sapphire Rapids. Current i10nm_edac only supports firmware decoder (ACPI DSM methods) for Sapphire Rapids. So add the driver decoder (decoding DDR memory errors via extracting error information from the IMC MC error codes) for Sapphire Rapids for better decoding performance. Co-developed-by: Qiuxu Zhuo Signed-off-by: Qiuxu Zhuo Signed-off-by: Youquan Song Signed-off-by: Tony Luck --- drivers/edac/i10nm_base.c | 102 ++++++++++++++++++++++++++------------ 1 file changed, 69 insertions(+), 33 deletions(-) diff --git a/drivers/edac/i10nm_base.c b/drivers/edac/i10nm_base.c index 46034310b78e..0a4691792801 100644 --- a/drivers/edac/i10nm_base.c +++ b/drivers/edac/i10nm_base.c @@ -434,20 +434,39 @@ static bool i10nm_check_2lm(struct res_config *cfg) } /* - * Check whether the error comes from DDRT by ICX/Tremont model specific error code. - * Refer to SDM vol3B 16.11.3 Intel IMC MC error codes for IA32_MCi_STATUS. + * Check whether the error comes from DDRT by ICX/Tremont/SPR model specific error code. + * Refer to SDM vol3B 17.11.3/17.13.2 Intel IMC MC error codes for IA32_MCi_STATUS. */ static bool i10nm_mscod_is_ddrt(u32 mscod) { - switch (mscod) { - case 0x0106: case 0x0107: - case 0x0800: case 0x0804: - case 0x0806 ... 0x0808: - case 0x080a ... 0x080e: - case 0x0810: case 0x0811: - case 0x0816: case 0x081e: - case 0x081f: - return true; + switch (res_cfg->type) { + case I10NM: + switch (mscod) { + case 0x0106: case 0x0107: + case 0x0800: case 0x0804: + case 0x0806 ... 0x0808: + case 0x080a ... 0x080e: + case 0x0810: case 0x0811: + case 0x0816: case 0x081e: + case 0x081f: + return true; + } + + break; + case SPR: + switch (mscod) { + case 0x0800: case 0x0804: + case 0x0806 ... 0x0808: + case 0x080a ... 0x080e: + case 0x0810: case 0x0811: + case 0x0816: case 0x081e: + case 0x081f: + return true; + } + + break; + default: + return false; } return false; @@ -455,6 +474,7 @@ static bool i10nm_mscod_is_ddrt(u32 mscod) static bool i10nm_mc_decode_available(struct mce *mce) { +#define ICX_IMCx_CHy 0x06666000 u8 bank; if (!decoding_via_mca || mem_cfg_2lm) @@ -468,21 +488,26 @@ static bool i10nm_mc_decode_available(struct mce *mce) switch (res_cfg->type) { case I10NM: - if (bank < 13 || bank > 26) + /* Check whether the bank is one of {13,14,17,18,21,22,25,26} */ + if (!(ICX_IMCx_CHy & (1 << bank))) return false; - - /* DDRT errors can't be decoded from MCA bank registers */ - if (MCI_MISC_ECC_MODE(mce->misc) == MCI_MISC_ECC_DDRT) + break; + case SPR: + if (bank < 13 || bank > 20) return false; - - if (i10nm_mscod_is_ddrt(MCI_STATUS_MSCOD(mce->status))) - return false; - - /* Check whether one of {13,14,17,18,21,22,25,26} */ - return ((bank - 13) & BIT(1)) == 0; + break; default: return false; } + + /* DDRT errors can't be decoded from MCA bank registers */ + if (MCI_MISC_ECC_MODE(mce->misc) == MCI_MISC_ECC_DDRT) + return false; + + if (i10nm_mscod_is_ddrt(MCI_STATUS_MSCOD(mce->status))) + return false; + + return true; } static bool i10nm_mc_decode(struct decoded_addr *res) @@ -504,9 +529,29 @@ static bool i10nm_mc_decode(struct decoded_addr *res) switch (res_cfg->type) { case I10NM: - bank = m->bank - 13; - res->imc = bank / 4; - res->channel = bank % 2; + bank = m->bank - 13; + res->imc = bank / 4; + res->channel = bank % 2; + res->column = GET_BITFIELD(m->misc, 9, 18) << 2; + res->row = GET_BITFIELD(m->misc, 19, 39); + res->bank_group = GET_BITFIELD(m->misc, 40, 41); + res->bank_address = GET_BITFIELD(m->misc, 42, 43); + res->bank_group |= GET_BITFIELD(m->misc, 44, 44) << 2; + res->rank = GET_BITFIELD(m->misc, 56, 58); + res->dimm = res->rank >> 2; + res->rank = res->rank % 4; + break; + case SPR: + bank = m->bank - 13; + res->imc = bank / 2; + res->channel = bank % 2; + res->column = GET_BITFIELD(m->misc, 9, 18) << 2; + res->row = GET_BITFIELD(m->misc, 19, 36); + res->bank_group = GET_BITFIELD(m->misc, 37, 38); + res->bank_address = GET_BITFIELD(m->misc, 39, 40); + res->bank_group |= GET_BITFIELD(m->misc, 41, 41) << 2; + res->rank = GET_BITFIELD(m->misc, 57, 57); + res->dimm = GET_BITFIELD(m->misc, 58, 58); break; default: return false; @@ -518,15 +563,6 @@ static bool i10nm_mc_decode(struct decoded_addr *res) return false; } - res->column = GET_BITFIELD(m->misc, 9, 18) << 2; - res->row = GET_BITFIELD(m->misc, 19, 39); - res->bank_group = GET_BITFIELD(m->misc, 40, 41); - res->bank_address = GET_BITFIELD(m->misc, 42, 43); - res->bank_group |= GET_BITFIELD(m->misc, 44, 44) << 2; - res->rank = GET_BITFIELD(m->misc, 56, 58); - res->dimm = res->rank >> 2; - res->rank = res->rank % 4; - return true; } From fdce765a13384cf411d456472d396d8db2b3114f Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Fri, 27 Jan 2023 17:03:58 +0000 Subject: [PATCH 68/94] EDAC/amd64: Don't set up EDAC PCI control on Family 17h+ EDAC PCI control is used to detect/report legacy PCI errors like "Parity" and "SERROR". Modern AMD systems use PCIe Advanced Error Reporting (AER), and legacy PCI errors should not be reported. Remove EDAC PCI control setup on AMD Family 17h and later systems. Signed-off-by: Yazen Ghannam Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/20230127170419.1824692-2-yazen.ghannam@amd.com --- drivers/edac/amd64_edac.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index e3318e5575a3..2cc7336a5121 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -4370,12 +4370,12 @@ static int __init amd64_edac_init(void) } /* register stuff with EDAC MCE */ - if (boot_cpu_data.x86 >= 0x17) + if (boot_cpu_data.x86 >= 0x17) { amd_register_ecc_decoder(decode_umc_error); - else + } else { amd_register_ecc_decoder(decode_bus_error); - - setup_pci_device(); + setup_pci_device(); + } #ifdef CONFIG_X86_32 amd64_err("%s on 32-bit is unsupported. USE AT YOUR OWN RISK!\n", EDAC_MOD_STR); From 6e241bc93c9f0ae6f76ed65b44132948ca70fd76 Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Fri, 27 Jan 2023 17:03:59 +0000 Subject: [PATCH 69/94] EDAC/amd64: Remove scrub rate control for Family 17h and later The scrub registers on AMD Family 17h and later may be inaccessible to the OS. Furthermore, hardware designers recommend that the scrubbing feature is managed by the firmware. Remove support for the sdram_scrub_rate interface for AMD Family 17h systems and later by not setting the scrub function pointers. The EDAC MC core will then not expose the scrub files in sysfs. Signed-off-by: Yazen Ghannam Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/20230127170419.1824692-3-yazen.ghannam@amd.com --- drivers/edac/amd64_edac.c | 33 +++++---------------------------- drivers/edac/amd64_edac.h | 2 -- 2 files changed, 5 insertions(+), 30 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 2cc7336a5121..07a89df0d4f4 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -182,21 +182,6 @@ static inline int amd64_read_dct_pci_cfg(struct amd64_pvt *pvt, u8 dct, * other archs, we might not have access to the caches directly. */ -static inline void __f17h_set_scrubval(struct amd64_pvt *pvt, u32 scrubval) -{ - /* - * Fam17h supports scrub values between 0x5 and 0x14. Also, the values - * are shifted down by 0x5, so scrubval 0x5 is written to the register - * as 0x0, scrubval 0x6 as 0x1, etc. - */ - if (scrubval >= 0x5 && scrubval <= 0x14) { - scrubval -= 0x5; - pci_write_bits32(pvt->F6, F17H_SCR_LIMIT_ADDR, scrubval, 0xF); - pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 1, 0x1); - } else { - pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 0, 0x1); - } -} /* * Scan the scrub rate mapping table for a close or matching bandwidth value to * issue. If requested is too big, then use last maximum value found. @@ -229,9 +214,7 @@ static int __set_scrub_rate(struct amd64_pvt *pvt, u32 new_bw, u32 min_rate) scrubval = scrubrates[i].scrubval; - if (pvt->umc) { - __f17h_set_scrubval(pvt, scrubval); - } else if (pvt->fam == 0x15 && pvt->model == 0x60) { + if (pvt->fam == 0x15 && pvt->model == 0x60) { f15h_select_dct(pvt, 0); pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F); f15h_select_dct(pvt, 1); @@ -271,16 +254,7 @@ static int get_scrub_rate(struct mem_ctl_info *mci) int i, retval = -EINVAL; u32 scrubval = 0; - if (pvt->umc) { - amd64_read_pci_cfg(pvt->F6, F17H_SCR_BASE_ADDR, &scrubval); - if (scrubval & BIT(0)) { - amd64_read_pci_cfg(pvt->F6, F17H_SCR_LIMIT_ADDR, &scrubval); - scrubval &= 0xF; - scrubval += 0x5; - } else { - scrubval = 0; - } - } else if (pvt->fam == 0x15) { + if (pvt->fam == 0x15) { /* Erratum #505 */ if (pvt->model < 0x10) f15h_select_dct(pvt, 0); @@ -3967,6 +3941,9 @@ static void setup_mci_misc_attrs(struct mem_ctl_info *mci) mci->dev_name = pci_name(pvt->F3); mci->ctl_page_to_phys = NULL; + if (pvt->fam >= 0x17) + return; + /* memory scrubber interface */ mci->set_sdram_scrub_rate = set_scrub_rate; mci->get_sdram_scrub_rate = get_scrub_rate; diff --git a/drivers/edac/amd64_edac.h b/drivers/edac/amd64_edac.h index 38e5ad95d010..48f1d97e1274 100644 --- a/drivers/edac/amd64_edac.h +++ b/drivers/edac/amd64_edac.h @@ -215,8 +215,6 @@ #define DCT_SEL_HI 0x114 #define F15H_M60H_SCRCTRL 0x1C8 -#define F17H_SCR_BASE_ADDR 0x48 -#define F17H_SCR_LIMIT_ADDR 0x4C /* * Function 3 - Misc Control From 6229235f7c6616c96ac06fd1094520b037410f46 Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Fri, 27 Jan 2023 17:04:00 +0000 Subject: [PATCH 70/94] EDAC/amd64: Remove PCI Function 6 PCI Function 6 is used on Family 17h and later to access scrub registers. With scrub access removed, this function has no other use. Remove all Function 6 PCI IDs and related code. Signed-off-by: Yazen Ghannam Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/20230127170419.1824692-4-yazen.ghannam@amd.com --- drivers/edac/amd64_edac.c | 22 +--------------------- drivers/edac/amd64_edac.h | 12 ++---------- 2 files changed, 3 insertions(+), 31 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 07a89df0d4f4..dce2179ad454 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -2906,7 +2906,6 @@ static struct amd64_family_type family_types[] = { [F17_CPUS] = { .ctl_name = "F17h", .f0_id = PCI_DEVICE_ID_AMD_17H_DF_F0, - .f6_id = PCI_DEVICE_ID_AMD_17H_DF_F6, .max_mcs = 2, .ops = { .early_channel_count = f17_early_channel_count, @@ -2916,7 +2915,6 @@ static struct amd64_family_type family_types[] = { [F17_M10H_CPUS] = { .ctl_name = "F17h_M10h", .f0_id = PCI_DEVICE_ID_AMD_17H_M10H_DF_F0, - .f6_id = PCI_DEVICE_ID_AMD_17H_M10H_DF_F6, .max_mcs = 2, .ops = { .early_channel_count = f17_early_channel_count, @@ -2926,7 +2924,6 @@ static struct amd64_family_type family_types[] = { [F17_M30H_CPUS] = { .ctl_name = "F17h_M30h", .f0_id = PCI_DEVICE_ID_AMD_17H_M30H_DF_F0, - .f6_id = PCI_DEVICE_ID_AMD_17H_M30H_DF_F6, .max_mcs = 8, .ops = { .early_channel_count = f17_early_channel_count, @@ -2936,7 +2933,6 @@ static struct amd64_family_type family_types[] = { [F17_M60H_CPUS] = { .ctl_name = "F17h_M60h", .f0_id = PCI_DEVICE_ID_AMD_17H_M60H_DF_F0, - .f6_id = PCI_DEVICE_ID_AMD_17H_M60H_DF_F6, .max_mcs = 2, .ops = { .early_channel_count = f17_early_channel_count, @@ -2946,7 +2942,6 @@ static struct amd64_family_type family_types[] = { [F17_M70H_CPUS] = { .ctl_name = "F17h_M70h", .f0_id = PCI_DEVICE_ID_AMD_17H_M70H_DF_F0, - .f6_id = PCI_DEVICE_ID_AMD_17H_M70H_DF_F6, .max_mcs = 2, .ops = { .early_channel_count = f17_early_channel_count, @@ -2956,7 +2951,6 @@ static struct amd64_family_type family_types[] = { [F19_CPUS] = { .ctl_name = "F19h", .f0_id = PCI_DEVICE_ID_AMD_19H_DF_F0, - .f6_id = PCI_DEVICE_ID_AMD_19H_DF_F6, .max_mcs = 8, .ops = { .early_channel_count = f17_early_channel_count, @@ -2966,7 +2960,6 @@ static struct amd64_family_type family_types[] = { [F19_M10H_CPUS] = { .ctl_name = "F19h_M10h", .f0_id = PCI_DEVICE_ID_AMD_19H_M10H_DF_F0, - .f6_id = PCI_DEVICE_ID_AMD_19H_M10H_DF_F6, .max_mcs = 12, .flags.zn_regs_v2 = 1, .ops = { @@ -2977,7 +2970,6 @@ static struct amd64_family_type family_types[] = { [F19_M50H_CPUS] = { .ctl_name = "F19h_M50h", .f0_id = PCI_DEVICE_ID_AMD_19H_M50H_DF_F0, - .f6_id = PCI_DEVICE_ID_AMD_19H_M50H_DF_F6, .max_mcs = 2, .ops = { .early_channel_count = f17_early_channel_count, @@ -3290,7 +3282,7 @@ log_error: /* * Use pvt->F3 which contains the F3 CPU PCI device to get the related * F1 (AddrMap) and F2 (Dct) devices. Return negative value on error. - * Reserve F0 and F6 on systems with a UMC. + * Reserve F0 on systems with a UMC. */ static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 pci_id1, u16 pci_id2) @@ -3302,21 +3294,11 @@ reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 pci_id1, u16 pci_id2) return -ENODEV; } - pvt->F6 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3); - if (!pvt->F6) { - pci_dev_put(pvt->F0); - pvt->F0 = NULL; - - edac_dbg(1, "F6 not found: device 0x%x\n", pci_id2); - return -ENODEV; - } - if (!pci_ctl_dev) pci_ctl_dev = &pvt->F0->dev; edac_dbg(1, "F0: %s\n", pci_name(pvt->F0)); edac_dbg(1, "F3: %s\n", pci_name(pvt->F3)); - edac_dbg(1, "F6: %s\n", pci_name(pvt->F6)); return 0; } @@ -3352,7 +3334,6 @@ static void free_mc_sibling_devs(struct amd64_pvt *pvt) { if (pvt->umc) { pci_dev_put(pvt->F0); - pci_dev_put(pvt->F6); } else { pci_dev_put(pvt->F1); pci_dev_put(pvt->F2); @@ -4078,7 +4059,6 @@ static int hw_info_get(struct amd64_pvt *pvt) return -ENOMEM; pci_id1 = fam_type->f0_id; - pci_id2 = fam_type->f6_id; } else { pci_id1 = fam_type->f1_id; pci_id2 = fam_type->f2_id; diff --git a/drivers/edac/amd64_edac.h b/drivers/edac/amd64_edac.h index 48f1d97e1274..2d5ea9ca3868 100644 --- a/drivers/edac/amd64_edac.h +++ b/drivers/edac/amd64_edac.h @@ -115,21 +115,13 @@ #define PCI_DEVICE_ID_AMD_16H_M30H_NB_F1 0x1581 #define PCI_DEVICE_ID_AMD_16H_M30H_NB_F2 0x1582 #define PCI_DEVICE_ID_AMD_17H_DF_F0 0x1460 -#define PCI_DEVICE_ID_AMD_17H_DF_F6 0x1466 #define PCI_DEVICE_ID_AMD_17H_M10H_DF_F0 0x15e8 -#define PCI_DEVICE_ID_AMD_17H_M10H_DF_F6 0x15ee #define PCI_DEVICE_ID_AMD_17H_M30H_DF_F0 0x1490 -#define PCI_DEVICE_ID_AMD_17H_M30H_DF_F6 0x1496 #define PCI_DEVICE_ID_AMD_17H_M60H_DF_F0 0x1448 -#define PCI_DEVICE_ID_AMD_17H_M60H_DF_F6 0x144e #define PCI_DEVICE_ID_AMD_17H_M70H_DF_F0 0x1440 -#define PCI_DEVICE_ID_AMD_17H_M70H_DF_F6 0x1446 #define PCI_DEVICE_ID_AMD_19H_DF_F0 0x1650 -#define PCI_DEVICE_ID_AMD_19H_DF_F6 0x1656 #define PCI_DEVICE_ID_AMD_19H_M10H_DF_F0 0x14ad -#define PCI_DEVICE_ID_AMD_19H_M10H_DF_F6 0x14b3 #define PCI_DEVICE_ID_AMD_19H_M50H_DF_F0 0x166a -#define PCI_DEVICE_ID_AMD_19H_M50H_DF_F6 0x1670 /* * Function 1 - Address Map @@ -354,7 +346,7 @@ struct amd64_pvt { struct low_ops *ops; /* pci_device handles which we utilize */ - struct pci_dev *F0, *F1, *F2, *F3, *F6; + struct pci_dev *F0, *F1, *F2, *F3; u16 mc_node_id; /* MC index of this MC node */ u8 fam; /* CPU family */ @@ -501,7 +493,7 @@ struct amd64_family_flags { struct amd64_family_type { const char *ctl_name; - u16 f0_id, f1_id, f2_id, f6_id; + u16 f0_id, f1_id, f2_id; /* Maximum number of memory controllers per die/node. */ u8 max_mcs; struct amd64_family_flags flags; From cf981562e6270397a2b0dc871a1f11ccc2a687e8 Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Fri, 27 Jan 2023 17:04:01 +0000 Subject: [PATCH 71/94] EDAC/amd64: Remove PCI Function 0 PCI Function 0 is used on Family 17h and later only to read the "dhar" value. This value is printed and provided through a module-specific debug sysfs file. The value is not used for any Family 17h and later code, and it does not have any apparent debug value on these systems. Remove "dhar", Function 0 PCI IDs, and all related code. Signed-off-by: Yazen Ghannam Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/20230127170419.1824692-5-yazen.ghannam@amd.com --- drivers/edac/amd64_edac.c | 38 +++++--------------------------------- drivers/edac/amd64_edac.h | 12 ++---------- 2 files changed, 7 insertions(+), 43 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index dce2179ad454..352cbcda53f9 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1428,9 +1428,6 @@ static void __dump_misc_regs_df(struct amd64_pvt *pvt) debug_display_dimm_sizes_df(pvt, i); } - - edac_dbg(1, "F0x104 (DRAM Hole Address): 0x%08x, base: 0x%08x\n", - pvt->dhar, dhar_base(pvt)); } /* Display and decode various NB registers for debug purposes. */ @@ -1465,6 +1462,8 @@ static void __dump_misc_regs(struct amd64_pvt *pvt) /* Only if NOT ganged does dclr1 have valid info */ if (!dct_ganging_enabled(pvt)) debug_dump_dramcfg_low(pvt, pvt->dclr1, 1); + + edac_dbg(1, " DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no"); } /* Display and decode various NB registers for debug purposes. */ @@ -1475,8 +1474,6 @@ static void dump_misc_regs(struct amd64_pvt *pvt) else __dump_misc_regs(pvt); - edac_dbg(1, " DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no"); - amd64_info("using x%u syndromes.\n", pvt->ecc_sym_sz); } @@ -2905,7 +2902,6 @@ static struct amd64_family_type family_types[] = { }, [F17_CPUS] = { .ctl_name = "F17h", - .f0_id = PCI_DEVICE_ID_AMD_17H_DF_F0, .max_mcs = 2, .ops = { .early_channel_count = f17_early_channel_count, @@ -2914,7 +2910,6 @@ static struct amd64_family_type family_types[] = { }, [F17_M10H_CPUS] = { .ctl_name = "F17h_M10h", - .f0_id = PCI_DEVICE_ID_AMD_17H_M10H_DF_F0, .max_mcs = 2, .ops = { .early_channel_count = f17_early_channel_count, @@ -2923,7 +2918,6 @@ static struct amd64_family_type family_types[] = { }, [F17_M30H_CPUS] = { .ctl_name = "F17h_M30h", - .f0_id = PCI_DEVICE_ID_AMD_17H_M30H_DF_F0, .max_mcs = 8, .ops = { .early_channel_count = f17_early_channel_count, @@ -2932,7 +2926,6 @@ static struct amd64_family_type family_types[] = { }, [F17_M60H_CPUS] = { .ctl_name = "F17h_M60h", - .f0_id = PCI_DEVICE_ID_AMD_17H_M60H_DF_F0, .max_mcs = 2, .ops = { .early_channel_count = f17_early_channel_count, @@ -2941,7 +2934,6 @@ static struct amd64_family_type family_types[] = { }, [F17_M70H_CPUS] = { .ctl_name = "F17h_M70h", - .f0_id = PCI_DEVICE_ID_AMD_17H_M70H_DF_F0, .max_mcs = 2, .ops = { .early_channel_count = f17_early_channel_count, @@ -2950,7 +2942,6 @@ static struct amd64_family_type family_types[] = { }, [F19_CPUS] = { .ctl_name = "F19h", - .f0_id = PCI_DEVICE_ID_AMD_19H_DF_F0, .max_mcs = 8, .ops = { .early_channel_count = f17_early_channel_count, @@ -2959,7 +2950,6 @@ static struct amd64_family_type family_types[] = { }, [F19_M10H_CPUS] = { .ctl_name = "F19h_M10h", - .f0_id = PCI_DEVICE_ID_AMD_19H_M10H_DF_F0, .max_mcs = 12, .flags.zn_regs_v2 = 1, .ops = { @@ -2969,7 +2959,6 @@ static struct amd64_family_type family_types[] = { }, [F19_M50H_CPUS] = { .ctl_name = "F19h_M50h", - .f0_id = PCI_DEVICE_ID_AMD_19H_M50H_DF_F0, .max_mcs = 2, .ops = { .early_channel_count = f17_early_channel_count, @@ -3282,26 +3271,12 @@ log_error: /* * Use pvt->F3 which contains the F3 CPU PCI device to get the related * F1 (AddrMap) and F2 (Dct) devices. Return negative value on error. - * Reserve F0 on systems with a UMC. */ static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 pci_id1, u16 pci_id2) { - if (pvt->umc) { - pvt->F0 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3); - if (!pvt->F0) { - edac_dbg(1, "F0 not found, device 0x%x\n", pci_id1); - return -ENODEV; - } - - if (!pci_ctl_dev) - pci_ctl_dev = &pvt->F0->dev; - - edac_dbg(1, "F0: %s\n", pci_name(pvt->F0)); - edac_dbg(1, "F3: %s\n", pci_name(pvt->F3)); - + if (pvt->umc) return 0; - } /* Reserve the ADDRESS MAP Device */ pvt->F1 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3); @@ -3333,7 +3308,7 @@ reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 pci_id1, u16 pci_id2) static void free_mc_sibling_devs(struct amd64_pvt *pvt) { if (pvt->umc) { - pci_dev_put(pvt->F0); + return; } else { pci_dev_put(pvt->F1); pci_dev_put(pvt->F2); @@ -3423,7 +3398,6 @@ static void read_mc_regs(struct amd64_pvt *pvt) if (pvt->umc) { __read_mc_regs_df(pvt); - amd64_read_pci_cfg(pvt->F0, DF_DHAR, &pvt->dhar); goto skip; } @@ -4057,8 +4031,6 @@ static int hw_info_get(struct amd64_pvt *pvt) pvt->umc = kcalloc(fam_type->max_mcs, sizeof(struct amd64_umc), GFP_KERNEL); if (!pvt->umc) return -ENOMEM; - - pci_id1 = fam_type->f0_id; } else { pci_id1 = fam_type->f1_id; pci_id2 = fam_type->f2_id; @@ -4075,7 +4047,7 @@ static int hw_info_get(struct amd64_pvt *pvt) static void hw_info_put(struct amd64_pvt *pvt) { - if (pvt->F0 || pvt->F1) + if (pvt->F1) free_mc_sibling_devs(pvt); kfree(pvt->umc); diff --git a/drivers/edac/amd64_edac.h b/drivers/edac/amd64_edac.h index 2d5ea9ca3868..398fb58dacbf 100644 --- a/drivers/edac/amd64_edac.h +++ b/drivers/edac/amd64_edac.h @@ -114,14 +114,6 @@ #define PCI_DEVICE_ID_AMD_16H_NB_F2 0x1532 #define PCI_DEVICE_ID_AMD_16H_M30H_NB_F1 0x1581 #define PCI_DEVICE_ID_AMD_16H_M30H_NB_F2 0x1582 -#define PCI_DEVICE_ID_AMD_17H_DF_F0 0x1460 -#define PCI_DEVICE_ID_AMD_17H_M10H_DF_F0 0x15e8 -#define PCI_DEVICE_ID_AMD_17H_M30H_DF_F0 0x1490 -#define PCI_DEVICE_ID_AMD_17H_M60H_DF_F0 0x1448 -#define PCI_DEVICE_ID_AMD_17H_M70H_DF_F0 0x1440 -#define PCI_DEVICE_ID_AMD_19H_DF_F0 0x1650 -#define PCI_DEVICE_ID_AMD_19H_M10H_DF_F0 0x14ad -#define PCI_DEVICE_ID_AMD_19H_M50H_DF_F0 0x166a /* * Function 1 - Address Map @@ -346,7 +338,7 @@ struct amd64_pvt { struct low_ops *ops; /* pci_device handles which we utilize */ - struct pci_dev *F0, *F1, *F2, *F3; + struct pci_dev *F1, *F2, *F3; u16 mc_node_id; /* MC index of this MC node */ u8 fam; /* CPU family */ @@ -493,7 +485,7 @@ struct amd64_family_flags { struct amd64_family_type { const char *ctl_name; - u16 f0_id, f1_id, f2_id; + u16 f1_id, f2_id; /* Maximum number of memory controllers per die/node. */ u8 max_mcs; struct amd64_family_flags flags; From c4605bde334367b22bbf43cbbef0d1b7c75433dc Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Fri, 27 Jan 2023 17:04:02 +0000 Subject: [PATCH 72/94] EDAC/amd64: Remove early_channel_count() The early_channel_count() function seems to have been useful in the past for knowing how many EDAC mci structures to populate. However, this is no longer needed as the maximum channel count for a system is used instead. Remove the early_channel_count() helper functions and related code. Use the size of the channel layer when iterating over channel structures. Signed-off-by: Yazen Ghannam Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/20230127170419.1824692-6-yazen.ghannam@amd.com --- drivers/edac/amd64_edac.c | 116 +------------------------------------- drivers/edac/amd64_edac.h | 2 - 2 files changed, 2 insertions(+), 116 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 352cbcda53f9..1c4bef1cdf28 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1703,24 +1703,6 @@ ddr3: pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3; } -/* Get the number of DCT channels the memory controller is using. */ -static int k8_early_channel_count(struct amd64_pvt *pvt) -{ - int flag; - - if (pvt->ext_model >= K8_REV_F) - /* RevF (NPT) and later */ - flag = pvt->dclr0 & WIDTH_128; - else - /* RevE and earlier */ - flag = pvt->dclr0 & REVE_WIDTH_128; - - /* not used */ - pvt->dclr1 = 0; - - return (flag) ? 2 : 1; -} - /* On F10h and later ErrAddr is MC4_ADDR[47:1] */ static u64 get_error_address(struct amd64_pvt *pvt, struct mce *m) { @@ -1972,69 +1954,6 @@ static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, } } -/* - * Get the number of DCT channels in use. - * - * Return: - * number of Memory Channels in operation - * Pass back: - * contents of the DCL0_LOW register - */ -static int f1x_early_channel_count(struct amd64_pvt *pvt) -{ - int i, j, channels = 0; - - /* On F10h, if we are in 128 bit mode, then we are using 2 channels */ - if (pvt->fam == 0x10 && (pvt->dclr0 & WIDTH_128)) - return 2; - - /* - * Need to check if in unganged mode: In such, there are 2 channels, - * but they are not in 128 bit mode and thus the above 'dclr0' status - * bit will be OFF. - * - * Need to check DCT0[0] and DCT1[0] to see if only one of them has - * their CSEnable bit on. If so, then SINGLE DIMM case. - */ - edac_dbg(0, "Data width is not 128 bits - need more decoding\n"); - - /* - * Check DRAM Bank Address Mapping values for each DIMM to see if there - * is more than just one DIMM present in unganged mode. Need to check - * both controllers since DIMMs can be placed in either one. - */ - for (i = 0; i < 2; i++) { - u32 dbam = (i ? pvt->dbam1 : pvt->dbam0); - - for (j = 0; j < 4; j++) { - if (DBAM_DIMM(j, dbam) > 0) { - channels++; - break; - } - } - } - - if (channels > 2) - channels = 2; - - amd64_info("MCT channel count: %d\n", channels); - - return channels; -} - -static int f17_early_channel_count(struct amd64_pvt *pvt) -{ - int i, channels = 0; - - /* SDP Control bit 31 (SdpInit) is clear for unused UMC channels */ - for_each_umc(i) - channels += !!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT); - - amd64_info("MCT channel count: %d\n", channels); - - return channels; -} - static int ddr3_cs_size(unsigned i, bool dct_width) { unsigned shift = 0; @@ -2829,7 +2748,6 @@ static struct amd64_family_type family_types[] = { .f2_id = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL, .max_mcs = 2, .ops = { - .early_channel_count = k8_early_channel_count, .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow, .dbam_to_cs = k8_dbam_to_chip_select, } @@ -2840,7 +2758,6 @@ static struct amd64_family_type family_types[] = { .f2_id = PCI_DEVICE_ID_AMD_10H_NB_DRAM, .max_mcs = 2, .ops = { - .early_channel_count = f1x_early_channel_count, .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, .dbam_to_cs = f10_dbam_to_chip_select, } @@ -2851,7 +2768,6 @@ static struct amd64_family_type family_types[] = { .f2_id = PCI_DEVICE_ID_AMD_15H_NB_F2, .max_mcs = 2, .ops = { - .early_channel_count = f1x_early_channel_count, .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, .dbam_to_cs = f15_dbam_to_chip_select, } @@ -2862,7 +2778,6 @@ static struct amd64_family_type family_types[] = { .f2_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F2, .max_mcs = 2, .ops = { - .early_channel_count = f1x_early_channel_count, .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, .dbam_to_cs = f16_dbam_to_chip_select, } @@ -2873,7 +2788,6 @@ static struct amd64_family_type family_types[] = { .f2_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F2, .max_mcs = 2, .ops = { - .early_channel_count = f1x_early_channel_count, .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, .dbam_to_cs = f15_m60h_dbam_to_chip_select, } @@ -2884,7 +2798,6 @@ static struct amd64_family_type family_types[] = { .f2_id = PCI_DEVICE_ID_AMD_16H_NB_F2, .max_mcs = 2, .ops = { - .early_channel_count = f1x_early_channel_count, .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, .dbam_to_cs = f16_dbam_to_chip_select, } @@ -2895,7 +2808,6 @@ static struct amd64_family_type family_types[] = { .f2_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F2, .max_mcs = 2, .ops = { - .early_channel_count = f1x_early_channel_count, .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, .dbam_to_cs = f16_dbam_to_chip_select, } @@ -2904,7 +2816,6 @@ static struct amd64_family_type family_types[] = { .ctl_name = "F17h", .max_mcs = 2, .ops = { - .early_channel_count = f17_early_channel_count, .dbam_to_cs = f17_addr_mask_to_cs_size, } }, @@ -2912,7 +2823,6 @@ static struct amd64_family_type family_types[] = { .ctl_name = "F17h_M10h", .max_mcs = 2, .ops = { - .early_channel_count = f17_early_channel_count, .dbam_to_cs = f17_addr_mask_to_cs_size, } }, @@ -2920,7 +2830,6 @@ static struct amd64_family_type family_types[] = { .ctl_name = "F17h_M30h", .max_mcs = 8, .ops = { - .early_channel_count = f17_early_channel_count, .dbam_to_cs = f17_addr_mask_to_cs_size, } }, @@ -2928,7 +2837,6 @@ static struct amd64_family_type family_types[] = { .ctl_name = "F17h_M60h", .max_mcs = 2, .ops = { - .early_channel_count = f17_early_channel_count, .dbam_to_cs = f17_addr_mask_to_cs_size, } }, @@ -2936,7 +2844,6 @@ static struct amd64_family_type family_types[] = { .ctl_name = "F17h_M70h", .max_mcs = 2, .ops = { - .early_channel_count = f17_early_channel_count, .dbam_to_cs = f17_addr_mask_to_cs_size, } }, @@ -2944,7 +2851,6 @@ static struct amd64_family_type family_types[] = { .ctl_name = "F19h", .max_mcs = 8, .ops = { - .early_channel_count = f17_early_channel_count, .dbam_to_cs = f17_addr_mask_to_cs_size, } }, @@ -2953,7 +2859,6 @@ static struct amd64_family_type family_types[] = { .max_mcs = 12, .flags.zn_regs_v2 = 1, .ops = { - .early_channel_count = f17_early_channel_count, .dbam_to_cs = f17_addr_mask_to_cs_size, } }, @@ -2961,7 +2866,6 @@ static struct amd64_family_type family_types[] = { .ctl_name = "F19h_M50h", .max_mcs = 2, .ops = { - .early_channel_count = f17_early_channel_count, .dbam_to_cs = f17_addr_mask_to_cs_size, } }, @@ -3620,7 +3524,7 @@ static int init_csrows(struct mem_ctl_info *mci) : EDAC_SECDED; } - for (j = 0; j < pvt->channel_count; j++) { + for (j = 0; j < fam_type->max_mcs; j++) { dimm = csrow->channels[j]->dimm; dimm->mtype = pvt->dram_type; dimm->edac_mode = edac_mode; @@ -4057,28 +3961,12 @@ static int init_one_instance(struct amd64_pvt *pvt) { struct mem_ctl_info *mci = NULL; struct edac_mc_layer layers[2]; - int ret = -EINVAL; + int ret = -ENOMEM; - /* - * We need to determine how many memory channels there are. Then use - * that information for calculating the size of the dynamic instance - * tables in the 'mci' structure. - */ - pvt->channel_count = pvt->ops->early_channel_count(pvt); - if (pvt->channel_count < 0) - return ret; - - ret = -ENOMEM; layers[0].type = EDAC_MC_LAYER_CHIP_SELECT; layers[0].size = pvt->csels[0].b_cnt; layers[0].is_virt_csrow = true; layers[1].type = EDAC_MC_LAYER_CHANNEL; - - /* - * Always allocate two channels since we can have setups with DIMMs on - * only one channel. Also, this simplifies handling later for the price - * of a couple of KBs tops. - */ layers[1].size = fam_type->max_mcs; layers[1].is_virt_csrow = false; diff --git a/drivers/edac/amd64_edac.h b/drivers/edac/amd64_edac.h index 398fb58dacbf..e4329dff8cf2 100644 --- a/drivers/edac/amd64_edac.h +++ b/drivers/edac/amd64_edac.h @@ -346,7 +346,6 @@ struct amd64_pvt { u8 stepping; /* ... stepping */ int ext_model; /* extended model value of this node */ - int channel_count; /* Raw registers */ u32 dclr0; /* DRAM Configuration Low DCT0 reg */ @@ -466,7 +465,6 @@ struct ecc_settings { * functions and per device encoding/decoding logic. */ struct low_ops { - int (*early_channel_count) (struct amd64_pvt *pvt); void (*map_sysaddr_to_csrow) (struct mem_ctl_info *mci, u64 sys_addr, struct err_info *); int (*dbam_to_cs) (struct amd64_pvt *pvt, u8 dct, From 5b268d8abaec6cbd4bd70d062e769098d96670aa Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Thu, 2 Feb 2023 16:12:14 +0100 Subject: [PATCH 73/94] time/debug: Fix memory leak with using debugfs_lookup() When calling debugfs_lookup() the result must have dput() called on it, otherwise the memory will leak over time. To make things simpler, just call debugfs_lookup_and_remove() instead which handles all of the logic at once. Signed-off-by: Greg Kroah-Hartman Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20230202151214.2306822-1-gregkh@linuxfoundation.org --- kernel/time/test_udelay.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/time/test_udelay.c b/kernel/time/test_udelay.c index 13b11eb62685..20d5df631570 100644 --- a/kernel/time/test_udelay.c +++ b/kernel/time/test_udelay.c @@ -149,7 +149,7 @@ module_init(udelay_test_init); static void __exit udelay_test_exit(void) { mutex_lock(&udelay_test_lock); - debugfs_remove(debugfs_lookup(DEBUGFS_FILENAME, NULL)); + debugfs_lookup_and_remove(DEBUGFS_FILENAME, NULL); mutex_unlock(&udelay_test_lock); } From 851026a2bf54e739a2e13723bf3c4513f3cbcdc9 Mon Sep 17 00:00:00 2001 From: "Borislav Petkov (AMD)" Date: Sat, 11 Feb 2023 00:45:41 +0100 Subject: [PATCH 74/94] x86/cacheinfo: Remove unused trace variable 15cd8812ab2c ("x86: Remove the CPU cache size printk's") removed the last use of the trace local var. Remove it too and the useless trace cache case. No functional changes. Reported-by: Jiapeng Chong Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/20230210234541.9694-1-bp@alien8.de Link: http://lore.kernel.org/r/20220705073349.1512-1-jiapeng.chong@linux.alibaba.com --- arch/x86/kernel/cpu/cacheinfo.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index f4e5aa27eec6..4063e8991211 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -734,7 +734,7 @@ void init_hygon_cacheinfo(struct cpuinfo_x86 *c) void init_intel_cacheinfo(struct cpuinfo_x86 *c) { /* Cache sizes */ - unsigned int trace = 0, l1i = 0, l1d = 0, l2 = 0, l3 = 0; + unsigned int l1i = 0, l1d = 0, l2 = 0, l3 = 0; unsigned int new_l1d = 0, new_l1i = 0; /* Cache sizes from cpuid(4) */ unsigned int new_l2 = 0, new_l3 = 0, i; /* Cache sizes from cpuid(4) */ unsigned int l2_id = 0, l3_id = 0, num_threads_sharing, index_msb; @@ -835,9 +835,6 @@ void init_intel_cacheinfo(struct cpuinfo_x86 *c) case LVL_3: l3 += cache_table[k].size; break; - case LVL_TRACE: - trace += cache_table[k].size; - break; } break; From 8fe6d84947582e2c076abc6253b80709fb047935 Mon Sep 17 00:00:00 2001 From: "Borislav Petkov (AMD)" Date: Sun, 18 Dec 2022 21:52:42 +0100 Subject: [PATCH 75/94] x86/tsc: Make recalibrate_cpu_khz() export GPL only A quick search doesn't reveal any use outside of the kernel - which would be questionable to begin with anyway - so make the export GPL only. Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/Y599miBzWRAuOwhg@zn.tnic --- arch/x86/kernel/tsc.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index a78e73da4a74..eaeffef93a12 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -912,8 +912,7 @@ void recalibrate_cpu_khz(void) cpu_khz_old, cpu_khz); #endif } - -EXPORT_SYMBOL(recalibrate_cpu_khz); +EXPORT_SYMBOL_GPL(recalibrate_cpu_khz); static unsigned long long cyc2ns_suspend; From 6b8d5dde5b6903baf82fc7400e0b3376b10805b4 Mon Sep 17 00:00:00 2001 From: "Borislav Petkov (AMD)" Date: Sun, 18 Dec 2022 21:53:36 +0100 Subject: [PATCH 76/94] x86/tsc: Do feature check as the very first thing Do the feature check as the very first thing in the function. Everything else comes after that and is meaningless work if the TSC CPUID bit is not even set. Switch to cpu_feature_enabled() too, while at it. No functional changes. Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/Y5990CUCuWd5jfBH@zn.tnic --- arch/x86/kernel/tsc.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index eaeffef93a12..aff1d79dd6ba 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -1509,6 +1509,11 @@ void __init tsc_early_init(void) void __init tsc_init(void) { + if (!cpu_feature_enabled(X86_FEATURE_TSC)) { + setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER); + return; + } + /* * native_calibrate_cpu_early can only calibrate using methods that are * available early in boot. @@ -1516,11 +1521,6 @@ void __init tsc_init(void) if (x86_platform.calibrate_cpu == native_calibrate_cpu_early) x86_platform.calibrate_cpu = native_calibrate_cpu; - if (!boot_cpu_has(X86_FEATURE_TSC)) { - setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER); - return; - } - if (!tsc_khz) { /* We failed to determine frequencies earlier, try again */ if (!determine_cpu_tsc_frequencies(false)) { From b3cbfb7927922e4586f890a70a22f13130247b4e Mon Sep 17 00:00:00 2001 From: Jagan Teki Date: Thu, 24 Nov 2022 00:01:18 +0530 Subject: [PATCH 77/94] dt-bindings: timer: rk-timer: Add rktimer for rv1126 Add rockchip timer compatible string for rockchip rv1126. Signed-off-by: Jagan Teki Acked-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20221123183124.6911-3-jagan@edgeble.ai Signed-off-by: Daniel Lezcano --- Documentation/devicetree/bindings/timer/rockchip,rk-timer.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/timer/rockchip,rk-timer.yaml b/Documentation/devicetree/bindings/timer/rockchip,rk-timer.yaml index b61ed1a431bb..65e59836a660 100644 --- a/Documentation/devicetree/bindings/timer/rockchip,rk-timer.yaml +++ b/Documentation/devicetree/bindings/timer/rockchip,rk-timer.yaml @@ -17,6 +17,7 @@ properties: - items: - enum: - rockchip,rv1108-timer + - rockchip,rv1126-timer - rockchip,rk3036-timer - rockchip,rk3128-timer - rockchip,rk3188-timer From 8b3b8fbb4896984b5564789a42240e4b3caddb61 Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Tue, 3 Jan 2023 19:41:00 +0530 Subject: [PATCH 78/94] RISC-V: time: initialize hrtimer based broadcast clock event device Similarly to commit 022eb8ae8b5e ("ARM: 8938/1: kernel: initialize broadcast hrtimer based clock event device"), RISC-V needs to initiate hrtimer based broadcast clock event device before C3STOP can be used. Otherwise, the introduction of C3STOP for the RISC-V arch timer in commit 232ccac1bd9b ("clocksource/drivers/riscv: Events are stopped during CPU suspend") leaves us without any broadcast timer registered. This prevents the kernel from entering oneshot mode, which breaks timer behaviour, for example clock_nanosleep(). A test app that sleeps each cpu for 6, 5, 4, 3 ms respectively, HZ=250 & C3STOP enabled, the sleep times are rounded up to the next jiffy: == CPU: 1 == == CPU: 2 == == CPU: 3 == == CPU: 4 == Mean: 7.974992 Mean: 7.976534 Mean: 7.962591 Mean: 3.952179 Std Dev: 0.154374 Std Dev: 0.156082 Std Dev: 0.171018 Std Dev: 0.076193 Hi: 9.472000 Hi: 10.495000 Hi: 8.864000 Hi: 4.736000 Lo: 6.087000 Lo: 6.380000 Lo: 4.872000 Lo: 3.403000 Samples: 521 Samples: 521 Samples: 521 Samples: 521 Link: https://lore.kernel.org/linux-riscv/YzYTNQRxLr7Q9JR0@spud/ Fixes: 232ccac1bd9b ("clocksource/drivers/riscv: Events are stopped during CPU suspend") Suggested-by: Samuel Holland Signed-off-by: Conor Dooley Signed-off-by: Anup Patel Reviewed-by: Samuel Holland Acked-by: Palmer Dabbelt Link: https://lore.kernel.org/r/20230103141102.772228-2-apatel@ventanamicro.com Signed-off-by: Daniel Lezcano --- arch/riscv/kernel/time.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/riscv/kernel/time.c b/arch/riscv/kernel/time.c index 8217b0f67c6c..1cf21db4fcc7 100644 --- a/arch/riscv/kernel/time.c +++ b/arch/riscv/kernel/time.c @@ -5,6 +5,7 @@ */ #include +#include #include #include #include @@ -29,6 +30,8 @@ void __init time_init(void) of_clk_init(NULL); timer_probe(); + + tick_setup_hrtimer_broadcast(); } void clocksource_arch_init(struct clocksource *cs) From e2bcf2d876fd7ca6ecca09794ac58d7e3a544794 Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Tue, 3 Jan 2023 19:41:01 +0530 Subject: [PATCH 79/94] dt-bindings: timer: Add bindings for the RISC-V timer device We add DT bindings for a separate RISC-V timer DT node which can be used to describe implementation specific behaviour (such as timer interrupt not triggered during non-retentive suspend). Signed-off-by: Anup Patel Reviewed-by: Conor Dooley Reviewed-by: Rob Herring Acked-by: Palmer Dabbelt Link: https://lore.kernel.org/r/20230103141102.772228-3-apatel@ventanamicro.com Signed-off-by: Daniel Lezcano --- .../bindings/timer/riscv,timer.yaml | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 Documentation/devicetree/bindings/timer/riscv,timer.yaml diff --git a/Documentation/devicetree/bindings/timer/riscv,timer.yaml b/Documentation/devicetree/bindings/timer/riscv,timer.yaml new file mode 100644 index 000000000000..38d67e1a5a79 --- /dev/null +++ b/Documentation/devicetree/bindings/timer/riscv,timer.yaml @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/timer/riscv,timer.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: RISC-V timer + +maintainers: + - Anup Patel + +description: |+ + RISC-V platforms always have a RISC-V timer device for the supervisor-mode + based on the time CSR defined by the RISC-V privileged specification. The + timer interrupts of this device are configured using the RISC-V SBI Time + extension or the RISC-V Sstc extension. + + The clock frequency of RISC-V timer device is specified via the + "timebase-frequency" DT property of "/cpus" DT node which is described + in Documentation/devicetree/bindings/riscv/cpus.yaml + +properties: + compatible: + enum: + - riscv,timer + + interrupts-extended: + minItems: 1 + maxItems: 4096 # Should be enough? + + riscv,timer-cannot-wake-cpu: + type: boolean + description: + If present, the timer interrupt cannot wake up the CPU from one or + more suspend/idle states. + +additionalProperties: false + +required: + - compatible + - interrupts-extended + +examples: + - | + timer { + compatible = "riscv,timer"; + interrupts-extended = <&cpu1intc 5>, + <&cpu2intc 5>, + <&cpu3intc 5>, + <&cpu4intc 5>; + }; +... From 8932a9533a9cdd1fa2924a061dc87277991507ca Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Tue, 3 Jan 2023 19:41:02 +0530 Subject: [PATCH 80/94] clocksource/drivers/timer-riscv: Set CLOCK_EVT_FEAT_C3STOP based on DT We should set CLOCK_EVT_FEAT_C3STOP for a clock_event_device only when riscv,timer-cannot-wake-cpu DT property is present in the RISC-V timer DT node. This way CLOCK_EVT_FEAT_C3STOP feature is set for clock_event_device based on RISC-V platform capabilities rather than having it set for all RISC-V platforms. Signed-off-by: Anup Patel Reviewed-by: Conor Dooley Acked-by: Palmer Dabbelt Link: https://lore.kernel.org/r/20230103141102.772228-4-apatel@ventanamicro.com Signed-off-by: Daniel Lezcano --- drivers/clocksource/timer-riscv.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/clocksource/timer-riscv.c b/drivers/clocksource/timer-riscv.c index a0d66fabf073..1b4b36df5484 100644 --- a/drivers/clocksource/timer-riscv.c +++ b/drivers/clocksource/timer-riscv.c @@ -28,6 +28,7 @@ #include static DEFINE_STATIC_KEY_FALSE(riscv_sstc_available); +static bool riscv_timer_cannot_wake_cpu; static int riscv_clock_next_event(unsigned long delta, struct clock_event_device *ce) @@ -85,6 +86,8 @@ static int riscv_timer_starting_cpu(unsigned int cpu) ce->cpumask = cpumask_of(cpu); ce->irq = riscv_clock_event_irq; + if (riscv_timer_cannot_wake_cpu) + ce->features |= CLOCK_EVT_FEAT_C3STOP; clockevents_config_and_register(ce, riscv_timebase, 100, 0x7fffffff); enable_percpu_irq(riscv_clock_event_irq, @@ -139,6 +142,13 @@ static int __init riscv_timer_init_dt(struct device_node *n) if (cpuid != smp_processor_id()) return 0; + child = of_find_compatible_node(NULL, NULL, "riscv,timer"); + if (child) { + riscv_timer_cannot_wake_cpu = of_property_read_bool(child, + "riscv,timer-cannot-wake-cpu"); + of_node_put(child); + } + domain = NULL; child = of_get_compatible_child(n, "riscv,cpu-intc"); if (!child) { From 674402b0098b66b8ba91fe93c0d27af703256098 Mon Sep 17 00:00:00 2001 From: Samuel Holland Date: Tue, 27 Dec 2022 18:44:44 -0600 Subject: [PATCH 81/94] clocksource/drivers/riscv: Increase the clock source rating RISC-V provides an architectural clock source via the time CSR. This clock source exposes a 64-bit counter synchronized across all CPUs. Because it is accessed using a CSR, it is much more efficient to read than MMIO clock sources. For example, on the Allwinner D1, reading the sun4i timer in a loop takes 131 cycles/iteration, while reading the RISC-V time CSR takes only 5 cycles/iteration. Adjust the RISC-V clock source rating so it is preferred over the various platform-specific MMIO clock sources. Signed-off-by: Samuel Holland Acked-by: Palmer Dabbelt Reviewed-by: Palmer Dabbelt Reviewed-by: Anup Patel Reviewed-by: Lad Prabhakar Link: https://lore.kernel.org/r/20221228004444.61568-1-samuel@sholland.org Signed-off-by: Daniel Lezcano --- drivers/clocksource/timer-riscv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/clocksource/timer-riscv.c b/drivers/clocksource/timer-riscv.c index 1b4b36df5484..adf7f98aab17 100644 --- a/drivers/clocksource/timer-riscv.c +++ b/drivers/clocksource/timer-riscv.c @@ -74,7 +74,7 @@ static u64 notrace riscv_sched_clock(void) static struct clocksource riscv_clocksource = { .name = "riscv_clocksource", - .rating = 300, + .rating = 400, .mask = CLOCKSOURCE_MASK(64), .flags = CLOCK_SOURCE_IS_CONTINUOUS, .read = riscv_clocksource_rdtime, From 8d17aca90bcf36833271de6531edb58a91c40e9f Mon Sep 17 00:00:00 2001 From: Jean Delvare Date: Sat, 21 Jan 2023 18:29:11 +0100 Subject: [PATCH 82/94] clocksource/drivers/timer-microchip-pit64b: Drop obsolete dependency on COMPILE_TEST Since commit 0166dc11be91 ("of: make CONFIG_OF user selectable"), it is possible to test-build any driver which depends on OF on any architecture by explicitly selecting OF. Therefore depending on COMPILE_TEST as an alternative is no longer needed. Signed-off-by: Jean Delvare Cc: Claudiu Beznea Cc: Daniel Lezcano Cc: Thomas Gleixner Reviewed-by: Claudiu Beznea Link: https://lore.kernel.org/r/20230121182911.4e47a5ff@endymion.delvare Signed-off-by: Daniel Lezcano --- drivers/clocksource/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index 4469e7f555e9..45085abbe197 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -706,7 +706,7 @@ config INGENIC_OST config MICROCHIP_PIT64B bool "Microchip PIT64B support" - depends on OF || COMPILE_TEST + depends on OF select TIMER_OF help This option enables Microchip PIT64B timer for Atmel From c3daa4754f3c57231bf47dcf4bdf897bc5c5f1f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Mon, 23 Jan 2023 23:02:21 +0100 Subject: [PATCH 83/94] clocksource/drivers/sh_cmt: Mark driver as non-removable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The comment in the remove callback suggests that the driver is not supposed to be unbound. However returning an error code in the remove callback doesn't accomplish that. Instead set the suppress_bind_attrs property (which makes it impossible to unbind the driver via sysfs). The only remaining way to unbind a sh_cmt device would be module unloading, but that doesn't apply here, as the driver cannot be built as a module. Also drop the useless remove callback. Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20230123220221.48164-1-u.kleine-koenig@pengutronix.de Signed-off-by: Daniel Lezcano --- drivers/clocksource/sh_cmt.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/drivers/clocksource/sh_cmt.c b/drivers/clocksource/sh_cmt.c index 7b952aa52c0b..8b2e079d9df2 100644 --- a/drivers/clocksource/sh_cmt.c +++ b/drivers/clocksource/sh_cmt.c @@ -1145,17 +1145,12 @@ static int sh_cmt_probe(struct platform_device *pdev) return 0; } -static int sh_cmt_remove(struct platform_device *pdev) -{ - return -EBUSY; /* cannot unregister clockevent and clocksource */ -} - static struct platform_driver sh_cmt_device_driver = { .probe = sh_cmt_probe, - .remove = sh_cmt_remove, .driver = { .name = "sh_cmt", .of_match_table = of_match_ptr(sh_cmt_of_table), + .suppress_bind_attrs = true, }, .id_table = sh_cmt_id_table, }; From 3aff0403f814df6ce2377a6ecf61dd7750a3925f Mon Sep 17 00:00:00 2001 From: Lad Prabhakar Date: Thu, 29 Dec 2022 22:46:01 +0000 Subject: [PATCH 84/94] clocksource/drivers/riscv: Get rid of clocksource_arch_init() callback Having a clocksource_arch_init() callback always sets vdso_clock_mode to VDSO_CLOCKMODE_ARCHTIMER if GENERIC_GETTIMEOFDAY is enabled, this is required for the riscv-timer. This works for platforms where just riscv-timer clocksource is present. On platforms where other clock sources are available we want them to register with vdso_clock_mode set to VDSO_CLOCKMODE_NONE. On the Renesas RZ/Five SoC OSTM block can be used as clocksource [0], to avoid multiple clock sources being registered as VDSO_CLOCKMODE_ARCHTIMER move setting of vdso_clock_mode in the riscv-timer driver instead of doing this in clocksource_arch_init() callback as done similarly for ARM/64 architecture. [0] drivers/clocksource/renesas-ostm.c Signed-off-by: Lad Prabhakar Tested-by: Samuel Holland Reviewed-by: Conor Dooley Reviewed-by: Samuel Holland Link: https://lore.kernel.org/r/20221229224601.103851-1-prabhakar.mahadev-lad.rj@bp.renesas.com Signed-off-by: Daniel Lezcano --- arch/riscv/Kconfig | 1 - arch/riscv/kernel/time.c | 9 --------- drivers/clocksource/timer-riscv.c | 5 +++++ 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index e2b656043abf..9c687da7756d 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -12,7 +12,6 @@ config 32BIT config RISCV def_bool y - select ARCH_CLOCKSOURCE_INIT select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 select ARCH_HAS_BINFMT_FLAT diff --git a/arch/riscv/kernel/time.c b/arch/riscv/kernel/time.c index 1cf21db4fcc7..babaf3b48ba8 100644 --- a/arch/riscv/kernel/time.c +++ b/arch/riscv/kernel/time.c @@ -33,12 +33,3 @@ void __init time_init(void) tick_setup_hrtimer_broadcast(); } - -void clocksource_arch_init(struct clocksource *cs) -{ -#ifdef CONFIG_GENERIC_GETTIMEOFDAY - cs->vdso_clock_mode = VDSO_CLOCKMODE_ARCHTIMER; -#else - cs->vdso_clock_mode = VDSO_CLOCKMODE_NONE; -#endif -} diff --git a/drivers/clocksource/timer-riscv.c b/drivers/clocksource/timer-riscv.c index adf7f98aab17..d8cb629d3aab 100644 --- a/drivers/clocksource/timer-riscv.c +++ b/drivers/clocksource/timer-riscv.c @@ -78,6 +78,11 @@ static struct clocksource riscv_clocksource = { .mask = CLOCKSOURCE_MASK(64), .flags = CLOCK_SOURCE_IS_CONTINUOUS, .read = riscv_clocksource_rdtime, +#if IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY) + .vdso_clock_mode = VDSO_CLOCKMODE_ARCHTIMER, +#else + .vdso_clock_mode = VDSO_CLOCKMODE_NONE, +#endif }; static int riscv_timer_starting_cpu(unsigned int cpu) From 27788e01a6507c882946b9aa87318c3185cd83e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?= Date: Wed, 25 Jan 2023 15:35:01 +0100 Subject: [PATCH 85/94] dt-bindings: timer: mediatek,mtk-timer: add MT8365 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add binding description for mediatek,mt8365-systimer Signed-off-by: Bernhard Rosenkränzer Acked-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20230125143503.1015424-8-bero@baylibre.com Signed-off-by: Daniel Lezcano --- Documentation/devicetree/bindings/timer/mediatek,mtk-timer.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/timer/mediatek,mtk-timer.txt b/Documentation/devicetree/bindings/timer/mediatek,mtk-timer.txt index 8bbb6e94508b..b3e797e8aa31 100644 --- a/Documentation/devicetree/bindings/timer/mediatek,mtk-timer.txt +++ b/Documentation/devicetree/bindings/timer/mediatek,mtk-timer.txt @@ -33,6 +33,7 @@ Required properties: For those SoCs that use CPUX * "mediatek,mt6795-systimer" for MT6795 compatible timers (CPUX) + * "mediatek,mt8365-systimer" for MT8365 compatible timers (CPUX) - reg: Should contain location and length for timer register. - clocks: Should contain system clock. From abd873afc889c0b4348ec4b567d83f97df8edaf6 Mon Sep 17 00:00:00 2001 From: Icenowy Zheng Date: Thu, 2 Feb 2023 15:28:14 +0800 Subject: [PATCH 86/94] dt-bindings: timer: sifive,clint: add comaptibles for T-Head's C9xx T-Head C906/C910 CLINT is not compliant to SiFive ones (and even not compliant to the newcoming ACLINT spec) because of lack of mtime register. Add a compatible string formatted like the C9xx-specific PLIC compatible, and do not allow a SiFive one as fallback because they're not really compliant. Signed-off-by: Icenowy Zheng Acked-by: Krzysztof Kozlowski Reviewed-by: Samuel Holland Acked-by: Conor Dooley Link: https://lore.kernel.org/r/20230202072814.319903-1-uwu@icenowy.me Signed-off-by: Daniel Lezcano --- Documentation/devicetree/bindings/timer/sifive,clint.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Documentation/devicetree/bindings/timer/sifive,clint.yaml b/Documentation/devicetree/bindings/timer/sifive,clint.yaml index bbad24165837..aada6957216c 100644 --- a/Documentation/devicetree/bindings/timer/sifive,clint.yaml +++ b/Documentation/devicetree/bindings/timer/sifive,clint.yaml @@ -20,6 +20,10 @@ description: property of "/cpus" DT node. The "timebase-frequency" DT property is described in Documentation/devicetree/bindings/riscv/cpus.yaml + T-Head C906/C910 CPU cores include an implementation of CLINT too, however + their implementation lacks a memory-mapped MTIME register, thus not + compatible with SiFive ones. + properties: compatible: oneOf: @@ -29,6 +33,10 @@ properties: - starfive,jh7100-clint - canaan,k210-clint - const: sifive,clint0 + - items: + - enum: + - allwinner,sun20i-d1-clint + - const: thead,c900-clint - items: - const: sifive,clint0 - const: riscv,clint0 From d19c8b2ed176c6aaf3dfefd149c740f73ce4875e Mon Sep 17 00:00:00 2001 From: Claudiu Beznea Date: Fri, 3 Feb 2023 15:05:36 +0200 Subject: [PATCH 87/94] clocksource/drivers/timer-microchip-pit64b: Select driver only on ARM Microchip PIT64B is currently available on ARM based devices. Thus select it only for ARM. This allows implementing delay timer. Signed-off-by: Claudiu Beznea Link: https://lore.kernel.org/r/20230203130537.1921608-2-claudiu.beznea@microchip.com Signed-off-by: Daniel Lezcano --- drivers/clocksource/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index 45085abbe197..ac22c1af056b 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -706,7 +706,7 @@ config INGENIC_OST config MICROCHIP_PIT64B bool "Microchip PIT64B support" - depends on OF + depends on OF && ARM select TIMER_OF help This option enables Microchip PIT64B timer for Atmel From f3af3dc7cc351bd22742eac992c72c76749ce8c3 Mon Sep 17 00:00:00 2001 From: Claudiu Beznea Date: Fri, 3 Feb 2023 15:05:37 +0200 Subject: [PATCH 88/94] clocksource/drivers/timer-microchip-pit64b: Add delay timer Add delay timer. Signed-off-by: Claudiu Beznea Link: https://lore.kernel.org/r/20230203130537.1921608-3-claudiu.beznea@microchip.com Signed-off-by: Daniel Lezcano --- drivers/clocksource/timer-microchip-pit64b.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/clocksource/timer-microchip-pit64b.c b/drivers/clocksource/timer-microchip-pit64b.c index d5f1436f33d9..57209bb38c70 100644 --- a/drivers/clocksource/timer-microchip-pit64b.c +++ b/drivers/clocksource/timer-microchip-pit64b.c @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -92,6 +93,8 @@ struct mchp_pit64b_clksrc { static void __iomem *mchp_pit64b_cs_base; /* Default cycles for clockevent timer. */ static u64 mchp_pit64b_ce_cycles; +/* Delay timer. */ +static struct delay_timer mchp_pit64b_dt; static inline u64 mchp_pit64b_cnt_read(void __iomem *base) { @@ -169,6 +172,11 @@ static u64 notrace mchp_pit64b_sched_read_clk(void) return mchp_pit64b_cnt_read(mchp_pit64b_cs_base); } +static unsigned long notrace mchp_pit64b_dt_read(void) +{ + return mchp_pit64b_cnt_read(mchp_pit64b_cs_base); +} + static int mchp_pit64b_clkevt_shutdown(struct clock_event_device *cedev) { struct mchp_pit64b_timer *timer = clkevt_to_mchp_pit64b_timer(cedev); @@ -376,6 +384,10 @@ static int __init mchp_pit64b_init_clksrc(struct mchp_pit64b_timer *timer, sched_clock_register(mchp_pit64b_sched_read_clk, 64, clk_rate); + mchp_pit64b_dt.read_current_timer = mchp_pit64b_dt_read; + mchp_pit64b_dt.freq = clk_rate; + register_current_timer_delay(&mchp_pit64b_dt); + return 0; } From 225b9596cb0227c1c1b1e4a836dad43595c3e61a Mon Sep 17 00:00:00 2001 From: Matt Evans Date: Wed, 1 Feb 2023 19:49:42 +0000 Subject: [PATCH 89/94] clocksource/drivers/riscv: Patch riscv_clock_next_event() jump before first use A static key is used to select between SBI and Sstc timer usage in riscv_clock_next_event(), but currently the direction is resolved after cpuhp_setup_state() is called (which sets the next event). The first event will therefore fall through the sbi_set_timer() path; this breaks Sstc-only systems. So, apply the jump patching before first use. Fixes: 9f7a8ff6391f ("RISC-V: Prefer sstc extension if available") Signed-off-by: Matt Evans Reviewed-by: Palmer Dabbelt Acked-by: Palmer Dabbelt Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/CDDAB2D0-264E-42F3-8E31-BA210BEB8EC1@rivosinc.com Signed-off-by: Daniel Lezcano --- drivers/clocksource/timer-riscv.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/clocksource/timer-riscv.c b/drivers/clocksource/timer-riscv.c index d8cb629d3aab..5f0f10c7e222 100644 --- a/drivers/clocksource/timer-riscv.c +++ b/drivers/clocksource/timer-riscv.c @@ -192,6 +192,11 @@ static int __init riscv_timer_init_dt(struct device_node *n) return error; } + if (riscv_isa_extension_available(NULL, SSTC)) { + pr_info("Timer interrupt in S-mode is available via sstc extension\n"); + static_branch_enable(&riscv_sstc_available); + } + error = cpuhp_setup_state(CPUHP_AP_RISCV_TIMER_STARTING, "clockevents/riscv/timer:starting", riscv_timer_starting_cpu, riscv_timer_dying_cpu); @@ -199,11 +204,6 @@ static int __init riscv_timer_init_dt(struct device_node *n) pr_err("cpu hp setup state failed for RISCV timer [%d]\n", error); - if (riscv_isa_extension_available(NULL, SSTC)) { - pr_info("Timer interrupt in S-mode is available via sstc extension\n"); - static_branch_enable(&riscv_sstc_available); - } - return error; } From d8c695d310a8607fd1d6a2750368467c4e0cef6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Tue, 7 Feb 2023 20:36:14 +0100 Subject: [PATCH 90/94] clocksource/drivers/sh_tmu: Mark driver as non-removable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The comment in the remove callback suggests that the driver is not supposed to be unbound. However returning an error code in the remove callback doesn't accomplish that. Instead set the suppress_bind_attrs property (which makes it impossible to unbind the driver via sysfs). The only remaining way to unbind a sh_tmu device would be module unloading, but that doesn't apply here, as the driver cannot be built as a module. Also drop the useless remove callback. Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20230207193614.472060-1-u.kleine-koenig@pengutronix.de Signed-off-by: Daniel Lezcano --- drivers/clocksource/sh_tmu.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/drivers/clocksource/sh_tmu.c b/drivers/clocksource/sh_tmu.c index b00dec0655cb..932f31a7c5be 100644 --- a/drivers/clocksource/sh_tmu.c +++ b/drivers/clocksource/sh_tmu.c @@ -632,11 +632,6 @@ static int sh_tmu_probe(struct platform_device *pdev) return 0; } -static int sh_tmu_remove(struct platform_device *pdev) -{ - return -EBUSY; /* cannot unregister clockevent and clocksource */ -} - static const struct platform_device_id sh_tmu_id_table[] = { { "sh-tmu", SH_TMU }, { "sh-tmu-sh3", SH_TMU_SH3 }, @@ -652,10 +647,10 @@ MODULE_DEVICE_TABLE(of, sh_tmu_of_table); static struct platform_driver sh_tmu_device_driver = { .probe = sh_tmu_probe, - .remove = sh_tmu_remove, .driver = { .name = "sh_tmu", .of_match_table = of_match_ptr(sh_tmu_of_table), + .suppress_bind_attrs = true, }, .id_table = sh_tmu_id_table, }; From cf16f631b0bbd98df4b2ccc13346f564412eae68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Tue, 7 Feb 2023 20:30:10 +0100 Subject: [PATCH 91/94] clocksource/drivers/em_sti: Mark driver as non-removable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The comment in the remove callback suggests that the driver is not supposed to be unbound. However returning an error code in the remove callback doesn't accomplish that. Instead set the suppress_bind_attrs property (which makes it impossible to unbind the driver via sysfs). The only remaining way to unbind a em_sti device would be module unloading, but that doesn't apply here, as the driver cannot be built as a module. Also drop the useless remove callback. Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20230207193010.469495-1-u.kleine-koenig@pengutronix.de Signed-off-by: Daniel Lezcano --- drivers/clocksource/em_sti.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/drivers/clocksource/em_sti.c b/drivers/clocksource/em_sti.c index ab190dffb1ed..c04b47bd4868 100644 --- a/drivers/clocksource/em_sti.c +++ b/drivers/clocksource/em_sti.c @@ -333,11 +333,6 @@ static int em_sti_probe(struct platform_device *pdev) return 0; } -static int em_sti_remove(struct platform_device *pdev) -{ - return -EBUSY; /* cannot unregister clockevent and clocksource */ -} - static const struct of_device_id em_sti_dt_ids[] = { { .compatible = "renesas,em-sti", }, {}, @@ -346,10 +341,10 @@ MODULE_DEVICE_TABLE(of, em_sti_dt_ids); static struct platform_driver em_sti_device_driver = { .probe = em_sti_probe, - .remove = em_sti_remove, .driver = { .name = "em_sti", .of_match_table = em_sti_dt_ids, + .suppress_bind_attrs = true, } }; From 5ccb51b06c8bfcfca99d3f68e671eead7318301d Mon Sep 17 00:00:00 2001 From: Yangtao Li Date: Thu, 9 Feb 2023 12:02:39 +0800 Subject: [PATCH 92/94] clocksource/drivers/timer-sun4i: Add CLOCK_EVT_FEAT_DYNIRQ Add CLOCK_EVT_FEAT_DYNIRQ to allow the IRQ could be runtime set affinity to the cores that needs wake up, otherwise saying core0 has to send IPI to wakeup core1. With CLOCK_EVT_FEAT_DYNIRQ set, when broadcast timer could wake up the cores, IPI is not needed. After enabling this feature, especially the scene where cpuidle is enabled can benefit. Signed-off-by: Yangtao Li Link: https://lore.kernel.org/r/20230209040239.24710-1-frank.li@vivo.com Signed-off-by: Daniel Lezcano --- drivers/clocksource/timer-sun4i.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/clocksource/timer-sun4i.c b/drivers/clocksource/timer-sun4i.c index e5a70aa1deb4..7bdcc60ad43c 100644 --- a/drivers/clocksource/timer-sun4i.c +++ b/drivers/clocksource/timer-sun4i.c @@ -144,7 +144,8 @@ static struct timer_of to = { .clkevt = { .name = "sun4i_tick", .rating = 350, - .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, + .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT | + CLOCK_EVT_FEAT_DYNIRQ, .set_state_shutdown = sun4i_clkevt_shutdown, .set_state_periodic = sun4i_clkevt_set_periodic, .set_state_oneshot = sun4i_clkevt_set_oneshot, From 28980db94742f9f2fb0f68ea35f2171b38007bae Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Tue, 14 Feb 2023 10:42:22 +0100 Subject: [PATCH 93/94] EDAC/amd64: Shut up an -Werror,-Wsometimes-uninitialized clang false positive Reportedly, clang cannot do interprocedural analysis: https://lore.kernel.org/r/20230213-amd64_edac-wsometimes-uninitialized-v1-1-5bde32b89e02@kernel.org and see that those arguments won't be used uninitialized. So, yeah, the code's fine even without this. Normally, such a "fix" won't be applied but that warning gets automatically enabled in -Wall builds and when CONFIG_WERROR is set in allmodconfig builds, the build fails. So shut it up with a minimal fix as this code will see more reorganization very soon. [ bp: Write commit message. ] Signed-off-by: Yazen Ghannam Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Nathan Chancellor Link: https://lore.kernel.org/r/Y%2BqdVHidnrrKvxiD@dev-arch.thelio-3990X --- drivers/edac/amd64_edac.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 1c4bef1cdf28..5b42533f306a 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -3928,7 +3928,7 @@ static const struct attribute_group *amd64_edac_attr_groups[] = { static int hw_info_get(struct amd64_pvt *pvt) { - u16 pci_id1, pci_id2; + u16 pci_id1 = 0, pci_id2 = 0; int ret; if (pvt->fam >= 0x17) { From fcb3a81d221750d2a54b4e3a82b3efbbeab8780c Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat (VMware)" Date: Fri, 27 Jan 2023 16:37:51 -0800 Subject: [PATCH 94/94] x86/hotplug: Remove incorrect comment about mwait_play_dead() The comment that says mwait_play_dead() returns only on failure is a bit misleading because mwait_play_dead() could actually return for valid reasons (such as mwait not being supported by the platform) that do not indicate a failure of the CPU offline operation. So, remove the comment. Suggested-by: Thomas Gleixner Signed-off-by: Srivatsa S. Bhat (VMware) Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20230128003751.141317-1-srivatsa@csail.mit.edu --- arch/x86/kernel/smpboot.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index 55cad72715d9..9013bb28255a 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -1833,7 +1833,7 @@ void native_play_dead(void) play_dead_common(); tboot_shutdown(TB_SHUTDOWN_WFS); - mwait_play_dead(); /* Only returns on failure */ + mwait_play_dead(); if (cpuidle_play_dead()) hlt_play_dead(); }