From 5b2ad5acaf5aa8a1ff441665967cf728a46ac967 Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Mon, 5 Dec 2022 19:43:43 -0600 Subject: [PATCH 001/239] dt-bindings: opp: opp-v2-kryo-cpu: Add missing 'cache-unified' property in example The examples' cache nodes are incomplete as 'cache-unified' is a required cache property for unified caches which an L2 cache certainly is. Signed-off-by: Rob Herring Signed-off-by: Viresh Kumar --- Documentation/devicetree/bindings/opp/opp-v2-kryo-cpu.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/devicetree/bindings/opp/opp-v2-kryo-cpu.yaml b/Documentation/devicetree/bindings/opp/opp-v2-kryo-cpu.yaml index 60cf3cbde4c5..b4947b326773 100644 --- a/Documentation/devicetree/bindings/opp/opp-v2-kryo-cpu.yaml +++ b/Documentation/devicetree/bindings/opp/opp-v2-kryo-cpu.yaml @@ -106,6 +106,7 @@ examples: L2_0: l2-cache { compatible = "cache"; cache-level = <2>; + cache-unified; }; }; @@ -140,6 +141,7 @@ examples: L2_1: l2-cache { compatible = "cache"; cache-level = <2>; + cache-unified; }; }; From 6ad90f71129219f9b87ce181189817bcbcde34b8 Mon Sep 17 00:00:00 2001 From: Stuart Hayes Date: Thu, 8 Dec 2022 17:25:36 -0600 Subject: [PATCH 002/239] ACPI: tables: Add support for NBFT Add support for the NVMe Boot Firmware Table (NBFT) to facilitate booting from NVM Express namespaces which are accessed via NVMe over Fabrics (NVMe-oF). Signed-off-by: Stuart Hayes Reviewed-by: John Meneghini Signed-off-by: Rafael J. Wysocki --- drivers/acpi/tables.c | 3 ++- include/acpi/actbl1.h | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c index 5fbc32b802d0..7b4680da57d7 100644 --- a/drivers/acpi/tables.c +++ b/drivers/acpi/tables.c @@ -555,7 +555,8 @@ static const char table_sigs[][ACPI_NAMESEG_SIZE] __initconst = { ACPI_SIG_WDDT, ACPI_SIG_WDRT, ACPI_SIG_DSDT, ACPI_SIG_FADT, ACPI_SIG_PSDT, ACPI_SIG_RSDT, ACPI_SIG_XSDT, ACPI_SIG_SSDT, ACPI_SIG_IORT, ACPI_SIG_NFIT, ACPI_SIG_HMAT, ACPI_SIG_PPTT, - ACPI_SIG_NHLT, ACPI_SIG_AEST, ACPI_SIG_CEDT, ACPI_SIG_AGDI }; + ACPI_SIG_NHLT, ACPI_SIG_AEST, ACPI_SIG_CEDT, ACPI_SIG_AGDI, + ACPI_SIG_NBFT }; #define ACPI_HEADER_SIZE sizeof(struct acpi_table_header) diff --git a/include/acpi/actbl1.h b/include/acpi/actbl1.h index 4175dce3967c..72884efc0d86 100644 --- a/include/acpi/actbl1.h +++ b/include/acpi/actbl1.h @@ -50,6 +50,8 @@ #define ACPI_SIG_S3PT "S3PT" /* S3 Performance (sub)Table */ #define ACPI_SIG_PCCS "PCC" /* PCC Shared Memory Region */ +#define ACPI_SIG_NBFT "NBFT" /* NVMe Boot Firmware Table */ + /* Reserved table signatures */ #define ACPI_SIG_MATR "MATR" /* Memory Address Translation Table */ From 73d15a7c13bfe1292db85ed3de8dfa711291819d Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Tue, 13 Dec 2022 13:34:54 +0100 Subject: [PATCH 003/239] ACPI: PMIC: Add pmic_i2c_address to BYT Crystal Cove support Add a pmic_i2c_address entry to intel_pmic_bytcrc.c, so that intel_soc_pmic_exec_mipi_pmic_seq_element() can be used on devices with a Bay Trail Crystal Cove PMIC OpRegion driver. Signed-off-by: Hans de Goede Reviewed-by: Andy Shevchenko Signed-off-by: Rafael J. Wysocki --- drivers/acpi/pmic/intel_pmic_bytcrc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/acpi/pmic/intel_pmic_bytcrc.c b/drivers/acpi/pmic/intel_pmic_bytcrc.c index 9ea79f210965..2b09f8da5400 100644 --- a/drivers/acpi/pmic/intel_pmic_bytcrc.c +++ b/drivers/acpi/pmic/intel_pmic_bytcrc.c @@ -283,6 +283,7 @@ static const struct intel_pmic_opregion_data intel_crc_pmic_opregion_data = { .power_table_count= ARRAY_SIZE(power_table), .thermal_table = thermal_table, .thermal_table_count = ARRAY_SIZE(thermal_table), + .pmic_i2c_address = 0x6e, }; static int intel_crc_pmic_opregion_probe(struct platform_device *pdev) From e1d9148582ab2c3dada5c5cf8ca7531ca269fee5 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Thu, 15 Dec 2022 09:51:20 -0600 Subject: [PATCH 004/239] ACPICA: Drop port I/O validation for some regions Microsoft introduced support in Windows XP for blocking port I/O to various regions. For Windows compatibility ACPICA has adopted the same protections and will disallow writes to those (presumably) the same regions. On some systems the AML included with the firmware will issue 4 byte long writes to 0x80. These writes aren't making it over because of this blockage. The first 4 byte write attempt is rejected, and then subsequently 1 byte at a time each offset is tried. The first at 0x80 works, but then the next 3 bytes are rejected. This manifests in bizarre failures for devices that expected the AML to write all 4 bytes. Trying the same AML on Windows 10 or 11 doesn't hit this failure and all 4 bytes are written. Either some of these regions were wrong or some point after Windows XP some of these regions blocks have been lifted. In the last 15 years there doesn't seem to be any reports popping up of this error in the Windows event viewer anymore. There is no documentation at Microsoft's developer site indicating that Windows ACPI interpreter blocks these regions. Between the lack of documentation and the fact that the writes actually do work in Windows 10 and 11, it's quite likely Windows doesn't actually enforce this anymore. So to help the issue, only enforce Windows XP specific entries if the latest _OSI supported is Windows XP. Continue to enforce the ALWAYS_ILLEGAL entries. Link: https://github.com/acpica/acpica/pull/817 Fixes: 7f0719039085 ("ACPICA: New: I/O port protection") Signed-off-by: Mario Limonciello Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpica/hwvalid.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/acpi/acpica/hwvalid.c b/drivers/acpi/acpica/hwvalid.c index 915b26448d2c..0d392e7b0747 100644 --- a/drivers/acpi/acpica/hwvalid.c +++ b/drivers/acpi/acpica/hwvalid.c @@ -23,8 +23,8 @@ acpi_hw_validate_io_request(acpi_io_address address, u32 bit_width); * * The table is used to implement the Microsoft port access rules that * first appeared in Windows XP. Some ports are always illegal, and some - * ports are only illegal if the BIOS calls _OSI with a win_XP string or - * later (meaning that the BIOS itelf is post-XP.) + * ports are only illegal if the BIOS calls _OSI with nothing newer than + * the specific _OSI strings. * * This provides ACPICA with the desired port protections and * Microsoft compatibility. @@ -145,7 +145,8 @@ acpi_hw_validate_io_request(acpi_io_address address, u32 bit_width) /* Port illegality may depend on the _OSI calls made by the BIOS */ - if (acpi_gbl_osi_data >= port_info->osi_dependency) { + if (port_info->osi_dependency == ACPI_ALWAYS_ILLEGAL || + acpi_gbl_osi_data == port_info->osi_dependency) { ACPI_DEBUG_PRINT((ACPI_DB_VALUES, "Denied AML access to port 0x%8.8X%8.8X/%X (%s 0x%.4X-0x%.4X)\n", ACPI_FORMAT_UINT64(address), From 91fdb91ccca2b48572a1ccf1d382fd599e3e1237 Mon Sep 17 00:00:00 2001 From: Sakari Ailus Date: Tue, 20 Dec 2022 17:01:26 +0200 Subject: [PATCH 005/239] ACPICA: Constify pathname argument for acpi_get_handle() acpi_get_handle() uses the pathname argument to find a handle related to that pathname but it does not need to modify it. Make it const, in order to be able to pass const pathname to it. Link: https://github.com/acpica/acpica/pull/773 Signed-off-by: Sakari Ailus Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpica/nsxfname.c | 2 +- include/acpi/acpixf.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/acpi/acpica/nsxfname.c b/drivers/acpi/acpica/nsxfname.c index b2cfdfef3194..a0592d15dd37 100644 --- a/drivers/acpi/acpica/nsxfname.c +++ b/drivers/acpi/acpica/nsxfname.c @@ -44,7 +44,7 @@ static char *acpi_ns_copy_device_id(struct acpi_pnp_device_id *dest, acpi_status acpi_get_handle(acpi_handle parent, - acpi_string pathname, acpi_handle *ret_handle) + const char *pathname, acpi_handle *ret_handle) { acpi_status status; struct acpi_namespace_node *node = NULL; diff --git a/include/acpi/acpixf.h b/include/acpi/acpixf.h index 9778408f8db4..8e364cbdd14a 100644 --- a/include/acpi/acpixf.h +++ b/include/acpi/acpixf.h @@ -526,7 +526,7 @@ ACPI_EXTERNAL_RETURN_STATUS(acpi_status struct acpi_buffer *ret_path_ptr)) ACPI_EXTERNAL_RETURN_STATUS(acpi_status acpi_get_handle(acpi_handle parent, - acpi_string pathname, + const char *pathname, acpi_handle *ret_handle)) ACPI_EXTERNAL_RETURN_STATUS(acpi_status acpi_attach_data(acpi_handle object, From fbf757e55afbd8b4d36f4808a6ae9107afeb79f3 Mon Sep 17 00:00:00 2001 From: Zhou jie Date: Fri, 23 Dec 2022 17:02:15 +0800 Subject: [PATCH 006/239] ACPI: processor: idle: Drop unnecessary (void *) conversion The (void *) type pointer does not need to be cast, so don't do that in lapic_timer_check_state(). Signed-off-by: Zhou jie [ rjw: Subject and changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/processor_idle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c index 7bf882fcd64b..5a01c2a1567f 100644 --- a/drivers/acpi/processor_idle.c +++ b/drivers/acpi/processor_idle.c @@ -147,7 +147,7 @@ static void lapic_timer_check_state(int state, struct acpi_processor *pr, static void __lapic_timer_propagate_broadcast(void *arg) { - struct acpi_processor *pr = (struct acpi_processor *) arg; + struct acpi_processor *pr = arg; if (pr->power.timer_broadcast_on_state < INT_MAX) tick_broadcast_enable(); From c02d5feb6e2f60affc6ba8606d8d614c071e2ba6 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Wed, 28 Dec 2022 22:21:49 +0100 Subject: [PATCH 007/239] ACPI: processor: perflib: Use the "no limit" frequency QoS When _PPC returns 0, it means that the CPU frequency is not limited by the platform firmware, so make acpi_processor_get_platform_limit() update the frequency QoS request used by it to "no limit" in that case. This addresses a problem with limiting CPU frequency artificially on some systems after CPU offline/online to the frequency that corresponds to the first entry in the _PSS return package. Reported-by: Pratyush Yadav Signed-off-by: Rafael J. Wysocki Reviewed-by: Pratyush Yadav Tested-by: Pratyush Yadav --- drivers/acpi/processor_perflib.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/drivers/acpi/processor_perflib.c b/drivers/acpi/processor_perflib.c index 970f04a958cd..0da85fc2a7e4 100644 --- a/drivers/acpi/processor_perflib.c +++ b/drivers/acpi/processor_perflib.c @@ -53,6 +53,8 @@ static int acpi_processor_get_platform_limit(struct acpi_processor *pr) { acpi_status status = 0; unsigned long long ppc = 0; + s32 qos_value; + int index; int ret; if (!pr) @@ -72,17 +74,27 @@ static int acpi_processor_get_platform_limit(struct acpi_processor *pr) } } - pr_debug("CPU %d: _PPC is %d - frequency %s limited\n", pr->id, - (int)ppc, ppc ? "" : "not"); + index = ppc; - pr->performance_platform_limit = (int)ppc; + pr_debug("CPU %d: _PPC is %d - frequency %s limited\n", pr->id, + index, index ? "is" : "is not"); + + pr->performance_platform_limit = index; if (ppc >= pr->performance->state_count || unlikely(!freq_qos_request_active(&pr->perflib_req))) return 0; - ret = freq_qos_update_request(&pr->perflib_req, - pr->performance->states[ppc].core_frequency * 1000); + /* + * If _PPC returns 0, it means that all of the available states can be + * used ("no limit"). + */ + if (index == 0) + qos_value = FREQ_QOS_MAX_DEFAULT_VALUE; + else + qos_value = pr->performance->states[index].core_frequency * 1000; + + ret = freq_qos_update_request(&pr->perflib_req, qos_value); if (ret < 0) { pr_warn("Failed to update perflib freq constraint: CPU%d (%d)\n", pr->id, ret); From 99387b016022c29234c4ebf9abd34358c6e56532 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Wed, 28 Dec 2022 22:24:10 +0100 Subject: [PATCH 008/239] ACPI: processor: perflib: Avoid updating frequency QoS unnecessarily Modify acpi_processor_get_platform_limit() to avoid updating its frequency QoS request when the _PPC return value has not changed by comparing that value to the previous _PPC return value stored in the performance_platform_limit field of the struct acpi_processor corresponding to the given CPU. While at it, do the _PPC return value check against the state count earlier, to avoid setting performance_platform_limit to an invalid value, and make acpi_processor_ppc_init() use FREQ_QOS_MAX_DEFAULT_VALUE as the "no limit" frequency QoS for consistency. Signed-off-by: Rafael J. Wysocki --- drivers/acpi/processor_perflib.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/drivers/acpi/processor_perflib.c b/drivers/acpi/processor_perflib.c index 0da85fc2a7e4..4265814c74f8 100644 --- a/drivers/acpi/processor_perflib.c +++ b/drivers/acpi/processor_perflib.c @@ -76,13 +76,16 @@ static int acpi_processor_get_platform_limit(struct acpi_processor *pr) index = ppc; + if (pr->performance_platform_limit == index || + ppc >= pr->performance->state_count) + return 0; + pr_debug("CPU %d: _PPC is %d - frequency %s limited\n", pr->id, index, index ? "is" : "is not"); pr->performance_platform_limit = index; - if (ppc >= pr->performance->state_count || - unlikely(!freq_qos_request_active(&pr->perflib_req))) + if (unlikely(!freq_qos_request_active(&pr->perflib_req))) return 0; /* @@ -178,9 +181,16 @@ void acpi_processor_ppc_init(struct cpufreq_policy *policy) if (!pr) continue; + /* + * Reset performance_platform_limit in case there is a stale + * value in it, so as to make it match the "no limit" QoS value + * below. + */ + pr->performance_platform_limit = 0; + ret = freq_qos_add_request(&policy->constraints, - &pr->perflib_req, - FREQ_QOS_MAX, INT_MAX); + &pr->perflib_req, FREQ_QOS_MAX, + FREQ_QOS_MAX_DEFAULT_VALUE); if (ret < 0) pr_err("Failed to add freq constraint for CPU%d (%d)\n", cpu, ret); From e8a0e30b742f76ebd0f3b196973df4bf65d8fbbb Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Wed, 28 Dec 2022 22:26:04 +0100 Subject: [PATCH 009/239] cpufreq: intel_pstate: Drop ACPI _PSS states table patching After making acpi_processor_get_platform_limit() use the "no limit" value for its frequency QoS request when _PPC returns 0, it is not necessary to replace the frequency corresponding to the first _PSS return package entry with the maximum turbo frequency of the given CPU in intel_pstate_init_acpi_perf_limits() any more, so drop the code doing that along with the comment explaining it. Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index fd73d6d2b808..cb4beec27555 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -452,20 +452,6 @@ static void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy) (u32) cpu->acpi_perf_data.states[i].control); } - /* - * The _PSS table doesn't contain whole turbo frequency range. - * This just contains +1 MHZ above the max non turbo frequency, - * with control value corresponding to max turbo ratio. But - * when cpufreq set policy is called, it will call with this - * max frequency, which will cause a reduced performance as - * this driver uses real max turbo frequency as the max - * frequency. So correct this frequency in _PSS table to - * correct max turbo frequency based on the turbo state. - * Also need to convert to MHz as _PSS freq is in MHz. - */ - if (!global.turbo_disabled) - cpu->acpi_perf_data.states[0].core_frequency = - policy->cpuinfo.max_freq / 1000; cpu->valid_pss_table = true; pr_debug("_PPC limits will be enforced\n"); From 446c85af665cf6d911c4821ac9507c8ba0cc7e75 Mon Sep 17 00:00:00 2001 From: Ammar Faizi Date: Fri, 23 Dec 2022 21:24:18 +0700 Subject: [PATCH 010/239] ACPI: Silence missing prototype warnings Compiling with clang-16: drivers/acpi/acpi_lpit.c:142:6: error: no previous prototype \ for function 'acpi_init_lpit' [-Werror,-Wmissing-prototypes] drivers/acpi/ioapic.c:212:6: error: no previous prototype \ for function 'pci_ioapic_remove' [-Werror,-Wmissing-prototypes] drivers/acpi/ioapic.c:229:5: error: no previous prototype \ for function 'acpi_ioapic_remove' [-Werror,-Wmissing-prototypes] Include "internal.h" to silence them. Signed-off-by: Ammar Faizi Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpi_lpit.c | 1 + drivers/acpi/ioapic.c | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/acpi/acpi_lpit.c b/drivers/acpi/acpi_lpit.c index 50540d4d4948..3843d2576d3f 100644 --- a/drivers/acpi/acpi_lpit.c +++ b/drivers/acpi/acpi_lpit.c @@ -10,6 +10,7 @@ #include #include #include +#include "internal.h" struct lpit_residency_info { struct acpi_generic_address gaddr; diff --git a/drivers/acpi/ioapic.c b/drivers/acpi/ioapic.c index a690c7b18623..6677955b4a8e 100644 --- a/drivers/acpi/ioapic.c +++ b/drivers/acpi/ioapic.c @@ -24,6 +24,7 @@ #include #include #include +#include "internal.h" struct acpi_pci_ioapic { acpi_handle root_handle; From 438500113f375c0b3185cc7ce62baa5f8088a5b2 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 4 Nov 2022 10:53:25 -0700 Subject: [PATCH 011/239] doc: Further updates to RCU's lockdep.rst This commit wordsmiths RCU's lockdep.rst. Signed-off-by: Paul E. McKenney --- Documentation/RCU/lockdep.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Documentation/RCU/lockdep.rst b/Documentation/RCU/lockdep.rst index 9308f1bdba05..2749f43ec1b0 100644 --- a/Documentation/RCU/lockdep.rst +++ b/Documentation/RCU/lockdep.rst @@ -69,9 +69,8 @@ checking of rcu_dereference() primitives: value of the pointer itself, for example, against NULL. The rcu_dereference_check() check expression can be any boolean -expression, but would normally include a lockdep expression. However, -any boolean expression can be used. For a moderately ornate example, -consider the following:: +expression, but would normally include a lockdep expression. For a +moderately ornate example, consider the following:: file = rcu_dereference_check(fdt->fd[fd], lockdep_is_held(&files->file_lock) || @@ -97,10 +96,10 @@ code, it could instead be written as follows:: atomic_read(&files->count) == 1); This would verify cases #2 and #3 above, and furthermore lockdep would -complain if this was used in an RCU read-side critical section unless one -of these two cases held. Because rcu_dereference_protected() omits all -barriers and compiler constraints, it generates better code than do the -other flavors of rcu_dereference(). On the other hand, it is illegal +complain even if this was used in an RCU read-side critical section unless +one of these two cases held. Because rcu_dereference_protected() omits +all barriers and compiler constraints, it generates better code than do +the other flavors of rcu_dereference(). On the other hand, it is illegal to use rcu_dereference_protected() if either the RCU-protected pointer or the RCU-protected data that it points to can change concurrently. From 8750dfe6fda4aca9cc02b3f652d14b14b49bccfb Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 4 Nov 2022 11:00:14 -0700 Subject: [PATCH 012/239] doc: Update NMI-RCU.rst This commit updates NMI-RCU.rst to highlight the ancient heritage of the example code and to discourage wanton compiler "optimizations". Signed-off-by: Paul E. McKenney --- Documentation/RCU/NMI-RCU.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/RCU/NMI-RCU.rst b/Documentation/RCU/NMI-RCU.rst index 2a92bc685ef1..dff60a80b386 100644 --- a/Documentation/RCU/NMI-RCU.rst +++ b/Documentation/RCU/NMI-RCU.rst @@ -8,7 +8,7 @@ Although RCU is usually used to protect read-mostly data structures, it is possible to use RCU to provide dynamic non-maskable interrupt handlers, as well as dynamic irq handlers. This document describes how to do this, drawing loosely from Zwane Mwaikambo's NMI-timer -work in "arch/x86/kernel/traps.c". +work in an old version of "arch/x86/kernel/traps.c". The relevant pieces of code are listed below, each followed by a brief explanation:: @@ -116,7 +116,7 @@ Answer to Quick Quiz: This same sad story can happen on other CPUs when using a compiler with aggressive pointer-value speculation - optimizations. + optimizations. (But please don't!) More important, the rcu_dereference_sched() makes it clear to someone reading the code that the pointer is From 42d689ec0016c24f2890f94ca0724cd467b3cb44 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 4 Nov 2022 11:44:45 -0700 Subject: [PATCH 013/239] doc: Update rcubarrier.rst This commit updates rcubarrier.txt to reflect RCU additions and changes over the past few years. [ paulmck: Apply Stephen Rothwell feedback. ] Signed-off-by: Paul E. McKenney --- Documentation/RCU/rcubarrier.rst | 192 +++++++++++++++++-------------- 1 file changed, 108 insertions(+), 84 deletions(-) diff --git a/Documentation/RCU/rcubarrier.rst b/Documentation/RCU/rcubarrier.rst index 3b4a24877496..5a643e5233d5 100644 --- a/Documentation/RCU/rcubarrier.rst +++ b/Documentation/RCU/rcubarrier.rst @@ -5,37 +5,12 @@ RCU and Unloadable Modules [Originally published in LWN Jan. 14, 2007: http://lwn.net/Articles/217484/] -RCU (read-copy update) is a synchronization mechanism that can be thought -of as a replacement for read-writer locking (among other things), but with -very low-overhead readers that are immune to deadlock, priority inversion, -and unbounded latency. RCU read-side critical sections are delimited -by rcu_read_lock() and rcu_read_unlock(), which, in non-CONFIG_PREEMPTION -kernels, generate no code whatsoever. - -This means that RCU writers are unaware of the presence of concurrent -readers, so that RCU updates to shared data must be undertaken quite -carefully, leaving an old version of the data structure in place until all -pre-existing readers have finished. These old versions are needed because -such readers might hold a reference to them. RCU updates can therefore be -rather expensive, and RCU is thus best suited for read-mostly situations. - -How can an RCU writer possibly determine when all readers are finished, -given that readers might well leave absolutely no trace of their -presence? There is a synchronize_rcu() primitive that blocks until all -pre-existing readers have completed. An updater wishing to delete an -element p from a linked list might do the following, while holding an -appropriate lock, of course:: - - list_del_rcu(p); - synchronize_rcu(); - kfree(p); - -But the above code cannot be used in IRQ context -- the call_rcu() -primitive must be used instead. This primitive takes a pointer to an -rcu_head struct placed within the RCU-protected data structure and -another pointer to a function that may be invoked later to free that -structure. Code to delete an element p from the linked list from IRQ -context might then be as follows:: +RCU updaters sometimes use call_rcu() to initiate an asynchronous wait for +a grace period to elapse. This primitive takes a pointer to an rcu_head +struct placed within the RCU-protected data structure and another pointer +to a function that may be invoked later to free that structure. Code to +delete an element p from the linked list from IRQ context might then be +as follows:: list_del_rcu(p); call_rcu(&p->rcu, p_callback); @@ -54,7 +29,7 @@ IRQ context. The function p_callback() might be defined as follows:: Unloading Modules That Use call_rcu() ------------------------------------- -But what if p_callback is defined in an unloadable module? +But what if the p_callback() function is defined in an unloadable module? If we unload the module while some RCU callbacks are pending, the CPUs executing these callbacks are going to be severely @@ -67,20 +42,21 @@ grace period to elapse, it does not wait for the callbacks to complete. One might be tempted to try several back-to-back synchronize_rcu() calls, but this is still not guaranteed to work. If there is a very -heavy RCU-callback load, then some of the callbacks might be deferred -in order to allow other processing to proceed. Such deferral is required -in realtime kernels in order to avoid excessive scheduling latencies. +heavy RCU-callback load, then some of the callbacks might be deferred in +order to allow other processing to proceed. For but one example, such +deferral is required in realtime kernels in order to avoid excessive +scheduling latencies. rcu_barrier() ------------- -We instead need the rcu_barrier() primitive. Rather than waiting for -a grace period to elapse, rcu_barrier() waits for all outstanding RCU -callbacks to complete. Please note that rcu_barrier() does **not** imply -synchronize_rcu(), in particular, if there are no RCU callbacks queued -anywhere, rcu_barrier() is within its rights to return immediately, -without waiting for a grace period to elapse. +This situation can be handled by the rcu_barrier() primitive. Rather +than waiting for a grace period to elapse, rcu_barrier() waits for all +outstanding RCU callbacks to complete. Please note that rcu_barrier() +does **not** imply synchronize_rcu(), in particular, if there are no RCU +callbacks queued anywhere, rcu_barrier() is within its rights to return +immediately, without waiting for anything, let alone a grace period. Pseudo-code using rcu_barrier() is as follows: @@ -89,19 +65,22 @@ Pseudo-code using rcu_barrier() is as follows: 3. Allow the module to be unloaded. There is also an srcu_barrier() function for SRCU, and you of course -must match the flavor of rcu_barrier() with that of call_rcu(). If your -module uses multiple flavors of call_rcu(), then it must also use multiple -flavors of rcu_barrier() when unloading that module. For example, if -it uses call_rcu(), call_srcu() on srcu_struct_1, and call_srcu() on -srcu_struct_2, then the following three lines of code will be required -when unloading:: +must match the flavor of srcu_barrier() with that of call_srcu(). +If your module uses multiple srcu_struct structures, then it must also +use multiple invocations of srcu_barrier() when unloading that module. +For example, if it uses call_rcu(), call_srcu() on srcu_struct_1, and +call_srcu() on srcu_struct_2, then the following three lines of code +will be required when unloading:: 1 rcu_barrier(); 2 srcu_barrier(&srcu_struct_1); 3 srcu_barrier(&srcu_struct_2); -The rcutorture module makes use of rcu_barrier() in its exit function -as follows:: +If latency is of the essence, workqueues could be used to run these +three functions concurrently. + +An ancient version of the rcutorture module makes use of rcu_barrier() +in its exit function as follows:: 1 static void 2 rcu_torture_cleanup(void) @@ -190,16 +169,17 @@ Quick Quiz #1: :ref:`Answer to Quick Quiz #1 ` Your module might have additional complications. For example, if your -module invokes call_rcu() from timers, you will need to first cancel all -the timers, and only then invoke rcu_barrier() to wait for any remaining +module invokes call_rcu() from timers, you will need to first refrain +from posting new timers, cancel (or wait for) all the already-posted +timers, and only then invoke rcu_barrier() to wait for any remaining RCU callbacks to complete. -Of course, if you module uses call_rcu(), you will need to invoke +Of course, if your module uses call_rcu(), you will need to invoke rcu_barrier() before unloading. Similarly, if your module uses call_srcu(), you will need to invoke srcu_barrier() before unloading, and on the same srcu_struct structure. If your module uses call_rcu() -**and** call_srcu(), then you will need to invoke rcu_barrier() **and** -srcu_barrier(). +**and** call_srcu(), then (as noted above) you will need to invoke +rcu_barrier() **and** srcu_barrier(). Implementing rcu_barrier() @@ -211,27 +191,40 @@ queues. His implementation queues an RCU callback on each of the per-CPU callback queues, and then waits until they have all started executing, at which point, all earlier RCU callbacks are guaranteed to have completed. -The original code for rcu_barrier() was as follows:: +The original code for rcu_barrier() was roughly as follows:: - 1 void rcu_barrier(void) - 2 { - 3 BUG_ON(in_interrupt()); - 4 /* Take cpucontrol mutex to protect against CPU hotplug */ - 5 mutex_lock(&rcu_barrier_mutex); - 6 init_completion(&rcu_barrier_completion); - 7 atomic_set(&rcu_barrier_cpu_count, 0); - 8 on_each_cpu(rcu_barrier_func, NULL, 0, 1); - 9 wait_for_completion(&rcu_barrier_completion); - 10 mutex_unlock(&rcu_barrier_mutex); - 11 } + 1 void rcu_barrier(void) + 2 { + 3 BUG_ON(in_interrupt()); + 4 /* Take cpucontrol mutex to protect against CPU hotplug */ + 5 mutex_lock(&rcu_barrier_mutex); + 6 init_completion(&rcu_barrier_completion); + 7 atomic_set(&rcu_barrier_cpu_count, 1); + 8 on_each_cpu(rcu_barrier_func, NULL, 0, 1); + 9 if (atomic_dec_and_test(&rcu_barrier_cpu_count)) + 10 complete(&rcu_barrier_completion); + 11 wait_for_completion(&rcu_barrier_completion); + 12 mutex_unlock(&rcu_barrier_mutex); + 13 } -Line 3 verifies that the caller is in process context, and lines 5 and 10 +Line 3 verifies that the caller is in process context, and lines 5 and 12 use rcu_barrier_mutex to ensure that only one rcu_barrier() is using the global completion and counters at a time, which are initialized on lines 6 and 7. Line 8 causes each CPU to invoke rcu_barrier_func(), which is shown below. Note that the final "1" in on_each_cpu()'s argument list ensures that all the calls to rcu_barrier_func() will have completed -before on_each_cpu() returns. Line 9 then waits for the completion. +before on_each_cpu() returns. Line 9 removes the initial count from +rcu_barrier_cpu_count, and if this count is now zero, line 10 finalizes +the completion, which prevents line 11 from blocking. Either way, +line 11 then waits (if needed) for the completion. + +.. _rcubarrier_quiz_2: + +Quick Quiz #2: + Why doesn't line 8 initialize rcu_barrier_cpu_count to zero, + thereby avoiding the need for lines 9 and 10? + +:ref:`Answer to Quick Quiz #2 ` This code was rewritten in 2008 and several times thereafter, but this still gives the general idea. @@ -253,7 +246,7 @@ to post an RCU callback, as follows:: Lines 3 and 4 locate RCU's internal per-CPU rcu_data structure, which contains the struct rcu_head that needed for the later call to call_rcu(). Line 7 picks up a pointer to this struct rcu_head, and line -8 increments a global counter. This counter will later be decremented +8 increments the global counter. This counter will later be decremented by the callback. Line 9 then registers the rcu_barrier_callback() on the current CPU's queue. @@ -267,27 +260,28 @@ reaches zero, as follows:: 4 complete(&rcu_barrier_completion); 5 } -.. _rcubarrier_quiz_2: +.. _rcubarrier_quiz_3: -Quick Quiz #2: +Quick Quiz #3: What happens if CPU 0's rcu_barrier_func() executes immediately (thus incrementing rcu_barrier_cpu_count to the value one), but the other CPU's rcu_barrier_func() invocations are delayed for a full grace period? Couldn't this result in rcu_barrier() returning prematurely? -:ref:`Answer to Quick Quiz #2 ` +:ref:`Answer to Quick Quiz #3 ` The current rcu_barrier() implementation is more complex, due to the need to avoid disturbing idle CPUs (especially on battery-powered systems) and the need to minimally disturb non-idle CPUs in real-time systems. -However, the code above illustrates the concepts. +In addition, a great many optimizations have been applied. However, +the code above illustrates the concepts. rcu_barrier() Summary --------------------- -The rcu_barrier() primitive has seen relatively little use, since most +The rcu_barrier() primitive is used relatively infrequently, since most code using RCU is in the core kernel rather than in modules. However, if you are using RCU from an unloadable module, you need to use rcu_barrier() so that your module may be safely unloaded. @@ -318,6 +312,39 @@ Answer: Interestingly enough, rcu_barrier() was not originally .. _answer_rcubarrier_quiz_2: Quick Quiz #2: + Why doesn't line 8 initialize rcu_barrier_cpu_count to zero, + thereby avoiding the need for lines 9 and 10? + +Answer: Suppose that the on_each_cpu() function shown on line 8 was + delayed, so that CPU 0's rcu_barrier_func() executed and + the corresponding grace period elapsed, all before CPU 1's + rcu_barrier_func() started executing. This would result in + rcu_barrier_cpu_count being decremented to zero, so that line + 11's wait_for_completion() would return immediately, failing to + wait for CPU 1's callbacks to be invoked. + + Note that this was not a problem when the rcu_barrier() code + was first added back in 2005. This is because on_each_cpu() + disables preemption, which acted as an RCU read-side critical + section, thus preventing CPU 0's grace period from completing + until on_each_cpu() had dealt with all of the CPUs. However, + with the advent of preemptible RCU, rcu_barrier() no longer + waited on nonpreemptible regions of code in preemptible kernels, + that being the job of the new rcu_barrier_sched() function. + + However, with the RCU flavor consolidation around v4.20, this + possibility was once again ruled out, because the consolidated + RCU once again waits on nonpreemptible regions of code. + + Nevertheless, that extra count might still be a good idea. + Relying on these sort of accidents of implementation can result + in later surprise bugs when the implementation changes. + +:ref:`Back to Quick Quiz #2 ` + +.. _answer_rcubarrier_quiz_3: + +Quick Quiz #3: What happens if CPU 0's rcu_barrier_func() executes immediately (thus incrementing rcu_barrier_cpu_count to the value one), but the other CPU's rcu_barrier_func() invocations @@ -336,18 +363,15 @@ Answer: This cannot happen. The reason is that on_each_cpu() has its last Therefore, on_each_cpu() disables preemption across its call to smp_call_function() and also across the local call to - rcu_barrier_func(). This prevents the local CPU from context - switching, again preventing grace periods from completing. This + rcu_barrier_func(). Because recent RCU implementations treat + preemption-disabled regions of code as RCU read-side critical + sections, this prevents grace periods from completing. This means that all CPUs have executed rcu_barrier_func() before the first rcu_barrier_callback() can possibly execute, in turn preventing rcu_barrier_cpu_count from prematurely reaching zero. - Currently, -rt implementations of RCU keep but a single global - queue for RCU callbacks, and thus do not suffer from this - problem. However, when the -rt RCU eventually does have per-CPU - callback queues, things will have to change. One simple change - is to add an rcu_read_lock() before line 8 of rcu_barrier() - and an rcu_read_unlock() after line 8 of this same function. If - you can think of a better change, please let me know! + But if on_each_cpu() ever decides to forgo disabling preemption, + as might well happen due to real-time latency considerations, + initializing rcu_barrier_cpu_count to one will save the day. -:ref:`Back to Quick Quiz #2 ` +:ref:`Back to Quick Quiz #3 ` From b33994ef2239348e144911f944aa0038bf2c214c Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 4 Nov 2022 13:06:54 -0700 Subject: [PATCH 014/239] doc: Update rcu_dereference.rst This commit updates rcu_dereference.rst to reflect RCU additions and changes over the past few years Signed-off-by: Paul E. McKenney --- Documentation/RCU/rcu_dereference.rst | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/Documentation/RCU/rcu_dereference.rst b/Documentation/RCU/rcu_dereference.rst index 81e828c8313b..3b739f6243c8 100644 --- a/Documentation/RCU/rcu_dereference.rst +++ b/Documentation/RCU/rcu_dereference.rst @@ -19,8 +19,9 @@ Follow these rules to keep your RCU code working properly: can reload the value, and won't your code have fun with two different values for a single pointer! Without rcu_dereference(), DEC Alpha can load a pointer, dereference that pointer, and - return data preceding initialization that preceded the store of - the pointer. + return data preceding initialization that preceded the store + of the pointer. (As noted later, in recent kernels READ_ONCE() + also prevents DEC Alpha from playing these tricks.) In addition, the volatile cast in rcu_dereference() prevents the compiler from deducing the resulting pointer value. Please see @@ -34,7 +35,7 @@ Follow these rules to keep your RCU code working properly: takes on the role of the lockless_dereference() primitive that was removed in v4.15. -- You are only permitted to use rcu_dereference on pointer values. +- You are only permitted to use rcu_dereference() on pointer values. The compiler simply knows too much about integral values to trust it to carry dependencies through integer operations. There are a very few exceptions, namely that you can temporarily @@ -240,6 +241,7 @@ precautions. To see this, consider the following code fragment:: struct foo *q; int r1, r2; + rcu_read_lock(); p = rcu_dereference(gp2); if (p == NULL) return; @@ -248,7 +250,10 @@ precautions. To see this, consider the following code fragment:: if (p == q) { /* The compiler decides that q->c is same as p->c. */ r2 = p->c; /* Could get 44 on weakly order system. */ + } else { + r2 = p->c - r1; /* Unconditional access to p->c. */ } + rcu_read_unlock(); do_something_with(r1, r2); } @@ -297,6 +302,7 @@ Then one approach is to use locking, for example, as follows:: struct foo *q; int r1, r2; + rcu_read_lock(); p = rcu_dereference(gp2); if (p == NULL) return; @@ -306,7 +312,12 @@ Then one approach is to use locking, for example, as follows:: if (p == q) { /* The compiler decides that q->c is same as p->c. */ r2 = p->c; /* Locking guarantees r2 == 144. */ + } else { + spin_lock(&q->lock); + r2 = q->c - r1; + spin_unlock(&q->lock); } + rcu_read_unlock(); spin_unlock(&p->lock); do_something_with(r1, r2); } @@ -364,7 +375,7 @@ the exact value of "p" even in the not-equals case. This allows the compiler to make the return values independent of the load from "gp", in turn destroying the ordering between this load and the loads of the return values. This can result in "p->b" returning pre-initialization -garbage values. +garbage values on weakly ordered systems. In short, rcu_dereference() is *not* optional when you are going to dereference the resulting pointer. @@ -430,7 +441,7 @@ member of the rcu_dereference() to use in various situations: SPARSE CHECKING OF RCU-PROTECTED POINTERS ----------------------------------------- -The sparse static-analysis tool checks for direct access to RCU-protected +The sparse static-analysis tool checks for non-RCU access to RCU-protected pointers, which can result in "interesting" bugs due to compiler optimizations involving invented loads and perhaps also load tearing. For example, suppose someone mistakenly does something like this:: From c004d231cac709f09987f2a6dd733013039d27c3 Mon Sep 17 00:00:00 2001 From: Zhao Mengmeng Date: Wed, 19 Oct 2022 08:36:50 -0400 Subject: [PATCH 015/239] rcu: Use hlist_nulls_next_rcu() in hlist_nulls_add_tail_rcu() In commit 8dbd76e79a16 ("tcp/dccp: fix possible race __inet_lookup_established()"), function hlist_nulls_add_tail_rcu() was added back, but the local variable *last* is of type hlist_nulls_node, so use hlist_nulls_next_rcu() instead of hlist_next_rcu(). Signed-off-by: Zhao Mengmeng Signed-off-by: Paul E. McKenney --- include/linux/rculist_nulls.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/rculist_nulls.h b/include/linux/rculist_nulls.h index d8afdb8784c1..ba4c00dd8005 100644 --- a/include/linux/rculist_nulls.h +++ b/include/linux/rculist_nulls.h @@ -139,7 +139,7 @@ static inline void hlist_nulls_add_tail_rcu(struct hlist_nulls_node *n, if (last) { n->next = last->next; n->pprev = &last->next; - rcu_assign_pointer(hlist_next_rcu(last), n); + rcu_assign_pointer(hlist_nulls_next_rcu(last), n); } else { hlist_nulls_add_head_rcu(n, h); } From 5a04848d005e051b8c063206b1a03363aca8ade4 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sun, 6 Nov 2022 16:33:38 -0800 Subject: [PATCH 016/239] rcu: Consolidate initialization and CPU-hotplug code This commit consolidates the initialization and CPU-hotplug code at the end of kernel/rcu/tree.c. This is strictly a code-motion commit. No functionality has changed. Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 314 +++++++++++++++++++++++----------------------- 1 file changed, 158 insertions(+), 156 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index cf34a961821a..d3b082233b74 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -144,14 +144,16 @@ static int rcu_scheduler_fully_active __read_mostly; static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp, unsigned long gps, unsigned long flags); -static void rcu_init_new_rnp(struct rcu_node *rnp_leaf); -static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf); static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu); static void invoke_rcu_core(void); static void rcu_report_exp_rdp(struct rcu_data *rdp); static void sync_sched_exp_online_cleanup(int cpu); static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp); static bool rcu_rdp_is_offloaded(struct rcu_data *rdp); +static bool rcu_rdp_cpu_online(struct rcu_data *rdp); +static bool rcu_init_invoked(void); +static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf); +static void rcu_init_new_rnp(struct rcu_node *rnp_leaf); /* * rcuc/rcub/rcuop kthread realtime priority. The "rcuop" @@ -214,27 +216,6 @@ EXPORT_SYMBOL_GPL(rcu_get_gp_kthreads_prio); */ #define PER_RCU_NODE_PERIOD 3 /* Number of grace periods between delays for debugging. */ -/* - * Compute the mask of online CPUs for the specified rcu_node structure. - * This will not be stable unless the rcu_node structure's ->lock is - * held, but the bit corresponding to the current CPU will be stable - * in most contexts. - */ -static unsigned long rcu_rnp_online_cpus(struct rcu_node *rnp) -{ - return READ_ONCE(rnp->qsmaskinitnext); -} - -/* - * Is the CPU corresponding to the specified rcu_data structure online - * from RCU's perspective? This perspective is given by that structure's - * ->qsmaskinitnext field rather than by the global cpu_online_mask. - */ -static bool rcu_rdp_cpu_online(struct rcu_data *rdp) -{ - return !!(rdp->grpmask & rcu_rnp_online_cpus(rdp->mynode)); -} - /* * Return true if an RCU grace period is in progress. The READ_ONCE()s * permit this function to be invoked without holding the root rcu_node @@ -734,46 +715,6 @@ void rcu_request_urgent_qs_task(struct task_struct *t) smp_store_release(per_cpu_ptr(&rcu_data.rcu_urgent_qs, cpu), true); } -#if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU) - -/* - * Is the current CPU online as far as RCU is concerned? - * - * Disable preemption to avoid false positives that could otherwise - * happen due to the current CPU number being sampled, this task being - * preempted, its old CPU being taken offline, resuming on some other CPU, - * then determining that its old CPU is now offline. - * - * Disable checking if in an NMI handler because we cannot safely - * report errors from NMI handlers anyway. In addition, it is OK to use - * RCU on an offline processor during initial boot, hence the check for - * rcu_scheduler_fully_active. - */ -bool rcu_lockdep_current_cpu_online(void) -{ - struct rcu_data *rdp; - bool ret = false; - - if (in_nmi() || !rcu_scheduler_fully_active) - return true; - preempt_disable_notrace(); - rdp = this_cpu_ptr(&rcu_data); - /* - * Strictly, we care here about the case where the current CPU is - * in rcu_cpu_starting() and thus has an excuse for rdp->grpmask - * not being up to date. So arch_spin_is_locked() might have a - * false positive if it's held by some *other* CPU, but that's - * OK because that just means a false *negative* on the warning. - */ - if (rcu_rdp_cpu_online(rdp) || arch_spin_is_locked(&rcu_state.ofl_lock)) - ret = true; - preempt_enable_notrace(); - return ret; -} -EXPORT_SYMBOL_GPL(rcu_lockdep_current_cpu_online); - -#endif /* #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU) */ - /* * When trying to report a quiescent state on behalf of some other CPU, * it is our responsibility to check for and handle potential overflow @@ -1350,13 +1291,6 @@ static void rcu_strict_gp_boundary(void *unused) invoke_rcu_core(); } -// Has rcu_init() been invoked? This is used (for example) to determine -// whether spinlocks may be acquired safely. -static bool rcu_init_invoked(void) -{ - return !!rcu_state.n_online_cpus; -} - // Make the polled API aware of the beginning of a grace period. static void rcu_poll_gp_seq_start(unsigned long *snap) { @@ -2091,92 +2025,6 @@ rcu_check_quiescent_state(struct rcu_data *rdp) rcu_report_qs_rdp(rdp); } -/* - * Near the end of the offline process. Trace the fact that this CPU - * is going offline. - */ -int rcutree_dying_cpu(unsigned int cpu) -{ - bool blkd; - struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); - struct rcu_node *rnp = rdp->mynode; - - if (!IS_ENABLED(CONFIG_HOTPLUG_CPU)) - return 0; - - blkd = !!(READ_ONCE(rnp->qsmask) & rdp->grpmask); - trace_rcu_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq), - blkd ? TPS("cpuofl-bgp") : TPS("cpuofl")); - return 0; -} - -/* - * All CPUs for the specified rcu_node structure have gone offline, - * and all tasks that were preempted within an RCU read-side critical - * section while running on one of those CPUs have since exited their RCU - * read-side critical section. Some other CPU is reporting this fact with - * the specified rcu_node structure's ->lock held and interrupts disabled. - * This function therefore goes up the tree of rcu_node structures, - * clearing the corresponding bits in the ->qsmaskinit fields. Note that - * the leaf rcu_node structure's ->qsmaskinit field has already been - * updated. - * - * This function does check that the specified rcu_node structure has - * all CPUs offline and no blocked tasks, so it is OK to invoke it - * prematurely. That said, invoking it after the fact will cost you - * a needless lock acquisition. So once it has done its work, don't - * invoke it again. - */ -static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf) -{ - long mask; - struct rcu_node *rnp = rnp_leaf; - - raw_lockdep_assert_held_rcu_node(rnp_leaf); - if (!IS_ENABLED(CONFIG_HOTPLUG_CPU) || - WARN_ON_ONCE(rnp_leaf->qsmaskinit) || - WARN_ON_ONCE(rcu_preempt_has_tasks(rnp_leaf))) - return; - for (;;) { - mask = rnp->grpmask; - rnp = rnp->parent; - if (!rnp) - break; - raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */ - rnp->qsmaskinit &= ~mask; - /* Between grace periods, so better already be zero! */ - WARN_ON_ONCE(rnp->qsmask); - if (rnp->qsmaskinit) { - raw_spin_unlock_rcu_node(rnp); - /* irqs remain disabled. */ - return; - } - raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ - } -} - -/* - * The CPU has been completely removed, and some other CPU is reporting - * this fact from process context. Do the remainder of the cleanup. - * There can only be one CPU hotplug operation at a time, so no need for - * explicit locking. - */ -int rcutree_dead_cpu(unsigned int cpu) -{ - struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); - struct rcu_node *rnp = rdp->mynode; /* Outgoing CPU's rdp & rnp. */ - - if (!IS_ENABLED(CONFIG_HOTPLUG_CPU)) - return 0; - - WRITE_ONCE(rcu_state.n_online_cpus, rcu_state.n_online_cpus - 1); - /* Adjust any no-longer-needed kthreads. */ - rcu_boost_kthread_setaffinity(rnp, -1); - // Stop-machine done, so allow nohz_full to disable tick. - tick_dep_clear(TICK_DEP_BIT_RCU); - return 0; -} - /* * Invoke any RCU callbacks that have made it to the end of their grace * period. Throttle as specified by rdp->blimit. @@ -4079,6 +3927,160 @@ retry: } EXPORT_SYMBOL_GPL(rcu_barrier); +/* + * Compute the mask of online CPUs for the specified rcu_node structure. + * This will not be stable unless the rcu_node structure's ->lock is + * held, but the bit corresponding to the current CPU will be stable + * in most contexts. + */ +static unsigned long rcu_rnp_online_cpus(struct rcu_node *rnp) +{ + return READ_ONCE(rnp->qsmaskinitnext); +} + +/* + * Is the CPU corresponding to the specified rcu_data structure online + * from RCU's perspective? This perspective is given by that structure's + * ->qsmaskinitnext field rather than by the global cpu_online_mask. + */ +static bool rcu_rdp_cpu_online(struct rcu_data *rdp) +{ + return !!(rdp->grpmask & rcu_rnp_online_cpus(rdp->mynode)); +} + +#if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU) + +/* + * Is the current CPU online as far as RCU is concerned? + * + * Disable preemption to avoid false positives that could otherwise + * happen due to the current CPU number being sampled, this task being + * preempted, its old CPU being taken offline, resuming on some other CPU, + * then determining that its old CPU is now offline. + * + * Disable checking if in an NMI handler because we cannot safely + * report errors from NMI handlers anyway. In addition, it is OK to use + * RCU on an offline processor during initial boot, hence the check for + * rcu_scheduler_fully_active. + */ +bool rcu_lockdep_current_cpu_online(void) +{ + struct rcu_data *rdp; + bool ret = false; + + if (in_nmi() || !rcu_scheduler_fully_active) + return true; + preempt_disable_notrace(); + rdp = this_cpu_ptr(&rcu_data); + /* + * Strictly, we care here about the case where the current CPU is + * in rcu_cpu_starting() and thus has an excuse for rdp->grpmask + * not being up to date. So arch_spin_is_locked() might have a + * false positive if it's held by some *other* CPU, but that's + * OK because that just means a false *negative* on the warning. + */ + if (rcu_rdp_cpu_online(rdp) || arch_spin_is_locked(&rcu_state.ofl_lock)) + ret = true; + preempt_enable_notrace(); + return ret; +} +EXPORT_SYMBOL_GPL(rcu_lockdep_current_cpu_online); + +#endif /* #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU) */ + +// Has rcu_init() been invoked? This is used (for example) to determine +// whether spinlocks may be acquired safely. +static bool rcu_init_invoked(void) +{ + return !!rcu_state.n_online_cpus; +} + +/* + * Near the end of the offline process. Trace the fact that this CPU + * is going offline. + */ +int rcutree_dying_cpu(unsigned int cpu) +{ + bool blkd; + struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); + struct rcu_node *rnp = rdp->mynode; + + if (!IS_ENABLED(CONFIG_HOTPLUG_CPU)) + return 0; + + blkd = !!(READ_ONCE(rnp->qsmask) & rdp->grpmask); + trace_rcu_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq), + blkd ? TPS("cpuofl-bgp") : TPS("cpuofl")); + return 0; +} + +/* + * All CPUs for the specified rcu_node structure have gone offline, + * and all tasks that were preempted within an RCU read-side critical + * section while running on one of those CPUs have since exited their RCU + * read-side critical section. Some other CPU is reporting this fact with + * the specified rcu_node structure's ->lock held and interrupts disabled. + * This function therefore goes up the tree of rcu_node structures, + * clearing the corresponding bits in the ->qsmaskinit fields. Note that + * the leaf rcu_node structure's ->qsmaskinit field has already been + * updated. + * + * This function does check that the specified rcu_node structure has + * all CPUs offline and no blocked tasks, so it is OK to invoke it + * prematurely. That said, invoking it after the fact will cost you + * a needless lock acquisition. So once it has done its work, don't + * invoke it again. + */ +static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf) +{ + long mask; + struct rcu_node *rnp = rnp_leaf; + + raw_lockdep_assert_held_rcu_node(rnp_leaf); + if (!IS_ENABLED(CONFIG_HOTPLUG_CPU) || + WARN_ON_ONCE(rnp_leaf->qsmaskinit) || + WARN_ON_ONCE(rcu_preempt_has_tasks(rnp_leaf))) + return; + for (;;) { + mask = rnp->grpmask; + rnp = rnp->parent; + if (!rnp) + break; + raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */ + rnp->qsmaskinit &= ~mask; + /* Between grace periods, so better already be zero! */ + WARN_ON_ONCE(rnp->qsmask); + if (rnp->qsmaskinit) { + raw_spin_unlock_rcu_node(rnp); + /* irqs remain disabled. */ + return; + } + raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ + } +} + +/* + * The CPU has been completely removed, and some other CPU is reporting + * this fact from process context. Do the remainder of the cleanup. + * There can only be one CPU hotplug operation at a time, so no need for + * explicit locking. + */ +int rcutree_dead_cpu(unsigned int cpu) +{ + struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); + struct rcu_node *rnp = rdp->mynode; /* Outgoing CPU's rdp & rnp. */ + + if (!IS_ENABLED(CONFIG_HOTPLUG_CPU)) + return 0; + + WRITE_ONCE(rcu_state.n_online_cpus, rcu_state.n_online_cpus - 1); + /* Adjust any no-longer-needed kthreads. */ + rcu_boost_kthread_setaffinity(rnp, -1); + // Stop-machine done, so allow nohz_full to disable tick. + tick_dep_clear(TICK_DEP_BIT_RCU); + return 0; +} + /* * Propagate ->qsinitmask bits up the rcu_node tree to account for the * first CPU in a given leaf rcu_node structure coming online. The caller From 253cbbff621407a6265ce7a6a03c3766f8846f02 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 14 Nov 2022 09:40:19 -0800 Subject: [PATCH 017/239] rcu: Throttle callback invocation based on number of ready callbacks Currently, rcu_do_batch() sizes its batches based on the total number of callbacks in the callback list. This can result in some strange choices, for example, if there was 12,800 callbacks in the list, but only 200 were ready to invoke, RCU would invoke 100 at a time (12,800 shifted down by seven bits). A more measured approach would use the number that were actually ready to invoke, an approach that has become feasible only recently given the per-segment ->seglen counts in ->cblist. This commit therefore bases the batch limit on the number of callbacks ready to invoke instead of on the total number of callbacks. Signed-off-by: Paul E. McKenney --- kernel/rcu/rcu_segcblist.c | 2 +- kernel/rcu/rcu_segcblist.h | 2 ++ kernel/rcu/tree.c | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/rcu_segcblist.c b/kernel/rcu/rcu_segcblist.c index c54ea2b6a36b..f71fac422c8f 100644 --- a/kernel/rcu/rcu_segcblist.c +++ b/kernel/rcu/rcu_segcblist.c @@ -89,7 +89,7 @@ static void rcu_segcblist_set_len(struct rcu_segcblist *rsclp, long v) } /* Get the length of a segment of the rcu_segcblist structure. */ -static long rcu_segcblist_get_seglen(struct rcu_segcblist *rsclp, int seg) +long rcu_segcblist_get_seglen(struct rcu_segcblist *rsclp, int seg) { return READ_ONCE(rsclp->seglen[seg]); } diff --git a/kernel/rcu/rcu_segcblist.h b/kernel/rcu/rcu_segcblist.h index 431cee212467..4fe877f5f654 100644 --- a/kernel/rcu/rcu_segcblist.h +++ b/kernel/rcu/rcu_segcblist.h @@ -15,6 +15,8 @@ static inline long rcu_cblist_n_cbs(struct rcu_cblist *rclp) return READ_ONCE(rclp->len); } +long rcu_segcblist_get_seglen(struct rcu_segcblist *rsclp, int seg); + /* Return number of callbacks in segmented callback list by summing seglen. */ long rcu_segcblist_n_segment_cbs(struct rcu_segcblist *rsclp); diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index d3b082233b74..7d3a59d4f37e 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2057,7 +2057,7 @@ static void rcu_do_batch(struct rcu_data *rdp) */ rcu_nocb_lock_irqsave(rdp, flags); WARN_ON_ONCE(cpu_is_offline(smp_processor_id())); - pending = rcu_segcblist_n_cbs(&rdp->cblist); + pending = rcu_segcblist_get_seglen(&rdp->cblist, RCU_DONE_TAIL); div = READ_ONCE(rcu_divisor); div = div < 0 ? 7 : div > sizeof(long) * 8 - 2 ? sizeof(long) * 8 - 2 : div; bl = max(rdp->blimit, pending >> div); From 95ff24ee7b809ff8d253cd5edf196f137ae08c44 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 25 Nov 2022 08:43:10 -0800 Subject: [PATCH 018/239] rcu: Upgrade header comment for poll_state_synchronize_rcu() This commit emphasizes the possibility of concurrent calls to synchronize_rcu() and synchronize_rcu_expedited() causing one or the other of the two grace periods being lost from the viewpoint of poll_state_synchronize_rcu(). If you cannot afford to lose grace periods this way, you should instead use the _full() variants of the polled RCU API, for example, poll_state_synchronize_rcu_full(). Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 7d3a59d4f37e..0147e69ea85a 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -3559,7 +3559,9 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu_full); * If @false is returned, it is the caller's responsibility to invoke this * function later on until it does return @true. Alternatively, the caller * can explicitly wait for a grace period, for example, by passing @oldstate - * to cond_synchronize_rcu() or by directly invoking synchronize_rcu(). + * to either cond_synchronize_rcu() or cond_synchronize_rcu_expedited() + * on the one hand or by directly invoking either synchronize_rcu() or + * synchronize_rcu_expedited() on the other. * * Yes, this function does not take counter wrap into account. * But counter wrap is harmless. If the counter wraps, we have waited for @@ -3570,6 +3572,12 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu_full); * completed. Alternatively, they can use get_completed_synchronize_rcu() * to get a guaranteed-completed grace-period state. * + * In addition, because oldstate compresses the grace-period state for + * both normal and expedited grace periods into a single unsigned long, + * it can miss a grace period when synchronize_rcu() runs concurrently + * with synchronize_rcu_expedited(). If this is unacceptable, please + * instead use the _full() variant of these polling APIs. + * * This function provides the same memory-ordering guarantees that * would be provided by a synchronize_rcu() that was invoked at the call * to the function that provided @oldstate, and that returned at the end From 0cae5ded535c3a80aed94f119bbd4ee3ae284a65 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 14 Dec 2022 11:41:44 -0800 Subject: [PATCH 019/239] rcu: Make RCU_LOCKDEP_WARN() avoid early lockdep checks Currently, RCU_LOCKDEP_WARN() checks the condition before checking to see if lockdep is still enabled. This is necessary to avoid the false-positive splats fixed by commit 3066820034b5dd ("rcu: Reject RCU_LOCKDEP_WARN() false positives"). However, the current state can result in false-positive splats during early boot before lockdep is fully initialized. This commit therefore checks debug_lockdep_rcu_enabled() both before and after checking the condition, thus avoiding both sets of false-positive error reports. Reported-by: Steven Rostedt Reported-by: Masami Hiramatsu (Google) Reported-by: Mathieu Desnoyers Signed-off-by: Paul E. McKenney Reviewed-by: Mathieu Desnoyers Cc: Boqun Feng Cc: Matthew Wilcox Cc: Thomas Gleixner --- include/linux/rcupdate.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 03abf883a281..aa86de01aab6 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -374,11 +374,18 @@ static inline int debug_lockdep_rcu_enabled(void) * RCU_LOCKDEP_WARN - emit lockdep splat if specified condition is met * @c: condition to check * @s: informative message + * + * This checks debug_lockdep_rcu_enabled() before checking (c) to + * prevent early boot splats due to lockdep not yet being initialized, + * and rechecks it after checking (c) to prevent false-positive splats + * due to races with lockdep being disabled. See commit 3066820034b5dd + * ("rcu: Reject RCU_LOCKDEP_WARN() false positives") for more detail. */ #define RCU_LOCKDEP_WARN(c, s) \ do { \ static bool __section(".data.unlikely") __warned; \ - if ((c) && debug_lockdep_rcu_enabled() && !__warned) { \ + if (debug_lockdep_rcu_enabled() && (c) && \ + debug_lockdep_rcu_enabled() && !__warned) { \ __warned = true; \ lockdep_rcu_suspicious(__FILE__, __LINE__, s); \ } \ From 2d7f00b2f01301d6e41fd4a28030dab0442265be Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 16 Dec 2022 15:55:48 -0800 Subject: [PATCH 020/239] rcu: Suppress smp_processor_id() complaint in synchronize_rcu_expedited_wait() The normal grace period's RCU CPU stall warnings are invoked from the scheduling-clock interrupt handler, and can thus invoke smp_processor_id() with impunity, which allows them to directly invoke dump_cpu_task(). In contrast, the expedited grace period's RCU CPU stall warnings are invoked from process context, which causes the dump_cpu_task() function's calls to smp_processor_id() to complain bitterly in debug kernels. This commit therefore causes synchronize_rcu_expedited_wait() to disable preemption around its call to dump_cpu_task(). Signed-off-by: Paul E. McKenney --- kernel/rcu/tree_exp.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index ed6c3cce28f2..927abaf6c822 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -667,7 +667,9 @@ static void synchronize_rcu_expedited_wait(void) mask = leaf_node_cpu_bit(rnp, cpu); if (!(READ_ONCE(rnp->expmask) & mask)) continue; + preempt_disable(); // For smp_processor_id() in dump_cpu_task(). dump_cpu_task(cpu); + preempt_enable(); } } jiffies_stall = 3 * rcu_exp_jiffies_till_stall_check() + 3; From 3d1adf7ada352b80e037509d26cdca156f75e830 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Thu, 15 Dec 2022 11:57:55 +0800 Subject: [PATCH 021/239] rcu: Make rcu_blocking_is_gp() stop early-boot might_sleep() Currently, rcu_blocking_is_gp() invokes might_sleep() even during early boot when interrupts are disabled and before the scheduler is scheduling. This is at best an accident waiting to happen. Therefore, this commit moves that might_sleep() under an rcu_scheduler_active check in order to ensure that might_sleep() is not invoked unless sleeping might actually happen. Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 0147e69ea85a..15f976506472 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -3333,9 +3333,10 @@ void __init kfree_rcu_scheduler_running(void) */ static int rcu_blocking_is_gp(void) { - if (rcu_scheduler_active != RCU_SCHEDULER_INACTIVE) + if (rcu_scheduler_active != RCU_SCHEDULER_INACTIVE) { + might_sleep(); return false; - might_sleep(); /* Check for RCU read-side critical section. */ + } return true; } From 748bf47a89d722c7e77f8700705e2189be14e99e Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 19 Dec 2022 17:02:20 -0800 Subject: [PATCH 022/239] rcu: Test synchronous RCU grace periods at the end of rcu_init() This commit tests synchronize_rcu() and synchronize_rcu_expedited() at the end of rcu_init(), in addition to the test already at the beginning of that function. These tests are run only in kernels built with CONFIG_PROVE_RCU=y. Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 2 ++ kernel/rcu/update.c | 1 + 2 files changed, 3 insertions(+) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 15f976506472..80b84ae285b4 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -4849,6 +4849,8 @@ void __init rcu_init(void) // Kick-start any polled grace periods that started early. if (!(per_cpu_ptr(&rcu_data, cpu)->mynode->exp_seq_poll_rq & 0x1)) (void)start_poll_synchronize_rcu_expedited(); + + rcu_test_sync_prims(); } #include "tree_stall.h" diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c index f5e6a2f95a2a..587b97c40191 100644 --- a/kernel/rcu/update.c +++ b/kernel/rcu/update.c @@ -220,6 +220,7 @@ void rcu_test_sync_prims(void) { if (!IS_ENABLED(CONFIG_PROVE_RCU)) return; + pr_info("Running RCU synchronous self tests\n"); synchronize_rcu(); synchronize_rcu_expedited(); } From 92987fe8bdd1cbec61919a394bb11316c5d860f4 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 19 Dec 2022 18:02:20 -0800 Subject: [PATCH 023/239] rcu: Allow expedited RCU CPU stall warnings to dump task stacks This commit introduces the rcupdate.rcu_exp_stall_task_details kernel boot parameter, which cause expedited RCU CPU stall warnings to dump the stacks of any tasks blocking the current expedited grace period. Reported-by: David Howells Signed-off-by: Paul E. McKenney --- .../admin-guide/kernel-parameters.txt | 5 +++ kernel/rcu/rcu.h | 1 + kernel/rcu/tree_exp.h | 41 +++++++++++++++++++ kernel/rcu/update.c | 2 + 4 files changed, 49 insertions(+) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 6cfa6e3996cf..aa453f9202d8 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -5113,6 +5113,11 @@ rcupdate.rcu_cpu_stall_timeout to be used (after conversion from seconds to milliseconds). + rcupdate.rcu_exp_stall_task_details= [KNL] + Print stack dumps of any tasks blocking the + current expedited RCU grace period during an + expedited RCU CPU stall warning. + rcupdate.rcu_expedited= [KNL] Use expedited grace-period primitives, for example, synchronize_rcu_expedited() instead diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index c5aa934de59b..fa640c45172e 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -224,6 +224,7 @@ extern int rcu_cpu_stall_ftrace_dump; extern int rcu_cpu_stall_suppress; extern int rcu_cpu_stall_timeout; extern int rcu_exp_cpu_stall_timeout; +extern bool rcu_exp_stall_task_details __read_mostly; int rcu_jiffies_till_stall_check(void); int rcu_exp_jiffies_till_stall_check(void); diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index 927abaf6c822..249c2967d9e6 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -11,6 +11,7 @@ static void rcu_exp_handler(void *unused); static int rcu_print_task_exp_stall(struct rcu_node *rnp); +static void rcu_exp_print_detail_task_stall_rnp(struct rcu_node *rnp); /* * Record the start of an expedited grace period. @@ -671,6 +672,7 @@ static void synchronize_rcu_expedited_wait(void) dump_cpu_task(cpu); preempt_enable(); } + rcu_exp_print_detail_task_stall_rnp(rnp); } jiffies_stall = 3 * rcu_exp_jiffies_till_stall_check() + 3; panic_on_rcu_stall(); @@ -813,6 +815,36 @@ static int rcu_print_task_exp_stall(struct rcu_node *rnp) return ndetected; } +/* + * Scan the current list of tasks blocked within RCU read-side critical + * sections, dumping the stack of each that is blocking the current + * expedited grace period. + */ +static void rcu_exp_print_detail_task_stall_rnp(struct rcu_node *rnp) +{ + unsigned long flags; + struct task_struct *t; + + if (!rcu_exp_stall_task_details) + return; + raw_spin_lock_irqsave_rcu_node(rnp, flags); + if (!READ_ONCE(rnp->exp_tasks)) { + raw_spin_unlock_irqrestore_rcu_node(rnp, flags); + return; + } + t = list_entry(rnp->exp_tasks->prev, + struct task_struct, rcu_node_entry); + list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) { + /* + * We could be printing a lot while holding a spinlock. + * Avoid triggering hard lockup. + */ + touch_nmi_watchdog(); + sched_show_task(t); + } + raw_spin_unlock_irqrestore_rcu_node(rnp, flags); +} + #else /* #ifdef CONFIG_PREEMPT_RCU */ /* Request an expedited quiescent state. */ @@ -885,6 +917,15 @@ static int rcu_print_task_exp_stall(struct rcu_node *rnp) return 0; } +/* + * Because preemptible RCU does not exist, we never have to print out + * tasks blocked within RCU read-side critical sections that are blocking + * the current expedited grace period. + */ +static void rcu_exp_print_detail_task_stall_rnp(struct rcu_node *rnp) +{ +} + #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ /** diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c index 587b97c40191..6ed5020aee6d 100644 --- a/kernel/rcu/update.c +++ b/kernel/rcu/update.c @@ -509,6 +509,8 @@ int rcu_cpu_stall_timeout __read_mostly = CONFIG_RCU_CPU_STALL_TIMEOUT; module_param(rcu_cpu_stall_timeout, int, 0644); int rcu_exp_cpu_stall_timeout __read_mostly = CONFIG_RCU_EXP_CPU_STALL_TIMEOUT; module_param(rcu_exp_cpu_stall_timeout, int, 0644); +bool rcu_exp_stall_task_details __read_mostly; +module_param(rcu_exp_stall_task_details, bool, 0644); #endif /* #ifdef CONFIG_RCU_STALL_COMMON */ // Suppress boot-time RCU CPU stall warnings and rcutorture writer stall From 04a522b7da3dbc083f8ae0aa1a6184b959a8f81c Mon Sep 17 00:00:00 2001 From: "Uladzislau Rezki (Sony)" Date: Tue, 25 Oct 2022 16:46:12 +0200 Subject: [PATCH 024/239] rcu: Refactor kvfree_call_rcu() and high-level helpers Currently a kvfree_call_rcu() takes an offset within a structure as a second parameter, so a helper such as a kvfree_rcu_arg_2() has to convert rcu_head and a freed ptr to an offset in order to pass it. That leads to an extra conversion on macro entry. Instead of converting, refactor the code in way that a pointer that has to be freed is passed directly to the kvfree_call_rcu(). This patch does not make any functional change and is transparent to all kvfree_rcu() users. Signed-off-by: Uladzislau Rezki (Sony) Signed-off-by: Paul E. McKenney --- include/linux/rcupdate.h | 5 ++--- include/linux/rcutiny.h | 12 ++++++------ include/linux/rcutree.h | 2 +- kernel/rcu/tiny.c | 9 +++------ kernel/rcu/tree.c | 29 ++++++++++++----------------- 5 files changed, 24 insertions(+), 33 deletions(-) diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 03abf883a281..f38d4469d7f3 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -1011,8 +1011,7 @@ do { \ \ if (___p) { \ BUILD_BUG_ON(!__is_kvfree_rcu_offset(offsetof(typeof(*(ptr)), rhf))); \ - kvfree_call_rcu(&((___p)->rhf), (rcu_callback_t)(unsigned long) \ - (offsetof(typeof(*(ptr)), rhf))); \ + kvfree_call_rcu(&((___p)->rhf), (void *) (___p)); \ } \ } while (0) @@ -1021,7 +1020,7 @@ do { \ typeof(ptr) ___p = (ptr); \ \ if (___p) \ - kvfree_call_rcu(NULL, (rcu_callback_t) (___p)); \ + kvfree_call_rcu(NULL, (void *) (___p)); \ } while (0) /* diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h index 68f9070aa111..7f17acf29dda 100644 --- a/include/linux/rcutiny.h +++ b/include/linux/rcutiny.h @@ -98,25 +98,25 @@ static inline void synchronize_rcu_expedited(void) */ extern void kvfree(const void *addr); -static inline void __kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) +static inline void __kvfree_call_rcu(struct rcu_head *head, void *ptr) { if (head) { - call_rcu(head, func); + call_rcu(head, (rcu_callback_t) ((void *) head - ptr)); return; } // kvfree_rcu(one_arg) call. might_sleep(); synchronize_rcu(); - kvfree((void *) func); + kvfree(ptr); } #ifdef CONFIG_KASAN_GENERIC -void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func); +void kvfree_call_rcu(struct rcu_head *head, void *ptr); #else -static inline void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) +static inline void kvfree_call_rcu(struct rcu_head *head, void *ptr) { - __kvfree_call_rcu(head, func); + __kvfree_call_rcu(head, ptr); } #endif diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index 4003bf6cfa1c..56bccb5a8fde 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h @@ -33,7 +33,7 @@ static inline void rcu_virt_note_context_switch(void) } void synchronize_rcu_expedited(void); -void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func); +void kvfree_call_rcu(struct rcu_head *head, void *ptr); void rcu_barrier(void); bool rcu_eqs_special_set(int cpu); diff --git a/kernel/rcu/tiny.c b/kernel/rcu/tiny.c index 72913ce21258..42f7589e51e0 100644 --- a/kernel/rcu/tiny.c +++ b/kernel/rcu/tiny.c @@ -246,15 +246,12 @@ bool poll_state_synchronize_rcu(unsigned long oldstate) EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu); #ifdef CONFIG_KASAN_GENERIC -void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) +void kvfree_call_rcu(struct rcu_head *head, void *ptr) { - if (head) { - void *ptr = (void *) head - (unsigned long) func; - + if (head) kasan_record_aux_stack_noalloc(ptr); - } - __kvfree_call_rcu(head, func); + __kvfree_call_rcu(head, ptr); } EXPORT_SYMBOL_GPL(kvfree_call_rcu); #endif diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index cf34a961821a..7d222acd85bf 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -3103,8 +3103,8 @@ static void kfree_rcu_work(struct work_struct *work) * This list is named "Channel 3". */ for (; head; head = next) { - unsigned long offset = (unsigned long)head->func; - void *ptr = (void *)head - offset; + void *ptr = (void *) head->func; + unsigned long offset = (void *) head - ptr; next = head->next; debug_rcu_head_unqueue((struct rcu_head *)ptr); @@ -3342,26 +3342,21 @@ add_ptr_to_bulk_krc_lock(struct kfree_rcu_cpu **krcp, * be free'd in workqueue context. This allows us to: batch requests together to * reduce the number of grace periods during heavy kfree_rcu()/kvfree_rcu() load. */ -void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) +void kvfree_call_rcu(struct rcu_head *head, void *ptr) { unsigned long flags; struct kfree_rcu_cpu *krcp; bool success; - void *ptr; - if (head) { - ptr = (void *) head - (unsigned long) func; - } else { - /* - * Please note there is a limitation for the head-less - * variant, that is why there is a clear rule for such - * objects: it can be used from might_sleep() context - * only. For other places please embed an rcu_head to - * your data. - */ + /* + * Please note there is a limitation for the head-less + * variant, that is why there is a clear rule for such + * objects: it can be used from might_sleep() context + * only. For other places please embed an rcu_head to + * your data. + */ + if (!head) might_sleep(); - ptr = (unsigned long *) func; - } // Queue the object but don't yet schedule the batch. if (debug_rcu_head_queue(ptr)) { @@ -3382,7 +3377,7 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) // Inline if kvfree_rcu(one_arg) call. goto unlock_return; - head->func = func; + head->func = ptr; head->next = krcp->head; krcp->head = head; success = true; From 27538e18b62fa38d38c593e8c9e050a31b6c8cea Mon Sep 17 00:00:00 2001 From: "Uladzislau Rezki (Sony)" Date: Tue, 29 Nov 2022 16:58:19 +0100 Subject: [PATCH 025/239] rcu/kvfree: Switch to a generic linked list API This commit improves the readability and maintainability of the kvfree_rcu() code by switching from an open-coded linked list to the standard Linux-kernel circular doubly linked list. This patch does not introduce any functional change. Signed-off-by: Uladzislau Rezki (Sony) Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 89 +++++++++++++++++++++++------------------------ 1 file changed, 43 insertions(+), 46 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 7d222acd85bf..4088b34ce961 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2876,13 +2876,13 @@ EXPORT_SYMBOL_GPL(call_rcu); /** * struct kvfree_rcu_bulk_data - single block to store kvfree_rcu() pointers + * @list: List node. All blocks are linked between each other * @nr_records: Number of active pointers in the array - * @next: Next bulk object in the block chain * @records: Array of the kvfree_rcu() pointers */ struct kvfree_rcu_bulk_data { + struct list_head list; unsigned long nr_records; - struct kvfree_rcu_bulk_data *next; void *records[]; }; @@ -2898,21 +2898,21 @@ struct kvfree_rcu_bulk_data { * struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests * @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period * @head_free: List of kfree_rcu() objects waiting for a grace period - * @bkvhead_free: Bulk-List of kvfree_rcu() objects waiting for a grace period + * @bulk_head_free: Bulk-List of kvfree_rcu() objects waiting for a grace period * @krcp: Pointer to @kfree_rcu_cpu structure */ struct kfree_rcu_cpu_work { struct rcu_work rcu_work; struct rcu_head *head_free; - struct kvfree_rcu_bulk_data *bkvhead_free[FREE_N_CHANNELS]; + struct list_head bulk_head_free[FREE_N_CHANNELS]; struct kfree_rcu_cpu *krcp; }; /** * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period * @head: List of kfree_rcu() objects not yet waiting for a grace period - * @bkvhead: Bulk-List of kvfree_rcu() objects not yet waiting for a grace period + * @bulk_head: Bulk-List of kvfree_rcu() objects not yet waiting for a grace period * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period * @lock: Synchronize access to this structure * @monitor_work: Promote @head to @head_free after KFREE_DRAIN_JIFFIES @@ -2936,7 +2936,7 @@ struct kfree_rcu_cpu_work { */ struct kfree_rcu_cpu { struct rcu_head *head; - struct kvfree_rcu_bulk_data *bkvhead[FREE_N_CHANNELS]; + struct list_head bulk_head[FREE_N_CHANNELS]; struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES]; raw_spinlock_t lock; struct delayed_work monitor_work; @@ -3031,12 +3031,13 @@ drain_page_cache(struct kfree_rcu_cpu *krcp) /* * This function is invoked in workqueue context after a grace period. - * It frees all the objects queued on ->bkvhead_free or ->head_free. + * It frees all the objects queued on ->bulk_head_free or ->head_free. */ static void kfree_rcu_work(struct work_struct *work) { unsigned long flags; - struct kvfree_rcu_bulk_data *bkvhead[FREE_N_CHANNELS], *bnext; + struct kvfree_rcu_bulk_data *bnode, *n; + struct list_head bulk_head[FREE_N_CHANNELS]; struct rcu_head *head, *next; struct kfree_rcu_cpu *krcp; struct kfree_rcu_cpu_work *krwp; @@ -3048,10 +3049,8 @@ static void kfree_rcu_work(struct work_struct *work) raw_spin_lock_irqsave(&krcp->lock, flags); // Channels 1 and 2. - for (i = 0; i < FREE_N_CHANNELS; i++) { - bkvhead[i] = krwp->bkvhead_free[i]; - krwp->bkvhead_free[i] = NULL; - } + for (i = 0; i < FREE_N_CHANNELS; i++) + list_replace_init(&krwp->bulk_head_free[i], &bulk_head[i]); // Channel 3. head = krwp->head_free; @@ -3060,36 +3059,33 @@ static void kfree_rcu_work(struct work_struct *work) // Handle the first two channels. for (i = 0; i < FREE_N_CHANNELS; i++) { - for (; bkvhead[i]; bkvhead[i] = bnext) { - bnext = bkvhead[i]->next; - debug_rcu_bhead_unqueue(bkvhead[i]); + list_for_each_entry_safe(bnode, n, &bulk_head[i], list) { + debug_rcu_bhead_unqueue(bnode); rcu_lock_acquire(&rcu_callback_map); if (i == 0) { // kmalloc() / kfree(). trace_rcu_invoke_kfree_bulk_callback( - rcu_state.name, bkvhead[i]->nr_records, - bkvhead[i]->records); + rcu_state.name, bnode->nr_records, + bnode->records); - kfree_bulk(bkvhead[i]->nr_records, - bkvhead[i]->records); + kfree_bulk(bnode->nr_records, bnode->records); } else { // vmalloc() / vfree(). - for (j = 0; j < bkvhead[i]->nr_records; j++) { + for (j = 0; j < bnode->nr_records; j++) { trace_rcu_invoke_kvfree_callback( - rcu_state.name, - bkvhead[i]->records[j], 0); + rcu_state.name, bnode->records[j], 0); - vfree(bkvhead[i]->records[j]); + vfree(bnode->records[j]); } } rcu_lock_release(&rcu_callback_map); raw_spin_lock_irqsave(&krcp->lock, flags); - if (put_cached_bnode(krcp, bkvhead[i])) - bkvhead[i] = NULL; + if (put_cached_bnode(krcp, bnode)) + bnode = NULL; raw_spin_unlock_irqrestore(&krcp->lock, flags); - if (bkvhead[i]) - free_page((unsigned long) bkvhead[i]); + if (bnode) + free_page((unsigned long) bnode); cond_resched_tasks_rcu_qs(); } @@ -3125,7 +3121,7 @@ need_offload_krc(struct kfree_rcu_cpu *krcp) int i; for (i = 0; i < FREE_N_CHANNELS; i++) - if (krcp->bkvhead[i]) + if (!list_empty(&krcp->bulk_head[i])) return true; return !!krcp->head; @@ -3162,21 +3158,20 @@ static void kfree_rcu_monitor(struct work_struct *work) for (i = 0; i < KFREE_N_BATCHES; i++) { struct kfree_rcu_cpu_work *krwp = &(krcp->krw_arr[i]); - // Try to detach bkvhead or head and attach it over any + // Try to detach bulk_head or head and attach it over any // available corresponding free channel. It can be that // a previous RCU batch is in progress, it means that // immediately to queue another one is not possible so // in that case the monitor work is rearmed. - if ((krcp->bkvhead[0] && !krwp->bkvhead_free[0]) || - (krcp->bkvhead[1] && !krwp->bkvhead_free[1]) || + if ((!list_empty(&krcp->bulk_head[0]) && list_empty(&krwp->bulk_head_free[0])) || + (!list_empty(&krcp->bulk_head[1]) && list_empty(&krwp->bulk_head_free[1])) || (krcp->head && !krwp->head_free)) { + // Channel 1 corresponds to the SLAB-pointer bulk path. // Channel 2 corresponds to vmalloc-pointer bulk path. for (j = 0; j < FREE_N_CHANNELS; j++) { - if (!krwp->bkvhead_free[j]) { - krwp->bkvhead_free[j] = krcp->bkvhead[j]; - krcp->bkvhead[j] = NULL; - } + if (list_empty(&krwp->bulk_head_free[j])) + list_replace_init(&krcp->bulk_head[j], &krwp->bulk_head_free[j]); } // Channel 3 corresponds to both SLAB and vmalloc @@ -3288,10 +3283,11 @@ add_ptr_to_bulk_krc_lock(struct kfree_rcu_cpu **krcp, return false; idx = !!is_vmalloc_addr(ptr); + bnode = list_first_entry_or_null(&(*krcp)->bulk_head[idx], + struct kvfree_rcu_bulk_data, list); /* Check if a new block is required. */ - if (!(*krcp)->bkvhead[idx] || - (*krcp)->bkvhead[idx]->nr_records == KVFREE_BULK_MAX_ENTR) { + if (!bnode || bnode->nr_records == KVFREE_BULK_MAX_ENTR) { bnode = get_cached_bnode(*krcp); if (!bnode && can_alloc) { krc_this_cpu_unlock(*krcp, *flags); @@ -3315,18 +3311,13 @@ add_ptr_to_bulk_krc_lock(struct kfree_rcu_cpu **krcp, if (!bnode) return false; - /* Initialize the new block. */ + // Initialize the new block and attach it. bnode->nr_records = 0; - bnode->next = (*krcp)->bkvhead[idx]; - - /* Attach it to the head. */ - (*krcp)->bkvhead[idx] = bnode; + list_add(&bnode->list, &(*krcp)->bulk_head[idx]); } /* Finally insert. */ - (*krcp)->bkvhead[idx]->records - [(*krcp)->bkvhead[idx]->nr_records++] = ptr; - + bnode->records[bnode->nr_records++] = ptr; return true; } @@ -4761,7 +4752,7 @@ struct workqueue_struct *rcu_gp_wq; static void __init kfree_rcu_batch_init(void) { int cpu; - int i; + int i, j; /* Clamp it to [0:100] seconds interval. */ if (rcu_delay_page_cache_fill_msec < 0 || @@ -4781,8 +4772,14 @@ static void __init kfree_rcu_batch_init(void) for (i = 0; i < KFREE_N_BATCHES; i++) { INIT_RCU_WORK(&krcp->krw_arr[i].rcu_work, kfree_rcu_work); krcp->krw_arr[i].krcp = krcp; + + for (j = 0; j < FREE_N_CHANNELS; j++) + INIT_LIST_HEAD(&krcp->krw_arr[i].bulk_head_free[j]); } + for (i = 0; i < FREE_N_CHANNELS; i++) + INIT_LIST_HEAD(&krcp->bulk_head[i]); + INIT_DELAYED_WORK(&krcp->monitor_work, kfree_rcu_monitor); INIT_DELAYED_WORK(&krcp->page_cache_work, fill_page_cache_func); krcp->initialized = true; From 8c15a9e8086508962b2b69456ed22dc517d91b15 Mon Sep 17 00:00:00 2001 From: "Uladzislau Rezki (Sony)" Date: Tue, 29 Nov 2022 16:58:20 +0100 Subject: [PATCH 026/239] rcu/kvfree: Move bulk/list reclaim to separate functions The kvfree_rcu() code maintains lists of pages of pointers, but also a singly linked list, with the latter being used when memory allocation fails. Traversal of these two types of lists is currently open coded. This commit simplifies the code by providing kvfree_rcu_bulk() and kvfree_rcu_list() functions, respectively, to traverse these two types of lists. This patch does not introduce any functional change. Signed-off-by: Uladzislau Rezki (Sony) Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 114 ++++++++++++++++++++++++++-------------------- 1 file changed, 65 insertions(+), 49 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 4088b34ce961..839e617f6c37 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -3029,6 +3029,65 @@ drain_page_cache(struct kfree_rcu_cpu *krcp) return freed; } +static void +kvfree_rcu_bulk(struct kfree_rcu_cpu *krcp, + struct kvfree_rcu_bulk_data *bnode, int idx) +{ + unsigned long flags; + int i; + + debug_rcu_bhead_unqueue(bnode); + + rcu_lock_acquire(&rcu_callback_map); + if (idx == 0) { // kmalloc() / kfree(). + trace_rcu_invoke_kfree_bulk_callback( + rcu_state.name, bnode->nr_records, + bnode->records); + + kfree_bulk(bnode->nr_records, bnode->records); + } else { // vmalloc() / vfree(). + for (i = 0; i < bnode->nr_records; i++) { + trace_rcu_invoke_kvfree_callback( + rcu_state.name, bnode->records[i], 0); + + vfree(bnode->records[i]); + } + } + rcu_lock_release(&rcu_callback_map); + + raw_spin_lock_irqsave(&krcp->lock, flags); + if (put_cached_bnode(krcp, bnode)) + bnode = NULL; + raw_spin_unlock_irqrestore(&krcp->lock, flags); + + if (bnode) + free_page((unsigned long) bnode); + + cond_resched_tasks_rcu_qs(); +} + +static void +kvfree_rcu_list(struct rcu_head *head) +{ + struct rcu_head *next; + + for (; head; head = next) { + void *ptr = (void *) head->func; + unsigned long offset = (void *) head - ptr; + + next = head->next; + debug_rcu_head_unqueue((struct rcu_head *)ptr); + rcu_lock_acquire(&rcu_callback_map); + trace_rcu_invoke_kvfree_callback(rcu_state.name, head, offset); + + if (!WARN_ON_ONCE(!__is_kvfree_rcu_offset(offset))) + kvfree(ptr); + + rcu_lock_release(&rcu_callback_map); + cond_resched_tasks_rcu_qs(); + } +} + /* * This function is invoked in workqueue context after a grace period. * It frees all the objects queued on ->bulk_head_free or ->head_free. @@ -3038,10 +3097,10 @@ static void kfree_rcu_work(struct work_struct *work) unsigned long flags; struct kvfree_rcu_bulk_data *bnode, *n; struct list_head bulk_head[FREE_N_CHANNELS]; - struct rcu_head *head, *next; + struct rcu_head *head; struct kfree_rcu_cpu *krcp; struct kfree_rcu_cpu_work *krwp; - int i, j; + int i; krwp = container_of(to_rcu_work(work), struct kfree_rcu_cpu_work, rcu_work); @@ -3058,38 +3117,9 @@ static void kfree_rcu_work(struct work_struct *work) raw_spin_unlock_irqrestore(&krcp->lock, flags); // Handle the first two channels. - for (i = 0; i < FREE_N_CHANNELS; i++) { - list_for_each_entry_safe(bnode, n, &bulk_head[i], list) { - debug_rcu_bhead_unqueue(bnode); - - rcu_lock_acquire(&rcu_callback_map); - if (i == 0) { // kmalloc() / kfree(). - trace_rcu_invoke_kfree_bulk_callback( - rcu_state.name, bnode->nr_records, - bnode->records); - - kfree_bulk(bnode->nr_records, bnode->records); - } else { // vmalloc() / vfree(). - for (j = 0; j < bnode->nr_records; j++) { - trace_rcu_invoke_kvfree_callback( - rcu_state.name, bnode->records[j], 0); - - vfree(bnode->records[j]); - } - } - rcu_lock_release(&rcu_callback_map); - - raw_spin_lock_irqsave(&krcp->lock, flags); - if (put_cached_bnode(krcp, bnode)) - bnode = NULL; - raw_spin_unlock_irqrestore(&krcp->lock, flags); - - if (bnode) - free_page((unsigned long) bnode); - - cond_resched_tasks_rcu_qs(); - } - } + for (i = 0; i < FREE_N_CHANNELS; i++) + list_for_each_entry_safe(bnode, n, &bulk_head[i], list) + kvfree_rcu_bulk(krcp, bnode, i); /* * This is used when the "bulk" path can not be used for the @@ -3098,21 +3128,7 @@ static void kfree_rcu_work(struct work_struct *work) * queued on a linked list through their rcu_head structures. * This list is named "Channel 3". */ - for (; head; head = next) { - void *ptr = (void *) head->func; - unsigned long offset = (void *) head - ptr; - - next = head->next; - debug_rcu_head_unqueue((struct rcu_head *)ptr); - rcu_lock_acquire(&rcu_callback_map); - trace_rcu_invoke_kvfree_callback(rcu_state.name, head, offset); - - if (!WARN_ON_ONCE(!__is_kvfree_rcu_offset(offset))) - kvfree(ptr); - - rcu_lock_release(&rcu_callback_map); - cond_resched_tasks_rcu_qs(); - } + kvfree_rcu_list(head); } static bool From 8fc5494ad5face62747a3937db66b00db1e5d80b Mon Sep 17 00:00:00 2001 From: "Uladzislau Rezki (Sony)" Date: Tue, 29 Nov 2022 16:58:21 +0100 Subject: [PATCH 027/239] rcu/kvfree: Move need_offload_krc() out of krcp->lock The need_offload_krc() function currently holds the krcp->lock in order to safely check krcp->head. This commit removes the need for this lock in that function by updating the krcp->head pointer using WRITE_ONCE() macro so that readers can carry out lockless loads of that pointer. Signed-off-by: Uladzislau Rezki (Sony) Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 839e617f6c37..0c42fce4efe3 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -3194,7 +3194,7 @@ static void kfree_rcu_monitor(struct work_struct *work) // objects queued on the linked list. if (!krwp->head_free) { krwp->head_free = krcp->head; - krcp->head = NULL; + WRITE_ONCE(krcp->head, NULL); } WRITE_ONCE(krcp->count, 0); @@ -3208,6 +3208,8 @@ static void kfree_rcu_monitor(struct work_struct *work) } } + raw_spin_unlock_irqrestore(&krcp->lock, flags); + // If there is nothing to detach, it means that our job is // successfully done here. In case of having at least one // of the channels that is still busy we should rearm the @@ -3215,8 +3217,6 @@ static void kfree_rcu_monitor(struct work_struct *work) // still in progress. if (need_offload_krc(krcp)) schedule_delayed_monitor_work(krcp); - - raw_spin_unlock_irqrestore(&krcp->lock, flags); } static enum hrtimer_restart @@ -3386,7 +3386,7 @@ void kvfree_call_rcu(struct rcu_head *head, void *ptr) head->func = ptr; head->next = krcp->head; - krcp->head = head; + WRITE_ONCE(krcp->head, head); success = true; } @@ -3463,15 +3463,12 @@ static struct shrinker kfree_rcu_shrinker = { void __init kfree_rcu_scheduler_running(void) { int cpu; - unsigned long flags; for_each_possible_cpu(cpu) { struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu); - raw_spin_lock_irqsave(&krcp->lock, flags); if (need_offload_krc(krcp)) schedule_delayed_monitor_work(krcp); - raw_spin_unlock_irqrestore(&krcp->lock, flags); } } From cc37d52076a91d8391bbd16249a5790a35292b85 Mon Sep 17 00:00:00 2001 From: "Uladzislau Rezki (Sony)" Date: Tue, 29 Nov 2022 16:58:22 +0100 Subject: [PATCH 028/239] rcu/kvfree: Use a polled API to speedup a reclaim process Currently all objects placed into a batch wait for a full grace period to elapse after that batch is ready to send to RCU. However, this can unnecessarily delay freeing of the first objects that were added to the batch. After all, several RCU grace periods might have elapsed since those objects were added, and if so, there is no point in further deferring their freeing. This commit therefore adds per-page grace-period snapshots which are obtained from get_state_synchronize_rcu(). When the batch is ready to be passed to call_rcu(), each page's snapshot is checked by passing it to poll_state_synchronize_rcu(). If a given page's RCU grace period has already elapsed, its objects are freed immediately by kvfree_rcu_bulk(). Otherwise, these objects are freed after a call to synchronize_rcu(). This approach requires that the pages be traversed in reverse order, that is, the oldest ones first. Test example: kvm.sh --memory 10G --torture rcuscale --allcpus --duration 1 \ --kconfig CONFIG_NR_CPUS=64 \ --kconfig CONFIG_RCU_NOCB_CPU=y \ --kconfig CONFIG_RCU_NOCB_CPU_DEFAULT_ALL=y \ --kconfig CONFIG_RCU_LAZY=n \ --bootargs "rcuscale.kfree_rcu_test=1 rcuscale.kfree_nthreads=16 \ rcuscale.holdoff=20 rcuscale.kfree_loops=10000 \ torture.disable_onoff_at_boot" --trust-make Before this commit: Total time taken by all kfree'ers: 8535693700 ns, loops: 10000, batches: 1188, memory footprint: 2248MB Total time taken by all kfree'ers: 8466933582 ns, loops: 10000, batches: 1157, memory footprint: 2820MB Total time taken by all kfree'ers: 5375602446 ns, loops: 10000, batches: 1130, memory footprint: 6502MB Total time taken by all kfree'ers: 7523283832 ns, loops: 10000, batches: 1006, memory footprint: 3343MB Total time taken by all kfree'ers: 6459171956 ns, loops: 10000, batches: 1150, memory footprint: 6549MB After this commit: Total time taken by all kfree'ers: 8560060176 ns, loops: 10000, batches: 1787, memory footprint: 61MB Total time taken by all kfree'ers: 8573885501 ns, loops: 10000, batches: 1777, memory footprint: 93MB Total time taken by all kfree'ers: 8320000202 ns, loops: 10000, batches: 1727, memory footprint: 66MB Total time taken by all kfree'ers: 8552718794 ns, loops: 10000, batches: 1790, memory footprint: 75MB Total time taken by all kfree'ers: 8601368792 ns, loops: 10000, batches: 1724, memory footprint: 62MB The reduction in memory footprint is well in excess of an order of magnitude. Signed-off-by: Uladzislau Rezki (Sony) Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 47 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 0c42fce4efe3..735312f78e98 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2877,11 +2877,13 @@ EXPORT_SYMBOL_GPL(call_rcu); /** * struct kvfree_rcu_bulk_data - single block to store kvfree_rcu() pointers * @list: List node. All blocks are linked between each other + * @gp_snap: Snapshot of RCU state for objects placed to this bulk * @nr_records: Number of active pointers in the array * @records: Array of the kvfree_rcu() pointers */ struct kvfree_rcu_bulk_data { struct list_head list; + unsigned long gp_snap; unsigned long nr_records; void *records[]; }; @@ -2898,13 +2900,15 @@ struct kvfree_rcu_bulk_data { * struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests * @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period * @head_free: List of kfree_rcu() objects waiting for a grace period + * @head_free_gp_snap: Snapshot of RCU state for objects placed to "@head_free" * @bulk_head_free: Bulk-List of kvfree_rcu() objects waiting for a grace period * @krcp: Pointer to @kfree_rcu_cpu structure */ struct kfree_rcu_cpu_work { - struct rcu_work rcu_work; + struct work_struct rcu_work; struct rcu_head *head_free; + unsigned long head_free_gp_snap; struct list_head bulk_head_free[FREE_N_CHANNELS]; struct kfree_rcu_cpu *krcp; }; @@ -3100,10 +3104,11 @@ static void kfree_rcu_work(struct work_struct *work) struct rcu_head *head; struct kfree_rcu_cpu *krcp; struct kfree_rcu_cpu_work *krwp; + unsigned long head_free_gp_snap; int i; - krwp = container_of(to_rcu_work(work), - struct kfree_rcu_cpu_work, rcu_work); + krwp = container_of(work, + struct kfree_rcu_cpu_work, rcu_work); krcp = krwp->krcp; raw_spin_lock_irqsave(&krcp->lock, flags); @@ -3114,12 +3119,29 @@ static void kfree_rcu_work(struct work_struct *work) // Channel 3. head = krwp->head_free; krwp->head_free = NULL; + head_free_gp_snap = krwp->head_free_gp_snap; raw_spin_unlock_irqrestore(&krcp->lock, flags); // Handle the first two channels. - for (i = 0; i < FREE_N_CHANNELS; i++) + for (i = 0; i < FREE_N_CHANNELS; i++) { + // Start from the tail page, so a GP is likely passed for it. + list_for_each_entry_safe_reverse(bnode, n, &bulk_head[i], list) { + // Not yet ready? Bail out since we need one more GP. + if (!poll_state_synchronize_rcu(bnode->gp_snap)) + break; + + list_del_init(&bnode->list); + kvfree_rcu_bulk(krcp, bnode, i); + } + + // Please note a request for one more extra GP can + // occur only once for all objects in this batch. + if (!list_empty(&bulk_head[i])) + synchronize_rcu(); + list_for_each_entry_safe(bnode, n, &bulk_head[i], list) kvfree_rcu_bulk(krcp, bnode, i); + } /* * This is used when the "bulk" path can not be used for the @@ -3128,7 +3150,10 @@ static void kfree_rcu_work(struct work_struct *work) * queued on a linked list through their rcu_head structures. * This list is named "Channel 3". */ - kvfree_rcu_list(head); + if (head) { + cond_synchronize_rcu(head_free_gp_snap); + kvfree_rcu_list(head); + } } static bool @@ -3195,6 +3220,11 @@ static void kfree_rcu_monitor(struct work_struct *work) if (!krwp->head_free) { krwp->head_free = krcp->head; WRITE_ONCE(krcp->head, NULL); + + // Take a snapshot for this krwp. Please note no more + // any objects can be added to attached head_free channel + // therefore fixate a GP for it here. + krwp->head_free_gp_snap = get_state_synchronize_rcu(); } WRITE_ONCE(krcp->count, 0); @@ -3204,7 +3234,7 @@ static void kfree_rcu_monitor(struct work_struct *work) // be that the work is in the pending state when // channels have been detached following by each // other. - queue_rcu_work(system_wq, &krwp->rcu_work); + queue_work(system_wq, &krwp->rcu_work); } } @@ -3332,8 +3362,9 @@ add_ptr_to_bulk_krc_lock(struct kfree_rcu_cpu **krcp, list_add(&bnode->list, &(*krcp)->bulk_head[idx]); } - /* Finally insert. */ + // Finally insert and update the GP for this page. bnode->records[bnode->nr_records++] = ptr; + bnode->gp_snap = get_state_synchronize_rcu(); return true; } @@ -4783,7 +4814,7 @@ static void __init kfree_rcu_batch_init(void) struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu); for (i = 0; i < KFREE_N_BATCHES; i++) { - INIT_RCU_WORK(&krcp->krw_arr[i].rcu_work, kfree_rcu_work); + INIT_WORK(&krcp->krw_arr[i].rcu_work, kfree_rcu_work); krcp->krw_arr[i].krcp = krcp; for (j = 0; j < FREE_N_CHANNELS; j++) From 9627456101ec9bb502daae7276e5141f66a9ddd1 Mon Sep 17 00:00:00 2001 From: "Uladzislau Rezki (Sony)" Date: Fri, 2 Dec 2022 14:18:37 +0100 Subject: [PATCH 029/239] rcu/kvfree: Use READ_ONCE() when access to krcp->head The need_offload_krc() function is now lock-free, which gives the compiler freedom to load old values from plain C-language loads from the kfree_rcu_cpu struture's ->head pointer. This commit therefore applied READ_ONCE() to these loads. Signed-off-by: Uladzislau Rezki (Sony) Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 735312f78e98..02551e0e1132 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -3165,7 +3165,7 @@ need_offload_krc(struct kfree_rcu_cpu *krcp) if (!list_empty(&krcp->bulk_head[i])) return true; - return !!krcp->head; + return !!READ_ONCE(krcp->head); } static void @@ -3206,7 +3206,7 @@ static void kfree_rcu_monitor(struct work_struct *work) // in that case the monitor work is rearmed. if ((!list_empty(&krcp->bulk_head[0]) && list_empty(&krwp->bulk_head_free[0])) || (!list_empty(&krcp->bulk_head[1]) && list_empty(&krwp->bulk_head_free[1])) || - (krcp->head && !krwp->head_free)) { + (READ_ONCE(krcp->head) && !krwp->head_free)) { // Channel 1 corresponds to the SLAB-pointer bulk path. // Channel 2 corresponds to vmalloc-pointer bulk path. From 4c33464ae85e59cba3f8048a34d571edf229823a Mon Sep 17 00:00:00 2001 From: "Uladzislau Rezki (Sony)" Date: Wed, 14 Dec 2022 13:06:29 +0100 Subject: [PATCH 030/239] rcu/kvfree: Carefully reset number of objects in krcp The schedule_delayed_monitor_work() function relies on the count of objects queued into any given kfree_rcu_cpu structure. This count is used to determine how quickly to schedule passing these objects to RCU. There are three pipes where pointers can be placed. When any pipe is offloaded, the kfree_rcu_cpu structure's ->count counter is set to zero, which is wrong because the other pipes might still be non-empty. This commit therefore maintains per-pipe counters, and introduces a krc_count() helper to access the aggregate value of those counters. Signed-off-by: Uladzislau Rezki (Sony) Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 02551e0e1132..52f4c7e87f88 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2921,7 +2921,8 @@ struct kfree_rcu_cpu_work { * @lock: Synchronize access to this structure * @monitor_work: Promote @head to @head_free after KFREE_DRAIN_JIFFIES * @initialized: The @rcu_work fields have been initialized - * @count: Number of objects for which GP not started + * @head_count: Number of objects in rcu_head singular list + * @bulk_count: Number of objects in bulk-list * @bkvcache: * A simple cache list that contains objects for reuse purpose. * In order to save some per-cpu space the list is singular. @@ -2939,13 +2940,19 @@ struct kfree_rcu_cpu_work { * the interactions with the slab allocators. */ struct kfree_rcu_cpu { + // Objects queued on a linked list + // through their rcu_head structures. struct rcu_head *head; + atomic_t head_count; + + // Objects queued on a bulk-list. struct list_head bulk_head[FREE_N_CHANNELS]; + atomic_t bulk_count[FREE_N_CHANNELS]; + struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES]; raw_spinlock_t lock; struct delayed_work monitor_work; bool initialized; - int count; struct delayed_work page_cache_work; atomic_t backoff_page_cache_fill; @@ -3168,12 +3175,23 @@ need_offload_krc(struct kfree_rcu_cpu *krcp) return !!READ_ONCE(krcp->head); } +static int krc_count(struct kfree_rcu_cpu *krcp) +{ + int sum = atomic_read(&krcp->head_count); + int i; + + for (i = 0; i < FREE_N_CHANNELS; i++) + sum += atomic_read(&krcp->bulk_count[i]); + + return sum; +} + static void schedule_delayed_monitor_work(struct kfree_rcu_cpu *krcp) { long delay, delay_left; - delay = READ_ONCE(krcp->count) >= KVFREE_BULK_MAX_ENTR ? 1:KFREE_DRAIN_JIFFIES; + delay = krc_count(krcp) >= KVFREE_BULK_MAX_ENTR ? 1:KFREE_DRAIN_JIFFIES; if (delayed_work_pending(&krcp->monitor_work)) { delay_left = krcp->monitor_work.timer.expires - jiffies; if (delay < delay_left) @@ -3211,8 +3229,10 @@ static void kfree_rcu_monitor(struct work_struct *work) // Channel 1 corresponds to the SLAB-pointer bulk path. // Channel 2 corresponds to vmalloc-pointer bulk path. for (j = 0; j < FREE_N_CHANNELS; j++) { - if (list_empty(&krwp->bulk_head_free[j])) + if (list_empty(&krwp->bulk_head_free[j])) { list_replace_init(&krcp->bulk_head[j], &krwp->bulk_head_free[j]); + atomic_set(&krcp->bulk_count[j], 0); + } } // Channel 3 corresponds to both SLAB and vmalloc @@ -3220,6 +3240,7 @@ static void kfree_rcu_monitor(struct work_struct *work) if (!krwp->head_free) { krwp->head_free = krcp->head; WRITE_ONCE(krcp->head, NULL); + atomic_set(&krcp->head_count, 0); // Take a snapshot for this krwp. Please note no more // any objects can be added to attached head_free channel @@ -3227,8 +3248,6 @@ static void kfree_rcu_monitor(struct work_struct *work) krwp->head_free_gp_snap = get_state_synchronize_rcu(); } - WRITE_ONCE(krcp->count, 0); - // One work is per one batch, so there are three // "free channels", the batch can handle. It can // be that the work is in the pending state when @@ -3365,6 +3384,8 @@ add_ptr_to_bulk_krc_lock(struct kfree_rcu_cpu **krcp, // Finally insert and update the GP for this page. bnode->records[bnode->nr_records++] = ptr; bnode->gp_snap = get_state_synchronize_rcu(); + atomic_inc(&(*krcp)->bulk_count[idx]); + return true; } @@ -3418,11 +3439,10 @@ void kvfree_call_rcu(struct rcu_head *head, void *ptr) head->func = ptr; head->next = krcp->head; WRITE_ONCE(krcp->head, head); + atomic_inc(&krcp->head_count); success = true; } - WRITE_ONCE(krcp->count, krcp->count + 1); - // Set timer to drain after KFREE_DRAIN_JIFFIES. if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING) schedule_delayed_monitor_work(krcp); @@ -3453,7 +3473,7 @@ kfree_rcu_shrink_count(struct shrinker *shrink, struct shrink_control *sc) for_each_possible_cpu(cpu) { struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu); - count += READ_ONCE(krcp->count); + count += krc_count(krcp); count += READ_ONCE(krcp->nr_bkv_objs); atomic_set(&krcp->backoff_page_cache_fill, 1); } @@ -3470,7 +3490,7 @@ kfree_rcu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) int count; struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu); - count = krcp->count; + count = krc_count(krcp); count += drain_page_cache(krcp); kfree_rcu_monitor(&krcp->monitor_work.work); From 2ca836b1da1777c75b7363a7ca2973e8ab11fc21 Mon Sep 17 00:00:00 2001 From: "Uladzislau Rezki (Sony)" Date: Wed, 14 Dec 2022 13:06:30 +0100 Subject: [PATCH 031/239] rcu/kvfree: Split ready for reclaim objects from a batch This patch splits the lists of objects so as to avoid sending any through RCU that have already been queued for more than one grace period. These long-term-resident objects are immediately freed. The remaining short-term-resident objects are queued for later freeing using queue_rcu_work(). This change avoids delaying workqueue handlers with synchronize_rcu() invocations. Yes, workqueue handlers are designed to handle blocking, but avoiding blocking when unnecessary improves performance during low-memory situations. Signed-off-by: Uladzislau Rezki (Sony) Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 87 +++++++++++++++++++++++++++++------------------ 1 file changed, 54 insertions(+), 33 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 52f4c7e87f88..0b4f7dd55157 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2900,15 +2900,13 @@ struct kvfree_rcu_bulk_data { * struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests * @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period * @head_free: List of kfree_rcu() objects waiting for a grace period - * @head_free_gp_snap: Snapshot of RCU state for objects placed to "@head_free" * @bulk_head_free: Bulk-List of kvfree_rcu() objects waiting for a grace period * @krcp: Pointer to @kfree_rcu_cpu structure */ struct kfree_rcu_cpu_work { - struct work_struct rcu_work; + struct rcu_work rcu_work; struct rcu_head *head_free; - unsigned long head_free_gp_snap; struct list_head bulk_head_free[FREE_N_CHANNELS]; struct kfree_rcu_cpu *krcp; }; @@ -2916,6 +2914,7 @@ struct kfree_rcu_cpu_work { /** * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period * @head: List of kfree_rcu() objects not yet waiting for a grace period + * @head_gp_snap: Snapshot of RCU state for objects placed to "@head" * @bulk_head: Bulk-List of kvfree_rcu() objects not yet waiting for a grace period * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period * @lock: Synchronize access to this structure @@ -2943,6 +2942,7 @@ struct kfree_rcu_cpu { // Objects queued on a linked list // through their rcu_head structures. struct rcu_head *head; + unsigned long head_gp_snap; atomic_t head_count; // Objects queued on a bulk-list. @@ -3111,10 +3111,9 @@ static void kfree_rcu_work(struct work_struct *work) struct rcu_head *head; struct kfree_rcu_cpu *krcp; struct kfree_rcu_cpu_work *krwp; - unsigned long head_free_gp_snap; int i; - krwp = container_of(work, + krwp = container_of(to_rcu_work(work), struct kfree_rcu_cpu_work, rcu_work); krcp = krwp->krcp; @@ -3126,26 +3125,11 @@ static void kfree_rcu_work(struct work_struct *work) // Channel 3. head = krwp->head_free; krwp->head_free = NULL; - head_free_gp_snap = krwp->head_free_gp_snap; raw_spin_unlock_irqrestore(&krcp->lock, flags); // Handle the first two channels. for (i = 0; i < FREE_N_CHANNELS; i++) { // Start from the tail page, so a GP is likely passed for it. - list_for_each_entry_safe_reverse(bnode, n, &bulk_head[i], list) { - // Not yet ready? Bail out since we need one more GP. - if (!poll_state_synchronize_rcu(bnode->gp_snap)) - break; - - list_del_init(&bnode->list); - kvfree_rcu_bulk(krcp, bnode, i); - } - - // Please note a request for one more extra GP can - // occur only once for all objects in this batch. - if (!list_empty(&bulk_head[i])) - synchronize_rcu(); - list_for_each_entry_safe(bnode, n, &bulk_head[i], list) kvfree_rcu_bulk(krcp, bnode, i); } @@ -3157,10 +3141,7 @@ static void kfree_rcu_work(struct work_struct *work) * queued on a linked list through their rcu_head structures. * This list is named "Channel 3". */ - if (head) { - cond_synchronize_rcu(head_free_gp_snap); - kvfree_rcu_list(head); - } + kvfree_rcu_list(head); } static bool @@ -3201,6 +3182,44 @@ schedule_delayed_monitor_work(struct kfree_rcu_cpu *krcp) queue_delayed_work(system_wq, &krcp->monitor_work, delay); } +static void +kvfree_rcu_drain_ready(struct kfree_rcu_cpu *krcp) +{ + struct list_head bulk_ready[FREE_N_CHANNELS]; + struct kvfree_rcu_bulk_data *bnode, *n; + struct rcu_head *head_ready = NULL; + unsigned long flags; + int i; + + raw_spin_lock_irqsave(&krcp->lock, flags); + for (i = 0; i < FREE_N_CHANNELS; i++) { + INIT_LIST_HEAD(&bulk_ready[i]); + + list_for_each_entry_safe_reverse(bnode, n, &krcp->bulk_head[i], list) { + if (!poll_state_synchronize_rcu(bnode->gp_snap)) + break; + + atomic_sub(bnode->nr_records, &krcp->bulk_count[i]); + list_move(&bnode->list, &bulk_ready[i]); + } + } + + if (krcp->head && poll_state_synchronize_rcu(krcp->head_gp_snap)) { + head_ready = krcp->head; + atomic_set(&krcp->head_count, 0); + WRITE_ONCE(krcp->head, NULL); + } + raw_spin_unlock_irqrestore(&krcp->lock, flags); + + for (i = 0; i < FREE_N_CHANNELS; i++) { + list_for_each_entry_safe(bnode, n, &bulk_ready[i], list) + kvfree_rcu_bulk(krcp, bnode, i); + } + + if (head_ready) + kvfree_rcu_list(head_ready); +} + /* * This function is invoked after the KFREE_DRAIN_JIFFIES timeout. */ @@ -3211,6 +3230,9 @@ static void kfree_rcu_monitor(struct work_struct *work) unsigned long flags; int i, j; + // Drain ready for reclaim. + kvfree_rcu_drain_ready(krcp); + raw_spin_lock_irqsave(&krcp->lock, flags); // Attempt to start a new batch. @@ -3230,8 +3252,9 @@ static void kfree_rcu_monitor(struct work_struct *work) // Channel 2 corresponds to vmalloc-pointer bulk path. for (j = 0; j < FREE_N_CHANNELS; j++) { if (list_empty(&krwp->bulk_head_free[j])) { - list_replace_init(&krcp->bulk_head[j], &krwp->bulk_head_free[j]); atomic_set(&krcp->bulk_count[j], 0); + list_replace_init(&krcp->bulk_head[j], + &krwp->bulk_head_free[j]); } } @@ -3239,13 +3262,8 @@ static void kfree_rcu_monitor(struct work_struct *work) // objects queued on the linked list. if (!krwp->head_free) { krwp->head_free = krcp->head; - WRITE_ONCE(krcp->head, NULL); atomic_set(&krcp->head_count, 0); - - // Take a snapshot for this krwp. Please note no more - // any objects can be added to attached head_free channel - // therefore fixate a GP for it here. - krwp->head_free_gp_snap = get_state_synchronize_rcu(); + WRITE_ONCE(krcp->head, NULL); } // One work is per one batch, so there are three @@ -3253,7 +3271,7 @@ static void kfree_rcu_monitor(struct work_struct *work) // be that the work is in the pending state when // channels have been detached following by each // other. - queue_work(system_wq, &krwp->rcu_work); + queue_rcu_work(system_wq, &krwp->rcu_work); } } @@ -3440,6 +3458,9 @@ void kvfree_call_rcu(struct rcu_head *head, void *ptr) head->next = krcp->head; WRITE_ONCE(krcp->head, head); atomic_inc(&krcp->head_count); + + // Take a snapshot for this krcp. + krcp->head_gp_snap = get_state_synchronize_rcu(); success = true; } @@ -4834,7 +4855,7 @@ static void __init kfree_rcu_batch_init(void) struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu); for (i = 0; i < KFREE_N_BATCHES; i++) { - INIT_WORK(&krcp->krw_arr[i].rcu_work, kfree_rcu_work); + INIT_RCU_WORK(&krcp->krw_arr[i].rcu_work, kfree_rcu_work); krcp->krw_arr[i].krcp = krcp; for (j = 0; j < FREE_N_CHANNELS; j++) From 66ea1029f9b839fbcc10d97ddc93a05c72b3acc1 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Thu, 10 Nov 2022 15:30:13 +0800 Subject: [PATCH 032/239] srcu: Release early_srcu resources when no longer in use Kernels built with the CONFIG_TREE_SRCU Kconfig option set and then booted with rcupdate.rcu_self_test=1 and srcutree.convert_to_big=1 will test Tree SRCU during early boot. The early_srcu structure's srcu_node array will be allocated when init_srcu_struct_fields() is invoked, but after the test completes this early_srcu structure will not be used. This commit therefore invokes cleanup_srcu_struct() to free that srcu_node structure. Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/update.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c index f5e6a2f95a2a..a5b4abbee643 100644 --- a/kernel/rcu/update.c +++ b/kernel/rcu/update.c @@ -586,6 +586,7 @@ static int rcu_verify_early_boot_tests(void) early_boot_test_counter++; srcu_barrier(&early_srcu); WARN_ON_ONCE(!poll_state_synchronize_srcu(&early_srcu, early_srcu_cookie)); + cleanup_srcu_struct(&early_srcu); } if (rcu_self_test_counter != early_boot_test_counter) { WARN_ON(1); From 7f24626d6dd844bfc6d1f492d214d29c86d02550 Mon Sep 17 00:00:00 2001 From: Pingfan Liu Date: Mon, 31 Oct 2022 09:52:37 +0800 Subject: [PATCH 033/239] srcu: Delegate work to the boot cpu if using SRCU_SIZE_SMALL Commit 994f706872e6 ("srcu: Make Tree SRCU able to operate without snp_node array") assumes that cpu 0 is always online. However, there really are situations when some other CPU is the boot CPU, for example, when booting a kdump kernel with the maxcpus=1 boot parameter. On PowerPC, the kdump kernel can hang as follows: ... [ 1.740036] systemd[1]: Hostname set to [ 243.686240] INFO: task systemd:1 blocked for more than 122 seconds. [ 243.686264] Not tainted 6.1.0-rc1 #1 [ 243.686272] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 243.686281] task:systemd state:D stack:0 pid:1 ppid:0 flags:0x00042000 [ 243.686296] Call Trace: [ 243.686301] [c000000016657640] [c000000016657670] 0xc000000016657670 (unreliable) [ 243.686317] [c000000016657830] [c00000001001dec0] __switch_to+0x130/0x220 [ 243.686333] [c000000016657890] [c000000010f607b8] __schedule+0x1f8/0x580 [ 243.686347] [c000000016657940] [c000000010f60bb4] schedule+0x74/0x140 [ 243.686361] [c0000000166579b0] [c000000010f699b8] schedule_timeout+0x168/0x1c0 [ 243.686374] [c000000016657a80] [c000000010f61de8] __wait_for_common+0x148/0x360 [ 243.686387] [c000000016657b20] [c000000010176bb0] __flush_work.isra.0+0x1c0/0x3d0 [ 243.686401] [c000000016657bb0] [c0000000105f2768] fsnotify_wait_marks_destroyed+0x28/0x40 [ 243.686415] [c000000016657bd0] [c0000000105f21b8] fsnotify_destroy_group+0x68/0x160 [ 243.686428] [c000000016657c40] [c0000000105f6500] inotify_release+0x30/0xa0 [ 243.686440] [c000000016657cb0] [c0000000105751a8] __fput+0xc8/0x350 [ 243.686452] [c000000016657d00] [c00000001017d524] task_work_run+0xe4/0x170 [ 243.686464] [c000000016657d50] [c000000010020e94] do_notify_resume+0x134/0x140 [ 243.686478] [c000000016657d80] [c00000001002eb18] interrupt_exit_user_prepare_main+0x198/0x270 [ 243.686493] [c000000016657de0] [c00000001002ec60] syscall_exit_prepare+0x70/0x180 [ 243.686505] [c000000016657e10] [c00000001000bf7c] system_call_vectored_common+0xfc/0x280 [ 243.686520] --- interrupt: 3000 at 0x7fffa47d5ba4 [ 243.686528] NIP: 00007fffa47d5ba4 LR: 0000000000000000 CTR: 0000000000000000 [ 243.686538] REGS: c000000016657e80 TRAP: 3000 Not tainted (6.1.0-rc1) [ 243.686548] MSR: 800000000000d033 CR: 42044440 XER: 00000000 [ 243.686572] IRQMASK: 0 [ 243.686572] GPR00: 0000000000000006 00007ffffa606710 00007fffa48e7200 0000000000000000 [ 243.686572] GPR04: 0000000000000002 000000000000000a 0000000000000000 0000000000000001 [ 243.686572] GPR08: 000001000c172dd0 0000000000000000 0000000000000000 0000000000000000 [ 243.686572] GPR12: 0000000000000000 00007fffa4ff4bc0 0000000000000000 0000000000000000 [ 243.686572] GPR16: 0000000000000000 0000000000000000 0000000000000000 0000000000000000 [ 243.686572] GPR20: 0000000132dfdc50 000000000000000e 0000000000189375 0000000000000000 [ 243.686572] GPR24: 00007ffffa606ae0 0000000000000005 000001000c185490 000001000c172570 [ 243.686572] GPR28: 000001000c172990 000001000c184850 000001000c172e00 00007fffa4fedd98 [ 243.686683] NIP [00007fffa47d5ba4] 0x7fffa47d5ba4 [ 243.686691] LR [0000000000000000] 0x0 [ 243.686698] --- interrupt: 3000 [ 243.686708] INFO: task kworker/u16:1:24 blocked for more than 122 seconds. [ 243.686717] Not tainted 6.1.0-rc1 #1 [ 243.686724] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 243.686733] task:kworker/u16:1 state:D stack:0 pid:24 ppid:2 flags:0x00000800 [ 243.686747] Workqueue: events_unbound fsnotify_mark_destroy_workfn [ 243.686758] Call Trace: [ 243.686762] [c0000000166736e0] [c00000004fd91000] 0xc00000004fd91000 (unreliable) [ 243.686775] [c0000000166738d0] [c00000001001dec0] __switch_to+0x130/0x220 [ 243.686788] [c000000016673930] [c000000010f607b8] __schedule+0x1f8/0x580 [ 243.686801] [c0000000166739e0] [c000000010f60bb4] schedule+0x74/0x140 [ 243.686814] [c000000016673a50] [c000000010f699b8] schedule_timeout+0x168/0x1c0 [ 243.686827] [c000000016673b20] [c000000010f61de8] __wait_for_common+0x148/0x360 [ 243.686840] [c000000016673bc0] [c000000010210840] __synchronize_srcu.part.0+0xa0/0xe0 [ 243.686855] [c000000016673c30] [c0000000105f2c64] fsnotify_mark_destroy_workfn+0xc4/0x1a0 [ 243.686868] [c000000016673ca0] [c000000010174ea8] process_one_work+0x2a8/0x570 [ 243.686882] [c000000016673d40] [c000000010175208] worker_thread+0x98/0x5e0 [ 243.686895] [c000000016673dc0] [c0000000101828d4] kthread+0x124/0x130 [ 243.686908] [c000000016673e10] [c00000001000cd40] ret_from_kernel_thread+0x5c/0x64 [ 366.566274] INFO: task systemd:1 blocked for more than 245 seconds. [ 366.566298] Not tainted 6.1.0-rc1 #1 [ 366.566305] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 366.566314] task:systemd state:D stack:0 pid:1 ppid:0 flags:0x00042000 [ 366.566329] Call Trace: ... The above splat occurs because PowerPC really does use maxcpus=1 instead of nr_cpus=1 in the kernel command line. Consequently, the (quite possibly non-zero) kdump CPU is the only online CPU in the kdump kernel. SRCU unconditionally queues a sdp->work on cpu 0, for which no worker thread has been created, so sdp->work will be never executed and __synchronize_srcu() will never be completed. This commit therefore replaces CPU ID 0 with get_boot_cpu_id() in key places in Tree SRCU. Since the CPU indicated by get_boot_cpu_id() is guaranteed to be online, this avoids the above splat. Signed-off-by: Pingfan Liu Cc: "Paul E. McKenney" Cc: Lai Jiangshan Cc: Josh Triplett Cc: Steven Rostedt Cc: Mathieu Desnoyers To: rcu@vger.kernel.org Signed-off-by: Paul E. McKenney --- kernel/rcu/srcutree.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index ca4b5dcec675..16953784a0bd 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -726,7 +726,7 @@ static void srcu_gp_start(struct srcu_struct *ssp) int state; if (smp_load_acquire(&ssp->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER) - sdp = per_cpu_ptr(ssp->sda, 0); + sdp = per_cpu_ptr(ssp->sda, get_boot_cpu_id()); else sdp = this_cpu_ptr(ssp->sda); lockdep_assert_held(&ACCESS_PRIVATE(ssp, lock)); @@ -837,7 +837,8 @@ static void srcu_gp_end(struct srcu_struct *ssp) /* Initiate callback invocation as needed. */ ss_state = smp_load_acquire(&ssp->srcu_size_state); if (ss_state < SRCU_SIZE_WAIT_BARRIER) { - srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, 0), cbdelay); + srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, get_boot_cpu_id()), + cbdelay); } else { idx = rcu_seq_ctr(gpseq) % ARRAY_SIZE(snp->srcu_have_cbs); srcu_for_each_node_breadth_first(ssp, snp) { @@ -1161,7 +1162,7 @@ static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp, idx = __srcu_read_lock_nmisafe(ssp); ss_state = smp_load_acquire(&ssp->srcu_size_state); if (ss_state < SRCU_SIZE_WAIT_CALL) - sdp = per_cpu_ptr(ssp->sda, 0); + sdp = per_cpu_ptr(ssp->sda, get_boot_cpu_id()); else sdp = raw_cpu_ptr(ssp->sda); spin_lock_irqsave_sdp_contention(sdp, &flags); @@ -1497,7 +1498,7 @@ void srcu_barrier(struct srcu_struct *ssp) idx = __srcu_read_lock_nmisafe(ssp); if (smp_load_acquire(&ssp->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER) - srcu_barrier_one_cpu(ssp, per_cpu_ptr(ssp->sda, 0)); + srcu_barrier_one_cpu(ssp, per_cpu_ptr(ssp->sda, get_boot_cpu_id())); else for_each_possible_cpu(cpu) srcu_barrier_one_cpu(ssp, per_cpu_ptr(ssp->sda, cpu)); From aa5210f524edcebf909e6576b515bc3e2d82af0d Mon Sep 17 00:00:00 2001 From: Pingfan Liu Date: Wed, 16 Nov 2022 09:52:43 +0800 Subject: [PATCH 034/239] srcu: Fix a misspelling in comment s/srcu_gq_seq/srcu_gp_seq/ Signed-off-by: Pingfan Liu Cc: Lai Jiangshan Cc: Josh Triplett Cc: Steven Rostedt Cc: Mathieu Desnoyers Cc: Reviewed-by: Mukesh Ojha Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- include/linux/srcutree.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h index c689a81752c9..558057b517b7 100644 --- a/include/linux/srcutree.h +++ b/include/linux/srcutree.h @@ -49,7 +49,7 @@ struct srcu_data { struct srcu_node { spinlock_t __private lock; unsigned long srcu_have_cbs[4]; /* GP seq for children having CBs, but only */ - /* if greater than ->srcu_gq_seq. */ + /* if greater than ->srcu_gp_seq. */ unsigned long srcu_data_have_cbs[4]; /* Which srcu_data structs have CBs for given GP? */ unsigned long srcu_gp_seq_needed_exp; /* Furthest future exp GP. */ struct srcu_node *srcu_parent; /* Next up in tree. */ From 50be0c0439fc1d8bda733ff26f6526e49970857a Mon Sep 17 00:00:00 2001 From: Pingfan Liu Date: Wed, 16 Nov 2022 09:52:44 +0800 Subject: [PATCH 035/239] srcu: Fix the comparision in srcu_invl_snp_seq() A grace-period sequence number contains two fields: counter and state. SRCU_SNP_INIT_SEQ provides a guaranteed invalid value for grace-period sequence numbers in newly allocated srcu_node structures' ->srcu_have_cbs[] and ->srcu_gp_seq_needed_exp fields. The point of the comparison in srcu_invl_snp_seq() is not to detect invalid grace-period sequence numbers in general, but rather to detect a newly allocated srcu_node structure whose ->srcu_have_cbs[] and ->srcu_gp_seq_needed_exp fields need to be brought into line with the srcu_struct structure's ->srcu_gp_seq field. This commit therefore causes srcu_invl_snp_seq() to compare both fields of the specified grace-period sequence number. Signed-off-by: Pingfan Liu Cc: Lai Jiangshan Cc: "Paul E. McKenney" Cc: Josh Triplett Cc: Steven Rostedt Cc: Mathieu Desnoyers Cc: Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- kernel/rcu/srcutree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 16953784a0bd..6af031200580 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -154,7 +154,7 @@ static void init_srcu_struct_data(struct srcu_struct *ssp) */ static inline bool srcu_invl_snp_seq(unsigned long s) { - return rcu_seq_state(s) == SRCU_SNP_INIT_SEQ; + return s == SRCU_SNP_INIT_SEQ; } /* From 0b1182bde303dc476ea9712c2c816be2e4f0cf81 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 23 Nov 2022 15:49:55 -0800 Subject: [PATCH 036/239] rcu: Add srcu_down_read() and srcu_up_read() A pair of matching srcu_read_lock() and srcu_read_unlock() invocations must take place within the same context, for example, within the same task. Otherwise, lockdep complains, as is the right thing to do for most use cases. However, there are use cases involving asynchronous I/O where the SRCU reader needs to begin on one task and end on another. This commit therefore supplies the semaphore-like srcu_down_read() and srcu_up_read(), which act like srcu_read_lock() and srcu_read_unlock(), but permitting srcu_up_read() to be invoked in a different context than was the matching srcu_down_read(). Neither srcu_down_read() nor srcu_up_read() may be invoked from an NMI handler. Reported-by: Jan Kara Signed-off-by: Paul E. McKenney Tested-by: Amir Goldstein --- include/linux/srcu.h | 45 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/include/linux/srcu.h b/include/linux/srcu.h index 9b9d0bbf1d3c..74796cd7e7a9 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h @@ -214,6 +214,34 @@ srcu_read_lock_notrace(struct srcu_struct *ssp) __acquires(ssp) return retval; } +/** + * srcu_down_read - register a new reader for an SRCU-protected structure. + * @ssp: srcu_struct in which to register the new reader. + * + * Enter a semaphore-like SRCU read-side critical section. Note that + * SRCU read-side critical sections may be nested. However, it is + * illegal to call anything that waits on an SRCU grace period for the + * same srcu_struct, whether directly or indirectly. Please note that + * one way to indirectly wait on an SRCU grace period is to acquire + * a mutex that is held elsewhere while calling synchronize_srcu() or + * synchronize_srcu_expedited(). But if you want lockdep to help you + * keep this stuff straight, you should instead use srcu_read_lock(). + * + * The semaphore-like nature of srcu_down_read() means that the matching + * srcu_up_read() can be invoked from some other context, for example, + * from some other task or from an irq handler. However, neither + * srcu_down_read() nor srcu_up_read() may be invoked from an NMI handler. + * + * Calls to srcu_down_read() may be nested, similar to the manner in + * which calls to down_read() may be nested. + */ +static inline int srcu_down_read(struct srcu_struct *ssp) __acquires(ssp) +{ + WARN_ON_ONCE(in_nmi()); + srcu_check_nmi_safety(ssp, false); + return __srcu_read_lock(ssp); +} + /** * srcu_read_unlock - unregister a old reader from an SRCU-protected structure. * @ssp: srcu_struct in which to unregister the old reader. @@ -254,6 +282,23 @@ srcu_read_unlock_notrace(struct srcu_struct *ssp, int idx) __releases(ssp) __srcu_read_unlock(ssp, idx); } +/** + * srcu_up_read - unregister a old reader from an SRCU-protected structure. + * @ssp: srcu_struct in which to unregister the old reader. + * @idx: return value from corresponding srcu_read_lock(). + * + * Exit an SRCU read-side critical section, but not necessarily from + * the same context as the maching srcu_down_read(). + */ +static inline void srcu_up_read(struct srcu_struct *ssp, int idx) + __releases(ssp) +{ + WARN_ON_ONCE(idx & ~0x1); + WARN_ON_ONCE(in_nmi()); + srcu_check_nmi_safety(ssp, false); + __srcu_read_unlock(ssp, idx); +} + /** * smp_mb__after_srcu_read_unlock - ensure full ordering after srcu_read_unlock * From efa3c40cfac0777071e30383d209534e09ee454b Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 25 Nov 2022 08:42:02 -0800 Subject: [PATCH 037/239] rcu: Add test code for semaphore-like SRCU readers This commit adds trivial test code for srcu_down_read() and srcu_up_read(). Signed-off-by: Paul E. McKenney --- kernel/rcu/update.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c index a5b4abbee643..a72f98c120f0 100644 --- a/kernel/rcu/update.c +++ b/kernel/rcu/update.c @@ -555,9 +555,12 @@ struct early_boot_kfree_rcu { static void early_boot_test_call_rcu(void) { static struct rcu_head head; + int idx; static struct rcu_head shead; struct early_boot_kfree_rcu *rhp; + idx = srcu_down_read(&early_srcu); + srcu_up_read(&early_srcu, idx); call_rcu(&head, test_callback); early_srcu_cookie = start_poll_synchronize_srcu(&early_srcu); call_srcu(&early_srcu, &shead, test_callback); From 1bafbfb3e1a18af7f404977ed0d218dc4f176f8e Mon Sep 17 00:00:00 2001 From: Pingfan Liu Date: Wed, 23 Nov 2022 21:56:37 +0800 Subject: [PATCH 038/239] srcu: Remove needless rcu_seq_done() check while holding read lock The srcu_gp_start_if_needed() function now read-holds the srcu_struct whose grace period is being started, which means that the corresponding SRCU grace period cannot end. This in turn means that the SRCU grace-period sequence number returned by rcu_seq_snap() cannot expire during this time. And that means that the calls to rcu_seq_done() in srcu_funnel_exp_start() and srcu_funnel_gp_start() can never return true. This commit therefore removes these rcu_seq_done() checks, but adds checks in kernels built with CONFIG_PROVE_RCU=y that splats if rcu_seq_done() does somehow return true. [ paulmck: Rearrange checks to handle kernels built with lockdep. ] Signed-off-by: Pingfan Liu Cc: Lai Jiangshan Cc: Frederic Weisbecker Cc: Josh Triplett Cc: Steven Rostedt Cc: Mathieu Desnoyers To: rcu@vger.kernel.org Signed-off-by: Paul E. McKenney --- kernel/rcu/srcutree.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 6af031200580..68b8d8b150db 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -915,7 +915,7 @@ static void srcu_funnel_exp_start(struct srcu_struct *ssp, struct srcu_node *snp if (snp) for (; snp != NULL; snp = snp->srcu_parent) { sgsne = READ_ONCE(snp->srcu_gp_seq_needed_exp); - if (rcu_seq_done(&ssp->srcu_gp_seq, s) || + if (WARN_ON_ONCE(rcu_seq_done(&ssp->srcu_gp_seq, s)) || (!srcu_invl_snp_seq(sgsne) && ULONG_CMP_GE(sgsne, s))) return; spin_lock_irqsave_rcu_node(snp, flags); @@ -942,6 +942,9 @@ static void srcu_funnel_exp_start(struct srcu_struct *ssp, struct srcu_node *snp * * Note that this function also does the work of srcu_funnel_exp_start(), * in some cases by directly invoking it. + * + * The srcu read lock should be hold around this function. And s is a seq snap + * after holding that lock. */ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp, unsigned long s, bool do_norm) @@ -962,7 +965,7 @@ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp, if (snp_leaf) /* Each pass through the loop does one level of the srcu_node tree. */ for (snp = snp_leaf; snp != NULL; snp = snp->srcu_parent) { - if (rcu_seq_done(&ssp->srcu_gp_seq, s) && snp != snp_leaf) + if (WARN_ON_ONCE(rcu_seq_done(&ssp->srcu_gp_seq, s)) && snp != snp_leaf) return; /* GP already done and CBs recorded. */ spin_lock_irqsave_rcu_node(snp, flags); snp_seq = snp->srcu_have_cbs[idx]; @@ -999,8 +1002,8 @@ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp, if (!do_norm && ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, s)) WRITE_ONCE(ssp->srcu_gp_seq_needed_exp, s); - /* If grace period not already done and none in progress, start it. */ - if (!rcu_seq_done(&ssp->srcu_gp_seq, s) && + /* If grace period not already in progress, start it. */ + if (!WARN_ON_ONCE(rcu_seq_done(&ssp->srcu_gp_seq, s)) && rcu_seq_state(ssp->srcu_gp_seq) == SRCU_STATE_IDLE) { WARN_ON_ONCE(ULONG_CMP_GE(ssp->srcu_gp_seq, ssp->srcu_gp_seq_needed)); srcu_gp_start(ssp); From 0cd4b50b12d96d668b0627c149b19b5784ad4898 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 14 Dec 2022 10:50:30 -0800 Subject: [PATCH 039/239] srcu: Yet more detail for srcu_readers_active_idx_check() comments The comment in srcu_readers_active_idx_check() following the smp_mb() is out of date, hailing from a simpler time when preemption was disabled across the bulk of __srcu_read_lock(). The fact that preemption was disabled meant that the number of tasks that had fetched the old index but not yet incremented counters was limited by the number of CPUs. In our more complex modern times, the number of CPUs is no longer a limit. This commit therefore updates this comment, additionally giving more memory-ordering detail. [ paulmck: Apply Nt->Nc feedback from Joel Fernandes. ] Reported-by: Boqun Feng Reported-by: Frederic Weisbecker Reported-by: "Joel Fernandes (Google)" Reported-by: Neeraj Upadhyay Reported-by: Uladzislau Rezki Signed-off-by: Paul E. McKenney --- kernel/rcu/srcutree.c | 67 ++++++++++++++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 16 deletions(-) diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 68b8d8b150db..d2d2e31c42b1 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -469,24 +469,59 @@ static bool srcu_readers_active_idx_check(struct srcu_struct *ssp, int idx) /* * If the locks are the same as the unlocks, then there must have - * been no readers on this index at some time in between. This does - * not mean that there are no more readers, as one could have read - * the current index but not have incremented the lock counter yet. + * been no readers on this index at some point in this function. + * But there might be more readers, as a task might have read + * the current ->srcu_idx but not yet have incremented its CPU's + * ->srcu_lock_count[idx] counter. In fact, it is possible + * that most of the tasks have been preempted between fetching + * ->srcu_idx and incrementing ->srcu_lock_count[idx]. And there + * could be almost (ULONG_MAX / sizeof(struct task_struct)) tasks + * in a system whose address space was fully populated with memory. + * Call this quantity Nt. * - * So suppose that the updater is preempted here for so long - * that more than ULONG_MAX non-nested readers come and go in - * the meantime. It turns out that this cannot result in overflow - * because if a reader modifies its unlock count after we read it - * above, then that reader's next load of ->srcu_idx is guaranteed - * to get the new value, which will cause it to operate on the - * other bank of counters, where it cannot contribute to the - * overflow of these counters. This means that there is a maximum - * of 2*NR_CPUS increments, which cannot overflow given current - * systems, especially not on 64-bit systems. + * So suppose that the updater is preempted at this point in the + * code for a long time. That now-preempted updater has already + * flipped ->srcu_idx (possibly during the preceding grace period), + * done an smp_mb() (again, possibly during the preceding grace + * period), and summed up the ->srcu_unlock_count[idx] counters. + * How many times can a given one of the aforementioned Nt tasks + * increment the old ->srcu_idx value's ->srcu_lock_count[idx] + * counter, in the absence of nesting? * - * OK, how about nesting? This does impose a limit on nesting - * of floor(ULONG_MAX/NR_CPUS/2), which should be sufficient, - * especially on 64-bit systems. + * It can clearly do so once, given that it has already fetched + * the old value of ->srcu_idx and is just about to use that value + * to index its increment of ->srcu_lock_count[idx]. But as soon as + * it leaves that SRCU read-side critical section, it will increment + * ->srcu_unlock_count[idx], which must follow the updater's above + * read from that same value. Thus, as soon the reading task does + * an smp_mb() and a later fetch from ->srcu_idx, that task will be + * guaranteed to get the new index. Except that the increment of + * ->srcu_unlock_count[idx] in __srcu_read_unlock() is after the + * smp_mb(), and the fetch from ->srcu_idx in __srcu_read_lock() + * is before the smp_mb(). Thus, that task might not see the new + * value of ->srcu_idx until the -second- __srcu_read_lock(), + * which in turn means that this task might well increment + * ->srcu_lock_count[idx] for the old value of ->srcu_idx twice, + * not just once. + * + * However, it is important to note that a given smp_mb() takes + * effect not just for the task executing it, but also for any + * later task running on that same CPU. + * + * That is, there can be almost Nt + Nc further increments of + * ->srcu_lock_count[idx] for the old index, where Nc is the number + * of CPUs. But this is OK because the size of the task_struct + * structure limits the value of Nt and current systems limit Nc + * to a few thousand. + * + * OK, but what about nesting? This does impose a limit on + * nesting of half of the size of the task_struct structure + * (measured in bytes), which should be sufficient. A late 2022 + * TREE01 rcutorture run reported this size to be no less than + * 9408 bytes, allowing up to 4704 levels of nesting, which is + * comfortably beyond excessive. Especially on 64-bit systems, + * which are unlikely to be configured with an address space fully + * populated with memory, at least not anytime soon. */ return srcu_readers_lock_idx(ssp, idx) == unlocks; } From dafc4d1603c27671adc2b41eb7e7827f8cc18961 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 21 Dec 2022 08:32:51 -0800 Subject: [PATCH 040/239] srcu: Update comment after the index flip Because there is not guaranteed to be a full memory barrier between the ->srcu_unlock_count increment of an srcu_read_unlock() and the ->srcu_lock_count increment of the next srcu_read_lock(), this next srcu_read_lock() is not guaranteed to see the effect of the index flip just prior to this comment. However, this next srcu_read_lock() will execute a full memory barrier, so the srcu_read_lock() after that is guaranteed to see that index flip. This guarantee is illustrated by the following diagram of events and the litmus test following that. ------------------------------------------------------------------------ READER UPDATER ------------- ---------- // idx is initially 0. srcu_flip() { smp_mb(); // RSCS srcu_read_unlock() { smp_mb(); idx++; // P smp_mb(); // QQ } srcu_readers_unlock_idx(0) { ,--counted------------ count all unlock[0]; // Q | unlock[0]++; // X } smp_mb(); srcu_read_lock() { READ(idx) = 0; ,---- count all lock[0]; // contributes imbalance of 1. lock[0]++; ----counted | smp_mb(); // PP } | } | | // RSCS not going to effect above scan | srcu_read_unlock() { | smp_mb(); | unlock[0]++; | } | / / srcu_read_lock() { | READ(idx); // Y -----cannot be counted because of P (has to sample idx as 1) lock[1]++; ... } ------------------------------------------------------------------------ This makes it similar to the store buffer pattern. Using X, Y, P and Q annotated above, we get: ------------------------------------------------------------------------ READER UPDATER X (write) P (write) smp_mb(); //PP smp_mb(); //QQ Y (read) Q (read) ------------------------------------------------------------------------ ASCII art courtesy of Joel Fernandes. Reported-by: Joel Fernandes Reported-by: Boqun Feng Reported-by: Frederic Weisbecker Reported-by: Neeraj Upadhyay Signed-off-by: Paul E. McKenney --- kernel/rcu/srcutree.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index d2d2e31c42b1..ab4ee58af84b 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -1098,10 +1098,11 @@ static void srcu_flip(struct srcu_struct *ssp) /* * Ensure that if the updater misses an __srcu_read_unlock() - * increment, that task's next __srcu_read_lock() will see the - * above counter update. Note that both this memory barrier - * and the one in srcu_readers_active_idx_check() provide the - * guarantee for __srcu_read_lock(). + * increment, that task's __srcu_read_lock() following its next + * __srcu_read_lock() or __srcu_read_unlock() will see the above + * counter update. Note that both this memory barrier and the + * one in srcu_readers_active_idx_check() provide the guarantee + * for __srcu_read_lock(). */ smp_mb(); /* D */ /* Pairs with C. */ } From 47904aed898a08f028572b9b5a5cc101ddfb2d82 Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Sat, 19 Nov 2022 17:25:03 +0800 Subject: [PATCH 041/239] genirq: Fix the return type of kstat_cpu_irqs_sum() The type of member ->irqs_sum is unsigned long, but kstat_cpu_irqs_sum() returns int, which can result in truncation. Therefore, change the kstat_cpu_irqs_sum() function's return value to unsigned long to avoid truncation. Fixes: f2c66cd8eedd ("/proc/stat: scalability of irq num per cpu") Reported-by: Elliott, Robert (Servers) Signed-off-by: Zhen Lei Cc: Tejun Heo Cc: "Peter Zijlstra (Intel)" Cc: Josh Don Cc: Andrew Morton Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- include/linux/kernel_stat.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index ddb5a358fd82..90e2fdc17d79 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -75,7 +75,7 @@ extern unsigned int kstat_irqs_usr(unsigned int irq); /* * Number of interrupts per cpu, since bootup */ -static inline unsigned int kstat_cpu_irqs_sum(unsigned int cpu) +static inline unsigned long kstat_cpu_irqs_sum(unsigned int cpu) { return kstat_cpu(cpu).irqs_sum; } From 9420fb934cf15bee1cb6999676fa2dbd2560efc2 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Mon, 21 Nov 2022 23:01:50 +0800 Subject: [PATCH 042/239] rcu-tasks: Use accurate runstart time for RCU Tasks boot-time testing Currently, test_rcu_tasks_callback() reads from the jiffies counter only once when this function is invoked. This introduces inaccuracies because of the latencies induced by the synchronize_rcu_tasks*() invocations. This commit therefore re-reads the jiffies counter at the beginning of each test, thus avoiding penalizing later tests for the latencies induced by earlier tests. Therefore, this commit at the start of each RCU Tasks test, re-fetch the jiffies time as the runstart time. Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index fe9840d90e96..c418aa1c038a 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1815,23 +1815,21 @@ static void test_rcu_tasks_callback(struct rcu_head *rhp) static void rcu_tasks_initiate_self_tests(void) { - unsigned long j = jiffies; - pr_info("Running RCU-tasks wait API self tests\n"); #ifdef CONFIG_TASKS_RCU - tests[0].runstart = j; + tests[0].runstart = jiffies; synchronize_rcu_tasks(); call_rcu_tasks(&tests[0].rh, test_rcu_tasks_callback); #endif #ifdef CONFIG_TASKS_RUDE_RCU - tests[1].runstart = j; + tests[1].runstart = jiffies; synchronize_rcu_tasks_rude(); call_rcu_tasks_rude(&tests[1].rh, test_rcu_tasks_callback); #endif #ifdef CONFIG_TASKS_TRACE_RCU - tests[2].runstart = j; + tests[2].runstart = jiffies; synchronize_rcu_tasks_trace(); call_rcu_tasks_trace(&tests[2].rh, test_rcu_tasks_callback); #endif From e4e1e8089c5fd948da12cb9f4adc93821036945f Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 25 Nov 2022 14:54:58 +0100 Subject: [PATCH 043/239] rcu-tasks: Improve comments explaining tasks_rcu_exit_srcu purpose Make sure we don't need to look again into the depths of git blame in order not to miss a subtle part about how rcu-tasks is dealing with exiting tasks. Suggested-by: Boqun Feng Suggested-by: Neeraj Upadhyay Suggested-by: Paul E. McKenney Cc: Oleg Nesterov Cc: Lai Jiangshan Cc: Eric W. Biederman Signed-off-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index c418aa1c038a..50d4c0ec7a89 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -827,11 +827,21 @@ static void rcu_tasks_pertask(struct task_struct *t, struct list_head *hop) static void rcu_tasks_postscan(struct list_head *hop) { /* - * Wait for tasks that are in the process of exiting. This - * does only part of the job, ensuring that all tasks that were - * previously exiting reach the point where they have disabled - * preemption, allowing the later synchronize_rcu() to finish - * the job. + * Exiting tasks may escape the tasklist scan. Those are vulnerable + * until their final schedule() with TASK_DEAD state. To cope with + * this, divide the fragile exit path part in two intersecting + * read side critical sections: + * + * 1) An _SRCU_ read side starting before calling exit_notify(), + * which may remove the task from the tasklist, and ending after + * the final preempt_disable() call in do_exit(). + * + * 2) An _RCU_ read side starting with the final preempt_disable() + * call in do_exit() and ending with the final call to schedule() + * with TASK_DEAD state. + * + * This handles the part 1). And postgp will handle part 2) with a + * call to synchronize_rcu(). */ synchronize_srcu(&tasks_rcu_exit_srcu); } @@ -898,7 +908,10 @@ static void rcu_tasks_postgp(struct rcu_tasks *rtp) * * In addition, this synchronize_rcu() waits for exiting tasks * to complete their final preempt_disable() region of execution, - * cleaning up after the synchronize_srcu() above. + * cleaning up after synchronize_srcu(&tasks_rcu_exit_srcu), + * enforcing the whole region before tasklist removal until + * the final schedule() with TASK_DEAD state to be an RCU TASKS + * read side critical section. */ synchronize_rcu(); } @@ -988,7 +1001,11 @@ void show_rcu_tasks_classic_gp_kthread(void) EXPORT_SYMBOL_GPL(show_rcu_tasks_classic_gp_kthread); #endif // !defined(CONFIG_TINY_RCU) -/* Do the srcu_read_lock() for the above synchronize_srcu(). */ +/* + * Contribute to protect against tasklist scan blind spot while the + * task is exiting and may be removed from the tasklist. See + * corresponding synchronize_srcu() for further details. + */ void exit_tasks_rcu_start(void) __acquires(&tasks_rcu_exit_srcu) { preempt_disable(); @@ -996,7 +1013,11 @@ void exit_tasks_rcu_start(void) __acquires(&tasks_rcu_exit_srcu) preempt_enable(); } -/* Do the srcu_read_unlock() for the above synchronize_srcu(). */ +/* + * Contribute to protect against tasklist scan blind spot while the + * task is exiting and may be removed from the tasklist. See + * corresponding synchronize_srcu() for further details. + */ void exit_tasks_rcu_finish(void) __releases(&tasks_rcu_exit_srcu) { struct task_struct *t = current; From 44757092958bdd749775022f915b7ac974384c2a Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 25 Nov 2022 14:54:59 +0100 Subject: [PATCH 044/239] rcu-tasks: Remove preemption disablement around srcu_read_[un]lock() calls Ever since the following commit: 5a41344a3d83 ("srcu: Simplify __srcu_read_unlock() via this_cpu_dec()") SRCU doesn't rely anymore on preemption to be disabled in order to modify the per-CPU counter. And even then it used to be done from the API itself. Therefore and after checking further, it appears to be safe to remove the preemption disablement around __srcu_read_[un]lock() in exit_tasks_rcu_start() and exit_tasks_rcu_finish() Suggested-by: Boqun Feng Suggested-by: Paul E. McKenney Suggested-by: Neeraj Upadhyay Cc: Lai Jiangshan Signed-off-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 50d4c0ec7a89..fbaed2637a7f 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1008,9 +1008,7 @@ EXPORT_SYMBOL_GPL(show_rcu_tasks_classic_gp_kthread); */ void exit_tasks_rcu_start(void) __acquires(&tasks_rcu_exit_srcu) { - preempt_disable(); current->rcu_tasks_idx = __srcu_read_lock(&tasks_rcu_exit_srcu); - preempt_enable(); } /* @@ -1022,9 +1020,7 @@ void exit_tasks_rcu_finish(void) __releases(&tasks_rcu_exit_srcu) { struct task_struct *t = current; - preempt_disable(); __srcu_read_unlock(&tasks_rcu_exit_srcu, t->rcu_tasks_idx); - preempt_enable(); exit_tasks_rcu_finish_trace(t); } From 28319d6dc5e2ffefa452c2377dd0f71621b5bff0 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 25 Nov 2022 14:55:00 +0100 Subject: [PATCH 045/239] rcu-tasks: Fix synchronize_rcu_tasks() VS zap_pid_ns_processes() RCU Tasks and PID-namespace unshare can interact in do_exit() in a complicated circular dependency: 1) TASK A calls unshare(CLONE_NEWPID), this creates a new PID namespace that every subsequent child of TASK A will belong to. But TASK A doesn't itself belong to that new PID namespace. 2) TASK A forks() and creates TASK B. TASK A stays attached to its PID namespace (let's say PID_NS1) and TASK B is the first task belonging to the new PID namespace created by unshare() (let's call it PID_NS2). 3) Since TASK B is the first task attached to PID_NS2, it becomes the PID_NS2 child reaper. 4) TASK A forks() again and creates TASK C which get attached to PID_NS2. Note how TASK C has TASK A as a parent (belonging to PID_NS1) but has TASK B (belonging to PID_NS2) as a pid_namespace child_reaper. 5) TASK B exits and since it is the child reaper for PID_NS2, it has to kill all other tasks attached to PID_NS2, and wait for all of them to die before getting reaped itself (zap_pid_ns_process()). 6) TASK A calls synchronize_rcu_tasks() which leads to synchronize_srcu(&tasks_rcu_exit_srcu). 7) TASK B is waiting for TASK C to get reaped. But TASK B is under a tasks_rcu_exit_srcu SRCU critical section (exit_notify() is between exit_tasks_rcu_start() and exit_tasks_rcu_finish()), blocking TASK A. 8) TASK C exits and since TASK A is its parent, it waits for it to reap TASK C, but it can't because TASK A waits for TASK B that waits for TASK C. Pid_namespace semantics can hardly be changed at this point. But the coverage of tasks_rcu_exit_srcu can be reduced instead. The current task is assumed not to be concurrently reapable at this stage of exit_notify() and therefore tasks_rcu_exit_srcu can be temporarily relaxed without breaking its constraints, providing a way out of the deadlock scenario. [ paulmck: Fix build failure by adding additional declaration. ] Fixes: 3f95aa81d265 ("rcu: Make TASKS_RCU handle tasks that are almost done exiting") Reported-by: Pengfei Xu Suggested-by: Boqun Feng Suggested-by: Neeraj Upadhyay Suggested-by: Paul E. McKenney Cc: Oleg Nesterov Cc: Lai Jiangshan Cc: Eric W . Biederman Signed-off-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- include/linux/rcupdate.h | 2 ++ kernel/pid_namespace.c | 17 +++++++++++++++++ kernel/rcu/tasks.h | 15 +++++++++++++-- 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 03abf883a281..c0c79beac3fe 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -238,6 +238,7 @@ void synchronize_rcu_tasks_rude(void); #define rcu_note_voluntary_context_switch(t) rcu_tasks_qs(t, false) void exit_tasks_rcu_start(void); +void exit_tasks_rcu_stop(void); void exit_tasks_rcu_finish(void); #else /* #ifdef CONFIG_TASKS_RCU_GENERIC */ #define rcu_tasks_classic_qs(t, preempt) do { } while (0) @@ -246,6 +247,7 @@ void exit_tasks_rcu_finish(void); #define call_rcu_tasks call_rcu #define synchronize_rcu_tasks synchronize_rcu static inline void exit_tasks_rcu_start(void) { } +static inline void exit_tasks_rcu_stop(void) { } static inline void exit_tasks_rcu_finish(void) { } #endif /* #else #ifdef CONFIG_TASKS_RCU_GENERIC */ diff --git a/kernel/pid_namespace.c b/kernel/pid_namespace.c index f4f8cb0435b4..fc21c5d5fd5d 100644 --- a/kernel/pid_namespace.c +++ b/kernel/pid_namespace.c @@ -244,7 +244,24 @@ void zap_pid_ns_processes(struct pid_namespace *pid_ns) set_current_state(TASK_INTERRUPTIBLE); if (pid_ns->pid_allocated == init_pids) break; + /* + * Release tasks_rcu_exit_srcu to avoid following deadlock: + * + * 1) TASK A unshare(CLONE_NEWPID) + * 2) TASK A fork() twice -> TASK B (child reaper for new ns) + * and TASK C + * 3) TASK B exits, kills TASK C, waits for TASK A to reap it + * 4) TASK A calls synchronize_rcu_tasks() + * -> synchronize_srcu(tasks_rcu_exit_srcu) + * 5) *DEADLOCK* + * + * It is considered safe to release tasks_rcu_exit_srcu here + * because we assume the current task can not be concurrently + * reaped at this point. + */ + exit_tasks_rcu_stop(); schedule(); + exit_tasks_rcu_start(); } __set_current_state(TASK_RUNNING); diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index fbaed2637a7f..5de61f12a164 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1016,16 +1016,27 @@ void exit_tasks_rcu_start(void) __acquires(&tasks_rcu_exit_srcu) * task is exiting and may be removed from the tasklist. See * corresponding synchronize_srcu() for further details. */ -void exit_tasks_rcu_finish(void) __releases(&tasks_rcu_exit_srcu) +void exit_tasks_rcu_stop(void) __releases(&tasks_rcu_exit_srcu) { struct task_struct *t = current; __srcu_read_unlock(&tasks_rcu_exit_srcu, t->rcu_tasks_idx); - exit_tasks_rcu_finish_trace(t); +} + +/* + * Contribute to protect against tasklist scan blind spot while the + * task is exiting and may be removed from the tasklist. See + * corresponding synchronize_srcu() for further details. + */ +void exit_tasks_rcu_finish(void) +{ + exit_tasks_rcu_stop(); + exit_tasks_rcu_finish_trace(current); } #else /* #ifdef CONFIG_TASKS_RCU */ void exit_tasks_rcu_start(void) { } +void exit_tasks_rcu_stop(void) { } void exit_tasks_rcu_finish(void) { exit_tasks_rcu_finish_trace(current); } #endif /* #else #ifdef CONFIG_TASKS_RCU */ From ea5c8987fef20a8cca07e428aa28bc64649c5104 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Thu, 1 Dec 2022 07:45:33 +0800 Subject: [PATCH 046/239] rcu-tasks: Make rude RCU-Tasks work well with CPU hotplug The synchronize_rcu_tasks_rude() function invokes rcu_tasks_rude_wait_gp() to wait one rude RCU-tasks grace period. The rcu_tasks_rude_wait_gp() function in turn checks if there is only a single online CPU. If so, it will immediately return, because a call to synchronize_rcu_tasks_rude() is by definition a grace period on a single-CPU system. (We could have blocked!) Unfortunately, this check uses num_online_cpus() without synchronization, which can result in too-short grace periods. To see this, consider the following scenario: CPU0 CPU1 (going offline) migration/1 task: cpu_stopper_thread -> take_cpu_down -> _cpu_disable (dec __num_online_cpus) ->cpuhp_invoke_callback preempt_disable access old_data0 task1 del old_data0 ..... synchronize_rcu_tasks_rude() task1 schedule out .... task2 schedule in rcu_tasks_rude_wait_gp() ->__num_online_cpus == 1 ->return .... task1 schedule in ->free old_data0 preempt_enable When CPU1 decrements __num_online_cpus, its value becomes 1. However, CPU1 has not finished going offline, and will take one last trip through the scheduler and the idle loop before it actually stops executing instructions. Because synchronize_rcu_tasks_rude() is mostly used for tracing, and because both the scheduler and the idle loop can be traced, this means that CPU0's prematurely ended grace period might disrupt the tracing on CPU1. Given that this disruption might include CPU1 executing instructions in memory that was just now freed (and maybe reallocated), this is a matter of some concern. This commit therefore removes that problematic single-CPU check from the rcu_tasks_rude_wait_gp() function. This dispenses with the single-CPU optimization, but there is no evidence indicating that this optimization is important. In addition, synchronize_rcu_tasks_generic() contains a similar optimization (albeit only for early boot), which also splats. (As in exactly why are you invoking synchronize_rcu_tasks_rude() so early in boot, anyway???) It is OK for the synchronize_rcu_tasks_rude() function's check to be unsynchronized because the only times that this check can evaluate to true is when there is only a single CPU running with preemption disabled. While in the area, this commit also fixes a minor bug in which a call to synchronize_rcu_tasks_rude() would instead be attributed to synchronize_rcu_tasks(). [ paulmck: Add "synchronize_" prefix and "()" suffix. ] Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 5de61f12a164..eee38b0d362a 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -560,8 +560,9 @@ static int __noreturn rcu_tasks_kthread(void *arg) static void synchronize_rcu_tasks_generic(struct rcu_tasks *rtp) { /* Complain if the scheduler has not started. */ - WARN_ONCE(rcu_scheduler_active == RCU_SCHEDULER_INACTIVE, - "synchronize_rcu_tasks called too soon"); + if (WARN_ONCE(rcu_scheduler_active == RCU_SCHEDULER_INACTIVE, + "synchronize_%s() called too soon", rtp->name)) + return; // If the grace-period kthread is running, use it. if (READ_ONCE(rtp->kthread_ptr)) { @@ -1064,9 +1065,6 @@ static void rcu_tasks_be_rude(struct work_struct *work) // Wait for one rude RCU-tasks grace period. static void rcu_tasks_rude_wait_gp(struct rcu_tasks *rtp) { - if (num_online_cpus() <= 1) - return; // Fastpath for only one CPU. - rtp->n_ipis += cpumask_weight(cpu_online_mask); schedule_on_each_cpu(rcu_tasks_be_rude); } From a4fcfbee8f6274f9b3f9a71dd5b03e6772ce33f3 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Sat, 3 Dec 2022 10:25:03 +0800 Subject: [PATCH 047/239] rcu-tasks: Handle queue-shrink/callback-enqueue race condition The rcu_tasks_need_gpcb() determines whether or not: (1) There are callbacks needing another grace period, (2) There are callbacks ready to be invoked, and (3) It would be a good time to shrink back down to a single-CPU callback list. This third case is interesting because some other CPU might be adding new callbacks, which might suddenly make this a very bad time to be shrinking. This is currently handled by requiring call_rcu_tasks_generic() to enqueue callbacks under the protection of rcu_read_lock() and requiring rcu_tasks_need_gpcb() to wait for an RCU grace period to elapse before finalizing the transition. This works well in practice. Unfortunately, the current code assumes that a grace period whose end is detected by the poll_state_synchronize_rcu() in the second "if" condition actually ended before the earlier code counted the callbacks queued on CPUs other than CPU 0 (local variable "ncbsnz"). Given the current code, it is possible that a long-delayed call_rcu_tasks_generic() invocation will queue a callback on a non-zero CPU after these CPUs have had their callbacks counted and zero has been stored to ncbsnz. Such a callback would trigger the WARN_ON_ONCE() in the second "if" statement. To see this, consider the following sequence of events: o CPU 0 invokes rcu_tasks_one_gp(), and counts fewer than rcu_task_collapse_lim callbacks. It sees at least one callback queued on some other CPU, thus setting ncbsnz to a non-zero value. o CPU 1 invokes call_rcu_tasks_generic() and loads 42 from ->percpu_enqueue_lim. It therefore decides to enqueue its callback onto CPU 1's callback list, but is delayed. o CPU 0 sees the rcu_task_cb_adjust is non-zero and that the number of callbacks does not exceed rcu_task_collapse_lim. It therefore checks percpu_enqueue_lim, and sees that its value is greater than the value one. CPU 0 therefore starts the shift back to a single callback list. It sets ->percpu_enqueue_lim to 1, but CPU 1 has already read the old value of 42. It also gets a grace-period state value from get_state_synchronize_rcu(). o CPU 0 sees that ncbsnz is non-zero in its second "if" statement, so it declines to finalize the shrink operation. o CPU 0 again invokes rcu_tasks_one_gp(), and counts fewer than rcu_task_collapse_lim callbacks. It also sees that there are no callback queued on any other CPU, and thus sets ncbsnz to zero. o CPU 1 resumes execution and enqueues its callback onto its own list. This invalidates the value of ncbsnz. o CPU 0 sees the rcu_task_cb_adjust is non-zero and that the number of callbacks does not exceed rcu_task_collapse_lim. It therefore checks percpu_enqueue_lim, but sees that its value is already unity. It therefore does not get a new grace-period state value. o CPU 0 sees that rcu_task_cb_adjust is non-zero, ncbsnz is zero, and that poll_state_synchronize_rcu() says that the grace period has completed. it therefore finalizes the shrink operation, setting ->percpu_dequeue_lim to the value one. o CPU 0 does a debug check, scanning the other CPUs' callback lists. It sees that CPU 1's list has a callback, so it (rightly) triggers the WARN_ON_ONCE(). After all, the new value of ->percpu_dequeue_lim says to not bother looking at CPU 1's callback list, which means that this callback will never be invoked. This can result in hangs and maybe even OOMs. Based on long experience with rcutorture, this is an extremely low-probability race condition, but it really can happen, especially in preemptible kernels or within guest OSes. This commit therefore checks for completion of the grace period before counting callbacks. With this change, in the above failure scenario CPU 0 would know not to prematurely end the shrink operation because the grace period would not have completed before the count operation started. [ paulmck: Adjust grace-period end rather than adding RCU reader. ] [ paulmck: Avoid spurious WARN_ON_ONCE() with ->percpu_dequeue_lim check. ] Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index eee38b0d362a..bfb5e1549f2b 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -384,6 +384,7 @@ static int rcu_tasks_need_gpcb(struct rcu_tasks *rtp) { int cpu; unsigned long flags; + bool gpdone = poll_state_synchronize_rcu(rtp->percpu_dequeue_gpseq); long n; long ncbs = 0; long ncbsnz = 0; @@ -425,21 +426,23 @@ static int rcu_tasks_need_gpcb(struct rcu_tasks *rtp) WRITE_ONCE(rtp->percpu_enqueue_shift, order_base_2(nr_cpu_ids)); smp_store_release(&rtp->percpu_enqueue_lim, 1); rtp->percpu_dequeue_gpseq = get_state_synchronize_rcu(); + gpdone = false; pr_info("Starting switch %s to CPU-0 callback queuing.\n", rtp->name); } raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags); } - if (rcu_task_cb_adjust && !ncbsnz && - poll_state_synchronize_rcu(rtp->percpu_dequeue_gpseq)) { + if (rcu_task_cb_adjust && !ncbsnz && gpdone) { raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags); if (rtp->percpu_enqueue_lim < rtp->percpu_dequeue_lim) { WRITE_ONCE(rtp->percpu_dequeue_lim, 1); pr_info("Completing switch %s to CPU-0 callback queuing.\n", rtp->name); } - for (cpu = rtp->percpu_dequeue_lim; cpu < nr_cpu_ids; cpu++) { - struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); + if (rtp->percpu_dequeue_lim == 1) { + for (cpu = rtp->percpu_dequeue_lim; cpu < nr_cpu_ids; cpu++) { + struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); - WARN_ON_ONCE(rcu_segcblist_n_cbs(&rtpcp->cblist)); + WARN_ON_ONCE(rcu_segcblist_n_cbs(&rtpcp->cblist)); + } } raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags); } From 95f93e97ed0f56f54f74293a0a82e05599d5078d Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sun, 6 Nov 2022 18:16:14 -0800 Subject: [PATCH 048/239] torture: Seed torture_random_state on CPU The DEFINE_TORTURE_RANDOM_PERCPU() macro defines per-CPU random-number generators for torture testing, but the seeds for each CPU's instance will be identical if they are first used at the same time. This commit therefore adds the CPU number to the mix when reseeding. Signed-off-by: Paul E. McKenney --- kernel/torture.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/torture.c b/kernel/torture.c index 789aeb0e1159..29afc62f2bfe 100644 --- a/kernel/torture.c +++ b/kernel/torture.c @@ -450,7 +450,7 @@ unsigned long torture_random(struct torture_random_state *trsp) { if (--trsp->trs_count < 0) { - trsp->trs_state += (unsigned long)local_clock(); + trsp->trs_state += (unsigned long)local_clock() + raw_smp_processor_id(); trsp->trs_count = TORTURE_RANDOM_REFRESH; } trsp->trs_state = trsp->trs_state * TORTURE_RANDOM_MULT + From 3c6496c86e484be94cd8d69e604566dfeaa726a2 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sun, 6 Nov 2022 20:58:15 -0800 Subject: [PATCH 049/239] refscale: Provide for initialization failure Current tests all have init() functions that are guaranteed to succeed. But upcoming tests will need to allocate memory, thus possibly failing. This commit therefore handles init() function failure. Signed-off-by: Paul E. McKenney --- kernel/rcu/refscale.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c index 435c884c02b5..7f12168627a1 100644 --- a/kernel/rcu/refscale.c +++ b/kernel/rcu/refscale.c @@ -124,7 +124,7 @@ static int exp_idx; // Operations vector for selecting different types of tests. struct ref_scale_ops { - void (*init)(void); + bool (*init)(void); void (*cleanup)(void); void (*readsection)(const int nloops); void (*delaysection)(const int nloops, const int udl, const int ndl); @@ -162,8 +162,9 @@ static void ref_rcu_delay_section(const int nloops, const int udl, const int ndl } } -static void rcu_sync_scale_init(void) +static bool rcu_sync_scale_init(void) { + return true; } static struct ref_scale_ops rcu_ops = { @@ -315,9 +316,10 @@ static struct ref_scale_ops refcnt_ops = { // Definitions for rwlock static rwlock_t test_rwlock; -static void ref_rwlock_init(void) +static bool ref_rwlock_init(void) { rwlock_init(&test_rwlock); + return true; } static void ref_rwlock_section(const int nloops) @@ -351,9 +353,10 @@ static struct ref_scale_ops rwlock_ops = { // Definitions for rwsem static struct rw_semaphore test_rwsem; -static void ref_rwsem_init(void) +static bool ref_rwsem_init(void) { init_rwsem(&test_rwsem); + return true; } static void ref_rwsem_section(const int nloops) @@ -833,7 +836,10 @@ ref_scale_init(void) goto unwind; } if (cur_ops->init) - cur_ops->init(); + if (!cur_ops->init()) { + firsterr = -EUCLEAN; + goto unwind; + } ref_scale_print_module_parms(cur_ops, "Start of test"); From 3e4c07b9f8217b016753985856238ac7684bc11f Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sun, 20 Nov 2022 19:56:27 -0800 Subject: [PATCH 050/239] torture: make kvm-find-errors.sh check for compressed vmlinux files Under some conditions, a given run's vmlinux file will be compressed, so that it is named vmlinux.xz rather than vmlinux. in such cases, kvm-find-errors.sh will complain about the nonexistence of vmlinux. This commit therefore causes kvm-find-errors.sh to check for vmlinux.xz as well as for vmlinux. Signed-off-by: Paul E. McKenney --- tools/testing/selftests/rcutorture/bin/kvm-find-errors.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/rcutorture/bin/kvm-find-errors.sh b/tools/testing/selftests/rcutorture/bin/kvm-find-errors.sh index 88983cba7956..d4dc059843a4 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm-find-errors.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm-find-errors.sh @@ -36,7 +36,7 @@ do then egrep "error:|warning:|^ld: .*undefined reference to" < $i > $i.diags files="$files $i.diags $i" - elif ! test -f ${scenariobasedir}/vmlinux && ! test -f "${rundir}/re-run" + elif ! test -f ${scenariobasedir}/vmlinux && ! test -f ${scenariobasedir}/vmlinux.xz && ! test -f "${rundir}/re-run" then echo No ${scenariobasedir}/vmlinux file > $i.diags files="$files $i.diags $i" From eeb4dd9e530de52fdf43283c50f371f0771725c9 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Wed, 23 Nov 2022 09:03:28 +0800 Subject: [PATCH 051/239] selftests: rcutorture: Use "grep -E" instead of "egrep" The latest version of grep is deprecating the egrep command, so that its output contains warnings as follows: egrep: warning: egrep is obsolescent; using grep -E Fix this using "grep -E" instead. sed -i "s/egrep/grep -E/g" `grep egrep -rwl tools/testing/selftests/rcutorture` Here are the steps to install the latest grep: wget http://ftp.gnu.org/gnu/grep/grep-3.8.tar.gz tar xf grep-3.8.tar.gz cd grep-3.8 && ./configure && make sudo make install export PATH=/usr/local/bin:$PATH Signed-off-by: Tiezhu Yang Signed-off-by: Paul E. McKenney --- .../selftests/rcutorture/bin/console-badness.sh | 2 +- tools/testing/selftests/rcutorture/bin/kvm-build.sh | 4 ++-- .../selftests/rcutorture/bin/kvm-find-errors.sh | 4 ++-- tools/testing/selftests/rcutorture/bin/kvm.sh | 4 ++-- .../testing/selftests/rcutorture/bin/parse-console.sh | 10 +++++----- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tools/testing/selftests/rcutorture/bin/console-badness.sh b/tools/testing/selftests/rcutorture/bin/console-badness.sh index 69f8a5958cef..aad51e7c0183 100755 --- a/tools/testing/selftests/rcutorture/bin/console-badness.sh +++ b/tools/testing/selftests/rcutorture/bin/console-badness.sh @@ -10,7 +10,7 @@ # # Authors: Paul E. McKenney -egrep 'Badness|WARNING:|Warn|BUG|===========|BUG: KCSAN:|Call Trace:|Oops:|detected stalls on CPUs/tasks:|self-detected stall on CPU|Stall ended before state dump start|\?\?\? Writer stall state|rcu_.*kthread starved for|!!!' | +grep -E 'Badness|WARNING:|Warn|BUG|===========|BUG: KCSAN:|Call Trace:|Oops:|detected stalls on CPUs/tasks:|self-detected stall on CPU|Stall ended before state dump start|\?\?\? Writer stall state|rcu_.*kthread starved for|!!!' | grep -v 'ODEBUG: ' | grep -v 'This means that this is a DEBUG kernel and it is' | grep -v 'Warning: unable to open an initial console' | diff --git a/tools/testing/selftests/rcutorture/bin/kvm-build.sh b/tools/testing/selftests/rcutorture/bin/kvm-build.sh index e28a82851f7c..11f8d232b0ee 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm-build.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm-build.sh @@ -44,10 +44,10 @@ fi ncpus="`getconf _NPROCESSORS_ONLN`" make -j$((2 * ncpus)) $TORTURE_KMAKE_ARG > $resdir/Make.out 2>&1 retval=$? -if test $retval -ne 0 || grep "rcu[^/]*": < $resdir/Make.out | egrep -q "Stop|Error|error:|warning:" || egrep -q "Stop|Error|error:" < $resdir/Make.out +if test $retval -ne 0 || grep "rcu[^/]*": < $resdir/Make.out | grep -E -q "Stop|Error|error:|warning:" || grep -E -q "Stop|Error|error:" < $resdir/Make.out then echo Kernel build error - egrep "Stop|Error|error:|warning:" < $resdir/Make.out + grep -E "Stop|Error|error:|warning:" < $resdir/Make.out echo Run aborted. exit 3 fi diff --git a/tools/testing/selftests/rcutorture/bin/kvm-find-errors.sh b/tools/testing/selftests/rcutorture/bin/kvm-find-errors.sh index d4dc059843a4..28981007465b 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm-find-errors.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm-find-errors.sh @@ -32,9 +32,9 @@ for i in ${rundir}/*/Make.out do scenariodir="`dirname $i`" scenariobasedir="`echo ${scenariodir} | sed -e 's/\.[0-9]*$//'`" - if egrep -q "error:|warning:|^ld: .*undefined reference to" < $i + if grep -E -q "error:|warning:|^ld: .*undefined reference to" < $i then - egrep "error:|warning:|^ld: .*undefined reference to" < $i > $i.diags + grep -E "error:|warning:|^ld: .*undefined reference to" < $i > $i.diags files="$files $i.diags $i" elif ! test -f ${scenariobasedir}/vmlinux && ! test -f ${scenariobasedir}/vmlinux.xz && ! test -f "${rundir}/re-run" then diff --git a/tools/testing/selftests/rcutorture/bin/kvm.sh b/tools/testing/selftests/rcutorture/bin/kvm.sh index 7710b1e1cdda..3025a949bc99 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm.sh @@ -585,7 +585,7 @@ awk < $T/cfgcpu.pack \ echo kvm-end-run-stats.sh "$resdir/$ds" "$starttime" >> $T/script # Extract the tests and their batches from the script. -egrep 'Start batch|Starting build\.' $T/script | grep -v ">>" | +grep -E 'Start batch|Starting build\.' $T/script | grep -v ">>" | sed -e 's/:.*$//' -e 's/^echo //' -e 's/-ovf//' | awk ' /^----Start/ { @@ -622,7 +622,7 @@ then elif test "$dryrun" = sched then # Extract the test run schedule from the script. - egrep 'Start batch|Starting build\.' $T/script | grep -v ">>" | + grep -E 'Start batch|Starting build\.' $T/script | grep -v ">>" | sed -e 's/:.*$//' -e 's/^echo //' nbuilds="`grep 'Starting build\.' $T/script | grep -v ">>" | sed -e 's/:.*$//' -e 's/^echo //' | diff --git a/tools/testing/selftests/rcutorture/bin/parse-console.sh b/tools/testing/selftests/rcutorture/bin/parse-console.sh index 822eb037a057..9ab0f6bc172c 100755 --- a/tools/testing/selftests/rcutorture/bin/parse-console.sh +++ b/tools/testing/selftests/rcutorture/bin/parse-console.sh @@ -65,7 +65,7 @@ then fi grep --binary-files=text 'torture:.*ver:' $file | - egrep --binary-files=text -v '\(null\)|rtc: 000000000* ' | + grep -E --binary-files=text -v '\(null\)|rtc: 000000000* ' | sed -e 's/^(initramfs)[^]]*] //' -e 's/^\[[^]]*] //' | sed -e 's/^.*ver: //' | awk ' @@ -128,17 +128,17 @@ then then summary="$summary Badness: $n_badness" fi - n_warn=`grep -v 'Warning: unable to open an initial console' $file | grep -v 'Warning: Failed to add ttynull console. No stdin, stdout, and stderr for the init process' | egrep -c 'WARNING:|Warn'` + n_warn=`grep -v 'Warning: unable to open an initial console' $file | grep -v 'Warning: Failed to add ttynull console. No stdin, stdout, and stderr for the init process' | grep -E -c 'WARNING:|Warn'` if test "$n_warn" -ne 0 then summary="$summary Warnings: $n_warn" fi - n_bugs=`egrep -c '\bBUG|Oops:' $file` + n_bugs=`grep -E -c '\bBUG|Oops:' $file` if test "$n_bugs" -ne 0 then summary="$summary Bugs: $n_bugs" fi - n_kcsan=`egrep -c 'BUG: KCSAN: ' $file` + n_kcsan=`grep -E -c 'BUG: KCSAN: ' $file` if test "$n_kcsan" -ne 0 then if test "$n_bugs" = "$n_kcsan" @@ -158,7 +158,7 @@ then then summary="$summary lockdep: $n_badness" fi - n_stalls=`egrep -c 'detected stalls on CPUs/tasks:|self-detected stall on CPU|Stall ended before state dump start|\?\?\? Writer stall state' $file` + n_stalls=`grep -E -c 'detected stalls on CPUs/tasks:|self-detected stall on CPU|Stall ended before state dump start|\?\?\? Writer stall state' $file` if test "$n_stalls" -ne 0 then summary="$summary Stalls: $n_stalls" From ac71c3dd11e809cb732ae34efa6d9fc29d4664e1 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 14 Dec 2022 16:37:27 -0800 Subject: [PATCH 052/239] torture: Permit double-quoted-string Kconfig options Currently, the presence of any quoted-string Kconfig option in the scenario files or the CFcommon file (aside from the special-cased CONFIG_INITRAMFS_SOURCE option) will result in an "improperly set" diagnostic. This commit updates configcheck.sh to strip double quotes in order to permit string-valued Kconfig options to be handled correctly. Signed-off-by: Paul E. McKenney --- tools/testing/selftests/rcutorture/bin/configcheck.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/rcutorture/bin/configcheck.sh b/tools/testing/selftests/rcutorture/bin/configcheck.sh index 83fac1852ab2..b92dfeb7fbbf 100755 --- a/tools/testing/selftests/rcutorture/bin/configcheck.sh +++ b/tools/testing/selftests/rcutorture/bin/configcheck.sh @@ -10,10 +10,9 @@ T="`mktemp -d ${TMPDIR-/tmp}/configcheck.sh.XXXXXX`" trap 'rm -rf $T' 0 -cat $1 > $T/.config +sed -e 's/"//g' < $1 > $T/.config -cat $2 | sed -e 's/\(.*\)=n/# \1 is not set/' -e 's/^#CHECK#//' | -grep -v '^CONFIG_INITRAMFS_SOURCE' | +sed -e 's/"//g' -e 's/\(.*\)=n/# \1 is not set/' -e 's/^#CHECK#//' < $2 | awk ' { print "if grep -q \"" $0 "\" < '"$T/.config"'"; From 5a6cd56ad79a490d40ead5df0dfc8502e8081db0 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 14 Dec 2022 20:53:00 -0800 Subject: [PATCH 053/239] rcu: Permit string-valued Kconfig options in kvm.sh This commit upgrades the kvm.sh script's --kconfig parameter to accept string-valued Kconfig options with double-quoted string values. Signed-off-by: Paul E. McKenney --- tools/testing/selftests/rcutorture/bin/kvm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/rcutorture/bin/kvm.sh b/tools/testing/selftests/rcutorture/bin/kvm.sh index 3025a949bc99..62f3b0f56e4d 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm.sh @@ -186,7 +186,7 @@ do fi ;; --kconfig|--kconfigs) - checkarg --kconfig "(Kconfig options)" $# "$2" '^CONFIG_[A-Z0-9_]\+=\([ynm]\|[0-9]\+\)\( CONFIG_[A-Z0-9_]\+=\([ynm]\|[0-9]\+\)\)*$' '^error$' + checkarg --kconfig "(Kconfig options)" $# "$2" '^CONFIG_[A-Z0-9_]\+=\([ynm]\|[0-9]\+\|"[^"]*"\)\( CONFIG_[A-Z0-9_]\+=\([ynm]\|[0-9]\+\|"[^"]*"\)\)*$' '^error$' TORTURE_KCONFIG_ARG="`echo "$TORTURE_KCONFIG_ARG $2" | sed -e 's/^ *//' -e 's/ *$//'`" shift ;; From 21786e5cb375a1e58a9175fee423e1d7f892d965 Mon Sep 17 00:00:00 2001 From: Daniel Vacek Date: Tue, 20 Dec 2022 16:14:15 +0100 Subject: [PATCH 054/239] cgroup/cpuset: no need to explicitly init a global static variable cpuset_rwsem is a static variable defined with DEFINE_STATIC_PERCPU_RWSEM(). It's initialized at build time and so there's no need for explicit runtime init leaking one percpu int. Signed-off-by: Daniel Vacek Reviewed-by: Aaron Tomlin Acked-by: Mukesh Ojha Signed-off-by: Tejun Heo --- kernel/cgroup/cpuset.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index a29c0b13706b..87fe410361b3 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -3281,8 +3281,6 @@ struct cgroup_subsys cpuset_cgrp_subsys = { int __init cpuset_init(void) { - BUG_ON(percpu_init_rwsem(&cpuset_rwsem)); - BUG_ON(!alloc_cpumask_var(&top_cpuset.cpus_allowed, GFP_KERNEL)); BUG_ON(!alloc_cpumask_var(&top_cpuset.effective_cpus, GFP_KERNEL)); BUG_ON(!zalloc_cpumask_var(&top_cpuset.subparts_cpus, GFP_KERNEL)); From 33e3f0a3358b8f9bb54b2661b9c1d37a75664c79 Mon Sep 17 00:00:00 2001 From: Richard Clark Date: Tue, 13 Dec 2022 12:39:36 +0800 Subject: [PATCH 055/239] workqueue: Add a new flag to spot the potential UAF error Currently if the user queues a new work item unintentionally into a wq after the destroy_workqueue(wq), the work still can be queued and scheduled without any noticeable kernel message before the end of a RCU grace period. As a debug-aid facility, this commit adds a new flag __WQ_DESTROYING to spot that issue by triggering a kernel WARN message. Signed-off-by: Richard Clark Reviewed-by: Lai Jiangshan Signed-off-by: Tejun Heo --- include/linux/workqueue.h | 1 + kernel/workqueue.c | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h index a0143dd24430..ac551b8ee7d9 100644 --- a/include/linux/workqueue.h +++ b/include/linux/workqueue.h @@ -335,6 +335,7 @@ enum { */ WQ_POWER_EFFICIENT = 1 << 7, + __WQ_DESTROYING = 1 << 15, /* internal: workqueue is destroying */ __WQ_DRAINING = 1 << 16, /* internal: workqueue is draining */ __WQ_ORDERED = 1 << 17, /* internal: workqueue is ordered */ __WQ_LEGACY = 1 << 18, /* internal: create*_workqueue() */ diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 07895deca271..5b06262a419c 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -1433,9 +1433,13 @@ static void __queue_work(int cpu, struct workqueue_struct *wq, lockdep_assert_irqs_disabled(); - /* if draining, only works from the same workqueue are allowed */ - if (unlikely(wq->flags & __WQ_DRAINING) && - WARN_ON_ONCE(!is_chained_work(wq))) + /* + * For a draining wq, only works from the same workqueue are + * allowed. The __WQ_DESTROYING helps to spot the issue that + * queues a new work item to a wq after destroy_workqueue(wq). + */ + if (unlikely(wq->flags & (__WQ_DESTROYING | __WQ_DRAINING) && + WARN_ON_ONCE(!is_chained_work(wq)))) return; rcu_read_lock(); retry: @@ -4414,6 +4418,11 @@ void destroy_workqueue(struct workqueue_struct *wq) */ workqueue_sysfs_unregister(wq); + /* mark the workqueue destruction is in progress */ + mutex_lock(&wq->mutex); + wq->flags |= __WQ_DESTROYING; + mutex_unlock(&wq->mutex); + /* drain it before proceeding with destruction */ drain_workqueue(wq); From cea7be909414d941a4616e6794f4a5282eb6e652 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 4 Jan 2023 16:38:02 -0800 Subject: [PATCH 056/239] drivers/opp: Remove "select SRCU" Now that the SRCU Kconfig option is unconditionally selected, there is no longer any point in selecting it. Therefore, remove the "select SRCU" Kconfig statements. Signed-off-by: Paul E. McKenney Cc: Viresh Kumar Cc: Nishanth Menon Cc: Stephen Boyd Cc: Signed-off-by: Viresh Kumar --- drivers/opp/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/opp/Kconfig b/drivers/opp/Kconfig index e8ce47b32735..d7c649a1a981 100644 --- a/drivers/opp/Kconfig +++ b/drivers/opp/Kconfig @@ -1,7 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only config PM_OPP bool - select SRCU help SOCs have a standard set of tuples consisting of frequency and voltage pairs that the device will support per voltage domain. This From 56eb276701cb67aa6c52e0713e66b46b4d94b38f Mon Sep 17 00:00:00 2001 From: Bagas Sanjaya Date: Thu, 5 Jan 2023 20:16:26 +0700 Subject: [PATCH 057/239] docs: cgroup-v1: replace custom note constructs with appropriate admonition blocks Admonition constructs on the documentation use definition lists, which isn't fit for the purpose. Replace them with appropriate blocks: * Use caution:: for outdated document notice * hint:: for memo * note:: for other constructs * warning:: for memory reclaim Signed-off-by: Bagas Sanjaya Signed-off-by: Tejun Heo --- .../admin-guide/cgroup-v1/memory.rst | 58 ++++++++++--------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/Documentation/admin-guide/cgroup-v1/memory.rst b/Documentation/admin-guide/cgroup-v1/memory.rst index 60370f2c67b9..d0d8c780cb6b 100644 --- a/Documentation/admin-guide/cgroup-v1/memory.rst +++ b/Documentation/admin-guide/cgroup-v1/memory.rst @@ -2,13 +2,13 @@ Memory Resource Controller ========================== -NOTE: +.. caution:: This document is hopelessly outdated and it asks for a complete rewrite. It still contains a useful information so we are keeping it here but make sure to check the current code if you need a deeper understanding. -NOTE: +.. note:: The Memory Resource Controller has generically been referred to as the memory controller in this document. Do not confuse memory controller used here with the memory controller that is used in hardware. @@ -274,12 +274,12 @@ The reclaim algorithm has not been modified for cgroups, except that pages that are selected for reclaiming come from the per-cgroup LRU list. -NOTE: - Reclaim does not work for the root cgroup, since we cannot set any - limits on the root cgroup. +.. note:: + Reclaim does not work for the root cgroup, since we cannot set any + limits on the root cgroup. -Note2: - When panic_on_oom is set to "2", the whole system will panic. +.. note:: + When panic_on_oom is set to "2", the whole system will panic. When oom event notifier is registered, event will be delivered. (See oom_control section) @@ -367,10 +367,10 @@ U != 0, K < U: never greater than the total memory, and freely set U at the cost of his QoS. -WARNING: - In the current implementation, memory reclaim will NOT be - triggered for a cgroup when it hits K while staying below U, which makes - this setup impractical. + .. warning:: + In the current implementation, memory reclaim will NOT be triggered for + a cgroup when it hits K while staying below U, which makes this setup + impractical. U != 0, K >= U: Since kmem charges will also be fed to the user counter and reclaim will be @@ -405,16 +405,16 @@ Since now we're in the 0 cgroup, we can alter the memory limit:: # echo 4M > /sys/fs/cgroup/memory/0/memory.limit_in_bytes -NOTE: - We can use a suffix (k, K, m, M, g or G) to indicate values in kilo, - mega or gigabytes. (Here, Kilo, Mega, Giga are Kibibytes, Mebibytes, - Gibibytes.) +.. note:: + We can use a suffix (k, K, m, M, g or G) to indicate values in kilo, + mega or gigabytes. (Here, Kilo, Mega, Giga are Kibibytes, Mebibytes, + Gibibytes.) -NOTE: - We can write "-1" to reset the ``*.limit_in_bytes(unlimited)``. +.. note:: + We can write "-1" to reset the ``*.limit_in_bytes(unlimited)``. -NOTE: - We cannot set limits on the root cgroup any more. +.. note:: + We cannot set limits on the root cgroup any more. :: @@ -574,12 +574,12 @@ recent_scanned_anon VM internal parameter. (see mm/vmscan.c) recent_scanned_file VM internal parameter. (see mm/vmscan.c) ========================= ======================================== -Memo: +.. hint:: recent_rotated means recent frequency of LRU rotation. recent_scanned means recent # of scans to LRU. showing for better debug please see the code for meanings. -Note: +.. note:: Only anonymous and swap cache memory is listed as part of 'rss' stat. This should not be confused with the true 'resident set size' or the amount of physical memory used by the cgroup. @@ -710,10 +710,11 @@ If we want to change this to 1G, we can at any time use:: # echo 1G > memory.soft_limit_in_bytes -NOTE1: +.. note:: Soft limits take effect over a long period of time, since they involve reclaiming memory for balancing between memory cgroups -NOTE2: + +.. note:: It is recommended to set the soft limit always below the hard limit, otherwise the hard limit will take precedence. @@ -735,17 +736,20 @@ If you want to enable it:: # echo (some positive value) > memory.move_charge_at_immigrate -Note: +.. note:: Each bits of move_charge_at_immigrate has its own meaning about what type of charges should be moved. See 8.2 for details. -Note: + +.. note:: Charges are moved only when you move mm->owner, in other words, a leader of a thread group. -Note: + +.. note:: If we cannot find enough space for the task in the destination cgroup, we try to make space by reclaiming memory. Task migration may fail if we cannot make enough space. -Note: + +.. note:: It can take several seconds if you move charges much. And if you want disable it again:: From 4ddb1a2aa1a3c4317b94417f1bace0e4f06f51b9 Mon Sep 17 00:00:00 2001 From: Bagas Sanjaya Date: Thu, 5 Jan 2023 20:16:27 +0700 Subject: [PATCH 058/239] docs: cgroup-v1: wrap remaining admonitions in admonition blocks Wrap two other admonitions in appropriate blocks in order for readers to pay more attention to block contents: * hint:: for editor's note * warning:: for move charges deprecation Signed-off-by: Bagas Sanjaya Signed-off-by: Tejun Heo --- Documentation/admin-guide/cgroup-v1/memory.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/admin-guide/cgroup-v1/memory.rst b/Documentation/admin-guide/cgroup-v1/memory.rst index d0d8c780cb6b..b0353c40639b 100644 --- a/Documentation/admin-guide/cgroup-v1/memory.rst +++ b/Documentation/admin-guide/cgroup-v1/memory.rst @@ -13,7 +13,7 @@ Memory Resource Controller memory controller in this document. Do not confuse memory controller used here with the memory controller that is used in hardware. -(For editors) In this document: +.. hint:: When we mention a cgroup (cgroupfs's directory) with memory controller, we call it "memory cgroup". When you see git-log and source code, you'll see patch's title and function names tend to use "memcg". From eb08489448fbc70eda1ba6cfdddd59de00ddf941 Mon Sep 17 00:00:00 2001 From: Bagas Sanjaya Date: Thu, 5 Jan 2023 20:16:28 +0700 Subject: [PATCH 059/239] docs: cgroup-v1: use code block for locking order schema The locking order schema is a figure (like diagram), which should have been formatted with literal code block for consistency with other figures. Signed-off-by: Bagas Sanjaya Signed-off-by: Tejun Heo --- Documentation/admin-guide/cgroup-v1/memory.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/admin-guide/cgroup-v1/memory.rst b/Documentation/admin-guide/cgroup-v1/memory.rst index b0353c40639b..2b807fc128c6 100644 --- a/Documentation/admin-guide/cgroup-v1/memory.rst +++ b/Documentation/admin-guide/cgroup-v1/memory.rst @@ -287,7 +287,7 @@ When oom event notifier is registered, event will be delivered. 2.6 Locking ----------- -Lock order is as follows: +Lock order is as follows:: Page lock (PG_locked bit of page->flags) mm->page_table_lock or split pte_lock From 71da431c30795716a1ca26158f608781b3eba33d Mon Sep 17 00:00:00 2001 From: Bagas Sanjaya Date: Thu, 5 Jan 2023 20:16:29 +0700 Subject: [PATCH 060/239] docs: cgroup-v1: fix footnotes The documentation contains external references, which some of them are marked as footnotes. Fix the syntax for them to be properly rendered as such. Non-footnote references aren't affected since the text for these is aligned the same to the footnotes. Signed-off-by: Bagas Sanjaya Signed-off-by: Tejun Heo --- .../admin-guide/cgroup-v1/memory.rst | 36 ++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/Documentation/admin-guide/cgroup-v1/memory.rst b/Documentation/admin-guide/cgroup-v1/memory.rst index 2b807fc128c6..8d1cedcf44f6 100644 --- a/Documentation/admin-guide/cgroup-v1/memory.rst +++ b/Documentation/admin-guide/cgroup-v1/memory.rst @@ -23,7 +23,7 @@ Benefits and Purpose of the memory controller ============================================= The memory controller isolates the memory behaviour of a group of tasks -from the rest of the system. The article on LWN [12] mentions some probable +from the rest of the system. The article on LWN [12]_ mentions some probable uses of the memory controller. The memory controller can be used to a. Isolate an application or a group of applications @@ -107,16 +107,16 @@ Brief summary of control files. ========== The memory controller has a long history. A request for comments for the memory -controller was posted by Balbir Singh [1]. At the time the RFC was posted +controller was posted by Balbir Singh [1]_. At the time the RFC was posted there were several implementations for memory control. The goal of the RFC was to build consensus and agreement for the minimal features required -for memory control. The first RSS controller was posted by Balbir Singh[2] -in Feb 2007. Pavel Emelianov [3][4][5] has since posted three versions of the -RSS controller. At OLS, at the resource management BoF, everyone suggested -that we handle both page cache and RSS together. Another request was raised -to allow user space handling of OOM. The current memory controller is +for memory control. The first RSS controller was posted by Balbir Singh [2]_ +in Feb 2007. Pavel Emelianov [3]_ [4]_ [5]_ has since posted three versions +of the RSS controller. At OLS, at the resource management BoF, everyone +suggested that we handle both page cache and RSS together. Another request was +raised to allow user space handling of OOM. The current memory controller is at version 6; it combines both mapped (RSS) and unmapped Page -Cache Control [11]. +Cache Control [11]_. 2. Memory Control ================= @@ -960,15 +960,16 @@ commented and discussed quite extensively in the community. References ========== -1. Singh, Balbir. RFC: Memory Controller, http://lwn.net/Articles/206697/ -2. Singh, Balbir. Memory Controller (RSS Control), +.. [1] Singh, Balbir. RFC: Memory Controller, http://lwn.net/Articles/206697/ +.. [2] Singh, Balbir. Memory Controller (RSS Control), http://lwn.net/Articles/222762/ -3. Emelianov, Pavel. Resource controllers based on process cgroups +.. [3] Emelianov, Pavel. Resource controllers based on process cgroups https://lore.kernel.org/r/45ED7DEC.7010403@sw.ru -4. Emelianov, Pavel. RSS controller based on process cgroups (v2) +.. [4] Emelianov, Pavel. RSS controller based on process cgroups (v2) https://lore.kernel.org/r/461A3010.90403@sw.ru -5. Emelianov, Pavel. RSS controller based on process cgroups (v3) +.. [5] Emelianov, Pavel. RSS controller based on process cgroups (v3) https://lore.kernel.org/r/465D9739.8070209@openvz.org + 6. Menage, Paul. Control Groups v10, http://lwn.net/Articles/236032/ 7. Vaidyanathan, Srinivasan, Control Groups: Pagecache accounting and control subsystem (v3), http://lwn.net/Articles/235534/ @@ -978,7 +979,8 @@ References https://lore.kernel.org/r/464D267A.50107@linux.vnet.ibm.com 10. Singh, Balbir. Memory controller v6 test results, https://lore.kernel.org/r/20070819094658.654.84837.sendpatchset@balbir-laptop -11. Singh, Balbir. Memory controller introduction (v6), - https://lore.kernel.org/r/20070817084228.26003.12568.sendpatchset@balbir-laptop -12. Corbet, Jonathan, Controlling memory use in cgroups, - http://lwn.net/Articles/243795/ + +.. [11] Singh, Balbir. Memory controller introduction (v6), + https://lore.kernel.org/r/20070817084228.26003.12568.sendpatchset@balbir-laptop +.. [12] Corbet, Jonathan, Controlling memory use in cgroups, + http://lwn.net/Articles/243795/ From f7423bb771d4acaad4e40addf9e09f5722f48b04 Mon Sep 17 00:00:00 2001 From: Bagas Sanjaya Date: Thu, 5 Jan 2023 20:16:30 +0700 Subject: [PATCH 061/239] docs: cgroup-v1: move hierarchy of accounting caption The caption for hierarchy of accounting figure is in the code block, which is quite odd. Move the caption into :caption: option of code-block:: directive instead. Signed-off-by: Bagas Sanjaya Signed-off-by: Tejun Heo --- Documentation/admin-guide/cgroup-v1/memory.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/cgroup-v1/memory.rst b/Documentation/admin-guide/cgroup-v1/memory.rst index 8d1cedcf44f6..4d96a5bbbfcf 100644 --- a/Documentation/admin-guide/cgroup-v1/memory.rst +++ b/Documentation/admin-guide/cgroup-v1/memory.rst @@ -147,7 +147,8 @@ specific data structure (mem_cgroup) associated with it. 2.2. Accounting --------------- -:: +.. code-block:: + :caption: Figure 1: Hierarchy of Accounting +--------------------+ | mem_cgroup | @@ -167,7 +168,6 @@ specific data structure (mem_cgroup) associated with it. | | | | +---------------+ +---------------+ - (Figure 1: Hierarchy of Accounting) Figure 1 shows the important aspects of the controller From b9d2a17b3290e251748c6a9f6d2b0a440401096c Mon Sep 17 00:00:00 2001 From: Bagas Sanjaya Date: Thu, 5 Jan 2023 20:16:31 +0700 Subject: [PATCH 062/239] docs: cgroup-v1: use bullet lists for list of stat file tables The stat file section contains three tables, where the leading texts for them are subsection heading. Organize them in the bullet list, while demoting headings into normal text. Signed-off-by: Bagas Sanjaya Signed-off-by: Tejun Heo --- .../admin-guide/cgroup-v1/memory.rst | 93 +++++++++---------- 1 file changed, 46 insertions(+), 47 deletions(-) diff --git a/Documentation/admin-guide/cgroup-v1/memory.rst b/Documentation/admin-guide/cgroup-v1/memory.rst index 4d96a5bbbfcf..162cc26dcddb 100644 --- a/Documentation/admin-guide/cgroup-v1/memory.rst +++ b/Documentation/admin-guide/cgroup-v1/memory.rst @@ -519,60 +519,59 @@ will be charged as a new owner of it. 5.2 stat file ------------- -memory.stat file includes following statistics +memory.stat file includes following statistics: -per-memory cgroup local status -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + * per-memory cgroup local status -=============== =============================================================== -cache # of bytes of page cache memory. -rss # of bytes of anonymous and swap cache memory (includes - transparent hugepages). -rss_huge # of bytes of anonymous transparent hugepages. -mapped_file # of bytes of mapped file (includes tmpfs/shmem) -pgpgin # of charging events to the memory cgroup. The charging - event happens each time a page is accounted as either mapped - anon page(RSS) or cache page(Page Cache) to the cgroup. -pgpgout # of uncharging events to the memory cgroup. The uncharging - event happens each time a page is unaccounted from the cgroup. -swap # of bytes of swap usage -dirty # of bytes that are waiting to get written back to the disk. -writeback # of bytes of file/anon cache that are queued for syncing to - disk. -inactive_anon # of bytes of anonymous and swap cache memory on inactive - LRU list. -active_anon # of bytes of anonymous and swap cache memory on active - LRU list. -inactive_file # of bytes of file-backed memory and MADV_FREE anonymous memory( - LazyFree pages) on inactive LRU list. -active_file # of bytes of file-backed memory on active LRU list. -unevictable # of bytes of memory that cannot be reclaimed (mlocked etc). -=============== =============================================================== + =============== =============================================================== + cache # of bytes of page cache memory. + rss # of bytes of anonymous and swap cache memory (includes + transparent hugepages). + rss_huge # of bytes of anonymous transparent hugepages. + mapped_file # of bytes of mapped file (includes tmpfs/shmem) + pgpgin # of charging events to the memory cgroup. The charging + event happens each time a page is accounted as either mapped + anon page(RSS) or cache page(Page Cache) to the cgroup. + pgpgout # of uncharging events to the memory cgroup. The uncharging + event happens each time a page is unaccounted from the + cgroup. + swap # of bytes of swap usage + dirty # of bytes that are waiting to get written back to the disk. + writeback # of bytes of file/anon cache that are queued for syncing to + disk. + inactive_anon # of bytes of anonymous and swap cache memory on inactive + LRU list. + active_anon # of bytes of anonymous and swap cache memory on active + LRU list. + inactive_file # of bytes of file-backed memory and MADV_FREE anonymous + memory (LazyFree pages) on inactive LRU list. + active_file # of bytes of file-backed memory on active LRU list. + unevictable # of bytes of memory that cannot be reclaimed (mlocked etc). + =============== =============================================================== -status considering hierarchy (see memory.use_hierarchy settings) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + * status considering hierarchy (see memory.use_hierarchy settings): -========================= =================================================== -hierarchical_memory_limit # of bytes of memory limit with regard to hierarchy - under which the memory cgroup is -hierarchical_memsw_limit # of bytes of memory+swap limit with regard to - hierarchy under which memory cgroup is. + ========================= =================================================== + hierarchical_memory_limit # of bytes of memory limit with regard to + hierarchy + under which the memory cgroup is + hierarchical_memsw_limit # of bytes of memory+swap limit with regard to + hierarchy under which memory cgroup is. -total_ # hierarchical version of , which in - addition to the cgroup's own value includes the - sum of all hierarchical children's values of - , i.e. total_cache -========================= =================================================== + total_ # hierarchical version of , which in + addition to the cgroup's own value includes the + sum of all hierarchical children's values of + , i.e. total_cache + ========================= =================================================== -The following additional stats are dependent on CONFIG_DEBUG_VM -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + * additional vm parameters (depends on CONFIG_DEBUG_VM): -========================= ======================================== -recent_rotated_anon VM internal parameter. (see mm/vmscan.c) -recent_rotated_file VM internal parameter. (see mm/vmscan.c) -recent_scanned_anon VM internal parameter. (see mm/vmscan.c) -recent_scanned_file VM internal parameter. (see mm/vmscan.c) -========================= ======================================== + ========================= ======================================== + recent_rotated_anon VM internal parameter. (see mm/vmscan.c) + recent_rotated_file VM internal parameter. (see mm/vmscan.c) + recent_scanned_anon VM internal parameter. (see mm/vmscan.c) + recent_scanned_file VM internal parameter. (see mm/vmscan.c) + ========================= ======================================== .. hint:: recent_rotated means recent frequency of LRU rotation. From 5fa16afc4b5ab0f030fe9732bac431557112b0c6 Mon Sep 17 00:00:00 2001 From: Bagas Sanjaya Date: Thu, 5 Jan 2023 20:16:32 +0700 Subject: [PATCH 063/239] docs: cgroup-v1: make swap extension subsections subsections Subsections text of swap extension section is marked up as bold text, whereas making them proper subsection is more appropriate. Signed-off-by: Bagas Sanjaya Signed-off-by: Tejun Heo --- Documentation/admin-guide/cgroup-v1/memory.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/cgroup-v1/memory.rst b/Documentation/admin-guide/cgroup-v1/memory.rst index 162cc26dcddb..0e583a6f7839 100644 --- a/Documentation/admin-guide/cgroup-v1/memory.rst +++ b/Documentation/admin-guide/cgroup-v1/memory.rst @@ -244,7 +244,8 @@ In this case, setting memsw.limit_in_bytes=3G will prevent bad use of swap. By using the memsw limit, you can avoid system OOM which can be caused by swap shortage. -**why 'memory+swap' rather than swap** +2.4.1 why 'memory+swap' rather than swap +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The global LRU(kswapd) can swap out arbitrary pages. Swap-out means to move account from memory to swap...there is no change in usage of @@ -252,7 +253,8 @@ memory+swap. In other words, when we want to limit the usage of swap without affecting global LRU, memory+swap limit is better than just limiting swap from an OS point of view. -**What happens when a cgroup hits memory.memsw.limit_in_bytes** +2.4.2. What happens when a cgroup hits memory.memsw.limit_in_bytes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ When a cgroup hits memory.memsw.limit_in_bytes, it's useless to do swap-out in this cgroup. Then, swap-out will not be done by cgroup routine and file From da3ad2e14f6314a0246ae3c88afef59c8b49a3e5 Mon Sep 17 00:00:00 2001 From: Bagas Sanjaya Date: Thu, 5 Jan 2023 20:16:33 +0700 Subject: [PATCH 064/239] docs: cgroup-v1: add internal cross-references The documentation contains references to other sections in the doc (internal). Add cross-references for them so that these can be accessed without having to manually search for them. Signed-off-by: Bagas Sanjaya Signed-off-by: Tejun Heo --- .../admin-guide/cgroup-v1/memory.rst | 43 +++++++++++++------ 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/Documentation/admin-guide/cgroup-v1/memory.rst b/Documentation/admin-guide/cgroup-v1/memory.rst index 0e583a6f7839..16d938abe69f 100644 --- a/Documentation/admin-guide/cgroup-v1/memory.rst +++ b/Documentation/admin-guide/cgroup-v1/memory.rst @@ -55,7 +55,8 @@ Features: - Root cgroup has no limit controls. Kernel memory support is a work in progress, and the current version provides - basically functionality. (See Section 2.7) + basically functionality. (See :ref:`section 2.7 + `) Brief summary of control files. @@ -221,8 +222,9 @@ behind this approach is that a cgroup that aggressively uses a shared page will eventually get charged for it (once it is uncharged from the cgroup that brought it in -- this will happen on memory pressure). -But see section 8.2: when moving a task to another cgroup, its pages may -be recharged to the new cgroup, if move_charge_at_immigrate has been chosen. +But see :ref:`section 8.2 ` when moving a +task to another cgroup, its pages may be recharged to the new cgroup, if +move_charge_at_immigrate has been chosen. 2.4 Swap Extension -------------------------------------- @@ -270,7 +272,7 @@ global VM. When a cgroup goes over its limit, we first try to reclaim memory from the cgroup so as to make space for the new pages that the cgroup has touched. If the reclaim is unsuccessful, an OOM routine is invoked to select and kill the bulkiest task in the -cgroup. (See 10. OOM Control below.) +cgroup. (See :ref:`10. OOM Control ` below.) The reclaim algorithm has not been modified for cgroups, except that pages that are selected for reclaiming come from the per-cgroup LRU @@ -284,7 +286,7 @@ list. When panic_on_oom is set to "2", the whole system will panic. When oom event notifier is registered, event will be delivered. -(See oom_control section) +(See :ref:`oom_control ` section) 2.6 Locking ----------- @@ -301,6 +303,8 @@ Per-node-per-memcgroup LRU (cgroup's private LRU) is guarded by lruvec->lru_lock; PG_lru bit of page->flags is cleared before isolating a page from its LRU under lruvec->lru_lock. +.. _cgroup-v1-memory-kernel-extension: + 2.7 Kernel Memory Extension ----------------------------------------------- @@ -460,6 +464,8 @@ test because it has noise of shared objects/status. But the above two are testing extreme situations. Trying usual test under memory controller is always helpful. +.. _cgroup-v1-memory-test-troubleshoot: + 4.1 Troubleshooting ------------------- @@ -472,8 +478,11 @@ terminated by the OOM killer. There are several causes for this: A sync followed by echo 1 > /proc/sys/vm/drop_caches will help get rid of some of the pages cached in the cgroup (page cache pages). -To know what happens, disabling OOM_Kill as per "10. OOM Control" (below) and -seeing what happens will be helpful. +To know what happens, disabling OOM_Kill as per :ref:`"10. OOM Control" +` (below) and seeing what happens will be +helpful. + +.. _cgroup-v1-memory-test-task-migration: 4.2 Task migration ------------------ @@ -484,15 +493,16 @@ remain charged to it, the charge is dropped when the page is freed or reclaimed. You can move charges of a task along with task migration. -See 8. "Move charges at task migration" +See :ref:`8. "Move charges at task migration" ` 4.3 Removing a cgroup --------------------- -A cgroup can be removed by rmdir, but as discussed in sections 4.1 and 4.2, a -cgroup might have some charge associated with it, even though all -tasks have migrated away from it. (because we charge against pages, not -against tasks.) +A cgroup can be removed by rmdir, but as discussed in :ref:`sections 4.1 +` and :ref:`4.2 +`, a cgroup might have some charge +associated with it, even though all tasks have migrated away from it. (because +we charge against pages, not against tasks.) We move the stats to parent, and no change on the charge except uncharging from the child. @@ -719,6 +729,8 @@ If we want to change this to 1G, we can at any time use:: It is recommended to set the soft limit always below the hard limit, otherwise the hard limit will take precedence. +.. _cgroup-v1-memory-move-charges: + 8. Move charges at task migration ================================= @@ -739,7 +751,8 @@ If you want to enable it:: .. note:: Each bits of move_charge_at_immigrate has its own meaning about what type - of charges should be moved. See 8.2 for details. + of charges should be moved. See :ref:`section 8.2 + ` for details. .. note:: Charges are moved only when you move mm->owner, in other words, @@ -757,6 +770,8 @@ And if you want disable it again:: # echo 0 > memory.move_charge_at_immigrate +.. _cgroup-v1-memory-movable-charges: + 8.2 Type of charges which can be moved -------------------------------------- @@ -806,6 +821,8 @@ threshold in any direction. It's applicable for root and non-root cgroup. +.. _cgroup-v1-memory-oom-control: + 10. OOM Control =============== From 980660cae7994ab03b31b2a32940c70e8421fc99 Mon Sep 17 00:00:00 2001 From: Bagas Sanjaya Date: Thu, 5 Jan 2023 20:16:34 +0700 Subject: [PATCH 065/239] docs: cgroup-v1: use numbered lists for user interface setup Setup instructions for memory resource controller UI uses a mix of section headings and normal paragraphs, whereas numbered lists are better fit for this purpose. While at it, also slightly reword the instructions and add reference to "Why are cgroups needed?" in the main cgroups documentation. Signed-off-by: Bagas Sanjaya Signed-off-by: Tejun Heo --- .../admin-guide/cgroup-v1/cgroups.rst | 2 ++ .../admin-guide/cgroup-v1/memory.rst | 26 ++++++++----------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/Documentation/admin-guide/cgroup-v1/cgroups.rst b/Documentation/admin-guide/cgroup-v1/cgroups.rst index b0688011ed06..9343148ee993 100644 --- a/Documentation/admin-guide/cgroup-v1/cgroups.rst +++ b/Documentation/admin-guide/cgroup-v1/cgroups.rst @@ -80,6 +80,8 @@ access. For example, cpusets (see Documentation/admin-guide/cgroup-v1/cpusets.rs you to associate a set of CPUs and a set of memory nodes with the tasks in each cgroup. +.. _cgroups-why-needed: + 1.2 Why are cgroups needed ? ---------------------------- diff --git a/Documentation/admin-guide/cgroup-v1/memory.rst b/Documentation/admin-guide/cgroup-v1/memory.rst index 16d938abe69f..27d89495ac88 100644 --- a/Documentation/admin-guide/cgroup-v1/memory.rst +++ b/Documentation/admin-guide/cgroup-v1/memory.rst @@ -387,30 +387,30 @@ U != 0, K >= U: 3. User Interface ================= -3.0. Configuration ------------------- +To use the user interface: -a. Enable CONFIG_CGROUPS -b. Enable CONFIG_MEMCG - -3.1. Prepare the cgroups (see cgroups.txt, Why are cgroups needed?) -------------------------------------------------------------------- - -:: +1. Enable CONFIG_CGROUPS and CONFIG_MEMCG options +2. Prepare the cgroups (see :ref:`Why are cgroups needed? + ` for the background information):: # mount -t tmpfs none /sys/fs/cgroup # mkdir /sys/fs/cgroup/memory # mount -t cgroup none /sys/fs/cgroup/memory -o memory -3.2. Make the new group and move bash into it:: +3. Make the new group and move bash into it:: # mkdir /sys/fs/cgroup/memory/0 # echo $$ > /sys/fs/cgroup/memory/0/tasks -Since now we're in the 0 cgroup, we can alter the memory limit:: +4. Since now we're in the 0 cgroup, we can alter the memory limit:: # echo 4M > /sys/fs/cgroup/memory/0/memory.limit_in_bytes + The limit can now be queried:: + + # cat /sys/fs/cgroup/memory/0/memory.limit_in_bytes + 4194304 + .. note:: We can use a suffix (k, K, m, M, g or G) to indicate values in kilo, mega or gigabytes. (Here, Kilo, Mega, Giga are Kibibytes, Mebibytes, @@ -422,10 +422,6 @@ Since now we're in the 0 cgroup, we can alter the memory limit:: .. note:: We cannot set limits on the root cgroup any more. -:: - - # cat /sys/fs/cgroup/memory/0/memory.limit_in_bytes - 4194304 We can check the usage:: From da82af04352b13a4880c97c4e00e8b12b1c67e6e Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 4 Nov 2022 14:03:15 -0700 Subject: [PATCH 066/239] doc: Update and wordsmith rculist_nulls.rst Do some wordsmithing and breaking up of RCU readers. Signed-off-by: Paul E. McKenney --- Documentation/RCU/rculist_nulls.rst | 115 ++++++++++++++-------------- 1 file changed, 57 insertions(+), 58 deletions(-) diff --git a/Documentation/RCU/rculist_nulls.rst b/Documentation/RCU/rculist_nulls.rst index ca4692775ad4..9a734bf54b76 100644 --- a/Documentation/RCU/rculist_nulls.rst +++ b/Documentation/RCU/rculist_nulls.rst @@ -14,19 +14,19 @@ Using 'nulls' ============= Using special makers (called 'nulls') is a convenient way -to solve following problem : +to solve following problem. -A typical RCU linked list managing objects which are -allocated with SLAB_TYPESAFE_BY_RCU kmem_cache can -use following algos : +Without 'nulls', a typical RCU linked list managing objects which are +allocated with SLAB_TYPESAFE_BY_RCU kmem_cache can use the following +algorithms: -1) Lookup algo --------------- +1) Lookup algorithm +------------------- :: - rcu_read_lock() begin: + rcu_read_lock() obj = lockless_lookup(key); if (obj) { if (!try_get_ref(obj)) // might fail for free objects @@ -38,6 +38,7 @@ use following algos : */ if (obj->key != key) { // not the object we expected put_ref(obj); + rcu_read_unlock(); goto begin; } } @@ -52,9 +53,9 @@ but a version with an additional memory barrier (smp_rmb()) { struct hlist_node *node, *next; for (pos = rcu_dereference((head)->first); - pos && ({ next = pos->next; smp_rmb(); prefetch(next); 1; }) && - ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); - pos = rcu_dereference(next)) + pos && ({ next = pos->next; smp_rmb(); prefetch(next); 1; }) && + ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); + pos = rcu_dereference(next)) if (obj->key == key) return obj; return NULL; @@ -64,9 +65,9 @@ And note the traditional hlist_for_each_entry_rcu() misses this smp_rmb():: struct hlist_node *node; for (pos = rcu_dereference((head)->first); - pos && ({ prefetch(pos->next); 1; }) && - ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); - pos = rcu_dereference(pos->next)) + pos && ({ prefetch(pos->next); 1; }) && + ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); + pos = rcu_dereference(pos->next)) if (obj->key == key) return obj; return NULL; @@ -82,36 +83,32 @@ Quoting Corey Minyard:: solved by pre-fetching the "next" field (with proper barriers) before checking the key." -2) Insert algo --------------- +2) Insertion algorithm +---------------------- We need to make sure a reader cannot read the new 'obj->obj_next' value -and previous value of 'obj->key'. Or else, an item could be deleted +and previous value of 'obj->key'. Otherwise, an item could be deleted from a chain, and inserted into another chain. If new chain was empty -before the move, 'next' pointer is NULL, and lockless reader can -not detect it missed following items in original chain. +before the move, 'next' pointer is NULL, and lockless reader can not +detect the fact that it missed following items in original chain. :: /* - * Please note that new inserts are done at the head of list, - * not in the middle or end. - */ + * Please note that new inserts are done at the head of list, + * not in the middle or end. + */ obj = kmem_cache_alloc(...); lock_chain(); // typically a spin_lock() obj->key = key; - /* - * we need to make sure obj->key is updated before obj->next - * or obj->refcnt - */ - smp_wmb(); - atomic_set(&obj->refcnt, 1); + atomic_set_release(&obj->refcnt, 1); // key before refcnt hlist_add_head_rcu(&obj->obj_node, list); unlock_chain(); // typically a spin_unlock() -3) Remove algo --------------- +3) Removal algorithm +-------------------- + Nothing special here, we can use a standard RCU hlist deletion. But thanks to SLAB_TYPESAFE_BY_RCU, beware a deleted object can be reused very very fast (before the end of RCU grace period) @@ -133,7 +130,7 @@ Avoiding extra smp_rmb() ======================== With hlist_nulls we can avoid extra smp_rmb() in lockless_lookup() -and extra smp_wmb() in insert function. +and extra _release() in insert function. For example, if we choose to store the slot number as the 'nulls' end-of-list marker for each slot of the hash table, we can detect @@ -142,59 +139,61 @@ to another chain) checking the final 'nulls' value if the lookup met the end of chain. If final 'nulls' value is not the slot number, then we must restart the lookup at the beginning. If the object was moved to the same chain, -then the reader doesn't care : It might eventually +then the reader doesn't care: It might occasionally scan the list again without harm. -1) lookup algo --------------- +1) lookup algorithm +------------------- :: head = &table[slot]; - rcu_read_lock(); begin: + rcu_read_lock(); hlist_nulls_for_each_entry_rcu(obj, node, head, member) { if (obj->key == key) { - if (!try_get_ref(obj)) // might fail for free objects - goto begin; - if (obj->key != key) { // not the object we expected - put_ref(obj); + if (!try_get_ref(obj)) { // might fail for free objects + rcu_read_unlock(); goto begin; } - goto out; + if (obj->key != key) { // not the object we expected + put_ref(obj); + rcu_read_unlock(); + goto begin; + } + goto out; + } + } + + // If the nulls value we got at the end of this lookup is + // not the expected one, we must restart lookup. + // We probably met an item that was moved to another chain. + if (get_nulls_value(node) != slot) { + put_ref(obj); + rcu_read_unlock(); + goto begin; } - /* - * if the nulls value we got at the end of this lookup is - * not the expected one, we must restart lookup. - * We probably met an item that was moved to another chain. - */ - if (get_nulls_value(node) != slot) - goto begin; obj = NULL; out: rcu_read_unlock(); -2) Insert function ------------------- +2) Insert algorithm +------------------- :: /* - * Please note that new inserts are done at the head of list, - * not in the middle or end. - */ + * Please note that new inserts are done at the head of list, + * not in the middle or end. + */ obj = kmem_cache_alloc(cachep); lock_chain(); // typically a spin_lock() obj->key = key; + atomic_set_release(&obj->refcnt, 1); // key before refcnt /* - * changes to obj->key must be visible before refcnt one - */ - smp_wmb(); - atomic_set(&obj->refcnt, 1); - /* - * insert obj in RCU way (readers might be traversing chain) - */ + * insert obj in RCU way (readers might be traversing chain) + */ hlist_nulls_add_head_rcu(&obj->obj_node, list); unlock_chain(); // typically a spin_unlock() From 647dd4cd7c2137194811d8585a7500dab3d328c0 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 4 Nov 2022 14:16:48 -0700 Subject: [PATCH 067/239] doc: Update rcu.rst This commit provides a couple of updates based on the inexorable passage of time. Signed-off-by: Paul E. McKenney --- Documentation/RCU/rcu.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation/RCU/rcu.rst b/Documentation/RCU/rcu.rst index 3cfe01ba9a49..381cb86f657d 100644 --- a/Documentation/RCU/rcu.rst +++ b/Documentation/RCU/rcu.rst @@ -77,12 +77,13 @@ Frequently Asked Questions search for the string "Patent" in Documentation/RCU/RTFP.txt to find them. Of these, one was allowed to lapse by the assignee, and the others have been contributed to the Linux kernel under GPL. + Many (but not all) have long since expired. There are now also LGPL implementations of user-level RCU available (https://liburcu.org/). - I hear that RCU needs work in order to support realtime kernels? - Realtime-friendly RCU can be enabled via the CONFIG_PREEMPT_RCU + Realtime-friendly RCU are enabled via the CONFIG_PREEMPTION kernel configuration parameter. - Where can I find more information on RCU? From 3abf176d64ac37d086271662dd47b8dfc9987152 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 4 Nov 2022 14:39:32 -0700 Subject: [PATCH 068/239] doc: Update stallwarn.rst This commit updates stallwarn.rst to reflect RCU additions and changes over the past few years. Signed-off-by: Paul E. McKenney --- Documentation/RCU/stallwarn.rst | 43 +++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/Documentation/RCU/stallwarn.rst b/Documentation/RCU/stallwarn.rst index e38c587067fc..dfa4db8c0931 100644 --- a/Documentation/RCU/stallwarn.rst +++ b/Documentation/RCU/stallwarn.rst @@ -25,10 +25,10 @@ warnings: - A CPU looping with bottom halves disabled. -- For !CONFIG_PREEMPTION kernels, a CPU looping anywhere in the kernel - without invoking schedule(). If the looping in the kernel is - really expected and desirable behavior, you might need to add - some calls to cond_resched(). +- For !CONFIG_PREEMPTION kernels, a CPU looping anywhere in the + kernel without potentially invoking schedule(). If the looping + in the kernel is really expected and desirable behavior, you + might need to add some calls to cond_resched(). - Booting Linux using a console connection that is too slow to keep up with the boot-time console-message rate. For example, @@ -108,16 +108,17 @@ warnings: - A bug in the RCU implementation. -- A hardware failure. This is quite unlikely, but has occurred - at least once in real life. A CPU failed in a running system, - becoming unresponsive, but not causing an immediate crash. - This resulted in a series of RCU CPU stall warnings, eventually - leading the realization that the CPU had failed. +- A hardware failure. This is quite unlikely, but is not at all + uncommon in large datacenter. In one memorable case some decades + back, a CPU failed in a running system, becoming unresponsive, + but not causing an immediate crash. This resulted in a series + of RCU CPU stall warnings, eventually leading the realization + that the CPU had failed. -The RCU, RCU-sched, and RCU-tasks implementations have CPU stall warning. -Note that SRCU does *not* have CPU stall warnings. Please note that -RCU only detects CPU stalls when there is a grace period in progress. -No grace period, no CPU stall warnings. +The RCU, RCU-sched, RCU-tasks, and RCU-tasks-trace implementations have +CPU stall warning. Note that SRCU does *not* have CPU stall warnings. +Please note that RCU only detects CPU stalls when there is a grace period +in progress. No grace period, no CPU stall warnings. To diagnose the cause of the stall, inspect the stack traces. The offending function will usually be near the top of the stack. @@ -205,16 +206,21 @@ RCU_STALL_RAT_DELAY rcupdate.rcu_task_stall_timeout ------------------------------- - This boot/sysfs parameter controls the RCU-tasks stall warning - interval. A value of zero or less suppresses RCU-tasks stall - warnings. A positive value sets the stall-warning interval - in seconds. An RCU-tasks stall warning starts with the line: + This boot/sysfs parameter controls the RCU-tasks and + RCU-tasks-trace stall warning intervals. A value of zero or less + suppresses RCU-tasks stall warnings. A positive value sets the + stall-warning interval in seconds. An RCU-tasks stall warning + starts with the line: INFO: rcu_tasks detected stalls on tasks: And continues with the output of sched_show_task() for each task stalling the current RCU-tasks grace period. + An RCU-tasks-trace stall warning starts (and continues) similarly: + + INFO: rcu_tasks_trace detected stalls on tasks + Interpreting RCU's CPU Stall-Detector "Splats" ============================================== @@ -248,7 +254,8 @@ dynticks counter, which will have an even-numbered value if the CPU is in dyntick-idle mode and an odd-numbered value otherwise. The hex number between the two "/"s is the value of the nesting, which will be a small non-negative number if in the idle loop (as shown above) and a -very large positive number otherwise. +very large positive number otherwise. The number following the final +"/" is the NMI nesting, which will be a small non-negative number. The "softirq=" portion of the message tracks the number of RCU softirq handlers that the stalled CPU has executed. The number before the "/" From 0c208a7930228a12a78cc2c11f6f09274872ec83 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 4 Nov 2022 15:34:03 -0700 Subject: [PATCH 069/239] doc: Update torture.rst This commit updates torture.rst with wordsmithing and the addition of a few more scripts. Signed-off-by: Paul E. McKenney --- Documentation/RCU/torture.rst | 89 +++++++++++++++++++++++++++++++++-- 1 file changed, 85 insertions(+), 4 deletions(-) diff --git a/Documentation/RCU/torture.rst b/Documentation/RCU/torture.rst index a90147713062..0316ba0c6922 100644 --- a/Documentation/RCU/torture.rst +++ b/Documentation/RCU/torture.rst @@ -206,7 +206,11 @@ values for memory may require disabling the callback-flooding tests using the --bootargs parameter discussed below. Sometimes additional debugging is useful, and in such cases the --kconfig -parameter to kvm.sh may be used, for example, ``--kconfig 'CONFIG_KASAN=y'``. +parameter to kvm.sh may be used, for example, ``--kconfig 'CONFIG_RCU_EQS_DEBUG=y'``. +In addition, there are the --gdb, --kasan, and --kcsan parameters. +Note that --gdb limits you to one scenario per kvm.sh run and requires +that you have another window open from which to run ``gdb`` as instructed +by the script. Kernel boot arguments can also be supplied, for example, to control rcutorture's module parameters. For example, to test a change to RCU's @@ -219,10 +223,17 @@ require disabling rcutorture's callback-flooding tests:: --bootargs 'rcutorture.fwd_progress=0' Sometimes all that is needed is a full set of kernel builds. This is -what the --buildonly argument does. +what the --buildonly parameter does. -Finally, the --trust-make argument allows each kernel build to reuse what -it can from the previous kernel build. +The --duration parameter can override the default run time of 30 minutes. +For example, ``--duration 2d`` would run for two days, ``--duration 3h`` +would run for three hours, ``--duration 5m`` would run for five minutes, +and ``--duration 45s`` would run for 45 seconds. This last can be useful +for tracking down rare boot-time failures. + +Finally, the --trust-make parameter allows each kernel build to reuse what +it can from the previous kernel build. Please note that without the +--trust-make parameter, your tags files may be demolished. There are additional more arcane arguments that are documented in the source code of the kvm.sh script. @@ -291,3 +302,73 @@ the following summary at the end of the run on a 12-CPU system:: TREE07 ------- 167347 GPs (30.9902/s) [rcu: g1079021 f0x0 ] n_max_cbs: 478732 CPU count limited from 16 to 12 TREE09 ------- 752238 GPs (139.303/s) [rcu: g13075057 f0x0 ] n_max_cbs: 99011 + + +Repeated Runs +============= + +Suppose that you are chasing down a rare boot-time failure. Although you +could use kvm.sh, doing so will rebuild the kernel on each run. If you +need (say) 1,000 runs to have confidence that you have fixed the bug, +these pointless rebuilds can become extremely annoying. + +This is why kvm-again.sh exists. + +Suppose that a previous kvm.sh run left its output in this directory:: + + tools/testing/selftests/rcutorture/res/2022.11.03-11.26.28 + +Then this run can be re-run without rebuilding as follow: + + kvm-again.sh tools/testing/selftests/rcutorture/res/2022.11.03-11.26.28 + +A few of the original run's kvm.sh parameters may be overridden, perhaps +most notably --duration and --bootargs. For example:: + + kvm-again.sh tools/testing/selftests/rcutorture/res/2022.11.03-11.26.28 \ + --duration 45s + +would re-run the previous test, but for only 45 seconds, thus facilitating +tracking down the aforementioned rare boot-time failure. + + +Distributed Runs +================ + +Although kvm.sh is quite useful, its testing is confined to a single +system. It is not all that hard to use your favorite framework to cause +(say) 5 instances of kvm.sh to run on your 5 systems, but this will very +likely unnecessarily rebuild kernels. In addition, manually distributing +the desired rcutorture scenarios across the available systems can be +painstaking and error-prone. + +And this is why the kvm-remote.sh script exists. + +If you the following command works:: + + ssh system0 date + +and if it also works for system1, system2, system3, system4, and system5, +and all of these systems have 64 CPUs, you can type:: + + kvm-remote.sh "system0 system1 system2 system3 system4 system5" \ + --cpus 64 --duration 8h --configs "5*CFLIST" + +This will build each default scenario's kernel on the local system, then +spread each of five instances of each scenario over the systems listed, +running each scenario for eight hours. At the end of the runs, the +results will be gathered, recorded, and printed. Most of the parameters +that kvm.sh will accept can be passed to kvm-remote.sh, but the list of +systems must come first. + +The kvm.sh ``--dryrun scenarios`` argument is useful for working out +how many scenarios may be run in one batch across a group of systems. + +You can also re-run a previous remote run in a manner similar to kvm.sh: + + kvm-remote.sh "system0 system1 system2 system3 system4 system5" \ + tools/testing/selftests/rcutorture/res/2022.11.03-11.26.28-remote \ + --duration 24h + +In this case, most of the kvm-again.sh parmeters may be supplied following +the pathname of the old run-results directory. From c8f2310e210ceedab86f1f696d27f7b5dd87959a Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 4 Nov 2022 16:45:55 -0700 Subject: [PATCH 070/239] doc: Update UP.rst This commit updates UP.rst to reflect changes over the past few years, including the advent of userspace RCU libraries for constrained systems. Signed-off-by: Paul E. McKenney --- Documentation/RCU/UP.rst | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Documentation/RCU/UP.rst b/Documentation/RCU/UP.rst index e26dda27430c..8b20fd45f255 100644 --- a/Documentation/RCU/UP.rst +++ b/Documentation/RCU/UP.rst @@ -38,7 +38,7 @@ by having call_rcu() directly invoke its arguments only if it was called from process context. However, this can fail in a similar manner. Suppose that an RCU-based algorithm again scans a linked list containing -elements A, B, and C in process contexts, but that it invokes a function +elements A, B, and C in process context, but that it invokes a function on each element as it is scanned. Suppose further that this function deletes element B from the list, then passes it to call_rcu() for deferred freeing. This may be a bit unconventional, but it is perfectly legal @@ -59,7 +59,8 @@ Example 3: Death by Deadlock Suppose that call_rcu() is invoked while holding a lock, and that the callback function must acquire this same lock. In this case, if call_rcu() were to directly invoke the callback, the result would -be self-deadlock. +be self-deadlock *even if* this invocation occurred from a later +call_rcu() invocation a full grace period later. In some cases, it would possible to restructure to code so that the call_rcu() is delayed until after the lock is released. However, @@ -85,6 +86,14 @@ Quick Quiz #2: :ref:`Answers to Quick Quiz ` +It is important to note that userspace RCU implementations *do* +permit call_rcu() to directly invoke callbacks, but only if a full +grace period has elapsed since those callbacks were queued. This is +the case because some userspace environments are extremely constrained. +Nevertheless, people writing userspace RCU implementations are strongly +encouraged to avoid invoking callbacks from call_rcu(), thus obtaining +the deadlock-avoidance benefits called out above. + Summary ------- From 3f58c55e239262447a758d41f0495b526ce10977 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 4 Nov 2022 16:55:03 -0700 Subject: [PATCH 071/239] doc: Update rcu.rst URL to RCU publications Also add the more recent thicket of Google Documents. Signed-off-by: Paul E. McKenney --- Documentation/RCU/rcu.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation/RCU/rcu.rst b/Documentation/RCU/rcu.rst index 381cb86f657d..bf6617b330a7 100644 --- a/Documentation/RCU/rcu.rst +++ b/Documentation/RCU/rcu.rst @@ -89,4 +89,5 @@ Frequently Asked Questions - Where can I find more information on RCU? See the Documentation/RCU/RTFP.txt file. - Or point your browser at (http://www.rdrop.com/users/paulmck/RCU/). + Or point your browser at (https://docs.google.com/document/d/1X0lThx8OK0ZgLMqVoXiR4ZrGURHrXK6NyLRbeXe3Xac/edit) + or (https://docs.google.com/document/d/1GCdQC8SDbb54W1shjEXqGZ0Rq8a6kIeYutdSIajfpLA/edit?usp=sharing). From 148750d736c44eecf10836ef7af397388e6194c3 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 4 Nov 2022 18:00:52 -0700 Subject: [PATCH 072/239] doc: Update whatisRCU.rst This commit updates whatisRCU.rst with wordsmithing and updates provokes by the passage of time. Signed-off-by: Paul E. McKenney --- Documentation/RCU/whatisRCU.rst | 189 +++++++++++++++++++++----------- 1 file changed, 123 insertions(+), 66 deletions(-) diff --git a/Documentation/RCU/whatisRCU.rst b/Documentation/RCU/whatisRCU.rst index 1c747ac3f2c8..2c5563a91998 100644 --- a/Documentation/RCU/whatisRCU.rst +++ b/Documentation/RCU/whatisRCU.rst @@ -16,18 +16,23 @@ to start learning about RCU: | 6. The RCU API, 2019 Edition https://lwn.net/Articles/777036/ | 2019 Big API Table https://lwn.net/Articles/777165/ +For those preferring video: + +| 1. Unraveling RCU Mysteries: Fundamentals https://www.linuxfoundation.org/webinars/unraveling-rcu-usage-mysteries +| 2. Unraveling RCU Mysteries: Additional Use Cases https://www.linuxfoundation.org/webinars/unraveling-rcu-usage-mysteries-additional-use-cases + What is RCU? RCU is a synchronization mechanism that was added to the Linux kernel during the 2.5 development effort that is optimized for read-mostly -situations. Although RCU is actually quite simple once you understand it, -getting there can sometimes be a challenge. Part of the problem is that -most of the past descriptions of RCU have been written with the mistaken -assumption that there is "one true way" to describe RCU. Instead, -the experience has been that different people must take different paths -to arrive at an understanding of RCU. This document provides several -different paths, as follows: +situations. Although RCU is actually quite simple, making effective use +of it requires you to think differently about your code. Another part +of the problem is the mistaken assumption that there is "one true way" to +describe and to use RCU. Instead, the experience has been that different +people must take different paths to arrive at an understanding of RCU, +depending on their experiences and use cases. This document provides +several different paths, as follows: :ref:`1. RCU OVERVIEW <1_whatisRCU>` @@ -157,34 +162,36 @@ rcu_read_lock() ^^^^^^^^^^^^^^^ void rcu_read_lock(void); - Used by a reader to inform the reclaimer that the reader is - entering an RCU read-side critical section. It is illegal - to block while in an RCU read-side critical section, though - kernels built with CONFIG_PREEMPT_RCU can preempt RCU - read-side critical sections. Any RCU-protected data structure - accessed during an RCU read-side critical section is guaranteed to - remain unreclaimed for the full duration of that critical section. - Reference counts may be used in conjunction with RCU to maintain - longer-term references to data structures. + This temporal primitive is used by a reader to inform the + reclaimer that the reader is entering an RCU read-side critical + section. It is illegal to block while in an RCU read-side + critical section, though kernels built with CONFIG_PREEMPT_RCU + can preempt RCU read-side critical sections. Any RCU-protected + data structure accessed during an RCU read-side critical section + is guaranteed to remain unreclaimed for the full duration of that + critical section. Reference counts may be used in conjunction + with RCU to maintain longer-term references to data structures. rcu_read_unlock() ^^^^^^^^^^^^^^^^^ void rcu_read_unlock(void); - Used by a reader to inform the reclaimer that the reader is - exiting an RCU read-side critical section. Note that RCU - read-side critical sections may be nested and/or overlapping. + This temporal primitives is used by a reader to inform the + reclaimer that the reader is exiting an RCU read-side critical + section. Note that RCU read-side critical sections may be nested + and/or overlapping. synchronize_rcu() ^^^^^^^^^^^^^^^^^ void synchronize_rcu(void); - Marks the end of updater code and the beginning of reclaimer - code. It does this by blocking until all pre-existing RCU - read-side critical sections on all CPUs have completed. - Note that synchronize_rcu() will **not** necessarily wait for - any subsequent RCU read-side critical sections to complete. - For example, consider the following sequence of events:: + This temporal primitive marks the end of updater code and the + beginning of reclaimer code. It does this by blocking until + all pre-existing RCU read-side critical sections on all CPUs + have completed. Note that synchronize_rcu() will **not** + necessarily wait for any subsequent RCU read-side critical + sections to complete. For example, consider the following + sequence of events:: CPU 0 CPU 1 CPU 2 ----------------- ------------------------- --------------- @@ -211,13 +218,13 @@ synchronize_rcu() to be useful in all but the most read-intensive situations, synchronize_rcu()'s overhead must also be quite small. - The call_rcu() API is a callback form of synchronize_rcu(), - and is described in more detail in a later section. Instead of - blocking, it registers a function and argument which are invoked - after all ongoing RCU read-side critical sections have completed. - This callback variant is particularly useful in situations where - it is illegal to block or where update-side performance is - critically important. + The call_rcu() API is an asynchronous callback form of + synchronize_rcu(), and is described in more detail in a later + section. Instead of blocking, it registers a function and + argument which are invoked after all ongoing RCU read-side + critical sections have completed. This callback variant is + particularly useful in situations where it is illegal to block + or where update-side performance is critically important. However, the call_rcu() API should not be used lightly, as use of the synchronize_rcu() API generally results in simpler code. @@ -236,11 +243,13 @@ rcu_assign_pointer() would be cool to be able to declare a function in this manner. (Compiler experts will no doubt disagree.) - The updater uses this function to assign a new value to an + The updater uses this spatial macro to assign a new value to an RCU-protected pointer, in order to safely communicate the change - in value from the updater to the reader. This macro does not - evaluate to an rvalue, but it does execute any memory-barrier - instructions required for a given CPU architecture. + in value from the updater to the reader. This is a spatial (as + opposed to temporal) macro. It does not evaluate to an rvalue, + but it does execute any memory-barrier instructions required + for a given CPU architecture. Its ordering properties are that + of a store-release operation. Perhaps just as important, it serves to document (1) which pointers are protected by RCU and (2) the point at which a @@ -255,14 +264,15 @@ rcu_dereference() Like rcu_assign_pointer(), rcu_dereference() must be implemented as a macro. - The reader uses rcu_dereference() to fetch an RCU-protected - pointer, which returns a value that may then be safely - dereferenced. Note that rcu_dereference() does not actually - dereference the pointer, instead, it protects the pointer for - later dereferencing. It also executes any needed memory-barrier - instructions for a given CPU architecture. Currently, only Alpha - needs memory barriers within rcu_dereference() -- on other CPUs, - it compiles to nothing, not even a compiler directive. + The reader uses the spatial rcu_dereference() macro to fetch + an RCU-protected pointer, which returns a value that may + then be safely dereferenced. Note that rcu_dereference() + does not actually dereference the pointer, instead, it + protects the pointer for later dereferencing. It also + executes any needed memory-barrier instructions for a given + CPU architecture. Currently, only Alpha needs memory barriers + within rcu_dereference() -- on other CPUs, it compiles to a + volatile load. Common coding practice uses rcu_dereference() to copy an RCU-protected pointer to a local variable, then dereferences @@ -355,12 +365,15 @@ reader, updater, and reclaimer. synchronize_rcu() & call_rcu() -The RCU infrastructure observes the time sequence of rcu_read_lock(), +The RCU infrastructure observes the temporal sequence of rcu_read_lock(), rcu_read_unlock(), synchronize_rcu(), and call_rcu() invocations in order to determine when (1) synchronize_rcu() invocations may return to their callers and (2) call_rcu() callbacks may be invoked. Efficient implementations of the RCU infrastructure make heavy use of batching in order to amortize their overhead over many uses of the corresponding APIs. +The rcu_assign_pointer() and rcu_dereference() invocations communicate +spatial changes via stores to and loads from the RCU-protected pointer in +question. There are at least three flavors of RCU usage in the Linux kernel. The diagram above shows the most common one. On the updater side, the rcu_assign_pointer(), @@ -392,7 +405,9 @@ b. RCU applied to networking data structures that may be subjected c. RCU applied to scheduler and interrupt/NMI-handler tasks. Again, most uses will be of (a). The (b) and (c) cases are important -for specialized uses, but are relatively uncommon. +for specialized uses, but are relatively uncommon. The SRCU, RCU-Tasks, +RCU-Tasks-Rude, and RCU-Tasks-Trace have similar relationships among +their assorted primitives. .. _3_whatisRCU: @@ -468,7 +483,7 @@ So, to sum up: - Within an RCU read-side critical section, use rcu_dereference() to dereference RCU-protected pointers. -- Use some solid scheme (such as locks or semaphores) to +- Use some solid design (such as locks or semaphores) to keep concurrent updates from interfering with each other. - Use rcu_assign_pointer() to update an RCU-protected pointer. @@ -579,6 +594,14 @@ to avoid having to write your own callback:: kfree_rcu(old_fp, rcu); +If the occasional sleep is permitted, the single-argument form may +be used, omitting the rcu_head structure from struct foo. + + kfree_rcu(old_fp); + +This variant of kfree_rcu() almost never blocks, but might do so by +invoking synchronize_rcu() in response to memory-allocation failure. + Again, see checklist.rst for additional rules governing the use of RCU. .. _5_whatisRCU: @@ -596,7 +619,7 @@ lacking both functionality and performance. However, they are useful in getting a feel for how RCU works. See kernel/rcu/update.c for a production-quality implementation, and see: - http://www.rdrop.com/users/paulmck/RCU + https://docs.google.com/document/d/1X0lThx8OK0ZgLMqVoXiR4ZrGURHrXK6NyLRbeXe3Xac/edit for papers describing the Linux kernel RCU implementation. The OLS'01 and OLS'02 papers are a good introduction, and the dissertation provides @@ -929,6 +952,8 @@ unfortunately any spinlock in a ``SLAB_TYPESAFE_BY_RCU`` object must be initialized after each and every call to kmem_cache_alloc(), which renders reference-free spinlock acquisition completely unsafe. Therefore, when using ``SLAB_TYPESAFE_BY_RCU``, make proper use of a reference counter. +(Those willing to use a kmem_cache constructor may also use locking, +including cache-friendly sequence locking.) With traditional reference counting -- such as that implemented by the kref library in Linux -- there is typically code that runs when the last @@ -1047,6 +1072,30 @@ sched:: rcu_read_lock_sched_held +RCU-Tasks:: + + Critical sections Grace period Barrier + + N/A call_rcu_tasks rcu_barrier_tasks + synchronize_rcu_tasks + + +RCU-Tasks-Rude:: + + Critical sections Grace period Barrier + + N/A call_rcu_tasks_rude rcu_barrier_tasks_rude + synchronize_rcu_tasks_rude + + +RCU-Tasks-Trace:: + + Critical sections Grace period Barrier + + rcu_read_lock_trace call_rcu_tasks_trace rcu_barrier_tasks_trace + rcu_read_unlock_trace synchronize_rcu_tasks_trace + + SRCU:: Critical sections Grace period Barrier @@ -1087,35 +1136,43 @@ list can be helpful: a. Will readers need to block? If so, you need SRCU. -b. What about the -rt patchset? If readers would need to block - in an non-rt kernel, you need SRCU. If readers would block - in a -rt kernel, but not in a non-rt kernel, SRCU is not - necessary. (The -rt patchset turns spinlocks into sleeplocks, - hence this distinction.) +b. Will readers need to block and are you doing tracing, for + example, ftrace or BPF? If so, you need RCU-tasks, + RCU-tasks-rude, and/or RCU-tasks-trace. -c. Do you need to treat NMI handlers, hardirq handlers, +c. What about the -rt patchset? If readers would need to block in + an non-rt kernel, you need SRCU. If readers would block when + acquiring spinlocks in a -rt kernel, but not in a non-rt kernel, + SRCU is not necessary. (The -rt patchset turns spinlocks into + sleeplocks, hence this distinction.) + +d. Do you need to treat NMI handlers, hardirq handlers, and code segments with preemption disabled (whether via preempt_disable(), local_irq_save(), local_bh_disable(), or some other mechanism) as if they were explicit RCU readers? - If so, RCU-sched is the only choice that will work for you. + If so, RCU-sched readers are the only choice that will work + for you, but since about v4.20 you use can use the vanilla RCU + update primitives. -d. Do you need RCU grace periods to complete even in the face - of softirq monopolization of one or more of the CPUs? For - example, is your code subject to network-based denial-of-service - attacks? If so, you should disable softirq across your readers, - for example, by using rcu_read_lock_bh(). +e. Do you need RCU grace periods to complete even in the face of + softirq monopolization of one or more of the CPUs? For example, + is your code subject to network-based denial-of-service attacks? + If so, you should disable softirq across your readers, for + example, by using rcu_read_lock_bh(). Since about v4.20 you + use can use the vanilla RCU update primitives. -e. Is your workload too update-intensive for normal use of +f. Is your workload too update-intensive for normal use of RCU, but inappropriate for other synchronization mechanisms? If so, consider SLAB_TYPESAFE_BY_RCU (which was originally named SLAB_DESTROY_BY_RCU). But please be careful! -f. Do you need read-side critical sections that are respected - even though they are in the middle of the idle loop, during - user-mode execution, or on an offlined CPU? If so, SRCU is the - only choice that will work for you. +g. Do you need read-side critical sections that are respected even + on CPUs that are deep in the idle loop, during entry to or exit + from user-mode execution, or on an offlined CPU? If so, SRCU + and RCU Tasks Trace are the only choices that will work for you, + with SRCU being strongly preferred in almost all cases. -g. Otherwise, use RCU. +h. Otherwise, use RCU. Of course, this all assumes that you have determined that RCU is in fact the right tool for your job. From 7a21ddf01af9da0a2d24d6ece0e2ffd60374a945 Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Sat, 19 Nov 2022 17:25:07 +0800 Subject: [PATCH 073/239] doc: Document CONFIG_RCU_CPU_STALL_CPUTIME=y stall information This commit documents the additional RCU CPU stall warning output produced by kernels built with CONFIG_RCU_CPU_STALL_CPUTIME=y or booted with rcupdate.rcu_cpu_stall_cputime=1. [ paulmck: Apply wordsmithing. ] Signed-off-by: Zhen Lei Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- Documentation/RCU/stallwarn.rst | 88 +++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/Documentation/RCU/stallwarn.rst b/Documentation/RCU/stallwarn.rst index dfa4db8c0931..c1e92dfef40d 100644 --- a/Documentation/RCU/stallwarn.rst +++ b/Documentation/RCU/stallwarn.rst @@ -390,3 +390,91 @@ for example, "P3421". It is entirely possible to see stall warnings from normal and from expedited grace periods at about the same time during the same run. + +RCU_CPU_STALL_CPUTIME +===================== + +In kernels built with CONFIG_RCU_CPU_STALL_CPUTIME=y or booted with +rcupdate.rcu_cpu_stall_cputime=1, the following additional information +is supplied with each RCU CPU stall warning:: + +rcu: hardirqs softirqs csw/system +rcu: number: 624 45 0 +rcu: cputime: 69 1 2425 ==> 2500(ms) + +These statistics are collected during the sampling period. The values +in row "number:" are the number of hard interrupts, number of soft +interrupts, and number of context switches on the stalled CPU. The +first three values in row "cputime:" indicate the CPU time in +milliseconds consumed by hard interrupts, soft interrupts, and tasks +on the stalled CPU. The last number is the measurement interval, again +in milliseconds. Because user-mode tasks normally do not cause RCU CPU +stalls, these tasks are typically kernel tasks, which is why only the +system CPU time are considered. + +The sampling period is shown as follows: +:<------------first timeout---------->:<-----second timeout----->: +:<--half timeout-->:<--half timeout-->: : +: :<--first period-->: : +: :<-----------second sampling period---------->: +: : : : +: snapshot time point 1st-stall 2nd-stall + + +The following describes four typical scenarios: + +1. A CPU looping with interrupts disabled.:: + + rcu: hardirqs softirqs csw/system + rcu: number: 0 0 0 + rcu: cputime: 0 0 0 ==> 2500(ms) + + Because interrupts have been disabled throughout the measurement + interval, there are no interrupts and no context switches. + Furthermore, because CPU time consumption was measured using interrupt + handlers, the system CPU consumption is misleadingly measured as zero. + This scenario will normally also have "(0 ticks this GP)" printed on + this CPU's summary line. + +2. A CPU looping with bottom halves disabled. + + This is similar to the previous example, but with non-zero number of + and CPU time consumed by hard interrupts, along with non-zero CPU + time consumed by in-kernel execution.:: + + rcu: hardirqs softirqs csw/system + rcu: number: 624 0 0 + rcu: cputime: 49 0 2446 ==> 2500(ms) + + The fact that there are zero softirqs gives a hint that these were + disabled, perhaps via local_bh_disable(). It is of course possible + that there were no softirqs, perhaps because all events that would + result in softirq execution are confined to other CPUs. In this case, + the diagnosis should continue as shown in the next example. + +3. A CPU looping with preemption disabled. + + Here, only the number of context switches is zero.:: + + rcu: hardirqs softirqs csw/system + rcu: number: 624 45 0 + rcu: cputime: 69 1 2425 ==> 2500(ms) + + This situation hints that the stalled CPU was looping with preemption + disabled. + +4. No looping, but massive hard and soft interrupts.:: + + rcu: hardirqs softirqs csw/system + rcu: number: xx xx 0 + rcu: cputime: xx xx 0 ==> 2500(ms) + + Here, the number and CPU time of hard interrupts are all non-zero, + but the number of context switches and the in-kernel CPU time consumed + are zero. The number and cputime of soft interrupts will usually be + non-zero, but could be zero, for example, if the CPU was spinning + within a single hard interrupt handler. + + If this type of RCU CPU stall warning can be reproduced, you can + narrow it down by looking at /proc/interrupts or by writing code to + trace each interrupt, for example, by referring to show_interrupts(). From a75f7b487c2b0c1dde149b82b494f97fd068d014 Mon Sep 17 00:00:00 2001 From: Akira Yokosawa Date: Wed, 23 Nov 2022 18:23:09 +0900 Subject: [PATCH 074/239] docs/RCU/rcubarrier: Adjust 'Answer' parts of QQs as definition-lists The "Answer" parts of QQs divert from proper format of definition-lists as described at [1] and are not rendered as such. Adjust them. Link: [1] https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#definition-lists Signed-off-by: Akira Yokosawa Signed-off-by: Paul E. McKenney --- Documentation/RCU/rcubarrier.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Documentation/RCU/rcubarrier.rst b/Documentation/RCU/rcubarrier.rst index 5a643e5233d5..9fb9ed777355 100644 --- a/Documentation/RCU/rcubarrier.rst +++ b/Documentation/RCU/rcubarrier.rst @@ -296,7 +296,8 @@ Quick Quiz #1: Is there any other situation where rcu_barrier() might be required? -Answer: Interestingly enough, rcu_barrier() was not originally +Answer: + Interestingly enough, rcu_barrier() was not originally implemented for module unloading. Nikita Danilov was using RCU in a filesystem, which resulted in a similar situation at filesystem-unmount time. Dipankar Sarma coded up rcu_barrier() @@ -315,7 +316,8 @@ Quick Quiz #2: Why doesn't line 8 initialize rcu_barrier_cpu_count to zero, thereby avoiding the need for lines 9 and 10? -Answer: Suppose that the on_each_cpu() function shown on line 8 was +Answer: + Suppose that the on_each_cpu() function shown on line 8 was delayed, so that CPU 0's rcu_barrier_func() executed and the corresponding grace period elapsed, all before CPU 1's rcu_barrier_func() started executing. This would result in @@ -351,7 +353,8 @@ Quick Quiz #3: are delayed for a full grace period? Couldn't this result in rcu_barrier() returning prematurely? -Answer: This cannot happen. The reason is that on_each_cpu() has its last +Answer: + This cannot happen. The reason is that on_each_cpu() has its last argument, the wait flag, set to "1". This flag is passed through to smp_call_function() and further to smp_call_function_on_cpu(), causing this latter to spin until the cross-CPU invocation of From eff864590bd57caeb8544b3f57bd3ab202efe1c2 Mon Sep 17 00:00:00 2001 From: Akira Yokosawa Date: Wed, 23 Nov 2022 18:29:00 +0900 Subject: [PATCH 075/239] docs/RCU/rcubarrier: Right-adjust line numbers in code snippets Line numbers in code snippets in rcubarrier.rst have beed left adjusted since commit 4af498306ffd ("doc: Convert to rcubarrier.txt to ReST"). This might have been because right adjusting them had confused Sphinx. The rules around a literal block in reST are: - Need a blank line above it. - A line with the same indent level as the line above it is regarded as the end of it. Those line numbers can be right adjusted by keeping indents at two- digit numbers. While at it, add some spaces between the column of line numbers and the code area for better readability. Signed-off-by: Akira Yokosawa Signed-off-by: Paul E. McKenney --- Documentation/RCU/rcubarrier.rst | 168 +++++++++++++++---------------- 1 file changed, 84 insertions(+), 84 deletions(-) diff --git a/Documentation/RCU/rcubarrier.rst b/Documentation/RCU/rcubarrier.rst index 9fb9ed777355..6da7f66da2a8 100644 --- a/Documentation/RCU/rcubarrier.rst +++ b/Documentation/RCU/rcubarrier.rst @@ -72,9 +72,9 @@ For example, if it uses call_rcu(), call_srcu() on srcu_struct_1, and call_srcu() on srcu_struct_2, then the following three lines of code will be required when unloading:: - 1 rcu_barrier(); - 2 srcu_barrier(&srcu_struct_1); - 3 srcu_barrier(&srcu_struct_2); + 1 rcu_barrier(); + 2 srcu_barrier(&srcu_struct_1); + 3 srcu_barrier(&srcu_struct_2); If latency is of the essence, workqueues could be used to run these three functions concurrently. @@ -82,69 +82,69 @@ three functions concurrently. An ancient version of the rcutorture module makes use of rcu_barrier() in its exit function as follows:: - 1 static void - 2 rcu_torture_cleanup(void) - 3 { - 4 int i; - 5 - 6 fullstop = 1; - 7 if (shuffler_task != NULL) { - 8 VERBOSE_PRINTK_STRING("Stopping rcu_torture_shuffle task"); - 9 kthread_stop(shuffler_task); - 10 } - 11 shuffler_task = NULL; + 1 static void + 2 rcu_torture_cleanup(void) + 3 { + 4 int i; + 5 + 6 fullstop = 1; + 7 if (shuffler_task != NULL) { + 8 VERBOSE_PRINTK_STRING("Stopping rcu_torture_shuffle task"); + 9 kthread_stop(shuffler_task); + 10 } + 11 shuffler_task = NULL; 12 - 13 if (writer_task != NULL) { - 14 VERBOSE_PRINTK_STRING("Stopping rcu_torture_writer task"); - 15 kthread_stop(writer_task); - 16 } - 17 writer_task = NULL; + 13 if (writer_task != NULL) { + 14 VERBOSE_PRINTK_STRING("Stopping rcu_torture_writer task"); + 15 kthread_stop(writer_task); + 16 } + 17 writer_task = NULL; 18 - 19 if (reader_tasks != NULL) { - 20 for (i = 0; i < nrealreaders; i++) { - 21 if (reader_tasks[i] != NULL) { - 22 VERBOSE_PRINTK_STRING( - 23 "Stopping rcu_torture_reader task"); - 24 kthread_stop(reader_tasks[i]); - 25 } - 26 reader_tasks[i] = NULL; - 27 } - 28 kfree(reader_tasks); - 29 reader_tasks = NULL; - 30 } - 31 rcu_torture_current = NULL; + 19 if (reader_tasks != NULL) { + 20 for (i = 0; i < nrealreaders; i++) { + 21 if (reader_tasks[i] != NULL) { + 22 VERBOSE_PRINTK_STRING( + 23 "Stopping rcu_torture_reader task"); + 24 kthread_stop(reader_tasks[i]); + 25 } + 26 reader_tasks[i] = NULL; + 27 } + 28 kfree(reader_tasks); + 29 reader_tasks = NULL; + 30 } + 31 rcu_torture_current = NULL; 32 - 33 if (fakewriter_tasks != NULL) { - 34 for (i = 0; i < nfakewriters; i++) { - 35 if (fakewriter_tasks[i] != NULL) { - 36 VERBOSE_PRINTK_STRING( - 37 "Stopping rcu_torture_fakewriter task"); - 38 kthread_stop(fakewriter_tasks[i]); - 39 } - 40 fakewriter_tasks[i] = NULL; - 41 } - 42 kfree(fakewriter_tasks); - 43 fakewriter_tasks = NULL; - 44 } + 33 if (fakewriter_tasks != NULL) { + 34 for (i = 0; i < nfakewriters; i++) { + 35 if (fakewriter_tasks[i] != NULL) { + 36 VERBOSE_PRINTK_STRING( + 37 "Stopping rcu_torture_fakewriter task"); + 38 kthread_stop(fakewriter_tasks[i]); + 39 } + 40 fakewriter_tasks[i] = NULL; + 41 } + 42 kfree(fakewriter_tasks); + 43 fakewriter_tasks = NULL; + 44 } 45 - 46 if (stats_task != NULL) { - 47 VERBOSE_PRINTK_STRING("Stopping rcu_torture_stats task"); - 48 kthread_stop(stats_task); - 49 } - 50 stats_task = NULL; + 46 if (stats_task != NULL) { + 47 VERBOSE_PRINTK_STRING("Stopping rcu_torture_stats task"); + 48 kthread_stop(stats_task); + 49 } + 50 stats_task = NULL; 51 - 52 /* Wait for all RCU callbacks to fire. */ - 53 rcu_barrier(); + 52 /* Wait for all RCU callbacks to fire. */ + 53 rcu_barrier(); 54 - 55 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */ + 55 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */ 56 - 57 if (cur_ops->cleanup != NULL) - 58 cur_ops->cleanup(); - 59 if (atomic_read(&n_rcu_torture_error)) - 60 rcu_torture_print_module_parms("End of test: FAILURE"); - 61 else - 62 rcu_torture_print_module_parms("End of test: SUCCESS"); - 63 } + 57 if (cur_ops->cleanup != NULL) + 58 cur_ops->cleanup(); + 59 if (atomic_read(&n_rcu_torture_error)) + 60 rcu_torture_print_module_parms("End of test: FAILURE"); + 61 else + 62 rcu_torture_print_module_parms("End of test: SUCCESS"); + 63 } Line 6 sets a global variable that prevents any RCU callbacks from re-posting themselves. This will not be necessary in most cases, since @@ -193,16 +193,16 @@ which point, all earlier RCU callbacks are guaranteed to have completed. The original code for rcu_barrier() was roughly as follows:: - 1 void rcu_barrier(void) - 2 { - 3 BUG_ON(in_interrupt()); - 4 /* Take cpucontrol mutex to protect against CPU hotplug */ - 5 mutex_lock(&rcu_barrier_mutex); - 6 init_completion(&rcu_barrier_completion); - 7 atomic_set(&rcu_barrier_cpu_count, 1); - 8 on_each_cpu(rcu_barrier_func, NULL, 0, 1); - 9 if (atomic_dec_and_test(&rcu_barrier_cpu_count)) - 10 complete(&rcu_barrier_completion); + 1 void rcu_barrier(void) + 2 { + 3 BUG_ON(in_interrupt()); + 4 /* Take cpucontrol mutex to protect against CPU hotplug */ + 5 mutex_lock(&rcu_barrier_mutex); + 6 init_completion(&rcu_barrier_completion); + 7 atomic_set(&rcu_barrier_cpu_count, 1); + 8 on_each_cpu(rcu_barrier_func, NULL, 0, 1); + 9 if (atomic_dec_and_test(&rcu_barrier_cpu_count)) + 10 complete(&rcu_barrier_completion); 11 wait_for_completion(&rcu_barrier_completion); 12 mutex_unlock(&rcu_barrier_mutex); 13 } @@ -232,16 +232,16 @@ still gives the general idea. The rcu_barrier_func() runs on each CPU, where it invokes call_rcu() to post an RCU callback, as follows:: - 1 static void rcu_barrier_func(void *notused) - 2 { - 3 int cpu = smp_processor_id(); - 4 struct rcu_data *rdp = &per_cpu(rcu_data, cpu); - 5 struct rcu_head *head; - 6 - 7 head = &rdp->barrier; - 8 atomic_inc(&rcu_barrier_cpu_count); - 9 call_rcu(head, rcu_barrier_callback); - 10 } + 1 static void rcu_barrier_func(void *notused) + 2 { + 3 int cpu = smp_processor_id(); + 4 struct rcu_data *rdp = &per_cpu(rcu_data, cpu); + 5 struct rcu_head *head; + 6 + 7 head = &rdp->barrier; + 8 atomic_inc(&rcu_barrier_cpu_count); + 9 call_rcu(head, rcu_barrier_callback); + 10 } Lines 3 and 4 locate RCU's internal per-CPU rcu_data structure, which contains the struct rcu_head that needed for the later call to @@ -254,11 +254,11 @@ The rcu_barrier_callback() function simply atomically decrements the rcu_barrier_cpu_count variable and finalizes the completion when it reaches zero, as follows:: - 1 static void rcu_barrier_callback(struct rcu_head *notused) - 2 { - 3 if (atomic_dec_and_test(&rcu_barrier_cpu_count)) - 4 complete(&rcu_barrier_completion); - 5 } + 1 static void rcu_barrier_callback(struct rcu_head *notused) + 2 { + 3 if (atomic_dec_and_test(&rcu_barrier_cpu_count)) + 4 complete(&rcu_barrier_completion); + 5 } .. _rcubarrier_quiz_3: From 5e013dc17e2ab2d59e85a4322e37398ca80671d6 Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Thu, 24 Nov 2022 14:22:03 +0800 Subject: [PATCH 076/239] doc: Fix htmldocs build warnings of stallwarn.rst Documentation/RCU/stallwarn.rst: 401: WARNING: Literal block expected; none found. 428: WARNING: Literal block expected; none found. 445: WARNING: Literal block expected; none found. 459: WARNING: Literal block expected; none found. 468: WARNING: Literal block expected; none found. The literal block needs to be indented, so this commit adds two spaces to each line. In addition, ':', which is used as a boundary in the literal block, is replaced by '|'. Link: https://lore.kernel.org/linux-next/20221123163255.48653674@canb.auug.org.au/ Fixes: 3d2788ba4573 ("doc: Document CONFIG_RCU_CPU_STALL_CPUTIME=y stall information") Reported-by: Stephen Rothwell Signed-off-by: Zhen Lei Tested-by: Akira Yokosawa Signed-off-by: Paul E. McKenney --- Documentation/RCU/stallwarn.rst | 56 ++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/Documentation/RCU/stallwarn.rst b/Documentation/RCU/stallwarn.rst index c1e92dfef40d..ca7b7cd806a1 100644 --- a/Documentation/RCU/stallwarn.rst +++ b/Documentation/RCU/stallwarn.rst @@ -398,9 +398,9 @@ In kernels built with CONFIG_RCU_CPU_STALL_CPUTIME=y or booted with rcupdate.rcu_cpu_stall_cputime=1, the following additional information is supplied with each RCU CPU stall warning:: -rcu: hardirqs softirqs csw/system -rcu: number: 624 45 0 -rcu: cputime: 69 1 2425 ==> 2500(ms) + rcu: hardirqs softirqs csw/system + rcu: number: 624 45 0 + rcu: cputime: 69 1 2425 ==> 2500(ms) These statistics are collected during the sampling period. The values in row "number:" are the number of hard interrupts, number of soft @@ -412,22 +412,24 @@ in milliseconds. Because user-mode tasks normally do not cause RCU CPU stalls, these tasks are typically kernel tasks, which is why only the system CPU time are considered. -The sampling period is shown as follows: -:<------------first timeout---------->:<-----second timeout----->: -:<--half timeout-->:<--half timeout-->: : -: :<--first period-->: : -: :<-----------second sampling period---------->: -: : : : -: snapshot time point 1st-stall 2nd-stall +The sampling period is shown as follows:: + |<------------first timeout---------->|<-----second timeout----->| + |<--half timeout-->|<--half timeout-->| | + | |<--first period-->| | + | |<-----------second sampling period---------->| + | | | | + snapshot time point 1st-stall 2nd-stall The following describes four typical scenarios: -1. A CPU looping with interrupts disabled.:: +1. A CPU looping with interrupts disabled. - rcu: hardirqs softirqs csw/system - rcu: number: 0 0 0 - rcu: cputime: 0 0 0 ==> 2500(ms) + :: + + rcu: hardirqs softirqs csw/system + rcu: number: 0 0 0 + rcu: cputime: 0 0 0 ==> 2500(ms) Because interrupts have been disabled throughout the measurement interval, there are no interrupts and no context switches. @@ -440,11 +442,11 @@ The following describes four typical scenarios: This is similar to the previous example, but with non-zero number of and CPU time consumed by hard interrupts, along with non-zero CPU - time consumed by in-kernel execution.:: + time consumed by in-kernel execution:: - rcu: hardirqs softirqs csw/system - rcu: number: 624 0 0 - rcu: cputime: 49 0 2446 ==> 2500(ms) + rcu: hardirqs softirqs csw/system + rcu: number: 624 0 0 + rcu: cputime: 49 0 2446 ==> 2500(ms) The fact that there are zero softirqs gives a hint that these were disabled, perhaps via local_bh_disable(). It is of course possible @@ -454,20 +456,22 @@ The following describes four typical scenarios: 3. A CPU looping with preemption disabled. - Here, only the number of context switches is zero.:: + Here, only the number of context switches is zero:: - rcu: hardirqs softirqs csw/system - rcu: number: 624 45 0 - rcu: cputime: 69 1 2425 ==> 2500(ms) + rcu: hardirqs softirqs csw/system + rcu: number: 624 45 0 + rcu: cputime: 69 1 2425 ==> 2500(ms) This situation hints that the stalled CPU was looping with preemption disabled. -4. No looping, but massive hard and soft interrupts.:: +4. No looping, but massive hard and soft interrupts. - rcu: hardirqs softirqs csw/system - rcu: number: xx xx 0 - rcu: cputime: xx xx 0 ==> 2500(ms) + :: + + rcu: hardirqs softirqs csw/system + rcu: number: xx xx 0 + rcu: cputime: xx xx 0 ==> 2500(ms) Here, the number and CPU time of hard interrupts are all non-zero, but the number of context switches and the in-kernel CPU time consumed From b2ba00c2a51793d916b662cb049b8f01c55d9e82 Mon Sep 17 00:00:00 2001 From: Stephen Rothwell Date: Thu, 5 Jan 2023 12:05:50 -0800 Subject: [PATCH 077/239] rxrpc: replace zero-lenth array with DECLARE_FLEX_ARRAY() helper 0-length arrays are deprecated, and cause problems with bounds checking. Replace with a flexible array: In file included from include/linux/string.h:253, from include/linux/bitmap.h:11, from include/linux/cpumask.h:12, from arch/x86/include/asm/paravirt.h:17, from arch/x86/include/asm/cpuid.h:62, from arch/x86/include/asm/processor.h:19, from arch/x86/include/asm/cpufeature.h:5, from arch/x86/include/asm/thread_info.h:53, from include/linux/thread_info.h:60, from arch/x86/include/asm/preempt.h:9, from include/linux/preempt.h:78, from include/linux/percpu.h:6, from include/linux/prandom.h:13, from include/linux/random.h:153, from include/linux/net.h:18, from net/rxrpc/output.c:10: In function 'fortify_memcpy_chk', inlined from 'rxrpc_fill_out_ack' at net/rxrpc/output.c:158:2: include/linux/fortify-string.h:520:25: error: call to '__write_overflow_field' declared with attribute warning: detected write beyond size of field (1st parameter); maybe use struct_group()? [-Werror=attribute-warning] 520 | __write_overflow_field(p_size_field, size); | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Link: https://lore.kernel.org/linux-next/20230105132535.0d65378f@canb.auug.org.au/ Cc: David Howells Cc: Marc Dionne Cc: "David S. Miller" Cc: Eric Dumazet Cc: Jakub Kicinski Cc: Paolo Abeni Cc: linux-afs@lists.infradead.org Cc: netdev@vger.kernel.org Signed-off-by: Stephen Rothwell Signed-off-by: Kees Cook --- net/rxrpc/ar-internal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/rxrpc/ar-internal.h b/net/rxrpc/ar-internal.h index 18092526d3c8..c7186484fc5f 100644 --- a/net/rxrpc/ar-internal.h +++ b/net/rxrpc/ar-internal.h @@ -783,7 +783,7 @@ struct rxrpc_txbuf { u8 data[RXRPC_JUMBO_DATALEN]; /* Data packet */ struct { struct rxrpc_ackpacket ack; - u8 acks[0]; + DECLARE_FLEX_ARRAY(u8, acks); }; }; } __aligned(64); From 439a1bcac648fe9b59210cde8991fb2acf37bdab Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Mon, 19 Sep 2022 15:53:13 -0700 Subject: [PATCH 078/239] fortify: Use __builtin_dynamic_object_size() when available Since the commits starting with c37495d6254c ("slab: add __alloc_size attributes for better bounds checking"), the compilers have runtime allocation size hints available in some places. This was immediately available to CONFIG_UBSAN_BOUNDS, but CONFIG_FORTIFY_SOURCE needed updating to explicitly make use of the hints via the associated __builtin_dynamic_object_size() helper. Detect and use the builtin when it is available, increasing the accuracy of the mitigation. When runtime sizes are not available, __builtin_dynamic_object_size() falls back to __builtin_object_size(), leaving the existing bounds checking unchanged. Additionally update the VMALLOC_LINEAR_OVERFLOW LKDTM test to make the hint invisible, otherwise the architectural defense is not exercised (the buffer overflow is detected in the memset() rather than when it crosses the edge of the allocation). Cc: Arnd Bergmann Cc: Greg Kroah-Hartman Cc: Nick Desaulniers Cc: Nathan Chancellor Cc: Tom Rix Cc: linux-hardening@vger.kernel.org Cc: llvm@lists.linux.dev Reviewed-by: Miguel Ojeda # include/linux/compiler_attributes.h Signed-off-by: Kees Cook --- drivers/misc/lkdtm/heap.c | 1 + include/linux/compiler_attributes.h | 5 +++++ include/linux/fortify-string.h | 7 +++++++ 3 files changed, 13 insertions(+) diff --git a/drivers/misc/lkdtm/heap.c b/drivers/misc/lkdtm/heap.c index 62516078a619..0ce4cbf6abda 100644 --- a/drivers/misc/lkdtm/heap.c +++ b/drivers/misc/lkdtm/heap.c @@ -31,6 +31,7 @@ static void lkdtm_VMALLOC_LINEAR_OVERFLOW(void) char *one, *two; one = vzalloc(PAGE_SIZE); + OPTIMIZER_HIDE_VAR(one); two = vzalloc(PAGE_SIZE); pr_info("Attempting vmalloc linear overflow ...\n"); diff --git a/include/linux/compiler_attributes.h b/include/linux/compiler_attributes.h index 898b3458b24a..56467f86a27c 100644 --- a/include/linux/compiler_attributes.h +++ b/include/linux/compiler_attributes.h @@ -297,6 +297,11 @@ * * clang: https://clang.llvm.org/docs/AttributeReference.html#pass-object-size-pass-dynamic-object-size */ +#if __has_attribute(__pass_dynamic_object_size__) +# define __pass_dynamic_object_size(type) __attribute__((__pass_dynamic_object_size__(type))) +#else +# define __pass_dynamic_object_size(type) +#endif #if __has_attribute(__pass_object_size__) # define __pass_object_size(type) __attribute__((__pass_object_size__(type))) #else diff --git a/include/linux/fortify-string.h b/include/linux/fortify-string.h index 7cad8bb031e9..c9de1f59ee80 100644 --- a/include/linux/fortify-string.h +++ b/include/linux/fortify-string.h @@ -90,10 +90,17 @@ extern char *__underlying_strncpy(char *p, const char *q, __kernel_size_t size) * size, rather than struct size), but there remain some stragglers using * type 0 that will be converted in the future. */ +#if __has_builtin(__builtin_dynamic_object_size) +#define POS __pass_dynamic_object_size(1) +#define POS0 __pass_dynamic_object_size(0) +#define __struct_size(p) __builtin_dynamic_object_size(p, 0) +#define __member_size(p) __builtin_dynamic_object_size(p, 1) +#else #define POS __pass_object_size(1) #define POS0 __pass_object_size(0) #define __struct_size(p) __builtin_object_size(p, 0) #define __member_size(p) __builtin_object_size(p, 1) +#endif #define __compiletime_lessthan(bounds, length) ( \ __builtin_constant_p((bounds) < (length)) && \ From a6889becb05394255c80b62103677e3b095726a9 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 8 Nov 2022 08:18:06 -0800 Subject: [PATCH 079/239] refscale: Add tests using SLAB_TYPESAFE_BY_RCU This commit adds three read-side-only tests of three use cases featuring SLAB_TYPESAFE_BY_RCU: One using per-object reference counting, one using per-object locking, and one using per-object sequence locking. [ paulmck: Apply feedback from kernel test robot. ] Signed-off-by: Paul E. McKenney --- kernel/rcu/refscale.c | 234 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 234 insertions(+) diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c index 7f12168627a1..afa3e1a2f690 100644 --- a/kernel/rcu/refscale.c +++ b/kernel/rcu/refscale.c @@ -76,6 +76,8 @@ torture_param(int, verbose_batched, 0, "Batch verbose debugging printk()s"); // Wait until there are multiple CPUs before starting test. torture_param(int, holdoff, IS_BUILTIN(CONFIG_RCU_REF_SCALE_TEST) ? 10 : 0, "Holdoff time before test start (s)"); +// Number of typesafe_lookup structures, that is, the degree of concurrency. +torture_param(long, lookup_instances, 0, "Number of typesafe_lookup structures."); // Number of loops per experiment, all readers execute operations concurrently. torture_param(long, loops, 10000, "Number of loops per experiment."); // Number of readers, with -1 defaulting to about 75% of the CPUs. @@ -526,6 +528,237 @@ static struct ref_scale_ops clock_ops = { .name = "clock" }; +//////////////////////////////////////////////////////////////////////// +// +// Methods leveraging SLAB_TYPESAFE_BY_RCU. +// + +// Item to look up in a typesafe manner. Array of pointers to these. +struct refscale_typesafe { + atomic_t rts_refctr; // Used by all flavors + spinlock_t rts_lock; + seqlock_t rts_seqlock; + unsigned int a; + unsigned int b; +}; + +static struct kmem_cache *typesafe_kmem_cachep; +static struct refscale_typesafe **rtsarray; +static long rtsarray_size; +static DEFINE_TORTURE_RANDOM_PERCPU(refscale_rand); +static bool (*rts_acquire)(struct refscale_typesafe *rtsp, unsigned int *start); +static bool (*rts_release)(struct refscale_typesafe *rtsp, unsigned int start); + +// Conditionally acquire an explicit in-structure reference count. +static bool typesafe_ref_acquire(struct refscale_typesafe *rtsp, unsigned int *start) +{ + return atomic_inc_not_zero(&rtsp->rts_refctr); +} + +// Unconditionally release an explicit in-structure reference count. +static bool typesafe_ref_release(struct refscale_typesafe *rtsp, unsigned int start) +{ + if (!atomic_dec_return(&rtsp->rts_refctr)) { + WRITE_ONCE(rtsp->a, rtsp->a + 1); + kmem_cache_free(typesafe_kmem_cachep, rtsp); + } + return true; +} + +// Unconditionally acquire an explicit in-structure spinlock. +static bool typesafe_lock_acquire(struct refscale_typesafe *rtsp, unsigned int *start) +{ + spin_lock(&rtsp->rts_lock); + return true; +} + +// Unconditionally release an explicit in-structure spinlock. +static bool typesafe_lock_release(struct refscale_typesafe *rtsp, unsigned int start) +{ + spin_unlock(&rtsp->rts_lock); + return true; +} + +// Unconditionally acquire an explicit in-structure sequence lock. +static bool typesafe_seqlock_acquire(struct refscale_typesafe *rtsp, unsigned int *start) +{ + *start = read_seqbegin(&rtsp->rts_seqlock); + return true; +} + +// Conditionally release an explicit in-structure sequence lock. Return +// true if this release was successful, that is, if no retry is required. +static bool typesafe_seqlock_release(struct refscale_typesafe *rtsp, unsigned int start) +{ + return !read_seqretry(&rtsp->rts_seqlock, start); +} + +// Do a read-side critical section with the specified delay in +// microseconds and nanoseconds inserted so as to increase probability +// of failure. +static void typesafe_delay_section(const int nloops, const int udl, const int ndl) +{ + unsigned int a; + unsigned int b; + int i; + long idx; + struct refscale_typesafe *rtsp; + unsigned int start; + + for (i = nloops; i >= 0; i--) { + preempt_disable(); + idx = torture_random(this_cpu_ptr(&refscale_rand)) % rtsarray_size; + preempt_enable(); +retry: + rcu_read_lock(); + rtsp = rcu_dereference(rtsarray[idx]); + a = READ_ONCE(rtsp->a); + if (!rts_acquire(rtsp, &start)) { + rcu_read_unlock(); + goto retry; + } + if (a != READ_ONCE(rtsp->a)) { + (void)rts_release(rtsp, start); + rcu_read_unlock(); + goto retry; + } + un_delay(udl, ndl); + // Remember, seqlock read-side release can fail. + if (!rts_release(rtsp, start)) { + rcu_read_unlock(); + goto retry; + } + b = READ_ONCE(rtsp->a); + WARN_ONCE(a != b, "Re-read of ->a changed from %u to %u.\n", a, b); + b = rtsp->b; + rcu_read_unlock(); + WARN_ON_ONCE(a * a != b); + } +} + +// Because the acquisition and release methods are expensive, there +// is no point in optimizing away the un_delay() function's two checks. +// Thus simply define typesafe_read_section() as a simple wrapper around +// typesafe_delay_section(). +static void typesafe_read_section(const int nloops) +{ + typesafe_delay_section(nloops, 0, 0); +} + +// Allocate and initialize one refscale_typesafe structure. +static struct refscale_typesafe *typesafe_alloc_one(void) +{ + struct refscale_typesafe *rtsp; + + rtsp = kmem_cache_alloc(typesafe_kmem_cachep, GFP_KERNEL); + if (!rtsp) + return NULL; + atomic_set(&rtsp->rts_refctr, 1); + WRITE_ONCE(rtsp->a, rtsp->a + 1); + WRITE_ONCE(rtsp->b, rtsp->a * rtsp->a); + return rtsp; +} + +// Slab-allocator constructor for refscale_typesafe structures created +// out of a new slab of system memory. +static void refscale_typesafe_ctor(void *rtsp_in) +{ + struct refscale_typesafe *rtsp = rtsp_in; + + spin_lock_init(&rtsp->rts_lock); + seqlock_init(&rtsp->rts_seqlock); + preempt_disable(); + rtsp->a = torture_random(this_cpu_ptr(&refscale_rand)); + preempt_enable(); +} + +static struct ref_scale_ops typesafe_ref_ops; +static struct ref_scale_ops typesafe_lock_ops; +static struct ref_scale_ops typesafe_seqlock_ops; + +// Initialize for a typesafe test. +static bool typesafe_init(void) +{ + long idx; + long si = lookup_instances; + + typesafe_kmem_cachep = kmem_cache_create("refscale_typesafe", + sizeof(struct refscale_typesafe), sizeof(void *), + SLAB_TYPESAFE_BY_RCU, refscale_typesafe_ctor); + if (!typesafe_kmem_cachep) + return false; + if (si < 0) + si = -si * nr_cpu_ids; + else if (si == 0) + si = nr_cpu_ids; + rtsarray_size = si; + rtsarray = kcalloc(si, sizeof(*rtsarray), GFP_KERNEL); + if (!rtsarray) + return false; + for (idx = 0; idx < rtsarray_size; idx++) { + rtsarray[idx] = typesafe_alloc_one(); + if (!rtsarray[idx]) + return false; + } + if (cur_ops == &typesafe_ref_ops) { + rts_acquire = typesafe_ref_acquire; + rts_release = typesafe_ref_release; + } else if (cur_ops == &typesafe_lock_ops) { + rts_acquire = typesafe_lock_acquire; + rts_release = typesafe_lock_release; + } else if (cur_ops == &typesafe_seqlock_ops) { + rts_acquire = typesafe_seqlock_acquire; + rts_release = typesafe_seqlock_release; + } else { + WARN_ON_ONCE(1); + return false; + } + return true; +} + +// Clean up after a typesafe test. +static void typesafe_cleanup(void) +{ + long idx; + + if (rtsarray) { + for (idx = 0; idx < rtsarray_size; idx++) + kmem_cache_free(typesafe_kmem_cachep, rtsarray[idx]); + kfree(rtsarray); + rtsarray = NULL; + rtsarray_size = 0; + } + kmem_cache_destroy(typesafe_kmem_cachep); + typesafe_kmem_cachep = NULL; + rts_acquire = NULL; + rts_release = NULL; +} + +// The typesafe_init() function distinguishes these structures by address. +static struct ref_scale_ops typesafe_ref_ops = { + .init = typesafe_init, + .cleanup = typesafe_cleanup, + .readsection = typesafe_read_section, + .delaysection = typesafe_delay_section, + .name = "typesafe_ref" +}; + +static struct ref_scale_ops typesafe_lock_ops = { + .init = typesafe_init, + .cleanup = typesafe_cleanup, + .readsection = typesafe_read_section, + .delaysection = typesafe_delay_section, + .name = "typesafe_lock" +}; + +static struct ref_scale_ops typesafe_seqlock_ops = { + .init = typesafe_init, + .cleanup = typesafe_cleanup, + .readsection = typesafe_read_section, + .delaysection = typesafe_delay_section, + .name = "typesafe_seqlock" +}; + static void rcu_scale_one_reader(void) { if (readdelay <= 0) @@ -815,6 +1048,7 @@ ref_scale_init(void) static struct ref_scale_ops *scale_ops[] = { &rcu_ops, &srcu_ops, RCU_TRACE_OPS RCU_TASKS_OPS &refcnt_ops, &rwlock_ops, &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops, &clock_ops, + &typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops, }; if (!torture_init_begin(scale_type, verbose)) From e01f3a1a589e314cf27bd2cb27d9c2c58e105a27 Mon Sep 17 00:00:00 2001 From: "Joel Fernandes (Google)" Date: Tue, 13 Dec 2022 20:48:38 +0000 Subject: [PATCH 080/239] locktorture: Allow non-rtmutex lock types to be boosted Currently RT boosting is only done for rtmutex_lock, however with proxy execution, we also have the mutex_lock participating in priorities. To exercise the testing better, add RT boosting to other lock testing types as well, using a new knob (rt_boost). Tested with boot parameters: locktorture.torture_type=mutex_lock locktorture.onoff_interval=1 locktorture.nwriters_stress=8 locktorture.stutter=0 locktorture.rt_boost=1 locktorture.rt_boost_factor=1 locktorture.nlocks=3 Signed-off-by: Joel Fernandes (Google) Signed-off-by: Paul E. McKenney --- kernel/locking/locktorture.c | 99 ++++++++++++++++++++---------------- 1 file changed, 56 insertions(+), 43 deletions(-) diff --git a/kernel/locking/locktorture.c b/kernel/locking/locktorture.c index 9c2fb613a55d..e2271e8fc302 100644 --- a/kernel/locking/locktorture.c +++ b/kernel/locking/locktorture.c @@ -46,6 +46,8 @@ torture_param(int, shutdown_secs, 0, "Shutdown time (j), <= zero to disable."); torture_param(int, stat_interval, 60, "Number of seconds between stats printk()s"); torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable"); +torture_param(int, rt_boost, 2, + "Do periodic rt-boost. 0=Disable, 1=Only for rt_mutex, 2=For all lock types."); torture_param(int, verbose, 1, "Enable verbose debugging printk()s"); @@ -127,15 +129,49 @@ static void torture_lock_busted_write_unlock(int tid __maybe_unused) /* BUGGY, do not use in real life!!! */ } -static void torture_boost_dummy(struct torture_random_state *trsp) +static void __torture_rt_boost(struct torture_random_state *trsp) { - /* Only rtmutexes care about priority */ + const unsigned int factor = 50000; /* yes, quite arbitrary */ + + if (!rt_task(current)) { + /* + * Boost priority once every ~50k operations. When the + * task tries to take the lock, the rtmutex it will account + * for the new priority, and do any corresponding pi-dance. + */ + if (trsp && !(torture_random(trsp) % + (cxt.nrealwriters_stress * factor))) { + sched_set_fifo(current); + } else /* common case, do nothing */ + return; + } else { + /* + * The task will remain boosted for another ~500k operations, + * then restored back to its original prio, and so forth. + * + * When @trsp is nil, we want to force-reset the task for + * stopping the kthread. + */ + if (!trsp || !(torture_random(trsp) % + (cxt.nrealwriters_stress * factor * 2))) { + sched_set_normal(current, 0); + } else /* common case, do nothing */ + return; + } +} + +static void torture_rt_boost(struct torture_random_state *trsp) +{ + if (rt_boost != 2) + return; + + __torture_rt_boost(trsp); } static struct lock_torture_ops lock_busted_ops = { .writelock = torture_lock_busted_write_lock, .write_delay = torture_lock_busted_write_delay, - .task_boost = torture_boost_dummy, + .task_boost = torture_rt_boost, .writeunlock = torture_lock_busted_write_unlock, .readlock = NULL, .read_delay = NULL, @@ -179,7 +215,7 @@ __releases(torture_spinlock) static struct lock_torture_ops spin_lock_ops = { .writelock = torture_spin_lock_write_lock, .write_delay = torture_spin_lock_write_delay, - .task_boost = torture_boost_dummy, + .task_boost = torture_rt_boost, .writeunlock = torture_spin_lock_write_unlock, .readlock = NULL, .read_delay = NULL, @@ -206,7 +242,7 @@ __releases(torture_spinlock) static struct lock_torture_ops spin_lock_irq_ops = { .writelock = torture_spin_lock_write_lock_irq, .write_delay = torture_spin_lock_write_delay, - .task_boost = torture_boost_dummy, + .task_boost = torture_rt_boost, .writeunlock = torture_lock_spin_write_unlock_irq, .readlock = NULL, .read_delay = NULL, @@ -275,7 +311,7 @@ __releases(torture_rwlock) static struct lock_torture_ops rw_lock_ops = { .writelock = torture_rwlock_write_lock, .write_delay = torture_rwlock_write_delay, - .task_boost = torture_boost_dummy, + .task_boost = torture_rt_boost, .writeunlock = torture_rwlock_write_unlock, .readlock = torture_rwlock_read_lock, .read_delay = torture_rwlock_read_delay, @@ -318,7 +354,7 @@ __releases(torture_rwlock) static struct lock_torture_ops rw_lock_irq_ops = { .writelock = torture_rwlock_write_lock_irq, .write_delay = torture_rwlock_write_delay, - .task_boost = torture_boost_dummy, + .task_boost = torture_rt_boost, .writeunlock = torture_rwlock_write_unlock_irq, .readlock = torture_rwlock_read_lock_irq, .read_delay = torture_rwlock_read_delay, @@ -358,7 +394,7 @@ __releases(torture_mutex) static struct lock_torture_ops mutex_lock_ops = { .writelock = torture_mutex_lock, .write_delay = torture_mutex_delay, - .task_boost = torture_boost_dummy, + .task_boost = torture_rt_boost, .writeunlock = torture_mutex_unlock, .readlock = NULL, .read_delay = NULL, @@ -456,7 +492,7 @@ static struct lock_torture_ops ww_mutex_lock_ops = { .exit = torture_ww_mutex_exit, .writelock = torture_ww_mutex_lock, .write_delay = torture_mutex_delay, - .task_boost = torture_boost_dummy, + .task_boost = torture_rt_boost, .writeunlock = torture_ww_mutex_unlock, .readlock = NULL, .read_delay = NULL, @@ -474,37 +510,6 @@ __acquires(torture_rtmutex) return 0; } -static void torture_rtmutex_boost(struct torture_random_state *trsp) -{ - const unsigned int factor = 50000; /* yes, quite arbitrary */ - - if (!rt_task(current)) { - /* - * Boost priority once every ~50k operations. When the - * task tries to take the lock, the rtmutex it will account - * for the new priority, and do any corresponding pi-dance. - */ - if (trsp && !(torture_random(trsp) % - (cxt.nrealwriters_stress * factor))) { - sched_set_fifo(current); - } else /* common case, do nothing */ - return; - } else { - /* - * The task will remain boosted for another ~500k operations, - * then restored back to its original prio, and so forth. - * - * When @trsp is nil, we want to force-reset the task for - * stopping the kthread. - */ - if (!trsp || !(torture_random(trsp) % - (cxt.nrealwriters_stress * factor * 2))) { - sched_set_normal(current, 0); - } else /* common case, do nothing */ - return; - } -} - static void torture_rtmutex_delay(struct torture_random_state *trsp) { const unsigned long shortdelay_us = 2; @@ -530,10 +535,18 @@ __releases(torture_rtmutex) rt_mutex_unlock(&torture_rtmutex); } +static void torture_rt_boost_rtmutex(struct torture_random_state *trsp) +{ + if (!rt_boost) + return; + + __torture_rt_boost(trsp); +} + static struct lock_torture_ops rtmutex_lock_ops = { .writelock = torture_rtmutex_lock, .write_delay = torture_rtmutex_delay, - .task_boost = torture_rtmutex_boost, + .task_boost = torture_rt_boost_rtmutex, .writeunlock = torture_rtmutex_unlock, .readlock = NULL, .read_delay = NULL, @@ -600,7 +613,7 @@ __releases(torture_rwsem) static struct lock_torture_ops rwsem_lock_ops = { .writelock = torture_rwsem_down_write, .write_delay = torture_rwsem_write_delay, - .task_boost = torture_boost_dummy, + .task_boost = torture_rt_boost, .writeunlock = torture_rwsem_up_write, .readlock = torture_rwsem_down_read, .read_delay = torture_rwsem_read_delay, @@ -652,7 +665,7 @@ static struct lock_torture_ops percpu_rwsem_lock_ops = { .exit = torture_percpu_rwsem_exit, .writelock = torture_percpu_rwsem_down_write, .write_delay = torture_rwsem_write_delay, - .task_boost = torture_boost_dummy, + .task_boost = torture_rt_boost, .writeunlock = torture_percpu_rwsem_up_write, .readlock = torture_percpu_rwsem_down_read, .read_delay = torture_rwsem_read_delay, From c24501b240741907ddfce52ffc186792db5ad3a5 Mon Sep 17 00:00:00 2001 From: "Joel Fernandes (Google)" Date: Tue, 13 Dec 2022 20:48:39 +0000 Subject: [PATCH 081/239] locktorture: Make the rt_boost factor a tunable The rt boosting in locktorture has a factor variable s currently large enough that boosting only happens once every minute or so. Add a tunable to reduce the factor so that boosting happens more often, to test paths and arrive at failure modes earlier. With this change, I can set the factor to like 50 and have the boosting happens every 10 seconds or so. Tested with boot parameters: locktorture.torture_type=mutex_lock locktorture.onoff_interval=1 locktorture.nwriters_stress=8 locktorture.stutter=0 locktorture.rt_boost=1 locktorture.rt_boost_factor=50 locktorture.nlocks=3 Signed-off-by: Joel Fernandes (Google) Reviewed-by: Davidlohr Bueso Signed-off-by: Paul E. McKenney --- kernel/locking/locktorture.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/kernel/locking/locktorture.c b/kernel/locking/locktorture.c index e2271e8fc302..f04b1978899d 100644 --- a/kernel/locking/locktorture.c +++ b/kernel/locking/locktorture.c @@ -48,6 +48,7 @@ torture_param(int, stat_interval, 60, torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable"); torture_param(int, rt_boost, 2, "Do periodic rt-boost. 0=Disable, 1=Only for rt_mutex, 2=For all lock types."); +torture_param(int, rt_boost_factor, 50, "A factor determining how often rt-boost happens."); torture_param(int, verbose, 1, "Enable verbose debugging printk()s"); @@ -131,12 +132,12 @@ static void torture_lock_busted_write_unlock(int tid __maybe_unused) static void __torture_rt_boost(struct torture_random_state *trsp) { - const unsigned int factor = 50000; /* yes, quite arbitrary */ + const unsigned int factor = rt_boost_factor; if (!rt_task(current)) { /* - * Boost priority once every ~50k operations. When the - * task tries to take the lock, the rtmutex it will account + * Boost priority once every rt_boost_factor operations. When + * the task tries to take the lock, the rtmutex it will account * for the new priority, and do any corresponding pi-dance. */ if (trsp && !(torture_random(trsp) % @@ -146,8 +147,9 @@ static void __torture_rt_boost(struct torture_random_state *trsp) return; } else { /* - * The task will remain boosted for another ~500k operations, - * then restored back to its original prio, and so forth. + * The task will remain boosted for another 10 * rt_boost_factor + * operations, then restored back to its original prio, and so + * forth. * * When @trsp is nil, we want to force-reset the task for * stopping the kthread. From 273661595c21f390d2522af06c7b552f907fe12a Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 16 Dec 2022 09:47:28 -0800 Subject: [PATCH 082/239] rcutorture: Drop sparse lock-acquisition annotations The sparse __acquires() and __releases() annotations provide very little value. The argument is ignored, so sparse cannot tell the differences between acquiring one lock and releasing another on the one hand and acquiring and releasing a given lock on the other. In addition, lockdep annotations provide much more precision, for but one example, actually knowing which lock is held. This commit therefore removes the __acquires() and __releases() annotations from rcutorture. Reported-by: Tejun Heo Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 634df26a2c27..8e6c023212cb 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -399,7 +399,7 @@ static int torture_readlock_not_held(void) return rcu_read_lock_bh_held() || rcu_read_lock_sched_held(); } -static int rcu_torture_read_lock(void) __acquires(RCU) +static int rcu_torture_read_lock(void) { rcu_read_lock(); return 0; @@ -441,7 +441,7 @@ rcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) } } -static void rcu_torture_read_unlock(int idx) __releases(RCU) +static void rcu_torture_read_unlock(int idx) { rcu_read_unlock(); } @@ -625,7 +625,7 @@ static struct srcu_struct srcu_ctld; static struct srcu_struct *srcu_ctlp = &srcu_ctl; static struct rcu_torture_ops srcud_ops; -static int srcu_torture_read_lock(void) __acquires(srcu_ctlp) +static int srcu_torture_read_lock(void) { if (cur_ops == &srcud_ops) return srcu_read_lock_nmisafe(srcu_ctlp); @@ -652,7 +652,7 @@ srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) } } -static void srcu_torture_read_unlock(int idx) __releases(srcu_ctlp) +static void srcu_torture_read_unlock(int idx) { if (cur_ops == &srcud_ops) srcu_read_unlock_nmisafe(srcu_ctlp, idx); @@ -814,13 +814,13 @@ static void synchronize_rcu_trivial(void) } } -static int rcu_torture_read_lock_trivial(void) __acquires(RCU) +static int rcu_torture_read_lock_trivial(void) { preempt_disable(); return 0; } -static void rcu_torture_read_unlock_trivial(int idx) __releases(RCU) +static void rcu_torture_read_unlock_trivial(int idx) { preempt_enable(); } From d52d3a2bf408ff86f3a79560b5cce80efb340239 Mon Sep 17 00:00:00 2001 From: "Joel Fernandes (Google)" Date: Sun, 1 Jan 2023 06:15:55 +0000 Subject: [PATCH 083/239] torture: Fix hang during kthread shutdown phase During rcutorture shutdown, the rcu_torture_cleanup() function calls torture_cleanup_begin(), which sets the fullstop global variable to FULLSTOP_RMMOD. This causes the rcutorture threads for readers and fakewriters to exit all of their "while" loops and start shutting down. They then call torture_kthread_stopping(), which in turn waits for kthread_stop() to be called. However, rcu_torture_cleanup() has not yet called kthread_stop() on those threads, and before it gets a chance to do so, multiple instances of torture_kthread_stopping() invoke schedule_timeout_interruptible(1) in a tight loop. Tracing confirms that TIMER_SOFTIRQ can then continuously execute timer callbacks. If that TIMER_SOFTIRQ preempts the task executing rcu_torture_cleanup(), that task might never invoke kthread_stop(). This commit improves this situation by increasing the timeout passed to schedule_timeout_interruptible() from one jiffy to 1/20th of a second. This change prevents TIMER_SOFTIRQ from monopolizing its CPU, thus allowing rcu_torture_cleanup() to carry out the needed kthread_stop() invocations. Testing has shown 100 runs of TREE07 passing reliably, as oppose to the tens-of-percent failure rates seen beforehand. Cc: Paul McKenney Cc: Frederic Weisbecker Cc: Zhouyi Zhou Cc: # 6.0.x Signed-off-by: Joel Fernandes (Google) Tested-by: Zhouyi Zhou Reviewed-by: Davidlohr Bueso Signed-off-by: Paul E. McKenney --- kernel/torture.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/torture.c b/kernel/torture.c index 29afc62f2bfe..1a0519b836ac 100644 --- a/kernel/torture.c +++ b/kernel/torture.c @@ -915,7 +915,7 @@ void torture_kthread_stopping(char *title) VERBOSE_TOROUT_STRING(buf); while (!kthread_should_stop()) { torture_shutdown_absorb(title); - schedule_timeout_uninterruptible(1); + schedule_timeout_uninterruptible(HZ / 20); } } EXPORT_SYMBOL_GPL(torture_kthread_stopping); From 3ca0a6ea8f6de0f21a9331ff3596a9c073fbdab0 Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Sat, 19 Nov 2022 17:25:04 +0800 Subject: [PATCH 084/239] sched: Add helper kstat_cpu_softirqs_sum() Add a kstat_cpu_softirqs_sum() function that is similar to kstat_cpu_irqs_sum(), but which counts software interrupts since boot on the specified CPU. Signed-off-by: Zhen Lei Cc: Josh Don Cc: Tejun Heo Cc: Peter Zijlstra Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- include/linux/kernel_stat.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index 90e2fdc17d79..898076e173a9 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -67,6 +67,17 @@ static inline unsigned int kstat_softirqs_cpu(unsigned int irq, int cpu) return kstat_cpu(cpu).softirqs[irq]; } +static inline unsigned int kstat_cpu_softirqs_sum(int cpu) +{ + int i; + unsigned int sum = 0; + + for (i = 0; i < NR_SOFTIRQS; i++) + sum += kstat_softirqs_cpu(i, cpu); + + return sum; +} + /* * Number of interrupts per specific IRQ source, since bootup */ From 7c182722a0a9447e31f9645de4f311e5bc59b480 Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Sat, 19 Nov 2022 17:25:05 +0800 Subject: [PATCH 085/239] sched: Add helper nr_context_switches_cpu() Add a function nr_context_switches_cpu() that returns number of context switches since boot on the specified CPU. This information will be used to diagnose RCU CPU stalls. Signed-off-by: Zhen Lei Cc: Ingo Molnar Cc: Peter Zijlstra Cc: Juri Lelli Cc: Vincent Guittot Cc: Dietmar Eggemann Cc: Steven Rostedt Cc: Ben Segall Cc: Mel Gorman Cc: Daniel Bristot de Oliveira Cc: Valentin Schneider Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- include/linux/kernel_stat.h | 1 + kernel/sched/core.c | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index 898076e173a9..9935f7ecbfb9 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -52,6 +52,7 @@ DECLARE_PER_CPU(struct kernel_cpustat, kernel_cpustat); #define kstat_cpu(cpu) per_cpu(kstat, cpu) #define kcpustat_cpu(cpu) per_cpu(kernel_cpustat, cpu) +extern unsigned long long nr_context_switches_cpu(int cpu); extern unsigned long long nr_context_switches(void); extern unsigned int kstat_irqs_cpu(unsigned int irq, int cpu); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 25b582b6ee5f..2e40a6c116e1 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5282,6 +5282,11 @@ bool single_task_running(void) } EXPORT_SYMBOL(single_task_running); +unsigned long long nr_context_switches_cpu(int cpu) +{ + return cpu_rq(cpu)->nr_switches; +} + unsigned long long nr_context_switches(void) { int i; From be42f00b73a0f50710d16eb7cb4efda0cce062dd Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Sat, 19 Nov 2022 17:25:06 +0800 Subject: [PATCH 086/239] rcu: Add RCU stall diagnosis information Because RCU CPU stall warnings are driven from the scheduling-clock interrupt handler, a workload consisting of a very large number of short-duration hardware interrupts can result in misleading stall-warning messages. On systems supporting only a single level of interrupts, that is, where interrupts handlers cannot be interrupted, this can produce misleading diagnostics. The stack traces will show the innocent-bystander interrupted task, not the interrupts that are at the very least exacerbating the stall. This situation can be improved by displaying the number of interrupts and the CPU time that they have consumed. Diagnosing other types of stalls can be eased by also providing the count of softirqs and the CPU time that they consumed as well as the number of context switches and the task-level CPU time consumed. Consider the following output given this change: rcu: INFO: rcu_preempt self-detected stall on CPU rcu: 0-....: (1250 ticks this GP) rcu: hardirqs softirqs csw/system rcu: number: 624 45 0 rcu: cputime: 69 1 2425 ==> 2500(ms) This output shows that the number of hard and soft interrupts is small, there are no context switches, and the system takes up a lot of time. This indicates that the current task is looping with preemption disabled. The impact on system performance is negligible because snapshot is recorded only once for all continuous RCU stalls. This added debugging information is suppressed by default and can be enabled by building the kernel with CONFIG_RCU_CPU_STALL_CPUTIME=y or by booting with rcupdate.rcu_cpu_stall_cputime=1. Signed-off-by: Zhen Lei Reviewed-by: Mukesh Ojha Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- .../admin-guide/kernel-parameters.txt | 6 ++++ kernel/rcu/Kconfig.debug | 13 ++++++++ kernel/rcu/rcu.h | 1 + kernel/rcu/tree.c | 18 +++++++++++ kernel/rcu/tree.h | 19 ++++++++++++ kernel/rcu/tree_stall.h | 31 +++++++++++++++++++ kernel/rcu/update.c | 2 ++ 7 files changed, 90 insertions(+) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 6cfa6e3996cf..43ca7f3ac96a 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -5113,6 +5113,12 @@ rcupdate.rcu_cpu_stall_timeout to be used (after conversion from seconds to milliseconds). + rcupdate.rcu_cpu_stall_cputime= [KNL] + Provide statistics on the cputime and count of + interrupts and tasks during the sampling period. For + multiple continuous RCU stalls, all sampling periods + begin at half of the first RCU stall timeout. + rcupdate.rcu_expedited= [KNL] Use expedited grace-period primitives, for example, synchronize_rcu_expedited() instead diff --git a/kernel/rcu/Kconfig.debug b/kernel/rcu/Kconfig.debug index 232e29fe3e5e..49da904df6aa 100644 --- a/kernel/rcu/Kconfig.debug +++ b/kernel/rcu/Kconfig.debug @@ -92,6 +92,19 @@ config RCU_EXP_CPU_STALL_TIMEOUT says to use the RCU_CPU_STALL_TIMEOUT value converted from seconds to milliseconds. +config RCU_CPU_STALL_CPUTIME + bool "Provide additional RCU stall debug information" + depends on RCU_STALL_COMMON + default n + help + Collect statistics during the sampling period, such as the number of + (hard interrupts, soft interrupts, task switches) and the cputime of + (hard interrupts, soft interrupts, kernel tasks) are added to the + RCU stall report. For multiple continuous RCU stalls, all sampling + periods begin at half of the first RCU stall timeout. + The boot option rcupdate.rcu_cpu_stall_cputime has the same function + as this one, but will override this if it exists. + config RCU_TRACE bool "Enable tracing for RCU" depends on DEBUG_KERNEL diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index c5aa934de59b..ff35920e1055 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -224,6 +224,7 @@ extern int rcu_cpu_stall_ftrace_dump; extern int rcu_cpu_stall_suppress; extern int rcu_cpu_stall_timeout; extern int rcu_exp_cpu_stall_timeout; +extern int rcu_cpu_stall_cputime; int rcu_jiffies_till_stall_check(void); int rcu_exp_jiffies_till_stall_check(void); diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index cf34a961821a..65552e6a6a5d 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -925,6 +925,24 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) rdp->rcu_iw_gp_seq = rnp->gp_seq; irq_work_queue_on(&rdp->rcu_iw, rdp->cpu); } + + if (rcu_cpu_stall_cputime && rdp->snap_record.gp_seq != rdp->gp_seq) { + int cpu = rdp->cpu; + struct rcu_snap_record *rsrp; + struct kernel_cpustat *kcsp; + + kcsp = &kcpustat_cpu(cpu); + + rsrp = &rdp->snap_record; + rsrp->cputime_irq = kcpustat_field(kcsp, CPUTIME_IRQ, cpu); + rsrp->cputime_softirq = kcpustat_field(kcsp, CPUTIME_SOFTIRQ, cpu); + rsrp->cputime_system = kcpustat_field(kcsp, CPUTIME_SYSTEM, cpu); + rsrp->nr_hardirqs = kstat_cpu_irqs_sum(rdp->cpu); + rsrp->nr_softirqs = kstat_cpu_softirqs_sum(rdp->cpu); + rsrp->nr_csw = nr_context_switches_cpu(rdp->cpu); + rsrp->jiffies = jiffies; + rsrp->gp_seq = rdp->gp_seq; + } } return 0; diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index fcb5d696eb17..192536916f9a 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -158,6 +158,23 @@ union rcu_noqs { u16 s; /* Set of bits, aggregate OR here. */ }; +/* + * Record the snapshot of the core stats at half of the first RCU stall timeout. + * The member gp_seq is used to ensure that all members are updated only once + * during the sampling period. The snapshot is taken only if this gp_seq is not + * equal to rdp->gp_seq. + */ +struct rcu_snap_record { + unsigned long gp_seq; /* Track rdp->gp_seq counter */ + u64 cputime_irq; /* Accumulated cputime of hard irqs */ + u64 cputime_softirq;/* Accumulated cputime of soft irqs */ + u64 cputime_system; /* Accumulated cputime of kernel tasks */ + unsigned long nr_hardirqs; /* Accumulated number of hard irqs */ + unsigned int nr_softirqs; /* Accumulated number of soft irqs */ + unsigned long long nr_csw; /* Accumulated number of task switches */ + unsigned long jiffies; /* Track jiffies value */ +}; + /* Per-CPU data for read-copy update. */ struct rcu_data { /* 1) quiescent-state and grace-period handling : */ @@ -262,6 +279,8 @@ struct rcu_data { short rcu_onl_gp_flags; /* ->gp_flags at last online. */ unsigned long last_fqs_resched; /* Time of last rcu_resched(). */ unsigned long last_sched_clock; /* Jiffies of last rcu_sched_clock_irq(). */ + struct rcu_snap_record snap_record; /* Snapshot of core stats at half of */ + /* the first RCU stall timeout */ long lazy_len; /* Length of buffered lazy callbacks. */ int cpu; diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index 5653560573e2..6de15fb10bc4 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -428,6 +428,35 @@ static bool rcu_is_rcuc_kthread_starving(struct rcu_data *rdp, unsigned long *jp return j > 2 * HZ; } +static void print_cpu_stat_info(int cpu) +{ + struct rcu_snap_record rsr, *rsrp; + struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); + struct kernel_cpustat *kcsp = &kcpustat_cpu(cpu); + + if (!rcu_cpu_stall_cputime) + return; + + rsrp = &rdp->snap_record; + if (rsrp->gp_seq != rdp->gp_seq) + return; + + rsr.cputime_irq = kcpustat_field(kcsp, CPUTIME_IRQ, cpu); + rsr.cputime_softirq = kcpustat_field(kcsp, CPUTIME_SOFTIRQ, cpu); + rsr.cputime_system = kcpustat_field(kcsp, CPUTIME_SYSTEM, cpu); + + pr_err("\t hardirqs softirqs csw/system\n"); + pr_err("\t number: %8ld %10d %12lld\n", + kstat_cpu_irqs_sum(cpu) - rsrp->nr_hardirqs, + kstat_cpu_softirqs_sum(cpu) - rsrp->nr_softirqs, + nr_context_switches_cpu(cpu) - rsrp->nr_csw); + pr_err("\tcputime: %8lld %10lld %12lld ==> %d(ms)\n", + div_u64(rsr.cputime_irq - rsrp->cputime_irq, NSEC_PER_MSEC), + div_u64(rsr.cputime_softirq - rsrp->cputime_softirq, NSEC_PER_MSEC), + div_u64(rsr.cputime_system - rsrp->cputime_system, NSEC_PER_MSEC), + jiffies_to_msecs(jiffies - rsrp->jiffies)); +} + /* * Print out diagnostic information for the specified stalled CPU. * @@ -484,6 +513,8 @@ static void print_cpu_stall_info(int cpu) data_race(rcu_state.n_force_qs) - rcu_state.n_force_qs_gpstart, rcuc_starved ? buf : "", falsepositive ? " (false positive?)" : ""); + + print_cpu_stat_info(cpu); } /* Complain about starvation of grace-period kthread. */ diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c index f5e6a2f95a2a..8d72cb7caead 100644 --- a/kernel/rcu/update.c +++ b/kernel/rcu/update.c @@ -508,6 +508,8 @@ int rcu_cpu_stall_timeout __read_mostly = CONFIG_RCU_CPU_STALL_TIMEOUT; module_param(rcu_cpu_stall_timeout, int, 0644); int rcu_exp_cpu_stall_timeout __read_mostly = CONFIG_RCU_EXP_CPU_STALL_TIMEOUT; module_param(rcu_exp_cpu_stall_timeout, int, 0644); +int rcu_cpu_stall_cputime __read_mostly = IS_ENABLED(CONFIG_RCU_CPU_STALL_CPUTIME); +module_param(rcu_cpu_stall_cputime, int, 0644); #endif /* #ifdef CONFIG_RCU_STALL_COMMON */ // Suppress boot-time RCU CPU stall warnings and rcutorture writer stall From 3ab955de929a53245c89deb7ff9992b6eebbe4e8 Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Sat, 19 Nov 2022 17:25:08 +0800 Subject: [PATCH 087/239] rcu: Align the output of RCU CPU stall warning messages Time stamps are added to the output in kernels built with CONFIG_PRINTK_TIME=y, which causes misaligned output. Therefore, replace pr_cont() with pr_err(), which fixes alignment and gets rid of a couple of despised pr_cont() calls. Before: [ 37.567343] rcu: INFO: rcu_preempt self-detected stall on CPU [ 37.567839] rcu: 0-....: (1500 ticks this GP) idle=*** [ 37.568270] (t=1501 jiffies g=4717 q=28 ncpus=4) [ 37.568668] CPU: 0 PID: 313 Comm: test0 Not tainted 6.1.0-rc4 #8 After: [ 36.762074] rcu: INFO: rcu_preempt self-detected stall on CPU [ 36.762543] rcu: 0-....: (1499 ticks this GP) idle=*** [ 36.763003] rcu: (t=1500 jiffies g=5097 q=27 ncpus=4) [ 36.763522] CPU: 0 PID: 313 Comm: test0 Not tainted 6.1.0-rc4 #9 Signed-off-by: Zhen Lei Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- kernel/rcu/tree_stall.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index 6de15fb10bc4..f360894f5599 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -619,7 +619,7 @@ static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps) for_each_possible_cpu(cpu) totqlen += rcu_get_n_cbs_cpu(cpu); - pr_cont("\t(detected by %d, t=%ld jiffies, g=%ld, q=%lu ncpus=%d)\n", + pr_err("\t(detected by %d, t=%ld jiffies, g=%ld, q=%lu ncpus=%d)\n", smp_processor_id(), (long)(jiffies - gps), (long)rcu_seq_current(&rcu_state.gp_seq), totqlen, rcu_state.n_online_cpus); if (ndetected) { @@ -680,7 +680,7 @@ static void print_cpu_stall(unsigned long gps) raw_spin_unlock_irqrestore_rcu_node(rdp->mynode, flags); for_each_possible_cpu(cpu) totqlen += rcu_get_n_cbs_cpu(cpu); - pr_cont("\t(t=%lu jiffies g=%ld q=%lu ncpus=%d)\n", + pr_err("\t(t=%lu jiffies g=%ld q=%lu ncpus=%d)\n", jiffies - gps, (long)rcu_seq_current(&rcu_state.gp_seq), totqlen, rcu_state.n_online_cpus); From c76feb0d5dfdb90b70fa820bb3181142bb01e980 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 6 Jan 2023 16:10:24 -0800 Subject: [PATCH 088/239] workqueue: Make show_pwq() use run-length encoding The show_pwq() function dumps out a pool_workqueue structure's activity, including the pending work-queue handlers: Showing busy workqueues and worker pools: workqueue events: flags=0x0 pwq 0: cpus=0 node=0 flags=0x1 nice=0 active=10/256 refcnt=11 in-flight: 7:test_work_func, 64:test_work_func, 249:test_work_func pending: test_work_func, test_work_func, test_work_func1, test_work_func1, test_work_func1, test_work_func1, test_work_func1 When large systems are facing certain types of hang conditions, it is not unusual for this "pending" list to contain runs of hundreds of identical function names. This "wall of text" is difficult to read, and worse yet, it can be interleaved with other output such as stack traces. Therefore, make show_pwq() use run-length encoding so that the above printout instead looks like this: Showing busy workqueues and worker pools: workqueue events: flags=0x0 pwq 0: cpus=0 node=0 flags=0x1 nice=0 active=10/256 refcnt=11 in-flight: 7:test_work_func, 64:test_work_func, 249:test_work_func pending: 2*test_work_func, 5*test_work_func1 When no comma would be printed, including the WORK_STRUCT_LINKED case, a new run is started unconditionally. This output is more readable, places less stress on the hardware, firmware, and software on the console-log path, and reduces interference with other output. Signed-off-by: Paul E. McKenney Cc: Tejun Heo Cc: Lai Jiangshan Cc: Dave Jones Cc: Rik van Riel Signed-off-by: Tejun Heo --- kernel/workqueue.c | 44 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 5b06262a419c..76b41850b158 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -4718,22 +4718,53 @@ static void pr_cont_pool_info(struct worker_pool *pool) pr_cont(" flags=0x%x nice=%d", pool->flags, pool->attrs->nice); } -static void pr_cont_work(bool comma, struct work_struct *work) +struct pr_cont_work_struct { + bool comma; + work_func_t func; + long ctr; +}; + +static void pr_cont_work_flush(bool comma, work_func_t func, struct pr_cont_work_struct *pcwsp) +{ + if (!pcwsp->ctr) + goto out_record; + if (func == pcwsp->func) { + pcwsp->ctr++; + return; + } + if (pcwsp->ctr == 1) + pr_cont("%s %ps", pcwsp->comma ? "," : "", pcwsp->func); + else + pr_cont("%s %ld*%ps", pcwsp->comma ? "," : "", pcwsp->ctr, pcwsp->func); + pcwsp->ctr = 0; +out_record: + if ((long)func == -1L) + return; + pcwsp->comma = comma; + pcwsp->func = func; + pcwsp->ctr = 1; +} + +static void pr_cont_work(bool comma, struct work_struct *work, struct pr_cont_work_struct *pcwsp) { if (work->func == wq_barrier_func) { struct wq_barrier *barr; barr = container_of(work, struct wq_barrier, work); + pr_cont_work_flush(comma, (work_func_t)-1, pcwsp); pr_cont("%s BAR(%d)", comma ? "," : "", task_pid_nr(barr->task)); } else { - pr_cont("%s %ps", comma ? "," : "", work->func); + if (!comma) + pr_cont_work_flush(comma, (work_func_t)-1, pcwsp); + pr_cont_work_flush(comma, work->func, pcwsp); } } static void show_pwq(struct pool_workqueue *pwq) { + struct pr_cont_work_struct pcws = { .ctr = 0, }; struct worker_pool *pool = pwq->pool; struct work_struct *work; struct worker *worker; @@ -4766,7 +4797,8 @@ static void show_pwq(struct pool_workqueue *pwq) worker->rescue_wq ? "(RESCUER)" : "", worker->current_func); list_for_each_entry(work, &worker->scheduled, entry) - pr_cont_work(false, work); + pr_cont_work(false, work, &pcws); + pr_cont_work_flush(comma, (work_func_t)-1L, &pcws); comma = true; } pr_cont("\n"); @@ -4786,9 +4818,10 @@ static void show_pwq(struct pool_workqueue *pwq) if (get_work_pwq(work) != pwq) continue; - pr_cont_work(comma, work); + pr_cont_work(comma, work, &pcws); comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED); } + pr_cont_work_flush(comma, (work_func_t)-1L, &pcws); pr_cont("\n"); } @@ -4797,9 +4830,10 @@ static void show_pwq(struct pool_workqueue *pwq) pr_info(" inactive:"); list_for_each_entry(work, &pwq->inactive_works, entry) { - pr_cont_work(comma, work); + pr_cont_work(comma, work, &pcws); comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED); } + pr_cont_work_flush(comma, (work_func_t)-1L, &pcws); pr_cont("\n"); } } From 84ec7c20363f46a4031695eceb0374c1c4d6b9ec Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 6 Dec 2022 09:38:39 -0800 Subject: [PATCH 089/239] rcu: Allow up to five minutes expedited RCU CPU stall-warning timeouts The maximum value of RCU CPU stall-warning timeouts has historically been five minutes (300 seconds). However, the recently introduced expedited RCU CPU stall-warning timeout is instead limited to 21 seconds. This causes problems for CI/fuzzing services such as syzkaller by obscuring the issue in question with expedited RCU CPU stall-warning timeout splats. This commit therefore sets the RCU_EXP_CPU_STALL_TIMEOUT Kconfig options upper bound to 300000 milliseconds, which is 300 seconds (AKA 5 minutes). [ paulmck: Apply feedback from Hillf Danton. ] [ paulmck: Apply feedback from Geert Uytterhoeven. ] Reported-by: Dave Chinner Reported-by: Dmitry Vyukov Tested-by: Dmitry Vyukov Signed-off-by: Paul E. McKenney --- kernel/rcu/Kconfig.debug | 2 +- kernel/rcu/tree_stall.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/Kconfig.debug b/kernel/rcu/Kconfig.debug index 49da904df6aa..2984de629f74 100644 --- a/kernel/rcu/Kconfig.debug +++ b/kernel/rcu/Kconfig.debug @@ -82,7 +82,7 @@ config RCU_CPU_STALL_TIMEOUT config RCU_EXP_CPU_STALL_TIMEOUT int "Expedited RCU CPU stall timeout in milliseconds" depends on RCU_STALL_COMMON - range 0 21000 + range 0 300000 default 0 help If a given expedited RCU grace period extends more than the diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index f360894f5599..b10b8349bb2a 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -39,7 +39,7 @@ int rcu_exp_jiffies_till_stall_check(void) // CONFIG_RCU_EXP_CPU_STALL_TIMEOUT, so check the allowed range. // The minimum clamped value is "2UL", because at least one full // tick has to be guaranteed. - till_stall_check = clamp(msecs_to_jiffies(cpu_stall_timeout), 2UL, 21UL * HZ); + till_stall_check = clamp(msecs_to_jiffies(cpu_stall_timeout), 2UL, 300UL * HZ); if (cpu_stall_timeout && jiffies_to_msecs(till_stall_check) != cpu_stall_timeout) WRITE_ONCE(rcu_exp_cpu_stall_timeout, jiffies_to_msecs(till_stall_check)); From ca843a4c79486e99a19b859ef0b9887854afe146 Mon Sep 17 00:00:00 2001 From: Daniil Tatianin Date: Sat, 7 Jan 2023 02:53:08 +0300 Subject: [PATCH 090/239] ACPICA: nsrepair: handle cases without a return value correctly Previously acpi_ns_simple_repair() would crash if expected_btypes contained any combination of ACPI_RTYPE_NONE with a different type, e.g | ACPI_RTYPE_INTEGER because of slightly incorrect logic in the !return_object branch, which wouldn't return AE_AML_NO_RETURN_VALUE for such cases. Found by Linux Verification Center (linuxtesting.org) with the SVACE static analysis tool. Link: https://github.com/acpica/acpica/pull/811 Fixes: 61db45ca2163 ("ACPICA: Restore code that repairs NULL package elements in return values.") Signed-off-by: Daniil Tatianin Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpica/nsrepair.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/acpi/acpica/nsrepair.c b/drivers/acpi/acpica/nsrepair.c index 367fcd201f96..ec512e06a48e 100644 --- a/drivers/acpi/acpica/nsrepair.c +++ b/drivers/acpi/acpica/nsrepair.c @@ -181,8 +181,9 @@ acpi_ns_simple_repair(struct acpi_evaluate_info *info, * Try to fix if there was no return object. Warning if failed to fix. */ if (!return_object) { - if (expected_btypes && (!(expected_btypes & ACPI_RTYPE_NONE))) { - if (package_index != ACPI_NOT_PACKAGE_ELEMENT) { + if (expected_btypes) { + if (!(expected_btypes & ACPI_RTYPE_NONE) && + package_index != ACPI_NOT_PACKAGE_ELEMENT) { ACPI_WARN_PREDEFINED((AE_INFO, info->full_pathname, ACPI_WARN_ALWAYS, @@ -196,14 +197,15 @@ acpi_ns_simple_repair(struct acpi_evaluate_info *info, if (ACPI_SUCCESS(status)) { return (AE_OK); /* Repair was successful */ } - } else { + } + + if (expected_btypes != ACPI_RTYPE_NONE) { ACPI_WARN_PREDEFINED((AE_INFO, info->full_pathname, ACPI_WARN_ALWAYS, "Missing expected return value")); + return (AE_AML_NO_RETURN_VALUE); } - - return (AE_AML_NO_RETURN_VALUE); } } From 27f850880192c19a26d89dab19e2bddcf276032f Mon Sep 17 00:00:00 2001 From: Kajetan Puchalski Date: Thu, 5 Jan 2023 14:51:58 +0000 Subject: [PATCH 091/239] cpuidle: teo: Optionally skip polling states in teo_find_shallower_state() Add a no_poll flag to teo_find_shallower_state() that will let the function optionally not consider polling states. This allows the caller to guard against the function inadvertently resulting in TEO putting the CPU in a polling state when that behaviour is undesirable. Signed-off-by: Kajetan Puchalski Signed-off-by: Rafael J. Wysocki --- drivers/cpuidle/governors/teo.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/cpuidle/governors/teo.c b/drivers/cpuidle/governors/teo.c index d9262db79cae..e2864474a98d 100644 --- a/drivers/cpuidle/governors/teo.c +++ b/drivers/cpuidle/governors/teo.c @@ -258,15 +258,17 @@ static s64 teo_middle_of_bin(int idx, struct cpuidle_driver *drv) * @dev: Target CPU. * @state_idx: Index of the capping idle state. * @duration_ns: Idle duration value to match. + * @no_poll: Don't consider polling states. */ static int teo_find_shallower_state(struct cpuidle_driver *drv, struct cpuidle_device *dev, int state_idx, - s64 duration_ns) + s64 duration_ns, bool no_poll) { int i; for (i = state_idx - 1; i >= 0; i--) { - if (dev->states_usage[i].disable) + if (dev->states_usage[i].disable || + (no_poll && drv->states[i].flags & CPUIDLE_FLAG_POLLING)) continue; state_idx = i; @@ -469,7 +471,7 @@ end: */ if (idx > idx0 && drv->states[idx].target_residency_ns > delta_tick) - idx = teo_find_shallower_state(drv, dev, idx, delta_tick); + idx = teo_find_shallower_state(drv, dev, idx, delta_tick, false); } return idx; From 9ce0f7c4bc64d820b02a1c53f7e8dba9539f942b Mon Sep 17 00:00:00 2001 From: Kajetan Puchalski Date: Thu, 5 Jan 2023 14:51:59 +0000 Subject: [PATCH 092/239] cpuidle: teo: Introduce util-awareness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Modern interactive systems, such as recent Android phones, tend to have power efficient shallow idle states. Selecting deeper idle states on a device while a latency-sensitive workload is running can adversely impact performance due to increased latency. Additionally, if the CPU wakes up from a deeper sleep before its target residency as is often the case, it results in a waste of energy on top of that. At the moment, none of the available idle governors take any scheduling information into account. They also tend to overestimate the idle duration quite often, which causes them to select excessively deep idle states, thus leading to increased wakeup latency and lower performance with no power saving. For 'menu' while web browsing on Android for instance, those types of wakeups ('too deep') account for over 24% of all wakeups. At the same time, on some platforms idle state 0 can be power efficient enough to warrant wanting to prefer it over idle state 1. This is because the power usage of the two states can be so close that sufficient amounts of too deep state 1 sleeps can completely offset the state 1 power saving to the point where it would've been more power efficient to just use state 0 instead. This is, of course, for systems where state 0 is not a polling state, such as arm-based devices. Sleeps that happened in state 0 while they could have used state 1 ('too shallow') only save less power than they otherwise could have. Too deep sleeps, on the other hand, harm performance and nullify the potential power saving from using state 1 in the first place. While taking this into account, it is clear that on balance it is preferable for an idle governor to have more too shallow sleeps instead of more too deep sleeps on those kinds of platforms. This patch specifically tunes TEO to prefer shallower idle states in order to reduce wakeup latency and achieve better performance. To this end, before selecting the next idle state it uses the avg_util signal of a CPU's runqueue in order to determine to what extent the CPU is being utilized. This util value is then compared to a threshold defined as a percentage of the CPU's capacity (capacity >> 6 ie. ~1.5% in the current implementation). If the util is above the threshold, the index of the idle state selected by TEO metrics will be reduced by 1, thus selecting a shallower state. If the util is below the threshold, the governor defaults to the TEO metrics mechanism to try to select the deepest available idle state based on the closest timer event and its own correctness. The main goal of this is to reduce latency and increase performance for some workloads. Under some workloads it will result in an increase in power usage (Geekbench 5) while for other workloads it will also result in a decrease in power usage compared to TEO (PCMark Web, Jankbench, Speedometer). It can provide drastically decreased latency and performance benefits in certain types of workloads that are sensitive to latency. Example test results: 1. GB5 (better score, latency & more power usage) | metric | menu | teo | teo-util-aware | | ------------------------------------- | -------------- | ----------------- | ----------------- | | gmean score | 2826.5 (0.0%) | 2764.8 (-2.18%) | 2865 (1.36%) | | gmean power usage [mW] | 2551.4 (0.0%) | 2606.8 (2.17%) | 2722.3 (6.7%) | | gmean too deep % | 14.99% | 9.65% | 4.02% | | gmean too shallow % | 2.5% | 5.96% | 14.59% | | gmean task wakeup latency (asynctask) | 78.16μs (0.0%) | 61.60μs (-21.19%) | 54.45μs (-30.34%) | 2. Jankbench (better score, latency & less power usage) | metric | menu | teo | teo-util-aware | | ------------------------------------- | -------------- | ----------------- | ----------------- | | gmean frame duration | 13.9 (0.0%) | 14.7 (6.0%) | 12.6 (-9.0%) | | gmean jank percentage | 1.5 (0.0%) | 2.1 (36.99%) | 1.3 (-17.37%) | | gmean power usage [mW] | 144.6 (0.0%) | 136.9 (-5.27%) | 121.3 (-16.08%) | | gmean too deep % | 26.00% | 11.00% | 2.54% | | gmean too shallow % | 4.74% | 11.89% | 21.93% | | gmean wakeup latency (RenderThread) | 139.5μs (0.0%) | 116.5μs (-16.49%) | 91.11μs (-34.7%) | | gmean wakeup latency (surfaceflinger) | 124.0μs (0.0%) | 151.9μs (22.47%) | 87.65μs (-29.33%) | Signed-off-by: Kajetan Puchalski [ rjw: Comment edits and white space adjustments ] Signed-off-by: Rafael J. Wysocki --- drivers/cpuidle/governors/teo.c | 94 ++++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 1 deletion(-) diff --git a/drivers/cpuidle/governors/teo.c b/drivers/cpuidle/governors/teo.c index e2864474a98d..987fc5f3997d 100644 --- a/drivers/cpuidle/governors/teo.c +++ b/drivers/cpuidle/governors/teo.c @@ -2,8 +2,13 @@ /* * Timer events oriented CPU idle governor * + * TEO governor: * Copyright (C) 2018 - 2021 Intel Corporation * Author: Rafael J. Wysocki + * + * Util-awareness mechanism: + * Copyright (C) 2022 Arm Ltd. + * Author: Kajetan Puchalski */ /** @@ -99,14 +104,55 @@ * select the given idle state instead of the candidate one. * * 3. By default, select the candidate state. + * + * Util-awareness mechanism: + * + * The idea behind the util-awareness extension is that there are two distinct + * scenarios for the CPU which should result in two different approaches to idle + * state selection - utilized and not utilized. + * + * In this case, 'utilized' means that the average runqueue util of the CPU is + * above a certain threshold. + * + * When the CPU is utilized while going into idle, more likely than not it will + * be woken up to do more work soon and so a shallower idle state should be + * selected to minimise latency and maximise performance. When the CPU is not + * being utilized, the usual metrics-based approach to selecting the deepest + * available idle state should be preferred to take advantage of the power + * saving. + * + * In order to achieve this, the governor uses a utilization threshold. + * The threshold is computed per-CPU as a percentage of the CPU's capacity + * by bit shifting the capacity value. Based on testing, the shift of 6 (~1.56%) + * seems to be getting the best results. + * + * Before selecting the next idle state, the governor compares the current CPU + * util to the precomputed util threshold. If it's below, it defaults to the + * TEO metrics mechanism. If it's above, the closest shallower idle state will + * be selected instead, as long as is not a polling state. */ #include #include #include +#include #include +#include #include +/* + * The number of bits to shift the CPU's capacity by in order to determine + * the utilized threshold. + * + * 6 was chosen based on testing as the number that achieved the best balance + * of power and performance on average. + * + * The resulting threshold is high enough to not be triggered by background + * noise and low enough to react quickly when activity starts to ramp up. + */ +#define UTIL_THRESHOLD_SHIFT 6 + + /* * The PULSE value is added to metrics when they grow and the DECAY_SHIFT value * is used for decreasing metrics on a regular basis. @@ -137,9 +183,11 @@ struct teo_bin { * @time_span_ns: Time between idle state selection and post-wakeup update. * @sleep_length_ns: Time till the closest timer event (at the selection time). * @state_bins: Idle state data bins for this CPU. - * @total: Grand total of the "intercepts" and "hits" mertics for all bins. + * @total: Grand total of the "intercepts" and "hits" metrics for all bins. * @next_recent_idx: Index of the next @recent_idx entry to update. * @recent_idx: Indices of bins corresponding to recent "intercepts". + * @util_threshold: Threshold above which the CPU is considered utilized + * @utilized: Whether the last sleep on the CPU happened while utilized */ struct teo_cpu { s64 time_span_ns; @@ -148,10 +196,29 @@ struct teo_cpu { unsigned int total; int next_recent_idx; int recent_idx[NR_RECENT]; + unsigned long util_threshold; + bool utilized; }; static DEFINE_PER_CPU(struct teo_cpu, teo_cpus); +/** + * teo_cpu_is_utilized - Check if the CPU's util is above the threshold + * @cpu: Target CPU + * @cpu_data: Governor CPU data for the target CPU + */ +#ifdef CONFIG_SMP +static bool teo_cpu_is_utilized(int cpu, struct teo_cpu *cpu_data) +{ + return sched_cpu_util(cpu) > cpu_data->util_threshold; +} +#else +static bool teo_cpu_is_utilized(int cpu, struct teo_cpu *cpu_data) +{ + return false; +} +#endif + /** * teo_update - Update CPU metrics after wakeup. * @drv: cpuidle driver containing state data. @@ -323,6 +390,22 @@ static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, goto end; } + cpu_data->utilized = teo_cpu_is_utilized(dev->cpu, cpu_data); + /* + * If the CPU is being utilized over the threshold and there are only 2 + * states to choose from, the metrics need not be considered, so choose + * the shallowest non-polling state and exit. + */ + if (drv->state_count < 3 && cpu_data->utilized) { + for (i = 0; i < drv->state_count; ++i) { + if (!dev->states_usage[i].disable && + !(drv->states[i].flags & CPUIDLE_FLAG_POLLING)) { + idx = i; + goto end; + } + } + } + /* * Find the deepest idle state whose target residency does not exceed * the current sleep length and the deepest idle state not deeper than @@ -454,6 +537,13 @@ static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, if (idx > constraint_idx) idx = constraint_idx; + /* + * If the CPU is being utilized over the threshold, choose a shallower + * non-polling state to improve latency + */ + if (cpu_data->utilized) + idx = teo_find_shallower_state(drv, dev, idx, duration_ns, true); + end: /* * Don't stop the tick if the selected state is a polling one or if the @@ -510,9 +600,11 @@ static int teo_enable_device(struct cpuidle_driver *drv, struct cpuidle_device *dev) { struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu); + unsigned long max_capacity = arch_scale_cpu_capacity(dev->cpu); int i; memset(cpu_data, 0, sizeof(*cpu_data)); + cpu_data->util_threshold = max_capacity >> UTIL_THRESHOLD_SHIFT; for (i = 0; i < NR_RECENT; i++) cpu_data->recent_idx[i] = -1; From 32a47817d07557ffca9992964c514fd79bda6fba Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Sat, 7 Jan 2023 18:12:17 -0800 Subject: [PATCH 093/239] cgroup/cpuset: fix a few kernel-doc warnings & coding style Fix kernel-doc notation warnings: kernel/cgroup/cpuset.c:1309: warning: Excess function parameter 'cpuset' description in 'update_parent_subparts_cpumask' kernel/cgroup/cpuset.c:3909: warning: expecting prototype for cpuset_mem_spread_node(). Prototype was for cpuset_spread_node() instead Also drop a blank line before EXPORT_SYMBOL_GPL() to be consistent with kernel coding style. Signed-off-by: Randy Dunlap Cc: Waiman Long Cc: Zefan Li Cc: Tejun Heo Cc: Johannes Weiner Cc: cgroups@vger.kernel.org Signed-off-by: Tejun Heo --- kernel/cgroup/cpuset.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 87fe410361b3..a585ced99e1e 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -1267,7 +1267,7 @@ static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs, int turning_on); /** * update_parent_subparts_cpumask - update subparts_cpus mask of parent cpuset - * @cpuset: The cpuset that requests change in partition root state + * @cs: The cpuset that requests change in partition root state * @cmd: Partition root state change command * @newmask: Optional new cpumask for partcmd_update * @tmp: Temporary addmask and delmask @@ -3877,8 +3877,7 @@ bool __cpuset_node_allowed(int node, gfp_t gfp_mask) } /** - * cpuset_mem_spread_node() - On which node to begin search for a file page - * cpuset_slab_spread_node() - On which node to begin search for a slab page + * cpuset_spread_node() - On which node to begin search for a page * * If a task is marked PF_SPREAD_PAGE or PF_SPREAD_SLAB (as for * tasks in a cpuset with is_spread_page or is_spread_slab set), @@ -3902,12 +3901,14 @@ bool __cpuset_node_allowed(int node, gfp_t gfp_mask) * is passed an offline node, it will fall back to the local node. * See kmem_cache_alloc_node(). */ - static int cpuset_spread_node(int *rotor) { return *rotor = next_node_in(*rotor, current->mems_allowed); } +/** + * cpuset_mem_spread_node() - On which node to begin search for a file page + */ int cpuset_mem_spread_node(void) { if (current->cpuset_mem_spread_rotor == NUMA_NO_NODE) @@ -3917,6 +3918,9 @@ int cpuset_mem_spread_node(void) return cpuset_spread_node(¤t->cpuset_mem_spread_rotor); } +/** + * cpuset_slab_spread_node() - On which node to begin search for a slab page + */ int cpuset_slab_spread_node(void) { if (current->cpuset_slab_spread_rotor == NUMA_NO_NODE) @@ -3925,7 +3929,6 @@ int cpuset_slab_spread_node(void) return cpuset_spread_node(¤t->cpuset_slab_spread_rotor); } - EXPORT_SYMBOL_GPL(cpuset_mem_spread_node); /** From 17549b0f184d870f2cfa4e5cfa79f4c4905ed757 Mon Sep 17 00:00:00 2001 From: Manfred Spraul Date: Fri, 16 Dec 2022 16:04:41 +0100 Subject: [PATCH 094/239] genirq: Add might_sleep() to disable_irq() With the introduction of threaded interrupt handlers, it is virtually never safe to call disable_irq() from non-premptible context. Thus: Update the documentation, add an explicit might_sleep() to catch any offenders. This is more obvious and straight forward than the implicit might_sleep() check deeper down in the disable_irq() call chain. Fixes: 3aa551c9b4c4 ("genirq: add threaded interrupt handler support") Signed-off-by: Manfred Spraul Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20221216150441.200533-3-manfred@colorfullife.com --- kernel/irq/manage.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index 5b7cf28df290..8ce75495e04f 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -723,10 +723,13 @@ EXPORT_SYMBOL(disable_irq_nosync); * to complete before returning. If you use this function while * holding a resource the IRQ handler may need you will deadlock. * - * This function may be called - with care - from IRQ context. + * Can only be called from preemptible code as it might sleep when + * an interrupt thread is associated to @irq. + * */ void disable_irq(unsigned int irq) { + might_sleep(); if (!__disable_irq_nosync(irq)) synchronize_irq(irq); } From 379af13b31fa8a36ad4abd59a5c511f25c5d4d42 Mon Sep 17 00:00:00 2001 From: Alexander Sverdlin Date: Mon, 12 Dec 2022 17:37:15 +0100 Subject: [PATCH 095/239] docs: locking: Discourage from calling disable_irq() in atomic Correct the example in the documentation so that disable_irq() is not being called in atomic context. disable_irq() calls sleeping synchronize_irq(), it's not allowed to call them in atomic context. Signed-off-by: Alexander Sverdlin Signed-off-by: Thomas Gleixner Reviewed-by: Manfred Spraul Cc: linux-doc@vger.kernel.org Link: https://lore.kernel.org/lkml/87k02wbs2n.ffs@tglx/ Link: https://lore.kernel.org/r/20221212163715.830315-1-alexander.sverdlin@siemens.com --- Documentation/kernel-hacking/locking.rst | 4 ++-- Documentation/translations/it_IT/kernel-hacking/locking.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/kernel-hacking/locking.rst b/Documentation/kernel-hacking/locking.rst index c756786e17ae..dff0646a717b 100644 --- a/Documentation/kernel-hacking/locking.rst +++ b/Documentation/kernel-hacking/locking.rst @@ -1277,11 +1277,11 @@ Manfred Spraul points out that you can still do this, even if the data is very occasionally accessed in user context or softirqs/tasklets. The irq handler doesn't use a lock, and all other accesses are done as so:: - spin_lock(&lock); + mutex_lock(&lock); disable_irq(irq); ... enable_irq(irq); - spin_unlock(&lock); + mutex_unlock(&lock); The disable_irq() prevents the irq handler from running (and waits for it to finish if it's currently running on other CPUs). diff --git a/Documentation/translations/it_IT/kernel-hacking/locking.rst b/Documentation/translations/it_IT/kernel-hacking/locking.rst index b8ecf41273c5..05d362b16bf0 100644 --- a/Documentation/translations/it_IT/kernel-hacking/locking.rst +++ b/Documentation/translations/it_IT/kernel-hacking/locking.rst @@ -1307,11 +1307,11 @@ se i dati vengono occasionalmente utilizzati da un contesto utente o da un'interruzione software. Il gestore d'interruzione non utilizza alcun *lock*, e tutti gli altri accessi verranno fatti così:: - spin_lock(&lock); + mutex_lock(&lock); disable_irq(irq); ... enable_irq(irq); - spin_unlock(&lock); + mutex_unlock(&lock); La funzione disable_irq() impedisce al gestore d'interruzioni d'essere eseguito (e aspetta che finisca nel caso fosse in esecuzione su From 0e2213fe0ab4c04da0e2354e84ec3b90e59939a4 Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Tue, 13 Dec 2022 15:08:43 +0100 Subject: [PATCH 096/239] irqchip: Use irq_domain_alloc_irqs() Use the irq_domain_alloc_irqs() wrapper instead of the full __irq_domain_alloc_irqs() interface, which was only intended for some legacy (x86) use cases. Signed-off-by: Johan Hovold Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20221213140844.15470-2-johan+linaro@kernel.org --- drivers/irqchip/irq-apple-aic.c | 4 +--- drivers/irqchip/irq-armada-370-xp.c | 3 +-- drivers/irqchip/irq-bcm2836.c | 5 +---- drivers/irqchip/irq-gic-v3.c | 4 +--- drivers/irqchip/irq-gic-v4.c | 9 +++------ drivers/irqchip/irq-gic.c | 4 +--- 6 files changed, 8 insertions(+), 21 deletions(-) diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index ae3437f03e6c..cf513b657654 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -924,9 +924,7 @@ static int __init aic_init_smp(struct aic_irq_chip *irqc, struct device_node *no ipi_domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE; irq_domain_update_bus_token(ipi_domain, DOMAIN_BUS_IPI); - base_ipi = __irq_domain_alloc_irqs(ipi_domain, -1, AIC_NR_SWIPI, - NUMA_NO_NODE, NULL, false, NULL); - + base_ipi = irq_domain_alloc_irqs(ipi_domain, AIC_NR_SWIPI, NUMA_NO_NODE, NULL); if (WARN_ON(!base_ipi)) { irq_domain_remove(ipi_domain); return -ENODEV; diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index ee18eb3e72b7..a55528469278 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -454,8 +454,7 @@ static __init void armada_xp_ipi_init(struct device_node *node) return; irq_domain_update_bus_token(ipi_domain, DOMAIN_BUS_IPI); - base_ipi = __irq_domain_alloc_irqs(ipi_domain, -1, IPI_DOORBELL_END, - NUMA_NO_NODE, NULL, false, NULL); + base_ipi = irq_domain_alloc_irqs(ipi_domain, IPI_DOORBELL_END, NUMA_NO_NODE, NULL); if (WARN_ON(!base_ipi)) return; diff --git a/drivers/irqchip/irq-bcm2836.c b/drivers/irqchip/irq-bcm2836.c index 51491c3c6fdd..e5f1059b989f 100644 --- a/drivers/irqchip/irq-bcm2836.c +++ b/drivers/irqchip/irq-bcm2836.c @@ -268,10 +268,7 @@ static void __init bcm2836_arm_irqchip_smp_init(void) ipi_domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE; irq_domain_update_bus_token(ipi_domain, DOMAIN_BUS_IPI); - base_ipi = __irq_domain_alloc_irqs(ipi_domain, -1, BITS_PER_MBOX, - NUMA_NO_NODE, NULL, - false, NULL); - + base_ipi = irq_domain_alloc_irqs(ipi_domain, BITS_PER_MBOX, NUMA_NO_NODE, NULL); if (WARN_ON(!base_ipi)) return; diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 997104d4338e..bb57ab8bff6a 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -1310,9 +1310,7 @@ static void __init gic_smp_init(void) gic_starting_cpu, NULL); /* Register all 8 non-secure SGIs */ - base_sgi = __irq_domain_alloc_irqs(gic_data.domain, -1, 8, - NUMA_NO_NODE, &sgi_fwspec, - false, NULL); + base_sgi = irq_domain_alloc_irqs(gic_data.domain, 8, NUMA_NO_NODE, &sgi_fwspec); if (WARN_ON(base_sgi <= 0)) return; diff --git a/drivers/irqchip/irq-gic-v4.c b/drivers/irqchip/irq-gic-v4.c index a6277dea4c7a..94d56a03b175 100644 --- a/drivers/irqchip/irq-gic-v4.c +++ b/drivers/irqchip/irq-gic-v4.c @@ -139,9 +139,7 @@ static int its_alloc_vcpu_sgis(struct its_vpe *vpe, int idx) if (!vpe->sgi_domain) goto err; - sgi_base = __irq_domain_alloc_irqs(vpe->sgi_domain, -1, 16, - NUMA_NO_NODE, vpe, - false, NULL); + sgi_base = irq_domain_alloc_irqs(vpe->sgi_domain, 16, NUMA_NO_NODE, vpe); if (sgi_base <= 0) goto err; @@ -176,9 +174,8 @@ int its_alloc_vcpu_irqs(struct its_vm *vm) vm->vpes[i]->idai = true; } - vpe_base_irq = __irq_domain_alloc_irqs(vm->domain, -1, vm->nr_vpes, - NUMA_NO_NODE, vm, - false, NULL); + vpe_base_irq = irq_domain_alloc_irqs(vm->domain, vm->nr_vpes, + NUMA_NO_NODE, vm); if (vpe_base_irq <= 0) goto err; diff --git a/drivers/irqchip/irq-gic.c b/drivers/irqchip/irq-gic.c index 210bc2f4d555..4fa4d8ac76d9 100644 --- a/drivers/irqchip/irq-gic.c +++ b/drivers/irqchip/irq-gic.c @@ -868,9 +868,7 @@ static __init void gic_smp_init(void) "irqchip/arm/gic:starting", gic_starting_cpu, NULL); - base_sgi = __irq_domain_alloc_irqs(gic_data[0].domain, -1, 8, - NUMA_NO_NODE, &sgi_fwspec, - false, NULL); + base_sgi = irq_domain_alloc_irqs(gic_data[0].domain, 8, NUMA_NO_NODE, &sgi_fwspec); if (WARN_ON(base_sgi <= 0)) return; From 99c621ef243bda726fb8d982a274ded96570b410 Mon Sep 17 00:00:00 2001 From: Lai Jiangshan Date: Thu, 12 Jan 2023 16:14:27 +0000 Subject: [PATCH 097/239] workqueue: Protects wq_unbound_cpumask with wq_pool_attach_mutex When unbind_workers() reads wq_unbound_cpumask to set the affinity of freshly-unbound kworkers, it only holds wq_pool_attach_mutex. This isn't sufficient as wq_unbound_cpumask is only protected by wq_pool_mutex. Make wq_unbound_cpumask protected with wq_pool_attach_mutex and also remove the need of temporary saved_cpumask. Fixes: 10a5a651e3af ("workqueue: Restrict kworker in the offline CPU pool running on housekeeping CPUs") Reported-by: Valentin Schneider Signed-off-by: Lai Jiangshan Signed-off-by: Tejun Heo --- kernel/workqueue.c | 41 ++++++++++++++++------------------------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 76b41850b158..55cca6ca1e78 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -326,7 +326,7 @@ static struct rcuwait manager_wait = __RCUWAIT_INITIALIZER(manager_wait); static LIST_HEAD(workqueues); /* PR: list of all workqueues */ static bool workqueue_freezing; /* PL: have wqs started freezing? */ -/* PL: allowable cpus for unbound wqs and work items */ +/* PL&A: allowable cpus for unbound wqs and work items */ static cpumask_var_t wq_unbound_cpumask; /* CPU where unbound work was last round robin scheduled from this CPU */ @@ -3956,7 +3956,8 @@ static void apply_wqattrs_cleanup(struct apply_wqattrs_ctx *ctx) /* allocate the attrs and pwqs for later installation */ static struct apply_wqattrs_ctx * apply_wqattrs_prepare(struct workqueue_struct *wq, - const struct workqueue_attrs *attrs) + const struct workqueue_attrs *attrs, + const cpumask_var_t unbound_cpumask) { struct apply_wqattrs_ctx *ctx; struct workqueue_attrs *new_attrs, *tmp_attrs; @@ -3972,14 +3973,15 @@ apply_wqattrs_prepare(struct workqueue_struct *wq, goto out_free; /* - * Calculate the attrs of the default pwq. + * Calculate the attrs of the default pwq with unbound_cpumask + * which is wq_unbound_cpumask or to set to wq_unbound_cpumask. * If the user configured cpumask doesn't overlap with the * wq_unbound_cpumask, we fallback to the wq_unbound_cpumask. */ copy_workqueue_attrs(new_attrs, attrs); - cpumask_and(new_attrs->cpumask, new_attrs->cpumask, wq_unbound_cpumask); + cpumask_and(new_attrs->cpumask, new_attrs->cpumask, unbound_cpumask); if (unlikely(cpumask_empty(new_attrs->cpumask))) - cpumask_copy(new_attrs->cpumask, wq_unbound_cpumask); + cpumask_copy(new_attrs->cpumask, unbound_cpumask); /* * We may create multiple pwqs with differing cpumasks. Make a @@ -4076,7 +4078,7 @@ static int apply_workqueue_attrs_locked(struct workqueue_struct *wq, wq->flags &= ~__WQ_ORDERED; } - ctx = apply_wqattrs_prepare(wq, attrs); + ctx = apply_wqattrs_prepare(wq, attrs, wq_unbound_cpumask); if (!ctx) return -ENOMEM; @@ -5377,7 +5379,7 @@ out_unlock: } #endif /* CONFIG_FREEZER */ -static int workqueue_apply_unbound_cpumask(void) +static int workqueue_apply_unbound_cpumask(const cpumask_var_t unbound_cpumask) { LIST_HEAD(ctxs); int ret = 0; @@ -5393,7 +5395,7 @@ static int workqueue_apply_unbound_cpumask(void) if (wq->flags & __WQ_ORDERED) continue; - ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs); + ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs, unbound_cpumask); if (!ctx) { ret = -ENOMEM; break; @@ -5408,6 +5410,11 @@ static int workqueue_apply_unbound_cpumask(void) apply_wqattrs_cleanup(ctx); } + if (!ret) { + mutex_lock(&wq_pool_attach_mutex); + cpumask_copy(wq_unbound_cpumask, unbound_cpumask); + mutex_unlock(&wq_pool_attach_mutex); + } return ret; } @@ -5426,7 +5433,6 @@ static int workqueue_apply_unbound_cpumask(void) int workqueue_set_unbound_cpumask(cpumask_var_t cpumask) { int ret = -EINVAL; - cpumask_var_t saved_cpumask; /* * Not excluding isolated cpus on purpose. @@ -5440,23 +5446,8 @@ int workqueue_set_unbound_cpumask(cpumask_var_t cpumask) goto out_unlock; } - if (!zalloc_cpumask_var(&saved_cpumask, GFP_KERNEL)) { - ret = -ENOMEM; - goto out_unlock; - } + ret = workqueue_apply_unbound_cpumask(cpumask); - /* save the old wq_unbound_cpumask. */ - cpumask_copy(saved_cpumask, wq_unbound_cpumask); - - /* update wq_unbound_cpumask at first and apply it to wqs. */ - cpumask_copy(wq_unbound_cpumask, cpumask); - ret = workqueue_apply_unbound_cpumask(); - - /* restore the wq_unbound_cpumask when failed. */ - if (ret < 0) - cpumask_copy(wq_unbound_cpumask, saved_cpumask); - - free_cpumask_var(saved_cpumask); out_unlock: apply_wqattrs_unlock(); } From 793777bc193b658f01924fd09b388eead26d741f Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Thu, 12 Jan 2023 16:14:28 +0000 Subject: [PATCH 098/239] workqueue: Factorize unbind/rebind_workers() logic Later patches will reuse this code, move it into reusable functions. Signed-off-by: Valentin Schneider Reviewed-by: Lai Jiangshan Signed-off-by: Tejun Heo --- kernel/workqueue.c | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 55cca6ca1e78..f1386bed3066 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -1976,6 +1976,23 @@ fail: return NULL; } +static void unbind_worker(struct worker *worker) +{ + lockdep_assert_held(&wq_pool_attach_mutex); + + kthread_set_per_cpu(worker->task, -1); + if (cpumask_intersects(wq_unbound_cpumask, cpu_active_mask)) + WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, wq_unbound_cpumask) < 0); + else + WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, cpu_possible_mask) < 0); +} + +static void rebind_worker(struct worker *worker, struct worker_pool *pool) +{ + kthread_set_per_cpu(worker->task, pool->cpu); + WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask) < 0); +} + /** * destroy_worker - destroy a workqueue worker * @worker: worker to be destroyed @@ -5051,13 +5068,8 @@ static void unbind_workers(int cpu) raw_spin_unlock_irq(&pool->lock); - for_each_pool_worker(worker, pool) { - kthread_set_per_cpu(worker->task, -1); - if (cpumask_intersects(wq_unbound_cpumask, cpu_active_mask)) - WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, wq_unbound_cpumask) < 0); - else - WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, cpu_possible_mask) < 0); - } + for_each_pool_worker(worker, pool) + unbind_worker(worker); mutex_unlock(&wq_pool_attach_mutex); } @@ -5082,11 +5094,8 @@ static void rebind_workers(struct worker_pool *pool) * of all workers first and then clear UNBOUND. As we're called * from CPU_ONLINE, the following shouldn't fail. */ - for_each_pool_worker(worker, pool) { - kthread_set_per_cpu(worker->task, pool->cpu); - WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, - pool->attrs->cpumask) < 0); - } + for_each_pool_worker(worker, pool) + rebind_worker(worker, pool); raw_spin_lock_irq(&pool->lock); From 3f959aa3b33829acfcd460c6c656d54dfebe8d1e Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Thu, 12 Jan 2023 16:14:29 +0000 Subject: [PATCH 099/239] workqueue: Convert the idle_timer to a timer + work_struct A later patch will require a sleepable context in the idle worker timeout function. Converting worker_pool.idle_timer to a delayed_work gives us just that, however this would imply turning all idle_timer expiries into scheduler events (waking up a worker to handle the dwork). Instead, implement a "custom dwork" where the timer callback does some extra checks before queuing the associated work. No change in functionality intended. Signed-off-by: Valentin Schneider Reviewed-by: Lai Jiangshan Signed-off-by: Tejun Heo --- kernel/workqueue.c | 52 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index f1386bed3066..e91816482e77 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -169,7 +169,9 @@ struct worker_pool { struct list_head idle_list; /* L: list of idle workers */ struct timer_list idle_timer; /* L: worker idle timeout */ - struct timer_list mayday_timer; /* L: SOS timer for workers */ + struct work_struct idle_cull_work; /* L: worker idle cleanup */ + + struct timer_list mayday_timer; /* L: SOS timer for workers */ /* a workers is either on busy_hash or idle_list, or the manager */ DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER); @@ -2023,9 +2025,54 @@ static void destroy_worker(struct worker *worker) wake_up_process(worker->task); } +/** + * idle_worker_timeout - check if some idle workers can now be deleted. + * @t: The pool's idle_timer that just expired + * + * The timer is armed in worker_enter_idle(). Note that it isn't disarmed in + * worker_leave_idle(), as a worker flicking between idle and active while its + * pool is at the too_many_workers() tipping point would cause too much timer + * housekeeping overhead. Since IDLE_WORKER_TIMEOUT is long enough, we just let + * it expire and re-evaluate things from there. + */ static void idle_worker_timeout(struct timer_list *t) { struct worker_pool *pool = from_timer(pool, t, idle_timer); + bool do_cull = false; + + if (work_pending(&pool->idle_cull_work)) + return; + + raw_spin_lock_irq(&pool->lock); + + if (too_many_workers(pool)) { + struct worker *worker; + unsigned long expires; + + /* idle_list is kept in LIFO order, check the last one */ + worker = list_entry(pool->idle_list.prev, struct worker, entry); + expires = worker->last_active + IDLE_WORKER_TIMEOUT; + do_cull = !time_before(jiffies, expires); + + if (!do_cull) + mod_timer(&pool->idle_timer, expires); + } + raw_spin_unlock_irq(&pool->lock); + + if (do_cull) + queue_work(system_unbound_wq, &pool->idle_cull_work); +} + +/** + * idle_cull_fn - cull workers that have been idle for too long. + * @work: the pool's work for handling these idle workers + * + * This goes through a pool's idle workers and gets rid of those that have been + * idle for at least IDLE_WORKER_TIMEOUT seconds. + */ +static void idle_cull_fn(struct work_struct *work) +{ + struct worker_pool *pool = container_of(work, struct worker_pool, idle_cull_work); raw_spin_lock_irq(&pool->lock); @@ -2033,7 +2080,6 @@ static void idle_worker_timeout(struct timer_list *t) struct worker *worker; unsigned long expires; - /* idle_list is kept in LIFO order, check the last one */ worker = list_entry(pool->idle_list.prev, struct worker, entry); expires = worker->last_active + IDLE_WORKER_TIMEOUT; @@ -3483,6 +3529,7 @@ static int init_worker_pool(struct worker_pool *pool) hash_init(pool->busy_hash); timer_setup(&pool->idle_timer, idle_worker_timeout, TIMER_DEFERRABLE); + INIT_WORK(&pool->idle_cull_work, idle_cull_fn); timer_setup(&pool->mayday_timer, pool_mayday_timeout, 0); @@ -3630,6 +3677,7 @@ static void put_unbound_pool(struct worker_pool *pool) /* shut down the timers */ del_timer_sync(&pool->idle_timer); + cancel_work_sync(&pool->idle_cull_work); del_timer_sync(&pool->mayday_timer); /* RCU protected to allow dereferences from get_work_pool() */ From 9ab03be42b8f9136dcc01a90ecc9ac71bc6149ef Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Thu, 12 Jan 2023 16:14:30 +0000 Subject: [PATCH 100/239] workqueue: Don't hold any lock while rcuwait'ing for !POOL_MANAGER_ACTIVE put_unbound_pool() currently passes wq_manager_inactive() as exit condition to rcuwait_wait_event(), which grabs pool->lock to check for pool->flags & POOL_MANAGER_ACTIVE A later patch will require destroy_worker() to be invoked with wq_pool_attach_mutex held, which needs to be acquired before pool->lock. A mutex cannot be acquired within rcuwait_wait_event(), as it could clobber the task state set by rcuwait_wait_event() Instead, restructure the waiting logic to acquire any necessary lock outside of rcuwait_wait_event(). Since further work cannot be inserted into unbound pwqs that have reached ->refcnt==0, this is bound to make forward progress as eventually the worklist will be drained and need_more_worker(pool) will remain false, preventing any worker from stealing the manager position from us. Suggested-by: Tejun Heo Signed-off-by: Valentin Schneider Signed-off-by: Tejun Heo --- kernel/workqueue.c | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index e91816482e77..a826956bc6c1 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -3608,18 +3608,6 @@ static void rcu_free_pool(struct rcu_head *rcu) kfree(pool); } -/* This returns with the lock held on success (pool manager is inactive). */ -static bool wq_manager_inactive(struct worker_pool *pool) -{ - raw_spin_lock_irq(&pool->lock); - - if (pool->flags & POOL_MANAGER_ACTIVE) { - raw_spin_unlock_irq(&pool->lock); - return false; - } - return true; -} - /** * put_unbound_pool - put a worker_pool * @pool: worker_pool to put @@ -3655,12 +3643,26 @@ static void put_unbound_pool(struct worker_pool *pool) * Become the manager and destroy all workers. This prevents * @pool's workers from blocking on attach_mutex. We're the last * manager and @pool gets freed with the flag set. - * Because of how wq_manager_inactive() works, we will hold the - * spinlock after a successful wait. + * + * Having a concurrent manager is quite unlikely to happen as we can + * only get here with + * pwq->refcnt == pool->refcnt == 0 + * which implies no work queued to the pool, which implies no worker can + * become the manager. However a worker could have taken the role of + * manager before the refcnts dropped to 0, since maybe_create_worker() + * drops pool->lock */ - rcuwait_wait_event(&manager_wait, wq_manager_inactive(pool), - TASK_UNINTERRUPTIBLE); - pool->flags |= POOL_MANAGER_ACTIVE; + while (true) { + rcuwait_wait_event(&manager_wait, + !(pool->flags & POOL_MANAGER_ACTIVE), + TASK_UNINTERRUPTIBLE); + raw_spin_lock_irq(&pool->lock); + if (!(pool->flags & POOL_MANAGER_ACTIVE)) { + pool->flags |= POOL_MANAGER_ACTIVE; + break; + } + raw_spin_unlock_irq(&pool->lock); + } while ((worker = first_idle_worker(pool))) destroy_worker(worker); From e02b93124855cd34b78e61ae44846c8cb5fddfc3 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Thu, 12 Jan 2023 16:14:31 +0000 Subject: [PATCH 101/239] workqueue: Unbind kworkers before sending them to exit() It has been reported that isolated CPUs can suffer from interference due to per-CPU kworkers waking up just to die. A surge of workqueue activity during initial setup of a latency-sensitive application (refresh_vm_stats() being one of the culprits) can cause extra per-CPU kworkers to be spawned. Then, said latency-sensitive task can be running merrily on an isolated CPU only to be interrupted sometime later by a kworker marked for death (cf. IDLE_WORKER_TIMEOUT, 5 minutes after last kworker activity). Prevent this by affining kworkers to the wq_unbound_cpumask (which doesn't contain isolated CPUs, cf. HK_TYPE_WQ) before waking them up after marking them with WORKER_DIE. Changing the affinity does require a sleepable context, leverage the newly introduced pool->idle_cull_work to get that. Remove dying workers from pool->workers and keep track of them in a separate list. This intentionally prevents for_each_loop_worker() from iterating over workers that are marked for death. Rename destroy_worker() to set_working_dying() to better reflect its effects and relationship with wake_dying_workers(). Signed-off-by: Valentin Schneider Signed-off-by: Tejun Heo --- kernel/workqueue.c | 72 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 60 insertions(+), 12 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index a826956bc6c1..5dc67aa9d696 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -179,6 +179,7 @@ struct worker_pool { struct worker *manager; /* L: purely informational */ struct list_head workers; /* A: attached workers */ + struct list_head dying_workers; /* A: workers about to die */ struct completion *detach_completion; /* all workers detached */ struct ida worker_ida; /* worker IDs for task name */ @@ -1906,7 +1907,7 @@ static void worker_detach_from_pool(struct worker *worker) list_del(&worker->node); worker->pool = NULL; - if (list_empty(&pool->workers)) + if (list_empty(&pool->workers) && list_empty(&pool->dying_workers)) detach_completion = pool->detach_completion; mutex_unlock(&wq_pool_attach_mutex); @@ -1995,21 +1996,44 @@ static void rebind_worker(struct worker *worker, struct worker_pool *pool) WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask) < 0); } +static void wake_dying_workers(struct list_head *cull_list) +{ + struct worker *worker, *tmp; + + list_for_each_entry_safe(worker, tmp, cull_list, entry) { + list_del_init(&worker->entry); + unbind_worker(worker); + /* + * If the worker was somehow already running, then it had to be + * in pool->idle_list when set_worker_dying() happened or we + * wouldn't have gotten here. + * + * Thus, the worker must either have observed the WORKER_DIE + * flag, or have set its state to TASK_IDLE. Either way, the + * below will be observed by the worker and is safe to do + * outside of pool->lock. + */ + wake_up_process(worker->task); + } +} + /** - * destroy_worker - destroy a workqueue worker + * set_worker_dying - Tag a worker for destruction * @worker: worker to be destroyed + * @list: transfer worker away from its pool->idle_list and into list * - * Destroy @worker and adjust @pool stats accordingly. The worker should - * be idle. + * Tag @worker for destruction and adjust @pool stats accordingly. The worker + * should be idle. * * CONTEXT: * raw_spin_lock_irq(pool->lock). */ -static void destroy_worker(struct worker *worker) +static void set_worker_dying(struct worker *worker, struct list_head *list) { struct worker_pool *pool = worker->pool; lockdep_assert_held(&pool->lock); + lockdep_assert_held(&wq_pool_attach_mutex); /* sanity check frenzy */ if (WARN_ON(worker->current_work) || @@ -2020,9 +2044,10 @@ static void destroy_worker(struct worker *worker) pool->nr_workers--; pool->nr_idle--; - list_del_init(&worker->entry); worker->flags |= WORKER_DIE; - wake_up_process(worker->task); + + list_move(&worker->entry, list); + list_move(&worker->node, &pool->dying_workers); } /** @@ -2069,11 +2094,24 @@ static void idle_worker_timeout(struct timer_list *t) * * This goes through a pool's idle workers and gets rid of those that have been * idle for at least IDLE_WORKER_TIMEOUT seconds. + * + * We don't want to disturb isolated CPUs because of a pcpu kworker being + * culled, so this also resets worker affinity. This requires a sleepable + * context, hence the split between timer callback and work item. */ static void idle_cull_fn(struct work_struct *work) { struct worker_pool *pool = container_of(work, struct worker_pool, idle_cull_work); + struct list_head cull_list; + INIT_LIST_HEAD(&cull_list); + /* + * Grabbing wq_pool_attach_mutex here ensures an already-running worker + * cannot proceed beyong worker_detach_from_pool() in its self-destruct + * path. This is required as a previously-preempted worker could run after + * set_worker_dying() has happened but before wake_dying_workers() did. + */ + mutex_lock(&wq_pool_attach_mutex); raw_spin_lock_irq(&pool->lock); while (too_many_workers(pool)) { @@ -2088,10 +2126,12 @@ static void idle_cull_fn(struct work_struct *work) break; } - destroy_worker(worker); + set_worker_dying(worker, &cull_list); } raw_spin_unlock_irq(&pool->lock); + wake_dying_workers(&cull_list); + mutex_unlock(&wq_pool_attach_mutex); } static void send_mayday(struct work_struct *work) @@ -2455,12 +2495,12 @@ woke_up: /* am I supposed to die? */ if (unlikely(worker->flags & WORKER_DIE)) { raw_spin_unlock_irq(&pool->lock); - WARN_ON_ONCE(!list_empty(&worker->entry)); set_pf_worker(false); set_task_comm(worker->task, "kworker/dying"); ida_free(&pool->worker_ida, worker->id); worker_detach_from_pool(worker); + WARN_ON_ONCE(!list_empty(&worker->entry)); kfree(worker); return 0; } @@ -3534,6 +3574,7 @@ static int init_worker_pool(struct worker_pool *pool) timer_setup(&pool->mayday_timer, pool_mayday_timeout, 0); INIT_LIST_HEAD(&pool->workers); + INIT_LIST_HEAD(&pool->dying_workers); ida_init(&pool->worker_ida); INIT_HLIST_NODE(&pool->hash_node); @@ -3622,8 +3663,11 @@ static void rcu_free_pool(struct rcu_head *rcu) static void put_unbound_pool(struct worker_pool *pool) { DECLARE_COMPLETION_ONSTACK(detach_completion); + struct list_head cull_list; struct worker *worker; + INIT_LIST_HEAD(&cull_list); + lockdep_assert_held(&wq_pool_mutex); if (--pool->refcnt) @@ -3656,21 +3700,25 @@ static void put_unbound_pool(struct worker_pool *pool) rcuwait_wait_event(&manager_wait, !(pool->flags & POOL_MANAGER_ACTIVE), TASK_UNINTERRUPTIBLE); + + mutex_lock(&wq_pool_attach_mutex); raw_spin_lock_irq(&pool->lock); if (!(pool->flags & POOL_MANAGER_ACTIVE)) { pool->flags |= POOL_MANAGER_ACTIVE; break; } raw_spin_unlock_irq(&pool->lock); + mutex_unlock(&wq_pool_attach_mutex); } while ((worker = first_idle_worker(pool))) - destroy_worker(worker); + set_worker_dying(worker, &cull_list); WARN_ON(pool->nr_workers || pool->nr_idle); raw_spin_unlock_irq(&pool->lock); - mutex_lock(&wq_pool_attach_mutex); - if (!list_empty(&pool->workers)) + wake_dying_workers(&cull_list); + + if (!list_empty(&pool->workers) || !list_empty(&pool->dying_workers)) pool->detach_completion = &detach_completion; mutex_unlock(&wq_pool_attach_mutex); From ccfe1fef9409ca80ffad6ce822a6d15eaee67c91 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Wed, 21 Dec 2022 11:15:43 -0800 Subject: [PATCH 102/239] rcu: Remove redundant call to rcu_boost_kthread_setaffinity() The rcu_boost_kthread_setaffinity() function is invoked at rcutree_online_cpu() and rcutree_offline_cpu() time, early in the online timeline and late in the offline timeline, respectively. It is also invoked from rcutree_dead_cpu(), however, in the absence of userspace manipulations (for which userspace must take responsibility), this call is redundant with that from rcutree_offline_cpu(). This redundancy can be demonstrated by printing out the relevant cpumasks This commit therefore removes the call to rcu_boost_kthread_setaffinity() from rcutree_dead_cpu(). Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney Reviewed-by: Frederic Weisbecker --- kernel/rcu/tree.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 80b84ae285b4..89313c7c17b6 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -4076,15 +4076,10 @@ static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf) */ int rcutree_dead_cpu(unsigned int cpu) { - struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); - struct rcu_node *rnp = rdp->mynode; /* Outgoing CPU's rdp & rnp. */ - if (!IS_ENABLED(CONFIG_HOTPLUG_CPU)) return 0; WRITE_ONCE(rcu_state.n_online_cpus, rcu_state.n_online_cpus - 1); - /* Adjust any no-longer-needed kthreads. */ - rcu_boost_kthread_setaffinity(rnp, -1); // Stop-machine done, so allow nohz_full to disable tick. tick_dep_clear(TICK_DEP_BIT_RCU); return 0; From c63a2e52d5e08f01140d7b76c08a78e15e801f03 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Fri, 13 Jan 2023 17:40:40 +0000 Subject: [PATCH 103/239] workqueue: Fold rebind_worker() within rebind_workers() !CONFIG_SMP builds complain about rebind_worker() being unused. Its only user, rebind_workers() is indeed only defined for CONFIG_SMP, so just fold the two lines back up there. Link: http://lore.kernel.org/r/20230113143102.2e94d74f@canb.auug.org.au Reported-by: Stephen Rothwell Signed-off-by: Valentin Schneider Signed-off-by: Tejun Heo --- kernel/workqueue.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 5dc67aa9d696..b8b541caed48 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -1990,12 +1990,6 @@ static void unbind_worker(struct worker *worker) WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, cpu_possible_mask) < 0); } -static void rebind_worker(struct worker *worker, struct worker_pool *pool) -{ - kthread_set_per_cpu(worker->task, pool->cpu); - WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask) < 0); -} - static void wake_dying_workers(struct list_head *cull_list) { struct worker *worker, *tmp; @@ -5192,8 +5186,11 @@ static void rebind_workers(struct worker_pool *pool) * of all workers first and then clear UNBOUND. As we're called * from CPU_ONLINE, the following shouldn't fail. */ - for_each_pool_worker(worker, pool) - rebind_worker(worker, pool); + for_each_pool_worker(worker, pool) { + kthread_set_per_cpu(worker->task, pool->cpu); + WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, + pool->attrs->cpumask) < 0); + } raw_spin_lock_irq(&pool->lock); From 4edc13ae891ab368a3026cde1dbf263331bc6e77 Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Fri, 9 Dec 2022 17:25:23 +0800 Subject: [PATCH 104/239] cpuidle-haltpoll: select haltpoll governor The haltpoll cpuidle driver should select the haltpoll governor, so as to ensure that they work together. Signed-off-by: Li RongQing [ rjw: Changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/cpuidle/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/cpuidle/Kconfig b/drivers/cpuidle/Kconfig index ff71dd662880..cac5997dca50 100644 --- a/drivers/cpuidle/Kconfig +++ b/drivers/cpuidle/Kconfig @@ -74,6 +74,7 @@ endmenu config HALTPOLL_CPUIDLE tristate "Halt poll cpuidle driver" depends on X86 && KVM_GUEST + select CPU_IDLE_GOV_HALTPOLL default y help This option enables halt poll cpuidle driver, which allows to poll From 450316dc4f41e857c928dfbcc495c3810d4b1928 Mon Sep 17 00:00:00 2001 From: Richard Fitzgerald Date: Tue, 13 Dec 2022 15:54:48 +0000 Subject: [PATCH 105/239] PM: runtime: Document that force_suspend() is incompatible with SMART_SUSPEND pm_runtime_force_suspend() cannot be used with DPM_FLAG_SMART_SUSPEND, so note this in the kerneldoc. If DPM_FLAG_SMART_SUSPEND is set and the PM core cannot skip system resume it will call pm_runtime_active() on the driver. This can lead to an inconsistent state where: pm_runtime_force_suspend() called ->runtime_suspend but device_resume_noirq() called pm_runtime_set_active() This leaves the driver actually suspended but marked as active. Signed-off-by: Richard Fitzgerald Signed-off-by: Rafael J. Wysocki --- drivers/base/power/runtime.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c index 50e726b6c2cf..b29be7d4d7d0 100644 --- a/drivers/base/power/runtime.c +++ b/drivers/base/power/runtime.c @@ -1864,6 +1864,10 @@ static bool pm_runtime_need_not_resume(struct device *dev) * sure the device is put into low power state and it should only be used during * system-wide PM transitions to sleep states. It assumes that the analogous * pm_runtime_force_resume() will be used to resume the device. + * + * Do not use with DPM_FLAG_SMART_SUSPEND as this can lead to an inconsistent + * state where this function has called the ->runtime_suspend callback but the + * PM core marks the driver as runtime active. */ int pm_runtime_force_suspend(struct device *dev) { From 0fb0624b15d21622c214617fda5c05a203b04564 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Sat, 7 Jan 2023 18:12:28 -0800 Subject: [PATCH 106/239] seccomp: fix kernel-doc function name warning Move the ACTION_ONLY() macro so that it is not between the kernel-doc notation and the function definition for seccomp_run_filters(), eliminating a kernel-doc warning: kernel/seccomp.c:400: warning: expecting prototype for seccomp_run_filters(). Prototype was for ACTION_ONLY() instead Signed-off-by: Randy Dunlap Cc: Kees Cook Cc: Andy Lutomirski Cc: Will Drewry Signed-off-by: Kees Cook Link: https://lore.kernel.org/r/20230108021228.15975-1-rdunlap@infradead.org --- kernel/seccomp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/seccomp.c b/kernel/seccomp.c index e9852d1b4a5e..cebf26445f9e 100644 --- a/kernel/seccomp.c +++ b/kernel/seccomp.c @@ -388,6 +388,7 @@ static inline bool seccomp_cache_check_allow(const struct seccomp_filter *sfilte } #endif /* SECCOMP_ARCH_NATIVE */ +#define ACTION_ONLY(ret) ((s32)((ret) & (SECCOMP_RET_ACTION_FULL))) /** * seccomp_run_filters - evaluates all seccomp filters against @sd * @sd: optional seccomp data to be passed to filters @@ -397,7 +398,6 @@ static inline bool seccomp_cache_check_allow(const struct seccomp_filter *sfilte * * Returns valid seccomp BPF response codes. */ -#define ACTION_ONLY(ret) ((s32)((ret) & (SECCOMP_RET_ACTION_FULL))) static u32 seccomp_run_filters(const struct seccomp_data *sd, struct seccomp_filter **match) { From 28a35ac2f5f8719a6451932d7ab6b6b3d264a3aa Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 10 Jan 2023 18:58:05 +0100 Subject: [PATCH 107/239] ACPI: PNP: Introduce list of known non-PNP devices In some cases, PNP device IDs from acpi_pnp_device_ids[] are returned by _CID for devices for which matching platform drivers are present in the kernel and should be bound to them. However, the IDs coming from _CID cause the PNP scan handler to attach to those devices which prevents platform device objects from being created for them. Address this by introducing a list of known non-PNP device IDs into acpi_pnp.c such that if a device ID is there in that list, it cannot be attached to by the PNP scan handler and add the platform runtime update and telemetry device IDs to that list to start with. Reported-by: Chen Yu Signed-off-by: Rafael J. Wysocki Tested-by: Zhang Yang --- drivers/acpi/acpi_pnp.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/drivers/acpi/acpi_pnp.c b/drivers/acpi/acpi_pnp.c index ffdcfcd4a10d..01abf26764b0 100644 --- a/drivers/acpi/acpi_pnp.c +++ b/drivers/acpi/acpi_pnp.c @@ -348,10 +348,22 @@ static bool acpi_pnp_match(const char *idstr, const struct acpi_device_id **matc return false; } +/* + * If one of the device IDs below is present in the list of device IDs of a + * given ACPI device object, the PNP scan handler will not attach to that + * object, because there is a proper non-PNP driver in the kernel for the + * device represented by it. + */ +static const struct acpi_device_id acpi_nonpnp_device_ids[] = { + {"INTC1080"}, + {"INTC1081"}, + {""}, +}; + static int acpi_pnp_attach(struct acpi_device *adev, const struct acpi_device_id *id) { - return 1; + return !!acpi_match_device_ids(adev, acpi_nonpnp_device_ids); } static struct acpi_scan_handler acpi_pnp_handler = { From cdf07f0ea48a3b52f924714d477366ac510ee870 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Tue, 27 Dec 2022 10:29:00 +0800 Subject: [PATCH 108/239] genirq/affinity: Remove the 'firstvec' parameter from irq_build_affinity_masks The 'firstvec' parameter is always same with the parameter of 'startvec', so use 'startvec' directly inside irq_build_affinity_masks(). Signed-off-by: Ming Lei Signed-off-by: Thomas Gleixner Reviewed-by: Christoph Hellwig Reviewed-by: John Garry Reviewed-by: Jens Axboe Link: https://lore.kernel.org/r/20221227022905.352674-2-ming.lei@redhat.com --- kernel/irq/affinity.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/kernel/irq/affinity.c b/kernel/irq/affinity.c index d9a5c1d65a79..3361e36ebaa1 100644 --- a/kernel/irq/affinity.c +++ b/kernel/irq/affinity.c @@ -337,10 +337,10 @@ static int __irq_build_affinity_masks(unsigned int startvec, * 2) spread other possible CPUs on these vectors */ static int irq_build_affinity_masks(unsigned int startvec, unsigned int numvecs, - unsigned int firstvec, struct irq_affinity_desc *masks) { unsigned int curvec = startvec, nr_present = 0, nr_others = 0; + unsigned int firstvec = startvec; cpumask_var_t *node_to_cpumask; cpumask_var_t nmsk, npresmsk; int ret = -ENOMEM; @@ -463,8 +463,7 @@ irq_create_affinity_masks(unsigned int nvecs, struct irq_affinity *affd) unsigned int this_vecs = affd->set_size[i]; int ret; - ret = irq_build_affinity_masks(curvec, this_vecs, - curvec, masks); + ret = irq_build_affinity_masks(curvec, this_vecs, masks); if (ret) { kfree(masks); return NULL; From 1f962d91a15af54301c63febb8ac2ba07aa3654f Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Tue, 27 Dec 2022 10:29:01 +0800 Subject: [PATCH 109/239] genirq/affinity: Pass affinity managed mask array to irq_build_affinity_masks Pass affinity managed mask array to irq_build_affinity_masks() so that the index of the first affinity managed vector is always zero. This allows to simplify the implementation a bit. Signed-off-by: Ming Lei Signed-off-by: Thomas Gleixner Reviewed-by: Christoph Hellwig Reviewed-by: John Garry Reviewed-by: Jens Axboe Link: https://lore.kernel.org/r/20221227022905.352674-3-ming.lei@redhat.com --- kernel/irq/affinity.c | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/kernel/irq/affinity.c b/kernel/irq/affinity.c index 3361e36ebaa1..da6379cd27fd 100644 --- a/kernel/irq/affinity.c +++ b/kernel/irq/affinity.c @@ -246,14 +246,13 @@ static void alloc_nodes_vectors(unsigned int numvecs, static int __irq_build_affinity_masks(unsigned int startvec, unsigned int numvecs, - unsigned int firstvec, cpumask_var_t *node_to_cpumask, const struct cpumask *cpu_mask, struct cpumask *nmsk, struct irq_affinity_desc *masks) { unsigned int i, n, nodes, cpus_per_vec, extra_vecs, done = 0; - unsigned int last_affv = firstvec + numvecs; + unsigned int last_affv = numvecs; unsigned int curvec = startvec; nodemask_t nodemsk = NODE_MASK_NONE; struct node_vectors *node_vectors; @@ -273,7 +272,7 @@ static int __irq_build_affinity_masks(unsigned int startvec, cpumask_and(nmsk, cpu_mask, node_to_cpumask[n]); cpumask_or(&masks[curvec].mask, &masks[curvec].mask, nmsk); if (++curvec == last_affv) - curvec = firstvec; + curvec = 0; } return numvecs; } @@ -321,7 +320,7 @@ static int __irq_build_affinity_masks(unsigned int startvec, * may start anywhere */ if (curvec >= last_affv) - curvec = firstvec; + curvec = 0; irq_spread_init_one(&masks[curvec].mask, nmsk, cpus_per_vec); } @@ -336,11 +335,10 @@ static int __irq_build_affinity_masks(unsigned int startvec, * 1) spread present CPU on these vectors * 2) spread other possible CPUs on these vectors */ -static int irq_build_affinity_masks(unsigned int startvec, unsigned int numvecs, +static int irq_build_affinity_masks(unsigned int numvecs, struct irq_affinity_desc *masks) { - unsigned int curvec = startvec, nr_present = 0, nr_others = 0; - unsigned int firstvec = startvec; + unsigned int curvec = 0, nr_present = 0, nr_others = 0; cpumask_var_t *node_to_cpumask; cpumask_var_t nmsk, npresmsk; int ret = -ENOMEM; @@ -360,9 +358,8 @@ static int irq_build_affinity_masks(unsigned int startvec, unsigned int numvecs, build_node_to_cpumask(node_to_cpumask); /* Spread on present CPUs starting from affd->pre_vectors */ - ret = __irq_build_affinity_masks(curvec, numvecs, firstvec, - node_to_cpumask, cpu_present_mask, - nmsk, masks); + ret = __irq_build_affinity_masks(curvec, numvecs, node_to_cpumask, + cpu_present_mask, nmsk, masks); if (ret < 0) goto fail_build_affinity; nr_present = ret; @@ -374,13 +371,12 @@ static int irq_build_affinity_masks(unsigned int startvec, unsigned int numvecs, * out vectors. */ if (nr_present >= numvecs) - curvec = firstvec; + curvec = 0; else - curvec = firstvec + nr_present; + curvec = nr_present; cpumask_andnot(npresmsk, cpu_possible_mask, cpu_present_mask); - ret = __irq_build_affinity_masks(curvec, numvecs, firstvec, - node_to_cpumask, npresmsk, nmsk, - masks); + ret = __irq_build_affinity_masks(curvec, numvecs, node_to_cpumask, + npresmsk, nmsk, masks); if (ret >= 0) nr_others = ret; @@ -463,7 +459,7 @@ irq_create_affinity_masks(unsigned int nvecs, struct irq_affinity *affd) unsigned int this_vecs = affd->set_size[i]; int ret; - ret = irq_build_affinity_masks(curvec, this_vecs, masks); + ret = irq_build_affinity_masks(this_vecs, &masks[curvec]); if (ret) { kfree(masks); return NULL; From e7bdd7f0cbd1c001bb9b4d3313edc5ee094bc3f8 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Tue, 27 Dec 2022 10:29:02 +0800 Subject: [PATCH 110/239] genirq/affinity: Don't pass irq_affinity_desc array to irq_build_affinity_masks Prepare for abstracting irq_build_affinity_masks() into a public function for assigning all CPUs evenly into several groups. Don't pass irq_affinity_desc array to irq_build_affinity_masks, instead return a cpumask array by storing each assigned group into one element of the array. This allows to provide a generic interface for grouping all CPUs evenly from a NUMA and CPU locality viewpoint, and the cost is one extra allocation in irq_build_affinity_masks(), which should be fine since it is done via GFP_KERNEL and irq_build_affinity_masks() is a slow path anyway. Signed-off-by: Ming Lei Signed-off-by: Thomas Gleixner Reviewed-by: Christoph Hellwig Reviewed-by: John Garry Reviewed-by: Jens Axboe Link: https://lore.kernel.org/r/20221227022905.352674-4-ming.lei@redhat.com --- kernel/irq/affinity.c | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/kernel/irq/affinity.c b/kernel/irq/affinity.c index da6379cd27fd..00bba1020ecb 100644 --- a/kernel/irq/affinity.c +++ b/kernel/irq/affinity.c @@ -249,7 +249,7 @@ static int __irq_build_affinity_masks(unsigned int startvec, cpumask_var_t *node_to_cpumask, const struct cpumask *cpu_mask, struct cpumask *nmsk, - struct irq_affinity_desc *masks) + struct cpumask *masks) { unsigned int i, n, nodes, cpus_per_vec, extra_vecs, done = 0; unsigned int last_affv = numvecs; @@ -270,7 +270,7 @@ static int __irq_build_affinity_masks(unsigned int startvec, for_each_node_mask(n, nodemsk) { /* Ensure that only CPUs which are in both masks are set */ cpumask_and(nmsk, cpu_mask, node_to_cpumask[n]); - cpumask_or(&masks[curvec].mask, &masks[curvec].mask, nmsk); + cpumask_or(&masks[curvec], &masks[curvec], nmsk); if (++curvec == last_affv) curvec = 0; } @@ -321,7 +321,7 @@ static int __irq_build_affinity_masks(unsigned int startvec, */ if (curvec >= last_affv) curvec = 0; - irq_spread_init_one(&masks[curvec].mask, nmsk, + irq_spread_init_one(&masks[curvec], nmsk, cpus_per_vec); } done += nv->nvectors; @@ -335,16 +335,16 @@ static int __irq_build_affinity_masks(unsigned int startvec, * 1) spread present CPU on these vectors * 2) spread other possible CPUs on these vectors */ -static int irq_build_affinity_masks(unsigned int numvecs, - struct irq_affinity_desc *masks) +static struct cpumask *irq_build_affinity_masks(unsigned int numvecs) { unsigned int curvec = 0, nr_present = 0, nr_others = 0; cpumask_var_t *node_to_cpumask; cpumask_var_t nmsk, npresmsk; int ret = -ENOMEM; + struct cpumask *masks = NULL; if (!zalloc_cpumask_var(&nmsk, GFP_KERNEL)) - return ret; + return NULL; if (!zalloc_cpumask_var(&npresmsk, GFP_KERNEL)) goto fail_nmsk; @@ -353,6 +353,10 @@ static int irq_build_affinity_masks(unsigned int numvecs, if (!node_to_cpumask) goto fail_npresmsk; + masks = kcalloc(numvecs, sizeof(*masks), GFP_KERNEL); + if (!masks) + goto fail_node_to_cpumask; + /* Stabilize the cpumasks */ cpus_read_lock(); build_node_to_cpumask(node_to_cpumask); @@ -386,6 +390,7 @@ static int irq_build_affinity_masks(unsigned int numvecs, if (ret >= 0) WARN_ON(nr_present + nr_others < numvecs); + fail_node_to_cpumask: free_node_to_cpumask(node_to_cpumask); fail_npresmsk: @@ -393,7 +398,11 @@ static int irq_build_affinity_masks(unsigned int numvecs, fail_nmsk: free_cpumask_var(nmsk); - return ret < 0 ? ret : 0; + if (ret < 0) { + kfree(masks); + return NULL; + } + return masks; } static void default_calc_sets(struct irq_affinity *affd, unsigned int affvecs) @@ -457,13 +466,18 @@ irq_create_affinity_masks(unsigned int nvecs, struct irq_affinity *affd) */ for (i = 0, usedvecs = 0; i < affd->nr_sets; i++) { unsigned int this_vecs = affd->set_size[i]; - int ret; + int j; + struct cpumask *result = irq_build_affinity_masks(this_vecs); - ret = irq_build_affinity_masks(this_vecs, &masks[curvec]); - if (ret) { + if (!result) { kfree(masks); return NULL; } + + for (j = 0; j < this_vecs; j++) + cpumask_copy(&masks[curvec + j].mask, &result[j]); + kfree(result); + curvec += this_vecs; usedvecs += this_vecs; } From 523f1ea76aad9025f9bd5258d77f4406fa9dbe5d Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Tue, 27 Dec 2022 10:29:03 +0800 Subject: [PATCH 111/239] genirq/affinity: Rename irq_build_affinity_masks as group_cpus_evenly Map irq vector into group, which allows to abstract the algorithm for a generic use case outside of the interrupt core. Rename irq_build_affinity_masks as group_cpus_evenly, so the API can be reused for blk-mq to make default queue mapping even though irq vectors aren't involved. No functional change, just rename vector as group. Signed-off-by: Ming Lei Signed-off-by: Thomas Gleixner Reviewed-by: Christoph Hellwig Reviewed-by: Jens Axboe Link: https://lore.kernel.org/r/20221227022905.352674-5-ming.lei@redhat.com --- kernel/irq/affinity.c | 242 +++++++++++++++++++++--------------------- 1 file changed, 121 insertions(+), 121 deletions(-) diff --git a/kernel/irq/affinity.c b/kernel/irq/affinity.c index 00bba1020ecb..54083331f1bc 100644 --- a/kernel/irq/affinity.c +++ b/kernel/irq/affinity.c @@ -9,13 +9,13 @@ #include #include -static void irq_spread_init_one(struct cpumask *irqmsk, struct cpumask *nmsk, - unsigned int cpus_per_vec) +static void grp_spread_init_one(struct cpumask *irqmsk, struct cpumask *nmsk, + unsigned int cpus_per_grp) { const struct cpumask *siblmsk; int cpu, sibl; - for ( ; cpus_per_vec > 0; ) { + for ( ; cpus_per_grp > 0; ) { cpu = cpumask_first(nmsk); /* Should not happen, but I'm too lazy to think about it */ @@ -24,18 +24,18 @@ static void irq_spread_init_one(struct cpumask *irqmsk, struct cpumask *nmsk, cpumask_clear_cpu(cpu, nmsk); cpumask_set_cpu(cpu, irqmsk); - cpus_per_vec--; + cpus_per_grp--; /* If the cpu has siblings, use them first */ siblmsk = topology_sibling_cpumask(cpu); - for (sibl = -1; cpus_per_vec > 0; ) { + for (sibl = -1; cpus_per_grp > 0; ) { sibl = cpumask_next(sibl, siblmsk); if (sibl >= nr_cpu_ids) break; if (!cpumask_test_and_clear_cpu(sibl, nmsk)) continue; cpumask_set_cpu(sibl, irqmsk); - cpus_per_vec--; + cpus_per_grp--; } } } @@ -95,48 +95,48 @@ static int get_nodes_in_cpumask(cpumask_var_t *node_to_cpumask, return nodes; } -struct node_vectors { +struct node_groups { unsigned id; union { - unsigned nvectors; + unsigned ngroups; unsigned ncpus; }; }; static int ncpus_cmp_func(const void *l, const void *r) { - const struct node_vectors *ln = l; - const struct node_vectors *rn = r; + const struct node_groups *ln = l; + const struct node_groups *rn = r; return ln->ncpus - rn->ncpus; } /* - * Allocate vector number for each node, so that for each node: + * Allocate group number for each node, so that for each node: * * 1) the allocated number is >= 1 * - * 2) the allocated numbver is <= active CPU number of this node + * 2) the allocated number is <= active CPU number of this node * - * The actual allocated total vectors may be less than @numvecs when - * active total CPU number is less than @numvecs. + * The actual allocated total groups may be less than @numgrps when + * active total CPU number is less than @numgrps. * * Active CPUs means the CPUs in '@cpu_mask AND @node_to_cpumask[]' * for each node. */ -static void alloc_nodes_vectors(unsigned int numvecs, - cpumask_var_t *node_to_cpumask, - const struct cpumask *cpu_mask, - const nodemask_t nodemsk, - struct cpumask *nmsk, - struct node_vectors *node_vectors) +static void alloc_nodes_groups(unsigned int numgrps, + cpumask_var_t *node_to_cpumask, + const struct cpumask *cpu_mask, + const nodemask_t nodemsk, + struct cpumask *nmsk, + struct node_groups *node_groups) { unsigned n, remaining_ncpus = 0; for (n = 0; n < nr_node_ids; n++) { - node_vectors[n].id = n; - node_vectors[n].ncpus = UINT_MAX; + node_groups[n].id = n; + node_groups[n].ncpus = UINT_MAX; } for_each_node_mask(n, nodemsk) { @@ -148,61 +148,61 @@ static void alloc_nodes_vectors(unsigned int numvecs, if (!ncpus) continue; remaining_ncpus += ncpus; - node_vectors[n].ncpus = ncpus; + node_groups[n].ncpus = ncpus; } - numvecs = min_t(unsigned, remaining_ncpus, numvecs); + numgrps = min_t(unsigned, remaining_ncpus, numgrps); - sort(node_vectors, nr_node_ids, sizeof(node_vectors[0]), + sort(node_groups, nr_node_ids, sizeof(node_groups[0]), ncpus_cmp_func, NULL); /* - * Allocate vectors for each node according to the ratio of this - * node's nr_cpus to remaining un-assigned ncpus. 'numvecs' is + * Allocate groups for each node according to the ratio of this + * node's nr_cpus to remaining un-assigned ncpus. 'numgrps' is * bigger than number of active numa nodes. Always start the * allocation from the node with minimized nr_cpus. * * This way guarantees that each active node gets allocated at - * least one vector, and the theory is simple: over-allocation - * is only done when this node is assigned by one vector, so - * other nodes will be allocated >= 1 vector, since 'numvecs' is + * least one group, and the theory is simple: over-allocation + * is only done when this node is assigned by one group, so + * other nodes will be allocated >= 1 groups, since 'numgrps' is * bigger than number of numa nodes. * - * One perfect invariant is that number of allocated vectors for + * One perfect invariant is that number of allocated groups for * each node is <= CPU count of this node: * * 1) suppose there are two nodes: A and B * ncpu(X) is CPU count of node X - * vecs(X) is the vector count allocated to node X via this + * grps(X) is the group count allocated to node X via this * algorithm * * ncpu(A) <= ncpu(B) * ncpu(A) + ncpu(B) = N - * vecs(A) + vecs(B) = V + * grps(A) + grps(B) = G * - * vecs(A) = max(1, round_down(V * ncpu(A) / N)) - * vecs(B) = V - vecs(A) + * grps(A) = max(1, round_down(G * ncpu(A) / N)) + * grps(B) = G - grps(A) * - * both N and V are integer, and 2 <= V <= N, suppose - * V = N - delta, and 0 <= delta <= N - 2 + * both N and G are integer, and 2 <= G <= N, suppose + * G = N - delta, and 0 <= delta <= N - 2 * - * 2) obviously vecs(A) <= ncpu(A) because: + * 2) obviously grps(A) <= ncpu(A) because: * - * if vecs(A) is 1, then vecs(A) <= ncpu(A) given + * if grps(A) is 1, then grps(A) <= ncpu(A) given * ncpu(A) >= 1 * * otherwise, - * vecs(A) <= V * ncpu(A) / N <= ncpu(A), given V <= N + * grps(A) <= G * ncpu(A) / N <= ncpu(A), given G <= N * - * 3) prove how vecs(B) <= ncpu(B): + * 3) prove how grps(B) <= ncpu(B): * - * if round_down(V * ncpu(A) / N) == 0, vecs(B) won't be - * over-allocated, so vecs(B) <= ncpu(B), + * if round_down(G * ncpu(A) / N) == 0, vecs(B) won't be + * over-allocated, so grps(B) <= ncpu(B), * * otherwise: * - * vecs(A) = - * round_down(V * ncpu(A) / N) = + * grps(A) = + * round_down(G * ncpu(A) / N) = * round_down((N - delta) * ncpu(A) / N) = * round_down((N * ncpu(A) - delta * ncpu(A)) / N) >= * round_down((N * ncpu(A) - delta * N) / N) = @@ -210,52 +210,50 @@ static void alloc_nodes_vectors(unsigned int numvecs, * * then: * - * vecs(A) - V >= ncpu(A) - delta - V + * grps(A) - G >= ncpu(A) - delta - G * => - * V - vecs(A) <= V + delta - ncpu(A) + * G - grps(A) <= G + delta - ncpu(A) * => - * vecs(B) <= N - ncpu(A) + * grps(B) <= N - ncpu(A) * => - * vecs(B) <= cpu(B) + * grps(B) <= cpu(B) * * For nodes >= 3, it can be thought as one node and another big * node given that is exactly what this algorithm is implemented, - * and we always re-calculate 'remaining_ncpus' & 'numvecs', and - * finally for each node X: vecs(X) <= ncpu(X). + * and we always re-calculate 'remaining_ncpus' & 'numgrps', and + * finally for each node X: grps(X) <= ncpu(X). * */ for (n = 0; n < nr_node_ids; n++) { - unsigned nvectors, ncpus; + unsigned ngroups, ncpus; - if (node_vectors[n].ncpus == UINT_MAX) + if (node_groups[n].ncpus == UINT_MAX) continue; - WARN_ON_ONCE(numvecs == 0); + WARN_ON_ONCE(numgrps == 0); - ncpus = node_vectors[n].ncpus; - nvectors = max_t(unsigned, 1, - numvecs * ncpus / remaining_ncpus); - WARN_ON_ONCE(nvectors > ncpus); + ncpus = node_groups[n].ncpus; + ngroups = max_t(unsigned, 1, + numgrps * ncpus / remaining_ncpus); + WARN_ON_ONCE(ngroups > ncpus); - node_vectors[n].nvectors = nvectors; + node_groups[n].ngroups = ngroups; remaining_ncpus -= ncpus; - numvecs -= nvectors; + numgrps -= ngroups; } } -static int __irq_build_affinity_masks(unsigned int startvec, - unsigned int numvecs, - cpumask_var_t *node_to_cpumask, - const struct cpumask *cpu_mask, - struct cpumask *nmsk, - struct cpumask *masks) +static int __group_cpus_evenly(unsigned int startgrp, unsigned int numgrps, + cpumask_var_t *node_to_cpumask, + const struct cpumask *cpu_mask, + struct cpumask *nmsk, struct cpumask *masks) { - unsigned int i, n, nodes, cpus_per_vec, extra_vecs, done = 0; - unsigned int last_affv = numvecs; - unsigned int curvec = startvec; + unsigned int i, n, nodes, cpus_per_grp, extra_grps, done = 0; + unsigned int last_grp = numgrps; + unsigned int curgrp = startgrp; nodemask_t nodemsk = NODE_MASK_NONE; - struct node_vectors *node_vectors; + struct node_groups *node_groups; if (cpumask_empty(cpu_mask)) return 0; @@ -264,34 +262,33 @@ static int __irq_build_affinity_masks(unsigned int startvec, /* * If the number of nodes in the mask is greater than or equal the - * number of vectors we just spread the vectors across the nodes. + * number of groups we just spread the groups across the nodes. */ - if (numvecs <= nodes) { + if (numgrps <= nodes) { for_each_node_mask(n, nodemsk) { /* Ensure that only CPUs which are in both masks are set */ cpumask_and(nmsk, cpu_mask, node_to_cpumask[n]); - cpumask_or(&masks[curvec], &masks[curvec], nmsk); - if (++curvec == last_affv) - curvec = 0; + cpumask_or(&masks[curgrp], &masks[curgrp], nmsk); + if (++curgrp == last_grp) + curgrp = 0; } - return numvecs; + return numgrps; } - node_vectors = kcalloc(nr_node_ids, - sizeof(struct node_vectors), + node_groups = kcalloc(nr_node_ids, + sizeof(struct node_groups), GFP_KERNEL); - if (!node_vectors) + if (!node_groups) return -ENOMEM; - /* allocate vector number for each node */ - alloc_nodes_vectors(numvecs, node_to_cpumask, cpu_mask, - nodemsk, nmsk, node_vectors); - + /* allocate group number for each node */ + alloc_nodes_groups(numgrps, node_to_cpumask, cpu_mask, + nodemsk, nmsk, node_groups); for (i = 0; i < nr_node_ids; i++) { unsigned int ncpus, v; - struct node_vectors *nv = &node_vectors[i]; + struct node_groups *nv = &node_groups[i]; - if (nv->nvectors == UINT_MAX) + if (nv->ngroups == UINT_MAX) continue; /* Get the cpus on this node which are in the mask */ @@ -300,44 +297,47 @@ static int __irq_build_affinity_masks(unsigned int startvec, if (!ncpus) continue; - WARN_ON_ONCE(nv->nvectors > ncpus); + WARN_ON_ONCE(nv->ngroups > ncpus); /* Account for rounding errors */ - extra_vecs = ncpus - nv->nvectors * (ncpus / nv->nvectors); + extra_grps = ncpus - nv->ngroups * (ncpus / nv->ngroups); - /* Spread allocated vectors on CPUs of the current node */ - for (v = 0; v < nv->nvectors; v++, curvec++) { - cpus_per_vec = ncpus / nv->nvectors; + /* Spread allocated groups on CPUs of the current node */ + for (v = 0; v < nv->ngroups; v++, curgrp++) { + cpus_per_grp = ncpus / nv->ngroups; - /* Account for extra vectors to compensate rounding errors */ - if (extra_vecs) { - cpus_per_vec++; - --extra_vecs; + /* Account for extra groups to compensate rounding errors */ + if (extra_grps) { + cpus_per_grp++; + --extra_grps; } /* - * wrapping has to be considered given 'startvec' + * wrapping has to be considered given 'startgrp' * may start anywhere */ - if (curvec >= last_affv) - curvec = 0; - irq_spread_init_one(&masks[curvec], nmsk, - cpus_per_vec); + if (curgrp >= last_grp) + curgrp = 0; + grp_spread_init_one(&masks[curgrp], nmsk, + cpus_per_grp); } - done += nv->nvectors; + done += nv->ngroups; } - kfree(node_vectors); + kfree(node_groups); return done; } /* - * build affinity in two stages: - * 1) spread present CPU on these vectors - * 2) spread other possible CPUs on these vectors + * build affinity in two stages for each group, and try to put close CPUs + * in viewpoint of CPU and NUMA locality into same group, and we run + * two-stage grouping: + * + * 1) allocate present CPUs on these groups evenly first + * 2) allocate other possible CPUs on these groups evenly */ -static struct cpumask *irq_build_affinity_masks(unsigned int numvecs) +static struct cpumask *group_cpus_evenly(unsigned int numgrps) { - unsigned int curvec = 0, nr_present = 0, nr_others = 0; + unsigned int curgrp = 0, nr_present = 0, nr_others = 0; cpumask_var_t *node_to_cpumask; cpumask_var_t nmsk, npresmsk; int ret = -ENOMEM; @@ -353,7 +353,7 @@ static struct cpumask *irq_build_affinity_masks(unsigned int numvecs) if (!node_to_cpumask) goto fail_npresmsk; - masks = kcalloc(numvecs, sizeof(*masks), GFP_KERNEL); + masks = kcalloc(numgrps, sizeof(*masks), GFP_KERNEL); if (!masks) goto fail_node_to_cpumask; @@ -361,26 +361,26 @@ static struct cpumask *irq_build_affinity_masks(unsigned int numvecs) cpus_read_lock(); build_node_to_cpumask(node_to_cpumask); - /* Spread on present CPUs starting from affd->pre_vectors */ - ret = __irq_build_affinity_masks(curvec, numvecs, node_to_cpumask, - cpu_present_mask, nmsk, masks); + /* grouping present CPUs first */ + ret = __group_cpus_evenly(curgrp, numgrps, node_to_cpumask, + cpu_present_mask, nmsk, masks); if (ret < 0) goto fail_build_affinity; nr_present = ret; /* - * Spread on non present CPUs starting from the next vector to be - * handled. If the spreading of present CPUs already exhausted the - * vector space, assign the non present CPUs to the already spread - * out vectors. + * Allocate non present CPUs starting from the next group to be + * handled. If the grouping of present CPUs already exhausted the + * group space, assign the non present CPUs to the already + * allocated out groups. */ - if (nr_present >= numvecs) - curvec = 0; + if (nr_present >= numgrps) + curgrp = 0; else - curvec = nr_present; + curgrp = nr_present; cpumask_andnot(npresmsk, cpu_possible_mask, cpu_present_mask); - ret = __irq_build_affinity_masks(curvec, numvecs, node_to_cpumask, - npresmsk, nmsk, masks); + ret = __group_cpus_evenly(curgrp, numgrps, node_to_cpumask, + npresmsk, nmsk, masks); if (ret >= 0) nr_others = ret; @@ -388,7 +388,7 @@ static struct cpumask *irq_build_affinity_masks(unsigned int numvecs) cpus_read_unlock(); if (ret >= 0) - WARN_ON(nr_present + nr_others < numvecs); + WARN_ON(nr_present + nr_others < numgrps); fail_node_to_cpumask: free_node_to_cpumask(node_to_cpumask); @@ -467,7 +467,7 @@ irq_create_affinity_masks(unsigned int nvecs, struct irq_affinity *affd) for (i = 0, usedvecs = 0; i < affd->nr_sets; i++) { unsigned int this_vecs = affd->set_size[i]; int j; - struct cpumask *result = irq_build_affinity_masks(this_vecs); + struct cpumask *result = group_cpus_evenly(this_vecs); if (!result) { kfree(masks); From f7b3ea8cf72f3d6060fe08e461805181e7450a13 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Tue, 27 Dec 2022 10:29:04 +0800 Subject: [PATCH 112/239] genirq/affinity: Move group_cpus_evenly() into lib/ group_cpus_evenly() has become a generic function which can be used for other subsystems than the interrupt subsystem, so move it into lib/. Signed-off-by: Ming Lei Signed-off-by: Thomas Gleixner Reviewed-by: Christoph Hellwig Reviewed-by: Jens Axboe Link: https://lore.kernel.org/r/20221227022905.352674-6-ming.lei@redhat.com --- MAINTAINERS | 2 + include/linux/group_cpus.h | 14 ++ kernel/irq/affinity.c | 398 +--------------------------------- lib/Makefile | 2 + lib/group_cpus.c | 427 +++++++++++++++++++++++++++++++++++++ 5 files changed, 446 insertions(+), 397 deletions(-) create mode 100644 include/linux/group_cpus.h create mode 100644 lib/group_cpus.c diff --git a/MAINTAINERS b/MAINTAINERS index a36df9ed283d..9a07bd4f097f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -10935,6 +10935,8 @@ L: linux-kernel@vger.kernel.org S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git irq/core F: kernel/irq/ +F: include/linux/group_cpus.h +F: lib/group_cpus.c IRQCHIP DRIVERS M: Thomas Gleixner diff --git a/include/linux/group_cpus.h b/include/linux/group_cpus.h new file mode 100644 index 000000000000..e42807ec61f6 --- /dev/null +++ b/include/linux/group_cpus.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2016 Thomas Gleixner. + * Copyright (C) 2016-2017 Christoph Hellwig. + */ + +#ifndef __LINUX_GROUP_CPUS_H +#define __LINUX_GROUP_CPUS_H +#include +#include + +struct cpumask *group_cpus_evenly(unsigned int numgrps); + +#endif diff --git a/kernel/irq/affinity.c b/kernel/irq/affinity.c index 54083331f1bc..44a4eba80315 100644 --- a/kernel/irq/affinity.c +++ b/kernel/irq/affinity.c @@ -7,403 +7,7 @@ #include #include #include -#include - -static void grp_spread_init_one(struct cpumask *irqmsk, struct cpumask *nmsk, - unsigned int cpus_per_grp) -{ - const struct cpumask *siblmsk; - int cpu, sibl; - - for ( ; cpus_per_grp > 0; ) { - cpu = cpumask_first(nmsk); - - /* Should not happen, but I'm too lazy to think about it */ - if (cpu >= nr_cpu_ids) - return; - - cpumask_clear_cpu(cpu, nmsk); - cpumask_set_cpu(cpu, irqmsk); - cpus_per_grp--; - - /* If the cpu has siblings, use them first */ - siblmsk = topology_sibling_cpumask(cpu); - for (sibl = -1; cpus_per_grp > 0; ) { - sibl = cpumask_next(sibl, siblmsk); - if (sibl >= nr_cpu_ids) - break; - if (!cpumask_test_and_clear_cpu(sibl, nmsk)) - continue; - cpumask_set_cpu(sibl, irqmsk); - cpus_per_grp--; - } - } -} - -static cpumask_var_t *alloc_node_to_cpumask(void) -{ - cpumask_var_t *masks; - int node; - - masks = kcalloc(nr_node_ids, sizeof(cpumask_var_t), GFP_KERNEL); - if (!masks) - return NULL; - - for (node = 0; node < nr_node_ids; node++) { - if (!zalloc_cpumask_var(&masks[node], GFP_KERNEL)) - goto out_unwind; - } - - return masks; - -out_unwind: - while (--node >= 0) - free_cpumask_var(masks[node]); - kfree(masks); - return NULL; -} - -static void free_node_to_cpumask(cpumask_var_t *masks) -{ - int node; - - for (node = 0; node < nr_node_ids; node++) - free_cpumask_var(masks[node]); - kfree(masks); -} - -static void build_node_to_cpumask(cpumask_var_t *masks) -{ - int cpu; - - for_each_possible_cpu(cpu) - cpumask_set_cpu(cpu, masks[cpu_to_node(cpu)]); -} - -static int get_nodes_in_cpumask(cpumask_var_t *node_to_cpumask, - const struct cpumask *mask, nodemask_t *nodemsk) -{ - int n, nodes = 0; - - /* Calculate the number of nodes in the supplied affinity mask */ - for_each_node(n) { - if (cpumask_intersects(mask, node_to_cpumask[n])) { - node_set(n, *nodemsk); - nodes++; - } - } - return nodes; -} - -struct node_groups { - unsigned id; - - union { - unsigned ngroups; - unsigned ncpus; - }; -}; - -static int ncpus_cmp_func(const void *l, const void *r) -{ - const struct node_groups *ln = l; - const struct node_groups *rn = r; - - return ln->ncpus - rn->ncpus; -} - -/* - * Allocate group number for each node, so that for each node: - * - * 1) the allocated number is >= 1 - * - * 2) the allocated number is <= active CPU number of this node - * - * The actual allocated total groups may be less than @numgrps when - * active total CPU number is less than @numgrps. - * - * Active CPUs means the CPUs in '@cpu_mask AND @node_to_cpumask[]' - * for each node. - */ -static void alloc_nodes_groups(unsigned int numgrps, - cpumask_var_t *node_to_cpumask, - const struct cpumask *cpu_mask, - const nodemask_t nodemsk, - struct cpumask *nmsk, - struct node_groups *node_groups) -{ - unsigned n, remaining_ncpus = 0; - - for (n = 0; n < nr_node_ids; n++) { - node_groups[n].id = n; - node_groups[n].ncpus = UINT_MAX; - } - - for_each_node_mask(n, nodemsk) { - unsigned ncpus; - - cpumask_and(nmsk, cpu_mask, node_to_cpumask[n]); - ncpus = cpumask_weight(nmsk); - - if (!ncpus) - continue; - remaining_ncpus += ncpus; - node_groups[n].ncpus = ncpus; - } - - numgrps = min_t(unsigned, remaining_ncpus, numgrps); - - sort(node_groups, nr_node_ids, sizeof(node_groups[0]), - ncpus_cmp_func, NULL); - - /* - * Allocate groups for each node according to the ratio of this - * node's nr_cpus to remaining un-assigned ncpus. 'numgrps' is - * bigger than number of active numa nodes. Always start the - * allocation from the node with minimized nr_cpus. - * - * This way guarantees that each active node gets allocated at - * least one group, and the theory is simple: over-allocation - * is only done when this node is assigned by one group, so - * other nodes will be allocated >= 1 groups, since 'numgrps' is - * bigger than number of numa nodes. - * - * One perfect invariant is that number of allocated groups for - * each node is <= CPU count of this node: - * - * 1) suppose there are two nodes: A and B - * ncpu(X) is CPU count of node X - * grps(X) is the group count allocated to node X via this - * algorithm - * - * ncpu(A) <= ncpu(B) - * ncpu(A) + ncpu(B) = N - * grps(A) + grps(B) = G - * - * grps(A) = max(1, round_down(G * ncpu(A) / N)) - * grps(B) = G - grps(A) - * - * both N and G are integer, and 2 <= G <= N, suppose - * G = N - delta, and 0 <= delta <= N - 2 - * - * 2) obviously grps(A) <= ncpu(A) because: - * - * if grps(A) is 1, then grps(A) <= ncpu(A) given - * ncpu(A) >= 1 - * - * otherwise, - * grps(A) <= G * ncpu(A) / N <= ncpu(A), given G <= N - * - * 3) prove how grps(B) <= ncpu(B): - * - * if round_down(G * ncpu(A) / N) == 0, vecs(B) won't be - * over-allocated, so grps(B) <= ncpu(B), - * - * otherwise: - * - * grps(A) = - * round_down(G * ncpu(A) / N) = - * round_down((N - delta) * ncpu(A) / N) = - * round_down((N * ncpu(A) - delta * ncpu(A)) / N) >= - * round_down((N * ncpu(A) - delta * N) / N) = - * cpu(A) - delta - * - * then: - * - * grps(A) - G >= ncpu(A) - delta - G - * => - * G - grps(A) <= G + delta - ncpu(A) - * => - * grps(B) <= N - ncpu(A) - * => - * grps(B) <= cpu(B) - * - * For nodes >= 3, it can be thought as one node and another big - * node given that is exactly what this algorithm is implemented, - * and we always re-calculate 'remaining_ncpus' & 'numgrps', and - * finally for each node X: grps(X) <= ncpu(X). - * - */ - for (n = 0; n < nr_node_ids; n++) { - unsigned ngroups, ncpus; - - if (node_groups[n].ncpus == UINT_MAX) - continue; - - WARN_ON_ONCE(numgrps == 0); - - ncpus = node_groups[n].ncpus; - ngroups = max_t(unsigned, 1, - numgrps * ncpus / remaining_ncpus); - WARN_ON_ONCE(ngroups > ncpus); - - node_groups[n].ngroups = ngroups; - - remaining_ncpus -= ncpus; - numgrps -= ngroups; - } -} - -static int __group_cpus_evenly(unsigned int startgrp, unsigned int numgrps, - cpumask_var_t *node_to_cpumask, - const struct cpumask *cpu_mask, - struct cpumask *nmsk, struct cpumask *masks) -{ - unsigned int i, n, nodes, cpus_per_grp, extra_grps, done = 0; - unsigned int last_grp = numgrps; - unsigned int curgrp = startgrp; - nodemask_t nodemsk = NODE_MASK_NONE; - struct node_groups *node_groups; - - if (cpumask_empty(cpu_mask)) - return 0; - - nodes = get_nodes_in_cpumask(node_to_cpumask, cpu_mask, &nodemsk); - - /* - * If the number of nodes in the mask is greater than or equal the - * number of groups we just spread the groups across the nodes. - */ - if (numgrps <= nodes) { - for_each_node_mask(n, nodemsk) { - /* Ensure that only CPUs which are in both masks are set */ - cpumask_and(nmsk, cpu_mask, node_to_cpumask[n]); - cpumask_or(&masks[curgrp], &masks[curgrp], nmsk); - if (++curgrp == last_grp) - curgrp = 0; - } - return numgrps; - } - - node_groups = kcalloc(nr_node_ids, - sizeof(struct node_groups), - GFP_KERNEL); - if (!node_groups) - return -ENOMEM; - - /* allocate group number for each node */ - alloc_nodes_groups(numgrps, node_to_cpumask, cpu_mask, - nodemsk, nmsk, node_groups); - for (i = 0; i < nr_node_ids; i++) { - unsigned int ncpus, v; - struct node_groups *nv = &node_groups[i]; - - if (nv->ngroups == UINT_MAX) - continue; - - /* Get the cpus on this node which are in the mask */ - cpumask_and(nmsk, cpu_mask, node_to_cpumask[nv->id]); - ncpus = cpumask_weight(nmsk); - if (!ncpus) - continue; - - WARN_ON_ONCE(nv->ngroups > ncpus); - - /* Account for rounding errors */ - extra_grps = ncpus - nv->ngroups * (ncpus / nv->ngroups); - - /* Spread allocated groups on CPUs of the current node */ - for (v = 0; v < nv->ngroups; v++, curgrp++) { - cpus_per_grp = ncpus / nv->ngroups; - - /* Account for extra groups to compensate rounding errors */ - if (extra_grps) { - cpus_per_grp++; - --extra_grps; - } - - /* - * wrapping has to be considered given 'startgrp' - * may start anywhere - */ - if (curgrp >= last_grp) - curgrp = 0; - grp_spread_init_one(&masks[curgrp], nmsk, - cpus_per_grp); - } - done += nv->ngroups; - } - kfree(node_groups); - return done; -} - -/* - * build affinity in two stages for each group, and try to put close CPUs - * in viewpoint of CPU and NUMA locality into same group, and we run - * two-stage grouping: - * - * 1) allocate present CPUs on these groups evenly first - * 2) allocate other possible CPUs on these groups evenly - */ -static struct cpumask *group_cpus_evenly(unsigned int numgrps) -{ - unsigned int curgrp = 0, nr_present = 0, nr_others = 0; - cpumask_var_t *node_to_cpumask; - cpumask_var_t nmsk, npresmsk; - int ret = -ENOMEM; - struct cpumask *masks = NULL; - - if (!zalloc_cpumask_var(&nmsk, GFP_KERNEL)) - return NULL; - - if (!zalloc_cpumask_var(&npresmsk, GFP_KERNEL)) - goto fail_nmsk; - - node_to_cpumask = alloc_node_to_cpumask(); - if (!node_to_cpumask) - goto fail_npresmsk; - - masks = kcalloc(numgrps, sizeof(*masks), GFP_KERNEL); - if (!masks) - goto fail_node_to_cpumask; - - /* Stabilize the cpumasks */ - cpus_read_lock(); - build_node_to_cpumask(node_to_cpumask); - - /* grouping present CPUs first */ - ret = __group_cpus_evenly(curgrp, numgrps, node_to_cpumask, - cpu_present_mask, nmsk, masks); - if (ret < 0) - goto fail_build_affinity; - nr_present = ret; - - /* - * Allocate non present CPUs starting from the next group to be - * handled. If the grouping of present CPUs already exhausted the - * group space, assign the non present CPUs to the already - * allocated out groups. - */ - if (nr_present >= numgrps) - curgrp = 0; - else - curgrp = nr_present; - cpumask_andnot(npresmsk, cpu_possible_mask, cpu_present_mask); - ret = __group_cpus_evenly(curgrp, numgrps, node_to_cpumask, - npresmsk, nmsk, masks); - if (ret >= 0) - nr_others = ret; - - fail_build_affinity: - cpus_read_unlock(); - - if (ret >= 0) - WARN_ON(nr_present + nr_others < numgrps); - - fail_node_to_cpumask: - free_node_to_cpumask(node_to_cpumask); - - fail_npresmsk: - free_cpumask_var(npresmsk); - - fail_nmsk: - free_cpumask_var(nmsk); - if (ret < 0) { - kfree(masks); - return NULL; - } - return masks; -} +#include static void default_calc_sets(struct irq_affinity *affd, unsigned int affvecs) { diff --git a/lib/Makefile b/lib/Makefile index 4d9461bfea42..a4665a802e87 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -353,6 +353,8 @@ obj-$(CONFIG_SBITMAP) += sbitmap.o obj-$(CONFIG_PARMAN) += parman.o +obj-y += group_cpus.o + # GCC library routines obj-$(CONFIG_GENERIC_LIB_ASHLDI3) += ashldi3.o obj-$(CONFIG_GENERIC_LIB_ASHRDI3) += ashrdi3.o diff --git a/lib/group_cpus.c b/lib/group_cpus.c new file mode 100644 index 000000000000..99f08c6cb9d9 --- /dev/null +++ b/lib/group_cpus.c @@ -0,0 +1,427 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2016 Thomas Gleixner. + * Copyright (C) 2016-2017 Christoph Hellwig. + */ +#include +#include +#include +#include +#include + +static void grp_spread_init_one(struct cpumask *irqmsk, struct cpumask *nmsk, + unsigned int cpus_per_grp) +{ + const struct cpumask *siblmsk; + int cpu, sibl; + + for ( ; cpus_per_grp > 0; ) { + cpu = cpumask_first(nmsk); + + /* Should not happen, but I'm too lazy to think about it */ + if (cpu >= nr_cpu_ids) + return; + + cpumask_clear_cpu(cpu, nmsk); + cpumask_set_cpu(cpu, irqmsk); + cpus_per_grp--; + + /* If the cpu has siblings, use them first */ + siblmsk = topology_sibling_cpumask(cpu); + for (sibl = -1; cpus_per_grp > 0; ) { + sibl = cpumask_next(sibl, siblmsk); + if (sibl >= nr_cpu_ids) + break; + if (!cpumask_test_and_clear_cpu(sibl, nmsk)) + continue; + cpumask_set_cpu(sibl, irqmsk); + cpus_per_grp--; + } + } +} + +static cpumask_var_t *alloc_node_to_cpumask(void) +{ + cpumask_var_t *masks; + int node; + + masks = kcalloc(nr_node_ids, sizeof(cpumask_var_t), GFP_KERNEL); + if (!masks) + return NULL; + + for (node = 0; node < nr_node_ids; node++) { + if (!zalloc_cpumask_var(&masks[node], GFP_KERNEL)) + goto out_unwind; + } + + return masks; + +out_unwind: + while (--node >= 0) + free_cpumask_var(masks[node]); + kfree(masks); + return NULL; +} + +static void free_node_to_cpumask(cpumask_var_t *masks) +{ + int node; + + for (node = 0; node < nr_node_ids; node++) + free_cpumask_var(masks[node]); + kfree(masks); +} + +static void build_node_to_cpumask(cpumask_var_t *masks) +{ + int cpu; + + for_each_possible_cpu(cpu) + cpumask_set_cpu(cpu, masks[cpu_to_node(cpu)]); +} + +static int get_nodes_in_cpumask(cpumask_var_t *node_to_cpumask, + const struct cpumask *mask, nodemask_t *nodemsk) +{ + int n, nodes = 0; + + /* Calculate the number of nodes in the supplied affinity mask */ + for_each_node(n) { + if (cpumask_intersects(mask, node_to_cpumask[n])) { + node_set(n, *nodemsk); + nodes++; + } + } + return nodes; +} + +struct node_groups { + unsigned id; + + union { + unsigned ngroups; + unsigned ncpus; + }; +}; + +static int ncpus_cmp_func(const void *l, const void *r) +{ + const struct node_groups *ln = l; + const struct node_groups *rn = r; + + return ln->ncpus - rn->ncpus; +} + +/* + * Allocate group number for each node, so that for each node: + * + * 1) the allocated number is >= 1 + * + * 2) the allocated number is <= active CPU number of this node + * + * The actual allocated total groups may be less than @numgrps when + * active total CPU number is less than @numgrps. + * + * Active CPUs means the CPUs in '@cpu_mask AND @node_to_cpumask[]' + * for each node. + */ +static void alloc_nodes_groups(unsigned int numgrps, + cpumask_var_t *node_to_cpumask, + const struct cpumask *cpu_mask, + const nodemask_t nodemsk, + struct cpumask *nmsk, + struct node_groups *node_groups) +{ + unsigned n, remaining_ncpus = 0; + + for (n = 0; n < nr_node_ids; n++) { + node_groups[n].id = n; + node_groups[n].ncpus = UINT_MAX; + } + + for_each_node_mask(n, nodemsk) { + unsigned ncpus; + + cpumask_and(nmsk, cpu_mask, node_to_cpumask[n]); + ncpus = cpumask_weight(nmsk); + + if (!ncpus) + continue; + remaining_ncpus += ncpus; + node_groups[n].ncpus = ncpus; + } + + numgrps = min_t(unsigned, remaining_ncpus, numgrps); + + sort(node_groups, nr_node_ids, sizeof(node_groups[0]), + ncpus_cmp_func, NULL); + + /* + * Allocate groups for each node according to the ratio of this + * node's nr_cpus to remaining un-assigned ncpus. 'numgrps' is + * bigger than number of active numa nodes. Always start the + * allocation from the node with minimized nr_cpus. + * + * This way guarantees that each active node gets allocated at + * least one group, and the theory is simple: over-allocation + * is only done when this node is assigned by one group, so + * other nodes will be allocated >= 1 groups, since 'numgrps' is + * bigger than number of numa nodes. + * + * One perfect invariant is that number of allocated groups for + * each node is <= CPU count of this node: + * + * 1) suppose there are two nodes: A and B + * ncpu(X) is CPU count of node X + * grps(X) is the group count allocated to node X via this + * algorithm + * + * ncpu(A) <= ncpu(B) + * ncpu(A) + ncpu(B) = N + * grps(A) + grps(B) = G + * + * grps(A) = max(1, round_down(G * ncpu(A) / N)) + * grps(B) = G - grps(A) + * + * both N and G are integer, and 2 <= G <= N, suppose + * G = N - delta, and 0 <= delta <= N - 2 + * + * 2) obviously grps(A) <= ncpu(A) because: + * + * if grps(A) is 1, then grps(A) <= ncpu(A) given + * ncpu(A) >= 1 + * + * otherwise, + * grps(A) <= G * ncpu(A) / N <= ncpu(A), given G <= N + * + * 3) prove how grps(B) <= ncpu(B): + * + * if round_down(G * ncpu(A) / N) == 0, vecs(B) won't be + * over-allocated, so grps(B) <= ncpu(B), + * + * otherwise: + * + * grps(A) = + * round_down(G * ncpu(A) / N) = + * round_down((N - delta) * ncpu(A) / N) = + * round_down((N * ncpu(A) - delta * ncpu(A)) / N) >= + * round_down((N * ncpu(A) - delta * N) / N) = + * cpu(A) - delta + * + * then: + * + * grps(A) - G >= ncpu(A) - delta - G + * => + * G - grps(A) <= G + delta - ncpu(A) + * => + * grps(B) <= N - ncpu(A) + * => + * grps(B) <= cpu(B) + * + * For nodes >= 3, it can be thought as one node and another big + * node given that is exactly what this algorithm is implemented, + * and we always re-calculate 'remaining_ncpus' & 'numgrps', and + * finally for each node X: grps(X) <= ncpu(X). + * + */ + for (n = 0; n < nr_node_ids; n++) { + unsigned ngroups, ncpus; + + if (node_groups[n].ncpus == UINT_MAX) + continue; + + WARN_ON_ONCE(numgrps == 0); + + ncpus = node_groups[n].ncpus; + ngroups = max_t(unsigned, 1, + numgrps * ncpus / remaining_ncpus); + WARN_ON_ONCE(ngroups > ncpus); + + node_groups[n].ngroups = ngroups; + + remaining_ncpus -= ncpus; + numgrps -= ngroups; + } +} + +static int __group_cpus_evenly(unsigned int startgrp, unsigned int numgrps, + cpumask_var_t *node_to_cpumask, + const struct cpumask *cpu_mask, + struct cpumask *nmsk, struct cpumask *masks) +{ + unsigned int i, n, nodes, cpus_per_grp, extra_grps, done = 0; + unsigned int last_grp = numgrps; + unsigned int curgrp = startgrp; + nodemask_t nodemsk = NODE_MASK_NONE; + struct node_groups *node_groups; + + if (cpumask_empty(cpu_mask)) + return 0; + + nodes = get_nodes_in_cpumask(node_to_cpumask, cpu_mask, &nodemsk); + + /* + * If the number of nodes in the mask is greater than or equal the + * number of groups we just spread the groups across the nodes. + */ + if (numgrps <= nodes) { + for_each_node_mask(n, nodemsk) { + /* Ensure that only CPUs which are in both masks are set */ + cpumask_and(nmsk, cpu_mask, node_to_cpumask[n]); + cpumask_or(&masks[curgrp], &masks[curgrp], nmsk); + if (++curgrp == last_grp) + curgrp = 0; + } + return numgrps; + } + + node_groups = kcalloc(nr_node_ids, + sizeof(struct node_groups), + GFP_KERNEL); + if (!node_groups) + return -ENOMEM; + + /* allocate group number for each node */ + alloc_nodes_groups(numgrps, node_to_cpumask, cpu_mask, + nodemsk, nmsk, node_groups); + for (i = 0; i < nr_node_ids; i++) { + unsigned int ncpus, v; + struct node_groups *nv = &node_groups[i]; + + if (nv->ngroups == UINT_MAX) + continue; + + /* Get the cpus on this node which are in the mask */ + cpumask_and(nmsk, cpu_mask, node_to_cpumask[nv->id]); + ncpus = cpumask_weight(nmsk); + if (!ncpus) + continue; + + WARN_ON_ONCE(nv->ngroups > ncpus); + + /* Account for rounding errors */ + extra_grps = ncpus - nv->ngroups * (ncpus / nv->ngroups); + + /* Spread allocated groups on CPUs of the current node */ + for (v = 0; v < nv->ngroups; v++, curgrp++) { + cpus_per_grp = ncpus / nv->ngroups; + + /* Account for extra groups to compensate rounding errors */ + if (extra_grps) { + cpus_per_grp++; + --extra_grps; + } + + /* + * wrapping has to be considered given 'startgrp' + * may start anywhere + */ + if (curgrp >= last_grp) + curgrp = 0; + grp_spread_init_one(&masks[curgrp], nmsk, + cpus_per_grp); + } + done += nv->ngroups; + } + kfree(node_groups); + return done; +} + +#ifdef CONFIG_SMP +/** + * group_cpus_evenly - Group all CPUs evenly per NUMA/CPU locality + * @numgrps: number of groups + * + * Return: cpumask array if successful, NULL otherwise. And each element + * includes CPUs assigned to this group + * + * Try to put close CPUs from viewpoint of CPU and NUMA locality into + * same group, and run two-stage grouping: + * 1) allocate present CPUs on these groups evenly first + * 2) allocate other possible CPUs on these groups evenly + * + * We guarantee in the resulted grouping that all CPUs are covered, and + * no same CPU is assigned to multiple groups + */ +struct cpumask *group_cpus_evenly(unsigned int numgrps) +{ + unsigned int curgrp = 0, nr_present = 0, nr_others = 0; + cpumask_var_t *node_to_cpumask; + cpumask_var_t nmsk, npresmsk; + int ret = -ENOMEM; + struct cpumask *masks = NULL; + + if (!zalloc_cpumask_var(&nmsk, GFP_KERNEL)) + return NULL; + + if (!zalloc_cpumask_var(&npresmsk, GFP_KERNEL)) + goto fail_nmsk; + + node_to_cpumask = alloc_node_to_cpumask(); + if (!node_to_cpumask) + goto fail_npresmsk; + + masks = kcalloc(numgrps, sizeof(*masks), GFP_KERNEL); + if (!masks) + goto fail_node_to_cpumask; + + /* Stabilize the cpumasks */ + cpus_read_lock(); + build_node_to_cpumask(node_to_cpumask); + + /* grouping present CPUs first */ + ret = __group_cpus_evenly(curgrp, numgrps, node_to_cpumask, + cpu_present_mask, nmsk, masks); + if (ret < 0) + goto fail_build_affinity; + nr_present = ret; + + /* + * Allocate non present CPUs starting from the next group to be + * handled. If the grouping of present CPUs already exhausted the + * group space, assign the non present CPUs to the already + * allocated out groups. + */ + if (nr_present >= numgrps) + curgrp = 0; + else + curgrp = nr_present; + cpumask_andnot(npresmsk, cpu_possible_mask, cpu_present_mask); + ret = __group_cpus_evenly(curgrp, numgrps, node_to_cpumask, + npresmsk, nmsk, masks); + if (ret >= 0) + nr_others = ret; + + fail_build_affinity: + cpus_read_unlock(); + + if (ret >= 0) + WARN_ON(nr_present + nr_others < numgrps); + + fail_node_to_cpumask: + free_node_to_cpumask(node_to_cpumask); + + fail_npresmsk: + free_cpumask_var(npresmsk); + + fail_nmsk: + free_cpumask_var(nmsk); + if (ret < 0) { + kfree(masks); + return NULL; + } + return masks; +} +#else +struct cpumask *group_cpus_evenly(unsigned int numgrps) +{ + struct cpumask *masks = kcalloc(numgrps, sizeof(*masks), GFP_KERNEL); + + if (!masks) + return NULL; + + /* assign all CPUs(cpu 0) to the 1st group only */ + cpumask_copy(&masks[0], cpu_possible_mask); + return masks; +} +#endif From 6a6dcae8f486c3f3298d0767d34505121c7b0b81 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Tue, 27 Dec 2022 10:29:05 +0800 Subject: [PATCH 113/239] blk-mq: Build default queue map via group_cpus_evenly() The default queue mapping builder of blk_mq_map_queues doesn't take NUMA topo into account, so the built mapping is pretty bad, since CPUs belonging to different NUMA node are assigned to same queue. It is observed that IOPS drops by ~30% when running two jobs on same hctx of null_blk from two CPUs belonging to two NUMA nodes compared with from same NUMA node. Address the issue by reusing group_cpus_evenly() for building queue mapping since group_cpus_evenly() does group cpus according to CPU/NUMA locality. Also performance data becomes more stable with this given correct queue mapping is applied wrt. numa locality viewpoint, for example, on one two nodes arm64 machine with 160 cpus, node 0(cpu 0~79), node 1(cpu 80~159): 1) modprobe null_blk nr_devices=1 submit_queues=2 2) run 'fio(t/io_uring -p 0 -n 4 -r 20 /dev/nullb0)', and observe that IOPS becomes much stable on multiple tests: - unpatched: IOPS is 2.5M ~ 4.5M - patched: IOPS is 4.3M ~ 5.0M Lots of drivers may benefit from the change, such as nvme pci poll, nvme tcp, ... Signed-off-by: Ming Lei Signed-off-by: Thomas Gleixner Reviewed-by: Christoph Hellwig Reviewed-by: John Garry Reviewed-by: Jens Axboe Link: https://lore.kernel.org/r/20221227022905.352674-7-ming.lei@redhat.com --- block/blk-mq-cpumap.c | 61 +++++++++---------------------------------- 1 file changed, 12 insertions(+), 49 deletions(-) diff --git a/block/blk-mq-cpumap.c b/block/blk-mq-cpumap.c index 9c2fce1a7b50..0c612c19feb8 100644 --- a/block/blk-mq-cpumap.c +++ b/block/blk-mq-cpumap.c @@ -10,66 +10,29 @@ #include #include #include +#include #include #include "blk.h" #include "blk-mq.h" -static int queue_index(struct blk_mq_queue_map *qmap, - unsigned int nr_queues, const int q) -{ - return qmap->queue_offset + (q % nr_queues); -} - -static int get_first_sibling(unsigned int cpu) -{ - unsigned int ret; - - ret = cpumask_first(topology_sibling_cpumask(cpu)); - if (ret < nr_cpu_ids) - return ret; - - return cpu; -} - void blk_mq_map_queues(struct blk_mq_queue_map *qmap) { - unsigned int *map = qmap->mq_map; - unsigned int nr_queues = qmap->nr_queues; - unsigned int cpu, first_sibling, q = 0; + const struct cpumask *masks; + unsigned int queue, cpu; - for_each_possible_cpu(cpu) - map[cpu] = -1; - - /* - * Spread queues among present CPUs first for minimizing - * count of dead queues which are mapped by all un-present CPUs - */ - for_each_present_cpu(cpu) { - if (q >= nr_queues) - break; - map[cpu] = queue_index(qmap, nr_queues, q++); + masks = group_cpus_evenly(qmap->nr_queues); + if (!masks) { + for_each_possible_cpu(cpu) + qmap->mq_map[cpu] = qmap->queue_offset; + return; } - for_each_possible_cpu(cpu) { - if (map[cpu] != -1) - continue; - /* - * First do sequential mapping between CPUs and queues. - * In case we still have CPUs to map, and we have some number of - * threads per cores then map sibling threads to the same queue - * for performance optimizations. - */ - if (q < nr_queues) { - map[cpu] = queue_index(qmap, nr_queues, q++); - } else { - first_sibling = get_first_sibling(cpu); - if (first_sibling == cpu) - map[cpu] = queue_index(qmap, nr_queues, q++); - else - map[cpu] = map[first_sibling]; - } + for (queue = 0; queue < qmap->nr_queues; queue++) { + for_each_cpu(cpu, &masks[queue]) + qmap->mq_map[cpu] = qmap->queue_offset + queue; } + kfree(masks); } EXPORT_SYMBOL_GPL(blk_mq_map_queues); From 6efdda8bec2900ce5166ee4ff4b1844b47b529cd Mon Sep 17 00:00:00 2001 From: "Joel Fernandes (Google)" Date: Thu, 12 Jan 2023 00:52:22 +0000 Subject: [PATCH 114/239] rcu: Track laziness during boot and suspend Boot and suspend/resume should not be slowed down in kernels built with CONFIG_RCU_LAZY=y. In particular, suspend can sometimes fail in such kernels. This commit therefore adds rcu_async_hurry(), rcu_async_relax(), and rcu_async_should_hurry() functions that track whether or not either a boot or a suspend/resume operation is in progress. This will enable a later commit to refrain from laziness during those times. Export rcu_async_should_hurry(), rcu_async_hurry(), and rcu_async_relax() for later use by rcutorture. [ paulmck: Apply feedback from Steve Rostedt. ] Fixes: 3cb278e73be5 ("rcu: Make call_rcu() lazy to save power") Signed-off-by: Joel Fernandes (Google) Signed-off-by: Paul E. McKenney --- kernel/rcu/rcu.h | 6 ++++++ kernel/rcu/tree.c | 2 ++ kernel/rcu/update.c | 40 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index fa640c45172e..95a0038c9218 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -448,14 +448,20 @@ do { \ /* Tiny RCU doesn't expedite, as its purpose in life is instead to be tiny. */ static inline bool rcu_gp_is_normal(void) { return true; } static inline bool rcu_gp_is_expedited(void) { return false; } +static inline bool rcu_async_should_hurry(void) { return false; } static inline void rcu_expedite_gp(void) { } static inline void rcu_unexpedite_gp(void) { } +static inline void rcu_async_hurry(void) { } +static inline void rcu_async_relax(void) { } static inline void rcu_request_urgent_qs_task(struct task_struct *t) { } #else /* #ifdef CONFIG_TINY_RCU */ bool rcu_gp_is_normal(void); /* Internal RCU use. */ bool rcu_gp_is_expedited(void); /* Internal RCU use. */ +bool rcu_async_should_hurry(void); /* Internal RCU use. */ void rcu_expedite_gp(void); void rcu_unexpedite_gp(void); +void rcu_async_hurry(void); +void rcu_async_relax(void); void rcupdate_announce_bootup_oddness(void); #ifdef CONFIG_TASKS_RCU_GENERIC void show_rcu_tasks_gp_kthreads(void); diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 89313c7c17b6..712c4e7b4675 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -4414,11 +4414,13 @@ static int rcu_pm_notify(struct notifier_block *self, switch (action) { case PM_HIBERNATION_PREPARE: case PM_SUSPEND_PREPARE: + rcu_async_hurry(); rcu_expedite_gp(); break; case PM_POST_HIBERNATION: case PM_POST_SUSPEND: rcu_unexpedite_gp(); + rcu_async_relax(); break; default: break; diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c index 6ed5020aee6d..6d1ea210af1f 100644 --- a/kernel/rcu/update.c +++ b/kernel/rcu/update.c @@ -144,8 +144,45 @@ bool rcu_gp_is_normal(void) } EXPORT_SYMBOL_GPL(rcu_gp_is_normal); -static atomic_t rcu_expedited_nesting = ATOMIC_INIT(1); +static atomic_t rcu_async_hurry_nesting = ATOMIC_INIT(1); +/* + * Should call_rcu() callbacks be processed with urgency or are + * they OK being executed with arbitrary delays? + */ +bool rcu_async_should_hurry(void) +{ + return !IS_ENABLED(CONFIG_RCU_LAZY) || + atomic_read(&rcu_async_hurry_nesting); +} +EXPORT_SYMBOL_GPL(rcu_async_should_hurry); +/** + * rcu_async_hurry - Make future async RCU callbacks not lazy. + * + * After a call to this function, future calls to call_rcu() + * will be processed in a timely fashion. + */ +void rcu_async_hurry(void) +{ + if (IS_ENABLED(CONFIG_RCU_LAZY)) + atomic_inc(&rcu_async_hurry_nesting); +} +EXPORT_SYMBOL_GPL(rcu_async_hurry); + +/** + * rcu_async_relax - Make future async RCU callbacks lazy. + * + * After a call to this function, future calls to call_rcu() + * will be processed in a lazy fashion. + */ +void rcu_async_relax(void) +{ + if (IS_ENABLED(CONFIG_RCU_LAZY)) + atomic_dec(&rcu_async_hurry_nesting); +} +EXPORT_SYMBOL_GPL(rcu_async_relax); + +static atomic_t rcu_expedited_nesting = ATOMIC_INIT(1); /* * Should normal grace-period primitives be expedited? Intended for * use within RCU. Note that this function takes the rcu_expedited @@ -195,6 +232,7 @@ static bool rcu_boot_ended __read_mostly; void rcu_end_inkernel_boot(void) { rcu_unexpedite_gp(); + rcu_async_relax(); if (rcu_normal_after_boot) WRITE_ONCE(rcu_normal, 1); rcu_boot_ended = true; From 188a569658584e93930ab60334c5a1079c0330d8 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 18 Jan 2023 12:14:01 +0100 Subject: [PATCH 115/239] genirq/affinity: Only build SMP-only helper functions on SMP kernels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit allnoconfig grew these new build warnings in lib/group_cpus.c: lib/group_cpus.c:247:12: warning: ‘__group_cpus_evenly’ defined but not used [-Wunused-function] lib/group_cpus.c:75:13: warning: ‘build_node_to_cpumask’ defined but not used [-Wunused-function] lib/group_cpus.c:66:13: warning: ‘free_node_to_cpumask’ defined but not used [-Wunused-function] lib/group_cpus.c:43:23: warning: ‘alloc_node_to_cpumask’ defined but not used [-Wunused-function] Widen the #ifdef CONFIG_SMP block to not expose unused helpers on non-SMP builds. Also annotate the preprocessor branches for better readability. Fixes: f7b3ea8cf72f ("genirq/affinity: Move group_cpus_evenly() into lib/") Cc: Ming Lei Cc: Thomas Gleixner Link: https://lore.kernel.org/r/20221227022905.352674-6-ming.lei@redhat.com Signed-off-by: Ingo Molnar --- lib/group_cpus.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/group_cpus.c b/lib/group_cpus.c index 99f08c6cb9d9..9c837a35fef7 100644 --- a/lib/group_cpus.c +++ b/lib/group_cpus.c @@ -9,6 +9,8 @@ #include #include +#ifdef CONFIG_SMP + static void grp_spread_init_one(struct cpumask *irqmsk, struct cpumask *nmsk, unsigned int cpus_per_grp) { @@ -327,7 +329,6 @@ static int __group_cpus_evenly(unsigned int startgrp, unsigned int numgrps, return done; } -#ifdef CONFIG_SMP /** * group_cpus_evenly - Group all CPUs evenly per NUMA/CPU locality * @numgrps: number of groups @@ -412,7 +413,7 @@ struct cpumask *group_cpus_evenly(unsigned int numgrps) } return masks; } -#else +#else /* CONFIG_SMP */ struct cpumask *group_cpus_evenly(unsigned int numgrps) { struct cpumask *masks = kcalloc(numgrps, sizeof(*masks), GFP_KERNEL); @@ -424,4 +425,4 @@ struct cpumask *group_cpus_evenly(unsigned int numgrps) cpumask_copy(&masks[0], cpu_possible_mask); return masks; } -#endif +#endif /* CONFIG_SMP */ From f2ac14b5f197e4a2dec51e5ceaa56682ff1592bc Mon Sep 17 00:00:00 2001 From: Armin Wolf Date: Sat, 14 Jan 2023 09:50:50 +0100 Subject: [PATCH 116/239] ACPI: battery: Fix missing NUL-termination with large strings When encountering a string bigger than the destination buffer (32 bytes), the string is not properly NUL-terminated, causing buffer overreads later. This for example happens on the Inspiron 3505, where the battery model name is larger than 32 bytes, which leads to sysfs showing the model name together with the serial number string (which is NUL-terminated and thus prevents worse). Fix this by using strscpy() which ensures that the result is always NUL-terminated. Fixes: 106449e870b3 ("ACPI: Battery: Allow extract string from integer") Signed-off-by: Armin Wolf Signed-off-by: Rafael J. Wysocki --- drivers/acpi/battery.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/acpi/battery.c b/drivers/acpi/battery.c index f4badcdde76e..fb64bd217d82 100644 --- a/drivers/acpi/battery.c +++ b/drivers/acpi/battery.c @@ -440,7 +440,7 @@ static int extract_package(struct acpi_battery *battery, if (element->type == ACPI_TYPE_STRING || element->type == ACPI_TYPE_BUFFER) - strncpy(ptr, element->string.pointer, 32); + strscpy(ptr, element->string.pointer, 32); else if (element->type == ACPI_TYPE_INTEGER) { strncpy(ptr, (u8 *)&element->integer.value, sizeof(u64)); From b5fc3ca3954fd0382bb576a5a295c03a089ac7e3 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 5 Jan 2023 13:57:10 -0800 Subject: [PATCH 117/239] ARM: ixp4xx: Replace 0-length arrays with flexible arrays Zero-length arrays are deprecated[1]. Replace npe_load_firmware's union of 0-length arrays with flexible arrays. Detected with GCC 13, using -fstrict-flex-arrays=3: drivers/soc/ixp4xx/ixp4xx-npe.c: In function 'npe_load_firmware': drivers/soc/ixp4xx/ixp4xx-npe.c:570:60: warning: array subscript i is outside array bounds of 'u32[0]' {aka 'unsigned int[]'} [-Warray-bounds=] 570 | image->data[i] = swab32(image->data[i]); include/uapi/linux/swab.h:115:54: note: in definition of macro '__swab32' 115 | #define __swab32(x) (__u32)__builtin_bswap32((__u32)(x)) | ^ drivers/soc/ixp4xx/ixp4xx-npe.c:570:42: note: in expansion of macro 'swab32' 570 | image->data[i] = swab32(image->data[i]); | ^~~~~~ drivers/soc/ixp4xx/ixp4xx-npe.c:522:29: note: while referencing 'data' 522 | u32 data[0]; | ^~~~ [1] https://www.kernel.org/doc/html/latest/process/deprecated.html#zero-length-and-one-element-arrays Cc: Krzysztof Halasa Cc: Linus Walleij Cc: Arnd Bergmann Cc: "Gustavo A. R. Silva" Cc: linux-arm-kernel@lists.infradead.org Signed-off-by: Kees Cook Reviewed-by: Linus Walleij Link: https://lore.kernel.org/r/20230105215706.never.027-kees@kernel.org --- drivers/soc/ixp4xx/ixp4xx-npe.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/soc/ixp4xx/ixp4xx-npe.c b/drivers/soc/ixp4xx/ixp4xx-npe.c index 58240e320c13..5be9988f30ce 100644 --- a/drivers/soc/ixp4xx/ixp4xx-npe.c +++ b/drivers/soc/ixp4xx/ixp4xx-npe.c @@ -519,15 +519,15 @@ int npe_load_firmware(struct npe *npe, const char *name, struct device *dev) u32 id; u32 size; union { - u32 data[0]; - struct dl_block blocks[0]; + DECLARE_FLEX_ARRAY(u32, data); + DECLARE_FLEX_ARRAY(struct dl_block, blocks); }; } *image; struct dl_codeblock { u32 npe_addr; u32 size; - u32 data[0]; + u32 data[]; } *cb; int i, j, err, data_size, instr_size, blocks, table_end; From b76ded214633cf5067ff51642a360eb87242c411 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Fri, 9 Dec 2022 11:57:42 -0800 Subject: [PATCH 118/239] LoadPin: Refactor read-only check into a helper In preparation for allowing mounts to shift when not enforced, move read-only checking into a separate helper. Cc: Paul Moore Cc: James Morris Cc: "Serge E. Hallyn" Cc: linux-security-module@vger.kernel.org Signed-off-by: Kees Cook Reviewed-by: Serge Hallyn Link: https://lore.kernel.org/r/20221209195746.1366607-1-keescook@chromium.org --- security/loadpin/loadpin.c | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/security/loadpin/loadpin.c b/security/loadpin/loadpin.c index 110a5ab2b46b..ca0eff3ce9d0 100644 --- a/security/loadpin/loadpin.c +++ b/security/loadpin/loadpin.c @@ -72,28 +72,21 @@ static struct ctl_table loadpin_sysctl_table[] = { { } }; -/* - * This must be called after early kernel init, since then the rootdev - * is available. - */ -static void check_pinning_enforcement(struct super_block *mnt_sb) +static void report_writable(struct super_block *mnt_sb, bool writable) { - bool ro = false; - /* * If load pinning is not enforced via a read-only block * device, allow sysctl to change modes for testing. */ if (mnt_sb->s_bdev) { - ro = bdev_read_only(mnt_sb->s_bdev); pr_info("%pg (%u:%u): %s\n", mnt_sb->s_bdev, MAJOR(mnt_sb->s_bdev->bd_dev), MINOR(mnt_sb->s_bdev->bd_dev), - ro ? "read-only" : "writable"); + writable ? "writable" : "read-only"); } else pr_info("mnt_sb lacks block device, treating as: writable\n"); - if (!ro) { + if (writable) { if (!register_sysctl_paths(loadpin_sysctl_path, loadpin_sysctl_table)) pr_notice("sysctl registration failed!\n"); @@ -103,12 +96,26 @@ static void check_pinning_enforcement(struct super_block *mnt_sb) pr_info("load pinning engaged.\n"); } #else -static void check_pinning_enforcement(struct super_block *mnt_sb) +static void report_writable(struct super_block *mnt_sb, bool writable) { pr_info("load pinning engaged.\n"); } #endif +/* + * This must be called after early kernel init, since then the rootdev + * is available. + */ +static bool sb_is_writable(struct super_block *mnt_sb) +{ + bool writable = true; + + if (mnt_sb->s_bdev) + writable = !bdev_read_only(mnt_sb->s_bdev); + + return writable; +} + static void loadpin_sb_free_security(struct super_block *mnt_sb) { /* @@ -126,6 +133,7 @@ static int loadpin_check(struct file *file, enum kernel_read_file_id id) { struct super_block *load_root; const char *origin = kernel_read_file_id_str(id); + bool load_root_writable; /* If the file id is excluded, ignore the pinning. */ if ((unsigned int)id < ARRAY_SIZE(ignore_read_file_id) && @@ -146,6 +154,7 @@ static int loadpin_check(struct file *file, enum kernel_read_file_id id) } load_root = file->f_path.mnt->mnt_sb; + load_root_writable = sb_is_writable(load_root); /* First loaded module/firmware defines the root for all others. */ spin_lock(&pinned_root_spinlock); @@ -162,7 +171,7 @@ static int loadpin_check(struct file *file, enum kernel_read_file_id id) * enforcing. This would be purely cosmetic. */ spin_unlock(&pinned_root_spinlock); - check_pinning_enforcement(pinned_root); + report_writable(pinned_root, load_root_writable); report_load(origin, file, "pinned"); } else { spin_unlock(&pinned_root_spinlock); From 60ba1028fc7b73e3cfbcfe7087a2e87e8b1fd208 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Fri, 9 Dec 2022 11:57:43 -0800 Subject: [PATCH 119/239] LoadPin: Refactor sysctl initialization In preparation for shifting root mount when not enforcing, split sysctl logic out into a separate helper, and unconditionally register the sysctl, but only make it writable when the device is writable. Cc: Paul Moore Cc: James Morris Cc: "Serge E. Hallyn" Cc: linux-security-module@vger.kernel.org Signed-off-by: Kees Cook Reviewed-by: Serge Hallyn Link: https://lore.kernel.org/r/20221209195746.1366607-2-keescook@chromium.org --- security/loadpin/loadpin.c | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/security/loadpin/loadpin.c b/security/loadpin/loadpin.c index ca0eff3ce9d0..5b15f8f7268d 100644 --- a/security/loadpin/loadpin.c +++ b/security/loadpin/loadpin.c @@ -52,7 +52,6 @@ static bool deny_reading_verity_digests; #endif #ifdef CONFIG_SYSCTL - static struct ctl_path loadpin_sysctl_path[] = { { .procname = "kernel", }, { .procname = "loadpin", }, @@ -66,18 +65,29 @@ static struct ctl_table loadpin_sysctl_table[] = { .maxlen = sizeof(int), .mode = 0644, .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ZERO, + .extra1 = SYSCTL_ONE, .extra2 = SYSCTL_ONE, }, { } }; -static void report_writable(struct super_block *mnt_sb, bool writable) +static void set_sysctl(bool is_writable) { /* * If load pinning is not enforced via a read-only block * device, allow sysctl to change modes for testing. */ + if (is_writable) + loadpin_sysctl_table[0].extra1 = SYSCTL_ZERO; + else + loadpin_sysctl_table[0].extra1 = SYSCTL_ONE; +} +#else +static inline void set_sysctl(bool is_writable) { } +#endif + +static void report_writable(struct super_block *mnt_sb, bool writable) +{ if (mnt_sb->s_bdev) { pr_info("%pg (%u:%u): %s\n", mnt_sb->s_bdev, MAJOR(mnt_sb->s_bdev->bd_dev), @@ -86,21 +96,9 @@ static void report_writable(struct super_block *mnt_sb, bool writable) } else pr_info("mnt_sb lacks block device, treating as: writable\n"); - if (writable) { - if (!register_sysctl_paths(loadpin_sysctl_path, - loadpin_sysctl_table)) - pr_notice("sysctl registration failed!\n"); - else - pr_info("enforcement can be disabled.\n"); - } else + if (!writable) pr_info("load pinning engaged.\n"); } -#else -static void report_writable(struct super_block *mnt_sb, bool writable) -{ - pr_info("load pinning engaged.\n"); -} -#endif /* * This must be called after early kernel init, since then the rootdev @@ -172,6 +170,7 @@ static int loadpin_check(struct file *file, enum kernel_read_file_id id) */ spin_unlock(&pinned_root_spinlock); report_writable(pinned_root, load_root_writable); + set_sysctl(load_root_writable); report_load(origin, file, "pinned"); } else { spin_unlock(&pinned_root_spinlock); @@ -259,6 +258,10 @@ static int __init loadpin_init(void) pr_info("ready to pin (currently %senforcing)\n", enforce ? "" : "not "); parse_exclude(); +#ifdef CONFIG_SYSCTL + if (!register_sysctl_paths(loadpin_sysctl_path, loadpin_sysctl_table)) + pr_notice("sysctl registration failed!\n"); +#endif security_add_hooks(loadpin_hooks, ARRAY_SIZE(loadpin_hooks), "loadpin"); return 0; From 2cfaa84efc25e52f116507a2e69781a40c4dda41 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Fri, 9 Dec 2022 11:57:44 -0800 Subject: [PATCH 120/239] LoadPin: Move pin reporting cleanly out of locking Refactor the pin reporting to be more cleanly outside the locking. It was already, but moving it around helps clear the path for the root to switch when not enforcing. Cc: Paul Moore Cc: James Morris Cc: "Serge E. Hallyn" Cc: linux-security-module@vger.kernel.org Signed-off-by: Kees Cook Reviewed-by: Serge Hallyn Link: https://lore.kernel.org/r/20221209195746.1366607-3-keescook@chromium.org --- security/loadpin/loadpin.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/security/loadpin/loadpin.c b/security/loadpin/loadpin.c index 5b15f8f7268d..ef12d77548ae 100644 --- a/security/loadpin/loadpin.c +++ b/security/loadpin/loadpin.c @@ -131,6 +131,7 @@ static int loadpin_check(struct file *file, enum kernel_read_file_id id) { struct super_block *load_root; const char *origin = kernel_read_file_id_str(id); + bool first_root_pin = false; bool load_root_writable; /* If the file id is excluded, ignore the pinning. */ @@ -162,18 +163,14 @@ static int loadpin_check(struct file *file, enum kernel_read_file_id id) */ if (!pinned_root) { pinned_root = load_root; - /* - * Unlock now since it's only pinned_root we care about. - * In the worst case, we will (correctly) report pinning - * failures before we have announced that pinning is - * enforcing. This would be purely cosmetic. - */ - spin_unlock(&pinned_root_spinlock); + first_root_pin = true; + } + spin_unlock(&pinned_root_spinlock); + + if (first_root_pin) { report_writable(pinned_root, load_root_writable); set_sysctl(load_root_writable); report_load(origin, file, "pinned"); - } else { - spin_unlock(&pinned_root_spinlock); } if (IS_ERR_OR_NULL(pinned_root) || From eba773596be9c21a8e979d7e653f721d1d0341a9 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Fri, 9 Dec 2022 11:57:45 -0800 Subject: [PATCH 121/239] LoadPin: Allow filesystem switch when not enforcing For LoadPin to be used at all in a classic distro environment, it needs to allow for switching filesystems (from the initramfs to the "real" root filesystem). To allow for this, if the "enforce" mode is not set at boot, reset the pinned filesystem tracking when the pinned filesystem gets unmounted instead of invalidating further loads. Once enforcement is set, it cannot be unset, and the pinning will stick. This means that distros can build with CONFIG_SECURITY_LOADPIN=y, but with CONFIG_SECURITY_LOADPIN_ENFORCE disabled, but after boot is running, the system can enable enforcement: $ sysctl -w kernel.loadpin.enforced=1 Cc: Paul Moore Cc: James Morris Cc: "Serge E. Hallyn" Cc: linux-security-module@vger.kernel.org Signed-off-by: Kees Cook Reviewed-by: Serge Hallyn Link: https://lore.kernel.org/r/20221209195746.1366607-4-keescook@chromium.org --- security/loadpin/loadpin.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/security/loadpin/loadpin.c b/security/loadpin/loadpin.c index ef12d77548ae..d73a281adf86 100644 --- a/security/loadpin/loadpin.c +++ b/security/loadpin/loadpin.c @@ -119,11 +119,16 @@ static void loadpin_sb_free_security(struct super_block *mnt_sb) /* * When unmounting the filesystem we were using for load * pinning, we acknowledge the superblock release, but make sure - * no other modules or firmware can be loaded. + * no other modules or firmware can be loaded when we are in + * enforcing mode. Otherwise, allow the root to be reestablished. */ if (!IS_ERR_OR_NULL(pinned_root) && mnt_sb == pinned_root) { - pinned_root = ERR_PTR(-EIO); - pr_info("umount pinned fs: refusing further loads\n"); + if (enforce) { + pinned_root = ERR_PTR(-EIO); + pr_info("umount pinned fs: refusing further loads\n"); + } else { + pinned_root = NULL; + } } } @@ -158,8 +163,9 @@ static int loadpin_check(struct file *file, enum kernel_read_file_id id) /* First loaded module/firmware defines the root for all others. */ spin_lock(&pinned_root_spinlock); /* - * pinned_root is only NULL at startup. Otherwise, it is either - * a valid reference, or an ERR_PTR. + * pinned_root is only NULL at startup or when the pinned root has + * been unmounted while we are not in enforcing mode. Otherwise, it + * is either a valid reference, or an ERR_PTR. */ if (!pinned_root) { pinned_root = load_root; From 6b37dfcb39f6b7d2e5070181d878a4c373b24bb0 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Mon, 2 Jan 2023 19:28:40 -0800 Subject: [PATCH 122/239] PM: hibernate: swap: don't use /** for non-kernel-doc comments kernel-doc complains about multiple occurrences of "/**" being used for something that is not a kernel-doc comment, so change all of these to just use "/*" comment style. The warning message for all of these is: FILE:LINE: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst kernel/power/swap.c:585: warning: ... Structure used for CRC32. kernel/power/swap.c:600: warning: ... * CRC32 update function that runs in its own thread. kernel/power/swap.c:627: warning: ... * Structure used for LZO data compression. kernel/power/swap.c:644: warning: ... * Compression function that runs in its own thread. kernel/power/swap.c:952: warning: ... * The following functions allow us to read data using a swap map kernel/power/swap.c:1111: warning: ... * Structure used for LZO data decompression. kernel/power/swap.c:1127: warning: ... * Decompression function that runs in its own thread. Also correct one spello/typo. Signed-off-by: Randy Dunlap Signed-off-by: Rafael J. Wysocki --- kernel/power/swap.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/kernel/power/swap.c b/kernel/power/swap.c index 277434b6c0bf..36a1df48280c 100644 --- a/kernel/power/swap.c +++ b/kernel/power/swap.c @@ -581,7 +581,7 @@ static int save_image(struct swap_map_handle *handle, return ret; } -/** +/* * Structure used for CRC32. */ struct crc_data { @@ -596,7 +596,7 @@ struct crc_data { unsigned char *unc[LZO_THREADS]; /* uncompressed data */ }; -/** +/* * CRC32 update function that runs in its own thread. */ static int crc32_threadfn(void *data) @@ -623,7 +623,7 @@ static int crc32_threadfn(void *data) } return 0; } -/** +/* * Structure used for LZO data compression. */ struct cmp_data { @@ -640,7 +640,7 @@ struct cmp_data { unsigned char wrk[LZO1X_1_MEM_COMPRESS]; /* compression workspace */ }; -/** +/* * Compression function that runs in its own thread. */ static int lzo_compress_threadfn(void *data) @@ -948,9 +948,9 @@ out_finish: return error; } -/** +/* * The following functions allow us to read data using a swap map - * in a file-alike way + * in a file-like way. */ static void release_swap_reader(struct swap_map_handle *handle) @@ -1107,7 +1107,7 @@ static int load_image(struct swap_map_handle *handle, return ret; } -/** +/* * Structure used for LZO data decompression. */ struct dec_data { @@ -1123,7 +1123,7 @@ struct dec_data { unsigned char cmp[LZO_CMP_SIZE]; /* compressed buffer */ }; -/** +/* * Decompression function that runs in its own thread. */ static int lzo_decompress_threadfn(void *data) From 1b6599f741a4525ca761ecde46e5885ff1e6ba58 Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Tue, 3 Jan 2023 20:57:26 +0800 Subject: [PATCH 123/239] powercap: fix possible name leak in powercap_register_zone() In the error path after calling dev_set_name(), the device name is leaked. To fix this, calling dev_set_name() before device_register(), and call put_device() if it returns error. All the resources is released in powercap_release(), so it can return from powercap_register_zone() directly. Fixes: 75d2364ea0ca ("PowerCap: Add class driver") Signed-off-by: Yang Yingliang Signed-off-by: Rafael J. Wysocki --- drivers/powercap/powercap_sys.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/drivers/powercap/powercap_sys.c b/drivers/powercap/powercap_sys.c index 1f968353d479..e180dee0f83d 100644 --- a/drivers/powercap/powercap_sys.c +++ b/drivers/powercap/powercap_sys.c @@ -530,9 +530,6 @@ struct powercap_zone *powercap_register_zone( power_zone->name = kstrdup(name, GFP_KERNEL); if (!power_zone->name) goto err_name_alloc; - dev_set_name(&power_zone->dev, "%s:%x", - dev_name(power_zone->dev.parent), - power_zone->id); power_zone->constraints = kcalloc(nr_constraints, sizeof(*power_zone->constraints), GFP_KERNEL); @@ -555,9 +552,16 @@ struct powercap_zone *powercap_register_zone( power_zone->dev_attr_groups[0] = &power_zone->dev_zone_attr_group; power_zone->dev_attr_groups[1] = NULL; power_zone->dev.groups = power_zone->dev_attr_groups; + dev_set_name(&power_zone->dev, "%s:%x", + dev_name(power_zone->dev.parent), + power_zone->id); result = device_register(&power_zone->dev); - if (result) - goto err_dev_ret; + if (result) { + put_device(&power_zone->dev); + mutex_unlock(&control_type->lock); + + return ERR_PTR(result); + } control_type->nr_zones++; mutex_unlock(&control_type->lock); From bdaad038cc3c620a769f2156e7c9aab8605411c2 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 4 Jan 2023 22:36:01 +0800 Subject: [PATCH 124/239] powercap: intel_rapl: add support for Meteor Lake Add Meteor Lake to the list of supported processor models in the Intel RAPL power capping driver. Signed-off-by: Zhang Rui Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl_common.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 26d00b1853b4..ca6ff27b4384 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -1113,6 +1113,8 @@ static const struct x86_cpu_id rapl_ids[] __initconst = { X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE, &rapl_defaults_core), X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_P, &rapl_defaults_core), X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_S, &rapl_defaults_core), + X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE, &rapl_defaults_core), + X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE_L, &rapl_defaults_core), X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &rapl_defaults_spr_server), X86_MATCH_INTEL_FAM6_MODEL(LAKEFIELD, &rapl_defaults_core), From 7adc6885259edd4ef5c9a7a62fd4270cf38fdbfb Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 4 Jan 2023 22:36:02 +0800 Subject: [PATCH 125/239] powercap: intel_rapl: add support for Emerald Rapids Add Emerald Rapids to the list of supported processor models in the Intel RAPL power capping driver. Signed-off-by: Zhang Rui Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl_common.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index ca6ff27b4384..9a9192fc8391 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -1116,6 +1116,7 @@ static const struct x86_cpu_id rapl_ids[] __initconst = { X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE, &rapl_defaults_core), X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE_L, &rapl_defaults_core), X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &rapl_defaults_spr_server), + X86_MATCH_INTEL_FAM6_MODEL(EMERALDRAPIDS_X, &rapl_defaults_spr_server), X86_MATCH_INTEL_FAM6_MODEL(LAKEFIELD, &rapl_defaults_core), X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT, &rapl_defaults_byt), From 716ff71ae234fd3ef1286aac8d8a19a0ed2d509d Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Fri, 6 Jan 2023 12:03:42 +0800 Subject: [PATCH 126/239] cpuidle-haltpoll: Replace default_idle() with arch_cpu_idle() When a KVM guest has MWAIT, mwait_idle() is used as the default idle function. However, the cpuidle-haltpoll driver calls default_idle() from default_enter_idle() directly and that one uses HLT instead of MWAIT, which may affect performance adversely, because MWAIT is preferred to HLT as explained by the changelog of commit aebef63cf7ff ("x86: Remove vendor checks from prefer_mwait_c1_over_halt"). Make default_enter_idle() call arch_cpu_idle(), which can use MWAIT, instead of default_idle() to address this issue. Suggested-by: Thomas Gleixner Suggested-by: Rafael J. Wysocki Signed-off-by: Li RongQing [ rjw: Changelog rewrite ] Signed-off-by: Rafael J. Wysocki --- arch/x86/kernel/process.c | 1 + drivers/cpuidle/cpuidle-haltpoll.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 40d156a31676..00a831d56c50 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -721,6 +721,7 @@ void arch_cpu_idle(void) { x86_idle(); } +EXPORT_SYMBOL_GPL(arch_cpu_idle); /* * We use this if we don't have any better idle routine.. diff --git a/drivers/cpuidle/cpuidle-haltpoll.c b/drivers/cpuidle/cpuidle-haltpoll.c index 3a39a7f48b77..e66df22f9695 100644 --- a/drivers/cpuidle/cpuidle-haltpoll.c +++ b/drivers/cpuidle/cpuidle-haltpoll.c @@ -32,7 +32,7 @@ static int default_enter_idle(struct cpuidle_device *dev, local_irq_enable(); return index; } - default_idle(); + arch_cpu_idle(); return index; } From 9a55ab6f02c98bfca1c9c9d73507c1744406d2ba Mon Sep 17 00:00:00 2001 From: Keguang Zhang Date: Thu, 12 Jan 2023 21:53:42 +0800 Subject: [PATCH 127/239] cpufreq: loongson1: Delete obsolete driver The generic DT based cpufreq driver works for Loongson-1, so delete the old custom driver. Signed-off-by: Keguang Zhang Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/Kconfig | 9 -- drivers/cpufreq/Makefile | 1 - drivers/cpufreq/loongson1-cpufreq.c | 222 ---------------------------- 3 files changed, 232 deletions(-) delete mode 100644 drivers/cpufreq/loongson1-cpufreq.c diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig index 2a84fc63371e..448b8ffb4ebd 100644 --- a/drivers/cpufreq/Kconfig +++ b/drivers/cpufreq/Kconfig @@ -270,15 +270,6 @@ config LOONGSON2_CPUFREQ Loongson2F and its successors support this feature. - If in doubt, say N. - -config LOONGSON1_CPUFREQ - tristate "Loongson1 CPUFreq Driver" - depends on LOONGSON1_LS1B - help - This option adds a CPUFreq driver for loongson1 processors which - support software configurable cpu frequency. - If in doubt, say N. endif diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile index 32a7029e25ed..4a806cc5265b 100644 --- a/drivers/cpufreq/Makefile +++ b/drivers/cpufreq/Makefile @@ -111,7 +111,6 @@ obj-$(CONFIG_POWERNV_CPUFREQ) += powernv-cpufreq.o obj-$(CONFIG_BMIPS_CPUFREQ) += bmips-cpufreq.o obj-$(CONFIG_IA64_ACPI_CPUFREQ) += ia64-acpi-cpufreq.o obj-$(CONFIG_LOONGSON2_CPUFREQ) += loongson2_cpufreq.o -obj-$(CONFIG_LOONGSON1_CPUFREQ) += loongson1-cpufreq.o obj-$(CONFIG_SH_CPU_FREQ) += sh-cpufreq.o obj-$(CONFIG_SPARC_US2E_CPUFREQ) += sparc-us2e-cpufreq.o obj-$(CONFIG_SPARC_US3_CPUFREQ) += sparc-us3-cpufreq.o diff --git a/drivers/cpufreq/loongson1-cpufreq.c b/drivers/cpufreq/loongson1-cpufreq.c deleted file mode 100644 index fb72d709db56..000000000000 --- a/drivers/cpufreq/loongson1-cpufreq.c +++ /dev/null @@ -1,222 +0,0 @@ -/* - * CPU Frequency Scaling for Loongson 1 SoC - * - * Copyright (C) 2014-2016 Zhang, Keguang - * - * This file is licensed under the terms of the GNU General Public - * License version 2. This program is licensed "as is" without any - * warranty of any kind, whether express or implied. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -struct ls1x_cpufreq { - struct device *dev; - struct clk *clk; /* CPU clk */ - struct clk *mux_clk; /* MUX of CPU clk */ - struct clk *pll_clk; /* PLL clk */ - struct clk *osc_clk; /* OSC clk */ - unsigned int max_freq; - unsigned int min_freq; -}; - -static struct ls1x_cpufreq *cpufreq; - -static int ls1x_cpufreq_notifier(struct notifier_block *nb, - unsigned long val, void *data) -{ - if (val == CPUFREQ_POSTCHANGE) - current_cpu_data.udelay_val = loops_per_jiffy; - - return NOTIFY_OK; -} - -static struct notifier_block ls1x_cpufreq_notifier_block = { - .notifier_call = ls1x_cpufreq_notifier -}; - -static int ls1x_cpufreq_target(struct cpufreq_policy *policy, - unsigned int index) -{ - struct device *cpu_dev = get_cpu_device(policy->cpu); - unsigned int old_freq, new_freq; - - old_freq = policy->cur; - new_freq = policy->freq_table[index].frequency; - - /* - * The procedure of reconfiguring CPU clk is as below. - * - * - Reparent CPU clk to OSC clk - * - Reset CPU clock (very important) - * - Reconfigure CPU DIV - * - Reparent CPU clk back to CPU DIV clk - */ - - clk_set_parent(policy->clk, cpufreq->osc_clk); - __raw_writel(__raw_readl(LS1X_CLK_PLL_DIV) | RST_CPU_EN | RST_CPU, - LS1X_CLK_PLL_DIV); - __raw_writel(__raw_readl(LS1X_CLK_PLL_DIV) & ~(RST_CPU_EN | RST_CPU), - LS1X_CLK_PLL_DIV); - clk_set_rate(cpufreq->mux_clk, new_freq * 1000); - clk_set_parent(policy->clk, cpufreq->mux_clk); - dev_dbg(cpu_dev, "%u KHz --> %u KHz\n", old_freq, new_freq); - - return 0; -} - -static int ls1x_cpufreq_init(struct cpufreq_policy *policy) -{ - struct device *cpu_dev = get_cpu_device(policy->cpu); - struct cpufreq_frequency_table *freq_tbl; - unsigned int pll_freq, freq; - int steps, i; - - pll_freq = clk_get_rate(cpufreq->pll_clk) / 1000; - - steps = 1 << DIV_CPU_WIDTH; - freq_tbl = kcalloc(steps, sizeof(*freq_tbl), GFP_KERNEL); - if (!freq_tbl) - return -ENOMEM; - - for (i = 0; i < (steps - 1); i++) { - freq = pll_freq / (i + 1); - if ((freq < cpufreq->min_freq) || (freq > cpufreq->max_freq)) - freq_tbl[i].frequency = CPUFREQ_ENTRY_INVALID; - else - freq_tbl[i].frequency = freq; - dev_dbg(cpu_dev, - "cpufreq table: index %d: frequency %d\n", i, - freq_tbl[i].frequency); - } - freq_tbl[i].frequency = CPUFREQ_TABLE_END; - - policy->clk = cpufreq->clk; - cpufreq_generic_init(policy, freq_tbl, 0); - - return 0; -} - -static int ls1x_cpufreq_exit(struct cpufreq_policy *policy) -{ - kfree(policy->freq_table); - return 0; -} - -static struct cpufreq_driver ls1x_cpufreq_driver = { - .name = "cpufreq-ls1x", - .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK, - .verify = cpufreq_generic_frequency_table_verify, - .target_index = ls1x_cpufreq_target, - .get = cpufreq_generic_get, - .init = ls1x_cpufreq_init, - .exit = ls1x_cpufreq_exit, - .attr = cpufreq_generic_attr, -}; - -static int ls1x_cpufreq_remove(struct platform_device *pdev) -{ - cpufreq_unregister_notifier(&ls1x_cpufreq_notifier_block, - CPUFREQ_TRANSITION_NOTIFIER); - cpufreq_unregister_driver(&ls1x_cpufreq_driver); - - return 0; -} - -static int ls1x_cpufreq_probe(struct platform_device *pdev) -{ - struct plat_ls1x_cpufreq *pdata = dev_get_platdata(&pdev->dev); - struct clk *clk; - int ret; - - if (!pdata || !pdata->clk_name || !pdata->osc_clk_name) { - dev_err(&pdev->dev, "platform data missing\n"); - return -EINVAL; - } - - cpufreq = - devm_kzalloc(&pdev->dev, sizeof(struct ls1x_cpufreq), GFP_KERNEL); - if (!cpufreq) - return -ENOMEM; - - cpufreq->dev = &pdev->dev; - - clk = devm_clk_get(&pdev->dev, pdata->clk_name); - if (IS_ERR(clk)) { - dev_err(&pdev->dev, "unable to get %s clock\n", - pdata->clk_name); - return PTR_ERR(clk); - } - cpufreq->clk = clk; - - clk = clk_get_parent(clk); - if (IS_ERR(clk)) { - dev_err(&pdev->dev, "unable to get parent of %s clock\n", - __clk_get_name(cpufreq->clk)); - return PTR_ERR(clk); - } - cpufreq->mux_clk = clk; - - clk = clk_get_parent(clk); - if (IS_ERR(clk)) { - dev_err(&pdev->dev, "unable to get parent of %s clock\n", - __clk_get_name(cpufreq->mux_clk)); - return PTR_ERR(clk); - } - cpufreq->pll_clk = clk; - - clk = devm_clk_get(&pdev->dev, pdata->osc_clk_name); - if (IS_ERR(clk)) { - dev_err(&pdev->dev, "unable to get %s clock\n", - pdata->osc_clk_name); - return PTR_ERR(clk); - } - cpufreq->osc_clk = clk; - - cpufreq->max_freq = pdata->max_freq; - cpufreq->min_freq = pdata->min_freq; - - ret = cpufreq_register_driver(&ls1x_cpufreq_driver); - if (ret) { - dev_err(&pdev->dev, - "failed to register CPUFreq driver: %d\n", ret); - return ret; - } - - ret = cpufreq_register_notifier(&ls1x_cpufreq_notifier_block, - CPUFREQ_TRANSITION_NOTIFIER); - - if (ret) { - dev_err(&pdev->dev, - "failed to register CPUFreq notifier: %d\n",ret); - cpufreq_unregister_driver(&ls1x_cpufreq_driver); - } - - return ret; -} - -static struct platform_driver ls1x_cpufreq_platdrv = { - .probe = ls1x_cpufreq_probe, - .remove = ls1x_cpufreq_remove, - .driver = { - .name = "ls1x-cpufreq", - }, -}; - -module_platform_driver(ls1x_cpufreq_platdrv); - -MODULE_ALIAS("platform:ls1x-cpufreq"); -MODULE_AUTHOR("Kelvin Cheung "); -MODULE_DESCRIPTION("Loongson1 CPUFreq driver"); -MODULE_LICENSE("GPL"); From 38a29e5834eba1e71bc4aab82b09ac065af62b80 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 12 Jan 2023 16:11:15 -0800 Subject: [PATCH 128/239] drivers/cpufreq: Remove "select SRCU" Now that the SRCU Kconfig option is unconditionally selected, there is no longer any point in selecting it. Therefore, remove the "select SRCU" Kconfig statements. Signed-off-by: Paul E. McKenney Acked-by: Viresh Kumar Reviewed-by: John Ogness Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig index 448b8ffb4ebd..76aa1336e2be 100644 --- a/drivers/cpufreq/Kconfig +++ b/drivers/cpufreq/Kconfig @@ -3,7 +3,6 @@ menu "CPU Frequency scaling" config CPU_FREQ bool "CPU Frequency scaling" - select SRCU help CPU Frequency scaling allows you to change the clock speed of CPUs on the fly. This is a nice method to save power, because From 52e0452b413d885d5ab7e3ae85287d67f5a286b2 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 12 Jan 2023 16:11:28 -0800 Subject: [PATCH 129/239] PM: sleep: Remove "select SRCU" Now that the SRCU Kconfig option is unconditionally selected, there is no longer any point in selecting it. Therefore, remove the "select SRCU" Kconfig statements. Signed-off-by: Paul E. McKenney Reviewed-by: John Ogness Signed-off-by: Rafael J. Wysocki --- kernel/power/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/kernel/power/Kconfig b/kernel/power/Kconfig index 60a1d3051cc7..4b31629c5be4 100644 --- a/kernel/power/Kconfig +++ b/kernel/power/Kconfig @@ -118,7 +118,6 @@ config PM_SLEEP def_bool y depends on SUSPEND || HIBERNATE_CALLBACKS select PM - select SRCU config PM_SLEEP_SMP def_bool y From c7cd6f04c0dfb6d44337f92b4c32126d20339873 Mon Sep 17 00:00:00 2001 From: Srinivas Pandruvada Date: Tue, 17 Jan 2023 10:22:40 -0800 Subject: [PATCH 130/239] powercap: idle_inject: Support 100% idle injection The users of the idle injection framework allow 100% idle injection. For example: thermal/cpuidle_cooling.c driver. When the ratio is set to 100%, the runtime_duration becomes zero. However, idle_inject_set_duration() in the idle injection framework silently ignores run_duration_us == 0 without any error (it is a void function). The caller will then assume that everything is fine and 100% idle is effective, but in reality the idle duration will not change. There are two options: - The caller may change their max state to 99% instead of 100% and document that 100% is not supported by the idle inject framework. - Add 100% idle support to the idle inject framework. Since there are other protections via RT throttling, this framework can allow 100% idle. The RT throttling will be activated at 95% idle by default. The caller disabling RT throttling and injecting 100% idle, should be aware that CPU can't be used at all. The idle inject timer is started for (run_duration_us + idle_duration_us) duration. Hence replace (run_duration_us && idle_duration_us) with (run_duration_us + idle_duration_us) in the function idle_inject_set_duration(). Also check for !(run_duration_us + idle_duration_us) to return -EINVAL in idle_inject_start(). Signed-off-by: Srinivas Pandruvada Acked-by: Daniel Lezcano [ rjw: Changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/powercap/idle_inject.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/powercap/idle_inject.c b/drivers/powercap/idle_inject.c index fe86a09e3b67..c03b5402c03b 100644 --- a/drivers/powercap/idle_inject.c +++ b/drivers/powercap/idle_inject.c @@ -155,10 +155,12 @@ void idle_inject_set_duration(struct idle_inject_device *ii_dev, unsigned int run_duration_us, unsigned int idle_duration_us) { - if (run_duration_us && idle_duration_us) { + if (run_duration_us + idle_duration_us) { WRITE_ONCE(ii_dev->run_duration_us, run_duration_us); WRITE_ONCE(ii_dev->idle_duration_us, idle_duration_us); } + if (!run_duration_us) + pr_debug("CPU is forced to 100 percent idle\n"); } /** @@ -201,7 +203,7 @@ int idle_inject_start(struct idle_inject_device *ii_dev) unsigned int idle_duration_us = READ_ONCE(ii_dev->idle_duration_us); unsigned int run_duration_us = READ_ONCE(ii_dev->run_duration_us); - if (!idle_duration_us || !run_duration_us) + if (!(idle_duration_us + run_duration_us)) return -EINVAL; pr_debug("Starting injecting idle cycles on CPUs '%*pbl'\n", From 74528edfbc664f9d2c927c4e5a44f1285598ed0f Mon Sep 17 00:00:00 2001 From: Artem Bityutskiy Date: Fri, 20 Jan 2023 11:15:28 +0200 Subject: [PATCH 131/239] intel_idle: add Emerald Rapids Xeon support Emerald Rapids (EMR) is the next Intel Xeon processor after Sapphire Rapids (SPR). EMR C-states are the same as SPR C-states, and we expect that EMR C-state characteristics (latency and target residency) will be the same as in SPR. Therefore, add EMR support by using SPR C-states table. Signed-off-by: Artem Bityutskiy Signed-off-by: Rafael J. Wysocki --- drivers/idle/intel_idle.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c index cfeb24d40d37..bb3d10099ba4 100644 --- a/drivers/idle/intel_idle.c +++ b/drivers/idle/intel_idle.c @@ -1430,6 +1430,7 @@ static const struct x86_cpu_id intel_idle_ids[] __initconst = { X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, &idle_cpu_adl_l), X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_N, &idle_cpu_adl_n), X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &idle_cpu_spr), + X86_MATCH_INTEL_FAM6_MODEL(EMERALDRAPIDS_X, &idle_cpu_spr), X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &idle_cpu_knl), X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &idle_cpu_knl), X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT, &idle_cpu_bxt), @@ -1862,6 +1863,7 @@ static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv) skx_idle_state_table_update(); break; case INTEL_FAM6_SAPPHIRERAPIDS_X: + case INTEL_FAM6_EMERALDRAPIDS_X: spr_idle_state_table_update(); break; case INTEL_FAM6_ALDERLAKE: From 36b20f82b699a9d15536058608b31f6dfe9336b4 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Thu, 19 Jan 2023 19:41:38 +0100 Subject: [PATCH 132/239] MAINTAINERS: Add x86 ACPI paths to the ACPI entry In order for things like get_maintainer.pl to print linux-acpi as a list to receive copies of ACPI-related patches, add paths to ACPI files in the arch/x86/ directory to the ACPI entry in MAINTAINERS. While at it, make the list of ACPI files listed in the suspend-to-RAM entry more precise. Signed-off-by: Rafael J. Wysocki Acked-by: Thomas Gleixner --- MAINTAINERS | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index f781f936ae35..b1b8cbf59ad9 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -361,6 +361,8 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm F: Documentation/ABI/testing/configfs-acpi F: Documentation/ABI/testing/sysfs-bus-acpi F: Documentation/firmware-guide/acpi/ +F: arch/x86/kernel/acpi/ +F: arch/x86/pci/acpi.c F: drivers/acpi/ F: drivers/pci/*/*acpi* F: drivers/pci/*acpi* @@ -20098,7 +20100,8 @@ L: linux-pm@vger.kernel.org S: Supported B: https://bugzilla.kernel.org F: Documentation/power/ -F: arch/x86/kernel/acpi/ +F: arch/x86/kernel/acpi/sleep* +F: arch/x86/kernel/acpi/wakeup* F: drivers/base/power/ F: include/linux/freezer.h F: include/linux/pm.h From cf7066b97e27b2319af1ae2ef6889c4a1704312d Mon Sep 17 00:00:00 2001 From: "Joel Fernandes (Google)" Date: Thu, 12 Jan 2023 00:52:23 +0000 Subject: [PATCH 133/239] rcu: Disable laziness if lazy-tracking says so During suspend, we see failures to suspend 1 in 300-500 suspends. Looking closer, it appears that asynchronous RCU callbacks are being queued as lazy even though synchronous callbacks are expedited. These delays appear to not be very welcome by the suspend/resume code as evidenced by these occasional suspend failures. This commit modifies call_rcu() to check if rcu_async_should_hurry(), which will return true if we are in suspend or in-kernel boot. [ paulmck: Alphabetize local variables. ] Ignoring the lazy hint makes the 3000 suspend/resume cycles pass reliably on a 12th gen 12-core Intel CPU, and there is some evidence that it also slightly speeds up boot performance. Fixes: 3cb278e73be5 ("rcu: Make call_rcu() lazy to save power") Signed-off-by: Joel Fernandes (Google) Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 712c4e7b4675..4374d7af2c11 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2575,10 +2575,11 @@ static void check_cb_ovld(struct rcu_data *rdp) } static void -__call_rcu_common(struct rcu_head *head, rcu_callback_t func, bool lazy) +__call_rcu_common(struct rcu_head *head, rcu_callback_t func, bool lazy_in) { static atomic_t doublefrees; unsigned long flags; + bool lazy; struct rcu_data *rdp; bool was_alldone; @@ -2603,6 +2604,7 @@ __call_rcu_common(struct rcu_head *head, rcu_callback_t func, bool lazy) kasan_record_aux_stack_noalloc(head); local_irq_save(flags); rdp = this_cpu_ptr(&rcu_data); + lazy = lazy_in && !rcu_async_should_hurry(); /* Add the callback to our list. */ if (unlikely(!rcu_segcblist_is_enabled(&rdp->cblist))) { From 4076ea2419cf15bc1e1580f8b24ddf675fbdb02c Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Sun, 27 Nov 2022 10:30:41 -0800 Subject: [PATCH 134/239] drm/nouveau/disp: Fix nvif_outp_acquire_dp() argument size Both Coverity and GCC with -Wstringop-overflow noticed that nvif_outp_acquire_dp() accidentally defined its second argument with 1 additional element: drivers/gpu/drm/nouveau/dispnv50/disp.c: In function 'nv50_pior_atomic_enable': drivers/gpu/drm/nouveau/dispnv50/disp.c:1813:17: error: 'nvif_outp_acquire_dp' accessing 16 bytes in a region of size 15 [-Werror=stringop-overflow=] 1813 | nvif_outp_acquire_dp(&nv_encoder->outp, nv_encoder->dp.dpcd, 0, 0, false, false); | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ drivers/gpu/drm/nouveau/dispnv50/disp.c:1813:17: note: referencing argument 2 of type 'u8[16]' {aka 'unsigned char[16]'} drivers/gpu/drm/nouveau/include/nvif/outp.h:24:5: note: in a call to function 'nvif_outp_acquire_dp' 24 | int nvif_outp_acquire_dp(struct nvif_outp *, u8 dpcd[16], | ^~~~~~~~~~~~~~~~~~~~ Avoid these warnings by defining the argument size using the matching define (DP_RECEIVER_CAP_SIZE, 15) instead of having it be a literal (and incorrect) value (16). Reported-by: coverity-bot Addresses-Coverity-ID: 1527269 ("Memory - corruptions") Addresses-Coverity-ID: 1527268 ("Memory - corruptions") Link: https://lore.kernel.org/lkml/202211100848.FFBA2432@keescook/ Link: https://lore.kernel.org/lkml/202211100848.F4C2819BB@keescook/ Fixes: 813443721331 ("drm/nouveau/disp: move DP link config into acquire") Reviewed-by: Lyude Paul Cc: Ben Skeggs Cc: Karol Herbst Cc: David Airlie Cc: Daniel Vetter Cc: Dave Airlie Cc: "Gustavo A. R. Silva" Cc: dri-devel@lists.freedesktop.org Cc: nouveau@lists.freedesktop.org Signed-off-by: Kees Cook Link: https://lore.kernel.org/r/20221127183036.never.139-kees@kernel.org --- drivers/gpu/drm/nouveau/include/nvif/outp.h | 3 ++- drivers/gpu/drm/nouveau/nvif/outp.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvif/outp.h b/drivers/gpu/drm/nouveau/include/nvif/outp.h index 45daadec3c0c..fa76a7b5e4b3 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/outp.h +++ b/drivers/gpu/drm/nouveau/include/nvif/outp.h @@ -3,6 +3,7 @@ #define __NVIF_OUTP_H__ #include #include +#include struct nvif_disp; struct nvif_outp { @@ -21,7 +22,7 @@ int nvif_outp_acquire_rgb_crt(struct nvif_outp *); int nvif_outp_acquire_tmds(struct nvif_outp *, int head, bool hdmi, u8 max_ac_packet, u8 rekey, u8 scdc, bool hda); int nvif_outp_acquire_lvds(struct nvif_outp *, bool dual, bool bpc8); -int nvif_outp_acquire_dp(struct nvif_outp *, u8 dpcd[16], +int nvif_outp_acquire_dp(struct nvif_outp *outp, u8 dpcd[DP_RECEIVER_CAP_SIZE], int link_nr, int link_bw, bool hda, bool mst); void nvif_outp_release(struct nvif_outp *); int nvif_outp_infoframe(struct nvif_outp *, u8 type, struct nvif_outp_infoframe_v0 *, u32 size); diff --git a/drivers/gpu/drm/nouveau/nvif/outp.c b/drivers/gpu/drm/nouveau/nvif/outp.c index 7da39f1eae9f..c24bc5eae3ec 100644 --- a/drivers/gpu/drm/nouveau/nvif/outp.c +++ b/drivers/gpu/drm/nouveau/nvif/outp.c @@ -127,7 +127,7 @@ nvif_outp_acquire(struct nvif_outp *outp, u8 proto, struct nvif_outp_acquire_v0 } int -nvif_outp_acquire_dp(struct nvif_outp *outp, u8 dpcd[16], +nvif_outp_acquire_dp(struct nvif_outp *outp, u8 dpcd[DP_RECEIVER_CAP_SIZE], int link_nr, int link_bw, bool hda, bool mst) { struct nvif_outp_acquire_v0 args; From 16a738f2f6b31f9edf0691e0f357539f7ac66662 Mon Sep 17 00:00:00 2001 From: Paulo Miguel Almeida Date: Tue, 20 Dec 2022 16:41:15 +1300 Subject: [PATCH 135/239] i915/gvt: Replace one-element array with flexible-array member One-element arrays are deprecated, and we are replacing them with flexible array members instead. So, replace one-element array with flexible-array member in struct gvt_firmware_header and refactor the rest of the code accordingly. Additionally, previous implementation was allocating 8 bytes more than required to represent firmware_header + cfg_space data + mmio data. This helps with the ongoing efforts to tighten the FORTIFY_SOURCE routines on memcpy() and help us make progress towards globally enabling -fstrict-flex-arrays=3 [1]. To make reviewing this patch easier, I'm pasting before/after struct sizes. pahole -C gvt_firmware_header before/drivers/gpu/drm/i915/gvt/firmware.o struct gvt_firmware_header { u64 magic; /* 0 8 */ u32 crc32; /* 8 4 */ u32 version; /* 12 4 */ u64 cfg_space_size; /* 16 8 */ u64 cfg_space_offset; /* 24 8 */ u64 mmio_size; /* 32 8 */ u64 mmio_offset; /* 40 8 */ unsigned char data[1]; /* 48 1 */ /* size: 56, cachelines: 1, members: 8 */ /* padding: 7 */ /* last cacheline: 56 bytes */ }; pahole -C gvt_firmware_header after/drivers/gpu/drm/i915/gvt/firmware.o struct gvt_firmware_header { u64 magic; /* 0 8 */ u32 crc32; /* 8 4 */ u32 version; /* 12 4 */ u64 cfg_space_size; /* 16 8 */ u64 cfg_space_offset; /* 24 8 */ u64 mmio_size; /* 32 8 */ u64 mmio_offset; /* 40 8 */ unsigned char data[]; /* 48 0 */ /* size: 48, cachelines: 1, members: 8 */ /* last cacheline: 48 bytes */ }; As you can see the additional byte of the fake-flexible array (data[1]) forced the compiler to pad the struct but those bytes aren't actually used as first & last bytes (of both cfg_space and mmio) are controlled by the <>_size and <>_offset members present in the gvt_firmware_header struct. Link: https://github.com/KSPP/linux/issues/79 Link: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=101836 [1] Signed-off-by: Paulo Miguel Almeida Reviewed-by: Zhenyu Wang Signed-off-by: Kees Cook Link: https://lore.kernel.org/r/Y6Eu2604cqtryP4g@mail.google.com --- drivers/gpu/drm/i915/gvt/firmware.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/i915/gvt/firmware.c b/drivers/gpu/drm/i915/gvt/firmware.c index a683c22d5b64..dce93738e98a 100644 --- a/drivers/gpu/drm/i915/gvt/firmware.c +++ b/drivers/gpu/drm/i915/gvt/firmware.c @@ -45,7 +45,7 @@ struct gvt_firmware_header { u64 cfg_space_offset; /* offset in the file */ u64 mmio_size; u64 mmio_offset; /* offset in the file */ - unsigned char data[1]; + unsigned char data[]; }; #define dev_to_drm_minor(d) dev_get_drvdata((d)) @@ -77,7 +77,7 @@ static int expose_firmware_sysfs(struct intel_gvt *gvt) unsigned long size, crc32_start; int ret; - size = sizeof(*h) + info->mmio_size + info->cfg_space_size; + size = offsetof(struct gvt_firmware_header, data) + info->mmio_size + info->cfg_space_size; firmware = vzalloc(size); if (!firmware) return -ENOMEM; From 118901ad1f25d2334255b3d50512fa20591531cd Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 4 Jan 2023 13:09:12 -0800 Subject: [PATCH 136/239] ext4: Fix function prototype mismatch for ext4_feat_ktype With clang's kernel control flow integrity (kCFI, CONFIG_CFI_CLANG), indirect call targets are validated against the expected function pointer prototype to make sure the call target is valid to help mitigate ROP attacks. If they are not identical, there is a failure at run time, which manifests as either a kernel panic or thread getting killed. ext4_feat_ktype was setting the "release" handler to "kfree", which doesn't have a matching function prototype. Add a simple wrapper with the correct prototype. This was found as a result of Clang's new -Wcast-function-type-strict flag, which is more sensitive than the simpler -Wcast-function-type, which only checks for type width mismatches. Note that this code is only reached when ext4 is a loadable module and it is being unloaded: CFI failure at kobject_put+0xbb/0x1b0 (target: kfree+0x0/0x180; expected type: 0x7c4aa698) ... RIP: 0010:kobject_put+0xbb/0x1b0 ... Call Trace: ext4_exit_sysfs+0x14/0x60 [ext4] cleanup_module+0x67/0xedb [ext4] Fixes: b99fee58a20a ("ext4: create ext4_feat kobject dynamically") Cc: Theodore Ts'o Cc: Eric Biggers Cc: stable@vger.kernel.org Build-tested-by: Gustavo A. R. Silva Reviewed-by: Gustavo A. R. Silva Reviewed-by: Nathan Chancellor Link: https://lore.kernel.org/r/20230103234616.never.915-kees@kernel.org Signed-off-by: Kees Cook Reviewed-by: Eric Biggers Link: https://lore.kernel.org/r/20230104210908.gonna.388-kees@kernel.org --- fs/ext4/sysfs.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fs/ext4/sysfs.c b/fs/ext4/sysfs.c index d233c24ea342..e2b8b3437c58 100644 --- a/fs/ext4/sysfs.c +++ b/fs/ext4/sysfs.c @@ -491,6 +491,11 @@ static void ext4_sb_release(struct kobject *kobj) complete(&sbi->s_kobj_unregister); } +static void ext4_feat_release(struct kobject *kobj) +{ + kfree(kobj); +} + static const struct sysfs_ops ext4_attr_ops = { .show = ext4_attr_show, .store = ext4_attr_store, @@ -505,7 +510,7 @@ static struct kobj_type ext4_sb_ktype = { static struct kobj_type ext4_feat_ktype = { .default_groups = ext4_feat_groups, .sysfs_ops = &ext4_attr_ops, - .release = (void (*)(struct kobject *))kfree, + .release = ext4_feat_release, }; void ext4_notify_error_sysfs(struct ext4_sb_info *sbi) From 36632d062975a9ff4410c90dd6d37922b68d0920 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 5 Jan 2023 11:05:11 -0800 Subject: [PATCH 137/239] io_uring: Replace 0-length array with flexible array Zero-length arrays are deprecated[1]. Replace struct io_uring_buf_ring's "bufs" with a flexible array member. (How is the size of this array verified?) Detected with GCC 13, using -fstrict-flex-arrays=3: In function 'io_ring_buffer_select', inlined from 'io_buffer_select' at io_uring/kbuf.c:183:10: io_uring/kbuf.c:141:23: warning: array subscript 255 is outside the bounds of an interior zero-length array 'struct io_uring_buf[0]' [-Wzero-length-bounds] 141 | buf = &br->bufs[head]; | ^~~~~~~~~~~~~~~ In file included from include/linux/io_uring.h:7, from io_uring/kbuf.c:10: include/uapi/linux/io_uring.h: In function 'io_buffer_select': include/uapi/linux/io_uring.h:628:41: note: while referencing 'bufs' 628 | struct io_uring_buf bufs[0]; | ^~~~ [1] https://www.kernel.org/doc/html/latest/process/deprecated.html#zero-length-and-one-element-arrays Fixes: c7fb19428d67 ("io_uring: add support for ring mapped supplied buffers") Cc: Jens Axboe Cc: Pavel Begunkov Cc: "Gustavo A. R. Silva" Cc: stable@vger.kernel.org Cc: io-uring@vger.kernel.org Signed-off-by: Kees Cook Reviewed-by: Gustavo A. R. Silva Link: https://lore.kernel.org/r/20230105190507.gonna.131-kees@kernel.org --- include/uapi/linux/io_uring.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 2780bce62faf..434f62e0fb72 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -625,7 +625,7 @@ struct io_uring_buf_ring { __u16 resv3; __u16 tail; }; - struct io_uring_buf bufs[0]; + __DECLARE_FLEX_ARRAY(struct io_uring_buf, bufs); }; }; From 8500689095a39a8c245ba62709f5e8735668e535 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 5 Jan 2023 15:46:01 -0800 Subject: [PATCH 138/239] net/i40e: Replace 0-length array with flexible array Zero-length arrays are deprecated[1]. Replace struct i40e_lump_tracking's "list" 0-length array with a flexible array. Detected with GCC 13, using -fstrict-flex-arrays=3: In function 'i40e_put_lump', inlined from 'i40e_clear_interrupt_scheme' at drivers/net/ethernet/intel/i40e/i40e_main.c:5145:2: drivers/net/ethernet/intel/i40e/i40e_main.c:278:27: warning: array subscript is outside array bounds of 'u16[0]' {aka 'short unsigned int[]'} [-Warray-bounds=] 278 | pile->list[i] = 0; | ~~~~~~~~~~^~~ drivers/net/ethernet/intel/i40e/i40e.h: In function 'i40e_clear_interrupt_scheme': drivers/net/ethernet/intel/i40e/i40e.h:179:13: note: while referencing 'list' 179 | u16 list[0]; | ^~~~ [1] https://www.kernel.org/doc/html/latest/process/deprecated.html#zero-length-and-one-element-arrays Cc: Jesse Brandeburg Cc: Tony Nguyen Cc: "David S. Miller" Cc: Eric Dumazet Cc: Jakub Kicinski Cc: Paolo Abeni Cc: "Gustavo A. R. Silva" Cc: intel-wired-lan@lists.osuosl.org Cc: netdev@vger.kernel.org Signed-off-by: Kees Cook Reviewed-by: Jiri Pirko Tested-by: Gurucharan G (A Contingent worker at Intel) Reviewed-by: Gustavo A. R. Silva Link: https://lore.kernel.org/r/20230105234557.never.799-kees@kernel.org --- drivers/net/ethernet/intel/i40e/i40e.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h index 60e351665c70..3a1c28ca5bb4 100644 --- a/drivers/net/ethernet/intel/i40e/i40e.h +++ b/drivers/net/ethernet/intel/i40e/i40e.h @@ -176,7 +176,7 @@ enum i40e_interrupt_policy { struct i40e_lump_tracking { u16 num_entries; - u16 list[0]; + u16 list[]; #define I40E_PILE_VALID_BIT 0x8000 #define I40E_IWARP_IRQ_PILE_ID (I40E_PILE_VALID_BIT - 2) }; From aa85923a954e7704bc9d3847dabeb8540aa98d13 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 5 Jan 2023 20:19:48 -0800 Subject: [PATCH 139/239] crypto: hisilicon: Wipe entire pool on error To work around a Clang __builtin_object_size bug that shows up under CONFIG_FORTIFY_SOURCE and UBSAN_BOUNDS, move the per-loop-iteration mem_block wipe into a single wipe of the entire pool structure after the loop. Reported-by: Nathan Chancellor Link: https://github.com/ClangBuiltLinux/linux/issues/1780 Cc: Weili Qian Cc: Zhou Wang Cc: Herbert Xu Cc: "David S. Miller" Cc: linux-crypto@vger.kernel.org Signed-off-by: Kees Cook Tested-by: Nathan Chancellor # build Link: https://lore.kernel.org/r/20230106041945.never.831-kees@kernel.org --- drivers/crypto/hisilicon/sgl.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/crypto/hisilicon/sgl.c b/drivers/crypto/hisilicon/sgl.c index 2b6f2281cfd6..0974b0041405 100644 --- a/drivers/crypto/hisilicon/sgl.c +++ b/drivers/crypto/hisilicon/sgl.c @@ -124,9 +124,8 @@ err_free_mem: for (j = 0; j < i; j++) { dma_free_coherent(dev, block_size, block[j].sgl, block[j].sgl_dma); - memset(block + j, 0, sizeof(*block)); } - kfree(pool); + kfree_sensitive(pool); return ERR_PTR(-ENOMEM); } EXPORT_SYMBOL_GPL(hisi_acc_create_sgl_pool); From a8c55407a7230798eb157ed2cf5398a6a2b123b6 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 27 Jan 2023 17:51:35 +0200 Subject: [PATCH 140/239] lib/string: Use strchr() in strpbrk() Use strchr() instead of open coding it as it's done elsewhere in the same file. Either we will have similar to what it was or possibly better performance in case architecture implements its own strchr(). Memory wise on x86_64 bloat-o-meter shows the following Function old new delta strsep 111 102 -9 Total: Before=2763, After=2754, chg -0.33% Signed-off-by: Andy Shevchenko Signed-off-by: Kees Cook Link: https://lore.kernel.org/r/20230127155135.27153-1-andriy.shevchenko@linux.intel.com --- lib/string.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/string.c b/lib/string.c index 4fb566ea610f..3d55ef890106 100644 --- a/lib/string.c +++ b/lib/string.c @@ -480,13 +480,11 @@ EXPORT_SYMBOL(strcspn); */ char *strpbrk(const char *cs, const char *ct) { - const char *sc1, *sc2; + const char *sc; - for (sc1 = cs; *sc1 != '\0'; ++sc1) { - for (sc2 = ct; *sc2 != '\0'; ++sc2) { - if (*sc1 == *sc2) - return (char *)sc1; - } + for (sc = cs; *sc != '\0'; ++sc) { + if (strchr(ct, *sc)) + return (char *)sc; } return NULL; } From 53fc7e80f3aa9c34d396bcfbcc03a4c0d2eaac96 Mon Sep 17 00:00:00 2001 From: Shuai Xue Date: Wed, 18 Jan 2023 14:35:04 +0800 Subject: [PATCH 141/239] ACPI: APEI: EINJ: Limit error type to 32-bit width The bit map of error types to inject is 32-bit width [1]. Add parameter check to reflect the fact. [1] ACPI Specification 6.4, Section 18.6.4. Error Types Signed-off-by: Shuai Xue Reviewed-by: Tony Luck Signed-off-by: Rafael J. Wysocki --- drivers/acpi/apei/einj.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/acpi/apei/einj.c b/drivers/acpi/apei/einj.c index ab86b2f4e719..b4373e575660 100644 --- a/drivers/acpi/apei/einj.c +++ b/drivers/acpi/apei/einj.c @@ -616,6 +616,10 @@ static int error_type_set(void *data, u64 val) u32 available_error_type = 0; u32 tval, vendor; + /* Only low 32 bits for error type are valid */ + if (val & GENMASK_ULL(63, 32)) + return -EINVAL; + /* * Vendor defined types have 0x80000000 bit set, and * are not enumerated by ACPI_EINJ_GET_ERROR_TYPE From a1a32ded2887fd421d67e9c9b67ae4504bdb08c9 Mon Sep 17 00:00:00 2001 From: Armin Wolf Date: Thu, 19 Jan 2023 15:21:14 +0100 Subject: [PATCH 142/239] ACPI: battery: Fix buffer overread if not NUL-terminated If a buffer containing ASCII characters is not NUL-terminated (which is perfectly legal according to the ACPI specification), the ACPI battery driver might not honor its length. Fix this by limiting the amount of data to be copied to the buffer length while also using strscpy() to make sure that the resulting string is always NUL-terminated. Also replace strncpy() vs strscpy(). Signed-off-by: Armin Wolf Signed-off-by: Rafael J. Wysocki --- drivers/acpi/battery.c | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/drivers/acpi/battery.c b/drivers/acpi/battery.c index fb64bd217d82..0ec12a7dbcca 100644 --- a/drivers/acpi/battery.c +++ b/drivers/acpi/battery.c @@ -437,16 +437,25 @@ static int extract_package(struct acpi_battery *battery, element = &package->package.elements[i]; if (offsets[i].mode) { u8 *ptr = (u8 *)battery + offsets[i].offset; + u32 len = 32; - if (element->type == ACPI_TYPE_STRING || - element->type == ACPI_TYPE_BUFFER) - strscpy(ptr, element->string.pointer, 32); - else if (element->type == ACPI_TYPE_INTEGER) { - strncpy(ptr, (u8 *)&element->integer.value, - sizeof(u64)); - ptr[sizeof(u64)] = 0; - } else + switch (element->type) { + case ACPI_TYPE_BUFFER: + if (len > element->buffer.length + 1) + len = element->buffer.length + 1; + + fallthrough; + case ACPI_TYPE_STRING: + strscpy(ptr, element->string.pointer, len); + + break; + case ACPI_TYPE_INTEGER: + strscpy(ptr, (u8 *)&element->integer.value, sizeof(u64) + 1); + + break; + default: *ptr = 0; /* don't have value */ + } } else { int *x = (int *)((u8 *)battery + offsets[i].offset); *x = (element->type == ACPI_TYPE_INTEGER) ? From 91507d25a67c561f97c34fdd9fdf04e9a1dd7355 Mon Sep 17 00:00:00 2001 From: Armin Wolf Date: Thu, 19 Jan 2023 15:21:15 +0100 Subject: [PATCH 143/239] ACPI: battery: Increase maximum string length On the Dell Inspiron 3505, the battery model name is represented as a hex string containing seven numbers, causing it to be larger than the current maximum string length (32). Increase this length to 64 to avoid truncating the string in such cases. Also introduce a common define for the length. Signed-off-by: Armin Wolf Signed-off-by: Rafael J. Wysocki --- drivers/acpi/battery.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/acpi/battery.c b/drivers/acpi/battery.c index 0ec12a7dbcca..9c67ed02d797 100644 --- a/drivers/acpi/battery.c +++ b/drivers/acpi/battery.c @@ -42,6 +42,8 @@ #define ACPI_BATTERY_STATE_CHARGING 0x2 #define ACPI_BATTERY_STATE_CRITICAL 0x4 +#define MAX_STRING_LENGTH 64 + MODULE_AUTHOR("Paul Diefenbaugh"); MODULE_AUTHOR("Alexey Starikovskiy "); MODULE_DESCRIPTION("ACPI Battery Driver"); @@ -118,10 +120,10 @@ struct acpi_battery { int capacity_granularity_1; int capacity_granularity_2; int alarm; - char model_number[32]; - char serial_number[32]; - char type[32]; - char oem_info[32]; + char model_number[MAX_STRING_LENGTH]; + char serial_number[MAX_STRING_LENGTH]; + char type[MAX_STRING_LENGTH]; + char oem_info[MAX_STRING_LENGTH]; int state; int power_unit; unsigned long flags; @@ -437,7 +439,7 @@ static int extract_package(struct acpi_battery *battery, element = &package->package.elements[i]; if (offsets[i].mode) { u8 *ptr = (u8 *)battery + offsets[i].offset; - u32 len = 32; + u32 len = MAX_STRING_LENGTH; switch (element->type) { case ACPI_TYPE_BUFFER: From 7de6c3fb6dfbf9b920caa8cad953d8ac406f6790 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Wed, 25 Jan 2023 12:38:11 +0100 Subject: [PATCH 144/239] ACPI: PMIC: Add comments with DSDT power opregion field names The DSDTs of CHT devices using the Dollar Cove TI PMIC, all use LDO1 - LDO14 names for the DSDT power opregion field names. Add comments with these fields to make it easier to see which PMIC registers are being set by ACPI code using these. Note that LDO4 is missing and the mapped registers jump from 0x43 to 0x45 to match. This matches with how the fields are declared in the DSDT where LDO4 is skipped too. Note there is no hole in the field addresses, LDO4 is simply just not defined on either side. Reviewed-by: Andy Shevchenko Signed-off-by: Hans de Goede Signed-off-by: Rafael J. Wysocki --- drivers/acpi/pmic/intel_pmic_chtdc_ti.c | 26 ++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/drivers/acpi/pmic/intel_pmic_chtdc_ti.c b/drivers/acpi/pmic/intel_pmic_chtdc_ti.c index 418eec523025..c84ef3d15181 100644 --- a/drivers/acpi/pmic/intel_pmic_chtdc_ti.c +++ b/drivers/acpi/pmic/intel_pmic_chtdc_ti.c @@ -20,19 +20,19 @@ #define CHTDC_TI_GPADC 0x5a static struct pmic_table chtdc_ti_power_table[] = { - { .address = 0x00, .reg = 0x41 }, - { .address = 0x04, .reg = 0x42 }, - { .address = 0x08, .reg = 0x43 }, - { .address = 0x0c, .reg = 0x45 }, - { .address = 0x10, .reg = 0x46 }, - { .address = 0x14, .reg = 0x47 }, - { .address = 0x18, .reg = 0x48 }, - { .address = 0x1c, .reg = 0x49 }, - { .address = 0x20, .reg = 0x4a }, - { .address = 0x24, .reg = 0x4b }, - { .address = 0x28, .reg = 0x4c }, - { .address = 0x2c, .reg = 0x4d }, - { .address = 0x30, .reg = 0x4e }, + { .address = 0x00, .reg = 0x41 }, /* LDO1 */ + { .address = 0x04, .reg = 0x42 }, /* LDO2 */ + { .address = 0x08, .reg = 0x43 }, /* LDO3 */ + { .address = 0x0c, .reg = 0x45 }, /* LDO5 */ + { .address = 0x10, .reg = 0x46 }, /* LDO6 */ + { .address = 0x14, .reg = 0x47 }, /* LDO7 */ + { .address = 0x18, .reg = 0x48 }, /* LDO8 */ + { .address = 0x1c, .reg = 0x49 }, /* LDO9 */ + { .address = 0x20, .reg = 0x4a }, /* LD10 */ + { .address = 0x24, .reg = 0x4b }, /* LD11 */ + { .address = 0x28, .reg = 0x4c }, /* LD12 */ + { .address = 0x2c, .reg = 0x4d }, /* LD13 */ + { .address = 0x30, .reg = 0x4e }, /* LD14 */ }; static struct pmic_table chtdc_ti_thermal_table[] = { From faffb0831ad683a33e3b0f2e70eb2957874a71bb Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Thu, 26 Jan 2023 22:39:39 -0800 Subject: [PATCH 145/239] Documentation: firmware-guide/ACPI: correct spelling Correct spelling problems for Documentation/firmware-guide/ as reported by codespell. Signed-off-by: Randy Dunlap Signed-off-by: Rafael J. Wysocki --- Documentation/firmware-guide/acpi/acpi-lid.rst | 2 +- Documentation/firmware-guide/acpi/namespace.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/firmware-guide/acpi/acpi-lid.rst b/Documentation/firmware-guide/acpi/acpi-lid.rst index 71b9af13a048..03cbad6c6730 100644 --- a/Documentation/firmware-guide/acpi/acpi-lid.rst +++ b/Documentation/firmware-guide/acpi/acpi-lid.rst @@ -34,7 +34,7 @@ state upon the last _LID evaluation. There won't be difference when the _LID control method is evaluated during the runtime, the problem is its initial returning value. When the AML tables implement this control method with cached value, the initial returning value is likely not reliable. -There are platforms always retun "closed" as initial lid state. +There are platforms always return "closed" as initial lid state. Restrictions of the lid state change notifications ================================================== diff --git a/Documentation/firmware-guide/acpi/namespace.rst b/Documentation/firmware-guide/acpi/namespace.rst index 6193582a2204..4ef963679a3d 100644 --- a/Documentation/firmware-guide/acpi/namespace.rst +++ b/Documentation/firmware-guide/acpi/namespace.rst @@ -31,7 +31,7 @@ Description Table). The XSDT always points to the FADT (Fixed ACPI Description Table) using its first entry, the data within the FADT includes various fixed-length entries that describe fixed ACPI features of the hardware. The FADT contains a pointer to the DSDT -(Differentiated System Descripition Table). The XSDT also contains +(Differentiated System Description Table). The XSDT also contains entries pointing to possibly multiple SSDTs (Secondary System Description Table). From 71bc571c64c13734c69c854e97fb97ac7bf7b54a Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Thu, 26 Jan 2023 22:39:51 -0800 Subject: [PATCH 146/239] Documentation: power: correct spelling Correct spelling problems for Documentation/power/ as reported by codespell. Signed-off-by: Randy Dunlap Signed-off-by: Rafael J. Wysocki --- Documentation/power/suspend-and-interrupts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/power/suspend-and-interrupts.rst b/Documentation/power/suspend-and-interrupts.rst index 4cda6617709a..dfbace2f4600 100644 --- a/Documentation/power/suspend-and-interrupts.rst +++ b/Documentation/power/suspend-and-interrupts.rst @@ -67,7 +67,7 @@ That may involve turning on a special signal handling logic within the platform during system sleep so as to trigger a system wakeup when needed. For example, the platform may include a dedicated interrupt controller used specifically for handling system wakeup events. Then, if a given interrupt line is supposed to -wake up the system from sleep sates, the corresponding input of that interrupt +wake up the system from sleep states, the corresponding input of that interrupt controller needs to be enabled to receive signals from the line in question. After wakeup, it generally is better to disable that input to prevent the dedicated controller from triggering interrupts unnecessarily. From 68d8ad3bd9c397f2bf009368cb13e48cb91ea018 Mon Sep 17 00:00:00 2001 From: Konrad Dybcio Date: Mon, 16 Jan 2023 10:38:42 +0100 Subject: [PATCH 147/239] dt-bindings: opp: v2-qcom-level: Let qcom,opp-fuse-level be a 2-long array In some instances (particularly with CPRh) we might want to specifiy more than one qcom,opp-fuse-level, as the same OPP subnodes may be used by different "CPR threads". We need to make sure that n = num_threads entries is legal and so far nobody seems to use more than two, so let's allow that. Acked-by: Rob Herring Signed-off-by: Konrad Dybcio Signed-off-by: Viresh Kumar --- Documentation/devicetree/bindings/opp/opp-v2-qcom-level.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/opp/opp-v2-qcom-level.yaml b/Documentation/devicetree/bindings/opp/opp-v2-qcom-level.yaml index b9ce2e099ce9..a30ef93213c0 100644 --- a/Documentation/devicetree/bindings/opp/opp-v2-qcom-level.yaml +++ b/Documentation/devicetree/bindings/opp/opp-v2-qcom-level.yaml @@ -30,7 +30,9 @@ patternProperties: this OPP node. Sometimes several corners/levels shares a certain fuse corner/level. A fuse corner/level contains e.g. ref uV, min uV, and max uV. - $ref: /schemas/types.yaml#/definitions/uint32 + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 2 required: - opp-level From cf3e0251868c56b10ccb3e0b3628fcfb0c9c4ec5 Mon Sep 17 00:00:00 2001 From: Ross Zwisler Date: Mon, 30 Jan 2023 11:19:11 -0700 Subject: [PATCH 148/239] PM: tools: use canonical ftrace path The canonical location for the tracefs filesystem is at /sys/kernel/tracing. But, from Documentation/trace/ftrace.rst: Before 4.1, all ftrace tracing control files were within the debugfs file system, which is typically located at /sys/kernel/debug/tracing. For backward compatibility, when mounting the debugfs file system, the tracefs file system will be automatically mounted at: /sys/kernel/debug/tracing A few scripts in tools/power still refer to this older debugfs path, so let's update them to avoid confusion. Signed-off-by: Ross Zwisler Signed-off-by: Rafael J. Wysocki --- tools/power/pm-graph/sleepgraph.py | 4 ++-- tools/power/x86/amd_pstate_tracer/amd_pstate_trace.py | 4 ++-- .../x86/intel_pstate_tracer/intel_pstate_tracer.py | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/power/pm-graph/sleepgraph.py b/tools/power/pm-graph/sleepgraph.py index c60c90f35d18..82c09cd25cc2 100755 --- a/tools/power/pm-graph/sleepgraph.py +++ b/tools/power/pm-graph/sleepgraph.py @@ -120,9 +120,9 @@ class SystemValues: cgexp = False testdir = '' outdir = '' - tpath = '/sys/kernel/debug/tracing/' + tpath = '/sys/kernel/tracing/' fpdtpath = '/sys/firmware/acpi/tables/FPDT' - epath = '/sys/kernel/debug/tracing/events/power/' + epath = '/sys/kernel/tracing/events/power/' pmdpath = '/sys/power/pm_debug_messages' s0ixpath = '/sys/module/intel_pmc_core/parameters/warn_on_s0ix_failures' s0ixres = '/sys/devices/system/cpu/cpuidle/low_power_idle_system_residency_us' diff --git a/tools/power/x86/amd_pstate_tracer/amd_pstate_trace.py b/tools/power/x86/amd_pstate_tracer/amd_pstate_trace.py index 2dea4032ac56..904df0ea0a1e 100755 --- a/tools/power/x86/amd_pstate_tracer/amd_pstate_trace.py +++ b/tools/power/x86/amd_pstate_tracer/amd_pstate_trace.py @@ -248,7 +248,7 @@ def signal_handler(signal, frame): ipt.free_trace_buffer() sys.exit(0) -trace_file = "/sys/kernel/debug/tracing/events/amd_cpu/enable" +trace_file = "/sys/kernel/tracing/events/amd_cpu/enable" signal.signal(signal.SIGINT, signal_handler) interval = "" @@ -319,7 +319,7 @@ print(cur_version) cleanup_data_files() if interval: - file_name = "/sys/kernel/debug/tracing/trace" + file_name = "/sys/kernel/tracing/trace" ipt.clear_trace_file() ipt.set_trace_buffer_size(memory) ipt.enable_trace(trace_file) diff --git a/tools/power/x86/intel_pstate_tracer/intel_pstate_tracer.py b/tools/power/x86/intel_pstate_tracer/intel_pstate_tracer.py index b46e9eb8f5aa..ec3323100e1a 100755 --- a/tools/power/x86/intel_pstate_tracer/intel_pstate_tracer.py +++ b/tools/power/x86/intel_pstate_tracer/intel_pstate_tracer.py @@ -373,7 +373,7 @@ def clear_trace_file(): """ Clear trace file """ try: - f_handle = open('/sys/kernel/debug/tracing/trace', 'w') + f_handle = open('/sys/kernel/tracing/trace', 'w') f_handle.close() except: print('IO error clearing trace file ') @@ -401,7 +401,7 @@ def set_trace_buffer_size(memory): """ Set trace buffer size """ try: - with open('/sys/kernel/debug/tracing/buffer_size_kb', 'w') as fp: + with open('/sys/kernel/tracing/buffer_size_kb', 'w') as fp: fp.write(memory) except: print('IO error setting trace buffer size ') @@ -411,7 +411,7 @@ def free_trace_buffer(): """ Free the trace buffer memory """ try: - open('/sys/kernel/debug/tracing/buffer_size_kb' + open('/sys/kernel/tracing/buffer_size_kb' , 'w').write("1") except: print('IO error freeing trace buffer ') @@ -495,7 +495,7 @@ def signal_handler(signal, frame): sys.exit(0) if __name__ == "__main__": - trace_file = "/sys/kernel/debug/tracing/events/power/pstate_sample/enable" + trace_file = "/sys/kernel/tracing/events/power/pstate_sample/enable" signal.signal(signal.SIGINT, signal_handler) interval = "" @@ -569,7 +569,7 @@ if __name__ == "__main__": cleanup_data_files() if interval: - filename = "/sys/kernel/debug/tracing/trace" + filename = "/sys/kernel/tracing/trace" clear_trace_file() set_trace_buffer_size(memory) enable_trace(trace_file) From 5a6b64adc18d9adfb497a529ff004d59b6df151f Mon Sep 17 00:00:00 2001 From: Sam James Date: Wed, 1 Feb 2023 23:00:09 +0000 Subject: [PATCH 149/239] gcc-plugins: drop -std=gnu++11 to fix GCC 13 build The latest GCC 13 snapshot (13.0.1 20230129) gives the following: ``` cc1: error: cannot load plugin ./scripts/gcc-plugins/randomize_layout_plugin.so :./scripts/gcc-plugins/randomize_layout_plugin.so: undefined symbol: tree_code_type ``` This ends up being because of https://gcc.gnu.org/git/gitweb.cgi?p=gcc.git;h=b0241ce6e37031 upstream in GCC which changes the visibility of some types used by the kernel's plugin infrastructure like tree_code_type. After discussion with the GCC folks, we found that the kernel needs to be building plugins with the same flags used to build GCC - and GCC defaults to gnu++17 right now. The minimum GCC version needed to build the kernel is GCC 5.1 and GCC 5.1 already defaults to gnu++14 anyway, so just drop the flag, as all GCCs that could be used to build GCC already default to an acceptable version which was >= the version we forced via flags until now. Bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108634 Signed-off-by: Sam James Signed-off-by: Kees Cook Link: https://lore.kernel.org/r/20230201230009.2252783-1-sam@gentoo.org --- scripts/gcc-plugins/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gcc-plugins/Makefile b/scripts/gcc-plugins/Makefile index b34d11e22636..320afd3cf8e8 100644 --- a/scripts/gcc-plugins/Makefile +++ b/scripts/gcc-plugins/Makefile @@ -29,7 +29,7 @@ GCC_PLUGINS_DIR = $(shell $(CC) -print-file-name=plugin) plugin_cxxflags = -Wp,-MMD,$(depfile) $(KBUILD_HOSTCXXFLAGS) -fPIC \ -include $(srctree)/include/linux/compiler-version.h \ -DPLUGIN_VERSION=$(call stringify,$(KERNELVERSION)) \ - -I $(GCC_PLUGINS_DIR)/include -I $(obj) -std=gnu++11 \ + -I $(GCC_PLUGINS_DIR)/include -I $(obj) \ -fno-rtti -fno-exceptions -fasynchronous-unwind-tables \ -ggdb -Wno-narrowing -Wno-unused-variable \ -Wno-format-diag From dc7c31b07adee585fe567bf44d8ef7f5a12e521a Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 22 Nov 2022 16:52:36 -0800 Subject: [PATCH 150/239] drivers/base: Remove CONFIG_SRCU Now that the SRCU Kconfig option is unconditionally selected, there is no longer any point in conditional compilation based on CONFIG_SRCU. Therefore, remove the #ifdef and throw away the #else clause. Signed-off-by: Paul E. McKenney Cc: Greg Kroah-Hartman Cc: "Rafael J. Wysocki" Acked-by: Greg Kroah-Hartman Acked-by: Rafael J. Wysocki Reviewed-by: John Ogness --- drivers/base/core.c | 42 ------------------------------------------ 1 file changed, 42 deletions(-) diff --git a/drivers/base/core.c b/drivers/base/core.c index a3e14143ec0c..bb36aca8d1b7 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -181,7 +181,6 @@ void fw_devlink_purge_absent_suppliers(struct fwnode_handle *fwnode) } EXPORT_SYMBOL_GPL(fw_devlink_purge_absent_suppliers); -#ifdef CONFIG_SRCU static DEFINE_MUTEX(device_links_lock); DEFINE_STATIC_SRCU(device_links_srcu); @@ -220,47 +219,6 @@ static void device_link_remove_from_lists(struct device_link *link) list_del_rcu(&link->s_node); list_del_rcu(&link->c_node); } -#else /* !CONFIG_SRCU */ -static DECLARE_RWSEM(device_links_lock); - -static inline void device_links_write_lock(void) -{ - down_write(&device_links_lock); -} - -static inline void device_links_write_unlock(void) -{ - up_write(&device_links_lock); -} - -int device_links_read_lock(void) -{ - down_read(&device_links_lock); - return 0; -} - -void device_links_read_unlock(int not_used) -{ - up_read(&device_links_lock); -} - -#ifdef CONFIG_DEBUG_LOCK_ALLOC -int device_links_read_lock_held(void) -{ - return lockdep_is_held(&device_links_lock); -} -#endif - -static inline void device_link_synchronize_removal(void) -{ -} - -static void device_link_remove_from_lists(struct device_link *link) -{ - list_del(&link->s_node); - list_del(&link->c_node); -} -#endif /* !CONFIG_SRCU */ static bool device_is_ancestor(struct device *dev, struct device *target) { From a870acc11230d7bf52b9bbeb6c096448f1176964 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 22 Nov 2022 16:59:03 -0800 Subject: [PATCH 151/239] drivers/dax: Remove "select SRCU" Now that the SRCU Kconfig option is unconditionally selected, there is no longer any point in selecting it. Therefore, remove the "select SRCU" Kconfig statements. Signed-off-by: Paul E. McKenney Cc: Dan Williams Cc: Vishal Verma Cc: Dave Jiang Cc: Acked-by: Dan Williams Reviewed-by: John Ogness --- drivers/dax/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/dax/Kconfig b/drivers/dax/Kconfig index 5fdf269a822e..2bf5123e4827 100644 --- a/drivers/dax/Kconfig +++ b/drivers/dax/Kconfig @@ -1,7 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only menuconfig DAX tristate "DAX: direct access to differentiated memory" - select SRCU default m if NVDIMM_DAX if DAX From 91193b27fb7a31c1f8e22a370989c007fbc3f10b Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 22 Nov 2022 17:22:10 -0800 Subject: [PATCH 152/239] drivers/hwtracing/stm: Remove "select SRCU" Now that the SRCU Kconfig option is unconditionally selected, there is no longer any point in selecting it. Therefore, remove the "select SRCU" Kconfig statements. Signed-off-by: Paul E. McKenney Cc: Alexander Shishkin Cc: Maxime Coquelin Cc: Alexandre Torgue Cc: Cc: Acked-by: Alexander Shishkin Reviewed-by: John Ogness --- drivers/hwtracing/stm/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/hwtracing/stm/Kconfig b/drivers/hwtracing/stm/Kconfig index aad594fe79cc..eda6b11d40a1 100644 --- a/drivers/hwtracing/stm/Kconfig +++ b/drivers/hwtracing/stm/Kconfig @@ -2,7 +2,6 @@ config STM tristate "System Trace Module devices" select CONFIGFS_FS - select SRCU help A System Trace Module (STM) is a device exporting data in System Trace Protocol (STP) format as defined by MIPI STP standards. From 9276cf8b30a5962d295a01c6fd912fdb9c138ff8 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 22 Nov 2022 17:24:14 -0800 Subject: [PATCH 153/239] drivers/md: Remove "select SRCU" Now that the SRCU Kconfig option is unconditionally selected, there is no longer any point in selecting it. Therefore, remove the "select SRCU" Kconfig statements. Signed-off-by: Paul E. McKenney Cc: Song Liu Cc: Alasdair Kergon Cc: Mike Snitzer Cc: Cc: Acked-by: Mike Snitzer Reviewed-by: John Ogness --- drivers/md/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig index 998a5cfdbc4e..5f1e2593fad7 100644 --- a/drivers/md/Kconfig +++ b/drivers/md/Kconfig @@ -6,7 +6,6 @@ menuconfig MD bool "Multiple devices driver support (RAID and LVM)" depends on BLOCK - select SRCU help Support multiple physical spindles through a single logical device. Required for RAID and logical volume management. From 520bb822d7d9cecab80bb4b16cf81782275cbc96 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 22 Nov 2022 17:25:33 -0800 Subject: [PATCH 154/239] drivers/net: Remove "select SRCU" Now that the SRCU Kconfig option is unconditionally selected, there is no longer any point in selecting it. Therefore, remove the "select SRCU" Kconfig statements. Signed-off-by: Paul E. McKenney Cc: "David S. Miller" Cc: Eric Dumazet Cc: Jakub Kicinski Cc: Paolo Abeni Cc: Acked-by: Jakub Kicinski Reviewed-by: John Ogness --- drivers/net/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig index 9e63b8c43f3e..12910338ea1a 100644 --- a/drivers/net/Kconfig +++ b/drivers/net/Kconfig @@ -334,7 +334,6 @@ config NETCONSOLE_DYNAMIC config NETPOLL def_bool NETCONSOLE - select SRCU config NET_POLL_CONTROLLER def_bool NETPOLL From a8f0ff9185ae0de54dc6ec1ba4d8c5bdd29cb71b Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 22 Nov 2022 17:48:03 -0800 Subject: [PATCH 155/239] drivers/pci/controller: Remove "select SRCU" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that the SRCU Kconfig option is unconditionally selected, there is no longer any point in selecting it. Therefore, remove the "select SRCU" Kconfig statements. Signed-off-by: Paul E. McKenney Cc: Lorenzo Pieralisi Cc: Rob Herring Cc: "Krzysztof Wilczyński" Cc: Bjorn Helgaas Cc: Acked-by: Lorenzo Pieralisi Reviewed-by: John Ogness --- drivers/pci/controller/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index 1569d9a3ada0..b09cdc59bfd0 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -258,7 +258,7 @@ config PCIE_MEDIATEK_GEN3 MediaTek SoCs. config VMD - depends on PCI_MSI && X86_64 && SRCU && !UML + depends on PCI_MSI && X86_64 && !UML tristate "Intel Volume Management Device Driver" help Adds support for the Intel Volume Management Device (VMD). VMD is a From 7b3a0473d10c64be7b2b4b4d69fa87128ebb6dd0 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 22 Nov 2022 18:20:28 -0800 Subject: [PATCH 156/239] fs: Remove CONFIG_SRCU Now that the SRCU Kconfig option is unconditionally selected, there is no longer any point in conditional compilation based on CONFIG_SRCU. Therefore, remove the #ifdef and throw away the #else clause. Signed-off-by: Paul E. McKenney Cc: Jeff Layton Cc: Chuck Lever Cc: Alexander Viro Cc: Reviewed-by: Jeff Layton Reviewed-by: John Ogness --- fs/locks.c | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/fs/locks.c b/fs/locks.c index 8f01bee17715..1909a9de242c 100644 --- a/fs/locks.c +++ b/fs/locks.c @@ -1889,7 +1889,6 @@ int generic_setlease(struct file *filp, long arg, struct file_lock **flp, } EXPORT_SYMBOL(generic_setlease); -#if IS_ENABLED(CONFIG_SRCU) /* * Kernel subsystems can register to be notified on any attempt to set * a new lease with the lease_notifier_chain. This is used by (e.g.) nfsd @@ -1923,30 +1922,6 @@ void lease_unregister_notifier(struct notifier_block *nb) } EXPORT_SYMBOL_GPL(lease_unregister_notifier); -#else /* !IS_ENABLED(CONFIG_SRCU) */ -static inline void -lease_notifier_chain_init(void) -{ -} - -static inline void -setlease_notifier(long arg, struct file_lock *lease) -{ -} - -int lease_register_notifier(struct notifier_block *nb) -{ - return 0; -} -EXPORT_SYMBOL_GPL(lease_register_notifier); - -void lease_unregister_notifier(struct notifier_block *nb) -{ -} -EXPORT_SYMBOL_GPL(lease_unregister_notifier); - -#endif /* IS_ENABLED(CONFIG_SRCU) */ - /** * vfs_setlease - sets a lease on an open file * @filp: file pointer From cfa71bb282d09556673a62fbdc3d23d9e4df3fa0 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 22 Nov 2022 17:49:29 -0800 Subject: [PATCH 157/239] fs/btrfs: Remove "select SRCU" Now that the SRCU Kconfig option is unconditionally selected, there is no longer any point in selecting it. Therefore, remove the "select SRCU" Kconfig statements. Signed-off-by: Paul E. McKenney Cc: Chris Mason Cc: Josef Bacik Cc: David Sterba Cc: Acked-by: David Sterba Reviewed-by: Qu Wenruo Reviewed-by: John Ogness --- fs/btrfs/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/btrfs/Kconfig b/fs/btrfs/Kconfig index 183e5c4aed34..37b6bab90c83 100644 --- a/fs/btrfs/Kconfig +++ b/fs/btrfs/Kconfig @@ -17,7 +17,6 @@ config BTRFS_FS select FS_IOMAP select RAID6_PQ select XOR_BLOCKS - select SRCU depends on PAGE_SIZE_LESS_THAN_256KB help From 818913feb814177384a15665f4001e1be33861a4 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 22 Nov 2022 17:52:19 -0800 Subject: [PATCH 158/239] fs/notify: Remove "select SRCU" Now that the SRCU Kconfig option is unconditionally selected, there is no longer any point in selecting it. Therefore, remove the "select SRCU" Kconfig statements. Signed-off-by: Paul E. McKenney Cc: Jan Kara Cc: Amir Goldstein Cc: Acked-by: Jan Kara Reviewed-by: John Ogness --- fs/notify/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/notify/Kconfig b/fs/notify/Kconfig index c020d26ba223..c6c72c90fd25 100644 --- a/fs/notify/Kconfig +++ b/fs/notify/Kconfig @@ -1,7 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only config FSNOTIFY def_bool n - select SRCU source "fs/notify/dnotify/Kconfig" source "fs/notify/inotify/Kconfig" From dbea8bcdeda89f1c3808e8eea0e8b5e18582ef8c Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 22 Nov 2022 17:53:17 -0800 Subject: [PATCH 159/239] fs/quota: Remove "select SRCU" Now that the SRCU Kconfig option is unconditionally selected, there is no longer any point in selecting it. Therefore, remove the "select SRCU" Kconfig statements. Signed-off-by: Paul E. McKenney Cc: Jan Kara Acked-by: Jan Kara Reviewed-by: John Ogness --- fs/quota/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/quota/Kconfig b/fs/quota/Kconfig index b59cd172b5f9..d5a85a8062d0 100644 --- a/fs/quota/Kconfig +++ b/fs/quota/Kconfig @@ -6,7 +6,6 @@ config QUOTA bool "Quota support" select QUOTACTL - select SRCU help If you say Y here, you will be able to set per user limits for disk usage (also called disk quotas). Currently, it works for the From bc636dcbf1c482f30a5d59415ba43e9907f97840 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 22 Nov 2022 18:10:15 -0800 Subject: [PATCH 160/239] init: Remove "select SRCU" Now that the SRCU Kconfig option is unconditionally selected, there is no longer any point in selecting it. Therefore, remove the "select SRCU" Kconfig statements. Signed-off-by: Paul E. McKenney Cc: Andrew Morton Cc: Nick Desaulniers Cc: Masahiro Yamada Cc: Johannes Weiner Cc: Vlastimil Babka Cc: Nathan Chancellor Cc: Arnd Bergmann Cc: Miguel Ojeda Cc: Masami Hiramatsu Cc: Dmitry Torokhov Cc: Aaron Tomlin Cc: Tejun Heo Cc: Christophe Leroy Reviewed-by: Nick Desaulniers Reviewed-by: John Ogness --- init/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/init/Kconfig b/init/Kconfig index 7e5c3ddc341d..af511c726d69 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1865,7 +1865,6 @@ config PERF_EVENTS default y if PROFILING depends on HAVE_PERF_EVENTS select IRQ_WORK - select SRCU help Enable kernel support for various performance events provided by software and hardware. From 5634469360ddc3337bd73c2cdcdf7849fb779026 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 22 Nov 2022 18:22:42 -0800 Subject: [PATCH 161/239] kernel/notifier: Remove CONFIG_SRCU MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that the SRCU Kconfig option is unconditionally selected, there is no longer any point in conditional compilation based on CONFIG_SRCU. Therefore, remove the #ifdef. Signed-off-by: Paul E. McKenney Cc: "Rafael J. Wysocki" Cc: "Michał Mirosław" Cc: Borislav Petkov Cc: Alan Stern Acked-by: Rafael J. Wysocki Reviewed-by: John Ogness --- kernel/notifier.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/kernel/notifier.c b/kernel/notifier.c index ab75637fd904..d353e4b5402d 100644 --- a/kernel/notifier.c +++ b/kernel/notifier.c @@ -456,7 +456,6 @@ int raw_notifier_call_chain(struct raw_notifier_head *nh, } EXPORT_SYMBOL_GPL(raw_notifier_call_chain); -#ifdef CONFIG_SRCU /* * SRCU notifier chain routines. Registration and unregistration * use a mutex, and call_chain is synchronized by SRCU (no locks). @@ -573,8 +572,6 @@ void srcu_init_notifier_head(struct srcu_notifier_head *nh) } EXPORT_SYMBOL_GPL(srcu_init_notifier_head); -#endif /* CONFIG_SRCU */ - static ATOMIC_NOTIFIER_HEAD(die_chain); int notrace notify_die(enum die_val val, const char *str, From 608723c41cd951fb32ade2f8371e61c270816175 Mon Sep 17 00:00:00 2001 From: "Uladzislau Rezki (Sony)" Date: Wed, 1 Feb 2023 16:08:07 +0100 Subject: [PATCH 162/239] rcu/kvfree: Add kvfree_rcu_mightsleep() and kfree_rcu_mightsleep() The kvfree_rcu() and kfree_rcu() APIs are hazardous in that if you forget the second argument, it works, but might sleep. This sleeping can be a correctness bug from atomic contexts, and even in non-atomic contexts it might introduce unacceptable latencies. This commit therefore adds kvfree_rcu_mightsleep() and kfree_rcu_mightsleep(), which will replace the single-argument kvfree_rcu() and kfree_rcu(), respectively. This commit enables a series of commits that switch from single-argument kvfree_rcu() and kfree_rcu() to their _mightsleep() counterparts. Once all of these commits land, the single-argument versions will be removed. Signed-off-by: Uladzislau Rezki (Sony) Signed-off-by: Paul E. McKenney --- include/linux/rcupdate.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index f38d4469d7f3..84433600885a 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -1004,6 +1004,9 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) #define kvfree_rcu(...) KVFREE_GET_MACRO(__VA_ARGS__, \ kvfree_rcu_arg_2, kvfree_rcu_arg_1)(__VA_ARGS__) +#define kvfree_rcu_mightsleep(ptr) kvfree_rcu_arg_1(ptr) +#define kfree_rcu_mightsleep(ptr) kvfree_rcu_mightsleep(ptr) + #define KVFREE_GET_MACRO(_1, _2, NAME, ...) NAME #define kvfree_rcu_arg_2(ptr, rhf) \ do { \ From 7bc1fcd399018245575974508c26e882da0bd915 Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Tue, 31 Jan 2023 17:00:06 +0800 Subject: [PATCH 163/239] ACPI: CPPC: Add AMD pstate energy performance preference cppc control Add support for setting and querying EPP preferences to the generic CPPC driver. This enables downstream drivers such as amd-pstate to discover and use these values. Downstream drivers that want to use the new symbols cppc_get_epp_caps and cppc_set_epp_perf for querying and setting EPP preferences will need to call cppc_set_epp_perf to enable the EPP function firstly. Acked-by: Huang Rui Reviewed-by: Mario Limonciello Reviewed-by: Wyes Karny Tested-by: Wyes Karny Signed-off-by: Perry Yuan Signed-off-by: Rafael J. Wysocki --- drivers/acpi/cppc_acpi.c | 67 ++++++++++++++++++++++++++++++++++++++++ include/acpi/cppc_acpi.h | 12 +++++++ 2 files changed, 79 insertions(+) diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index 0f17b1c32718..02d83c807271 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -1153,6 +1153,19 @@ int cppc_get_nominal_perf(int cpunum, u64 *nominal_perf) return cppc_get_perf(cpunum, NOMINAL_PERF, nominal_perf); } +/** + * cppc_get_epp_perf - Get the epp register value. + * @cpunum: CPU from which to get epp preference value. + * @epp_perf: Return address. + * + * Return: 0 for success, -EIO otherwise. + */ +int cppc_get_epp_perf(int cpunum, u64 *epp_perf) +{ + return cppc_get_perf(cpunum, ENERGY_PERF, epp_perf); +} +EXPORT_SYMBOL_GPL(cppc_get_epp_perf); + /** * cppc_get_perf_caps - Get a CPU's performance capabilities. * @cpunum: CPU from which to get capabilities info. @@ -1365,6 +1378,60 @@ out_err: } EXPORT_SYMBOL_GPL(cppc_get_perf_ctrs); +/* + * Set Energy Performance Preference Register value through + * Performance Controls Interface + */ +int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls, bool enable) +{ + int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpu); + struct cpc_register_resource *epp_set_reg; + struct cpc_register_resource *auto_sel_reg; + struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpu); + struct cppc_pcc_data *pcc_ss_data = NULL; + int ret; + + if (!cpc_desc) { + pr_debug("No CPC descriptor for CPU:%d\n", cpu); + return -ENODEV; + } + + auto_sel_reg = &cpc_desc->cpc_regs[AUTO_SEL_ENABLE]; + epp_set_reg = &cpc_desc->cpc_regs[ENERGY_PERF]; + + if (CPC_IN_PCC(epp_set_reg) || CPC_IN_PCC(auto_sel_reg)) { + if (pcc_ss_id < 0) { + pr_debug("Invalid pcc_ss_id for CPU:%d\n", cpu); + return -ENODEV; + } + + if (CPC_SUPPORTED(auto_sel_reg)) { + ret = cpc_write(cpu, auto_sel_reg, enable); + if (ret) + return ret; + } + + if (CPC_SUPPORTED(epp_set_reg)) { + ret = cpc_write(cpu, epp_set_reg, perf_ctrls->energy_perf); + if (ret) + return ret; + } + + pcc_ss_data = pcc_data[pcc_ss_id]; + + down_write(&pcc_ss_data->pcc_lock); + /* after writing CPC, transfer the ownership of PCC to platform */ + ret = send_pcc_cmd(pcc_ss_id, CMD_WRITE); + up_write(&pcc_ss_data->pcc_lock); + } else { + ret = -ENOTSUPP; + pr_debug("_CPC in PCC is not supported\n"); + } + + return ret; +} +EXPORT_SYMBOL_GPL(cppc_set_epp_perf); + /** * cppc_set_enable - Set to enable CPPC on the processor by writing the * Continuous Performance Control package EnableRegister field. diff --git a/include/acpi/cppc_acpi.h b/include/acpi/cppc_acpi.h index c5614444031f..6b487a5bd638 100644 --- a/include/acpi/cppc_acpi.h +++ b/include/acpi/cppc_acpi.h @@ -108,12 +108,14 @@ struct cppc_perf_caps { u32 lowest_nonlinear_perf; u32 lowest_freq; u32 nominal_freq; + u32 energy_perf; }; struct cppc_perf_ctrls { u32 max_perf; u32 min_perf; u32 desired_perf; + u32 energy_perf; }; struct cppc_perf_fb_ctrs { @@ -149,6 +151,8 @@ extern bool cpc_ffh_supported(void); extern bool cpc_supported_by_cpu(void); extern int cpc_read_ffh(int cpunum, struct cpc_reg *reg, u64 *val); extern int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val); +extern int cppc_get_epp_perf(int cpunum, u64 *epp_perf); +extern int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls, bool enable); #else /* !CONFIG_ACPI_CPPC_LIB */ static inline int cppc_get_desired_perf(int cpunum, u64 *desired_perf) { @@ -202,6 +206,14 @@ static inline int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val) { return -ENOTSUPP; } +static inline int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls, bool enable) +{ + return -ENOTSUPP; +} +static inline int cppc_get_epp_perf(int cpunum, u64 *epp_perf) +{ + return -ENOTSUPP; +} #endif /* !CONFIG_ACPI_CPPC_LIB */ #endif /* _CPPC_ACPI_H*/ From e22abc6bb97cee240200d037a16b73951df16f9a Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Tue, 31 Jan 2023 17:00:07 +0800 Subject: [PATCH 164/239] Documentation: amd-pstate: add EPP profiles introduction The amd-pstate driver supports a feature called energy performance preference (EPP). Add information to the documentation to explain how users can interact with the sysfs files for this feature. 1) See all EPP profiles $ sudo cat /sys/devices/system/cpu/cpu0/cpufreq/energy_performance_available_preferences default performance balance_performance balance_power power 2) Check current EPP profile $ sudo cat /sys/devices/system/cpu/cpu0/cpufreq/energy_performance_preference performance 3) Set new EPP profile $ sudo bash -c "echo power > /sys/devices/system/cpu/cpu0/cpufreq/energy_performance_preference" Acked-by: Huang Rui Reviewed-by: Mario Limonciello Reviewed-by: Wyes Karny Tested-by: Wyes Karny Signed-off-by: Perry Yuan Signed-off-by: Rafael J. Wysocki --- Documentation/admin-guide/pm/amd-pstate.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst index 5376d53faaa8..98a2bb44f80c 100644 --- a/Documentation/admin-guide/pm/amd-pstate.rst +++ b/Documentation/admin-guide/pm/amd-pstate.rst @@ -262,6 +262,25 @@ lowest non-linear performance in `AMD CPPC Performance Capability `_.) This attribute is read-only. +``energy_performance_available_preferences`` + +A list of all the supported EPP preferences that could be used for +``energy_performance_preference`` on this system. +These profiles represent different hints that are provided +to the low-level firmware about the user's desired energy vs efficiency +tradeoff. ``default`` represents the epp value is set by platform +firmware. This attribute is read-only. + +``energy_performance_preference`` + +The current energy performance preference can be read from this attribute. +and user can change current preference according to energy or performance needs +Please get all support profiles list from +``energy_performance_available_preferences`` attribute, all the profiles are +integer values defined between 0 to 255 when EPP feature is enabled by platform +firmware, if EPP feature is disabled, driver will ignore the written value +This attribute is read-write. + Other performance and frequency values can be read back from ``/sys/devices/system/cpu/cpuX/acpi_cppc/``, see :ref:`cppc_sysfs`. From 36c5014e5460963ad7766487c0e22a7ff28681fc Mon Sep 17 00:00:00 2001 From: Wyes Karny Date: Tue, 31 Jan 2023 17:00:08 +0800 Subject: [PATCH 165/239] cpufreq: amd-pstate: optimize driver working mode selection in amd_pstate_param() The amd-pstate driver may support multiple working modes. Introduce a variable to keep track of which mode is currently enabled. Here we use cppc_state var to indicate which mode is enabled. This change will help to simplify the the amd_pstate_param() to choose which mode used for the following driver registration. Acked-by: Huang Rui Reviewed-by: Mario Limonciello Tested-by: Wyes Karny Signed-off-by: Perry Yuan Signed-off-by: Wyes Karny Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/amd-pstate.c | 39 +++++++++++++++++++++++++++--------- include/linux/amd-pstate.h | 17 ++++++++++++++++ 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index c17bd845f5fc..65c16edbbb20 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -60,7 +60,18 @@ * module parameter to be able to enable it manually for debugging. */ static struct cpufreq_driver amd_pstate_driver; -static int cppc_load __initdata; +static int cppc_state = AMD_PSTATE_DISABLE; + +static inline int get_mode_idx_from_str(const char *str, size_t size) +{ + int i; + + for (i=0; i < AMD_PSTATE_MAX; i++) { + if (!strncmp(str, amd_pstate_mode_string[i], size)) + return i; + } + return -EINVAL; +} static inline int pstate_enable(bool enable) { @@ -626,10 +637,10 @@ static int __init amd_pstate_init(void) /* * by default the pstate driver is disabled to load * enable the amd_pstate passive mode driver explicitly - * with amd_pstate=passive in kernel command line + * with amd_pstate=passive or other modes in kernel command line */ - if (!cppc_load) { - pr_debug("driver load is disabled, boot with amd_pstate=passive to enable this\n"); + if (cppc_state == AMD_PSTATE_DISABLE) { + pr_debug("driver load is disabled, boot with specific mode to enable this\n"); return -ENODEV; } @@ -671,16 +682,24 @@ device_initcall(amd_pstate_init); static int __init amd_pstate_param(char *str) { + size_t size; + int mode_idx; + if (!str) return -EINVAL; - if (!strcmp(str, "disable")) { - cppc_load = 0; - pr_info("driver is explicitly disabled\n"); - } else if (!strcmp(str, "passive")) - cppc_load = 1; + size = strlen(str); + mode_idx = get_mode_idx_from_str(str, size); - return 0; + if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) { + cppc_state = mode_idx; + if (cppc_state == AMD_PSTATE_DISABLE) + pr_info("driver is explicitly disabled\n"); + + return 0; + } + + return -EINVAL; } early_param("amd_pstate", amd_pstate_param); diff --git a/include/linux/amd-pstate.h b/include/linux/amd-pstate.h index 1c4b8659f171..dae2ce0f6735 100644 --- a/include/linux/amd-pstate.h +++ b/include/linux/amd-pstate.h @@ -74,4 +74,21 @@ struct amd_cpudata { bool boost_supported; }; +/* + * enum amd_pstate_mode - driver working mode of amd pstate + */ +enum amd_pstate_mode { + AMD_PSTATE_DISABLE = 0, + AMD_PSTATE_PASSIVE, + AMD_PSTATE_ACTIVE, + AMD_PSTATE_MAX, +}; + +static const char * const amd_pstate_mode_string[] = { + [AMD_PSTATE_DISABLE] = "disable", + [AMD_PSTATE_PASSIVE] = "passive", + [AMD_PSTATE_ACTIVE] = "active", + NULL, +}; + #endif /* _LINUX_AMD_PSTATE_H */ From ffa5096a7c338641f70fb06d4778e8cf400181a8 Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Tue, 31 Jan 2023 17:00:09 +0800 Subject: [PATCH 166/239] cpufreq: amd-pstate: implement Pstate EPP support for the AMD processors Add EPP driver support for AMD SoCs which support a dedicated MSR for CPPC. EPP is used by the DPM controller to configure the frequency that a core operates at during short periods of activity. The SoC EPP targets are configured on a scale from 0 to 255 where 0 represents maximum performance and 255 represents maximum efficiency. The amd-pstate driver exports profile string names to userspace that are tied to specific EPP values. The balance_performance string (0x80) provides the best balance for efficiency versus power on most systems, but users can choose other strings to meet their needs as well. $ cat /sys/devices/system/cpu/cpufreq/policy0/energy_performance_available_preferences default performance balance_performance balance_power power $ cat /sys/devices/system/cpu/cpufreq/policy0/energy_performance_preference balance_performance To enable the driver,it needs to add `amd_pstate=active` to kernel command line and kernel will load the active mode epp driver Acked-by: Huang Rui Reviewed-by: Mario Limonciello Reviewed-by: Wyes Karny Tested-by: Wyes Karny Signed-off-by: Perry Yuan Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/amd-pstate.c | 420 ++++++++++++++++++++++++++++++++++- include/linux/amd-pstate.h | 16 +- 2 files changed, 429 insertions(+), 7 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 65c16edbbb20..bca86b5b8b12 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -59,9 +59,52 @@ * we disable it by default to go acpi-cpufreq on these processors and add a * module parameter to be able to enable it manually for debugging. */ +static struct cpufreq_driver *current_pstate_driver; static struct cpufreq_driver amd_pstate_driver; +static struct cpufreq_driver amd_pstate_epp_driver; static int cppc_state = AMD_PSTATE_DISABLE; +/* + * AMD Energy Preference Performance (EPP) + * The EPP is used in the CCLK DPM controller to drive + * the frequency that a core is going to operate during + * short periods of activity. EPP values will be utilized for + * different OS profiles (balanced, performance, power savings) + * display strings corresponding to EPP index in the + * energy_perf_strings[] + * index String + *------------------------------------- + * 0 default + * 1 performance + * 2 balance_performance + * 3 balance_power + * 4 power + */ +enum energy_perf_value_index { + EPP_INDEX_DEFAULT = 0, + EPP_INDEX_PERFORMANCE, + EPP_INDEX_BALANCE_PERFORMANCE, + EPP_INDEX_BALANCE_POWERSAVE, + EPP_INDEX_POWERSAVE, +}; + +static const char * const energy_perf_strings[] = { + [EPP_INDEX_DEFAULT] = "default", + [EPP_INDEX_PERFORMANCE] = "performance", + [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", + [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", + [EPP_INDEX_POWERSAVE] = "power", + NULL +}; + +static unsigned int epp_values[] = { + [EPP_INDEX_DEFAULT] = 0, + [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE, + [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE, + [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, + [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, + }; + static inline int get_mode_idx_from_str(const char *str, size_t size) { int i; @@ -73,6 +116,114 @@ static inline int get_mode_idx_from_str(const char *str, size_t size) return -EINVAL; } +static DEFINE_MUTEX(amd_pstate_limits_lock); +static DEFINE_MUTEX(amd_pstate_driver_lock); + +static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached) +{ + u64 epp; + int ret; + + if (boot_cpu_has(X86_FEATURE_CPPC)) { + if (!cppc_req_cached) { + epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, + &cppc_req_cached); + if (epp) + return epp; + } + epp = (cppc_req_cached >> 24) & 0xFF; + } else { + ret = cppc_get_epp_perf(cpudata->cpu, &epp); + if (ret < 0) { + pr_debug("Could not retrieve energy perf value (%d)\n", ret); + return -EIO; + } + } + + return (s16)(epp & 0xff); +} + +static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata) +{ + s16 epp; + int index = -EINVAL; + + epp = amd_pstate_get_epp(cpudata, 0); + if (epp < 0) + return epp; + + switch (epp) { + case AMD_CPPC_EPP_PERFORMANCE: + index = EPP_INDEX_PERFORMANCE; + break; + case AMD_CPPC_EPP_BALANCE_PERFORMANCE: + index = EPP_INDEX_BALANCE_PERFORMANCE; + break; + case AMD_CPPC_EPP_BALANCE_POWERSAVE: + index = EPP_INDEX_BALANCE_POWERSAVE; + break; + case AMD_CPPC_EPP_POWERSAVE: + index = EPP_INDEX_POWERSAVE; + break; + default: + break; + } + + return index; +} + +static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp) +{ + int ret; + struct cppc_perf_ctrls perf_ctrls; + + if (boot_cpu_has(X86_FEATURE_CPPC)) { + u64 value = READ_ONCE(cpudata->cppc_req_cached); + + value &= ~GENMASK_ULL(31, 24); + value |= (u64)epp << 24; + WRITE_ONCE(cpudata->cppc_req_cached, value); + + ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); + if (!ret) + cpudata->epp_cached = epp; + } else { + perf_ctrls.energy_perf = epp; + ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); + if (ret) { + pr_debug("failed to set energy perf value (%d)\n", ret); + return ret; + } + cpudata->epp_cached = epp; + } + + return ret; +} + +static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, + int pref_index) +{ + int epp = -EINVAL; + int ret; + + if (!pref_index) { + pr_debug("EPP pref_index is invalid\n"); + return -EINVAL; + } + + if (epp == -EINVAL) + epp = epp_values[pref_index]; + + if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { + pr_debug("EPP cannot be set under performance policy\n"); + return -EBUSY; + } + + ret = amd_pstate_set_epp(cpudata, epp); + + return ret; +} + static inline int pstate_enable(bool enable) { return wrmsrl_safe(MSR_AMD_CPPC_ENABLE, enable); @@ -81,11 +232,21 @@ static inline int pstate_enable(bool enable) static int cppc_enable(bool enable) { int cpu, ret = 0; + struct cppc_perf_ctrls perf_ctrls; for_each_present_cpu(cpu) { ret = cppc_set_enable(cpu, enable); if (ret) return ret; + + /* Enable autonomous mode for EPP */ + if (cppc_state == AMD_PSTATE_ACTIVE) { + /* Set desired perf as zero to allow EPP firmware control */ + perf_ctrls.desired_perf = 0; + ret = cppc_set_perf(cpu, &perf_ctrls); + if (ret) + return ret; + } } return ret; @@ -429,7 +590,7 @@ static void amd_pstate_boost_init(struct amd_cpudata *cpudata) return; cpudata->boost_supported = true; - amd_pstate_driver.boost_enabled = true; + current_pstate_driver->boost_enabled = true; } static void amd_perf_ctl_reset(unsigned int cpu) @@ -603,10 +764,61 @@ static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, return sprintf(&buf[0], "%u\n", perf); } +static ssize_t show_energy_performance_available_preferences( + struct cpufreq_policy *policy, char *buf) +{ + int i = 0; + int offset = 0; + + while (energy_perf_strings[i] != NULL) + offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i++]); + + sysfs_emit_at(buf, offset, "\n"); + + return offset; +} + +static ssize_t store_energy_performance_preference( + struct cpufreq_policy *policy, const char *buf, size_t count) +{ + struct amd_cpudata *cpudata = policy->driver_data; + char str_preference[21]; + ssize_t ret; + + ret = sscanf(buf, "%20s", str_preference); + if (ret != 1) + return -EINVAL; + + ret = match_string(energy_perf_strings, -1, str_preference); + if (ret < 0) + return -EINVAL; + + mutex_lock(&amd_pstate_limits_lock); + ret = amd_pstate_set_energy_pref_index(cpudata, ret); + mutex_unlock(&amd_pstate_limits_lock); + + return ret ?: count; +} + +static ssize_t show_energy_performance_preference( + struct cpufreq_policy *policy, char *buf) +{ + struct amd_cpudata *cpudata = policy->driver_data; + int preference; + + preference = amd_pstate_get_energy_pref_index(cpudata); + if (preference < 0) + return preference; + + return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]); +} + cpufreq_freq_attr_ro(amd_pstate_max_freq); cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); cpufreq_freq_attr_ro(amd_pstate_highest_perf); +cpufreq_freq_attr_rw(energy_performance_preference); +cpufreq_freq_attr_ro(energy_performance_available_preferences); static struct freq_attr *amd_pstate_attr[] = { &amd_pstate_max_freq, @@ -615,6 +827,186 @@ static struct freq_attr *amd_pstate_attr[] = { NULL, }; +static struct freq_attr *amd_pstate_epp_attr[] = { + &amd_pstate_max_freq, + &amd_pstate_lowest_nonlinear_freq, + &amd_pstate_highest_perf, + &energy_performance_preference, + &energy_performance_available_preferences, + NULL, +}; + +static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) +{ + int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; + struct amd_cpudata *cpudata; + struct device *dev; + int rc; + u64 value; + + /* + * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, + * which is ideal for initialization process. + */ + amd_perf_ctl_reset(policy->cpu); + dev = get_cpu_device(policy->cpu); + if (!dev) + goto free_cpudata1; + + cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); + if (!cpudata) + return -ENOMEM; + + cpudata->cpu = policy->cpu; + cpudata->epp_policy = 0; + + rc = amd_pstate_init_perf(cpudata); + if (rc) + goto free_cpudata1; + + min_freq = amd_get_min_freq(cpudata); + max_freq = amd_get_max_freq(cpudata); + nominal_freq = amd_get_nominal_freq(cpudata); + lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); + if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { + dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", + min_freq, max_freq); + ret = -EINVAL; + goto free_cpudata1; + } + + policy->cpuinfo.min_freq = min_freq; + policy->cpuinfo.max_freq = max_freq; + /* It will be updated by governor */ + policy->cur = policy->cpuinfo.min_freq; + + /* Initial processor data capability frequencies */ + cpudata->max_freq = max_freq; + cpudata->min_freq = min_freq; + cpudata->nominal_freq = nominal_freq; + cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; + + policy->driver_data = cpudata; + + cpudata->epp_cached = amd_pstate_get_epp(cpudata, 0); + + policy->min = policy->cpuinfo.min_freq; + policy->max = policy->cpuinfo.max_freq; + + /* + * Set the policy to powersave to provide a valid fallback value in case + * the default cpufreq governor is neither powersave nor performance. + */ + policy->policy = CPUFREQ_POLICY_POWERSAVE; + + if (boot_cpu_has(X86_FEATURE_CPPC)) { + policy->fast_switch_possible = true; + ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); + if (ret) + return ret; + WRITE_ONCE(cpudata->cppc_req_cached, value); + + ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value); + if (ret) + return ret; + WRITE_ONCE(cpudata->cppc_cap1_cached, value); + } + amd_pstate_boost_init(cpudata); + + return 0; + +free_cpudata1: + kfree(cpudata); + return ret; +} + +static int amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) +{ + pr_debug("CPU %d exiting\n", policy->cpu); + policy->fast_switch_possible = false; + return 0; +} + +static void amd_pstate_epp_init(unsigned int cpu) +{ + struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); + struct amd_cpudata *cpudata = policy->driver_data; + u32 max_perf, min_perf; + u64 value; + s16 epp; + + max_perf = READ_ONCE(cpudata->highest_perf); + min_perf = READ_ONCE(cpudata->lowest_perf); + + value = READ_ONCE(cpudata->cppc_req_cached); + + if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) + min_perf = max_perf; + + /* Initial min/max values for CPPC Performance Controls Register */ + value &= ~AMD_CPPC_MIN_PERF(~0L); + value |= AMD_CPPC_MIN_PERF(min_perf); + + value &= ~AMD_CPPC_MAX_PERF(~0L); + value |= AMD_CPPC_MAX_PERF(max_perf); + + /* CPPC EPP feature require to set zero to the desire perf bit */ + value &= ~AMD_CPPC_DES_PERF(~0L); + value |= AMD_CPPC_DES_PERF(0); + + if (cpudata->epp_policy == cpudata->policy) + goto skip_epp; + + cpudata->epp_policy = cpudata->policy; + + if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { + epp = amd_pstate_get_epp(cpudata, value); + if (epp < 0) + goto skip_epp; + /* force the epp value to be zero for performance policy */ + epp = 0; + } else { + /* Get BIOS pre-defined epp value */ + epp = amd_pstate_get_epp(cpudata, value); + if (epp) + goto skip_epp; + } + /* Set initial EPP value */ + if (boot_cpu_has(X86_FEATURE_CPPC)) { + value &= ~GENMASK_ULL(31, 24); + value |= (u64)epp << 24; + } + +skip_epp: + WRITE_ONCE(cpudata->cppc_req_cached, value); + amd_pstate_set_epp(cpudata, epp); + cpufreq_cpu_put(policy); +} + +static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata = policy->driver_data; + + if (!policy->cpuinfo.max_freq) + return -ENODEV; + + pr_debug("set_policy: cpuinfo.max %u policy->max %u\n", + policy->cpuinfo.max_freq, policy->max); + + cpudata->policy = policy->policy; + + amd_pstate_epp_init(policy->cpu); + + return 0; +} + +static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy) +{ + cpufreq_verify_within_cpu_limits(policy); + pr_debug("policy_max =%d, policy_min=%d\n", policy->max, policy->min); + return 0; +} + static struct cpufreq_driver amd_pstate_driver = { .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, .verify = amd_pstate_verify, @@ -628,6 +1020,16 @@ static struct cpufreq_driver amd_pstate_driver = { .attr = amd_pstate_attr, }; +static struct cpufreq_driver amd_pstate_epp_driver = { + .flags = CPUFREQ_CONST_LOOPS, + .verify = amd_pstate_epp_verify_policy, + .setpolicy = amd_pstate_epp_set_policy, + .init = amd_pstate_epp_cpu_init, + .exit = amd_pstate_epp_cpu_exit, + .name = "amd_pstate_epp", + .attr = amd_pstate_epp_attr, +}; + static int __init amd_pstate_init(void) { int ret; @@ -656,7 +1058,8 @@ static int __init amd_pstate_init(void) /* capability check */ if (boot_cpu_has(X86_FEATURE_CPPC)) { pr_debug("AMD CPPC MSR based functionality is supported\n"); - amd_pstate_driver.adjust_perf = amd_pstate_adjust_perf; + if (cppc_state == AMD_PSTATE_PASSIVE) + current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; } else { pr_debug("AMD CPPC shared memory based functionality is supported\n"); static_call_update(amd_pstate_enable, cppc_enable); @@ -667,14 +1070,13 @@ static int __init amd_pstate_init(void) /* enable amd pstate feature */ ret = amd_pstate_enable(true); if (ret) { - pr_err("failed to enable amd-pstate with return %d\n", ret); + pr_err("failed to enable with return %d\n", ret); return ret; } - ret = cpufreq_register_driver(&amd_pstate_driver); + ret = cpufreq_register_driver(current_pstate_driver); if (ret) - pr_err("failed to register amd_pstate_driver with return %d\n", - ret); + pr_err("failed to register with return %d\n", ret); return ret; } @@ -696,6 +1098,12 @@ static int __init amd_pstate_param(char *str) if (cppc_state == AMD_PSTATE_DISABLE) pr_info("driver is explicitly disabled\n"); + if (cppc_state == AMD_PSTATE_ACTIVE) + current_pstate_driver = &amd_pstate_epp_driver; + + if (cppc_state == AMD_PSTATE_PASSIVE) + current_pstate_driver = &amd_pstate_driver; + return 0; } diff --git a/include/linux/amd-pstate.h b/include/linux/amd-pstate.h index dae2ce0f6735..72ea7cf85ca3 100644 --- a/include/linux/amd-pstate.h +++ b/include/linux/amd-pstate.h @@ -12,6 +12,11 @@ #include +#define AMD_CPPC_EPP_PERFORMANCE 0x00 +#define AMD_CPPC_EPP_BALANCE_PERFORMANCE 0x80 +#define AMD_CPPC_EPP_BALANCE_POWERSAVE 0xBF +#define AMD_CPPC_EPP_POWERSAVE 0xFF + /********************************************************************* * AMD P-state INTERFACE * *********************************************************************/ @@ -47,6 +52,10 @@ struct amd_aperf_mperf { * @prev: Last Aperf/Mperf/tsc count value read from register * @freq: current cpu frequency value * @boost_supported: check whether the Processor or SBIOS supports boost mode + * @epp_policy: Last saved policy used to set energy-performance preference + * @epp_cached: Cached CPPC energy-performance preference value + * @policy: Cpufreq policy value + * @cppc_cap1_cached Cached MSR_AMD_CPPC_CAP1 register value * * The amd_cpudata is key private data for each CPU thread in AMD P-State, and * represents all the attributes and goals that AMD P-State requests at runtime. @@ -72,6 +81,12 @@ struct amd_cpudata { u64 freq; bool boost_supported; + + /* EPP feature related attributes*/ + s16 epp_policy; + s16 epp_cached; + u32 policy; + u64 cppc_cap1_cached; }; /* @@ -90,5 +105,4 @@ static const char * const amd_pstate_mode_string[] = { [AMD_PSTATE_ACTIVE] = "active", NULL, }; - #endif /* _LINUX_AMD_PSTATE_H */ From d4da12f8033a123353eccf993cb95ee5bff21e7c Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Tue, 31 Jan 2023 17:00:10 +0800 Subject: [PATCH 167/239] cpufreq: amd-pstate: implement amd pstate cpu online and offline callback Adds online and offline driver callback support to allow cpu cores go offline and help to restore the previous working states when core goes back online later for EPP driver mode. Acked-by: Huang Rui Reviewed-by: Mario Limonciello Reviewed-by: Wyes Karny Tested-by: Wyes Karny Signed-off-by: Perry Yuan Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/amd-pstate.c | 82 ++++++++++++++++++++++++++++++++++++ include/linux/amd-pstate.h | 1 + 2 files changed, 83 insertions(+) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index bca86b5b8b12..26f6ac83d87e 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1000,6 +1000,86 @@ static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) return 0; } +static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata) +{ + struct cppc_perf_ctrls perf_ctrls; + u64 value, max_perf; + int ret; + + ret = amd_pstate_enable(true); + if (ret) + pr_err("failed to enable amd pstate during resume, return %d\n", ret); + + value = READ_ONCE(cpudata->cppc_req_cached); + max_perf = READ_ONCE(cpudata->highest_perf); + + if (boot_cpu_has(X86_FEATURE_CPPC)) { + wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); + } else { + perf_ctrls.max_perf = max_perf; + perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached); + cppc_set_perf(cpudata->cpu, &perf_ctrls); + } +} + +static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata = policy->driver_data; + + pr_debug("AMD CPU Core %d going online\n", cpudata->cpu); + + if (cppc_state == AMD_PSTATE_ACTIVE) { + amd_pstate_epp_reenable(cpudata); + cpudata->suspended = false; + } + + return 0; +} + +static void amd_pstate_epp_offline(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata = policy->driver_data; + struct cppc_perf_ctrls perf_ctrls; + int min_perf; + u64 value; + + min_perf = READ_ONCE(cpudata->lowest_perf); + value = READ_ONCE(cpudata->cppc_req_cached); + + mutex_lock(&amd_pstate_limits_lock); + if (boot_cpu_has(X86_FEATURE_CPPC)) { + cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN; + + /* Set max perf same as min perf */ + value &= ~AMD_CPPC_MAX_PERF(~0L); + value |= AMD_CPPC_MAX_PERF(min_perf); + value &= ~AMD_CPPC_MIN_PERF(~0L); + value |= AMD_CPPC_MIN_PERF(min_perf); + wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); + } else { + perf_ctrls.desired_perf = 0; + perf_ctrls.max_perf = min_perf; + perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(HWP_EPP_BALANCE_POWERSAVE); + cppc_set_perf(cpudata->cpu, &perf_ctrls); + } + mutex_unlock(&amd_pstate_limits_lock); +} + +static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata = policy->driver_data; + + pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu); + + if (cpudata->suspended) + return 0; + + if (cppc_state == AMD_PSTATE_ACTIVE) + amd_pstate_epp_offline(policy); + + return 0; +} + static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy) { cpufreq_verify_within_cpu_limits(policy); @@ -1026,6 +1106,8 @@ static struct cpufreq_driver amd_pstate_epp_driver = { .setpolicy = amd_pstate_epp_set_policy, .init = amd_pstate_epp_cpu_init, .exit = amd_pstate_epp_cpu_exit, + .offline = amd_pstate_epp_cpu_offline, + .online = amd_pstate_epp_cpu_online, .name = "amd_pstate_epp", .attr = amd_pstate_epp_attr, }; diff --git a/include/linux/amd-pstate.h b/include/linux/amd-pstate.h index 72ea7cf85ca3..f5f22418e64b 100644 --- a/include/linux/amd-pstate.h +++ b/include/linux/amd-pstate.h @@ -87,6 +87,7 @@ struct amd_cpudata { s16 epp_cached; u32 policy; u64 cppc_cap1_cached; + bool suspended; }; /* From 50ddd2f7826927e6dc111a43b3a183f53c260fa4 Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Tue, 31 Jan 2023 17:00:11 +0800 Subject: [PATCH 168/239] cpufreq: amd-pstate: implement suspend and resume callbacks add suspend and resume support for the AMD processors by amd_pstate_epp driver instance. When the CPPC is suspended, EPP driver will set EPP profile to 'power' profile and set max/min perf to lowest perf value. When resume happens, it will restore the MSR registers with previous cached value. Acked-by: Huang Rui Reviewed-by: Mario Limonciello Reviewed-by: Wyes Karny Tested-by: Wyes Karny Signed-off-by: Perry Yuan Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/amd-pstate.c | 40 ++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 26f6ac83d87e..4e3770e0d4d3 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1087,6 +1087,44 @@ static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy) return 0; } +static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata = policy->driver_data; + int ret; + + /* avoid suspending when EPP is not enabled */ + if (cppc_state != AMD_PSTATE_ACTIVE) + return 0; + + /* set this flag to avoid setting core offline*/ + cpudata->suspended = true; + + /* disable CPPC in lowlevel firmware */ + ret = amd_pstate_enable(false); + if (ret) + pr_err("failed to suspend, return %d\n", ret); + + return 0; +} + +static int amd_pstate_epp_resume(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata = policy->driver_data; + + if (cpudata->suspended) { + mutex_lock(&amd_pstate_limits_lock); + + /* enable amd pstate from suspend state*/ + amd_pstate_epp_reenable(cpudata); + + mutex_unlock(&amd_pstate_limits_lock); + + cpudata->suspended = false; + } + + return 0; +} + static struct cpufreq_driver amd_pstate_driver = { .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, .verify = amd_pstate_verify, @@ -1108,6 +1146,8 @@ static struct cpufreq_driver amd_pstate_epp_driver = { .exit = amd_pstate_epp_cpu_exit, .offline = amd_pstate_epp_cpu_offline, .online = amd_pstate_epp_cpu_online, + .suspend = amd_pstate_epp_suspend, + .resume = amd_pstate_epp_resume, .name = "amd_pstate_epp", .attr = amd_pstate_epp_attr, }; From abd61c08ef349af08df0bf587d33f5bde5996a89 Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Tue, 31 Jan 2023 17:00:12 +0800 Subject: [PATCH 169/239] cpufreq: amd-pstate: add driver working mode switch support While amd-pstate driver was loaded with specific driver mode, it will need to check which mode is enabled for the pstate driver,add this sysfs entry to show the current status $ cat /sys/devices/system/cpu/amd-pstate/status active Meanwhile, user can switch the pstate driver mode with writing mode string to sysfs entry as below. Enable passive mode: $ sudo bash -c "echo passive > /sys/devices/system/cpu/amd-pstate/status" Enable active mode (EPP driver mode): $ sudo bash -c "echo active > /sys/devices/system/cpu/amd-pstate/status" Acked-by: Huang Rui Reviewed-by: Mario Limonciello Reviewed-by: Wyes Karny Tested-by: Wyes Karny Signed-off-by: Perry Yuan Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/amd-pstate.c | 118 +++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 4e3770e0d4d3..1ae2e0d56ed1 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -63,6 +63,7 @@ static struct cpufreq_driver *current_pstate_driver; static struct cpufreq_driver amd_pstate_driver; static struct cpufreq_driver amd_pstate_epp_driver; static int cppc_state = AMD_PSTATE_DISABLE; +struct kobject *amd_pstate_kobj; /* * AMD Energy Preference Performance (EPP) @@ -673,6 +674,8 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) policy->driver_data = cpudata; amd_pstate_boost_init(cpudata); + if (!current_pstate_driver->adjust_perf) + current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; return 0; @@ -813,12 +816,99 @@ static ssize_t show_energy_performance_preference( return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]); } +static ssize_t amd_pstate_show_status(char *buf) +{ + if (!current_pstate_driver) + return sysfs_emit(buf, "disable\n"); + + return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]); +} + +static void amd_pstate_driver_cleanup(void) +{ + current_pstate_driver = NULL; +} + +static int amd_pstate_update_status(const char *buf, size_t size) +{ + int ret; + int mode_idx; + + if (size > 7 || size < 6) + return -EINVAL; + mode_idx = get_mode_idx_from_str(buf, size); + + switch(mode_idx) { + case AMD_PSTATE_DISABLE: + if (!current_pstate_driver) + return -EINVAL; + if (cppc_state == AMD_PSTATE_ACTIVE) + return -EBUSY; + ret = cpufreq_unregister_driver(current_pstate_driver); + amd_pstate_driver_cleanup(); + break; + case AMD_PSTATE_PASSIVE: + if (current_pstate_driver) { + if (current_pstate_driver == &amd_pstate_driver) + return 0; + cpufreq_unregister_driver(current_pstate_driver); + cppc_state = AMD_PSTATE_PASSIVE; + current_pstate_driver = &amd_pstate_driver; + } + + ret = cpufreq_register_driver(current_pstate_driver); + break; + case AMD_PSTATE_ACTIVE: + if (current_pstate_driver) { + if (current_pstate_driver == &amd_pstate_epp_driver) + return 0; + cpufreq_unregister_driver(current_pstate_driver); + current_pstate_driver = &amd_pstate_epp_driver; + cppc_state = AMD_PSTATE_ACTIVE; + } + + ret = cpufreq_register_driver(current_pstate_driver); + break; + default: + ret = -EINVAL; + break; + } + + return ret; +} + +static ssize_t show_status(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + ssize_t ret; + + mutex_lock(&amd_pstate_driver_lock); + ret = amd_pstate_show_status(buf); + mutex_unlock(&amd_pstate_driver_lock); + + return ret; +} + +static ssize_t store_status(struct kobject *a, struct kobj_attribute *b, + const char *buf, size_t count) +{ + char *p = memchr(buf, '\n', count); + int ret; + + mutex_lock(&amd_pstate_driver_lock); + ret = amd_pstate_update_status(buf, p ? p - buf : count); + mutex_unlock(&amd_pstate_driver_lock); + + return ret < 0 ? ret : count; +} + cpufreq_freq_attr_ro(amd_pstate_max_freq); cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); cpufreq_freq_attr_ro(amd_pstate_highest_perf); cpufreq_freq_attr_rw(energy_performance_preference); cpufreq_freq_attr_ro(energy_performance_available_preferences); +define_one_global_rw(status); static struct freq_attr *amd_pstate_attr[] = { &amd_pstate_max_freq, @@ -836,6 +926,15 @@ static struct freq_attr *amd_pstate_epp_attr[] = { NULL, }; +static struct attribute *pstate_global_attributes[] = { + &status.attr, + NULL +}; + +static const struct attribute_group amd_pstate_global_attr_group = { + .attrs = pstate_global_attributes, +}; + static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) { int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; @@ -1200,6 +1299,25 @@ static int __init amd_pstate_init(void) if (ret) pr_err("failed to register with return %d\n", ret); + amd_pstate_kobj = kobject_create_and_add("amd_pstate", &cpu_subsys.dev_root->kobj); + if (!amd_pstate_kobj) { + ret = -EINVAL; + pr_err("global sysfs registration failed.\n"); + goto kobject_free; + } + + ret = sysfs_create_group(amd_pstate_kobj, &amd_pstate_global_attr_group); + if (ret) { + pr_err("sysfs attribute export failed with error %d.\n", ret); + goto global_attr_free; + } + + return ret; + +global_attr_free: + kobject_put(amd_pstate_kobj); +kobject_free: + cpufreq_unregister_driver(current_pstate_driver); return ret; } device_initcall(amd_pstate_init); From 92e6088427c5da7ef8dc92d6ab2f0f8f6a01fab7 Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Tue, 31 Jan 2023 17:00:13 +0800 Subject: [PATCH 170/239] Documentation: amd-pstate: add amd pstate driver mode introduction The amd-pstate driver has two operation modes supported: * CPPC Autonomous (active) mode * CPPC non-autonomous (passive) mode. active mode and passive mode can be chosen by different kernel parameters. Acked-by: Huang Rui Reviewed-by: Mario Limonciello Reviewed-by: Wyes Karny Tested-by: Wyes Karny Signed-off-by: Perry Yuan Signed-off-by: Rafael J. Wysocki --- Documentation/admin-guide/pm/amd-pstate.rst | 26 +++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst index 98a2bb44f80c..b6aee69f564f 100644 --- a/Documentation/admin-guide/pm/amd-pstate.rst +++ b/Documentation/admin-guide/pm/amd-pstate.rst @@ -299,8 +299,30 @@ module which supports the new AMD P-States mechanism on most of the future AMD platforms. The AMD P-States mechanism is the more performance and energy efficiency frequency management method on AMD processors. -Kernel Module Options for ``amd-pstate`` -========================================= + +AMD Pstate Driver Operation Modes +================================= + +``amd_pstate`` CPPC has two operation modes: CPPC Autonomous(active) mode and +CPPC non-autonomous(passive) mode. +active mode and passive mode can be chosen by different kernel parameters. +When in Autonomous mode, CPPC ignores requests done in the Desired Performance +Target register and takes into account only the values set to the Minimum requested +performance, Maximum requested performance, and Energy Performance Preference +registers. When Autonomous is disabled, it only considers the Desired Performance Target. + +Active Mode +------------ + +``amd_pstate=active`` + +This is the low-level firmware control mode which is implemented by ``amd_pstate_epp`` +driver with ``amd_pstate=active`` passed to the kernel in the command line. +In this mode, ``amd_pstate_epp`` driver provides a hint to the hardware if software +wants to bias toward performance (0x0) or energy efficiency (0xff) to the CPPC firmware. +then CPPC power algorithm will calculate the runtime workload and adjust the realtime +cores frequency according to the power supply and thermal, core voltage and some other +hardware conditions. Passive Mode ------------ From 5014603e409b01001bfbeae090a16733f61a7640 Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Tue, 31 Jan 2023 17:00:14 +0800 Subject: [PATCH 171/239] Documentation: introduce amd pstate active mode kernel command line options AMD Pstate driver support another firmware based autonomous mode with "amd_pstate=active" added to the kernel command line. In autonomous mode SMU firmware decides frequencies at runtime based on workload utilization, usage in other IPs, infrastructure limits such as power, thermals and so on. Acked-by: Huang Rui Reviewed-by: Mario Limonciello Reviewed-by: Wyes Karny Tested-by: Wyes Karny Signed-off-by: Perry Yuan Signed-off-by: Rafael J. Wysocki --- Documentation/admin-guide/kernel-parameters.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 6cfa6e3996cf..e3618dfdb36a 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -7020,3 +7020,10 @@ management firmware translates the requests into actual hardware states (core frequency, data fabric and memory clocks etc.) + active + Use amd_pstate_epp driver instance as the scaling driver, + driver provides a hint to the hardware if software wants + to bias toward performance (0x0) or energy efficiency (0xff) + to the CPPC firmware. then CPPC power algorithm will + calculate the runtime workload and adjust the realtime cores + frequency. From 3ec32b6d17c5b229c6f5d05849932af1f0c6f523 Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Tue, 31 Jan 2023 17:00:15 +0800 Subject: [PATCH 172/239] cpufreq: amd-pstate: convert sprintf with sysfs_emit() replace the sprintf with a more generic sysfs_emit function No intended potential function impact Acked-by: Huang Rui Reviewed-by: Mario Limonciello Reviewed-by: Wyes Karny Tested-by: Wyes Karny Signed-off-by: Perry Yuan Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/amd-pstate.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 1ae2e0d56ed1..168a28bed6ee 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -736,7 +736,7 @@ static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, if (max_freq < 0) return max_freq; - return sprintf(&buf[0], "%u\n", max_freq); + return sysfs_emit(buf, "%u\n", max_freq); } static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, @@ -749,7 +749,7 @@ static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *poli if (freq < 0) return freq; - return sprintf(&buf[0], "%u\n", freq); + return sysfs_emit(buf, "%u\n", freq); } /* @@ -764,7 +764,7 @@ static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, perf = READ_ONCE(cpudata->highest_perf); - return sprintf(&buf[0], "%u\n", perf); + return sysfs_emit(buf, "%u\n", perf); } static ssize_t show_energy_performance_available_preferences( From b9e6a2d47b2565eb450d3ee900fba49cc9b25cbd Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Tue, 31 Jan 2023 17:00:16 +0800 Subject: [PATCH 173/239] Documentation: amd-pstate: introduce new global sysfs attributes The amd-pstate driver supports switching working modes at runtime. Users can view and change modes by interacting with the "status" sysfs attribute. 1) check driver mode: $ cat /sys/devices/system/cpu/amd-pstate/status 2) switch mode: `# echo "passive" | sudo tee /sys/devices/system/cpu/amd-pstate/status` or `# echo "active" | sudo tee /sys/devices/system/cpu/amd-pstate/status` Acked-by: Huang Rui Reviewed-by: Mario Limonciello Reviewed-by: Wyes Karny Tested-by: Wyes Karny Signed-off-by: Perry Yuan Signed-off-by: Rafael J. Wysocki --- Documentation/admin-guide/pm/amd-pstate.rst | 29 +++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst index b6aee69f564f..5304adf2fc2f 100644 --- a/Documentation/admin-guide/pm/amd-pstate.rst +++ b/Documentation/admin-guide/pm/amd-pstate.rst @@ -339,6 +339,35 @@ processor must provide at least nominal performance requested and go higher if c operating conditions allow. +User Space Interface in ``sysfs`` +================================= + +Global Attributes +----------------- + +``amd-pstate`` exposes several global attributes (files) in ``sysfs`` to +control its functionality at the system level. They are located in the +``/sys/devices/system/cpu/amd-pstate/`` directory and affect all CPUs. + +``status`` + Operation mode of the driver: "active", "passive" or "disable". + + "active" + The driver is functional and in the ``active mode`` + + "passive" + The driver is functional and in the ``passive mode`` + + "disable" + The driver is unregistered and not functional now. + + This attribute can be written to in order to change the driver's + operation mode or to unregister it. The string written to it must be + one of the possible values of it and, if successful, writing one of + these values to the sysfs file will cause the driver to switch over + to the operation mode represented by that string - or to be + unregistered in the "disable" case. + ``cpupower`` tool support for ``amd-pstate`` =============================================== From e740604232dc5c3097808f3e91fd02d9316010c5 Mon Sep 17 00:00:00 2001 From: Ryan Chen Date: Mon, 30 Jan 2023 16:54:30 +0800 Subject: [PATCH 174/239] irqchip/aspeed-scu-ic: Correctly initialise status and enable registers The status and enable registers are never initialised with sensible default values. Fix those. Signed-off-by: Ryan Chen [maz: commit message] Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230130085430.635583-1-ryan_chen@aspeedtech.com --- drivers/irqchip/irq-aspeed-scu-ic.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-aspeed-scu-ic.c b/drivers/irqchip/irq-aspeed-scu-ic.c index 279e92cf0b16..94a7223e95df 100644 --- a/drivers/irqchip/irq-aspeed-scu-ic.c +++ b/drivers/irqchip/irq-aspeed-scu-ic.c @@ -17,8 +17,9 @@ #define ASPEED_SCU_IC_REG 0x018 #define ASPEED_SCU_IC_SHIFT 0 -#define ASPEED_SCU_IC_ENABLE GENMASK(6, ASPEED_SCU_IC_SHIFT) +#define ASPEED_SCU_IC_ENABLE GENMASK(15, ASPEED_SCU_IC_SHIFT) #define ASPEED_SCU_IC_NUM_IRQS 7 +#define ASPEED_SCU_IC_STATUS GENMASK(28, 16) #define ASPEED_SCU_IC_STATUS_SHIFT 16 #define ASPEED_AST2600_SCU_IC0_REG 0x560 @@ -155,6 +156,8 @@ static int aspeed_scu_ic_of_init_common(struct aspeed_scu_ic *scu_ic, rc = PTR_ERR(scu_ic->scu); goto err; } + regmap_write_bits(scu_ic->scu, scu_ic->reg, ASPEED_SCU_IC_STATUS, ASPEED_SCU_IC_STATUS); + regmap_write_bits(scu_ic->scu, scu_ic->reg, ASPEED_SCU_IC_ENABLE, 0); irq = irq_of_parse_and_map(node, 0); if (!irq) { From fc98adb9a8435cdb4e8349138ac0b728df80ade9 Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Wed, 7 Dec 2022 22:06:43 +0800 Subject: [PATCH 175/239] irqchip/loongson-liointc: Save/restore int_edge/int_pol registers during S3/S4 If int_edge/int_pol registers are configured to not be the default values, we should save/restore them during S3/S4. Signed-off-by: Yingkun Meng Signed-off-by: Huacai Chen Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20221207140643.1600743-1-chenhuacai@loongson.cn --- drivers/irqchip/irq-loongson-liointc.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/drivers/irqchip/irq-loongson-liointc.c b/drivers/irqchip/irq-loongson-liointc.c index 85b754f7f4e6..8d00a9ad5b00 100644 --- a/drivers/irqchip/irq-loongson-liointc.c +++ b/drivers/irqchip/irq-loongson-liointc.c @@ -55,6 +55,8 @@ struct liointc_priv { struct liointc_handler_data handler[LIOINTC_NUM_PARENT]; void __iomem *core_isr[LIOINTC_NUM_CORES]; u8 map_cache[LIOINTC_CHIP_IRQ]; + u32 int_pol; + u32 int_edge; bool has_lpc_irq_errata; }; @@ -138,6 +140,14 @@ static int liointc_set_type(struct irq_data *data, unsigned int type) return 0; } +static void liointc_suspend(struct irq_chip_generic *gc) +{ + struct liointc_priv *priv = gc->private; + + priv->int_pol = readl(gc->reg_base + LIOINTC_REG_INTC_POL); + priv->int_edge = readl(gc->reg_base + LIOINTC_REG_INTC_EDGE); +} + static void liointc_resume(struct irq_chip_generic *gc) { struct liointc_priv *priv = gc->private; @@ -150,6 +160,8 @@ static void liointc_resume(struct irq_chip_generic *gc) /* Restore map cache */ for (i = 0; i < LIOINTC_CHIP_IRQ; i++) writeb(priv->map_cache[i], gc->reg_base + i); + writel(priv->int_pol, gc->reg_base + LIOINTC_REG_INTC_POL); + writel(priv->int_edge, gc->reg_base + LIOINTC_REG_INTC_EDGE); /* Restore mask cache */ writel(gc->mask_cache, gc->reg_base + LIOINTC_REG_INTC_ENABLE); irq_gc_unlock_irqrestore(gc, flags); @@ -269,6 +281,7 @@ static int liointc_init(phys_addr_t addr, unsigned long size, int revision, gc->private = priv; gc->reg_base = base; gc->domain = domain; + gc->suspend = liointc_suspend; gc->resume = liointc_resume; ct = gc->chip_types; From 835a486cd9f55790dee9f6b67ce0057d49f15da5 Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Tue, 3 Jan 2023 19:42:15 +0530 Subject: [PATCH 176/239] genirq: Add mechanism to multiplex a single HW IPI All RISC-V platforms have a single HW IPI provided by the INTC local interrupt controller. The HW method to trigger INTC IPI can be through external irqchip (e.g. RISC-V AIA), through platform specific device (e.g. SiFive CLINT timer), or through firmware (e.g. SBI IPI call). To support multiple IPIs on RISC-V, add a generic IPI multiplexing mechanism which help us create multiple virtual IPIs using a single HW IPI. This generic IPI multiplexing is inspired by the Apple AIC irqchip driver and it is shared by various RISC-V irqchip drivers. Signed-off-by: Anup Patel Reviewed-by: Hector Martin Tested-by: Hector Martin Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230103141221.772261-4-apatel@ventanamicro.com --- include/linux/irq.h | 3 + kernel/irq/Kconfig | 5 ++ kernel/irq/Makefile | 1 + kernel/irq/ipi-mux.c | 207 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 216 insertions(+) create mode 100644 kernel/irq/ipi-mux.c diff --git a/include/linux/irq.h b/include/linux/irq.h index c3eb89606c2b..b1b28affb32a 100644 --- a/include/linux/irq.h +++ b/include/linux/irq.h @@ -1266,6 +1266,9 @@ int __ipi_send_mask(struct irq_desc *desc, const struct cpumask *dest); int ipi_send_single(unsigned int virq, unsigned int cpu); int ipi_send_mask(unsigned int virq, const struct cpumask *dest); +void ipi_mux_process(void); +int ipi_mux_create(unsigned int nr_ipi, void (*mux_send)(unsigned int cpu)); + #ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER /* * Registers a generic IRQ handling function as the top-level IRQ handler in diff --git a/kernel/irq/Kconfig b/kernel/irq/Kconfig index b64c44ae4c25..2531f3496ab6 100644 --- a/kernel/irq/Kconfig +++ b/kernel/irq/Kconfig @@ -86,6 +86,11 @@ config GENERIC_IRQ_IPI depends on SMP select IRQ_DOMAIN_HIERARCHY +# Generic IRQ IPI Mux support +config GENERIC_IRQ_IPI_MUX + bool + depends on SMP + # Generic MSI hierarchical interrupt domain support config GENERIC_MSI_IRQ bool diff --git a/kernel/irq/Makefile b/kernel/irq/Makefile index b4f53717d143..f19d3080bf11 100644 --- a/kernel/irq/Makefile +++ b/kernel/irq/Makefile @@ -15,6 +15,7 @@ obj-$(CONFIG_GENERIC_IRQ_MIGRATION) += cpuhotplug.o obj-$(CONFIG_PM_SLEEP) += pm.o obj-$(CONFIG_GENERIC_MSI_IRQ) += msi.o obj-$(CONFIG_GENERIC_IRQ_IPI) += ipi.o +obj-$(CONFIG_GENERIC_IRQ_IPI_MUX) += ipi-mux.o obj-$(CONFIG_SMP) += affinity.o obj-$(CONFIG_GENERIC_IRQ_DEBUGFS) += debugfs.o obj-$(CONFIG_GENERIC_IRQ_MATRIX_ALLOCATOR) += matrix.o diff --git a/kernel/irq/ipi-mux.c b/kernel/irq/ipi-mux.c new file mode 100644 index 000000000000..3a403c3a785d --- /dev/null +++ b/kernel/irq/ipi-mux.c @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Multiplex several virtual IPIs over a single HW IPI. + * + * Copyright The Asahi Linux Contributors + * Copyright (c) 2022 Ventana Micro Systems Inc. + */ + +#define pr_fmt(fmt) "ipi-mux: " fmt +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct ipi_mux_cpu { + atomic_t enable; + atomic_t bits; +}; + +static struct ipi_mux_cpu __percpu *ipi_mux_pcpu; +static struct irq_domain *ipi_mux_domain; +static void (*ipi_mux_send)(unsigned int cpu); + +static void ipi_mux_mask(struct irq_data *d) +{ + struct ipi_mux_cpu *icpu = this_cpu_ptr(ipi_mux_pcpu); + + atomic_andnot(BIT(irqd_to_hwirq(d)), &icpu->enable); +} + +static void ipi_mux_unmask(struct irq_data *d) +{ + struct ipi_mux_cpu *icpu = this_cpu_ptr(ipi_mux_pcpu); + u32 ibit = BIT(irqd_to_hwirq(d)); + + atomic_or(ibit, &icpu->enable); + + /* + * The atomic_or() above must complete before the atomic_read() + * below to avoid racing ipi_mux_send_mask(). + */ + smp_mb__after_atomic(); + + /* If a pending IPI was unmasked, raise a parent IPI immediately. */ + if (atomic_read(&icpu->bits) & ibit) + ipi_mux_send(smp_processor_id()); +} + +static void ipi_mux_send_mask(struct irq_data *d, const struct cpumask *mask) +{ + struct ipi_mux_cpu *icpu = this_cpu_ptr(ipi_mux_pcpu); + u32 ibit = BIT(irqd_to_hwirq(d)); + unsigned long pending; + int cpu; + + for_each_cpu(cpu, mask) { + icpu = per_cpu_ptr(ipi_mux_pcpu, cpu); + + /* + * This sequence is the mirror of the one in ipi_mux_unmask(); + * see the comment there. Additionally, release semantics + * ensure that the vIPI flag set is ordered after any shared + * memory accesses that precede it. This therefore also pairs + * with the atomic_fetch_andnot in ipi_mux_process(). + */ + pending = atomic_fetch_or_release(ibit, &icpu->bits); + + /* + * The atomic_fetch_or_release() above must complete + * before the atomic_read() below to avoid racing with + * ipi_mux_unmask(). + */ + smp_mb__after_atomic(); + + /* + * The flag writes must complete before the physical IPI is + * issued to another CPU. This is implied by the control + * dependency on the result of atomic_read() below, which is + * itself already ordered after the vIPI flag write. + */ + if (!(pending & ibit) && (atomic_read(&icpu->enable) & ibit)) + ipi_mux_send(cpu); + } +} + +static const struct irq_chip ipi_mux_chip = { + .name = "IPI Mux", + .irq_mask = ipi_mux_mask, + .irq_unmask = ipi_mux_unmask, + .ipi_send_mask = ipi_mux_send_mask, +}; + +static int ipi_mux_domain_alloc(struct irq_domain *d, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + int i; + + for (i = 0; i < nr_irqs; i++) { + irq_set_percpu_devid(virq + i); + irq_domain_set_info(d, virq + i, i, &ipi_mux_chip, NULL, + handle_percpu_devid_irq, NULL, NULL); + } + + return 0; +} + +static const struct irq_domain_ops ipi_mux_domain_ops = { + .alloc = ipi_mux_domain_alloc, + .free = irq_domain_free_irqs_top, +}; + +/** + * ipi_mux_process - Process multiplexed virtual IPIs + */ +void ipi_mux_process(void) +{ + struct ipi_mux_cpu *icpu = this_cpu_ptr(ipi_mux_pcpu); + irq_hw_number_t hwirq; + unsigned long ipis; + unsigned int en; + + /* + * Reading enable mask does not need to be ordered as long as + * this function is called from interrupt handler because only + * the CPU itself can change it's own enable mask. + */ + en = atomic_read(&icpu->enable); + + /* + * Clear the IPIs we are about to handle. This pairs with the + * atomic_fetch_or_release() in ipi_mux_send_mask(). + */ + ipis = atomic_fetch_andnot(en, &icpu->bits) & en; + + for_each_set_bit(hwirq, &ipis, BITS_PER_TYPE(int)) + generic_handle_domain_irq(ipi_mux_domain, hwirq); +} + +/** + * ipi_mux_create - Create virtual IPIs multiplexed on top of a single + * parent IPI. + * @nr_ipi: number of virtual IPIs to create. This should + * be <= BITS_PER_TYPE(int) + * @mux_send: callback to trigger parent IPI for a particular CPU + * + * Returns first virq of the newly created virtual IPIs upon success + * or <=0 upon failure + */ +int ipi_mux_create(unsigned int nr_ipi, void (*mux_send)(unsigned int cpu)) +{ + struct fwnode_handle *fwnode; + struct irq_domain *domain; + int rc; + + if (ipi_mux_domain) + return -EEXIST; + + if (BITS_PER_TYPE(int) < nr_ipi || !mux_send) + return -EINVAL; + + ipi_mux_pcpu = alloc_percpu(typeof(*ipi_mux_pcpu)); + if (!ipi_mux_pcpu) + return -ENOMEM; + + fwnode = irq_domain_alloc_named_fwnode("IPI-Mux"); + if (!fwnode) { + pr_err("unable to create IPI Mux fwnode\n"); + rc = -ENOMEM; + goto fail_free_cpu; + } + + domain = irq_domain_create_linear(fwnode, nr_ipi, + &ipi_mux_domain_ops, NULL); + if (!domain) { + pr_err("unable to add IPI Mux domain\n"); + rc = -ENOMEM; + goto fail_free_fwnode; + } + + domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE; + irq_domain_update_bus_token(domain, DOMAIN_BUS_IPI); + + rc = __irq_domain_alloc_irqs(domain, -1, nr_ipi, + NUMA_NO_NODE, NULL, false, NULL); + if (rc <= 0) { + pr_err("unable to alloc IRQs from IPI Mux domain\n"); + goto fail_free_domain; + } + + ipi_mux_domain = domain; + ipi_mux_send = mux_send; + + return rc; + +fail_free_domain: + irq_domain_remove(domain); +fail_free_fwnode: + irq_domain_free_fwnode(fwnode); +fail_free_cpu: + free_percpu(ipi_mux_pcpu); + return rc; +} From c19f897194288ec286bb52001b9ee9551876a614 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 3 Jan 2023 19:42:21 +0530 Subject: [PATCH 177/239] irqchip/apple-aic: Move over to core ipi-mux Now that the complexity of the AIC IPI mux has been copied into the core code for the benefit of the riscv architecture, shrink the AIC driver by the same amount by using that infrastructure. Signed-off-by: Marc Zyngier Signed-off-by: Anup Patel Acked-by: Hector Martin Link: https://lore.kernel.org/r/20230103141221.772261-10-apatel@ventanamicro.com --- drivers/irqchip/Kconfig | 1 + drivers/irqchip/irq-apple-aic.c | 161 ++------------------------------ 2 files changed, 9 insertions(+), 153 deletions(-) diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index caa952c40ff9..cbf504794e7c 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -658,6 +658,7 @@ config APPLE_AIC bool "Apple Interrupt Controller (AIC)" depends on ARM64 depends on ARCH_APPLE || COMPILE_TEST + select GENERIC_IRQ_IPI_MUX help Support for the Apple Interrupt Controller found on Apple Silicon SoCs, such as the M1. diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index ae3437f03e6c..eabb3b92965b 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -292,7 +292,6 @@ struct aic_irq_chip { void __iomem *base; void __iomem *event; struct irq_domain *hw_domain; - struct irq_domain *ipi_domain; struct { cpumask_t aff; } *fiq_aff[AIC_NR_FIQ]; @@ -307,9 +306,6 @@ struct aic_irq_chip { static DEFINE_PER_CPU(uint32_t, aic_fiq_unmasked); -static DEFINE_PER_CPU(atomic_t, aic_vipi_flag); -static DEFINE_PER_CPU(atomic_t, aic_vipi_enable); - static struct aic_irq_chip *aic_irqc; static void aic_handle_ipi(struct pt_regs *regs); @@ -751,98 +747,8 @@ static void aic_ipi_send_fast(int cpu) isb(); } -static void aic_ipi_mask(struct irq_data *d) -{ - u32 irq_bit = BIT(irqd_to_hwirq(d)); - - /* No specific ordering requirements needed here. */ - atomic_andnot(irq_bit, this_cpu_ptr(&aic_vipi_enable)); -} - -static void aic_ipi_unmask(struct irq_data *d) -{ - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - u32 irq_bit = BIT(irqd_to_hwirq(d)); - - atomic_or(irq_bit, this_cpu_ptr(&aic_vipi_enable)); - - /* - * The atomic_or() above must complete before the atomic_read() - * below to avoid racing aic_ipi_send_mask(). - */ - smp_mb__after_atomic(); - - /* - * If a pending vIPI was unmasked, raise a HW IPI to ourselves. - * No barriers needed here since this is a self-IPI. - */ - if (atomic_read(this_cpu_ptr(&aic_vipi_flag)) & irq_bit) { - if (static_branch_likely(&use_fast_ipi)) - aic_ipi_send_fast(smp_processor_id()); - else - aic_ic_write(ic, AIC_IPI_SEND, AIC_IPI_SEND_CPU(smp_processor_id())); - } -} - -static void aic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) -{ - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - u32 irq_bit = BIT(irqd_to_hwirq(d)); - u32 send = 0; - int cpu; - unsigned long pending; - - for_each_cpu(cpu, mask) { - /* - * This sequence is the mirror of the one in aic_ipi_unmask(); - * see the comment there. Additionally, release semantics - * ensure that the vIPI flag set is ordered after any shared - * memory accesses that precede it. This therefore also pairs - * with the atomic_fetch_andnot in aic_handle_ipi(). - */ - pending = atomic_fetch_or_release(irq_bit, per_cpu_ptr(&aic_vipi_flag, cpu)); - - /* - * The atomic_fetch_or_release() above must complete before the - * atomic_read() below to avoid racing aic_ipi_unmask(). - */ - smp_mb__after_atomic(); - - if (!(pending & irq_bit) && - (atomic_read(per_cpu_ptr(&aic_vipi_enable, cpu)) & irq_bit)) { - if (static_branch_likely(&use_fast_ipi)) - aic_ipi_send_fast(cpu); - else - send |= AIC_IPI_SEND_CPU(cpu); - } - } - - /* - * The flag writes must complete before the physical IPI is issued - * to another CPU. This is implied by the control dependency on - * the result of atomic_read_acquire() above, which is itself - * already ordered after the vIPI flag write. - */ - if (send) - aic_ic_write(ic, AIC_IPI_SEND, send); -} - -static struct irq_chip ipi_chip = { - .name = "AIC-IPI", - .irq_mask = aic_ipi_mask, - .irq_unmask = aic_ipi_unmask, - .ipi_send_mask = aic_ipi_send_mask, -}; - -/* - * IPI IRQ domain - */ - static void aic_handle_ipi(struct pt_regs *regs) { - int i; - unsigned long enabled, firing; - /* * Ack the IPI. We need to order this after the AIC event read, but * that is enforced by normal MMIO ordering guarantees. @@ -857,27 +763,7 @@ static void aic_handle_ipi(struct pt_regs *regs) aic_ic_write(aic_irqc, AIC_IPI_ACK, AIC_IPI_OTHER); } - /* - * The mask read does not need to be ordered. Only we can change - * our own mask anyway, so no races are possible here, as long as - * we are properly in the interrupt handler (which is covered by - * the barrier that is part of the top-level AIC handler's readl()). - */ - enabled = atomic_read(this_cpu_ptr(&aic_vipi_enable)); - - /* - * Clear the IPIs we are about to handle. This pairs with the - * atomic_fetch_or_release() in aic_ipi_send_mask(), and needs to be - * ordered after the aic_ic_write() above (to avoid dropping vIPIs) and - * before IPI handling code (to avoid races handling vIPIs before they - * are signaled). The former is taken care of by the release semantics - * of the write portion, while the latter is taken care of by the - * acquire semantics of the read portion. - */ - firing = atomic_fetch_andnot(enabled, this_cpu_ptr(&aic_vipi_flag)) & enabled; - - for_each_set_bit(i, &firing, AIC_NR_SWIPI) - generic_handle_domain_irq(aic_irqc->ipi_domain, i); + ipi_mux_process(); /* * No ordering needed here; at worst this just changes the timing of @@ -887,55 +773,24 @@ static void aic_handle_ipi(struct pt_regs *regs) aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); } -static int aic_ipi_alloc(struct irq_domain *d, unsigned int virq, - unsigned int nr_irqs, void *args) +static void aic_ipi_send_single(unsigned int cpu) { - int i; - - for (i = 0; i < nr_irqs; i++) { - irq_set_percpu_devid(virq + i); - irq_domain_set_info(d, virq + i, i, &ipi_chip, d->host_data, - handle_percpu_devid_irq, NULL, NULL); - } - - return 0; + if (static_branch_likely(&use_fast_ipi)) + aic_ipi_send_fast(cpu); + else + aic_ic_write(aic_irqc, AIC_IPI_SEND, AIC_IPI_SEND_CPU(cpu)); } -static void aic_ipi_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs) -{ - /* Not freeing IPIs */ -} - -static const struct irq_domain_ops aic_ipi_domain_ops = { - .alloc = aic_ipi_alloc, - .free = aic_ipi_free, -}; - static int __init aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) { - struct irq_domain *ipi_domain; int base_ipi; - ipi_domain = irq_domain_create_linear(irqc->hw_domain->fwnode, AIC_NR_SWIPI, - &aic_ipi_domain_ops, irqc); - if (WARN_ON(!ipi_domain)) + base_ipi = ipi_mux_create(AIC_NR_SWIPI, aic_ipi_send_single); + if (WARN_ON(base_ipi <= 0)) return -ENODEV; - ipi_domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE; - irq_domain_update_bus_token(ipi_domain, DOMAIN_BUS_IPI); - - base_ipi = __irq_domain_alloc_irqs(ipi_domain, -1, AIC_NR_SWIPI, - NUMA_NO_NODE, NULL, false, NULL); - - if (WARN_ON(!base_ipi)) { - irq_domain_remove(ipi_domain); - return -ENODEV; - } - set_smp_ipi_range(base_ipi, AIC_NR_SWIPI); - irqc->ipi_domain = ipi_domain; - return 0; } From 6caa5a2b78f5f53c433d3a3781e53325da22f0ac Mon Sep 17 00:00:00 2001 From: Miaoqian Lin Date: Mon, 2 Jan 2023 16:13:18 +0400 Subject: [PATCH 178/239] irqchip: Fix refcount leak in platform_irqchip_probe of_irq_find_parent() returns a node pointer with refcount incremented, We should use of_node_put() on it when not needed anymore. Add missing of_node_put() to avoid refcount leak. Fixes: f8410e626569 ("irqchip: Add IRQCHIP_PLATFORM_DRIVER_BEGIN/END and IRQCHIP_MATCH helper macros") Signed-off-by: Miaoqian Lin Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230102121318.3990586-1-linmq006@gmail.com --- drivers/irqchip/irqchip.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/irqchip/irqchip.c b/drivers/irqchip/irqchip.c index 3570f0a588c4..7899607fbee8 100644 --- a/drivers/irqchip/irqchip.c +++ b/drivers/irqchip/irqchip.c @@ -38,8 +38,10 @@ int platform_irqchip_probe(struct platform_device *pdev) struct device_node *par_np = of_irq_find_parent(np); of_irq_init_cb_t irq_init_cb = of_device_get_match_data(&pdev->dev); - if (!irq_init_cb) + if (!irq_init_cb) { + of_node_put(par_np); return -EINVAL; + } if (par_np == np) par_np = NULL; @@ -52,8 +54,10 @@ int platform_irqchip_probe(struct platform_device *pdev) * interrupt controller. The actual initialization callback of this * interrupt controller can check for specific domains as necessary. */ - if (par_np && !irq_find_matching_host(par_np, DOMAIN_BUS_ANY)) + if (par_np && !irq_find_matching_host(par_np, DOMAIN_BUS_ANY)) { + of_node_put(par_np); return -EPROBE_DEFER; + } return irq_init_cb(np, par_np); } From 071d068b89e95d1b078aa6bbcb9d0961b77d6aa1 Mon Sep 17 00:00:00 2001 From: Miaoqian Lin Date: Mon, 2 Jan 2023 12:28:10 +0400 Subject: [PATCH 179/239] irqchip/alpine-msi: Fix refcount leak in alpine_msix_init_domains of_irq_find_parent() returns a node pointer with refcount incremented, We should use of_node_put() on it when not needed anymore. Add missing of_node_put() to avoid refcount leak. Fixes: e6b78f2c3e14 ("irqchip: Add the Alpine MSIX interrupt controller") Signed-off-by: Miaoqian Lin Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230102082811.3947760-1-linmq006@gmail.com --- drivers/irqchip/irq-alpine-msi.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/irqchip/irq-alpine-msi.c b/drivers/irqchip/irq-alpine-msi.c index 5ddb8e578ac6..fc1ef7de3797 100644 --- a/drivers/irqchip/irq-alpine-msi.c +++ b/drivers/irqchip/irq-alpine-msi.c @@ -199,6 +199,7 @@ static int alpine_msix_init_domains(struct alpine_msix_data *priv, } gic_domain = irq_find_host(gic_node); + of_node_put(gic_node); if (!gic_domain) { pr_err("Failed to find the GIC domain\n"); return -ENXIO; From 9419e700021a393f67be36abd0c4f3acc6139041 Mon Sep 17 00:00:00 2001 From: Miaoqian Lin Date: Mon, 2 Jan 2023 12:42:08 +0400 Subject: [PATCH 180/239] irqchip/irq-mvebu-gicp: Fix refcount leak in mvebu_gicp_probe of_irq_find_parent() returns a node pointer with refcount incremented, We should use of_node_put() on it when not needed anymore. Add missing of_node_put() to avoid refcount leak. Fixes: a68a63cb4dfc ("irqchip/irq-mvebu-gicp: Add new driver for Marvell GICP") Signed-off-by: Miaoqian Lin Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230102084208.3951758-1-linmq006@gmail.com --- drivers/irqchip/irq-mvebu-gicp.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/irqchip/irq-mvebu-gicp.c b/drivers/irqchip/irq-mvebu-gicp.c index fe88a782173d..c43a345061d5 100644 --- a/drivers/irqchip/irq-mvebu-gicp.c +++ b/drivers/irqchip/irq-mvebu-gicp.c @@ -221,6 +221,7 @@ static int mvebu_gicp_probe(struct platform_device *pdev) } parent_domain = irq_find_host(irq_parent_dn); + of_node_put(irq_parent_dn); if (!parent_domain) { dev_err(&pdev->dev, "failed to find parent IRQ domain\n"); return -ENODEV; From 02298b7bae12936ca313975b02e7f98b06670d37 Mon Sep 17 00:00:00 2001 From: Miaoqian Lin Date: Mon, 2 Jan 2023 12:56:10 +0400 Subject: [PATCH 181/239] irqchip/ti-sci: Fix refcount leak in ti_sci_intr_irq_domain_probe of_irq_find_parent() returns a node pointer with refcount incremented, We should use of_node_put() on it when not needed anymore. Add missing of_node_put() to avoid refcount leak. Fixes: cd844b0715ce ("irqchip/ti-sci-intr: Add support for Interrupt Router driver") Signed-off-by: Miaoqian Lin Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230102085611.3955984-1-linmq006@gmail.com --- drivers/irqchip/irq-ti-sci-intr.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/irqchip/irq-ti-sci-intr.c b/drivers/irqchip/irq-ti-sci-intr.c index fe8fad22bcf9..020ddf29efb8 100644 --- a/drivers/irqchip/irq-ti-sci-intr.c +++ b/drivers/irqchip/irq-ti-sci-intr.c @@ -236,6 +236,7 @@ static int ti_sci_intr_irq_domain_probe(struct platform_device *pdev) } parent_domain = irq_find_host(parent_node); + of_node_put(parent_node); if (!parent_domain) { dev_err(dev, "Failed to find IRQ parent domain\n"); return -ENODEV; From 9c1a7bfc2993112cfb3056b18301fcafe5c2fde5 Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Wed, 11 Jan 2023 11:05:54 +0100 Subject: [PATCH 182/239] irqchip/ls-scfg-msi: Simplify Kconfig dependencies Having both PCI_MSI and PCI is redundant. Drop PCI. Signed-off-by: Lukas Bulwahn [maz: cut commit message extra verbosity] Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230111100554.24500-1-lukas.bulwahn@gmail.com --- drivers/irqchip/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index caa952c40ff9..f3d88e9c87a3 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -389,7 +389,7 @@ config LS_EXTIRQ config LS_SCFG_MSI def_bool y if SOC_LS1021A || ARCH_LAYERSCAPE - depends on PCI && PCI_MSI + depends on PCI_MSI config PARTITION_PERCPU bool From 7214015c7f975542d227db6eaec2db2ecc88f2c1 Mon Sep 17 00:00:00 2001 From: Yi-Wei Wang Date: Tue, 24 Jan 2023 11:53:23 +0000 Subject: [PATCH 183/239] cpufreq: tegra194: Enable CPUFREQ thermal cooling Populate the flag CPUFREQ_IS_COOLING_DEV for the Tegra194 CPUFREQ driver to register it as a cooling device. This enables CPU frequency throttling for CPUs when the passive trip points are crossed. Signed-off-by: Yi-Wei Wang Signed-off-by: Jon Hunter Signed-off-by: Viresh Kumar --- drivers/cpufreq/tegra194-cpufreq.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/tegra194-cpufreq.c b/drivers/cpufreq/tegra194-cpufreq.c index 4596c3e323aa..5890e25d7f77 100644 --- a/drivers/cpufreq/tegra194-cpufreq.c +++ b/drivers/cpufreq/tegra194-cpufreq.c @@ -411,7 +411,8 @@ static int tegra194_cpufreq_set_target(struct cpufreq_policy *policy, static struct cpufreq_driver tegra194_cpufreq_driver = { .name = "tegra194", - .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_INITIAL_FREQ_CHECK, + .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_INITIAL_FREQ_CHECK | + CPUFREQ_IS_COOLING_DEV, .verify = cpufreq_generic_frequency_table_verify, .target_index = tegra194_cpufreq_set_target, .get = tegra194_get_speed, From 09608d62ae5cd9559549637cccb4092d8ecad3a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?N=C3=ADcolas=20F=2E=20R=2E=20A=2E=20Prado?= Date: Thu, 26 Jan 2023 10:48:56 -0500 Subject: [PATCH 184/239] cpufreq: mediatek-hw: Register to module device table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Register the compatibles for this module on the module device table so it can be automatically loaded when a matching device is found on the system. Signed-off-by: Nícolas F. R. A. Prado Signed-off-by: Viresh Kumar --- drivers/cpufreq/mediatek-cpufreq-hw.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/cpufreq/mediatek-cpufreq-hw.c b/drivers/cpufreq/mediatek-cpufreq-hw.c index f80339779084..115b0eda38c1 100644 --- a/drivers/cpufreq/mediatek-cpufreq-hw.c +++ b/drivers/cpufreq/mediatek-cpufreq-hw.c @@ -324,6 +324,7 @@ static const struct of_device_id mtk_cpufreq_hw_match[] = { { .compatible = "mediatek,cpufreq-hw", .data = &cpufreq_mtk_offsets }, {} }; +MODULE_DEVICE_TABLE(of, mtk_cpufreq_hw_match); static struct platform_driver mtk_cpufreq_hw_driver = { .probe = mtk_cpufreq_hw_driver_probe, From fa68d9c5ff765318adf8e46ff1d89539a8546a0d Mon Sep 17 00:00:00 2001 From: Luca Weiss Date: Sun, 16 Oct 2022 11:00:30 +0200 Subject: [PATCH 185/239] dt-bindings: cpufreq: cpufreq-qcom-hw: Add missing compatibles Document the cpufreq-epss compatibles currently used in the tree, plus the sc7280 which will be added in a separate commit. Signed-off-by: Luca Weiss Signed-off-by: Viresh Kumar --- .../devicetree/bindings/cpufreq/cpufreq-qcom-hw.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.yaml b/Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.yaml index 903b31129f01..b69b71d497cc 100644 --- a/Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.yaml +++ b/Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.yaml @@ -26,8 +26,12 @@ properties: items: - enum: - qcom,qdu1000-cpufreq-epss + - qcom,sc7280-cpufreq-epss + - qcom,sc8280xp-cpufreq-epss - qcom,sm6375-cpufreq-epss - qcom,sm8250-cpufreq-epss + - qcom,sm8350-cpufreq-epss + - qcom,sm8450-cpufreq-epss - const: qcom,cpufreq-epss reg: From 8e6cb91f946a059c3714c6d7f0e43e313f389183 Mon Sep 17 00:00:00 2001 From: Abel Vesa Date: Mon, 30 Jan 2023 14:30:46 +0200 Subject: [PATCH 186/239] dt-bindings: cpufreq: cpufreq-qcom-hw: Add SM8550 compatible Add compatible for EPSS CPUFREQ-HW on SM8550. Also document the interrupts. Signed-off-by: Abel Vesa Acked-by: Rob Herring Signed-off-by: Viresh Kumar --- Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.yaml b/Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.yaml index b69b71d497cc..4b166fddb958 100644 --- a/Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.yaml +++ b/Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.yaml @@ -32,6 +32,7 @@ properties: - qcom,sm8250-cpufreq-epss - qcom,sm8350-cpufreq-epss - qcom,sm8450-cpufreq-epss + - qcom,sm8550-cpufreq-epss - const: qcom,cpufreq-epss reg: From 3d812a0f27baa2d094f2c18298d48b012878dc0b Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 6 Feb 2023 17:21:15 +0000 Subject: [PATCH 187/239] genirq/ipi-mux: Use irq_domain_alloc_irqs() Using __irq_domain_alloc_irqs() is an unnecessary complexity. Use irq_domain_alloc_irqs(), which is simpler and makes the code more readable. Reported-by: Stephen Rothwell Signed-off-by: Marc Zyngier --- kernel/irq/ipi-mux.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kernel/irq/ipi-mux.c b/kernel/irq/ipi-mux.c index 3a403c3a785d..fa4fc18c6131 100644 --- a/kernel/irq/ipi-mux.c +++ b/kernel/irq/ipi-mux.c @@ -185,8 +185,7 @@ int ipi_mux_create(unsigned int nr_ipi, void (*mux_send)(unsigned int cpu)) domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE; irq_domain_update_bus_token(domain, DOMAIN_BUS_IPI); - rc = __irq_domain_alloc_irqs(domain, -1, nr_ipi, - NUMA_NO_NODE, NULL, false, NULL); + rc = irq_domain_alloc_irqs(domain, nr_ipi, NUMA_NO_NODE, NULL); if (rc <= 0) { pr_err("unable to alloc IRQs from IPI Mux domain\n"); goto fail_free_domain; From 2d11eae42d52a131f06061015e49dc0f085c5bfc Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Thu, 2 Feb 2023 13:44:49 +0100 Subject: [PATCH 188/239] ACPI: video: Fix Lenovo Ideapad Z570 DMI match Multiple Ideapad Z570 variants need acpi_backlight=native to force native use on these pre Windows 8 machines since acpi_video backlight control does not work here. The original DMI quirk matches on a product_name of "102434U" but other variants may have different product_name-s such as e.g. "1024D9U". Move to checking product_version instead as is more or less standard for Lenovo DMI quirks for similar reasons. Signed-off-by: Hans de Goede Signed-off-by: Rafael J. Wysocki --- drivers/acpi/video_detect.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/acpi/video_detect.c b/drivers/acpi/video_detect.c index a8c02608dde4..710ac640267d 100644 --- a/drivers/acpi/video_detect.c +++ b/drivers/acpi/video_detect.c @@ -434,7 +434,7 @@ static const struct dmi_system_id video_detect_dmi_table[] = { /* Lenovo Ideapad Z570 */ .matches = { DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), - DMI_MATCH(DMI_PRODUCT_NAME, "102434U"), + DMI_MATCH(DMI_PRODUCT_VERSION, "Ideapad Z570"), }, }, { From eca4c0eea53432ec4b711b2a8ad282cbad231b4f Mon Sep 17 00:00:00 2001 From: Qi Zheng Date: Wed, 8 Feb 2023 12:00:37 +0800 Subject: [PATCH 189/239] OPP: fix error checking in opp_migrate_dentry() Since commit ff9fb72bc077 ("debugfs: return error values, not NULL") changed return value of debugfs_rename() in error cases from %NULL to %ERR_PTR(-ERROR), we should also check error values instead of NULL. Fixes: ff9fb72bc077 ("debugfs: return error values, not NULL") Signed-off-by: Qi Zheng Signed-off-by: Viresh Kumar --- drivers/opp/debugfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/opp/debugfs.c b/drivers/opp/debugfs.c index 96a30a032c5f..2c7fb683441e 100644 --- a/drivers/opp/debugfs.c +++ b/drivers/opp/debugfs.c @@ -235,7 +235,7 @@ static void opp_migrate_dentry(struct opp_device *opp_dev, dentry = debugfs_rename(rootdir, opp_dev->dentry, rootdir, opp_table->dentry_name); - if (!dentry) { + if (IS_ERR(dentry)) { dev_err(dev, "%s: Failed to rename link from: %s to %s\n", __func__, dev_name(opp_dev->dev), dev_name(dev)); return; From 48df133578c70185a95a49390d42df1996ddba2a Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Fri, 27 Jan 2023 14:39:21 -0800 Subject: [PATCH 190/239] coda: Avoid partial allocation of sig_inputArgs GCC does not like having a partially allocated object, since it cannot reason about it for bounds checking when it is passed to other code. Instead, fully allocate sig_inputArgs. (Alternatively, sig_inputArgs should be defined as a struct coda_in_hdr, if it is actually not using any other part of the union.) Seen under GCC 13: ../fs/coda/upcall.c: In function 'coda_upcall': ../fs/coda/upcall.c:801:22: warning: array subscript 'union inputArgs[0]' is partly outside array bounds of 'unsigned char[20]' [-Warray-bounds=] 801 | sig_inputArgs->ih.opcode = CODA_SIGNAL; | ^~ Cc: Jan Harkes Cc: coda@cs.cmu.edu Cc: codalist@coda.cs.cmu.edu Signed-off-by: Kees Cook Link: https://lore.kernel.org/r/20230127223921.never.882-kees@kernel.org --- fs/coda/upcall.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/coda/upcall.c b/fs/coda/upcall.c index 59f6cfd06f96..cd6a3721f6f6 100644 --- a/fs/coda/upcall.c +++ b/fs/coda/upcall.c @@ -791,7 +791,7 @@ static int coda_upcall(struct venus_comm *vcp, sig_req = kmalloc(sizeof(struct upc_req), GFP_KERNEL); if (!sig_req) goto exit; - sig_inputArgs = kvzalloc(sizeof(struct coda_in_hdr), GFP_KERNEL); + sig_inputArgs = kvzalloc(sizeof(*sig_inputArgs), GFP_KERNEL); if (!sig_inputArgs) { kfree(sig_req); goto exit; From 25b84002afb9dc9a91a7ea67166879c13ad82422 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 2 Feb 2023 22:07:49 +0000 Subject: [PATCH 191/239] arm64: Support Clang UBSAN trap codes for better reporting When building with CONFIG_UBSAN_TRAP=y on arm64, Clang encodes the UBSAN check (handler) type in the esr. Extract this and actually report these traps as coming from the specific UBSAN check that tripped. Before: Internal error: BRK handler: 00000000f20003e8 [#1] PREEMPT SMP After: Internal error: UBSAN: shift out of bounds: 00000000f2005514 [#1] PREEMPT SMP Acked-by: Mark Rutland Reviewed-by: Ard Biesheuvel Acked-by: Mukesh Ojha Reviewed-by: Fangrui Song Cc: Catalin Marinas Cc: Will Deacon Cc: John Stultz Cc: Yongqin Liu Cc: Sami Tolvanen Cc: Yury Norov Cc: Andrey Konovalov Cc: Marco Elver Cc: linux-arm-kernel@lists.infradead.org Cc: llvm@lists.linux.dev Signed-off-by: Kees Cook --- arch/arm64/include/asm/brk-imm.h | 3 ++ arch/arm64/kernel/traps.c | 21 ++++++++++ include/linux/ubsan.h | 9 +++++ lib/Makefile | 2 - lib/ubsan.c | 68 ++++++++++++++++++++++++++++++++ lib/ubsan.h | 32 +++++++++++++++ 6 files changed, 133 insertions(+), 2 deletions(-) create mode 100644 include/linux/ubsan.h diff --git a/arch/arm64/include/asm/brk-imm.h b/arch/arm64/include/asm/brk-imm.h index 6e000113e508..1abdcd508a11 100644 --- a/arch/arm64/include/asm/brk-imm.h +++ b/arch/arm64/include/asm/brk-imm.h @@ -17,6 +17,7 @@ * 0x401: for compile time BRK instruction * 0x800: kernel-mode BUG() and WARN() traps * 0x9xx: tag-based KASAN trap (allowed values 0x900 - 0x9ff) + * 0x55xx: Undefined Behavior Sanitizer traps ('U' << 8) * 0x8xxx: Control-Flow Integrity traps */ #define KPROBES_BRK_IMM 0x004 @@ -28,6 +29,8 @@ #define BUG_BRK_IMM 0x800 #define KASAN_BRK_IMM 0x900 #define KASAN_BRK_MASK 0x0ff +#define UBSAN_BRK_IMM 0x5500 +#define UBSAN_BRK_MASK 0x00ff #define CFI_BRK_IMM_TARGET GENMASK(4, 0) #define CFI_BRK_IMM_TYPE GENMASK(9, 5) diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index 4c0caa589e12..87f42eb1c950 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -1074,6 +1075,19 @@ static struct break_hook kasan_break_hook = { }; #endif +#ifdef CONFIG_UBSAN_TRAP +static int ubsan_handler(struct pt_regs *regs, unsigned long esr) +{ + die(report_ubsan_failure(regs, esr & UBSAN_BRK_MASK), regs, esr); + return DBG_HOOK_HANDLED; +} + +static struct break_hook ubsan_break_hook = { + .fn = ubsan_handler, + .imm = UBSAN_BRK_IMM, + .mask = UBSAN_BRK_MASK, +}; +#endif #define esr_comment(esr) ((esr) & ESR_ELx_BRK64_ISS_COMMENT_MASK) @@ -1091,6 +1105,10 @@ int __init early_brk64(unsigned long addr, unsigned long esr, #ifdef CONFIG_KASAN_SW_TAGS if ((esr_comment(esr) & ~KASAN_BRK_MASK) == KASAN_BRK_IMM) return kasan_handler(regs, esr) != DBG_HOOK_HANDLED; +#endif +#ifdef CONFIG_UBSAN_TRAP + if ((esr_comment(esr) & ~UBSAN_BRK_MASK) == UBSAN_BRK_IMM) + return ubsan_handler(regs, esr) != DBG_HOOK_HANDLED; #endif return bug_handler(regs, esr) != DBG_HOOK_HANDLED; } @@ -1104,6 +1122,9 @@ void __init trap_init(void) register_kernel_break_hook(&fault_break_hook); #ifdef CONFIG_KASAN_SW_TAGS register_kernel_break_hook(&kasan_break_hook); +#endif +#ifdef CONFIG_UBSAN_TRAP + register_kernel_break_hook(&ubsan_break_hook); #endif debug_traps_init(); } diff --git a/include/linux/ubsan.h b/include/linux/ubsan.h new file mode 100644 index 000000000000..bff7445498de --- /dev/null +++ b/include/linux/ubsan.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_UBSAN_H +#define _LINUX_UBSAN_H + +#ifdef CONFIG_UBSAN_TRAP +const char *report_ubsan_failure(struct pt_regs *regs, u32 check_type); +#endif + +#endif diff --git a/lib/Makefile b/lib/Makefile index 4d9461bfea42..81b988bf9448 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -340,9 +340,7 @@ quiet_cmd_build_OID_registry = GEN $@ clean-files += oid_registry_data.c obj-$(CONFIG_UCS2_STRING) += ucs2_string.o -ifneq ($(CONFIG_UBSAN_TRAP),y) obj-$(CONFIG_UBSAN) += ubsan.o -endif UBSAN_SANITIZE_ubsan.o := n KASAN_SANITIZE_ubsan.o := n diff --git a/lib/ubsan.c b/lib/ubsan.c index 60c7099857a0..6620e0719c9d 100644 --- a/lib/ubsan.c +++ b/lib/ubsan.c @@ -14,10 +14,76 @@ #include #include #include +#include #include #include "ubsan.h" +#ifdef CONFIG_UBSAN_TRAP +/* + * Only include matches for UBSAN checks that are actually compiled in. + * The mappings of struct SanitizerKind (the -fsanitize=xxx args) to + * enum SanitizerHandler (the traps) in Clang is in clang/lib/CodeGen/. + */ +const char *report_ubsan_failure(struct pt_regs *regs, u32 check_type) +{ + switch (check_type) { +#ifdef CONFIG_UBSAN_BOUNDS + /* + * SanitizerKind::ArrayBounds and SanitizerKind::LocalBounds + * emit SanitizerHandler::OutOfBounds. + */ + case ubsan_out_of_bounds: + return "UBSAN: array index out of bounds"; +#endif +#ifdef CONFIG_UBSAN_SHIFT + /* + * SanitizerKind::ShiftBase and SanitizerKind::ShiftExponent + * emit SanitizerHandler::ShiftOutOfBounds. + */ + case ubsan_shift_out_of_bounds: + return "UBSAN: shift out of bounds"; +#endif +#ifdef CONFIG_UBSAN_DIV_ZERO + /* + * SanitizerKind::IntegerDivideByZero emits + * SanitizerHandler::DivremOverflow. + */ + case ubsan_divrem_overflow: + return "UBSAN: divide/remainder overflow"; +#endif +#ifdef CONFIG_UBSAN_UNREACHABLE + /* + * SanitizerKind::Unreachable emits + * SanitizerHandler::BuiltinUnreachable. + */ + case ubsan_builtin_unreachable: + return "UBSAN: unreachable code"; +#endif +#if defined(CONFIG_UBSAN_BOOL) || defined(CONFIG_UBSAN_ENUM) + /* + * SanitizerKind::Bool and SanitizerKind::Enum emit + * SanitizerHandler::LoadInvalidValue. + */ + case ubsan_load_invalid_value: + return "UBSAN: loading invalid value"; +#endif +#ifdef CONFIG_UBSAN_ALIGNMENT + /* + * SanitizerKind::Alignment emits SanitizerHandler::TypeMismatch + * or SanitizerHandler::AlignmentAssumption. + */ + case ubsan_alignment_assumption: + return "UBSAN: alignment assumption"; + case ubsan_type_mismatch: + return "UBSAN: type mismatch"; +#endif + default: + return "UBSAN: unrecognized failure code"; + } +} + +#else static const char * const type_check_kinds[] = { "load of", "store to", @@ -384,3 +450,5 @@ void __ubsan_handle_alignment_assumption(void *_data, unsigned long ptr, ubsan_epilogue(); } EXPORT_SYMBOL(__ubsan_handle_alignment_assumption); + +#endif /* !CONFIG_UBSAN_TRAP */ diff --git a/lib/ubsan.h b/lib/ubsan.h index 9a0b71c5ff9f..cc5cb94895a6 100644 --- a/lib/ubsan.h +++ b/lib/ubsan.h @@ -2,6 +2,38 @@ #ifndef _LIB_UBSAN_H #define _LIB_UBSAN_H +/* + * ABI defined by Clang's UBSAN enum SanitizerHandler: + * https://github.com/llvm/llvm-project/blob/release/16.x/clang/lib/CodeGen/CodeGenFunction.h#L113 + */ +enum ubsan_checks { + ubsan_add_overflow, + ubsan_builtin_unreachable, + ubsan_cfi_check_fail, + ubsan_divrem_overflow, + ubsan_dynamic_type_cache_miss, + ubsan_float_cast_overflow, + ubsan_function_type_mismatch, + ubsan_implicit_conversion, + ubsan_invalid_builtin, + ubsan_invalid_objc_cast, + ubsan_load_invalid_value, + ubsan_missing_return, + ubsan_mul_overflow, + ubsan_negate_overflow, + ubsan_nullability_arg, + ubsan_nullability_return, + ubsan_nonnull_arg, + ubsan_nonnull_return, + ubsan_out_of_bounds, + ubsan_pointer_overflow, + ubsan_shift_out_of_bounds, + ubsan_sub_overflow, + ubsan_type_mismatch, + ubsan_alignment_assumption, + ubsan_vla_bound_not_positive, +}; + enum { type_kind_int = 0, type_kind_float = 1, From 04ffde1319a715bd0550ded3580d4ea3bc003776 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Tue, 31 Jan 2023 17:37:59 -0800 Subject: [PATCH 192/239] uaccess: Add minimum bounds check on kernel buffer size While there is logic about the difference between ksize and usize, copy_struct_from_user() didn't check the size of the destination buffer (when it was known) against ksize. Add this check so there is an upper bounds check on the possible memset() call, otherwise lower bounds checks made by callers will trigger bounds warnings under -Warray-bounds. Seen under GCC 13: In function 'copy_struct_from_user', inlined from 'iommufd_fops_ioctl' at ../drivers/iommu/iommufd/main.c:333:8: ../include/linux/fortify-string.h:59:33: warning: '__builtin_memset' offset [57, 4294967294] is out of the bounds [0, 56] of object 'buf' with type 'union ucmd_buffer' [-Warray-bounds=] 59 | #define __underlying_memset __builtin_memset | ^ ../include/linux/fortify-string.h:453:9: note: in expansion of macro '__underlying_memset' 453 | __underlying_memset(p, c, __fortify_size); \ | ^~~~~~~~~~~~~~~~~~~ ../include/linux/fortify-string.h:461:25: note: in expansion of macro '__fortify_memset_chk' 461 | #define memset(p, c, s) __fortify_memset_chk(p, c, s, \ | ^~~~~~~~~~~~~~~~~~~~ ../include/linux/uaccess.h:334:17: note: in expansion of macro 'memset' 334 | memset(dst + size, 0, rest); | ^~~~~~ ../drivers/iommu/iommufd/main.c: In function 'iommufd_fops_ioctl': ../drivers/iommu/iommufd/main.c:311:27: note: 'buf' declared here 311 | union ucmd_buffer buf; | ^~~ Cc: Christian Brauner Cc: Rasmus Villemoes Cc: Arnd Bergmann Cc: Dinh Nguyen Cc: Catalin Marinas Cc: Andrew Morton Cc: Geert Uytterhoeven Cc: Alexander Potapenko Acked-by: Aleksa Sarai Signed-off-by: Kees Cook Link: https://lore.kernel.org/lkml/20230203193523.never.667-kees@kernel.org/ --- include/linux/uaccess.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index afb18f198843..ab9728138ad6 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -329,6 +329,10 @@ copy_struct_from_user(void *dst, size_t ksize, const void __user *src, size_t size = min(ksize, usize); size_t rest = max(ksize, usize) - size; + /* Double check if ksize is larger than a known object size. */ + if (WARN_ON_ONCE(ksize > __builtin_object_size(dst, 1))) + return -E2BIG; + /* Deal with trailing bytes. */ if (usize < ksize) { memset(dst + size, 0, rest); From 78f7a3fd6dc66cb788c21d7705977ed13c879351 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 7 Feb 2023 22:51:33 -0800 Subject: [PATCH 193/239] randstruct: disable Clang 15 support The randstruct support released in Clang 15 is unsafe to use due to a bug that can cause miscompilations: "-frandomize-layout-seed inconsistently randomizes all-function-pointers structs" (https://github.com/llvm/llvm-project/issues/60349). It has been fixed on the Clang 16 release branch, so add a Clang version check. Fixes: 035f7f87b729 ("randstruct: Enable Clang support") Cc: stable@vger.kernel.org Signed-off-by: Eric Biggers Acked-by: Nick Desaulniers Reviewed-by: Nathan Chancellor Reviewed-by: Bill Wendling Signed-off-by: Kees Cook Link: https://lore.kernel.org/r/20230208065133.220589-1-ebiggers@kernel.org --- security/Kconfig.hardening | 3 +++ 1 file changed, 3 insertions(+) diff --git a/security/Kconfig.hardening b/security/Kconfig.hardening index 53baa95cb644..0f295961e773 100644 --- a/security/Kconfig.hardening +++ b/security/Kconfig.hardening @@ -281,6 +281,9 @@ endmenu config CC_HAS_RANDSTRUCT def_bool $(cc-option,-frandomize-layout-seed-file=/dev/null) + # Randstruct was first added in Clang 15, but it isn't safe to use until + # Clang 16 due to https://github.com/llvm/llvm-project/issues/60349 + depends on !CC_IS_CLANG || CLANG_VERSION >= 160000 choice prompt "Randomize layout of sensitive kernel structures" From dd329e1e21b54c73f58a440b6164d04d8a7fc542 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Tue, 7 Feb 2023 20:59:09 +0100 Subject: [PATCH 194/239] cpufreq: Make cpufreq_unregister_driver() return void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All but a few drivers ignore the return value of cpufreq_unregister_driver(). Those few that don't only call it after cpufreq_register_driver() succeeded, in which case the call doesn't fail. Make the function return no value and add a WARN_ON for the case that the function is called in an invalid situation (i.e. without a previous successful call to cpufreq_register_driver()). Signed-off-by: Uwe Kleine-König Acked-by: Florian Fainelli # brcmstb-avs-cpufreq.c Acked-by: Viresh Kumar Reviewed-by: AngeloGioacchino Del Regno Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/amd-pstate.c | 4 ++-- drivers/cpufreq/brcmstb-avs-cpufreq.c | 5 +---- drivers/cpufreq/cpufreq.c | 8 +++----- drivers/cpufreq/davinci-cpufreq.c | 4 +++- drivers/cpufreq/mediatek-cpufreq-hw.c | 4 +++- drivers/cpufreq/omap-cpufreq.c | 4 +++- drivers/cpufreq/qcom-cpufreq-hw.c | 4 +++- include/linux/cpufreq.h | 2 +- 8 files changed, 19 insertions(+), 16 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 168a28bed6ee..70debd5a9f40 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -831,7 +831,7 @@ static void amd_pstate_driver_cleanup(void) static int amd_pstate_update_status(const char *buf, size_t size) { - int ret; + int ret = 0; int mode_idx; if (size > 7 || size < 6) @@ -844,7 +844,7 @@ static int amd_pstate_update_status(const char *buf, size_t size) return -EINVAL; if (cppc_state == AMD_PSTATE_ACTIVE) return -EBUSY; - ret = cpufreq_unregister_driver(current_pstate_driver); + cpufreq_unregister_driver(current_pstate_driver); amd_pstate_driver_cleanup(); break; case AMD_PSTATE_PASSIVE: diff --git a/drivers/cpufreq/brcmstb-avs-cpufreq.c b/drivers/cpufreq/brcmstb-avs-cpufreq.c index 4153150e20db..ffea6402189d 100644 --- a/drivers/cpufreq/brcmstb-avs-cpufreq.c +++ b/drivers/cpufreq/brcmstb-avs-cpufreq.c @@ -751,10 +751,7 @@ static int brcm_avs_cpufreq_probe(struct platform_device *pdev) static int brcm_avs_cpufreq_remove(struct platform_device *pdev) { - int ret; - - ret = cpufreq_unregister_driver(&brcm_avs_driver); - WARN_ON(ret); + cpufreq_unregister_driver(&brcm_avs_driver); brcm_avs_prepare_uninit(pdev); diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 7e56a42750ea..85a0bea2dbf1 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -2904,12 +2904,12 @@ EXPORT_SYMBOL_GPL(cpufreq_register_driver); * Returns zero if successful, and -EINVAL if the cpufreq_driver is * currently not initialised. */ -int cpufreq_unregister_driver(struct cpufreq_driver *driver) +void cpufreq_unregister_driver(struct cpufreq_driver *driver) { unsigned long flags; - if (!cpufreq_driver || (driver != cpufreq_driver)) - return -EINVAL; + if (WARN_ON(!cpufreq_driver || (driver != cpufreq_driver))) + return; pr_debug("unregistering driver %s\n", driver->name); @@ -2926,8 +2926,6 @@ int cpufreq_unregister_driver(struct cpufreq_driver *driver) write_unlock_irqrestore(&cpufreq_driver_lock, flags); cpus_read_unlock(); - - return 0; } EXPORT_SYMBOL_GPL(cpufreq_unregister_driver); diff --git a/drivers/cpufreq/davinci-cpufreq.c b/drivers/cpufreq/davinci-cpufreq.c index 9e97f60f8199..2d23015e2abd 100644 --- a/drivers/cpufreq/davinci-cpufreq.c +++ b/drivers/cpufreq/davinci-cpufreq.c @@ -138,7 +138,9 @@ static int __exit davinci_cpufreq_remove(struct platform_device *pdev) if (cpufreq.asyncclk) clk_put(cpufreq.asyncclk); - return cpufreq_unregister_driver(&davinci_driver); + cpufreq_unregister_driver(&davinci_driver); + + return 0; } static struct platform_driver davinci_cpufreq_driver = { diff --git a/drivers/cpufreq/mediatek-cpufreq-hw.c b/drivers/cpufreq/mediatek-cpufreq-hw.c index f80339779084..f21a9e3df53d 100644 --- a/drivers/cpufreq/mediatek-cpufreq-hw.c +++ b/drivers/cpufreq/mediatek-cpufreq-hw.c @@ -317,7 +317,9 @@ static int mtk_cpufreq_hw_driver_probe(struct platform_device *pdev) static int mtk_cpufreq_hw_driver_remove(struct platform_device *pdev) { - return cpufreq_unregister_driver(&cpufreq_mtk_hw_driver); + cpufreq_unregister_driver(&cpufreq_mtk_hw_driver); + + return 0; } static const struct of_device_id mtk_cpufreq_hw_match[] = { diff --git a/drivers/cpufreq/omap-cpufreq.c b/drivers/cpufreq/omap-cpufreq.c index 1b50df06c6bc..81649a1969b6 100644 --- a/drivers/cpufreq/omap-cpufreq.c +++ b/drivers/cpufreq/omap-cpufreq.c @@ -184,7 +184,9 @@ static int omap_cpufreq_probe(struct platform_device *pdev) static int omap_cpufreq_remove(struct platform_device *pdev) { - return cpufreq_unregister_driver(&omap_driver); + cpufreq_unregister_driver(&omap_driver); + + return 0; } static struct platform_driver omap_cpufreq_platdrv = { diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c index 957cf6bb8c05..b6227fae3285 100644 --- a/drivers/cpufreq/qcom-cpufreq-hw.c +++ b/drivers/cpufreq/qcom-cpufreq-hw.c @@ -768,7 +768,9 @@ of_exit: static int qcom_cpufreq_hw_driver_remove(struct platform_device *pdev) { - return cpufreq_unregister_driver(&cpufreq_qcom_hw_driver); + cpufreq_unregister_driver(&cpufreq_qcom_hw_driver); + + return 0; } static struct platform_driver qcom_cpufreq_hw_driver = { diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 6a94a6eaad27..65623233ab2f 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -448,7 +448,7 @@ struct cpufreq_driver { #define CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING BIT(6) int cpufreq_register_driver(struct cpufreq_driver *driver_data); -int cpufreq_unregister_driver(struct cpufreq_driver *driver_data); +void cpufreq_unregister_driver(struct cpufreq_driver *driver_data); bool cpufreq_driver_test_flags(u16 flags); const char *cpufreq_get_current_driver(void); From 7cca9a9851a5fb44808949539af6c0428e48a267 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 7 Feb 2023 17:12:51 +0100 Subject: [PATCH 195/239] cpufreq: amd-pstate: avoid uninitialized variable use The new epp support causes warnings about three separate but related bugs: 1) failing before allocation should just return an error: drivers/cpufreq/amd-pstate.c:951:6: error: variable 'ret' is used uninitialized whenever 'if' condition is true [-Werror,-Wsometimes-uninitialized] if (!dev) ^~~~ drivers/cpufreq/amd-pstate.c:1018:9: note: uninitialized use occurs here return ret; ^~~ 2) wrong variable to store return code: drivers/cpufreq/amd-pstate.c:963:6: error: variable 'ret' is used uninitialized whenever 'if' condition is true [-Werror,-Wsometimes-uninitialized] if (rc) ^~ drivers/cpufreq/amd-pstate.c:1019:9: note: uninitialized use occurs here return ret; ^~~ drivers/cpufreq/amd-pstate.c:963:2: note: remove the 'if' if its condition is always false if (rc) ^~~~~~~ 3) calling amd_pstate_set_epp() in cleanup path after determining that it should not be called: drivers/cpufreq/amd-pstate.c:1055:6: error: variable 'epp' is used uninitialized whenever 'if' condition is true [-Werror,-Wsometimes-uninitialized] if (cpudata->epp_policy == cpudata->policy) ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ drivers/cpufreq/amd-pstate.c:1080:30: note: uninitialized use occurs here amd_pstate_set_epp(cpudata, epp); ^~~ All three are trivial to fix, but most likely there are additional bugs in this function when the error handling was not really tested. Fixes: ffa5096a7c33 ("cpufreq: amd-pstate: implement Pstate EPP support for the AMD processors") Signed-off-by: Arnd Bergmann Tested-by: Wyes Karny Reviewed-by: Yuan Perry Acked-by: Huang Rui Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/amd-pstate.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 70debd5a9f40..b8862afef4e4 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -940,7 +940,6 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; struct amd_cpudata *cpudata; struct device *dev; - int rc; u64 value; /* @@ -950,7 +949,7 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) amd_perf_ctl_reset(policy->cpu); dev = get_cpu_device(policy->cpu); if (!dev) - goto free_cpudata1; + return -ENODEV; cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); if (!cpudata) @@ -959,8 +958,8 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) cpudata->cpu = policy->cpu; cpudata->epp_policy = 0; - rc = amd_pstate_init_perf(cpudata); - if (rc) + ret = amd_pstate_init_perf(cpudata); + if (ret) goto free_cpudata1; min_freq = amd_get_min_freq(cpudata); @@ -1076,9 +1075,9 @@ static void amd_pstate_epp_init(unsigned int cpu) value |= (u64)epp << 24; } + amd_pstate_set_epp(cpudata, epp); skip_epp: WRITE_ONCE(cpudata->cppc_req_cached, value); - amd_pstate_set_epp(cpudata, epp); cpufreq_cpu_put(policy); } From 5d8f384a9b4fc50f6a18405f1c08e5a87a77b5b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 8 Feb 2023 10:26:54 +0100 Subject: [PATCH 196/239] cpufreq: davinci: Fix clk use after free MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The remove function first frees the clks and only then calls cpufreq_unregister_driver(). If one of the cpufreq callbacks is called just before cpufreq_unregister_driver() is run, the freed clks might be used. Fixes: 6601b8030de3 ("davinci: add generic CPUFreq driver for DaVinci") Signed-off-by: Uwe Kleine-König Acked-by: Viresh Kumar Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/davinci-cpufreq.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/davinci-cpufreq.c b/drivers/cpufreq/davinci-cpufreq.c index 2d23015e2abd..ebb3a8102681 100644 --- a/drivers/cpufreq/davinci-cpufreq.c +++ b/drivers/cpufreq/davinci-cpufreq.c @@ -133,13 +133,13 @@ static int __init davinci_cpufreq_probe(struct platform_device *pdev) static int __exit davinci_cpufreq_remove(struct platform_device *pdev) { + cpufreq_unregister_driver(&davinci_driver); + clk_put(cpufreq.armclk); if (cpufreq.asyncclk) clk_put(cpufreq.asyncclk); - cpufreq_unregister_driver(&davinci_driver); - return 0; } From 108fcad91109bd7e9374ae9d509085f5ec55799b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Tue, 7 Feb 2023 19:58:18 +0000 Subject: [PATCH 197/239] cpufreq: Make kobj_type structure constant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since commit ee6d3dd4ed48 ("driver core: make kobj_type constant.") the driver core allows the usage of const struct kobj_type. Take advantage of this to constify the structure definition to prevent modification at runtime. Signed-off-by: Thomas Weißschuh Acked-by: Viresh Kumar Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/cpufreq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 85a0bea2dbf1..6d8fd3b8dcb5 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -993,7 +993,7 @@ static const struct sysfs_ops sysfs_ops = { .store = store, }; -static struct kobj_type ktype_cpufreq = { +static const struct kobj_type ktype_cpufreq = { .sysfs_ops = &sysfs_ops, .default_groups = cpufreq_groups, .release = cpufreq_sysfs_release, From 0b6200e1e9f53dabdc30d0f6c51af9a5f664d32b Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Thu, 2 Feb 2023 15:15:45 +0100 Subject: [PATCH 198/239] PM: domains: fix memory leak with using debugfs_lookup() When calling debugfs_lookup() the result must have dput() called on it, otherwise the memory will leak over time. To make things simpler, just call debugfs_lookup_and_remove() instead which handles all of the logic at once. Signed-off-by: Greg Kroah-Hartman Reviewed-by: Ulf Hansson Signed-off-by: Rafael J. Wysocki --- drivers/base/power/domain.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c index 967bcf9d415e..6097644ebdc5 100644 --- a/drivers/base/power/domain.c +++ b/drivers/base/power/domain.c @@ -220,13 +220,10 @@ static void genpd_debug_add(struct generic_pm_domain *genpd); static void genpd_debug_remove(struct generic_pm_domain *genpd) { - struct dentry *d; - if (!genpd_debugfs_dir) return; - d = debugfs_lookup(genpd->name, genpd_debugfs_dir); - debugfs_remove(d); + debugfs_lookup_and_remove(genpd->name, genpd_debugfs_dir); } static void genpd_update_accounting(struct generic_pm_domain *genpd) From a0e8c13ccd6a9a636d27353da62c2410c4eca337 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Thu, 2 Feb 2023 16:15:15 +0100 Subject: [PATCH 199/239] PM: EM: fix memory leak with using debugfs_lookup() When calling debugfs_lookup() the result must have dput() called on it, otherwise the memory will leak over time. To make things simpler, just call debugfs_lookup_and_remove() instead which handles all of the logic at once. Signed-off-by: Greg Kroah-Hartman Signed-off-by: Rafael J. Wysocki --- kernel/power/energy_model.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index f82111837b8d..7b44f5b89fa1 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -87,10 +87,7 @@ static void em_debug_create_pd(struct device *dev) static void em_debug_remove_pd(struct device *dev) { - struct dentry *debug_dir; - - debug_dir = debugfs_lookup(dev_name(dev), rootdir); - debugfs_remove_recursive(debug_dir); + debugfs_lookup_and_remove(dev_name(dev), rootdir); } static int __init em_debug_init(void) From 7787943a3a8ade6594a68db28c166adbb1d3708c Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 6 Feb 2023 20:33:06 +0100 Subject: [PATCH 200/239] cpuidle: add ARCH_SUSPEND_POSSIBLE dependencies Some ARMv4 processors don't support suspend, which leads to a build failure with the tegra and qualcomm cpuidle driver: WARNING: unmet direct dependencies detected for ARM_CPU_SUSPEND Depends on [n]: ARCH_SUSPEND_POSSIBLE [=n] Selected by [y]: - ARM_TEGRA_CPUIDLE [=y] && CPU_IDLE [=y] && (ARM [=y] || ARM64) && (ARCH_TEGRA [=n] || COMPILE_TEST [=y]) && !ARM64 && MMU [=y] arch/arm/kernel/sleep.o: in function `__cpu_suspend': (.text+0x68): undefined reference to `cpu_sa110_suspend_size' (.text+0x68): undefined reference to `cpu_fa526_suspend_size' Add an explicit dependency to make randconfig builds avoid this combination. Fixes: faae6c9f2e68 ("cpuidle: tegra: Enable compile testing") Fixes: a871be6b8eee ("cpuidle: Convert Qualcomm SPM driver to a generic CPUidle driver") Link: https://lore.kernel.org/all/20211013160125.772873-1-arnd@kernel.org/ Cc: All applicable Reviewed-by: Dmitry Osipenko Signed-off-by: Arnd Bergmann Acked-by: Thierry Reding Signed-off-by: Rafael J. Wysocki --- drivers/cpuidle/Kconfig.arm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/cpuidle/Kconfig.arm b/drivers/cpuidle/Kconfig.arm index 747aa537389b..f0714a32921e 100644 --- a/drivers/cpuidle/Kconfig.arm +++ b/drivers/cpuidle/Kconfig.arm @@ -102,6 +102,7 @@ config ARM_MVEBU_V7_CPUIDLE config ARM_TEGRA_CPUIDLE bool "CPU Idle Driver for NVIDIA Tegra SoCs" depends on (ARCH_TEGRA || COMPILE_TEST) && !ARM64 && MMU + depends on ARCH_SUSPEND_POSSIBLE select ARCH_NEEDS_CPU_IDLE_COUPLED if SMP select ARM_CPU_SUSPEND help @@ -110,6 +111,7 @@ config ARM_TEGRA_CPUIDLE config ARM_QCOM_SPM_CPUIDLE bool "CPU Idle Driver for Qualcomm Subsystem Power Manager (SPM)" depends on (ARCH_QCOM || COMPILE_TEST) && !ARM64 && MMU + depends on ARCH_SUSPEND_POSSIBLE select ARM_CPU_SUSPEND select CPU_IDLE_MULTIPLE_DRIVERS select DT_IDLE_STATES From e898b07deb3ce801b21113979a05f4b3166c7cb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Tue, 7 Feb 2023 19:55:19 +0000 Subject: [PATCH 201/239] cpuidle: sysfs: make kobj_type structures constant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since commit ee6d3dd4ed48 ("driver core: make kobj_type constant.") the driver core allows the usage of const struct kobj_type. Take advantage of this to constify the structure definitions to prevent modification at runtime. Signed-off-by: Thomas Weißschuh Signed-off-by: Rafael J. Wysocki --- drivers/cpuidle/sysfs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/cpuidle/sysfs.c b/drivers/cpuidle/sysfs.c index 2b496a53cbca..48948b171749 100644 --- a/drivers/cpuidle/sysfs.c +++ b/drivers/cpuidle/sysfs.c @@ -200,7 +200,7 @@ static void cpuidle_sysfs_release(struct kobject *kobj) complete(&kdev->kobj_unregister); } -static struct kobj_type ktype_cpuidle = { +static const struct kobj_type ktype_cpuidle = { .sysfs_ops = &cpuidle_sysfs_ops, .release = cpuidle_sysfs_release, }; @@ -447,7 +447,7 @@ static void cpuidle_state_sysfs_release(struct kobject *kobj) complete(&state_obj->kobj_unregister); } -static struct kobj_type ktype_state_cpuidle = { +static const struct kobj_type ktype_state_cpuidle = { .sysfs_ops = &cpuidle_state_sysfs_ops, .default_groups = cpuidle_state_default_groups, .release = cpuidle_state_sysfs_release, @@ -594,7 +594,7 @@ static struct attribute *cpuidle_driver_default_attrs[] = { }; ATTRIBUTE_GROUPS(cpuidle_driver_default); -static struct kobj_type ktype_driver_cpuidle = { +static const struct kobj_type ktype_driver_cpuidle = { .sysfs_ops = &cpuidle_driver_sysfs_ops, .default_groups = cpuidle_driver_default_groups, .release = cpuidle_driver_sysfs_release, From fda7be2068973195343d14c1f760adcd481455c9 Mon Sep 17 00:00:00 2001 From: Yang Li Date: Wed, 8 Feb 2023 10:31:58 +0800 Subject: [PATCH 202/239] ACPI: CPPC: Fix some kernel-doc comments Add the description of @pcc_ss_id in pcc_data_alloc(). Add the description of @cpu_num in cppc_get_transition_latency(). clear the below warnings: drivers/acpi/cppc_acpi.c:607: warning: Function parameter or member 'pcc_ss_id' not described in 'pcc_data_alloc' drivers/acpi/cppc_acpi.c:1616: warning: Function parameter or member 'cpu_num' not described in 'cppc_get_transition_latency' Reported-by: Abaci Robot Link: https://bugzilla.openanolis.cn/show_bug.cgi?id=3983 Signed-off-by: Yang Li [ rjw: Dropped redundant empty code lines, minor edits ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/cppc_acpi.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index 0f17b1c32718..1d00a73fb518 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -595,6 +595,7 @@ bool __weak cpc_supported_by_cpu(void) /** * pcc_data_alloc() - Allocate the pcc_data memory for pcc subspace + * @pcc_ss_id: PCC Subspace index as in the PCC client ACPI package. * * Check and allocate the cppc_pcc_data memory. * In some processor configurations it is possible that same subspace @@ -1536,6 +1537,7 @@ EXPORT_SYMBOL_GPL(cppc_set_perf); /** * cppc_get_transition_latency - returns frequency transition latency in ns + * @cpu_num: CPU number for per_cpu(). * * ACPI CPPC does not explicitly specify how a platform can specify the * transition latency for performance change requests. The closest we have From 41204a607679ccca7eabff9f2871b969d6ef2ce3 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 7 Feb 2023 20:59:59 +0100 Subject: [PATCH 203/239] cpuidle: driver: Update microsecond values of state parameters as needed If the cpuidle driver provides the target residency and exit latency in nanoseconds, the corresponding values in microseconds need to be set to reflect the provided numbers in order for the sysfs interface to show them correctly, so make __cpuidle_driver_init() do that. Signed-off-by: Rafael J. Wysocki Tested-by: Artem Bityutskiy --- drivers/cpuidle/driver.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/cpuidle/driver.c b/drivers/cpuidle/driver.c index f70aa17e2a8e..d9cda7f6ccb9 100644 --- a/drivers/cpuidle/driver.c +++ b/drivers/cpuidle/driver.c @@ -183,11 +183,15 @@ static void __cpuidle_driver_init(struct cpuidle_driver *drv) s->target_residency_ns = s->target_residency * NSEC_PER_USEC; else if (s->target_residency_ns < 0) s->target_residency_ns = 0; + else + s->target_residency = div_u64(s->target_residency_ns, NSEC_PER_USEC); if (s->exit_latency > 0) s->exit_latency_ns = s->exit_latency * NSEC_PER_USEC; else if (s->exit_latency_ns < 0) s->exit_latency_ns = 0; + else + s->exit_latency = div_u64(s->exit_latency_ns, NSEC_PER_USEC); } } From cf835b005b2857c2fd763a006c1957f332e5254b Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Sat, 11 Feb 2023 11:17:10 +0800 Subject: [PATCH 204/239] powercap: intel_rapl: Fix handling for large time window When setting the power limit time window, software updates the 'y' bits and 'f' bits in the power limit register, and the value hardware takes follows the formula below Time window = 2 ^ y * (1 + f / 4) * Time_Unit When handling large time window input from userspace, using left shifting breaks in two cases: 1. when ilog2(value) is bigger than 31, in expression "1 << y", left shifting by more than 31 bits has undefined behavior. This breaks 'y'. For example, on an Alderlake platform, "1 << 32" returns 1. 2. when ilog2(value) equals 31, "1 << 31" returns negative value because '1' is recognized as signed int. And this breaks 'f'. Given that 'y' has 5 bits and hardware can never take a value larger than 31, fix the first problem by clamp the time window to the maximum possible value that the hardware can take. Fix the second problem by using unsigned bit left shift. Note that hardware has its own maximum time window limitation, which may be lower than the time window value retrieved from the power limit register. When this happens, hardware clamps the input to its maximum time window limitation. That is why a software clamp is preferred to handle the problem on hand. Signed-off-by: Zhang Rui [ rjw: Adjusted the comment added by this change ] Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl_common.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 9a9192fc8391..8970c7b80884 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -999,7 +999,15 @@ static u64 rapl_compute_time_window_core(struct rapl_package *rp, u64 value, do_div(value, rp->time_unit); y = ilog2(value); - f = div64_u64(4 * (value - (1 << y)), 1 << y); + + /* + * The target hardware field is 7 bits wide, so return all ones + * if the exponent is too large. + */ + if (y > 0x1f) + return 0x7f; + + f = div64_u64(4 * (value - (1ULL << y)), 1ULL << y); value = (y & 0x1f) | ((f & 0x3) << 5); } return value; From e947925f10c250ba3ad2d905006b2e632510310c Mon Sep 17 00:00:00 2001 From: Keguang Zhang Date: Fri, 10 Feb 2023 19:09:34 +0800 Subject: [PATCH 205/239] MIPS: loongson32: Drop obsolete cpufreq platform device The obsolete cpufreq driver was removed, drop the platform device and data accordingly. Link: https://lore.kernel.org/all/20230112135342.3927338-1-keguang.zhang@gmail.com Signed-off-by: Keguang Zhang Signed-off-by: Rafael J. Wysocki --- .../mips/include/asm/mach-loongson32/cpufreq.h | 18 ------------------ .../include/asm/mach-loongson32/platform.h | 1 - arch/mips/loongson32/common/platform.c | 16 ---------------- arch/mips/loongson32/ls1b/board.c | 1 - 4 files changed, 36 deletions(-) delete mode 100644 arch/mips/include/asm/mach-loongson32/cpufreq.h diff --git a/arch/mips/include/asm/mach-loongson32/cpufreq.h b/arch/mips/include/asm/mach-loongson32/cpufreq.h deleted file mode 100644 index e422a32883ae..000000000000 --- a/arch/mips/include/asm/mach-loongson32/cpufreq.h +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (c) 2014 Zhang, Keguang - * - * Loongson 1 CPUFreq platform support. - */ - -#ifndef __ASM_MACH_LOONGSON32_CPUFREQ_H -#define __ASM_MACH_LOONGSON32_CPUFREQ_H - -struct plat_ls1x_cpufreq { - const char *clk_name; /* CPU clk */ - const char *osc_clk_name; /* OSC clk */ - unsigned int max_freq; /* in kHz */ - unsigned int min_freq; /* in kHz */ -}; - -#endif /* __ASM_MACH_LOONGSON32_CPUFREQ_H */ diff --git a/arch/mips/include/asm/mach-loongson32/platform.h b/arch/mips/include/asm/mach-loongson32/platform.h index eb83e2741887..86e1a6aab4e5 100644 --- a/arch/mips/include/asm/mach-loongson32/platform.h +++ b/arch/mips/include/asm/mach-loongson32/platform.h @@ -12,7 +12,6 @@ #include extern struct platform_device ls1x_uart_pdev; -extern struct platform_device ls1x_cpufreq_pdev; extern struct platform_device ls1x_eth0_pdev; extern struct platform_device ls1x_eth1_pdev; extern struct platform_device ls1x_ehci_pdev; diff --git a/arch/mips/loongson32/common/platform.c b/arch/mips/loongson32/common/platform.c index 311dc1580bbd..64d7979394e6 100644 --- a/arch/mips/loongson32/common/platform.c +++ b/arch/mips/loongson32/common/platform.c @@ -15,7 +15,6 @@ #include #include -#include #include #include @@ -62,21 +61,6 @@ void __init ls1x_serial_set_uartclk(struct platform_device *pdev) p->uartclk = clk_get_rate(clk); } -/* CPUFreq */ -static struct plat_ls1x_cpufreq ls1x_cpufreq_pdata = { - .clk_name = "cpu_clk", - .osc_clk_name = "osc_clk", - .max_freq = 266 * 1000, - .min_freq = 33 * 1000, -}; - -struct platform_device ls1x_cpufreq_pdev = { - .name = "ls1x-cpufreq", - .dev = { - .platform_data = &ls1x_cpufreq_pdata, - }, -}; - /* Synopsys Ethernet GMAC */ static struct stmmac_mdio_bus_data ls1x_mdio_bus_data = { .phy_mask = 0, diff --git a/arch/mips/loongson32/ls1b/board.c b/arch/mips/loongson32/ls1b/board.c index 727e06718dab..fed8d432ef20 100644 --- a/arch/mips/loongson32/ls1b/board.c +++ b/arch/mips/loongson32/ls1b/board.c @@ -35,7 +35,6 @@ static const struct gpio_led_platform_data ls1x_led_pdata __initconst = { static struct platform_device *ls1b_platform_devices[] __initdata = { &ls1x_uart_pdev, - &ls1x_cpufreq_pdev, &ls1x_eth0_pdev, &ls1x_eth1_pdev, &ls1x_ehci_pdev, From f9901f64536c39f699e6ed228c5e64e7e7ce8708 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Wed, 25 Jan 2023 12:34:18 +0100 Subject: [PATCH 206/239] cpuidle: psci: Do not suspend topology CPUs on PREEMPT_RT The runtime Power Management of CPU topology is not compatible with PREEMPT_RT: 1. Core cpuidle path disables IRQs. 2. Core cpuidle calls cpuidle-psci. 3. cpuidle-psci in __psci_enter_domain_idle_state() calls pm_runtime_put_sync_suspend() and pm_runtime_get_sync() which use spinlocks (which are sleeping on PREEMPT_RT). Deep sleep modes are not a priority of Realtime kernels because the latencies might become unpredictable. On the other hand the PSCI CPU idle power domain is a parent of other devices and power domain controllers, thus it cannot be simply skipped (e.g. on Qualcomm SM8250). Disable the idle callbacks in cpuidle-psci and mark the domain as always on. This is a trade-off between making PREEMPT_RT working and still having a proper power domain hierarchy in the system. Signed-off-by: Krzysztof Kozlowski Reviewed-by: Ulf Hansson Tested-by: Adrien Thierry Signed-off-by: Rafael J. Wysocki --- drivers/cpuidle/Kconfig.arm | 8 ++++++++ drivers/cpuidle/cpuidle-psci-domain.c | 7 +++++-- drivers/cpuidle/cpuidle-psci.c | 3 +++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/drivers/cpuidle/Kconfig.arm b/drivers/cpuidle/Kconfig.arm index f0714a32921e..a1ee475d180d 100644 --- a/drivers/cpuidle/Kconfig.arm +++ b/drivers/cpuidle/Kconfig.arm @@ -24,6 +24,14 @@ config ARM_PSCI_CPUIDLE It provides an idle driver that is capable of detecting and managing idle states through the PSCI firmware interface. + The driver has limitations when used with PREEMPT_RT: + - If the idle states are described with the non-hierarchical layout, + all idle states are still available. + + - If the idle states are described with the hierarchical layout, + only the idle states defined per CPU are available, but not the ones + being shared among a group of CPUs (aka cluster idle states). + config ARM_PSCI_CPUIDLE_DOMAIN bool "PSCI CPU idle Domain" depends on ARM_PSCI_CPUIDLE diff --git a/drivers/cpuidle/cpuidle-psci-domain.c b/drivers/cpuidle/cpuidle-psci-domain.c index c80cf9ddabd8..6ad2954948a5 100644 --- a/drivers/cpuidle/cpuidle-psci-domain.c +++ b/drivers/cpuidle/cpuidle-psci-domain.c @@ -64,8 +64,11 @@ static int psci_pd_init(struct device_node *np, bool use_osi) pd->flags |= GENPD_FLAG_IRQ_SAFE | GENPD_FLAG_CPU_DOMAIN; - /* Allow power off when OSI has been successfully enabled. */ - if (use_osi) + /* + * Allow power off when OSI has been successfully enabled. + * PREEMPT_RT is not yet ready to enter domain idle states. + */ + if (use_osi && !IS_ENABLED(CONFIG_PREEMPT_RT)) pd->power_off = psci_pd_power_off; else pd->flags |= GENPD_FLAG_ALWAYS_ON; diff --git a/drivers/cpuidle/cpuidle-psci.c b/drivers/cpuidle/cpuidle-psci.c index 57bc3e3ae391..68467a325dcb 100644 --- a/drivers/cpuidle/cpuidle-psci.c +++ b/drivers/cpuidle/cpuidle-psci.c @@ -231,6 +231,9 @@ static int psci_dt_cpu_init_topology(struct cpuidle_driver *drv, if (!psci_has_osi_support()) return 0; + if (IS_ENABLED(CONFIG_PREEMPT_RT)) + return 0; + data->dev = psci_dt_attach_cpu(cpu); if (IS_ERR_OR_NULL(data->dev)) return PTR_ERR_OR_ZERO(data->dev); From 41a337b40e983db4f0e1602308109f2b93687a06 Mon Sep 17 00:00:00 2001 From: Richard Fitzgerald Date: Mon, 13 Feb 2023 16:50:05 +0000 Subject: [PATCH 207/239] PM: Add EXPORT macros for exporting PM functions Add a pair of macros for exporting functions only if CONFIG_PM is enabled. The naming follows the style of the standard EXPORT_SYMBOL_*() macros that they replace. Sometimes a module wants to export PM functions directly to other drivers, not a complete struct dev_pm_ops. A typical example is where a core library exports the generic (shared) implementation and calling code wraps one or more of these in custom code. Signed-off-by: Richard Fitzgerald Signed-off-by: Rafael J. Wysocki --- include/linux/pm.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/linux/pm.h b/include/linux/pm.h index 93cd34f00822..035d9649eba4 100644 --- a/include/linux/pm.h +++ b/include/linux/pm.h @@ -379,9 +379,13 @@ const struct dev_pm_ops name = { \ const struct dev_pm_ops name; \ __EXPORT_SYMBOL(name, sec, ns); \ const struct dev_pm_ops name +#define EXPORT_PM_FN_GPL(name) EXPORT_SYMBOL_GPL(name) +#define EXPORT_PM_FN_NS_GPL(name, ns) EXPORT_SYMBOL_NS_GPL(name, ns) #else #define _EXPORT_DEV_PM_OPS(name, sec, ns) \ static __maybe_unused const struct dev_pm_ops __static_##name +#define EXPORT_PM_FN_GPL(name) +#define EXPORT_PM_FN_NS_GPL(name, ns) #endif #define EXPORT_DEV_PM_OPS(name) _EXPORT_DEV_PM_OPS(name, "", "") From cb18703c179713056bd7e3bdfc2260ab4e8658f0 Mon Sep 17 00:00:00 2001 From: Adam Niederer Date: Sat, 11 Feb 2023 15:13:33 -0500 Subject: [PATCH 208/239] ACPI: resource: Add IRQ overrides for MAINGEAR Vector Pro 2 models Fix a regression introduced by commit 9946e39fe8d0 ("ACPI: resource: skip IRQ override on AMD Zen platforms") on MAINGEAR Vector Pro 2 systems, which causes the built-in keyboard to not work. This restores the functionality by adding an IRQ override. No other IRQs were being overridden before, so this should be all that is needed for these systems. I have personally tested this on the 15" model (MG-VCP2-15A3070T), and I have confirmation that the issue is present on the 17" model (MG-VCP2-17A3070T). Fixes: 9946e39fe8d0 ("ACPI: resource: skip IRQ override on AMD Zen platforms") Signed-off-by: Adam Niederer Signed-off-by: Rafael J. Wysocki --- drivers/acpi/resource.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/drivers/acpi/resource.c b/drivers/acpi/resource.c index 192d1784e409..1d9d3364bc2b 100644 --- a/drivers/acpi/resource.c +++ b/drivers/acpi/resource.c @@ -478,6 +478,24 @@ static const struct dmi_system_id schenker_gm_rg[] = { { } }; +static const struct dmi_system_id maingear_laptop[] = { + { + .ident = "MAINGEAR Vector Pro 2 15", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Micro Electronics Inc"), + DMI_MATCH(DMI_PRODUCT_NAME, "MG-VCP2-15A3070T"), + } + }, + { + .ident = "MAINGEAR Vector Pro 2 17", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Micro Electronics Inc"), + DMI_MATCH(DMI_PRODUCT_NAME, "MG-VCP2-17A3070T"), + }, + }, + { } +}; + struct irq_override_cmp { const struct dmi_system_id *system; unsigned char irq; @@ -493,6 +511,7 @@ static const struct irq_override_cmp override_table[] = { { lenovo_laptop, 6, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_LOW, 0, true }, { lenovo_laptop, 10, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_LOW, 0, true }, { schenker_gm_rg, 1, ACPI_EDGE_SENSITIVE, ACPI_ACTIVE_LOW, 1, true }, + { maingear_laptop, 1, ACPI_EDGE_SENSITIVE, ACPI_ACTIVE_LOW, 1, true }, }; static bool acpi_dev_irq_override(u32 gsi, u8 triggering, u8 polarity, From 17bb7046e7ce038a73ee97eaa804e0300c5199e2 Mon Sep 17 00:00:00 2001 From: Werner Sembach Date: Mon, 13 Feb 2023 19:16:53 +0100 Subject: [PATCH 209/239] ACPI: resource: Do IRQ override on all TongFang GMxRGxx Apply commit 7592b79ba4a9 ("ACPI: resource: do IRQ override on XMG Core 15") override for all vendors using this mainboard. Signed-off-by: Werner Sembach Fixes: 9946e39fe8d0 ("ACPI: resource: skip IRQ override on AMD Zen platforms") Signed-off-by: Rafael J. Wysocki --- drivers/acpi/resource.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/acpi/resource.c b/drivers/acpi/resource.c index 1d9d3364bc2b..a222bda7e15b 100644 --- a/drivers/acpi/resource.c +++ b/drivers/acpi/resource.c @@ -467,11 +467,10 @@ static const struct dmi_system_id lenovo_laptop[] = { { } }; -static const struct dmi_system_id schenker_gm_rg[] = { +static const struct dmi_system_id tongfang_gm_rg[] = { { - .ident = "XMG CORE 15 (M22)", + .ident = "TongFang GMxRGxx/XMG CORE 15 (M22)/TUXEDO Stellaris 15 Gen4 AMD", .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "SchenkerTechnologiesGmbH"), DMI_MATCH(DMI_BOARD_NAME, "GMxRGxx"), }, }, @@ -510,7 +509,7 @@ static const struct irq_override_cmp override_table[] = { { asus_laptop, 1, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_LOW, 0, false }, { lenovo_laptop, 6, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_LOW, 0, true }, { lenovo_laptop, 10, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_LOW, 0, true }, - { schenker_gm_rg, 1, ACPI_EDGE_SENSITIVE, ACPI_ACTIVE_LOW, 1, true }, + { tongfang_gm_rg, 1, ACPI_EDGE_SENSITIVE, ACPI_ACTIVE_LOW, 1, true }, { maingear_laptop, 1, ACPI_EDGE_SENSITIVE, ACPI_ACTIVE_LOW, 1, true }, }; From 5a9e358f1d1f5a2c2dfd8055b0e9e44d58a54a49 Mon Sep 17 00:00:00 2001 From: Lukas Wunner Date: Wed, 8 Feb 2023 06:59:51 +0100 Subject: [PATCH 210/239] ACPICA: Fix typo in CDAT DSMAS struct definition ACPICA commit 9d8bd58d5f3495ce76d1b9767ec0b92251cbc366 Link: https://github.com/acpica/acpica/commit/9d8bd58d5f34 Signed-off-by: Lukas Wunner Reviewed-by: Ira Weiny Signed-off-by: Rafael J. Wysocki --- include/acpi/actbl1.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/acpi/actbl1.h b/include/acpi/actbl1.h index 4175dce3967c..e8297cefde09 100644 --- a/include/acpi/actbl1.h +++ b/include/acpi/actbl1.h @@ -344,7 +344,7 @@ enum acpi_cdat_type { /* Subtable 0: Device Scoped Memory Affinity Structure (DSMAS) */ -struct acpi_cadt_dsmas { +struct acpi_cdat_dsmas { u8 dsmad_handle; u8 flags; u16 reserved; From b06730a571a9ff1ba5bd6b20bf9e50e5a12f1ec6 Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Mon, 13 Feb 2023 11:42:43 +0100 Subject: [PATCH 211/239] irqdomain: Fix association race The sanity check for an already mapped virq is done outside of the irq_domain_mutex-protected section which means that an (unlikely) racing association may not be detected. Fix this by factoring out the association implementation, which will also be used in a follow-on change to fix a shared-interrupt mapping race. Fixes: ddaf144c61da ("irqdomain: Refactor irq_domain_associate_many()") Cc: stable@vger.kernel.org # 3.11 Tested-by: Hsin-Yi Wang Tested-by: Mark-PK Tsai Signed-off-by: Johan Hovold Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230213104302.17307-2-johan+linaro@kernel.org --- kernel/irq/irqdomain.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index 8fe1da9614ee..6661de18550e 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -559,8 +559,8 @@ static void irq_domain_disassociate(struct irq_domain *domain, unsigned int irq) irq_domain_clear_mapping(domain, hwirq); } -int irq_domain_associate(struct irq_domain *domain, unsigned int virq, - irq_hw_number_t hwirq) +static int irq_domain_associate_locked(struct irq_domain *domain, unsigned int virq, + irq_hw_number_t hwirq) { struct irq_data *irq_data = irq_get_irq_data(virq); int ret; @@ -573,7 +573,6 @@ int irq_domain_associate(struct irq_domain *domain, unsigned int virq, if (WARN(irq_data->domain, "error: virq%i is already associated", virq)) return -EINVAL; - mutex_lock(&irq_domain_mutex); irq_data->hwirq = hwirq; irq_data->domain = domain; if (domain->ops->map) { @@ -590,7 +589,6 @@ int irq_domain_associate(struct irq_domain *domain, unsigned int virq, } irq_data->domain = NULL; irq_data->hwirq = 0; - mutex_unlock(&irq_domain_mutex); return ret; } @@ -601,12 +599,23 @@ int irq_domain_associate(struct irq_domain *domain, unsigned int virq, domain->mapcount++; irq_domain_set_mapping(domain, hwirq, irq_data); - mutex_unlock(&irq_domain_mutex); irq_clear_status_flags(virq, IRQ_NOREQUEST); return 0; } + +int irq_domain_associate(struct irq_domain *domain, unsigned int virq, + irq_hw_number_t hwirq) +{ + int ret; + + mutex_lock(&irq_domain_mutex); + ret = irq_domain_associate_locked(domain, virq, hwirq); + mutex_unlock(&irq_domain_mutex); + + return ret; +} EXPORT_SYMBOL_GPL(irq_domain_associate); void irq_domain_associate_many(struct irq_domain *domain, unsigned int irq_base, From 3f883c38f5628f46b30bccf090faec054088e262 Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Mon, 13 Feb 2023 11:42:44 +0100 Subject: [PATCH 212/239] irqdomain: Fix disassociation race The global irq_domain_mutex is held when mapping interrupts from non-hierarchical domains but currently not when disposing them. This specifically means that updates of the domain mapcount is racy (currently only used for statistics in debugfs). Make sure to hold the global irq_domain_mutex also when disposing mappings from non-hierarchical domains. Fixes: 9dc6be3d4193 ("genirq/irqdomain: Add map counter") Cc: stable@vger.kernel.org # 4.13 Tested-by: Hsin-Yi Wang Tested-by: Mark-PK Tsai Signed-off-by: Johan Hovold Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230213104302.17307-3-johan+linaro@kernel.org --- kernel/irq/irqdomain.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index 6661de18550e..f77549a2a178 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -538,6 +538,9 @@ static void irq_domain_disassociate(struct irq_domain *domain, unsigned int irq) return; hwirq = irq_data->hwirq; + + mutex_lock(&irq_domain_mutex); + irq_set_status_flags(irq, IRQ_NOREQUEST); /* remove chip and handler */ @@ -557,6 +560,8 @@ static void irq_domain_disassociate(struct irq_domain *domain, unsigned int irq) /* Clear reverse map for this hwirq */ irq_domain_clear_mapping(domain, hwirq); + + mutex_unlock(&irq_domain_mutex); } static int irq_domain_associate_locked(struct irq_domain *domain, unsigned int virq, From e3b7ab025e931accdc2c12acf9b75c6197f1c062 Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Mon, 13 Feb 2023 11:42:45 +0100 Subject: [PATCH 213/239] irqdomain: Drop bogus fwspec-mapping error handling In case a newly allocated IRQ ever ends up not having any associated struct irq_data it would not even be possible to dispose the mapping. Replace the bogus disposal with a WARN_ON(). This will also be used to fix a shared-interrupt mapping race, hence the CC-stable tag. Fixes: 1e2a7d78499e ("irqdomain: Don't set type when mapping an IRQ") Cc: stable@vger.kernel.org # 4.8 Tested-by: Hsin-Yi Wang Tested-by: Mark-PK Tsai Signed-off-by: Johan Hovold Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230213104302.17307-4-johan+linaro@kernel.org --- kernel/irq/irqdomain.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index f77549a2a178..9f5b96cf6c5c 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -847,13 +847,8 @@ unsigned int irq_create_fwspec_mapping(struct irq_fwspec *fwspec) } irq_data = irq_get_irq_data(virq); - if (!irq_data) { - if (irq_domain_is_hierarchy(domain)) - irq_domain_free_irqs(virq, 1); - else - irq_dispose_mapping(virq); + if (WARN_ON(!irq_data)) return 0; - } /* Store trigger type */ irqd_set_trigger_type(irq_data, type); From 6e6f75c9c98d2d246d90411ff2b6f0cd271f4cba Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Mon, 13 Feb 2023 11:42:46 +0100 Subject: [PATCH 214/239] irqdomain: Look for existing mapping only once Avoid looking for an existing mapping twice when creating a new mapping using irq_create_fwspec_mapping() by factoring out the actual allocation which is shared with irq_create_mapping_affinity(). The new helper function will also be used to fix a shared-interrupt mapping race, hence the Fixes tag. Fixes: b62b2cf5759b ("irqdomain: Fix handling of type settings for existing mappings") Cc: stable@vger.kernel.org # 4.8 Tested-by: Hsin-Yi Wang Tested-by: Mark-PK Tsai Signed-off-by: Johan Hovold Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230213104302.17307-5-johan+linaro@kernel.org --- kernel/irq/irqdomain.c | 74 +++++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 34 deletions(-) diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index 9f5b96cf6c5c..9f95047e4bc7 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -682,44 +682,15 @@ unsigned int irq_create_direct_mapping(struct irq_domain *domain) EXPORT_SYMBOL_GPL(irq_create_direct_mapping); #endif -/** - * irq_create_mapping_affinity() - Map a hardware interrupt into linux irq space - * @domain: domain owning this hardware interrupt or NULL for default domain - * @hwirq: hardware irq number in that domain space - * @affinity: irq affinity - * - * Only one mapping per hardware interrupt is permitted. Returns a linux - * irq number. - * If the sense/trigger is to be specified, set_irq_type() should be called - * on the number returned from that call. - */ -unsigned int irq_create_mapping_affinity(struct irq_domain *domain, - irq_hw_number_t hwirq, - const struct irq_affinity_desc *affinity) +static unsigned int __irq_create_mapping_affinity(struct irq_domain *domain, + irq_hw_number_t hwirq, + const struct irq_affinity_desc *affinity) { - struct device_node *of_node; + struct device_node *of_node = irq_domain_get_of_node(domain); int virq; pr_debug("irq_create_mapping(0x%p, 0x%lx)\n", domain, hwirq); - /* Look for default domain if necessary */ - if (domain == NULL) - domain = irq_default_domain; - if (domain == NULL) { - WARN(1, "%s(, %lx) called with NULL domain\n", __func__, hwirq); - return 0; - } - pr_debug("-> using domain @%p\n", domain); - - of_node = irq_domain_get_of_node(domain); - - /* Check if mapping already exists */ - virq = irq_find_mapping(domain, hwirq); - if (virq) { - pr_debug("-> existing mapping on virq %d\n", virq); - return virq; - } - /* Allocate a virtual interrupt number */ virq = irq_domain_alloc_descs(-1, 1, hwirq, of_node_to_nid(of_node), affinity); @@ -738,6 +709,41 @@ unsigned int irq_create_mapping_affinity(struct irq_domain *domain, return virq; } + +/** + * irq_create_mapping_affinity() - Map a hardware interrupt into linux irq space + * @domain: domain owning this hardware interrupt or NULL for default domain + * @hwirq: hardware irq number in that domain space + * @affinity: irq affinity + * + * Only one mapping per hardware interrupt is permitted. Returns a linux + * irq number. + * If the sense/trigger is to be specified, set_irq_type() should be called + * on the number returned from that call. + */ +unsigned int irq_create_mapping_affinity(struct irq_domain *domain, + irq_hw_number_t hwirq, + const struct irq_affinity_desc *affinity) +{ + int virq; + + /* Look for default domain if necessary */ + if (domain == NULL) + domain = irq_default_domain; + if (domain == NULL) { + WARN(1, "%s(, %lx) called with NULL domain\n", __func__, hwirq); + return 0; + } + + /* Check if mapping already exists */ + virq = irq_find_mapping(domain, hwirq); + if (virq) { + pr_debug("existing mapping on virq %d\n", virq); + return virq; + } + + return __irq_create_mapping_affinity(domain, hwirq, affinity); +} EXPORT_SYMBOL_GPL(irq_create_mapping_affinity); static int irq_domain_translate(struct irq_domain *d, @@ -841,7 +847,7 @@ unsigned int irq_create_fwspec_mapping(struct irq_fwspec *fwspec) return 0; } else { /* Create mapping */ - virq = irq_create_mapping(domain, hwirq); + virq = __irq_create_mapping_affinity(domain, hwirq, NULL); if (!virq) return virq; } From d55f7f4c58c07beb5050a834bf57ae2ede599c7e Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Mon, 13 Feb 2023 11:42:47 +0100 Subject: [PATCH 215/239] irqdomain: Refactor __irq_domain_alloc_irqs() Refactor __irq_domain_alloc_irqs() so that it can be called internally while holding the irq_domain_mutex. This will be used to fix a shared-interrupt mapping race, hence the Fixes tag. Fixes: b62b2cf5759b ("irqdomain: Fix handling of type settings for existing mappings") Cc: stable@vger.kernel.org # 4.8 Tested-by: Hsin-Yi Wang Tested-by: Mark-PK Tsai Signed-off-by: Johan Hovold Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230213104302.17307-6-johan+linaro@kernel.org --- kernel/irq/irqdomain.c | 90 +++++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 41 deletions(-) diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index 9f95047e4bc7..78fb4800c0d2 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -1441,6 +1441,52 @@ int irq_domain_alloc_irqs_hierarchy(struct irq_domain *domain, return domain->ops->alloc(domain, irq_base, nr_irqs, arg); } +static int irq_domain_alloc_irqs_locked(struct irq_domain *domain, int irq_base, + unsigned int nr_irqs, int node, void *arg, + bool realloc, const struct irq_affinity_desc *affinity) +{ + int i, ret, virq; + + if (realloc && irq_base >= 0) { + virq = irq_base; + } else { + virq = irq_domain_alloc_descs(irq_base, nr_irqs, 0, node, + affinity); + if (virq < 0) { + pr_debug("cannot allocate IRQ(base %d, count %d)\n", + irq_base, nr_irqs); + return virq; + } + } + + if (irq_domain_alloc_irq_data(domain, virq, nr_irqs)) { + pr_debug("cannot allocate memory for IRQ%d\n", virq); + ret = -ENOMEM; + goto out_free_desc; + } + + ret = irq_domain_alloc_irqs_hierarchy(domain, virq, nr_irqs, arg); + if (ret < 0) + goto out_free_irq_data; + + for (i = 0; i < nr_irqs; i++) { + ret = irq_domain_trim_hierarchy(virq + i); + if (ret) + goto out_free_irq_data; + } + + for (i = 0; i < nr_irqs; i++) + irq_domain_insert_irq(virq + i); + + return virq; + +out_free_irq_data: + irq_domain_free_irq_data(virq, nr_irqs); +out_free_desc: + irq_free_descs(virq, nr_irqs); + return ret; +} + /** * __irq_domain_alloc_irqs - Allocate IRQs from domain * @domain: domain to allocate from @@ -1467,7 +1513,7 @@ int __irq_domain_alloc_irqs(struct irq_domain *domain, int irq_base, unsigned int nr_irqs, int node, void *arg, bool realloc, const struct irq_affinity_desc *affinity) { - int i, ret, virq; + int ret; if (domain == NULL) { domain = irq_default_domain; @@ -1475,49 +1521,11 @@ int __irq_domain_alloc_irqs(struct irq_domain *domain, int irq_base, return -EINVAL; } - if (realloc && irq_base >= 0) { - virq = irq_base; - } else { - virq = irq_domain_alloc_descs(irq_base, nr_irqs, 0, node, - affinity); - if (virq < 0) { - pr_debug("cannot allocate IRQ(base %d, count %d)\n", - irq_base, nr_irqs); - return virq; - } - } - - if (irq_domain_alloc_irq_data(domain, virq, nr_irqs)) { - pr_debug("cannot allocate memory for IRQ%d\n", virq); - ret = -ENOMEM; - goto out_free_desc; - } - mutex_lock(&irq_domain_mutex); - ret = irq_domain_alloc_irqs_hierarchy(domain, virq, nr_irqs, arg); - if (ret < 0) { - mutex_unlock(&irq_domain_mutex); - goto out_free_irq_data; - } - - for (i = 0; i < nr_irqs; i++) { - ret = irq_domain_trim_hierarchy(virq + i); - if (ret) { - mutex_unlock(&irq_domain_mutex); - goto out_free_irq_data; - } - } - - for (i = 0; i < nr_irqs; i++) - irq_domain_insert_irq(virq + i); + ret = irq_domain_alloc_irqs_locked(domain, irq_base, nr_irqs, node, arg, + realloc, affinity); mutex_unlock(&irq_domain_mutex); - return virq; - -out_free_irq_data: - irq_domain_free_irq_data(virq, nr_irqs); -out_free_desc: - irq_free_descs(virq, nr_irqs); return ret; } EXPORT_SYMBOL_GPL(__irq_domain_alloc_irqs); From 601363cc08da25747feb87c55573dd54de91d66a Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Mon, 13 Feb 2023 11:42:48 +0100 Subject: [PATCH 216/239] irqdomain: Fix mapping-creation race Parallel probing of devices that share interrupts (e.g. when a driver uses asynchronous probing) can currently result in two mappings for the same hardware interrupt to be created due to missing serialisation. Make sure to hold the irq_domain_mutex when creating mappings so that looking for an existing mapping before creating a new one is done atomically. Fixes: 765230b5f084 ("driver-core: add asynchronous probing support for drivers") Fixes: b62b2cf5759b ("irqdomain: Fix handling of type settings for existing mappings") Link: https://lore.kernel.org/r/YuJXMHoT4ijUxnRb@hovoldconsulting.com Cc: stable@vger.kernel.org # 4.8 Cc: Dmitry Torokhov Cc: Jon Hunter Tested-by: Hsin-Yi Wang Tested-by: Mark-PK Tsai Signed-off-by: Johan Hovold Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230213104302.17307-7-johan+linaro@kernel.org --- kernel/irq/irqdomain.c | 64 ++++++++++++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 18 deletions(-) diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index 78fb4800c0d2..df0cbad1b0d7 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -25,6 +25,9 @@ static DEFINE_MUTEX(irq_domain_mutex); static struct irq_domain *irq_default_domain; +static int irq_domain_alloc_irqs_locked(struct irq_domain *domain, int irq_base, + unsigned int nr_irqs, int node, void *arg, + bool realloc, const struct irq_affinity_desc *affinity); static void irq_domain_check_hierarchy(struct irq_domain *domain); struct irqchip_fwid { @@ -682,9 +685,9 @@ unsigned int irq_create_direct_mapping(struct irq_domain *domain) EXPORT_SYMBOL_GPL(irq_create_direct_mapping); #endif -static unsigned int __irq_create_mapping_affinity(struct irq_domain *domain, - irq_hw_number_t hwirq, - const struct irq_affinity_desc *affinity) +static unsigned int irq_create_mapping_affinity_locked(struct irq_domain *domain, + irq_hw_number_t hwirq, + const struct irq_affinity_desc *affinity) { struct device_node *of_node = irq_domain_get_of_node(domain); int virq; @@ -699,7 +702,7 @@ static unsigned int __irq_create_mapping_affinity(struct irq_domain *domain, return 0; } - if (irq_domain_associate(domain, virq, hwirq)) { + if (irq_domain_associate_locked(domain, virq, hwirq)) { irq_free_desc(virq); return 0; } @@ -735,14 +738,20 @@ unsigned int irq_create_mapping_affinity(struct irq_domain *domain, return 0; } + mutex_lock(&irq_domain_mutex); + /* Check if mapping already exists */ virq = irq_find_mapping(domain, hwirq); if (virq) { pr_debug("existing mapping on virq %d\n", virq); - return virq; + goto out; } - return __irq_create_mapping_affinity(domain, hwirq, affinity); + virq = irq_create_mapping_affinity_locked(domain, hwirq, affinity); +out: + mutex_unlock(&irq_domain_mutex); + + return virq; } EXPORT_SYMBOL_GPL(irq_create_mapping_affinity); @@ -809,6 +818,8 @@ unsigned int irq_create_fwspec_mapping(struct irq_fwspec *fwspec) if (WARN_ON(type & ~IRQ_TYPE_SENSE_MASK)) type &= IRQ_TYPE_SENSE_MASK; + mutex_lock(&irq_domain_mutex); + /* * If we've already configured this interrupt, * don't do it again, or hell will break loose. @@ -821,7 +832,7 @@ unsigned int irq_create_fwspec_mapping(struct irq_fwspec *fwspec) * interrupt number. */ if (type == IRQ_TYPE_NONE || type == irq_get_trigger_type(virq)) - return virq; + goto out; /* * If the trigger type has not been set yet, then set @@ -829,35 +840,45 @@ unsigned int irq_create_fwspec_mapping(struct irq_fwspec *fwspec) */ if (irq_get_trigger_type(virq) == IRQ_TYPE_NONE) { irq_data = irq_get_irq_data(virq); - if (!irq_data) - return 0; + if (!irq_data) { + virq = 0; + goto out; + } irqd_set_trigger_type(irq_data, type); - return virq; + goto out; } pr_warn("type mismatch, failed to map hwirq-%lu for %s!\n", hwirq, of_node_full_name(to_of_node(fwspec->fwnode))); - return 0; + virq = 0; + goto out; } if (irq_domain_is_hierarchy(domain)) { - virq = irq_domain_alloc_irqs(domain, 1, NUMA_NO_NODE, fwspec); - if (virq <= 0) - return 0; + virq = irq_domain_alloc_irqs_locked(domain, -1, 1, NUMA_NO_NODE, + fwspec, false, NULL); + if (virq <= 0) { + virq = 0; + goto out; + } } else { /* Create mapping */ - virq = __irq_create_mapping_affinity(domain, hwirq, NULL); + virq = irq_create_mapping_affinity_locked(domain, hwirq, NULL); if (!virq) - return virq; + goto out; } irq_data = irq_get_irq_data(virq); - if (WARN_ON(!irq_data)) - return 0; + if (WARN_ON(!irq_data)) { + virq = 0; + goto out; + } /* Store trigger type */ irqd_set_trigger_type(irq_data, type); +out: + mutex_unlock(&irq_domain_mutex); return virq; } @@ -1888,6 +1909,13 @@ void irq_domain_set_info(struct irq_domain *domain, unsigned int virq, irq_set_handler_data(virq, handler_data); } +static int irq_domain_alloc_irqs_locked(struct irq_domain *domain, int irq_base, + unsigned int nr_irqs, int node, void *arg, + bool realloc, const struct irq_affinity_desc *affinity) +{ + return -EINVAL; +} + static void irq_domain_check_hierarchy(struct irq_domain *domain) { } From 8932c32c3053accd50702b36e944ac2016cd103c Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 13 Feb 2023 11:42:49 +0100 Subject: [PATCH 217/239] irqdomain: Fix domain registration race Hierarchical domains created using irq_domain_create_hierarchy() are currently added to the domain list before having been fully initialised. This specifically means that a racing allocation request might fail to allocate irq data for the inner domains of a hierarchy in case the parent domain pointer has not yet been set up. Note that this is not really any issue for irqchip drivers that are registered early (e.g. via IRQCHIP_DECLARE() or IRQCHIP_ACPI_DECLARE()) but could potentially cause trouble with drivers that are registered later (e.g. modular drivers using IRQCHIP_PLATFORM_DRIVER_BEGIN(), gpiochip drivers, etc.). Fixes: afb7da83b9f4 ("irqdomain: Introduce helper function irq_domain_add_hierarchy()") Cc: stable@vger.kernel.org # 3.19 Signed-off-by: Marc Zyngier [ johan: add commit message ] Signed-off-by: Johan Hovold Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230213104302.17307-8-johan+linaro@kernel.org --- kernel/irq/irqdomain.c | 62 +++++++++++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index df0cbad1b0d7..a6d1b108b8f7 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -126,23 +126,12 @@ void irq_domain_free_fwnode(struct fwnode_handle *fwnode) } EXPORT_SYMBOL_GPL(irq_domain_free_fwnode); -/** - * __irq_domain_add() - Allocate a new irq_domain data structure - * @fwnode: firmware node for the interrupt controller - * @size: Size of linear map; 0 for radix mapping only - * @hwirq_max: Maximum number of interrupts supported by controller - * @direct_max: Maximum value of direct maps; Use ~0 for no limit; 0 for no - * direct mapping - * @ops: domain callbacks - * @host_data: Controller private data pointer - * - * Allocates and initializes an irq_domain structure. - * Returns pointer to IRQ domain, or NULL on failure. - */ -struct irq_domain *__irq_domain_add(struct fwnode_handle *fwnode, unsigned int size, - irq_hw_number_t hwirq_max, int direct_max, - const struct irq_domain_ops *ops, - void *host_data) +static struct irq_domain *__irq_domain_create(struct fwnode_handle *fwnode, + unsigned int size, + irq_hw_number_t hwirq_max, + int direct_max, + const struct irq_domain_ops *ops, + void *host_data) { struct irqchip_fwid *fwid; struct irq_domain *domain; @@ -230,12 +219,44 @@ struct irq_domain *__irq_domain_add(struct fwnode_handle *fwnode, unsigned int s irq_domain_check_hierarchy(domain); + return domain; +} + +static void __irq_domain_publish(struct irq_domain *domain) +{ mutex_lock(&irq_domain_mutex); debugfs_add_domain_dir(domain); list_add(&domain->link, &irq_domain_list); mutex_unlock(&irq_domain_mutex); pr_debug("Added domain %s\n", domain->name); +} + +/** + * __irq_domain_add() - Allocate a new irq_domain data structure + * @fwnode: firmware node for the interrupt controller + * @size: Size of linear map; 0 for radix mapping only + * @hwirq_max: Maximum number of interrupts supported by controller + * @direct_max: Maximum value of direct maps; Use ~0 for no limit; 0 for no + * direct mapping + * @ops: domain callbacks + * @host_data: Controller private data pointer + * + * Allocates and initializes an irq_domain structure. + * Returns pointer to IRQ domain, or NULL on failure. + */ +struct irq_domain *__irq_domain_add(struct fwnode_handle *fwnode, unsigned int size, + irq_hw_number_t hwirq_max, int direct_max, + const struct irq_domain_ops *ops, + void *host_data) +{ + struct irq_domain *domain; + + domain = __irq_domain_create(fwnode, size, hwirq_max, direct_max, + ops, host_data); + if (domain) + __irq_domain_publish(domain); + return domain; } EXPORT_SYMBOL_GPL(__irq_domain_add); @@ -1138,12 +1159,15 @@ struct irq_domain *irq_domain_create_hierarchy(struct irq_domain *parent, struct irq_domain *domain; if (size) - domain = irq_domain_create_linear(fwnode, size, ops, host_data); + domain = __irq_domain_create(fwnode, size, size, 0, ops, host_data); else - domain = irq_domain_create_tree(fwnode, ops, host_data); + domain = __irq_domain_create(fwnode, 0, ~0, 0, ops, host_data); + if (domain) { domain->parent = parent; domain->flags |= flags; + + __irq_domain_publish(domain); } return domain; From 47d1932f37de99bae3345bb93f098ac8750ab0fb Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Mon, 13 Feb 2023 11:42:50 +0100 Subject: [PATCH 218/239] irqdomain: Drop revmap mutex The revmap mutex is essentially only used to maintain the integrity of the radix tree during updates (lookups use RCU). As the global irq_domain_mutex is now held in all paths that update the revmap structures there is strictly no longer any need for the dedicated mutex, which can be removed. Drop the revmap mutex and add lockdep assertions to the revmap helpers to make sure that the global lock is always held when updating the revmap. Tested-by: Hsin-Yi Wang Tested-by: Mark-PK Tsai Signed-off-by: Johan Hovold Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230213104302.17307-9-johan+linaro@kernel.org --- include/linux/irqdomain.h | 2 -- kernel/irq/irqdomain.c | 13 ++++++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/include/linux/irqdomain.h b/include/linux/irqdomain.h index a372086750ca..16399de00b48 100644 --- a/include/linux/irqdomain.h +++ b/include/linux/irqdomain.h @@ -143,7 +143,6 @@ struct irq_domain_chip_generic; * Revmap data, used internally by the irq domain code: * @revmap_size: Size of the linear map table @revmap[] * @revmap_tree: Radix map tree for hwirqs that don't fit in the linear map - * @revmap_mutex: Lock for the revmap * @revmap: Linear table of irq_data pointers */ struct irq_domain { @@ -171,7 +170,6 @@ struct irq_domain { irq_hw_number_t hwirq_max; unsigned int revmap_size; struct radix_tree_root revmap_tree; - struct mutex revmap_mutex; struct irq_data __rcu *revmap[]; }; diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index a6d1b108b8f7..c7113e776543 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -206,7 +206,6 @@ static struct irq_domain *__irq_domain_create(struct fwnode_handle *fwnode, /* Fill structure */ INIT_RADIX_TREE(&domain->revmap_tree, GFP_KERNEL); - mutex_init(&domain->revmap_mutex); domain->ops = ops; domain->host_data = host_data; domain->hwirq_max = hwirq_max; @@ -526,30 +525,30 @@ static bool irq_domain_is_nomap(struct irq_domain *domain) static void irq_domain_clear_mapping(struct irq_domain *domain, irq_hw_number_t hwirq) { + lockdep_assert_held(&irq_domain_mutex); + if (irq_domain_is_nomap(domain)) return; - mutex_lock(&domain->revmap_mutex); if (hwirq < domain->revmap_size) rcu_assign_pointer(domain->revmap[hwirq], NULL); else radix_tree_delete(&domain->revmap_tree, hwirq); - mutex_unlock(&domain->revmap_mutex); } static void irq_domain_set_mapping(struct irq_domain *domain, irq_hw_number_t hwirq, struct irq_data *irq_data) { + lockdep_assert_held(&irq_domain_mutex); + if (irq_domain_is_nomap(domain)) return; - mutex_lock(&domain->revmap_mutex); if (hwirq < domain->revmap_size) rcu_assign_pointer(domain->revmap[hwirq], irq_data); else radix_tree_insert(&domain->revmap_tree, hwirq, irq_data); - mutex_unlock(&domain->revmap_mutex); } static void irq_domain_disassociate(struct irq_domain *domain, unsigned int irq) @@ -1580,11 +1579,12 @@ static void irq_domain_fix_revmap(struct irq_data *d) { void __rcu **slot; + lockdep_assert_held(&irq_domain_mutex); + if (irq_domain_is_nomap(d->domain)) return; /* Fix up the revmap. */ - mutex_lock(&d->domain->revmap_mutex); if (d->hwirq < d->domain->revmap_size) { /* Not using radix tree */ rcu_assign_pointer(d->domain->revmap[d->hwirq], d); @@ -1593,7 +1593,6 @@ static void irq_domain_fix_revmap(struct irq_data *d) if (slot) radix_tree_replace_slot(&d->domain->revmap_tree, slot, d); } - mutex_unlock(&d->domain->revmap_mutex); } /** From 28a9ff23d8b56db09cb01cef174a205ea5e2ca49 Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Mon, 13 Feb 2023 11:42:51 +0100 Subject: [PATCH 219/239] irqdomain: Drop dead domain-name assignment Since commit d59f6617eef0 ("genirq: Allow fwnode to carry name information only") an IRQ domain is always given a name during allocation (e.g. used for the debugfs entry). Drop the leftover name assignment when allocating the first IRQ. Tested-by: Hsin-Yi Wang Tested-by: Mark-PK Tsai Signed-off-by: Johan Hovold Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230213104302.17307-10-johan+linaro@kernel.org --- kernel/irq/irqdomain.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index c7113e776543..6bd6b610568c 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -619,10 +619,6 @@ static int irq_domain_associate_locked(struct irq_domain *domain, unsigned int v irq_data->hwirq = 0; return ret; } - - /* If not already assigned, give the domain the chip's name */ - if (!domain->name && irq_data->chip) - domain->name = irq_data->chip->name; } domain->mapcount++; @@ -1182,10 +1178,6 @@ static void irq_domain_insert_irq(int virq) domain->mapcount++; irq_domain_set_mapping(domain, data->hwirq, data); - - /* If not already assigned, give the domain the chip's name */ - if (!domain->name && data->chip) - domain->name = data->chip->name; } irq_clear_status_flags(virq, IRQ_NOREQUEST); From 4e0d86df9344bfd1951eb2571e4ef8f3d37000a4 Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Mon, 13 Feb 2023 11:42:52 +0100 Subject: [PATCH 220/239] irqdomain: Drop leftover brackets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop some unnecessary brackets that were left in place when the corresponding code was updated. Reviewed-by: Philippe Mathieu-Daudé Tested-by: Hsin-Yi Wang Tested-by: Mark-PK Tsai Signed-off-by: Johan Hovold Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230213104302.17307-11-johan+linaro@kernel.org --- kernel/irq/irqdomain.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index 6bd6b610568c..3a3213d730ee 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -210,9 +210,8 @@ static struct irq_domain *__irq_domain_create(struct fwnode_handle *fwnode, domain->host_data = host_data; domain->hwirq_max = hwirq_max; - if (direct_max) { + if (direct_max) domain->flags |= IRQ_DOMAIN_FLAG_NO_MAP; - } domain->revmap_size = size; @@ -652,9 +651,8 @@ void irq_domain_associate_many(struct irq_domain *domain, unsigned int irq_base, pr_debug("%s(%s, irqbase=%i, hwbase=%i, count=%i)\n", __func__, of_node_full_name(of_node), irq_base, (int)hwirq_base, count); - for (i = 0; i < count; i++) { + for (i = 0; i < count; i++) irq_domain_associate(domain, irq_base + i, hwirq_base + i); - } } EXPORT_SYMBOL_GPL(irq_domain_associate_many); From 930a1bbbef01cdcd682d9c2b4bc9e36b9618fed3 Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Mon, 13 Feb 2023 11:42:53 +0100 Subject: [PATCH 221/239] irqdomain: Clean up irq_domain_push/pop_irq() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The irq_domain_push_irq() interface is used to add a new (outmost) level to a hierarchical domain after IRQs have been allocated. Possibly due to differing mental images of hierarchical domains, the names used for the irq_data variables make these functions much harder to understand than what they need to be. Rename the struct irq_data pointer to the data embedded in the descriptor as simply 'irq_data' and refer to the data allocated by this interface as 'parent_irq_data' so that the names reflect how hierarchical domains are implemented. Reviewed-by: Philippe Mathieu-Daudé Tested-by: Hsin-Yi Wang Tested-by: Mark-PK Tsai Signed-off-by: Johan Hovold Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230213104302.17307-12-johan+linaro@kernel.org --- kernel/irq/irqdomain.c | 65 +++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index 3a3213d730ee..6d480dc6ab53 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -1598,8 +1598,8 @@ static void irq_domain_fix_revmap(struct irq_data *d) */ int irq_domain_push_irq(struct irq_domain *domain, int virq, void *arg) { - struct irq_data *child_irq_data; - struct irq_data *root_irq_data = irq_get_irq_data(virq); + struct irq_data *irq_data = irq_get_irq_data(virq); + struct irq_data *parent_irq_data; struct irq_desc *desc; int rv = 0; @@ -1624,45 +1624,44 @@ int irq_domain_push_irq(struct irq_domain *domain, int virq, void *arg) if (WARN_ON(!irq_domain_is_hierarchy(domain))) return -EINVAL; - if (!root_irq_data) + if (!irq_data) return -EINVAL; - if (domain->parent != root_irq_data->domain) + if (domain->parent != irq_data->domain) return -EINVAL; - child_irq_data = kzalloc_node(sizeof(*child_irq_data), GFP_KERNEL, - irq_data_get_node(root_irq_data)); - if (!child_irq_data) + parent_irq_data = kzalloc_node(sizeof(*parent_irq_data), GFP_KERNEL, + irq_data_get_node(irq_data)); + if (!parent_irq_data) return -ENOMEM; mutex_lock(&irq_domain_mutex); /* Copy the original irq_data. */ - *child_irq_data = *root_irq_data; + *parent_irq_data = *irq_data; /* - * Overwrite the root_irq_data, which is embedded in struct - * irq_desc, with values for this domain. + * Overwrite the irq_data, which is embedded in struct irq_desc, with + * values for this domain. */ - root_irq_data->parent_data = child_irq_data; - root_irq_data->domain = domain; - root_irq_data->mask = 0; - root_irq_data->hwirq = 0; - root_irq_data->chip = NULL; - root_irq_data->chip_data = NULL; + irq_data->parent_data = parent_irq_data; + irq_data->domain = domain; + irq_data->mask = 0; + irq_data->hwirq = 0; + irq_data->chip = NULL; + irq_data->chip_data = NULL; /* May (probably does) set hwirq, chip, etc. */ rv = irq_domain_alloc_irqs_hierarchy(domain, virq, 1, arg); if (rv) { /* Restore the original irq_data. */ - *root_irq_data = *child_irq_data; - kfree(child_irq_data); + *irq_data = *parent_irq_data; + kfree(parent_irq_data); goto error; } - irq_domain_fix_revmap(child_irq_data); - irq_domain_set_mapping(domain, root_irq_data->hwirq, root_irq_data); - + irq_domain_fix_revmap(parent_irq_data); + irq_domain_set_mapping(domain, irq_data->hwirq, irq_data); error: mutex_unlock(&irq_domain_mutex); @@ -1680,8 +1679,8 @@ EXPORT_SYMBOL_GPL(irq_domain_push_irq); */ int irq_domain_pop_irq(struct irq_domain *domain, int virq) { - struct irq_data *root_irq_data = irq_get_irq_data(virq); - struct irq_data *child_irq_data; + struct irq_data *irq_data = irq_get_irq_data(virq); + struct irq_data *parent_irq_data; struct irq_data *tmp_irq_data; struct irq_desc *desc; @@ -1703,37 +1702,37 @@ int irq_domain_pop_irq(struct irq_domain *domain, int virq) if (domain == NULL) return -EINVAL; - if (!root_irq_data) + if (!irq_data) return -EINVAL; tmp_irq_data = irq_domain_get_irq_data(domain, virq); /* We can only "pop" if this domain is at the top of the list */ - if (WARN_ON(root_irq_data != tmp_irq_data)) + if (WARN_ON(irq_data != tmp_irq_data)) return -EINVAL; - if (WARN_ON(root_irq_data->domain != domain)) + if (WARN_ON(irq_data->domain != domain)) return -EINVAL; - child_irq_data = root_irq_data->parent_data; - if (WARN_ON(!child_irq_data)) + parent_irq_data = irq_data->parent_data; + if (WARN_ON(!parent_irq_data)) return -EINVAL; mutex_lock(&irq_domain_mutex); - root_irq_data->parent_data = NULL; + irq_data->parent_data = NULL; - irq_domain_clear_mapping(domain, root_irq_data->hwirq); + irq_domain_clear_mapping(domain, irq_data->hwirq); irq_domain_free_irqs_hierarchy(domain, virq, 1); /* Restore the original irq_data. */ - *root_irq_data = *child_irq_data; + *irq_data = *parent_irq_data; - irq_domain_fix_revmap(root_irq_data); + irq_domain_fix_revmap(irq_data); mutex_unlock(&irq_domain_mutex); - kfree(child_irq_data); + kfree(parent_irq_data); return 0; } From bc1bc1b309b42ff3faea957df7ac9382366c5eef Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Mon, 13 Feb 2023 11:42:54 +0100 Subject: [PATCH 222/239] x86/ioapic: Use irq_domain_create_hierarchy() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the irq_domain_create_hierarchy() helper to create the hierarchical domain, which both serves as documentation and avoids poking at irqdomain internals. Reviewed-by: Philippe Mathieu-Daudé Tested-by: Hsin-Yi Wang Tested-by: Mark-PK Tsai Signed-off-by: Johan Hovold Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230213104302.17307-13-johan+linaro@kernel.org --- arch/x86/kernel/apic/io_apic.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index a868b76cd3d4..1f83b052bb74 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -2364,9 +2364,8 @@ static int mp_irqdomain_create(int ioapic) return -ENODEV; } - ip->irqdomain = irq_domain_create_linear(fn, hwirqs, cfg->ops, - (void *)(long)ioapic); - + ip->irqdomain = irq_domain_create_hierarchy(parent, 0, hwirqs, fn, cfg->ops, + (void *)(long)ioapic); if (!ip->irqdomain) { /* Release fw handle if it was allocated above */ if (!cfg->dev) @@ -2374,8 +2373,6 @@ static int mp_irqdomain_create(int ioapic) return -ENOMEM; } - ip->irqdomain->parent = parent; - if (cfg->type == IOAPIC_DOMAIN_LEGACY || cfg->type == IOAPIC_DOMAIN_STRICT) ioapic_dynirq_base = max(ioapic_dynirq_base, From a14e7fdd43040a2a5da2cc979063b3c958015aa9 Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Mon, 13 Feb 2023 11:42:55 +0100 Subject: [PATCH 223/239] x86/uv: Use irq_domain_create_hierarchy() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the irq_domain_create_hierarchy() helper to create the hierarchical domain, which both serves as documentation and avoids poking at irqdomain internals. Reviewed-by: Philippe Mathieu-Daudé Tested-by: Hsin-Yi Wang Tested-by: Mark-PK Tsai Signed-off-by: Johan Hovold Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230213104302.17307-14-johan+linaro@kernel.org --- arch/x86/platform/uv/uv_irq.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/arch/x86/platform/uv/uv_irq.c b/arch/x86/platform/uv/uv_irq.c index 1a536a187d74..ee21d6a36a80 100644 --- a/arch/x86/platform/uv/uv_irq.c +++ b/arch/x86/platform/uv/uv_irq.c @@ -166,10 +166,9 @@ static struct irq_domain *uv_get_irq_domain(void) if (!fn) goto out; - uv_domain = irq_domain_create_tree(fn, &uv_domain_ops, NULL); - if (uv_domain) - uv_domain->parent = x86_vector_domain; - else + uv_domain = irq_domain_create_hierarchy(x86_vector_domain, 0, 0, fn, + &uv_domain_ops, NULL); + if (!uv_domain) irq_domain_free_fwnode(fn); out: mutex_unlock(&uv_lock); From 6c889231e04dffa4b668c1f0fbc26db679600147 Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Mon, 13 Feb 2023 11:42:56 +0100 Subject: [PATCH 224/239] irqchip/alpine-msi: Use irq_domain_add_hierarchy() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the irq_domain_add_hierarchy() helper to create the hierarchical domain, which both serves as documentation and avoids poking at irqdomain internals. Reviewed-by: Philippe Mathieu-Daudé Tested-by: Hsin-Yi Wang Tested-by: Mark-PK Tsai Signed-off-by: Johan Hovold Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230213104302.17307-15-johan+linaro@kernel.org --- drivers/irqchip/irq-alpine-msi.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/drivers/irqchip/irq-alpine-msi.c b/drivers/irqchip/irq-alpine-msi.c index 5ddb8e578ac6..604459372fdd 100644 --- a/drivers/irqchip/irq-alpine-msi.c +++ b/drivers/irqchip/irq-alpine-msi.c @@ -204,16 +204,14 @@ static int alpine_msix_init_domains(struct alpine_msix_data *priv, return -ENXIO; } - middle_domain = irq_domain_add_tree(NULL, - &alpine_msix_middle_domain_ops, - priv); + middle_domain = irq_domain_add_hierarchy(gic_domain, 0, 0, NULL, + &alpine_msix_middle_domain_ops, + priv); if (!middle_domain) { pr_err("Failed to create the MSIX middle domain\n"); return -ENOMEM; } - middle_domain->parent = gic_domain; - msi_domain = pci_msi_create_irq_domain(of_node_to_fwnode(node), &alpine_msix_domain_info, middle_domain); From e6e8cd62a56f9ed0dcdf8d01147709b59a111418 Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Mon, 13 Feb 2023 11:42:57 +0100 Subject: [PATCH 225/239] irqchip/gic-v2m: Use irq_domain_create_hierarchy() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the irq_domain_create_hierarchy() helper to create the hierarchical domain, which both serves as documentation and avoids poking at irqdomain internals. Reviewed-by: Philippe Mathieu-Daudé Tested-by: Hsin-Yi Wang Tested-by: Mark-PK Tsai Signed-off-by: Johan Hovold Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230213104302.17307-16-johan+linaro@kernel.org --- drivers/irqchip/irq-gic-v2m.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/irqchip/irq-gic-v2m.c b/drivers/irqchip/irq-gic-v2m.c index f4d7eeb13951..f1e75b35a52a 100644 --- a/drivers/irqchip/irq-gic-v2m.c +++ b/drivers/irqchip/irq-gic-v2m.c @@ -287,15 +287,14 @@ static __init int gicv2m_allocate_domains(struct irq_domain *parent) if (!v2m) return 0; - inner_domain = irq_domain_create_tree(v2m->fwnode, - &gicv2m_domain_ops, v2m); + inner_domain = irq_domain_create_hierarchy(parent, 0, 0, v2m->fwnode, + &gicv2m_domain_ops, v2m); if (!inner_domain) { pr_err("Failed to create GICv2m domain\n"); return -ENOMEM; } irq_domain_update_bus_token(inner_domain, DOMAIN_BUS_NEXUS); - inner_domain->parent = parent; pci_domain = pci_msi_create_irq_domain(v2m->fwnode, &gicv2m_msi_domain_info, inner_domain); From 1e46e040decede1723a5b11f0689942134c29d9a Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Mon, 13 Feb 2023 11:42:58 +0100 Subject: [PATCH 226/239] irqchip/gic-v3-its: Use irq_domain_create_hierarchy() Use the irq_domain_create_hierarchy() helper to create the hierarchical domain, which both serves as documentation and avoids poking at irqdomain internals. Note that the domain host_data was first set to the struct its_node during allocation only to immediately be overwritten with the struct msi_domain_info. Tested-by: Hsin-Yi Wang Tested-by: Mark-PK Tsai Signed-off-by: Johan Hovold Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230213104302.17307-17-johan+linaro@kernel.org --- drivers/irqchip/irq-gic-v3-its.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 973ede0197e3..5634d29b644d 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -4909,18 +4909,19 @@ static int its_init_domain(struct fwnode_handle *handle, struct its_node *its) if (!info) return -ENOMEM; - inner_domain = irq_domain_create_tree(handle, &its_domain_ops, its); + info->ops = &its_msi_domain_ops; + info->data = its; + + inner_domain = irq_domain_create_hierarchy(its_parent, + its->msi_domain_flags, 0, + handle, &its_domain_ops, + info); if (!inner_domain) { kfree(info); return -ENOMEM; } - inner_domain->parent = its_parent; irq_domain_update_bus_token(inner_domain, DOMAIN_BUS_NEXUS); - inner_domain->flags |= its->msi_domain_flags; - info->ops = &its_msi_domain_ops; - info->data = its; - inner_domain->host_data = info; return 0; } From 331f9aac03267f76a15776260eda4f47b81731f7 Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Mon, 13 Feb 2023 11:42:59 +0100 Subject: [PATCH 227/239] irqchip/gic-v3-mbi: Use irq_domain_create_hierarchy() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the irq_domain_create_hierarchy() helper to create the hierarchical domain, which both serves as documentation and avoids poking at irqdomain internals. Reviewed-by: Philippe Mathieu-Daudé Tested-by: Hsin-Yi Wang Tested-by: Mark-PK Tsai Signed-off-by: Johan Hovold Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230213104302.17307-18-johan+linaro@kernel.org --- drivers/irqchip/irq-gic-v3-mbi.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-mbi.c b/drivers/irqchip/irq-gic-v3-mbi.c index e1efdec9e9ac..dbb8b1efda44 100644 --- a/drivers/irqchip/irq-gic-v3-mbi.c +++ b/drivers/irqchip/irq-gic-v3-mbi.c @@ -233,13 +233,12 @@ static int mbi_allocate_domains(struct irq_domain *parent) struct irq_domain *nexus_domain, *pci_domain, *plat_domain; int err; - nexus_domain = irq_domain_create_tree(parent->fwnode, - &mbi_domain_ops, NULL); + nexus_domain = irq_domain_create_hierarchy(parent, 0, 0, parent->fwnode, + &mbi_domain_ops, NULL); if (!nexus_domain) return -ENOMEM; irq_domain_update_bus_token(nexus_domain, DOMAIN_BUS_NEXUS); - nexus_domain->parent = parent; err = mbi_allocate_pci_domain(nexus_domain, &pci_domain); From 6159c470f812eab0a2f1900c70acbb3ca7b9e14a Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Mon, 13 Feb 2023 11:43:00 +0100 Subject: [PATCH 228/239] irqchip/loongson-pch-msi: Use irq_domain_create_hierarchy() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the irq_domain_create_hierarchy() helper to create the hierarchical domain, which both serves as documentation and avoids poking at irqdomain internals. Reviewed-by: Philippe Mathieu-Daudé Tested-by: Hsin-Yi Wang Tested-by: Mark-PK Tsai Signed-off-by: Johan Hovold Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230213104302.17307-19-johan+linaro@kernel.org --- drivers/irqchip/irq-loongson-pch-msi.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/drivers/irqchip/irq-loongson-pch-msi.c b/drivers/irqchip/irq-loongson-pch-msi.c index a72ede90ffc6..6e1e1f011bb2 100644 --- a/drivers/irqchip/irq-loongson-pch-msi.c +++ b/drivers/irqchip/irq-loongson-pch-msi.c @@ -163,16 +163,15 @@ static int pch_msi_init_domains(struct pch_msi_data *priv, { struct irq_domain *middle_domain, *msi_domain; - middle_domain = irq_domain_create_linear(domain_handle, - priv->num_irqs, - &pch_msi_middle_domain_ops, - priv); + middle_domain = irq_domain_create_hierarchy(parent, 0, priv->num_irqs, + domain_handle, + &pch_msi_middle_domain_ops, + priv); if (!middle_domain) { pr_err("Failed to create the MSI middle domain\n"); return -ENOMEM; } - middle_domain->parent = parent; irq_domain_update_bus_token(middle_domain, DOMAIN_BUS_NEXUS); msi_domain = pci_msi_create_irq_domain(domain_handle, From f743f54fa8d2bcb3f2891b783687d91b76a144f5 Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Mon, 13 Feb 2023 11:43:01 +0100 Subject: [PATCH 229/239] irqchip/mvebu-odmi: Use irq_domain_create_hierarchy() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the irq_domain_create_hierarchy() helper to create the hierarchical domain, which both serves as documentation and avoids poking at irqdomain internals. Reviewed-by: Philippe Mathieu-Daudé Tested-by: Hsin-Yi Wang Tested-by: Mark-PK Tsai Signed-off-by: Johan Hovold Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230213104302.17307-20-johan+linaro@kernel.org --- drivers/irqchip/irq-mvebu-odmi.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/irqchip/irq-mvebu-odmi.c b/drivers/irqchip/irq-mvebu-odmi.c index dc4145abdd6f..108091533e10 100644 --- a/drivers/irqchip/irq-mvebu-odmi.c +++ b/drivers/irqchip/irq-mvebu-odmi.c @@ -161,7 +161,7 @@ static struct msi_domain_info odmi_msi_domain_info = { static int __init mvebu_odmi_init(struct device_node *node, struct device_node *parent) { - struct irq_domain *inner_domain, *plat_domain; + struct irq_domain *parent_domain, *inner_domain, *plat_domain; int ret, i; if (of_property_read_u32(node, "marvell,odmi-frames", &odmis_count)) @@ -197,16 +197,17 @@ static int __init mvebu_odmi_init(struct device_node *node, } } - inner_domain = irq_domain_create_linear(of_node_to_fwnode(node), - odmis_count * NODMIS_PER_FRAME, - &odmi_domain_ops, NULL); + parent_domain = irq_find_host(parent); + + inner_domain = irq_domain_create_hierarchy(parent_domain, 0, + odmis_count * NODMIS_PER_FRAME, + of_node_to_fwnode(node), + &odmi_domain_ops, NULL); if (!inner_domain) { ret = -ENOMEM; goto err_unmap; } - inner_domain->parent = irq_find_host(parent); - plat_domain = platform_msi_create_irq_domain(of_node_to_fwnode(node), &odmi_msi_domain_info, inner_domain); From 9dbb8e3452aba34e6fa4f63054b3adc66aceb7ec Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Mon, 13 Feb 2023 11:43:02 +0100 Subject: [PATCH 230/239] irqdomain: Switch to per-domain locking The IRQ domain structures are currently protected by the global irq_domain_mutex. Switch to using more fine-grained per-domain locking, which can speed up parallel probing by reducing lock contention. On a recent arm64 laptop, the total time spent waiting for the locks during boot drops from 160 to 40 ms on average, while the maximum aggregate wait time drops from 550 to 90 ms over ten runs for example. Note that the domain lock of the root domain (innermost domain) must be used for hierarchical domains. For non-hierarchical domains (as for root domains), the new root pointer is set to the domain itself so that &domain->root->mutex always points to the right lock. Also note that hierarchical domains should be constructed using irq_domain_create_hierarchy() (or irq_domain_add_hierarchy()) to avoid having racing allocations access a not fully initialised domain. As a safeguard, the lockdep assertion in irq_domain_set_mapping() will catch any offenders that also fail to set the root domain pointer. Tested-by: Hsin-Yi Wang Tested-by: Mark-PK Tsai Signed-off-by: Johan Hovold Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230213104302.17307-21-johan+linaro@kernel.org --- include/linux/irqdomain.h | 4 +++ kernel/irq/irqdomain.c | 59 ++++++++++++++++++++++++++------------- 2 files changed, 43 insertions(+), 20 deletions(-) diff --git a/include/linux/irqdomain.h b/include/linux/irqdomain.h index 16399de00b48..d320d15d4fba 100644 --- a/include/linux/irqdomain.h +++ b/include/linux/irqdomain.h @@ -125,6 +125,8 @@ struct irq_domain_chip_generic; * core code. * @flags: Per irq_domain flags * @mapcount: The number of mapped interrupts + * @mutex: Domain lock, hierarchical domains use root domain's lock + * @root: Pointer to root domain, or containing structure if non-hierarchical * * Optional elements: * @fwnode: Pointer to firmware node associated with the irq_domain. Pretty easy @@ -152,6 +154,8 @@ struct irq_domain { void *host_data; unsigned int flags; unsigned int mapcount; + struct mutex mutex; + struct irq_domain *root; /* Optional data */ struct fwnode_handle *fwnode; diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index 6d480dc6ab53..1983f1beeec7 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -215,6 +215,17 @@ static struct irq_domain *__irq_domain_create(struct fwnode_handle *fwnode, domain->revmap_size = size; + /* + * Hierarchical domains use the domain lock of the root domain + * (innermost domain). + * + * For non-hierarchical domains (as for root domains), the root + * pointer is set to the domain itself so that &domain->root->mutex + * always points to the right lock. + */ + mutex_init(&domain->mutex); + domain->root = domain; + irq_domain_check_hierarchy(domain); return domain; @@ -524,7 +535,7 @@ static bool irq_domain_is_nomap(struct irq_domain *domain) static void irq_domain_clear_mapping(struct irq_domain *domain, irq_hw_number_t hwirq) { - lockdep_assert_held(&irq_domain_mutex); + lockdep_assert_held(&domain->root->mutex); if (irq_domain_is_nomap(domain)) return; @@ -539,7 +550,11 @@ static void irq_domain_set_mapping(struct irq_domain *domain, irq_hw_number_t hwirq, struct irq_data *irq_data) { - lockdep_assert_held(&irq_domain_mutex); + /* + * This also makes sure that all domains point to the same root when + * called from irq_domain_insert_irq() for each domain in a hierarchy. + */ + lockdep_assert_held(&domain->root->mutex); if (irq_domain_is_nomap(domain)) return; @@ -561,7 +576,7 @@ static void irq_domain_disassociate(struct irq_domain *domain, unsigned int irq) hwirq = irq_data->hwirq; - mutex_lock(&irq_domain_mutex); + mutex_lock(&domain->root->mutex); irq_set_status_flags(irq, IRQ_NOREQUEST); @@ -583,7 +598,7 @@ static void irq_domain_disassociate(struct irq_domain *domain, unsigned int irq) /* Clear reverse map for this hwirq */ irq_domain_clear_mapping(domain, hwirq); - mutex_unlock(&irq_domain_mutex); + mutex_unlock(&domain->root->mutex); } static int irq_domain_associate_locked(struct irq_domain *domain, unsigned int virq, @@ -633,9 +648,9 @@ int irq_domain_associate(struct irq_domain *domain, unsigned int virq, { int ret; - mutex_lock(&irq_domain_mutex); + mutex_lock(&domain->root->mutex); ret = irq_domain_associate_locked(domain, virq, hwirq); - mutex_unlock(&irq_domain_mutex); + mutex_unlock(&domain->root->mutex); return ret; } @@ -752,7 +767,7 @@ unsigned int irq_create_mapping_affinity(struct irq_domain *domain, return 0; } - mutex_lock(&irq_domain_mutex); + mutex_lock(&domain->root->mutex); /* Check if mapping already exists */ virq = irq_find_mapping(domain, hwirq); @@ -763,7 +778,7 @@ unsigned int irq_create_mapping_affinity(struct irq_domain *domain, virq = irq_create_mapping_affinity_locked(domain, hwirq, affinity); out: - mutex_unlock(&irq_domain_mutex); + mutex_unlock(&domain->root->mutex); return virq; } @@ -832,7 +847,7 @@ unsigned int irq_create_fwspec_mapping(struct irq_fwspec *fwspec) if (WARN_ON(type & ~IRQ_TYPE_SENSE_MASK)) type &= IRQ_TYPE_SENSE_MASK; - mutex_lock(&irq_domain_mutex); + mutex_lock(&domain->root->mutex); /* * If we've already configured this interrupt, @@ -892,7 +907,7 @@ unsigned int irq_create_fwspec_mapping(struct irq_fwspec *fwspec) /* Store trigger type */ irqd_set_trigger_type(irq_data, type); out: - mutex_unlock(&irq_domain_mutex); + mutex_unlock(&domain->root->mutex); return virq; } @@ -1157,6 +1172,7 @@ struct irq_domain *irq_domain_create_hierarchy(struct irq_domain *parent, domain = __irq_domain_create(fwnode, 0, ~0, 0, ops, host_data); if (domain) { + domain->root = parent->root; domain->parent = parent; domain->flags |= flags; @@ -1555,10 +1571,10 @@ int __irq_domain_alloc_irqs(struct irq_domain *domain, int irq_base, return -EINVAL; } - mutex_lock(&irq_domain_mutex); + mutex_lock(&domain->root->mutex); ret = irq_domain_alloc_irqs_locked(domain, irq_base, nr_irqs, node, arg, realloc, affinity); - mutex_unlock(&irq_domain_mutex); + mutex_unlock(&domain->root->mutex); return ret; } @@ -1569,7 +1585,7 @@ static void irq_domain_fix_revmap(struct irq_data *d) { void __rcu **slot; - lockdep_assert_held(&irq_domain_mutex); + lockdep_assert_held(&d->domain->root->mutex); if (irq_domain_is_nomap(d->domain)) return; @@ -1635,7 +1651,7 @@ int irq_domain_push_irq(struct irq_domain *domain, int virq, void *arg) if (!parent_irq_data) return -ENOMEM; - mutex_lock(&irq_domain_mutex); + mutex_lock(&domain->root->mutex); /* Copy the original irq_data. */ *parent_irq_data = *irq_data; @@ -1663,7 +1679,7 @@ int irq_domain_push_irq(struct irq_domain *domain, int virq, void *arg) irq_domain_fix_revmap(parent_irq_data); irq_domain_set_mapping(domain, irq_data->hwirq, irq_data); error: - mutex_unlock(&irq_domain_mutex); + mutex_unlock(&domain->root->mutex); return rv; } @@ -1718,7 +1734,7 @@ int irq_domain_pop_irq(struct irq_domain *domain, int virq) if (WARN_ON(!parent_irq_data)) return -EINVAL; - mutex_lock(&irq_domain_mutex); + mutex_lock(&domain->root->mutex); irq_data->parent_data = NULL; @@ -1730,7 +1746,7 @@ int irq_domain_pop_irq(struct irq_domain *domain, int virq) irq_domain_fix_revmap(irq_data); - mutex_unlock(&irq_domain_mutex); + mutex_unlock(&domain->root->mutex); kfree(parent_irq_data); @@ -1746,17 +1762,20 @@ EXPORT_SYMBOL_GPL(irq_domain_pop_irq); void irq_domain_free_irqs(unsigned int virq, unsigned int nr_irqs) { struct irq_data *data = irq_get_irq_data(virq); + struct irq_domain *domain; int i; if (WARN(!data || !data->domain || !data->domain->ops->free, "NULL pointer, cannot free irq\n")) return; - mutex_lock(&irq_domain_mutex); + domain = data->domain; + + mutex_lock(&domain->root->mutex); for (i = 0; i < nr_irqs; i++) irq_domain_remove_irq(virq + i); - irq_domain_free_irqs_hierarchy(data->domain, virq, nr_irqs); - mutex_unlock(&irq_domain_mutex); + irq_domain_free_irqs_hierarchy(domain, virq, nr_irqs); + mutex_unlock(&domain->root->mutex); irq_domain_free_irq_data(virq, nr_irqs); irq_free_descs(virq, nr_irqs); From 26e27f4e382f126d80e230df2a76d5f1c5a1ac42 Mon Sep 17 00:00:00 2001 From: Christian Marangi Date: Wed, 8 Feb 2023 16:39:11 +0100 Subject: [PATCH 231/239] dt-bindings: cpufreq: qcom-cpufreq-nvmem: specify supported opp tables Add additional info on what opp tables the defined devices in this schema supports (operating-points-v2-kryo-cpu and operating-points-v2-qcom-level) and reference them. Signed-off-by: Christian Marangi Reviewed-by: Krzysztof Kozlowski Signed-off-by: Viresh Kumar --- .../bindings/cpufreq/qcom-cpufreq-nvmem.yaml | 35 ++++++++++++++----- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/Documentation/devicetree/bindings/cpufreq/qcom-cpufreq-nvmem.yaml b/Documentation/devicetree/bindings/cpufreq/qcom-cpufreq-nvmem.yaml index 9c086eac6ca7..7c42d9439abd 100644 --- a/Documentation/devicetree/bindings/cpufreq/qcom-cpufreq-nvmem.yaml +++ b/Documentation/devicetree/bindings/cpufreq/qcom-cpufreq-nvmem.yaml @@ -55,15 +55,32 @@ properties: patternProperties: '^opp-table(-[a-z0-9]+)?$': - if: - properties: - compatible: - const: operating-points-v2-kryo-cpu - then: - patternProperties: - '^opp-?[0-9]+$': - required: - - required-opps + allOf: + - if: + properties: + compatible: + const: operating-points-v2-kryo-cpu + then: + $ref: /schemas/opp/opp-v2-kryo-cpu.yaml# + + - if: + properties: + compatible: + const: operating-points-v2-kryo-cpu + then: + patternProperties: + '^opp-?[0-9]+$': + required: + - required-opps + + - if: + properties: + compatible: + const: operating-points-v2-qcom-level + then: + $ref: /schemas/opp/opp-v2-qcom-level.yaml# + + unevaluatedProperties: false additionalProperties: true From 389de9c5a677c0d950528ee7d1871366ad2a6fd8 Mon Sep 17 00:00:00 2001 From: Christian Marangi Date: Wed, 8 Feb 2023 16:39:12 +0100 Subject: [PATCH 232/239] dt-bindings: cpufreq: qcom-cpufreq-nvmem: make cpr bindings optional The qcom-cpufreq-nvmem driver supports 2 kind of devices: - pre-cpr that doesn't have power-domains and base everything on nvmem cells and multiple named microvolt bindings. Doesn't need required-opp binding in the opp nodes as they are only used for genpd based devices. - cpr-based that require power-domain in the cpu nodes and use various source to decide the correct voltage and freq Require required-opp binding since they need to be linked to the related opp-level. When the schema was introduced, it was wrongly set to always require these binding but this is not the case for pre-cpr devices. Make the power-domain and the required-opp optional and set them required only for qcs404 based devices. Signed-off-by: Christian Marangi Reviewed-by: Krzysztof Kozlowski Signed-off-by: Viresh Kumar --- .../bindings/cpufreq/qcom-cpufreq-nvmem.yaml | 74 +++++++++++-------- 1 file changed, 44 insertions(+), 30 deletions(-) diff --git a/Documentation/devicetree/bindings/cpufreq/qcom-cpufreq-nvmem.yaml b/Documentation/devicetree/bindings/cpufreq/qcom-cpufreq-nvmem.yaml index 7c42d9439abd..6f5e7904181f 100644 --- a/Documentation/devicetree/bindings/cpufreq/qcom-cpufreq-nvmem.yaml +++ b/Documentation/devicetree/bindings/cpufreq/qcom-cpufreq-nvmem.yaml @@ -17,6 +17,9 @@ description: | on the CPU OPP in use. The CPUFreq driver sets the CPR power domain level according to the required OPPs defined in the CPU OPP tables. + For old implementation efuses are parsed to select the correct opp table and + voltage and CPR is not supported/used. + select: properties: compatible: @@ -33,26 +36,6 @@ select: required: - compatible -properties: - cpus: - type: object - - patternProperties: - '^cpu@[0-9a-f]+$': - type: object - - properties: - power-domains: - maxItems: 1 - - power-domain-names: - items: - - const: cpr - - required: - - power-domains - - power-domain-names - patternProperties: '^opp-table(-[a-z0-9]+)?$': allOf: @@ -63,16 +46,6 @@ patternProperties: then: $ref: /schemas/opp/opp-v2-kryo-cpu.yaml# - - if: - properties: - compatible: - const: operating-points-v2-kryo-cpu - then: - patternProperties: - '^opp-?[0-9]+$': - required: - - required-opps - - if: properties: compatible: @@ -82,6 +55,47 @@ patternProperties: unevaluatedProperties: false +allOf: + - if: + properties: + compatible: + contains: + enum: + - qcom,qcs404 + + then: + properties: + cpus: + type: object + + patternProperties: + '^cpu@[0-9a-f]+$': + type: object + + properties: + power-domains: + maxItems: 1 + + power-domain-names: + items: + - const: cpr + + required: + - power-domains + - power-domain-names + + patternProperties: + '^opp-table(-[a-z0-9]+)?$': + if: + properties: + compatible: + const: operating-points-v2-kryo-cpu + then: + patternProperties: + '^opp-?[0-9]+$': + required: + - required-opps + additionalProperties: true examples: From ba38f3cbe7db2cec802c6d60e2bef57a3ff095a4 Mon Sep 17 00:00:00 2001 From: Christian Marangi Date: Wed, 8 Feb 2023 16:39:13 +0100 Subject: [PATCH 233/239] dt-bindings: opp: opp-v2-kryo-cpu: enlarge opp-supported-hw maximum Enlarge opp-supported-hw maximum value. In recent SoC we started matching more bit and we currently match mask of 112. The old maximum of 7 was good for old SoC that didn't had complex id, but now this is limiting and we need to enlarge it to support more variants. Document all the various mask that can be used and limit them to only reasonable values instead of using a generic maximum limit. Signed-off-by: Christian Marangi Reviewed-by: Krzysztof Kozlowski Signed-off-by: Viresh Kumar --- .../devicetree/bindings/opp/opp-v2-kryo-cpu.yaml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/opp/opp-v2-kryo-cpu.yaml b/Documentation/devicetree/bindings/opp/opp-v2-kryo-cpu.yaml index 60cf3cbde4c5..c1bd185d8a0b 100644 --- a/Documentation/devicetree/bindings/opp/opp-v2-kryo-cpu.yaml +++ b/Documentation/devicetree/bindings/opp/opp-v2-kryo-cpu.yaml @@ -50,12 +50,22 @@ patternProperties: opp-supported-hw: description: | A single 32 bit bitmap value, representing compatible HW. - Bitmap: + Bitmap for MSM8996 format: 0: MSM8996, speedbin 0 1: MSM8996, speedbin 1 2: MSM8996, speedbin 2 - 3-31: unused - maximum: 0x7 + 3: MSM8996, speedbin 3 + 4-31: unused + + Bitmap for MSM8996SG format (speedbin shifted of 4 left): + 0-3: unused + 4: MSM8996SG, speedbin 0 + 5: MSM8996SG, speedbin 1 + 6: MSM8996SG, speedbin 2 + 7-31: unused + enum: [0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, + 0x9, 0xd, 0xe, 0xf, + 0x10, 0x20, 0x30, 0x70] clock-latency-ns: true From 3082d7efd76a55f74f263dc01f059f5b5221e8cc Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 13 Feb 2023 22:40:05 +0200 Subject: [PATCH 234/239] Documentation: firmware-guide: gpio-properties: Clarify Explicit and Implicit Clarify the Explicit and Implicit meanings in the table of Pull Bias. While at it, distinguish pull bias keywords used in ACPI by using bold font in the table of the respective terms. Signed-off-by: Andy Shevchenko Signed-off-by: Rafael J. Wysocki --- .../firmware-guide/acpi/gpio-properties.rst | 35 +++++++++++++------ 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/Documentation/firmware-guide/acpi/gpio-properties.rst b/Documentation/firmware-guide/acpi/gpio-properties.rst index eaec732cc77c..db0c0b1f3700 100644 --- a/Documentation/firmware-guide/acpi/gpio-properties.rst +++ b/Documentation/firmware-guide/acpi/gpio-properties.rst @@ -67,17 +67,30 @@ state of the output pin which driver should use during its initialization. Linux tries to use common sense here and derives the state from the bias and polarity settings. The table below shows the expectations: -========= ============= ============== -Pull Bias Polarity Requested... -========= ============= ============== -Implicit x AS IS (assumed firmware configured for us) -Explicit x (no _DSD) as Pull Bias (Up == High, Down == Low), - assuming non-active (Polarity = !Pull Bias) -Down Low as low, assuming active -Down High as low, assuming non-active -Up Low as high, assuming non-active -Up High as high, assuming active -========= ============= ============== ++-------------+-------------+-----------------------------------------------+ +| Pull Bias | Polarity | Requested... | ++=============+=============+===============================================+ +| Implicit | ++-------------+-------------+-----------------------------------------------+ +| **Default** | x | AS IS (assumed firmware configured it for us) | ++-------------+-------------+-----------------------------------------------+ +| Explicit | ++-------------+-------------+-----------------------------------------------+ +| **None** | x | AS IS (assumed firmware configured it for us) | +| | | with no Pull Bias | ++-------------+-------------+-----------------------------------------------+ +| **Up** | x (no _DSD) | | +| +-------------+ as high, assuming non-active | +| | Low | | +| +-------------+-----------------------------------------------+ +| | High | as high, assuming active | ++-------------+-------------+-----------------------------------------------+ +| **Down** | x (no _DSD) | | +| +-------------+ as low, assuming non-active | +| | High | | +| +-------------+-----------------------------------------------+ +| | Low | as low, assuming active | ++-------------+-------------+-----------------------------------------------+ That said, for our above example the both GPIOs, since the bias setting is explicit and _DSD is present, will be treated as active with a high From a527b0111798ed7b9f49830989eaabfe537e09ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Tue, 14 Feb 2023 03:23:52 +0000 Subject: [PATCH 235/239] ACPI: make kobj_type structures constant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since commit ee6d3dd4ed48 ("driver core: make kobj_type constant.") the driver core allows the usage of const struct kobj_type. Take advantage of this to constify the structure definitions to prevent modification at runtime. Signed-off-by: Thomas Weißschuh Signed-off-by: Rafael J. Wysocki --- drivers/acpi/cppc_acpi.c | 2 +- drivers/acpi/device_sysfs.c | 2 +- drivers/acpi/sysfs.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index 0f17b1c32718..a8f58b32d66f 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -193,7 +193,7 @@ static struct attribute *cppc_attrs[] = { }; ATTRIBUTE_GROUPS(cppc); -static struct kobj_type cppc_ktype = { +static const struct kobj_type cppc_ktype = { .sysfs_ops = &kobj_sysfs_ops, .default_groups = cppc_groups, }; diff --git a/drivers/acpi/device_sysfs.c b/drivers/acpi/device_sysfs.c index 120873dad2cc..c3aa15571f16 100644 --- a/drivers/acpi/device_sysfs.c +++ b/drivers/acpi/device_sysfs.c @@ -78,7 +78,7 @@ static void acpi_data_node_release(struct kobject *kobj) complete(&dn->kobj_done); } -static struct kobj_type acpi_data_node_ktype = { +static const struct kobj_type acpi_data_node_ktype = { .sysfs_ops = &acpi_data_node_sysfs_ops, .default_groups = acpi_data_node_default_groups, .release = acpi_data_node_release, diff --git a/drivers/acpi/sysfs.c b/drivers/acpi/sysfs.c index 7db3b530279b..7f4ff56c9d42 100644 --- a/drivers/acpi/sysfs.c +++ b/drivers/acpi/sysfs.c @@ -953,7 +953,7 @@ static struct attribute *hotplug_profile_attrs[] = { }; ATTRIBUTE_GROUPS(hotplug_profile); -static struct kobj_type acpi_hotplug_profile_ktype = { +static const struct kobj_type acpi_hotplug_profile_ktype = { .sysfs_ops = &kobj_sysfs_ops, .default_groups = hotplug_profile_groups, }; From 6e9d12125fcac048c78f497044370c1da85b552a Mon Sep 17 00:00:00 2001 From: Wyes Karny Date: Tue, 14 Feb 2023 07:58:11 +0000 Subject: [PATCH 236/239] cpufreq: amd-pstate: Fix invalid write to MSR_AMD_CPPC_REQ `amd_pstate_set_epp` function uses `cppc_req_cached` and `epp` variable to update the MSR_AMD_CPPC_REQ register for AMD MSR systems. The recent commit 7cca9a9851a5 ("cpufreq: amd-pstate: avoid uninitialized variable use") changed the sequence of updating cppc_req_cached and writing the MSR_AMD_CPPC_REQ. Therefore while switching from powersave to performance governor and vice-versa in active mode MSR_AMD_CPPC_REQ is set with the previous cached value. To fix this: first update the `cppc_req_cached` variable and then call `amd_pstate_set_epp` function. - Before commit 7cca9a9851a5 ("cpufreq: amd-pstate: avoid uninitialized variable use"): With powersave governor: [ 1.652743] amd_pstate_epp_init: writing to cppc_req_cached = 0x1eff [ 1.652744] amd_pstate_set_epp: writing cppc_req_cached = 0x1eff [ 1.652746] amd_pstate_set_epp: writing min_perf = 30, des_perf = 0, max_perf = 255, epp = 0 Changing to performance governor: [ 300.493842] amd_pstate_epp_init: writing to cppc_req_cached = 0xffff [ 300.493846] amd_pstate_set_epp: writing cppc_req_cached = 0xffff [ 300.493847] amd_pstate_set_epp: writing min_perf = 255, des_perf = 0, max_perf = 255, epp = 0 - After commit 7cca9a9851a5 ("cpufreq: amd-pstate: avoid uninitialized variable use"): With powersave governor: [ 1.646037] amd_pstate_set_epp: writing cppc_req_cached = 0xffff [ 1.646038] amd_pstate_set_epp: writing min_perf = 255, des_perf = 0, max_perf = 255, epp = 0 [ 1.646042] amd_pstate_epp_init: writing to cppc_req_cached = 0x1eff Changing to performance governor: [ 687.117401] amd_pstate_set_epp: writing cppc_req_cached = 0x1eff [ 687.117405] amd_pstate_set_epp: writing min_perf = 30, des_perf = 0, max_perf = 255, epp = 0 [ 687.117419] amd_pstate_epp_init: writing to cppc_req_cached = 0xffff - After this fix: With powersave governor: [ 2.525717] amd_pstate_epp_init: writing to cppc_req_cached = 0x1eff [ 2.525720] amd_pstate_set_epp: writing cppc_req_cached = 0x1eff [ 2.525722] amd_pstate_set_epp: writing min_perf = 30, des_perf = 0, max_perf = 255, epp = 0 Changing to performance governor: [ 3440.152468] amd_pstate_epp_init: writing to cppc_req_cached = 0xffff [ 3440.152473] amd_pstate_set_epp: writing cppc_req_cached = 0xffff [ 3440.152474] amd_pstate_set_epp: writing min_perf = 255, des_perf = 0, max_perf = 255, epp = 0 Fixes: 7cca9a9851a5 ("cpufreq: amd-pstate: avoid uninitialized variable use") Signed-off-by: Wyes Karny Acked-by: Huang Rui Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/amd-pstate.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index b8862afef4e4..45c88894fd8e 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1057,27 +1057,28 @@ static void amd_pstate_epp_init(unsigned int cpu) cpudata->epp_policy = cpudata->policy; - if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { - epp = amd_pstate_get_epp(cpudata, value); - if (epp < 0) - goto skip_epp; - /* force the epp value to be zero for performance policy */ - epp = 0; - } else { - /* Get BIOS pre-defined epp value */ - epp = amd_pstate_get_epp(cpudata, value); - if (epp) - goto skip_epp; + /* Get BIOS pre-defined epp value */ + epp = amd_pstate_get_epp(cpudata, value); + if (epp < 0) { + /** + * This return value can only be negative for shared_memory + * systems where EPP register read/write not supported. + */ + goto skip_epp; } + + if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) + epp = 0; + /* Set initial EPP value */ if (boot_cpu_has(X86_FEATURE_CPPC)) { value &= ~GENMASK_ULL(31, 24); value |= (u64)epp << 24; } + WRITE_ONCE(cpudata->cppc_req_cached, value); amd_pstate_set_epp(cpudata, epp); skip_epp: - WRITE_ONCE(cpudata->cppc_req_cached, value); cpufreq_cpu_put(policy); } From 73dd3206811af32695e46ca71e2b0da5dac81ad9 Mon Sep 17 00:00:00 2001 From: Bagas Sanjaya Date: Wed, 15 Feb 2023 19:32:53 +0700 Subject: [PATCH 237/239] Documentation: amd-pstate: disambiguate user space sections kernel test robot reported htmldocs warning: Documentation/admin-guide/pm/amd-pstate.rst:343: WARNING: duplicate label admin-guide/pm/amd-pstate:user space interface in ``sysfs``, other instance in Documentation/admin-guide/pm/amd-pstate.rst The documentation contains two sections with the same "User Space Interface in ``sysfs``" title. The first one deals with per-policy sysfs and the second one is about general attributes (currently only global attributes are documented). Disambiguate title text of both sections to fix the warning. Link: https://lore.kernel.org/linux-doc/202302151041.0SWs1RHK-lkp@intel.com/ Fixes: b9e6a2d47b2565 ("Documentation: amd-pstate: introduce new global sysfs attributes") Reported-by: kernel test robot Signed-off-by: Bagas Sanjaya Reviewed-by: Wyes Karny Acked-by: Huang Rui Signed-off-by: Rafael J. Wysocki --- Documentation/admin-guide/pm/amd-pstate.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst index 5304adf2fc2f..d143e72cf93e 100644 --- a/Documentation/admin-guide/pm/amd-pstate.rst +++ b/Documentation/admin-guide/pm/amd-pstate.rst @@ -230,8 +230,8 @@ with :c:macro:`MSR_AMD_CPPC_ENABLE` or ``cppc_set_enable``, it will respond to the request from AMD P-States. -User Space Interface in ``sysfs`` -================================== +User Space Interface in ``sysfs`` - Per-policy control +====================================================== ``amd-pstate`` exposes several global attributes (files) in ``sysfs`` to control its functionality at the system level. They are located in the @@ -339,8 +339,8 @@ processor must provide at least nominal performance requested and go higher if c operating conditions allow. -User Space Interface in ``sysfs`` -================================= +User Space Interface in ``sysfs`` - General +=========================================== Global Attributes ----------------- From 94debe03e8afa1267f95a9001786a6aa506b9ff3 Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Fri, 16 Dec 2022 15:09:33 -0800 Subject: [PATCH 238/239] irqchip/irq-brcmstb-l2: Set IRQ_LEVEL for level triggered interrupts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When support for the level triggered interrupt controller flavor was added with c0ca7262088e, we forgot to update the flags to be set to contain IRQ_LEVEL. While the flow handler is correct, the output from /proc/interrupts does not show such interrupts as being level triggered when they are, correct that. Fixes: c0ca7262088e ("irqchip/brcmstb-l2: Add support for the BCM7271 L2 controller") Signed-off-by: Florian Fainelli Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20221216230934.2478345-2-f.fainelli@gmail.com --- drivers/irqchip/irq-brcmstb-l2.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-brcmstb-l2.c b/drivers/irqchip/irq-brcmstb-l2.c index e4efc08ac594..091b0fe7e324 100644 --- a/drivers/irqchip/irq-brcmstb-l2.c +++ b/drivers/irqchip/irq-brcmstb-l2.c @@ -161,6 +161,7 @@ static int __init brcmstb_l2_intc_of_init(struct device_node *np, *init_params) { unsigned int clr = IRQ_NOREQUEST | IRQ_NOPROBE | IRQ_NOAUTOEN; + unsigned int set = 0; struct brcmstb_l2_intc_data *data; struct irq_chip_type *ct; int ret; @@ -208,9 +209,12 @@ static int __init brcmstb_l2_intc_of_init(struct device_node *np, if (IS_ENABLED(CONFIG_MIPS) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) flags |= IRQ_GC_BE_IO; + if (init_params->handler == handle_level_irq) + set |= IRQ_LEVEL; + /* Allocate a single Generic IRQ chip for this node */ ret = irq_alloc_domain_generic_chips(data->domain, 32, 1, - np->full_name, init_params->handler, clr, 0, flags); + np->full_name, init_params->handler, clr, set, flags); if (ret) { pr_err("failed to allocate generic irq chip\n"); goto out_free_domain; From 13a157b38ca5b4f9eed81442b8821db293755961 Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Fri, 16 Dec 2022 15:09:34 -0800 Subject: [PATCH 239/239] irqchip/irq-bcm7120-l2: Set IRQ_LEVEL for level triggered interrupts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When support for the interrupt controller was added with a5042de2688d, we forgot to update the flags to be set to contain IRQ_LEVEL. While the flow handler is correct, the output from /proc/interrupts does not show such interrupts as being level triggered when they are, correct that. Fixes: a5042de2688d ("irqchip: bcm7120-l2: Add Broadcom BCM7120-style Level 2 interrupt controller") Signed-off-by: Florian Fainelli Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20221216230934.2478345-3-f.fainelli@gmail.com --- drivers/irqchip/irq-bcm7120-l2.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-bcm7120-l2.c b/drivers/irqchip/irq-bcm7120-l2.c index bb6609cebdbc..1e9dab6e0d86 100644 --- a/drivers/irqchip/irq-bcm7120-l2.c +++ b/drivers/irqchip/irq-bcm7120-l2.c @@ -279,7 +279,8 @@ static int __init bcm7120_l2_intc_probe(struct device_node *dn, flags |= IRQ_GC_BE_IO; ret = irq_alloc_domain_generic_chips(data->domain, IRQS_PER_WORD, 1, - dn->full_name, handle_level_irq, clr, 0, flags); + dn->full_name, handle_level_irq, clr, + IRQ_LEVEL, flags); if (ret) { pr_err("failed to allocate generic irq chip\n"); goto out_free_domain;