tmakin/ack-tegra
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge c1fe867b5b ("Merge tag 'timers-core-2026-04-12' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip") into android-mainline

Browse Source 
Steps on the way to v7.1-rc1

Change-Id: I4f117d1c9b9afb406a79c34ea3c35ec1c2abd1b9
Signed-off-by: Carlos Llamas <cmllamas@google.com>
This commit is contained in:
Carlos Llamas authored and android-test-infra-autosubmit@system.gserviceaccount.com committed 2026-05-06 10:14:14 -07:00
2 parents 729dad03ba c1fe867b5b
commit 9cbc1c9479
74 files changed
+3392 -1824
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Documentation/admin-guide/kernel-parameters.txt
+1 -6
View File
@@ -7997,12 +7997,7 @@ Kernel parameters
(HPET or PM timer) on systems whose TSC frequency was
obtained from HW or FW using either an MSR or CPUID(0x15).
Warn if the difference is more than 500 ppm.
[x86] watchdog: Use TSC as the watchdog clocksource with
which to check other HW timers (HPET or PM timer), but
only on systems where TSC has been deemed trustworthy.
This will be suppressed by an earlier tsc=nowatchdog and
can be overridden by a later tsc=nowatchdog. A console
message will flag any such suppression or overriding.
[x86] watchdog: Enforce the clocksource watchdog on TSC
tsc_early_khz= [X86,EARLY] Skip early TSC calibration and use the given
value instead. Useful when the early TSC frequency discovery
Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml
+21 -9
View File
@@ -4,10 +4,10 @@
$id: http://devicetree.org/schemas/interrupt-controller/apple,aic2.yaml#
$schema: http://devicetree.org/meta-schemas/core.yaml#
title: Apple Interrupt Controller 2
title: Apple Interrupt Controller 2 and 3
maintainers:
- Hector Martin <marcan@marcan.st>
- Janne Grunau <j@jannau.net>
description: |
The Apple Interrupt Controller 2 is a simple interrupt controller present on
@@ -28,14 +28,24 @@ description: |
which do not go through a discrete interrupt controller. It also handles
FIQ-based Fast IPIs.
The Apple Interrupt Controller 3 is in its base functionality very similar to
the Apple Interrupt Controller 2 and uses the same device tree bindings. It is
found on Apple ARM SoCs platforms starting with t8122 (M3).
properties:
compatible:
items:
- enum:
- apple,t8112-aic
- apple,t6000-aic
- apple,t6020-aic
- const: apple,aic2
oneOf:
- items:
- enum:
- apple,t6000-aic
- apple,t6020-aic
- apple,t8112-aic
- const: apple,aic2
- items:
- enum:
- apple,t6030-aic3
- const: apple,t8122-aic3
- const: apple,t8122-aic3
interrupt-controller: true
@@ -117,7 +127,9 @@ allOf:
properties:
compatible:
contains:
const: apple,t8112-aic
enum:
- apple,t8112-aic
- apple,t8122-aic3
then:
properties:
'#interrupt-cells':
Documentation/devicetree/bindings/interrupt-controller/loongson,pch-lpc.yaml
+52
View File
@@ -0,0 +1,52 @@
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
%YAML 1.2
---
$id: http://devicetree.org/schemas/interrupt-controller/loongson,pch-lpc.yaml#
$schema: http://devicetree.org/meta-schemas/core.yaml#
title: Loongson PCH LPC Controller
maintainers:
- Jiaxun Yang <jiaxun.yang@flygoat.com>
description:
This interrupt controller is found in the Loongson LS7A family of PCH for
accepting interrupts sent by LPC-connected peripherals and signalling PIC
via a single interrupt line when interrupts are available.
properties:
compatible:
const: loongson,ls7a-lpc
reg:
maxItems: 1
interrupt-controller: true
interrupts:
maxItems: 1
'#interrupt-cells':
const: 2
required:
- compatible
- reg
- interrupt-controller
- interrupts
- '#interrupt-cells'
additionalProperties: false
examples:
- |
#include <dt-bindings/interrupt-controller/irq.h>
lpc: interrupt-controller@10002000 {
compatible = "loongson,ls7a-lpc";
reg = <0x10002000 0x400>;
interrupt-controller;
#interrupt-cells = <2>;
interrupt-parent = <&pic>;
interrupts = <19 IRQ_TYPE_LEVEL_HIGH>;
};
...
Documentation/devicetree/bindings/interrupt-controller/renesas,rzg2l-irqc.yaml
+56 -101
View File
@@ -30,7 +30,9 @@ properties:
- renesas,r9a08g045-irqc # RZ/G3S
- const: renesas,rzg2l-irqc
- const: renesas,r9a07g043f-irqc # RZ/Five
- enum:
- renesas,r9a07g043f-irqc # RZ/Five
- renesas,r9a08g046-irqc # RZ/G3L
'#interrupt-cells':
description: The first cell should contain a macro RZG2L_{NMI,IRQX} included in the
@@ -48,107 +50,35 @@ properties:
interrupts:
minItems: 45
items:
- description: NMI interrupt
- description: IRQ0 interrupt
- description: IRQ1 interrupt
- description: IRQ2 interrupt
- description: IRQ3 interrupt
- description: IRQ4 interrupt
- description: IRQ5 interrupt
- description: IRQ6 interrupt
- description: IRQ7 interrupt
- description: GPIO interrupt, TINT0
- description: GPIO interrupt, TINT1
- description: GPIO interrupt, TINT2
- description: GPIO interrupt, TINT3
- description: GPIO interrupt, TINT4
- description: GPIO interrupt, TINT5
- description: GPIO interrupt, TINT6
- description: GPIO interrupt, TINT7
- description: GPIO interrupt, TINT8
- description: GPIO interrupt, TINT9
- description: GPIO interrupt, TINT10
- description: GPIO interrupt, TINT11
- description: GPIO interrupt, TINT12
- description: GPIO interrupt, TINT13
- description: GPIO interrupt, TINT14
- description: GPIO interrupt, TINT15
- description: GPIO interrupt, TINT16
- description: GPIO interrupt, TINT17
- description: GPIO interrupt, TINT18
- description: GPIO interrupt, TINT19
- description: GPIO interrupt, TINT20
- description: GPIO interrupt, TINT21
- description: GPIO interrupt, TINT22
- description: GPIO interrupt, TINT23
- description: GPIO interrupt, TINT24
- description: GPIO interrupt, TINT25
- description: GPIO interrupt, TINT26
- description: GPIO interrupt, TINT27
- description: GPIO interrupt, TINT28
- description: GPIO interrupt, TINT29
- description: GPIO interrupt, TINT30
- description: GPIO interrupt, TINT31
- description: Bus error interrupt
- description: ECCRAM0 or combined ECCRAM0/1 1bit error interrupt
- description: ECCRAM0 or combined ECCRAM0/1 2bit error interrupt
- description: ECCRAM0 or combined ECCRAM0/1 error overflow interrupt
- description: ECCRAM1 1bit error interrupt
- description: ECCRAM1 2bit error interrupt
- description: ECCRAM1 error overflow interrupt
maxItems: 61
interrupt-names:
minItems: 45
maxItems: 61
items:
- const: nmi
- const: irq0
- const: irq1
- const: irq2
- const: irq3
- const: irq4
- const: irq5
- const: irq6
- const: irq7
- const: tint0
- const: tint1
- const: tint2
- const: tint3
- const: tint4
- const: tint5
- const: tint6
- const: tint7
- const: tint8
- const: tint9
- const: tint10
- const: tint11
- const: tint12
- const: tint13
- const: tint14
- const: tint15
- const: tint16
- const: tint17
- const: tint18
- const: tint19
- const: tint20
- const: tint21
- const: tint22
- const: tint23
- const: tint24
- const: tint25
- const: tint26
- const: tint27
- const: tint28
- const: tint29
- const: tint30
- const: tint31
- const: bus-err
- const: ec7tie1-0
- const: ec7tie2-0
- const: ec7tiovf-0
- const: ec7tie1-1
- const: ec7tie2-1
- const: ec7tiovf-1
oneOf:
- description: NMI interrupt
const: nmi
- description: External IRQ interrupt
pattern: '^irq([0-9]|1[0-5])$'
- description: GPIO interrupt
pattern: '^tint([0-9]|1[0-9]|2[0-9]|3[0-1])$'
- description: Bus error interrupt
const: bus-err
- description: ECCRAM0 or combined ECCRAM0/1 1bit error interrupt
const: ec7tie1-0
- description: ECCRAM0 or combined ECCRAM0/1 2bit error interrupt
const: ec7tie2-0
- description: ECCRAM0 or combined ECCRAM0/1 error overflow interrupt
const: ec7tiovf-0
- description: ECCRAM1 1bit error interrupt
const: ec7tie1-1
- description: ECCRAM1 2bit error interrupt
const: ec7tie2-1
- description: ECCRAM1 error overflow interrupt
const: ec7tiovf-1
- description: Integrated GPT Error interrupt
pattern: '^ovfunf([0-7])$'
clocks:
maxItems: 2
@@ -180,6 +110,24 @@ required:
allOf:
- $ref: /schemas/interrupt-controller.yaml#
- if:
properties:
compatible:
contains:
enum:
- renesas,r9a07g043f-irqc
- renesas,r9a07g043u-irqc
- renesas,r9a07g044-irqc
- renesas,r9a07g054-irqc
then:
properties:
interrupts:
minItems: 48
maxItems: 48
interrupt-names:
minItems: 48
maxItems: 48
- if:
properties:
compatible:
@@ -192,12 +140,19 @@ allOf:
maxItems: 45
interrupt-names:
maxItems: 45
else:
- if:
properties:
compatible:
contains:
enum:
- renesas,r9a08g046-irqc
then:
properties:
interrupts:
minItems: 48
minItems: 61
interrupt-names:
minItems: 48
minItems: 61
unevaluatedProperties: false
MAINTAINERS
+1
View File
@@ -26677,6 +26677,7 @@ F: include/linux/timekeeping.h
F: include/linux/timex.h
F: include/uapi/linux/time.h
F: include/uapi/linux/timex.h
F: kernel/time/.kunitconfig
F: kernel/time/alarmtimer.c
F: kernel/time/clocksource*
F: kernel/time/ntp*
arch/loongarch/kernel/irq.c
+6
View File
@@ -11,6 +11,7 @@
#include <linux/irqchip.h>
#include <linux/kernel_stat.h>
#include <linux/proc_fs.h>
#include <linux/minmax.h>
#include <linux/mm.h>
#include <linux/sched.h>
#include <linux/seq_file.h>
@@ -99,6 +100,11 @@ int __init arch_probe_nr_irqs(void)
return NR_IRQS_LEGACY;
}
unsigned int arch_dynirq_lower_bound(unsigned int from)
{
return MAX(from, NR_IRQS_LEGACY);
}
void __init init_IRQ(void)
{
int i;
arch/mips/Kconfig
-1
View File
@@ -1131,7 +1131,6 @@ config CSRC_IOASIC
bool
config CSRC_R4K
select CLOCKSOURCE_WATCHDOG if CPU_FREQ
bool
config CSRC_SB1250
arch/mips/loongson64/init.c
+6
View File
@@ -7,6 +7,7 @@
#include <linux/irqchip.h>
#include <linux/logic_pio.h>
#include <linux/memblock.h>
#include <linux/minmax.h>
#include <linux/of.h>
#include <linux/of_address.h>
#include <asm/bootinfo.h>
@@ -227,3 +228,8 @@ void __init arch_init_irq(void)
reserve_pio_range();
irqchip_init();
}
unsigned int arch_dynirq_lower_bound(unsigned int from)
{
return MAX(from, NR_IRQS_LEGACY);
}
arch/mips/pic32/Kconfig
-1
View File
@@ -20,7 +20,6 @@ config PIC32MZDA
select LIBFDT
select USE_OF
select PINCTRL
select PIC32_EVIC
help
Support for the Microchip PIC32MZDA microcontroller.
arch/parisc/kernel/time.c
+1 -4
View File
@@ -210,12 +210,9 @@ static struct clocksource clocksource_cr16 = {
.read = read_cr16,
.mask = CLOCKSOURCE_MASK(BITS_PER_LONG),
.flags = CLOCK_SOURCE_IS_CONTINUOUS |
CLOCK_SOURCE_VALID_FOR_HRES |
CLOCK_SOURCE_MUST_VERIFY |
CLOCK_SOURCE_VERIFY_PERCPU,
CLOCK_SOURCE_VALID_FOR_HRES,
};
/*
* timer interrupt and sched_clock() initialization
*/
arch/x86/Kconfig
+2
View File
@@ -141,6 +141,7 @@ config X86
select ARCH_USE_SYM_ANNOTATIONS
select ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
select ARCH_WANT_DEFAULT_BPF_JIT if X86_64
select ARCH_WANTS_CLOCKSOURCE_READ_INLINE if X86_64
select ARCH_WANTS_DYNAMIC_TASK_STRUCT
select ARCH_WANTS_NO_INSTR
select ARCH_WANT_GENERAL_HUGETLB
@@ -163,6 +164,7 @@ config X86
select EDAC_SUPPORT
select GENERIC_CLOCKEVENTS_BROADCAST if X86_64 || (X86_32 && X86_LOCAL_APIC)
select GENERIC_CLOCKEVENTS_BROADCAST_IDLE if GENERIC_CLOCKEVENTS_BROADCAST
select GENERIC_CLOCKEVENTS_COUPLED_INLINE if X86_64
select GENERIC_CLOCKEVENTS_MIN_ADJUST
select GENERIC_CMOS_UPDATE
select GENERIC_CPU_AUTOPROBE
arch/x86/include/asm/clock_inlined.h
+22
View File
@@ -0,0 +1,22 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_CLOCK_INLINED_H
#define _ASM_X86_CLOCK_INLINED_H
#include <asm/tsc.h>
struct clocksource;
static __always_inline u64 arch_inlined_clocksource_read(struct clocksource *cs)
{
return (u64)rdtsc_ordered();
}
struct clock_event_device;
static __always_inline void
arch_inlined_clockevent_set_next_coupled(u64 cycles, struct clock_event_device *evt)
{
native_wrmsrq(MSR_IA32_TSC_DEADLINE, cycles);
}
#endif
arch/x86/include/asm/time.h
-1
View File
@@ -7,7 +7,6 @@
extern void hpet_time_init(void);
extern bool pit_timer_init(void);
extern bool tsc_clocksource_watchdog_disabled(void);
extern struct clock_event_device *global_clock_event;
arch/x86/kernel/apic/apic.c
+23 -18
View File
@@ -412,23 +412,21 @@ EXPORT_SYMBOL_GPL(setup_APIC_eilvt);
/*
* Program the next event, relative to now
*/
static int lapic_next_event(unsigned long delta,
struct clock_event_device *evt)
static int lapic_next_event(unsigned long delta, struct clock_event_device *evt)
{
apic_write(APIC_TMICT, delta);
return 0;
}
static int lapic_next_deadline(unsigned long delta,
struct clock_event_device *evt)
static int lapic_next_deadline(unsigned long delta, struct clock_event_device *evt)
{
u64 tsc;
/*
* There is no weak_wrmsr_fence() required here as all of this is purely
* CPU local. Avoid the [ml]fence overhead.
*/
u64 tsc = rdtsc();
/* This MSR is special and need a special fence: */
weak_wrmsr_fence();
tsc = rdtsc();
wrmsrq(MSR_IA32_TSC_DEADLINE, tsc + (((u64) delta) * TSC_DIVISOR));
native_wrmsrq(MSR_IA32_TSC_DEADLINE, tsc + (((u64) delta) * TSC_DIVISOR));
return 0;
}
@@ -452,7 +450,7 @@ static int lapic_timer_shutdown(struct clock_event_device *evt)
* the timer _and_ zero the counter registers:
*/
if (v & APIC_LVT_TIMER_TSCDEADLINE)
wrmsrq(MSR_IA32_TSC_DEADLINE, 0);
native_wrmsrq(MSR_IA32_TSC_DEADLINE, 0);
else
apic_write(APIC_TMICT, 0);
@@ -549,6 +547,11 @@ static __init bool apic_validate_deadline_timer(void)
if (!boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
return false;
/* XEN_PV does not support it, but be paranoia about it */
if (boot_cpu_has(X86_FEATURE_XENPV))
goto clear;
if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
return true;
@@ -561,9 +564,11 @@ static __init bool apic_validate_deadline_timer(void)
if (boot_cpu_data.microcode >= rev)
return true;
setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER);
pr_err(FW_BUG "TSC_DEADLINE disabled due to Errata; "
"please update microcode to version: 0x%x (or later)\n", rev);
clear:
setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER);
return false;
}
@@ -586,14 +591,14 @@ static void setup_APIC_timer(void)
if (this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) {
levt->name = "lapic-deadline";
levt->features &= ~(CLOCK_EVT_FEAT_PERIODIC |
CLOCK_EVT_FEAT_DUMMY);
levt->features &= ~(CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_DUMMY);
levt->features |= CLOCK_EVT_FEAT_CLOCKSOURCE_COUPLED;
levt->cs_id = CSID_X86_TSC;
levt->set_next_event = lapic_next_deadline;
clockevents_config_and_register(levt,
tsc_khz * (1000 / TSC_DIVISOR),
0xF, ~0UL);
} else
clockevents_config_and_register(levt, tsc_khz * (1000 / TSC_DIVISOR), 0xF, ~0UL);
} else {
clockevents_register_device(levt);
}
apic_update_vector(smp_processor_id(), LOCAL_TIMER_VECTOR, true);
}
arch/x86/kernel/hpet.c
+1 -3
View File
@@ -854,7 +854,7 @@ static struct clocksource clocksource_hpet = {
.rating = 250,
.read = read_hpet,
.mask = HPET_MASK,
.flags = CLOCK_SOURCE_IS_CONTINUOUS,
.flags = CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_CALIBRATED,
.resume = hpet_resume_counter,
};
@@ -1082,8 +1082,6 @@ int __init hpet_enable(void)
if (!hpet_counting())
goto out_nohpet;
if (tsc_clocksource_watchdog_disabled())
clocksource_hpet.flags |= CLOCK_SOURCE_MUST_VERIFY;
clocksource_register_hz(&clocksource_hpet, (u32)hpet_freq);
if (id & HPET_ID_LEGSUP) {
arch/x86/kernel/tsc.c
+29 -32
View File
@@ -322,12 +322,16 @@ int __init notsc_setup(char *str)
return 1;
}
#endif
__setup("notsc", notsc_setup);
enum {
TSC_WATCHDOG_AUTO,
TSC_WATCHDOG_OFF,
TSC_WATCHDOG_ON,
};
static int no_sched_irq_time;
static int no_tsc_watchdog;
static int tsc_as_watchdog;
static int tsc_watchdog;
static int __init tsc_setup(char *str)
{
@@ -337,25 +341,14 @@ static int __init tsc_setup(char *str)
no_sched_irq_time = 1;
if (!strcmp(str, "unstable"))
mark_tsc_unstable("boot parameter");
if (!strcmp(str, "nowatchdog")) {
no_tsc_watchdog = 1;
if (tsc_as_watchdog)
pr_alert("%s: Overriding earlier tsc=watchdog with tsc=nowatchdog\n",
__func__);
tsc_as_watchdog = 0;
}
if (!strcmp(str, "nowatchdog"))
tsc_watchdog = TSC_WATCHDOG_OFF;
if (!strcmp(str, "recalibrate"))
tsc_force_recalibrate = 1;
if (!strcmp(str, "watchdog")) {
if (no_tsc_watchdog)
pr_alert("%s: tsc=watchdog overridden by earlier tsc=nowatchdog\n",
__func__);
else
tsc_as_watchdog = 1;
}
if (!strcmp(str, "watchdog"))
tsc_watchdog = TSC_WATCHDOG_ON;
return 1;
}
__setup("tsc=", tsc_setup);
#define MAX_RETRIES 5
@@ -1175,7 +1168,6 @@ static int tsc_cs_enable(struct clocksource *cs)
static struct clocksource clocksource_tsc_early = {
.name = "tsc-early",
.rating = 299,
.uncertainty_margin = 32 * NSEC_PER_MSEC,
.read = read_tsc,
.mask = CLOCKSOURCE_MASK(64),
.flags = CLOCK_SOURCE_IS_CONTINUOUS |
@@ -1200,9 +1192,9 @@ static struct clocksource clocksource_tsc = {
.read = read_tsc,
.mask = CLOCKSOURCE_MASK(64),
.flags = CLOCK_SOURCE_IS_CONTINUOUS |
CLOCK_SOURCE_VALID_FOR_HRES |
CLOCK_SOURCE_CAN_INLINE_READ |
CLOCK_SOURCE_MUST_VERIFY |
CLOCK_SOURCE_VERIFY_PERCPU,
CLOCK_SOURCE_HAS_COUPLED_CLOCK_EVENT,
.id = CSID_X86_TSC,
.vdso_clock_mode = VDSO_CLOCKMODE_TSC,
.enable = tsc_cs_enable,
@@ -1230,16 +1222,12 @@ EXPORT_SYMBOL_GPL(mark_tsc_unstable);
static void __init tsc_disable_clocksource_watchdog(void)
{
if (tsc_watchdog == TSC_WATCHDOG_ON)
return;
clocksource_tsc_early.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
}
bool tsc_clocksource_watchdog_disabled(void)
{
return !(clocksource_tsc.flags & CLOCK_SOURCE_MUST_VERIFY) &&
tsc_as_watchdog && !no_tsc_watchdog;
}
static void __init check_system_tsc_reliable(void)
{
#if defined(CONFIG_MGEODEGX1) || defined(CONFIG_MGEODE_LX) || defined(CONFIG_X86_GENERIC)
@@ -1394,6 +1382,8 @@ restart:
(unsigned long)tsc_khz / 1000,
(unsigned long)tsc_khz % 1000);
clocksource_tsc.flags |= CLOCK_SOURCE_CALIBRATED;
/* Inform the TSC deadline clockevent devices about the recalibration */
lapic_update_tsc_freq();
@@ -1409,6 +1399,15 @@ out:
have_art = true;
clocksource_tsc.base = &art_base_clk;
}
/*
* Transfer the valid for high resolution flag if it was set on the
* early TSC already. That guarantees that there is no intermediate
* clocksource selected once the early TSC is unregistered.
*/
if (clocksource_tsc_early.flags & CLOCK_SOURCE_VALID_FOR_HRES)
clocksource_tsc.flags |= CLOCK_SOURCE_VALID_FOR_HRES;
clocksource_register_khz(&clocksource_tsc, tsc_khz);
unreg:
clocksource_unregister(&clocksource_tsc_early);
@@ -1460,12 +1459,10 @@ static bool __init determine_cpu_tsc_frequencies(bool early)
if (early) {
cpu_khz = x86_platform.calibrate_cpu();
if (tsc_early_khz) {
if (tsc_early_khz)
tsc_khz = tsc_early_khz;
} else {
else
tsc_khz = x86_platform.calibrate_tsc();
clocksource_tsc.freq_khz = tsc_khz;
}
} else {
/* We should not be here with non-native cpu calibration */
WARN_ON(x86_platform.calibrate_cpu != native_calibrate_cpu);
@@ -1569,7 +1566,7 @@ void __init tsc_init(void)
return;
}
if (tsc_clocksource_reliable || no_tsc_watchdog)
if (tsc_clocksource_reliable || tsc_watchdog == TSC_WATCHDOG_OFF)
tsc_disable_clocksource_watchdog();
clocksource_register_khz(&clocksource_tsc_early, tsc_khz);
drivers/clocksource/Kconfig
-1
View File
@@ -596,7 +596,6 @@ config CLKSRC_VERSATILE
config CLKSRC_MIPS_GIC
bool
depends on MIPS_GIC
select CLOCKSOURCE_WATCHDOG
select TIMER_OF
config CLKSRC_PXA
drivers/clocksource/acpi_pm.c
+1 -3
View File
@@ -98,7 +98,7 @@ static struct clocksource clocksource_acpi_pm = {
.rating = 200,
.read = acpi_pm_read,
.mask = (u64)ACPI_PM_MASK,
.flags = CLOCK_SOURCE_IS_CONTINUOUS,
.flags = CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_CALIBRATED,
.suspend = acpi_pm_suspend,
.resume = acpi_pm_resume,
};
@@ -243,8 +243,6 @@ static int __init init_acpi_pm_clocksource(void)
return -ENODEV;
}
if (tsc_clocksource_watchdog_disabled())
clocksource_acpi_pm.flags |= CLOCK_SOURCE_MUST_VERIFY;
return clocksource_register_hz(&clocksource_acpi_pm, PMTMR_TICKS_PER_SEC);
}
drivers/irqchip/Kconfig
+7 -5
View File
@@ -252,9 +252,12 @@ config ORION_IRQCHIP
select IRQ_DOMAIN
config PIC32_EVIC
bool
def_bool MACH_PIC32 || COMPILE_TEST
select GENERIC_IRQ_CHIP
select IRQ_DOMAIN
help
Enable support for the interrupt controller on the Microchip PIC32
family of platforms.
config JCORE_AIC
bool "J-Core integrated AIC" if COMPILE_TEST
@@ -541,11 +544,11 @@ config CSKY_APB_INTC
config IMX_IRQSTEER
bool "i.MX IRQSTEER support"
depends on ARCH_MXC || COMPILE_TEST
default ARCH_MXC
depends on ARCH_MXC || ARCH_S32 || COMPILE_TEST
default y if ARCH_MXC || ARCH_S32
select IRQ_DOMAIN
help
Support for the i.MX IRQSTEER interrupt multiplexer/remapper.
Support for the i.MX and S32 IRQSTEER interrupt multiplexer/remapper.
config IMX_INTMUX
bool "i.MX INTMUX support" if COMPILE_TEST
@@ -761,7 +764,6 @@ config LOONGSON_PCH_MSI
config LOONGSON_PCH_LPC
bool "Loongson PCH LPC Controller"
depends on LOONGARCH
depends on MACH_LOONGSON64 || LOONGARCH
default MACH_LOONGSON64
select IRQ_DOMAIN_HIERARCHY
drivers/irqchip/irq-apple-aic.c
+21 -3
View File
@@ -134,8 +134,12 @@
#define AIC2_IRQ_CFG 0x2000
/* AIC v3 registers (MMIO) */
#define AIC3_IRQ_CFG 0x10000
/*
* AIC2 registers are laid out like this, starting at AIC2_IRQ_CFG:
* AIC3 registers use the same layout but start at AIC3_IRQ_CFG:
*
* Repeat for each die:
* IRQ_CFG: u32 * MAX_IRQS
@@ -293,6 +297,15 @@ static const struct aic_info aic2_info __initconst = {
.local_fast_ipi = true,
};
static const struct aic_info aic3_info __initconst = {
.version = 3,
.irq_cfg = AIC3_IRQ_CFG,
.fast_ipi = true,
.local_fast_ipi = true,
};
static const struct of_device_id aic_info_match[] = {
{
.compatible = "apple,t8103-aic",
@@ -310,6 +323,10 @@ static const struct of_device_id aic_info_match[] = {
.compatible = "apple,aic2",
.data = &aic2_info,
},
{
.compatible = "apple,t8122-aic3",
.data = &aic3_info,
},
{}
};
@@ -620,7 +637,7 @@ static int aic_irq_domain_map(struct irq_domain *id, unsigned int irq,
u32 type = FIELD_GET(AIC_EVENT_TYPE, hw);
struct irq_chip *chip = &aic_chip;
if (ic->info.version == 2)
if (ic->info.version == 2 || ic->info.version == 3)
chip = &aic2_chip;
if (type == AIC_EVENT_TYPE_IRQ) {
@@ -991,7 +1008,7 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p
break;
}
case 2: {
case 2 ... 3: {
u32 info1, info3;
info1 = aic_ic_read(irqc, AIC2_INFO1);
@@ -1065,7 +1082,7 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p
off += irqc->info.die_stride;
}
if (irqc->info.version == 2) {
if (irqc->info.version == 2 || irqc->info.version == 3) {
u32 config = aic_ic_read(irqc, AIC2_CONFIG);
config |= AIC2_CONFIG_ENABLE;
@@ -1116,3 +1133,4 @@ err_unmap:
IRQCHIP_DECLARE(apple_aic, "apple,aic", aic_of_ic_init);
IRQCHIP_DECLARE(apple_aic2, "apple,aic2", aic_of_ic_init);
IRQCHIP_DECLARE(apple_aic3, "apple,t8122-aic3", aic_of_ic_init);
drivers/irqchip/irq-gic-v3.c
+9 -1
View File
@@ -1607,15 +1607,23 @@ static int gic_irq_domain_translate(struct irq_domain *d,
switch (fwspec->param[0]) {
case 0: /* SPI */
if (fwspec->param[1] > 987)
pr_warn_once("SPI %u out of range (use ESPI?)\n", fwspec->param[1]);
*hwirq = fwspec->param[1] + 32;
break;
case 1: /* PPI */
if (fwspec->param[1] > 15)
pr_warn_once("PPI %u out of range (use EPPI?)\n", fwspec->param[1]);
*hwirq = fwspec->param[1] + 16;
break;
case 2: /* ESPI */
if (fwspec->param[1] > 1023)
pr_warn_once("ESPI %u out of range\n", fwspec->param[1]);
*hwirq = fwspec->param[1] + ESPI_BASE_INTID;
break;
case 3: /* EPPI */
if (fwspec->param[1] > 63)
pr_warn_once("EPPI %u out of range\n", fwspec->param[1]);
*hwirq = fwspec->param[1] + EPPI_BASE_INTID;
break;
case GIC_IRQ_TYPE_LPI: /* LPI */
@@ -2256,7 +2264,7 @@ static int __init gic_of_init(struct device_node *node, struct device_node *pare
out_unmap_rdist:
for (i = 0; i < nr_redist_regions; i++)
if (rdist_regs[i].redist_base && !IS_ERR(rdist_regs[i].redist_base))
if (!IS_ERR_OR_NULL(rdist_regs[i].redist_base))
iounmap(rdist_regs[i].redist_base);
kfree(rdist_regs);
out_unmap_dist:
drivers/irqchip/irq-imx-irqsteer.c
+41 -14
View File
@@ -26,19 +26,38 @@
#define CHAN_MAX_OUTPUT_INT 0xF
struct irqsteer_data {
void __iomem *regs;
struct clk *ipg_clk;
int irq[CHAN_MAX_OUTPUT_INT];
int irq_count;
raw_spinlock_t lock;
int reg_num;
int channel;
struct irq_domain *domain;
u32 *saved_reg;
struct device *dev;
/* SoC does not implement the CHANCTRL register */
#define IRQSTEER_QUIRK_NO_CHANCTRL BIT(0)
struct irqsteer_devtype_data {
u32 quirks;
};
struct irqsteer_data {
void __iomem *regs;
struct clk *ipg_clk;
int irq[CHAN_MAX_OUTPUT_INT];
int irq_count;
raw_spinlock_t lock;
int reg_num;
int channel;
struct irq_domain *domain;
u32 *saved_reg;
struct device *dev;
const struct irqsteer_devtype_data *devtype_data;
};
static const struct irqsteer_devtype_data imx_data = { };
static const struct irqsteer_devtype_data s32n79_data = {
.quirks = IRQSTEER_QUIRK_NO_CHANCTRL,
};
static bool irqsteer_has_chanctrl(const struct irqsteer_devtype_data *data)
{
return !(data->quirks & IRQSTEER_QUIRK_NO_CHANCTRL);
}
static int imx_irqsteer_get_reg_index(struct irqsteer_data *data,
unsigned long irqnum)
{
@@ -188,6 +207,10 @@ static int imx_irqsteer_probe(struct platform_device *pdev)
if (ret)
return ret;
data->devtype_data = device_get_match_data(&pdev->dev);
if (!data->devtype_data)
return dev_err_probe(&pdev->dev, -ENODEV, "failed to match device data\n");
/*
* There is one output irq for each group of 64 inputs.
* One register bit map can represent 32 input interrupts.
@@ -210,7 +233,8 @@ static int imx_irqsteer_probe(struct platform_device *pdev)
}
/* steer all IRQs into configured channel */
writel_relaxed(BIT(data->channel), data->regs + CHANCTRL);
if (irqsteer_has_chanctrl(data->devtype_data))
writel_relaxed(BIT(data->channel), data->regs + CHANCTRL);
data->domain = irq_domain_create_linear(dev_fwnode(&pdev->dev), data->reg_num * 32,
&imx_irqsteer_domain_ops, data);
@@ -279,7 +303,9 @@ static void imx_irqsteer_restore_regs(struct irqsteer_data *data)
{
int i;
writel_relaxed(BIT(data->channel), data->regs + CHANCTRL);
if (irqsteer_has_chanctrl(data->devtype_data))
writel_relaxed(BIT(data->channel), data->regs + CHANCTRL);
for (i = 0; i < data->reg_num; i++)
writel_relaxed(data->saved_reg[i],
data->regs + CHANMASK(i, data->reg_num));
@@ -319,7 +345,8 @@ static const struct dev_pm_ops imx_irqsteer_pm_ops = {
};
static const struct of_device_id imx_irqsteer_dt_ids[] = {
{ .compatible = "fsl,imx-irqsteer", },
{ .compatible = "fsl,imx-irqsteer", .data = &imx_data },
{ .compatible = "nxp,s32n79-irqsteer", .data = &s32n79_data },
{},
};
drivers/irqchip/irq-loongson-pch-lpc.c
+72 -20
View File
@@ -13,6 +13,8 @@
#include <linux/irqchip/chained_irq.h>
#include <linux/irqdomain.h>
#include <linux/kernel.h>
#include <linux/of_address.h>
#include <linux/of_irq.h>
#include <linux/syscore_ops.h>
#include "irq-loongson.h"
@@ -175,13 +177,10 @@ static struct syscore pch_lpc_syscore = {
.ops = &pch_lpc_syscore_ops,
};
int __init pch_lpc_acpi_init(struct irq_domain *parent,
struct acpi_madt_lpc_pic *acpi_pchlpc)
static int __init pch_lpc_init(phys_addr_t addr, unsigned long size,
struct fwnode_handle *irq_handle, int parent_irq)
{
int parent_irq;
struct pch_lpc *priv;
struct irq_fwspec fwspec;
struct fwnode_handle *irq_handle;
priv = kzalloc_obj(*priv);
if (!priv)
@@ -189,7 +188,7 @@ int __init pch_lpc_acpi_init(struct irq_domain *parent,
raw_spin_lock_init(&priv->lpc_lock);
priv->base = ioremap(acpi_pchlpc->address, acpi_pchlpc->size);
priv->base = ioremap(addr, size);
if (!priv->base)
goto free_priv;
@@ -198,12 +197,6 @@ int __init pch_lpc_acpi_init(struct irq_domain *parent,
goto iounmap_base;
}
irq_handle = irq_domain_alloc_named_fwnode("lpcintc");
if (!irq_handle) {
pr_err("Unable to allocate domain handle\n");
goto iounmap_base;
}
/*
* The LPC interrupt controller is a legacy i8259-compatible device,
* which requires a static 1:1 mapping for IRQs 0-15.
@@ -213,15 +206,10 @@ int __init pch_lpc_acpi_init(struct irq_domain *parent,
&pch_lpc_domain_ops, priv);
if (!priv->lpc_domain) {
pr_err("Failed to create IRQ domain\n");
goto free_irq_handle;
goto iounmap_base;
}
pch_lpc_reset(priv);
fwspec.fwnode = parent->fwnode;
fwspec.param[0] = acpi_pchlpc->cascade + GSI_MIN_PCH_IRQ;
fwspec.param[1] = IRQ_TYPE_LEVEL_HIGH;
fwspec.param_count = 2;
parent_irq = irq_create_fwspec_mapping(&fwspec);
irq_set_chained_handler_and_data(parent_irq, lpc_irq_dispatch, priv);
pch_lpc_priv = priv;
@@ -230,8 +218,6 @@ int __init pch_lpc_acpi_init(struct irq_domain *parent,
return 0;
free_irq_handle:
irq_domain_free_fwnode(irq_handle);
iounmap_base:
iounmap(priv->base);
free_priv:
@@ -239,3 +225,69 @@ free_priv:
return -ENOMEM;
}
#ifdef CONFIG_ACPI
int __init pch_lpc_acpi_init(struct irq_domain *parent, struct acpi_madt_lpc_pic *acpi_pchlpc)
{
struct fwnode_handle *irq_handle;
struct irq_fwspec fwspec;
int parent_irq, ret;
irq_handle = irq_domain_alloc_named_fwnode("lpcintc");
if (!irq_handle) {
pr_err("Unable to allocate domain handle\n");
return -ENOMEM;
}
fwspec.fwnode = parent->fwnode;
fwspec.param[0] = acpi_pchlpc->cascade + GSI_MIN_PCH_IRQ;
fwspec.param[1] = IRQ_TYPE_LEVEL_HIGH;
fwspec.param_count = 2;
parent_irq = irq_create_fwspec_mapping(&fwspec);
if (parent_irq <= 0) {
pr_err("Unable to map LPC parent interrupt\n");
irq_domain_free_fwnode(irq_handle);
return -ENOMEM;
}
ret = pch_lpc_init(acpi_pchlpc->address, acpi_pchlpc->size, irq_handle, parent_irq);
if (ret) {
irq_dispose_mapping(parent_irq);
irq_domain_free_fwnode(irq_handle);
return ret;
}
return 0;
}
#endif /* CONFIG_ACPI */
#ifdef CONFIG_OF
static int __init pch_lpc_of_init(struct device_node *node, struct device_node *parent)
{
struct fwnode_handle *irq_handle;
struct resource res;
int parent_irq, ret;
if (of_address_to_resource(node, 0, &res))
return -EINVAL;
parent_irq = irq_of_parse_and_map(node, 0);
if (!parent_irq) {
pr_err("Failed to get the parent IRQ for LPC IRQs\n");
return -EINVAL;
}
irq_handle = of_fwnode_handle(node);
ret = pch_lpc_init(res.start, resource_size(&res), irq_handle,
parent_irq);
if (ret) {
irq_dispose_mapping(parent_irq);
return ret;
}
return 0;
}
IRQCHIP_DECLARE(pch_lpc, "loongson,ls7a-lpc", pch_lpc_of_init);
#endif /* CONFIG_OF */
drivers/irqchip/irq-msi-lib.c
+3
View File
@@ -48,6 +48,9 @@ bool msi_lib_init_dev_msi_info(struct device *dev, struct irq_domain *domain,
return false;
}
if (WARN_ON_ONCE(!chip->irq_write_msi_msg))
return false;
required_flags = pops->required_flags;
/* Is the target domain bus token supported? */
drivers/irqchip/irq-mvebu-odmi.c
+1 -1
View File
@@ -217,7 +217,7 @@ err_unmap:
for (i = 0; i < odmis_count; i++) {
struct odmi_data *odmi = &odmis[i];
if (odmi->base && !IS_ERR(odmi->base))
if (!IS_ERR_OR_NULL(odmi->base))
iounmap(odmis[i].base);
}
bitmap_free(odmis_bm);
drivers/irqchip/irq-pic32-evic.c
+7 -1
View File
@@ -15,8 +15,10 @@
#include <linux/irq.h>
#include <linux/platform_data/pic32.h>
#ifdef CONFIG_MIPS
#include <asm/irq.h>
#include <asm/traps.h>
#endif
#define REG_INTCON 0x0000
#define REG_INTSTAT 0x0020
@@ -40,6 +42,7 @@ struct evic_chip_data {
static struct irq_domain *evic_irq_domain;
static void __iomem *evic_base;
#ifdef CONFIG_MIPS
asmlinkage void __weak plat_irq_dispatch(void)
{
unsigned int hwirq;
@@ -47,6 +50,9 @@ asmlinkage void __weak plat_irq_dispatch(void)
hwirq = readl(evic_base + REG_INTSTAT) & 0xFF;
do_domain_IRQ(evic_irq_domain, hwirq);
}
#else
static __maybe_unused void (*board_bind_eic_interrupt)(int irq, int regset);
#endif
static struct evic_chip_data *irqd_to_priv(struct irq_data *data)
{
@@ -196,7 +202,7 @@ static void __init pic32_ext_irq_of_init(struct irq_domain *domain)
of_property_for_each_u32(node, pname, hwirq) {
if (i >= ARRAY_SIZE(priv->ext_irqs)) {
pr_warn("More than %d external irq, skip rest\n",
pr_warn("More than %zu external irq, skip rest\n",
ARRAY_SIZE(priv->ext_irqs));
break;
}
drivers/irqchip/irq-renesas-rzg2l.c
+465 -117
View File
@@ -20,18 +20,21 @@
#include <linux/spinlock.h>
#include <linux/syscore_ops.h>
#define IRQC_NMI 0
#define IRQC_IRQ_START 1
#define IRQC_IRQ_COUNT 8
#define IRQC_TINT_START (IRQC_IRQ_START + IRQC_IRQ_COUNT)
#define IRQC_TINT_COUNT 32
#define IRQC_NUM_IRQ (IRQC_TINT_START + IRQC_TINT_COUNT)
#define IRQC_SHARED_IRQ_COUNT 8
#define IRQC_IRQ_SHARED_START (IRQC_IRQ_START + IRQC_SHARED_IRQ_COUNT)
#define NSCR 0x0
#define NITSR 0x4
#define ISCR 0x10
#define IITSR 0x14
#define TSCR 0x20
#define TITSR(n) (0x24 + (n) * 4)
#define TITSR0_MAX_INT 16
#define TITSEL_WIDTH 0x2
#define INTTSEL 0x2c
#define TSSR(n) (0x30 + ((n) * 4))
#define TIEN BIT(7)
#define TSSEL_SHIFT(n) (8 * (n))
@@ -43,6 +46,10 @@
#define TSSR_OFFSET(n) ((n) % 4)
#define TSSR_INDEX(n) ((n) / 4)
#define NSCR_NSTAT 0
#define NITSR_NTSEL_EDGE_FALLING 0
#define NITSR_NTSEL_EDGE_RISING 1
#define TITSR_TITSEL_EDGE_RISING 0
#define TITSR_TITSEL_EDGE_FALLING 1
#define TITSR_TITSEL_LEVEL_HIGH 2
@@ -55,33 +62,62 @@
#define IITSR_IITSEL_EDGE_BOTH 3
#define IITSR_IITSEL_MASK(n) IITSR_IITSEL((n), 3)
#define INTTSEL_TINTSEL(n) BIT(n)
#define INTTSEL_TINTSEL_START 24
#define TINT_EXTRACT_HWIRQ(x) FIELD_GET(GENMASK(15, 0), (x))
#define TINT_EXTRACT_GPIOINT(x) FIELD_GET(GENMASK(31, 16), (x))
/**
* struct rzg2l_irqc_reg_cache - registers cache (necessary for suspend/resume)
* @iitsr: IITSR register
* @titsr: TITSR registers
* @nitsr: NITSR register
* @iitsr: IITSR register
* @inttsel: INTTSEL register
* @titsr: TITSR registers
*/
struct rzg2l_irqc_reg_cache {
u32 nitsr;
u32 iitsr;
u32 inttsel;
u32 titsr[2];
};
/**
* struct rzg2l_hw_info - Interrupt Control Unit controller hardware info structure.
* @tssel_lut: TINT lookup table
* @irq_count: Number of IRQC interrupts
* @tint_start: Start of TINT interrupts
* @num_irq: Total Number of interrupts
* @shared_irq_cnt: Number of shared interrupts
*/
struct rzg2l_hw_info {
const u8 *tssel_lut;
unsigned int irq_count;
unsigned int tint_start;
unsigned int num_irq;
unsigned int shared_irq_cnt;
};
/**
* struct rzg2l_irqc_priv - IRQ controller private data structure
* @base: Controller's base address
* @irqchip: Pointer to struct irq_chip
* @irq_chip: Pointer to struct irq_chip for irq
* @tint_chip: Pointer to struct irq_chip for tint
* @fwspec: IRQ firmware specific data
* @lock: Lock to serialize access to hardware registers
* @info: Hardware specific data
* @cache: Registers cache for suspend/resume
* @used_irqs: Bitmap to manage the shared interrupts
*/
static struct rzg2l_irqc_priv {
void __iomem *base;
const struct irq_chip *irqchip;
struct irq_fwspec fwspec[IRQC_NUM_IRQ];
const struct irq_chip *irq_chip;
const struct irq_chip *tint_chip;
struct irq_fwspec *fwspec;
raw_spinlock_t lock;
struct rzg2l_hw_info info;
struct rzg2l_irqc_reg_cache cache;
DECLARE_BITMAP(used_irqs, IRQC_SHARED_IRQ_COUNT);
} *rzg2l_irqc_data;
static struct rzg2l_irqc_priv *irq_data_to_priv(struct irq_data *data)
@@ -89,6 +125,28 @@ static struct rzg2l_irqc_priv *irq_data_to_priv(struct irq_data *data)
return data->domain->host_data;
}
static void rzg2l_clear_nmi_int(struct rzg2l_irqc_priv *priv)
{
u32 bit = BIT(NSCR_NSTAT);
u32 reg;
/*
* No locking required as the register is not shared
* with other interrupts.
*
* Writing is allowed only when NSTAT is 1
*/
reg = readl_relaxed(priv->base + NSCR);
if (reg & bit) {
writel_relaxed(reg & ~bit, priv->base + NSCR);
/*
* Enforce that the posted write is flushed to prevent that the
* just handled interrupt is raised again.
*/
readl_relaxed(priv->base + NSCR);
}
}
static void rzg2l_clear_irq_int(struct rzg2l_irqc_priv *priv, unsigned int hwirq)
{
unsigned int hw_irq = hwirq - IRQC_IRQ_START;
@@ -114,7 +172,7 @@ static void rzg2l_clear_irq_int(struct rzg2l_irqc_priv *priv, unsigned int hwirq
static void rzg2l_clear_tint_int(struct rzg2l_irqc_priv *priv, unsigned int hwirq)
{
u32 bit = BIT(hwirq - IRQC_TINT_START);
u32 bit = BIT(hwirq - priv->info.tint_start);
u32 reg;
reg = readl_relaxed(priv->base + TSCR);
@@ -128,17 +186,33 @@ static void rzg2l_clear_tint_int(struct rzg2l_irqc_priv *priv, unsigned int hwir
}
}
static void rzg2l_irqc_eoi(struct irq_data *d)
static void rzg2l_irqc_nmi_eoi(struct irq_data *d)
{
struct rzg2l_irqc_priv *priv = irq_data_to_priv(d);
rzg2l_clear_nmi_int(priv);
irq_chip_eoi_parent(d);
}
static void rzg2l_irqc_irq_eoi(struct irq_data *d)
{
struct rzg2l_irqc_priv *priv = irq_data_to_priv(d);
unsigned int hw_irq = irqd_to_hwirq(d);
raw_spin_lock(&priv->lock);
if (hw_irq >= IRQC_IRQ_START && hw_irq <= IRQC_IRQ_COUNT)
scoped_guard(raw_spinlock, &priv->lock)
rzg2l_clear_irq_int(priv, hw_irq);
else if (hw_irq >= IRQC_TINT_START && hw_irq < IRQC_NUM_IRQ)
irq_chip_eoi_parent(d);
}
static void rzg2l_irqc_tint_eoi(struct irq_data *d)
{
struct rzg2l_irqc_priv *priv = irq_data_to_priv(d);
unsigned int hw_irq = irqd_to_hwirq(d);
scoped_guard(raw_spinlock, &priv->lock)
rzg2l_clear_tint_int(priv, hw_irq);
raw_spin_unlock(&priv->lock);
irq_chip_eoi_parent(d);
}
@@ -161,7 +235,7 @@ static void rzfive_irqc_unmask_irq_interrupt(struct rzg2l_irqc_priv *priv,
static void rzfive_irqc_mask_tint_interrupt(struct rzg2l_irqc_priv *priv,
unsigned int hwirq)
{
u32 bit = BIT(hwirq - IRQC_TINT_START);
u32 bit = BIT(hwirq - priv->info.tint_start);
writel_relaxed(readl_relaxed(priv->base + TMSK) | bit, priv->base + TMSK);
}
@@ -169,125 +243,170 @@ static void rzfive_irqc_mask_tint_interrupt(struct rzg2l_irqc_priv *priv,
static void rzfive_irqc_unmask_tint_interrupt(struct rzg2l_irqc_priv *priv,
unsigned int hwirq)
{
u32 bit = BIT(hwirq - IRQC_TINT_START);
u32 bit = BIT(hwirq - priv->info.tint_start);
writel_relaxed(readl_relaxed(priv->base + TMSK) & ~bit, priv->base + TMSK);
}
static void rzfive_irqc_mask(struct irq_data *d)
static void rzfive_irqc_irq_mask(struct irq_data *d)
{
struct rzg2l_irqc_priv *priv = irq_data_to_priv(d);
unsigned int hwirq = irqd_to_hwirq(d);
raw_spin_lock(&priv->lock);
if (hwirq >= IRQC_IRQ_START && hwirq <= IRQC_IRQ_COUNT)
scoped_guard(raw_spinlock, &priv->lock)
rzfive_irqc_mask_irq_interrupt(priv, hwirq);
else if (hwirq >= IRQC_TINT_START && hwirq < IRQC_NUM_IRQ)
rzfive_irqc_mask_tint_interrupt(priv, hwirq);
raw_spin_unlock(&priv->lock);
irq_chip_mask_parent(d);
}
static void rzfive_irqc_unmask(struct irq_data *d)
static void rzfive_irqc_tint_mask(struct irq_data *d)
{
struct rzg2l_irqc_priv *priv = irq_data_to_priv(d);
unsigned int hwirq = irqd_to_hwirq(d);
raw_spin_lock(&priv->lock);
if (hwirq >= IRQC_IRQ_START && hwirq <= IRQC_IRQ_COUNT)
scoped_guard(raw_spinlock, &priv->lock)
rzfive_irqc_mask_tint_interrupt(priv, hwirq);
irq_chip_mask_parent(d);
}
static void rzfive_irqc_irq_unmask(struct irq_data *d)
{
struct rzg2l_irqc_priv *priv = irq_data_to_priv(d);
unsigned int hwirq = irqd_to_hwirq(d);
scoped_guard(raw_spinlock, &priv->lock)
rzfive_irqc_unmask_irq_interrupt(priv, hwirq);
else if (hwirq >= IRQC_TINT_START && hwirq < IRQC_NUM_IRQ)
rzfive_irqc_unmask_tint_interrupt(priv, hwirq);
raw_spin_unlock(&priv->lock);
irq_chip_unmask_parent(d);
}
static void rzfive_tint_irq_endisable(struct irq_data *d, bool enable)
static void rzfive_irqc_tint_unmask(struct irq_data *d)
{
struct rzg2l_irqc_priv *priv = irq_data_to_priv(d);
unsigned int hwirq = irqd_to_hwirq(d);
if (hwirq >= IRQC_TINT_START && hwirq < IRQC_NUM_IRQ) {
u32 offset = hwirq - IRQC_TINT_START;
u32 tssr_offset = TSSR_OFFSET(offset);
u8 tssr_index = TSSR_INDEX(offset);
u32 reg;
scoped_guard(raw_spinlock, &priv->lock)
rzfive_irqc_unmask_tint_interrupt(priv, hwirq);
raw_spin_lock(&priv->lock);
if (enable)
rzfive_irqc_unmask_tint_interrupt(priv, hwirq);
else
rzfive_irqc_mask_tint_interrupt(priv, hwirq);
reg = readl_relaxed(priv->base + TSSR(tssr_index));
if (enable)
reg |= TIEN << TSSEL_SHIFT(tssr_offset);
else
reg &= ~(TIEN << TSSEL_SHIFT(tssr_offset));
writel_relaxed(reg, priv->base + TSSR(tssr_index));
raw_spin_unlock(&priv->lock);
} else {
raw_spin_lock(&priv->lock);
if (enable)
rzfive_irqc_unmask_irq_interrupt(priv, hwirq);
else
rzfive_irqc_mask_irq_interrupt(priv, hwirq);
raw_spin_unlock(&priv->lock);
}
irq_chip_unmask_parent(d);
}
static void rzfive_irq_endisable(struct irq_data *d, bool enable)
{
struct rzg2l_irqc_priv *priv = irq_data_to_priv(d);
unsigned int hwirq = irqd_to_hwirq(d);
guard(raw_spinlock)(&priv->lock);
if (enable)
rzfive_irqc_unmask_irq_interrupt(priv, hwirq);
else
rzfive_irqc_mask_irq_interrupt(priv, hwirq);
}
static void rzfive_tint_endisable(struct irq_data *d, bool enable)
{
struct rzg2l_irqc_priv *priv = irq_data_to_priv(d);
unsigned int hwirq = irqd_to_hwirq(d);
unsigned int offset = hwirq - priv->info.tint_start;
unsigned int tssr_offset = TSSR_OFFSET(offset);
unsigned int tssr_index = TSSR_INDEX(offset);
u32 reg;
guard(raw_spinlock)(&priv->lock);
if (enable)
rzfive_irqc_unmask_tint_interrupt(priv, hwirq);
else
rzfive_irqc_mask_tint_interrupt(priv, hwirq);
reg = readl_relaxed(priv->base + TSSR(tssr_index));
if (enable)
reg |= TIEN << TSSEL_SHIFT(tssr_offset);
else
reg &= ~(TIEN << TSSEL_SHIFT(tssr_offset));
writel_relaxed(reg, priv->base + TSSR(tssr_index));
}
static void rzfive_irqc_irq_disable(struct irq_data *d)
{
irq_chip_disable_parent(d);
rzfive_tint_irq_endisable(d, false);
rzfive_irq_endisable(d, false);
}
static void rzfive_irqc_irq_enable(struct irq_data *d)
{
rzfive_tint_irq_endisable(d, true);
rzfive_irq_endisable(d, true);
irq_chip_enable_parent(d);
}
static void rzfive_irqc_tint_disable(struct irq_data *d)
{
irq_chip_disable_parent(d);
rzfive_tint_endisable(d, false);
}
static void rzfive_irqc_tint_enable(struct irq_data *d)
{
rzfive_tint_endisable(d, true);
irq_chip_enable_parent(d);
}
static void rzg2l_tint_irq_endisable(struct irq_data *d, bool enable)
{
struct rzg2l_irqc_priv *priv = irq_data_to_priv(d);
unsigned int hw_irq = irqd_to_hwirq(d);
unsigned int offset = hw_irq - priv->info.tint_start;
unsigned int tssr_offset = TSSR_OFFSET(offset);
unsigned int tssr_index = TSSR_INDEX(offset);
u32 reg;
if (hw_irq >= IRQC_TINT_START && hw_irq < IRQC_NUM_IRQ) {
struct rzg2l_irqc_priv *priv = irq_data_to_priv(d);
u32 offset = hw_irq - IRQC_TINT_START;
u32 tssr_offset = TSSR_OFFSET(offset);
u8 tssr_index = TSSR_INDEX(offset);
u32 reg;
raw_spin_lock(&priv->lock);
reg = readl_relaxed(priv->base + TSSR(tssr_index));
if (enable)
reg |= TIEN << TSSEL_SHIFT(tssr_offset);
else
reg &= ~(TIEN << TSSEL_SHIFT(tssr_offset));
writel_relaxed(reg, priv->base + TSSR(tssr_index));
raw_spin_unlock(&priv->lock);
}
guard(raw_spinlock)(&priv->lock);
reg = readl_relaxed(priv->base + TSSR(tssr_index));
if (enable)
reg |= TIEN << TSSEL_SHIFT(tssr_offset);
else
reg &= ~(TIEN << TSSEL_SHIFT(tssr_offset));
writel_relaxed(reg, priv->base + TSSR(tssr_index));
}
static void rzg2l_irqc_irq_disable(struct irq_data *d)
static void rzg2l_irqc_tint_disable(struct irq_data *d)
{
irq_chip_disable_parent(d);
rzg2l_tint_irq_endisable(d, false);
}
static void rzg2l_irqc_irq_enable(struct irq_data *d)
static void rzg2l_irqc_tint_enable(struct irq_data *d)
{
rzg2l_tint_irq_endisable(d, true);
irq_chip_enable_parent(d);
}
static int rzg2l_nmi_set_type(struct irq_data *d, unsigned int type)
{
struct rzg2l_irqc_priv *priv = irq_data_to_priv(d);
u32 sense;
switch (type & IRQ_TYPE_SENSE_MASK) {
case IRQ_TYPE_EDGE_FALLING:
sense = NITSR_NTSEL_EDGE_FALLING;
break;
case IRQ_TYPE_EDGE_RISING:
sense = NITSR_NTSEL_EDGE_RISING;
break;
default:
return -EINVAL;
}
writel_relaxed(sense, priv->base + NITSR);
return 0;
}
static int rzg2l_irq_set_type(struct irq_data *d, unsigned int type)
{
struct rzg2l_irqc_priv *priv = irq_data_to_priv(d);
unsigned int hwirq = irqd_to_hwirq(d);
u32 iitseln = hwirq - IRQC_IRQ_START;
unsigned int iitseln = hwirq - IRQC_IRQ_START;
bool clear_irq_int = false;
u16 sense, tmp;
unsigned int sense, tmp;
switch (type & IRQ_TYPE_SENSE_MASK) {
case IRQ_TYPE_LEVEL_LOW:
@@ -313,14 +432,13 @@ static int rzg2l_irq_set_type(struct irq_data *d, unsigned int type)
return -EINVAL;
}
raw_spin_lock(&priv->lock);
guard(raw_spinlock)(&priv->lock);
tmp = readl_relaxed(priv->base + IITSR);
tmp &= ~IITSR_IITSEL_MASK(iitseln);
tmp |= IITSR_IITSEL(iitseln, sense);
if (clear_irq_int)
rzg2l_clear_irq_int(priv, hwirq);
writel_relaxed(tmp, priv->base + IITSR);
raw_spin_unlock(&priv->lock);
return 0;
}
@@ -331,6 +449,11 @@ static u32 rzg2l_disable_tint_and_set_tint_source(struct irq_data *d, struct rzg
u32 tint = (u32)(uintptr_t)irq_data_get_irq_chip_data(d);
u32 tien = reg & (TIEN << TSSEL_SHIFT(tssr_offset));
if (priv->info.tssel_lut)
tint = priv->info.tssel_lut[tint];
else
tint = (u32)(uintptr_t)irq_data_get_irq_chip_data(d);
/* Clear the relevant byte in reg */
reg &= ~(TSSEL_MASK << TSSEL_SHIFT(tssr_offset));
/* Set TINT and leave TIEN clear */
@@ -344,10 +467,10 @@ static int rzg2l_tint_set_edge(struct irq_data *d, unsigned int type)
{
struct rzg2l_irqc_priv *priv = irq_data_to_priv(d);
unsigned int hwirq = irqd_to_hwirq(d);
u32 titseln = hwirq - IRQC_TINT_START;
u32 tssr_offset = TSSR_OFFSET(titseln);
u8 tssr_index = TSSR_INDEX(titseln);
u8 index, sense;
unsigned int titseln = hwirq - priv->info.tint_start;
unsigned int tssr_offset = TSSR_OFFSET(titseln);
unsigned int tssr_index = TSSR_INDEX(titseln);
unsigned int index, sense;
u32 reg, tssr;
switch (type & IRQ_TYPE_SENSE_MASK) {
@@ -383,15 +506,31 @@ static int rzg2l_tint_set_edge(struct irq_data *d, unsigned int type)
return 0;
}
static int rzg2l_irqc_set_type(struct irq_data *d, unsigned int type)
static int rzg2l_irqc_irq_set_type(struct irq_data *d, unsigned int type)
{
unsigned int hw_irq = irqd_to_hwirq(d);
int ret = -EINVAL;
int ret = rzg2l_irq_set_type(d, type);
if (hw_irq >= IRQC_IRQ_START && hw_irq <= IRQC_IRQ_COUNT)
ret = rzg2l_irq_set_type(d, type);
else if (hw_irq >= IRQC_TINT_START && hw_irq < IRQC_NUM_IRQ)
ret = rzg2l_tint_set_edge(d, type);
if (ret)
return ret;
return irq_chip_set_type_parent(d, IRQ_TYPE_LEVEL_HIGH);
}
static int rzg2l_irqc_tint_set_type(struct irq_data *d, unsigned int type)
{
int ret = rzg2l_tint_set_edge(d, type);
if (ret)
return ret;
return irq_chip_set_type_parent(d, IRQ_TYPE_LEVEL_HIGH);
}
static int rzg2l_irqc_nmi_set_type(struct irq_data *d, unsigned int type)
{
int ret;
ret = rzg2l_nmi_set_type(d, type);
if (ret)
return ret;
@@ -403,7 +542,10 @@ static int rzg2l_irqc_irq_suspend(void *data)
struct rzg2l_irqc_reg_cache *cache = &rzg2l_irqc_data->cache;
void __iomem *base = rzg2l_irqc_data->base;
cache->nitsr = readl_relaxed(base + NITSR);
cache->iitsr = readl_relaxed(base + IITSR);
if (rzg2l_irqc_data->info.shared_irq_cnt)
cache->inttsel = readl_relaxed(base + INTTSEL);
for (u8 i = 0; i < 2; i++)
cache->titsr[i] = readl_relaxed(base + TITSR(i));
@@ -422,7 +564,10 @@ static void rzg2l_irqc_irq_resume(void *data)
*/
for (u8 i = 0; i < 2; i++)
writel_relaxed(cache->titsr[i], base + TITSR(i));
if (rzg2l_irqc_data->info.shared_irq_cnt)
writel_relaxed(cache->inttsel, base + INTTSEL);
writel_relaxed(cache->iitsr, base + IITSR);
writel_relaxed(cache->nitsr, base + NITSR);
}
static const struct syscore_ops rzg2l_irqc_syscore_ops = {
@@ -434,44 +579,162 @@ static struct syscore rzg2l_irqc_syscore = {
.ops = &rzg2l_irqc_syscore_ops,
};
static const struct irq_chip rzg2l_irqc_chip = {
static const struct irq_chip rzg2l_irqc_nmi_chip = {
.name = "rzg2l-irqc",
.irq_eoi = rzg2l_irqc_eoi,
.irq_eoi = rzg2l_irqc_nmi_eoi,
.irq_mask = irq_chip_mask_parent,
.irq_unmask = irq_chip_unmask_parent,
.irq_disable = rzg2l_irqc_irq_disable,
.irq_enable = rzg2l_irqc_irq_enable,
.irq_disable = irq_chip_disable_parent,
.irq_enable = irq_chip_enable_parent,
.irq_get_irqchip_state = irq_chip_get_parent_state,
.irq_set_irqchip_state = irq_chip_set_parent_state,
.irq_retrigger = irq_chip_retrigger_hierarchy,
.irq_set_type = rzg2l_irqc_set_type,
.irq_set_type = rzg2l_irqc_nmi_set_type,
.irq_set_affinity = irq_chip_set_affinity_parent,
.flags = IRQCHIP_MASK_ON_SUSPEND |
IRQCHIP_SET_TYPE_MASKED |
IRQCHIP_SKIP_SET_WAKE,
};
static const struct irq_chip rzfive_irqc_chip = {
static const struct irq_chip rzg2l_irqc_irq_chip = {
.name = "rzg2l-irqc",
.irq_eoi = rzg2l_irqc_irq_eoi,
.irq_mask = irq_chip_mask_parent,
.irq_unmask = irq_chip_unmask_parent,
.irq_disable = irq_chip_disable_parent,
.irq_enable = irq_chip_enable_parent,
.irq_get_irqchip_state = irq_chip_get_parent_state,
.irq_set_irqchip_state = irq_chip_set_parent_state,
.irq_retrigger = irq_chip_retrigger_hierarchy,
.irq_set_type = rzg2l_irqc_irq_set_type,
.irq_set_affinity = irq_chip_set_affinity_parent,
.flags = IRQCHIP_MASK_ON_SUSPEND |
IRQCHIP_SET_TYPE_MASKED |
IRQCHIP_SKIP_SET_WAKE,
};
static const struct irq_chip rzg2l_irqc_tint_chip = {
.name = "rzg2l-irqc",
.irq_eoi = rzg2l_irqc_tint_eoi,
.irq_mask = irq_chip_mask_parent,
.irq_unmask = irq_chip_unmask_parent,
.irq_disable = rzg2l_irqc_tint_disable,
.irq_enable = rzg2l_irqc_tint_enable,
.irq_get_irqchip_state = irq_chip_get_parent_state,
.irq_set_irqchip_state = irq_chip_set_parent_state,
.irq_retrigger = irq_chip_retrigger_hierarchy,
.irq_set_type = rzg2l_irqc_tint_set_type,
.irq_set_affinity = irq_chip_set_affinity_parent,
.flags = IRQCHIP_MASK_ON_SUSPEND |
IRQCHIP_SET_TYPE_MASKED |
IRQCHIP_SKIP_SET_WAKE,
};
static const struct irq_chip rzfive_irqc_irq_chip = {
.name = "rzfive-irqc",
.irq_eoi = rzg2l_irqc_eoi,
.irq_mask = rzfive_irqc_mask,
.irq_unmask = rzfive_irqc_unmask,
.irq_eoi = rzg2l_irqc_irq_eoi,
.irq_mask = rzfive_irqc_irq_mask,
.irq_unmask = rzfive_irqc_irq_unmask,
.irq_disable = rzfive_irqc_irq_disable,
.irq_enable = rzfive_irqc_irq_enable,
.irq_get_irqchip_state = irq_chip_get_parent_state,
.irq_set_irqchip_state = irq_chip_set_parent_state,
.irq_retrigger = irq_chip_retrigger_hierarchy,
.irq_set_type = rzg2l_irqc_set_type,
.irq_set_type = rzg2l_irqc_irq_set_type,
.irq_set_affinity = irq_chip_set_affinity_parent,
.flags = IRQCHIP_MASK_ON_SUSPEND |
IRQCHIP_SET_TYPE_MASKED |
IRQCHIP_SKIP_SET_WAKE,
};
static const struct irq_chip rzfive_irqc_tint_chip = {
.name = "rzfive-irqc",
.irq_eoi = rzg2l_irqc_tint_eoi,
.irq_mask = rzfive_irqc_tint_mask,
.irq_unmask = rzfive_irqc_tint_unmask,
.irq_disable = rzfive_irqc_tint_disable,
.irq_enable = rzfive_irqc_tint_enable,
.irq_get_irqchip_state = irq_chip_get_parent_state,
.irq_set_irqchip_state = irq_chip_set_parent_state,
.irq_retrigger = irq_chip_retrigger_hierarchy,
.irq_set_type = rzg2l_irqc_tint_set_type,
.irq_set_affinity = irq_chip_set_affinity_parent,
.flags = IRQCHIP_MASK_ON_SUSPEND |
IRQCHIP_SET_TYPE_MASKED |
IRQCHIP_SKIP_SET_WAKE,
};
static bool rzg2l_irqc_is_shared_irqc(const struct rzg2l_hw_info info, unsigned int hw_irq)
{
return ((hw_irq >= (info.tint_start - info.shared_irq_cnt)) && hw_irq < info.tint_start);
}
static bool rzg2l_irqc_is_shared_tint(const struct rzg2l_hw_info info, unsigned int hw_irq)
{
return ((hw_irq >= (info.num_irq - info.shared_irq_cnt)) && hw_irq < info.num_irq);
}
static bool rzg2l_irqc_is_shared_and_get_irq_num(struct rzg2l_irqc_priv *priv,
irq_hw_number_t hwirq, unsigned int *irq_num)
{
bool is_shared = false;
if (rzg2l_irqc_is_shared_irqc(priv->info, hwirq)) {
*irq_num = hwirq - IRQC_IRQ_SHARED_START;
is_shared = true;
} else if (rzg2l_irqc_is_shared_tint(priv->info, hwirq)) {
*irq_num = hwirq - IRQC_TINT_COUNT - IRQC_IRQ_SHARED_START;
is_shared = true;
}
return is_shared;
}
static void rzg2l_irqc_set_inttsel(struct rzg2l_irqc_priv *priv, unsigned int offset,
unsigned int select_irq)
{
u32 reg;
guard(raw_spinlock_irqsave)(&priv->lock);
reg = readl_relaxed(priv->base + INTTSEL);
if (select_irq)
reg |= INTTSEL_TINTSEL(offset);
else
reg &= ~INTTSEL_TINTSEL(offset);
writel_relaxed(reg, priv->base + INTTSEL);
}
static int rzg2l_irqc_shared_irq_alloc(struct rzg2l_irqc_priv *priv, irq_hw_number_t hwirq)
{
unsigned int irq_num;
if (rzg2l_irqc_is_shared_and_get_irq_num(priv, hwirq, &irq_num)) {
if (test_and_set_bit(irq_num, priv->used_irqs))
return -EBUSY;
if (hwirq < priv->info.tint_start)
rzg2l_irqc_set_inttsel(priv, INTTSEL_TINTSEL_START + irq_num, 1);
else
rzg2l_irqc_set_inttsel(priv, INTTSEL_TINTSEL_START + irq_num, 0);
}
return 0;
}
static void rzg2l_irqc_shared_irq_free(struct rzg2l_irqc_priv *priv, irq_hw_number_t hwirq)
{
unsigned int irq_num;
if (rzg2l_irqc_is_shared_and_get_irq_num(priv, hwirq, &irq_num) &&
test_and_clear_bit(irq_num, priv->used_irqs))
rzg2l_irqc_set_inttsel(priv, INTTSEL_TINTSEL_START + irq_num, 0);
}
static int rzg2l_irqc_alloc(struct irq_domain *domain, unsigned int virq,
unsigned int nr_irqs, void *arg)
{
struct rzg2l_irqc_priv *priv = domain->host_data;
const struct irq_chip *chip;
unsigned long tint = 0;
irq_hw_number_t hwirq;
unsigned int type;
@@ -488,28 +751,57 @@ static int rzg2l_irqc_alloc(struct irq_domain *domain, unsigned int virq,
* from 16-31 bits. TINT from the pinctrl driver needs to be programmed
* in IRQC registers to enable a given gpio pin as interrupt.
*/
if (hwirq > IRQC_IRQ_COUNT) {
if (hwirq == IRQC_NMI) {
chip = &rzg2l_irqc_nmi_chip;
} else if (hwirq > priv->info.irq_count) {
tint = TINT_EXTRACT_GPIOINT(hwirq);
hwirq = TINT_EXTRACT_HWIRQ(hwirq);
if (hwirq < IRQC_TINT_START)
return -EINVAL;
chip = priv->tint_chip;
} else {
chip = priv->irq_chip;
}
if (hwirq > (IRQC_NUM_IRQ - 1))
if (hwirq >= priv->info.num_irq)
return -EINVAL;
ret = irq_domain_set_hwirq_and_chip(domain, virq, hwirq, priv->irqchip,
(void *)(uintptr_t)tint);
if (ret)
return ret;
if (priv->info.shared_irq_cnt) {
ret = rzg2l_irqc_shared_irq_alloc(priv, hwirq);
if (ret)
return ret;
}
return irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, &priv->fwspec[hwirq]);
ret = irq_domain_set_hwirq_and_chip(domain, virq, hwirq, chip, (void *)(uintptr_t)tint);
if (ret)
goto shared_irq_free;
ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, &priv->fwspec[hwirq]);
if (ret)
goto shared_irq_free;
return 0;
shared_irq_free:
if (priv->info.shared_irq_cnt)
rzg2l_irqc_shared_irq_free(priv, hwirq);
return ret;
}
static void rzg2l_irqc_free(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs)
{
struct irq_data *d = irq_domain_get_irq_data(domain, virq);
struct rzg2l_irqc_priv *priv = domain->host_data;
irq_hw_number_t hwirq = irqd_to_hwirq(d);
irq_domain_free_irqs_common(domain, virq, nr_irqs);
if (priv->info.shared_irq_cnt)
rzg2l_irqc_shared_irq_free(priv, hwirq);
}
static const struct irq_domain_ops rzg2l_irqc_domain_ops = {
.alloc = rzg2l_irqc_alloc,
.free = irq_domain_free_irqs_common,
.free = rzg2l_irqc_free,
.translate = irq_domain_translate_twocell,
};
@@ -520,7 +812,7 @@ static int rzg2l_irqc_parse_interrupts(struct rzg2l_irqc_priv *priv,
unsigned int i;
int ret;
for (i = 0; i < IRQC_NUM_IRQ; i++) {
for (i = 0; i < priv->info.num_irq; i++) {
ret = of_irq_parse_one(np, i, &map);
if (ret)
return ret;
@@ -532,7 +824,9 @@ static int rzg2l_irqc_parse_interrupts(struct rzg2l_irqc_priv *priv,
}
static int rzg2l_irqc_common_probe(struct platform_device *pdev, struct device_node *parent,
const struct irq_chip *irq_chip)
const struct irq_chip *irq_chip,
const struct irq_chip *tint_chip,
const struct rzg2l_hw_info info)
{
struct irq_domain *irq_domain, *parent_domain;
struct device_node *node = pdev->dev.of_node;
@@ -548,12 +842,20 @@ static int rzg2l_irqc_common_probe(struct platform_device *pdev, struct device_n
if (!rzg2l_irqc_data)
return -ENOMEM;
rzg2l_irqc_data->irqchip = irq_chip;
rzg2l_irqc_data->irq_chip = irq_chip;
rzg2l_irqc_data->tint_chip = tint_chip;
rzg2l_irqc_data->base = devm_of_iomap(dev, dev->of_node, 0, NULL);
if (IS_ERR(rzg2l_irqc_data->base))
return PTR_ERR(rzg2l_irqc_data->base);
rzg2l_irqc_data->info = info;
rzg2l_irqc_data->fwspec = devm_kcalloc(&pdev->dev, info.num_irq,
sizeof(*rzg2l_irqc_data->fwspec), GFP_KERNEL);
if (!rzg2l_irqc_data->fwspec)
return -ENOMEM;
ret = rzg2l_irqc_parse_interrupts(rzg2l_irqc_data, node);
if (ret)
return dev_err_probe(dev, ret, "cannot parse interrupts: %d\n", ret);
@@ -574,10 +876,10 @@ static int rzg2l_irqc_common_probe(struct platform_device *pdev, struct device_n
raw_spin_lock_init(&rzg2l_irqc_data->lock);
irq_domain = irq_domain_create_hierarchy(parent_domain, 0, IRQC_NUM_IRQ, dev_fwnode(dev),
irq_domain = irq_domain_create_hierarchy(parent_domain, 0, info.num_irq, dev_fwnode(dev),
&rzg2l_irqc_domain_ops, rzg2l_irqc_data);
if (!irq_domain) {
pm_runtime_put(dev);
pm_runtime_put_sync(dev);
return -ENOMEM;
}
@@ -586,18 +888,64 @@ static int rzg2l_irqc_common_probe(struct platform_device *pdev, struct device_n
return 0;
}
/* Mapping based on port index on Table 4.2-1 and GPIOINT on Table 4.6-7 */
static const u8 rzg3l_tssel_lut[] = {
83, 84, /* P20-P21 */
7, 8, 9, 10, 11, 12, 13, /* P30-P36 */
85, 86, 87, 88, 89, 90, 91, /* P50-P56 */
92, 93, 94, 95, 96, 97, 98, /* P60-P66 */
99, 100, 101, 102, 103, 104, 105, 106, /* P70-P77 */
107, 108, 109, 110, 111, 112, /* P80-P85 */
45, 46, 47, 48, 49, 50, 51, 52, /* PA0-PA7 */
53, 54, 55, 56, 57, 58, 59, 60, /* PB0-PB7 */
61, 62, 63, /* PC0-PC2 */
64, 65, 66, 67, 68, 69, 70, 71, /* PD0-PD7 */
72, 73, 74, 75, 76, 77, 78, 79, /* PE0-PE7 */
80, 81, 82, /* PF0-PF2 */
27, 28, 29, 30, 31, 32, 33, 34, /* PG0-PG7 */
35, 36, 37, 38, 39, 40, /* PH0-PH5 */
2, 3, 4, 5, 6, /* PJ0-PJ4 */
41, 42, 43, 44, /* PK0-PK3 */
14, 15, 16, 17, 26, /* PL0-PL4 */
18, 19, 20, 21, 22, 23, 24, 25, /* PM0-PM7 */
0, 1 /* PS0-PS1 */
};
static const struct rzg2l_hw_info rzg3l_hw_params = {
.tssel_lut = rzg3l_tssel_lut,
.irq_count = 16,
.tint_start = IRQC_IRQ_START + 16,
.num_irq = IRQC_IRQ_START + 16 + IRQC_TINT_COUNT,
.shared_irq_cnt = IRQC_SHARED_IRQ_COUNT,
};
static const struct rzg2l_hw_info rzg2l_hw_params = {
.irq_count = 8,
.tint_start = IRQC_IRQ_START + 8,
.num_irq = IRQC_IRQ_START + 8 + IRQC_TINT_COUNT,
};
static int rzg2l_irqc_probe(struct platform_device *pdev, struct device_node *parent)
{
return rzg2l_irqc_common_probe(pdev, parent, &rzg2l_irqc_chip);
return rzg2l_irqc_common_probe(pdev, parent, &rzg2l_irqc_irq_chip, &rzg2l_irqc_tint_chip,
rzg2l_hw_params);
}
static int rzg3l_irqc_probe(struct platform_device *pdev, struct device_node *parent)
{
return rzg2l_irqc_common_probe(pdev, parent, &rzg2l_irqc_irq_chip, &rzg2l_irqc_tint_chip,
rzg3l_hw_params);
}
static int rzfive_irqc_probe(struct platform_device *pdev, struct device_node *parent)
{
return rzg2l_irqc_common_probe(pdev, parent, &rzfive_irqc_chip);
return rzg2l_irqc_common_probe(pdev, parent, &rzfive_irqc_irq_chip, &rzfive_irqc_tint_chip,
rzg2l_hw_params);
}
IRQCHIP_PLATFORM_DRIVER_BEGIN(rzg2l_irqc)
IRQCHIP_MATCH("renesas,rzg2l-irqc", rzg2l_irqc_probe)
IRQCHIP_MATCH("renesas,r9a08g046-irqc", rzg3l_irqc_probe)
IRQCHIP_MATCH("renesas,r9a07g043f-irqc", rzfive_irqc_probe)
IRQCHIP_PLATFORM_DRIVER_END(rzg2l_irqc)
MODULE_AUTHOR("Lad Prabhakar <prabhakar.mahadev-lad.rj@bp.renesas.com>");
drivers/irqchip/irq-renesas-rzv2h.c
+371 -102
View File
@@ -12,6 +12,7 @@
#include <linux/bitfield.h>
#include <linux/cleanup.h>
#include <linux/err.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/irqchip.h>
#include <linux/irqchip/irq-renesas-rzv2h.h>
@@ -25,9 +26,17 @@
/* DT "interrupts" indexes */
#define ICU_IRQ_START 1
#define ICU_IRQ_COUNT 16
#define ICU_TINT_START (ICU_IRQ_START + ICU_IRQ_COUNT)
#define ICU_IRQ_LAST (ICU_IRQ_START + ICU_IRQ_COUNT - 1)
#define ICU_TINT_START (ICU_IRQ_LAST + 1)
#define ICU_TINT_COUNT 32
#define ICU_NUM_IRQ (ICU_TINT_START + ICU_TINT_COUNT)
#define ICU_TINT_LAST (ICU_TINT_START + ICU_TINT_COUNT - 1)
#define ICU_CA55_INT_START (ICU_TINT_LAST + 1)
#define ICU_CA55_INT_COUNT 4
#define ICU_CA55_INT_LAST (ICU_CA55_INT_START + ICU_CA55_INT_COUNT - 1)
#define ICU_ERR_INT_START (ICU_CA55_INT_LAST + 1)
#define ICU_ERR_INT_COUNT 1
#define ICU_ERR_INT_LAST (ICU_ERR_INT_START + ICU_ERR_INT_COUNT - 1)
#define ICU_NUM_IRQ (ICU_ERR_INT_LAST + 1)
/* Registers */
#define ICU_NSCNT 0x00
@@ -40,6 +49,15 @@
#define ICU_TSCLR 0x24
#define ICU_TITSR(k) (0x28 + (k) * 4)
#define ICU_TSSR(k) (0x30 + (k) * 4)
#define ICU_BEISR(k) (0x70 + (k) * 4)
#define ICU_BECLR(k) (0x80 + (k) * 4)
#define ICU_EREISR(k) (0x90 + (k) * 4)
#define ICU_ERCLR(k) (0xE0 + (k) * 4)
#define ICU_SWINT 0x130
#define ICU_ERINTA55CTL(k) (0x338 + (k) * 4)
#define ICU_ERINTA55CRL(k) (0x348 + (k) * 4)
#define ICU_ERINTA55MSK(k) (0x358 + (k) * 4)
#define ICU_SWPE 0x370
#define ICU_DMkSELy(k, y) (0x420 + (k) * 0x20 + (y) * 4)
#define ICU_DMACKSELk(k) (0x500 + (k) * 4)
@@ -90,6 +108,10 @@
#define ICU_RZG3E_TSSEL_MAX_VAL 0x8c
#define ICU_RZV2H_TSSEL_MAX_VAL 0x55
#define ICU_SWPE_NUM 16
#define ICU_NUM_BE 4
#define ICU_NUM_A55ERR 4
/**
* struct rzv2h_irqc_reg_cache - registers cache (necessary for suspend/resume)
* @nitsr: ICU_NITSR register
@@ -108,12 +130,16 @@ struct rzv2h_irqc_reg_cache {
* @t_offs: TINT offset
* @max_tssel: TSSEL max value
* @field_width: TSSR field width
* @ecc_start: Start index of ECC RAM interrupts
* @ecc_end: End index of ECC RAM interrupts
*/
struct rzv2h_hw_info {
const u8 *tssel_lut;
u16 t_offs;
u8 max_tssel;
u8 field_width;
u8 ecc_start;
u8 ecc_end;
};
/* DMAC */
@@ -167,32 +193,47 @@ static inline struct rzv2h_icu_priv *irq_data_to_priv(struct irq_data *data)
return data->domain->host_data;
}
static void rzv2h_icu_eoi(struct irq_data *d)
static void rzv2h_icu_tint_eoi(struct irq_data *d)
{
struct rzv2h_icu_priv *priv = irq_data_to_priv(d);
unsigned int hw_irq = irqd_to_hwirq(d);
unsigned int tintirq_nr;
u32 bit;
scoped_guard(raw_spinlock, &priv->lock) {
if (hw_irq >= ICU_TINT_START) {
tintirq_nr = hw_irq - ICU_TINT_START;
bit = BIT(tintirq_nr);
if (!irqd_is_level_type(d))
writel_relaxed(bit, priv->base + priv->info->t_offs + ICU_TSCLR);
} else if (hw_irq >= ICU_IRQ_START) {
tintirq_nr = hw_irq - ICU_IRQ_START;
bit = BIT(tintirq_nr);
if (!irqd_is_level_type(d))
writel_relaxed(bit, priv->base + ICU_ISCLR);
} else {
writel_relaxed(ICU_NSCLR_NCLR, priv->base + ICU_NSCLR);
}
if (!irqd_is_level_type(d)) {
tintirq_nr = hw_irq - ICU_TINT_START;
bit = BIT(tintirq_nr);
writel_relaxed(bit, priv->base + priv->info->t_offs + ICU_TSCLR);
}
irq_chip_eoi_parent(d);
}
static void rzv2h_icu_irq_eoi(struct irq_data *d)
{
struct rzv2h_icu_priv *priv = irq_data_to_priv(d);
unsigned int hw_irq = irqd_to_hwirq(d);
unsigned int tintirq_nr;
u32 bit;
if (!irqd_is_level_type(d)) {
tintirq_nr = hw_irq - ICU_IRQ_START;
bit = BIT(tintirq_nr);
writel_relaxed(bit, priv->base + ICU_ISCLR);
}
irq_chip_eoi_parent(d);
}
static void rzv2h_icu_nmi_eoi(struct irq_data *d)
{
struct rzv2h_icu_priv *priv = irq_data_to_priv(d);
writel_relaxed(ICU_NSCLR_NCLR, priv->base + ICU_NSCLR);
irq_chip_eoi_parent(d);
}
static void rzv2h_tint_irq_endisable(struct irq_data *d, bool enable)
{
struct rzv2h_icu_priv *priv = irq_data_to_priv(d);
@@ -200,9 +241,6 @@ static void rzv2h_tint_irq_endisable(struct irq_data *d, bool enable)
u32 tint_nr, tssel_n, k, tssr;
u8 nr_tint;
if (hw_irq < ICU_TINT_START)
return;
tint_nr = hw_irq - ICU_TINT_START;
nr_tint = 32 / priv->info->field_width;
k = tint_nr / nr_tint;
@@ -225,13 +263,13 @@ static void rzv2h_tint_irq_endisable(struct irq_data *d, bool enable)
writel_relaxed(BIT(tint_nr), priv->base + priv->info->t_offs + ICU_TSCLR);
}
static void rzv2h_icu_irq_disable(struct irq_data *d)
static void rzv2h_icu_tint_disable(struct irq_data *d)
{
irq_chip_disable_parent(d);
rzv2h_tint_irq_endisable(d, false);
}
static void rzv2h_icu_irq_enable(struct irq_data *d)
static void rzv2h_icu_tint_enable(struct irq_data *d)
{
rzv2h_tint_irq_endisable(d, true);
irq_chip_enable_parent(d);
@@ -257,7 +295,7 @@ static int rzv2h_nmi_set_type(struct irq_data *d, unsigned int type)
writel_relaxed(sense, priv->base + ICU_NITSR);
return 0;
return irq_chip_set_type_parent(d, IRQ_TYPE_LEVEL_HIGH);
}
static void rzv2h_clear_irq_int(struct rzv2h_icu_priv *priv, unsigned int hwirq)
@@ -307,14 +345,15 @@ static int rzv2h_irq_set_type(struct irq_data *d, unsigned int type)
return -EINVAL;
}
guard(raw_spinlock)(&priv->lock);
iitsr = readl_relaxed(priv->base + ICU_IITSR);
iitsr &= ~ICU_IITSR_IITSEL_MASK(irq_nr);
iitsr |= ICU_IITSR_IITSEL_PREP(sense, irq_nr);
rzv2h_clear_irq_int(priv, hwirq);
writel_relaxed(iitsr, priv->base + ICU_IITSR);
scoped_guard(raw_spinlock, &priv->lock) {
iitsr = readl_relaxed(priv->base + ICU_IITSR);
iitsr &= ~ICU_IITSR_IITSEL_MASK(irq_nr);
iitsr |= ICU_IITSR_IITSEL_PREP(sense, irq_nr);
rzv2h_clear_irq_int(priv, hwirq);
writel_relaxed(iitsr, priv->base + ICU_IITSR);
}
return 0;
return irq_chip_set_type_parent(d, IRQ_TYPE_LEVEL_HIGH);
}
static void rzv2h_clear_tint_int(struct rzv2h_icu_priv *priv, unsigned int hwirq)
@@ -389,49 +428,82 @@ static int rzv2h_tint_set_type(struct irq_data *d, unsigned int type)
titsr_k = ICU_TITSR_K(tint_nr);
titsel_n = ICU_TITSR_TITSEL_N(tint_nr);
guard(raw_spinlock)(&priv->lock);
scoped_guard(raw_spinlock, &priv->lock) {
tssr = readl_relaxed(priv->base + priv->info->t_offs + ICU_TSSR(tssr_k));
titsr = readl_relaxed(priv->base + priv->info->t_offs + ICU_TITSR(titsr_k));
tssr = readl_relaxed(priv->base + priv->info->t_offs + ICU_TSSR(tssr_k));
titsr = readl_relaxed(priv->base + priv->info->t_offs + ICU_TITSR(titsr_k));
tssr_cur = field_get(ICU_TSSR_TSSEL_MASK(tssel_n, priv->info->field_width), tssr);
titsr_cur = field_get(ICU_TITSR_TITSEL_MASK(titsel_n), titsr);
if (tssr_cur == tint && titsr_cur == sense)
goto set_parent_type;
tssr_cur = field_get(ICU_TSSR_TSSEL_MASK(tssel_n, priv->info->field_width), tssr);
titsr_cur = field_get(ICU_TITSR_TITSEL_MASK(titsel_n), titsr);
if (tssr_cur == tint && titsr_cur == sense)
tssr &= ~(ICU_TSSR_TSSEL_MASK(tssel_n, priv->info->field_width) | tien);
tssr |= ICU_TSSR_TSSEL_PREP(tint, tssel_n, priv->info->field_width);
writel_relaxed(tssr, priv->base + priv->info->t_offs + ICU_TSSR(tssr_k));
titsr &= ~ICU_TITSR_TITSEL_MASK(titsel_n);
titsr |= ICU_TITSR_TITSEL_PREP(sense, titsel_n);
writel_relaxed(titsr, priv->base + priv->info->t_offs + ICU_TITSR(titsr_k));
rzv2h_clear_tint_int(priv, hwirq);
writel_relaxed(tssr | tien, priv->base + priv->info->t_offs + ICU_TSSR(tssr_k));
}
set_parent_type:
return irq_chip_set_type_parent(d, IRQ_TYPE_LEVEL_HIGH);
}
static int rzv2h_icu_swint_set_irqchip_state(struct irq_data *d, enum irqchip_irq_state which,
bool state)
{
unsigned int hwirq = irqd_to_hwirq(d);
struct rzv2h_icu_priv *priv;
unsigned int bit;
if (which != IRQCHIP_STATE_PENDING)
return irq_chip_set_parent_state(d, which, state);
if (!state)
return 0;
tssr &= ~(ICU_TSSR_TSSEL_MASK(tssel_n, priv->info->field_width) | tien);
tssr |= ICU_TSSR_TSSEL_PREP(tint, tssel_n, priv->info->field_width);
priv = irq_data_to_priv(d);
bit = BIT(hwirq - ICU_CA55_INT_START);
writel_relaxed(tssr, priv->base + priv->info->t_offs + ICU_TSSR(tssr_k));
titsr &= ~ICU_TITSR_TITSEL_MASK(titsel_n);
titsr |= ICU_TITSR_TITSEL_PREP(sense, titsel_n);
writel_relaxed(titsr, priv->base + priv->info->t_offs + ICU_TITSR(titsr_k));
rzv2h_clear_tint_int(priv, hwirq);
writel_relaxed(tssr | tien, priv->base + priv->info->t_offs + ICU_TSSR(tssr_k));
/* Trigger the software interrupt */
writel_relaxed(bit, priv->base + ICU_SWINT);
return 0;
}
static int rzv2h_icu_set_type(struct irq_data *d, unsigned int type)
static int rzv2h_icu_swpe_set_irqchip_state(struct irq_data *d, enum irqchip_irq_state which,
bool state)
{
unsigned int hw_irq = irqd_to_hwirq(d);
int ret;
struct rzv2h_icu_priv *priv;
unsigned int bit;
static u8 swpe;
if (hw_irq >= ICU_TINT_START)
ret = rzv2h_tint_set_type(d, type);
else if (hw_irq >= ICU_IRQ_START)
ret = rzv2h_irq_set_type(d, type);
else
ret = rzv2h_nmi_set_type(d, type);
if (which != IRQCHIP_STATE_PENDING)
return irq_chip_set_parent_state(d, which, state);
if (ret)
return ret;
if (!state)
return 0;
return irq_chip_set_type_parent(d, IRQ_TYPE_LEVEL_HIGH);
priv = irq_data_to_priv(d);
bit = BIT(swpe);
/*
* SWPE has 16 bits; the bit position is rotated on each trigger
* and wraps around once all bits have been used.
*/
if (++swpe >= ICU_SWPE_NUM)
swpe = 0;
/* Trigger the pseudo error interrupt */
writel_relaxed(bit, priv->base + ICU_SWPE);
return 0;
}
static int rzv2h_irqc_irq_suspend(void *data)
@@ -472,27 +544,98 @@ static struct syscore rzv2h_irqc_syscore = {
.ops = &rzv2h_irqc_syscore_ops,
};
static const struct irq_chip rzv2h_icu_chip = {
static const struct irq_chip rzv2h_icu_tint_chip = {
.name = "rzv2h-icu",
.irq_eoi = rzv2h_icu_eoi,
.irq_eoi = rzv2h_icu_tint_eoi,
.irq_mask = irq_chip_mask_parent,
.irq_unmask = irq_chip_unmask_parent,
.irq_disable = rzv2h_icu_irq_disable,
.irq_enable = rzv2h_icu_irq_enable,
.irq_disable = rzv2h_icu_tint_disable,
.irq_enable = rzv2h_icu_tint_enable,
.irq_get_irqchip_state = irq_chip_get_parent_state,
.irq_set_irqchip_state = irq_chip_set_parent_state,
.irq_retrigger = irq_chip_retrigger_hierarchy,
.irq_set_type = rzv2h_icu_set_type,
.irq_set_type = rzv2h_tint_set_type,
.irq_set_affinity = irq_chip_set_affinity_parent,
.flags = IRQCHIP_MASK_ON_SUSPEND |
IRQCHIP_SET_TYPE_MASKED |
IRQCHIP_SKIP_SET_WAKE,
};
static const struct irq_chip rzv2h_icu_irq_chip = {
.name = "rzv2h-icu",
.irq_eoi = rzv2h_icu_irq_eoi,
.irq_mask = irq_chip_mask_parent,
.irq_unmask = irq_chip_unmask_parent,
.irq_disable = irq_chip_disable_parent,
.irq_enable = irq_chip_enable_parent,
.irq_get_irqchip_state = irq_chip_get_parent_state,
.irq_set_irqchip_state = irq_chip_set_parent_state,
.irq_retrigger = irq_chip_retrigger_hierarchy,
.irq_set_type = rzv2h_irq_set_type,
.irq_set_affinity = irq_chip_set_affinity_parent,
.flags = IRQCHIP_MASK_ON_SUSPEND |
IRQCHIP_SET_TYPE_MASKED |
IRQCHIP_SKIP_SET_WAKE,
};
static const struct irq_chip rzv2h_icu_nmi_chip = {
.name = "rzv2h-icu",
.irq_eoi = rzv2h_icu_nmi_eoi,
.irq_mask = irq_chip_mask_parent,
.irq_unmask = irq_chip_unmask_parent,
.irq_disable = irq_chip_disable_parent,
.irq_enable = irq_chip_enable_parent,
.irq_get_irqchip_state = irq_chip_get_parent_state,
.irq_set_irqchip_state = irq_chip_set_parent_state,
.irq_retrigger = irq_chip_retrigger_hierarchy,
.irq_set_type = rzv2h_nmi_set_type,
.irq_set_affinity = irq_chip_set_affinity_parent,
.flags = IRQCHIP_MASK_ON_SUSPEND |
IRQCHIP_SET_TYPE_MASKED |
IRQCHIP_SKIP_SET_WAKE,
};
static const struct irq_chip rzv2h_icu_swint_chip = {
.name = "rzv2h-icu",
.irq_eoi = irq_chip_eoi_parent,
.irq_mask = irq_chip_mask_parent,
.irq_unmask = irq_chip_unmask_parent,
.irq_disable = irq_chip_disable_parent,
.irq_enable = irq_chip_enable_parent,
.irq_get_irqchip_state = irq_chip_get_parent_state,
.irq_set_irqchip_state = rzv2h_icu_swint_set_irqchip_state,
.irq_retrigger = irq_chip_retrigger_hierarchy,
.irq_set_type = irq_chip_set_type_parent,
.irq_set_affinity = irq_chip_set_affinity_parent,
.flags = IRQCHIP_MASK_ON_SUSPEND |
IRQCHIP_SET_TYPE_MASKED |
IRQCHIP_SKIP_SET_WAKE,
};
static const struct irq_chip rzv2h_icu_swpe_err_chip = {
.name = "rzv2h-icu",
.irq_eoi = irq_chip_eoi_parent,
.irq_mask = irq_chip_mask_parent,
.irq_unmask = irq_chip_unmask_parent,
.irq_disable = irq_chip_disable_parent,
.irq_enable = irq_chip_enable_parent,
.irq_get_irqchip_state = irq_chip_get_parent_state,
.irq_set_irqchip_state = rzv2h_icu_swpe_set_irqchip_state,
.irq_retrigger = irq_chip_retrigger_hierarchy,
.irq_set_type = irq_chip_set_type_parent,
.irq_set_affinity = irq_chip_set_affinity_parent,
.flags = IRQCHIP_MASK_ON_SUSPEND |
IRQCHIP_SET_TYPE_MASKED |
IRQCHIP_SKIP_SET_WAKE,
};
#define hwirq_within(hwirq, which) ((hwirq) >= which##_START && (hwirq) <= which##_LAST)
static int rzv2h_icu_alloc(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs,
void *arg)
{
struct rzv2h_icu_priv *priv = domain->host_data;
const struct irq_chip *chip;
unsigned long tint = 0;
irq_hw_number_t hwirq;
unsigned int type;
@@ -508,19 +651,27 @@ static int rzv2h_icu_alloc(struct irq_domain *domain, unsigned int virq, unsigne
* hwirq is embedded in bits 0-15.
* TINT is embedded in bits 16-31.
*/
if (hwirq >= ICU_TINT_START) {
tint = ICU_TINT_EXTRACT_GPIOINT(hwirq);
tint = ICU_TINT_EXTRACT_GPIOINT(hwirq);
if (tint || hwirq_within(hwirq, ICU_TINT)) {
hwirq = ICU_TINT_EXTRACT_HWIRQ(hwirq);
if (hwirq < ICU_TINT_START)
if (!hwirq_within(hwirq, ICU_TINT))
return -EINVAL;
chip = &rzv2h_icu_tint_chip;
} else if (hwirq_within(hwirq, ICU_IRQ)) {
chip = &rzv2h_icu_irq_chip;
} else if (hwirq_within(hwirq, ICU_CA55_INT)) {
chip = &rzv2h_icu_swint_chip;
} else if (hwirq_within(hwirq, ICU_ERR_INT)) {
chip = &rzv2h_icu_swpe_err_chip;
} else {
chip = &rzv2h_icu_nmi_chip;
}
if (hwirq > (ICU_NUM_IRQ - 1))
return -EINVAL;
ret = irq_domain_set_hwirq_and_chip(domain, virq, hwirq, &rzv2h_icu_chip,
(void *)(uintptr_t)tint);
ret = irq_domain_set_hwirq_and_chip(domain, virq, hwirq, chip, (void *)(uintptr_t)tint);
if (ret)
return ret;
@@ -550,62 +701,160 @@ static int rzv2h_icu_parse_interrupts(struct rzv2h_icu_priv *priv, struct device
return 0;
}
static irqreturn_t rzv2h_icu_error_irq(int irq, void *data)
{
struct rzv2h_icu_priv *priv = data;
const struct rzv2h_hw_info *hw_info = priv->info;
void __iomem *base = priv->base;
unsigned int k;
u32 st;
/* 1) Bus errors (BEISR0..3) */
for (k = 0; k < ICU_NUM_BE; k++) {
st = readl(base + ICU_BEISR(k));
if (!st)
continue;
writel_relaxed(st, base + ICU_BECLR(k));
pr_warn("rzv2h-icu: BUS error k=%u status=0x%08x\n", k, st);
}
/* 2) ECC RAM errors (EREISR0..X) */
for (k = hw_info->ecc_start; k <= hw_info->ecc_end; k++) {
st = readl(base + ICU_EREISR(k));
if (!st)
continue;
writel_relaxed(st, base + ICU_ERCLR(k));
pr_warn("rzv2h-icu: ECC error k=%u status=0x%08x\n", k, st);
}
/* 3) IP/CA55 error interrupt status (ERINTA55CTL0..3) */
for (k = 0; k < ICU_NUM_A55ERR; k++) {
st = readl(base + ICU_ERINTA55CTL(k));
if (!st)
continue;
/* there is no relation with status bits so clear all the interrupts */
writel_relaxed(0xffffffff, base + ICU_ERINTA55CRL(k));
pr_warn("rzv2h-icu: IP/CA55 error k=%u status=0x%08x\n", k, st);
}
return IRQ_HANDLED;
}
static irqreturn_t rzv2h_icu_swint_irq(int irq, void *data)
{
unsigned int cpu = (uintptr_t)data;
pr_info("SWINT interrupt for CA55 core %u\n", cpu);
return IRQ_HANDLED;
}
static int rzv2h_icu_setup_irqs(struct platform_device *pdev, struct irq_domain *irq_domain)
{
const struct rzv2h_hw_info *hw_info = rzv2h_icu_data->info;
bool irq_inject = IS_ENABLED(CONFIG_GENERIC_IRQ_INJECTION);
void __iomem *base = rzv2h_icu_data->base;
struct device *dev = &pdev->dev;
struct irq_fwspec fwspec;
unsigned int i, virq;
int ret;
for (i = 0; i < ICU_CA55_INT_COUNT && irq_inject; i++) {
fwspec.fwnode = irq_domain->fwnode;
fwspec.param_count = 2;
fwspec.param[0] = ICU_CA55_INT_START + i;
fwspec.param[1] = IRQ_TYPE_EDGE_RISING;
virq = irq_create_fwspec_mapping(&fwspec);
if (!virq) {
return dev_err_probe(dev, -EINVAL,
"failed to create int-ca55-%u IRQ mapping\n", i);
}
ret = devm_request_irq(dev, virq, rzv2h_icu_swint_irq, 0, dev_name(dev),
(void *)(uintptr_t)i);
if (ret)
return dev_err_probe(dev, ret, "Failed to request int-ca55-%u IRQ\n", i);
}
/* Unmask and clear all IP/CA55 error interrupts */
for (i = 0; i < ICU_NUM_A55ERR; i++) {
writel_relaxed(0xffffff, base + ICU_ERINTA55CRL(i));
writel_relaxed(0x0, base + ICU_ERINTA55MSK(i));
}
/* Clear all Bus errors */
for (i = 0; i < ICU_NUM_BE; i++)
writel_relaxed(0xffffffff, base + ICU_BECLR(i));
/* Clear all ECCRAM errors */
for (i = hw_info->ecc_start; i <= hw_info->ecc_end; i++)
writel_relaxed(0xffffffff, base + ICU_ERCLR(i));
fwspec.fwnode = irq_domain->fwnode;
fwspec.param_count = 2;
fwspec.param[0] = ICU_ERR_INT_START;
fwspec.param[1] = IRQ_TYPE_LEVEL_HIGH;
virq = irq_create_fwspec_mapping(&fwspec);
if (!virq)
return dev_err_probe(dev, -EINVAL, "failed to create icu-error-ca55 IRQ mapping\n");
ret = devm_request_irq(dev, virq, rzv2h_icu_error_irq, 0, dev_name(dev), rzv2h_icu_data);
if (ret)
return dev_err_probe(dev, ret, "Failed to request icu-error-ca55 IRQ\n");
return 0;
}
static int rzv2h_icu_probe_common(struct platform_device *pdev, struct device_node *parent,
const struct rzv2h_hw_info *hw_info)
{
struct irq_domain *irq_domain, *parent_domain;
struct device_node *node = pdev->dev.of_node;
struct device *dev = &pdev->dev;
struct reset_control *resetn;
int ret;
parent_domain = irq_find_host(parent);
if (!parent_domain) {
dev_err(&pdev->dev, "cannot find parent domain\n");
return -ENODEV;
}
if (!parent_domain)
return dev_err_probe(dev, -ENODEV, "cannot find parent domain\n");
rzv2h_icu_data = devm_kzalloc(&pdev->dev, sizeof(*rzv2h_icu_data), GFP_KERNEL);
rzv2h_icu_data = devm_kzalloc(dev, sizeof(*rzv2h_icu_data), GFP_KERNEL);
if (!rzv2h_icu_data)
return -ENOMEM;
platform_set_drvdata(pdev, rzv2h_icu_data);
rzv2h_icu_data->base = devm_of_iomap(&pdev->dev, pdev->dev.of_node, 0, NULL);
rzv2h_icu_data->base = devm_of_iomap(dev, node, 0, NULL);
if (IS_ERR(rzv2h_icu_data->base))
return PTR_ERR(rzv2h_icu_data->base);
ret = rzv2h_icu_parse_interrupts(rzv2h_icu_data, node);
if (ret) {
dev_err(&pdev->dev, "cannot parse interrupts: %d\n", ret);
return ret;
}
if (ret)
return dev_err_probe(dev, ret, "cannot parse interrupts\n");
resetn = devm_reset_control_get_exclusive_deasserted(&pdev->dev, NULL);
if (IS_ERR(resetn)) {
ret = PTR_ERR(resetn);
dev_err(&pdev->dev, "failed to acquire deasserted reset: %d\n", ret);
return ret;
}
resetn = devm_reset_control_get_exclusive_deasserted(dev, NULL);
if (IS_ERR(resetn))
return dev_err_probe(dev, PTR_ERR(resetn), "failed to acquire deasserted reset\n");
ret = devm_pm_runtime_enable(&pdev->dev);
if (ret < 0) {
dev_err(&pdev->dev, "devm_pm_runtime_enable failed, %d\n", ret);
return ret;
}
ret = devm_pm_runtime_enable(dev);
if (ret < 0)
return dev_err_probe(dev, ret, "devm_pm_runtime_enable failed\n");
ret = pm_runtime_resume_and_get(&pdev->dev);
if (ret < 0) {
dev_err(&pdev->dev, "pm_runtime_resume_and_get failed: %d\n", ret);
return ret;
}
ret = pm_runtime_resume_and_get(dev);
if (ret < 0)
return dev_err_probe(dev, ret, "pm_runtime_resume_and_get failed\n");
raw_spin_lock_init(&rzv2h_icu_data->lock);
irq_domain = irq_domain_create_hierarchy(parent_domain, 0, ICU_NUM_IRQ,
dev_fwnode(&pdev->dev), &rzv2h_icu_domain_ops,
dev_fwnode(dev), &rzv2h_icu_domain_ops,
rzv2h_icu_data);
if (!irq_domain) {
dev_err(&pdev->dev, "failed to add irq domain\n");
dev_err(dev, "failed to add irq domain\n");
ret = -ENOMEM;
goto pm_put;
}
@@ -614,15 +863,18 @@ static int rzv2h_icu_probe_common(struct platform_device *pdev, struct device_no
register_syscore(&rzv2h_irqc_syscore);
ret = rzv2h_icu_setup_irqs(pdev, irq_domain);
if (ret)
goto pm_put;
/*
* coccicheck complains about a missing put_device call before returning, but it's a false
* positive. We still need &pdev->dev after successfully returning from this function.
* positive. We still need dev after successfully returning from this function.
*/
return 0;
pm_put:
pm_runtime_put_sync(&pdev->dev);
pm_runtime_put_sync(dev);
return ret;
}
@@ -657,12 +909,24 @@ static const struct rzv2h_hw_info rzg3e_hw_params = {
.t_offs = ICU_RZG3E_TINT_OFFSET,
.max_tssel = ICU_RZG3E_TSSEL_MAX_VAL,
.field_width = 16,
.ecc_start = 1,
.ecc_end = 4,
};
static const struct rzv2h_hw_info rzv2n_hw_params = {
.t_offs = 0,
.max_tssel = ICU_RZV2H_TSSEL_MAX_VAL,
.field_width = 8,
.ecc_start = 0,
.ecc_end = 2,
};
static const struct rzv2h_hw_info rzv2h_hw_params = {
.t_offs = 0,
.max_tssel = ICU_RZV2H_TSSEL_MAX_VAL,
.field_width = 8,
.ecc_start = 0,
.ecc_end = 11,
};
static int rzg3e_icu_probe(struct platform_device *pdev, struct device_node *parent)
@@ -670,6 +934,11 @@ static int rzg3e_icu_probe(struct platform_device *pdev, struct device_node *par
return rzv2h_icu_probe_common(pdev, parent, &rzg3e_hw_params);
}
static int rzv2n_icu_probe(struct platform_device *pdev, struct device_node *parent)
{
return rzv2h_icu_probe_common(pdev, parent, &rzv2n_hw_params);
}
static int rzv2h_icu_probe(struct platform_device *pdev, struct device_node *parent)
{
return rzv2h_icu_probe_common(pdev, parent, &rzv2h_hw_params);
@@ -677,7 +946,7 @@ static int rzv2h_icu_probe(struct platform_device *pdev, struct device_node *par
IRQCHIP_PLATFORM_DRIVER_BEGIN(rzv2h_icu)
IRQCHIP_MATCH("renesas,r9a09g047-icu", rzg3e_icu_probe)
IRQCHIP_MATCH("renesas,r9a09g056-icu", rzv2h_icu_probe)
IRQCHIP_MATCH("renesas,r9a09g056-icu", rzv2n_icu_probe)
IRQCHIP_MATCH("renesas,r9a09g057-icu", rzv2h_icu_probe)
IRQCHIP_PLATFORM_DRIVER_END(rzv2h_icu)
MODULE_AUTHOR("Fabrizio Castro <fabrizio.castro.jz@renesas.com>");
include/asm-generic/thread_info_tif.h
+4 -1
View File
@@ -41,11 +41,14 @@
#define _TIF_PATCH_PENDING BIT(TIF_PATCH_PENDING)
#ifdef HAVE_TIF_RESTORE_SIGMASK
# define TIF_RESTORE_SIGMASK 10 // Restore signal mask in do_signal() */
# define TIF_RESTORE_SIGMASK 10 // Restore signal mask in do_signal()
# define _TIF_RESTORE_SIGMASK BIT(TIF_RESTORE_SIGMASK)
#endif
#define TIF_RSEQ 11 // Run RSEQ fast path
#define _TIF_RSEQ BIT(TIF_RSEQ)
#define TIF_HRTIMER_REARM 12 // re-arm the timer
#define _TIF_HRTIMER_REARM BIT(TIF_HRTIMER_REARM)
#endif /* _ASM_GENERIC_THREAD_INFO_TIF_H_ */
include/linux/clockchips.h
+7 -3
View File
@@ -43,9 +43,9 @@ enum clock_event_state {
/*
* Clock event features
*/
# define CLOCK_EVT_FEAT_PERIODIC 0x000001
# define CLOCK_EVT_FEAT_ONESHOT 0x000002
# define CLOCK_EVT_FEAT_KTIME 0x000004
# define CLOCK_EVT_FEAT_PERIODIC 0x000001
# define CLOCK_EVT_FEAT_ONESHOT 0x000002
# define CLOCK_EVT_FEAT_CLOCKSOURCE_COUPLED 0x000004
/*
* x86(64) specific (mis)features:
@@ -73,6 +73,7 @@ enum clock_event_state {
* level handler of the event source
* @set_next_event: set next event function using a clocksource delta
* @set_next_ktime: set next event function using a direct ktime value
* @set_next_coupled: set next event function for clocksource coupled mode
* @next_event: local storage for the next event in oneshot mode
* @max_delta_ns: maximum delta value in ns
* @min_delta_ns: minimum delta value in ns
@@ -80,6 +81,7 @@ enum clock_event_state {
* @shift: nanoseconds to cycles divisor (power of two)
* @state_use_accessors:current state of the device, assigned by the core code
* @features: features
* @cs_id: Clocksource ID to denote the clocksource for coupled mode
* @next_event_forced: True if the last programming was a forced event
* @retries: number of forced programming retries
* @set_state_periodic: switch state to periodic
@@ -102,6 +104,7 @@ struct clock_event_device {
void (*event_handler)(struct clock_event_device *);
int (*set_next_event)(unsigned long evt, struct clock_event_device *);
int (*set_next_ktime)(ktime_t expires, struct clock_event_device *);
void (*set_next_coupled)(u64 cycles, struct clock_event_device *);
ktime_t next_event;
u64 max_delta_ns;
u64 min_delta_ns;
@@ -109,6 +112,7 @@ struct clock_event_device {
u32 shift;
enum clock_event_state state_use_accessors;
unsigned int features;
enum clocksource_ids cs_id;
unsigned int next_event_forced;
unsigned long retries;
include/linux/clocksource.h
+8 -19
View File
@@ -44,8 +44,6 @@ struct module;
* @shift: Cycle to nanosecond divisor (power of two)
* @max_idle_ns: Maximum idle time permitted by the clocksource (nsecs)
* @maxadj: Maximum adjustment value to mult (~11%)
* @uncertainty_margin: Maximum uncertainty in nanoseconds per half second.
* Zero says to use default WATCHDOG_THRESHOLD.
* @archdata: Optional arch-specific data
* @max_cycles: Maximum safe cycle value which won't overflow on
* multiplication
@@ -105,7 +103,6 @@ struct clocksource {
u32 shift;
u64 max_idle_ns;
u32 maxadj;
u32 uncertainty_margin;
#ifdef CONFIG_ARCH_CLOCKSOURCE_DATA
struct arch_clocksource_data archdata;
#endif
@@ -133,6 +130,7 @@ struct clocksource {
struct list_head wd_list;
u64 cs_last;
u64 wd_last;
unsigned int wd_cpu;
#endif
struct module *owner;
};
@@ -142,13 +140,19 @@ struct clocksource {
*/
#define CLOCK_SOURCE_IS_CONTINUOUS 0x01
#define CLOCK_SOURCE_MUST_VERIFY 0x02
#define CLOCK_SOURCE_CALIBRATED 0x04
#define CLOCK_SOURCE_WATCHDOG 0x10
#define CLOCK_SOURCE_VALID_FOR_HRES 0x20
#define CLOCK_SOURCE_UNSTABLE 0x40
#define CLOCK_SOURCE_SUSPEND_NONSTOP 0x80
#define CLOCK_SOURCE_RESELECT 0x100
#define CLOCK_SOURCE_VERIFY_PERCPU 0x200
#define CLOCK_SOURCE_CAN_INLINE_READ 0x200
#define CLOCK_SOURCE_HAS_COUPLED_CLOCK_EVENT 0x400
#define CLOCK_SOURCE_WDTEST 0x800
#define CLOCK_SOURCE_WDTEST_PERCPU 0x1000
/* simplify initialization of mask field */
#define CLOCKSOURCE_MASK(bits) GENMASK_ULL((bits) - 1, 0)
@@ -298,21 +302,6 @@ static inline void timer_probe(void) {}
#define TIMER_ACPI_DECLARE(name, table_id, fn) \
ACPI_DECLARE_PROBE_ENTRY(timer, name, table_id, 0, NULL, 0, fn)
static inline unsigned int clocksource_get_max_watchdog_retry(void)
{
/*
* When system is in the boot phase or under heavy workload, there
* can be random big latencies during the clocksource/watchdog
* read, so allow retries to filter the noise latency. As the
* latency's frequency and maximum value goes up with the number of
* CPUs, scale the number of retries with the number of online
* CPUs.
*/
return (ilog2(num_online_cpus()) / 2) + 1;
}
void clocksource_verify_percpu(struct clocksource *cs);
/**
* struct clocksource_base - hardware abstraction for clock on which a clocksource
* is based
include/linux/hrtimer.h
+20 -44
View File
@@ -13,6 +13,7 @@
#define _LINUX_HRTIMER_H
#include <linux/hrtimer_defs.h>
#include <linux/hrtimer_rearm.h>
#include <linux/hrtimer_types.h>
#include <linux/init.h>
#include <linux/list.h>
@@ -31,6 +32,13 @@
* soft irq context
* HRTIMER_MODE_HARD - Timer callback function will be executed in
* hard irq context even on PREEMPT_RT.
* HRTIMER_MODE_LAZY_REARM - Avoid reprogramming if the timer was the
* first expiring timer and is moved into the
* future. Special mode for the HRTICK timer to
* avoid extensive reprogramming of the hardware,
* which is expensive in virtual machines. Risks
* a pointless expiry, but that's better than
* reprogramming on every context switch,
*/
enum hrtimer_mode {
HRTIMER_MODE_ABS = 0x00,
@@ -38,6 +46,7 @@ enum hrtimer_mode {
HRTIMER_MODE_PINNED = 0x02,
HRTIMER_MODE_SOFT = 0x04,
HRTIMER_MODE_HARD = 0x08,
HRTIMER_MODE_LAZY_REARM = 0x10,
HRTIMER_MODE_ABS_PINNED = HRTIMER_MODE_ABS | HRTIMER_MODE_PINNED,
HRTIMER_MODE_REL_PINNED = HRTIMER_MODE_REL | HRTIMER_MODE_PINNED,
@@ -55,33 +64,6 @@ enum hrtimer_mode {
HRTIMER_MODE_REL_PINNED_HARD = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_HARD,
};
/*
* Values to track state of the timer
*
* Possible states:
*
* 0x00 inactive
* 0x01 enqueued into rbtree
*
* The callback state is not part of the timer->state because clearing it would
* mean touching the timer after the callback, this makes it impossible to free
* the timer from the callback function.
*
* Therefore we track the callback state in:
*
* timer->base->cpu_base->running == timer
*
* On SMP it is possible to have a "callback function running and enqueued"
* status. It happens for example when a posix timer expired and the callback
* queued a signal. Between dropping the lock which protects the posix timer
* and reacquiring the base lock of the hrtimer, another CPU can deliver the
* signal and rearm the timer.
*
* All state transitions are protected by cpu_base->lock.
*/
#define HRTIMER_STATE_INACTIVE 0x00
#define HRTIMER_STATE_ENQUEUED 0x01
/**
* struct hrtimer_sleeper - simple sleeper structure
* @timer: embedded timer structure
@@ -134,11 +116,6 @@ static inline ktime_t hrtimer_get_softexpires(const struct hrtimer *timer)
return timer->_softexpires;
}
static inline s64 hrtimer_get_expires_ns(const struct hrtimer *timer)
{
return ktime_to_ns(timer->node.expires);
}
ktime_t hrtimer_cb_get_time(const struct hrtimer *timer);
static inline ktime_t hrtimer_expires_remaining(const struct hrtimer *timer)
@@ -146,24 +123,23 @@ static inline ktime_t hrtimer_expires_remaining(const struct hrtimer *timer)
return ktime_sub(timer->node.expires, hrtimer_cb_get_time(timer));
}
static inline int hrtimer_is_hres_active(struct hrtimer *timer)
{
return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ?
timer->base->cpu_base->hres_active : 0;
}
#ifdef CONFIG_HIGH_RES_TIMERS
extern unsigned int hrtimer_resolution;
struct clock_event_device;
extern void hrtimer_interrupt(struct clock_event_device *dev);
extern unsigned int hrtimer_resolution;
extern struct static_key_false hrtimer_highres_enabled_key;
#else
static inline bool hrtimer_highres_enabled(void)
{
return static_branch_likely(&hrtimer_highres_enabled_key);
}
#else /* CONFIG_HIGH_RES_TIMERS */
#define hrtimer_resolution (unsigned int)LOW_RES_NSEC
#endif
static inline bool hrtimer_highres_enabled(void) { return false; }
#endif /* !CONFIG_HIGH_RES_TIMERS */
static inline ktime_t
__hrtimer_expires_remaining_adjusted(const struct hrtimer *timer, ktime_t now)
@@ -293,8 +269,8 @@ extern bool hrtimer_active(const struct hrtimer *timer);
*/
static inline bool hrtimer_is_queued(struct hrtimer *timer)
{
/* The READ_ONCE pairs with the update functions of timer->state */
return !!(READ_ONCE(timer->state) & HRTIMER_STATE_ENQUEUED);
/* The READ_ONCE pairs with the update functions of timer->is_queued */
return READ_ONCE(timer->is_queued);
}
/*
include/linux/hrtimer_defs.h
+43 -40
View File
@@ -19,21 +19,23 @@
* timer to a base on another cpu.
* @clockid: clock id for per_cpu support
* @seq: seqcount around __run_hrtimer
* @expires_next: Absolute time of the next event in this clock base
* @running: pointer to the currently running hrtimer
* @active: red black tree root node for the active timers
* @offset: offset of this clock to the monotonic base
*/
struct hrtimer_clock_base {
struct hrtimer_cpu_base *cpu_base;
unsigned int index;
clockid_t clockid;
seqcount_raw_spinlock_t seq;
struct hrtimer *running;
struct timerqueue_head active;
ktime_t offset;
struct hrtimer_cpu_base *cpu_base;
const unsigned int index;
const clockid_t clockid;
seqcount_raw_spinlock_t seq;
ktime_t expires_next;
struct hrtimer *running;
struct timerqueue_linked_head active;
ktime_t offset;
} __hrtimer_clock_base_align;
enum hrtimer_base_type {
enum hrtimer_base_type {
HRTIMER_BASE_MONOTONIC,
HRTIMER_BASE_REALTIME,
HRTIMER_BASE_BOOTTIME,
@@ -42,37 +44,36 @@ enum hrtimer_base_type {
HRTIMER_BASE_REALTIME_SOFT,
HRTIMER_BASE_BOOTTIME_SOFT,
HRTIMER_BASE_TAI_SOFT,
HRTIMER_MAX_CLOCK_BASES,
HRTIMER_MAX_CLOCK_BASES
};
/**
* struct hrtimer_cpu_base - the per cpu clock bases
* @lock: lock protecting the base and associated clock bases
* and timers
* @cpu: cpu number
* @active_bases: Bitfield to mark bases with active timers
* @clock_was_set_seq: Sequence counter of clock was set events
* @hres_active: State of high resolution mode
* @in_hrtirq: hrtimer_interrupt() is currently executing
* @hang_detected: The last hrtimer interrupt detected a hang
* @softirq_activated: displays, if the softirq is raised - update of softirq
* related settings is not required then.
* @nr_events: Total number of hrtimer interrupt events
* @nr_retries: Total number of hrtimer interrupt retries
* @nr_hangs: Total number of hrtimer interrupt hangs
* @max_hang_time: Maximum time spent in hrtimer_interrupt
* @softirq_expiry_lock: Lock which is taken while softirq based hrtimer are
* expired
* @online: CPU is online from an hrtimers point of view
* @timer_waiters: A hrtimer_cancel() invocation waits for the timer
* callback to finish.
* @expires_next: absolute time of the next event, is required for remote
* hrtimer enqueue; it is the total first expiry time (hard
* and soft hrtimer are taken into account)
* @next_timer: Pointer to the first expiring timer
* @softirq_expires_next: Time to check, if soft queues needs also to be expired
* @softirq_next_timer: Pointer to the first expiring softirq based timer
* @clock_base: array of clock bases for this cpu
* @lock: lock protecting the base and associated clock bases and timers
* @cpu: cpu number
* @active_bases: Bitfield to mark bases with active timers
* @clock_was_set_seq: Sequence counter of clock was set events
* @hres_active: State of high resolution mode
* @deferred_rearm: A deferred rearm is pending
* @deferred_needs_update: The deferred rearm must re-evaluate the first timer
* @hang_detected: The last hrtimer interrupt detected a hang
* @softirq_activated: displays, if the softirq is raised - update of softirq
* related settings is not required then.
* @nr_events: Total number of hrtimer interrupt events
* @nr_retries: Total number of hrtimer interrupt retries
* @nr_hangs: Total number of hrtimer interrupt hangs
* @max_hang_time: Maximum time spent in hrtimer_interrupt
* @softirq_expiry_lock: Lock which is taken while softirq based hrtimer are expired
* @online: CPU is online from an hrtimers point of view
* @timer_waiters: A hrtimer_cancel() waiters for the timer callback to finish.
* @expires_next: Absolute time of the next event, is required for remote
* hrtimer enqueue; it is the total first expiry time (hard
* and soft hrtimer are taken into account)
* @next_timer: Pointer to the first expiring timer
* @softirq_expires_next: Time to check, if soft queues needs also to be expired
* @softirq_next_timer: Pointer to the first expiring softirq based timer
* @deferred_expires_next: Cached expires next value for deferred rearm
* @clock_base: Array of clock bases for this cpu
*
* Note: next_timer is just an optimization for __remove_hrtimer().
* Do not dereference the pointer because it is not reliable on
@@ -83,11 +84,12 @@ struct hrtimer_cpu_base {
unsigned int cpu;
unsigned int active_bases;
unsigned int clock_was_set_seq;
unsigned int hres_active : 1,
in_hrtirq : 1,
hang_detected : 1,
softirq_activated : 1,
online : 1;
bool hres_active;
bool deferred_rearm;
bool deferred_needs_update;
bool hang_detected;
bool softirq_activated;
bool online;
#ifdef CONFIG_HIGH_RES_TIMERS
unsigned int nr_events;
unsigned short nr_retries;
@@ -102,6 +104,7 @@ struct hrtimer_cpu_base {
struct hrtimer *next_timer;
ktime_t softirq_expires_next;
struct hrtimer *softirq_next_timer;
ktime_t deferred_expires_next;
struct hrtimer_clock_base clock_base[HRTIMER_MAX_CLOCK_BASES];
call_single_data_t csd;
} ____cacheline_aligned;
include/linux/hrtimer_rearm.h
+83
View File
@@ -0,0 +1,83 @@
// SPDX-License-Identifier: GPL-2.0
#ifndef _LINUX_HRTIMER_REARM_H
#define _LINUX_HRTIMER_REARM_H
#ifdef CONFIG_HRTIMER_REARM_DEFERRED
#include <linux/thread_info.h>
void __hrtimer_rearm_deferred(void);
/*
* This is purely CPU local, so check the TIF bit first to avoid the overhead of
* the atomic test_and_clear_bit() operation for the common case where the bit
* is not set.
*/
static __always_inline bool hrtimer_test_and_clear_rearm_deferred_tif(unsigned long tif_work)
{
lockdep_assert_irqs_disabled();
if (unlikely(tif_work & _TIF_HRTIMER_REARM)) {
clear_thread_flag(TIF_HRTIMER_REARM);
return true;
}
return false;
}
#define TIF_REARM_MASK (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY | _TIF_HRTIMER_REARM)
/* Invoked from the exit to user before invoking exit_to_user_mode_loop() */
static __always_inline bool
hrtimer_rearm_deferred_user_irq(unsigned long *tif_work, const unsigned long tif_mask)
{
/* Help the compiler to optimize the function out for syscall returns */
if (!(tif_mask & _TIF_HRTIMER_REARM))
return false;
/*
* Rearm the timer if none of the resched flags is set before going into
* the loop which re-enables interrupts.
*/
if (unlikely((*tif_work & TIF_REARM_MASK) == _TIF_HRTIMER_REARM)) {
clear_thread_flag(TIF_HRTIMER_REARM);
__hrtimer_rearm_deferred();
/* Don't go into the loop if HRTIMER_REARM was the only flag */
*tif_work &= ~TIF_HRTIMER_REARM;
return !*tif_work;
}
return false;
}
/* Invoked from the time slice extension decision function */
static __always_inline void hrtimer_rearm_deferred_tif(unsigned long tif_work)
{
if (hrtimer_test_and_clear_rearm_deferred_tif(tif_work))
__hrtimer_rearm_deferred();
}
/*
* This is to be called on all irqentry_exit() paths that will enable
* interrupts.
*/
static __always_inline void hrtimer_rearm_deferred(void)
{
hrtimer_rearm_deferred_tif(read_thread_flags());
}
/*
* Invoked from the scheduler on entry to __schedule() so it can defer
* rearming after the load balancing callbacks which might change hrtick.
*/
static __always_inline bool hrtimer_test_and_clear_rearm_deferred(void)
{
return hrtimer_test_and_clear_rearm_deferred_tif(read_thread_flags());
}
#else /* CONFIG_HRTIMER_REARM_DEFERRED */
static __always_inline void __hrtimer_rearm_deferred(void) { }
static __always_inline void hrtimer_rearm_deferred(void) { }
static __always_inline void hrtimer_rearm_deferred_tif(unsigned long tif_work) { }
static __always_inline bool
hrtimer_rearm_deferred_user_irq(unsigned long *tif_work, const unsigned long tif_mask) { return false; }
static __always_inline bool hrtimer_test_and_clear_rearm_deferred(void) { return false; }
#endif /* !CONFIG_HRTIMER_REARM_DEFERRED */
#endif
include/linux/hrtimer_types.h
+11 -8
View File
@@ -17,7 +17,7 @@ enum hrtimer_restart {
/**
* struct hrtimer - the basic hrtimer structure
* @node: timerqueue node, which also manages node.expires,
* @node: Linked timerqueue node, which also manages node.expires,
* the absolute expiry time in the hrtimers internal
* representation. The time is related to the clock on
* which the timer is based. Is setup by adding
@@ -28,23 +28,26 @@ enum hrtimer_restart {
* was armed.
* @function: timer expiry callback function
* @base: pointer to the timer base (per cpu and per clock)
* @state: state information (See bit values above)
* @is_queued: Indicates whether a timer is enqueued or not
* @is_rel: Set if the timer was armed relative
* @is_soft: Set if hrtimer will be expired in soft interrupt context.
* @is_hard: Set if hrtimer will be expired in hard interrupt context
* even on RT.
* @is_lazy: Set if the timer is frequently rearmed to avoid updates
* of the clock event device
*
* The hrtimer structure must be initialized by hrtimer_setup()
*/
struct hrtimer {
struct timerqueue_node node;
struct timerqueue_linked_node node;
struct hrtimer_clock_base *base;
bool is_queued;
bool is_rel;
bool is_soft;
bool is_hard;
bool is_lazy;
ktime_t _softexpires;
enum hrtimer_restart (*__private function)(struct hrtimer *);
struct hrtimer_clock_base *base;
u8 state;
u8 is_rel;
u8 is_soft;
u8 is_hard;
};
#endif /* _LINUX_HRTIMER_TYPES_H */
include/linux/irq-entry-common.h
+19 -6
View File
@@ -3,6 +3,7 @@
#define __LINUX_IRQENTRYCOMMON_H
#include <linux/context_tracking.h>
#include <linux/hrtimer_rearm.h>
#include <linux/kmsan.h>
#include <linux/rseq_entry.h>
#include <linux/static_call_types.h>
@@ -33,6 +34,14 @@
_TIF_PATCH_PENDING | _TIF_NOTIFY_SIGNAL | _TIF_RSEQ | \
ARCH_EXIT_TO_USER_MODE_WORK)
#ifdef CONFIG_HRTIMER_REARM_DEFERRED
# define EXIT_TO_USER_MODE_WORK_SYSCALL (EXIT_TO_USER_MODE_WORK)
# define EXIT_TO_USER_MODE_WORK_IRQ (EXIT_TO_USER_MODE_WORK | _TIF_HRTIMER_REARM)
#else
# define EXIT_TO_USER_MODE_WORK_SYSCALL (EXIT_TO_USER_MODE_WORK)
# define EXIT_TO_USER_MODE_WORK_IRQ (EXIT_TO_USER_MODE_WORK)
#endif
/**
* arch_enter_from_user_mode - Architecture specific sanity check for user mode regs
* @regs: Pointer to currents pt_regs
@@ -203,6 +212,7 @@ unsigned long exit_to_user_mode_loop(struct pt_regs *regs, unsigned long ti_work
/**
* __exit_to_user_mode_prepare - call exit_to_user_mode_loop() if required
* @regs: Pointer to pt_regs on entry stack
* @work_mask: Which TIF bits need to be evaluated
*
* 1) check that interrupts are disabled
* 2) call tick_nohz_user_enter_prepare()
@@ -212,7 +222,8 @@ unsigned long exit_to_user_mode_loop(struct pt_regs *regs, unsigned long ti_work
*
* Don't invoke directly, use the syscall/irqentry_ prefixed variants below
*/
static __always_inline void __exit_to_user_mode_prepare(struct pt_regs *regs)
static __always_inline void __exit_to_user_mode_prepare(struct pt_regs *regs,
const unsigned long work_mask)
{
unsigned long ti_work;
@@ -222,8 +233,10 @@ static __always_inline void __exit_to_user_mode_prepare(struct pt_regs *regs)
tick_nohz_user_enter_prepare();
ti_work = read_thread_flags();
if (unlikely(ti_work & EXIT_TO_USER_MODE_WORK))
ti_work = exit_to_user_mode_loop(regs, ti_work);
if (unlikely(ti_work & work_mask)) {
if (!hrtimer_rearm_deferred_user_irq(&ti_work, work_mask))
ti_work = exit_to_user_mode_loop(regs, ti_work);
}
arch_exit_to_user_mode_prepare(regs, ti_work);
}
@@ -239,7 +252,7 @@ static __always_inline void __exit_to_user_mode_validate(void)
/* Temporary workaround to keep ARM64 alive */
static __always_inline void exit_to_user_mode_prepare_legacy(struct pt_regs *regs)
{
__exit_to_user_mode_prepare(regs);
__exit_to_user_mode_prepare(regs, EXIT_TO_USER_MODE_WORK);
rseq_exit_to_user_mode_legacy();
__exit_to_user_mode_validate();
}
@@ -253,7 +266,7 @@ static __always_inline void exit_to_user_mode_prepare_legacy(struct pt_regs *reg
*/
static __always_inline void syscall_exit_to_user_mode_prepare(struct pt_regs *regs)
{
__exit_to_user_mode_prepare(regs);
__exit_to_user_mode_prepare(regs, EXIT_TO_USER_MODE_WORK_SYSCALL);
rseq_syscall_exit_to_user_mode();
__exit_to_user_mode_validate();
}
@@ -267,7 +280,7 @@ static __always_inline void syscall_exit_to_user_mode_prepare(struct pt_regs *re
*/
static __always_inline void irqentry_exit_to_user_mode_prepare(struct pt_regs *regs)
{
__exit_to_user_mode_prepare(regs);
__exit_to_user_mode_prepare(regs, EXIT_TO_USER_MODE_WORK_IRQ);
rseq_irqentry_exit_to_user_mode();
__exit_to_user_mode_validate();
}
include/linux/jiffies.h
+1 -5
View File
@@ -67,10 +67,6 @@ extern void register_refined_jiffies(long clock_tick_rate);
/* USER_TICK_USEC is the time between ticks in usec assuming fake USER_HZ */
#define USER_TICK_USEC ((1000000UL + USER_HZ/2) / USER_HZ)
#ifndef __jiffy_arch_data
#define __jiffy_arch_data
#endif
/*
* The 64-bit value is not atomic on 32-bit systems - you MUST NOT read it
* without sampling the sequence number in jiffies_lock.
@@ -83,7 +79,7 @@ extern void register_refined_jiffies(long clock_tick_rate);
* See arch/ARCH/kernel/vmlinux.lds.S
*/
extern u64 __cacheline_aligned_in_smp jiffies_64;
extern unsigned long volatile __cacheline_aligned_in_smp __jiffy_arch_data jiffies;
extern unsigned long volatile __cacheline_aligned_in_smp jiffies;
#if (BITS_PER_LONG < 64)
u64 get_jiffies_64(void);
include/linux/rbtree.h
+72 -9
View File
@@ -35,10 +35,15 @@
#define RB_CLEAR_NODE(node) \
((node)->__rb_parent_color = (unsigned long)(node))
#define RB_EMPTY_LINKED_NODE(lnode) RB_EMPTY_NODE(&(lnode)->node)
#define RB_CLEAR_LINKED_NODE(lnode) ({ \
RB_CLEAR_NODE(&(lnode)->node); \
(lnode)->prev = (lnode)->next = NULL; \
})
extern void rb_insert_color(struct rb_node *, struct rb_root *);
extern void rb_erase(struct rb_node *, struct rb_root *);
extern bool rb_erase_linked(struct rb_node_linked *, struct rb_root_linked *);
/* Find logical next and previous nodes in a tree */
extern struct rb_node *rb_next(const struct rb_node *);
@@ -213,15 +218,10 @@ rb_add_cached(struct rb_node *node, struct rb_root_cached *tree,
return leftmost ? node : NULL;
}
/**
* rb_add() - insert @node into @tree
* @node: node to insert
* @tree: tree to insert @node into
* @less: operator defining the (partial) node order
*/
static __always_inline void
rb_add(struct rb_node *node, struct rb_root *tree,
bool (*less)(struct rb_node *, const struct rb_node *))
__rb_add(struct rb_node *node, struct rb_root *tree,
bool (*less)(struct rb_node *, const struct rb_node *),
void (*linkop)(struct rb_node *, struct rb_node *, struct rb_node **))
{
struct rb_node **link = &tree->rb_node;
struct rb_node *parent = NULL;
@@ -234,10 +234,73 @@ rb_add(struct rb_node *node, struct rb_root *tree,
link = &parent->rb_right;
}
linkop(node, parent, link);
rb_link_node(node, parent, link);
rb_insert_color(node, tree);
}
#define __node_2_linked_node(_n) \
rb_entry((_n), struct rb_node_linked, node)
static inline void
rb_link_linked_node(struct rb_node *node, struct rb_node *parent, struct rb_node **link)
{
if (!parent)
return;
struct rb_node_linked *nnew = __node_2_linked_node(node);
struct rb_node_linked *npar = __node_2_linked_node(parent);
if (link == &parent->rb_left) {
nnew->prev = npar->prev;
nnew->next = npar;
npar->prev = nnew;
if (nnew->prev)
nnew->prev->next = nnew;
} else {
nnew->next = npar->next;
nnew->prev = npar;
npar->next = nnew;
if (nnew->next)
nnew->next->prev = nnew;
}
}
/**
* rb_add_linked() - insert @node into the leftmost linked tree @tree
* @node: node to insert
* @tree: linked tree to insert @node into
* @less: operator defining the (partial) node order
*
* Returns @true when @node is the new leftmost, @false otherwise.
*/
static __always_inline bool
rb_add_linked(struct rb_node_linked *node, struct rb_root_linked *tree,
bool (*less)(struct rb_node *, const struct rb_node *))
{
__rb_add(&node->node, &tree->rb_root, less, rb_link_linked_node);
if (!node->prev)
tree->rb_leftmost = node;
return !node->prev;
}
/* Empty linkop function which is optimized away by the compiler */
static __always_inline void
rb_link_noop(struct rb_node *n, struct rb_node *p, struct rb_node **l) { }
/**
* rb_add() - insert @node into @tree
* @node: node to insert
* @tree: tree to insert @node into
* @less: operator defining the (partial) node order
*/
static __always_inline void
rb_add(struct rb_node *node, struct rb_root *tree,
bool (*less)(struct rb_node *, const struct rb_node *))
{
__rb_add(node, tree, less, rb_link_noop);
}
/**
* rb_find_add_cached() - find equivalent @node in @tree, or add @node
* @node: node to look-for / insert
include/linux/rbtree_types.h
+16
View File
@@ -9,6 +9,12 @@ struct rb_node {
} __attribute__((aligned(sizeof(long))));
/* The alignment might seem pointless, but allegedly CRIS needs it */
struct rb_node_linked {
struct rb_node node;
struct rb_node_linked *prev;
struct rb_node_linked *next;
};
struct rb_root {
struct rb_node *rb_node;
};
@@ -28,7 +34,17 @@ struct rb_root_cached {
struct rb_node *rb_leftmost;
};
/*
* Leftmost tree with links. This would allow a trivial rb_rightmost update,
* but that has been omitted due to the lack of users.
*/
struct rb_root_linked {
struct rb_root rb_root;
struct rb_node_linked *rb_leftmost;
};
#define RB_ROOT (struct rb_root) { NULL, }
#define RB_ROOT_CACHED (struct rb_root_cached) { {NULL, }, NULL }
#define RB_ROOT_LINKED (struct rb_root_linked) { {NULL, }, NULL }
#endif
include/linux/rseq_entry.h
+13 -3
View File
@@ -40,6 +40,7 @@ DECLARE_PER_CPU(struct rseq_stats, rseq_stats);
#endif /* !CONFIG_RSEQ_STATS */
#ifdef CONFIG_RSEQ
#include <linux/hrtimer_rearm.h>
#include <linux/jump_label.h>
#include <linux/rseq.h>
#include <linux/sched/signal.h>
@@ -110,7 +111,7 @@ static __always_inline void rseq_slice_clear_grant(struct task_struct *t)
t->rseq.slice.state.granted = false;
}
static __always_inline bool rseq_grant_slice_extension(bool work_pending)
static __always_inline bool __rseq_grant_slice_extension(bool work_pending)
{
struct task_struct *curr = current;
struct rseq_slice_ctrl usr_ctrl;
@@ -215,11 +216,20 @@ efault:
return false;
}
static __always_inline bool rseq_grant_slice_extension(unsigned long ti_work, unsigned long mask)
{
if (unlikely(__rseq_grant_slice_extension(ti_work & mask))) {
hrtimer_rearm_deferred_tif(ti_work);
return true;
}
return false;
}
#else /* CONFIG_RSEQ_SLICE_EXTENSION */
static __always_inline bool rseq_slice_extension_enabled(void) { return false; }
static __always_inline bool rseq_arm_slice_extension_timer(void) { return false; }
static __always_inline void rseq_slice_clear_grant(struct task_struct *t) { }
static __always_inline bool rseq_grant_slice_extension(bool work_pending) { return false; }
static __always_inline bool rseq_grant_slice_extension(unsigned long ti_work, unsigned long mask) { return false; }
#endif /* !CONFIG_RSEQ_SLICE_EXTENSION */
bool rseq_debug_update_user_cs(struct task_struct *t, struct pt_regs *regs, unsigned long csaddr);
@@ -778,7 +788,7 @@ static inline void rseq_syscall_exit_to_user_mode(void) { }
static inline void rseq_irqentry_exit_to_user_mode(void) { }
static inline void rseq_exit_to_user_mode_legacy(void) { }
static inline void rseq_debug_syscall_return(struct pt_regs *regs) { }
static inline bool rseq_grant_slice_extension(bool work_pending) { return false; }
static inline bool rseq_grant_slice_extension(unsigned long ti_work, unsigned long mask) { return false; }
#endif /* !CONFIG_RSEQ */
#endif /* _LINUX_RSEQ_ENTRY_H */
include/linux/timekeeper_internal.h
+8
View File
@@ -72,6 +72,10 @@ struct tk_read_base {
* @id: The timekeeper ID
* @tkr_raw: The readout base structure for CLOCK_MONOTONIC_RAW
* @raw_sec: CLOCK_MONOTONIC_RAW time in seconds
* @cs_id: The ID of the current clocksource
* @cs_ns_to_cyc_mult: Multiplicator for nanoseconds to cycles conversion
* @cs_ns_to_cyc_shift: Shift value for nanoseconds to cycles conversion
* @cs_ns_to_cyc_maxns: Maximum nanoseconds to cyles conversion range
* @clock_was_set_seq: The sequence number of clock was set events
* @cs_was_changed_seq: The sequence number of clocksource change events
* @clock_valid: Indicator for valid clock
@@ -159,6 +163,10 @@ struct timekeeper {
u64 raw_sec;
/* Cachline 3 and 4 (timekeeping internal variables): */
enum clocksource_ids cs_id;
u32 cs_ns_to_cyc_mult;
u32 cs_ns_to_cyc_shift;
u64 cs_ns_to_cyc_maxns;
unsigned int clock_was_set_seq;
u8 cs_was_changed_seq;
u8 clock_valid;
include/linux/timerqueue.h
+48 -8
View File
@@ -5,12 +5,11 @@
#include <linux/rbtree.h>
#include <linux/timerqueue_types.h>
extern bool timerqueue_add(struct timerqueue_head *head,
struct timerqueue_node *node);
extern bool timerqueue_del(struct timerqueue_head *head,
struct timerqueue_node *node);
extern struct timerqueue_node *timerqueue_iterate_next(
struct timerqueue_node *node);
bool timerqueue_add(struct timerqueue_head *head, struct timerqueue_node *node);
bool timerqueue_del(struct timerqueue_head *head, struct timerqueue_node *node);
struct timerqueue_node *timerqueue_iterate_next(struct timerqueue_node *node);
bool timerqueue_linked_add(struct timerqueue_linked_head *head, struct timerqueue_linked_node *node);
/**
* timerqueue_getnext - Returns the timer with the earliest expiration time
@@ -19,8 +18,7 @@ extern struct timerqueue_node *timerqueue_iterate_next(
*
* Returns a pointer to the timer node that has the earliest expiration time.
*/
static inline
struct timerqueue_node *timerqueue_getnext(struct timerqueue_head *head)
static inline struct timerqueue_node *timerqueue_getnext(struct timerqueue_head *head)
{
struct rb_node *leftmost = rb_first_cached(&head->rb_root);
@@ -41,4 +39,46 @@ static inline void timerqueue_init_head(struct timerqueue_head *head)
{
head->rb_root = RB_ROOT_CACHED;
}
/* Timer queues with linked nodes */
static __always_inline
struct timerqueue_linked_node *timerqueue_linked_first(struct timerqueue_linked_head *head)
{
return rb_entry_safe(head->rb_root.rb_leftmost, struct timerqueue_linked_node, node);
}
static __always_inline
struct timerqueue_linked_node *timerqueue_linked_next(struct timerqueue_linked_node *node)
{
return rb_entry_safe(node->node.next, struct timerqueue_linked_node, node);
}
static __always_inline
struct timerqueue_linked_node *timerqueue_linked_prev(struct timerqueue_linked_node *node)
{
return rb_entry_safe(node->node.prev, struct timerqueue_linked_node, node);
}
static __always_inline
bool timerqueue_linked_del(struct timerqueue_linked_head *head, struct timerqueue_linked_node *node)
{
return rb_erase_linked(&node->node, &head->rb_root);
}
static __always_inline void timerqueue_linked_init(struct timerqueue_linked_node *node)
{
RB_CLEAR_LINKED_NODE(&node->node);
}
static __always_inline bool timerqueue_linked_node_queued(struct timerqueue_linked_node *node)
{
return !RB_EMPTY_LINKED_NODE(&node->node);
}
static __always_inline void timerqueue_linked_init_head(struct timerqueue_linked_head *head)
{
head->rb_root = RB_ROOT_LINKED;
}
#endif /* _LINUX_TIMERQUEUE_H */
include/linux/timerqueue_types.h
+12 -3
View File
@@ -6,12 +6,21 @@
#include <linux/types.h>
struct timerqueue_node {
struct rb_node node;
ktime_t expires;
struct rb_node node;
ktime_t expires;
};
struct timerqueue_head {
struct rb_root_cached rb_root;
struct rb_root_cached rb_root;
};
struct timerqueue_linked_node {
struct rb_node_linked node;
ktime_t expires;
};
struct timerqueue_linked_head {
struct rb_root_linked rb_root;
};
#endif /* _LINUX_TIMERQUEUE_TYPES_H */
include/linux/trace_events.h
+8 -5
View File
@@ -22,20 +22,23 @@ union bpf_attr;
const char *trace_print_flags_seq(struct trace_seq *p, const char *delim,
unsigned long flags,
const struct trace_print_flags *flag_array);
const struct trace_print_flags *flag_array,
size_t flag_array_size);
const char *trace_print_symbols_seq(struct trace_seq *p, unsigned long val,
const struct trace_print_flags *symbol_array);
const struct trace_print_flags *symbol_array,
size_t symbol_array_size);
#if BITS_PER_LONG == 32
const char *trace_print_flags_seq_u64(struct trace_seq *p, const char *delim,
unsigned long long flags,
const struct trace_print_flags_u64 *flag_array);
const struct trace_print_flags_u64 *flag_array,
size_t flag_array_size);
const char *trace_print_symbols_seq_u64(struct trace_seq *p,
unsigned long long val,
const struct trace_print_flags_u64
*symbol_array);
const struct trace_print_flags_u64 *symbol_array,
size_t symbol_array_size);
#endif
struct trace_iterator;
include/trace/events/timer.h
+34 -8
View File
@@ -218,12 +218,13 @@ TRACE_EVENT(hrtimer_setup,
* hrtimer_start - called when the hrtimer is started
* @hrtimer: pointer to struct hrtimer
* @mode: the hrtimers mode
* @was_armed: Was armed when hrtimer_start*() was invoked
*/
TRACE_EVENT(hrtimer_start,
TP_PROTO(struct hrtimer *hrtimer, enum hrtimer_mode mode),
TP_PROTO(struct hrtimer *hrtimer, enum hrtimer_mode mode, bool was_armed),
TP_ARGS(hrtimer, mode),
TP_ARGS(hrtimer, mode, was_armed),
TP_STRUCT__entry(
__field( void *, hrtimer )
@@ -231,6 +232,7 @@ TRACE_EVENT(hrtimer_start,
__field( s64, expires )
__field( s64, softexpires )
__field( enum hrtimer_mode, mode )
__field( bool, was_armed )
),
TP_fast_assign(
@@ -239,26 +241,26 @@ TRACE_EVENT(hrtimer_start,
__entry->expires = hrtimer_get_expires(hrtimer);
__entry->softexpires = hrtimer_get_softexpires(hrtimer);
__entry->mode = mode;
__entry->was_armed = was_armed;
),
TP_printk("hrtimer=%p function=%ps expires=%llu softexpires=%llu "
"mode=%s", __entry->hrtimer, __entry->function,
"mode=%s was_armed=%d", __entry->hrtimer, __entry->function,
(unsigned long long) __entry->expires,
(unsigned long long) __entry->softexpires,
decode_hrtimer_mode(__entry->mode))
decode_hrtimer_mode(__entry->mode), __entry->was_armed)
);
/**
* hrtimer_expire_entry - called immediately before the hrtimer callback
* @hrtimer: pointer to struct hrtimer
* @now: pointer to variable which contains current time of the
* timers base.
* @now: variable which contains current time of the timers base.
*
* Allows to determine the timer latency.
*/
TRACE_EVENT(hrtimer_expire_entry,
TP_PROTO(struct hrtimer *hrtimer, ktime_t *now),
TP_PROTO(struct hrtimer *hrtimer, ktime_t now),
TP_ARGS(hrtimer, now),
@@ -270,7 +272,7 @@ TRACE_EVENT(hrtimer_expire_entry,
TP_fast_assign(
__entry->hrtimer = hrtimer;
__entry->now = *now;
__entry->now = now;
__entry->function = ACCESS_PRIVATE(hrtimer, function);
),
@@ -321,6 +323,30 @@ DEFINE_EVENT(hrtimer_class, hrtimer_cancel,
TP_ARGS(hrtimer)
);
/**
* hrtimer_rearm - Invoked when the clockevent device is rearmed
* @next_event: The next expiry time (CLOCK_MONOTONIC)
*/
TRACE_EVENT(hrtimer_rearm,
TP_PROTO(ktime_t next_event, bool deferred),
TP_ARGS(next_event, deferred),
TP_STRUCT__entry(
__field( s64, next_event )
__field( bool, deferred )
),
TP_fast_assign(
__entry->next_event = next_event;
__entry->deferred = deferred;
),
TP_printk("next_event=%llu deferred=%d",
(unsigned long long) __entry->next_event, __entry->deferred)
);
/**
* itimer_state - called when itimer is started or canceled
* @which: name of the interval timer
include/trace/stages/stage3_trace_output.h
+20 -20
View File
@@ -64,36 +64,36 @@
#define __get_rel_sockaddr(field) ((struct sockaddr *)__get_rel_dynamic_array(field))
#undef __print_flags
#define __print_flags(flag, delim, flag_array...) \
({ \
static const struct trace_print_flags __flags[] = \
{ flag_array, { -1, NULL }}; \
trace_print_flags_seq(p, delim, flag, __flags); \
#define __print_flags(flag, delim, flag_array...) \
({ \
static const struct trace_print_flags __flags[] = \
{ flag_array }; \
trace_print_flags_seq(p, delim, flag, __flags, ARRAY_SIZE(__flags)); \
})
#undef __print_symbolic
#define __print_symbolic(value, symbol_array...) \
({ \
static const struct trace_print_flags symbols[] = \
{ symbol_array, { -1, NULL }}; \
trace_print_symbols_seq(p, value, symbols); \
#define __print_symbolic(value, symbol_array...) \
({ \
static const struct trace_print_flags symbols[] = \
{ symbol_array }; \
trace_print_symbols_seq(p, value, symbols, ARRAY_SIZE(symbols)); \
})
#undef __print_flags_u64
#undef __print_symbolic_u64
#if BITS_PER_LONG == 32
#define __print_flags_u64(flag, delim, flag_array...) \
({ \
static const struct trace_print_flags_u64 __flags[] = \
{ flag_array, { -1, NULL } }; \
trace_print_flags_seq_u64(p, delim, flag, __flags); \
#define __print_flags_u64(flag, delim, flag_array...) \
({ \
static const struct trace_print_flags_u64 __flags[] = \
{ flag_array }; \
trace_print_flags_seq_u64(p, delim, flag, __flags, ARRAY_SIZE(__flags)); \
})
#define __print_symbolic_u64(value, symbol_array...) \
({ \
static const struct trace_print_flags_u64 symbols[] = \
{ symbol_array, { -1, NULL } }; \
trace_print_symbols_seq_u64(p, value, symbols); \
#define __print_symbolic_u64(value, symbol_array...) \
({ \
static const struct trace_print_flags_u64 symbols[] = \
{ symbol_array }; \
trace_print_symbols_seq_u64(p, value, symbols, ARRAY_SIZE(symbols)); \
})
#else
#define __print_flags_u64(flag, delim, flag_array...) \
kernel/entry/common.c
+3 -1
View File
@@ -50,7 +50,7 @@ static __always_inline unsigned long __exit_to_user_mode_loop(struct pt_regs *re
local_irq_enable_exit_to_user(ti_work);
if (ti_work & (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY)) {
if (!rseq_grant_slice_extension(ti_work & TIF_SLICE_EXT_DENY))
if (!rseq_grant_slice_extension(ti_work, TIF_SLICE_EXT_DENY))
schedule();
}
@@ -225,6 +225,7 @@ noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state)
*/
if (state.exit_rcu) {
instrumentation_begin();
hrtimer_rearm_deferred();
/* Tell the tracer that IRET will enable interrupts */
trace_hardirqs_on_prepare();
lockdep_hardirqs_on_prepare();
@@ -238,6 +239,7 @@ noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state)
if (IS_ENABLED(CONFIG_PREEMPTION))
irqentry_exit_cond_resched();
hrtimer_rearm_deferred();
/* Covers both tracing and lockdep */
trace_hardirqs_on();
instrumentation_end();
kernel/sched/core.c
+77 -18
View File
@@ -884,7 +884,14 @@ EXPORT_SYMBOL_GPL(update_rq_clock);
* Use HR-timers to deliver accurate preemption points.
*/
static void hrtick_clear(struct rq *rq)
enum {
HRTICK_SCHED_NONE = 0,
HRTICK_SCHED_DEFER = BIT(1),
HRTICK_SCHED_START = BIT(2),
HRTICK_SCHED_REARM_HRTIMER = BIT(3)
};
static void __used hrtick_clear(struct rq *rq)
{
if (hrtimer_active(&rq->hrtick_timer))
hrtimer_cancel(&rq->hrtick_timer);
@@ -909,12 +916,24 @@ static enum hrtimer_restart hrtick(struct hrtimer *timer)
return HRTIMER_NORESTART;
}
static void __hrtick_restart(struct rq *rq)
static inline bool hrtick_needs_rearm(struct hrtimer *timer, ktime_t expires)
{
/*
* Queued is false when the timer is not started or currently
* running the callback. In both cases, restart. If queued check
* whether the expiry time actually changes substantially.
*/
return !hrtimer_is_queued(timer) ||
abs(expires - hrtimer_get_expires(timer)) > 5000;
}
static void hrtick_cond_restart(struct rq *rq)
{
struct hrtimer *timer = &rq->hrtick_timer;
ktime_t time = rq->hrtick_time;
hrtimer_start(timer, time, HRTIMER_MODE_ABS_PINNED_HARD);
if (hrtick_needs_rearm(timer, time))
hrtimer_start(timer, time, HRTIMER_MODE_ABS_PINNED_HARD);
}
/*
@@ -926,7 +945,7 @@ static void __hrtick_start(void *arg)
struct rq_flags rf;
rq_lock(rq, &rf);
__hrtick_restart(rq);
hrtick_cond_restart(rq);
rq_unlock(rq, &rf);
}
@@ -937,7 +956,6 @@ static void __hrtick_start(void *arg)
*/
void hrtick_start(struct rq *rq, u64 delay)
{
struct hrtimer *timer = &rq->hrtick_timer;
s64 delta;
/*
@@ -945,27 +963,67 @@ void hrtick_start(struct rq *rq, u64 delay)
* doesn't make sense and can cause timer DoS.
*/
delta = max_t(s64, delay, 10000LL);
rq->hrtick_time = ktime_add_ns(hrtimer_cb_get_time(timer), delta);
/*
* If this is in the middle of schedule() only note the delay
* and let hrtick_schedule_exit() deal with it.
*/
if (rq->hrtick_sched) {
rq->hrtick_sched |= HRTICK_SCHED_START;
rq->hrtick_delay = delta;
return;
}
rq->hrtick_time = ktime_add_ns(ktime_get(), delta);
if (!hrtick_needs_rearm(&rq->hrtick_timer, rq->hrtick_time))
return;
if (rq == this_rq())
__hrtick_restart(rq);
hrtimer_start(&rq->hrtick_timer, rq->hrtick_time, HRTIMER_MODE_ABS_PINNED_HARD);
else
smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd);
}
static inline void hrtick_schedule_enter(struct rq *rq)
{
rq->hrtick_sched = HRTICK_SCHED_DEFER;
if (hrtimer_test_and_clear_rearm_deferred())
rq->hrtick_sched |= HRTICK_SCHED_REARM_HRTIMER;
}
static inline void hrtick_schedule_exit(struct rq *rq)
{
if (rq->hrtick_sched & HRTICK_SCHED_START) {
rq->hrtick_time = ktime_add_ns(ktime_get(), rq->hrtick_delay);
hrtick_cond_restart(rq);
} else if (idle_rq(rq)) {
/*
* No need for using hrtimer_is_active(). The timer is CPU local
* and interrupts are disabled, so the callback cannot be
* running and the queued state is valid.
*/
if (hrtimer_is_queued(&rq->hrtick_timer))
hrtimer_cancel(&rq->hrtick_timer);
}
if (rq->hrtick_sched & HRTICK_SCHED_REARM_HRTIMER)
__hrtimer_rearm_deferred();
rq->hrtick_sched = HRTICK_SCHED_NONE;
}
static void hrtick_rq_init(struct rq *rq)
{
INIT_CSD(&rq->hrtick_csd, __hrtick_start, rq);
hrtimer_setup(&rq->hrtick_timer, hrtick, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
rq->hrtick_sched = HRTICK_SCHED_NONE;
hrtimer_setup(&rq->hrtick_timer, hrtick, CLOCK_MONOTONIC,
HRTIMER_MODE_REL_HARD | HRTIMER_MODE_LAZY_REARM);
}
#else /* !CONFIG_SCHED_HRTICK: */
static inline void hrtick_clear(struct rq *rq)
{
}
static inline void hrtick_rq_init(struct rq *rq)
{
}
static inline void hrtick_clear(struct rq *rq) { }
static inline void hrtick_rq_init(struct rq *rq) { }
static inline void hrtick_schedule_enter(struct rq *rq) { }
static inline void hrtick_schedule_exit(struct rq *rq) { }
#endif /* !CONFIG_SCHED_HRTICK */
/*
@@ -5127,6 +5185,7 @@ static inline void finish_lock_switch(struct rq *rq)
*/
spin_acquire(&__rq_lockp(rq)->dep_map, 0, 0, _THIS_IP_);
__balance_callbacks(rq, NULL);
hrtick_schedule_exit(rq);
raw_spin_rq_unlock_irq(rq);
}
@@ -6892,9 +6951,6 @@ static void __sched notrace __schedule(int sched_mode)
schedule_debug(prev, preempt);
if (sched_feat(HRTICK) || sched_feat(HRTICK_DL))
hrtick_clear(rq);
klp_sched_try_switch(prev);
local_irq_disable();
@@ -6921,6 +6977,8 @@ static void __sched notrace __schedule(int sched_mode)
rq_lock(rq, &rf);
smp_mb__after_spinlock();
hrtick_schedule_enter(rq);
/* Promote REQ to ACT */
rq->clock_update_flags <<= 1;
update_rq_clock(rq);
@@ -7024,6 +7082,7 @@ keep_resched:
rq_unpin_lock(rq, &rf);
__balance_callbacks(rq, NULL);
hrtick_schedule_exit(rq);
raw_spin_rq_unlock_irq(rq);
}
trace_sched_exit_tp(is_switch);
kernel/sched/deadline.c
+1 -1
View File
@@ -1097,7 +1097,7 @@ static int start_dl_timer(struct sched_dl_entity *dl_se)
act = ns_to_ktime(dl_next_period(dl_se));
}
now = hrtimer_cb_get_time(timer);
now = ktime_get();
delta = ktime_to_ns(now) - rq_clock(rq);
act = ktime_add_ns(act, delta);
kernel/sched/fair.c
+32 -23
View File
@@ -5642,7 +5642,7 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued)
* validating it and just reschedule.
*/
if (queued) {
resched_curr_lazy(rq_of(cfs_rq));
resched_curr(rq_of(cfs_rq));
return;
}
#endif
@@ -6848,27 +6848,41 @@ static inline void sched_fair_update_stop_tick(struct rq *rq, struct task_struct
static void hrtick_start_fair(struct rq *rq, struct task_struct *p)
{
struct sched_entity *se = &p->se;
unsigned long scale = 1024;
unsigned long util = 0;
u64 vdelta;
u64 delta;
WARN_ON_ONCE(task_rq(p) != rq);
if (rq->cfs.h_nr_queued > 1) {
u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime;
u64 slice = se->slice;
s64 delta = slice - ran;
if (rq->cfs.h_nr_queued <= 1)
return;
if (delta < 0) {
if (task_current_donor(rq, p))
resched_curr(rq);
return;
}
hrtick_start(rq, delta);
/*
* Compute time until virtual deadline
*/
vdelta = se->deadline - se->vruntime;
if ((s64)vdelta < 0) {
if (task_current_donor(rq, p))
resched_curr(rq);
return;
}
delta = (se->load.weight * vdelta) / NICE_0_LOAD;
/*
* Correct for instantaneous load of other classes.
*/
util += cpu_util_irq(rq);
if (util && util < 1024) {
scale *= 1024;
scale /= (1024 - util);
}
hrtick_start(rq, (scale * delta) / 1024);
}
/*
* called from enqueue/dequeue and updates the hrtick when the
* current task is from our class and nr_running is low enough
* to matter.
* Called on enqueue to start the hrtick when h_nr_queued becomes more than 1.
*/
static void hrtick_update(struct rq *rq)
{
@@ -6877,6 +6891,9 @@ static void hrtick_update(struct rq *rq)
if (!hrtick_enabled_fair(rq) || donor->sched_class != &fair_sched_class)
return;
if (hrtick_active(rq))
return;
hrtick_start_fair(rq, donor);
}
#else /* !CONFIG_SCHED_HRTICK: */
@@ -7206,9 +7223,6 @@ static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags)
WARN_ON_ONCE(!task_sleep);
WARN_ON_ONCE(p->on_rq != 1);
/* Fix-up what dequeue_task_fair() skipped */
hrtick_update(rq);
/*
* Fix-up what block_task() skipped.
*
@@ -7242,8 +7256,6 @@ static bool dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
/*
* Must not reference @p after dequeue_entities(DEQUEUE_DELAYED).
*/
hrtick_update(rq);
return true;
}
@@ -13562,11 +13574,8 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued)
entity_tick(cfs_rq, se, queued);
}
if (queued) {
if (!need_resched())
hrtick_start_fair(rq, curr);
if (queued)
return;
}
if (static_branch_unlikely(&sched_numa_balancing))
task_tick_numa(rq, curr);
kernel/sched/features.h
+5
View File
@@ -63,8 +63,13 @@ SCHED_FEAT(DELAY_ZERO, true)
*/
SCHED_FEAT(WAKEUP_PREEMPTION, true)
#ifdef CONFIG_HRTIMER_REARM_DEFERRED
SCHED_FEAT(HRTICK, true)
SCHED_FEAT(HRTICK_DL, true)
#else
SCHED_FEAT(HRTICK, false)
SCHED_FEAT(HRTICK_DL, false)
#endif
/*
* Decrement CPU capacity based on time not spent running tasks
kernel/sched/sched.h
+15 -28
View File
@@ -1297,6 +1297,8 @@ struct rq {
call_single_data_t hrtick_csd;
struct hrtimer hrtick_timer;
ktime_t hrtick_time;
ktime_t hrtick_delay;
unsigned int hrtick_sched;
#endif
#ifdef CONFIG_SCHEDSTATS
@@ -3053,46 +3055,31 @@ extern unsigned int sysctl_numa_balancing_hot_threshold;
* - enabled by features
* - hrtimer is actually high res
*/
static inline int hrtick_enabled(struct rq *rq)
static inline bool hrtick_enabled(struct rq *rq)
{
if (!cpu_active(cpu_of(rq)))
return 0;
return hrtimer_is_hres_active(&rq->hrtick_timer);
return cpu_active(cpu_of(rq)) && hrtimer_highres_enabled();
}
static inline int hrtick_enabled_fair(struct rq *rq)
static inline bool hrtick_enabled_fair(struct rq *rq)
{
if (!sched_feat(HRTICK))
return 0;
return hrtick_enabled(rq);
return sched_feat(HRTICK) && hrtick_enabled(rq);
}
static inline int hrtick_enabled_dl(struct rq *rq)
static inline bool hrtick_enabled_dl(struct rq *rq)
{
if (!sched_feat(HRTICK_DL))
return 0;
return hrtick_enabled(rq);
return sched_feat(HRTICK_DL) && hrtick_enabled(rq);
}
extern void hrtick_start(struct rq *rq, u64 delay);
static inline bool hrtick_active(struct rq *rq)
{
return hrtimer_active(&rq->hrtick_timer);
}
#else /* !CONFIG_SCHED_HRTICK: */
static inline int hrtick_enabled_fair(struct rq *rq)
{
return 0;
}
static inline int hrtick_enabled_dl(struct rq *rq)
{
return 0;
}
static inline int hrtick_enabled(struct rq *rq)
{
return 0;
}
static inline bool hrtick_enabled_fair(struct rq *rq) { return false; }
static inline bool hrtick_enabled_dl(struct rq *rq) { return false; }
static inline bool hrtick_enabled(struct rq *rq) { return false; }
#endif /* !CONFIG_SCHED_HRTICK */
#ifndef arch_scale_freq_tick
kernel/softirq.c
+14 -1
View File
@@ -704,6 +704,13 @@ void irq_enter_rcu(void)
{
__irq_enter_raw();
/*
* If this is a nested interrupt that hits the exit_to_user_mode_loop
* where it has enabled interrupts but before it has hit schedule() we
* could have hrtimers in an undefined state. Fix it up here.
*/
hrtimer_rearm_deferred();
if (tick_nohz_full_cpu(smp_processor_id()) ||
(is_idle_task(current) && (irq_count() == HARDIRQ_OFFSET)))
tick_irq_enter();
@@ -760,8 +767,14 @@ static inline void __irq_exit_rcu(void)
#endif
account_hardirq_exit(current);
preempt_count_sub(HARDIRQ_OFFSET);
if (!in_interrupt() && local_softirq_pending())
if (!in_interrupt() && local_softirq_pending()) {
/*
* If we left hrtimers unarmed, make sure to arm them now,
* before enabling interrupts to run SoftIRQ.
*/
hrtimer_rearm_deferred();
invoke_softirq();
}
if (IS_ENABLED(CONFIG_IRQ_FORCED_THREADING) && force_irqthreads() &&
local_timers_pending_force_th() && !(in_nmi() | in_hardirq()))
kernel/time/.kunitconfig
+2
View File
@@ -0,0 +1,2 @@
CONFIG_KUNIT=y
CONFIG_TIME_KUNIT_TEST=y
kernel/time/Kconfig
+16 -12
View File
@@ -17,6 +17,9 @@ config ARCH_CLOCKSOURCE_DATA
config ARCH_CLOCKSOURCE_INIT
bool
config ARCH_WANTS_CLOCKSOURCE_READ_INLINE
bool
# Timekeeping vsyscall support
config GENERIC_TIME_VSYSCALL
bool
@@ -44,10 +47,23 @@ config GENERIC_CLOCKEVENTS_BROADCAST_IDLE
config GENERIC_CLOCKEVENTS_MIN_ADJUST
bool
config GENERIC_CLOCKEVENTS_COUPLED
bool
config GENERIC_CLOCKEVENTS_COUPLED_INLINE
select GENERIC_CLOCKEVENTS_COUPLED
bool
# Generic update of CMOS clock
config GENERIC_CMOS_UPDATE
bool
# Deferred rearming of the hrtimer interrupt
config HRTIMER_REARM_DEFERRED
def_bool y
depends on GENERIC_ENTRY && HAVE_GENERIC_TIF_BITS
depends on HIGH_RES_TIMERS && SCHED_HRTICK
# Select to handle posix CPU timers from task_work
# and not from the timer interrupt context
config HAVE_POSIX_CPU_TIMERS_TASK_WORK
@@ -196,18 +212,6 @@ config HIGH_RES_TIMERS
hardware is not capable then this option only increases
the size of the kernel image.
config CLOCKSOURCE_WATCHDOG_MAX_SKEW_US
int "Clocksource watchdog maximum allowable skew (in microseconds)"
depends on CLOCKSOURCE_WATCHDOG
range 50 1000
default 125
help
Specify the maximum amount of allowable watchdog skew in
microseconds before reporting the clocksource to be unstable.
The default is based on a half-second clocksource watchdog
interval and NTP's maximum frequency drift of 500 parts
per million. If the clocksource is good enough for NTP,
it is good enough for the clocksource watchdog!
endif
config POSIX_AUX_CLOCKS
kernel/time/alarmtimer.c
+8 -4
View File
@@ -234,19 +234,23 @@ static int alarmtimer_suspend(struct device *dev)
if (!rtc)
return 0;
/* Find the soonest timer to expire*/
/* Find the soonest timer to expire */
for (i = 0; i < ALARM_NUMTYPE; i++) {
struct alarm_base *base = &alarm_bases[i];
struct timerqueue_node *next;
ktime_t next_expires;
ktime_t delta;
scoped_guard(spinlock_irqsave, &base->lock)
scoped_guard(spinlock_irqsave, &base->lock) {
next = timerqueue_getnext(&base->timerqueue);
if (next)
next_expires = next->expires;
}
if (!next)
continue;
delta = ktime_sub(next->expires, base->get_ktime());
delta = ktime_sub(next_expires, base->get_ktime());
if (!min || (delta < min)) {
expires = next->expires;
expires = next_expires;
min = delta;
type = i;
}
kernel/time/clockevents.c
+41 -7
View File
@@ -293,6 +293,38 @@ static int clockevents_program_min_delta(struct clock_event_device *dev)
#endif /* CONFIG_GENERIC_CLOCKEVENTS_MIN_ADJUST */
#ifdef CONFIG_GENERIC_CLOCKEVENTS_COUPLED
#ifdef CONFIG_GENERIC_CLOCKEVENTS_COUPLED_INLINE
#include <asm/clock_inlined.h>
#else
static __always_inline void
arch_inlined_clockevent_set_next_coupled(u64 u64 cycles, struct clock_event_device *dev) { }
#endif
static inline bool clockevent_set_next_coupled(struct clock_event_device *dev, ktime_t expires)
{
u64 cycles;
if (unlikely(!(dev->features & CLOCK_EVT_FEAT_CLOCKSOURCE_COUPLED)))
return false;
if (unlikely(!ktime_expiry_to_cycles(dev->cs_id, expires, &cycles)))
return false;
if (IS_ENABLED(CONFIG_GENERIC_CLOCKEVENTS_COUPLED_INLINE))
arch_inlined_clockevent_set_next_coupled(cycles, dev);
else
dev->set_next_coupled(cycles, dev);
return true;
}
#else
static inline bool clockevent_set_next_coupled(struct clock_event_device *dev, ktime_t expires)
{
return false;
}
#endif
/**
* clockevents_program_event - Reprogram the clock event device.
* @dev: device to program
@@ -301,11 +333,10 @@ static int clockevents_program_min_delta(struct clock_event_device *dev)
*
* Returns 0 on success, -ETIME when the event is in the past.
*/
int clockevents_program_event(struct clock_event_device *dev, ktime_t expires,
bool force)
int clockevents_program_event(struct clock_event_device *dev, ktime_t expires, bool force)
{
unsigned long long clc;
int64_t delta;
u64 cycles;
if (WARN_ON_ONCE(expires < 0))
return -ETIME;
@@ -319,10 +350,13 @@ int clockevents_program_event(struct clock_event_device *dev, ktime_t expires,
WARN_ONCE(!clockevent_state_oneshot(dev), "Current state: %d\n",
clockevent_get_state(dev));
/* Shortcut for clockevent devices that can deal with ktime. */
if (dev->features & CLOCK_EVT_FEAT_KTIME)
/* ktime_t based reprogramming for the broadcast hrtimer device */
if (unlikely(dev->features & CLOCK_EVT_FEAT_HRTIMER))
return dev->set_next_ktime(expires, dev);
if (likely(clockevent_set_next_coupled(dev, expires)))
return 0;
delta = ktime_to_ns(ktime_sub(expires, ktime_get()));
/* Required for tick_periodic() during early boot */
@@ -331,8 +365,8 @@ int clockevents_program_event(struct clock_event_device *dev, ktime_t expires,
if (delta > (int64_t)dev->min_delta_ns) {
delta = min(delta, (int64_t) dev->max_delta_ns);
clc = ((unsigned long long) delta * dev->mult) >> dev->shift;
if (!dev->set_next_event((unsigned long) clc, dev))
cycles = ((u64)delta * dev->mult) >> dev->shift;
if (!dev->set_next_event((unsigned long) cycles, dev))
return 0;
}
kernel/time/clocksource-wdtest.c
+145 -151
View File
@@ -3,202 +3,196 @@
* Unit test for the clocksource watchdog.
*
* Copyright (C) 2021 Facebook, Inc.
* Copyright (C) 2026 Intel Corp.
*
* Author: Paul E. McKenney <paulmck@kernel.org>
* Author: Thomas Gleixner <tglx@kernel.org>
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/device.h>
#include <linux/clocksource.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/sched.h> /* for spin_unlock_irq() using preempt_count() m68k */
#include <linux/tick.h>
#include <linux/kthread.h>
#include <linux/delay.h>
#include <linux/prandom.h>
#include <linux/cpu.h>
#include <linux/module.h>
#include <linux/kthread.h>
#include "tick-internal.h"
#include "timekeeping_internal.h"
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Clocksource watchdog unit test");
MODULE_AUTHOR("Paul E. McKenney <paulmck@kernel.org>");
MODULE_AUTHOR("Thomas Gleixner <tglx@kernel.org>");
static int holdoff = IS_BUILTIN(CONFIG_TEST_CLOCKSOURCE_WATCHDOG) ? 10 : 0;
module_param(holdoff, int, 0444);
MODULE_PARM_DESC(holdoff, "Time to wait to start test (s).");
/* Watchdog kthread's task_struct pointer for debug purposes. */
static struct task_struct *wdtest_task;
static u64 wdtest_jiffies_read(struct clocksource *cs)
{
return (u64)jiffies;
}
static struct clocksource clocksource_wdtest_jiffies = {
.name = "wdtest-jiffies",
.rating = 1, /* lowest valid rating*/
.uncertainty_margin = TICK_NSEC,
.read = wdtest_jiffies_read,
.mask = CLOCKSOURCE_MASK(32),
.flags = CLOCK_SOURCE_MUST_VERIFY,
.mult = TICK_NSEC << JIFFIES_SHIFT, /* details above */
.shift = JIFFIES_SHIFT,
.max_cycles = 10,
enum wdtest_states {
WDTEST_INJECT_NONE,
WDTEST_INJECT_DELAY,
WDTEST_INJECT_POSITIVE,
WDTEST_INJECT_NEGATIVE,
WDTEST_INJECT_PERCPU = 0x100,
};
static int wdtest_ktime_read_ndelays;
static bool wdtest_ktime_read_fuzz;
static enum wdtest_states wdtest_state;
static unsigned long wdtest_test_count;
static ktime_t wdtest_last_ts, wdtest_offset;
#define SHIFT_4000PPM 8
static ktime_t wdtest_get_offset(struct clocksource *cs)
{
if (wdtest_state < WDTEST_INJECT_PERCPU)
return wdtest_test_count & 0x1 ? 0 : wdtest_offset >> SHIFT_4000PPM;
/* Only affect the readout of the "remote" CPU */
return cs->wd_cpu == smp_processor_id() ? 0 : NSEC_PER_MSEC;
}
static u64 wdtest_ktime_read(struct clocksource *cs)
{
int wkrn = READ_ONCE(wdtest_ktime_read_ndelays);
static int sign = 1;
u64 ret;
ktime_t now = ktime_get_raw_fast_ns();
ktime_t intv = now - wdtest_last_ts;
if (wkrn) {
udelay(cs->uncertainty_margin / 250);
WRITE_ONCE(wdtest_ktime_read_ndelays, wkrn - 1);
/*
* Only increment the test counter once per watchdog interval and
* store the interval for the offset calculation of this step. This
* guarantees a consistent behaviour even if the other side needs
* to repeat due to a watchdog read timeout.
*/
if (intv > (NSEC_PER_SEC / 4)) {
WRITE_ONCE(wdtest_test_count, wdtest_test_count + 1);
wdtest_last_ts = now;
wdtest_offset = intv;
}
ret = ktime_get_real_fast_ns();
if (READ_ONCE(wdtest_ktime_read_fuzz)) {
sign = -sign;
ret = ret + sign * 100 * NSEC_PER_MSEC;
switch (wdtest_state & ~WDTEST_INJECT_PERCPU) {
case WDTEST_INJECT_POSITIVE:
return now + wdtest_get_offset(cs);
case WDTEST_INJECT_NEGATIVE:
return now - wdtest_get_offset(cs);
case WDTEST_INJECT_DELAY:
udelay(500);
return now;
default:
return now;
}
return ret;
}
static void wdtest_ktime_cs_mark_unstable(struct clocksource *cs)
{
pr_info("--- Marking %s unstable due to clocksource watchdog.\n", cs->name);
}
#define KTIME_FLAGS (CLOCK_SOURCE_IS_CONTINUOUS | \
CLOCK_SOURCE_VALID_FOR_HRES | \
CLOCK_SOURCE_MUST_VERIFY | \
CLOCK_SOURCE_VERIFY_PERCPU)
#define KTIME_FLAGS (CLOCK_SOURCE_IS_CONTINUOUS | \
CLOCK_SOURCE_CALIBRATED | \
CLOCK_SOURCE_MUST_VERIFY | \
CLOCK_SOURCE_WDTEST)
static struct clocksource clocksource_wdtest_ktime = {
.name = "wdtest-ktime",
.rating = 300,
.rating = 10,
.read = wdtest_ktime_read,
.mask = CLOCKSOURCE_MASK(64),
.flags = KTIME_FLAGS,
.mark_unstable = wdtest_ktime_cs_mark_unstable,
.list = LIST_HEAD_INIT(clocksource_wdtest_ktime.list),
};
/* Reset the clocksource if needed. */
static void wdtest_ktime_clocksource_reset(void)
static void wdtest_clocksource_reset(enum wdtest_states which, bool percpu)
{
if (clocksource_wdtest_ktime.flags & CLOCK_SOURCE_UNSTABLE) {
clocksource_unregister(&clocksource_wdtest_ktime);
clocksource_wdtest_ktime.flags = KTIME_FLAGS;
schedule_timeout_uninterruptible(HZ / 10);
clocksource_register_khz(&clocksource_wdtest_ktime, 1000 * 1000);
}
}
/* Run the specified series of watchdog tests. */
static int wdtest_func(void *arg)
{
unsigned long j1, j2;
int i, max_retries;
char *s;
schedule_timeout_uninterruptible(holdoff * HZ);
/*
* Verify that jiffies-like clocksources get the manually
* specified uncertainty margin.
*/
pr_info("--- Verify jiffies-like uncertainty margin.\n");
__clocksource_register(&clocksource_wdtest_jiffies);
WARN_ON_ONCE(clocksource_wdtest_jiffies.uncertainty_margin != TICK_NSEC);
j1 = clocksource_wdtest_jiffies.read(&clocksource_wdtest_jiffies);
schedule_timeout_uninterruptible(HZ);
j2 = clocksource_wdtest_jiffies.read(&clocksource_wdtest_jiffies);
WARN_ON_ONCE(j1 == j2);
clocksource_unregister(&clocksource_wdtest_jiffies);
/*
* Verify that tsc-like clocksources are assigned a reasonable
* uncertainty margin.
*/
pr_info("--- Verify tsc-like uncertainty margin.\n");
clocksource_register_khz(&clocksource_wdtest_ktime, 1000 * 1000);
WARN_ON_ONCE(clocksource_wdtest_ktime.uncertainty_margin < NSEC_PER_USEC);
j1 = clocksource_wdtest_ktime.read(&clocksource_wdtest_ktime);
udelay(1);
j2 = clocksource_wdtest_ktime.read(&clocksource_wdtest_ktime);
pr_info("--- tsc-like times: %lu - %lu = %lu.\n", j2, j1, j2 - j1);
WARN_ONCE(time_before(j2, j1 + NSEC_PER_USEC),
"Expected at least 1000ns, got %lu.\n", j2 - j1);
/* Verify tsc-like stability with various numbers of errors injected. */
max_retries = clocksource_get_max_watchdog_retry();
for (i = 0; i <= max_retries + 1; i++) {
if (i <= 1 && i < max_retries)
s = "";
else if (i <= max_retries)
s = ", expect message";
else
s = ", expect clock skew";
pr_info("--- Watchdog with %dx error injection, %d retries%s.\n", i, max_retries, s);
WRITE_ONCE(wdtest_ktime_read_ndelays, i);
schedule_timeout_uninterruptible(2 * HZ);
WARN_ON_ONCE(READ_ONCE(wdtest_ktime_read_ndelays));
WARN_ON_ONCE((i <= max_retries) !=
!(clocksource_wdtest_ktime.flags & CLOCK_SOURCE_UNSTABLE));
wdtest_ktime_clocksource_reset();
}
/* Verify tsc-like stability with clock-value-fuzz error injection. */
pr_info("--- Watchdog clock-value-fuzz error injection, expect clock skew and per-CPU mismatches.\n");
WRITE_ONCE(wdtest_ktime_read_fuzz, true);
schedule_timeout_uninterruptible(2 * HZ);
WARN_ON_ONCE(!(clocksource_wdtest_ktime.flags & CLOCK_SOURCE_UNSTABLE));
clocksource_verify_percpu(&clocksource_wdtest_ktime);
WRITE_ONCE(wdtest_ktime_read_fuzz, false);
clocksource_unregister(&clocksource_wdtest_ktime);
pr_info("--- Done with test.\n");
pr_info("Test: State %d percpu %d\n", which, percpu);
wdtest_state = which;
if (percpu)
wdtest_state |= WDTEST_INJECT_PERCPU;
wdtest_test_count = 0;
wdtest_last_ts = 0;
clocksource_wdtest_ktime.rating = 10;
clocksource_wdtest_ktime.flags = KTIME_FLAGS;
if (percpu)
clocksource_wdtest_ktime.flags |= CLOCK_SOURCE_WDTEST_PERCPU;
clocksource_register_khz(&clocksource_wdtest_ktime, 1000 * 1000);
}
static bool wdtest_execute(enum wdtest_states which, bool percpu, unsigned int expect,
unsigned long calls)
{
wdtest_clocksource_reset(which, percpu);
for (; READ_ONCE(wdtest_test_count) < calls; msleep(100)) {
unsigned int flags = READ_ONCE(clocksource_wdtest_ktime.flags);
if (kthread_should_stop())
return false;
if (flags & CLOCK_SOURCE_UNSTABLE) {
if (expect & CLOCK_SOURCE_UNSTABLE)
return true;
pr_warn("Fail: Unexpected unstable\n");
return false;
}
if (flags & CLOCK_SOURCE_VALID_FOR_HRES) {
if (expect & CLOCK_SOURCE_VALID_FOR_HRES)
return true;
pr_warn("Fail: Unexpected valid for highres\n");
return false;
}
}
if (!expect)
return true;
pr_warn("Fail: Timed out\n");
return false;
}
static bool wdtest_run(bool percpu)
{
if (!wdtest_execute(WDTEST_INJECT_NONE, percpu, CLOCK_SOURCE_VALID_FOR_HRES, 8))
return false;
if (!wdtest_execute(WDTEST_INJECT_DELAY, percpu, 0, 4))
return false;
if (!wdtest_execute(WDTEST_INJECT_POSITIVE, percpu, CLOCK_SOURCE_UNSTABLE, 8))
return false;
if (!wdtest_execute(WDTEST_INJECT_NEGATIVE, percpu, CLOCK_SOURCE_UNSTABLE, 8))
return false;
return true;
}
static int wdtest_func(void *arg)
{
clocksource_register_khz(&clocksource_wdtest_ktime, 1000 * 1000);
if (wdtest_run(false)) {
if (wdtest_run(true))
pr_info("Success: All tests passed\n");
}
clocksource_unregister(&clocksource_wdtest_ktime);
if (!IS_MODULE(CONFIG_TEST_CLOCKSOURCE_WATCHDOG))
return 0;
while (!kthread_should_stop())
schedule_timeout_interruptible(3600 * HZ);
return 0;
}
static void wdtest_print_module_parms(void)
{
pr_alert("--- holdoff=%d\n", holdoff);
}
/* Cleanup function. */
static void clocksource_wdtest_cleanup(void)
{
}
static struct task_struct *wdtest_thread;
static int __init clocksource_wdtest_init(void)
{
int ret = 0;
struct task_struct *t = kthread_run(wdtest_func, NULL, "wdtest");
wdtest_print_module_parms();
/* Create watchdog-test task. */
wdtest_task = kthread_run(wdtest_func, NULL, "wdtest");
if (IS_ERR(wdtest_task)) {
ret = PTR_ERR(wdtest_task);
pr_warn("%s: Failed to create wdtest kthread.\n", __func__);
wdtest_task = NULL;
return ret;
if (IS_ERR(t)) {
pr_warn("Failed to create wdtest kthread.\n");
return PTR_ERR(t);
}
wdtest_thread = t;
return 0;
}
module_init(clocksource_wdtest_init);
static void clocksource_wdtest_cleanup(void)
{
if (wdtest_thread)
kthread_stop(wdtest_thread);
}
module_exit(clocksource_wdtest_cleanup);
kernel/time/clocksource.c
+474 -403
View File
@@ -7,15 +7,17 @@
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/device.h>
#include <linux/clocksource.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/sched.h> /* for spin_unlock_irq() using preempt_count() m68k */
#include <linux/tick.h>
#include <linux/kthread.h>
#include <linux/prandom.h>
#include <linux/cpu.h>
#include <linux/delay.h>
#include <linux/device.h>
#include <linux/init.h>
#include <linux/kthread.h>
#include <linux/module.h>
#include <linux/prandom.h>
#include <linux/sched.h>
#include <linux/tick.h>
#include <linux/topology.h>
#include "tick-internal.h"
#include "timekeeping_internal.h"
@@ -107,48 +109,6 @@ static char override_name[CS_NAME_LEN];
static int finished_booting;
static u64 suspend_start;
/*
* Interval: 0.5sec.
*/
#define WATCHDOG_INTERVAL (HZ >> 1)
#define WATCHDOG_INTERVAL_MAX_NS ((2 * WATCHDOG_INTERVAL) * (NSEC_PER_SEC / HZ))
/*
* Threshold: 0.0312s, when doubled: 0.0625s.
*/
#define WATCHDOG_THRESHOLD (NSEC_PER_SEC >> 5)
/*
* Maximum permissible delay between two readouts of the watchdog
* clocksource surrounding a read of the clocksource being validated.
* This delay could be due to SMIs, NMIs, or to VCPU preemptions. Used as
* a lower bound for cs->uncertainty_margin values when registering clocks.
*
* The default of 500 parts per million is based on NTP's limits.
* If a clocksource is good enough for NTP, it is good enough for us!
*
* In other words, by default, even if a clocksource is extremely
* precise (for example, with a sub-nanosecond period), the maximum
* permissible skew between the clocksource watchdog and the clocksource
* under test is not permitted to go below the 500ppm minimum defined
* by MAX_SKEW_USEC. This 500ppm minimum may be overridden using the
* CLOCKSOURCE_WATCHDOG_MAX_SKEW_US Kconfig option.
*/
#ifdef CONFIG_CLOCKSOURCE_WATCHDOG_MAX_SKEW_US
#define MAX_SKEW_USEC CONFIG_CLOCKSOURCE_WATCHDOG_MAX_SKEW_US
#else
#define MAX_SKEW_USEC (125 * WATCHDOG_INTERVAL / HZ)
#endif
/*
* Default for maximum permissible skew when cs->uncertainty_margin is
* not specified, and the lower bound even when cs->uncertainty_margin
* is specified. This is also the default that is used when registering
* clocks with unspecified cs->uncertainty_margin, so this macro is used
* even in CONFIG_CLOCKSOURCE_WATCHDOG=n kernels.
*/
#define WATCHDOG_MAX_SKEW (MAX_SKEW_USEC * NSEC_PER_USEC)
#ifdef CONFIG_CLOCKSOURCE_WATCHDOG
static void clocksource_watchdog_work(struct work_struct *work);
static void clocksource_select(void);
@@ -160,7 +120,42 @@ static DECLARE_WORK(watchdog_work, clocksource_watchdog_work);
static DEFINE_SPINLOCK(watchdog_lock);
static int watchdog_running;
static atomic_t watchdog_reset_pending;
static int64_t watchdog_max_interval;
/* Watchdog interval: 0.5sec. */
#define WATCHDOG_INTERVAL (HZ >> 1)
#define WATCHDOG_INTERVAL_NS (WATCHDOG_INTERVAL * (NSEC_PER_SEC / HZ))
/* Maximum time between two reference watchdog readouts */
#define WATCHDOG_READOUT_MAX_NS (50U * NSEC_PER_USEC)
/*
* Maximum time between two remote readouts for NUMA=n. On NUMA enabled systems
* the timeout is calculated from the numa distance.
*/
#define WATCHDOG_DEFAULT_TIMEOUT_NS (50U * NSEC_PER_USEC)
/*
* Remote timeout NUMA distance multiplier. The local distance is 10. The
* default remote distance is 20. ACPI tables provide more accurate numbers
* which are guaranteed to be greater than the local distance.
*
* This results in a 5us base value, which is equivalent to the above !NUMA
* default.
*/
#define WATCHDOG_NUMA_MULTIPLIER_NS ((u64)(WATCHDOG_DEFAULT_TIMEOUT_NS / LOCAL_DISTANCE))
/* Limit the NUMA timeout in case the distance values are insanely big */
#define WATCHDOG_NUMA_MAX_TIMEOUT_NS ((u64)(500U * NSEC_PER_USEC))
/* Shift values to calculate the approximate $N ppm of a given delta. */
#define SHIFT_500PPM 11
#define SHIFT_4000PPM 8
/* Number of attempts to read the watchdog */
#define WATCHDOG_FREQ_RETRIES 3
/* Five reads local and remote for inter CPU skew detection */
#define WATCHDOG_REMOTE_MAX_SEQ 10
static inline void clocksource_watchdog_lock(unsigned long *flags)
{
@@ -241,177 +236,6 @@ void clocksource_mark_unstable(struct clocksource *cs)
spin_unlock_irqrestore(&watchdog_lock, flags);
}
static int verify_n_cpus = 8;
module_param(verify_n_cpus, int, 0644);
enum wd_read_status {
WD_READ_SUCCESS,
WD_READ_UNSTABLE,
WD_READ_SKIP
};
static enum wd_read_status cs_watchdog_read(struct clocksource *cs, u64 *csnow, u64 *wdnow)
{
int64_t md = watchdog->uncertainty_margin;
unsigned int nretries, max_retries;
int64_t wd_delay, wd_seq_delay;
u64 wd_end, wd_end2;
max_retries = clocksource_get_max_watchdog_retry();
for (nretries = 0; nretries <= max_retries; nretries++) {
local_irq_disable();
*wdnow = watchdog->read(watchdog);
*csnow = cs->read(cs);
wd_end = watchdog->read(watchdog);
wd_end2 = watchdog->read(watchdog);
local_irq_enable();
wd_delay = cycles_to_nsec_safe(watchdog, *wdnow, wd_end);
if (wd_delay <= md + cs->uncertainty_margin) {
if (nretries > 1 && nretries >= max_retries) {
pr_warn("timekeeping watchdog on CPU%d: %s retried %d times before success\n",
smp_processor_id(), watchdog->name, nretries);
}
return WD_READ_SUCCESS;
}
/*
* Now compute delay in consecutive watchdog read to see if
* there is too much external interferences that cause
* significant delay in reading both clocksource and watchdog.
*
* If consecutive WD read-back delay > md, report
* system busy, reinit the watchdog and skip the current
* watchdog test.
*/
wd_seq_delay = cycles_to_nsec_safe(watchdog, wd_end, wd_end2);
if (wd_seq_delay > md)
goto skip_test;
}
pr_warn("timekeeping watchdog on CPU%d: wd-%s-wd excessive read-back delay of %lldns vs. limit of %ldns, wd-wd read-back delay only %lldns, attempt %d, marking %s unstable\n",
smp_processor_id(), cs->name, wd_delay, WATCHDOG_MAX_SKEW, wd_seq_delay, nretries, cs->name);
return WD_READ_UNSTABLE;
skip_test:
pr_info("timekeeping watchdog on CPU%d: %s wd-wd read-back delay of %lldns\n",
smp_processor_id(), watchdog->name, wd_seq_delay);
pr_info("wd-%s-wd read-back delay of %lldns, clock-skew test skipped!\n",
cs->name, wd_delay);
return WD_READ_SKIP;
}
static u64 csnow_mid;
static cpumask_t cpus_ahead;
static cpumask_t cpus_behind;
static cpumask_t cpus_chosen;
static void clocksource_verify_choose_cpus(void)
{
int cpu, i, n = verify_n_cpus;
if (n < 0 || n >= num_online_cpus()) {
/* Check all of the CPUs. */
cpumask_copy(&cpus_chosen, cpu_online_mask);
cpumask_clear_cpu(smp_processor_id(), &cpus_chosen);
return;
}
/* If no checking desired, or no other CPU to check, leave. */
cpumask_clear(&cpus_chosen);
if (n == 0 || num_online_cpus() <= 1)
return;
/* Make sure to select at least one CPU other than the current CPU. */
cpu = cpumask_any_but(cpu_online_mask, smp_processor_id());
if (WARN_ON_ONCE(cpu >= nr_cpu_ids))
return;
cpumask_set_cpu(cpu, &cpus_chosen);
/* Force a sane value for the boot parameter. */
if (n > nr_cpu_ids)
n = nr_cpu_ids;
/*
* Randomly select the specified number of CPUs. If the same
* CPU is selected multiple times, that CPU is checked only once,
* and no replacement CPU is selected. This gracefully handles
* situations where verify_n_cpus is greater than the number of
* CPUs that are currently online.
*/
for (i = 1; i < n; i++) {
cpu = cpumask_random(cpu_online_mask);
if (!WARN_ON_ONCE(cpu >= nr_cpu_ids))
cpumask_set_cpu(cpu, &cpus_chosen);
}
/* Don't verify ourselves. */
cpumask_clear_cpu(smp_processor_id(), &cpus_chosen);
}
static void clocksource_verify_one_cpu(void *csin)
{
struct clocksource *cs = (struct clocksource *)csin;
csnow_mid = cs->read(cs);
}
void clocksource_verify_percpu(struct clocksource *cs)
{
int64_t cs_nsec, cs_nsec_max = 0, cs_nsec_min = LLONG_MAX;
u64 csnow_begin, csnow_end;
int cpu, testcpu;
s64 delta;
if (verify_n_cpus == 0)
return;
cpumask_clear(&cpus_ahead);
cpumask_clear(&cpus_behind);
cpus_read_lock();
migrate_disable();
clocksource_verify_choose_cpus();
if (cpumask_empty(&cpus_chosen)) {
migrate_enable();
cpus_read_unlock();
pr_warn("Not enough CPUs to check clocksource '%s'.\n", cs->name);
return;
}
testcpu = smp_processor_id();
pr_info("Checking clocksource %s synchronization from CPU %d to CPUs %*pbl.\n",
cs->name, testcpu, cpumask_pr_args(&cpus_chosen));
preempt_disable();
for_each_cpu(cpu, &cpus_chosen) {
if (cpu == testcpu)
continue;
csnow_begin = cs->read(cs);
smp_call_function_single(cpu, clocksource_verify_one_cpu, cs, 1);
csnow_end = cs->read(cs);
delta = (s64)((csnow_mid - csnow_begin) & cs->mask);
if (delta < 0)
cpumask_set_cpu(cpu, &cpus_behind);
delta = (csnow_end - csnow_mid) & cs->mask;
if (delta < 0)
cpumask_set_cpu(cpu, &cpus_ahead);
cs_nsec = cycles_to_nsec_safe(cs, csnow_begin, csnow_end);
if (cs_nsec > cs_nsec_max)
cs_nsec_max = cs_nsec;
if (cs_nsec < cs_nsec_min)
cs_nsec_min = cs_nsec;
}
preempt_enable();
migrate_enable();
cpus_read_unlock();
if (!cpumask_empty(&cpus_ahead))
pr_warn(" CPUs %*pbl ahead of CPU %d for clocksource %s.\n",
cpumask_pr_args(&cpus_ahead), testcpu, cs->name);
if (!cpumask_empty(&cpus_behind))
pr_warn(" CPUs %*pbl behind CPU %d for clocksource %s.\n",
cpumask_pr_args(&cpus_behind), testcpu, cs->name);
pr_info(" CPU %d check durations %lldns - %lldns for clocksource %s.\n",
testcpu, cs_nsec_min, cs_nsec_max, cs->name);
}
EXPORT_SYMBOL_GPL(clocksource_verify_percpu);
static inline void clocksource_reset_watchdog(void)
{
struct clocksource *cs;
@@ -420,25 +244,414 @@ static inline void clocksource_reset_watchdog(void)
cs->flags &= ~CLOCK_SOURCE_WATCHDOG;
}
enum wd_result {
WD_SUCCESS,
WD_FREQ_NO_WATCHDOG,
WD_FREQ_TIMEOUT,
WD_FREQ_RESET,
WD_FREQ_SKEWED,
WD_CPU_TIMEOUT,
WD_CPU_SKEWED,
};
struct watchdog_cpu_data {
/* Keep first as it is 32 byte aligned */
call_single_data_t csd;
atomic_t remote_inprogress;
enum wd_result result;
u64 cpu_ts[2];
struct clocksource *cs;
/* Ensure that the sequence is in a separate cache line */
atomic_t seq ____cacheline_aligned;
/* Set by the control CPU according to NUMA distance */
u64 timeout_ns;
};
struct watchdog_data {
raw_spinlock_t lock;
enum wd_result result;
u64 wd_seq;
u64 wd_delta;
u64 cs_delta;
u64 cpu_ts[2];
unsigned int curr_cpu;
} ____cacheline_aligned_in_smp;
static void watchdog_check_skew_remote(void *unused);
static DEFINE_PER_CPU_ALIGNED(struct watchdog_cpu_data, watchdog_cpu_data) = {
.csd = CSD_INIT(watchdog_check_skew_remote, NULL),
};
static struct watchdog_data watchdog_data = {
.lock = __RAW_SPIN_LOCK_UNLOCKED(watchdog_data.lock),
};
static inline void watchdog_set_result(struct watchdog_cpu_data *wd, enum wd_result result)
{
guard(raw_spinlock)(&watchdog_data.lock);
if (!wd->result) {
atomic_set(&wd->seq, WATCHDOG_REMOTE_MAX_SEQ);
WRITE_ONCE(wd->result, result);
}
}
/* Wait for the sequence number to hand over control. */
static bool watchdog_wait_seq(struct watchdog_cpu_data *wd, u64 start, int seq)
{
for(int cnt = 0; atomic_read(&wd->seq) < seq; cnt++) {
/* Bail if the other side set an error result */
if (READ_ONCE(wd->result) != WD_SUCCESS)
return false;
/* Prevent endless loops if the other CPU does not react. */
if (cnt == 5000) {
u64 nsecs = ktime_get_raw_fast_ns();
if (nsecs - start >=wd->timeout_ns) {
watchdog_set_result(wd, WD_CPU_TIMEOUT);
return false;
}
cnt = 0;
}
cpu_relax();
}
return seq < WATCHDOG_REMOTE_MAX_SEQ;
}
static void watchdog_check_skew(struct watchdog_cpu_data *wd, int index)
{
u64 prev, now, delta, start = ktime_get_raw_fast_ns();
int local = index, remote = (index + 1) & 0x1;
struct clocksource *cs = wd->cs;
/* Set the local timestamp so that the first iteration works correctly */
wd->cpu_ts[local] = cs->read(cs);
/* Signal arrival */
atomic_inc(&wd->seq);
for (int seq = local + 2; seq < WATCHDOG_REMOTE_MAX_SEQ; seq += 2) {
if (!watchdog_wait_seq(wd, start, seq))
return;
/* Capture local timestamp before possible non-local coherency overhead */
now = cs->read(cs);
/* Store local timestamp before reading remote to limit coherency stalls */
wd->cpu_ts[local] = now;
prev = wd->cpu_ts[remote];
delta = (now - prev) & cs->mask;
if (delta > cs->max_raw_delta) {
watchdog_set_result(wd, WD_CPU_SKEWED);
return;
}
/* Hand over to the remote CPU */
atomic_inc(&wd->seq);
}
}
static void watchdog_check_skew_remote(void *unused)
{
struct watchdog_cpu_data *wd = this_cpu_ptr(&watchdog_cpu_data);
atomic_inc(&wd->remote_inprogress);
watchdog_check_skew(wd, 1);
atomic_dec(&wd->remote_inprogress);
}
static inline bool wd_csd_locked(struct watchdog_cpu_data *wd)
{
return READ_ONCE(wd->csd.node.u_flags) & CSD_FLAG_LOCK;
}
/*
* This is only invoked for remote CPUs. See watchdog_check_cpu_skew().
*/
static inline u64 wd_get_remote_timeout(unsigned int remote_cpu)
{
unsigned int n1, n2;
u64 ns;
if (nr_node_ids == 1)
return WATCHDOG_DEFAULT_TIMEOUT_NS;
n1 = cpu_to_node(smp_processor_id());
n2 = cpu_to_node(remote_cpu);
ns = WATCHDOG_NUMA_MULTIPLIER_NS * node_distance(n1, n2);
return min(ns, WATCHDOG_NUMA_MAX_TIMEOUT_NS);
}
static void __watchdog_check_cpu_skew(struct clocksource *cs, unsigned int cpu)
{
struct watchdog_cpu_data *wd;
wd = per_cpu_ptr(&watchdog_cpu_data, cpu);
if (atomic_read(&wd->remote_inprogress) || wd_csd_locked(wd)) {
watchdog_data.result = WD_CPU_TIMEOUT;
return;
}
atomic_set(&wd->seq, 0);
wd->result = WD_SUCCESS;
wd->cs = cs;
/* Store the current CPU ID for the watchdog test unit */
cs->wd_cpu = smp_processor_id();
wd->timeout_ns = wd_get_remote_timeout(cpu);
/* Kick the remote CPU into the watchdog function */
if (WARN_ON_ONCE(smp_call_function_single_async(cpu, &wd->csd))) {
watchdog_data.result = WD_CPU_TIMEOUT;
return;
}
scoped_guard(irq)
watchdog_check_skew(wd, 0);
scoped_guard(raw_spinlock_irq, &watchdog_data.lock) {
watchdog_data.result = wd->result;
memcpy(watchdog_data.cpu_ts, wd->cpu_ts, sizeof(wd->cpu_ts));
}
}
static void watchdog_check_cpu_skew(struct clocksource *cs)
{
unsigned int cpu = watchdog_data.curr_cpu;
cpu = cpumask_next_wrap(cpu, cpu_online_mask);
watchdog_data.curr_cpu = cpu;
/* Skip the current CPU. Handles num_online_cpus() == 1 as well */
if (cpu == smp_processor_id())
return;
/* Don't interfere with the test mechanics */
if ((cs->flags & CLOCK_SOURCE_WDTEST) && !(cs->flags & CLOCK_SOURCE_WDTEST_PERCPU))
return;
__watchdog_check_cpu_skew(cs, cpu);
}
static bool watchdog_check_freq(struct clocksource *cs, bool reset_pending)
{
unsigned int ppm_shift = SHIFT_4000PPM;
u64 wd_ts0, wd_ts1, cs_ts;
watchdog_data.result = WD_SUCCESS;
if (!watchdog) {
watchdog_data.result = WD_FREQ_NO_WATCHDOG;
return false;
}
if (cs->flags & CLOCK_SOURCE_WDTEST_PERCPU)
return true;
/*
* If both the clocksource and the watchdog claim they are
* calibrated use 500ppm limit. Uncalibrated clocksources need a
* larger allowance because thefirmware supplied frequencies can be
* way off.
*/
if (watchdog->flags & CLOCK_SOURCE_CALIBRATED && cs->flags & CLOCK_SOURCE_CALIBRATED)
ppm_shift = SHIFT_500PPM;
for (int retries = 0; retries < WATCHDOG_FREQ_RETRIES; retries++) {
s64 wd_last, cs_last, wd_seq, wd_delta, cs_delta, max_delta;
scoped_guard(irq) {
wd_ts0 = watchdog->read(watchdog);
cs_ts = cs->read(cs);
wd_ts1 = watchdog->read(watchdog);
}
wd_last = cs->wd_last;
cs_last = cs->cs_last;
/* Validate the watchdog readout window */
wd_seq = cycles_to_nsec_safe(watchdog, wd_ts0, wd_ts1);
if (wd_seq > WATCHDOG_READOUT_MAX_NS) {
/* Store for printout in case all retries fail */
watchdog_data.wd_seq = wd_seq;
continue;
}
/* Store for subsequent processing */
cs->wd_last = wd_ts0;
cs->cs_last = cs_ts;
/* First round or reset pending? */
if (!(cs->flags & CLOCK_SOURCE_WATCHDOG) || reset_pending)
goto reset;
/* Calculate the nanosecond deltas from the last invocation */
wd_delta = cycles_to_nsec_safe(watchdog, wd_last, wd_ts0);
cs_delta = cycles_to_nsec_safe(cs, cs_last, cs_ts);
watchdog_data.wd_delta = wd_delta;
watchdog_data.cs_delta = cs_delta;
/*
* Ensure that the deltas are within the readout limits of
* the clocksource and the watchdog. Long delays can cause
* clocksources to overflow.
*/
max_delta = max(wd_delta, cs_delta);
if (max_delta > cs->max_idle_ns || max_delta > watchdog->max_idle_ns)
goto reset;
/*
* Calculate and validate the skew against the allowed PPM
* value of the maximum delta plus the watchdog readout
* time.
*/
if (abs(wd_delta - cs_delta) < (max_delta >> ppm_shift) + wd_seq)
return true;
watchdog_data.result = WD_FREQ_SKEWED;
return false;
}
watchdog_data.result = WD_FREQ_TIMEOUT;
return false;
reset:
cs->flags |= CLOCK_SOURCE_WATCHDOG;
watchdog_data.result = WD_FREQ_RESET;
return false;
}
/* Synchronization for sched clock */
static void clocksource_tick_stable(struct clocksource *cs)
{
if (cs == curr_clocksource && cs->tick_stable)
cs->tick_stable(cs);
}
/* Conditionaly enable high resolution mode */
static void clocksource_enable_highres(struct clocksource *cs)
{
if ((cs->flags & CLOCK_SOURCE_VALID_FOR_HRES) ||
!(cs->flags & CLOCK_SOURCE_IS_CONTINUOUS) ||
!watchdog || !(watchdog->flags & CLOCK_SOURCE_IS_CONTINUOUS))
return;
/* Mark it valid for high-res. */
cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES;
/*
* Can't schedule work before finished_booting is
* true. clocksource_done_booting will take care of it.
*/
if (!finished_booting)
return;
if (cs->flags & CLOCK_SOURCE_WDTEST)
return;
/*
* If this is not the current clocksource let the watchdog thread
* reselect it. Due to the change to high res this clocksource
* might be preferred now. If it is the current clocksource let the
* tick code know about that change.
*/
if (cs != curr_clocksource) {
cs->flags |= CLOCK_SOURCE_RESELECT;
schedule_work(&watchdog_work);
} else {
tick_clock_notify();
}
}
static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 2);
static void watchdog_print_freq_timeout(struct clocksource *cs)
{
if (!__ratelimit(&ratelimit_state))
return;
pr_info("Watchdog %s read timed out. Readout sequence took: %lluns\n",
watchdog->name, watchdog_data.wd_seq);
}
static void watchdog_print_freq_skew(struct clocksource *cs)
{
pr_warn("Marking clocksource %s unstable due to frequency skew\n", cs->name);
pr_warn("Watchdog %20s interval: %16lluns\n", watchdog->name, watchdog_data.wd_delta);
pr_warn("Clocksource %20s interval: %16lluns\n", cs->name, watchdog_data.cs_delta);
}
static void watchdog_handle_remote_timeout(struct clocksource *cs)
{
pr_info_once("Watchdog remote CPU %u read timed out\n", watchdog_data.curr_cpu);
}
static void watchdog_print_remote_skew(struct clocksource *cs)
{
pr_warn("Marking clocksource %s unstable due to inter CPU skew\n", cs->name);
if (watchdog_data.cpu_ts[0] < watchdog_data.cpu_ts[1]) {
pr_warn("CPU%u %16llu < CPU%u %16llu (cycles)\n", smp_processor_id(),
watchdog_data.cpu_ts[0], watchdog_data.curr_cpu, watchdog_data.cpu_ts[1]);
} else {
pr_warn("CPU%u %16llu < CPU%u %16llu (cycles)\n", watchdog_data.curr_cpu,
watchdog_data.cpu_ts[1], smp_processor_id(), watchdog_data.cpu_ts[0]);
}
}
static void watchdog_check_result(struct clocksource *cs)
{
switch (watchdog_data.result) {
case WD_SUCCESS:
clocksource_tick_stable(cs);
clocksource_enable_highres(cs);
return;
case WD_FREQ_TIMEOUT:
watchdog_print_freq_timeout(cs);
/* Try again later and invalidate the reference timestamps. */
cs->flags &= ~CLOCK_SOURCE_WATCHDOG;
return;
case WD_FREQ_NO_WATCHDOG:
case WD_FREQ_RESET:
/*
* Nothing to do when the reference timestamps were reset
* or no watchdog clocksource registered.
*/
return;
case WD_FREQ_SKEWED:
watchdog_print_freq_skew(cs);
break;
case WD_CPU_TIMEOUT:
/* Remote check timed out. Try again next cycle. */
watchdog_handle_remote_timeout(cs);
return;
case WD_CPU_SKEWED:
watchdog_print_remote_skew(cs);
break;
}
__clocksource_unstable(cs);
}
static void clocksource_watchdog(struct timer_list *unused)
{
int64_t wd_nsec, cs_nsec, interval;
u64 csnow, wdnow, cslast, wdlast;
int next_cpu, reset_pending;
struct clocksource *cs;
enum wd_read_status read_ret;
unsigned long extra_wait = 0;
u32 md;
bool reset_pending;
spin_lock(&watchdog_lock);
guard(spinlock)(&watchdog_lock);
if (!watchdog_running)
goto out;
return;
reset_pending = atomic_read(&watchdog_reset_pending);
list_for_each_entry(cs, &watchdog_list, wd_list) {
/* Clocksource already marked unstable? */
if (cs->flags & CLOCK_SOURCE_UNSTABLE) {
if (finished_booting)
@@ -446,170 +659,40 @@ static void clocksource_watchdog(struct timer_list *unused)
continue;
}
read_ret = cs_watchdog_read(cs, &csnow, &wdnow);
if (read_ret == WD_READ_UNSTABLE) {
/* Clock readout unreliable, so give it up. */
__clocksource_unstable(cs);
continue;
/* Compare against watchdog clocksource if available */
if (watchdog_check_freq(cs, reset_pending)) {
/* Check for inter CPU skew */
watchdog_check_cpu_skew(cs);
}
/*
* When WD_READ_SKIP is returned, it means the system is likely
* under very heavy load, where the latency of reading
* watchdog/clocksource is very big, and affect the accuracy of
* watchdog check. So give system some space and suspend the
* watchdog check for 5 minutes.
*/
if (read_ret == WD_READ_SKIP) {
/*
* As the watchdog timer will be suspended, and
* cs->last could keep unchanged for 5 minutes, reset
* the counters.
*/
clocksource_reset_watchdog();
extra_wait = HZ * 300;
break;
}
/* Clocksource initialized ? */
if (!(cs->flags & CLOCK_SOURCE_WATCHDOG) ||
atomic_read(&watchdog_reset_pending)) {
cs->flags |= CLOCK_SOURCE_WATCHDOG;
cs->wd_last = wdnow;
cs->cs_last = csnow;
continue;
}
wd_nsec = cycles_to_nsec_safe(watchdog, cs->wd_last, wdnow);
cs_nsec = cycles_to_nsec_safe(cs, cs->cs_last, csnow);
wdlast = cs->wd_last; /* save these in case we print them */
cslast = cs->cs_last;
cs->cs_last = csnow;
cs->wd_last = wdnow;
if (atomic_read(&watchdog_reset_pending))
continue;
/*
* The processing of timer softirqs can get delayed (usually
* on account of ksoftirqd not getting to run in a timely
* manner), which causes the watchdog interval to stretch.
* Skew detection may fail for longer watchdog intervals
* on account of fixed margins being used.
* Some clocksources, e.g. acpi_pm, cannot tolerate
* watchdog intervals longer than a few seconds.
*/
interval = max(cs_nsec, wd_nsec);
if (unlikely(interval > WATCHDOG_INTERVAL_MAX_NS)) {
if (system_state > SYSTEM_SCHEDULING &&
interval > 2 * watchdog_max_interval) {
watchdog_max_interval = interval;
pr_warn("Long readout interval, skipping watchdog check: cs_nsec: %lld wd_nsec: %lld\n",
cs_nsec, wd_nsec);
}
watchdog_timer.expires = jiffies;
continue;
}
/* Check the deviation from the watchdog clocksource. */
md = cs->uncertainty_margin + watchdog->uncertainty_margin;
if (abs(cs_nsec - wd_nsec) > md) {
s64 cs_wd_msec;
s64 wd_msec;
u32 wd_rem;
pr_warn("timekeeping watchdog on CPU%d: Marking clocksource '%s' as unstable because the skew is too large:\n",
smp_processor_id(), cs->name);
pr_warn(" '%s' wd_nsec: %lld wd_now: %llx wd_last: %llx mask: %llx\n",
watchdog->name, wd_nsec, wdnow, wdlast, watchdog->mask);
pr_warn(" '%s' cs_nsec: %lld cs_now: %llx cs_last: %llx mask: %llx\n",
cs->name, cs_nsec, csnow, cslast, cs->mask);
cs_wd_msec = div_s64_rem(cs_nsec - wd_nsec, 1000 * 1000, &wd_rem);
wd_msec = div_s64_rem(wd_nsec, 1000 * 1000, &wd_rem);
pr_warn(" Clocksource '%s' skewed %lld ns (%lld ms) over watchdog '%s' interval of %lld ns (%lld ms)\n",
cs->name, cs_nsec - wd_nsec, cs_wd_msec, watchdog->name, wd_nsec, wd_msec);
if (curr_clocksource == cs)
pr_warn(" '%s' is current clocksource.\n", cs->name);
else if (curr_clocksource)
pr_warn(" '%s' (not '%s') is current clocksource.\n", curr_clocksource->name, cs->name);
else
pr_warn(" No current clocksource.\n");
__clocksource_unstable(cs);
continue;
}
if (cs == curr_clocksource && cs->tick_stable)
cs->tick_stable(cs);
if (!(cs->flags & CLOCK_SOURCE_VALID_FOR_HRES) &&
(cs->flags & CLOCK_SOURCE_IS_CONTINUOUS) &&
(watchdog->flags & CLOCK_SOURCE_IS_CONTINUOUS)) {
/* Mark it valid for high-res. */
cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES;
/*
* clocksource_done_booting() will sort it if
* finished_booting is not set yet.
*/
if (!finished_booting)
continue;
/*
* If this is not the current clocksource let
* the watchdog thread reselect it. Due to the
* change to high res this clocksource might
* be preferred now. If it is the current
* clocksource let the tick code know about
* that change.
*/
if (cs != curr_clocksource) {
cs->flags |= CLOCK_SOURCE_RESELECT;
schedule_work(&watchdog_work);
} else {
tick_clock_notify();
}
}
watchdog_check_result(cs);
}
/*
* We only clear the watchdog_reset_pending, when we did a
* full cycle through all clocksources.
*/
/* Clear after the full clocksource walk */
if (reset_pending)
atomic_dec(&watchdog_reset_pending);
/*
* Cycle through CPUs to check if the CPUs stay synchronized
* to each other.
*/
next_cpu = cpumask_next_wrap(raw_smp_processor_id(), cpu_online_mask);
/*
* Arm timer if not already pending: could race with concurrent
* pair clocksource_stop_watchdog() clocksource_start_watchdog().
*/
/* Could have been rearmed by a stop/start cycle */
if (!timer_pending(&watchdog_timer)) {
watchdog_timer.expires += WATCHDOG_INTERVAL + extra_wait;
add_timer_on(&watchdog_timer, next_cpu);
watchdog_timer.expires += WATCHDOG_INTERVAL;
add_timer_local(&watchdog_timer);
}
out:
spin_unlock(&watchdog_lock);
}
static inline void clocksource_start_watchdog(void)
{
if (watchdog_running || !watchdog || list_empty(&watchdog_list))
if (watchdog_running || list_empty(&watchdog_list))
return;
timer_setup(&watchdog_timer, clocksource_watchdog, 0);
timer_setup(&watchdog_timer, clocksource_watchdog, TIMER_PINNED);
watchdog_timer.expires = jiffies + WATCHDOG_INTERVAL;
add_timer_on(&watchdog_timer, cpumask_first(cpu_online_mask));
add_timer_on(&watchdog_timer, get_boot_cpu_id());
watchdog_running = 1;
}
static inline void clocksource_stop_watchdog(void)
{
if (!watchdog_running || (watchdog && !list_empty(&watchdog_list)))
if (!watchdog_running || !list_empty(&watchdog_list))
return;
timer_delete(&watchdog_timer);
watchdog_running = 0;
@@ -651,6 +734,13 @@ static void clocksource_select_watchdog(bool fallback)
if (cs->flags & CLOCK_SOURCE_MUST_VERIFY)
continue;
/*
* If it's not continuous, don't put the fox in charge of
* the henhouse.
*/
if (!(cs->flags & CLOCK_SOURCE_IS_CONTINUOUS))
continue;
/* Skip current if we were requested for a fallback. */
if (fallback && cs == old_wd)
continue;
@@ -690,12 +780,6 @@ static int __clocksource_watchdog_kthread(void)
unsigned long flags;
int select = 0;
/* Do any required per-CPU skew verification. */
if (curr_clocksource &&
curr_clocksource->flags & CLOCK_SOURCE_UNSTABLE &&
curr_clocksource->flags & CLOCK_SOURCE_VERIFY_PERCPU)
clocksource_verify_percpu(curr_clocksource);
spin_lock_irqsave(&watchdog_lock, flags);
list_for_each_entry_safe(cs, tmp, &watchdog_list, wd_list) {
if (cs->flags & CLOCK_SOURCE_UNSTABLE) {
@@ -1016,6 +1100,8 @@ static struct clocksource *clocksource_find_best(bool oneshot, bool skipcur)
continue;
if (oneshot && !(cs->flags & CLOCK_SOURCE_VALID_FOR_HRES))
continue;
if (cs->flags & CLOCK_SOURCE_WDTEST)
continue;
return cs;
}
return NULL;
@@ -1040,6 +1126,8 @@ static void __clocksource_select(bool skipcur)
continue;
if (strcmp(cs->name, override_name) != 0)
continue;
if (cs->flags & CLOCK_SOURCE_WDTEST)
continue;
/*
* Check to make sure we don't switch to a non-highres
* capable clocksource if the tick code is in oneshot
@@ -1169,31 +1257,10 @@ void __clocksource_update_freq_scale(struct clocksource *cs, u32 scale, u32 freq
clocks_calc_mult_shift(&cs->mult, &cs->shift, freq,
NSEC_PER_SEC / scale, sec * scale);
}
/*
* If the uncertainty margin is not specified, calculate it. If
* both scale and freq are non-zero, calculate the clock period, but
* bound below at 2*WATCHDOG_MAX_SKEW, that is, 500ppm by default.
* However, if either of scale or freq is zero, be very conservative
* and take the tens-of-milliseconds WATCHDOG_THRESHOLD value
* for the uncertainty margin. Allow stupidly small uncertainty
* margins to be specified by the caller for testing purposes,
* but warn to discourage production use of this capability.
*
* Bottom line: The sum of the uncertainty margins of the
* watchdog clocksource and the clocksource under test will be at
* least 500ppm by default. For more information, please see the
* comment preceding CONFIG_CLOCKSOURCE_WATCHDOG_MAX_SKEW_US above.
*/
if (scale && freq && !cs->uncertainty_margin) {
cs->uncertainty_margin = NSEC_PER_SEC / (scale * freq);
if (cs->uncertainty_margin < 2 * WATCHDOG_MAX_SKEW)
cs->uncertainty_margin = 2 * WATCHDOG_MAX_SKEW;
} else if (!cs->uncertainty_margin) {
cs->uncertainty_margin = WATCHDOG_THRESHOLD;
/* Update cs::freq_khz */
cs->freq_khz = div_u64((u64)freq * scale, 1000);
}
WARN_ON_ONCE(cs->uncertainty_margin < 2 * WATCHDOG_MAX_SKEW);
/*
* Ensure clocksources that have large 'mult' values don't overflow
@@ -1241,6 +1308,10 @@ int __clocksource_register_scale(struct clocksource *cs, u32 scale, u32 freq)
if (WARN_ON_ONCE((unsigned int)cs->id >= CSID_MAX))
cs->id = CSID_GENERIC;
if (WARN_ON_ONCE(!freq && cs->flags & CLOCK_SOURCE_HAS_COUPLED_CLOCK_EVENT))
cs->flags &= ~CLOCK_SOURCE_HAS_COUPLED_CLOCK_EVENT;
if (cs->vdso_clock_mode < 0 ||
cs->vdso_clock_mode >= VDSO_CLOCKMODE_MAX) {
pr_warn("clocksource %s registered with invalid VDSO mode %d. Disabling VDSO support.\n",
kernel/time/hrtimer.c
+652 -489
View File
File diff suppressed because it is too large. Load diff
kernel/time/jiffies.c
-1
View File
@@ -32,7 +32,6 @@ static u64 jiffies_read(struct clocksource *cs)
static struct clocksource clocksource_jiffies = {
.name = "jiffies",
.rating = 1, /* lowest valid rating*/
.uncertainty_margin = 32 * NSEC_PER_MSEC,
.read = jiffies_read,
.mask = CLOCKSOURCE_MASK(32),
.mult = TICK_NSEC << JIFFIES_SHIFT, /* details above */
kernel/time/posix-timers.c
+1 -1
View File
@@ -1092,7 +1092,7 @@ void exit_itimers(struct task_struct *tsk)
}
/*
* There should be no timers on the ignored list. itimer_delete() has
* There should be no timers on the ignored list. posix_timer_delete() has
* mopped them up.
*/
if (!WARN_ON_ONCE(!hlist_empty(&tsk->signal->ignored_posix_timers)))
kernel/time/tick-broadcast-hrtimer.c
-1
View File
@@ -78,7 +78,6 @@ static struct clock_event_device ce_broadcast_hrtimer = {
.set_state_shutdown = bc_shutdown,
.set_next_ktime = bc_set_next,
.features = CLOCK_EVT_FEAT_ONESHOT |
CLOCK_EVT_FEAT_KTIME |
CLOCK_EVT_FEAT_HRTIMER,
.rating = 0,
.bound_on = -1,
kernel/time/tick-sched.c
+20 -7
View File
@@ -867,19 +867,32 @@ u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
}
EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
/* Simplified variant of hrtimer_forward_now() */
static ktime_t tick_forward_now(ktime_t expires, ktime_t now)
{
ktime_t delta = now - expires;
if (likely(delta < TICK_NSEC))
return expires + TICK_NSEC;
expires += TICK_NSEC * ktime_divns(delta, TICK_NSEC);
if (expires > now)
return expires;
return expires + TICK_NSEC;
}
static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
{
hrtimer_cancel(&ts->sched_timer);
hrtimer_set_expires(&ts->sched_timer, ts->last_tick);
ktime_t expires = ts->last_tick;
/* Forward the time to expire in the future */
hrtimer_forward(&ts->sched_timer, now, TICK_NSEC);
if (now >= expires)
expires = tick_forward_now(expires, now);
if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES)) {
hrtimer_start_expires(&ts->sched_timer,
HRTIMER_MODE_ABS_PINNED_HARD);
hrtimer_start(&ts->sched_timer, expires, HRTIMER_MODE_ABS_PINNED_HARD);
} else {
tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
hrtimer_set_expires(&ts->sched_timer, expires);
tick_program_event(expires, 1);
}
/*
kernel/time/timekeeping.c
+181 -26
View File
@@ -3,34 +3,30 @@
* Kernel timekeeping code and accessor functions. Based on code from
* timer.c, moved in commit 8524070b7982.
*/
#include <linux/timekeeper_internal.h>
#include <linux/module.h>
#include <linux/interrupt.h>
#include <linux/kobject.h>
#include <linux/percpu.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/nmi.h>
#include <linux/sched.h>
#include <linux/sched/loadavg.h>
#include <linux/sched/clock.h>
#include <linux/syscore_ops.h>
#include <linux/audit.h>
#include <linux/clocksource.h>
#include <linux/compiler.h>
#include <linux/jiffies.h>
#include <linux/kobject.h>
#include <linux/module.h>
#include <linux/nmi.h>
#include <linux/pvclock_gtod.h>
#include <linux/random.h>
#include <linux/sched/clock.h>
#include <linux/sched/loadavg.h>
#include <linux/static_key.h>
#include <linux/stop_machine.h>
#include <linux/syscore_ops.h>
#include <linux/tick.h>
#include <linux/time.h>
#include <linux/timex.h>
#include <linux/tick.h>
#include <linux/stop_machine.h>
#include <linux/pvclock_gtod.h>
#include <linux/compiler.h>
#include <linux/audit.h>
#include <linux/random.h>
#include <linux/timekeeper_internal.h>
#include <vdso/auxclock.h>
#include "tick-internal.h"
#include "ntp_internal.h"
#include "timekeeping_internal.h"
#include "ntp_internal.h"
#define TK_CLEAR_NTP (1 << 0)
#define TK_CLOCK_WAS_SET (1 << 1)
@@ -275,6 +271,11 @@ static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta)
tk->monotonic_to_boot = ktime_to_timespec64(tk->offs_boot);
}
#ifdef CONFIG_ARCH_WANTS_CLOCKSOURCE_READ_INLINE
#include <asm/clock_inlined.h>
static DEFINE_STATIC_KEY_FALSE(clocksource_read_inlined);
/*
* tk_clock_read - atomic clocksource read() helper
*
@@ -288,12 +289,35 @@ static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta)
* a read of the fast-timekeeper tkrs (which is protected by its own locking
* and update logic).
*/
static inline u64 tk_clock_read(const struct tk_read_base *tkr)
static __always_inline u64 tk_clock_read(const struct tk_read_base *tkr)
{
struct clocksource *clock = READ_ONCE(tkr->clock);
if (static_branch_likely(&clocksource_read_inlined))
return arch_inlined_clocksource_read(clock);
return clock->read(clock);
}
static inline void clocksource_disable_inline_read(void)
{
static_branch_disable(&clocksource_read_inlined);
}
static inline void clocksource_enable_inline_read(void)
{
static_branch_enable(&clocksource_read_inlined);
}
#else
static __always_inline u64 tk_clock_read(const struct tk_read_base *tkr)
{
struct clocksource *clock = READ_ONCE(tkr->clock);
return clock->read(clock);
}
static inline void clocksource_disable_inline_read(void) { }
static inline void clocksource_enable_inline_read(void) { }
#endif
/**
* tk_setup_internals - Set up internals to use clocksource clock.
@@ -367,6 +391,27 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
tk->tkr_raw.mult = clock->mult;
tk->ntp_err_mult = 0;
tk->skip_second_overflow = 0;
tk->cs_id = clock->id;
/* Coupled clockevent data */
if (IS_ENABLED(CONFIG_GENERIC_CLOCKEVENTS_COUPLED) &&
clock->flags & CLOCK_SOURCE_HAS_COUPLED_CLOCK_EVENT) {
/*
* Aim for an one hour maximum delta and use KHz to handle
* clocksources with a frequency above 4GHz correctly as
* the frequency argument of clocks_calc_mult_shift() is u32.
*/
clocks_calc_mult_shift(&tk->cs_ns_to_cyc_mult, &tk->cs_ns_to_cyc_shift,
NSEC_PER_MSEC, clock->freq_khz, 3600 * 1000);
/*
* Initialize the conversion limit as the previous clocksource
* might have the same shift/mult pair so the quick check in
* tk_update_ns_to_cyc() fails to update it after a clocksource
* change leaving it effectivly zero.
*/
tk->cs_ns_to_cyc_maxns = div_u64(clock->mask, tk->cs_ns_to_cyc_mult);
}
}
/* Timekeeper helper functions. */
@@ -375,7 +420,7 @@ static noinline u64 delta_to_ns_safe(const struct tk_read_base *tkr, u64 delta)
return mul_u64_u32_add_u64_shr(delta, tkr->mult, tkr->xtime_nsec, tkr->shift);
}
static inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 cycles)
static __always_inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 cycles)
{
/* Calculate the delta since the last update_wall_time() */
u64 mask = tkr->mask, delta = (cycles - tkr->cycle_last) & mask;
@@ -696,6 +741,36 @@ static inline void tk_update_ktime_data(struct timekeeper *tk)
tk->tkr_raw.base = ns_to_ktime(tk->raw_sec * NSEC_PER_SEC);
}
static inline void tk_update_ns_to_cyc(struct timekeeper *tks, struct timekeeper *tkc)
{
struct tk_read_base *tkrs = &tks->tkr_mono;
struct tk_read_base *tkrc = &tkc->tkr_mono;
unsigned int shift;
if (!IS_ENABLED(CONFIG_GENERIC_CLOCKEVENTS_COUPLED) ||
!(tkrs->clock->flags & CLOCK_SOURCE_HAS_COUPLED_CLOCK_EVENT))
return;
if (tkrs->mult == tkrc->mult && tkrs->shift == tkrc->shift)
return;
/*
* The conversion math is simple:
*
* CS::MULT (1 << NS_TO_CYC_SHIFT)
* --------------- = ----------------------
* (1 << CS:SHIFT) NS_TO_CYC_MULT
*
* Ergo:
*
* NS_TO_CYC_MULT = (1 << (CS::SHIFT + NS_TO_CYC_SHIFT)) / CS::MULT
*
* NS_TO_CYC_SHIFT has been set up in tk_setup_internals()
*/
shift = tkrs->shift + tks->cs_ns_to_cyc_shift;
tks->cs_ns_to_cyc_mult = (u32)div_u64(1ULL << shift, tkrs->mult);
tks->cs_ns_to_cyc_maxns = div_u64(tkrs->clock->mask, tks->cs_ns_to_cyc_mult);
}
/*
* Restore the shadow timekeeper from the real timekeeper.
*/
@@ -730,6 +805,7 @@ static void timekeeping_update_from_shadow(struct tk_data *tkd, unsigned int act
tk->tkr_mono.base_real = tk->tkr_mono.base + tk->offs_real;
if (tk->id == TIMEKEEPER_CORE) {
tk_update_ns_to_cyc(tk, &tkd->timekeeper);
update_vsyscall(tk);
update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET);
@@ -784,6 +860,71 @@ static void timekeeping_forward_now(struct timekeeper *tk)
tk_update_coarse_nsecs(tk);
}
/*
* ktime_expiry_to_cycles - Convert a expiry time to clocksource cycles
* @id: Clocksource ID which is required for validity
* @expires_ns: Absolute CLOCK_MONOTONIC expiry time (nsecs) to be converted
* @cycles: Pointer to storage for corresponding absolute cycles value
*
* Convert a CLOCK_MONOTONIC based absolute expiry time to a cycles value
* based on the correlated clocksource of the clockevent device by using
* the base nanoseconds and cycles values of the last timekeeper update and
* converting the delta between @expires_ns and base nanoseconds to cycles.
*
* This only works for clockevent devices which are using a less than or
* equal comparator against the clocksource.
*
* Utilizing this avoids two clocksource reads for such devices, the
* ktime_get() in clockevents_program_event() to calculate the delta expiry
* value and the readout in the device::set_next_event() callback to
* convert the delta back to a absolute comparator value.
*
* Returns: True if @id matches the current clocksource ID, false otherwise
*/
bool ktime_expiry_to_cycles(enum clocksource_ids id, ktime_t expires_ns, u64 *cycles)
{
struct timekeeper *tk = &tk_core.timekeeper;
struct tk_read_base *tkrm = &tk->tkr_mono;
ktime_t base_ns, delta_ns, max_ns;
u64 base_cycles, delta_cycles;
unsigned int seq;
u32 mult, shift;
/*
* Racy check to avoid the seqcount overhead when ID does not match. If
* the relevant clocksource is installed concurrently, then this will
* just delay the switch over to this mechanism until the next event is
* programmed. If the ID is not matching the clock events code will use
* the regular relative set_next_event() callback as before.
*/
if (data_race(tk->cs_id) != id)
return false;
do {
seq = read_seqcount_begin(&tk_core.seq);
if (tk->cs_id != id)
return false;
base_cycles = tkrm->cycle_last;
base_ns = tkrm->base + (tkrm->xtime_nsec >> tkrm->shift);
mult = tk->cs_ns_to_cyc_mult;
shift = tk->cs_ns_to_cyc_shift;
max_ns = tk->cs_ns_to_cyc_maxns;
} while (read_seqcount_retry(&tk_core.seq, seq));
/* Prevent negative deltas and multiplication overflows */
delta_ns = min(expires_ns - base_ns, max_ns);
delta_ns = max(delta_ns, 0);
/* Convert to cycles */
delta_cycles = ((u64)delta_ns * mult) >> shift;
*cycles = base_cycles + delta_cycles;
return true;
}
/**
* ktime_get_real_ts64 - Returns the time of day in a timespec64.
* @ts: pointer to the timespec to be set
@@ -848,7 +989,7 @@ u32 ktime_get_resolution_ns(void)
}
EXPORT_SYMBOL_GPL(ktime_get_resolution_ns);
static ktime_t *offsets[TK_OFFS_MAX] = {
static const ktime_t *const offsets[TK_OFFS_MAX] = {
[TK_OFFS_REAL] = &tk_core.timekeeper.offs_real,
[TK_OFFS_BOOT] = &tk_core.timekeeper.offs_boot,
[TK_OFFS_TAI] = &tk_core.timekeeper.offs_tai,
@@ -857,8 +998,9 @@ static ktime_t *offsets[TK_OFFS_MAX] = {
ktime_t ktime_get_with_offset(enum tk_offsets offs)
{
struct timekeeper *tk = &tk_core.timekeeper;
const ktime_t *offset = offsets[offs];
unsigned int seq;
ktime_t base, *offset = offsets[offs];
ktime_t base;
u64 nsecs;
WARN_ON(timekeeping_suspended);
@@ -878,8 +1020,9 @@ EXPORT_SYMBOL_GPL(ktime_get_with_offset);
ktime_t ktime_get_coarse_with_offset(enum tk_offsets offs)
{
struct timekeeper *tk = &tk_core.timekeeper;
ktime_t base, *offset = offsets[offs];
const ktime_t *offset = offsets[offs];
unsigned int seq;
ktime_t base;
u64 nsecs;
WARN_ON(timekeeping_suspended);
@@ -902,7 +1045,7 @@ EXPORT_SYMBOL_GPL(ktime_get_coarse_with_offset);
*/
ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs)
{
ktime_t *offset = offsets[offs];
const ktime_t *offset = offsets[offs];
unsigned int seq;
ktime_t tconv;
@@ -1631,7 +1774,19 @@ int timekeeping_notify(struct clocksource *clock)
if (tk->tkr_mono.clock == clock)
return 0;
/* Disable inlined reads accross the clocksource switch */
clocksource_disable_inline_read();
stop_machine(change_clocksource, clock, NULL);
/*
* If the clocksource has been selected and supports inlined reads
* enable the branch.
*/
if (tk->tkr_mono.clock == clock && clock->flags & CLOCK_SOURCE_CAN_INLINE_READ)
clocksource_enable_inline_read();
tick_clock_notify();
return tk->tkr_mono.clock == clock ? 0 : -1;
}
@@ -2834,7 +2989,7 @@ static void tk_aux_update_clocksource(void)
continue;
timekeeping_forward_now(tks);
tk_setup_internals(tks, tk_core.timekeeper.tkr_mono.clock);
tk_setup_internals(tks, tk_core.timekeeper.tkr_raw.clock);
timekeeping_update_from_shadow(tkd, TK_UPDATE_ALL);
}
}
kernel/time/timekeeping.h
+2
View File
@@ -9,6 +9,8 @@ extern ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq,
ktime_t *offs_boot,
ktime_t *offs_tai);
bool ktime_expiry_to_cycles(enum clocksource_ids id, ktime_t expires_ns, u64 *cycles);
extern int timekeeping_valid_for_hres(void);
extern u64 timekeeping_max_deferment(void);
extern void timekeeping_warp_clock(void);
kernel/time/timer.c
+3 -2
View File
@@ -2322,6 +2322,7 @@ u64 timer_base_try_to_set_idle(unsigned long basej, u64 basem, bool *idle)
*/
void timer_clear_idle(void)
{
int this_cpu = smp_processor_id();
/*
* We do this unlocked. The worst outcome is a remote pinned timer
* enqueue sending a pointless IPI, but taking the lock would just
@@ -2330,9 +2331,9 @@ void timer_clear_idle(void)
* path. Required for BASE_LOCAL only.
*/
__this_cpu_write(timer_bases[BASE_LOCAL].is_idle, false);
if (tick_nohz_full_cpu(smp_processor_id()))
if (tick_nohz_full_cpu(this_cpu))
__this_cpu_write(timer_bases[BASE_GLOBAL].is_idle, false);
trace_timer_base_idle(false, smp_processor_id());
trace_timer_base_idle(false, this_cpu);
/* Activate without holding the timer_base->lock */
tmigr_cpu_activate();
kernel/time/timer_list.c
+7 -9
View File
@@ -47,7 +47,7 @@ print_timer(struct seq_file *m, struct hrtimer *taddr, struct hrtimer *timer,
int idx, u64 now)
{
SEQ_printf(m, " #%d: <%p>, %ps", idx, taddr, ACCESS_PRIVATE(timer, function));
SEQ_printf(m, ", S:%02x", timer->state);
SEQ_printf(m, ", S:%02x", timer->is_queued);
SEQ_printf(m, "\n");
SEQ_printf(m, " # expires at %Lu-%Lu nsecs [in %Ld to %Ld nsecs]\n",
(unsigned long long)ktime_to_ns(hrtimer_get_softexpires(timer)),
@@ -56,13 +56,11 @@ print_timer(struct seq_file *m, struct hrtimer *taddr, struct hrtimer *timer,
(long long)(ktime_to_ns(hrtimer_get_expires(timer)) - now));
}
static void
print_active_timers(struct seq_file *m, struct hrtimer_clock_base *base,
u64 now)
static void print_active_timers(struct seq_file *m, struct hrtimer_clock_base *base, u64 now)
{
struct timerqueue_linked_node *curr;
struct hrtimer *timer, tmp;
unsigned long next = 0, i;
struct timerqueue_node *curr;
unsigned long flags;
next_one:
@@ -72,13 +70,13 @@ next_one:
raw_spin_lock_irqsave(&base->cpu_base->lock, flags);
curr = timerqueue_getnext(&base->active);
curr = timerqueue_linked_first(&base->active);
/*
* Crude but we have to do this O(N*N) thing, because
* we have to unlock the base when printing:
*/
while (curr && i < next) {
curr = timerqueue_iterate_next(curr);
curr = timerqueue_linked_next(curr);
i++;
}
@@ -103,8 +101,8 @@ print_base(struct seq_file *m, struct hrtimer_clock_base *base, u64 now)
SEQ_printf(m, " .resolution: %u nsecs\n", hrtimer_resolution);
#ifdef CONFIG_HIGH_RES_TIMERS
SEQ_printf(m, " .offset: %Lu nsecs\n",
(unsigned long long) ktime_to_ns(base->offset));
SEQ_printf(m, " .offset: %Ld nsecs\n",
(long long) base->offset);
#endif
SEQ_printf(m, "active timers:\n");
print_active_timers(m, base, now + ktime_to_ns(base->offset));
kernel/trace/trace_events_synth.c
+2 -2
View File
@@ -395,7 +395,7 @@ static enum print_line_t print_synth_event(struct trace_iterator *iter,
n_u64++;
} else {
struct trace_print_flags __flags[] = {
__def_gfpflag_names, {-1, NULL} };
__def_gfpflag_names };
char *space = (i == se->n_fields - 1 ? "" : " ");
print_synth_event_num_val(s, print_fmt,
@@ -408,7 +408,7 @@ static enum print_line_t print_synth_event(struct trace_iterator *iter,
trace_seq_puts(s, " (");
trace_print_flags_seq(s, "|",
entry->fields[n_u64].as_u64,
__flags);
__flags, ARRAY_SIZE(__flags));
trace_seq_putc(s, ')');
}
n_u64++;
kernel/trace/trace_output.c
+12 -8
View File
@@ -69,14 +69,15 @@ enum print_line_t trace_print_printk_msg_only(struct trace_iterator *iter)
const char *
trace_print_flags_seq(struct trace_seq *p, const char *delim,
unsigned long flags,
const struct trace_print_flags *flag_array)
const struct trace_print_flags *flag_array,
size_t flag_array_size)
{
unsigned long mask;
const char *str;
const char *ret = trace_seq_buffer_ptr(p);
int i, first = 1;
for (i = 0; flag_array[i].name && flags; i++) {
for (i = 0; i < flag_array_size && flags; i++) {
mask = flag_array[i].mask;
if ((flags & mask) != mask)
@@ -106,12 +107,13 @@ EXPORT_SYMBOL(trace_print_flags_seq);
const char *
trace_print_symbols_seq(struct trace_seq *p, unsigned long val,
const struct trace_print_flags *symbol_array)
const struct trace_print_flags *symbol_array,
size_t symbol_array_size)
{
int i;
const char *ret = trace_seq_buffer_ptr(p);
for (i = 0; symbol_array[i].name; i++) {
for (i = 0; i < symbol_array_size; i++) {
if (val != symbol_array[i].mask)
continue;
@@ -133,14 +135,15 @@ EXPORT_SYMBOL(trace_print_symbols_seq);
const char *
trace_print_flags_seq_u64(struct trace_seq *p, const char *delim,
unsigned long long flags,
const struct trace_print_flags_u64 *flag_array)
const struct trace_print_flags_u64 *flag_array,
size_t flag_array_size)
{
unsigned long long mask;
const char *str;
const char *ret = trace_seq_buffer_ptr(p);
int i, first = 1;
for (i = 0; flag_array[i].name && flags; i++) {
for (i = 0; i < flag_array_size && flags; i++) {
mask = flag_array[i].mask;
if ((flags & mask) != mask)
@@ -170,12 +173,13 @@ EXPORT_SYMBOL(trace_print_flags_seq_u64);
const char *
trace_print_symbols_seq_u64(struct trace_seq *p, unsigned long long val,
const struct trace_print_flags_u64 *symbol_array)
const struct trace_print_flags_u64 *symbol_array,
size_t symbol_array_size)
{
int i;
const char *ret = trace_seq_buffer_ptr(p);
for (i = 0; symbol_array[i].name; i++) {
for (i = 0; i < symbol_array_size; i++) {
if (val != symbol_array[i].mask)
continue;
kernel/trace/trace_syscalls.c
+1 -2
View File
@@ -174,7 +174,6 @@ sys_enter_openat_print(struct syscall_trace_enter *trace, struct syscall_metadat
{ O_NOFOLLOW, "O_NOFOLLOW" },
{ O_NOATIME, "O_NOATIME" },
{ O_CLOEXEC, "O_CLOEXEC" },
{ -1, NULL }
};
trace_seq_printf(s, "%s(", entry->name);
@@ -205,7 +204,7 @@ sys_enter_openat_print(struct syscall_trace_enter *trace, struct syscall_metadat
trace_seq_puts(s, "O_RDONLY|");
}
trace_print_flags_seq(s, "|", bits, __flags);
trace_print_flags_seq(s, "|", bits, __flags, ARRAY_SIZE(__flags));
/*
* trace_print_flags_seq() adds a '\0' to the
* buffer, but this needs to append more to the seq.
lib/rbtree.c
+17
View File
@@ -446,6 +446,23 @@ void rb_erase(struct rb_node *node, struct rb_root *root)
}
EXPORT_SYMBOL(rb_erase);
bool rb_erase_linked(struct rb_node_linked *node, struct rb_root_linked *root)
{
if (node->prev)
node->prev->next = node->next;
else
root->rb_leftmost = node->next;
if (node->next)
node->next->prev = node->prev;
rb_erase(&node->node, &root->rb_root);
RB_CLEAR_LINKED_NODE(node);
return !!root->rb_leftmost;
}
EXPORT_SYMBOL_GPL(rb_erase_linked);
/*
* Augmented rbtree manipulation functions.
*
lib/timerqueue.c
+14
View File
@@ -82,3 +82,17 @@ struct timerqueue_node *timerqueue_iterate_next(struct timerqueue_node *node)
return container_of(next, struct timerqueue_node, node);
}
EXPORT_SYMBOL_GPL(timerqueue_iterate_next);
#define __node_2_tq_linked(_n) \
container_of(rb_entry((_n), struct rb_node_linked, node), struct timerqueue_linked_node, node)
static __always_inline bool __tq_linked_less(struct rb_node *a, const struct rb_node *b)
{
return __node_2_tq_linked(a)->expires < __node_2_tq_linked(b)->expires;
}
bool timerqueue_linked_add(struct timerqueue_linked_head *head, struct timerqueue_linked_node *node)
{
return rb_add_linked(&node->node, &head->rb_root, __tq_linked_less);
}
EXPORT_SYMBOL_GPL(timerqueue_linked_add);
scripts/gdb/linux/timerlist.py
+1 -1
View File
@@ -20,7 +20,7 @@ def ktime_get():
We can't read the hardware timer itself to add any nanoseconds
that need to be added since we last stored the time in the
timekeeper. But this is probably good enough for debug purposes."""
tk_core = gdb.parse_and_eval("&tk_core")
tk_core = gdb.parse_and_eval("&timekeeper_data[TIMEKEEPER_CORE]")
return tk_core['timekeeper']['tkr_mono']['base']