From 0af02a8e356fc6d3b1eebb32fae7d35625127835 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 10 Jun 2024 18:42:06 +0200 Subject: [PATCH 001/180] selftests/timers/posix_timers: Simplify error handling No point in returning to main() on fatal errors. Just exit right away. Signed-off-by: Thomas Gleixner Signed-off-by: Frederic Weisbecker Reviewed-by: Anna-Maria Behnsen --- tools/testing/selftests/timers/posix_timers.c | 151 ++++++------------ 1 file changed, 52 insertions(+), 99 deletions(-) diff --git a/tools/testing/selftests/timers/posix_timers.c b/tools/testing/selftests/timers/posix_timers.c index 07c81c0093c0..38db82cf2a3a 100644 --- a/tools/testing/selftests/timers/posix_timers.c +++ b/tools/testing/selftests/timers/posix_timers.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -19,6 +20,20 @@ #define DELAY 2 #define USECS_PER_SEC 1000000 +static void __fatal_error(const char *test, const char *name, const char *what) +{ + char buf[64]; + + strerror_r(errno, buf, sizeof(buf)); + + if (name && strlen(name)) + ksft_exit_fail_msg("%s %s %s %s\n", test, name, what, buf); + else + ksft_exit_fail_msg("%s %s %s\n", test, what, buf); +} + +#define fatal_error(name, what) __fatal_error(__func__, name, what) + static volatile int done; /* Busy loop in userspace to elapse ITIMER_VIRTUAL */ @@ -74,24 +89,13 @@ static int check_diff(struct timeval start, struct timeval end) return 0; } -static int check_itimer(int which) +static void check_itimer(int which, const char *name) { - const char *name; - int err; struct timeval start, end; struct itimerval val = { .it_value.tv_sec = DELAY, }; - if (which == ITIMER_VIRTUAL) - name = "ITIMER_VIRTUAL"; - else if (which == ITIMER_PROF) - name = "ITIMER_PROF"; - else if (which == ITIMER_REAL) - name = "ITIMER_REAL"; - else - return -1; - done = 0; if (which == ITIMER_VIRTUAL) @@ -101,17 +105,11 @@ static int check_itimer(int which) else if (which == ITIMER_REAL) signal(SIGALRM, sig_handler); - err = gettimeofday(&start, NULL); - if (err < 0) { - ksft_perror("Can't call gettimeofday()"); - return -1; - } + if (gettimeofday(&start, NULL) < 0) + fatal_error(name, "gettimeofday()"); - err = setitimer(which, &val, NULL); - if (err < 0) { - ksft_perror("Can't set timer"); - return -1; - } + if (setitimer(which, &val, NULL) < 0) + fatal_error(name, "setitimer()"); if (which == ITIMER_VIRTUAL) user_loop(); @@ -120,68 +118,41 @@ static int check_itimer(int which) else if (which == ITIMER_REAL) idle_loop(); - err = gettimeofday(&end, NULL); - if (err < 0) { - ksft_perror("Can't call gettimeofday()"); - return -1; - } + if (gettimeofday(&end, NULL) < 0) + fatal_error(name, "gettimeofday()"); ksft_test_result(check_diff(start, end) == 0, "%s\n", name); - - return 0; } -static int check_timer_create(int which) +static void check_timer_create(int which, const char *name) { - const char *type; - int err; - timer_t id; struct timeval start, end; struct itimerspec val = { .it_value.tv_sec = DELAY, }; - - if (which == CLOCK_THREAD_CPUTIME_ID) { - type = "thread"; - } else if (which == CLOCK_PROCESS_CPUTIME_ID) { - type = "process"; - } else { - ksft_print_msg("Unknown timer_create() type %d\n", which); - return -1; - } + timer_t id; done = 0; - err = timer_create(which, NULL, &id); - if (err < 0) { - ksft_perror("Can't create timer"); - return -1; - } - signal(SIGALRM, sig_handler); - err = gettimeofday(&start, NULL); - if (err < 0) { - ksft_perror("Can't call gettimeofday()"); - return -1; - } + if (timer_create(which, NULL, &id) < 0) + fatal_error(name, "timer_create()"); - err = timer_settime(id, 0, &val, NULL); - if (err < 0) { - ksft_perror("Can't set timer"); - return -1; - } + if (signal(SIGALRM, sig_handler) == SIG_ERR) + fatal_error(name, "signal()"); + + if (gettimeofday(&start, NULL) < 0) + fatal_error(name, "gettimeofday()"); + + if (timer_settime(id, 0, &val, NULL) < 0) + fatal_error(name, "timer_settime()"); user_loop(); - err = gettimeofday(&end, NULL); - if (err < 0) { - ksft_perror("Can't call gettimeofday()"); - return -1; - } + if (gettimeofday(&end, NULL) < 0) + fatal_error(name, "gettimeofday()"); ksft_test_result(check_diff(start, end) == 0, - "timer_create() per %s\n", type); - - return 0; + "timer_create() per %s\n", name); } static pthread_t ctd_thread; @@ -209,15 +180,14 @@ static void *ctd_thread_func(void *arg) ctd_count = 100; if (timer_create(CLOCK_PROCESS_CPUTIME_ID, NULL, &id)) - return "Can't create timer\n"; + fatal_error(NULL, "timer_create()"); if (timer_settime(id, 0, &val, NULL)) - return "Can't set timer\n"; - + fatal_error(NULL, "timer_settime()"); while (ctd_count > 0 && !ctd_failed) ; if (timer_delete(id)) - return "Can't delete timer\n"; + fatal_error(NULL, "timer_delete()"); return NULL; } @@ -225,19 +195,16 @@ static void *ctd_thread_func(void *arg) /* * Test that only the running thread receives the timer signal. */ -static int check_timer_distribution(void) +static void check_timer_distribution(void) { - const char *errmsg; + if (signal(SIGALRM, ctd_sighandler) == SIG_ERR) + fatal_error(NULL, "signal()"); - signal(SIGALRM, ctd_sighandler); - - errmsg = "Can't create thread\n"; if (pthread_create(&ctd_thread, NULL, ctd_thread_func, NULL)) - goto err; + fatal_error(NULL, "pthread_create()"); - errmsg = "Can't join thread\n"; - if (pthread_join(ctd_thread, (void **)&errmsg) || errmsg) - goto err; + if (pthread_join(ctd_thread, NULL)) + fatal_error(NULL, "pthread_join()"); if (!ctd_failed) ksft_test_result_pass("check signal distribution\n"); @@ -245,10 +212,6 @@ static int check_timer_distribution(void) ksft_test_result_fail("check signal distribution\n"); else ksft_test_result_skip("check signal distribution (old kernel)\n"); - return 0; -err: - ksft_print_msg("%s", errmsg); - return -1; } int main(int argc, char **argv) @@ -259,17 +222,10 @@ int main(int argc, char **argv) ksft_print_msg("Testing posix timers. False negative may happen on CPU execution \n"); ksft_print_msg("based timers if other threads run on the CPU...\n"); - if (check_itimer(ITIMER_VIRTUAL) < 0) - ksft_exit_fail(); - - if (check_itimer(ITIMER_PROF) < 0) - ksft_exit_fail(); - - if (check_itimer(ITIMER_REAL) < 0) - ksft_exit_fail(); - - if (check_timer_create(CLOCK_THREAD_CPUTIME_ID) < 0) - ksft_exit_fail(); + check_itimer(ITIMER_VIRTUAL, "ITIMER_VIRTUAL"); + check_itimer(ITIMER_PROF, "ITIMER_PROF"); + check_itimer(ITIMER_REAL, "ITIMER_REAL"); + check_timer_create(CLOCK_THREAD_CPUTIME_ID, "CLOCK_THREAD_CPUTIME_ID"); /* * It's unfortunately hard to reliably test a timer expiration @@ -280,11 +236,8 @@ int main(int argc, char **argv) * to ensure true parallelism. So test only one thread until we * find a better solution. */ - if (check_timer_create(CLOCK_PROCESS_CPUTIME_ID) < 0) - ksft_exit_fail(); - - if (check_timer_distribution() < 0) - ksft_exit_fail(); + check_timer_create(CLOCK_PROCESS_CPUTIME_ID, "CLOCK_PROCESS_CPUTIME_ID"); + check_timer_distribution(); ksft_finished(); } From 45c4225c3dcc7db2c0cdbf889cc7a9c72a53f742 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 10 Jun 2024 18:42:07 +0200 Subject: [PATCH 002/180] selftests/timers/posix_timers: Add SIG_IGN test Add a test case to validate correct behaviour vs. SIG_IGN. The posix specification states: "Setting a signal action to SIG_IGN for a signal that is pending shall cause the pending signal to be discarded, whether or not it is blocked." The kernel implements this in the signal handling code, but due to the way how posix timers are handling SIG_IGN for periodic timers, the behaviour after installing a real handler again is inconsistent and suprising. The following sequence is expected to deliver a signal: install_handler(SIG); block_signal(SIG); timer_create(...); <- Should send SIG timer_settime(value=100ms, interval=100ms); sleep(1); <- Timer expires and queues signal, timer is not rearmed as that should happen in the signal delivery path ignore_signal(SIG); <- Discards queued signal install_handler(SIG); <- Restore handler, should rearm but does not sleep(1); unblock_signal(SIG); <- Should deliver one signal with overrun count set in siginfo This fails because nothing rearms the timer when the signal handler is restored. Add a test for this case which fails until the signal and posix timer code is fixed. Signed-off-by: Thomas Gleixner Signed-off-by: Frederic Weisbecker --- tools/testing/selftests/timers/posix_timers.c | 127 +++++++++++++++++- 1 file changed, 125 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/timers/posix_timers.c b/tools/testing/selftests/timers/posix_timers.c index 38db82cf2a3a..8a6139a1d27a 100644 --- a/tools/testing/selftests/timers/posix_timers.c +++ b/tools/testing/selftests/timers/posix_timers.c @@ -6,8 +6,9 @@ * * Kernel loop code stolen from Steven Rostedt */ - +#define _GNU_SOURCE #include +#include #include #include #include @@ -214,10 +215,129 @@ static void check_timer_distribution(void) ksft_test_result_skip("check signal distribution (old kernel)\n"); } +struct tmrsig { + int signals; + int overruns; +}; + +static void siginfo_handler(int sig, siginfo_t *si, void *uc) +{ + struct tmrsig *tsig = si ? si->si_ptr : NULL; + + if (tsig) { + tsig->signals++; + tsig->overruns += si->si_overrun; + } +} + +static void *ignore_thread(void *arg) +{ + unsigned int *tid = arg; + sigset_t set; + + sigemptyset(&set); + sigaddset(&set, SIGUSR1); + if (sigprocmask(SIG_BLOCK, &set, NULL)) + fatal_error(NULL, "sigprocmask(SIG_BLOCK)"); + + *tid = gettid(); + sleep(100); + + if (sigprocmask(SIG_UNBLOCK, &set, NULL)) + fatal_error(NULL, "sigprocmask(SIG_UNBLOCK)"); + return NULL; +} + +static void check_sig_ign(int thread) +{ + struct tmrsig tsig = { }; + struct itimerspec its; + unsigned int tid = 0; + struct sigaction sa; + struct sigevent sev; + pthread_t pthread; + timer_t timerid; + sigset_t set; + + if (thread) { + if (pthread_create(&pthread, NULL, ignore_thread, &tid)) + fatal_error(NULL, "pthread_create()"); + sleep(1); + } + + sa.sa_flags = SA_SIGINFO; + sa.sa_sigaction = siginfo_handler; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGUSR1, &sa, NULL)) + fatal_error(NULL, "sigaction()"); + + /* Block the signal */ + sigemptyset(&set); + sigaddset(&set, SIGUSR1); + if (sigprocmask(SIG_BLOCK, &set, NULL)) + fatal_error(NULL, "sigprocmask(SIG_BLOCK)"); + + memset(&sev, 0, sizeof(sev)); + sev.sigev_notify = SIGEV_SIGNAL; + sev.sigev_signo = SIGUSR1; + sev.sigev_value.sival_ptr = &tsig; + if (thread) { + sev.sigev_notify = SIGEV_THREAD_ID; + sev._sigev_un._tid = tid; + } + + if (timer_create(CLOCK_MONOTONIC, &sev, &timerid)) + fatal_error(NULL, "timer_create()"); + + /* Start the timer to expire in 100ms and 100ms intervals */ + its.it_value.tv_sec = 0; + its.it_value.tv_nsec = 100000000; + its.it_interval.tv_sec = 0; + its.it_interval.tv_nsec = 100000000; + timer_settime(timerid, 0, &its, NULL); + + sleep(1); + + /* Set the signal to be ignored */ + if (signal(SIGUSR1, SIG_IGN) == SIG_ERR) + fatal_error(NULL, "signal(SIG_IGN)"); + + sleep(1); + + if (thread) { + /* Stop the thread first. No signal should be delivered to it */ + if (pthread_cancel(pthread)) + fatal_error(NULL, "pthread_cancel()"); + if (pthread_join(pthread, NULL)) + fatal_error(NULL, "pthread_join()"); + } + + /* Restore the handler */ + if (sigaction(SIGUSR1, &sa, NULL)) + fatal_error(NULL, "sigaction()"); + + sleep(1); + + /* Unblock it, which should deliver the signal in the !thread case*/ + if (sigprocmask(SIG_UNBLOCK, &set, NULL)) + fatal_error(NULL, "sigprocmask(SIG_UNBLOCK)"); + + if (timer_delete(timerid)) + fatal_error(NULL, "timer_delete()"); + + if (!thread) { + ksft_test_result(tsig.signals == 1 && tsig.overruns == 29, + "check_sig_ign SIGEV_SIGNAL\n"); + } else { + ksft_test_result(tsig.signals == 0 && tsig.overruns == 0, + "check_sig_ign SIGEV_THREAD_ID\n"); + } +} + int main(int argc, char **argv) { ksft_print_header(); - ksft_set_plan(6); + ksft_set_plan(8); ksft_print_msg("Testing posix timers. False negative may happen on CPU execution \n"); ksft_print_msg("based timers if other threads run on the CPU...\n"); @@ -239,5 +359,8 @@ int main(int argc, char **argv) check_timer_create(CLOCK_PROCESS_CPUTIME_ID, "CLOCK_PROCESS_CPUTIME_ID"); check_timer_distribution(); + check_sig_ign(0); + check_sig_ign(1); + ksft_finished(); } From e65bb03e442751ccf1631382516dcca5b3a20122 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 10 Jun 2024 18:42:09 +0200 Subject: [PATCH 003/180] selftests/timers/posix_timers: Validate signal rules Add a test case to validate correct behaviour vs. timer reprogramming and deletion. The handling of queued signals in case of timer reprogramming or deletion is inconsistent at best. POSIX does not really specify the behaviour for that: - "The effect of disarming or resetting a timer with pending expiration notifications is unspecified." - "The disposition of pending signals for the deleted timer is unspecified." In both cases it is reasonable to expect that pending signals are discarded. Especially in the reprogramming case it does not make sense to account for previous overruns or to deliver a signal for a timer which has been disarmed. Add tests to validate that no unexpected signals are delivered. They fail for now until the signal and posix timer code is updated. Signed-off-by: Thomas Gleixner Signed-off-by: Frederic Weisbecker --- tools/testing/selftests/timers/posix_timers.c | 108 +++++++++++++++++- 1 file changed, 107 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/timers/posix_timers.c b/tools/testing/selftests/timers/posix_timers.c index 8a6139a1d27a..41b43d1327eb 100644 --- a/tools/testing/selftests/timers/posix_timers.c +++ b/tools/testing/selftests/timers/posix_timers.c @@ -334,10 +334,114 @@ static void check_sig_ign(int thread) } } +static void check_rearm(void) +{ + struct tmrsig tsig = { }; + struct itimerspec its; + struct sigaction sa; + struct sigevent sev; + timer_t timerid; + sigset_t set; + + sa.sa_flags = SA_SIGINFO; + sa.sa_sigaction = siginfo_handler; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGUSR1, &sa, NULL)) + fatal_error(NULL, "sigaction()"); + + /* Block the signal */ + sigemptyset(&set); + sigaddset(&set, SIGUSR1); + if (sigprocmask(SIG_BLOCK, &set, NULL)) + fatal_error(NULL, "sigprocmask(SIG_BLOCK)"); + + memset(&sev, 0, sizeof(sev)); + sev.sigev_notify = SIGEV_SIGNAL; + sev.sigev_signo = SIGUSR1; + sev.sigev_value.sival_ptr = &tsig; + if (timer_create(CLOCK_MONOTONIC, &sev, &timerid)) + fatal_error(NULL, "timer_create()"); + + /* Start the timer to expire in 100ms and 100ms intervals */ + its.it_value.tv_sec = 0; + its.it_value.tv_nsec = 100000000; + its.it_interval.tv_sec = 0; + its.it_interval.tv_nsec = 100000000; + if (timer_settime(timerid, 0, &its, NULL)) + fatal_error(NULL, "timer_settime()"); + + sleep(1); + + /* Reprogram the timer to single shot */ + its.it_value.tv_sec = 10; + its.it_value.tv_nsec = 0; + its.it_interval.tv_sec = 0; + its.it_interval.tv_nsec = 0; + if (timer_settime(timerid, 0, &its, NULL)) + fatal_error(NULL, "timer_settime()"); + + /* Unblock it, which should not deliver a signal */ + if (sigprocmask(SIG_UNBLOCK, &set, NULL)) + fatal_error(NULL, "sigprocmask(SIG_UNBLOCK)"); + + if (timer_delete(timerid)) + fatal_error(NULL, "timer_delete()"); + + ksft_test_result(!tsig.signals, "check_rearm\n"); +} + +static void check_delete(void) +{ + struct tmrsig tsig = { }; + struct itimerspec its; + struct sigaction sa; + struct sigevent sev; + timer_t timerid; + sigset_t set; + + sa.sa_flags = SA_SIGINFO; + sa.sa_sigaction = siginfo_handler; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGUSR1, &sa, NULL)) + fatal_error(NULL, "sigaction()"); + + /* Block the signal */ + sigemptyset(&set); + sigaddset(&set, SIGUSR1); + if (sigprocmask(SIG_BLOCK, &set, NULL)) + fatal_error(NULL, "sigprocmask(SIG_BLOCK)"); + + memset(&sev, 0, sizeof(sev)); + sev.sigev_notify = SIGEV_SIGNAL; + sev.sigev_signo = SIGUSR1; + sev.sigev_value.sival_ptr = &tsig; + if (timer_create(CLOCK_MONOTONIC, &sev, &timerid)) + fatal_error(NULL, "timer_create()"); + + /* Start the timer to expire in 100ms and 100ms intervals */ + its.it_value.tv_sec = 0; + its.it_value.tv_nsec = 100000000; + its.it_interval.tv_sec = 0; + its.it_interval.tv_nsec = 100000000; + if (timer_settime(timerid, 0, &its, NULL)) + fatal_error(NULL, "timer_settime()"); + + sleep(1); + + if (timer_delete(timerid)) + fatal_error(NULL, "timer_delete()"); + + /* Unblock it, which should not deliver a signal */ + if (sigprocmask(SIG_UNBLOCK, &set, NULL)) + fatal_error(NULL, "sigprocmask(SIG_UNBLOCK)"); + + ksft_test_result(!tsig.signals, "check_delete\n"); +} + int main(int argc, char **argv) { ksft_print_header(); - ksft_set_plan(8); + ksft_set_plan(10); ksft_print_msg("Testing posix timers. False negative may happen on CPU execution \n"); ksft_print_msg("based timers if other threads run on the CPU...\n"); @@ -361,6 +465,8 @@ int main(int argc, char **argv) check_sig_ign(0); check_sig_ign(1); + check_rearm(); + check_delete(); ksft_finished(); } From 2c2b56132bb74b6b801dcb82f57489ae1cf81a91 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 10 Jun 2024 18:42:10 +0200 Subject: [PATCH 004/180] selftests/timers/posix-timers: Validate SIGEV_NONE Posix timers with a delivery mode of SIGEV_NONE deliver no signals but the remaining expiry time must be readable via timer_gettime() for both one shot and interval timers. That's implemented correctly for regular posix timers but broken for posix CPU timers. Add a self test so the fixes can be verified. Signed-off-by: Thomas Gleixner Signed-off-by: Frederic Weisbecker --- tools/testing/selftests/timers/posix_timers.c | 53 ++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/timers/posix_timers.c b/tools/testing/selftests/timers/posix_timers.c index 41b43d1327eb..097a132615f6 100644 --- a/tools/testing/selftests/timers/posix_timers.c +++ b/tools/testing/selftests/timers/posix_timers.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -20,6 +21,7 @@ #define DELAY 2 #define USECS_PER_SEC 1000000 +#define NSECS_PER_SEC 1000000000 static void __fatal_error(const char *test, const char *name, const char *what) { @@ -438,10 +440,57 @@ static void check_delete(void) ksft_test_result(!tsig.signals, "check_delete\n"); } +static inline int64_t calcdiff_ns(struct timespec t1, struct timespec t2) +{ + int64_t diff; + + diff = NSECS_PER_SEC * (int64_t)((int) t1.tv_sec - (int) t2.tv_sec); + diff += ((int) t1.tv_nsec - (int) t2.tv_nsec); + return diff; +} + +static void check_sigev_none(int which, const char *name) +{ + struct timespec start, now; + struct itimerspec its; + struct sigevent sev; + timer_t timerid; + + memset(&sev, 0, sizeof(sev)); + sev.sigev_notify = SIGEV_NONE; + + if (timer_create(which, &sev, &timerid)) + fatal_error(name, "timer_create()"); + + /* Start the timer to expire in 100ms and 100ms intervals */ + its.it_value.tv_sec = 0; + its.it_value.tv_nsec = 100000000; + its.it_interval.tv_sec = 0; + its.it_interval.tv_nsec = 100000000; + timer_settime(timerid, 0, &its, NULL); + + if (clock_gettime(which, &start)) + fatal_error(name, "clock_gettime()"); + + do { + if (clock_gettime(which, &now)) + fatal_error(name, "clock_gettime()"); + } while (calcdiff_ns(now, start) < NSECS_PER_SEC); + + if (timer_gettime(timerid, &its)) + fatal_error(name, "timer_gettime()"); + + if (timer_delete(timerid)) + fatal_error(name, "timer_delete()"); + + ksft_test_result(its.it_value.tv_sec || its.it_value.tv_nsec, + "check_sigev_none %s\n", name); +} + int main(int argc, char **argv) { ksft_print_header(); - ksft_set_plan(10); + ksft_set_plan(12); ksft_print_msg("Testing posix timers. False negative may happen on CPU execution \n"); ksft_print_msg("based timers if other threads run on the CPU...\n"); @@ -467,6 +516,8 @@ int main(int argc, char **argv) check_sig_ign(1); check_rearm(); check_delete(); + check_sigev_none(CLOCK_MONOTONIC, "CLOCK_MONOTONIC"); + check_sigev_none(CLOCK_PROCESS_CPUTIME_ID, "CLOCK_PROCESS_CPUTIME_ID"); ksft_finished(); } From f924f868ed05d954d79db419e97ba11293110d52 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 10 Jun 2024 18:42:11 +0200 Subject: [PATCH 005/180] selftests/timers/posix-timers: Validate timer_gettime() timer_gettime() must return the correct expiry time for interval timers even when the timer is not armed, which is the case when a signal is pending but blocked. Works correctly for regular posix timers, but not for posix CPU timers. Add a selftest to validate the fixes. Signed-off-by: Thomas Gleixner Signed-off-by: Frederic Weisbecker --- tools/testing/selftests/timers/posix_timers.c | 58 ++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/timers/posix_timers.c b/tools/testing/selftests/timers/posix_timers.c index 097a132615f6..4c993dbbdbf2 100644 --- a/tools/testing/selftests/timers/posix_timers.c +++ b/tools/testing/selftests/timers/posix_timers.c @@ -487,10 +487,63 @@ static void check_sigev_none(int which, const char *name) "check_sigev_none %s\n", name); } +static void check_gettime(int which, const char *name) +{ + struct itimerspec its, prev; + struct timespec start, now; + struct sigevent sev; + timer_t timerid; + int wraps = 0; + sigset_t set; + + /* Block the signal */ + sigemptyset(&set); + sigaddset(&set, SIGUSR1); + if (sigprocmask(SIG_BLOCK, &set, NULL)) + fatal_error(name, "sigprocmask(SIG_BLOCK)"); + + memset(&sev, 0, sizeof(sev)); + sev.sigev_notify = SIGEV_SIGNAL; + sev.sigev_signo = SIGUSR1; + + if (timer_create(which, &sev, &timerid)) + fatal_error(name, "timer_create()"); + + /* Start the timer to expire in 100ms and 100ms intervals */ + its.it_value.tv_sec = 0; + its.it_value.tv_nsec = 100000000; + its.it_interval.tv_sec = 0; + its.it_interval.tv_nsec = 100000000; + if (timer_settime(timerid, 0, &its, NULL)) + fatal_error(name, "timer_settime()"); + + if (timer_gettime(timerid, &prev)) + fatal_error(name, "timer_gettime()"); + + if (clock_gettime(which, &start)) + fatal_error(name, "clock_gettime()"); + + do { + if (clock_gettime(which, &now)) + fatal_error(name, "clock_gettime()"); + if (timer_gettime(timerid, &its)) + fatal_error(name, "timer_gettime()"); + if (its.it_value.tv_nsec > prev.it_value.tv_nsec) + wraps++; + prev = its; + + } while (calcdiff_ns(now, start) < NSECS_PER_SEC); + + if (timer_delete(timerid)) + fatal_error(name, "timer_delete()"); + + ksft_test_result(wraps > 1, "check_gettime %s\n", name); +} + int main(int argc, char **argv) { ksft_print_header(); - ksft_set_plan(12); + ksft_set_plan(15); ksft_print_msg("Testing posix timers. False negative may happen on CPU execution \n"); ksft_print_msg("based timers if other threads run on the CPU...\n"); @@ -518,6 +571,9 @@ int main(int argc, char **argv) check_delete(); check_sigev_none(CLOCK_MONOTONIC, "CLOCK_MONOTONIC"); check_sigev_none(CLOCK_PROCESS_CPUTIME_ID, "CLOCK_PROCESS_CPUTIME_ID"); + check_gettime(CLOCK_MONOTONIC, "CLOCK_MONOTONIC"); + check_gettime(CLOCK_PROCESS_CPUTIME_ID, "CLOCK_PROCESS_CPUTIME_ID"); + check_gettime(CLOCK_THREAD_CPUTIME_ID, "CLOCK_THREAD_CPUTIME_ID"); ksft_finished(); } From 73339b82f86521eeadbb781d8d7e04813cbd0998 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 10 Jun 2024 18:42:12 +0200 Subject: [PATCH 006/180] selftests/timers/posix-timers: Validate overrun after unblock When a timer signal is blocked and later unblocked then one signal should be delivered with the correct number of overruns since the timer was queued. Validate that behaviour. Signed-off-by: Thomas Gleixner Signed-off-by: Frederic Weisbecker --- tools/testing/selftests/timers/posix_timers.c | 61 ++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/timers/posix_timers.c b/tools/testing/selftests/timers/posix_timers.c index 4c993dbbdbf2..16bd49492efa 100644 --- a/tools/testing/selftests/timers/posix_timers.c +++ b/tools/testing/selftests/timers/posix_timers.c @@ -540,10 +540,66 @@ static void check_gettime(int which, const char *name) ksft_test_result(wraps > 1, "check_gettime %s\n", name); } +static void check_overrun(int which, const char *name) +{ + struct timespec start, now; + struct tmrsig tsig = { }; + struct itimerspec its; + struct sigaction sa; + struct sigevent sev; + timer_t timerid; + sigset_t set; + + sa.sa_flags = SA_SIGINFO; + sa.sa_sigaction = siginfo_handler; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGUSR1, &sa, NULL)) + fatal_error(name, "sigaction()"); + + /* Block the signal */ + sigemptyset(&set); + sigaddset(&set, SIGUSR1); + if (sigprocmask(SIG_BLOCK, &set, NULL)) + fatal_error(name, "sigprocmask(SIG_BLOCK)"); + + memset(&sev, 0, sizeof(sev)); + sev.sigev_notify = SIGEV_SIGNAL; + sev.sigev_signo = SIGUSR1; + sev.sigev_value.sival_ptr = &tsig; + if (timer_create(which, &sev, &timerid)) + fatal_error(name, "timer_create()"); + + /* Start the timer to expire in 100ms and 100ms intervals */ + its.it_value.tv_sec = 0; + its.it_value.tv_nsec = 100000000; + its.it_interval.tv_sec = 0; + its.it_interval.tv_nsec = 100000000; + if (timer_settime(timerid, 0, &its, NULL)) + fatal_error(name, "timer_settime()"); + + if (clock_gettime(which, &start)) + fatal_error(name, "clock_gettime()"); + + do { + if (clock_gettime(which, &now)) + fatal_error(name, "clock_gettime()"); + } while (calcdiff_ns(now, start) < NSECS_PER_SEC); + + /* Unblock it, which should deliver a signal */ + if (sigprocmask(SIG_UNBLOCK, &set, NULL)) + fatal_error(name, "sigprocmask(SIG_UNBLOCK)"); + + if (timer_delete(timerid)) + fatal_error(name, "timer_delete()"); + + ksft_test_result(tsig.signals == 1 && tsig.overruns == 9, + "check_overrun %s\n", name); +} + int main(int argc, char **argv) { ksft_print_header(); - ksft_set_plan(15); + ksft_set_plan(18); ksft_print_msg("Testing posix timers. False negative may happen on CPU execution \n"); ksft_print_msg("based timers if other threads run on the CPU...\n"); @@ -574,6 +630,9 @@ int main(int argc, char **argv) check_gettime(CLOCK_MONOTONIC, "CLOCK_MONOTONIC"); check_gettime(CLOCK_PROCESS_CPUTIME_ID, "CLOCK_PROCESS_CPUTIME_ID"); check_gettime(CLOCK_THREAD_CPUTIME_ID, "CLOCK_THREAD_CPUTIME_ID"); + check_overrun(CLOCK_MONOTONIC, "CLOCK_MONOTONIC"); + check_overrun(CLOCK_PROCESS_CPUTIME_ID, "CLOCK_PROCESS_CPUTIME_ID"); + check_overrun(CLOCK_THREAD_CPUTIME_ID, "CLOCK_THREAD_CPUTIME_ID"); ksft_finished(); } From d859704bf18519739c231403d53461e008eea4bf Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 10 Jun 2024 18:42:14 +0200 Subject: [PATCH 007/180] posix-cpu-timers: Split up posix_cpu_timer_get() In preparation for addressing issues in the timer_get() and timer_set() functions of posix CPU timers. No functional change. Signed-off-by: Thomas Gleixner Signed-off-by: Frederic Weisbecker Reviewed-by: Anna-Maria Behnsen Acked-by: Peter Zijlstra (Intel) --- kernel/time/posix-cpu-timers.c | 51 ++++++++++++++++------------------ 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index e9c6f9d0e42c..558be8d53e47 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -785,33 +785,9 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, return ret; } -static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp) +static void __posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp, u64 now) { - clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock); - struct cpu_timer *ctmr = &timer->it.cpu; - u64 now, expires = cpu_timer_getexpires(ctmr); - struct task_struct *p; - - rcu_read_lock(); - p = cpu_timer_task_rcu(timer); - if (!p) - goto out; - - /* - * Easy part: convert the reload time. - */ - itp->it_interval = ktime_to_timespec64(timer->it_interval); - - if (!expires) - goto out; - - /* - * Sample the clock to take the difference with the expiry time. - */ - if (CPUCLOCK_PERTHREAD(timer->it_clock)) - now = cpu_clock_sample(clkid, p); - else - now = cpu_clock_sample_group(clkid, p, false); + u64 expires = cpu_timer_getexpires(&timer->it.cpu); if (now < expires) { itp->it_value = ns_to_timespec64(expires - now); @@ -823,7 +799,28 @@ static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp itp->it_value.tv_nsec = 1; itp->it_value.tv_sec = 0; } -out: +} + +static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp) +{ + clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock); + struct task_struct *p; + u64 now; + + rcu_read_lock(); + p = cpu_timer_task_rcu(timer); + if (p) { + itp->it_interval = ktime_to_timespec64(timer->it_interval); + + if (cpu_timer_getexpires(&timer->it.cpu)) { + if (CPUCLOCK_PERTHREAD(timer->it_clock)) + now = cpu_clock_sample(clkid, p); + else + now = cpu_clock_sample_group(clkid, p, false); + + __posix_cpu_timer_get(timer, itp, now); + } + } rcu_read_unlock(); } From b3e866b2dffbc36b31be7811ebded91ce82ecd10 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 10 Jun 2024 18:42:15 +0200 Subject: [PATCH 008/180] posix-cpu-timers: Save interval only for armed timers There is no point to return the interval for timers which have been disarmed. Signed-off-by: Thomas Gleixner Signed-off-by: Frederic Weisbecker Acked-by: Peter Zijlstra (Intel) --- kernel/time/posix-cpu-timers.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 558be8d53e47..5aac0886bbc7 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -809,17 +809,15 @@ static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp rcu_read_lock(); p = cpu_timer_task_rcu(timer); - if (p) { + if (p && cpu_timer_getexpires(&timer->it.cpu)) { itp->it_interval = ktime_to_timespec64(timer->it_interval); - if (cpu_timer_getexpires(&timer->it.cpu)) { - if (CPUCLOCK_PERTHREAD(timer->it_clock)) - now = cpu_clock_sample(clkid, p); - else - now = cpu_clock_sample_group(clkid, p, false); + if (CPUCLOCK_PERTHREAD(timer->it_clock)) + now = cpu_clock_sample(clkid, p); + else + now = cpu_clock_sample_group(clkid, p, false); - __posix_cpu_timer_get(timer, itp, now); - } + __posix_cpu_timer_get(timer, itp, now); } rcu_read_unlock(); } From 1c5028425793ea98fcb403852331664662d97226 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 10 Jun 2024 18:42:16 +0200 Subject: [PATCH 009/180] posix-cpu-timers: Handle interval timers correctly in timer_get() timer_gettime() must return the remaining time to the next expiry of a timer or 0 if the timer is not armed and no signal pending, but posix CPU timers fail to forward a timer which is already expired. Add the required logic to address that. Signed-off-by: Thomas Gleixner Signed-off-by: Frederic Weisbecker Acked-by: Peter Zijlstra (Intel) --- kernel/time/posix-cpu-timers.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 5aac0886bbc7..92222ca89f49 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -787,8 +787,24 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, static void __posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp, u64 now) { - u64 expires = cpu_timer_getexpires(&timer->it.cpu); + u64 expires, iv = timer->it_interval; + /* + * Make sure that interval timers are moved forward for the + * following cases: + * - Timers which expired, but the signal has not yet been + * delivered + */ + if (iv && (timer->it_requeue_pending & REQUEUE_PENDING)) + expires = bump_cpu_timer(timer, now); + else + expires = cpu_timer_getexpires(&timer->it.cpu); + + /* + * Expired interval timers cannot have a remaining time <= 0. + * The kernel has to move them forward so that the next + * timer expiry is > @now. + */ if (now < expires) { itp->it_value = ns_to_timespec64(expires - now); } else { From d786b8ba9f01b361817033d06766b2dadf619c60 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 10 Jun 2024 18:42:17 +0200 Subject: [PATCH 010/180] posix-cpu-timers: Handle SIGEV_NONE timers correctly in timer_get() Expired SIGEV_NONE oneshot timers must return 0 nsec for the expiry time in timer_get(), but the posix CPU timer implementation returns 1 nsec. Add the missing conditional. Signed-off-by: Thomas Gleixner Signed-off-by: Frederic Weisbecker Acked-by: Peter Zijlstra (Intel) --- kernel/time/posix-cpu-timers.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 92222ca89f49..d70879d502f9 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -787,15 +787,17 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, static void __posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp, u64 now) { + bool sigev_none = timer->it_sigev_notify == SIGEV_NONE; u64 expires, iv = timer->it_interval; /* * Make sure that interval timers are moved forward for the * following cases: + * - SIGEV_NONE timers which are never armed * - Timers which expired, but the signal has not yet been * delivered */ - if (iv && (timer->it_requeue_pending & REQUEUE_PENDING)) + if (iv && ((timer->it_requeue_pending & REQUEUE_PENDING) || sigev_none)) expires = bump_cpu_timer(timer, now); else expires = cpu_timer_getexpires(&timer->it.cpu); @@ -809,11 +811,13 @@ static void __posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *i itp->it_value = ns_to_timespec64(expires - now); } else { /* - * The timer should have expired already, but the firing - * hasn't taken place yet. Say it's just about to expire. + * A single shot SIGEV_NONE timer must return 0, when it is + * expired! Timers which have a real signal delivery mode + * must return a remaining time greater than 0 because the + * signal has not yet been delivered. */ - itp->it_value.tv_nsec = 1; - itp->it_value.tv_sec = 0; + if (!sigev_none) + itp->it_value.tv_nsec = 1; } } From 5f9d4a1065949a64c107f93a89dc2b62f806c833 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sun, 23 Jun 2024 13:16:02 +0200 Subject: [PATCH 011/180] posix-cpu-timers: Handle SIGEV_NONE timers correctly in timer_set() Expired SIGEV_NONE oneshot timers must return 0 nsec for the expiry time in timer_get(), but the posix CPU timer implementation returns 1 nsec. Add the missing conditional. This will be cleaned up in a follow up patch. Signed-off-by: Thomas Gleixner Signed-off-by: Frederic Weisbecker Acked-by: Peter Zijlstra (Intel) --- kernel/time/posix-cpu-timers.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index d70879d502f9..cf8904423cba 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -623,6 +623,7 @@ static void cpu_timer_fire(struct k_itimer *timer) static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, struct itimerspec64 *new, struct itimerspec64 *old) { + bool sigev_none = timer->it_sigev_notify == SIGEV_NONE; clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock); u64 old_expires, new_expires, old_incr, val; struct cpu_timer *ctmr = &timer->it.cpu; @@ -706,7 +707,16 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, old_expires = exp - val; old->it_value = ns_to_timespec64(old_expires); } else { - old->it_value.tv_nsec = 1; + /* + * A single shot SIGEV_NONE timer must return 0, when it is + * expired! Timers which have a real signal delivery mode + * must return a remaining time greater than 0 because the + * signal has not yet been delivered. + */ + if (sigev_none) + old->it_value.tv_nsec = 0; + else + old->it_value.tv_nsec = 1; old->it_value.tv_sec = 0; } } From d471ff397c3571cfa648a20e7018aa04f8873cb3 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sun, 23 Jun 2024 13:17:08 +0200 Subject: [PATCH 012/180] posix-cpu-timers: Replace old expiry retrieval in posix_cpu_timer_set() Reuse the split out __posix_cpu_timer_get() function which does already the right thing. No functional change. Signed-off-by: Thomas Gleixner Signed-off-by: Frederic Weisbecker Reviewed-by: Anna-Maria Behnsen Acked-by: Peter Zijlstra (Intel) --- kernel/time/posix-cpu-timers.c | 37 ++++++---------------------------- 1 file changed, 6 insertions(+), 31 deletions(-) diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index cf8904423cba..040d5c36184e 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -614,6 +614,8 @@ static void cpu_timer_fire(struct k_itimer *timer) } } +static void __posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp, u64 now); + /* * Guts of sys_timer_settime for CPU timers. * This is called with the timer locked and interrupts disabled. @@ -623,7 +625,6 @@ static void cpu_timer_fire(struct k_itimer *timer) static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, struct itimerspec64 *new, struct itimerspec64 *old) { - bool sigev_none = timer->it_sigev_notify == SIGEV_NONE; clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock); u64 old_expires, new_expires, old_incr, val; struct cpu_timer *ctmr = &timer->it.cpu; @@ -689,37 +690,11 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, else val = cpu_clock_sample_group(clkid, p, true); + /* Retrieve the previous expiry value if requested. */ if (old) { - if (old_expires == 0) { - old->it_value.tv_sec = 0; - old->it_value.tv_nsec = 0; - } else { - /* - * Update the timer in case it has overrun already. - * If it has, we'll report it as having overrun and - * with the next reloaded timer already ticking, - * though we are swallowing that pending - * notification here to install the new setting. - */ - u64 exp = bump_cpu_timer(timer, val); - - if (val < exp) { - old_expires = exp - val; - old->it_value = ns_to_timespec64(old_expires); - } else { - /* - * A single shot SIGEV_NONE timer must return 0, when it is - * expired! Timers which have a real signal delivery mode - * must return a remaining time greater than 0 because the - * signal has not yet been delivered. - */ - if (sigev_none) - old->it_value.tv_nsec = 0; - else - old->it_value.tv_nsec = 1; - old->it_value.tv_sec = 0; - } - } + old->it_value = (struct timespec64){ }; + if (old_expires) + __posix_cpu_timer_get(timer, old, val); } if (unlikely(ret)) { From bd29d773ea8d2c7fc36dc3f70d4c9d1cf404aa4b Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 10 Jun 2024 18:42:21 +0200 Subject: [PATCH 013/180] posix-cpu-timers: Do not arm SIGEV_NONE timers There is no point in arming SIGEV_NONE timers as they never deliver a signal. timer_gettime() is handling the expiry time correctly and that's all SIGEV_NONE timers care about. Prevent arming them and remove the expiry handler code which just disarms them. Signed-off-by: Thomas Gleixner Signed-off-by: Frederic Weisbecker Reviewed-by: Anna-Maria Behnsen Acked-by: Peter Zijlstra (Intel) --- kernel/time/posix-cpu-timers.c | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 040d5c36184e..62e6b907c21d 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -584,12 +584,7 @@ static void cpu_timer_fire(struct k_itimer *timer) { struct cpu_timer *ctmr = &timer->it.cpu; - if ((timer->it_sigev_notify & ~SIGEV_THREAD_ID) == SIGEV_NONE) { - /* - * User don't want any signal. - */ - cpu_timer_setexpires(ctmr, 0); - } else if (unlikely(timer->sigq == NULL)) { + if (unlikely(timer->sigq == NULL)) { /* * This a special case for clock_nanosleep, * not a normal timer from sys_timer_create. @@ -625,6 +620,7 @@ static void __posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *i static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, struct itimerspec64 *new, struct itimerspec64 *old) { + bool sigev_none = timer->it_sigev_notify == SIGEV_NONE; clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock); u64 old_expires, new_expires, old_incr, val; struct cpu_timer *ctmr = &timer->it.cpu; @@ -688,7 +684,7 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, if (CPUCLOCK_PERTHREAD(timer->it_clock)) val = cpu_clock_sample(clkid, p); else - val = cpu_clock_sample_group(clkid, p, true); + val = cpu_clock_sample_group(clkid, p, !sigev_none); /* Retrieve the previous expiry value if requested. */ if (old) { @@ -708,19 +704,20 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, goto out; } - if (new_expires != 0 && !(timer_flags & TIMER_ABSTIME)) { + /* Convert relative expiry time to absolute */ + if (new_expires && !(timer_flags & TIMER_ABSTIME)) new_expires += val; - } + + /* Set the new expiry time (might be 0) */ + cpu_timer_setexpires(ctmr, new_expires); /* - * Install the new expiry time (or zero). - * For a timer with no notification action, we don't actually - * arm the timer (we'll just fake it for timer_gettime). + * Arm the timer if it is not disabled, the new expiry value has + * not yet expired and the timer requires signal delivery. + * SIGEV_NONE timers are never armed. */ - cpu_timer_setexpires(ctmr, new_expires); - if (new_expires != 0 && val < new_expires) { + if (!sigev_none && new_expires && val < new_expires) arm_timer(timer, p); - } unlock_task_sighand(p, &flags); /* @@ -739,7 +736,7 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, timer->it_overrun_last = 0; timer->it_overrun = -1; - if (val >= new_expires) { + if (!sigev_none && val >= new_expires) { if (new_expires != 0) { /* * The designated time already passed, so we notify From c44462661e4c5967c5df451393af727d1886c8ea Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 10 Jun 2024 18:42:22 +0200 Subject: [PATCH 014/180] posix-cpu-timers: Use @now instead of @val for clarity posix_cpu_timer_set() uses @val as variable for the current time. That's confusing at best. Use @now as anywhere else and rewrite the confusing comment about clock sampling. No functional change. Signed-off-by: Thomas Gleixner Signed-off-by: Frederic Weisbecker Acked-by: Peter Zijlstra (Intel) --- kernel/time/posix-cpu-timers.c | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 62e6b907c21d..b1a9a2ffddc9 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -622,7 +622,7 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, { bool sigev_none = timer->it_sigev_notify == SIGEV_NONE; clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock); - u64 old_expires, new_expires, old_incr, val; + u64 old_expires, new_expires, old_incr, now; struct cpu_timer *ctmr = &timer->it.cpu; struct sighand_struct *sighand; struct task_struct *p; @@ -674,23 +674,19 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, } /* - * We need to sample the current value to convert the new - * value from to relative and absolute, and to convert the - * old value from absolute to relative. To set a process - * timer, we need a sample to balance the thread expiry - * times (in arm_timer). With an absolute time, we must - * check if it's already passed. In short, we need a sample. + * Sample the current clock for saving the previous setting + * and for rearming the timer. */ if (CPUCLOCK_PERTHREAD(timer->it_clock)) - val = cpu_clock_sample(clkid, p); + now = cpu_clock_sample(clkid, p); else - val = cpu_clock_sample_group(clkid, p, !sigev_none); + now = cpu_clock_sample_group(clkid, p, !sigev_none); /* Retrieve the previous expiry value if requested. */ if (old) { old->it_value = (struct timespec64){ }; if (old_expires) - __posix_cpu_timer_get(timer, old, val); + __posix_cpu_timer_get(timer, old, now); } if (unlikely(ret)) { @@ -706,7 +702,7 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, /* Convert relative expiry time to absolute */ if (new_expires && !(timer_flags & TIMER_ABSTIME)) - new_expires += val; + new_expires += now; /* Set the new expiry time (might be 0) */ cpu_timer_setexpires(ctmr, new_expires); @@ -716,7 +712,7 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, * not yet expired and the timer requires signal delivery. * SIGEV_NONE timers are never armed. */ - if (!sigev_none && new_expires && val < new_expires) + if (!sigev_none && new_expires && now < new_expires) arm_timer(timer, p); unlock_task_sighand(p, &flags); @@ -736,7 +732,7 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, timer->it_overrun_last = 0; timer->it_overrun = -1; - if (!sigev_none && val >= new_expires) { + if (!sigev_none && now >= new_expires) { if (new_expires != 0) { /* * The designated time already passed, so we notify From 286bfaccea76e0bd3805ac6e77c8ec4a18ecb3fe Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 10 Jun 2024 18:42:23 +0200 Subject: [PATCH 015/180] posix-cpu-timers: Remove incorrect comment in posix_cpu_timer_set() A leftover from historical code which describes fiction. Signed-off-by: Thomas Gleixner Signed-off-by: Frederic Weisbecker Acked-by: Peter Zijlstra (Intel) --- kernel/time/posix-cpu-timers.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index b1a9a2ffddc9..eb28150c8d63 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -689,13 +689,8 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, __posix_cpu_timer_get(timer, old, now); } + /* Retry if the timer expiry is running concurrently */ if (unlikely(ret)) { - /* - * We are colliding with the timer actually firing. - * Punt after filling in the timer's old value, and - * disable this firing since we are already reporting - * it as an overrun (thanks to bump_cpu_timer above). - */ unlock_task_sighand(p, &flags); goto out; } From c20b99e3243f9e72b6fa0e260766adcba115f25b Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 10 Jun 2024 18:42:25 +0200 Subject: [PATCH 016/180] posix-cpu-timers: Simplify posix_cpu_timer_set() Avoid the late sighand lock/unlock dance when a timer is not armed to enforce reevaluation of the timer base so that the process wide CPU timer sampling can be disabled. Do it right at the point where the arming decision is made which already has sighand locked. Signed-off-by: Thomas Gleixner Signed-off-by: Frederic Weisbecker Reviewed-by: Anna-Maria Behnsen Acked-by: Peter Zijlstra (Intel) --- kernel/time/posix-cpu-timers.c | 44 +++++++++++++--------------------- 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index eb28150c8d63..c6fe017193b8 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -705,10 +705,16 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, /* * Arm the timer if it is not disabled, the new expiry value has * not yet expired and the timer requires signal delivery. - * SIGEV_NONE timers are never armed. + * SIGEV_NONE timers are never armed. In case the timer is not + * armed, enforce the reevaluation of the timer base so that the + * process wide cputime counter can be disabled eventually. */ - if (!sigev_none && new_expires && now < new_expires) - arm_timer(timer, p); + if (likely(!sigev_none)) { + if (new_expires && now < new_expires) + arm_timer(timer, p); + else + trigger_base_recalc_expires(timer, p); + } unlock_task_sighand(p, &flags); /* @@ -727,30 +733,14 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, timer->it_overrun_last = 0; timer->it_overrun = -1; - if (!sigev_none && now >= new_expires) { - if (new_expires != 0) { - /* - * The designated time already passed, so we notify - * immediately, even if the thread never runs to - * accumulate more time on this clock. - */ - cpu_timer_fire(timer); - } - - /* - * Make sure we don't keep around the process wide cputime - * counter or the tick dependency if they are not necessary. - */ - sighand = lock_task_sighand(p, &flags); - if (!sighand) - goto out; - - if (!cpu_timer_queued(ctmr)) - trigger_base_recalc_expires(timer, p); - - unlock_task_sighand(p, &flags); - } - out: + /* + * If the new expiry time was already in the past the timer was not + * queued. Fire it immediately even if the thread never runs to + * accumulate more time on this clock. + */ + if (!sigev_none && new_expires && now >= new_expires) + cpu_timer_fire(timer); +out: rcu_read_unlock(); if (old) old->it_interval = ns_to_timespec64(old_incr); From bfa408f03fc74bcfe8f275a434294bde06eabb00 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 10 Jun 2024 18:42:26 +0200 Subject: [PATCH 017/180] posix-timers: Retrieve interval in common timer_settime() code No point in doing this all over the place. Signed-off-by: Thomas Gleixner Signed-off-by: Frederic Weisbecker Reviewed-by: Anna-Maria Behnsen Acked-by: Peter Zijlstra (Intel) --- kernel/time/posix-cpu-timers.c | 10 ++-------- kernel/time/posix-timers.c | 5 ++++- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index c6fe017193b8..410797706ccf 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -622,8 +622,8 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, { bool sigev_none = timer->it_sigev_notify == SIGEV_NONE; clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock); - u64 old_expires, new_expires, old_incr, now; struct cpu_timer *ctmr = &timer->it.cpu; + u64 old_expires, new_expires, now; struct sighand_struct *sighand; struct task_struct *p; unsigned long flags; @@ -660,10 +660,7 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, return -ESRCH; } - /* - * Disarm any old timer after extracting its expiry time. - */ - old_incr = timer->it_interval; + /* Retrieve the current expiry time before disarming the timer */ old_expires = cpu_timer_getexpires(ctmr); if (unlikely(timer->it.cpu.firing)) { @@ -742,9 +739,6 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, cpu_timer_fire(timer); out: rcu_read_unlock(); - if (old) - old->it_interval = ns_to_timespec64(old_incr); - return ret; } diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c index b924f0f096fa..056966b9d7da 100644 --- a/kernel/time/posix-timers.c +++ b/kernel/time/posix-timers.c @@ -904,7 +904,7 @@ static int do_timer_settime(timer_t timer_id, int tmr_flags, const struct k_clock *kc; struct k_itimer *timr; unsigned long flags; - int error = 0; + int error; if (!timespec64_valid(&new_spec64->it_interval) || !timespec64_valid(&new_spec64->it_value)) @@ -918,6 +918,9 @@ retry: if (!timr) return -EINVAL; + if (old_spec64) + old_spec64->it_interval = ktime_to_timespec64(timr->it_interval); + kc = timr->kclock; if (WARN_ON_ONCE(!kc || !kc->timer_set)) error = -EINVAL; From aca1dc0ce128a9b12640c39c0e035266bf9c9fa5 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 10 Jun 2024 18:42:27 +0200 Subject: [PATCH 018/180] posix-timers: Clear overrun in common_timer_set() Keeping the overrun count of the previous setup around is just wrong. The new setting has nothing to do with the previous one and has to start from a clean slate. Signed-off-by: Thomas Gleixner Signed-off-by: Frederic Weisbecker Acked-by: Peter Zijlstra (Intel) --- kernel/time/posix-timers.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c index 056966b9d7da..53a993e4dbfa 100644 --- a/kernel/time/posix-timers.c +++ b/kernel/time/posix-timers.c @@ -881,6 +881,7 @@ int common_timer_set(struct k_itimer *timr, int flags, timr->it_requeue_pending = (timr->it_requeue_pending + 2) & ~REQUEUE_PENDING; timr->it_overrun_last = 0; + timr->it_overrun = -1LL; /* Switch off the timer when it_value is zero */ if (!new_setting->it_value.tv_sec && !new_setting->it_value.tv_nsec) From 52dea0a15cc888e89f0144dca7b712fb56d12a28 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 10 Jun 2024 18:42:28 +0200 Subject: [PATCH 019/180] posix-timers: Convert timer list to hlist No requirement for a real list. Spare a few bytes. Signed-off-by: Thomas Gleixner Signed-off-by: Frederic Weisbecker Acked-by: Peter Zijlstra (Intel) --- fs/proc/base.c | 6 +++--- include/linux/posix-timers.h | 2 +- include/linux/sched/signal.h | 2 +- init/init_task.c | 2 +- kernel/fork.c | 2 +- kernel/time/posix-timers.c | 19 ++++++++----------- 6 files changed, 15 insertions(+), 18 deletions(-) diff --git a/fs/proc/base.c b/fs/proc/base.c index 72a1acd03675..dd579332a7f8 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -2456,13 +2456,13 @@ static void *timers_start(struct seq_file *m, loff_t *pos) if (!tp->sighand) return ERR_PTR(-ESRCH); - return seq_list_start(&tp->task->signal->posix_timers, *pos); + return seq_hlist_start(&tp->task->signal->posix_timers, *pos); } static void *timers_next(struct seq_file *m, void *v, loff_t *pos) { struct timers_private *tp = m->private; - return seq_list_next(v, &tp->task->signal->posix_timers, pos); + return seq_hlist_next(v, &tp->task->signal->posix_timers, pos); } static void timers_stop(struct seq_file *m, void *v) @@ -2491,7 +2491,7 @@ static int show_timer(struct seq_file *m, void *v) [SIGEV_THREAD] = "thread", }; - timer = list_entry((struct list_head *)v, struct k_itimer, list); + timer = hlist_entry((struct hlist_node *)v, struct k_itimer, list); notify = timer->it_sigev_notify; seq_printf(m, "ID: %d\n", timer->it_id); diff --git a/include/linux/posix-timers.h b/include/linux/posix-timers.h index dc7b738de299..453691710839 100644 --- a/include/linux/posix-timers.h +++ b/include/linux/posix-timers.h @@ -158,7 +158,7 @@ static inline void posix_cputimers_init_work(void) { } * @rcu: RCU head for freeing the timer. */ struct k_itimer { - struct list_head list; + struct hlist_node list; struct hlist_node t_hash; spinlock_t it_lock; const struct k_clock *kclock; diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index 0a0e23c45406..622fdef78e14 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -137,7 +137,7 @@ struct signal_struct { /* POSIX.1b Interval Timers */ unsigned int next_posix_timer_id; - struct list_head posix_timers; + struct hlist_head posix_timers; /* ITIMER_REAL timer for the process */ struct hrtimer real_timer; diff --git a/init/init_task.c b/init/init_task.c index eeb110c65fe2..5d0399bc8d2f 100644 --- a/init/init_task.c +++ b/init/init_task.c @@ -29,7 +29,7 @@ static struct signal_struct init_signals = { .cred_guard_mutex = __MUTEX_INITIALIZER(init_signals.cred_guard_mutex), .exec_update_lock = __RWSEM_INITIALIZER(init_signals.exec_update_lock), #ifdef CONFIG_POSIX_TIMERS - .posix_timers = LIST_HEAD_INIT(init_signals.posix_timers), + .posix_timers = HLIST_HEAD_INIT, .cputimer = { .cputime_atomic = INIT_CPUTIME_ATOMIC, }, diff --git a/kernel/fork.c b/kernel/fork.c index cc760491f201..c1b343cba560 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1861,7 +1861,7 @@ static int copy_signal(unsigned long clone_flags, struct task_struct *tsk) prev_cputime_init(&sig->prev_cputime); #ifdef CONFIG_POSIX_TIMERS - INIT_LIST_HEAD(&sig->posix_timers); + INIT_HLIST_HEAD(&sig->posix_timers); hrtimer_init(&sig->real_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); sig->real_timer.function = it_real_fn; #endif diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c index 53a993e4dbfa..fa75e9493fd6 100644 --- a/kernel/time/posix-timers.c +++ b/kernel/time/posix-timers.c @@ -515,7 +515,7 @@ static int do_timer_create(clockid_t which_clock, struct sigevent *event, spin_lock_irq(¤t->sighand->siglock); /* This makes the timer valid in the hash table */ WRITE_ONCE(new_timer->it_signal, current->signal); - list_add(&new_timer->list, ¤t->signal->posix_timers); + hlist_add_head(&new_timer->list, ¤t->signal->posix_timers); spin_unlock_irq(¤t->sighand->siglock); /* * After unlocking sighand::siglock @new_timer is subject to @@ -1025,7 +1025,7 @@ retry_delete: } spin_lock(¤t->sighand->siglock); - list_del(&timer->list); + hlist_del(&timer->list); spin_unlock(¤t->sighand->siglock); /* * A concurrent lookup could check timer::it_signal lockless. It @@ -1075,7 +1075,7 @@ retry_delete: goto retry_delete; } - list_del(&timer->list); + hlist_del(&timer->list); /* * Setting timer::it_signal to NULL is technically not required @@ -1096,22 +1096,19 @@ retry_delete: */ void exit_itimers(struct task_struct *tsk) { - struct list_head timers; - struct k_itimer *tmr; + struct hlist_head timers; - if (list_empty(&tsk->signal->posix_timers)) + if (hlist_empty(&tsk->signal->posix_timers)) return; /* Protect against concurrent read via /proc/$PID/timers */ spin_lock_irq(&tsk->sighand->siglock); - list_replace_init(&tsk->signal->posix_timers, &timers); + hlist_move_list(&tsk->signal->posix_timers, &timers); spin_unlock_irq(&tsk->sighand->siglock); /* The timers are not longer accessible via tsk::signal */ - while (!list_empty(&timers)) { - tmr = list_first_entry(&timers, struct k_itimer, list); - itimer_delete(tmr); - } + while (!hlist_empty(&timers)) + itimer_delete(hlist_entry(timers.first, struct k_itimer, list)); } SYSCALL_DEFINE2(clock_settime, const clockid_t, which_clock, From 20f13385b5836d00d64698748565fc6d3ce9b419 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 10 Jun 2024 18:42:30 +0200 Subject: [PATCH 020/180] posix-timers: Consolidate timer setup hrtimer based and CPU timers have their own way to install the new interval and to reset overrun and signal handling related data. Create a helper function and do the same operation for all variants. This also makes the handling of the interval consistent. It's only stored when the timer is actually armed, i.e. timer->it_value != 0. Before that it was stored unconditionally for posix CPU timers and conditionally for the other posix timers. Signed-off-by: Thomas Gleixner Signed-off-by: Frederic Weisbecker Acked-by: Peter Zijlstra (Intel) --- kernel/time/posix-cpu-timers.c | 15 +-------------- kernel/time/posix-timers.c | 25 +++++++++++++++++++------ kernel/time/posix-timers.h | 1 + 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 410797706ccf..43cf3f63f31b 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -714,21 +714,8 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, } unlock_task_sighand(p, &flags); - /* - * Install the new reload setting, and - * set up the signal and overrun bookkeeping. - */ - timer->it_interval = timespec64_to_ktime(new->it_interval); - /* - * This acts as a modification timestamp for the timer, - * so any automatic reload attempt will punt on seeing - * that we have reset the timer manually. - */ - timer->it_requeue_pending = (timer->it_requeue_pending + 2) & - ~REQUEUE_PENDING; - timer->it_overrun_last = 0; - timer->it_overrun = -1; + posix_timer_set_common(timer, new); /* * If the new expiry time was already in the past the timer was not diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c index fa75e9493fd6..679be902be7c 100644 --- a/kernel/time/posix-timers.c +++ b/kernel/time/posix-timers.c @@ -856,6 +856,23 @@ static struct k_itimer *timer_wait_running(struct k_itimer *timer, return lock_timer(timer_id, flags); } +/* + * Set up the new interval and reset the signal delivery data + */ +void posix_timer_set_common(struct k_itimer *timer, struct itimerspec64 *new_setting) +{ + if (new_setting->it_value.tv_sec || new_setting->it_value.tv_nsec) + timer->it_interval = timespec64_to_ktime(new_setting->it_interval); + else + timer->it_interval = 0; + + /* Prevent reloading in case there is a signal pending */ + timer->it_requeue_pending = (timer->it_requeue_pending + 2) & ~REQUEUE_PENDING; + /* Reset overrun accounting */ + timer->it_overrun_last = 0; + timer->it_overrun = -1LL; +} + /* Set a POSIX.1b interval timer. */ int common_timer_set(struct k_itimer *timr, int flags, struct itimerspec64 *new_setting, @@ -878,16 +895,12 @@ int common_timer_set(struct k_itimer *timr, int flags, return TIMER_RETRY; timr->it_active = 0; - timr->it_requeue_pending = (timr->it_requeue_pending + 2) & - ~REQUEUE_PENDING; - timr->it_overrun_last = 0; - timr->it_overrun = -1LL; + posix_timer_set_common(timr, new_setting); - /* Switch off the timer when it_value is zero */ + /* Keep timer disarmed when it_value is zero */ if (!new_setting->it_value.tv_sec && !new_setting->it_value.tv_nsec) return 0; - timr->it_interval = timespec64_to_ktime(new_setting->it_interval); expires = timespec64_to_ktime(new_setting->it_value); if (flags & TIMER_ABSTIME) expires = timens_ktime_to_host(timr->it_clock, expires); diff --git a/kernel/time/posix-timers.h b/kernel/time/posix-timers.h index f32a2ebba9b8..630a77b2ec6a 100644 --- a/kernel/time/posix-timers.h +++ b/kernel/time/posix-timers.h @@ -42,4 +42,5 @@ void common_timer_get(struct k_itimer *timr, struct itimerspec64 *cur_setting); int common_timer_set(struct k_itimer *timr, int flags, struct itimerspec64 *new_setting, struct itimerspec64 *old_setting); +void posix_timer_set_common(struct k_itimer *timer, struct itimerspec64 *new_setting); int common_timer_del(struct k_itimer *timer); From 24aea4cc483240ead3fdf581045a636dc7ea1352 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 10 Jun 2024 18:42:31 +0200 Subject: [PATCH 021/180] posix-cpu-timers: Make k_itimer::it_active consistent Posix CPU timers are not updating k_itimer::it_active which makes it impossible to base decisions in the common posix timer code on it. Update it when queueing or dequeueing posix CPU timers. Signed-off-by: Thomas Gleixner Signed-off-by: Frederic Weisbecker Reviewed-by: Anna-Maria Behnsen Acked-by: Peter Zijlstra (Intel) --- kernel/time/posix-cpu-timers.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 43cf3f63f31b..bcc2b83e6666 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -453,6 +453,7 @@ static void disarm_timer(struct k_itimer *timer, struct task_struct *p) struct cpu_timer *ctmr = &timer->it.cpu; struct posix_cputimer_base *base; + timer->it_active = 0; if (!cpu_timer_dequeue(ctmr)) return; @@ -559,6 +560,7 @@ static void arm_timer(struct k_itimer *timer, struct task_struct *p) struct cpu_timer *ctmr = &timer->it.cpu; u64 newexp = cpu_timer_getexpires(ctmr); + timer->it_active = 1; if (!cpu_timer_enqueue(&base->tqhead, ctmr)) return; @@ -584,6 +586,7 @@ static void cpu_timer_fire(struct k_itimer *timer) { struct cpu_timer *ctmr = &timer->it.cpu; + timer->it_active = 0; if (unlikely(timer->sigq == NULL)) { /* * This a special case for clock_nanosleep, @@ -668,6 +671,7 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, ret = TIMER_RETRY; } else { cpu_timer_dequeue(ctmr); + timer->it_active = 0; } /* From 566e2d82536cf77b5ed7c03f887e7c36bf86feaa Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 10 Jun 2024 18:42:32 +0200 Subject: [PATCH 022/180] posix-timers: Consolidate signal queueing Rename posix_timer_event() to posix_timer_queue_signal() as this is what the function is about. Consolidate the requeue pending and deactivation updates into that function as there is no point in doing this in all incarnations of posix timers. Signed-off-by: Thomas Gleixner Signed-off-by: Frederic Weisbecker Acked-by: Peter Zijlstra (Intel) --- kernel/time/alarmtimer.c | 7 +------ kernel/time/posix-cpu-timers.c | 4 ++-- kernel/time/posix-timers.c | 21 +++++++++++---------- kernel/time/posix-timers.h | 2 +- 4 files changed, 15 insertions(+), 19 deletions(-) diff --git a/kernel/time/alarmtimer.c b/kernel/time/alarmtimer.c index 5abfa4390673..76bd4fda3472 100644 --- a/kernel/time/alarmtimer.c +++ b/kernel/time/alarmtimer.c @@ -574,15 +574,10 @@ static enum alarmtimer_restart alarm_handle_timer(struct alarm *alarm, it.alarm.alarmtimer); enum alarmtimer_restart result = ALARMTIMER_NORESTART; unsigned long flags; - int si_private = 0; spin_lock_irqsave(&ptr->it_lock, flags); - ptr->it_active = 0; - if (ptr->it_interval) - si_private = ++ptr->it_requeue_pending; - - if (posix_timer_event(ptr, si_private) && ptr->it_interval) { + if (posix_timer_queue_signal(ptr) && ptr->it_interval) { /* * Handle ignored signals and rearm the timer. This will go * away once we handle ignored signals proper. Ensure that diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index bcc2b83e6666..6bcee4704059 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -598,9 +598,9 @@ static void cpu_timer_fire(struct k_itimer *timer) /* * One-shot timer. Clear it as soon as it's fired. */ - posix_timer_event(timer, 0); + posix_timer_queue_signal(timer); cpu_timer_setexpires(ctmr, 0); - } else if (posix_timer_event(timer, ++timer->it_requeue_pending)) { + } else if (posix_timer_queue_signal(timer)) { /* * The signal did not get queued because the signal * was ignored, so we won't get any callback to diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c index 679be902be7c..1cc830ef93a7 100644 --- a/kernel/time/posix-timers.c +++ b/kernel/time/posix-timers.c @@ -277,10 +277,17 @@ void posixtimer_rearm(struct kernel_siginfo *info) unlock_timer(timr, flags); } -int posix_timer_event(struct k_itimer *timr, int si_private) +int posix_timer_queue_signal(struct k_itimer *timr) { + int ret, si_private = 0; enum pid_type type; - int ret; + + lockdep_assert_held(&timr->it_lock); + + timr->it_active = 0; + if (timr->it_interval) + si_private = ++timr->it_requeue_pending; + /* * FIXME: if ->sigq is queued we can race with * dequeue_signal()->posixtimer_rearm(). @@ -309,19 +316,13 @@ int posix_timer_event(struct k_itimer *timr, int si_private) */ static enum hrtimer_restart posix_timer_fn(struct hrtimer *timer) { + struct k_itimer *timr = container_of(timer, struct k_itimer, it.real.timer); enum hrtimer_restart ret = HRTIMER_NORESTART; - struct k_itimer *timr; unsigned long flags; - int si_private = 0; - timr = container_of(timer, struct k_itimer, it.real.timer); spin_lock_irqsave(&timr->it_lock, flags); - timr->it_active = 0; - if (timr->it_interval != 0) - si_private = ++timr->it_requeue_pending; - - if (posix_timer_event(timr, si_private)) { + if (posix_timer_queue_signal(timr)) { /* * The signal was not queued due to SIG_IGN. As a * consequence the timer is not going to be rearmed from diff --git a/kernel/time/posix-timers.h b/kernel/time/posix-timers.h index 630a77b2ec6a..4784ea65f685 100644 --- a/kernel/time/posix-timers.h +++ b/kernel/time/posix-timers.h @@ -36,7 +36,7 @@ extern const struct k_clock clock_process; extern const struct k_clock clock_thread; extern const struct k_clock alarm_clock; -int posix_timer_event(struct k_itimer *timr, int si_private); +int posix_timer_queue_signal(struct k_itimer *timr); void common_timer_get(struct k_itimer *timr, struct itimerspec64 *cur_setting); int common_timer_set(struct k_itimer *timr, int flags, From a2b80ce87a87fc18c594e74d13031d5e347b69cb Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 10 Jun 2024 18:42:33 +0200 Subject: [PATCH 023/180] signal: Remove task argument from dequeue_signal() The task pointer which is handed to dequeue_signal() is always current. The argument along with the first comment about signalfd in that function is confusing at best. Remove it and use current internally. Update the stale comment for dequeue_signal() while at it. Signed-off-by: Thomas Gleixner Signed-off-by: Frederic Weisbecker Reviewed-by: Oleg Nesterov Acked-by: Peter Zijlstra (Intel) --- fs/signalfd.c | 4 ++-- include/linux/sched/signal.h | 5 ++--- kernel/signal.c | 23 ++++++++++------------- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/fs/signalfd.c b/fs/signalfd.c index ec7b2da2477a..d0333bce015e 100644 --- a/fs/signalfd.c +++ b/fs/signalfd.c @@ -159,7 +159,7 @@ static ssize_t signalfd_dequeue(struct signalfd_ctx *ctx, kernel_siginfo_t *info DECLARE_WAITQUEUE(wait, current); spin_lock_irq(¤t->sighand->siglock); - ret = dequeue_signal(current, &ctx->sigmask, info, &type); + ret = dequeue_signal(&ctx->sigmask, info, &type); switch (ret) { case 0: if (!nonblock) @@ -174,7 +174,7 @@ static ssize_t signalfd_dequeue(struct signalfd_ctx *ctx, kernel_siginfo_t *info add_wait_queue(¤t->sighand->signalfd_wqh, &wait); for (;;) { set_current_state(TASK_INTERRUPTIBLE); - ret = dequeue_signal(current, &ctx->sigmask, info, &type); + ret = dequeue_signal(&ctx->sigmask, info, &type); if (ret != 0) break; if (signal_pending(current)) { diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index 622fdef78e14..c8ed09ac29ac 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -276,8 +276,7 @@ static inline void signal_set_stop_flags(struct signal_struct *sig, extern void flush_signals(struct task_struct *); extern void ignore_signals(struct task_struct *); extern void flush_signal_handlers(struct task_struct *, int force_default); -extern int dequeue_signal(struct task_struct *task, sigset_t *mask, - kernel_siginfo_t *info, enum pid_type *type); +extern int dequeue_signal(sigset_t *mask, kernel_siginfo_t *info, enum pid_type *type); static inline int kernel_dequeue_signal(void) { @@ -287,7 +286,7 @@ static inline int kernel_dequeue_signal(void) int ret; spin_lock_irq(&task->sighand->siglock); - ret = dequeue_signal(task, &task->blocked, &__info, &__type); + ret = dequeue_signal(&task->blocked, &__info, &__type); spin_unlock_irq(&task->sighand->siglock); return ret; diff --git a/kernel/signal.c b/kernel/signal.c index 60c737e423a1..897765b254f9 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -618,20 +618,18 @@ static int __dequeue_signal(struct sigpending *pending, sigset_t *mask, } /* - * Dequeue a signal and return the element to the caller, which is - * expected to free it. - * - * All callers have to hold the siglock. + * Try to dequeue a signal. If a deliverable signal is found fill in the + * caller provided siginfo and return the signal number. Otherwise return + * 0. */ -int dequeue_signal(struct task_struct *tsk, sigset_t *mask, - kernel_siginfo_t *info, enum pid_type *type) +int dequeue_signal(sigset_t *mask, kernel_siginfo_t *info, enum pid_type *type) { + struct task_struct *tsk = current; bool resched_timer = false; int signr; - /* We only dequeue private signals from ourselves, we don't let - * signalfd steal them - */ + lockdep_assert_held(&tsk->sighand->siglock); + *type = PIDTYPE_PID; signr = __dequeue_signal(&tsk->pending, mask, info, &resched_timer); if (!signr) { @@ -2793,8 +2791,7 @@ relock: type = PIDTYPE_PID; signr = dequeue_synchronous_signal(&ksig->info); if (!signr) - signr = dequeue_signal(current, ¤t->blocked, - &ksig->info, &type); + signr = dequeue_signal(¤t->blocked, &ksig->info, &type); if (!signr) break; /* will return 0 */ @@ -3648,7 +3645,7 @@ static int do_sigtimedwait(const sigset_t *which, kernel_siginfo_t *info, signotset(&mask); spin_lock_irq(&tsk->sighand->siglock); - sig = dequeue_signal(tsk, &mask, info, &type); + sig = dequeue_signal(&mask, info, &type); if (!sig && timeout) { /* * None ready, temporarily unblock those we're interested @@ -3667,7 +3664,7 @@ static int do_sigtimedwait(const sigset_t *which, kernel_siginfo_t *info, spin_lock_irq(&tsk->sighand->siglock); __set_task_blocked(tsk, &tsk->real_blocked); sigemptyset(&tsk->real_blocked); - sig = dequeue_signal(tsk, &mask, info, &type); + sig = dequeue_signal(&mask, info, &type); } spin_unlock_irq(&tsk->sighand->siglock); From 7f8af7bac5380f2d95a63a6f19964e22437166e1 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 10 Jun 2024 18:42:34 +0200 Subject: [PATCH 024/180] signal: Replace BUG_ON()s These really can be handled gracefully without killing the machine. Signed-off-by: Thomas Gleixner Signed-off-by: Frederic Weisbecker Reviewed-by: Oleg Nesterov Acked-by: Peter Zijlstra (Intel) --- kernel/signal.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/kernel/signal.c b/kernel/signal.c index 897765b254f9..6f3a5aa39b09 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -1938,10 +1938,11 @@ struct sigqueue *sigqueue_alloc(void) void sigqueue_free(struct sigqueue *q) { - unsigned long flags; spinlock_t *lock = ¤t->sighand->siglock; + unsigned long flags; - BUG_ON(!(q->flags & SIGQUEUE_PREALLOC)); + if (WARN_ON_ONCE(!(q->flags & SIGQUEUE_PREALLOC))) + return; /* * We must hold ->siglock while testing q->list * to serialize with collect_signal() or with @@ -1969,7 +1970,10 @@ int send_sigqueue(struct sigqueue *q, struct pid *pid, enum pid_type type) unsigned long flags; int ret, result; - BUG_ON(!(q->flags & SIGQUEUE_PREALLOC)); + if (WARN_ON_ONCE(!(q->flags & SIGQUEUE_PREALLOC))) + return 0; + if (WARN_ON_ONCE(q->info.si_code != SI_TIMER)) + return 0; ret = -1; rcu_read_lock(); @@ -2004,7 +2008,6 @@ int send_sigqueue(struct sigqueue *q, struct pid *pid, enum pid_type type) * If an SI_TIMER entry is already queue just increment * the overrun count. */ - BUG_ON(q->info.si_code != SI_TIMER); q->info.si_overrun++; result = TRACE_SIGNAL_ALREADY_PENDING; goto out; From 5e389e9868878c8aeb3ed60789eb62242506c9f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Mon, 8 Jul 2024 17:17:52 +0200 Subject: [PATCH 025/180] irqchip/armada-370-xp: Drop _OFFS suffix from some register constants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some register constants have the _OFFS suffix and some do not. Drop it to be more consistent. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Reviewed-by: Ilpo Järvinen Link: https://lore.kernel.org/all/20240708151801.11592-2-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 105 +++++++++++++--------------- 1 file changed, 48 insertions(+), 57 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index dce2b80bf439..66d6a2ebc8a5 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -66,15 +66,14 @@ * device * * The "global interrupt mask/unmask" is modified using the - * ARMADA_370_XP_INT_SET_ENABLE_OFFS and - * ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS registers, which are relative - * to "main_int_base". + * ARMADA_370_XP_INT_SET_ENABLE and ARMADA_370_XP_INT_CLEAR_ENABLE + * registers, which are relative to "main_int_base". * * The "per-CPU mask/unmask" is modified using the - * ARMADA_370_XP_INT_SET_MASK_OFFS and - * ARMADA_370_XP_INT_CLEAR_MASK_OFFS registers, which are relative to - * "per_cpu_int_base". This base address points to a special address, - * which automatically accesses the registers of the current CPU. + * ARMADA_370_XP_INT_SET_MASK and ARMADA_370_XP_INT_CLEAR_MASK + * registers, which are relative to "per_cpu_int_base". This base + * address points to a special address, which automatically accesses + * the registers of the current CPU. * * The per-CPU mask/unmask can also be adjusted using the global * per-interrupt ARMADA_370_XP_INT_SOURCE_CTL register, which we use @@ -118,21 +117,21 @@ /* Registers relative to main_int_base */ #define ARMADA_370_XP_INT_CONTROL (0x00) -#define ARMADA_370_XP_SW_TRIG_INT_OFFS (0x04) -#define ARMADA_370_XP_INT_SET_ENABLE_OFFS (0x30) -#define ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS (0x34) +#define ARMADA_370_XP_SW_TRIG_INT (0x04) +#define ARMADA_370_XP_INT_SET_ENABLE (0x30) +#define ARMADA_370_XP_INT_CLEAR_ENABLE (0x34) #define ARMADA_370_XP_INT_SOURCE_CTL(irq) (0x100 + irq*4) #define ARMADA_370_XP_INT_SOURCE_CPU_MASK 0xF #define ARMADA_370_XP_INT_IRQ_FIQ_MASK(cpuid) ((BIT(0) | BIT(8)) << cpuid) /* Registers relative to per_cpu_int_base */ -#define ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS (0x08) -#define ARMADA_370_XP_IN_DRBEL_MSK_OFFS (0x0c) +#define ARMADA_370_XP_IN_DRBEL_CAUSE (0x08) +#define ARMADA_370_XP_IN_DRBEL_MSK (0x0c) #define ARMADA_375_PPI_CAUSE (0x10) -#define ARMADA_370_XP_CPU_INTACK_OFFS (0x44) -#define ARMADA_370_XP_INT_SET_MASK_OFFS (0x48) -#define ARMADA_370_XP_INT_CLEAR_MASK_OFFS (0x4C) -#define ARMADA_370_XP_INT_FABRIC_MASK_OFFS (0x54) +#define ARMADA_370_XP_CPU_INTACK (0x44) +#define ARMADA_370_XP_INT_SET_MASK (0x48) +#define ARMADA_370_XP_INT_CLEAR_MASK (0x4C) +#define ARMADA_370_XP_INT_FABRIC_MASK (0x54) #define ARMADA_370_XP_INT_CAUSE_PERF(cpu) (1 << cpu) #define ARMADA_370_XP_MAX_PER_CPU_IRQS (28) @@ -220,11 +219,9 @@ static void armada_370_xp_irq_mask(struct irq_data *d) irq_hw_number_t hwirq = irqd_to_hwirq(d); if (!is_percpu_irq(hwirq)) - writel(hwirq, main_int_base + - ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS); + writel(hwirq, main_int_base + ARMADA_370_XP_INT_CLEAR_ENABLE); else - writel(hwirq, per_cpu_int_base + - ARMADA_370_XP_INT_SET_MASK_OFFS); + writel(hwirq, per_cpu_int_base + ARMADA_370_XP_INT_SET_MASK); } static void armada_370_xp_irq_unmask(struct irq_data *d) @@ -232,11 +229,9 @@ static void armada_370_xp_irq_unmask(struct irq_data *d) irq_hw_number_t hwirq = irqd_to_hwirq(d); if (!is_percpu_irq(hwirq)) - writel(hwirq, main_int_base + - ARMADA_370_XP_INT_SET_ENABLE_OFFS); + writel(hwirq, main_int_base + ARMADA_370_XP_INT_SET_ENABLE); else - writel(hwirq, per_cpu_int_base + - ARMADA_370_XP_INT_CLEAR_MASK_OFFS); + writel(hwirq, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK); } #ifdef CONFIG_PCI_MSI @@ -329,19 +324,18 @@ static void armada_370_xp_msi_reenable_percpu(void) u32 reg; /* Enable MSI doorbell mask and combined cpu local interrupt */ - reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); + reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK); reg |= msi_doorbell_mask(); - writel(reg, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); + writel(reg, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK); /* Unmask local doorbell interrupt */ - writel(1, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); + writel(1, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK); } static int armada_370_xp_msi_init(struct device_node *node, phys_addr_t main_int_phys_base) { - msi_doorbell_addr = main_int_phys_base + - ARMADA_370_XP_SW_TRIG_INT_OFFS; + msi_doorbell_addr = main_int_phys_base + ARMADA_370_XP_SW_TRIG_INT; armada_370_xp_msi_inner_domain = irq_domain_add_linear(NULL, msi_doorbell_size(), @@ -362,7 +356,7 @@ static int armada_370_xp_msi_init(struct device_node *node, /* Unmask low 16 MSI irqs on non-IPI platforms */ if (!is_ipi_available()) - writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); + writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK); return 0; } @@ -391,7 +385,7 @@ static void armada_xp_mpic_perf_init(void) /* Enable Performance Counter Overflow interrupts */ writel(ARMADA_370_XP_INT_CAUSE_PERF(cpuid), - per_cpu_int_base + ARMADA_370_XP_INT_FABRIC_MASK_OFFS); + per_cpu_int_base + ARMADA_370_XP_INT_FABRIC_MASK); } #ifdef CONFIG_SMP @@ -400,17 +394,17 @@ static struct irq_domain *ipi_domain; static void armada_370_xp_ipi_mask(struct irq_data *d) { u32 reg; - reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); + reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK); reg &= ~BIT(d->hwirq); - writel(reg, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); + writel(reg, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK); } static void armada_370_xp_ipi_unmask(struct irq_data *d) { u32 reg; - reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); + reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK); reg |= BIT(d->hwirq); - writel(reg, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); + writel(reg, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK); } static void armada_370_xp_ipi_send_mask(struct irq_data *d, @@ -431,12 +425,12 @@ static void armada_370_xp_ipi_send_mask(struct irq_data *d, /* submit softirq */ writel((map << 8) | d->hwirq, main_int_base + - ARMADA_370_XP_SW_TRIG_INT_OFFS); + ARMADA_370_XP_SW_TRIG_INT); } static void armada_370_xp_ipi_ack(struct irq_data *d) { - writel(~BIT(d->hwirq), per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS); + writel(~BIT(d->hwirq), per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE); } static struct irq_chip ipi_irqchip = { @@ -539,19 +533,19 @@ static void armada_xp_mpic_smp_cpu_init(void) nr_irqs = (control >> 2) & 0x3ff; for (i = 0; i < nr_irqs; i++) - writel(i, per_cpu_int_base + ARMADA_370_XP_INT_SET_MASK_OFFS); + writel(i, per_cpu_int_base + ARMADA_370_XP_INT_SET_MASK); if (!is_ipi_available()) return; /* Disable all IPIs */ - writel(0, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); + writel(0, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK); /* Clear pending IPIs */ - writel(0, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS); + writel(0, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE); /* Unmask IPI interrupt */ - writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); + writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK); } static void armada_xp_mpic_reenable_percpu(void) @@ -622,9 +616,9 @@ static int armada_370_xp_mpic_irq_map(struct irq_domain *h, armada_370_xp_irq_mask(irq_get_irq_data(virq)); if (!is_percpu_irq(hw)) writel(hw, per_cpu_int_base + - ARMADA_370_XP_INT_CLEAR_MASK_OFFS); + ARMADA_370_XP_INT_CLEAR_MASK); else - writel(hw, main_int_base + ARMADA_370_XP_INT_SET_ENABLE_OFFS); + writel(hw, main_int_base + ARMADA_370_XP_INT_SET_ENABLE); irq_set_status_flags(virq, IRQ_LEVEL); if (is_percpu_irq(hw)) { @@ -651,12 +645,10 @@ static void armada_370_xp_handle_msi_irq(struct pt_regs *regs, bool is_chained) { u32 msimask, msinr; - msimask = readl_relaxed(per_cpu_int_base + - ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS); + msimask = readl_relaxed(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE); msimask &= msi_doorbell_mask(); - writel(~msimask, per_cpu_int_base + - ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS); + writel(~msimask, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE); for (msinr = msi_doorbell_start(); msinr < msi_doorbell_end(); msinr++) { @@ -712,7 +704,7 @@ armada_370_xp_handle_irq(struct pt_regs *regs) do { irqstat = readl_relaxed(per_cpu_int_base + - ARMADA_370_XP_CPU_INTACK_OFFS); + ARMADA_370_XP_CPU_INTACK); irqnr = irqstat & 0x3FF; if (irqnr > 1022) @@ -735,7 +727,7 @@ armada_370_xp_handle_irq(struct pt_regs *regs) int ipi; ipimask = readl_relaxed(per_cpu_int_base + - ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS) + ARMADA_370_XP_IN_DRBEL_CAUSE) & IPI_DOORBELL_MASK; for_each_set_bit(ipi, &ipimask, IPI_DOORBELL_END) @@ -748,8 +740,7 @@ armada_370_xp_handle_irq(struct pt_regs *regs) static int armada_370_xp_mpic_suspend(void) { - doorbell_mask_reg = readl(per_cpu_int_base + - ARMADA_370_XP_IN_DRBEL_MSK_OFFS); + doorbell_mask_reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK); return 0; } @@ -774,13 +765,13 @@ static void armada_370_xp_mpic_resume(void) if (!is_percpu_irq(irq)) { /* Non per-CPU interrupts */ writel(irq, per_cpu_int_base + - ARMADA_370_XP_INT_CLEAR_MASK_OFFS); + ARMADA_370_XP_INT_CLEAR_MASK); if (!irqd_irq_disabled(data)) armada_370_xp_irq_unmask(data); } else { /* Per-CPU interrupts */ writel(irq, main_int_base + - ARMADA_370_XP_INT_SET_ENABLE_OFFS); + ARMADA_370_XP_INT_SET_ENABLE); /* * Re-enable on the current CPU, @@ -794,7 +785,7 @@ static void armada_370_xp_mpic_resume(void) /* Reconfigure doorbells for IPIs and MSIs */ writel(doorbell_mask_reg, - per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); + per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK); if (is_ipi_available()) { src0 = doorbell_mask_reg & IPI_DOORBELL_MASK; @@ -805,9 +796,9 @@ static void armada_370_xp_mpic_resume(void) } if (src0) - writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); + writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK); if (src1) - writel(1, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); + writel(1, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK); if (is_ipi_available()) ipi_resume(); @@ -847,7 +838,7 @@ static int __init armada_370_xp_mpic_of_init(struct device_node *node, nr_irqs = (control >> 2) & 0x3ff; for (i = 0; i < nr_irqs; i++) - writel(i, main_int_base + ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS); + writel(i, main_int_base + ARMADA_370_XP_INT_CLEAR_ENABLE); armada_370_xp_mpic_domain = irq_domain_add_linear(node, nr_irqs, From 9fa3e59a003bb82977ade5011ca6255f5ec83c5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Mon, 8 Jul 2024 17:17:53 +0200 Subject: [PATCH 026/180] irqchip/armada-370-xp: Change register constant suffix from _MSK to _MASK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is one occurrence of suffix _MSK in register constants, others have _MASK instead. Change the one to _MASK for consistency. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Reviewed-by: Andrew Lunn Reviewed-by: Ilpo Järvinen Link: https://lore.kernel.org/all/20240708151801.11592-3-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 66d6a2ebc8a5..588a9e2e1887 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -126,7 +126,7 @@ /* Registers relative to per_cpu_int_base */ #define ARMADA_370_XP_IN_DRBEL_CAUSE (0x08) -#define ARMADA_370_XP_IN_DRBEL_MSK (0x0c) +#define ARMADA_370_XP_IN_DRBEL_MASK (0x0c) #define ARMADA_375_PPI_CAUSE (0x10) #define ARMADA_370_XP_CPU_INTACK (0x44) #define ARMADA_370_XP_INT_SET_MASK (0x48) @@ -324,9 +324,9 @@ static void armada_370_xp_msi_reenable_percpu(void) u32 reg; /* Enable MSI doorbell mask and combined cpu local interrupt */ - reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK); + reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MASK); reg |= msi_doorbell_mask(); - writel(reg, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK); + writel(reg, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MASK); /* Unmask local doorbell interrupt */ writel(1, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK); @@ -394,17 +394,17 @@ static struct irq_domain *ipi_domain; static void armada_370_xp_ipi_mask(struct irq_data *d) { u32 reg; - reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK); + reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MASK); reg &= ~BIT(d->hwirq); - writel(reg, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK); + writel(reg, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MASK); } static void armada_370_xp_ipi_unmask(struct irq_data *d) { u32 reg; - reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK); + reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MASK); reg |= BIT(d->hwirq); - writel(reg, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK); + writel(reg, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MASK); } static void armada_370_xp_ipi_send_mask(struct irq_data *d, @@ -539,7 +539,7 @@ static void armada_xp_mpic_smp_cpu_init(void) return; /* Disable all IPIs */ - writel(0, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK); + writel(0, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MASK); /* Clear pending IPIs */ writel(0, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE); @@ -740,7 +740,7 @@ armada_370_xp_handle_irq(struct pt_regs *regs) static int armada_370_xp_mpic_suspend(void) { - doorbell_mask_reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK); + doorbell_mask_reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MASK); return 0; } @@ -785,7 +785,7 @@ static void armada_370_xp_mpic_resume(void) /* Reconfigure doorbells for IPIs and MSIs */ writel(doorbell_mask_reg, - per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK); + per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MASK); if (is_ipi_available()) { src0 = doorbell_mask_reg & IPI_DOORBELL_MASK; From f04ef167b350722f1623308c085c3ce119894035 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Mon, 8 Jul 2024 17:17:54 +0200 Subject: [PATCH 027/180] irqchip/armada-370-xp: Change spaces to tabs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change spaces to tabs in register constants definitions. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Reviewed-by: Andrew Lunn Reviewed-by: Ilpo Järvinen Link: https://lore.kernel.org/all/20240708151801.11592-4-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 588a9e2e1887..427ba5fd6adc 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -137,13 +137,13 @@ #define ARMADA_370_XP_MAX_PER_CPU_IRQS (28) /* IPI and MSI interrupt definitions for IPI platforms */ -#define IPI_DOORBELL_START (0) -#define IPI_DOORBELL_END (8) -#define IPI_DOORBELL_MASK 0xFF -#define PCI_MSI_DOORBELL_START (16) -#define PCI_MSI_DOORBELL_NR (16) -#define PCI_MSI_DOORBELL_END (32) -#define PCI_MSI_DOORBELL_MASK 0xFFFF0000 +#define IPI_DOORBELL_START (0) +#define IPI_DOORBELL_END (8) +#define IPI_DOORBELL_MASK 0xFF +#define PCI_MSI_DOORBELL_START (16) +#define PCI_MSI_DOORBELL_NR (16) +#define PCI_MSI_DOORBELL_END (32) +#define PCI_MSI_DOORBELL_MASK 0xFFFF0000 /* MSI interrupt definitions for non-IPI platforms */ #define PCI_MSI_FULL_DOORBELL_START 0 From 2613b94d2dc5fc6b80ea8175ac3dbf579e6e1bac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Mon, 8 Jul 2024 17:17:55 +0200 Subject: [PATCH 028/180] irqchip/armada-370-xp: Use BIT() and GENMASK() macros MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the BIT() and GENMASK() macros where appropriate. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Reviewed-by: Andrew Lunn Link: https://lore.kernel.org/all/20240708151801.11592-5-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 427ba5fd6adc..18aca9b5d3b3 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -121,7 +121,7 @@ #define ARMADA_370_XP_INT_SET_ENABLE (0x30) #define ARMADA_370_XP_INT_CLEAR_ENABLE (0x34) #define ARMADA_370_XP_INT_SOURCE_CTL(irq) (0x100 + irq*4) -#define ARMADA_370_XP_INT_SOURCE_CPU_MASK 0xF +#define ARMADA_370_XP_INT_SOURCE_CPU_MASK GENMASK(3, 0) #define ARMADA_370_XP_INT_IRQ_FIQ_MASK(cpuid) ((BIT(0) | BIT(8)) << cpuid) /* Registers relative to per_cpu_int_base */ @@ -132,18 +132,18 @@ #define ARMADA_370_XP_INT_SET_MASK (0x48) #define ARMADA_370_XP_INT_CLEAR_MASK (0x4C) #define ARMADA_370_XP_INT_FABRIC_MASK (0x54) -#define ARMADA_370_XP_INT_CAUSE_PERF(cpu) (1 << cpu) +#define ARMADA_370_XP_INT_CAUSE_PERF(cpu) BIT(cpu) #define ARMADA_370_XP_MAX_PER_CPU_IRQS (28) /* IPI and MSI interrupt definitions for IPI platforms */ #define IPI_DOORBELL_START (0) #define IPI_DOORBELL_END (8) -#define IPI_DOORBELL_MASK 0xFF +#define IPI_DOORBELL_MASK GENMASK(7, 0) #define PCI_MSI_DOORBELL_START (16) #define PCI_MSI_DOORBELL_NR (16) #define PCI_MSI_DOORBELL_END (32) -#define PCI_MSI_DOORBELL_MASK 0xFFFF0000 +#define PCI_MSI_DOORBELL_MASK GENMASK(31, 16) /* MSI interrupt definitions for non-IPI platforms */ #define PCI_MSI_FULL_DOORBELL_START 0 @@ -415,7 +415,7 @@ static void armada_370_xp_ipi_send_mask(struct irq_data *d, /* Convert our logical CPU mask into a physical one. */ for_each_cpu(cpu, mask) - map |= 1 << cpu_logical_map(cpu); + map |= BIT(cpu_logical_map(cpu)); /* * Ensure that stores to Normal memory are visible to the From 9236717b97e3f5f0a5c77e40a85b8355b6025311 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Mon, 8 Jul 2024 17:17:56 +0200 Subject: [PATCH 029/180] irqchip/armada-370-xp: Cosmetic fix parentheses in register constant definitions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop parentheses where not needed and add them where it makes sense in register constant definitions. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Reviewed-by: Andrew Lunn Reviewed-by: Ilpo Järvinen Link: https://lore.kernel.org/all/20240708151801.11592-6-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 38 ++++++++++++++--------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 18aca9b5d3b3..14d213e9b0d2 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -116,33 +116,33 @@ */ /* Registers relative to main_int_base */ -#define ARMADA_370_XP_INT_CONTROL (0x00) -#define ARMADA_370_XP_SW_TRIG_INT (0x04) -#define ARMADA_370_XP_INT_SET_ENABLE (0x30) -#define ARMADA_370_XP_INT_CLEAR_ENABLE (0x34) -#define ARMADA_370_XP_INT_SOURCE_CTL(irq) (0x100 + irq*4) +#define ARMADA_370_XP_INT_CONTROL 0x00 +#define ARMADA_370_XP_SW_TRIG_INT 0x04 +#define ARMADA_370_XP_INT_SET_ENABLE 0x30 +#define ARMADA_370_XP_INT_CLEAR_ENABLE 0x34 +#define ARMADA_370_XP_INT_SOURCE_CTL(irq) (0x100 + (irq) * 4) #define ARMADA_370_XP_INT_SOURCE_CPU_MASK GENMASK(3, 0) -#define ARMADA_370_XP_INT_IRQ_FIQ_MASK(cpuid) ((BIT(0) | BIT(8)) << cpuid) +#define ARMADA_370_XP_INT_IRQ_FIQ_MASK(cpuid) ((BIT(0) | BIT(8)) << (cpuid)) /* Registers relative to per_cpu_int_base */ -#define ARMADA_370_XP_IN_DRBEL_CAUSE (0x08) -#define ARMADA_370_XP_IN_DRBEL_MASK (0x0c) -#define ARMADA_375_PPI_CAUSE (0x10) -#define ARMADA_370_XP_CPU_INTACK (0x44) -#define ARMADA_370_XP_INT_SET_MASK (0x48) -#define ARMADA_370_XP_INT_CLEAR_MASK (0x4C) -#define ARMADA_370_XP_INT_FABRIC_MASK (0x54) +#define ARMADA_370_XP_IN_DRBEL_CAUSE 0x08 +#define ARMADA_370_XP_IN_DRBEL_MASK 0x0c +#define ARMADA_375_PPI_CAUSE 0x10 +#define ARMADA_370_XP_CPU_INTACK 0x44 +#define ARMADA_370_XP_INT_SET_MASK 0x48 +#define ARMADA_370_XP_INT_CLEAR_MASK 0x4C +#define ARMADA_370_XP_INT_FABRIC_MASK 0x54 #define ARMADA_370_XP_INT_CAUSE_PERF(cpu) BIT(cpu) -#define ARMADA_370_XP_MAX_PER_CPU_IRQS (28) +#define ARMADA_370_XP_MAX_PER_CPU_IRQS 28 /* IPI and MSI interrupt definitions for IPI platforms */ -#define IPI_DOORBELL_START (0) -#define IPI_DOORBELL_END (8) +#define IPI_DOORBELL_START 0 +#define IPI_DOORBELL_END 8 #define IPI_DOORBELL_MASK GENMASK(7, 0) -#define PCI_MSI_DOORBELL_START (16) -#define PCI_MSI_DOORBELL_NR (16) -#define PCI_MSI_DOORBELL_END (32) +#define PCI_MSI_DOORBELL_START 16 +#define PCI_MSI_DOORBELL_NR 16 +#define PCI_MSI_DOORBELL_END 32 #define PCI_MSI_DOORBELL_MASK GENMASK(31, 16) /* MSI interrupt definitions for non-IPI platforms */ From e812dd60b6cca3211e7d83528c8bf077a51730b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Mon, 8 Jul 2024 17:17:57 +0200 Subject: [PATCH 030/180] irqchip/armada-370-xp: Change register constants prefix to MPIC_ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change the long ARMADA_370_XP_ prefix in register constants (ARMADA_375_ in one case) to MPIC_. The rationale is that it is shorter and more generic (this controller is called MPIC and is also used on Armada 38x and 39x). Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Reviewed-by: Andrew Lunn Link: https://lore.kernel.org/all/20240708151801.11592-7-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 148 +++++++++++++--------------- 1 file changed, 69 insertions(+), 79 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 14d213e9b0d2..8f52de6d8921 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -66,18 +66,17 @@ * device * * The "global interrupt mask/unmask" is modified using the - * ARMADA_370_XP_INT_SET_ENABLE and ARMADA_370_XP_INT_CLEAR_ENABLE + * MPIC_INT_SET_ENABLE and MPIC_INT_CLEAR_ENABLE * registers, which are relative to "main_int_base". * - * The "per-CPU mask/unmask" is modified using the - * ARMADA_370_XP_INT_SET_MASK and ARMADA_370_XP_INT_CLEAR_MASK - * registers, which are relative to "per_cpu_int_base". This base - * address points to a special address, which automatically accesses - * the registers of the current CPU. + * The "per-CPU mask/unmask" is modified using the MPIC_INT_SET_MASK + * and MPIC_INT_CLEAR_MASK registers, which are relative to + * "per_cpu_int_base". This base address points to a special address, + * which automatically accesses the registers of the current CPU. * * The per-CPU mask/unmask can also be adjusted using the global - * per-interrupt ARMADA_370_XP_INT_SOURCE_CTL register, which we use - * to configure interrupt affinity. + * per-interrupt MPIC_INT_SOURCE_CTL register, which we use to + * configure interrupt affinity. * * Due to this model, all interrupts need to be mask/unmasked at two * different levels: at the global level and at the per-CPU level. @@ -91,9 +90,8 @@ * the current CPU, running the ->map() code. This allows to have * the interrupt unmasked at this level in non-SMP * configurations. In SMP configurations, the ->set_affinity() - * callback is called, which using the - * ARMADA_370_XP_INT_SOURCE_CTL() readjusts the per-CPU mask/unmask - * for the interrupt. + * callback is called, which using the MPIC_INT_SOURCE_CTL() + * readjusts the per-CPU mask/unmask for the interrupt. * * The ->mask() and ->unmask() operations only mask/unmask the * interrupt at the "global" level. @@ -116,25 +114,25 @@ */ /* Registers relative to main_int_base */ -#define ARMADA_370_XP_INT_CONTROL 0x00 -#define ARMADA_370_XP_SW_TRIG_INT 0x04 -#define ARMADA_370_XP_INT_SET_ENABLE 0x30 -#define ARMADA_370_XP_INT_CLEAR_ENABLE 0x34 -#define ARMADA_370_XP_INT_SOURCE_CTL(irq) (0x100 + (irq) * 4) -#define ARMADA_370_XP_INT_SOURCE_CPU_MASK GENMASK(3, 0) -#define ARMADA_370_XP_INT_IRQ_FIQ_MASK(cpuid) ((BIT(0) | BIT(8)) << (cpuid)) +#define MPIC_INT_CONTROL 0x00 +#define MPIC_SW_TRIG_INT 0x04 +#define MPIC_INT_SET_ENABLE 0x30 +#define MPIC_INT_CLEAR_ENABLE 0x34 +#define MPIC_INT_SOURCE_CTL(irq) (0x100 + (irq) * 4) +#define MPIC_INT_SOURCE_CPU_MASK GENMASK(3, 0) +#define MPIC_INT_IRQ_FIQ_MASK(cpuid) ((BIT(0) | BIT(8)) << (cpuid)) /* Registers relative to per_cpu_int_base */ -#define ARMADA_370_XP_IN_DRBEL_CAUSE 0x08 -#define ARMADA_370_XP_IN_DRBEL_MASK 0x0c -#define ARMADA_375_PPI_CAUSE 0x10 -#define ARMADA_370_XP_CPU_INTACK 0x44 -#define ARMADA_370_XP_INT_SET_MASK 0x48 -#define ARMADA_370_XP_INT_CLEAR_MASK 0x4C -#define ARMADA_370_XP_INT_FABRIC_MASK 0x54 -#define ARMADA_370_XP_INT_CAUSE_PERF(cpu) BIT(cpu) +#define MPIC_IN_DRBEL_CAUSE 0x08 +#define MPIC_IN_DRBEL_MASK 0x0c +#define MPIC_PPI_CAUSE 0x10 +#define MPIC_CPU_INTACK 0x44 +#define MPIC_INT_SET_MASK 0x48 +#define MPIC_INT_CLEAR_MASK 0x4C +#define MPIC_INT_FABRIC_MASK 0x54 +#define MPIC_INT_CAUSE_PERF(cpu) BIT(cpu) -#define ARMADA_370_XP_MAX_PER_CPU_IRQS 28 +#define MPIC_MAX_PER_CPU_IRQS 28 /* IPI and MSI interrupt definitions for IPI platforms */ #define IPI_DOORBELL_START 0 @@ -203,7 +201,7 @@ static inline unsigned int msi_doorbell_end(void) static inline bool is_percpu_irq(irq_hw_number_t irq) { - if (irq <= ARMADA_370_XP_MAX_PER_CPU_IRQS) + if (irq <= MPIC_MAX_PER_CPU_IRQS) return true; return false; @@ -219,9 +217,9 @@ static void armada_370_xp_irq_mask(struct irq_data *d) irq_hw_number_t hwirq = irqd_to_hwirq(d); if (!is_percpu_irq(hwirq)) - writel(hwirq, main_int_base + ARMADA_370_XP_INT_CLEAR_ENABLE); + writel(hwirq, main_int_base + MPIC_INT_CLEAR_ENABLE); else - writel(hwirq, per_cpu_int_base + ARMADA_370_XP_INT_SET_MASK); + writel(hwirq, per_cpu_int_base + MPIC_INT_SET_MASK); } static void armada_370_xp_irq_unmask(struct irq_data *d) @@ -229,9 +227,9 @@ static void armada_370_xp_irq_unmask(struct irq_data *d) irq_hw_number_t hwirq = irqd_to_hwirq(d); if (!is_percpu_irq(hwirq)) - writel(hwirq, main_int_base + ARMADA_370_XP_INT_SET_ENABLE); + writel(hwirq, main_int_base + MPIC_INT_SET_ENABLE); else - writel(hwirq, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK); + writel(hwirq, per_cpu_int_base + MPIC_INT_CLEAR_MASK); } #ifdef CONFIG_PCI_MSI @@ -324,18 +322,18 @@ static void armada_370_xp_msi_reenable_percpu(void) u32 reg; /* Enable MSI doorbell mask and combined cpu local interrupt */ - reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MASK); + reg = readl(per_cpu_int_base + MPIC_IN_DRBEL_MASK); reg |= msi_doorbell_mask(); - writel(reg, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MASK); + writel(reg, per_cpu_int_base + MPIC_IN_DRBEL_MASK); /* Unmask local doorbell interrupt */ - writel(1, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK); + writel(1, per_cpu_int_base + MPIC_INT_CLEAR_MASK); } static int armada_370_xp_msi_init(struct device_node *node, phys_addr_t main_int_phys_base) { - msi_doorbell_addr = main_int_phys_base + ARMADA_370_XP_SW_TRIG_INT; + msi_doorbell_addr = main_int_phys_base + MPIC_SW_TRIG_INT; armada_370_xp_msi_inner_domain = irq_domain_add_linear(NULL, msi_doorbell_size(), @@ -356,7 +354,7 @@ static int armada_370_xp_msi_init(struct device_node *node, /* Unmask low 16 MSI irqs on non-IPI platforms */ if (!is_ipi_available()) - writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK); + writel(0, per_cpu_int_base + MPIC_INT_CLEAR_MASK); return 0; } @@ -384,8 +382,8 @@ static void armada_xp_mpic_perf_init(void) cpuid = cpu_logical_map(smp_processor_id()); /* Enable Performance Counter Overflow interrupts */ - writel(ARMADA_370_XP_INT_CAUSE_PERF(cpuid), - per_cpu_int_base + ARMADA_370_XP_INT_FABRIC_MASK); + writel(MPIC_INT_CAUSE_PERF(cpuid), + per_cpu_int_base + MPIC_INT_FABRIC_MASK); } #ifdef CONFIG_SMP @@ -394,17 +392,17 @@ static struct irq_domain *ipi_domain; static void armada_370_xp_ipi_mask(struct irq_data *d) { u32 reg; - reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MASK); + reg = readl(per_cpu_int_base + MPIC_IN_DRBEL_MASK); reg &= ~BIT(d->hwirq); - writel(reg, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MASK); + writel(reg, per_cpu_int_base + MPIC_IN_DRBEL_MASK); } static void armada_370_xp_ipi_unmask(struct irq_data *d) { u32 reg; - reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MASK); + reg = readl(per_cpu_int_base + MPIC_IN_DRBEL_MASK); reg |= BIT(d->hwirq); - writel(reg, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MASK); + writel(reg, per_cpu_int_base + MPIC_IN_DRBEL_MASK); } static void armada_370_xp_ipi_send_mask(struct irq_data *d, @@ -424,13 +422,12 @@ static void armada_370_xp_ipi_send_mask(struct irq_data *d, dsb(); /* submit softirq */ - writel((map << 8) | d->hwirq, main_int_base + - ARMADA_370_XP_SW_TRIG_INT); + writel((map << 8) | d->hwirq, main_int_base + MPIC_SW_TRIG_INT); } static void armada_370_xp_ipi_ack(struct irq_data *d) { - writel(~BIT(d->hwirq), per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE); + writel(~BIT(d->hwirq), per_cpu_int_base + MPIC_IN_DRBEL_CAUSE); } static struct irq_chip ipi_irqchip = { @@ -515,9 +512,8 @@ static int armada_xp_set_affinity(struct irq_data *d, /* Select a single core from the affinity mask which is online */ cpu = cpumask_any_and(mask_val, cpu_online_mask); - atomic_io_modify(main_int_base + ARMADA_370_XP_INT_SOURCE_CTL(hwirq), - ARMADA_370_XP_INT_SOURCE_CPU_MASK, - BIT(cpu_logical_map(cpu))); + atomic_io_modify(main_int_base + MPIC_INT_SOURCE_CTL(hwirq), + MPIC_INT_SOURCE_CPU_MASK, BIT(cpu_logical_map(cpu))); irq_data_update_effective_affinity(d, cpumask_of(cpu)); @@ -529,23 +525,23 @@ static void armada_xp_mpic_smp_cpu_init(void) u32 control; int nr_irqs, i; - control = readl(main_int_base + ARMADA_370_XP_INT_CONTROL); + control = readl(main_int_base + MPIC_INT_CONTROL); nr_irqs = (control >> 2) & 0x3ff; for (i = 0; i < nr_irqs; i++) - writel(i, per_cpu_int_base + ARMADA_370_XP_INT_SET_MASK); + writel(i, per_cpu_int_base + MPIC_INT_SET_MASK); if (!is_ipi_available()) return; /* Disable all IPIs */ - writel(0, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MASK); + writel(0, per_cpu_int_base + MPIC_IN_DRBEL_MASK); /* Clear pending IPIs */ - writel(0, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE); + writel(0, per_cpu_int_base + MPIC_IN_DRBEL_CAUSE); /* Unmask IPI interrupt */ - writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK); + writel(0, per_cpu_int_base + MPIC_INT_CLEAR_MASK); } static void armada_xp_mpic_reenable_percpu(void) @@ -553,7 +549,7 @@ static void armada_xp_mpic_reenable_percpu(void) unsigned int irq; /* Re-enable per-CPU interrupts that were enabled before suspend */ - for (irq = 0; irq < ARMADA_370_XP_MAX_PER_CPU_IRQS; irq++) { + for (irq = 0; irq < MPIC_MAX_PER_CPU_IRQS; irq++) { struct irq_data *data; int virq; @@ -615,10 +611,9 @@ static int armada_370_xp_mpic_irq_map(struct irq_domain *h, armada_370_xp_irq_mask(irq_get_irq_data(virq)); if (!is_percpu_irq(hw)) - writel(hw, per_cpu_int_base + - ARMADA_370_XP_INT_CLEAR_MASK); + writel(hw, per_cpu_int_base + MPIC_INT_CLEAR_MASK); else - writel(hw, main_int_base + ARMADA_370_XP_INT_SET_ENABLE); + writel(hw, main_int_base + MPIC_INT_SET_ENABLE); irq_set_status_flags(virq, IRQ_LEVEL); if (is_percpu_irq(hw)) { @@ -645,10 +640,10 @@ static void armada_370_xp_handle_msi_irq(struct pt_regs *regs, bool is_chained) { u32 msimask, msinr; - msimask = readl_relaxed(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE); + msimask = readl_relaxed(per_cpu_int_base + MPIC_IN_DRBEL_CAUSE); msimask &= msi_doorbell_mask(); - writel(~msimask, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE); + writel(~msimask, per_cpu_int_base + MPIC_IN_DRBEL_CAUSE); for (msinr = msi_doorbell_start(); msinr < msi_doorbell_end(); msinr++) { @@ -673,17 +668,16 @@ static void armada_370_xp_mpic_handle_cascade_irq(struct irq_desc *desc) chained_irq_enter(chip, desc); - irqmap = readl_relaxed(per_cpu_int_base + ARMADA_375_PPI_CAUSE); + irqmap = readl_relaxed(per_cpu_int_base + MPIC_PPI_CAUSE); cpuid = cpu_logical_map(smp_processor_id()); for_each_set_bit(irqn, &irqmap, BITS_PER_LONG) { - irqsrc = readl_relaxed(main_int_base + - ARMADA_370_XP_INT_SOURCE_CTL(irqn)); + irqsrc = readl_relaxed(main_int_base + MPIC_INT_SOURCE_CTL(irqn)); /* Check if the interrupt is not masked on current CPU. * Test IRQ (0-1) and FIQ (8-9) mask bits. */ - if (!(irqsrc & ARMADA_370_XP_INT_IRQ_FIQ_MASK(cpuid))) + if (!(irqsrc & MPIC_INT_IRQ_FIQ_MASK(cpuid))) continue; if (irqn == 0 || irqn == 1) { @@ -703,8 +697,7 @@ armada_370_xp_handle_irq(struct pt_regs *regs) u32 irqstat, irqnr; do { - irqstat = readl_relaxed(per_cpu_int_base + - ARMADA_370_XP_CPU_INTACK); + irqstat = readl_relaxed(per_cpu_int_base + MPIC_CPU_INTACK); irqnr = irqstat & 0x3FF; if (irqnr > 1022) @@ -727,7 +720,7 @@ armada_370_xp_handle_irq(struct pt_regs *regs) int ipi; ipimask = readl_relaxed(per_cpu_int_base + - ARMADA_370_XP_IN_DRBEL_CAUSE) + MPIC_IN_DRBEL_CAUSE) & IPI_DOORBELL_MASK; for_each_set_bit(ipi, &ipimask, IPI_DOORBELL_END) @@ -740,7 +733,7 @@ armada_370_xp_handle_irq(struct pt_regs *regs) static int armada_370_xp_mpic_suspend(void) { - doorbell_mask_reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MASK); + doorbell_mask_reg = readl(per_cpu_int_base + MPIC_IN_DRBEL_MASK); return 0; } @@ -751,7 +744,7 @@ static void armada_370_xp_mpic_resume(void) irq_hw_number_t irq; /* Re-enable interrupts */ - nirqs = (readl(main_int_base + ARMADA_370_XP_INT_CONTROL) >> 2) & 0x3ff; + nirqs = (readl(main_int_base + MPIC_INT_CONTROL) >> 2) & 0x3ff; for (irq = 0; irq < nirqs; irq++) { struct irq_data *data; int virq; @@ -764,14 +757,12 @@ static void armada_370_xp_mpic_resume(void) if (!is_percpu_irq(irq)) { /* Non per-CPU interrupts */ - writel(irq, per_cpu_int_base + - ARMADA_370_XP_INT_CLEAR_MASK); + writel(irq, per_cpu_int_base + MPIC_INT_CLEAR_MASK); if (!irqd_irq_disabled(data)) armada_370_xp_irq_unmask(data); } else { /* Per-CPU interrupts */ - writel(irq, main_int_base + - ARMADA_370_XP_INT_SET_ENABLE); + writel(irq, main_int_base + MPIC_INT_SET_ENABLE); /* * Re-enable on the current CPU, @@ -784,8 +775,7 @@ static void armada_370_xp_mpic_resume(void) } /* Reconfigure doorbells for IPIs and MSIs */ - writel(doorbell_mask_reg, - per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MASK); + writel(doorbell_mask_reg, per_cpu_int_base + MPIC_IN_DRBEL_MASK); if (is_ipi_available()) { src0 = doorbell_mask_reg & IPI_DOORBELL_MASK; @@ -796,9 +786,9 @@ static void armada_370_xp_mpic_resume(void) } if (src0) - writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK); + writel(0, per_cpu_int_base + MPIC_INT_CLEAR_MASK); if (src1) - writel(1, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK); + writel(1, per_cpu_int_base + MPIC_INT_CLEAR_MASK); if (is_ipi_available()) ipi_resume(); @@ -834,11 +824,11 @@ static int __init armada_370_xp_mpic_of_init(struct device_node *node, resource_size(&per_cpu_int_res)); BUG_ON(!per_cpu_int_base); - control = readl(main_int_base + ARMADA_370_XP_INT_CONTROL); + control = readl(main_int_base + MPIC_INT_CONTROL); nr_irqs = (control >> 2) & 0x3ff; for (i = 0; i < nr_irqs; i++) - writel(i, main_int_base + ARMADA_370_XP_INT_CLEAR_ENABLE); + writel(i, main_int_base + MPIC_INT_CLEAR_ENABLE); armada_370_xp_mpic_domain = irq_domain_add_linear(node, nr_irqs, From 0cbbf7c15d197ac370387c08d900abe142153cd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Mon, 8 Jul 2024 17:17:58 +0200 Subject: [PATCH 031/180] irqchip/armada-370-xp: Use correct type for cpu variable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use unsigned int instead of int for variable storing the cpu number. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Reviewed-by: Andrew Lunn Link: https://lore.kernel.org/all/20240708151801.11592-8-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 8f52de6d8921..b9631cc25c0b 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -409,7 +409,7 @@ static void armada_370_xp_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) { unsigned long map = 0; - int cpu; + unsigned int cpu; /* Convert our logical CPU mask into a physical one. */ for_each_cpu(cpu, mask) @@ -507,7 +507,7 @@ static int armada_xp_set_affinity(struct irq_data *d, const struct cpumask *mask_val, bool force) { irq_hw_number_t hwirq = irqd_to_hwirq(d); - int cpu; + unsigned int cpu; /* Select a single core from the affinity mask which is online */ cpu = cpumask_any_and(mask_val, cpu_online_mask); From ccef3a991b7c972cccce4aee8e62f70e3a706e78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Mon, 8 Jul 2024 17:17:59 +0200 Subject: [PATCH 032/180] irqchip/armada-370-xp: Simplify is_percpu_irq() code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Simplify the code in the is_percpu_irq() function. Instead of if (condition) return true; return false; simply return condition. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Reviewed-by: Andrew Lunn Reviewed-by: Ilpo Järvinen Link: https://lore.kernel.org/all/20240708151801.11592-9-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index b9631cc25c0b..cfd6dc803150 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -201,10 +201,7 @@ static inline unsigned int msi_doorbell_end(void) static inline bool is_percpu_irq(irq_hw_number_t irq) { - if (irq <= MPIC_MAX_PER_CPU_IRQS) - return true; - - return false; + return irq <= MPIC_MAX_PER_CPU_IRQS; } /* From 045c4bb864489fda99309dfa902346570d576a39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Mon, 8 Jul 2024 17:18:00 +0200 Subject: [PATCH 033/180] irqchip/armada-370-xp: Change to SPDX license identifier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change the license identifier to SPDX style. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Reviewed-by: Andrew Lunn Link: https://lore.kernel.org/all/20240708151801.11592-10-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index cfd6dc803150..3d15d0bb7605 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Marvell Armada 370 and Armada XP SoC IRQ handling * @@ -7,10 +8,6 @@ * Gregory CLEMENT * Thomas Petazzoni * Ben Dooks - * - * This file is licensed under the terms of the GNU General Public - * License version 2. This program is licensed "as is" without any - * warranty of any kind, whether express or implied. */ #include From 644799f920c906666b5393c33dcf3008ace1ef6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Mon, 8 Jul 2024 17:18:01 +0200 Subject: [PATCH 034/180] irqchip/armada-370-xp: Declare iterators in for loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Where possible, declare iterators in for cycle. This is possible since kernel uses -std=gnu11. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Reviewed-by: Andrew Lunn Link: https://lore.kernel.org/all/20240708151801.11592-11-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 3d15d0bb7605..22e1a493abae 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -276,7 +276,7 @@ static struct irq_chip armada_370_xp_msi_bottom_irq_chip = { static int armada_370_xp_msi_alloc(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs, void *args) { - int hwirq, i; + int hwirq; mutex_lock(&msi_used_lock); hwirq = bitmap_find_free_region(msi_used, msi_doorbell_size(), @@ -286,7 +286,7 @@ static int armada_370_xp_msi_alloc(struct irq_domain *domain, unsigned int virq, if (hwirq < 0) return -ENOSPC; - for (i = 0; i < nr_irqs; i++) { + for (int i = 0; i < nr_irqs; i++) { irq_domain_set_info(domain, virq + i, hwirq + i, &armada_370_xp_msi_bottom_irq_chip, domain->host_data, handle_simple_irq, @@ -436,9 +436,7 @@ static int armada_370_xp_ipi_alloc(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs, void *args) { - int i; - - for (i = 0; i < nr_irqs; i++) { + for (int i = 0; i < nr_irqs; i++) { irq_set_percpu_devid(virq + i); irq_domain_set_info(d, virq + i, i, &ipi_irqchip, d->host_data, @@ -463,9 +461,7 @@ static const struct irq_domain_ops ipi_domain_ops = { static void ipi_resume(void) { - int i; - - for (i = 0; i < IPI_DOORBELL_END; i++) { + for (int i = 0; i < IPI_DOORBELL_END; i++) { int irq; irq = irq_find_mapping(ipi_domain, i); @@ -517,12 +513,12 @@ static int armada_xp_set_affinity(struct irq_data *d, static void armada_xp_mpic_smp_cpu_init(void) { u32 control; - int nr_irqs, i; + int nr_irqs; control = readl(main_int_base + MPIC_INT_CONTROL); nr_irqs = (control >> 2) & 0x3ff; - for (i = 0; i < nr_irqs; i++) + for (int i = 0; i < nr_irqs; i++) writel(i, per_cpu_int_base + MPIC_INT_SET_MASK); if (!is_ipi_available()) @@ -540,10 +536,8 @@ static void armada_xp_mpic_smp_cpu_init(void) static void armada_xp_mpic_reenable_percpu(void) { - unsigned int irq; - /* Re-enable per-CPU interrupts that were enabled before suspend */ - for (irq = 0; irq < MPIC_MAX_PER_CPU_IRQS; irq++) { + for (unsigned int irq = 0; irq < MPIC_MAX_PER_CPU_IRQS; irq++) { struct irq_data *data; int virq; @@ -735,11 +729,10 @@ static void armada_370_xp_mpic_resume(void) { bool src0, src1; int nirqs; - irq_hw_number_t irq; /* Re-enable interrupts */ nirqs = (readl(main_int_base + MPIC_INT_CONTROL) >> 2) & 0x3ff; - for (irq = 0; irq < nirqs; irq++) { + for (irq_hw_number_t irq = 0; irq < nirqs; irq++) { struct irq_data *data; int virq; @@ -797,7 +790,7 @@ static int __init armada_370_xp_mpic_of_init(struct device_node *node, struct device_node *parent) { struct resource main_int_res, per_cpu_int_res; - int nr_irqs, i; + int nr_irqs; u32 control; BUG_ON(of_address_to_resource(node, 0, &main_int_res)); @@ -821,7 +814,7 @@ static int __init armada_370_xp_mpic_of_init(struct device_node *node, control = readl(main_int_base + MPIC_INT_CONTROL); nr_irqs = (control >> 2) & 0x3ff; - for (i = 0; i < nr_irqs; i++) + for (int i = 0; i < nr_irqs; i++) writel(i, main_int_base + MPIC_INT_CLEAR_ENABLE); armada_370_xp_mpic_domain = From 55689986d7eaed09b6569b1e06b29044cd3cb590 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Thu, 11 Jul 2024 13:57:39 +0200 Subject: [PATCH 035/180] irqchip/armada-370-xp: Rename variable for consistency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename the irq variable to virq in the ipi_resume() function for consistency with the rest of the code. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Reviewed-by: Andrew Lunn Reviewed-by: Ilpo Järvinen Link: https://lore.kernel.org/all/20240711115748.30268-2-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 22e1a493abae..7016b206bddd 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -462,14 +462,14 @@ static const struct irq_domain_ops ipi_domain_ops = { static void ipi_resume(void) { for (int i = 0; i < IPI_DOORBELL_END; i++) { - int irq; + int virq; - irq = irq_find_mapping(ipi_domain, i); - if (irq <= 0) + virq = irq_find_mapping(ipi_domain, i); + if (virq <= 0) continue; - if (irq_percpu_is_enabled(irq)) { + if (irq_percpu_is_enabled(virq)) { struct irq_data *d; - d = irq_domain_get_irq_data(ipi_domain, irq); + d = irq_domain_get_irq_data(ipi_domain, virq); armada_370_xp_ipi_unmask(d); } } From e4cd7c553a00e2904689cac543e314b1962c6a8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Thu, 11 Jul 2024 13:57:40 +0200 Subject: [PATCH 036/180] irqchip/armada-370-xp: Use unsigned int type for virqs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The return type of irq_find_mapping() and irq_linear_revmap() is unsigned int. Use the unsigned int type for the variables storing the return value. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Reviewed-by: Andrew Lunn Reviewed-by: Ilpo Järvinen Link: https://lore.kernel.org/all/20240711115748.30268-3-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 7016b206bddd..b29f3bbfb1c3 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -462,10 +462,10 @@ static const struct irq_domain_ops ipi_domain_ops = { static void ipi_resume(void) { for (int i = 0; i < IPI_DOORBELL_END; i++) { - int virq; + unsigned int virq; virq = irq_find_mapping(ipi_domain, i); - if (virq <= 0) + if (!virq) continue; if (irq_percpu_is_enabled(virq)) { struct irq_data *d; @@ -539,7 +539,7 @@ static void armada_xp_mpic_reenable_percpu(void) /* Re-enable per-CPU interrupts that were enabled before suspend */ for (unsigned int irq = 0; irq < MPIC_MAX_PER_CPU_IRQS; irq++) { struct irq_data *data; - int virq; + unsigned int virq; virq = irq_linear_revmap(armada_370_xp_mpic_domain, irq); if (virq == 0) @@ -734,7 +734,7 @@ static void armada_370_xp_mpic_resume(void) nirqs = (readl(main_int_base + MPIC_INT_CONTROL) >> 2) & 0x3ff; for (irq_hw_number_t irq = 0; irq < nirqs; irq++) { struct irq_data *data; - int virq; + unsigned int virq; virq = irq_linear_revmap(armada_370_xp_mpic_domain, irq); if (virq == 0) From 88d49ee30ca52ba9a0dd593860a1323426791710 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Thu, 11 Jul 2024 13:57:41 +0200 Subject: [PATCH 037/180] irqchip/armada-370-xp: Use !virq instead of virq == 0 in condition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use !virq instead of virq == 0 when checking for availability of the virq. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Reviewed-by: Andrew Lunn Reviewed-by: Ilpo Järvinen Link: https://lore.kernel.org/all/20240711115748.30268-4-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index b29f3bbfb1c3..c007610413fe 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -542,7 +542,7 @@ static void armada_xp_mpic_reenable_percpu(void) unsigned int virq; virq = irq_linear_revmap(armada_370_xp_mpic_domain, irq); - if (virq == 0) + if (!virq) continue; data = irq_get_irq_data(virq); @@ -737,7 +737,7 @@ static void armada_370_xp_mpic_resume(void) unsigned int virq; virq = irq_linear_revmap(armada_370_xp_mpic_domain, irq); - if (virq == 0) + if (!virq) continue; data = irq_get_irq_data(virq); From 0381be072f301088637ab6b01f2c8f0e5745e0e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Thu, 11 Jul 2024 13:57:42 +0200 Subject: [PATCH 038/180] irqchip/armada-370-xp: Simplify ipi_resume() code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactor the ipi_resume() function to drop one indentation level. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Reviewed-by: Andrew Lunn Reviewed-by: Ilpo Järvinen Link: https://lore.kernel.org/all/20240711115748.30268-5-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index c007610413fe..316c27c97951 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -462,16 +462,14 @@ static const struct irq_domain_ops ipi_domain_ops = { static void ipi_resume(void) { for (int i = 0; i < IPI_DOORBELL_END; i++) { - unsigned int virq; + unsigned int virq = irq_find_mapping(ipi_domain, i); + struct irq_data *d; - virq = irq_find_mapping(ipi_domain, i); - if (!virq) + if (!virq || !irq_percpu_is_enabled(virq)) continue; - if (irq_percpu_is_enabled(virq)) { - struct irq_data *d; - d = irq_domain_get_irq_data(ipi_domain, virq); - armada_370_xp_ipi_unmask(d); - } + + d = irq_domain_get_irq_data(ipi_domain, virq); + armada_370_xp_ipi_unmask(d); } } From 5302e767ebfc1c297bf76f6ef65888249b831a73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Thu, 11 Jul 2024 13:57:43 +0200 Subject: [PATCH 039/180] irqchip/armada-370-xp: Improve indentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add some blank lines and other indentation improvements. Checkpatch now stops complaining. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Reviewed-by: Andrew Lunn Reviewed-by: Ilpo Järvinen Link: https://lore.kernel.org/all/20240711115748.30268-6-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 56 ++++++++++++++--------------- 1 file changed, 26 insertions(+), 30 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 316c27c97951..a66d3459f7fa 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -229,9 +229,9 @@ static void armada_370_xp_irq_unmask(struct irq_data *d) #ifdef CONFIG_PCI_MSI static struct irq_chip armada_370_xp_msi_irq_chip = { - .name = "MPIC MSI", - .irq_mask = pci_msi_mask_irq, - .irq_unmask = pci_msi_unmask_irq, + .name = "MPIC MSI", + .irq_mask = pci_msi_mask_irq, + .irq_unmask = pci_msi_unmask_irq, }; static struct msi_domain_info armada_370_xp_msi_domain_info = { @@ -386,6 +386,7 @@ static struct irq_domain *ipi_domain; static void armada_370_xp_ipi_mask(struct irq_data *d) { u32 reg; + reg = readl(per_cpu_int_base + MPIC_IN_DRBEL_MASK); reg &= ~BIT(d->hwirq); writel(reg, per_cpu_int_base + MPIC_IN_DRBEL_MASK); @@ -394,6 +395,7 @@ static void armada_370_xp_ipi_mask(struct irq_data *d) static void armada_370_xp_ipi_unmask(struct irq_data *d) { u32 reg; + reg = readl(per_cpu_int_base + MPIC_IN_DRBEL_MASK); reg |= BIT(d->hwirq); writel(reg, per_cpu_int_base + MPIC_IN_DRBEL_MASK); @@ -432,24 +434,20 @@ static struct irq_chip ipi_irqchip = { .ipi_send_mask = armada_370_xp_ipi_send_mask, }; -static int armada_370_xp_ipi_alloc(struct irq_domain *d, - unsigned int virq, - unsigned int nr_irqs, void *args) +static int armada_370_xp_ipi_alloc(struct irq_domain *d, unsigned int virq, + unsigned int nr_irqs, void *args) { for (int i = 0; i < nr_irqs; i++) { irq_set_percpu_devid(virq + i); - irq_domain_set_info(d, virq + i, i, &ipi_irqchip, - d->host_data, - handle_percpu_devid_irq, - NULL, NULL); + irq_domain_set_info(d, virq + i, i, &ipi_irqchip, d->host_data, + handle_percpu_devid_irq, NULL, NULL); } return 0; } -static void armada_370_xp_ipi_free(struct irq_domain *d, - unsigned int virq, - unsigned int nr_irqs) +static void armada_370_xp_ipi_free(struct irq_domain *d, unsigned int virq, + unsigned int nr_irqs) { /* Not freeing IPIs */ } @@ -477,8 +475,7 @@ static __init void armada_xp_ipi_init(struct device_node *node) { int base_ipi; - ipi_domain = irq_domain_create_linear(of_node_to_fwnode(node), - IPI_DOORBELL_END, + ipi_domain = irq_domain_create_linear(of_node_to_fwnode(node), IPI_DOORBELL_END, &ipi_domain_ops, NULL); if (WARN_ON(!ipi_domain)) return; @@ -562,6 +559,7 @@ static int armada_xp_mpic_starting_cpu(unsigned int cpu) armada_xp_mpic_perf_init(); armada_xp_mpic_smp_cpu_init(); armada_xp_mpic_reenable_percpu(); + return 0; } @@ -570,6 +568,7 @@ static int mpic_cascaded_starting_cpu(unsigned int cpu) armada_xp_mpic_perf_init(); armada_xp_mpic_reenable_percpu(); enable_percpu_irq(parent_irq, IRQ_TYPE_NONE); + return 0; } #else @@ -579,9 +578,9 @@ static void ipi_resume(void) {} static struct irq_chip armada_370_xp_irq_chip = { .name = "MPIC", - .irq_mask = armada_370_xp_irq_mask, - .irq_mask_ack = armada_370_xp_irq_mask, - .irq_unmask = armada_370_xp_irq_unmask, + .irq_mask = armada_370_xp_irq_mask, + .irq_mask_ack = armada_370_xp_irq_mask, + .irq_unmask = armada_370_xp_irq_unmask, #ifdef CONFIG_SMP .irq_set_affinity = armada_xp_set_affinity, #endif @@ -605,10 +604,9 @@ static int armada_370_xp_mpic_irq_map(struct irq_domain *h, if (is_percpu_irq(hw)) { irq_set_percpu_devid(virq); irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip, - handle_percpu_devid_irq); + handle_percpu_devid_irq); } else { - irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip, - handle_level_irq); + irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip, handle_level_irq); irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(virq))); } irq_set_probe(virq); @@ -617,8 +615,8 @@ static int armada_370_xp_mpic_irq_map(struct irq_domain *h, } static const struct irq_domain_ops armada_370_xp_mpic_irq_ops = { - .map = armada_370_xp_mpic_irq_map, - .xlate = irq_domain_xlate_onecell, + .map = armada_370_xp_mpic_irq_map, + .xlate = irq_domain_xlate_onecell, }; #ifdef CONFIG_PCI_MSI @@ -705,21 +703,20 @@ armada_370_xp_handle_irq(struct pt_regs *regs) unsigned long ipimask; int ipi; - ipimask = readl_relaxed(per_cpu_int_base + - MPIC_IN_DRBEL_CAUSE) - & IPI_DOORBELL_MASK; + ipimask = readl_relaxed(per_cpu_int_base + MPIC_IN_DRBEL_CAUSE) & + IPI_DOORBELL_MASK; for_each_set_bit(ipi, &ipimask, IPI_DOORBELL_END) generic_handle_domain_irq(ipi_domain, ipi); } #endif - } while (1); } static int armada_370_xp_mpic_suspend(void) { doorbell_mask_reg = readl(per_cpu_int_base + MPIC_IN_DRBEL_MASK); + return 0; } @@ -815,9 +812,8 @@ static int __init armada_370_xp_mpic_of_init(struct device_node *node, for (int i = 0; i < nr_irqs; i++) writel(i, main_int_base + MPIC_INT_CLEAR_ENABLE); - armada_370_xp_mpic_domain = - irq_domain_add_linear(node, nr_irqs, - &armada_370_xp_mpic_irq_ops, NULL); + armada_370_xp_mpic_domain = irq_domain_add_linear(node, nr_irqs, + &armada_370_xp_mpic_irq_ops, NULL); BUG_ON(!armada_370_xp_mpic_domain); irq_domain_update_bus_token(armada_370_xp_mpic_domain, DOMAIN_BUS_WIRED); From f63f54a2b8ff0815788d0c73cbbd5a96a3d467eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Thu, 11 Jul 2024 13:57:44 +0200 Subject: [PATCH 040/180] irqchip/armada-370-xp: Change symbol prefixes to mpic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change symbol prefixes from armada_370_xp_ or others to mpic_. The rationale is that it is shorter and more generic (this controller is called MPIC and is also used on Armada 38x and 39x). Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Reviewed-by: Andrew Lunn Reviewed-by: Ilpo Järvinen Link: https://lore.kernel.org/all/20240711115748.30268-7-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 305 +++++++++++++--------------- 1 file changed, 142 insertions(+), 163 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index a66d3459f7fa..27588347189e 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -150,18 +150,18 @@ static void __iomem *per_cpu_int_base; static void __iomem *main_int_base; -static struct irq_domain *armada_370_xp_mpic_domain; +static struct irq_domain *mpic_domain; static u32 doorbell_mask_reg; static int parent_irq; #ifdef CONFIG_PCI_MSI -static struct irq_domain *armada_370_xp_msi_domain; -static struct irq_domain *armada_370_xp_msi_inner_domain; +static struct irq_domain *mpic_msi_domain; +static struct irq_domain *mpic_msi_inner_domain; static DECLARE_BITMAP(msi_used, PCI_MSI_FULL_DOORBELL_NR); static DEFINE_MUTEX(msi_used_lock); static phys_addr_t msi_doorbell_addr; #endif -static inline bool is_ipi_available(void) +static inline bool mpic_is_ipi_available(void) { /* * We distinguish IPI availability in the IC by the IC not having a @@ -174,29 +174,25 @@ static inline bool is_ipi_available(void) static inline u32 msi_doorbell_mask(void) { - return is_ipi_available() ? PCI_MSI_DOORBELL_MASK : - PCI_MSI_FULL_DOORBELL_MASK; + return mpic_is_ipi_available() ? PCI_MSI_DOORBELL_MASK : PCI_MSI_FULL_DOORBELL_MASK; } static inline unsigned int msi_doorbell_start(void) { - return is_ipi_available() ? PCI_MSI_DOORBELL_START : - PCI_MSI_FULL_DOORBELL_START; + return mpic_is_ipi_available() ? PCI_MSI_DOORBELL_START : PCI_MSI_FULL_DOORBELL_START; } static inline unsigned int msi_doorbell_size(void) { - return is_ipi_available() ? PCI_MSI_DOORBELL_NR : - PCI_MSI_FULL_DOORBELL_NR; + return mpic_is_ipi_available() ? PCI_MSI_DOORBELL_NR : PCI_MSI_FULL_DOORBELL_NR; } static inline unsigned int msi_doorbell_end(void) { - return is_ipi_available() ? PCI_MSI_DOORBELL_END : - PCI_MSI_FULL_DOORBELL_END; + return mpic_is_ipi_available() ? PCI_MSI_DOORBELL_END : PCI_MSI_FULL_DOORBELL_END; } -static inline bool is_percpu_irq(irq_hw_number_t irq) +static inline bool mpic_is_percpu_irq(irq_hw_number_t irq) { return irq <= MPIC_MAX_PER_CPU_IRQS; } @@ -206,21 +202,21 @@ static inline bool is_percpu_irq(irq_hw_number_t irq) * For shared global interrupts, mask/unmask global enable bit * For CPU interrupts, mask/unmask the calling CPU's bit */ -static void armada_370_xp_irq_mask(struct irq_data *d) +static void mpic_irq_mask(struct irq_data *d) { irq_hw_number_t hwirq = irqd_to_hwirq(d); - if (!is_percpu_irq(hwirq)) + if (!mpic_is_percpu_irq(hwirq)) writel(hwirq, main_int_base + MPIC_INT_CLEAR_ENABLE); else writel(hwirq, per_cpu_int_base + MPIC_INT_SET_MASK); } -static void armada_370_xp_irq_unmask(struct irq_data *d) +static void mpic_irq_unmask(struct irq_data *d) { irq_hw_number_t hwirq = irqd_to_hwirq(d); - if (!is_percpu_irq(hwirq)) + if (!mpic_is_percpu_irq(hwirq)) writel(hwirq, main_int_base + MPIC_INT_SET_ENABLE); else writel(hwirq, per_cpu_int_base + MPIC_INT_CLEAR_MASK); @@ -228,19 +224,19 @@ static void armada_370_xp_irq_unmask(struct irq_data *d) #ifdef CONFIG_PCI_MSI -static struct irq_chip armada_370_xp_msi_irq_chip = { +static struct irq_chip mpic_msi_irq_chip = { .name = "MPIC MSI", .irq_mask = pci_msi_mask_irq, .irq_unmask = pci_msi_unmask_irq, }; -static struct msi_domain_info armada_370_xp_msi_domain_info = { +static struct msi_domain_info mpic_msi_domain_info = { .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX), - .chip = &armada_370_xp_msi_irq_chip, + .chip = &mpic_msi_irq_chip, }; -static void armada_370_xp_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) +static void mpic_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) { unsigned int cpu = cpumask_first(irq_data_get_effective_affinity_mask(data)); @@ -249,8 +245,7 @@ static void armada_370_xp_compose_msi_msg(struct irq_data *data, struct msi_msg msg->data = BIT(cpu + 8) | (data->hwirq + msi_doorbell_start()); } -static int armada_370_xp_msi_set_affinity(struct irq_data *irq_data, - const struct cpumask *mask, bool force) +static int mpic_msi_set_affinity(struct irq_data *irq_data, const struct cpumask *mask, bool force) { unsigned int cpu; @@ -267,14 +262,14 @@ static int armada_370_xp_msi_set_affinity(struct irq_data *irq_data, return IRQ_SET_MASK_OK; } -static struct irq_chip armada_370_xp_msi_bottom_irq_chip = { +static struct irq_chip mpic_msi_bottom_irq_chip = { .name = "MPIC MSI", - .irq_compose_msi_msg = armada_370_xp_compose_msi_msg, - .irq_set_affinity = armada_370_xp_msi_set_affinity, + .irq_compose_msi_msg = mpic_compose_msi_msg, + .irq_set_affinity = mpic_msi_set_affinity, }; -static int armada_370_xp_msi_alloc(struct irq_domain *domain, unsigned int virq, - unsigned int nr_irqs, void *args) +static int mpic_msi_alloc(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs, + void *args) { int hwirq; @@ -288,7 +283,7 @@ static int armada_370_xp_msi_alloc(struct irq_domain *domain, unsigned int virq, for (int i = 0; i < nr_irqs; i++) { irq_domain_set_info(domain, virq + i, hwirq + i, - &armada_370_xp_msi_bottom_irq_chip, + &mpic_msi_bottom_irq_chip, domain->host_data, handle_simple_irq, NULL, NULL); } @@ -296,8 +291,7 @@ static int armada_370_xp_msi_alloc(struct irq_domain *domain, unsigned int virq, return 0; } -static void armada_370_xp_msi_free(struct irq_domain *domain, - unsigned int virq, unsigned int nr_irqs) +static void mpic_msi_free(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs) { struct irq_data *d = irq_domain_get_irq_data(domain, virq); @@ -306,12 +300,12 @@ static void armada_370_xp_msi_free(struct irq_domain *domain, mutex_unlock(&msi_used_lock); } -static const struct irq_domain_ops armada_370_xp_msi_domain_ops = { - .alloc = armada_370_xp_msi_alloc, - .free = armada_370_xp_msi_free, +static const struct irq_domain_ops mpic_msi_domain_ops = { + .alloc = mpic_msi_alloc, + .free = mpic_msi_free, }; -static void armada_370_xp_msi_reenable_percpu(void) +static void mpic_msi_reenable_percpu(void) { u32 reg; @@ -324,45 +318,41 @@ static void armada_370_xp_msi_reenable_percpu(void) writel(1, per_cpu_int_base + MPIC_INT_CLEAR_MASK); } -static int armada_370_xp_msi_init(struct device_node *node, - phys_addr_t main_int_phys_base) +static int mpic_msi_init(struct device_node *node, phys_addr_t main_int_phys_base) { msi_doorbell_addr = main_int_phys_base + MPIC_SW_TRIG_INT; - armada_370_xp_msi_inner_domain = - irq_domain_add_linear(NULL, msi_doorbell_size(), - &armada_370_xp_msi_domain_ops, NULL); - if (!armada_370_xp_msi_inner_domain) + mpic_msi_inner_domain = irq_domain_add_linear(NULL, msi_doorbell_size(), + &mpic_msi_domain_ops, NULL); + if (!mpic_msi_inner_domain) return -ENOMEM; - armada_370_xp_msi_domain = - pci_msi_create_irq_domain(of_node_to_fwnode(node), - &armada_370_xp_msi_domain_info, - armada_370_xp_msi_inner_domain); - if (!armada_370_xp_msi_domain) { - irq_domain_remove(armada_370_xp_msi_inner_domain); + mpic_msi_domain = pci_msi_create_irq_domain(of_node_to_fwnode(node), &mpic_msi_domain_info, + mpic_msi_inner_domain); + if (!mpic_msi_domain) { + irq_domain_remove(mpic_msi_inner_domain); return -ENOMEM; } - armada_370_xp_msi_reenable_percpu(); + mpic_msi_reenable_percpu(); /* Unmask low 16 MSI irqs on non-IPI platforms */ - if (!is_ipi_available()) + if (!mpic_is_ipi_available()) writel(0, per_cpu_int_base + MPIC_INT_CLEAR_MASK); return 0; } #else -static __maybe_unused void armada_370_xp_msi_reenable_percpu(void) {} +static __maybe_unused void mpic_msi_reenable_percpu(void) {} -static inline int armada_370_xp_msi_init(struct device_node *node, - phys_addr_t main_int_phys_base) +static inline int mpic_msi_init(struct device_node *node, + phys_addr_t main_int_phys_base) { return 0; } #endif -static void armada_xp_mpic_perf_init(void) +static void mpic_perf_init(void) { unsigned long cpuid; @@ -381,9 +371,9 @@ static void armada_xp_mpic_perf_init(void) } #ifdef CONFIG_SMP -static struct irq_domain *ipi_domain; +static struct irq_domain *mpic_ipi_domain; -static void armada_370_xp_ipi_mask(struct irq_data *d) +static void mpic_ipi_mask(struct irq_data *d) { u32 reg; @@ -392,7 +382,7 @@ static void armada_370_xp_ipi_mask(struct irq_data *d) writel(reg, per_cpu_int_base + MPIC_IN_DRBEL_MASK); } -static void armada_370_xp_ipi_unmask(struct irq_data *d) +static void mpic_ipi_unmask(struct irq_data *d) { u32 reg; @@ -401,8 +391,7 @@ static void armada_370_xp_ipi_unmask(struct irq_data *d) writel(reg, per_cpu_int_base + MPIC_IN_DRBEL_MASK); } -static void armada_370_xp_ipi_send_mask(struct irq_data *d, - const struct cpumask *mask) +static void mpic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) { unsigned long map = 0; unsigned int cpu; @@ -421,75 +410,73 @@ static void armada_370_xp_ipi_send_mask(struct irq_data *d, writel((map << 8) | d->hwirq, main_int_base + MPIC_SW_TRIG_INT); } -static void armada_370_xp_ipi_ack(struct irq_data *d) +static void mpic_ipi_ack(struct irq_data *d) { writel(~BIT(d->hwirq), per_cpu_int_base + MPIC_IN_DRBEL_CAUSE); } -static struct irq_chip ipi_irqchip = { +static struct irq_chip mpic_ipi_irqchip = { .name = "IPI", - .irq_ack = armada_370_xp_ipi_ack, - .irq_mask = armada_370_xp_ipi_mask, - .irq_unmask = armada_370_xp_ipi_unmask, - .ipi_send_mask = armada_370_xp_ipi_send_mask, + .irq_ack = mpic_ipi_ack, + .irq_mask = mpic_ipi_mask, + .irq_unmask = mpic_ipi_unmask, + .ipi_send_mask = mpic_ipi_send_mask, }; -static int armada_370_xp_ipi_alloc(struct irq_domain *d, unsigned int virq, - unsigned int nr_irqs, void *args) +static int mpic_ipi_alloc(struct irq_domain *d, unsigned int virq, + unsigned int nr_irqs, void *args) { for (int i = 0; i < nr_irqs; i++) { irq_set_percpu_devid(virq + i); - irq_domain_set_info(d, virq + i, i, &ipi_irqchip, d->host_data, + irq_domain_set_info(d, virq + i, i, &mpic_ipi_irqchip, d->host_data, handle_percpu_devid_irq, NULL, NULL); } return 0; } -static void armada_370_xp_ipi_free(struct irq_domain *d, unsigned int virq, - unsigned int nr_irqs) +static void mpic_ipi_free(struct irq_domain *d, unsigned int virq, + unsigned int nr_irqs) { /* Not freeing IPIs */ } -static const struct irq_domain_ops ipi_domain_ops = { - .alloc = armada_370_xp_ipi_alloc, - .free = armada_370_xp_ipi_free, +static const struct irq_domain_ops mpic_ipi_domain_ops = { + .alloc = mpic_ipi_alloc, + .free = mpic_ipi_free, }; -static void ipi_resume(void) +static void mpic_ipi_resume(void) { for (int i = 0; i < IPI_DOORBELL_END; i++) { - unsigned int virq = irq_find_mapping(ipi_domain, i); + unsigned int virq = irq_find_mapping(mpic_ipi_domain, i); struct irq_data *d; if (!virq || !irq_percpu_is_enabled(virq)) continue; - d = irq_domain_get_irq_data(ipi_domain, virq); - armada_370_xp_ipi_unmask(d); + d = irq_domain_get_irq_data(mpic_ipi_domain, virq); + mpic_ipi_unmask(d); } } -static __init void armada_xp_ipi_init(struct device_node *node) +static __init void mpic_ipi_init(struct device_node *node) { int base_ipi; - ipi_domain = irq_domain_create_linear(of_node_to_fwnode(node), IPI_DOORBELL_END, - &ipi_domain_ops, NULL); - if (WARN_ON(!ipi_domain)) + mpic_ipi_domain = irq_domain_create_linear(of_node_to_fwnode(node), IPI_DOORBELL_END, + &mpic_ipi_domain_ops, NULL); + if (WARN_ON(!mpic_ipi_domain)) return; - irq_domain_update_bus_token(ipi_domain, DOMAIN_BUS_IPI); - base_ipi = irq_domain_alloc_irqs(ipi_domain, IPI_DOORBELL_END, NUMA_NO_NODE, NULL); + irq_domain_update_bus_token(mpic_ipi_domain, DOMAIN_BUS_IPI); + base_ipi = irq_domain_alloc_irqs(mpic_ipi_domain, IPI_DOORBELL_END, NUMA_NO_NODE, NULL); if (WARN_ON(!base_ipi)) return; - set_smp_ipi_range(base_ipi, IPI_DOORBELL_END); } -static int armada_xp_set_affinity(struct irq_data *d, - const struct cpumask *mask_val, bool force) +static int mpic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, bool force) { irq_hw_number_t hwirq = irqd_to_hwirq(d); unsigned int cpu; @@ -505,7 +492,7 @@ static int armada_xp_set_affinity(struct irq_data *d, return IRQ_SET_MASK_OK; } -static void armada_xp_mpic_smp_cpu_init(void) +static void mpic_smp_cpu_init(void) { u32 control; int nr_irqs; @@ -516,7 +503,7 @@ static void armada_xp_mpic_smp_cpu_init(void) for (int i = 0; i < nr_irqs; i++) writel(i, per_cpu_int_base + MPIC_INT_SET_MASK); - if (!is_ipi_available()) + if (!mpic_is_ipi_available()) return; /* Disable all IPIs */ @@ -529,14 +516,14 @@ static void armada_xp_mpic_smp_cpu_init(void) writel(0, per_cpu_int_base + MPIC_INT_CLEAR_MASK); } -static void armada_xp_mpic_reenable_percpu(void) +static void mpic_reenable_percpu(void) { /* Re-enable per-CPU interrupts that were enabled before suspend */ for (unsigned int irq = 0; irq < MPIC_MAX_PER_CPU_IRQS; irq++) { struct irq_data *data; unsigned int virq; - virq = irq_linear_revmap(armada_370_xp_mpic_domain, irq); + virq = irq_linear_revmap(mpic_domain, irq); if (!virq) continue; @@ -545,82 +532,80 @@ static void armada_xp_mpic_reenable_percpu(void) if (!irq_percpu_is_enabled(virq)) continue; - armada_370_xp_irq_unmask(data); + mpic_irq_unmask(data); } - if (is_ipi_available()) - ipi_resume(); + if (mpic_is_ipi_available()) + mpic_ipi_resume(); - armada_370_xp_msi_reenable_percpu(); + mpic_msi_reenable_percpu(); } -static int armada_xp_mpic_starting_cpu(unsigned int cpu) +static int mpic_starting_cpu(unsigned int cpu) { - armada_xp_mpic_perf_init(); - armada_xp_mpic_smp_cpu_init(); - armada_xp_mpic_reenable_percpu(); + mpic_perf_init(); + mpic_smp_cpu_init(); + mpic_reenable_percpu(); return 0; } static int mpic_cascaded_starting_cpu(unsigned int cpu) { - armada_xp_mpic_perf_init(); - armada_xp_mpic_reenable_percpu(); + mpic_perf_init(); + mpic_reenable_percpu(); enable_percpu_irq(parent_irq, IRQ_TYPE_NONE); return 0; } #else -static void armada_xp_mpic_smp_cpu_init(void) {} -static void ipi_resume(void) {} +static void mpic_smp_cpu_init(void) {} +static void mpic_ipi_resume(void) {} #endif -static struct irq_chip armada_370_xp_irq_chip = { +static struct irq_chip mpic_irq_chip = { .name = "MPIC", - .irq_mask = armada_370_xp_irq_mask, - .irq_mask_ack = armada_370_xp_irq_mask, - .irq_unmask = armada_370_xp_irq_unmask, + .irq_mask = mpic_irq_mask, + .irq_mask_ack = mpic_irq_mask, + .irq_unmask = mpic_irq_unmask, #ifdef CONFIG_SMP - .irq_set_affinity = armada_xp_set_affinity, + .irq_set_affinity = mpic_set_affinity, #endif .flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND, }; -static int armada_370_xp_mpic_irq_map(struct irq_domain *h, - unsigned int virq, irq_hw_number_t hw) +static int mpic_irq_map(struct irq_domain *h, unsigned int virq, + irq_hw_number_t hw) { /* IRQs 0 and 1 cannot be mapped, they are handled internally */ if (hw <= 1) return -EINVAL; - armada_370_xp_irq_mask(irq_get_irq_data(virq)); - if (!is_percpu_irq(hw)) + mpic_irq_mask(irq_get_irq_data(virq)); + if (!mpic_is_percpu_irq(hw)) writel(hw, per_cpu_int_base + MPIC_INT_CLEAR_MASK); else writel(hw, main_int_base + MPIC_INT_SET_ENABLE); irq_set_status_flags(virq, IRQ_LEVEL); - if (is_percpu_irq(hw)) { + if (mpic_is_percpu_irq(hw)) { irq_set_percpu_devid(virq); - irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip, - handle_percpu_devid_irq); + irq_set_chip_and_handler(virq, &mpic_irq_chip, handle_percpu_devid_irq); } else { - irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip, handle_level_irq); + irq_set_chip_and_handler(virq, &mpic_irq_chip, handle_level_irq); irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(virq))); } irq_set_probe(virq); - return 0; } -static const struct irq_domain_ops armada_370_xp_mpic_irq_ops = { - .map = armada_370_xp_mpic_irq_map, +static const struct irq_domain_ops mpic_irq_ops = { + .map = mpic_irq_map, .xlate = irq_domain_xlate_onecell, }; #ifdef CONFIG_PCI_MSI -static void armada_370_xp_handle_msi_irq(struct pt_regs *regs, bool is_chained) +static void mpic_handle_msi_irq(struct pt_regs *regs, bool is_chained) { u32 msimask, msinr; @@ -638,14 +623,14 @@ static void armada_370_xp_handle_msi_irq(struct pt_regs *regs, bool is_chained) irq = msinr - msi_doorbell_start(); - generic_handle_domain_irq(armada_370_xp_msi_inner_domain, irq); + generic_handle_domain_irq(mpic_msi_inner_domain, irq); } } #else -static void armada_370_xp_handle_msi_irq(struct pt_regs *r, bool b) {} +static void mpic_handle_msi_irq(struct pt_regs *r, bool b) {} #endif -static void armada_370_xp_mpic_handle_cascade_irq(struct irq_desc *desc) +static void mpic_handle_cascade_irq(struct irq_desc *desc) { struct irq_chip *chip = irq_desc_get_chip(desc); unsigned long irqmap, irqn, irqsrc, cpuid; @@ -665,18 +650,17 @@ static void armada_370_xp_mpic_handle_cascade_irq(struct irq_desc *desc) continue; if (irqn == 0 || irqn == 1) { - armada_370_xp_handle_msi_irq(NULL, true); + mpic_handle_msi_irq(NULL, true); continue; } - generic_handle_domain_irq(armada_370_xp_mpic_domain, irqn); + generic_handle_domain_irq(mpic_domain, irqn); } chained_irq_exit(chip, desc); } -static void __exception_irq_entry -armada_370_xp_handle_irq(struct pt_regs *regs) +static void __exception_irq_entry mpic_handle_irq(struct pt_regs *regs) { u32 irqstat, irqnr; @@ -688,14 +672,13 @@ armada_370_xp_handle_irq(struct pt_regs *regs) break; if (irqnr > 1) { - generic_handle_domain_irq(armada_370_xp_mpic_domain, - irqnr); + generic_handle_domain_irq(mpic_domain, irqnr); continue; } /* MSI handling */ if (irqnr == 1) - armada_370_xp_handle_msi_irq(regs, false); + mpic_handle_msi_irq(regs, false); #ifdef CONFIG_SMP /* IPI Handling */ @@ -704,62 +687,60 @@ armada_370_xp_handle_irq(struct pt_regs *regs) int ipi; ipimask = readl_relaxed(per_cpu_int_base + MPIC_IN_DRBEL_CAUSE) & - IPI_DOORBELL_MASK; + IPI_DOORBELL_MASK; for_each_set_bit(ipi, &ipimask, IPI_DOORBELL_END) - generic_handle_domain_irq(ipi_domain, ipi); + generic_handle_domain_irq(mpic_ipi_domain, ipi); } #endif } while (1); } -static int armada_370_xp_mpic_suspend(void) +static int mpic_suspend(void) { doorbell_mask_reg = readl(per_cpu_int_base + MPIC_IN_DRBEL_MASK); return 0; } -static void armada_370_xp_mpic_resume(void) +static void mpic_resume(void) { bool src0, src1; int nirqs; - /* Re-enable interrupts */ nirqs = (readl(main_int_base + MPIC_INT_CONTROL) >> 2) & 0x3ff; for (irq_hw_number_t irq = 0; irq < nirqs; irq++) { struct irq_data *data; unsigned int virq; - virq = irq_linear_revmap(armada_370_xp_mpic_domain, irq); + virq = irq_linear_revmap(mpic_domain, irq); if (!virq) continue; data = irq_get_irq_data(virq); - if (!is_percpu_irq(irq)) { + if (!mpic_is_percpu_irq(irq)) { /* Non per-CPU interrupts */ writel(irq, per_cpu_int_base + MPIC_INT_CLEAR_MASK); if (!irqd_irq_disabled(data)) - armada_370_xp_irq_unmask(data); + mpic_irq_unmask(data); } else { /* Per-CPU interrupts */ writel(irq, main_int_base + MPIC_INT_SET_ENABLE); /* - * Re-enable on the current CPU, - * armada_xp_mpic_reenable_percpu() will take - * care of secondary CPUs when they come up. + * Re-enable on the current CPU, mpic_reenable_percpu() + * will take care of secondary CPUs when they come up. */ if (irq_percpu_is_enabled(virq)) - armada_370_xp_irq_unmask(data); + mpic_irq_unmask(data); } } /* Reconfigure doorbells for IPIs and MSIs */ writel(doorbell_mask_reg, per_cpu_int_base + MPIC_IN_DRBEL_MASK); - if (is_ipi_available()) { + if (mpic_is_ipi_available()) { src0 = doorbell_mask_reg & IPI_DOORBELL_MASK; src1 = doorbell_mask_reg & PCI_MSI_DOORBELL_MASK; } else { @@ -772,17 +753,17 @@ static void armada_370_xp_mpic_resume(void) if (src1) writel(1, per_cpu_int_base + MPIC_INT_CLEAR_MASK); - if (is_ipi_available()) - ipi_resume(); + if (mpic_is_ipi_available()) + mpic_ipi_resume(); } -static struct syscore_ops armada_370_xp_mpic_syscore_ops = { - .suspend = armada_370_xp_mpic_suspend, - .resume = armada_370_xp_mpic_resume, +static struct syscore_ops mpic_syscore_ops = { + .suspend = mpic_suspend, + .resume = mpic_resume, }; -static int __init armada_370_xp_mpic_of_init(struct device_node *node, - struct device_node *parent) +static int __init mpic_of_init(struct device_node *node, + struct device_node *parent) { struct resource main_int_res, per_cpu_int_res; int nr_irqs; @@ -812,10 +793,9 @@ static int __init armada_370_xp_mpic_of_init(struct device_node *node, for (int i = 0; i < nr_irqs; i++) writel(i, main_int_base + MPIC_INT_CLEAR_ENABLE); - armada_370_xp_mpic_domain = irq_domain_add_linear(node, nr_irqs, - &armada_370_xp_mpic_irq_ops, NULL); - BUG_ON(!armada_370_xp_mpic_domain); - irq_domain_update_bus_token(armada_370_xp_mpic_domain, DOMAIN_BUS_WIRED); + mpic_domain = irq_domain_add_linear(node, nr_irqs, &mpic_irq_ops, NULL); + BUG_ON(!mpic_domain); + irq_domain_update_bus_token(mpic_domain, DOMAIN_BUS_WIRED); /* * Initialize parent_irq before calling any other functions, since it is @@ -824,19 +804,19 @@ static int __init armada_370_xp_mpic_of_init(struct device_node *node, parent_irq = irq_of_parse_and_map(node, 0); /* Setup for the boot CPU */ - armada_xp_mpic_perf_init(); - armada_xp_mpic_smp_cpu_init(); + mpic_perf_init(); + mpic_smp_cpu_init(); - armada_370_xp_msi_init(node, main_int_res.start); + mpic_msi_init(node, main_int_res.start); if (parent_irq <= 0) { - irq_set_default_host(armada_370_xp_mpic_domain); - set_handle_irq(armada_370_xp_handle_irq); + irq_set_default_host(mpic_domain); + set_handle_irq(mpic_handle_irq); #ifdef CONFIG_SMP - armada_xp_ipi_init(node); + mpic_ipi_init(node); cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_ARMADA_XP_STARTING, "irqchip/armada/ipi:starting", - armada_xp_mpic_starting_cpu, NULL); + mpic_starting_cpu, NULL); #endif } else { #ifdef CONFIG_SMP @@ -844,13 +824,12 @@ static int __init armada_370_xp_mpic_of_init(struct device_node *node, "irqchip/armada/cascade:starting", mpic_cascaded_starting_cpu, NULL); #endif - irq_set_chained_handler(parent_irq, - armada_370_xp_mpic_handle_cascade_irq); + irq_set_chained_handler(parent_irq, mpic_handle_cascade_irq); } - register_syscore_ops(&armada_370_xp_mpic_syscore_ops); + register_syscore_ops(&mpic_syscore_ops); return 0; } -IRQCHIP_DECLARE(armada_370_xp_mpic, "marvell,mpic", armada_370_xp_mpic_of_init); +IRQCHIP_DECLARE(marvell_mpic, "marvell,mpic", mpic_of_init); From 5ecafc9a640f7c1e5690375cf3a82848d669abb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Thu, 11 Jul 2024 13:57:45 +0200 Subject: [PATCH 041/180] irqchip/armada-370-xp: Don't read number of supported interrupts multiple times MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use mpic_domain::hwirq_max at runtime instead of reading the same value over and over from the MPIC_INT_CONTROL register. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Reviewed-by: Andrew Lunn Reviewed-by: Ilpo Järvinen Link: https://lore.kernel.org/all/20240711115748.30268-8-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 27588347189e..e43eb26ab6e7 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -494,13 +494,7 @@ static int mpic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, static void mpic_smp_cpu_init(void) { - u32 control; - int nr_irqs; - - control = readl(main_int_base + MPIC_INT_CONTROL); - nr_irqs = (control >> 2) & 0x3ff; - - for (int i = 0; i < nr_irqs; i++) + for (int i = 0; i < mpic_domain->hwirq_max; i++) writel(i, per_cpu_int_base + MPIC_INT_SET_MASK); if (!mpic_is_ipi_available()) @@ -706,10 +700,9 @@ static int mpic_suspend(void) static void mpic_resume(void) { bool src0, src1; - int nirqs; + /* Re-enable interrupts */ - nirqs = (readl(main_int_base + MPIC_INT_CONTROL) >> 2) & 0x3ff; - for (irq_hw_number_t irq = 0; irq < nirqs; irq++) { + for (irq_hw_number_t irq = 0; irq < mpic_domain->hwirq_max; irq++) { struct irq_data *data; unsigned int virq; From 92128c74e41868e42e6944f83d9d2130c9aa8a22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Thu, 11 Jul 2024 13:57:46 +0200 Subject: [PATCH 042/180] irqchip/armada-370-xp: Use FIELD_GET() and named register constant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use FIELD_GET() and named register mask constant when reading the number of supported interrupts / current interrupt. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Reviewed-by: Andrew Lunn Reviewed-by: Ilpo Järvinen Link: https://lore.kernel.org/all/20240711115748.30268-9-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index e43eb26ab6e7..179a30a4a128 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -10,6 +10,7 @@ * Ben Dooks */ +#include #include #include #include @@ -112,6 +113,7 @@ /* Registers relative to main_int_base */ #define MPIC_INT_CONTROL 0x00 +#define MPIC_INT_CONTROL_NUMINT_MASK GENMASK(12, 2) #define MPIC_SW_TRIG_INT 0x04 #define MPIC_INT_SET_ENABLE 0x30 #define MPIC_INT_CLEAR_ENABLE 0x34 @@ -124,6 +126,7 @@ #define MPIC_IN_DRBEL_MASK 0x0c #define MPIC_PPI_CAUSE 0x10 #define MPIC_CPU_INTACK 0x44 +#define MPIC_CPU_INTACK_IID_MASK GENMASK(9, 0) #define MPIC_INT_SET_MASK 0x48 #define MPIC_INT_CLEAR_MASK 0x4C #define MPIC_INT_FABRIC_MASK 0x54 @@ -660,7 +663,7 @@ static void __exception_irq_entry mpic_handle_irq(struct pt_regs *regs) do { irqstat = readl_relaxed(per_cpu_int_base + MPIC_CPU_INTACK); - irqnr = irqstat & 0x3FF; + irqnr = FIELD_GET(MPIC_CPU_INTACK_IID_MASK, irqstat); if (irqnr > 1022) break; @@ -759,8 +762,7 @@ static int __init mpic_of_init(struct device_node *node, struct device_node *parent) { struct resource main_int_res, per_cpu_int_res; - int nr_irqs; - u32 control; + unsigned int nr_irqs; BUG_ON(of_address_to_resource(node, 0, &main_int_res)); BUG_ON(of_address_to_resource(node, 1, &per_cpu_int_res)); @@ -780,8 +782,7 @@ static int __init mpic_of_init(struct device_node *node, resource_size(&per_cpu_int_res)); BUG_ON(!per_cpu_int_base); - control = readl(main_int_base + MPIC_INT_CONTROL); - nr_irqs = (control >> 2) & 0x3ff; + nr_irqs = FIELD_GET(MPIC_INT_CONTROL_NUMINT_MASK, readl(main_int_base + MPIC_INT_CONTROL)); for (int i = 0; i < nr_irqs; i++) writel(i, main_int_base + MPIC_INT_CLEAR_ENABLE); From 63697bc7199ee2bacc8324b59951046a7b3ea991 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Thu, 11 Jul 2024 13:57:47 +0200 Subject: [PATCH 043/180] irqchip/armada-370-xp: Refactor mpic_handle_msi_irq() code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactor the mpic_handle_msi_irq() function to make it simpler: - drop the function arguments, they are not needed - rename the variable holding the doorbell cause register to "cause" - rename the iterating variable to "i" - use for_each_set_bit() (requires retyping "cause" to unsigned long) Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Reviewed-by: Andrew Lunn Reviewed-by: Ilpo Järvinen Link: https://lore.kernel.org/all/20240711115748.30268-10-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 32 +++++++++++------------------ 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 179a30a4a128..5c2631f6f7b0 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -602,29 +602,21 @@ static const struct irq_domain_ops mpic_irq_ops = { }; #ifdef CONFIG_PCI_MSI -static void mpic_handle_msi_irq(struct pt_regs *regs, bool is_chained) +static void mpic_handle_msi_irq(void) { - u32 msimask, msinr; + unsigned long cause; + unsigned int i; - msimask = readl_relaxed(per_cpu_int_base + MPIC_IN_DRBEL_CAUSE); - msimask &= msi_doorbell_mask(); + cause = readl_relaxed(per_cpu_int_base + MPIC_IN_DRBEL_CAUSE); + cause &= msi_doorbell_mask(); + writel(~cause, per_cpu_int_base + MPIC_IN_DRBEL_CAUSE); - writel(~msimask, per_cpu_int_base + MPIC_IN_DRBEL_CAUSE); - - for (msinr = msi_doorbell_start(); - msinr < msi_doorbell_end(); msinr++) { - unsigned int irq; - - if (!(msimask & BIT(msinr))) - continue; - - irq = msinr - msi_doorbell_start(); - - generic_handle_domain_irq(mpic_msi_inner_domain, irq); - } + for_each_set_bit(i, &cause, BITS_PER_LONG) + generic_handle_domain_irq(mpic_msi_inner_domain, + i - msi_doorbell_start()); } #else -static void mpic_handle_msi_irq(struct pt_regs *r, bool b) {} +static void mpic_handle_msi_irq(void) {} #endif static void mpic_handle_cascade_irq(struct irq_desc *desc) @@ -647,7 +639,7 @@ static void mpic_handle_cascade_irq(struct irq_desc *desc) continue; if (irqn == 0 || irqn == 1) { - mpic_handle_msi_irq(NULL, true); + mpic_handle_msi_irq(); continue; } @@ -675,7 +667,7 @@ static void __exception_irq_entry mpic_handle_irq(struct pt_regs *regs) /* MSI handling */ if (irqnr == 1) - mpic_handle_msi_irq(regs, false); + mpic_handle_msi_irq(); #ifdef CONFIG_SMP /* IPI Handling */ From baf01c726b7f99b72f2abfa54e249d766cbd59a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Thu, 11 Jul 2024 13:57:48 +0200 Subject: [PATCH 044/180] irqchip/armada-370-xp: Refactor handling IPI interrupts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactor the handling of IPI interrupts - put into own function mpic_handle_ipi_irq(), similar to mpic_handle_msi_irq() - rename the variable holding the doorbell cause register to "cause" - retype and rename the variable holding the IPI HW IRQ number to "irq_hw_number_t i" Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Reviewed-by: Andrew Lunn Reviewed-by: Ilpo Järvinen Link: https://lore.kernel.org/all/20240711115748.30268-11-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 30 +++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 5c2631f6f7b0..d42c7a1750ac 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -619,6 +619,22 @@ static void mpic_handle_msi_irq(void) static void mpic_handle_msi_irq(void) {} #endif +#ifdef CONFIG_SMP +static void mpic_handle_ipi_irq(void) +{ + unsigned long cause; + irq_hw_number_t i; + + cause = readl_relaxed(per_cpu_int_base + MPIC_IN_DRBEL_CAUSE); + cause &= IPI_DOORBELL_MASK; + + for_each_set_bit(i, &cause, IPI_DOORBELL_END) + generic_handle_domain_irq(mpic_ipi_domain, i); +} +#else +static inline void mpic_handle_ipi_irq(void) {} +#endif + static void mpic_handle_cascade_irq(struct irq_desc *desc) { struct irq_chip *chip = irq_desc_get_chip(desc); @@ -669,19 +685,9 @@ static void __exception_irq_entry mpic_handle_irq(struct pt_regs *regs) if (irqnr == 1) mpic_handle_msi_irq(); -#ifdef CONFIG_SMP /* IPI Handling */ - if (irqnr == 0) { - unsigned long ipimask; - int ipi; - - ipimask = readl_relaxed(per_cpu_int_base + MPIC_IN_DRBEL_CAUSE) & - IPI_DOORBELL_MASK; - - for_each_set_bit(ipi, &ipimask, IPI_DOORBELL_END) - generic_handle_domain_irq(mpic_ipi_domain, ipi); - } -#endif + if (irqnr == 0) + mpic_handle_ipi_irq(); } while (1); } From 66fc31034f96cbdef7687b1c55d600367e70287e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Thu, 11 Jul 2024 18:08:58 +0200 Subject: [PATCH 045/180] irqchip/armada-370-xp: Use consistent variable names for hwirqs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use consistent variable names for hwirqs: when iterating, use "i", otherwise use "hwirq". Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240711160907.31012-2-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 56 ++++++++++++++--------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index d42c7a1750ac..8f95da0977d7 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -117,7 +117,7 @@ #define MPIC_SW_TRIG_INT 0x04 #define MPIC_INT_SET_ENABLE 0x30 #define MPIC_INT_CLEAR_ENABLE 0x34 -#define MPIC_INT_SOURCE_CTL(irq) (0x100 + (irq) * 4) +#define MPIC_INT_SOURCE_CTL(hwirq) (0x100 + (hwirq) * 4) #define MPIC_INT_SOURCE_CPU_MASK GENMASK(3, 0) #define MPIC_INT_IRQ_FIQ_MASK(cpuid) ((BIT(0) | BIT(8)) << (cpuid)) @@ -195,9 +195,9 @@ static inline unsigned int msi_doorbell_end(void) return mpic_is_ipi_available() ? PCI_MSI_DOORBELL_END : PCI_MSI_FULL_DOORBELL_END; } -static inline bool mpic_is_percpu_irq(irq_hw_number_t irq) +static inline bool mpic_is_percpu_irq(irq_hw_number_t hwirq) { - return irq <= MPIC_MAX_PER_CPU_IRQS; + return hwirq <= MPIC_MAX_PER_CPU_IRQS; } /* @@ -516,11 +516,11 @@ static void mpic_smp_cpu_init(void) static void mpic_reenable_percpu(void) { /* Re-enable per-CPU interrupts that were enabled before suspend */ - for (unsigned int irq = 0; irq < MPIC_MAX_PER_CPU_IRQS; irq++) { + for (unsigned int i = 0; i < MPIC_MAX_PER_CPU_IRQS; i++) { struct irq_data *data; unsigned int virq; - virq = irq_linear_revmap(mpic_domain, irq); + virq = irq_linear_revmap(mpic_domain, i); if (!virq) continue; @@ -572,20 +572,20 @@ static struct irq_chip mpic_irq_chip = { }; static int mpic_irq_map(struct irq_domain *h, unsigned int virq, - irq_hw_number_t hw) + irq_hw_number_t hwirq) { /* IRQs 0 and 1 cannot be mapped, they are handled internally */ - if (hw <= 1) + if (hwirq <= 1) return -EINVAL; mpic_irq_mask(irq_get_irq_data(virq)); - if (!mpic_is_percpu_irq(hw)) - writel(hw, per_cpu_int_base + MPIC_INT_CLEAR_MASK); + if (!mpic_is_percpu_irq(hwirq)) + writel(hwirq, per_cpu_int_base + MPIC_INT_CLEAR_MASK); else - writel(hw, main_int_base + MPIC_INT_SET_ENABLE); + writel(hwirq, main_int_base + MPIC_INT_SET_ENABLE); irq_set_status_flags(virq, IRQ_LEVEL); - if (mpic_is_percpu_irq(hw)) { + if (mpic_is_percpu_irq(hwirq)) { irq_set_percpu_devid(virq); irq_set_chip_and_handler(virq, &mpic_irq_chip, handle_percpu_devid_irq); } else { @@ -638,15 +638,15 @@ static inline void mpic_handle_ipi_irq(void) {} static void mpic_handle_cascade_irq(struct irq_desc *desc) { struct irq_chip *chip = irq_desc_get_chip(desc); - unsigned long irqmap, irqn, irqsrc, cpuid; + unsigned long irqmap, i, irqsrc, cpuid; chained_irq_enter(chip, desc); irqmap = readl_relaxed(per_cpu_int_base + MPIC_PPI_CAUSE); cpuid = cpu_logical_map(smp_processor_id()); - for_each_set_bit(irqn, &irqmap, BITS_PER_LONG) { - irqsrc = readl_relaxed(main_int_base + MPIC_INT_SOURCE_CTL(irqn)); + for_each_set_bit(i, &irqmap, BITS_PER_LONG) { + irqsrc = readl_relaxed(main_int_base + MPIC_INT_SOURCE_CTL(i)); /* Check if the interrupt is not masked on current CPU. * Test IRQ (0-1) and FIQ (8-9) mask bits. @@ -654,12 +654,12 @@ static void mpic_handle_cascade_irq(struct irq_desc *desc) if (!(irqsrc & MPIC_INT_IRQ_FIQ_MASK(cpuid))) continue; - if (irqn == 0 || irqn == 1) { + if (i == 0 || i == 1) { mpic_handle_msi_irq(); continue; } - generic_handle_domain_irq(mpic_domain, irqn); + generic_handle_domain_irq(mpic_domain, i); } chained_irq_exit(chip, desc); @@ -667,26 +667,26 @@ static void mpic_handle_cascade_irq(struct irq_desc *desc) static void __exception_irq_entry mpic_handle_irq(struct pt_regs *regs) { - u32 irqstat, irqnr; + u32 irqstat, i; do { irqstat = readl_relaxed(per_cpu_int_base + MPIC_CPU_INTACK); - irqnr = FIELD_GET(MPIC_CPU_INTACK_IID_MASK, irqstat); + i = FIELD_GET(MPIC_CPU_INTACK_IID_MASK, irqstat); - if (irqnr > 1022) + if (i > 1022) break; - if (irqnr > 1) { - generic_handle_domain_irq(mpic_domain, irqnr); + if (i > 1) { + generic_handle_domain_irq(mpic_domain, i); continue; } /* MSI handling */ - if (irqnr == 1) + if (i == 1) mpic_handle_msi_irq(); /* IPI Handling */ - if (irqnr == 0) + if (i == 0) mpic_handle_ipi_irq(); } while (1); } @@ -703,24 +703,24 @@ static void mpic_resume(void) bool src0, src1; /* Re-enable interrupts */ - for (irq_hw_number_t irq = 0; irq < mpic_domain->hwirq_max; irq++) { + for (irq_hw_number_t i = 0; i < mpic_domain->hwirq_max; i++) { struct irq_data *data; unsigned int virq; - virq = irq_linear_revmap(mpic_domain, irq); + virq = irq_linear_revmap(mpic_domain, i); if (!virq) continue; data = irq_get_irq_data(virq); - if (!mpic_is_percpu_irq(irq)) { + if (!mpic_is_percpu_irq(i)) { /* Non per-CPU interrupts */ - writel(irq, per_cpu_int_base + MPIC_INT_CLEAR_MASK); + writel(i, per_cpu_int_base + MPIC_INT_CLEAR_MASK); if (!irqd_irq_disabled(data)) mpic_irq_unmask(data); } else { /* Per-CPU interrupts */ - writel(irq, main_int_base + MPIC_INT_SET_ENABLE); + writel(i, main_int_base + MPIC_INT_SET_ENABLE); /* * Re-enable on the current CPU, mpic_reenable_percpu() From a5d32b7475fffe2506fa374b1d6b4a74fa13020c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Thu, 11 Jul 2024 18:08:59 +0200 Subject: [PATCH 046/180] irqchip/armada-370-xp: Use consistent types when iterating interrupts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When iterating, use either the irq_hw_number_t type or the unsigned int type for the iterator variable, depending on whether the variable represents HW IRQ number or whether it is added to a IRQ number. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240711160907.31012-3-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 8f95da0977d7..db9594b1024f 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -284,7 +284,7 @@ static int mpic_msi_alloc(struct irq_domain *domain, unsigned int virq, unsigned if (hwirq < 0) return -ENOSPC; - for (int i = 0; i < nr_irqs; i++) { + for (unsigned int i = 0; i < nr_irqs; i++) { irq_domain_set_info(domain, virq + i, hwirq + i, &mpic_msi_bottom_irq_chip, domain->host_data, handle_simple_irq, @@ -429,7 +429,7 @@ static struct irq_chip mpic_ipi_irqchip = { static int mpic_ipi_alloc(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs, void *args) { - for (int i = 0; i < nr_irqs; i++) { + for (unsigned int i = 0; i < nr_irqs; i++) { irq_set_percpu_devid(virq + i); irq_domain_set_info(d, virq + i, i, &mpic_ipi_irqchip, d->host_data, handle_percpu_devid_irq, NULL, NULL); @@ -451,7 +451,7 @@ static const struct irq_domain_ops mpic_ipi_domain_ops = { static void mpic_ipi_resume(void) { - for (int i = 0; i < IPI_DOORBELL_END; i++) { + for (irq_hw_number_t i = 0; i < IPI_DOORBELL_END; i++) { unsigned int virq = irq_find_mapping(mpic_ipi_domain, i); struct irq_data *d; @@ -497,7 +497,7 @@ static int mpic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, static void mpic_smp_cpu_init(void) { - for (int i = 0; i < mpic_domain->hwirq_max; i++) + for (irq_hw_number_t i = 0; i < mpic_domain->hwirq_max; i++) writel(i, per_cpu_int_base + MPIC_INT_SET_MASK); if (!mpic_is_ipi_available()) @@ -516,7 +516,7 @@ static void mpic_smp_cpu_init(void) static void mpic_reenable_percpu(void) { /* Re-enable per-CPU interrupts that were enabled before suspend */ - for (unsigned int i = 0; i < MPIC_MAX_PER_CPU_IRQS; i++) { + for (irq_hw_number_t i = 0; i < MPIC_MAX_PER_CPU_IRQS; i++) { struct irq_data *data; unsigned int virq; @@ -638,7 +638,8 @@ static inline void mpic_handle_ipi_irq(void) {} static void mpic_handle_cascade_irq(struct irq_desc *desc) { struct irq_chip *chip = irq_desc_get_chip(desc); - unsigned long irqmap, i, irqsrc, cpuid; + unsigned long irqmap, irqsrc, cpuid; + irq_hw_number_t i; chained_irq_enter(chip, desc); @@ -667,7 +668,8 @@ static void mpic_handle_cascade_irq(struct irq_desc *desc) static void __exception_irq_entry mpic_handle_irq(struct pt_regs *regs) { - u32 irqstat, i; + irq_hw_number_t i; + u32 irqstat; do { irqstat = readl_relaxed(per_cpu_int_base + MPIC_CPU_INTACK); @@ -782,7 +784,7 @@ static int __init mpic_of_init(struct device_node *node, nr_irqs = FIELD_GET(MPIC_INT_CONTROL_NUMINT_MASK, readl(main_int_base + MPIC_INT_CONTROL)); - for (int i = 0; i < nr_irqs; i++) + for (irq_hw_number_t i = 0; i < nr_irqs; i++) writel(i, main_int_base + MPIC_INT_CLEAR_ENABLE); mpic_domain = irq_domain_add_linear(node, nr_irqs, &mpic_irq_ops, NULL); From 0d4b1fcd378ea61ff76bedf0d484eac69c028c57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Thu, 11 Jul 2024 18:09:00 +0200 Subject: [PATCH 047/180] irqchip/armada-370-xp: Use consistent name for struct irq_data variables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Always use variable name "d" for struct irq_data *, for consistency. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240711160907.31012-4-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index db9594b1024f..98f90a3d62d8 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -239,16 +239,16 @@ static struct msi_domain_info mpic_msi_domain_info = { .chip = &mpic_msi_irq_chip, }; -static void mpic_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) +static void mpic_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) { - unsigned int cpu = cpumask_first(irq_data_get_effective_affinity_mask(data)); + unsigned int cpu = cpumask_first(irq_data_get_effective_affinity_mask(d)); msg->address_lo = lower_32_bits(msi_doorbell_addr); msg->address_hi = upper_32_bits(msi_doorbell_addr); - msg->data = BIT(cpu + 8) | (data->hwirq + msi_doorbell_start()); + msg->data = BIT(cpu + 8) | (d->hwirq + msi_doorbell_start()); } -static int mpic_msi_set_affinity(struct irq_data *irq_data, const struct cpumask *mask, bool force) +static int mpic_msi_set_affinity(struct irq_data *d, const struct cpumask *mask, bool force) { unsigned int cpu; @@ -260,7 +260,7 @@ static int mpic_msi_set_affinity(struct irq_data *irq_data, const struct cpumask if (cpu >= nr_cpu_ids) return -EINVAL; - irq_data_update_effective_affinity(irq_data, cpumask_of(cpu)); + irq_data_update_effective_affinity(d, cpumask_of(cpu)); return IRQ_SET_MASK_OK; } @@ -517,19 +517,19 @@ static void mpic_reenable_percpu(void) { /* Re-enable per-CPU interrupts that were enabled before suspend */ for (irq_hw_number_t i = 0; i < MPIC_MAX_PER_CPU_IRQS; i++) { - struct irq_data *data; + struct irq_data *d; unsigned int virq; virq = irq_linear_revmap(mpic_domain, i); if (!virq) continue; - data = irq_get_irq_data(virq); + d = irq_get_irq_data(virq); if (!irq_percpu_is_enabled(virq)) continue; - mpic_irq_unmask(data); + mpic_irq_unmask(d); } if (mpic_is_ipi_available()) @@ -706,20 +706,20 @@ static void mpic_resume(void) /* Re-enable interrupts */ for (irq_hw_number_t i = 0; i < mpic_domain->hwirq_max; i++) { - struct irq_data *data; + struct irq_data *d; unsigned int virq; virq = irq_linear_revmap(mpic_domain, i); if (!virq) continue; - data = irq_get_irq_data(virq); + d = irq_get_irq_data(virq); if (!mpic_is_percpu_irq(i)) { /* Non per-CPU interrupts */ writel(i, per_cpu_int_base + MPIC_INT_CLEAR_MASK); - if (!irqd_irq_disabled(data)) - mpic_irq_unmask(data); + if (!irqd_irq_disabled(d)) + mpic_irq_unmask(d); } else { /* Per-CPU interrupts */ writel(i, main_int_base + MPIC_INT_SET_ENABLE); @@ -729,7 +729,7 @@ static void mpic_resume(void) * will take care of secondary CPUs when they come up. */ if (irq_percpu_is_enabled(virq)) - mpic_irq_unmask(data); + mpic_irq_unmask(d); } } From 15a50eeaadc169243b00ec90087f689a8a28848e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Thu, 11 Jul 2024 18:09:01 +0200 Subject: [PATCH 048/180] irqchip/armada-370-xp: Simplify mpic_reenable_percpu() and mpic_resume() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactor the mpic_reenable_percpu() and mpic_resume() functions to make them a little bit simpler. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240711160907.31012-5-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 98f90a3d62d8..66e14f1183d1 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -517,18 +517,13 @@ static void mpic_reenable_percpu(void) { /* Re-enable per-CPU interrupts that were enabled before suspend */ for (irq_hw_number_t i = 0; i < MPIC_MAX_PER_CPU_IRQS; i++) { + unsigned int virq = irq_linear_revmap(mpic_domain, i); struct irq_data *d; - unsigned int virq; - virq = irq_linear_revmap(mpic_domain, i); - if (!virq) + if (!virq || !irq_percpu_is_enabled(virq)) continue; d = irq_get_irq_data(virq); - - if (!irq_percpu_is_enabled(virq)) - continue; - mpic_irq_unmask(d); } @@ -706,10 +701,9 @@ static void mpic_resume(void) /* Re-enable interrupts */ for (irq_hw_number_t i = 0; i < mpic_domain->hwirq_max; i++) { + unsigned int virq = irq_linear_revmap(mpic_domain, i); struct irq_data *d; - unsigned int virq; - virq = irq_linear_revmap(mpic_domain, i); if (!virq) continue; From 081b64cc872707f80a23e41f0ab12852716551b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Thu, 11 Jul 2024 18:09:03 +0200 Subject: [PATCH 049/180] irqchip/armada-370-xp: Drop redundant continue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop redundant continue from mpic_handle_irq(). Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240711160907.31012-7-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 66e14f1183d1..4abe0eac184e 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -673,10 +673,8 @@ static void __exception_irq_entry mpic_handle_irq(struct pt_regs *regs) if (i > 1022) break; - if (i > 1) { + if (i > 1) generic_handle_domain_irq(mpic_domain, i); - continue; - } /* MSI handling */ if (i == 1) From ac0ae59db6f521223b477677d2ff51e26815b114 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Thu, 11 Jul 2024 18:09:04 +0200 Subject: [PATCH 050/180] irqchip/armada-370-xp: Rename variable for consistency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename the variable holding the cause register to "cause" in mpic_handle_cascade_irq(). Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240711160907.31012-8-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 4abe0eac184e..5cde229c9e39 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -633,15 +633,15 @@ static inline void mpic_handle_ipi_irq(void) {} static void mpic_handle_cascade_irq(struct irq_desc *desc) { struct irq_chip *chip = irq_desc_get_chip(desc); - unsigned long irqmap, irqsrc, cpuid; + unsigned long cause, irqsrc, cpuid; irq_hw_number_t i; chained_irq_enter(chip, desc); - irqmap = readl_relaxed(per_cpu_int_base + MPIC_PPI_CAUSE); + cause = readl_relaxed(per_cpu_int_base + MPIC_PPI_CAUSE); cpuid = cpu_logical_map(smp_processor_id()); - for_each_set_bit(i, &irqmap, BITS_PER_LONG) { + for_each_set_bit(i, &cause, BITS_PER_LONG) { irqsrc = readl_relaxed(main_int_base + MPIC_INT_SOURCE_CTL(i)); /* Check if the interrupt is not masked on current CPU. From 625f0582f05d1f496ecd598323df1c8bfcdcbd6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Thu, 11 Jul 2024 18:09:05 +0200 Subject: [PATCH 051/180] irqchip/armada-370-xp: Use u32 type instead of unsigned long where possieble MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For consistency across the driver, use the u32 type instead of unsigned long for holding register values and return value of cpu_logical_map(). One exception is when the variable is referenced for passing into for_each_set_bit(), in which case it has to be unsigned long. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240711160907.31012-9-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 5cde229c9e39..8b28188953b0 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -357,7 +357,7 @@ static inline int mpic_msi_init(struct device_node *node, static void mpic_perf_init(void) { - unsigned long cpuid; + u32 cpuid; /* * This Performance Counter Overflow interrupt is specific for @@ -396,8 +396,8 @@ static void mpic_ipi_unmask(struct irq_data *d) static void mpic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) { - unsigned long map = 0; unsigned int cpu; + u32 map = 0; /* Convert our logical CPU mask into a physical one. */ for_each_cpu(cpu, mask) @@ -633,7 +633,8 @@ static inline void mpic_handle_ipi_irq(void) {} static void mpic_handle_cascade_irq(struct irq_desc *desc) { struct irq_chip *chip = irq_desc_get_chip(desc); - unsigned long cause, irqsrc, cpuid; + unsigned long cause; + u32 irqsrc, cpuid; irq_hw_number_t i; chained_irq_enter(chip, desc); From 654caa9db6649dbecdfa55ea29c9cbf4603fb402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Thu, 11 Jul 2024 18:09:06 +0200 Subject: [PATCH 052/180] irqchip/armada-370-xp: Refactor initial memory regions mapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactor the initial memory regions mapping: - put into its own function - return error numbers on failure - use WARN_ON() instead of BUG_ON() Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240711160907.31012-10-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 62 +++++++++++++++++++---------- 1 file changed, 42 insertions(+), 20 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 8b28188953b0..9c66c251cee5 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -751,29 +752,50 @@ static struct syscore_ops mpic_syscore_ops = { .resume = mpic_resume, }; -static int __init mpic_of_init(struct device_node *node, - struct device_node *parent) +static int __init mpic_map_region(struct device_node *np, int index, + void __iomem **base, phys_addr_t *phys_base) { - struct resource main_int_res, per_cpu_int_res; + struct resource res; + int err; + + err = of_address_to_resource(np, index, &res); + if (WARN_ON(err)) + goto fail; + + if (WARN_ON(!request_mem_region(res.start, resource_size(&res), np->full_name))) { + err = -EBUSY; + goto fail; + } + + *base = ioremap(res.start, resource_size(&res)); + if (WARN_ON(!*base)) { + err = -ENOMEM; + goto fail; + } + + if (phys_base) + *phys_base = res.start; + + return 0; + +fail: + pr_err("%pOF: Unable to map resource %d: %pE\n", np, index, ERR_PTR(err)); + return err; +} + +static int __init mpic_of_init(struct device_node *node, struct device_node *parent) +{ + phys_addr_t phys_base; unsigned int nr_irqs; + int err; - BUG_ON(of_address_to_resource(node, 0, &main_int_res)); - BUG_ON(of_address_to_resource(node, 1, &per_cpu_int_res)); + err = mpic_map_region(node, 0, &main_int_base, &phys_base); + if (err) + return err; - BUG_ON(!request_mem_region(main_int_res.start, - resource_size(&main_int_res), - node->full_name)); - BUG_ON(!request_mem_region(per_cpu_int_res.start, - resource_size(&per_cpu_int_res), - node->full_name)); - - main_int_base = ioremap(main_int_res.start, - resource_size(&main_int_res)); - BUG_ON(!main_int_base); - - per_cpu_int_base = ioremap(per_cpu_int_res.start, - resource_size(&per_cpu_int_res)); - BUG_ON(!per_cpu_int_base); + err = mpic_map_region(node, 1, &per_cpu_int_base, NULL); + if (err) + return err; nr_irqs = FIELD_GET(MPIC_INT_CONTROL_NUMINT_MASK, readl(main_int_base + MPIC_INT_CONTROL)); @@ -794,7 +816,7 @@ static int __init mpic_of_init(struct device_node *node, mpic_perf_init(); mpic_smp_cpu_init(); - mpic_msi_init(node, main_int_res.start); + mpic_msi_init(node, phys_base); if (parent_irq <= 0) { irq_set_default_host(mpic_domain); From 1d07c9a3e71c97ee1bbc1f119e104bf3746c51f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Thu, 11 Jul 2024 18:09:07 +0200 Subject: [PATCH 053/180] irqchip/armada-370-xp: Print error and return error code on initialization failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Print error and return error code on main / IPI / MSI domain initialization failure. Use WARN_ON() instead of BUG_ON(). Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240711160907.31012-11-kabel@kernel.org --- drivers/irqchip/irq-armada-370-xp.c | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 9c66c251cee5..b11612acec78 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -464,20 +464,23 @@ static void mpic_ipi_resume(void) } } -static __init void mpic_ipi_init(struct device_node *node) +static __init int mpic_ipi_init(struct device_node *node) { int base_ipi; mpic_ipi_domain = irq_domain_create_linear(of_node_to_fwnode(node), IPI_DOORBELL_END, &mpic_ipi_domain_ops, NULL); if (WARN_ON(!mpic_ipi_domain)) - return; + return -ENOMEM; irq_domain_update_bus_token(mpic_ipi_domain, DOMAIN_BUS_IPI); base_ipi = irq_domain_alloc_irqs(mpic_ipi_domain, IPI_DOORBELL_END, NUMA_NO_NODE, NULL); if (WARN_ON(!base_ipi)) - return; + return -ENOMEM; + set_smp_ipi_range(base_ipi, IPI_DOORBELL_END); + + return 0; } static int mpic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, bool force) @@ -803,7 +806,11 @@ static int __init mpic_of_init(struct device_node *node, struct device_node *par writel(i, main_int_base + MPIC_INT_CLEAR_ENABLE); mpic_domain = irq_domain_add_linear(node, nr_irqs, &mpic_irq_ops, NULL); - BUG_ON(!mpic_domain); + if (!mpic_domain) { + pr_err("%pOF: Unable to add IRQ domain\n", node); + return -ENOMEM; + } + irq_domain_update_bus_token(mpic_domain, DOMAIN_BUS_WIRED); /* @@ -816,13 +823,22 @@ static int __init mpic_of_init(struct device_node *node, struct device_node *par mpic_perf_init(); mpic_smp_cpu_init(); - mpic_msi_init(node, phys_base); + err = mpic_msi_init(node, phys_base); + if (err) { + pr_err("%pOF: Unable to initialize MSI domain\n", node); + return err; + } if (parent_irq <= 0) { irq_set_default_host(mpic_domain); set_handle_irq(mpic_handle_irq); #ifdef CONFIG_SMP - mpic_ipi_init(node); + err = mpic_ipi_init(node); + if (err) { + pr_err("%pOF: Unable to initialize IPI domain\n", node); + return err; + } + cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_ARMADA_XP_STARTING, "irqchip/armada/ipi:starting", mpic_starting_cpu, NULL); From b8fb82e4ffec3da153a6100d4cd6229fbfd3a22c Mon Sep 17 00:00:00 2001 From: Jinjie Ruan Date: Mon, 29 Jul 2024 19:26:06 +0800 Subject: [PATCH 054/180] irqchip: Remove asmlinkage for handlers registered with set_handle_irq() All architectures with use set_handle_irq() to set the root chip interrupt handler call that handler from C code, so there's no need for these handlers to be marked asmlinkage. Remove asmlinkage for all handlers registered with set_handle_irq(). Suggested-by: Thomas Gleixner Signed-off-by: Jinjie Ruan Signed-off-by: Thomas Gleixner Acked-by: Mark Rutland Link: https://lore.kernel.org/all/20240729112606.1581732-1-ruanjinjie@huawei.com --- drivers/irqchip/irq-atmel-aic.c | 3 +-- drivers/irqchip/irq-atmel-aic5.c | 3 +-- drivers/irqchip/irq-clps711x.c | 2 +- drivers/irqchip/irq-davinci-cp-intc.c | 3 +-- drivers/irqchip/irq-ftintc010.c | 2 +- drivers/irqchip/irq-gic-v3.c | 2 +- drivers/irqchip/irq-ixp4xx.c | 3 +-- drivers/irqchip/irq-omap-intc.c | 3 +-- drivers/irqchip/irq-sa11x0.c | 3 +-- drivers/irqchip/irq-versatile-fpga.c | 2 +- 10 files changed, 10 insertions(+), 16 deletions(-) diff --git a/drivers/irqchip/irq-atmel-aic.c b/drivers/irqchip/irq-atmel-aic.c index 4631f6847953..3839ad79ad31 100644 --- a/drivers/irqchip/irq-atmel-aic.c +++ b/drivers/irqchip/irq-atmel-aic.c @@ -57,8 +57,7 @@ static struct irq_domain *aic_domain; -static asmlinkage void __exception_irq_entry -aic_handle(struct pt_regs *regs) +static void __exception_irq_entry aic_handle(struct pt_regs *regs) { struct irq_domain_chip_generic *dgc = aic_domain->gc; struct irq_chip_generic *gc = dgc->gc[0]; diff --git a/drivers/irqchip/irq-atmel-aic5.c b/drivers/irqchip/irq-atmel-aic5.c index 145535bd7560..c0f55dc7b050 100644 --- a/drivers/irqchip/irq-atmel-aic5.c +++ b/drivers/irqchip/irq-atmel-aic5.c @@ -67,8 +67,7 @@ static struct irq_domain *aic5_domain; -static asmlinkage void __exception_irq_entry -aic5_handle(struct pt_regs *regs) +static void __exception_irq_entry aic5_handle(struct pt_regs *regs) { struct irq_chip_generic *bgc = irq_get_domain_generic_chip(aic5_domain, 0); u32 irqnr; diff --git a/drivers/irqchip/irq-clps711x.c b/drivers/irqchip/irq-clps711x.c index e731e0784f7e..806ebb1de201 100644 --- a/drivers/irqchip/irq-clps711x.c +++ b/drivers/irqchip/irq-clps711x.c @@ -69,7 +69,7 @@ static struct { struct irq_domain_ops ops; } *clps711x_intc; -static asmlinkage void __exception_irq_entry clps711x_irqh(struct pt_regs *regs) +static void __exception_irq_entry clps711x_irqh(struct pt_regs *regs) { u32 irqstat; diff --git a/drivers/irqchip/irq-davinci-cp-intc.c b/drivers/irqchip/irq-davinci-cp-intc.c index 7482c8ed34b2..f4f8e9fadbbf 100644 --- a/drivers/irqchip/irq-davinci-cp-intc.c +++ b/drivers/irqchip/irq-davinci-cp-intc.c @@ -116,8 +116,7 @@ static struct irq_chip davinci_cp_intc_irq_chip = { .flags = IRQCHIP_SKIP_SET_WAKE, }; -static asmlinkage void __exception_irq_entry -davinci_cp_intc_handle_irq(struct pt_regs *regs) +static void __exception_irq_entry davinci_cp_intc_handle_irq(struct pt_regs *regs) { int gpir, irqnr, none; diff --git a/drivers/irqchip/irq-ftintc010.c b/drivers/irqchip/irq-ftintc010.c index 359efc1d1be7..b91c358ea6db 100644 --- a/drivers/irqchip/irq-ftintc010.c +++ b/drivers/irqchip/irq-ftintc010.c @@ -125,7 +125,7 @@ static struct irq_chip ft010_irq_chip = { /* Local static for the IRQ entry call */ static struct ft010_irq_data firq; -static asmlinkage void __exception_irq_entry ft010_irqchip_handle_irq(struct pt_regs *regs) +static void __exception_irq_entry ft010_irqchip_handle_irq(struct pt_regs *regs) { struct ft010_irq_data *f = &firq; int irq; diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index c19083bfb943..0efa3443c323 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -930,7 +930,7 @@ static void __gic_handle_irq_from_irqsoff(struct pt_regs *regs) __gic_handle_nmi(irqnr, regs); } -static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) +static void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) { if (unlikely(gic_supports_nmi() && !interrupts_enabled(regs))) __gic_handle_irq_from_irqsoff(regs); diff --git a/drivers/irqchip/irq-ixp4xx.c b/drivers/irqchip/irq-ixp4xx.c index 5fba907b9052..f23b02f62a5c 100644 --- a/drivers/irqchip/irq-ixp4xx.c +++ b/drivers/irqchip/irq-ixp4xx.c @@ -105,8 +105,7 @@ static void ixp4xx_irq_unmask(struct irq_data *d) } } -static asmlinkage void __exception_irq_entry -ixp4xx_handle_irq(struct pt_regs *regs) +static void __exception_irq_entry ixp4xx_handle_irq(struct pt_regs *regs) { struct ixp4xx_irq *ixi = &ixirq; unsigned long status; diff --git a/drivers/irqchip/irq-omap-intc.c b/drivers/irqchip/irq-omap-intc.c index dc82162ba763..ad84a2f03368 100644 --- a/drivers/irqchip/irq-omap-intc.c +++ b/drivers/irqchip/irq-omap-intc.c @@ -325,8 +325,7 @@ static int __init omap_init_irq(u32 base, struct device_node *node) return ret; } -static asmlinkage void __exception_irq_entry -omap_intc_handle_irq(struct pt_regs *regs) +static void __exception_irq_entry omap_intc_handle_irq(struct pt_regs *regs) { extern unsigned long irq_err_count; u32 irqnr; diff --git a/drivers/irqchip/irq-sa11x0.c b/drivers/irqchip/irq-sa11x0.c index 31c202a1ae62..9d0b80271949 100644 --- a/drivers/irqchip/irq-sa11x0.c +++ b/drivers/irqchip/irq-sa11x0.c @@ -127,8 +127,7 @@ static int __init sa1100irq_init_devicefs(void) device_initcall(sa1100irq_init_devicefs); -static asmlinkage void __exception_irq_entry -sa1100_handle_irq(struct pt_regs *regs) +static void __exception_irq_entry sa1100_handle_irq(struct pt_regs *regs) { uint32_t icip, icmr, mask; diff --git a/drivers/irqchip/irq-versatile-fpga.c b/drivers/irqchip/irq-versatile-fpga.c index 5018a06060e6..ca471c6fee99 100644 --- a/drivers/irqchip/irq-versatile-fpga.c +++ b/drivers/irqchip/irq-versatile-fpga.c @@ -128,7 +128,7 @@ static int handle_one_fpga(struct fpga_irq_data *f, struct pt_regs *regs) * Keep iterating over all registered FPGA IRQ controllers until there are * no pending interrupts. */ -static asmlinkage void __exception_irq_entry fpga_handle_irq(struct pt_regs *regs) +static void __exception_irq_entry fpga_handle_irq(struct pt_regs *regs) { int i, handled; From bb4531976523c6e394188c4f4a7eeaf5e9efdd48 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Fri, 2 Aug 2024 14:26:01 +0530 Subject: [PATCH 055/180] irqchip/gic-v4.1: Replace bare number with ID_AA64PFR0_EL1_GIC_V4P1 Use ID_AA64PFR0_EL1_GIC_V4P1 instead of '3' in gic_cpuif_has_vsgi() to check for the GIC version. Signed-off-by: Anshuman Khandual Signed-off-by: Thomas Gleixner Reviewed-by: Zenghui Yu Acked-by: Marc Zyngier Link: https://lore.kernel.org/all/20240802085601.1824057-1-anshuman.khandual@arm.com --- drivers/irqchip/irq-gic-v4.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-gic-v4.c b/drivers/irqchip/irq-gic-v4.c index ca32ac19d284..58c28895f8c4 100644 --- a/drivers/irqchip/irq-gic-v4.c +++ b/drivers/irqchip/irq-gic-v4.c @@ -97,7 +97,7 @@ bool gic_cpuif_has_vsgi(void) fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64PFR0_EL1_GIC_SHIFT); - return fld >= 0x3; + return fld >= ID_AA64PFR0_EL1_GIC_V4P1; } #else bool gic_cpuif_has_vsgi(void) From c0e81a455e23f77683178b8ae32651df5841f065 Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Tue, 16 Jul 2024 22:14:58 +0800 Subject: [PATCH 056/180] cpu/hotplug: Make HOTPLUG_PARALLEL independent of HOTPLUG_SMT Provide stub functions for SMT related parallel bring up functions so that HOTPLUG_PARALLEL can work without HOTPLUG_SMT. Signed-off-by: Jiaxun Yang Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240716-loongarch-hotplug-v3-1-af59b3bb35c8@flygoat.com --- kernel/cpu.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/kernel/cpu.c b/kernel/cpu.c index 1209ddaec026..c89e0e91379a 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -1808,6 +1808,7 @@ static int __init parallel_bringup_parse_param(char *arg) } early_param("cpuhp.parallel", parallel_bringup_parse_param); +#ifdef CONFIG_HOTPLUG_SMT static inline bool cpuhp_smt_aware(void) { return cpu_smt_max_threads > 1; @@ -1817,6 +1818,16 @@ static inline const struct cpumask *cpuhp_get_primary_thread_mask(void) { return cpu_primary_thread_mask; } +#else +static inline bool cpuhp_smt_aware(void) +{ + return false; +} +static inline const struct cpumask *cpuhp_get_primary_thread_mask(void) +{ + return cpu_none_mask; +} +#endif /* * On architectures which have enabled parallel bringup this invokes all BP From 2dce993165088dbe728faa21547e3b74213b6732 Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Tue, 16 Jul 2024 22:14:59 +0800 Subject: [PATCH 057/180] cpu/hotplug: Provide weak fallback for arch_cpuhp_init_parallel_bringup() CONFIG_HOTPLUG_PARALLEL expects the architecture to implement arch_cpuhp_init_parallel_bringup() to decide whether paralllel hotplug is possible and to do the necessary architecture specific initialization. There are architectures which can enable it unconditionally and do not require architecture specific initialization. Provide a weak fallback for arch_cpuhp_init_parallel_bringup() so that such architectures are not forced to implement empty stub functions. Signed-off-by: Jiaxun Yang Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240716-loongarch-hotplug-v3-2-af59b3bb35c8@flygoat.com --- kernel/cpu.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kernel/cpu.c b/kernel/cpu.c index c89e0e91379a..16323610cd20 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -1829,6 +1829,11 @@ static inline const struct cpumask *cpuhp_get_primary_thread_mask(void) } #endif +bool __weak arch_cpuhp_init_parallel_bringup(void) +{ + return true; +} + /* * On architectures which have enabled parallel bringup this invokes all BP * prepare states for each of the to be onlined APs first. The last state From 17915131ae4660658aa779f89e9f444319861561 Mon Sep 17 00:00:00 2001 From: Borislav Petkov Date: Fri, 2 Aug 2024 08:46:14 -0700 Subject: [PATCH 058/180] clocksource: Improve comments for watchdog skew bounds Add more detail on the rationale for bounding the clocksource ->uncertainty_margin below at about 500ppm. Signed-off-by: Borislav Petkov Signed-off-by: Paul E. McKenney Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240802154618.4149953-1-paulmck@kernel.org --- kernel/time/clocksource.c | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index d0538a75f4c6..581cdbb53844 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -125,6 +125,13 @@ static u64 suspend_start; * * The default of 500 parts per million is based on NTP's limits. * If a clocksource is good enough for NTP, it is good enough for us! + * + * In other words, by default, even if a clocksource is extremely + * precise (for example, with a sub-nanosecond period), the maximum + * permissible skew between the clocksource watchdog and the clocksource + * under test is not permitted to go below the 500ppm minimum defined + * by MAX_SKEW_USEC. This 500ppm minimum may be overridden using the + * CLOCKSOURCE_WATCHDOG_MAX_SKEW_US Kconfig option. */ #ifdef CONFIG_CLOCKSOURCE_WATCHDOG_MAX_SKEW_US #define MAX_SKEW_USEC CONFIG_CLOCKSOURCE_WATCHDOG_MAX_SKEW_US @@ -1146,14 +1153,19 @@ void __clocksource_update_freq_scale(struct clocksource *cs, u32 scale, u32 freq } /* - * If the uncertainty margin is not specified, calculate it. - * If both scale and freq are non-zero, calculate the clock - * period, but bound below at 2*WATCHDOG_MAX_SKEW. However, - * if either of scale or freq is zero, be very conservative and - * take the tens-of-milliseconds WATCHDOG_THRESHOLD value for the - * uncertainty margin. Allow stupidly small uncertainty margins - * to be specified by the caller for testing purposes, but warn - * to discourage production use of this capability. + * If the uncertainty margin is not specified, calculate it. If + * both scale and freq are non-zero, calculate the clock period, but + * bound below at 2*WATCHDOG_MAX_SKEW, that is, 500ppm by default. + * However, if either of scale or freq is zero, be very conservative + * and take the tens-of-milliseconds WATCHDOG_THRESHOLD value + * for the uncertainty margin. Allow stupidly small uncertainty + * margins to be specified by the caller for testing purposes, + * but warn to discourage production use of this capability. + * + * Bottom line: The sum of the uncertainty margins of the + * watchdog clocksource and the clocksource under test will be at + * least 500ppm by default. For more information, please see the + * comment preceding CONFIG_CLOCKSOURCE_WATCHDOG_MAX_SKEW_US above. */ if (scale && freq && !cs->uncertainty_margin) { cs->uncertainty_margin = NSEC_PER_SEC / (scale * freq); From f33a5d4bd9c2e545857b2cf7481eb721bcab867c Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 2 Aug 2024 08:46:16 -0700 Subject: [PATCH 059/180] clocksource: Fix comments on WATCHDOG_THRESHOLD & WATCHDOG_MAX_SKEW The WATCHDOG_THRESHOLD macro is no longer used to supply a default value for ->uncertainty_margin, but WATCHDOG_MAX_SKEW now is. Therefore, update the comments to reflect this change. Reported-by: Borislav Petkov Signed-off-by: Paul E. McKenney Signed-off-by: Thomas Gleixner Acked-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/all/20240802154618.4149953-3-paulmck@kernel.org --- kernel/time/clocksource.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index 581cdbb53844..ee0ad5e4d517 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -113,7 +113,6 @@ static u64 suspend_start; /* * Threshold: 0.0312s, when doubled: 0.0625s. - * Also a default for cs->uncertainty_margin when registering clocks. */ #define WATCHDOG_THRESHOLD (NSEC_PER_SEC >> 5) @@ -139,6 +138,13 @@ static u64 suspend_start; #define MAX_SKEW_USEC (125 * WATCHDOG_INTERVAL / HZ) #endif +/* + * Default for maximum permissible skew when cs->uncertainty_margin is + * not specified, and the lower bound even when cs->uncertainty_margin + * is specified. This is also the default that is used when registering + * clocks with unspecifed cs->uncertainty_margin, so this macro is used + * even in CONFIG_CLOCKSOURCE_WATCHDOG=n kernels. + */ #define WATCHDOG_MAX_SKEW (MAX_SKEW_USEC * NSEC_PER_USEC) #ifdef CONFIG_CLOCKSOURCE_WATCHDOG From 4ac1dd3245b9067f929ab30141bb0475e9e32fc5 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 2 Aug 2024 08:46:17 -0700 Subject: [PATCH 060/180] clocksource: Set cs_watchdog_read() checks based on .uncertainty_margin Right now, cs_watchdog_read() does clocksource sanity checks based on WATCHDOG_MAX_SKEW, which sets a floor on any clocksource's .uncertainty_margin. These sanity checks can therefore act inappropriately for clocksources with large uncertainty margins. One reason for a clocksource to have a large .uncertainty_margin is when that clocksource has long read-out latency, given that it does not make sense for the .uncertainty_margin to be smaller than the read-out latency. With the current checks, cs_watchdog_read() could reject all normal reads from a clocksource with long read-out latencies, such as those from legacy clocksources that are no longer implemented in hardware. Therefore, recast the cs_watchdog_read() checks in terms of the .uncertainty_margin values of the clocksources involved in the timespan in question. The first covers two watchdog reads and one cs read, so use twice the watchdog .uncertainty_margin plus that of the cs. The second covers only a pair of watchdog reads, so use twice the watchdog .uncertainty_margin. Reported-by: Borislav Petkov Signed-off-by: Paul E. McKenney Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240802154618.4149953-4-paulmck@kernel.org --- kernel/time/clocksource.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index ee0ad5e4d517..23336eecb4f4 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -244,6 +244,7 @@ enum wd_read_status { static enum wd_read_status cs_watchdog_read(struct clocksource *cs, u64 *csnow, u64 *wdnow) { + int64_t md = 2 * watchdog->uncertainty_margin; unsigned int nretries, max_retries; int64_t wd_delay, wd_seq_delay; u64 wd_end, wd_end2; @@ -258,7 +259,7 @@ static enum wd_read_status cs_watchdog_read(struct clocksource *cs, u64 *csnow, local_irq_enable(); wd_delay = cycles_to_nsec_safe(watchdog, *wdnow, wd_end); - if (wd_delay <= WATCHDOG_MAX_SKEW) { + if (wd_delay <= md + cs->uncertainty_margin) { if (nretries > 1 && nretries >= max_retries) { pr_warn("timekeeping watchdog on CPU%d: %s retried %d times before success\n", smp_processor_id(), watchdog->name, nretries); @@ -271,12 +272,12 @@ static enum wd_read_status cs_watchdog_read(struct clocksource *cs, u64 *csnow, * there is too much external interferences that cause * significant delay in reading both clocksource and watchdog. * - * If consecutive WD read-back delay > WATCHDOG_MAX_SKEW/2, - * report system busy, reinit the watchdog and skip the current + * If consecutive WD read-back delay > md, report + * system busy, reinit the watchdog and skip the current * watchdog test. */ wd_seq_delay = cycles_to_nsec_safe(watchdog, wd_end, wd_end2); - if (wd_seq_delay > WATCHDOG_MAX_SKEW/2) + if (wd_seq_delay > md) goto skip_test; } From 3431392d5e8a7d420c06048260d521c1dd08e931 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Wed, 7 Aug 2024 18:40:53 +0200 Subject: [PATCH 061/180] irqchip/armada-370-xp: Drop IPI_DOORBELL_START and rename IPI_DOORBELL_END MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop IPI_DOORBELL_START since it is not used and rename IPI_DOORBELL_END to IPI_DOORBELL_NR. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner --- drivers/irqchip/irq-armada-370-xp.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index b11612acec78..9a431d04ed19 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -136,8 +136,7 @@ #define MPIC_MAX_PER_CPU_IRQS 28 /* IPI and MSI interrupt definitions for IPI platforms */ -#define IPI_DOORBELL_START 0 -#define IPI_DOORBELL_END 8 +#define IPI_DOORBELL_NR 8 #define IPI_DOORBELL_MASK GENMASK(7, 0) #define PCI_MSI_DOORBELL_START 16 #define PCI_MSI_DOORBELL_NR 16 @@ -452,7 +451,7 @@ static const struct irq_domain_ops mpic_ipi_domain_ops = { static void mpic_ipi_resume(void) { - for (irq_hw_number_t i = 0; i < IPI_DOORBELL_END; i++) { + for (irq_hw_number_t i = 0; i < IPI_DOORBELL_NR; i++) { unsigned int virq = irq_find_mapping(mpic_ipi_domain, i); struct irq_data *d; @@ -468,17 +467,17 @@ static __init int mpic_ipi_init(struct device_node *node) { int base_ipi; - mpic_ipi_domain = irq_domain_create_linear(of_node_to_fwnode(node), IPI_DOORBELL_END, + mpic_ipi_domain = irq_domain_create_linear(of_node_to_fwnode(node), IPI_DOORBELL_NR, &mpic_ipi_domain_ops, NULL); if (WARN_ON(!mpic_ipi_domain)) return -ENOMEM; irq_domain_update_bus_token(mpic_ipi_domain, DOMAIN_BUS_IPI); - base_ipi = irq_domain_alloc_irqs(mpic_ipi_domain, IPI_DOORBELL_END, NUMA_NO_NODE, NULL); + base_ipi = irq_domain_alloc_irqs(mpic_ipi_domain, IPI_DOORBELL_NR, NUMA_NO_NODE, NULL); if (WARN_ON(!base_ipi)) return -ENOMEM; - set_smp_ipi_range(base_ipi, IPI_DOORBELL_END); + set_smp_ipi_range(base_ipi, IPI_DOORBELL_NR); return 0; } @@ -627,7 +626,7 @@ static void mpic_handle_ipi_irq(void) cause = readl_relaxed(per_cpu_int_base + MPIC_IN_DRBEL_CAUSE); cause &= IPI_DOORBELL_MASK; - for_each_set_bit(i, &cause, IPI_DOORBELL_END) + for_each_set_bit(i, &cause, IPI_DOORBELL_NR) generic_handle_domain_irq(mpic_ipi_domain, i); } #else From 0dbf9b6025e3d6092ad883541c090118e5361d98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Wed, 7 Aug 2024 18:40:54 +0200 Subject: [PATCH 062/180] irqchip/armada-370-xp: Drop msi_doorbell_end() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop the msi_doorbell_end() function and related constants, it is not used anymore. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner --- drivers/irqchip/irq-armada-370-xp.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 9a431d04ed19..fcfc5f86fbff 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -140,13 +140,11 @@ #define IPI_DOORBELL_MASK GENMASK(7, 0) #define PCI_MSI_DOORBELL_START 16 #define PCI_MSI_DOORBELL_NR 16 -#define PCI_MSI_DOORBELL_END 32 #define PCI_MSI_DOORBELL_MASK GENMASK(31, 16) /* MSI interrupt definitions for non-IPI platforms */ #define PCI_MSI_FULL_DOORBELL_START 0 #define PCI_MSI_FULL_DOORBELL_NR 32 -#define PCI_MSI_FULL_DOORBELL_END 32 #define PCI_MSI_FULL_DOORBELL_MASK GENMASK(31, 0) #define PCI_MSI_FULL_DOORBELL_SRC0_MASK GENMASK(15, 0) #define PCI_MSI_FULL_DOORBELL_SRC1_MASK GENMASK(31, 16) @@ -190,11 +188,6 @@ static inline unsigned int msi_doorbell_size(void) return mpic_is_ipi_available() ? PCI_MSI_DOORBELL_NR : PCI_MSI_FULL_DOORBELL_NR; } -static inline unsigned int msi_doorbell_end(void) -{ - return mpic_is_ipi_available() ? PCI_MSI_DOORBELL_END : PCI_MSI_FULL_DOORBELL_END; -} - static inline bool mpic_is_percpu_irq(irq_hw_number_t hwirq) { return hwirq <= MPIC_MAX_PER_CPU_IRQS; From 37e130c224fd0da168570003355fcbd091a87030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Wed, 7 Aug 2024 18:40:55 +0200 Subject: [PATCH 063/180] irqchip/armada-370-xp: Add the __init attribute to mpic_msi_init() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the __init attribute to the mpic_msi_init() function. It is only called from the device initializer, and so can be dropped after boot is complete. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner --- drivers/irqchip/irq-armada-370-xp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index fcfc5f86fbff..f5a693745785 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -314,7 +314,7 @@ static void mpic_msi_reenable_percpu(void) writel(1, per_cpu_int_base + MPIC_INT_CLEAR_MASK); } -static int mpic_msi_init(struct device_node *node, phys_addr_t main_int_phys_base) +static int __init mpic_msi_init(struct device_node *node, phys_addr_t main_int_phys_base) { msi_doorbell_addr = main_int_phys_base + MPIC_SW_TRIG_INT; From a4d4d4a642da83d869d2851c8c3732c699cbc08e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Wed, 7 Aug 2024 18:40:56 +0200 Subject: [PATCH 064/180] irqchip/armada-370-xp: Put __init attribute after return type in mpic_ipi_init() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For consistency with the rest of the driver, put the __init attribute after the return type of the mpic_ipi_init() function. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner --- drivers/irqchip/irq-armada-370-xp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index f5a693745785..07004ecec165 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -456,7 +456,7 @@ static void mpic_ipi_resume(void) } } -static __init int mpic_ipi_init(struct device_node *node) +static int __init mpic_ipi_init(struct device_node *node) { int base_ipi; From 68fe2c59853611e2551370f0ab4b80b04a0748ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Wed, 7 Aug 2024 18:40:57 +0200 Subject: [PATCH 065/180] irqchip/armada-370-xp: Put static variables into driver private structure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In preparation for converting the driver to modern style put all the interrupt controller private static variables into driver private structure. Access to these variables changes as: main_int_base mpic->base per_cpu_int_base mpic->per_cpu mpic_domain mpic->domain parent_irq mpic->parent_irq ... Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner --- drivers/irqchip/irq-armada-370-xp.c | 225 +++++++++++++++------------- 1 file changed, 123 insertions(+), 102 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 07004ecec165..00f38428d2ba 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -66,11 +66,11 @@ * * The "global interrupt mask/unmask" is modified using the * MPIC_INT_SET_ENABLE and MPIC_INT_CLEAR_ENABLE - * registers, which are relative to "main_int_base". + * registers, which are relative to "mpic->base". * * The "per-CPU mask/unmask" is modified using the MPIC_INT_SET_MASK * and MPIC_INT_CLEAR_MASK registers, which are relative to - * "per_cpu_int_base". This base address points to a special address, + * "mpic->per_cpu". This base address points to a special address, * which automatically accesses the registers of the current CPU. * * The per-CPU mask/unmask can also be adjusted using the global @@ -112,7 +112,7 @@ * at the per-CPU level. */ -/* Registers relative to main_int_base */ +/* Registers relative to mpic->base */ #define MPIC_INT_CONTROL 0x00 #define MPIC_INT_CONTROL_NUMINT_MASK GENMASK(12, 2) #define MPIC_SW_TRIG_INT 0x04 @@ -122,7 +122,7 @@ #define MPIC_INT_SOURCE_CPU_MASK GENMASK(3, 0) #define MPIC_INT_IRQ_FIQ_MASK(cpuid) ((BIT(0) | BIT(8)) << (cpuid)) -/* Registers relative to per_cpu_int_base */ +/* Registers relative to mpic->per_cpu */ #define MPIC_IN_DRBEL_CAUSE 0x08 #define MPIC_IN_DRBEL_MASK 0x0c #define MPIC_PPI_CAUSE 0x10 @@ -149,18 +149,40 @@ #define PCI_MSI_FULL_DOORBELL_SRC0_MASK GENMASK(15, 0) #define PCI_MSI_FULL_DOORBELL_SRC1_MASK GENMASK(31, 16) -static void __iomem *per_cpu_int_base; -static void __iomem *main_int_base; -static struct irq_domain *mpic_domain; -static u32 doorbell_mask_reg; -static int parent_irq; -#ifdef CONFIG_PCI_MSI -static struct irq_domain *mpic_msi_domain; -static struct irq_domain *mpic_msi_inner_domain; -static DECLARE_BITMAP(msi_used, PCI_MSI_FULL_DOORBELL_NR); -static DEFINE_MUTEX(msi_used_lock); -static phys_addr_t msi_doorbell_addr; +/** + * struct mpic - MPIC private data structure + * @base: MPIC registers base address + * @per_cpu: per-CPU registers base address + * @parent_irq: parent IRQ if MPIC is not top-level interrupt controller + * @domain: MPIC main interrupt domain + * @ipi_domain: IPI domain + * @msi_domain: MSI domain + * @msi_inner_domain: MSI inner domain + * @msi_used: bitmap of used MSI numbers + * @msi_lock: mutex serializing access to @msi_used + * @msi_doorbell_addr: physical address of MSI doorbell register + * @doorbell_mask: doorbell mask of MSIs and IPIs, stored on suspend, restored on resume + */ +struct mpic { + void __iomem *base; + void __iomem *per_cpu; + int parent_irq; + struct irq_domain *domain; +#ifdef CONFIG_SMP + struct irq_domain *ipi_domain; #endif +#ifdef CONFIG_PCI_MSI + struct irq_domain *msi_domain; + struct irq_domain *msi_inner_domain; + DECLARE_BITMAP(msi_used, PCI_MSI_FULL_DOORBELL_NR); + struct mutex msi_lock; + phys_addr_t msi_doorbell_addr; +#endif + u32 doorbell_mask; +}; + +static struct mpic mpic_data; +static struct mpic * const mpic = &mpic_data; static inline bool mpic_is_ipi_available(void) { @@ -170,7 +192,7 @@ static inline bool mpic_is_ipi_available(void) * interrupt controller (e.g. GIC) that takes care of inter-processor * interrupts. */ - return parent_irq <= 0; + return mpic->parent_irq <= 0; } static inline u32 msi_doorbell_mask(void) @@ -203,9 +225,9 @@ static void mpic_irq_mask(struct irq_data *d) irq_hw_number_t hwirq = irqd_to_hwirq(d); if (!mpic_is_percpu_irq(hwirq)) - writel(hwirq, main_int_base + MPIC_INT_CLEAR_ENABLE); + writel(hwirq, mpic->base + MPIC_INT_CLEAR_ENABLE); else - writel(hwirq, per_cpu_int_base + MPIC_INT_SET_MASK); + writel(hwirq, mpic->per_cpu + MPIC_INT_SET_MASK); } static void mpic_irq_unmask(struct irq_data *d) @@ -213,9 +235,9 @@ static void mpic_irq_unmask(struct irq_data *d) irq_hw_number_t hwirq = irqd_to_hwirq(d); if (!mpic_is_percpu_irq(hwirq)) - writel(hwirq, main_int_base + MPIC_INT_SET_ENABLE); + writel(hwirq, mpic->base + MPIC_INT_SET_ENABLE); else - writel(hwirq, per_cpu_int_base + MPIC_INT_CLEAR_MASK); + writel(hwirq, mpic->per_cpu + MPIC_INT_CLEAR_MASK); } #ifdef CONFIG_PCI_MSI @@ -236,8 +258,8 @@ static void mpic_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) { unsigned int cpu = cpumask_first(irq_data_get_effective_affinity_mask(d)); - msg->address_lo = lower_32_bits(msi_doorbell_addr); - msg->address_hi = upper_32_bits(msi_doorbell_addr); + msg->address_lo = lower_32_bits(mpic->msi_doorbell_addr); + msg->address_hi = upper_32_bits(mpic->msi_doorbell_addr); msg->data = BIT(cpu + 8) | (d->hwirq + msi_doorbell_start()); } @@ -269,10 +291,10 @@ static int mpic_msi_alloc(struct irq_domain *domain, unsigned int virq, unsigned { int hwirq; - mutex_lock(&msi_used_lock); - hwirq = bitmap_find_free_region(msi_used, msi_doorbell_size(), + mutex_lock(&mpic->msi_lock); + hwirq = bitmap_find_free_region(mpic->msi_used, msi_doorbell_size(), order_base_2(nr_irqs)); - mutex_unlock(&msi_used_lock); + mutex_unlock(&mpic->msi_lock); if (hwirq < 0) return -ENOSPC; @@ -291,9 +313,9 @@ static void mpic_msi_free(struct irq_domain *domain, unsigned int virq, unsigned { struct irq_data *d = irq_domain_get_irq_data(domain, virq); - mutex_lock(&msi_used_lock); - bitmap_release_region(msi_used, d->hwirq, order_base_2(nr_irqs)); - mutex_unlock(&msi_used_lock); + mutex_lock(&mpic->msi_lock); + bitmap_release_region(mpic->msi_used, d->hwirq, order_base_2(nr_irqs)); + mutex_unlock(&mpic->msi_lock); } static const struct irq_domain_ops mpic_msi_domain_ops = { @@ -306,27 +328,29 @@ static void mpic_msi_reenable_percpu(void) u32 reg; /* Enable MSI doorbell mask and combined cpu local interrupt */ - reg = readl(per_cpu_int_base + MPIC_IN_DRBEL_MASK); + reg = readl(mpic->per_cpu + MPIC_IN_DRBEL_MASK); reg |= msi_doorbell_mask(); - writel(reg, per_cpu_int_base + MPIC_IN_DRBEL_MASK); + writel(reg, mpic->per_cpu + MPIC_IN_DRBEL_MASK); /* Unmask local doorbell interrupt */ - writel(1, per_cpu_int_base + MPIC_INT_CLEAR_MASK); + writel(1, mpic->per_cpu + MPIC_INT_CLEAR_MASK); } static int __init mpic_msi_init(struct device_node *node, phys_addr_t main_int_phys_base) { - msi_doorbell_addr = main_int_phys_base + MPIC_SW_TRIG_INT; + mpic->msi_doorbell_addr = main_int_phys_base + MPIC_SW_TRIG_INT; - mpic_msi_inner_domain = irq_domain_add_linear(NULL, msi_doorbell_size(), - &mpic_msi_domain_ops, NULL); - if (!mpic_msi_inner_domain) + mutex_init(&mpic->msi_lock); + + mpic->msi_inner_domain = irq_domain_add_linear(NULL, msi_doorbell_size(), + &mpic_msi_domain_ops, NULL); + if (!mpic->msi_inner_domain) return -ENOMEM; - mpic_msi_domain = pci_msi_create_irq_domain(of_node_to_fwnode(node), &mpic_msi_domain_info, - mpic_msi_inner_domain); - if (!mpic_msi_domain) { - irq_domain_remove(mpic_msi_inner_domain); + mpic->msi_domain = pci_msi_create_irq_domain(of_node_to_fwnode(node), &mpic_msi_domain_info, + mpic->msi_inner_domain); + if (!mpic->msi_domain) { + irq_domain_remove(mpic->msi_inner_domain); return -ENOMEM; } @@ -334,7 +358,7 @@ static int __init mpic_msi_init(struct device_node *node, phys_addr_t main_int_p /* Unmask low 16 MSI irqs on non-IPI platforms */ if (!mpic_is_ipi_available()) - writel(0, per_cpu_int_base + MPIC_INT_CLEAR_MASK); + writel(0, mpic->per_cpu + MPIC_INT_CLEAR_MASK); return 0; } @@ -362,29 +386,26 @@ static void mpic_perf_init(void) cpuid = cpu_logical_map(smp_processor_id()); /* Enable Performance Counter Overflow interrupts */ - writel(MPIC_INT_CAUSE_PERF(cpuid), - per_cpu_int_base + MPIC_INT_FABRIC_MASK); + writel(MPIC_INT_CAUSE_PERF(cpuid), mpic->per_cpu + MPIC_INT_FABRIC_MASK); } #ifdef CONFIG_SMP -static struct irq_domain *mpic_ipi_domain; - static void mpic_ipi_mask(struct irq_data *d) { u32 reg; - reg = readl(per_cpu_int_base + MPIC_IN_DRBEL_MASK); + reg = readl(mpic->per_cpu + MPIC_IN_DRBEL_MASK); reg &= ~BIT(d->hwirq); - writel(reg, per_cpu_int_base + MPIC_IN_DRBEL_MASK); + writel(reg, mpic->per_cpu + MPIC_IN_DRBEL_MASK); } static void mpic_ipi_unmask(struct irq_data *d) { u32 reg; - reg = readl(per_cpu_int_base + MPIC_IN_DRBEL_MASK); + reg = readl(mpic->per_cpu + MPIC_IN_DRBEL_MASK); reg |= BIT(d->hwirq); - writel(reg, per_cpu_int_base + MPIC_IN_DRBEL_MASK); + writel(reg, mpic->per_cpu + MPIC_IN_DRBEL_MASK); } static void mpic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) @@ -403,12 +424,12 @@ static void mpic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) dsb(); /* submit softirq */ - writel((map << 8) | d->hwirq, main_int_base + MPIC_SW_TRIG_INT); + writel((map << 8) | d->hwirq, mpic->base + MPIC_SW_TRIG_INT); } static void mpic_ipi_ack(struct irq_data *d) { - writel(~BIT(d->hwirq), per_cpu_int_base + MPIC_IN_DRBEL_CAUSE); + writel(~BIT(d->hwirq), mpic->per_cpu + MPIC_IN_DRBEL_CAUSE); } static struct irq_chip mpic_ipi_irqchip = { @@ -445,13 +466,13 @@ static const struct irq_domain_ops mpic_ipi_domain_ops = { static void mpic_ipi_resume(void) { for (irq_hw_number_t i = 0; i < IPI_DOORBELL_NR; i++) { - unsigned int virq = irq_find_mapping(mpic_ipi_domain, i); + unsigned int virq = irq_find_mapping(mpic->ipi_domain, i); struct irq_data *d; if (!virq || !irq_percpu_is_enabled(virq)) continue; - d = irq_domain_get_irq_data(mpic_ipi_domain, virq); + d = irq_domain_get_irq_data(mpic->ipi_domain, virq); mpic_ipi_unmask(d); } } @@ -460,13 +481,13 @@ static int __init mpic_ipi_init(struct device_node *node) { int base_ipi; - mpic_ipi_domain = irq_domain_create_linear(of_node_to_fwnode(node), IPI_DOORBELL_NR, - &mpic_ipi_domain_ops, NULL); - if (WARN_ON(!mpic_ipi_domain)) + mpic->ipi_domain = irq_domain_create_linear(of_node_to_fwnode(node), IPI_DOORBELL_NR, + &mpic_ipi_domain_ops, NULL); + if (WARN_ON(!mpic->ipi_domain)) return -ENOMEM; - irq_domain_update_bus_token(mpic_ipi_domain, DOMAIN_BUS_IPI); - base_ipi = irq_domain_alloc_irqs(mpic_ipi_domain, IPI_DOORBELL_NR, NUMA_NO_NODE, NULL); + irq_domain_update_bus_token(mpic->ipi_domain, DOMAIN_BUS_IPI); + base_ipi = irq_domain_alloc_irqs(mpic->ipi_domain, IPI_DOORBELL_NR, NUMA_NO_NODE, NULL); if (WARN_ON(!base_ipi)) return -ENOMEM; @@ -483,7 +504,7 @@ static int mpic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, /* Select a single core from the affinity mask which is online */ cpu = cpumask_any_and(mask_val, cpu_online_mask); - atomic_io_modify(main_int_base + MPIC_INT_SOURCE_CTL(hwirq), + atomic_io_modify(mpic->base + MPIC_INT_SOURCE_CTL(hwirq), MPIC_INT_SOURCE_CPU_MASK, BIT(cpu_logical_map(cpu))); irq_data_update_effective_affinity(d, cpumask_of(cpu)); @@ -493,27 +514,27 @@ static int mpic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, static void mpic_smp_cpu_init(void) { - for (irq_hw_number_t i = 0; i < mpic_domain->hwirq_max; i++) - writel(i, per_cpu_int_base + MPIC_INT_SET_MASK); + for (irq_hw_number_t i = 0; i < mpic->domain->hwirq_max; i++) + writel(i, mpic->per_cpu + MPIC_INT_SET_MASK); if (!mpic_is_ipi_available()) return; /* Disable all IPIs */ - writel(0, per_cpu_int_base + MPIC_IN_DRBEL_MASK); + writel(0, mpic->per_cpu + MPIC_IN_DRBEL_MASK); /* Clear pending IPIs */ - writel(0, per_cpu_int_base + MPIC_IN_DRBEL_CAUSE); + writel(0, mpic->per_cpu + MPIC_IN_DRBEL_CAUSE); /* Unmask IPI interrupt */ - writel(0, per_cpu_int_base + MPIC_INT_CLEAR_MASK); + writel(0, mpic->per_cpu + MPIC_INT_CLEAR_MASK); } static void mpic_reenable_percpu(void) { /* Re-enable per-CPU interrupts that were enabled before suspend */ for (irq_hw_number_t i = 0; i < MPIC_MAX_PER_CPU_IRQS; i++) { - unsigned int virq = irq_linear_revmap(mpic_domain, i); + unsigned int virq = irq_linear_revmap(mpic->domain, i); struct irq_data *d; if (!virq || !irq_percpu_is_enabled(virq)) @@ -542,7 +563,7 @@ static int mpic_cascaded_starting_cpu(unsigned int cpu) { mpic_perf_init(); mpic_reenable_percpu(); - enable_percpu_irq(parent_irq, IRQ_TYPE_NONE); + enable_percpu_irq(mpic->parent_irq, IRQ_TYPE_NONE); return 0; } @@ -571,9 +592,9 @@ static int mpic_irq_map(struct irq_domain *h, unsigned int virq, mpic_irq_mask(irq_get_irq_data(virq)); if (!mpic_is_percpu_irq(hwirq)) - writel(hwirq, per_cpu_int_base + MPIC_INT_CLEAR_MASK); + writel(hwirq, mpic->per_cpu + MPIC_INT_CLEAR_MASK); else - writel(hwirq, main_int_base + MPIC_INT_SET_ENABLE); + writel(hwirq, mpic->base + MPIC_INT_SET_ENABLE); irq_set_status_flags(virq, IRQ_LEVEL); if (mpic_is_percpu_irq(hwirq)) { @@ -598,12 +619,12 @@ static void mpic_handle_msi_irq(void) unsigned long cause; unsigned int i; - cause = readl_relaxed(per_cpu_int_base + MPIC_IN_DRBEL_CAUSE); + cause = readl_relaxed(mpic->per_cpu + MPIC_IN_DRBEL_CAUSE); cause &= msi_doorbell_mask(); - writel(~cause, per_cpu_int_base + MPIC_IN_DRBEL_CAUSE); + writel(~cause, mpic->per_cpu + MPIC_IN_DRBEL_CAUSE); for_each_set_bit(i, &cause, BITS_PER_LONG) - generic_handle_domain_irq(mpic_msi_inner_domain, + generic_handle_domain_irq(mpic->msi_inner_domain, i - msi_doorbell_start()); } #else @@ -616,11 +637,11 @@ static void mpic_handle_ipi_irq(void) unsigned long cause; irq_hw_number_t i; - cause = readl_relaxed(per_cpu_int_base + MPIC_IN_DRBEL_CAUSE); + cause = readl_relaxed(mpic->per_cpu + MPIC_IN_DRBEL_CAUSE); cause &= IPI_DOORBELL_MASK; for_each_set_bit(i, &cause, IPI_DOORBELL_NR) - generic_handle_domain_irq(mpic_ipi_domain, i); + generic_handle_domain_irq(mpic->ipi_domain, i); } #else static inline void mpic_handle_ipi_irq(void) {} @@ -635,11 +656,11 @@ static void mpic_handle_cascade_irq(struct irq_desc *desc) chained_irq_enter(chip, desc); - cause = readl_relaxed(per_cpu_int_base + MPIC_PPI_CAUSE); + cause = readl_relaxed(mpic->per_cpu + MPIC_PPI_CAUSE); cpuid = cpu_logical_map(smp_processor_id()); for_each_set_bit(i, &cause, BITS_PER_LONG) { - irqsrc = readl_relaxed(main_int_base + MPIC_INT_SOURCE_CTL(i)); + irqsrc = readl_relaxed(mpic->base + MPIC_INT_SOURCE_CTL(i)); /* Check if the interrupt is not masked on current CPU. * Test IRQ (0-1) and FIQ (8-9) mask bits. @@ -652,7 +673,7 @@ static void mpic_handle_cascade_irq(struct irq_desc *desc) continue; } - generic_handle_domain_irq(mpic_domain, i); + generic_handle_domain_irq(mpic->domain, i); } chained_irq_exit(chip, desc); @@ -664,14 +685,14 @@ static void __exception_irq_entry mpic_handle_irq(struct pt_regs *regs) u32 irqstat; do { - irqstat = readl_relaxed(per_cpu_int_base + MPIC_CPU_INTACK); + irqstat = readl_relaxed(mpic->per_cpu + MPIC_CPU_INTACK); i = FIELD_GET(MPIC_CPU_INTACK_IID_MASK, irqstat); if (i > 1022) break; if (i > 1) - generic_handle_domain_irq(mpic_domain, i); + generic_handle_domain_irq(mpic->domain, i); /* MSI handling */ if (i == 1) @@ -685,7 +706,7 @@ static void __exception_irq_entry mpic_handle_irq(struct pt_regs *regs) static int mpic_suspend(void) { - doorbell_mask_reg = readl(per_cpu_int_base + MPIC_IN_DRBEL_MASK); + mpic->doorbell_mask = readl(mpic->per_cpu + MPIC_IN_DRBEL_MASK); return 0; } @@ -695,8 +716,8 @@ static void mpic_resume(void) bool src0, src1; /* Re-enable interrupts */ - for (irq_hw_number_t i = 0; i < mpic_domain->hwirq_max; i++) { - unsigned int virq = irq_linear_revmap(mpic_domain, i); + for (irq_hw_number_t i = 0; i < mpic->domain->hwirq_max; i++) { + unsigned int virq = irq_linear_revmap(mpic->domain, i); struct irq_data *d; if (!virq) @@ -706,12 +727,12 @@ static void mpic_resume(void) if (!mpic_is_percpu_irq(i)) { /* Non per-CPU interrupts */ - writel(i, per_cpu_int_base + MPIC_INT_CLEAR_MASK); + writel(i, mpic->per_cpu + MPIC_INT_CLEAR_MASK); if (!irqd_irq_disabled(d)) mpic_irq_unmask(d); } else { /* Per-CPU interrupts */ - writel(i, main_int_base + MPIC_INT_SET_ENABLE); + writel(i, mpic->base + MPIC_INT_SET_ENABLE); /* * Re-enable on the current CPU, mpic_reenable_percpu() @@ -723,20 +744,20 @@ static void mpic_resume(void) } /* Reconfigure doorbells for IPIs and MSIs */ - writel(doorbell_mask_reg, per_cpu_int_base + MPIC_IN_DRBEL_MASK); + writel(mpic->doorbell_mask, mpic->per_cpu + MPIC_IN_DRBEL_MASK); if (mpic_is_ipi_available()) { - src0 = doorbell_mask_reg & IPI_DOORBELL_MASK; - src1 = doorbell_mask_reg & PCI_MSI_DOORBELL_MASK; + src0 = mpic->doorbell_mask & IPI_DOORBELL_MASK; + src1 = mpic->doorbell_mask & PCI_MSI_DOORBELL_MASK; } else { - src0 = doorbell_mask_reg & PCI_MSI_FULL_DOORBELL_SRC0_MASK; - src1 = doorbell_mask_reg & PCI_MSI_FULL_DOORBELL_SRC1_MASK; + src0 = mpic->doorbell_mask & PCI_MSI_FULL_DOORBELL_SRC0_MASK; + src1 = mpic->doorbell_mask & PCI_MSI_FULL_DOORBELL_SRC1_MASK; } if (src0) - writel(0, per_cpu_int_base + MPIC_INT_CLEAR_MASK); + writel(0, mpic->per_cpu + MPIC_INT_CLEAR_MASK); if (src1) - writel(1, per_cpu_int_base + MPIC_INT_CLEAR_MASK); + writel(1, mpic->per_cpu + MPIC_INT_CLEAR_MASK); if (mpic_is_ipi_available()) mpic_ipi_resume(); @@ -784,32 +805,32 @@ static int __init mpic_of_init(struct device_node *node, struct device_node *par unsigned int nr_irqs; int err; - err = mpic_map_region(node, 0, &main_int_base, &phys_base); + err = mpic_map_region(node, 0, &mpic->base, &phys_base); if (err) return err; - err = mpic_map_region(node, 1, &per_cpu_int_base, NULL); + err = mpic_map_region(node, 1, &mpic->per_cpu, NULL); if (err) return err; - nr_irqs = FIELD_GET(MPIC_INT_CONTROL_NUMINT_MASK, readl(main_int_base + MPIC_INT_CONTROL)); + nr_irqs = FIELD_GET(MPIC_INT_CONTROL_NUMINT_MASK, readl(mpic->base + MPIC_INT_CONTROL)); for (irq_hw_number_t i = 0; i < nr_irqs; i++) - writel(i, main_int_base + MPIC_INT_CLEAR_ENABLE); + writel(i, mpic->base + MPIC_INT_CLEAR_ENABLE); - mpic_domain = irq_domain_add_linear(node, nr_irqs, &mpic_irq_ops, NULL); - if (!mpic_domain) { + mpic->domain = irq_domain_add_linear(node, nr_irqs, &mpic_irq_ops, NULL); + if (!mpic->domain) { pr_err("%pOF: Unable to add IRQ domain\n", node); return -ENOMEM; } - irq_domain_update_bus_token(mpic_domain, DOMAIN_BUS_WIRED); + irq_domain_update_bus_token(mpic->domain, DOMAIN_BUS_WIRED); /* - * Initialize parent_irq before calling any other functions, since it is - * used to distinguish between IPI and non-IPI platforms. + * Initialize mpic->parent_irq before calling any other functions, since + * it is used to distinguish between IPI and non-IPI platforms. */ - parent_irq = irq_of_parse_and_map(node, 0); + mpic->parent_irq = irq_of_parse_and_map(node, 0); /* Setup for the boot CPU */ mpic_perf_init(); @@ -821,8 +842,8 @@ static int __init mpic_of_init(struct device_node *node, struct device_node *par return err; } - if (parent_irq <= 0) { - irq_set_default_host(mpic_domain); + if (mpic->parent_irq <= 0) { + irq_set_default_host(mpic->domain); set_handle_irq(mpic_handle_irq); #ifdef CONFIG_SMP err = mpic_ipi_init(node); @@ -841,7 +862,7 @@ static int __init mpic_of_init(struct device_node *node, struct device_node *par "irqchip/armada/cascade:starting", mpic_cascaded_starting_cpu, NULL); #endif - irq_set_chained_handler(parent_irq, mpic_handle_cascade_irq); + irq_set_chained_handler(mpic->parent_irq, mpic_handle_cascade_irq); } register_syscore_ops(&mpic_syscore_ops); From ee5d09cf14a10010af163ac98e21aa282de6c4cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Wed, 7 Aug 2024 18:40:58 +0200 Subject: [PATCH 066/180] irqchip/armada-370-xp: Put MSI doorbell limits into the mpic structure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Put the MSI doorbell limits msi_doorbell_start, msi_doorbell_size and msi_doorbell_mask into the driver private structure and get rid of the corresponding functions. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner --- drivers/irqchip/irq-armada-370-xp.c | 44 ++++++++++++++--------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 00f38428d2ba..1c95a61c18cb 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -161,6 +161,10 @@ * @msi_used: bitmap of used MSI numbers * @msi_lock: mutex serializing access to @msi_used * @msi_doorbell_addr: physical address of MSI doorbell register + * @msi_doorbell_mask: mask of available doorbell bits for MSIs (either PCI_MSI_DOORBELL_MASK or + * PCI_MSI_FULL_DOORBELL_MASK) + * @msi_doorbell_start: first set bit in @msi_doorbell_mask + * @msi_doorbell_size: number of set bits in @msi_doorbell_mask * @doorbell_mask: doorbell mask of MSIs and IPIs, stored on suspend, restored on resume */ struct mpic { @@ -177,6 +181,8 @@ struct mpic { DECLARE_BITMAP(msi_used, PCI_MSI_FULL_DOORBELL_NR); struct mutex msi_lock; phys_addr_t msi_doorbell_addr; + u32 msi_doorbell_mask; + unsigned int msi_doorbell_start, msi_doorbell_size; #endif u32 doorbell_mask; }; @@ -195,21 +201,6 @@ static inline bool mpic_is_ipi_available(void) return mpic->parent_irq <= 0; } -static inline u32 msi_doorbell_mask(void) -{ - return mpic_is_ipi_available() ? PCI_MSI_DOORBELL_MASK : PCI_MSI_FULL_DOORBELL_MASK; -} - -static inline unsigned int msi_doorbell_start(void) -{ - return mpic_is_ipi_available() ? PCI_MSI_DOORBELL_START : PCI_MSI_FULL_DOORBELL_START; -} - -static inline unsigned int msi_doorbell_size(void) -{ - return mpic_is_ipi_available() ? PCI_MSI_DOORBELL_NR : PCI_MSI_FULL_DOORBELL_NR; -} - static inline bool mpic_is_percpu_irq(irq_hw_number_t hwirq) { return hwirq <= MPIC_MAX_PER_CPU_IRQS; @@ -260,7 +251,7 @@ static void mpic_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) msg->address_lo = lower_32_bits(mpic->msi_doorbell_addr); msg->address_hi = upper_32_bits(mpic->msi_doorbell_addr); - msg->data = BIT(cpu + 8) | (d->hwirq + msi_doorbell_start()); + msg->data = BIT(cpu + 8) | (d->hwirq + mpic->msi_doorbell_start); } static int mpic_msi_set_affinity(struct irq_data *d, const struct cpumask *mask, bool force) @@ -292,7 +283,7 @@ static int mpic_msi_alloc(struct irq_domain *domain, unsigned int virq, unsigned int hwirq; mutex_lock(&mpic->msi_lock); - hwirq = bitmap_find_free_region(mpic->msi_used, msi_doorbell_size(), + hwirq = bitmap_find_free_region(mpic->msi_used, mpic->msi_doorbell_size, order_base_2(nr_irqs)); mutex_unlock(&mpic->msi_lock); @@ -329,7 +320,7 @@ static void mpic_msi_reenable_percpu(void) /* Enable MSI doorbell mask and combined cpu local interrupt */ reg = readl(mpic->per_cpu + MPIC_IN_DRBEL_MASK); - reg |= msi_doorbell_mask(); + reg |= mpic->msi_doorbell_mask; writel(reg, mpic->per_cpu + MPIC_IN_DRBEL_MASK); /* Unmask local doorbell interrupt */ @@ -342,7 +333,17 @@ static int __init mpic_msi_init(struct device_node *node, phys_addr_t main_int_p mutex_init(&mpic->msi_lock); - mpic->msi_inner_domain = irq_domain_add_linear(NULL, msi_doorbell_size(), + if (mpic_is_ipi_available()) { + mpic->msi_doorbell_start = PCI_MSI_DOORBELL_START; + mpic->msi_doorbell_size = PCI_MSI_DOORBELL_NR; + mpic->msi_doorbell_mask = PCI_MSI_DOORBELL_MASK; + } else { + mpic->msi_doorbell_start = PCI_MSI_FULL_DOORBELL_START; + mpic->msi_doorbell_size = PCI_MSI_FULL_DOORBELL_NR; + mpic->msi_doorbell_mask = PCI_MSI_FULL_DOORBELL_MASK; + } + + mpic->msi_inner_domain = irq_domain_add_linear(NULL, mpic->msi_doorbell_size, &mpic_msi_domain_ops, NULL); if (!mpic->msi_inner_domain) return -ENOMEM; @@ -620,12 +621,11 @@ static void mpic_handle_msi_irq(void) unsigned int i; cause = readl_relaxed(mpic->per_cpu + MPIC_IN_DRBEL_CAUSE); - cause &= msi_doorbell_mask(); + cause &= mpic->msi_doorbell_mask; writel(~cause, mpic->per_cpu + MPIC_IN_DRBEL_CAUSE); for_each_set_bit(i, &cause, BITS_PER_LONG) - generic_handle_domain_irq(mpic->msi_inner_domain, - i - msi_doorbell_start()); + generic_handle_domain_irq(mpic->msi_inner_domain, i - mpic->msi_doorbell_start); } #else static void mpic_handle_msi_irq(void) {} From 77eef29b642f07f56af28a7126b5666f705ca8d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Wed, 7 Aug 2024 18:40:59 +0200 Subject: [PATCH 067/180] irqchip/armada-370-xp: Pass around the driver private structure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In continuation of converting the driver to modern style, drop the global pointer to the driver private structure and instead pass it around the functions and callbacks, wherever possible. (There are 3 cases where it is not possible: mpic_cascaded_starting_cpu() and the syscore operations mpic_suspend() and mpic_resume()). Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner --- drivers/irqchip/irq-armada-370-xp.c | 115 +++++++++++++++++----------- 1 file changed, 70 insertions(+), 45 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 1c95a61c18cb..5710ce206cca 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -188,9 +188,8 @@ struct mpic { }; static struct mpic mpic_data; -static struct mpic * const mpic = &mpic_data; -static inline bool mpic_is_ipi_available(void) +static inline bool mpic_is_ipi_available(struct mpic *mpic) { /* * We distinguish IPI availability in the IC by the IC not having a @@ -213,6 +212,7 @@ static inline bool mpic_is_percpu_irq(irq_hw_number_t hwirq) */ static void mpic_irq_mask(struct irq_data *d) { + struct mpic *mpic = irq_data_get_irq_chip_data(d); irq_hw_number_t hwirq = irqd_to_hwirq(d); if (!mpic_is_percpu_irq(hwirq)) @@ -223,6 +223,7 @@ static void mpic_irq_mask(struct irq_data *d) static void mpic_irq_unmask(struct irq_data *d) { + struct mpic *mpic = irq_data_get_irq_chip_data(d); irq_hw_number_t hwirq = irqd_to_hwirq(d); if (!mpic_is_percpu_irq(hwirq)) @@ -248,6 +249,7 @@ static struct msi_domain_info mpic_msi_domain_info = { static void mpic_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) { unsigned int cpu = cpumask_first(irq_data_get_effective_affinity_mask(d)); + struct mpic *mpic = irq_data_get_irq_chip_data(d); msg->address_lo = lower_32_bits(mpic->msi_doorbell_addr); msg->address_hi = upper_32_bits(mpic->msi_doorbell_addr); @@ -280,6 +282,7 @@ static struct irq_chip mpic_msi_bottom_irq_chip = { static int mpic_msi_alloc(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs, void *args) { + struct mpic *mpic = domain->host_data; int hwirq; mutex_lock(&mpic->msi_lock); @@ -303,6 +306,7 @@ static int mpic_msi_alloc(struct irq_domain *domain, unsigned int virq, unsigned static void mpic_msi_free(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs) { struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct mpic *mpic = domain->host_data; mutex_lock(&mpic->msi_lock); bitmap_release_region(mpic->msi_used, d->hwirq, order_base_2(nr_irqs)); @@ -314,7 +318,7 @@ static const struct irq_domain_ops mpic_msi_domain_ops = { .free = mpic_msi_free, }; -static void mpic_msi_reenable_percpu(void) +static void mpic_msi_reenable_percpu(struct mpic *mpic) { u32 reg; @@ -327,13 +331,14 @@ static void mpic_msi_reenable_percpu(void) writel(1, mpic->per_cpu + MPIC_INT_CLEAR_MASK); } -static int __init mpic_msi_init(struct device_node *node, phys_addr_t main_int_phys_base) +static int __init mpic_msi_init(struct mpic *mpic, struct device_node *node, + phys_addr_t main_int_phys_base) { mpic->msi_doorbell_addr = main_int_phys_base + MPIC_SW_TRIG_INT; mutex_init(&mpic->msi_lock); - if (mpic_is_ipi_available()) { + if (mpic_is_ipi_available(mpic)) { mpic->msi_doorbell_start = PCI_MSI_DOORBELL_START; mpic->msi_doorbell_size = PCI_MSI_DOORBELL_NR; mpic->msi_doorbell_mask = PCI_MSI_DOORBELL_MASK; @@ -344,7 +349,7 @@ static int __init mpic_msi_init(struct device_node *node, phys_addr_t main_int_p } mpic->msi_inner_domain = irq_domain_add_linear(NULL, mpic->msi_doorbell_size, - &mpic_msi_domain_ops, NULL); + &mpic_msi_domain_ops, mpic); if (!mpic->msi_inner_domain) return -ENOMEM; @@ -355,25 +360,25 @@ static int __init mpic_msi_init(struct device_node *node, phys_addr_t main_int_p return -ENOMEM; } - mpic_msi_reenable_percpu(); + mpic_msi_reenable_percpu(mpic); /* Unmask low 16 MSI irqs on non-IPI platforms */ - if (!mpic_is_ipi_available()) + if (!mpic_is_ipi_available(mpic)) writel(0, mpic->per_cpu + MPIC_INT_CLEAR_MASK); return 0; } #else -static __maybe_unused void mpic_msi_reenable_percpu(void) {} +static __maybe_unused void mpic_msi_reenable_percpu(struct mpic *mpic) {} -static inline int mpic_msi_init(struct device_node *node, +static inline int mpic_msi_init(struct mpic *mpic, struct device_node *node, phys_addr_t main_int_phys_base) { return 0; } #endif -static void mpic_perf_init(void) +static void mpic_perf_init(struct mpic *mpic) { u32 cpuid; @@ -393,6 +398,7 @@ static void mpic_perf_init(void) #ifdef CONFIG_SMP static void mpic_ipi_mask(struct irq_data *d) { + struct mpic *mpic = irq_data_get_irq_chip_data(d); u32 reg; reg = readl(mpic->per_cpu + MPIC_IN_DRBEL_MASK); @@ -402,6 +408,7 @@ static void mpic_ipi_mask(struct irq_data *d) static void mpic_ipi_unmask(struct irq_data *d) { + struct mpic *mpic = irq_data_get_irq_chip_data(d); u32 reg; reg = readl(mpic->per_cpu + MPIC_IN_DRBEL_MASK); @@ -411,6 +418,7 @@ static void mpic_ipi_unmask(struct irq_data *d) static void mpic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) { + struct mpic *mpic = irq_data_get_irq_chip_data(d); unsigned int cpu; u32 map = 0; @@ -430,6 +438,8 @@ static void mpic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) static void mpic_ipi_ack(struct irq_data *d) { + struct mpic *mpic = irq_data_get_irq_chip_data(d); + writel(~BIT(d->hwirq), mpic->per_cpu + MPIC_IN_DRBEL_CAUSE); } @@ -464,7 +474,7 @@ static const struct irq_domain_ops mpic_ipi_domain_ops = { .free = mpic_ipi_free, }; -static void mpic_ipi_resume(void) +static void mpic_ipi_resume(struct mpic *mpic) { for (irq_hw_number_t i = 0; i < IPI_DOORBELL_NR; i++) { unsigned int virq = irq_find_mapping(mpic->ipi_domain, i); @@ -478,12 +488,12 @@ static void mpic_ipi_resume(void) } } -static int __init mpic_ipi_init(struct device_node *node) +static int __init mpic_ipi_init(struct mpic *mpic, struct device_node *node) { int base_ipi; mpic->ipi_domain = irq_domain_create_linear(of_node_to_fwnode(node), IPI_DOORBELL_NR, - &mpic_ipi_domain_ops, NULL); + &mpic_ipi_domain_ops, mpic); if (WARN_ON(!mpic->ipi_domain)) return -ENOMEM; @@ -499,6 +509,7 @@ static int __init mpic_ipi_init(struct device_node *node) static int mpic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, bool force) { + struct mpic *mpic = irq_data_get_irq_chip_data(d); irq_hw_number_t hwirq = irqd_to_hwirq(d); unsigned int cpu; @@ -513,12 +524,12 @@ static int mpic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, return IRQ_SET_MASK_OK; } -static void mpic_smp_cpu_init(void) +static void mpic_smp_cpu_init(struct mpic *mpic) { for (irq_hw_number_t i = 0; i < mpic->domain->hwirq_max; i++) writel(i, mpic->per_cpu + MPIC_INT_SET_MASK); - if (!mpic_is_ipi_available()) + if (!mpic_is_ipi_available(mpic)) return; /* Disable all IPIs */ @@ -531,7 +542,7 @@ static void mpic_smp_cpu_init(void) writel(0, mpic->per_cpu + MPIC_INT_CLEAR_MASK); } -static void mpic_reenable_percpu(void) +static void mpic_reenable_percpu(struct mpic *mpic) { /* Re-enable per-CPU interrupts that were enabled before suspend */ for (irq_hw_number_t i = 0; i < MPIC_MAX_PER_CPU_IRQS; i++) { @@ -545,32 +556,36 @@ static void mpic_reenable_percpu(void) mpic_irq_unmask(d); } - if (mpic_is_ipi_available()) - mpic_ipi_resume(); + if (mpic_is_ipi_available(mpic)) + mpic_ipi_resume(mpic); - mpic_msi_reenable_percpu(); + mpic_msi_reenable_percpu(mpic); } static int mpic_starting_cpu(unsigned int cpu) { - mpic_perf_init(); - mpic_smp_cpu_init(); - mpic_reenable_percpu(); + struct mpic *mpic = irq_get_default_host()->host_data; + + mpic_perf_init(mpic); + mpic_smp_cpu_init(mpic); + mpic_reenable_percpu(mpic); return 0; } static int mpic_cascaded_starting_cpu(unsigned int cpu) { - mpic_perf_init(); - mpic_reenable_percpu(); + struct mpic *mpic = &mpic_data; + + mpic_perf_init(mpic); + mpic_reenable_percpu(mpic); enable_percpu_irq(mpic->parent_irq, IRQ_TYPE_NONE); return 0; } #else -static void mpic_smp_cpu_init(void) {} -static void mpic_ipi_resume(void) {} +static void mpic_smp_cpu_init(struct mpic *mpic) {} +static void mpic_ipi_resume(struct mpic *mpic) {} #endif static struct irq_chip mpic_irq_chip = { @@ -584,13 +599,16 @@ static struct irq_chip mpic_irq_chip = { .flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND, }; -static int mpic_irq_map(struct irq_domain *h, unsigned int virq, - irq_hw_number_t hwirq) +static int mpic_irq_map(struct irq_domain *domain, unsigned int virq, irq_hw_number_t hwirq) { + struct mpic *mpic = domain->host_data; + /* IRQs 0 and 1 cannot be mapped, they are handled internally */ if (hwirq <= 1) return -EINVAL; + irq_set_chip_data(virq, mpic); + mpic_irq_mask(irq_get_irq_data(virq)); if (!mpic_is_percpu_irq(hwirq)) writel(hwirq, mpic->per_cpu + MPIC_INT_CLEAR_MASK); @@ -615,7 +633,7 @@ static const struct irq_domain_ops mpic_irq_ops = { }; #ifdef CONFIG_PCI_MSI -static void mpic_handle_msi_irq(void) +static void mpic_handle_msi_irq(struct mpic *mpic) { unsigned long cause; unsigned int i; @@ -628,11 +646,11 @@ static void mpic_handle_msi_irq(void) generic_handle_domain_irq(mpic->msi_inner_domain, i - mpic->msi_doorbell_start); } #else -static void mpic_handle_msi_irq(void) {} +static void mpic_handle_msi_irq(struct mpic *mpic) {} #endif #ifdef CONFIG_SMP -static void mpic_handle_ipi_irq(void) +static void mpic_handle_ipi_irq(struct mpic *mpic) { unsigned long cause; irq_hw_number_t i; @@ -644,11 +662,12 @@ static void mpic_handle_ipi_irq(void) generic_handle_domain_irq(mpic->ipi_domain, i); } #else -static inline void mpic_handle_ipi_irq(void) {} +static inline void mpic_handle_ipi_irq(struct mpic *mpic) {} #endif static void mpic_handle_cascade_irq(struct irq_desc *desc) { + struct mpic *mpic = irq_desc_get_handler_data(desc); struct irq_chip *chip = irq_desc_get_chip(desc); unsigned long cause; u32 irqsrc, cpuid; @@ -669,7 +688,7 @@ static void mpic_handle_cascade_irq(struct irq_desc *desc) continue; if (i == 0 || i == 1) { - mpic_handle_msi_irq(); + mpic_handle_msi_irq(mpic); continue; } @@ -681,6 +700,7 @@ static void mpic_handle_cascade_irq(struct irq_desc *desc) static void __exception_irq_entry mpic_handle_irq(struct pt_regs *regs) { + struct mpic *mpic = irq_get_default_host()->host_data; irq_hw_number_t i; u32 irqstat; @@ -696,16 +716,18 @@ static void __exception_irq_entry mpic_handle_irq(struct pt_regs *regs) /* MSI handling */ if (i == 1) - mpic_handle_msi_irq(); + mpic_handle_msi_irq(mpic); /* IPI Handling */ if (i == 0) - mpic_handle_ipi_irq(); + mpic_handle_ipi_irq(mpic); } while (1); } static int mpic_suspend(void) { + struct mpic *mpic = &mpic_data; + mpic->doorbell_mask = readl(mpic->per_cpu + MPIC_IN_DRBEL_MASK); return 0; @@ -713,6 +735,7 @@ static int mpic_suspend(void) static void mpic_resume(void) { + struct mpic *mpic = &mpic_data; bool src0, src1; /* Re-enable interrupts */ @@ -746,7 +769,7 @@ static void mpic_resume(void) /* Reconfigure doorbells for IPIs and MSIs */ writel(mpic->doorbell_mask, mpic->per_cpu + MPIC_IN_DRBEL_MASK); - if (mpic_is_ipi_available()) { + if (mpic_is_ipi_available(mpic)) { src0 = mpic->doorbell_mask & IPI_DOORBELL_MASK; src1 = mpic->doorbell_mask & PCI_MSI_DOORBELL_MASK; } else { @@ -759,8 +782,8 @@ static void mpic_resume(void) if (src1) writel(1, mpic->per_cpu + MPIC_INT_CLEAR_MASK); - if (mpic_is_ipi_available()) - mpic_ipi_resume(); + if (mpic_is_ipi_available(mpic)) + mpic_ipi_resume(mpic); } static struct syscore_ops mpic_syscore_ops = { @@ -801,6 +824,7 @@ fail: static int __init mpic_of_init(struct device_node *node, struct device_node *parent) { + struct mpic *mpic = &mpic_data; phys_addr_t phys_base; unsigned int nr_irqs; int err; @@ -818,7 +842,7 @@ static int __init mpic_of_init(struct device_node *node, struct device_node *par for (irq_hw_number_t i = 0; i < nr_irqs; i++) writel(i, mpic->base + MPIC_INT_CLEAR_ENABLE); - mpic->domain = irq_domain_add_linear(node, nr_irqs, &mpic_irq_ops, NULL); + mpic->domain = irq_domain_add_linear(node, nr_irqs, &mpic_irq_ops, mpic); if (!mpic->domain) { pr_err("%pOF: Unable to add IRQ domain\n", node); return -ENOMEM; @@ -833,10 +857,10 @@ static int __init mpic_of_init(struct device_node *node, struct device_node *par mpic->parent_irq = irq_of_parse_and_map(node, 0); /* Setup for the boot CPU */ - mpic_perf_init(); - mpic_smp_cpu_init(); + mpic_perf_init(mpic); + mpic_smp_cpu_init(mpic); - err = mpic_msi_init(node, phys_base); + err = mpic_msi_init(mpic, node, phys_base); if (err) { pr_err("%pOF: Unable to initialize MSI domain\n", node); return err; @@ -846,7 +870,7 @@ static int __init mpic_of_init(struct device_node *node, struct device_node *par irq_set_default_host(mpic->domain); set_handle_irq(mpic_handle_irq); #ifdef CONFIG_SMP - err = mpic_ipi_init(node); + err = mpic_ipi_init(mpic, node); if (err) { pr_err("%pOF: Unable to initialize IPI domain\n", node); return err; @@ -862,7 +886,8 @@ static int __init mpic_of_init(struct device_node *node, struct device_node *par "irqchip/armada/cascade:starting", mpic_cascaded_starting_cpu, NULL); #endif - irq_set_chained_handler(mpic->parent_irq, mpic_handle_cascade_irq); + irq_set_chained_handler_and_data(mpic->parent_irq, + mpic_handle_cascade_irq, mpic); } register_syscore_ops(&mpic_syscore_ops); From 6abd809a543936ca005fd37efa32906c78409aea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Wed, 7 Aug 2024 18:41:00 +0200 Subject: [PATCH 068/180] irqchip/armada-370-xp: Dynamically allocate the driver private structure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dynamically allocate the driver private structure. This concludes the conversion of this driver to modern style. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner --- drivers/irqchip/irq-armada-370-xp.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 5710ce206cca..f8658a232f21 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -187,7 +187,7 @@ struct mpic { u32 doorbell_mask; }; -static struct mpic mpic_data; +static struct mpic *mpic_data __ro_after_init; static inline bool mpic_is_ipi_available(struct mpic *mpic) { @@ -575,7 +575,7 @@ static int mpic_starting_cpu(unsigned int cpu) static int mpic_cascaded_starting_cpu(unsigned int cpu) { - struct mpic *mpic = &mpic_data; + struct mpic *mpic = mpic_data; mpic_perf_init(mpic); mpic_reenable_percpu(mpic); @@ -726,7 +726,7 @@ static void __exception_irq_entry mpic_handle_irq(struct pt_regs *regs) static int mpic_suspend(void) { - struct mpic *mpic = &mpic_data; + struct mpic *mpic = mpic_data; mpic->doorbell_mask = readl(mpic->per_cpu + MPIC_IN_DRBEL_MASK); @@ -735,7 +735,7 @@ static int mpic_suspend(void) static void mpic_resume(void) { - struct mpic *mpic = &mpic_data; + struct mpic *mpic = mpic_data; bool src0, src1; /* Re-enable interrupts */ @@ -824,11 +824,17 @@ fail: static int __init mpic_of_init(struct device_node *node, struct device_node *parent) { - struct mpic *mpic = &mpic_data; phys_addr_t phys_base; unsigned int nr_irqs; + struct mpic *mpic; int err; + mpic = kzalloc(sizeof(*mpic), GFP_KERNEL); + if (WARN_ON(!mpic)) + return -ENOMEM; + + mpic_data = mpic; + err = mpic_map_region(node, 0, &mpic->base, &phys_base); if (err) return err; From 2793f68749c1fb035e255cdffb10108ef023f608 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Wed, 7 Aug 2024 18:41:01 +0200 Subject: [PATCH 069/180] irqchip/armada-370-xp: Fix reenabling last per-CPU interrupt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The number of per-CPU interrupts is 29 (0 to 28). This is described by the constant MPIC_MAX_PER_CPU_IRQS, set to 28 (the maximum per-CPU interrupt). Commit 0fa4ce746d1d ("irqchip/armada-370-xp: Re-enable per-CPU interrupts at resume time") used the constant incorrectly in the for-loop, it used the operator < instead of <=, causing it to iterate only the first 28 interrupts (0 to 27), ignoring the last, 28th, per-CPU interrupt. To avoid this kind of confusions, fix this issue by renaming the constant to MPIC_PER_CPU_IRQS_NR and set it to 29, the number of per-CPU IRQs. Update its use in mpic_is_percpu_irq() accordingly. Fixes: 0fa4ce746d1d ("irqchip/armada-370-xp: Re-enable per-CPU interrupts at resume time") Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner Cc: # The 29th interrupt is not used in any device-tree --- drivers/irqchip/irq-armada-370-xp.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index f8658a232f21..83afc3a27812 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -133,7 +133,7 @@ #define MPIC_INT_FABRIC_MASK 0x54 #define MPIC_INT_CAUSE_PERF(cpu) BIT(cpu) -#define MPIC_MAX_PER_CPU_IRQS 28 +#define MPIC_PER_CPU_IRQS_NR 29 /* IPI and MSI interrupt definitions for IPI platforms */ #define IPI_DOORBELL_NR 8 @@ -202,7 +202,7 @@ static inline bool mpic_is_ipi_available(struct mpic *mpic) static inline bool mpic_is_percpu_irq(irq_hw_number_t hwirq) { - return hwirq <= MPIC_MAX_PER_CPU_IRQS; + return hwirq < MPIC_PER_CPU_IRQS_NR; } /* @@ -545,7 +545,7 @@ static void mpic_smp_cpu_init(struct mpic *mpic) static void mpic_reenable_percpu(struct mpic *mpic) { /* Re-enable per-CPU interrupts that were enabled before suspend */ - for (irq_hw_number_t i = 0; i < MPIC_MAX_PER_CPU_IRQS; i++) { + for (irq_hw_number_t i = 0; i < MPIC_PER_CPU_IRQS_NR; i++) { unsigned int virq = irq_linear_revmap(mpic->domain, i); struct irq_data *d; From 4042a965a5e62c8d298d642cbf72b14f41687319 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Wed, 7 Aug 2024 18:41:02 +0200 Subject: [PATCH 070/180] irqchip/armada-370-xp: Iterate only valid bits of the per-CPU interrupt cause register MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use MPIC_PER_CPU_IRQS_NR (29) bound instead of BITS_PER_LONG (32) when iterating the bits of the per-CPU interrupt cause register, since there are only 29 per-CPU interrupts. The top 3 bits are always zero anyway. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner --- drivers/irqchip/irq-armada-370-xp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 83afc3a27812..36d1bac8a99f 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -678,7 +678,7 @@ static void mpic_handle_cascade_irq(struct irq_desc *desc) cause = readl_relaxed(mpic->per_cpu + MPIC_PPI_CAUSE); cpuid = cpu_logical_map(smp_processor_id()); - for_each_set_bit(i, &cause, BITS_PER_LONG) { + for_each_set_bit(i, &cause, MPIC_PER_CPU_IRQS_NR) { irqsrc = readl_relaxed(mpic->base + MPIC_INT_SOURCE_CTL(i)); /* Check if the interrupt is not masked on current CPU. From d6ca3f440239fb4fa85228ead4c5e8b286645b7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Wed, 7 Aug 2024 18:41:03 +0200 Subject: [PATCH 071/180] irqchip/armada-370-xp: Allow mapping only per-CPU interrupts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On platforms where MPIC is not the top-level interrupt controller the driver currently only supports handling of the per-CPU interrupts (the first 29 interrupts). This is obvious from the code of mpic_handle_cascade_irq(), which reads only one cause register. Bound the number of available interrupts in the interrupt domain to 29 for these platforms. The corresponding device-trees refer only to per-CPU interrupts via MPIC, the other interrupts are referred to via GIC. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner --- drivers/irqchip/irq-armada-370-xp.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 36d1bac8a99f..4f3f99af12b2 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -848,6 +848,19 @@ static int __init mpic_of_init(struct device_node *node, struct device_node *par for (irq_hw_number_t i = 0; i < nr_irqs; i++) writel(i, mpic->base + MPIC_INT_CLEAR_ENABLE); + /* + * Initialize mpic->parent_irq before calling any other functions, since + * it is used to distinguish between IPI and non-IPI platforms. + */ + mpic->parent_irq = irq_of_parse_and_map(node, 0); + + /* + * On non-IPI platforms the driver currently supports only the per-CPU + * interrupts (the first 29 interrupts). See mpic_handle_cascade_irq(). + */ + if (!mpic_is_ipi_available(mpic)) + nr_irqs = MPIC_PER_CPU_IRQS_NR; + mpic->domain = irq_domain_add_linear(node, nr_irqs, &mpic_irq_ops, mpic); if (!mpic->domain) { pr_err("%pOF: Unable to add IRQ domain\n", node); @@ -856,12 +869,6 @@ static int __init mpic_of_init(struct device_node *node, struct device_node *par irq_domain_update_bus_token(mpic->domain, DOMAIN_BUS_WIRED); - /* - * Initialize mpic->parent_irq before calling any other functions, since - * it is used to distinguish between IPI and non-IPI platforms. - */ - mpic->parent_irq = irq_of_parse_and_map(node, 0); - /* Setup for the boot CPU */ mpic_perf_init(mpic); mpic_smp_cpu_init(mpic); From b77c6a73e10ae16b19999bebc6ca1413739dfe86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Beh=C3=BAn?= Date: Wed, 7 Aug 2024 18:41:04 +0200 Subject: [PATCH 072/180] irqchip/armada-370-xp: Use mpic_is_ipi_available() in mpic_of_init() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mpic_of_init() contains the last case where the open coded IPI support condition needs to be replaced with mpic_is_ipi_available() to keep the code consistent. Signed-off-by: Marek Behún Signed-off-by: Thomas Gleixner --- drivers/irqchip/irq-armada-370-xp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 4f3f99af12b2..d7c5ef248474 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -879,7 +879,7 @@ static int __init mpic_of_init(struct device_node *node, struct device_node *par return err; } - if (mpic->parent_irq <= 0) { + if (mpic_is_ipi_available(mpic)) { irq_set_default_host(mpic->domain); set_handle_irq(mpic_handle_irq); #ifdef CONFIG_SMP From 76bee035c6add05841addc3f31b41cd726b912c4 Mon Sep 17 00:00:00 2001 From: Zhang Zekun Date: Thu, 8 Aug 2024 11:15:52 +0800 Subject: [PATCH 073/180] irqchip/mbigen: Simplify code logic with for_each_child_of_node_scoped() for_each_child_of_node_scoped() handles the device_node automaticlly, so switching over to it removes the device node cleanups and allows to return directly from the loop. Signed-off-by: Zhang Zekun Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240808031552.3156-1-zhangzekun11@huawei.com --- drivers/irqchip/irq-mbigen.c | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/drivers/irqchip/irq-mbigen.c b/drivers/irqchip/irq-mbigen.c index 093fd42893a7..12919836dadb 100644 --- a/drivers/irqchip/irq-mbigen.c +++ b/drivers/irqchip/irq-mbigen.c @@ -222,37 +222,27 @@ static int mbigen_of_create_domain(struct platform_device *pdev, struct mbigen_device *mgn_chip) { struct platform_device *child; - struct device_node *np; u32 num_pins; - int ret = 0; - for_each_child_of_node(pdev->dev.of_node, np) { + for_each_child_of_node_scoped(pdev->dev.of_node, np) { if (!of_property_read_bool(np, "interrupt-controller")) continue; child = of_platform_device_create(np, NULL, NULL); - if (!child) { - ret = -ENOMEM; - break; - } + if (!child) + return -ENOMEM; if (of_property_read_u32(child->dev.of_node, "num-pins", &num_pins) < 0) { dev_err(&pdev->dev, "No num-pins property\n"); - ret = -EINVAL; - break; + return -EINVAL; } - if (!mbigen_create_device_domain(&child->dev, num_pins, mgn_chip)) { - ret = -ENOMEM; - break; - } + if (!mbigen_create_device_domain(&child->dev, num_pins, mgn_chip)) + return -ENOMEM; } - if (ret) - of_node_put(np); - - return ret; + return 0; } #ifdef CONFIG_ACPI From 15e46124ec937bb0ab530634dce4550947f53133 Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Thu, 8 Aug 2024 12:41:16 +0200 Subject: [PATCH 074/180] genirq/irq_sim: Remove unused irq_sim_work_ctx:: Irq_base Since commit 337cbeb2c13e ("genirq/irq_sim: Simplify the API"), irq_sim_work_ctx::irq_base is unused. Drop it. Found by https://github.com/jirislaby/clang-struct. Signed-off-by: Jiri Slaby (SUSE) Signed-off-by: Thomas Gleixner Acked-by: Bartosz Golaszewski Link: https://lore.kernel.org/all/20240808104118.430670-1-jirislaby@kernel.org --- kernel/irq/irq_sim.c | 1 - 1 file changed, 1 deletion(-) diff --git a/kernel/irq/irq_sim.c b/kernel/irq/irq_sim.c index 3d4036db15ac..1a3d483548e2 100644 --- a/kernel/irq/irq_sim.c +++ b/kernel/irq/irq_sim.c @@ -13,7 +13,6 @@ struct irq_sim_work_ctx { struct irq_work work; - int irq_base; unsigned int irq_count; unsigned long *pending; struct irq_domain *domain; From a09cdb8f564613769142a60400bb5160864c3269 Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Thu, 8 Aug 2024 12:41:17 +0200 Subject: [PATCH 075/180] genirq: Remove unused irq_chip_generic:: {type,polarity}_cache The type_cache and polarity_cache members of struct irq_chip_generic are unused. Remove them both along with their kernel-doc. Found by https://github.com/jirislaby/clang-struct. Signed-off-by: Jiri Slaby (SUSE) Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240808104118.430670-2-jirislaby@kernel.org --- include/linux/irq.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/include/linux/irq.h b/include/linux/irq.h index 1f5dbf1f92c9..00490d6ead65 100644 --- a/include/linux/irq.h +++ b/include/linux/irq.h @@ -1040,8 +1040,6 @@ struct irq_chip_type { * @irq_base: Interrupt base nr for this chip * @irq_cnt: Number of interrupts handled by this chip * @mask_cache: Cached mask register shared between all chip types - * @type_cache: Cached type register - * @polarity_cache: Cached polarity register * @wake_enabled: Interrupt can wakeup from suspend * @wake_active: Interrupt is marked as an wakeup from suspend source * @num_ct: Number of available irq_chip_type instances (usually 1) @@ -1068,8 +1066,6 @@ struct irq_chip_generic { unsigned int irq_base; unsigned int irq_cnt; u32 mask_cache; - u32 type_cache; - u32 polarity_cache; u32 wake_enabled; u32 wake_active; unsigned int num_ct; From 60029162a0458832ab2bcfc6fd4986bfd9ca0f55 Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Thu, 8 Aug 2024 12:41:18 +0200 Subject: [PATCH 076/180] genirq: Remove irq_chip_regs:: Polarity The polarity member of struct irq_chip_regs is unused. Remove it along with its kernel-doc. Found by https://github.com/jirislaby/clang-struct. Signed-off-by: Jiri Slaby (SUSE) Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240808104118.430670-3-jirislaby@kernel.org --- include/linux/irq.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/linux/irq.h b/include/linux/irq.h index 00490d6ead65..fa711f80957b 100644 --- a/include/linux/irq.h +++ b/include/linux/irq.h @@ -991,7 +991,6 @@ void irq_init_desc(unsigned int irq); * @ack: Ack register offset to reg_base * @eoi: Eoi register offset to reg_base * @type: Type configuration register offset to reg_base - * @polarity: Polarity configuration register offset to reg_base */ struct irq_chip_regs { unsigned long enable; @@ -1000,7 +999,6 @@ struct irq_chip_regs { unsigned long ack; unsigned long eoi; unsigned long type; - unsigned long polarity; }; /** From 70114e7f7585ef078c2b7033ee14218f95f55e22 Mon Sep 17 00:00:00 2001 From: Matti Vaittinen Date: Thu, 8 Aug 2024 15:34:02 +0300 Subject: [PATCH 077/180] irqdomain: Simplify simple and legacy domain creation irq_domain_create_simple() and irq_domain_create_legacy() use __irq_domain_instantiate(), but have extra handling of allocating interrupt descriptors and associating interrupts in them. Some of that is duplicated. There are also call sites which have conditonals to invoke different interrupt domain creator functions, where one of them is usually irq_domain_create_legacy(). Alternatively they associate the interrupts for the legacy case after creating the domain. Moving the extra logic of irq_domain_create_simple()/legacy() into __irq_domain_instantiate() allows to consolidate that. Introduce hwirq_base and virq_base members in the irq_domain_info structure, which allows to transport the required information and add the conditional interrupt descriptor allocation and interrupt association into __irq_domain_instantiate(). This reduces irq_domain_create_legacy() and irq_domain_create_simple() to trivial wrappers which fill in the info structure and allows call sites which must support the legacy case along with more modern mechanism to select the domain type via the parameters of the info struct. [ tglx: Massaged change log ] Suggested-by: Thomas Gleixner Signed-off-by: Matti Vaittinen Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/32d07bd79eb2b5416e24da9e9e8fe5955423dcf9.1723120028.git.mazziesaccount@gmail.com --- include/linux/irqdomain.h | 5 +++ kernel/irq/irqdomain.c | 74 ++++++++++++++++++++++----------------- 2 files changed, 46 insertions(+), 33 deletions(-) diff --git a/include/linux/irqdomain.h b/include/linux/irqdomain.h index de6105f68fec..bfcffa2c7047 100644 --- a/include/linux/irqdomain.h +++ b/include/linux/irqdomain.h @@ -291,6 +291,9 @@ struct irq_domain_chip_generic_info; * @hwirq_max: Maximum number of interrupts supported by controller * @direct_max: Maximum value of direct maps; * Use ~0 for no limit; 0 for no direct mapping + * @hwirq_base: The first hardware interrupt number (legacy domains only) + * @virq_base: The first Linux interrupt number for legacy domains to + * immediately associate the interrupts after domain creation * @bus_token: Domain bus token * @ops: Domain operation callbacks * @host_data: Controller private data pointer @@ -307,6 +310,8 @@ struct irq_domain_info { unsigned int size; irq_hw_number_t hwirq_max; int direct_max; + unsigned int hwirq_base; + unsigned int virq_base; enum irq_domain_bus_token bus_token; const struct irq_domain_ops *ops; void *host_data; diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index cea8f6874b1f..7625e424f85a 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -267,13 +267,20 @@ static void irq_domain_free(struct irq_domain *domain) kfree(domain); } -/** - * irq_domain_instantiate() - Instantiate a new irq domain data structure - * @info: Domain information pointer pointing to the information for this domain - * - * Return: A pointer to the instantiated irq domain or an ERR_PTR value. - */ -struct irq_domain *irq_domain_instantiate(const struct irq_domain_info *info) +static void irq_domain_instantiate_descs(const struct irq_domain_info *info) +{ + if (!IS_ENABLED(CONFIG_SPARSE_IRQ)) + return; + + if (irq_alloc_descs(info->virq_base, info->virq_base, info->size, + of_node_to_nid(to_of_node(info->fwnode))) < 0) { + pr_info("Cannot allocate irq_descs @ IRQ%d, assuming pre-allocated\n", + info->virq_base); + } +} + +static struct irq_domain *__irq_domain_instantiate(const struct irq_domain_info *info, + bool cond_alloc_descs) { struct irq_domain *domain; int err; @@ -306,6 +313,15 @@ struct irq_domain *irq_domain_instantiate(const struct irq_domain_info *info) __irq_domain_publish(domain); + if (cond_alloc_descs && info->virq_base > 0) + irq_domain_instantiate_descs(info); + + /* Legacy interrupt domains have a fixed Linux interrupt number */ + if (info->virq_base > 0) { + irq_domain_associate_many(domain, info->virq_base, info->hwirq_base, + info->size - info->hwirq_base); + } + return domain; err_domain_gc_remove: @@ -315,6 +331,17 @@ err_domain_free: irq_domain_free(domain); return ERR_PTR(err); } + +/** + * irq_domain_instantiate() - Instantiate a new irq domain data structure + * @info: Domain information pointer pointing to the information for this domain + * + * Return: A pointer to the instantiated irq domain or an ERR_PTR value. + */ +struct irq_domain *irq_domain_instantiate(const struct irq_domain_info *info) +{ + return __irq_domain_instantiate(info, false); +} EXPORT_SYMBOL_GPL(irq_domain_instantiate); /** @@ -413,28 +440,13 @@ struct irq_domain *irq_domain_create_simple(struct fwnode_handle *fwnode, .fwnode = fwnode, .size = size, .hwirq_max = size, + .virq_base = first_irq, .ops = ops, .host_data = host_data, }; - struct irq_domain *domain; + struct irq_domain *domain = __irq_domain_instantiate(&info, true); - domain = irq_domain_instantiate(&info); - if (IS_ERR(domain)) - return NULL; - - if (first_irq > 0) { - if (IS_ENABLED(CONFIG_SPARSE_IRQ)) { - /* attempt to allocated irq_descs */ - int rc = irq_alloc_descs(first_irq, first_irq, size, - of_node_to_nid(to_of_node(fwnode))); - if (rc < 0) - pr_info("Cannot allocate irq_descs @ IRQ%d, assuming pre-allocated\n", - first_irq); - } - irq_domain_associate_many(domain, first_irq, 0, size); - } - - return domain; + return IS_ERR(domain) ? NULL : domain; } EXPORT_SYMBOL_GPL(irq_domain_create_simple); @@ -476,18 +488,14 @@ struct irq_domain *irq_domain_create_legacy(struct fwnode_handle *fwnode, .fwnode = fwnode, .size = first_hwirq + size, .hwirq_max = first_hwirq + size, + .hwirq_base = first_hwirq, + .virq_base = first_irq, .ops = ops, .host_data = host_data, }; - struct irq_domain *domain; + struct irq_domain *domain = irq_domain_instantiate(&info); - domain = irq_domain_instantiate(&info); - if (IS_ERR(domain)) - return NULL; - - irq_domain_associate_many(domain, first_irq, first_hwirq, size); - - return domain; + return IS_ERR(domain) ? NULL : domain; } EXPORT_SYMBOL_GPL(irq_domain_create_legacy); From 1bf2c92829274e7c815d06d7b3196a967ff70917 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Thu, 8 Aug 2024 22:19:41 +0200 Subject: [PATCH 078/180] irqdomain: Cleanup domain name allocation irq_domain_set_name() is truly unreadable gunk. Clean it up before adding more. Signed-off-by: Thomas Gleixner Reviewed-by: Matti Vaittinen Link: https://lore.kernel.org/all/874j7uvkbm.ffs@tglx --- kernel/irq/irqdomain.c | 106 +++++++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 51 deletions(-) diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index 7625e424f85a..72ab60187103 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -128,72 +128,76 @@ void irq_domain_free_fwnode(struct fwnode_handle *fwnode) } EXPORT_SYMBOL_GPL(irq_domain_free_fwnode); -static int irq_domain_set_name(struct irq_domain *domain, - const struct fwnode_handle *fwnode, - enum irq_domain_bus_token bus_token) +static int alloc_name(struct irq_domain *domain, char *base, enum irq_domain_bus_token bus_token) +{ + domain->name = bus_token ? kasprintf(GFP_KERNEL, "%s-%d", base, bus_token) : + kasprintf(GFP_KERNEL, "%s", base); + if (!domain->name) + return -ENOMEM; + + domain->flags |= IRQ_DOMAIN_NAME_ALLOCATED; + return 0; +} + +static int alloc_fwnode_name(struct irq_domain *domain, const struct fwnode_handle *fwnode, + enum irq_domain_bus_token bus_token) +{ + char *name = bus_token ? kasprintf(GFP_KERNEL, "%pfw-%d", fwnode, bus_token) : + kasprintf(GFP_KERNEL, "%pfw", fwnode); + + if (!name) + return -ENOMEM; + + /* + * fwnode paths contain '/', which debugfs is legitimately unhappy + * about. Replace them with ':', which does the trick and is not as + * offensive as '\'... + */ + domain->name = strreplace(name, '/', ':'); + domain->flags |= IRQ_DOMAIN_NAME_ALLOCATED; + return 0; +} + +static int alloc_unknown_name(struct irq_domain *domain, enum irq_domain_bus_token bus_token) { static atomic_t unknown_domains; - struct irqchip_fwid *fwid; + int id = atomic_inc_return(&unknown_domains); + domain->name = bus_token ? kasprintf(GFP_KERNEL, "unknown-%d-%d", id, bus_token) : + kasprintf(GFP_KERNEL, "unknown-%d", id); + + if (!domain->name) + return -ENOMEM; + domain->flags |= IRQ_DOMAIN_NAME_ALLOCATED; + return 0; +} + +static int irq_domain_set_name(struct irq_domain *domain, const struct fwnode_handle *fwnode, + enum irq_domain_bus_token bus_token) +{ if (is_fwnode_irqchip(fwnode)) { - fwid = container_of(fwnode, struct irqchip_fwid, fwnode); + struct irqchip_fwid *fwid = container_of(fwnode, struct irqchip_fwid, fwnode); switch (fwid->type) { case IRQCHIP_FWNODE_NAMED: case IRQCHIP_FWNODE_NAMED_ID: - domain->name = bus_token ? - kasprintf(GFP_KERNEL, "%s-%d", - fwid->name, bus_token) : - kstrdup(fwid->name, GFP_KERNEL); - if (!domain->name) - return -ENOMEM; - domain->flags |= IRQ_DOMAIN_NAME_ALLOCATED; - break; + return alloc_name(domain, fwid->name, bus_token); default: domain->name = fwid->name; - if (bus_token) { - domain->name = kasprintf(GFP_KERNEL, "%s-%d", - fwid->name, bus_token); - if (!domain->name) - return -ENOMEM; - domain->flags |= IRQ_DOMAIN_NAME_ALLOCATED; - } - break; + if (bus_token) + return alloc_name(domain, fwid->name, bus_token); } - } else if (is_of_node(fwnode) || is_acpi_device_node(fwnode) || - is_software_node(fwnode)) { - char *name; - /* - * fwnode paths contain '/', which debugfs is legitimately - * unhappy about. Replace them with ':', which does - * the trick and is not as offensive as '\'... - */ - name = bus_token ? - kasprintf(GFP_KERNEL, "%pfw-%d", fwnode, bus_token) : - kasprintf(GFP_KERNEL, "%pfw", fwnode); - if (!name) - return -ENOMEM; - - domain->name = strreplace(name, '/', ':'); - domain->flags |= IRQ_DOMAIN_NAME_ALLOCATED; + } else if (is_of_node(fwnode) || is_acpi_device_node(fwnode) || is_software_node(fwnode)) { + return alloc_fwnode_name(domain, fwnode, bus_token); } - if (!domain->name) { - if (fwnode) - pr_err("Invalid fwnode type for irqdomain\n"); - domain->name = bus_token ? - kasprintf(GFP_KERNEL, "unknown-%d-%d", - atomic_inc_return(&unknown_domains), - bus_token) : - kasprintf(GFP_KERNEL, "unknown-%d", - atomic_inc_return(&unknown_domains)); - if (!domain->name) - return -ENOMEM; - domain->flags |= IRQ_DOMAIN_NAME_ALLOCATED; - } + if (domain->name) + return 0; - return 0; + if (fwnode) + pr_err("Invalid fwnode type for irqdomain\n"); + return alloc_unknown_name(domain, bus_token); } static struct irq_domain *__irq_domain_create(const struct irq_domain_info *info) From 1e7c05292531e5b6bebe409cd531ed4ec0b2ff56 Mon Sep 17 00:00:00 2001 From: Matti Vaittinen Date: Thu, 8 Aug 2024 22:23:06 +0200 Subject: [PATCH 079/180] irqdomain: Allow giving name suffix for domain Devices can provide multiple interrupt lines. One reason for this is that a device has multiple subfunctions, each providing its own interrupt line. Another reason is that a device can be designed to be used (also) on a system where some of the interrupts can be routed to another processor. A line often further acts as a demultiplex for specific interrupts and has it's respective set of interrupt (status, mask, ack, ...) registers. Regmap supports the handling of these registers and demultiplexing interrupts, but the interrupt domain code ends up assigning the same name for the per interrupt line domains. This causes a naming collision in the debugFS code and leads to confusion, as /proc/interrupts shows two separate interrupts with the same domain name and hardware interrupt number. Instead of adding a workaround in regmap or driver code, allow giving a name suffix for the domain name when the domain is created. Add a name_suffix field in the irq_domain_info structure and make irq_domain_instantiate() use this suffix if it is given when a domain is created. [ tglx: Adopt it to the cleanup patch and fixup the invalid NULL return ] Signed-off-by: Matti Vaittinen Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/871q2yvk5x.ffs@tglx --- include/linux/irqdomain.h | 3 +++ kernel/irq/irqdomain.c | 30 +++++++++++++++++++++++------- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/include/linux/irqdomain.h b/include/linux/irqdomain.h index bfcffa2c7047..e432b6a12a32 100644 --- a/include/linux/irqdomain.h +++ b/include/linux/irqdomain.h @@ -295,6 +295,8 @@ struct irq_domain_chip_generic_info; * @virq_base: The first Linux interrupt number for legacy domains to * immediately associate the interrupts after domain creation * @bus_token: Domain bus token + * @name_suffix: Optional name suffix to avoid collisions when multiple + * domains are added using same fwnode * @ops: Domain operation callbacks * @host_data: Controller private data pointer * @dgc_info: Geneneric chip information structure pointer used to @@ -313,6 +315,7 @@ struct irq_domain_info { unsigned int hwirq_base; unsigned int virq_base; enum irq_domain_bus_token bus_token; + const char *name_suffix; const struct irq_domain_ops *ops; void *host_data; #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index 72ab60187103..01001eb615ec 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -140,11 +140,14 @@ static int alloc_name(struct irq_domain *domain, char *base, enum irq_domain_bus } static int alloc_fwnode_name(struct irq_domain *domain, const struct fwnode_handle *fwnode, - enum irq_domain_bus_token bus_token) + enum irq_domain_bus_token bus_token, const char *suffix) { - char *name = bus_token ? kasprintf(GFP_KERNEL, "%pfw-%d", fwnode, bus_token) : - kasprintf(GFP_KERNEL, "%pfw", fwnode); + const char *sep = suffix ? "-" : ""; + const char *suf = suffix ? : ""; + char *name; + name = bus_token ? kasprintf(GFP_KERNEL, "%pfw-%s%s%d", fwnode, suf, sep, bus_token) : + kasprintf(GFP_KERNEL, "%pfw-%s", fwnode, suf); if (!name) return -ENOMEM; @@ -172,12 +175,25 @@ static int alloc_unknown_name(struct irq_domain *domain, enum irq_domain_bus_tok return 0; } -static int irq_domain_set_name(struct irq_domain *domain, const struct fwnode_handle *fwnode, - enum irq_domain_bus_token bus_token) +static int irq_domain_set_name(struct irq_domain *domain, const struct irq_domain_info *info) { + enum irq_domain_bus_token bus_token = info->bus_token; + const struct fwnode_handle *fwnode = info->fwnode; + if (is_fwnode_irqchip(fwnode)) { struct irqchip_fwid *fwid = container_of(fwnode, struct irqchip_fwid, fwnode); + /* + * The name_suffix is only intended to be used to avoid a name + * collision when multiple domains are created for a single + * device and the name is picked using a real device node. + * (Typical use-case is regmap-IRQ controllers for devices + * providing more than one physical IRQ.) There should be no + * need to use name_suffix with irqchip-fwnode. + */ + if (info->name_suffix) + return -EINVAL; + switch (fwid->type) { case IRQCHIP_FWNODE_NAMED: case IRQCHIP_FWNODE_NAMED_ID: @@ -189,7 +205,7 @@ static int irq_domain_set_name(struct irq_domain *domain, const struct fwnode_ha } } else if (is_of_node(fwnode) || is_acpi_device_node(fwnode) || is_software_node(fwnode)) { - return alloc_fwnode_name(domain, fwnode, bus_token); + return alloc_fwnode_name(domain, fwnode, bus_token, info->name_suffix); } if (domain->name) @@ -215,7 +231,7 @@ static struct irq_domain *__irq_domain_create(const struct irq_domain_info *info if (!domain) return ERR_PTR(-ENOMEM); - err = irq_domain_set_name(domain, info->fwnode, info->bus_token); + err = irq_domain_set_name(domain, info); if (err) { kfree(domain); return ERR_PTR(err); From c0ece64497992473aabbcbb007e2afecc8d750a2 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 12 Aug 2024 22:29:39 +0300 Subject: [PATCH 080/180] irqdomain: Clarify checks for bus_token The code uses if (bus_token) and if (bus_token == DOMAIN_BUS_ANY). Since bus_token is an enum, the latter is more robust against changes. Convert all !bus_token checks to explicitely check for DOMAIN_BUS_ANY. Signed-off-by: Andy Shevchenko Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240812193101.1266625-2-andriy.shevchenko@linux.intel.com --- kernel/irq/irqdomain.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index 01001eb615ec..18d253e10e87 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -130,8 +130,10 @@ EXPORT_SYMBOL_GPL(irq_domain_free_fwnode); static int alloc_name(struct irq_domain *domain, char *base, enum irq_domain_bus_token bus_token) { - domain->name = bus_token ? kasprintf(GFP_KERNEL, "%s-%d", base, bus_token) : - kasprintf(GFP_KERNEL, "%s", base); + if (bus_token == DOMAIN_BUS_ANY) + domain->name = kasprintf(GFP_KERNEL, "%s", base); + else + domain->name = kasprintf(GFP_KERNEL, "%s-%d", base, bus_token); if (!domain->name) return -ENOMEM; @@ -146,8 +148,10 @@ static int alloc_fwnode_name(struct irq_domain *domain, const struct fwnode_hand const char *suf = suffix ? : ""; char *name; - name = bus_token ? kasprintf(GFP_KERNEL, "%pfw-%s%s%d", fwnode, suf, sep, bus_token) : - kasprintf(GFP_KERNEL, "%pfw-%s", fwnode, suf); + if (bus_token == DOMAIN_BUS_ANY) + name = kasprintf(GFP_KERNEL, "%pfw-%s", fwnode, suf); + else + name = kasprintf(GFP_KERNEL, "%pfw-%s%s%d", fwnode, suf, sep, bus_token); if (!name) return -ENOMEM; @@ -166,11 +170,13 @@ static int alloc_unknown_name(struct irq_domain *domain, enum irq_domain_bus_tok static atomic_t unknown_domains; int id = atomic_inc_return(&unknown_domains); - domain->name = bus_token ? kasprintf(GFP_KERNEL, "unknown-%d-%d", id, bus_token) : - kasprintf(GFP_KERNEL, "unknown-%d", id); - + if (bus_token == DOMAIN_BUS_ANY) + domain->name = kasprintf(GFP_KERNEL, "unknown-%d", id); + else + domain->name = kasprintf(GFP_KERNEL, "unknown-%d-%d", id, bus_token); if (!domain->name) return -ENOMEM; + domain->flags |= IRQ_DOMAIN_NAME_ALLOCATED; return 0; } @@ -200,7 +206,7 @@ static int irq_domain_set_name(struct irq_domain *domain, const struct irq_domai return alloc_name(domain, fwid->name, bus_token); default: domain->name = fwid->name; - if (bus_token) + if (bus_token != DOMAIN_BUS_ANY) return alloc_name(domain, fwid->name, bus_token); } From 7b9414cb2d370b7c5149b37f585b077af2ae211b Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 12 Aug 2024 22:29:40 +0300 Subject: [PATCH 081/180] irqdomain: Remove stray '-' in the domain name When the domain suffix is not supplied alloc_fwnode_name() unconditionally adds a separator. Fix the format strings to get rid of the stray '-' separator. Fixes: 1e7c05292531 ("irqdomain: Allow giving name suffix for domain") Signed-off-by: Andy Shevchenko Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240812193101.1266625-3-andriy.shevchenko@linux.intel.com --- kernel/irq/irqdomain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index 18d253e10e87..1acc5308fcb7 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -149,9 +149,9 @@ static int alloc_fwnode_name(struct irq_domain *domain, const struct fwnode_hand char *name; if (bus_token == DOMAIN_BUS_ANY) - name = kasprintf(GFP_KERNEL, "%pfw-%s", fwnode, suf); + name = kasprintf(GFP_KERNEL, "%pfw%s%s", fwnode, sep, suf); else - name = kasprintf(GFP_KERNEL, "%pfw-%s%s%d", fwnode, suf, sep, bus_token); + name = kasprintf(GFP_KERNEL, "%pfw%s%s-%d", fwnode, sep, suf, bus_token); if (!name) return -ENOMEM; From 38cd4cee73a87c40a3e9ef31c0ca7b179fd282f0 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Mon, 12 Aug 2024 12:51:04 +0200 Subject: [PATCH 082/180] timers: Add sparse annotation for timer_sync_wait_running(). timer_sync_wait_running() first releases two locks and then acquires them again. This is unexpected and sparse complains about it. Add sparse annotation for timer_sync_wait_running() to note that the locking is expected. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240812105326.2240000-2-bigeasy@linutronix.de --- kernel/time/timer.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/time/timer.c b/kernel/time/timer.c index 64b0d8a0aa0f..429232d8659a 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -1561,6 +1561,8 @@ static inline void timer_base_unlock_expiry(struct timer_base *base) * the waiter to acquire the lock and make progress. */ static void timer_sync_wait_running(struct timer_base *base) + __releases(&base->lock) __releases(&base->expiry_lock) + __acquires(&base->expiry_lock) __acquires(&base->lock) { if (atomic_read(&base->timer_waiters)) { raw_spin_unlock_irq(&base->lock); From 330dd6d9c0fce69718b53ca0bc4f2e3920f7f600 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Mon, 12 Aug 2024 12:51:05 +0200 Subject: [PATCH 083/180] hrtimer: Annotate hrtimer_cpu_base_.*_expiry() for sparse. The two hrtimer_cpu_base_.*_expiry() functions are wrappers around the locking functions and sparse complains about the missing counterpart. Add sparse annotation to denote that this bevaviour is expected. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240812105326.2240000-3-bigeasy@linutronix.de --- kernel/time/hrtimer.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index b8ee320208d4..f56ef23aad90 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1351,11 +1351,13 @@ static void hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base) } static void hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base) + __acquires(&base->softirq_expiry_lock) { spin_lock(&base->softirq_expiry_lock); } static void hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base) + __releases(&base->softirq_expiry_lock) { spin_unlock(&base->softirq_expiry_lock); } From 24d02c4e53e2f02da16b2ae8a1bc92553110ca25 Mon Sep 17 00:00:00 2001 From: Matti Vaittinen Date: Tue, 13 Aug 2024 14:34:27 +0300 Subject: [PATCH 084/180] irqdomain: Always associate interrupts for legacy domains The unification of irq_domain_create_legacy() missed the fact that interrupts must be associated even when the Linux interrupt number provided in the first_irq argument is 0. This breaks all call sites of irq_domain_create_legacy() which supply 0 as the first_irq argument. Enforce the association for legacy domains in __irq_domain_instantiate() to cure this. [ tglx: Massaged it slightly. ] Fixes: 70114e7f7585 ("irqdomain: Simplify simple and legacy domain creation") Reported-by: Jiaxun Yang Signed-off-by Matti Vaittinen Signed-off-by: Thomas Gleixner Tested-by: Jiaxun Yang Link: https://lore.kernel.org/all/c3379142-10bc-4f14-b8ac-a46927aeac38@gmail.com --- kernel/irq/irqdomain.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index 1acc5308fcb7..5df8780100bb 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -306,7 +306,7 @@ static void irq_domain_instantiate_descs(const struct irq_domain_info *info) } static struct irq_domain *__irq_domain_instantiate(const struct irq_domain_info *info, - bool cond_alloc_descs) + bool cond_alloc_descs, bool force_associate) { struct irq_domain *domain; int err; @@ -342,8 +342,12 @@ static struct irq_domain *__irq_domain_instantiate(const struct irq_domain_info if (cond_alloc_descs && info->virq_base > 0) irq_domain_instantiate_descs(info); - /* Legacy interrupt domains have a fixed Linux interrupt number */ - if (info->virq_base > 0) { + /* + * Legacy interrupt domains have a fixed Linux interrupt number + * associated. Other interrupt domains can request association by + * providing a Linux interrupt number > 0. + */ + if (force_associate || info->virq_base > 0) { irq_domain_associate_many(domain, info->virq_base, info->hwirq_base, info->size - info->hwirq_base); } @@ -366,7 +370,7 @@ err_domain_free: */ struct irq_domain *irq_domain_instantiate(const struct irq_domain_info *info) { - return __irq_domain_instantiate(info, false); + return __irq_domain_instantiate(info, false, false); } EXPORT_SYMBOL_GPL(irq_domain_instantiate); @@ -470,7 +474,7 @@ struct irq_domain *irq_domain_create_simple(struct fwnode_handle *fwnode, .ops = ops, .host_data = host_data, }; - struct irq_domain *domain = __irq_domain_instantiate(&info, true); + struct irq_domain *domain = __irq_domain_instantiate(&info, true, false); return IS_ERR(domain) ? NULL : domain; } @@ -519,7 +523,7 @@ struct irq_domain *irq_domain_create_legacy(struct fwnode_handle *fwnode, .ops = ops, .host_data = host_data, }; - struct irq_domain *domain = irq_domain_instantiate(&info); + struct irq_domain *domain = __irq_domain_instantiate(&info, false, true); return IS_ERR(domain) ? NULL : domain; } From e68ac2b488495fa4d127d6105ce633849859957a Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Thu, 15 Aug 2024 11:15:40 -0600 Subject: [PATCH 085/180] softirq: Remove unused 'action' parameter from action callback When soft interrupt actions are called, they are passed a pointer to the struct softirq action which contains the action's function pointer. This pointer isn't useful, as the action callback already knows what function it is. And since each callback handles a specific soft interrupt, the callback also knows which soft interrupt number is running. No soft interrupt action callback actually uses this parameter, so remove it from the function pointer signature. This clarifies that soft interrupt actions are global routines and makes it slightly cheaper to call them. Signed-off-by: Caleb Sander Mateos Signed-off-by: Thomas Gleixner Reviewed-by: Jens Axboe Link: https://lore.kernel.org/all/20240815171549.3260003-1-csander@purestorage.com --- block/blk-mq.c | 2 +- include/linux/interrupt.h | 4 ++-- kernel/rcu/tiny.c | 2 +- kernel/rcu/tree.c | 2 +- kernel/sched/fair.c | 2 +- kernel/softirq.c | 15 +++++++-------- kernel/time/hrtimer.c | 2 +- kernel/time/timer.c | 2 +- lib/irq_poll.c | 2 +- net/core/dev.c | 4 ++-- 10 files changed, 18 insertions(+), 19 deletions(-) diff --git a/block/blk-mq.c b/block/blk-mq.c index e3c3c0c21b55..aa28157b1aaf 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -1128,7 +1128,7 @@ static void blk_complete_reqs(struct llist_head *list) rq->q->mq_ops->complete(rq); } -static __latent_entropy void blk_done_softirq(struct softirq_action *h) +static __latent_entropy void blk_done_softirq(void) { blk_complete_reqs(this_cpu_ptr(&blk_cpu_done)); } diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index 3f30c88e0b4c..694de61e0b38 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -594,7 +594,7 @@ extern const char * const softirq_to_name[NR_SOFTIRQS]; struct softirq_action { - void (*action)(struct softirq_action *); + void (*action)(void); }; asmlinkage void do_softirq(void); @@ -609,7 +609,7 @@ static inline void do_softirq_post_smp_call_flush(unsigned int unused) } #endif -extern void open_softirq(int nr, void (*action)(struct softirq_action *)); +extern void open_softirq(int nr, void (*action)(void)); extern void softirq_init(void); extern void __raise_softirq_irqoff(unsigned int nr); diff --git a/kernel/rcu/tiny.c b/kernel/rcu/tiny.c index 4402d6f5f857..b3b3ce34df63 100644 --- a/kernel/rcu/tiny.c +++ b/kernel/rcu/tiny.c @@ -105,7 +105,7 @@ static inline bool rcu_reclaim_tiny(struct rcu_head *head) } /* Invoke the RCU callbacks whose grace period has elapsed. */ -static __latent_entropy void rcu_process_callbacks(struct softirq_action *unused) +static __latent_entropy void rcu_process_callbacks(void) { struct rcu_head *next, *list; unsigned long flags; diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index e641cc681901..93bd665637c0 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2855,7 +2855,7 @@ static __latent_entropy void rcu_core(void) queue_work_on(rdp->cpu, rcu_gp_wq, &rdp->strict_work); } -static void rcu_core_si(struct softirq_action *h) +static void rcu_core_si(void) { rcu_core(); } diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 9057584ec06d..8dc9385f6da4 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -12483,7 +12483,7 @@ out: * - indirectly from a remote scheduler_tick() for NOHZ idle balancing * through the SMP cross-call nohz_csd_func() */ -static __latent_entropy void sched_balance_softirq(struct softirq_action *h) +static __latent_entropy void sched_balance_softirq(void) { struct rq *this_rq = this_rq(); enum cpu_idle_type idle = this_rq->idle_balance; diff --git a/kernel/softirq.c b/kernel/softirq.c index 02582017759a..d082e7840f88 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -551,7 +551,7 @@ restart: kstat_incr_softirqs_this_cpu(vec_nr); trace_softirq_entry(vec_nr); - h->action(h); + h->action(); trace_softirq_exit(vec_nr); if (unlikely(prev_count != preempt_count())) { pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n", @@ -700,7 +700,7 @@ void __raise_softirq_irqoff(unsigned int nr) or_softirq_pending(1UL << nr); } -void open_softirq(int nr, void (*action)(struct softirq_action *)) +void open_softirq(int nr, void (*action)(void)) { softirq_vec[nr].action = action; } @@ -760,8 +760,7 @@ static bool tasklet_clear_sched(struct tasklet_struct *t) return false; } -static void tasklet_action_common(struct softirq_action *a, - struct tasklet_head *tl_head, +static void tasklet_action_common(struct tasklet_head *tl_head, unsigned int softirq_nr) { struct tasklet_struct *list; @@ -805,16 +804,16 @@ static void tasklet_action_common(struct softirq_action *a, } } -static __latent_entropy void tasklet_action(struct softirq_action *a) +static __latent_entropy void tasklet_action(void) { workqueue_softirq_action(false); - tasklet_action_common(a, this_cpu_ptr(&tasklet_vec), TASKLET_SOFTIRQ); + tasklet_action_common(this_cpu_ptr(&tasklet_vec), TASKLET_SOFTIRQ); } -static __latent_entropy void tasklet_hi_action(struct softirq_action *a) +static __latent_entropy void tasklet_hi_action(void) { workqueue_softirq_action(true); - tasklet_action_common(a, this_cpu_ptr(&tasklet_hi_vec), HI_SOFTIRQ); + tasklet_action_common(this_cpu_ptr(&tasklet_hi_vec), HI_SOFTIRQ); } void tasklet_setup(struct tasklet_struct *t, diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index b8ee320208d4..836157e09e25 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1757,7 +1757,7 @@ static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now, } } -static __latent_entropy void hrtimer_run_softirq(struct softirq_action *h) +static __latent_entropy void hrtimer_run_softirq(void) { struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); unsigned long flags; diff --git a/kernel/time/timer.c b/kernel/time/timer.c index 64b0d8a0aa0f..760bbeb1f331 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -2440,7 +2440,7 @@ static void run_timer_base(int index) /* * This function runs timers and the timer-tq in bottom half context. */ -static __latent_entropy void run_timer_softirq(struct softirq_action *h) +static __latent_entropy void run_timer_softirq(void) { run_timer_base(BASE_LOCAL); if (IS_ENABLED(CONFIG_NO_HZ_COMMON)) { diff --git a/lib/irq_poll.c b/lib/irq_poll.c index 2d5329a42105..08b242bbdbdf 100644 --- a/lib/irq_poll.c +++ b/lib/irq_poll.c @@ -75,7 +75,7 @@ void irq_poll_complete(struct irq_poll *iop) } EXPORT_SYMBOL(irq_poll_complete); -static void __latent_entropy irq_poll_softirq(struct softirq_action *h) +static void __latent_entropy irq_poll_softirq(void) { struct list_head *list = this_cpu_ptr(&blk_cpu_iopoll); int rearm = 0, budget = irq_poll_budget; diff --git a/net/core/dev.c b/net/core/dev.c index 6ea1d20676fb..e24a3bcb496d 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -5247,7 +5247,7 @@ int netif_rx(struct sk_buff *skb) } EXPORT_SYMBOL(netif_rx); -static __latent_entropy void net_tx_action(struct softirq_action *h) +static __latent_entropy void net_tx_action(void) { struct softnet_data *sd = this_cpu_ptr(&softnet_data); @@ -6920,7 +6920,7 @@ static int napi_threaded_poll(void *data) return 0; } -static __latent_entropy void net_rx_action(struct softirq_action *h) +static __latent_entropy void net_rx_action(void) { struct softnet_data *sd = this_cpu_ptr(&softnet_data); unsigned long time_limit = jiffies + From 0b3af7591dbfd16ca45740cd90eb34be8b9a7175 Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Thu, 15 Aug 2024 19:26:07 +0800 Subject: [PATCH 086/180] irqchip/loongson-pch-msi: Switch to MSI parent domains Remove the global PCI/MSI irqdomain implementation and provide the required MSI parent functionality by filling in msi_parent_ops, so the PCI/MSI code can detect the new parent and setup per-device MSI domains. Signed-off-by: Huacai Chen Signed-off-by: Tianyang Zhang Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240815112608.26925-2-zhangtianyang@loongson.cn --- drivers/irqchip/Kconfig | 1 + drivers/irqchip/irq-loongson-pch-msi.c | 58 ++++++++++---------------- 2 files changed, 24 insertions(+), 35 deletions(-) diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index d078bdc48c38..341cd9ca5a05 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -685,6 +685,7 @@ config LOONGSON_PCH_MSI depends on PCI default MACH_LOONGSON64 select IRQ_DOMAIN_HIERARCHY + select IRQ_MSI_LIB select PCI_MSI help Support for the Loongson PCH MSI Controller. diff --git a/drivers/irqchip/irq-loongson-pch-msi.c b/drivers/irqchip/irq-loongson-pch-msi.c index dd4d699170f4..2242f63c66fc 100644 --- a/drivers/irqchip/irq-loongson-pch-msi.c +++ b/drivers/irqchip/irq-loongson-pch-msi.c @@ -15,6 +15,8 @@ #include #include +#include "irq-msi-lib.h" + static int nr_pics; struct pch_msi_data { @@ -27,26 +29,6 @@ struct pch_msi_data { static struct fwnode_handle *pch_msi_handle[MAX_IO_PICS]; -static void pch_msi_mask_msi_irq(struct irq_data *d) -{ - pci_msi_mask_irq(d); - irq_chip_mask_parent(d); -} - -static void pch_msi_unmask_msi_irq(struct irq_data *d) -{ - irq_chip_unmask_parent(d); - pci_msi_unmask_irq(d); -} - -static struct irq_chip pch_msi_irq_chip = { - .name = "PCH PCI MSI", - .irq_mask = pch_msi_mask_msi_irq, - .irq_unmask = pch_msi_unmask_msi_irq, - .irq_ack = irq_chip_ack_parent, - .irq_set_affinity = irq_chip_set_affinity_parent, -}; - static int pch_msi_allocate_hwirq(struct pch_msi_data *priv, int num_req) { int first; @@ -85,12 +67,6 @@ static void pch_msi_compose_msi_msg(struct irq_data *data, msg->data = data->hwirq; } -static struct msi_domain_info pch_msi_domain_info = { - .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX, - .chip = &pch_msi_irq_chip, -}; - static struct irq_chip middle_irq_chip = { .name = "PCH MSI", .irq_mask = irq_chip_mask_parent, @@ -155,13 +131,31 @@ static void pch_msi_middle_domain_free(struct irq_domain *domain, static const struct irq_domain_ops pch_msi_middle_domain_ops = { .alloc = pch_msi_middle_domain_alloc, .free = pch_msi_middle_domain_free, + .select = msi_lib_irq_domain_select, +}; + +#define PCH_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS | \ + MSI_FLAG_PCI_MSI_MASK_PARENT) + +#define PCH_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK | \ + MSI_FLAG_PCI_MSIX | \ + MSI_FLAG_MULTI_PCI_MSI) + +static struct msi_parent_ops pch_msi_parent_ops = { + .required_flags = PCH_MSI_FLAGS_REQUIRED, + .supported_flags = PCH_MSI_FLAGS_SUPPORTED, + .bus_select_mask = MATCH_PCI_MSI, + .bus_select_token = DOMAIN_BUS_NEXUS, + .prefix = "PCH-", + .init_dev_msi_info = msi_lib_init_dev_msi_info, }; static int pch_msi_init_domains(struct pch_msi_data *priv, struct irq_domain *parent, struct fwnode_handle *domain_handle) { - struct irq_domain *middle_domain, *msi_domain; + struct irq_domain *middle_domain; middle_domain = irq_domain_create_hierarchy(parent, 0, priv->num_irqs, domain_handle, @@ -174,14 +168,8 @@ static int pch_msi_init_domains(struct pch_msi_data *priv, irq_domain_update_bus_token(middle_domain, DOMAIN_BUS_NEXUS); - msi_domain = pci_msi_create_irq_domain(domain_handle, - &pch_msi_domain_info, - middle_domain); - if (!msi_domain) { - pr_err("Failed to create PCI MSI domain\n"); - irq_domain_remove(middle_domain); - return -ENOMEM; - } + middle_domain->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT; + middle_domain->msi_parent_ops = &pch_msi_parent_ops; return 0; } From eda25860bf6e71932311f1b5acdf8fc05cd84ff3 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:27 +0206 Subject: [PATCH 087/180] printk: Add notation to console_srcu locking kernel/printk/printk.c:284:5: sparse: sparse: context imbalance in 'console_srcu_read_lock' - wrong count at exit include/linux/srcu.h:301:9: sparse: sparse: context imbalance in 'console_srcu_read_unlock' - unexpected unlock Fixes: 6c4afa79147e ("printk: Prepare for SRCU console list protection") Signed-off-by: John Ogness Reviewed-by: Petr Mladek Acked-by: Paul E. McKenney Link: https://lore.kernel.org/r/20240820063001.36405-2-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/printk.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index c22b07049c38..93c67eb7ca9e 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -282,6 +282,7 @@ EXPORT_SYMBOL(console_list_unlock); * Return: A cookie to pass to console_srcu_read_unlock(). */ int console_srcu_read_lock(void) + __acquires(&console_srcu) { return srcu_read_lock_nmisafe(&console_srcu); } @@ -295,6 +296,7 @@ EXPORT_SYMBOL(console_srcu_read_lock); * Counterpart to console_srcu_read_lock() */ void console_srcu_read_unlock(int cookie) + __releases(&console_srcu) { srcu_read_unlock_nmisafe(&console_srcu, cookie); } From f37b105faef637cdaff334c0369b451410566ca0 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:28 +0206 Subject: [PATCH 088/180] printk: nbcon: Consolidate alloc() and init() Rather than splitting the nbcon allocation and initialization into two pieces, perform all initialization in nbcon_alloc(). Later, the initial sequence is calculated and can be explicitly set using nbcon_seq_force(). This removes the need for the strong rules of nbcon_init() that even included a BUG_ON(). Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-3-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/internal.h | 2 -- kernel/printk/nbcon.c | 37 +++++++++++-------------------------- kernel/printk/printk.c | 2 +- 3 files changed, 12 insertions(+), 29 deletions(-) diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index 19dcc5832651..398ecb40d279 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -75,7 +75,6 @@ u16 printk_parse_prefix(const char *text, int *level, u64 nbcon_seq_read(struct console *con); void nbcon_seq_force(struct console *con, u64 seq); bool nbcon_alloc(struct console *con); -void nbcon_init(struct console *con); void nbcon_free(struct console *con); #else @@ -96,7 +95,6 @@ static inline bool printk_percpu_data_ready(void) { return false; } static inline u64 nbcon_seq_read(struct console *con) { return 0; } static inline void nbcon_seq_force(struct console *con, u64 seq) { } static inline bool nbcon_alloc(struct console *con) { return false; } -static inline void nbcon_init(struct console *con) { } static inline void nbcon_free(struct console *con) { } #endif /* CONFIG_PRINTK */ diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index c8093bcc01fe..670692dc9b10 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -929,17 +929,22 @@ update_con: } /** - * nbcon_alloc - Allocate buffers needed by the nbcon console - * @con: Console to allocate buffers for + * nbcon_alloc - Allocate and init the nbcon console specific data + * @con: Console to initialize * - * Return: True on success. False otherwise and the console cannot - * be used. + * Return: True if the console was fully allocated and initialized. + * Otherwise @con must not be registered. * - * This is not part of nbcon_init() because buffer allocation must - * be performed earlier in the console registration process. + * When allocation and init was successful, the console must be properly + * freed using nbcon_free() once it is no longer needed. */ bool nbcon_alloc(struct console *con) { + struct nbcon_state state = { }; + + nbcon_state_set(con, &state); + atomic_long_set(&ACCESS_PRIVATE(con, nbcon_seq), 0); + if (con->flags & CON_BOOT) { /* * Boot console printing is synchronized with legacy console @@ -958,26 +963,6 @@ bool nbcon_alloc(struct console *con) return true; } -/** - * nbcon_init - Initialize the nbcon console specific data - * @con: Console to initialize - * - * nbcon_alloc() *must* be called and succeed before this function - * is called. - * - * This function expects that the legacy @con->seq has been set. - */ -void nbcon_init(struct console *con) -{ - struct nbcon_state state = { }; - - /* nbcon_alloc() must have been called and successful! */ - BUG_ON(!con->pbufs); - - nbcon_seq_force(con, con->seq); - nbcon_state_set(con, &state); -} - /** * nbcon_free - Free and cleanup the nbcon console specific data * @con: Console to free/cleanup nbcon data diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 93c67eb7ca9e..a47017c932be 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -3618,7 +3618,7 @@ void register_console(struct console *newcon) console_init_seq(newcon, bootcon_registered); if (newcon->flags & CON_NBCON) - nbcon_init(newcon); + nbcon_seq_force(newcon, newcon->seq); /* * Put this console in the list - keep the From d3ff380d47b6312d537c00829008528d1caad639 Mon Sep 17 00:00:00 2001 From: Petr Mladek Date: Tue, 20 Aug 2024 08:35:29 +0206 Subject: [PATCH 089/180] printk: Properly deal with nbcon consoles on seq init If a non-boot console is registering and boot consoles exist, the consoles are flushed before being unregistered. This allows the non-boot console to continue where the boot console left off. If for whatever reason flushing fails, the lowest seq found from any of the enabled boot consoles is used. Until now con->seq was checked. However, if it is an nbcon boot console, the function nbcon_seq_read() must be used to read seq because con->seq is not updated for nbcon consoles. Check if it is an nbcon boot console and if so call nbcon_seq_read() to read seq. Also, avoid usage of con->seq as temporary storage of the starting record. Instead, rename console_init_seq() to get_init_console_seq() and just return the value. For nbcon consoles set the sequence via nbcon_seq_force(), for legacy consoles set con->seq. The cleaned design should make sure that the value stays and is set before the console is added to the console list. It also unifies the sequence number initialization for legacy and nbcon consoles. Reviewed-by: John Ogness Link: https://lore.kernel.org/r/20240820063001.36405-4-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/nbcon.c | 3 --- kernel/printk/printk.c | 41 +++++++++++++++++++++++++++++------------ 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index 670692dc9b10..776746d20fc0 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -172,9 +172,6 @@ void nbcon_seq_force(struct console *con, u64 seq) u64 valid_seq = max_t(u64, seq, prb_first_valid_seq(prb)); atomic_long_set(&ACCESS_PRIVATE(con, nbcon_seq), __u64seq_to_ulseq(valid_seq)); - - /* Clear con->seq since nbcon consoles use con->nbcon_seq instead. */ - con->seq = 0; } /** diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index a47017c932be..20c39505f5aa 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -3448,19 +3448,21 @@ static void try_enable_default_console(struct console *newcon) newcon->flags |= CON_CONSDEV; } -static void console_init_seq(struct console *newcon, bool bootcon_registered) +/* Return the starting sequence number for a newly registered console. */ +static u64 get_init_console_seq(struct console *newcon, bool bootcon_registered) { struct console *con; bool handover; + u64 init_seq; if (newcon->flags & (CON_PRINTBUFFER | CON_BOOT)) { /* Get a consistent copy of @syslog_seq. */ mutex_lock(&syslog_lock); - newcon->seq = syslog_seq; + init_seq = syslog_seq; mutex_unlock(&syslog_lock); } else { /* Begin with next message added to ringbuffer. */ - newcon->seq = prb_next_seq(prb); + init_seq = prb_next_seq(prb); /* * If any enabled boot consoles are due to be unregistered @@ -3481,7 +3483,7 @@ static void console_init_seq(struct console *newcon, bool bootcon_registered) * Flush all consoles and set the console to start at * the next unprinted sequence number. */ - if (!console_flush_all(true, &newcon->seq, &handover)) { + if (!console_flush_all(true, &init_seq, &handover)) { /* * Flushing failed. Just choose the lowest * sequence of the enabled boot consoles. @@ -3494,19 +3496,30 @@ static void console_init_seq(struct console *newcon, bool bootcon_registered) if (handover) console_lock(); - newcon->seq = prb_next_seq(prb); + init_seq = prb_next_seq(prb); for_each_console(con) { - if ((con->flags & CON_BOOT) && - (con->flags & CON_ENABLED) && - con->seq < newcon->seq) { - newcon->seq = con->seq; + u64 seq; + + if (!(con->flags & CON_BOOT) || + !(con->flags & CON_ENABLED)) { + continue; } + + if (con->flags & CON_NBCON) + seq = nbcon_seq_read(con); + else + seq = con->seq; + + if (seq < init_seq) + init_seq = seq; } } console_unlock(); } } + + return init_seq; } #define console_first() \ @@ -3538,6 +3551,7 @@ void register_console(struct console *newcon) struct console *con; bool bootcon_registered = false; bool realcon_registered = false; + u64 init_seq; int err; console_list_lock(); @@ -3615,10 +3629,13 @@ void register_console(struct console *newcon) } newcon->dropped = 0; - console_init_seq(newcon, bootcon_registered); + init_seq = get_init_console_seq(newcon, bootcon_registered); - if (newcon->flags & CON_NBCON) - nbcon_seq_force(newcon, newcon->seq); + if (newcon->flags & CON_NBCON) { + nbcon_seq_force(newcon, init_seq); + } else { + newcon->seq = init_seq; + } /* * Put this console in the list - keep the From 0e1d5731d3c1e2214249ef36dcd13ad51ad304cf Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Tue, 20 Aug 2024 08:35:30 +0206 Subject: [PATCH 090/180] printk: Check printk_deferred_enter()/_exit() usage Add validation that printk_deferred_enter()/_exit() are called in non-migration contexts. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-5-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- include/linux/printk.h | 9 +++++---- kernel/printk/internal.h | 3 +++ kernel/printk/printk_safe.c | 12 ++++++++++++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/include/linux/printk.h b/include/linux/printk.h index b937cefcb31c..eee8e97da681 100644 --- a/include/linux/printk.h +++ b/include/linux/printk.h @@ -161,15 +161,16 @@ int _printk(const char *fmt, ...); */ __printf(1, 2) __cold int _printk_deferred(const char *fmt, ...); -extern void __printk_safe_enter(void); -extern void __printk_safe_exit(void); +extern void __printk_deferred_enter(void); +extern void __printk_deferred_exit(void); + /* * The printk_deferred_enter/exit macros are available only as a hack for * some code paths that need to defer all printk console printing. Interrupts * must be disabled for the deferred duration. */ -#define printk_deferred_enter __printk_safe_enter -#define printk_deferred_exit __printk_safe_exit +#define printk_deferred_enter() __printk_deferred_enter() +#define printk_deferred_exit() __printk_deferred_exit() /* * Please don't use printk_ratelimit(), because it shares ratelimiting state diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index 398ecb40d279..dc8bc0890fd2 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -53,6 +53,9 @@ int vprintk_store(int facility, int level, __printf(1, 0) int vprintk_default(const char *fmt, va_list args); __printf(1, 0) int vprintk_deferred(const char *fmt, va_list args); +void __printk_safe_enter(void); +void __printk_safe_exit(void); + bool printk_percpu_data_ready(void); #define printk_safe_enter_irqsave(flags) \ diff --git a/kernel/printk/printk_safe.c b/kernel/printk/printk_safe.c index 6d10927a07d8..4421ccac3113 100644 --- a/kernel/printk/printk_safe.c +++ b/kernel/printk/printk_safe.c @@ -26,6 +26,18 @@ void __printk_safe_exit(void) this_cpu_dec(printk_context); } +void __printk_deferred_enter(void) +{ + cant_migrate(); + __printk_safe_enter(); +} + +void __printk_deferred_exit(void) +{ + cant_migrate(); + __printk_safe_exit(); +} + asmlinkage int vprintk(const char *fmt, va_list args) { #ifdef CONFIG_KGDB_KDB From 8c9dab2c55ad74680728c72949971b20d70b56ca Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:31 +0206 Subject: [PATCH 091/180] printk: nbcon: Clarify rules of the owner/waiter matching The functions nbcon_owner_matches() and nbcon_waiter_matches() use a minimal set of data to determine if a context matches. The existing kerneldoc and comments were not clear enough and caused the printk folks to re-prove that the functions are indeed reliable in all cases. Update and expand the explanations so that it is clear that the implementations are sufficient for all cases. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-6-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/nbcon.c | 56 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index 776746d20fc0..931b8f086902 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -228,6 +228,13 @@ static int nbcon_context_try_acquire_direct(struct nbcon_context *ctxt, struct nbcon_state new; do { + /* + * Panic does not imply that the console is owned. However, it + * is critical that non-panic CPUs during panic are unable to + * acquire ownership in order to satisfy the assumptions of + * nbcon_waiter_matches(). In particular, the assumption that + * lower priorities are ignored during panic. + */ if (other_cpu_in_panic()) return -EPERM; @@ -259,18 +266,29 @@ static bool nbcon_waiter_matches(struct nbcon_state *cur, int expected_prio) /* * The request context is well defined by the @req_prio because: * - * - Only a context with a higher priority can take over the request. + * - Only a context with a priority higher than the owner can become + * a waiter. + * - Only a context with a priority higher than the waiter can + * directly take over the request. * - There are only three priorities. * - Only one CPU is allowed to request PANIC priority. * - Lower priorities are ignored during panic() until reboot. * * As a result, the following scenario is *not* possible: * - * 1. Another context with a higher priority directly takes ownership. - * 2. The higher priority context releases the ownership. - * 3. A lower priority context takes the ownership. - * 4. Another context with the same priority as this context + * 1. This context is currently a waiter. + * 2. Another context with a higher priority than this context + * directly takes ownership. + * 3. The higher priority context releases the ownership. + * 4. Another lower priority context takes the ownership. + * 5. Another context with the same priority as this context * creates a request and starts waiting. + * + * Event #1 implies this context is EMERGENCY. + * Event #2 implies the new context is PANIC. + * Event #3 occurs when panic() has flushed the console. + * Events #4 and #5 are not possible due to the other_cpu_in_panic() + * check in nbcon_context_try_acquire_direct(). */ return (cur->req_prio == expected_prio); @@ -578,11 +596,29 @@ static bool nbcon_owner_matches(struct nbcon_state *cur, int expected_cpu, int expected_prio) { /* - * Since consoles can only be acquired by higher priorities, - * owning contexts are uniquely identified by @prio. However, - * since contexts can unexpectedly lose ownership, it is - * possible that later another owner appears with the same - * priority. For this reason @cpu is also needed. + * A similar function, nbcon_waiter_matches(), only deals with + * EMERGENCY and PANIC priorities. However, this function must also + * deal with the NORMAL priority, which requires additional checks + * and constraints. + * + * For the case where preemption and interrupts are disabled, it is + * enough to also verify that the owning CPU has not changed. + * + * For the case where preemption or interrupts are enabled, an + * external synchronization method *must* be used. In particular, + * the driver-specific locking mechanism used in device_lock() + * (including disabling migration) should be used. It prevents + * scenarios such as: + * + * 1. [Task A] owns a context with NBCON_PRIO_NORMAL on [CPU X] and + * is scheduled out. + * 2. Another context takes over the lock with NBCON_PRIO_EMERGENCY + * and releases it. + * 3. [Task B] acquires a context with NBCON_PRIO_NORMAL on [CPU X] + * and is scheduled out. + * 4. [Task A] gets running on [CPU X] and sees that the console is + * still owned by a task on [CPU X] with NBON_PRIO_NORMAL. Thus + * [Task A] thinks it is the owner when it is not. */ if (cur->prio != expected_prio) From b7049d88c1763d5f133b0e93e39da8e89a9f944b Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:32 +0206 Subject: [PATCH 092/180] printk: nbcon: Remove return value for write_atomic() The return value of write_atomic() does not provide any useful information. On the contrary, it makes things more complicated for the caller to appropriately deal with the information. Change write_atomic() to not have a return value. If the message did not get printed due to loss of ownership, the caller will notice this on its own. If ownership was not lost, it will be assumed that the driver successfully printed the message and the sequence number for that console will be incremented. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-7-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- include/linux/console.h | 2 +- kernel/printk/nbcon.c | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/include/linux/console.h b/include/linux/console.h index 31a8f5b85f5d..577b157fe4fb 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -345,7 +345,7 @@ struct console { struct hlist_node node; /* nbcon console specific members */ - bool (*write_atomic)(struct console *con, + void (*write_atomic)(struct console *con, struct nbcon_write_context *wctxt); atomic_t __private nbcon_state; atomic_long_t __private nbcon_seq; diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index 931b8f086902..f279f839741a 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -885,7 +885,6 @@ static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt) unsigned long con_dropped; struct nbcon_state cur; unsigned long dropped; - bool done; /* * The printk buffers are filled within an unsafe section. This @@ -925,16 +924,16 @@ static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt) wctxt->unsafe_takeover = cur.unsafe_takeover; if (con->write_atomic) { - done = con->write_atomic(con, wctxt); + con->write_atomic(con, wctxt); } else { - nbcon_context_release(ctxt); + /* + * This function should never be called for legacy consoles. + * Handle it as if ownership was lost and try to continue. + */ WARN_ON_ONCE(1); - done = false; - } - - /* If not done, the emit was aborted. */ - if (!done) + nbcon_context_release(ctxt); return false; + } /* * Since any dropped message was successfully output, reset the From 7d56936c1e5208b5eeea2a9a2f70e85248f6da49 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:33 +0206 Subject: [PATCH 093/180] printk: nbcon: Add detailed doc for write_atomic() The write_atomic() callback has special requirements and is allowed to use special helper functions. Provide detailed documentation of the callback so that a developer has a chance of implementing it correctly. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-8-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- include/linux/console.h | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/include/linux/console.h b/include/linux/console.h index 577b157fe4fb..35c64ee3827b 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -303,7 +303,7 @@ struct nbcon_write_context { /** * struct console - The console descriptor structure * @name: The name of the console driver - * @write: Write callback to output messages (Optional) + * @write: Legacy write callback to output messages (Optional) * @read: Read callback for console input (Optional) * @device: The underlying TTY device driver (Optional) * @unblank: Callback to unblank the console (Optional) @@ -320,7 +320,6 @@ struct nbcon_write_context { * @data: Driver private data * @node: hlist node for the console list * - * @write_atomic: Write callback for atomic context * @nbcon_state: State for nbcon consoles * @nbcon_seq: Sequence number of the next record for nbcon to print * @pbufs: Pointer to nbcon private buffer @@ -345,8 +344,34 @@ struct console { struct hlist_node node; /* nbcon console specific members */ - void (*write_atomic)(struct console *con, - struct nbcon_write_context *wctxt); + + /** + * @write_atomic: + * + * NBCON callback to write out text in any context. (Optional) + * + * This callback is called with the console already acquired. However, + * a higher priority context is allowed to take it over by default. + * + * The callback must call nbcon_enter_unsafe() and nbcon_exit_unsafe() + * around any code where the takeover is not safe, for example, when + * manipulating the serial port registers. + * + * nbcon_enter_unsafe() will fail if the context has lost the console + * ownership in the meantime. In this case, the callback is no longer + * allowed to go forward. It must back out immediately and carefully. + * The buffer content is also no longer trusted since it no longer + * belongs to the context. + * + * The callback should allow the takeover whenever it is safe. It + * increases the chance to see messages when the system is in trouble. + * + * The callback can be called from any context (including NMI). + * Therefore it must avoid usage of any locking and instead rely + * on the console ownership for synchronization. + */ + void (*write_atomic)(struct console *con, struct nbcon_write_context *wctxt); + atomic_t __private nbcon_state; atomic_long_t __private nbcon_seq; struct printk_buffers *pbufs; From 7a16a771890746b4060333d665ebe674945a3cfa Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:34 +0206 Subject: [PATCH 094/180] printk: nbcon: Add callbacks to synchronize with driver Console drivers typically must deal with access to the hardware via user input/output (such as an interactive login shell) and output of kernel messages via printk() calls. To provide the necessary synchronization, usually some driver-specific locking mechanism is used (for example, the port spinlock for uart serial consoles). Until now, usage of this driver-specific locking has been hidden from the printk-subsystem and implemented within the various console callbacks. However, nbcon consoles would need to use it even in the generic code. Add device_lock() and device_unlock() callback which will need to get implemented by nbcon consoles. The callbacks will use whatever synchronization mechanism the driver is using for itself. The minimum requirement is to prevent CPU migration. It would allow a context friendly acquiring of nbcon console ownership in non-emergency and non-panic context. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-9-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- include/linux/console.h | 43 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/include/linux/console.h b/include/linux/console.h index 35c64ee3827b..46b3c210b931 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -372,6 +372,49 @@ struct console { */ void (*write_atomic)(struct console *con, struct nbcon_write_context *wctxt); + /** + * @device_lock: + * + * NBCON callback to begin synchronization with driver code. + * + * Console drivers typically must deal with access to the hardware + * via user input/output (such as an interactive login shell) and + * output of kernel messages via printk() calls. This callback is + * called by the printk-subsystem whenever it needs to synchronize + * with hardware access by the driver. It should be implemented to + * use whatever synchronization mechanism the driver is using for + * itself (for example, the port lock for uart serial consoles). + * + * The callback is always called from task context. It may use any + * synchronization method required by the driver. + * + * IMPORTANT: The callback MUST disable migration. The console driver + * may be using a synchronization mechanism that already takes + * care of this (such as spinlocks). Otherwise this function must + * explicitly call migrate_disable(). + * + * The flags argument is provided as a convenience to the driver. It + * will be passed again to device_unlock(). It can be ignored if the + * driver does not need it. + */ + void (*device_lock)(struct console *con, unsigned long *flags); + + /** + * @device_unlock: + * + * NBCON callback to finish synchronization with driver code. + * + * It is the counterpart to device_lock(). + * + * This callback is always called from task context. It must + * appropriately re-enable migration (depending on how device_lock() + * disabled migration). + * + * The flags argument is the value of the same variable that was + * passed to device_lock(). + */ + void (*device_unlock)(struct console *con, unsigned long flags); + atomic_t __private nbcon_state; atomic_long_t __private nbcon_seq; struct printk_buffers *pbufs; From e55c3bcf380c7593d768f31994eff63935081d06 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:35 +0206 Subject: [PATCH 095/180] printk: nbcon: Use driver synchronization while (un)registering Console drivers typically have to deal with access to the hardware via user input/output (such as an interactive login shell) and output of kernel messages via printk() calls. They use some classic driver-specific locking mechanism in most situations. But console->write_atomic() callbacks, used by nbcon consoles, are synchronized only by acquiring the console context. The synchronization via the console context ownership is possible only when the console driver is registered. It is when a particular device driver is connected with a particular console driver. The two synchronization mechanisms must be synchronized between each other. It is tricky because the console context ownership is quite special. It might be taken over by a higher priority context. Also CPU migration must be disabled. The most tricky part is to (dis)connect these two mechanisms during the console (un)registration. Use the driver-specific locking callbacks: device_lock(), device_unlock(). They allow taking the device-specific lock while the device is being (un)registered by the related console driver. For example, these callbacks lock/unlock the port lock for serial port drivers. Note that the driver-specific locking is only needed during (un)register if it is an nbcon console with the write_atomic() callback implemented. If write_atomic() is not implemented, the driver should never attempt to access the hardware without first acquiring its driver-specific lock. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-10-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/printk.c | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 20c39505f5aa..4cd2c50dd06d 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -3548,9 +3548,11 @@ static int unregister_console_locked(struct console *console); */ void register_console(struct console *newcon) { - struct console *con; + bool use_device_lock = (newcon->flags & CON_NBCON) && newcon->write_atomic; bool bootcon_registered = false; bool realcon_registered = false; + struct console *con; + unsigned long flags; u64 init_seq; int err; @@ -3637,6 +3639,19 @@ void register_console(struct console *newcon) newcon->seq = init_seq; } + /* + * If another context is actively using the hardware of this new + * console, it will not be aware of the nbcon synchronization. This + * is a risk that two contexts could access the hardware + * simultaneously if this new console is used for atomic printing + * and the other context is still using the hardware. + * + * Use the driver synchronization to ensure that the hardware is not + * in use while this new console transitions to being registered. + */ + if (use_device_lock) + newcon->device_lock(newcon, &flags); + /* * Put this console in the list - keep the * preferred driver at the head of the list. @@ -3661,6 +3676,10 @@ void register_console(struct console *newcon) * register_console() completes. */ + /* This new console is now registered. */ + if (use_device_lock) + newcon->device_unlock(newcon, flags); + console_sysfs_notify(); /* @@ -3689,6 +3708,8 @@ EXPORT_SYMBOL(register_console); /* Must be called under console_list_lock(). */ static int unregister_console_locked(struct console *console) { + bool use_device_lock = (console->flags & CON_NBCON) && console->write_atomic; + unsigned long flags; int res; lockdep_assert_console_list_lock_held(); @@ -3707,8 +3728,18 @@ static int unregister_console_locked(struct console *console) if (!console_is_registered_locked(console)) return -ENODEV; + /* + * Use the driver synchronization to ensure that the hardware is not + * in use while this console transitions to being unregistered. + */ + if (use_device_lock) + console->device_lock(console, &flags); + hlist_del_init_rcu(&console->node); + if (use_device_lock) + console->device_unlock(console, flags); + /* * * If this isn't the last console and it has CON_CONSDEV set, we From 77e73c0687f5bc884fa1b0cb97aca912bcc01957 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:36 +0206 Subject: [PATCH 096/180] serial: core: Provide low-level functions to lock port It will be necessary at times for the uart nbcon console drivers to acquire the port lock directly (without the additional nbcon functionality of the port lock wrappers). These are special cases such as the implementation of the device_lock()/device_unlock() callbacks or for internal port lock wrapper synchronization. Provide low-level variants __uart_port_lock_irqsave() and __uart_port_unlock_irqrestore() for this purpose. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Acked-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20240820063001.36405-11-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- include/linux/serial_core.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h index aea25eef9a1a..8872cd21e70a 100644 --- a/include/linux/serial_core.h +++ b/include/linux/serial_core.h @@ -590,6 +590,24 @@ struct uart_port { void *private_data; /* generic platform data pointer */ }; +/* + * Only for console->device_lock()/_unlock() callbacks and internal + * port lock wrapper synchronization. + */ +static inline void __uart_port_lock_irqsave(struct uart_port *up, unsigned long *flags) +{ + spin_lock_irqsave(&up->lock, *flags); +} + +/* + * Only for console->device_lock()/_unlock() callbacks and internal + * port lock wrapper synchronization. + */ +static inline void __uart_port_unlock_irqrestore(struct uart_port *up, unsigned long flags) +{ + spin_unlock_irqrestore(&up->lock, flags); +} + /** * uart_port_lock - Lock the UART port * @up: Pointer to UART port structure From eabd4600dafacf9895184259373f2445ff748654 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:37 +0206 Subject: [PATCH 097/180] serial: core: Introduce wrapper to set @uart_port->cons MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce uart_port_set_cons() as a wrapper to set @cons of a uart_port. The wrapper sets @cons under the port lock in order to prevent @cons from disappearing while another context is holding the port lock. This is necessary for a follow-up commit relating to the port lock wrappers, which rely on @cons not changing between lock and unlock. Signed-off-by: John Ogness Tested-by: Théo Lebrun # EyeQ5, AMBA-PL011 Acked-by: Greg Kroah-Hartman Reviewed-by: Petr Mladek Reviewed-by: Ilpo Järvinen Link: https://lore.kernel.org/r/20240820063001.36405-12-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- drivers/tty/serial/8250/8250_core.c | 6 +++--- drivers/tty/serial/amba-pl011.c | 2 +- drivers/tty/serial/serial_core.c | 16 ++++++++-------- include/linux/serial_core.h | 17 +++++++++++++++++ 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/drivers/tty/serial/8250/8250_core.c b/drivers/tty/serial/8250/8250_core.c index 29e4b83e0376..5f9f06911795 100644 --- a/drivers/tty/serial/8250/8250_core.c +++ b/drivers/tty/serial/8250/8250_core.c @@ -423,11 +423,11 @@ static int univ8250_console_setup(struct console *co, char *options) port = &serial8250_ports[co->index].port; /* link port to console */ - port->cons = co; + uart_port_set_cons(port, co); retval = serial8250_console_setup(port, options, false); if (retval != 0) - port->cons = NULL; + uart_port_set_cons(port, NULL); return retval; } @@ -485,7 +485,7 @@ static int univ8250_console_match(struct console *co, char *name, int idx, continue; co->index = i; - port->cons = co; + uart_port_set_cons(port, co); return serial8250_console_setup(port, options, true); } diff --git a/drivers/tty/serial/amba-pl011.c b/drivers/tty/serial/amba-pl011.c index 8b1644f5411e..7d0134ecd82f 100644 --- a/drivers/tty/serial/amba-pl011.c +++ b/drivers/tty/serial/amba-pl011.c @@ -2480,7 +2480,7 @@ static int pl011_console_match(struct console *co, char *name, int idx, continue; co->index = i; - port->cons = co; + uart_port_set_cons(port, co); return pl011_console_setup(co, options); } diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c index 5bea3af46abc..1e3e28e364df 100644 --- a/drivers/tty/serial/serial_core.c +++ b/drivers/tty/serial/serial_core.c @@ -3176,8 +3176,15 @@ static int serial_core_add_one_port(struct uart_driver *drv, struct uart_port *u state->uart_port = uport; uport->state = state; + /* + * If this port is in use as a console then the spinlock is already + * initialised. + */ + if (!uart_console_registered(uport)) + uart_port_spin_lock_init(uport); + state->pm_state = UART_PM_STATE_UNDEFINED; - uport->cons = drv->cons; + uart_port_set_cons(uport, drv->cons); uport->minor = drv->tty_driver->minor_start + uport->line; uport->name = kasprintf(GFP_KERNEL, "%s%d", drv->dev_name, drv->tty_driver->name_base + uport->line); @@ -3186,13 +3193,6 @@ static int serial_core_add_one_port(struct uart_driver *drv, struct uart_port *u goto out; } - /* - * If this port is in use as a console then the spinlock is already - * initialised. - */ - if (!uart_console_registered(uport)) - uart_port_spin_lock_init(uport); - if (uport->cons && uport->dev) of_console_check(uport->dev->of_node, uport->cons->name, uport->line); diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h index 8872cd21e70a..2cf03ff2056a 100644 --- a/include/linux/serial_core.h +++ b/include/linux/serial_core.h @@ -608,6 +608,23 @@ static inline void __uart_port_unlock_irqrestore(struct uart_port *up, unsigned spin_unlock_irqrestore(&up->lock, flags); } +/** + * uart_port_set_cons - Safely set the @cons field for a uart + * @up: The uart port to set + * @con: The new console to set to + * + * This function must be used to set @up->cons. It uses the port lock to + * synchronize with the port lock wrappers in order to ensure that the console + * cannot change or disappear while another context is holding the port lock. + */ +static inline void uart_port_set_cons(struct uart_port *up, struct console *con) +{ + unsigned long flags; + + __uart_port_lock_irqsave(up, &flags); + up->cons = con; + __uart_port_unlock_irqrestore(up, flags); +} /** * uart_port_lock - Lock the UART port * @up: Pointer to UART port structure From dc219d8d858d95549543aa7a41b6e64a0da9e406 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:38 +0206 Subject: [PATCH 098/180] console: Improve console_srcu_read_flags() comments It was not clear when exactly console_srcu_read_flags() must be used vs. directly reading @console->flags. Refactor and clarify that console_srcu_read_flags() is only needed if the console is registered or the caller is in a context where the registration status of the console may change (due to another context). The function requires the caller holds @console_srcu, which will ensure that the caller sees an appropriate @flags value for the registered console and that exit/cleanup routines will not run if the console is in the process of unregistration. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-13-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- include/linux/console.h | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/include/linux/console.h b/include/linux/console.h index 46b3c210b931..aafe3121b74e 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -446,28 +446,34 @@ extern void console_list_unlock(void) __releases(console_mutex); extern struct hlist_head console_list; /** - * console_srcu_read_flags - Locklessly read the console flags + * console_srcu_read_flags - Locklessly read flags of a possibly registered + * console * @con: struct console pointer of console to read flags from * - * This function provides the necessary READ_ONCE() and data_race() - * notation for locklessly reading the console flags. The READ_ONCE() - * in this function matches the WRITE_ONCE() when @flags are modified - * for registered consoles with console_srcu_write_flags(). + * Locklessly reading @con->flags provides a consistent read value because + * there is at most one CPU modifying @con->flags and that CPU is using only + * read-modify-write operations to do so. * - * Only use this function to read console flags when locklessly - * iterating the console list via srcu. + * Requires console_srcu_read_lock to be held, which implies that @con might + * be a registered console. The purpose of holding console_srcu_read_lock is + * to guarantee that the console state is valid (CON_SUSPENDED/CON_ENABLED) + * and that no exit/cleanup routines will run if the console is currently + * undergoing unregistration. + * + * If the caller is holding the console_list_lock or it is _certain_ that + * @con is not and will not become registered, the caller may read + * @con->flags directly instead. * * Context: Any context. + * Return: The current value of the @con->flags field. */ static inline short console_srcu_read_flags(const struct console *con) { WARN_ON_ONCE(!console_srcu_read_lock_is_held()); /* - * Locklessly reading console->flags provides a consistent - * read value because there is at most one CPU modifying - * console->flags and that CPU is using only read-modify-write - * operations to do so. + * The READ_ONCE() matches the WRITE_ONCE() when @flags are modified + * for registered consoles with console_srcu_write_flags(). */ return data_race(READ_ONCE(con->flags)); } From adf6f37d142e14896115929f3892bbc0a86e25bf Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:39 +0206 Subject: [PATCH 099/180] nbcon: Add API to acquire context for non-printing operations Provide functions nbcon_device_try_acquire() and nbcon_device_release() which will try to acquire the nbcon console ownership with NBCON_PRIO_NORMAL and mark it unsafe for handover/takeover. These functions are to be used together with the device-specific locking when performing non-printing activities on the console device. They will allow synchronization against the atomic_write() callback which will be serialized, for higher priority contexts, only by acquiring the console context ownership. Pitfalls: The API requires to be called in a context with migration disabled because it uses per-CPU variables internally. The context is set unsafe for a takeover all the time. It guarantees full serialization against any atomic_write() caller except for the final flush in panic() which might try an unsafe takeover. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-14-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- include/linux/console.h | 2 ++ include/linux/printk.h | 14 ++++++++++ kernel/printk/nbcon.c | 58 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/include/linux/console.h b/include/linux/console.h index aafe3121b74e..3706f944de46 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -322,6 +322,7 @@ struct nbcon_write_context { * * @nbcon_state: State for nbcon consoles * @nbcon_seq: Sequence number of the next record for nbcon to print + * @nbcon_device_ctxt: Context available for non-printing operations * @pbufs: Pointer to nbcon private buffer */ struct console { @@ -417,6 +418,7 @@ struct console { atomic_t __private nbcon_state; atomic_long_t __private nbcon_seq; + struct nbcon_context __private nbcon_device_ctxt; struct printk_buffers *pbufs; }; diff --git a/include/linux/printk.h b/include/linux/printk.h index eee8e97da681..9687089f5ace 100644 --- a/include/linux/printk.h +++ b/include/linux/printk.h @@ -9,6 +9,8 @@ #include #include +struct console; + extern const char linux_banner[]; extern const char linux_proc_banner[]; @@ -198,6 +200,8 @@ extern asmlinkage void dump_stack_lvl(const char *log_lvl) __cold; extern asmlinkage void dump_stack(void) __cold; void printk_trigger_flush(void); void console_try_replay_all(void); +extern bool nbcon_device_try_acquire(struct console *con); +extern void nbcon_device_release(struct console *con); #else static inline __printf(1, 0) int vprintk(const char *s, va_list args) @@ -280,6 +284,16 @@ static inline void printk_trigger_flush(void) static inline void console_try_replay_all(void) { } + +static inline bool nbcon_device_try_acquire(struct console *con) +{ + return false; +} + +static inline void nbcon_device_release(struct console *con) +{ +} + #endif bool this_cpu_in_panic(void); diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index f279f839741a..61f0ae6a4809 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -5,7 +5,9 @@ #include #include #include +#include #include +#include #include "internal.h" /* * Printk console printing implementation for consoles which does not depend @@ -546,6 +548,7 @@ static struct printk_buffers panic_nbcon_pbufs; * nbcon_context_try_acquire - Try to acquire nbcon console * @ctxt: The context of the caller * + * Context: Under @ctxt->con->device_lock() or local_irq_save(). * Return: True if the console was acquired. False otherwise. * * If the caller allowed an unsafe hostile takeover, on success the @@ -553,7 +556,6 @@ static struct printk_buffers panic_nbcon_pbufs; * in an unsafe state. Otherwise, on success the caller may assume * the console is not in an unsafe state. */ -__maybe_unused static bool nbcon_context_try_acquire(struct nbcon_context *ctxt) { unsigned int cpu = smp_processor_id(); @@ -1011,3 +1013,57 @@ void nbcon_free(struct console *con) con->pbufs = NULL; } + +/** + * nbcon_device_try_acquire - Try to acquire nbcon console and enter unsafe + * section + * @con: The nbcon console to acquire + * + * Context: Under the locking mechanism implemented in + * @con->device_lock() including disabling migration. + * Return: True if the console was acquired. False otherwise. + * + * Console drivers will usually use their own internal synchronization + * mechasism to synchronize between console printing and non-printing + * activities (such as setting baud rates). However, nbcon console drivers + * supporting atomic consoles may also want to mark unsafe sections when + * performing non-printing activities in order to synchronize against their + * atomic_write() callback. + * + * This function acquires the nbcon console using priority NBCON_PRIO_NORMAL + * and marks it unsafe for handover/takeover. + */ +bool nbcon_device_try_acquire(struct console *con) +{ + struct nbcon_context *ctxt = &ACCESS_PRIVATE(con, nbcon_device_ctxt); + + cant_migrate(); + + memset(ctxt, 0, sizeof(*ctxt)); + ctxt->console = con; + ctxt->prio = NBCON_PRIO_NORMAL; + + if (!nbcon_context_try_acquire(ctxt)) + return false; + + if (!nbcon_context_enter_unsafe(ctxt)) + return false; + + return true; +} +EXPORT_SYMBOL_GPL(nbcon_device_try_acquire); + +/** + * nbcon_device_release - Exit unsafe section and release the nbcon console + * @con: The nbcon console acquired in nbcon_device_try_acquire() + */ +void nbcon_device_release(struct console *con) +{ + struct nbcon_context *ctxt = &ACCESS_PRIVATE(con, nbcon_device_ctxt); + + if (!nbcon_context_exit_unsafe(ctxt)) + return; + + nbcon_context_release(ctxt); +} +EXPORT_SYMBOL_GPL(nbcon_device_release); From 4126f149c48b2ae757d11ea24675f7071ab22ebf Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:40 +0206 Subject: [PATCH 100/180] serial: core: Acquire nbcon context in port->lock wrapper Currently the port->lock wrappers uart_port_lock(), uart_port_unlock() (and their variants) only lock/unlock the spin_lock. If the port is an nbcon console that has implemented the write_atomic() callback, the wrappers must also acquire/release the console context and mark the region as unsafe. This allows general port->lock synchronization to be synchronized against the nbcon write_atomic() callback. Note that __uart_port_using_nbcon() relies on the port->lock being held while a console is added and removed from the console list (i.e. all uart nbcon drivers *must* take the port->lock in their device_lock() callbacks). Signed-off-by: John Ogness Acked-by: Greg Kroah-Hartman Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-15-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- include/linux/serial_core.h | 82 ++++++++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 2 deletions(-) diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h index 2cf03ff2056a..4ab65874a850 100644 --- a/include/linux/serial_core.h +++ b/include/linux/serial_core.h @@ -11,6 +11,8 @@ #include #include #include +#include +#include #include #include #include @@ -625,6 +627,60 @@ static inline void uart_port_set_cons(struct uart_port *up, struct console *con) up->cons = con; __uart_port_unlock_irqrestore(up, flags); } + +/* Only for internal port lock wrapper usage. */ +static inline bool __uart_port_using_nbcon(struct uart_port *up) +{ + lockdep_assert_held_once(&up->lock); + + if (likely(!uart_console(up))) + return false; + + /* + * @up->cons is only modified under the port lock. Therefore it is + * certain that it cannot disappear here. + * + * @up->cons->node is added/removed from the console list under the + * port lock. Therefore it is certain that the registration status + * cannot change here, thus @up->cons->flags can be read directly. + */ + if (hlist_unhashed_lockless(&up->cons->node) || + !(up->cons->flags & CON_NBCON) || + !up->cons->write_atomic) { + return false; + } + + return true; +} + +/* Only for internal port lock wrapper usage. */ +static inline bool __uart_port_nbcon_try_acquire(struct uart_port *up) +{ + if (!__uart_port_using_nbcon(up)) + return true; + + return nbcon_device_try_acquire(up->cons); +} + +/* Only for internal port lock wrapper usage. */ +static inline void __uart_port_nbcon_acquire(struct uart_port *up) +{ + if (!__uart_port_using_nbcon(up)) + return; + + while (!nbcon_device_try_acquire(up->cons)) + cpu_relax(); +} + +/* Only for internal port lock wrapper usage. */ +static inline void __uart_port_nbcon_release(struct uart_port *up) +{ + if (!__uart_port_using_nbcon(up)) + return; + + nbcon_device_release(up->cons); +} + /** * uart_port_lock - Lock the UART port * @up: Pointer to UART port structure @@ -632,6 +688,7 @@ static inline void uart_port_set_cons(struct uart_port *up, struct console *con) static inline void uart_port_lock(struct uart_port *up) { spin_lock(&up->lock); + __uart_port_nbcon_acquire(up); } /** @@ -641,6 +698,7 @@ static inline void uart_port_lock(struct uart_port *up) static inline void uart_port_lock_irq(struct uart_port *up) { spin_lock_irq(&up->lock); + __uart_port_nbcon_acquire(up); } /** @@ -651,6 +709,7 @@ static inline void uart_port_lock_irq(struct uart_port *up) static inline void uart_port_lock_irqsave(struct uart_port *up, unsigned long *flags) { spin_lock_irqsave(&up->lock, *flags); + __uart_port_nbcon_acquire(up); } /** @@ -661,7 +720,15 @@ static inline void uart_port_lock_irqsave(struct uart_port *up, unsigned long *f */ static inline bool uart_port_trylock(struct uart_port *up) { - return spin_trylock(&up->lock); + if (!spin_trylock(&up->lock)) + return false; + + if (!__uart_port_nbcon_try_acquire(up)) { + spin_unlock(&up->lock); + return false; + } + + return true; } /** @@ -673,7 +740,15 @@ static inline bool uart_port_trylock(struct uart_port *up) */ static inline bool uart_port_trylock_irqsave(struct uart_port *up, unsigned long *flags) { - return spin_trylock_irqsave(&up->lock, *flags); + if (!spin_trylock_irqsave(&up->lock, *flags)) + return false; + + if (!__uart_port_nbcon_try_acquire(up)) { + spin_unlock_irqrestore(&up->lock, *flags); + return false; + } + + return true; } /** @@ -682,6 +757,7 @@ static inline bool uart_port_trylock_irqsave(struct uart_port *up, unsigned long */ static inline void uart_port_unlock(struct uart_port *up) { + __uart_port_nbcon_release(up); spin_unlock(&up->lock); } @@ -691,6 +767,7 @@ static inline void uart_port_unlock(struct uart_port *up) */ static inline void uart_port_unlock_irq(struct uart_port *up) { + __uart_port_nbcon_release(up); spin_unlock_irq(&up->lock); } @@ -701,6 +778,7 @@ static inline void uart_port_unlock_irq(struct uart_port *up) */ static inline void uart_port_unlock_irqrestore(struct uart_port *up, unsigned long flags) { + __uart_port_nbcon_release(up); spin_unlock_irqrestore(&up->lock, flags); } From 1c17ebb7907a809a92f978995c27a53ead2526ee Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:41 +0206 Subject: [PATCH 101/180] printk: nbcon: Do not rely on proxy headers The headers kernel.h, serial_core.h, and console.h allow for the definitions of many types and functions from other headers. Rather than relying on these as proxy headers, explicitly include all headers providing needed definitions. Also sort the list alphabetically to be able to easily detect duplicates. Suggested-by: Andy Shevchenko Signed-off-by: John Ogness Reviewed-by: Andy Shevchenko Acked-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-16-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/internal.h | 8 ++++++-- kernel/printk/nbcon.c | 13 ++++++++++++- kernel/printk/printk_ringbuffer.h | 2 ++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index dc8bc0890fd2..ccb916688178 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -2,11 +2,12 @@ /* * internal.h - printk internal definitions */ -#include #include -#include "printk_ringbuffer.h" +#include +#include #if defined(CONFIG_PRINTK) && defined(CONFIG_SYSCTL) +struct ctl_table; void __init printk_sysctl_init(void); int devkmsg_sysctl_set_loglvl(const struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); @@ -43,6 +44,9 @@ enum printk_info_flags { LOG_CONT = 8, /* text is a fragment of a continuation line */ }; +struct printk_ringbuffer; +struct dev_printk_info; + extern struct printk_ringbuffer *prb; __printf(4, 0) diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index 61f0ae6a4809..e8ddcb6f7053 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -2,13 +2,24 @@ // Copyright (C) 2022 Linutronix GmbH, John Ogness // Copyright (C) 2022 Intel, Thomas Gleixner -#include +#include +#include #include #include +#include #include +#include +#include +#include +#include +#include #include +#include +#include #include +#include #include "internal.h" +#include "printk_ringbuffer.h" /* * Printk console printing implementation for consoles which does not depend * on the legacy style console_lock mechanism. diff --git a/kernel/printk/printk_ringbuffer.h b/kernel/printk/printk_ringbuffer.h index 52626d0f1fa3..bd2a892deac1 100644 --- a/kernel/printk/printk_ringbuffer.h +++ b/kernel/printk/printk_ringbuffer.h @@ -5,6 +5,8 @@ #include #include +#include +#include /* * Meta information about each stored message. From 864c25c83d834bdf2cb5a24c768d37236b418410 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:42 +0206 Subject: [PATCH 102/180] printk: Make console_is_usable() available to nbcon.c Move console_is_usable() as-is into internal.h so that it can be used by nbcon printing functions as well. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-17-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/internal.h | 32 ++++++++++++++++++++++++++++++++ kernel/printk/printk.c | 30 ------------------------------ 2 files changed, 32 insertions(+), 30 deletions(-) diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index ccb916688178..5d9deb56b582 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -84,6 +84,36 @@ void nbcon_seq_force(struct console *con, u64 seq); bool nbcon_alloc(struct console *con); void nbcon_free(struct console *con); +/* + * Check if the given console is currently capable and allowed to print + * records. + * + * Requires the console_srcu_read_lock. + */ +static inline bool console_is_usable(struct console *con) +{ + short flags = console_srcu_read_flags(con); + + if (!(flags & CON_ENABLED)) + return false; + + if ((flags & CON_SUSPENDED)) + return false; + + if (!con->write) + return false; + + /* + * Console drivers may assume that per-cpu resources have been + * allocated. So unless they're explicitly marked as being able to + * cope (CON_ANYTIME) don't call them until this CPU is officially up. + */ + if (!cpu_online(raw_smp_processor_id()) && !(flags & CON_ANYTIME)) + return false; + + return true; +} + #else #define PRINTK_PREFIX_MAX 0 @@ -104,6 +134,8 @@ static inline void nbcon_seq_force(struct console *con, u64 seq) { } static inline bool nbcon_alloc(struct console *con) { return false; } static inline void nbcon_free(struct console *con) { } +static inline bool console_is_usable(struct console *con) { return false; } + #endif /* CONFIG_PRINTK */ extern struct printk_buffers printk_shared_pbufs; diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 4cd2c50dd06d..b9c8fff9a493 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -2767,36 +2767,6 @@ int is_console_locked(void) } EXPORT_SYMBOL(is_console_locked); -/* - * Check if the given console is currently capable and allowed to print - * records. - * - * Requires the console_srcu_read_lock. - */ -static inline bool console_is_usable(struct console *con) -{ - short flags = console_srcu_read_flags(con); - - if (!(flags & CON_ENABLED)) - return false; - - if ((flags & CON_SUSPENDED)) - return false; - - if (!con->write) - return false; - - /* - * Console drivers may assume that per-cpu resources have been - * allocated. So unless they're explicitly marked as being able to - * cope (CON_ANYTIME) don't call them until this CPU is officially up. - */ - if (!cpu_online(raw_smp_processor_id()) && !(flags & CON_ANYTIME)) - return false; - - return true; -} - static void __console_unlock(void) { console_locked = 0; From 20846d1ce2adacd2b1f8672e24d6acb26b2e757b Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:43 +0206 Subject: [PATCH 103/180] printk: Let console_is_usable() handle nbcon The nbcon consoles use a different printing callback. For nbcon consoles, check for the write_atomic() callback instead of write(). Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-18-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/internal.h | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index 5d9deb56b582..448a5fcd5228 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -86,6 +86,8 @@ void nbcon_free(struct console *con); /* * Check if the given console is currently capable and allowed to print + * records. Note that this function does not consider the current context, + * which can also play a role in deciding if @con can be used to print * records. * * Requires the console_srcu_read_lock. @@ -100,8 +102,13 @@ static inline bool console_is_usable(struct console *con) if ((flags & CON_SUSPENDED)) return false; - if (!con->write) - return false; + if (flags & CON_NBCON) { + if (!con->write_atomic) + return false; + } else { + if (!con->write) + return false; + } /* * Console drivers may assume that per-cpu resources have been From fc400d5f63570afdadd718ae962cf5aa0feeace6 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:44 +0206 Subject: [PATCH 104/180] printk: Add @flags argument for console_is_usable() The caller of console_is_usable() usually needs @console->flags for its own checks. Rather than having console_is_usable() read its own copy, make the caller pass in the @flags. This also ensures that the caller saw the same @flags value. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-19-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/internal.h | 8 ++------ kernel/printk/printk.c | 5 +++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index 448a5fcd5228..fe8d84d78f1c 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -89,13 +89,9 @@ void nbcon_free(struct console *con); * records. Note that this function does not consider the current context, * which can also play a role in deciding if @con can be used to print * records. - * - * Requires the console_srcu_read_lock. */ -static inline bool console_is_usable(struct console *con) +static inline bool console_is_usable(struct console *con, short flags) { - short flags = console_srcu_read_flags(con); - if (!(flags & CON_ENABLED)) return false; @@ -141,7 +137,7 @@ static inline void nbcon_seq_force(struct console *con, u64 seq) { } static inline bool nbcon_alloc(struct console *con) { return false; } static inline void nbcon_free(struct console *con) { } -static inline bool console_is_usable(struct console *con) { return false; } +static inline bool console_is_usable(struct console *con, short flags) { return false; } #endif /* CONFIG_PRINTK */ diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index b9c8fff9a493..ffb56c2150b0 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -3012,9 +3012,10 @@ static bool console_flush_all(bool do_cond_resched, u64 *next_seq, bool *handove cookie = console_srcu_read_lock(); for_each_console_srcu(con) { + short flags = console_srcu_read_flags(con); bool progress; - if (!console_is_usable(con)) + if (!console_is_usable(con, flags)) continue; any_usable = true; @@ -3925,7 +3926,7 @@ static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre * that they make forward progress, so only increment * @diff for usable consoles. */ - if (!console_is_usable(c)) + if (!console_is_usable(c, flags)) continue; if (flags & CON_NBCON) { From 06683a6649895ccf279c35ca2fb77fd7afb7a6c5 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:45 +0206 Subject: [PATCH 105/180] printk: nbcon: Add helper to assign priority based on CPU state Add a helper function to use the current state of the CPU to determine which priority to assign to the printing context. The EMERGENCY priority handling is added in a follow-up commit. It will use a per-CPU variable. Note: nbcon_device_try_acquire(), which is used by console drivers to acquire the nbcon console for non-printing activities, is hard-coded to always use NORMAL priority. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-20-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/internal.h | 2 ++ kernel/printk/nbcon.c | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index fe8d84d78f1c..72f229382cfa 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -83,6 +83,7 @@ u64 nbcon_seq_read(struct console *con); void nbcon_seq_force(struct console *con, u64 seq); bool nbcon_alloc(struct console *con); void nbcon_free(struct console *con); +enum nbcon_prio nbcon_get_default_prio(void); /* * Check if the given console is currently capable and allowed to print @@ -136,6 +137,7 @@ static inline u64 nbcon_seq_read(struct console *con) { return 0; } static inline void nbcon_seq_force(struct console *con, u64 seq) { } static inline bool nbcon_alloc(struct console *con) { return false; } static inline void nbcon_free(struct console *con) { } +static inline enum nbcon_prio nbcon_get_default_prio(void) { return NBCON_PRIO_NONE; } static inline bool console_is_usable(struct console *con, short flags) { return false; } diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index e8ddcb6f7053..c6a9aa9f62f6 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -973,6 +973,25 @@ update_con: return nbcon_context_exit_unsafe(ctxt); } +/** + * nbcon_get_default_prio - The appropriate nbcon priority to use for nbcon + * printing on the current CPU + * + * Context: Any context. + * Return: The nbcon_prio to use for acquiring an nbcon console in this + * context for printing. + * + * The function is safe for reading per-CPU data in any context because + * preemption is disabled if the current CPU is in the panic state. + */ +enum nbcon_prio nbcon_get_default_prio(void) +{ + if (this_cpu_in_panic()) + return NBCON_PRIO_PANIC; + + return NBCON_PRIO_NORMAL; +} + /** * nbcon_alloc - Allocate and init the nbcon console specific data * @con: Console to initialize From d3a9f82ec5c095d6eb1eb94ecaa494470b4cef70 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 20 Aug 2024 08:35:46 +0206 Subject: [PATCH 106/180] printk: nbcon: Provide function to flush using write_atomic() Provide nbcon_atomic_flush_pending() to perform flushing of all registered nbcon consoles using their write_atomic() callback. Unlike console_flush_all(), nbcon_atomic_flush_pending() will only flush up through the newest record at the time of the call. This prevents a CPU from printing unbounded when other CPUs are adding records. If new records are added while flushing, it is expected that the dedicated printer threads will print those records. If the printer thread is not available (which is always the case at this point in the rework), nbcon_atomic_flush_pending() _will_ flush all records in the ringbuffer. Unlike console_flush_all(), nbcon_atomic_flush_pending() will fully flush one console before flushing the next. This helps to guarantee that a block of pending records (such as a stack trace in an emergency situation) can be printed atomically at once before releasing console ownership. nbcon_atomic_flush_pending() is safe in any context because it uses write_atomic() and acquires with unsafe_takeover disabled. Co-developed-by: John Ogness Signed-off-by: John Ogness Signed-off-by: Thomas Gleixner (Intel) Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-21-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/internal.h | 2 + kernel/printk/nbcon.c | 151 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 152 insertions(+), 1 deletion(-) diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index 72f229382cfa..0dc9b92b5dd0 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -84,6 +84,7 @@ void nbcon_seq_force(struct console *con, u64 seq); bool nbcon_alloc(struct console *con); void nbcon_free(struct console *con); enum nbcon_prio nbcon_get_default_prio(void); +void nbcon_atomic_flush_pending(void); /* * Check if the given console is currently capable and allowed to print @@ -138,6 +139,7 @@ static inline void nbcon_seq_force(struct console *con, u64 seq) { } static inline bool nbcon_alloc(struct console *con) { return false; } static inline void nbcon_free(struct console *con) { } static inline enum nbcon_prio nbcon_get_default_prio(void) { return NBCON_PRIO_NONE; } +static inline void nbcon_atomic_flush_pending(void) { } static inline bool console_is_usable(struct console *con, short flags) { return false; } diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index c6a9aa9f62f6..3982d68979d6 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -886,7 +886,6 @@ EXPORT_SYMBOL_GPL(nbcon_exit_unsafe); * When true is returned, @wctxt->ctxt.backlog indicates whether there are * still records pending in the ringbuffer, */ -__maybe_unused static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt) { struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt); @@ -992,6 +991,156 @@ enum nbcon_prio nbcon_get_default_prio(void) return NBCON_PRIO_NORMAL; } +/* + * __nbcon_atomic_flush_pending_con - Flush specified nbcon console using its + * write_atomic() callback + * @con: The nbcon console to flush + * @stop_seq: Flush up until this record + * + * Return: 0 if @con was flushed up to @stop_seq Otherwise, error code on + * failure. + * + * Errors: + * + * -EPERM: Unable to acquire console ownership. + * + * -EAGAIN: Another context took over ownership while printing. + * + * -ENOENT: A record before @stop_seq is not available. + * + * If flushing up to @stop_seq was not successful, it only makes sense for the + * caller to try again when -EAGAIN was returned. When -EPERM is returned, + * this context is not allowed to acquire the console. When -ENOENT is + * returned, it cannot be expected that the unfinalized record will become + * available. + */ +static int __nbcon_atomic_flush_pending_con(struct console *con, u64 stop_seq) +{ + struct nbcon_write_context wctxt = { }; + struct nbcon_context *ctxt = &ACCESS_PRIVATE(&wctxt, ctxt); + int err = 0; + + ctxt->console = con; + ctxt->spinwait_max_us = 2000; + ctxt->prio = nbcon_get_default_prio(); + + if (!nbcon_context_try_acquire(ctxt)) + return -EPERM; + + while (nbcon_seq_read(con) < stop_seq) { + /* + * nbcon_emit_next_record() returns false when the console was + * handed over or taken over. In both cases the context is no + * longer valid. + */ + if (!nbcon_emit_next_record(&wctxt)) + return -EAGAIN; + + if (!ctxt->backlog) { + /* Are there reserved but not yet finalized records? */ + if (nbcon_seq_read(con) < stop_seq) + err = -ENOENT; + break; + } + } + + nbcon_context_release(ctxt); + return err; +} + +/** + * nbcon_atomic_flush_pending_con - Flush specified nbcon console using its + * write_atomic() callback + * @con: The nbcon console to flush + * @stop_seq: Flush up until this record + * + * This will stop flushing before @stop_seq if another context has ownership. + * That context is then responsible for the flushing. Likewise, if new records + * are added while this context was flushing and there is no other context + * to handle the printing, this context must also flush those records. + */ +static void nbcon_atomic_flush_pending_con(struct console *con, u64 stop_seq) +{ + unsigned long flags; + int err; + +again: + /* + * Atomic flushing does not use console driver synchronization (i.e. + * it does not hold the port lock for uart consoles). Therefore IRQs + * must be disabled to avoid being interrupted and then calling into + * a driver that will deadlock trying to acquire console ownership. + */ + local_irq_save(flags); + + err = __nbcon_atomic_flush_pending_con(con, stop_seq); + + local_irq_restore(flags); + + /* + * If there was a new owner (-EPERM, -EAGAIN), that context is + * responsible for completing. + * + * Do not wait for records not yet finalized (-ENOENT) to avoid a + * possible deadlock. They will either get flushed by the writer or + * eventually skipped on panic CPU. + */ + if (err) + return; + + /* + * If flushing was successful but more records are available, this + * context must flush those remaining records because there is no + * other context that will do it. + */ + if (prb_read_valid(prb, nbcon_seq_read(con), NULL)) { + stop_seq = prb_next_reserve_seq(prb); + goto again; + } +} + +/** + * __nbcon_atomic_flush_pending - Flush all nbcon consoles using their + * write_atomic() callback + * @stop_seq: Flush up until this record + */ +static void __nbcon_atomic_flush_pending(u64 stop_seq) +{ + struct console *con; + int cookie; + + cookie = console_srcu_read_lock(); + for_each_console_srcu(con) { + short flags = console_srcu_read_flags(con); + + if (!(flags & CON_NBCON)) + continue; + + if (!console_is_usable(con, flags)) + continue; + + if (nbcon_seq_read(con) >= stop_seq) + continue; + + nbcon_atomic_flush_pending_con(con, stop_seq); + } + console_srcu_read_unlock(cookie); +} + +/** + * nbcon_atomic_flush_pending - Flush all nbcon consoles using their + * write_atomic() callback + * + * Flush the backlog up through the currently newest record. Any new + * records added while flushing will not be flushed if there is another + * context available to handle the flushing. This is to avoid one CPU + * printing unbounded because other CPUs continue to add records. + */ +void nbcon_atomic_flush_pending(void) +{ + __nbcon_atomic_flush_pending(prb_next_reserve_seq(prb)); +} + /** * nbcon_alloc - Allocate and init the nbcon console specific data * @con: Console to initialize From 97ea9bccfcbe4c97f127e736823cc8d984d781bf Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:47 +0206 Subject: [PATCH 107/180] printk: Track registered boot consoles Unfortunately it is not known if a boot console and a regular (legacy or nbcon) console use the same hardware. For this reason they must not be allowed to print simultaneously. For legacy consoles this is not an issue because they are already synchronized with the boot consoles using the console lock. However nbcon consoles can be triggered separately. Add a global flag @have_boot_console to identify if any boot consoles are registered. This will be used in follow-up commits to ensure that boot consoles and nbcon consoles cannot print simultaneously. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-22-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/printk.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index ffb56c2150b0..b8634a153d1d 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -463,6 +463,14 @@ static int console_msg_format = MSG_FORMAT_DEFAULT; /* syslog_lock protects syslog_* variables and write access to clear_seq. */ static DEFINE_MUTEX(syslog_lock); +/* + * Specifies if a boot console is registered. If boot consoles are present, + * nbcon consoles cannot print simultaneously and must be synchronized by + * the console lock. This is because boot consoles and nbcon consoles may + * have mapped the same hardware. + */ +static bool have_boot_console; + #ifdef CONFIG_PRINTK DECLARE_WAIT_QUEUE_HEAD(log_wait); /* All 3 protected by @syslog_lock. */ @@ -3610,6 +3618,9 @@ void register_console(struct console *newcon) newcon->seq = init_seq; } + if (newcon->flags & CON_BOOT) + have_boot_console = true; + /* * If another context is actively using the hardware of this new * console, it will not be aware of the nbcon synchronization. This @@ -3680,7 +3691,9 @@ EXPORT_SYMBOL(register_console); static int unregister_console_locked(struct console *console) { bool use_device_lock = (console->flags & CON_NBCON) && console->write_atomic; + bool found_boot_con = false; unsigned long flags; + struct console *c; int res; lockdep_assert_console_list_lock_held(); @@ -3738,6 +3751,17 @@ static int unregister_console_locked(struct console *console) if (console->exit) res = console->exit(console); + /* + * With this console gone, the global flags tracking registered + * console types may have changed. Update them. + */ + for_each_console(c) { + if (c->flags & CON_BOOT) + found_boot_con = true; + } + if (!found_boot_con) + have_boot_console = found_boot_con; + return res; } From c158834b223fbfab3a14855ac203b8d9cddbbefd Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:48 +0206 Subject: [PATCH 108/180] printk: nbcon: Use nbcon consoles in console_flush_all() Allow nbcon consoles to print messages in the legacy printk() caller context (printing via unlock) by integrating them into console_flush_all(). The write_atomic() callback is used for printing. Provide nbcon_legacy_emit_next_record(), which acts as the nbcon variant of console_emit_next_record(). Call this variant within console_flush_all() for nbcon consoles. Since nbcon consoles use their own @nbcon_seq variable to track the next record to print, this also must be appropriately handled in console_flush_all(). Note that the legacy printing logic uses @handover to detect handovers for printing all consoles. For nbcon consoles, handovers/takeovers occur on a per-console basis and thus do not cause the console_flush_all() loop to abort. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-23-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/internal.h | 6 +++ kernel/printk/nbcon.c | 87 ++++++++++++++++++++++++++++++++++++++++ kernel/printk/printk.c | 17 +++++--- 3 files changed, 105 insertions(+), 5 deletions(-) diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index 0dc9b92b5dd0..44468f3828f3 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -78,6 +78,8 @@ void defer_console_output(void); u16 printk_parse_prefix(const char *text, int *level, enum printk_info_flags *flags); +void console_lock_spinning_enable(void); +int console_lock_spinning_disable_and_check(int cookie); u64 nbcon_seq_read(struct console *con); void nbcon_seq_force(struct console *con, u64 seq); @@ -85,6 +87,8 @@ bool nbcon_alloc(struct console *con); void nbcon_free(struct console *con); enum nbcon_prio nbcon_get_default_prio(void); void nbcon_atomic_flush_pending(void); +bool nbcon_legacy_emit_next_record(struct console *con, bool *handover, + int cookie); /* * Check if the given console is currently capable and allowed to print @@ -140,6 +144,8 @@ static inline bool nbcon_alloc(struct console *con) { return false; } static inline void nbcon_free(struct console *con) { } static inline enum nbcon_prio nbcon_get_default_prio(void) { return NBCON_PRIO_NONE; } static inline void nbcon_atomic_flush_pending(void) { } +static inline bool nbcon_legacy_emit_next_record(struct console *con, bool *handover, + int cookie) { return false; } static inline bool console_is_usable(struct console *con, short flags) { return false; } diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index 3982d68979d6..d09c084c9af4 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -992,6 +992,93 @@ enum nbcon_prio nbcon_get_default_prio(void) } /* + * nbcon_atomic_emit_one - Print one record for an nbcon console using the + * write_atomic() callback + * @wctxt: An initialized write context struct to use for this context + * + * Return: True, when a record has been printed and there are still + * pending records. The caller might want to continue flushing. + * + * False, when there is no pending record, or when the console + * context cannot be acquired, or the ownership has been lost. + * The caller should give up. Either the job is done, cannot be + * done, or will be handled by the owning context. + * + * This is an internal helper to handle the locking of the console before + * calling nbcon_emit_next_record(). + */ +static bool nbcon_atomic_emit_one(struct nbcon_write_context *wctxt) +{ + struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt); + + if (!nbcon_context_try_acquire(ctxt)) + return false; + + /* + * nbcon_emit_next_record() returns false when the console was + * handed over or taken over. In both cases the context is no + * longer valid. + * + * The higher priority printing context takes over responsibility + * to print the pending records. + */ + if (!nbcon_emit_next_record(wctxt)) + return false; + + nbcon_context_release(ctxt); + + return ctxt->backlog; +} + +/** + * nbcon_legacy_emit_next_record - Print one record for an nbcon console + * in legacy contexts + * @con: The console to print on + * @handover: Will be set to true if a printk waiter has taken over the + * console_lock, in which case the caller is no longer holding + * both the console_lock and the SRCU read lock. Otherwise it + * is set to false. + * @cookie: The cookie from the SRCU read lock. + * + * Context: Any context except NMI. + * Return: True, when a record has been printed and there are still + * pending records. The caller might want to continue flushing. + * + * False, when there is no pending record, or when the console + * context cannot be acquired, or the ownership has been lost. + * The caller should give up. Either the job is done, cannot be + * done, or will be handled by the owning context. + * + * This function is meant to be called by console_flush_all() to print records + * on nbcon consoles from legacy context (printing via console unlocking). + * Essentially it is the nbcon version of console_emit_next_record(). + */ +bool nbcon_legacy_emit_next_record(struct console *con, bool *handover, + int cookie) +{ + struct nbcon_write_context wctxt = { }; + struct nbcon_context *ctxt = &ACCESS_PRIVATE(&wctxt, ctxt); + unsigned long flags; + bool progress; + + /* Use the same procedure as console_emit_next_record(). */ + printk_safe_enter_irqsave(flags); + console_lock_spinning_enable(); + stop_critical_timings(); + + ctxt->console = con; + ctxt->prio = nbcon_get_default_prio(); + + progress = nbcon_atomic_emit_one(&wctxt); + + start_critical_timings(); + *handover = console_lock_spinning_disable_and_check(cookie); + printk_safe_exit_irqrestore(flags); + + return progress; +} + +/** * __nbcon_atomic_flush_pending_con - Flush specified nbcon console using its * write_atomic() callback * @con: The nbcon console to flush diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index b8634a153d1d..f08bf5e82fc7 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -1860,7 +1860,7 @@ static bool console_waiter; * there may be a waiter spinning (like a spinlock). Also it must be * ready to hand over the lock at the end of the section. */ -static void console_lock_spinning_enable(void) +void console_lock_spinning_enable(void) { /* * Do not use spinning in panic(). The panic CPU wants to keep the lock. @@ -1899,7 +1899,7 @@ lockdep: * * Return: 1 if the lock rights were passed, 0 otherwise. */ -static int console_lock_spinning_disable_and_check(int cookie) +int console_lock_spinning_disable_and_check(int cookie) { int waiter; @@ -3021,13 +3021,20 @@ static bool console_flush_all(bool do_cond_resched, u64 *next_seq, bool *handove cookie = console_srcu_read_lock(); for_each_console_srcu(con) { short flags = console_srcu_read_flags(con); + u64 printk_seq; bool progress; if (!console_is_usable(con, flags)) continue; any_usable = true; - progress = console_emit_next_record(con, handover, cookie); + if (flags & CON_NBCON) { + progress = nbcon_legacy_emit_next_record(con, handover, cookie); + printk_seq = nbcon_seq_read(con); + } else { + progress = console_emit_next_record(con, handover, cookie); + printk_seq = con->seq; + } /* * If a handover has occurred, the SRCU read lock @@ -3037,8 +3044,8 @@ static bool console_flush_all(bool do_cond_resched, u64 *next_seq, bool *handove return false; /* Track the next of the highest seq flushed. */ - if (con->seq > *next_seq) - *next_seq = con->seq; + if (printk_seq > *next_seq) + *next_seq = printk_seq; if (!progress) continue; From 70411bf8d22ab3d2f794f199d81d70b62d3a85fa Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:49 +0206 Subject: [PATCH 109/180] printk: Add is_printk_legacy_deferred() If printk has been explicitly deferred or is called from NMI context, legacy console printing must be deferred to an irq_work context. Introduce a helper function is_printk_legacy_deferred() for a CPU to query if it must defer legacy console printing. In follow-up commits this helper will be needed at other call sites as well. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-24-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/internal.h | 2 ++ kernel/printk/printk_safe.c | 11 ++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index 44468f3828f3..84706c1c934b 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -75,6 +75,7 @@ bool printk_percpu_data_ready(void); } while (0) void defer_console_output(void); +bool is_printk_legacy_deferred(void); u16 printk_parse_prefix(const char *text, int *level, enum printk_info_flags *flags); @@ -138,6 +139,7 @@ static inline bool console_is_usable(struct console *con, short flags) #define printk_safe_exit_irqrestore(flags) local_irq_restore(flags) static inline bool printk_percpu_data_ready(void) { return false; } +static inline bool is_printk_legacy_deferred(void) { return false; } static inline u64 nbcon_seq_read(struct console *con) { return 0; } static inline void nbcon_seq_force(struct console *con, u64 seq) { } static inline bool nbcon_alloc(struct console *con) { return false; } diff --git a/kernel/printk/printk_safe.c b/kernel/printk/printk_safe.c index 4421ccac3113..86439fd20aab 100644 --- a/kernel/printk/printk_safe.c +++ b/kernel/printk/printk_safe.c @@ -38,6 +38,15 @@ void __printk_deferred_exit(void) __printk_safe_exit(); } +bool is_printk_legacy_deferred(void) +{ + /* + * The per-CPU variable @printk_context can be read safely in any + * context. CPU migration is always disabled when set. + */ + return (this_cpu_read(printk_context) || in_nmi()); +} + asmlinkage int vprintk(const char *fmt, va_list args) { #ifdef CONFIG_KGDB_KDB @@ -50,7 +59,7 @@ asmlinkage int vprintk(const char *fmt, va_list args) * Use the main logbuf even in NMI. But avoid calling console * drivers that might have their own locks. */ - if (this_cpu_read(printk_context) || in_nmi()) + if (is_printk_legacy_deferred()) return vprintk_deferred(fmt, args); /* No obstacles. */ From 8ba77712a7501ca941603d6d5ed650cd0d42cafb Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:50 +0206 Subject: [PATCH 110/180] printk: nbcon: Flush new records on device_release() There may be new records that were added while a driver was holding the nbcon context for non-printing purposes. These new records must be flushed by the nbcon_device_release() context because no other context will do it. If boot consoles are registered, the legacy loop is used (either direct or per irq_work) to handle the flushing. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-25-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/internal.h | 2 ++ kernel/printk/nbcon.c | 20 ++++++++++++++++++++ kernel/printk/printk.c | 2 +- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index 84706c1c934b..7679e18f24b3 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -153,6 +153,8 @@ static inline bool console_is_usable(struct console *con, short flags) { return #endif /* CONFIG_PRINTK */ +extern bool have_boot_console; + extern struct printk_buffers printk_shared_pbufs; /** diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index d09c084c9af4..269aeed18064 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -1326,10 +1326,30 @@ EXPORT_SYMBOL_GPL(nbcon_device_try_acquire); void nbcon_device_release(struct console *con) { struct nbcon_context *ctxt = &ACCESS_PRIVATE(con, nbcon_device_ctxt); + int cookie; if (!nbcon_context_exit_unsafe(ctxt)) return; nbcon_context_release(ctxt); + + /* + * This context must flush any new records added while the console + * was locked. The console_srcu_read_lock must be taken to ensure + * the console is usable throughout flushing. + */ + cookie = console_srcu_read_lock(); + if (console_is_usable(con, console_srcu_read_flags(con)) && + prb_read_valid(prb, nbcon_seq_read(con), NULL)) { + if (!have_boot_console) { + __nbcon_atomic_flush_pending_con(con, prb_next_reserve_seq(prb)); + } else if (!is_printk_legacy_deferred()) { + if (console_trylock()) + console_unlock(); + } else { + printk_trigger_flush(); + } + } + console_srcu_read_unlock(cookie); } EXPORT_SYMBOL_GPL(nbcon_device_release); diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index f08bf5e82fc7..7c9f8f6e1738 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -469,7 +469,7 @@ static DEFINE_MUTEX(syslog_lock); * the console lock. This is because boot consoles and nbcon consoles may * have mapped the same hardware. */ -static bool have_boot_console; +bool have_boot_console; #ifdef CONFIG_PRINTK DECLARE_WAIT_QUEUE_HEAD(log_wait); From d2e85ca7a736d2d889bac9aef6ede0a67f6870b2 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:51 +0206 Subject: [PATCH 111/180] printk: Flush nbcon consoles first on panic In console_flush_on_panic(), flush the nbcon consoles before flushing legacy consoles. The legacy write() callbacks are not fully safe when oops_in_progress is set. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-26-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/printk.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 7c9f8f6e1738..c6e633329e4d 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -3269,6 +3269,9 @@ void console_flush_on_panic(enum con_flush_mode mode) if (mode == CONSOLE_REPLAY_ALL) __console_rewind_all(); + if (!have_boot_console) + nbcon_atomic_flush_pending(); + console_flush_all(false, &next_seq, &handover); } From 5dde3b7354133846079fa51e55e74ef90a836759 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:52 +0206 Subject: [PATCH 112/180] printk: nbcon: Add unsafe flushing on panic Add nbcon_atomic_flush_unsafe() to flush all nbcon consoles using the write_atomic() callback and allowing unsafe hostile takeovers. Call this at the end of panic() as a final attempt to flush any pending messages. Note that legacy consoles use unsafe methods for flushing from the beginning of panic (see bust_spinlocks()). Therefore, systems using both legacy and nbcon consoles may still fail to see panic messages due to unsafe legacy console usage. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-27-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- include/linux/printk.h | 5 +++++ kernel/panic.c | 1 + kernel/printk/nbcon.c | 32 +++++++++++++++++++++++++------- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/include/linux/printk.h b/include/linux/printk.h index 9687089f5ace..2e083f01f8a3 100644 --- a/include/linux/printk.h +++ b/include/linux/printk.h @@ -202,6 +202,7 @@ void printk_trigger_flush(void); void console_try_replay_all(void); extern bool nbcon_device_try_acquire(struct console *con); extern void nbcon_device_release(struct console *con); +void nbcon_atomic_flush_unsafe(void); #else static inline __printf(1, 0) int vprintk(const char *s, va_list args) @@ -294,6 +295,10 @@ static inline void nbcon_device_release(struct console *con) { } +static inline void nbcon_atomic_flush_unsafe(void) +{ +} + #endif bool this_cpu_in_panic(void); diff --git a/kernel/panic.c b/kernel/panic.c index 2a0449144f82..df37c913b010 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -463,6 +463,7 @@ void panic(const char *fmt, ...) * Explicitly flush the kernel log buffer one last time. */ console_flush_on_panic(CONSOLE_FLUSH_PENDING); + nbcon_atomic_flush_unsafe(); local_irq_enable(); for (i = 0; ; i += PANIC_TIMER_STEP) { diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index 269aeed18064..afdb16c1c733 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -1083,6 +1083,7 @@ bool nbcon_legacy_emit_next_record(struct console *con, bool *handover, * write_atomic() callback * @con: The nbcon console to flush * @stop_seq: Flush up until this record + * @allow_unsafe_takeover: True, to allow unsafe hostile takeovers * * Return: 0 if @con was flushed up to @stop_seq Otherwise, error code on * failure. @@ -1101,7 +1102,8 @@ bool nbcon_legacy_emit_next_record(struct console *con, bool *handover, * returned, it cannot be expected that the unfinalized record will become * available. */ -static int __nbcon_atomic_flush_pending_con(struct console *con, u64 stop_seq) +static int __nbcon_atomic_flush_pending_con(struct console *con, u64 stop_seq, + bool allow_unsafe_takeover) { struct nbcon_write_context wctxt = { }; struct nbcon_context *ctxt = &ACCESS_PRIVATE(&wctxt, ctxt); @@ -1110,6 +1112,7 @@ static int __nbcon_atomic_flush_pending_con(struct console *con, u64 stop_seq) ctxt->console = con; ctxt->spinwait_max_us = 2000; ctxt->prio = nbcon_get_default_prio(); + ctxt->allow_unsafe_takeover = allow_unsafe_takeover; if (!nbcon_context_try_acquire(ctxt)) return -EPERM; @@ -1140,13 +1143,15 @@ static int __nbcon_atomic_flush_pending_con(struct console *con, u64 stop_seq) * write_atomic() callback * @con: The nbcon console to flush * @stop_seq: Flush up until this record + * @allow_unsafe_takeover: True, to allow unsafe hostile takeovers * * This will stop flushing before @stop_seq if another context has ownership. * That context is then responsible for the flushing. Likewise, if new records * are added while this context was flushing and there is no other context * to handle the printing, this context must also flush those records. */ -static void nbcon_atomic_flush_pending_con(struct console *con, u64 stop_seq) +static void nbcon_atomic_flush_pending_con(struct console *con, u64 stop_seq, + bool allow_unsafe_takeover) { unsigned long flags; int err; @@ -1160,7 +1165,7 @@ again: */ local_irq_save(flags); - err = __nbcon_atomic_flush_pending_con(con, stop_seq); + err = __nbcon_atomic_flush_pending_con(con, stop_seq, allow_unsafe_takeover); local_irq_restore(flags); @@ -1190,8 +1195,9 @@ again: * __nbcon_atomic_flush_pending - Flush all nbcon consoles using their * write_atomic() callback * @stop_seq: Flush up until this record + * @allow_unsafe_takeover: True, to allow unsafe hostile takeovers */ -static void __nbcon_atomic_flush_pending(u64 stop_seq) +static void __nbcon_atomic_flush_pending(u64 stop_seq, bool allow_unsafe_takeover) { struct console *con; int cookie; @@ -1209,7 +1215,7 @@ static void __nbcon_atomic_flush_pending(u64 stop_seq) if (nbcon_seq_read(con) >= stop_seq) continue; - nbcon_atomic_flush_pending_con(con, stop_seq); + nbcon_atomic_flush_pending_con(con, stop_seq, allow_unsafe_takeover); } console_srcu_read_unlock(cookie); } @@ -1225,7 +1231,19 @@ static void __nbcon_atomic_flush_pending(u64 stop_seq) */ void nbcon_atomic_flush_pending(void) { - __nbcon_atomic_flush_pending(prb_next_reserve_seq(prb)); + __nbcon_atomic_flush_pending(prb_next_reserve_seq(prb), false); +} + +/** + * nbcon_atomic_flush_unsafe - Flush all nbcon consoles using their + * write_atomic() callback and allowing unsafe hostile takeovers + * + * Flush the backlog up through the currently newest record. Unsafe hostile + * takeovers will be performed, if necessary. + */ +void nbcon_atomic_flush_unsafe(void) +{ + __nbcon_atomic_flush_pending(prb_next_reserve_seq(prb), true); } /** @@ -1342,7 +1360,7 @@ void nbcon_device_release(struct console *con) if (console_is_usable(con, console_srcu_read_flags(con)) && prb_read_valid(prb, nbcon_seq_read(con), NULL)) { if (!have_boot_console) { - __nbcon_atomic_flush_pending_con(con, prb_next_reserve_seq(prb)); + __nbcon_atomic_flush_pending_con(con, prb_next_reserve_seq(prb), false); } else if (!is_printk_legacy_deferred()) { if (console_trylock()) console_unlock(); From 60013065fdc677df7b71f9a0bac501020e3bbd4f Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:53 +0206 Subject: [PATCH 113/180] printk: Avoid console_lock dance if no legacy or boot consoles Currently the console lock is used to attempt legacy-type printing even if there are no legacy or boot consoles registered. If no such consoles are registered, the console lock does not need to be taken. Add tracking of legacy console registration and use it with boot console tracking to avoid unnecessary code paths, i.e. do not use the console lock if there are no boot consoles and no legacy consoles. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-28-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/printk.c | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index c6e633329e4d..b3ddcf39a53c 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -463,6 +463,13 @@ static int console_msg_format = MSG_FORMAT_DEFAULT; /* syslog_lock protects syslog_* variables and write access to clear_seq. */ static DEFINE_MUTEX(syslog_lock); +/* + * Specifies if a legacy console is registered. If legacy consoles are + * present, it is necessary to perform the console lock/unlock dance + * whenever console flushing should occur. + */ +static bool have_legacy_console; + /* * Specifies if a boot console is registered. If boot consoles are present, * nbcon consoles cannot print simultaneously and must be synchronized by @@ -471,6 +478,14 @@ static DEFINE_MUTEX(syslog_lock); */ bool have_boot_console; +/* + * Specifies if the console lock/unlock dance is needed for console + * printing. If @have_boot_console is true, the nbcon consoles will + * be printed serially along with the legacy consoles because nbcon + * consoles cannot print simultaneously with boot consoles. + */ +#define printing_via_unlock (have_legacy_console || have_boot_console) + #ifdef CONFIG_PRINTK DECLARE_WAIT_QUEUE_HEAD(log_wait); /* All 3 protected by @syslog_lock. */ @@ -2339,7 +2354,7 @@ asmlinkage int vprintk_emit(int facility, int level, printed_len = vprintk_store(facility, level, dev_info, fmt, args); /* If called from the scheduler, we can not call up(). */ - if (!in_sched) { + if (!in_sched && printing_via_unlock) { /* * The caller may be holding system-critical or * timing-sensitive locks. Disable preemption during @@ -2359,7 +2374,7 @@ asmlinkage int vprintk_emit(int facility, int level, preempt_enable(); } - if (in_sched) + if (in_sched && printing_via_unlock) defer_console_output(); else wake_up_klogd(); @@ -2718,7 +2733,7 @@ void resume_console(void) */ static int console_cpu_notify(unsigned int cpu) { - if (!cpuhp_tasks_frozen) { + if (!cpuhp_tasks_frozen && printing_via_unlock) { /* If trylock fails, someone else is doing the printing */ if (console_trylock()) console_unlock(); @@ -3625,6 +3640,7 @@ void register_console(struct console *newcon) if (newcon->flags & CON_NBCON) { nbcon_seq_force(newcon, init_seq); } else { + have_legacy_console = true; newcon->seq = init_seq; } @@ -3701,6 +3717,7 @@ EXPORT_SYMBOL(register_console); static int unregister_console_locked(struct console *console) { bool use_device_lock = (console->flags & CON_NBCON) && console->write_atomic; + bool found_legacy_con = false; bool found_boot_con = false; unsigned long flags; struct console *c; @@ -3768,9 +3785,13 @@ static int unregister_console_locked(struct console *console) for_each_console(c) { if (c->flags & CON_BOOT) found_boot_con = true; + if (!(c->flags & CON_NBCON)) + found_legacy_con = true; } if (!found_boot_con) have_boot_console = found_boot_con; + if (!found_legacy_con) + have_legacy_console = found_legacy_con; return res; } @@ -3931,8 +3952,10 @@ static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre seq = prb_next_reserve_seq(prb); /* Flush the consoles so that records up to @seq are printed. */ - console_lock(); - console_unlock(); + if (printing_via_unlock) { + console_lock(); + console_unlock(); + } for (;;) { unsigned long begin_jiffies; @@ -3945,6 +3968,12 @@ static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre * console->seq. Releasing console_lock flushes more * records in case @seq is still not printed on all * usable consoles. + * + * Holding the console_lock is not necessary if there + * are no legacy or boot consoles. However, such a + * console could register at any time. Always hold the + * console_lock as a precaution rather than + * synchronizing against register_console(). */ console_lock(); From bebd87ae27e052e390c203893c7ee46f54b0bf9e Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:54 +0206 Subject: [PATCH 114/180] printk: Track nbcon consoles Add a global flag @have_nbcon_console to identify if any nbcon consoles are registered. This will be used in follow-up commits to preserve legacy behavior when no nbcon consoles are registered. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-29-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/printk.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index b3ddcf39a53c..e30107d216a5 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -470,6 +470,11 @@ static DEFINE_MUTEX(syslog_lock); */ static bool have_legacy_console; +/* + * Specifies if an nbcon console is registered. + */ +static bool have_nbcon_console; + /* * Specifies if a boot console is registered. If boot consoles are present, * nbcon consoles cannot print simultaneously and must be synchronized by @@ -3638,6 +3643,7 @@ void register_console(struct console *newcon) init_seq = get_init_console_seq(newcon, bootcon_registered); if (newcon->flags & CON_NBCON) { + have_nbcon_console = true; nbcon_seq_force(newcon, init_seq); } else { have_legacy_console = true; @@ -3718,6 +3724,7 @@ static int unregister_console_locked(struct console *console) { bool use_device_lock = (console->flags & CON_NBCON) && console->write_atomic; bool found_legacy_con = false; + bool found_nbcon_con = false; bool found_boot_con = false; unsigned long flags; struct console *c; @@ -3785,13 +3792,18 @@ static int unregister_console_locked(struct console *console) for_each_console(c) { if (c->flags & CON_BOOT) found_boot_con = true; - if (!(c->flags & CON_NBCON)) + + if (c->flags & CON_NBCON) + found_nbcon_con = true; + else found_legacy_con = true; } if (!found_boot_con) have_boot_console = found_boot_con; if (!found_legacy_con) have_legacy_console = found_legacy_con; + if (!found_nbcon_con) + have_nbcon_console = found_nbcon_con; return res; } From e35a8884270bae11196eedf3b0a5bf22619f11f2 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:55 +0206 Subject: [PATCH 115/180] printk: Coordinate direct printing in panic If legacy and nbcon consoles are registered and the nbcon consoles are allowed to flush (i.e. no boot consoles registered), the legacy consoles will no longer perform direct printing on the panic CPU until after the backtrace has been stored. This will give the safe nbcon consoles a chance to print the panic messages before allowing the unsafe legacy consoles to print. If no nbcon consoles are registered or they are not allowed to flush because boot consoles are registered, there is no change in behavior (i.e. legacy consoles will always attempt to print from the printk() caller context). Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-30-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- include/linux/printk.h | 5 ++++ kernel/panic.c | 2 ++ kernel/printk/internal.h | 1 + kernel/printk/printk.c | 55 +++++++++++++++++++++++++++++++++++----- 4 files changed, 56 insertions(+), 7 deletions(-) diff --git a/include/linux/printk.h b/include/linux/printk.h index 2e083f01f8a3..eca9bb2ee637 100644 --- a/include/linux/printk.h +++ b/include/linux/printk.h @@ -200,6 +200,7 @@ extern asmlinkage void dump_stack_lvl(const char *log_lvl) __cold; extern asmlinkage void dump_stack(void) __cold; void printk_trigger_flush(void); void console_try_replay_all(void); +void printk_legacy_allow_panic_sync(void); extern bool nbcon_device_try_acquire(struct console *con); extern void nbcon_device_release(struct console *con); void nbcon_atomic_flush_unsafe(void); @@ -286,6 +287,10 @@ static inline void console_try_replay_all(void) { } +static inline void printk_legacy_allow_panic_sync(void) +{ +} + static inline bool nbcon_device_try_acquire(struct console *con) { return false; diff --git a/kernel/panic.c b/kernel/panic.c index df37c913b010..93096d5abcc7 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -374,6 +374,8 @@ void panic(const char *fmt, ...) panic_other_cpus_shutdown(_crash_kexec_post_notifiers); + printk_legacy_allow_panic_sync(); + /* * Run any panic handlers, including those that might need to * add information to the kmsg dump output. diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index 7679e18f24b3..6b61350a3026 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -154,6 +154,7 @@ static inline bool console_is_usable(struct console *con, short flags) { return #endif /* CONFIG_PRINTK */ extern bool have_boot_console; +extern bool legacy_allow_panic_sync; extern struct printk_buffers printk_shared_pbufs; diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index e30107d216a5..fc3f8970eea2 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -471,7 +471,9 @@ static DEFINE_MUTEX(syslog_lock); static bool have_legacy_console; /* - * Specifies if an nbcon console is registered. + * Specifies if an nbcon console is registered. If nbcon consoles are present, + * synchronous printing of legacy consoles will not occur during panic until + * the backtrace has been stored to the ringbuffer. */ static bool have_nbcon_console; @@ -483,6 +485,9 @@ static bool have_nbcon_console; */ bool have_boot_console; +/* See printk_legacy_allow_panic_sync() for details. */ +bool legacy_allow_panic_sync; + /* * Specifies if the console lock/unlock dance is needed for console * printing. If @have_boot_console is true, the nbcon consoles will @@ -2330,12 +2335,28 @@ out: return ret; } +/* + * This acts as a one-way switch to allow legacy consoles to print from + * the printk() caller context on a panic CPU. It also attempts to flush + * the legacy consoles in this context. + */ +void printk_legacy_allow_panic_sync(void) +{ + legacy_allow_panic_sync = true; + + if (printing_via_unlock && !is_printk_legacy_deferred()) { + if (console_trylock()) + console_unlock(); + } +} + asmlinkage int vprintk_emit(int facility, int level, const struct dev_printk_info *dev_info, const char *fmt, va_list args) { + bool do_trylock_unlock = printing_via_unlock; + bool defer_legacy = false; int printed_len; - bool in_sched = false; /* Suppress unimportant messages after panic happens */ if (unlikely(suppress_printk)) @@ -2349,17 +2370,35 @@ asmlinkage int vprintk_emit(int facility, int level, if (other_cpu_in_panic() && !panic_triggering_all_cpu_backtrace) return 0; + /* If called from the scheduler, we can not call up(). */ if (level == LOGLEVEL_SCHED) { level = LOGLEVEL_DEFAULT; - in_sched = true; + defer_legacy = do_trylock_unlock; + do_trylock_unlock = false; } printk_delay(level); printed_len = vprintk_store(facility, level, dev_info, fmt, args); - /* If called from the scheduler, we can not call up(). */ - if (!in_sched && printing_via_unlock) { + if (have_nbcon_console && !have_boot_console) { + nbcon_atomic_flush_pending(); + + /* + * In panic, the legacy consoles are not allowed to print from + * the printk calling context unless explicitly allowed. This + * gives the safe nbcon consoles a chance to print out all the + * panic messages first. This restriction only applies if + * there are nbcon consoles registered and they are allowed to + * flush. + */ + if (this_cpu_in_panic() && !legacy_allow_panic_sync) { + do_trylock_unlock = false; + defer_legacy = false; + } + } + + if (do_trylock_unlock) { /* * The caller may be holding system-critical or * timing-sensitive locks. Disable preemption during @@ -2379,7 +2418,7 @@ asmlinkage int vprintk_emit(int facility, int level, preempt_enable(); } - if (in_sched && printing_via_unlock) + if (defer_legacy) defer_console_output(); else wake_up_klogd(); @@ -3292,7 +3331,9 @@ void console_flush_on_panic(enum con_flush_mode mode) if (!have_boot_console) nbcon_atomic_flush_pending(); - console_flush_all(false, &next_seq, &handover); + /* Flush legacy consoles once allowed, even when dangerous. */ + if (legacy_allow_panic_sync) + console_flush_all(false, &next_seq, &handover); } /* From 6690d6b52726bcb2b743466a1833e0b9f049b9d7 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:56 +0206 Subject: [PATCH 116/180] printk: Add helper for flush type logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are many call sites where console flushing occur. Depending on the system state and types of consoles, the flush methods to use are different. A flush call site generally must consider: @have_boot_console @have_nbcon_console @have_legacy_console @legacy_allow_panic_sync is_printk_preferred() and take into account the current CPU state: NBCON_PRIO_NORMAL NBCON_PRIO_EMERGENCY NBCON_PRIO_PANIC in order to decide if it should: flush nbcon directly via atomic_write() callback flush legacy directly via console_unlock flush legacy via offload to irq_work All of these call sites use their own logic to make this decision, which is complicated and error prone. Especially later when two more flush methods will be introduced: flush nbcon via offload to kthread flush legacy via offload to kthread Introduce a new internal struct console_flush_type that specifies which console flushing methods should be used in the context of the caller. Introduce a helper function to fill out console_flush_type to be used for flushing call sites. Replace the logic of all flushing call sites to use the new helper. This change standardizes behavior, leading to both fixes and optimizations across various call sites. For instance, in console_cpu_notify(), the new logic ensures that nbcon consoles are flushed when they aren’t managed by the legacy loop. Similarly, in console_flush_on_panic(), the system no longer needs to flush nbcon consoles if none are present. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-31-john.ogness@linutronix.de [pmladek@suse.com: Updated the commit message.] Signed-off-by: Petr Mladek --- kernel/printk/internal.h | 73 ++++++++++++++++++++++++++++++++++++++++ kernel/printk/nbcon.c | 12 +++++-- kernel/printk/printk.c | 68 +++++++++++++++++-------------------- 3 files changed, 112 insertions(+), 41 deletions(-) diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index 6b61350a3026..ba2e0f1940bd 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -154,8 +154,81 @@ static inline bool console_is_usable(struct console *con, short flags) { return #endif /* CONFIG_PRINTK */ extern bool have_boot_console; +extern bool have_nbcon_console; +extern bool have_legacy_console; extern bool legacy_allow_panic_sync; +/** + * struct console_flush_type - Define available console flush methods + * @nbcon_atomic: Flush directly using nbcon_atomic() callback + * @legacy_direct: Call the legacy loop in this context + * @legacy_offload: Offload the legacy loop into IRQ + * + * Note that the legacy loop also flushes the nbcon consoles. + */ +struct console_flush_type { + bool nbcon_atomic; + bool legacy_direct; + bool legacy_offload; +}; + +/* + * Identify which console flushing methods should be used in the context of + * the caller. + */ +static inline void printk_get_console_flush_type(struct console_flush_type *ft) +{ + memset(ft, 0, sizeof(*ft)); + + switch (nbcon_get_default_prio()) { + case NBCON_PRIO_NORMAL: + if (have_nbcon_console && !have_boot_console) + ft->nbcon_atomic = true; + + /* Legacy consoles are flushed directly when possible. */ + if (have_legacy_console || have_boot_console) { + if (!is_printk_legacy_deferred()) + ft->legacy_direct = true; + else + ft->legacy_offload = true; + } + break; + + case NBCON_PRIO_PANIC: + /* + * In panic, the nbcon consoles will directly print. But + * only allowed if there are no boot consoles. + */ + if (have_nbcon_console && !have_boot_console) + ft->nbcon_atomic = true; + + if (have_legacy_console || have_boot_console) { + /* + * This is the same decision as NBCON_PRIO_NORMAL + * except that offloading never occurs in panic. + * + * Note that console_flush_on_panic() will flush + * legacy consoles anyway, even if unsafe. + */ + if (!is_printk_legacy_deferred()) + ft->legacy_direct = true; + + /* + * In panic, if nbcon atomic printing occurs, + * the legacy consoles must remain silent until + * explicitly allowed. + */ + if (ft->nbcon_atomic && !legacy_allow_panic_sync) + ft->legacy_direct = false; + } + break; + + default: + WARN_ON_ONCE(1); + break; + } +} + extern struct printk_buffers printk_shared_pbufs; /** diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index afdb16c1c733..18488d6c17c0 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -1344,6 +1344,7 @@ EXPORT_SYMBOL_GPL(nbcon_device_try_acquire); void nbcon_device_release(struct console *con) { struct nbcon_context *ctxt = &ACCESS_PRIVATE(con, nbcon_device_ctxt); + struct console_flush_type ft; int cookie; if (!nbcon_context_exit_unsafe(ctxt)) @@ -1359,12 +1360,17 @@ void nbcon_device_release(struct console *con) cookie = console_srcu_read_lock(); if (console_is_usable(con, console_srcu_read_flags(con)) && prb_read_valid(prb, nbcon_seq_read(con), NULL)) { - if (!have_boot_console) { + /* + * If nbcon_atomic flushing is not available, fallback to + * using the legacy loop. + */ + printk_get_console_flush_type(&ft); + if (ft.nbcon_atomic) { __nbcon_atomic_flush_pending_con(con, prb_next_reserve_seq(prb), false); - } else if (!is_printk_legacy_deferred()) { + } else if (ft.legacy_direct) { if (console_trylock()) console_unlock(); - } else { + } else if (ft.legacy_offload) { printk_trigger_flush(); } } diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index fc3f8970eea2..6accd1704e73 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -468,14 +468,14 @@ static DEFINE_MUTEX(syslog_lock); * present, it is necessary to perform the console lock/unlock dance * whenever console flushing should occur. */ -static bool have_legacy_console; +bool have_legacy_console; /* * Specifies if an nbcon console is registered. If nbcon consoles are present, * synchronous printing of legacy consoles will not occur during panic until * the backtrace has been stored to the ringbuffer. */ -static bool have_nbcon_console; +bool have_nbcon_console; /* * Specifies if a boot console is registered. If boot consoles are present, @@ -488,14 +488,6 @@ bool have_boot_console; /* See printk_legacy_allow_panic_sync() for details. */ bool legacy_allow_panic_sync; -/* - * Specifies if the console lock/unlock dance is needed for console - * printing. If @have_boot_console is true, the nbcon consoles will - * be printed serially along with the legacy consoles because nbcon - * consoles cannot print simultaneously with boot consoles. - */ -#define printing_via_unlock (have_legacy_console || have_boot_console) - #ifdef CONFIG_PRINTK DECLARE_WAIT_QUEUE_HEAD(log_wait); /* All 3 protected by @syslog_lock. */ @@ -2342,9 +2334,12 @@ out: */ void printk_legacy_allow_panic_sync(void) { + struct console_flush_type ft; + legacy_allow_panic_sync = true; - if (printing_via_unlock && !is_printk_legacy_deferred()) { + printk_get_console_flush_type(&ft); + if (ft.legacy_direct) { if (console_trylock()) console_unlock(); } @@ -2354,8 +2349,7 @@ asmlinkage int vprintk_emit(int facility, int level, const struct dev_printk_info *dev_info, const char *fmt, va_list args) { - bool do_trylock_unlock = printing_via_unlock; - bool defer_legacy = false; + struct console_flush_type ft; int printed_len; /* Suppress unimportant messages after panic happens */ @@ -2370,35 +2364,23 @@ asmlinkage int vprintk_emit(int facility, int level, if (other_cpu_in_panic() && !panic_triggering_all_cpu_backtrace) return 0; + printk_get_console_flush_type(&ft); + /* If called from the scheduler, we can not call up(). */ if (level == LOGLEVEL_SCHED) { level = LOGLEVEL_DEFAULT; - defer_legacy = do_trylock_unlock; - do_trylock_unlock = false; + ft.legacy_offload |= ft.legacy_direct; + ft.legacy_direct = false; } printk_delay(level); printed_len = vprintk_store(facility, level, dev_info, fmt, args); - if (have_nbcon_console && !have_boot_console) { + if (ft.nbcon_atomic) nbcon_atomic_flush_pending(); - /* - * In panic, the legacy consoles are not allowed to print from - * the printk calling context unless explicitly allowed. This - * gives the safe nbcon consoles a chance to print out all the - * panic messages first. This restriction only applies if - * there are nbcon consoles registered and they are allowed to - * flush. - */ - if (this_cpu_in_panic() && !legacy_allow_panic_sync) { - do_trylock_unlock = false; - defer_legacy = false; - } - } - - if (do_trylock_unlock) { + if (ft.legacy_direct) { /* * The caller may be holding system-critical or * timing-sensitive locks. Disable preemption during @@ -2418,7 +2400,7 @@ asmlinkage int vprintk_emit(int facility, int level, preempt_enable(); } - if (defer_legacy) + if (ft.legacy_offload) defer_console_output(); else wake_up_klogd(); @@ -2777,10 +2759,16 @@ void resume_console(void) */ static int console_cpu_notify(unsigned int cpu) { - if (!cpuhp_tasks_frozen && printing_via_unlock) { - /* If trylock fails, someone else is doing the printing */ - if (console_trylock()) - console_unlock(); + struct console_flush_type ft; + + if (!cpuhp_tasks_frozen) { + printk_get_console_flush_type(&ft); + if (ft.nbcon_atomic) + nbcon_atomic_flush_pending(); + if (ft.legacy_direct) { + if (console_trylock()) + console_unlock(); + } } return 0; } @@ -3305,6 +3293,7 @@ static void __console_rewind_all(void) */ void console_flush_on_panic(enum con_flush_mode mode) { + struct console_flush_type ft; bool handover; u64 next_seq; @@ -3328,7 +3317,8 @@ void console_flush_on_panic(enum con_flush_mode mode) if (mode == CONSOLE_REPLAY_ALL) __console_rewind_all(); - if (!have_boot_console) + printk_get_console_flush_type(&ft); + if (ft.nbcon_atomic) nbcon_atomic_flush_pending(); /* Flush legacy consoles once allowed, even when dangerous. */ @@ -3992,6 +3982,7 @@ static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre { unsigned long timeout_jiffies = msecs_to_jiffies(timeout_ms); unsigned long remaining_jiffies = timeout_jiffies; + struct console_flush_type ft; struct console *c; u64 last_diff = 0; u64 printk_seq; @@ -4005,7 +3996,8 @@ static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre seq = prb_next_reserve_seq(prb); /* Flush the consoles so that records up to @seq are printed. */ - if (printing_via_unlock) { + printk_get_console_flush_type(&ft); + if (ft.legacy_direct) { console_lock(); console_unlock(); } From ecb5e1aa82c86642ec1eaafefd4e317dfba3a238 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 20 Aug 2024 08:35:57 +0206 Subject: [PATCH 117/180] printk: nbcon: Implement emergency sections In emergency situations (something has gone wrong but the system continues to operate), usually important information (such as a backtrace) is generated via printk(). This information should be pushed out to the consoles ASAP. Add per-CPU emergency nesting tracking because an emergency can arise while in an emergency situation. Add functions to mark the beginning and end of emergency sections where the urgent messages are generated. Perform direct console flushing at the emergency priority if the current CPU is in an emergency state and it is safe to do so. Note that the emergency state is not system-wide. While one CPU is in an emergency state, another CPU may attempt to print console messages at normal priority. Also note that printk() already attempts to flush consoles in the caller context for normal priority. However, follow-up changes will introduce printing kthreads, in which case the normal priority printk() calls will offload to the kthreads. Co-developed-by: John Ogness Signed-off-by: John Ogness Signed-off-by: Thomas Gleixner (Intel) Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-32-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- include/linux/console.h | 4 +++ kernel/printk/internal.h | 1 + kernel/printk/nbcon.c | 75 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 79 insertions(+), 1 deletion(-) diff --git a/include/linux/console.h b/include/linux/console.h index 3706f944de46..9a13f91b0c43 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -553,10 +553,14 @@ static inline bool console_is_registered(const struct console *con) hlist_for_each_entry(con, &console_list, node) #ifdef CONFIG_PRINTK +extern void nbcon_cpu_emergency_enter(void); +extern void nbcon_cpu_emergency_exit(void); extern bool nbcon_can_proceed(struct nbcon_write_context *wctxt); extern bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt); extern bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt); #else +static inline void nbcon_cpu_emergency_enter(void) { } +static inline void nbcon_cpu_emergency_exit(void) { } static inline bool nbcon_can_proceed(struct nbcon_write_context *wctxt) { return false; } static inline bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt) { return false; } static inline bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt) { return false; } diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index ba2e0f1940bd..8e36d8695f81 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -182,6 +182,7 @@ static inline void printk_get_console_flush_type(struct console_flush_type *ft) switch (nbcon_get_default_prio()) { case NBCON_PRIO_NORMAL: + case NBCON_PRIO_EMERGENCY: if (have_nbcon_console && !have_boot_console) ft->nbcon_atomic = true; diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index 18488d6c17c0..92ac5c590927 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -972,6 +972,36 @@ update_con: return nbcon_context_exit_unsafe(ctxt); } +/* Track the nbcon emergency nesting per CPU. */ +static DEFINE_PER_CPU(unsigned int, nbcon_pcpu_emergency_nesting); +static unsigned int early_nbcon_pcpu_emergency_nesting __initdata; + +/** + * nbcon_get_cpu_emergency_nesting - Get the per CPU emergency nesting pointer + * + * Context: For reading, any context. For writing, any context which could + * not be migrated to another CPU. + * Return: Either a pointer to the per CPU emergency nesting counter of + * the current CPU or to the init data during early boot. + * + * The function is safe for reading per-CPU variables in any context because + * preemption is disabled if the current CPU is in the emergency state. See + * also nbcon_cpu_emergency_enter(). + */ +static __ref unsigned int *nbcon_get_cpu_emergency_nesting(void) +{ + /* + * The value of __printk_percpu_data_ready gets set in normal + * context and before SMP initialization. As a result it could + * never change while inside an nbcon emergency section. + */ + if (!printk_percpu_data_ready()) + return &early_nbcon_pcpu_emergency_nesting; + + /* Open code this_cpu_ptr() without checking migration. */ + return per_cpu_ptr(&nbcon_pcpu_emergency_nesting, raw_smp_processor_id()); +} + /** * nbcon_get_default_prio - The appropriate nbcon priority to use for nbcon * printing on the current CPU @@ -981,13 +1011,20 @@ update_con: * context for printing. * * The function is safe for reading per-CPU data in any context because - * preemption is disabled if the current CPU is in the panic state. + * preemption is disabled if the current CPU is in the emergency or panic + * state. */ enum nbcon_prio nbcon_get_default_prio(void) { + unsigned int *cpu_emergency_nesting; + if (this_cpu_in_panic()) return NBCON_PRIO_PANIC; + cpu_emergency_nesting = nbcon_get_cpu_emergency_nesting(); + if (*cpu_emergency_nesting) + return NBCON_PRIO_EMERGENCY; + return NBCON_PRIO_NORMAL; } @@ -1246,6 +1283,42 @@ void nbcon_atomic_flush_unsafe(void) __nbcon_atomic_flush_pending(prb_next_reserve_seq(prb), true); } +/** + * nbcon_cpu_emergency_enter - Enter an emergency section where printk() + * messages for that CPU are flushed directly + * + * Context: Any context. Disables preemption. + * + * When within an emergency section, printk() calls will attempt to flush any + * pending messages in the ringbuffer. + */ +void nbcon_cpu_emergency_enter(void) +{ + unsigned int *cpu_emergency_nesting; + + preempt_disable(); + + cpu_emergency_nesting = nbcon_get_cpu_emergency_nesting(); + (*cpu_emergency_nesting)++; +} + +/** + * nbcon_cpu_emergency_exit - Exit an emergency section + * + * Context: Within an emergency section. Enables preemption. + */ +void nbcon_cpu_emergency_exit(void) +{ + unsigned int *cpu_emergency_nesting; + + cpu_emergency_nesting = nbcon_get_cpu_emergency_nesting(); + + if (!WARN_ON_ONCE(*cpu_emergency_nesting == 0)) + (*cpu_emergency_nesting)--; + + preempt_enable(); +} + /** * nbcon_alloc - Allocate and init the nbcon console specific data * @con: Console to initialize From 4833794db61c8cc4de4563a2754c7aedaffbb684 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 20 Aug 2024 08:35:58 +0206 Subject: [PATCH 118/180] panic: Mark emergency section in warn Mark the full contents of __warn() as an emergency section. In this section, every printk() call will attempt to directly flush to the consoles using the EMERGENCY priority. Co-developed-by: John Ogness Signed-off-by: John Ogness Signed-off-by: Thomas Gleixner (Intel) Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-33-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/panic.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kernel/panic.c b/kernel/panic.c index 93096d5abcc7..1a10b6e2a855 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -718,6 +718,8 @@ struct warn_args { void __warn(const char *file, int line, void *caller, unsigned taint, struct pt_regs *regs, struct warn_args *args) { + nbcon_cpu_emergency_enter(); + disable_trace_on_warning(); if (file) @@ -753,6 +755,8 @@ void __warn(const char *file, int line, void *caller, unsigned taint, /* Just a warning, don't kill lockdep. */ add_taint(taint, LOCKDEP_STILL_OK); + + nbcon_cpu_emergency_exit(); } #ifdef CONFIG_BUG From 4bdfa0d8e920c391e6cc0aa1feef8ed91d81f724 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:35:59 +0206 Subject: [PATCH 119/180] panic: Mark emergency section in oops Mark an emergency section beginning with oops_enter() until the end of oops_exit(). In this section, every printk() call will attempt to directly flush to the consoles using the EMERGENCY priority. The very end of oops_exit() performs a kmsg_dump(). This is not included in the emergency section because it is another flushing mechanism that should occur after the consoles have flushed the oops messages. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-34-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/panic.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/panic.c b/kernel/panic.c index 1a10b6e2a855..753d12f4dc8f 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -685,6 +685,7 @@ bool oops_may_print(void) */ void oops_enter(void) { + nbcon_cpu_emergency_enter(); tracing_off(); /* can't trust the integrity of the kernel anymore: */ debug_locks_off(); @@ -707,6 +708,7 @@ void oops_exit(void) { do_oops_enter_exit(); print_oops_end_marker(); + nbcon_cpu_emergency_exit(); kmsg_dump(KMSG_DUMP_OOPS); } From 8c03273a509c88b12c53a9ddf07702f83983e4de Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:36:00 +0206 Subject: [PATCH 120/180] rcu: Mark emergency sections in rcu stalls Mark emergency sections wherever multiple lines of rcu stall information are generated. In an emergency section, every printk() call will attempt to directly flush to the consoles using the EMERGENCY priority. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Acked-by: Paul E. McKenney Link: https://lore.kernel.org/r/20240820063001.36405-35-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/rcu/tree_exp.h | 7 +++++++ kernel/rcu/tree_stall.h | 9 +++++++++ 2 files changed, 16 insertions(+) diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index 4acd29d16fdb..f6b35a0585a8 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -7,6 +7,7 @@ * Authors: Paul E. McKenney */ +#include #include static void rcu_exp_handler(void *unused); @@ -590,6 +591,9 @@ static void synchronize_rcu_expedited_wait(void) return; if (rcu_stall_is_suppressed()) continue; + + nbcon_cpu_emergency_enter(); + j = jiffies; rcu_stall_notifier_call_chain(RCU_STALL_NOTIFY_EXP, (void *)(j - jiffies_start)); trace_rcu_stall_warning(rcu_state.name, TPS("ExpeditedStall")); @@ -643,6 +647,9 @@ static void synchronize_rcu_expedited_wait(void) rcu_exp_print_detail_task_stall_rnp(rnp); } jiffies_stall = 3 * rcu_exp_jiffies_till_stall_check() + 3; + + nbcon_cpu_emergency_exit(); + panic_on_rcu_stall(); } } diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index 4b0e9d7c4c68..b3a6943127bc 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -7,6 +7,7 @@ * Author: Paul E. McKenney */ +#include #include #include @@ -605,6 +606,8 @@ static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps) if (rcu_stall_is_suppressed()) return; + nbcon_cpu_emergency_enter(); + /* * OK, time to rat on our buddy... * See Documentation/RCU/stallwarn.rst for info on how to debug @@ -657,6 +660,8 @@ static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps) rcu_check_gp_kthread_expired_fqs_timer(); rcu_check_gp_kthread_starvation(); + nbcon_cpu_emergency_exit(); + panic_on_rcu_stall(); rcu_force_quiescent_state(); /* Kick them all. */ @@ -677,6 +682,8 @@ static void print_cpu_stall(unsigned long gps) if (rcu_stall_is_suppressed()) return; + nbcon_cpu_emergency_enter(); + /* * OK, time to rat on ourselves... * See Documentation/RCU/stallwarn.rst for info on how to debug @@ -706,6 +713,8 @@ static void print_cpu_stall(unsigned long gps) jiffies + 3 * rcu_jiffies_till_stall_check() + 3); raw_spin_unlock_irqrestore_rcu_node(rnp, flags); + nbcon_cpu_emergency_exit(); + panic_on_rcu_stall(); /* From 59cd94ef80094857f0d0085daa2e32badc4cddf4 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 20 Aug 2024 08:36:01 +0206 Subject: [PATCH 121/180] lockdep: Mark emergency sections in lockdep splats Mark emergency sections wherever multiple lines of lock debugging output are generated. In an emergency section, every printk() call will attempt to directly flush to the consoles using the EMERGENCY priority. Note that debug_show_all_locks() and lockdep_print_held_locks() rely on their callers to enter the emergency section. This is because these functions can also be called in non-emergency situations (such as sysrq). Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240820063001.36405-36-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/locking/lockdep.c | 83 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 81 insertions(+), 2 deletions(-) diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index 0349f957e672..7963deac33c3 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -56,6 +56,7 @@ #include #include #include +#include #include @@ -573,8 +574,10 @@ static struct lock_trace *save_trace(void) if (!debug_locks_off_graph_unlock()) return NULL; + nbcon_cpu_emergency_enter(); print_lockdep_off("BUG: MAX_STACK_TRACE_ENTRIES too low!"); dump_stack(); + nbcon_cpu_emergency_exit(); return NULL; } @@ -887,11 +890,13 @@ look_up_lock_class(const struct lockdep_map *lock, unsigned int subclass) if (unlikely(subclass >= MAX_LOCKDEP_SUBCLASSES)) { instrumentation_begin(); debug_locks_off(); + nbcon_cpu_emergency_enter(); printk(KERN_ERR "BUG: looking up invalid subclass: %u\n", subclass); printk(KERN_ERR "turning off the locking correctness validator.\n"); dump_stack(); + nbcon_cpu_emergency_exit(); instrumentation_end(); return NULL; } @@ -968,11 +973,13 @@ static bool assign_lock_key(struct lockdep_map *lock) else { /* Debug-check: all keys must be persistent! */ debug_locks_off(); + nbcon_cpu_emergency_enter(); pr_err("INFO: trying to register non-static key.\n"); pr_err("The code is fine but needs lockdep annotation, or maybe\n"); pr_err("you didn't initialize this object before use?\n"); pr_err("turning off the locking correctness validator.\n"); dump_stack(); + nbcon_cpu_emergency_exit(); return false; } @@ -1316,8 +1323,10 @@ register_lock_class(struct lockdep_map *lock, unsigned int subclass, int force) return NULL; } + nbcon_cpu_emergency_enter(); print_lockdep_off("BUG: MAX_LOCKDEP_KEYS too low!"); dump_stack(); + nbcon_cpu_emergency_exit(); return NULL; } nr_lock_classes++; @@ -1349,11 +1358,13 @@ register_lock_class(struct lockdep_map *lock, unsigned int subclass, int force) if (verbose(class)) { graph_unlock(); + nbcon_cpu_emergency_enter(); printk("\nnew class %px: %s", class->key, class->name); if (class->name_version > 1) printk(KERN_CONT "#%d", class->name_version); printk(KERN_CONT "\n"); dump_stack(); + nbcon_cpu_emergency_exit(); if (!graph_lock()) { return NULL; @@ -1392,8 +1403,10 @@ static struct lock_list *alloc_list_entry(void) if (!debug_locks_off_graph_unlock()) return NULL; + nbcon_cpu_emergency_enter(); print_lockdep_off("BUG: MAX_LOCKDEP_ENTRIES too low!"); dump_stack(); + nbcon_cpu_emergency_exit(); return NULL; } nr_list_entries++; @@ -2039,6 +2052,8 @@ static noinline void print_circular_bug(struct lock_list *this, depth = get_lock_depth(target); + nbcon_cpu_emergency_enter(); + print_circular_bug_header(target, depth, check_src, check_tgt); parent = get_lock_parent(target); @@ -2057,6 +2072,8 @@ static noinline void print_circular_bug(struct lock_list *this, printk("\nstack backtrace:\n"); dump_stack(); + + nbcon_cpu_emergency_exit(); } static noinline void print_bfs_bug(int ret) @@ -2569,6 +2586,8 @@ print_bad_irq_dependency(struct task_struct *curr, if (!debug_locks_off_graph_unlock() || debug_locks_silent) return; + nbcon_cpu_emergency_enter(); + pr_warn("\n"); pr_warn("=====================================================\n"); pr_warn("WARNING: %s-safe -> %s-unsafe lock order detected\n", @@ -2618,11 +2637,13 @@ print_bad_irq_dependency(struct task_struct *curr, pr_warn(" and %s-irq-unsafe lock:\n", irqclass); next_root->trace = save_trace(); if (!next_root->trace) - return; + goto out; print_shortest_lock_dependencies(forwards_entry, next_root); pr_warn("\nstack backtrace:\n"); dump_stack(); +out: + nbcon_cpu_emergency_exit(); } static const char *state_names[] = { @@ -2987,6 +3008,8 @@ print_deadlock_bug(struct task_struct *curr, struct held_lock *prev, if (!debug_locks_off_graph_unlock() || debug_locks_silent) return; + nbcon_cpu_emergency_enter(); + pr_warn("\n"); pr_warn("============================================\n"); pr_warn("WARNING: possible recursive locking detected\n"); @@ -3009,6 +3032,8 @@ print_deadlock_bug(struct task_struct *curr, struct held_lock *prev, pr_warn("\nstack backtrace:\n"); dump_stack(); + + nbcon_cpu_emergency_exit(); } /* @@ -3606,6 +3631,8 @@ static void print_collision(struct task_struct *curr, struct held_lock *hlock_next, struct lock_chain *chain) { + nbcon_cpu_emergency_enter(); + pr_warn("\n"); pr_warn("============================\n"); pr_warn("WARNING: chain_key collision\n"); @@ -3622,6 +3649,8 @@ static void print_collision(struct task_struct *curr, pr_warn("\nstack backtrace:\n"); dump_stack(); + + nbcon_cpu_emergency_exit(); } #endif @@ -3712,8 +3741,10 @@ static inline int add_chain_cache(struct task_struct *curr, if (!debug_locks_off_graph_unlock()) return 0; + nbcon_cpu_emergency_enter(); print_lockdep_off("BUG: MAX_LOCKDEP_CHAINS too low!"); dump_stack(); + nbcon_cpu_emergency_exit(); return 0; } chain->chain_key = chain_key; @@ -3730,8 +3761,10 @@ static inline int add_chain_cache(struct task_struct *curr, if (!debug_locks_off_graph_unlock()) return 0; + nbcon_cpu_emergency_enter(); print_lockdep_off("BUG: MAX_LOCKDEP_CHAIN_HLOCKS too low!"); dump_stack(); + nbcon_cpu_emergency_exit(); return 0; } @@ -3970,6 +4003,8 @@ print_usage_bug(struct task_struct *curr, struct held_lock *this, if (!debug_locks_off() || debug_locks_silent) return; + nbcon_cpu_emergency_enter(); + pr_warn("\n"); pr_warn("================================\n"); pr_warn("WARNING: inconsistent lock state\n"); @@ -3998,6 +4033,8 @@ print_usage_bug(struct task_struct *curr, struct held_lock *this, pr_warn("\nstack backtrace:\n"); dump_stack(); + + nbcon_cpu_emergency_exit(); } /* @@ -4032,6 +4069,8 @@ print_irq_inversion_bug(struct task_struct *curr, if (!debug_locks_off_graph_unlock() || debug_locks_silent) return; + nbcon_cpu_emergency_enter(); + pr_warn("\n"); pr_warn("========================================================\n"); pr_warn("WARNING: possible irq lock inversion dependency detected\n"); @@ -4072,11 +4111,13 @@ print_irq_inversion_bug(struct task_struct *curr, pr_warn("\nthe shortest dependencies between 2nd lock and 1st lock:\n"); root->trace = save_trace(); if (!root->trace) - return; + goto out; print_shortest_lock_dependencies(other, root); pr_warn("\nstack backtrace:\n"); dump_stack(); +out: + nbcon_cpu_emergency_exit(); } /* @@ -4153,6 +4194,8 @@ void print_irqtrace_events(struct task_struct *curr) { const struct irqtrace_events *trace = &curr->irqtrace; + nbcon_cpu_emergency_enter(); + printk("irq event stamp: %u\n", trace->irq_events); printk("hardirqs last enabled at (%u): [<%px>] %pS\n", trace->hardirq_enable_event, (void *)trace->hardirq_enable_ip, @@ -4166,6 +4209,8 @@ void print_irqtrace_events(struct task_struct *curr) printk("softirqs last disabled at (%u): [<%px>] %pS\n", trace->softirq_disable_event, (void *)trace->softirq_disable_ip, (void *)trace->softirq_disable_ip); + + nbcon_cpu_emergency_exit(); } static int HARDIRQ_verbose(struct lock_class *class) @@ -4686,10 +4731,12 @@ unlock: * We must printk outside of the graph_lock: */ if (ret == 2) { + nbcon_cpu_emergency_enter(); printk("\nmarked lock as {%s}:\n", usage_str[new_bit]); print_lock(this); print_irqtrace_events(curr); dump_stack(); + nbcon_cpu_emergency_exit(); } return ret; @@ -4730,6 +4777,8 @@ print_lock_invalid_wait_context(struct task_struct *curr, if (debug_locks_silent) return 0; + nbcon_cpu_emergency_enter(); + pr_warn("\n"); pr_warn("=============================\n"); pr_warn("[ BUG: Invalid wait context ]\n"); @@ -4749,6 +4798,8 @@ print_lock_invalid_wait_context(struct task_struct *curr, pr_warn("stack backtrace:\n"); dump_stack(); + nbcon_cpu_emergency_exit(); + return 0; } @@ -4956,6 +5007,8 @@ print_lock_nested_lock_not_held(struct task_struct *curr, if (debug_locks_silent) return; + nbcon_cpu_emergency_enter(); + pr_warn("\n"); pr_warn("==================================\n"); pr_warn("WARNING: Nested lock was not taken\n"); @@ -4976,6 +5029,8 @@ print_lock_nested_lock_not_held(struct task_struct *curr, pr_warn("\nstack backtrace:\n"); dump_stack(); + + nbcon_cpu_emergency_exit(); } static int __lock_is_held(const struct lockdep_map *lock, int read); @@ -5024,11 +5079,13 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, debug_class_ops_inc(class); if (very_verbose(class)) { + nbcon_cpu_emergency_enter(); printk("\nacquire class [%px] %s", class->key, class->name); if (class->name_version > 1) printk(KERN_CONT "#%d", class->name_version); printk(KERN_CONT "\n"); dump_stack(); + nbcon_cpu_emergency_exit(); } /* @@ -5155,6 +5212,7 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, #endif if (unlikely(curr->lockdep_depth >= MAX_LOCK_DEPTH)) { debug_locks_off(); + nbcon_cpu_emergency_enter(); print_lockdep_off("BUG: MAX_LOCK_DEPTH too low!"); printk(KERN_DEBUG "depth: %i max: %lu!\n", curr->lockdep_depth, MAX_LOCK_DEPTH); @@ -5162,6 +5220,7 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, lockdep_print_held_locks(current); debug_show_all_locks(); dump_stack(); + nbcon_cpu_emergency_exit(); return 0; } @@ -5181,6 +5240,8 @@ static void print_unlock_imbalance_bug(struct task_struct *curr, if (debug_locks_silent) return; + nbcon_cpu_emergency_enter(); + pr_warn("\n"); pr_warn("=====================================\n"); pr_warn("WARNING: bad unlock balance detected!\n"); @@ -5197,6 +5258,8 @@ static void print_unlock_imbalance_bug(struct task_struct *curr, pr_warn("\nstack backtrace:\n"); dump_stack(); + + nbcon_cpu_emergency_exit(); } static noinstr int match_held_lock(const struct held_lock *hlock, @@ -5901,6 +5964,8 @@ static void print_lock_contention_bug(struct task_struct *curr, if (debug_locks_silent) return; + nbcon_cpu_emergency_enter(); + pr_warn("\n"); pr_warn("=================================\n"); pr_warn("WARNING: bad contention detected!\n"); @@ -5917,6 +5982,8 @@ static void print_lock_contention_bug(struct task_struct *curr, pr_warn("\nstack backtrace:\n"); dump_stack(); + + nbcon_cpu_emergency_exit(); } static void @@ -6536,6 +6603,8 @@ print_freed_lock_bug(struct task_struct *curr, const void *mem_from, if (debug_locks_silent) return; + nbcon_cpu_emergency_enter(); + pr_warn("\n"); pr_warn("=========================\n"); pr_warn("WARNING: held lock freed!\n"); @@ -6548,6 +6617,8 @@ print_freed_lock_bug(struct task_struct *curr, const void *mem_from, pr_warn("\nstack backtrace:\n"); dump_stack(); + + nbcon_cpu_emergency_exit(); } static inline int not_in_range(const void* mem_from, unsigned long mem_len, @@ -6594,6 +6665,8 @@ static void print_held_locks_bug(void) if (debug_locks_silent) return; + nbcon_cpu_emergency_enter(); + pr_warn("\n"); pr_warn("====================================\n"); pr_warn("WARNING: %s/%d still has locks held!\n", @@ -6603,6 +6676,8 @@ static void print_held_locks_bug(void) lockdep_print_held_locks(current); pr_warn("\nstack backtrace:\n"); dump_stack(); + + nbcon_cpu_emergency_exit(); } void debug_check_no_locks_held(void) @@ -6660,6 +6735,7 @@ asmlinkage __visible void lockdep_sys_exit(void) if (unlikely(curr->lockdep_depth)) { if (!debug_locks_off()) return; + nbcon_cpu_emergency_enter(); pr_warn("\n"); pr_warn("================================================\n"); pr_warn("WARNING: lock held when returning to user space!\n"); @@ -6668,6 +6744,7 @@ asmlinkage __visible void lockdep_sys_exit(void) pr_warn("%s/%d is leaving the kernel with locks still held!\n", curr->comm, curr->pid); lockdep_print_held_locks(curr); + nbcon_cpu_emergency_exit(); } /* @@ -6684,6 +6761,7 @@ void lockdep_rcu_suspicious(const char *file, const int line, const char *s) bool rcu = warn_rcu_enter(); /* Note: the following can be executed concurrently, so be careful. */ + nbcon_cpu_emergency_enter(); pr_warn("\n"); pr_warn("=============================\n"); pr_warn("WARNING: suspicious RCU usage\n"); @@ -6722,6 +6800,7 @@ void lockdep_rcu_suspicious(const char *file, const int line, const char *s) lockdep_print_held_locks(curr); pr_warn("\nstack backtrace:\n"); dump_stack(); + nbcon_cpu_emergency_exit(); warn_rcu_exit(rcu); } EXPORT_SYMBOL_GPL(lockdep_rcu_suspicious); From ed4fb6d7ef68111bb539283561953e5c6e9a6e38 Mon Sep 17 00:00:00 2001 From: Felix Moessbauer Date: Wed, 14 Aug 2024 14:10:32 +0200 Subject: [PATCH 122/180] hrtimer: Use and report correct timerslack values for realtime tasks The timerslack_ns setting is used to specify how much the hardware timers should be delayed, to potentially dispatch multiple timers in a single interrupt. This is a performance optimization. Timers of realtime tasks (having a realtime scheduling policy) should not be delayed. This logic was inconsitently applied to the hrtimers, leading to delays of realtime tasks which used timed waits for events (e.g. condition variables). Due to the downstream override of the slack for rt tasks, the procfs reported incorrect (non-zero) timerslack_ns values. This is changed by setting the timer_slack_ns task attribute to 0 for all tasks with a rt policy. By that, downstream users do not need to specially handle rt tasks (w.r.t. the slack), and the procfs entry shows the correct value of "0". Setting non-zero slack values (either via procfs or PR_SET_TIMERSLACK) on tasks with a rt policy is ignored, as stated in "man 2 PR_SET_TIMERSLACK": Timer slack is not applied to threads that are scheduled under a real-time scheduling policy (see sched_setscheduler(2)). The special handling of timerslack on rt tasks in downstream users is removed as well. Signed-off-by: Felix Moessbauer Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240814121032.368444-2-felix.moessbauer@siemens.com --- fs/proc/base.c | 9 +++++---- fs/select.c | 11 ++++------- kernel/sched/syscalls.c | 8 ++++++++ kernel/sys.c | 2 ++ kernel/time/hrtimer.c | 18 +++--------------- 5 files changed, 22 insertions(+), 26 deletions(-) diff --git a/fs/proc/base.c b/fs/proc/base.c index dd579332a7f8..afe573e8d8f1 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -2569,10 +2569,11 @@ static ssize_t timerslack_ns_write(struct file *file, const char __user *buf, } task_lock(p); - if (slack_ns == 0) - p->timer_slack_ns = p->default_timer_slack_ns; - else - p->timer_slack_ns = slack_ns; + if (task_is_realtime(p)) + slack_ns = 0; + else if (slack_ns == 0) + slack_ns = p->default_timer_slack_ns; + p->timer_slack_ns = slack_ns; task_unlock(p); out: diff --git a/fs/select.c b/fs/select.c index 9515c3fa1a03..ad171b7a5c11 100644 --- a/fs/select.c +++ b/fs/select.c @@ -77,19 +77,16 @@ u64 select_estimate_accuracy(struct timespec64 *tv) { u64 ret; struct timespec64 now; + u64 slack = current->timer_slack_ns; - /* - * Realtime tasks get a slack of 0 for obvious reasons. - */ - - if (rt_task(current)) + if (slack == 0) return 0; ktime_get_ts64(&now); now = timespec64_sub(*tv, now); ret = __estimate_accuracy(&now); - if (ret < current->timer_slack_ns) - return current->timer_slack_ns; + if (ret < slack) + return slack; return ret; } diff --git a/kernel/sched/syscalls.c b/kernel/sched/syscalls.c index ae1b42775ef9..195d2f2834a9 100644 --- a/kernel/sched/syscalls.c +++ b/kernel/sched/syscalls.c @@ -406,6 +406,14 @@ static void __setscheduler_params(struct task_struct *p, else if (fair_policy(policy)) p->static_prio = NICE_TO_PRIO(attr->sched_nice); + /* rt-policy tasks do not have a timerslack */ + if (task_is_realtime(p)) { + p->timer_slack_ns = 0; + } else if (p->timer_slack_ns == 0) { + /* when switching back to non-rt policy, restore timerslack */ + p->timer_slack_ns = p->default_timer_slack_ns; + } + /* * __sched_setscheduler() ensures attr->sched_priority == 0 when * !rt_policy. Always setting this ensures that things like diff --git a/kernel/sys.c b/kernel/sys.c index 3a2df1bd9f64..e3c4cffb520c 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -2557,6 +2557,8 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, error = current->timer_slack_ns; break; case PR_SET_TIMERSLACK: + if (task_is_realtime(current)) + break; if (arg2 <= 0) current->timer_slack_ns = current->default_timer_slack_ns; diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index f56ef23aad90..a023946f8558 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -2074,14 +2074,9 @@ long hrtimer_nanosleep(ktime_t rqtp, const enum hrtimer_mode mode, struct restart_block *restart; struct hrtimer_sleeper t; int ret = 0; - u64 slack; - - slack = current->timer_slack_ns; - if (rt_task(current)) - slack = 0; hrtimer_init_sleeper_on_stack(&t, clockid, mode); - hrtimer_set_expires_range_ns(&t.timer, rqtp, slack); + hrtimer_set_expires_range_ns(&t.timer, rqtp, current->timer_slack_ns); ret = do_nanosleep(&t, mode); if (ret != -ERESTART_RESTARTBLOCK) goto out; @@ -2251,7 +2246,7 @@ void __init hrtimers_init(void) /** * schedule_hrtimeout_range_clock - sleep until timeout * @expires: timeout value (ktime_t) - * @delta: slack in expires timeout (ktime_t) for SCHED_OTHER tasks + * @delta: slack in expires timeout (ktime_t) * @mode: timer mode * @clock_id: timer clock to be used */ @@ -2278,13 +2273,6 @@ schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta, return -EINTR; } - /* - * Override any slack passed by the user if under - * rt contraints. - */ - if (rt_task(current)) - delta = 0; - hrtimer_init_sleeper_on_stack(&t, clock_id, mode); hrtimer_set_expires_range_ns(&t.timer, *expires, delta); hrtimer_sleeper_start_expires(&t, mode); @@ -2304,7 +2292,7 @@ EXPORT_SYMBOL_GPL(schedule_hrtimeout_range_clock); /** * schedule_hrtimeout_range - sleep until timeout * @expires: timeout value (ktime_t) - * @delta: slack in expires timeout (ktime_t) for SCHED_OTHER tasks + * @delta: slack in expires timeout (ktime_t) * @mode: timer mode * * Make the current task sleep until the given expiry time has From 06fac729a6d54e2c6650b38734f84383aafb3acc Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Fri, 23 Aug 2024 18:39:32 +0800 Subject: [PATCH 123/180] LoongArch: Move irqchip function prototypes to irq-loongson.h Some irqchip functions are only for internal use by irqchip drivers, so move their prototypes from asm/irq.h to drivers/irqchip/irq-loongson.h. All related driver files include the new irq-loongson.h. Signed-off-by: Huacai Chen Signed-off-by: Tianyang Zhang Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240823103936.25092-1-zhangtianyang@loongson.cn --- arch/loongarch/include/asm/irq.h | 14 -------------- drivers/irqchip/irq-loongarch-cpu.c | 2 ++ drivers/irqchip/irq-loongson-eiointc.c | 2 ++ drivers/irqchip/irq-loongson-htvec.c | 2 ++ drivers/irqchip/irq-loongson-liointc.c | 2 ++ drivers/irqchip/irq-loongson-pch-lpc.c | 2 ++ drivers/irqchip/irq-loongson-pch-msi.c | 1 + drivers/irqchip/irq-loongson-pch-pic.c | 2 ++ drivers/irqchip/irq-loongson.h | 25 +++++++++++++++++++++++++ 9 files changed, 38 insertions(+), 14 deletions(-) create mode 100644 drivers/irqchip/irq-loongson.h diff --git a/arch/loongarch/include/asm/irq.h b/arch/loongarch/include/asm/irq.h index 480418bc5071..65503c9eb529 100644 --- a/arch/loongarch/include/asm/irq.h +++ b/arch/loongarch/include/asm/irq.h @@ -88,20 +88,6 @@ struct acpi_madt_bio_pic; struct acpi_madt_msi_pic; struct acpi_madt_lpc_pic; -int liointc_acpi_init(struct irq_domain *parent, - struct acpi_madt_lio_pic *acpi_liointc); -int eiointc_acpi_init(struct irq_domain *parent, - struct acpi_madt_eio_pic *acpi_eiointc); - -int htvec_acpi_init(struct irq_domain *parent, - struct acpi_madt_ht_pic *acpi_htvec); -int pch_lpc_acpi_init(struct irq_domain *parent, - struct acpi_madt_lpc_pic *acpi_pchlpc); -int pch_msi_acpi_init(struct irq_domain *parent, - struct acpi_madt_msi_pic *acpi_pchmsi); -int pch_pic_acpi_init(struct irq_domain *parent, - struct acpi_madt_bio_pic *acpi_pchpic); -int find_pch_pic(u32 gsi); struct fwnode_handle *get_pch_msi_handle(int pci_segment); extern struct acpi_madt_lio_pic *acpi_liointc; diff --git a/drivers/irqchip/irq-loongarch-cpu.c b/drivers/irqchip/irq-loongarch-cpu.c index 9d8f2c406043..83f7492290a8 100644 --- a/drivers/irqchip/irq-loongarch-cpu.c +++ b/drivers/irqchip/irq-loongarch-cpu.c @@ -13,6 +13,8 @@ #include #include +#include "irq-loongson.h" + static struct irq_domain *irq_domain; struct fwnode_handle *cpuintc_handle; diff --git a/drivers/irqchip/irq-loongson-eiointc.c b/drivers/irqchip/irq-loongson-eiointc.c index b1f2080be2be..34b5ca2f5e62 100644 --- a/drivers/irqchip/irq-loongson-eiointc.c +++ b/drivers/irqchip/irq-loongson-eiointc.c @@ -17,6 +17,8 @@ #include #include +#include "irq-loongson.h" + #define EIOINTC_REG_NODEMAP 0x14a0 #define EIOINTC_REG_IPMAP 0x14c0 #define EIOINTC_REG_ENABLE 0x1600 diff --git a/drivers/irqchip/irq-loongson-htvec.c b/drivers/irqchip/irq-loongson-htvec.c index 0bff728b25e3..5da02c7ad0b3 100644 --- a/drivers/irqchip/irq-loongson-htvec.c +++ b/drivers/irqchip/irq-loongson-htvec.c @@ -17,6 +17,8 @@ #include #include +#include "irq-loongson.h" + /* Registers */ #define HTVEC_EN_OFF 0x20 #define HTVEC_MAX_PARENT_IRQ 8 diff --git a/drivers/irqchip/irq-loongson-liointc.c b/drivers/irqchip/irq-loongson-liointc.c index 7c4fe7ab4b83..2b1bd4a96665 100644 --- a/drivers/irqchip/irq-loongson-liointc.c +++ b/drivers/irqchip/irq-loongson-liointc.c @@ -22,6 +22,8 @@ #include #endif +#include "irq-loongson.h" + #define LIOINTC_CHIP_IRQ 32 #define LIOINTC_NUM_PARENT 4 #define LIOINTC_NUM_CORES 4 diff --git a/drivers/irqchip/irq-loongson-pch-lpc.c b/drivers/irqchip/irq-loongson-pch-lpc.c index 9b35492fb6be..2d4c3ec128b8 100644 --- a/drivers/irqchip/irq-loongson-pch-lpc.c +++ b/drivers/irqchip/irq-loongson-pch-lpc.c @@ -15,6 +15,8 @@ #include #include +#include "irq-loongson.h" + /* Registers */ #define LPC_INT_CTL 0x00 #define LPC_INT_ENA 0x04 diff --git a/drivers/irqchip/irq-loongson-pch-msi.c b/drivers/irqchip/irq-loongson-pch-msi.c index 2242f63c66fc..d43731878800 100644 --- a/drivers/irqchip/irq-loongson-pch-msi.c +++ b/drivers/irqchip/irq-loongson-pch-msi.c @@ -16,6 +16,7 @@ #include #include "irq-msi-lib.h" +#include "irq-loongson.h" static int nr_pics; diff --git a/drivers/irqchip/irq-loongson-pch-pic.c b/drivers/irqchip/irq-loongson-pch-pic.c index cbaef65e804c..69efda35a8e7 100644 --- a/drivers/irqchip/irq-loongson-pch-pic.c +++ b/drivers/irqchip/irq-loongson-pch-pic.c @@ -17,6 +17,8 @@ #include #include +#include "irq-loongson.h" + /* Registers */ #define PCH_PIC_MASK 0x20 #define PCH_PIC_HTMSI_EN 0x40 diff --git a/drivers/irqchip/irq-loongson.h b/drivers/irqchip/irq-loongson.h new file mode 100644 index 000000000000..b155f1258ed5 --- /dev/null +++ b/drivers/irqchip/irq-loongson.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2024 Loongson Technology Corporation Limited + */ + +#ifndef _DRIVERS_IRQCHIP_IRQ_LOONGSON_H +#define _DRIVERS_IRQCHIP_IRQ_LOONGSON_H + +int find_pch_pic(u32 gsi); + +int liointc_acpi_init(struct irq_domain *parent, + struct acpi_madt_lio_pic *acpi_liointc); +int eiointc_acpi_init(struct irq_domain *parent, + struct acpi_madt_eio_pic *acpi_eiointc); + +int htvec_acpi_init(struct irq_domain *parent, + struct acpi_madt_ht_pic *acpi_htvec); +int pch_lpc_acpi_init(struct irq_domain *parent, + struct acpi_madt_lpc_pic *acpi_pchlpc); +int pch_pic_acpi_init(struct irq_domain *parent, + struct acpi_madt_bio_pic *acpi_pchpic); +int pch_msi_acpi_init(struct irq_domain *parent, + struct acpi_madt_msi_pic *acpi_pchmsi); + +#endif /* _DRIVERS_IRQCHIP_IRQ_LOONGSON_H */ From 843ed9317be1d0c3f4245418644fc7e55f465419 Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Fri, 23 Aug 2024 18:39:33 +0800 Subject: [PATCH 124/180] LoongArch: Architectural preparation for AVEC irqchip Add architectural preparation for AVEC irqchip, including: 1. CPUCFG feature bits definition for AVEC; 2. Detection of AVEC irqchip in cpu_probe(); 3. New IPI type definition (IPI_CLEAR_VECTOR) for AVEC; 4. Provide arch_probe_nr_irqs() for large NR_IRQS; 5. Other related changes about the number of interrupts. Signed-off-by: Huacai Chen Signed-off-by: Tianyang Zhang Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240823103936.25092-2-zhangtianyang@loongson.cn --- arch/loongarch/include/asm/cpu-features.h | 1 + arch/loongarch/include/asm/cpu.h | 2 ++ arch/loongarch/include/asm/hardirq.h | 3 ++- arch/loongarch/include/asm/irq.h | 15 +++++++++++++-- arch/loongarch/include/asm/loongarch.h | 18 ++++++++++++++---- arch/loongarch/include/asm/smp.h | 2 ++ arch/loongarch/kernel/cpu-probe.c | 3 ++- arch/loongarch/kernel/irq.c | 12 ++++++++++++ 8 files changed, 48 insertions(+), 8 deletions(-) diff --git a/arch/loongarch/include/asm/cpu-features.h b/arch/loongarch/include/asm/cpu-features.h index 2eafe6a6aca8..16a716f88a5c 100644 --- a/arch/loongarch/include/asm/cpu-features.h +++ b/arch/loongarch/include/asm/cpu-features.h @@ -65,5 +65,6 @@ #define cpu_has_guestid cpu_opt(LOONGARCH_CPU_GUESTID) #define cpu_has_hypervisor cpu_opt(LOONGARCH_CPU_HYPERVISOR) #define cpu_has_ptw cpu_opt(LOONGARCH_CPU_PTW) +#define cpu_has_avecint cpu_opt(LOONGARCH_CPU_AVECINT) #endif /* __ASM_CPU_FEATURES_H */ diff --git a/arch/loongarch/include/asm/cpu.h b/arch/loongarch/include/asm/cpu.h index 48b9f7168bcc..843f9c4ec980 100644 --- a/arch/loongarch/include/asm/cpu.h +++ b/arch/loongarch/include/asm/cpu.h @@ -99,6 +99,7 @@ enum cpu_type_enum { #define CPU_FEATURE_GUESTID 24 /* CPU has GuestID feature */ #define CPU_FEATURE_HYPERVISOR 25 /* CPU has hypervisor (running in VM) */ #define CPU_FEATURE_PTW 26 /* CPU has hardware page table walker */ +#define CPU_FEATURE_AVECINT 27 /* CPU has avec interrupt */ #define LOONGARCH_CPU_CPUCFG BIT_ULL(CPU_FEATURE_CPUCFG) #define LOONGARCH_CPU_LAM BIT_ULL(CPU_FEATURE_LAM) @@ -127,5 +128,6 @@ enum cpu_type_enum { #define LOONGARCH_CPU_GUESTID BIT_ULL(CPU_FEATURE_GUESTID) #define LOONGARCH_CPU_HYPERVISOR BIT_ULL(CPU_FEATURE_HYPERVISOR) #define LOONGARCH_CPU_PTW BIT_ULL(CPU_FEATURE_PTW) +#define LOONGARCH_CPU_AVECINT BIT_ULL(CPU_FEATURE_AVECINT) #endif /* _ASM_CPU_H */ diff --git a/arch/loongarch/include/asm/hardirq.h b/arch/loongarch/include/asm/hardirq.h index 1d7feb719515..10da8d6961cb 100644 --- a/arch/loongarch/include/asm/hardirq.h +++ b/arch/loongarch/include/asm/hardirq.h @@ -12,12 +12,13 @@ extern void ack_bad_irq(unsigned int irq); #define ack_bad_irq ack_bad_irq -#define NR_IPI 3 +#define NR_IPI 4 enum ipi_msg_type { IPI_RESCHEDULE, IPI_CALL_FUNCTION, IPI_IRQ_WORK, + IPI_CLEAR_VECTOR, }; typedef struct { diff --git a/arch/loongarch/include/asm/irq.h b/arch/loongarch/include/asm/irq.h index 65503c9eb529..c4835800c8b9 100644 --- a/arch/loongarch/include/asm/irq.h +++ b/arch/loongarch/include/asm/irq.h @@ -39,11 +39,22 @@ void spurious_interrupt(void); #define NR_IRQS_LEGACY 16 +/* + * 256 Vectors Mapping for AVECINTC: + * + * 0 - 15: Mapping classic IPs, e.g. IP0-12. + * 16 - 255: Mapping vectors for external IRQ. + * + */ +#define NR_VECTORS 256 +#define NR_LEGACY_VECTORS 16 +#define IRQ_MATRIX_BITS NR_VECTORS + #define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace void arch_trigger_cpumask_backtrace(const struct cpumask *mask, int exclude_cpu); #define MAX_IO_PICS 2 -#define NR_IRQS (64 + (256 * MAX_IO_PICS)) +#define NR_IRQS (64 + NR_VECTORS * (NR_CPUS + MAX_IO_PICS)) struct acpi_vector_group { int node; @@ -65,7 +76,7 @@ extern struct acpi_vector_group msi_group[MAX_IO_PICS]; #define LOONGSON_LPC_LAST_IRQ (LOONGSON_LPC_IRQ_BASE + 15) #define LOONGSON_CPU_IRQ_BASE 16 -#define LOONGSON_CPU_LAST_IRQ (LOONGSON_CPU_IRQ_BASE + 14) +#define LOONGSON_CPU_LAST_IRQ (LOONGSON_CPU_IRQ_BASE + 15) #define LOONGSON_PCH_IRQ_BASE 64 #define LOONGSON_PCH_ACPI_IRQ (LOONGSON_PCH_IRQ_BASE + 47) diff --git a/arch/loongarch/include/asm/loongarch.h b/arch/loongarch/include/asm/loongarch.h index 04a78010fc72..631d249b3ef2 100644 --- a/arch/loongarch/include/asm/loongarch.h +++ b/arch/loongarch/include/asm/loongarch.h @@ -253,8 +253,8 @@ #define CSR_ESTAT_EXC_WIDTH 6 #define CSR_ESTAT_EXC (_ULCAST_(0x3f) << CSR_ESTAT_EXC_SHIFT) #define CSR_ESTAT_IS_SHIFT 0 -#define CSR_ESTAT_IS_WIDTH 14 -#define CSR_ESTAT_IS (_ULCAST_(0x3fff) << CSR_ESTAT_IS_SHIFT) +#define CSR_ESTAT_IS_WIDTH 15 +#define CSR_ESTAT_IS (_ULCAST_(0x7fff) << CSR_ESTAT_IS_SHIFT) #define LOONGARCH_CSR_ERA 0x6 /* ERA */ @@ -649,6 +649,13 @@ #define LOONGARCH_CSR_CTAG 0x98 /* TagLo + TagHi */ +#define LOONGARCH_CSR_ISR0 0xa0 +#define LOONGARCH_CSR_ISR1 0xa1 +#define LOONGARCH_CSR_ISR2 0xa2 +#define LOONGARCH_CSR_ISR3 0xa3 + +#define LOONGARCH_CSR_IRR 0xa4 + #define LOONGARCH_CSR_PRID 0xc0 /* Shadow MCSR : 0xc0 ~ 0xff */ @@ -1011,7 +1018,7 @@ /* * CSR_ECFG IM */ -#define ECFG0_IM 0x00001fff +#define ECFG0_IM 0x00005fff #define ECFGB_SIP0 0 #define ECFGF_SIP0 (_ULCAST_(1) << ECFGB_SIP0) #define ECFGB_SIP1 1 @@ -1054,6 +1061,7 @@ #define IOCSRF_EIODECODE BIT_ULL(9) #define IOCSRF_FLATMODE BIT_ULL(10) #define IOCSRF_VM BIT_ULL(11) +#define IOCSRF_AVEC BIT_ULL(15) #define LOONGARCH_IOCSR_VENDOR 0x10 @@ -1065,6 +1073,7 @@ #define IOCSR_MISC_FUNC_SOFT_INT BIT_ULL(10) #define IOCSR_MISC_FUNC_TIMER_RESET BIT_ULL(21) #define IOCSR_MISC_FUNC_EXT_IOI_EN BIT_ULL(48) +#define IOCSR_MISC_FUNC_AVEC_EN BIT_ULL(51) #define LOONGARCH_IOCSR_CPUTEMP 0x428 @@ -1387,9 +1396,10 @@ __BUILD_CSR_OP(tlbidx) #define INT_TI 11 /* Timer */ #define INT_IPI 12 #define INT_NMI 13 +#define INT_AVEC 14 /* ExcCodes corresponding to interrupts */ -#define EXCCODE_INT_NUM (INT_NMI + 1) +#define EXCCODE_INT_NUM (INT_AVEC + 1) #define EXCCODE_INT_START 64 #define EXCCODE_INT_END (EXCCODE_INT_START + EXCCODE_INT_NUM - 1) diff --git a/arch/loongarch/include/asm/smp.h b/arch/loongarch/include/asm/smp.h index 50db503f44e3..3383c9d24e94 100644 --- a/arch/loongarch/include/asm/smp.h +++ b/arch/loongarch/include/asm/smp.h @@ -70,10 +70,12 @@ extern int __cpu_logical_map[NR_CPUS]; #define ACTION_RESCHEDULE 1 #define ACTION_CALL_FUNCTION 2 #define ACTION_IRQ_WORK 3 +#define ACTION_CLEAR_VECTOR 4 #define SMP_BOOT_CPU BIT(ACTION_BOOT_CPU) #define SMP_RESCHEDULE BIT(ACTION_RESCHEDULE) #define SMP_CALL_FUNCTION BIT(ACTION_CALL_FUNCTION) #define SMP_IRQ_WORK BIT(ACTION_IRQ_WORK) +#define SMP_CLEAR_VECTOR BIT(ACTION_CLEAR_VECTOR) struct secondary_data { unsigned long stack; diff --git a/arch/loongarch/kernel/cpu-probe.c b/arch/loongarch/kernel/cpu-probe.c index 55320813ee08..14f0449f5452 100644 --- a/arch/loongarch/kernel/cpu-probe.c +++ b/arch/loongarch/kernel/cpu-probe.c @@ -106,7 +106,6 @@ static void cpu_probe_common(struct cpuinfo_loongarch *c) elf_hwcap |= HWCAP_LOONGARCH_CRC32; } - config = read_cpucfg(LOONGARCH_CPUCFG2); if (config & CPUCFG2_LAM) { c->options |= LOONGARCH_CPU_LAM; @@ -174,6 +173,8 @@ static void cpu_probe_common(struct cpuinfo_loongarch *c) c->options |= LOONGARCH_CPU_FLATMODE; if (config & IOCSRF_EIODECODE) c->options |= LOONGARCH_CPU_EIODECODE; + if (config & IOCSRF_AVEC) + c->options |= LOONGARCH_CPU_AVECINT; if (config & IOCSRF_VM) c->options |= LOONGARCH_CPU_HYPERVISOR; diff --git a/arch/loongarch/kernel/irq.c b/arch/loongarch/kernel/irq.c index f4991c03514f..414f5249d70a 100644 --- a/arch/loongarch/kernel/irq.c +++ b/arch/loongarch/kernel/irq.c @@ -87,6 +87,18 @@ static void __init init_vec_parent_group(void) acpi_table_parse(ACPI_SIG_MCFG, early_pci_mcfg_parse); } +int __init arch_probe_nr_irqs(void) +{ + int nr_io_pics = bitmap_weight(loongson_sysconf.cores_io_master, NR_CPUS); + + if (!cpu_has_avecint) + nr_irqs = (64 + NR_VECTORS * nr_io_pics); + else + nr_irqs = (64 + NR_VECTORS * (nr_cpu_ids + nr_io_pics)); + + return NR_IRQS_LEGACY; +} + void __init init_IRQ(void) { int i; From 9e83dd3ebb14fadccb936308b7b101c75da76324 Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Fri, 23 Aug 2024 18:39:34 +0800 Subject: [PATCH 125/180] irqchip/loongson-eiointc: Rename CPUHP_AP_IRQ_LOONGARCH_STARTING Rename CPUHP_AP_IRQ_LOONGARCH_STARTING to CPUHP_AP_IRQ_EIOINTC_STARTING because the upcoming AVECINTC irqchip driver will introduce a new state and so both are clearly identifiable. Signed-off-by: Huacai Chen Signed-off-by: Tianyang Zhang Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240823103936.25092-3-zhangtianyang@loongson.cn --- drivers/irqchip/irq-loongson-eiointc.c | 4 ++-- include/linux/cpuhotplug.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/irqchip/irq-loongson-eiointc.c b/drivers/irqchip/irq-loongson-eiointc.c index 34b5ca2f5e62..c756b7aa8055 100644 --- a/drivers/irqchip/irq-loongson-eiointc.c +++ b/drivers/irqchip/irq-loongson-eiointc.c @@ -398,8 +398,8 @@ static int __init eiointc_init(struct eiointc_priv *priv, int parent_irq, if (nr_pics == 1) { register_syscore_ops(&eiointc_syscore_ops); - cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_LOONGARCH_STARTING, - "irqchip/loongarch/intc:starting", + cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_EIOINTC_STARTING, + "irqchip/loongarch/eiointc:starting", eiointc_router_init, NULL); } diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 51ba681b915a..e49807f7805c 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -145,7 +145,7 @@ enum cpuhp_state { CPUHP_AP_IRQ_ARMADA_XP_STARTING, CPUHP_AP_IRQ_BCM2836_STARTING, CPUHP_AP_IRQ_MIPS_GIC_STARTING, - CPUHP_AP_IRQ_LOONGARCH_STARTING, + CPUHP_AP_IRQ_EIOINTC_STARTING, CPUHP_AP_IRQ_SIFIVE_PLIC_STARTING, CPUHP_AP_IRQ_RISCV_IMSIC_STARTING, CPUHP_AP_ARM_MVEBU_COHERENCY, From a1d4646d34c6642194a421ca9afbd060b0f9aa00 Mon Sep 17 00:00:00 2001 From: Tianyang Zhang Date: Fri, 23 Aug 2024 18:43:36 +0800 Subject: [PATCH 126/180] irqchip/loongson-pch-msi: Prepare get_pch_msi_handle() for AVECINTC On Loongson-3C6000 and higher systems with AVECINTC irqchip, there may be multiple PCI segments, but only one PCH-MSI irq domain. In this case, let get_pch_msi_handle() return the first domain handle. Co-developed-by: Jianmin Lv Signed-off-by: Jianmin Lv Co-developed-by: Liupu Wang Signed-off-by: Liupu Wang Co-developed-by: Huacai Chen Signed-off-by: Huacai Chen Signed-off-by: Tianyang Zhang Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240823104337.25577-1-zhangtianyang@loongson.cn --- drivers/irqchip/irq-loongson-pch-msi.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/irqchip/irq-loongson-pch-msi.c b/drivers/irqchip/irq-loongson-pch-msi.c index d43731878800..0dc14550791d 100644 --- a/drivers/irqchip/irq-loongson-pch-msi.c +++ b/drivers/irqchip/irq-loongson-pch-msi.c @@ -255,17 +255,17 @@ IRQCHIP_DECLARE(pch_msi, "loongson,pch-msi-1.0", pch_msi_of_init); #ifdef CONFIG_ACPI struct fwnode_handle *get_pch_msi_handle(int pci_segment) { - int i; + if (cpu_has_avecint) + return pch_msi_handle[0]; - for (i = 0; i < MAX_IO_PICS; i++) { + for (int i = 0; i < MAX_IO_PICS; i++) { if (msi_group[i].pci_segment == pci_segment) return pch_msi_handle[i]; } - return NULL; + return pch_msi_handle[0]; } -int __init pch_msi_acpi_init(struct irq_domain *parent, - struct acpi_madt_msi_pic *acpi_pchmsi) +int __init pch_msi_acpi_init(struct irq_domain *parent, struct acpi_madt_msi_pic *acpi_pchmsi) { int ret; struct fwnode_handle *domain_handle; From ae16f05c928a1336d5d9d19fd805d7bf29c3f0c8 Mon Sep 17 00:00:00 2001 From: Tianyang Zhang Date: Fri, 23 Aug 2024 18:43:37 +0800 Subject: [PATCH 127/180] irqchip/loongarch-avec: Add AVEC irqchip support Introduce the advanced extended interrupt controllers (AVECINTC). This feature will allow each core to have 256 independent interrupt vectors and MSI interrupts can be independently routed to any vector on any CPU. The whole topology of irqchips in LoongArch machines looks like this if AVECINTC is supported: +-----+ +-----------------------+ +-------+ | IPI | --> | CPUINTC | <-- | Timer | +-----+ +-----------------------+ +-------+ ^ ^ ^ | | | +---------+ +----------+ +---------+ +-------+ | EIOINTC | | AVECINTC | | LIOINTC | <-- | UARTs | +---------+ +----------+ +---------+ +-------+ ^ ^ | | +---------+ +---------+ | PCH-PIC | | PCH-MSI | +---------+ +---------+ ^ ^ ^ | | | +---------+ +---------+ +---------+ | Devices | | PCH-LPC | | Devices | +---------+ +---------+ +---------+ ^ | +---------+ | Devices | +---------+ Co-developed-by: Jianmin Lv Signed-off-by: Jianmin Lv Co-developed-by: Liupu Wang Signed-off-by: Liupu Wang Co-developed-by: Huacai Chen Signed-off-by: Huacai Chen Signed-off-by: Tianyang Zhang Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240823104337.25577-2-zhangtianyang@loongson.cn --- arch/loongarch/Kconfig | 1 + arch/loongarch/include/asm/irq.h | 2 + arch/loongarch/kernel/paravirt.c | 5 + arch/loongarch/kernel/smp.c | 6 + drivers/irqchip/Makefile | 2 +- drivers/irqchip/irq-loongarch-avec.c | 425 +++++++++++++++++++++++++ drivers/irqchip/irq-loongarch-cpu.c | 5 +- drivers/irqchip/irq-loongson-eiointc.c | 3 + drivers/irqchip/irq-loongson-pch-msi.c | 14 + drivers/irqchip/irq-loongson.h | 2 + include/linux/cpuhotplug.h | 1 + 11 files changed, 464 insertions(+), 2 deletions(-) create mode 100644 drivers/irqchip/irq-loongarch-avec.c diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 70f169210b52..0e3abf7b0bd3 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -85,6 +85,7 @@ config LOONGARCH select GENERIC_ENTRY select GENERIC_GETTIMEOFDAY select GENERIC_IOREMAP if !ARCH_IOREMAP + select GENERIC_IRQ_MATRIX_ALLOCATOR select GENERIC_IRQ_MULTI_HANDLER select GENERIC_IRQ_PROBE select GENERIC_IRQ_SHOW diff --git a/arch/loongarch/include/asm/irq.h b/arch/loongarch/include/asm/irq.h index c4835800c8b9..9c2ca785faa9 100644 --- a/arch/loongarch/include/asm/irq.h +++ b/arch/loongarch/include/asm/irq.h @@ -99,6 +99,8 @@ struct acpi_madt_bio_pic; struct acpi_madt_msi_pic; struct acpi_madt_lpc_pic; +void complete_irq_moving(void); + struct fwnode_handle *get_pch_msi_handle(int pci_segment); extern struct acpi_madt_lio_pic *acpi_liointc; diff --git a/arch/loongarch/kernel/paravirt.c b/arch/loongarch/kernel/paravirt.c index 9c9b75b76f62..4d736a4e488d 100644 --- a/arch/loongarch/kernel/paravirt.c +++ b/arch/loongarch/kernel/paravirt.c @@ -134,6 +134,11 @@ static irqreturn_t pv_ipi_interrupt(int irq, void *dev) info->ipi_irqs[IPI_IRQ_WORK]++; } + if (action & SMP_CLEAR_VECTOR) { + complete_irq_moving(); + info->ipi_irqs[IPI_CLEAR_VECTOR]++; + } + return IRQ_HANDLED; } diff --git a/arch/loongarch/kernel/smp.c b/arch/loongarch/kernel/smp.c index ca405ab86aae..4adbbef3450a 100644 --- a/arch/loongarch/kernel/smp.c +++ b/arch/loongarch/kernel/smp.c @@ -72,6 +72,7 @@ static const char *ipi_types[NR_IPI] __tracepoint_string = { [IPI_RESCHEDULE] = "Rescheduling interrupts", [IPI_CALL_FUNCTION] = "Function call interrupts", [IPI_IRQ_WORK] = "IRQ work interrupts", + [IPI_CLEAR_VECTOR] = "Clear vector interrupts", }; void show_ipi_list(struct seq_file *p, int prec) @@ -248,6 +249,11 @@ static irqreturn_t loongson_ipi_interrupt(int irq, void *dev) per_cpu(irq_stat, cpu).ipi_irqs[IPI_IRQ_WORK]++; } + if (action & SMP_CLEAR_VECTOR) { + complete_irq_moving(); + per_cpu(irq_stat, cpu).ipi_irqs[IPI_CLEAR_VECTOR]++; + } + return IRQ_HANDLED; } diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index 15635812b2d6..e3679ec2b9f7 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -110,7 +110,7 @@ obj-$(CONFIG_LS1X_IRQ) += irq-ls1x.o obj-$(CONFIG_TI_SCI_INTR_IRQCHIP) += irq-ti-sci-intr.o obj-$(CONFIG_TI_SCI_INTA_IRQCHIP) += irq-ti-sci-inta.o obj-$(CONFIG_TI_PRUSS_INTC) += irq-pruss-intc.o -obj-$(CONFIG_IRQ_LOONGARCH_CPU) += irq-loongarch-cpu.o +obj-$(CONFIG_IRQ_LOONGARCH_CPU) += irq-loongarch-cpu.o irq-loongarch-avec.o obj-$(CONFIG_LOONGSON_LIOINTC) += irq-loongson-liointc.o obj-$(CONFIG_LOONGSON_EIOINTC) += irq-loongson-eiointc.o obj-$(CONFIG_LOONGSON_HTPIC) += irq-loongson-htpic.o diff --git a/drivers/irqchip/irq-loongarch-avec.c b/drivers/irqchip/irq-loongarch-avec.c new file mode 100644 index 000000000000..0f6e465dd309 --- /dev/null +++ b/drivers/irqchip/irq-loongarch-avec.c @@ -0,0 +1,425 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2024 Loongson Technologies, Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "irq-msi-lib.h" +#include "irq-loongson.h" + +#define VECTORS_PER_REG 64 +#define IRR_VECTOR_MASK 0xffUL +#define IRR_INVALID_MASK 0x80000000UL +#define AVEC_MSG_OFFSET 0x100000 + +#ifdef CONFIG_SMP +struct pending_list { + struct list_head head; +}; + +static struct cpumask intersect_mask; +static DEFINE_PER_CPU(struct pending_list, pending_list); +#endif + +static DEFINE_PER_CPU(struct irq_desc * [NR_VECTORS], irq_map); + +struct avecintc_chip { + raw_spinlock_t lock; + struct fwnode_handle *fwnode; + struct irq_domain *domain; + struct irq_matrix *vector_matrix; + phys_addr_t msi_base_addr; +}; + +static struct avecintc_chip loongarch_avec; + +struct avecintc_data { + struct list_head entry; + unsigned int cpu; + unsigned int vec; + unsigned int prev_cpu; + unsigned int prev_vec; + unsigned int moving; +}; + +static inline void avecintc_ack_irq(struct irq_data *d) +{ +} + +static inline void avecintc_mask_irq(struct irq_data *d) +{ +} + +static inline void avecintc_unmask_irq(struct irq_data *d) +{ +} + +#ifdef CONFIG_SMP +static inline void pending_list_init(int cpu) +{ + struct pending_list *plist = per_cpu_ptr(&pending_list, cpu); + + INIT_LIST_HEAD(&plist->head); +} + +static void avecintc_sync(struct avecintc_data *adata) +{ + struct pending_list *plist; + + if (cpu_online(adata->prev_cpu)) { + plist = per_cpu_ptr(&pending_list, adata->prev_cpu); + list_add_tail(&adata->entry, &plist->head); + adata->moving = 1; + mp_ops.send_ipi_single(adata->prev_cpu, ACTION_CLEAR_VECTOR); + } +} + +static int avecintc_set_affinity(struct irq_data *data, const struct cpumask *dest, bool force) +{ + int cpu, ret, vector; + struct avecintc_data *adata; + + scoped_guard(raw_spinlock, &loongarch_avec.lock) { + adata = irq_data_get_irq_chip_data(data); + + if (adata->moving) + return -EBUSY; + + if (cpu_online(adata->cpu) && cpumask_test_cpu(adata->cpu, dest)) + return 0; + + cpumask_and(&intersect_mask, dest, cpu_online_mask); + + ret = irq_matrix_alloc(loongarch_avec.vector_matrix, &intersect_mask, false, &cpu); + if (ret < 0) + return ret; + + vector = ret; + adata->cpu = cpu; + adata->vec = vector; + per_cpu_ptr(irq_map, adata->cpu)[adata->vec] = irq_data_to_desc(data); + avecintc_sync(adata); + } + + irq_data_update_effective_affinity(data, cpumask_of(cpu)); + + return IRQ_SET_MASK_OK; +} + +static int avecintc_cpu_online(unsigned int cpu) +{ + if (!loongarch_avec.vector_matrix) + return 0; + + guard(raw_spinlock)(&loongarch_avec.lock); + + irq_matrix_online(loongarch_avec.vector_matrix); + + pending_list_init(cpu); + + return 0; +} + +static int avecintc_cpu_offline(unsigned int cpu) +{ + struct pending_list *plist = per_cpu_ptr(&pending_list, cpu); + + if (!loongarch_avec.vector_matrix) + return 0; + + guard(raw_spinlock)(&loongarch_avec.lock); + + if (!list_empty(&plist->head)) + pr_warn("CPU#%d vector is busy\n", cpu); + + irq_matrix_offline(loongarch_avec.vector_matrix); + + return 0; +} + +void complete_irq_moving(void) +{ + struct pending_list *plist = this_cpu_ptr(&pending_list); + struct avecintc_data *adata, *tdata; + int cpu, vector, bias; + uint64_t isr; + + guard(raw_spinlock)(&loongarch_avec.lock); + + list_for_each_entry_safe(adata, tdata, &plist->head, entry) { + cpu = adata->prev_cpu; + vector = adata->prev_vec; + bias = vector / VECTORS_PER_REG; + switch (bias) { + case 0: + isr = csr_read64(LOONGARCH_CSR_ISR0); + break; + case 1: + isr = csr_read64(LOONGARCH_CSR_ISR1); + break; + case 2: + isr = csr_read64(LOONGARCH_CSR_ISR2); + break; + case 3: + isr = csr_read64(LOONGARCH_CSR_ISR3); + break; + } + + if (isr & (1UL << (vector % VECTORS_PER_REG))) { + mp_ops.send_ipi_single(cpu, ACTION_CLEAR_VECTOR); + continue; + } + list_del(&adata->entry); + irq_matrix_free(loongarch_avec.vector_matrix, cpu, vector, false); + this_cpu_write(irq_map[vector], NULL); + adata->moving = 0; + adata->prev_cpu = adata->cpu; + adata->prev_vec = adata->vec; + } +} +#endif + +static void avecintc_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) +{ + struct avecintc_data *adata = irq_data_get_irq_chip_data(d); + + msg->address_hi = 0x0; + msg->address_lo = (loongarch_avec.msi_base_addr | (adata->vec & 0xff) << 4) + | ((cpu_logical_map(adata->cpu & 0xffff)) << 12); + msg->data = 0x0; +} + +static struct irq_chip avec_irq_controller = { + .name = "AVECINTC", + .irq_ack = avecintc_ack_irq, + .irq_mask = avecintc_mask_irq, + .irq_unmask = avecintc_unmask_irq, +#ifdef CONFIG_SMP + .irq_set_affinity = avecintc_set_affinity, +#endif + .irq_compose_msi_msg = avecintc_compose_msi_msg, +}; + +static void avecintc_irq_dispatch(struct irq_desc *desc) +{ + struct irq_chip *chip = irq_desc_get_chip(desc); + struct irq_desc *d; + + chained_irq_enter(chip, desc); + + while (true) { + unsigned long vector = csr_read64(LOONGARCH_CSR_IRR); + if (vector & IRR_INVALID_MASK) + break; + + vector &= IRR_VECTOR_MASK; + + d = this_cpu_read(irq_map[vector]); + if (d) { + generic_handle_irq_desc(d); + } else { + spurious_interrupt(); + pr_warn("Unexpected IRQ occurs on CPU#%d [vector %ld]\n", smp_processor_id(), vector); + } + } + + chained_irq_exit(chip, desc); +} + +static int avecintc_alloc_vector(struct irq_data *irqd, struct avecintc_data *adata) +{ + int cpu, ret; + + guard(raw_spinlock_irqsave)(&loongarch_avec.lock); + + ret = irq_matrix_alloc(loongarch_avec.vector_matrix, cpu_online_mask, false, &cpu); + if (ret < 0) + return ret; + + adata->prev_cpu = adata->cpu = cpu; + adata->prev_vec = adata->vec = ret; + per_cpu_ptr(irq_map, adata->cpu)[adata->vec] = irq_data_to_desc(irqd); + + return 0; +} + +static int avecintc_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + for (unsigned int i = 0; i < nr_irqs; i++) { + struct irq_data *irqd = irq_domain_get_irq_data(domain, virq + i); + struct avecintc_data *adata = kzalloc(sizeof(*adata), GFP_KERNEL); + int ret; + + if (!adata) + return -ENOMEM; + + ret = avecintc_alloc_vector(irqd, adata); + if (ret < 0) { + kfree(adata); + return ret; + } + + irq_domain_set_info(domain, virq + i, virq + i, &avec_irq_controller, + adata, handle_edge_irq, NULL, NULL); + irqd_set_single_target(irqd); + irqd_set_affinity_on_activate(irqd); + } + + return 0; +} + +static void avecintc_free_vector(struct irq_data *irqd, struct avecintc_data *adata) +{ + guard(raw_spinlock_irqsave)(&loongarch_avec.lock); + + per_cpu(irq_map, adata->cpu)[adata->vec] = NULL; + irq_matrix_free(loongarch_avec.vector_matrix, adata->cpu, adata->vec, false); + +#ifdef CONFIG_SMP + if (!adata->moving) + return; + + per_cpu(irq_map, adata->prev_cpu)[adata->prev_vec] = NULL; + irq_matrix_free(loongarch_avec.vector_matrix, adata->prev_cpu, adata->prev_vec, false); + list_del_init(&adata->entry); +#endif +} + +static void avecintc_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + for (unsigned int i = 0; i < nr_irqs; i++) { + struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); + + if (d) { + struct avecintc_data *adata = irq_data_get_irq_chip_data(d); + + avecintc_free_vector(d, adata); + irq_domain_reset_irq_data(d); + kfree(adata); + } + } +} + +static const struct irq_domain_ops avecintc_domain_ops = { + .alloc = avecintc_domain_alloc, + .free = avecintc_domain_free, + .select = msi_lib_irq_domain_select, +}; + +static int __init irq_matrix_init(void) +{ + loongarch_avec.vector_matrix = irq_alloc_matrix(NR_VECTORS, 0, NR_VECTORS); + if (!loongarch_avec.vector_matrix) + return -ENOMEM; + + for (int i = 0; i < NR_LEGACY_VECTORS; i++) + irq_matrix_assign_system(loongarch_avec.vector_matrix, i, false); + + irq_matrix_online(loongarch_avec.vector_matrix); + + return 0; +} + +static int __init avecintc_init(struct irq_domain *parent) +{ + int ret, parent_irq; + unsigned long value; + + raw_spin_lock_init(&loongarch_avec.lock); + + loongarch_avec.fwnode = irq_domain_alloc_named_fwnode("AVECINTC"); + if (!loongarch_avec.fwnode) { + pr_err("Unable to allocate domain handle\n"); + ret = -ENOMEM; + goto out; + } + + loongarch_avec.domain = irq_domain_create_tree(loongarch_avec.fwnode, + &avecintc_domain_ops, NULL); + if (!loongarch_avec.domain) { + pr_err("Unable to create IRQ domain\n"); + ret = -ENOMEM; + goto out_free_handle; + } + + parent_irq = irq_create_mapping(parent, INT_AVEC); + if (!parent_irq) { + pr_err("Failed to mapping hwirq\n"); + ret = -EINVAL; + goto out_remove_domain; + } + + ret = irq_matrix_init(); + if (ret < 0) { + pr_err("Failed to init irq matrix\n"); + goto out_remove_domain; + } + irq_set_chained_handler_and_data(parent_irq, avecintc_irq_dispatch, NULL); + +#ifdef CONFIG_SMP + pending_list_init(0); + cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_AVECINTC_STARTING, + "irqchip/loongarch/avecintc:starting", + avecintc_cpu_online, avecintc_cpu_offline); +#endif + value = iocsr_read64(LOONGARCH_IOCSR_MISC_FUNC); + value |= IOCSR_MISC_FUNC_AVEC_EN; + iocsr_write64(value, LOONGARCH_IOCSR_MISC_FUNC); + + return ret; + +out_remove_domain: + irq_domain_remove(loongarch_avec.domain); +out_free_handle: + irq_domain_free_fwnode(loongarch_avec.fwnode); +out: + return ret; +} + +static int __init pch_msi_parse_madt(union acpi_subtable_headers *header, + const unsigned long end) +{ + struct acpi_madt_msi_pic *pchmsi_entry = (struct acpi_madt_msi_pic *)header; + + loongarch_avec.msi_base_addr = pchmsi_entry->msg_address - AVEC_MSG_OFFSET; + + return pch_msi_acpi_init_avec(loongarch_avec.domain); +} + +static inline int __init acpi_cascade_irqdomain_init(void) +{ + return acpi_table_parse_madt(ACPI_MADT_TYPE_MSI_PIC, pch_msi_parse_madt, 1); +} + +int __init avecintc_acpi_init(struct irq_domain *parent) +{ + int ret = avecintc_init(parent); + if (ret < 0) { + pr_err("Failed to init IRQ domain\n"); + return ret; + } + + ret = acpi_cascade_irqdomain_init(); + if (ret < 0) { + pr_err("Failed to init cascade IRQ domain\n"); + return ret; + } + + return ret; +} diff --git a/drivers/irqchip/irq-loongarch-cpu.c b/drivers/irqchip/irq-loongarch-cpu.c index 83f7492290a8..bcbd7fd33178 100644 --- a/drivers/irqchip/irq-loongarch-cpu.c +++ b/drivers/irqchip/irq-loongarch-cpu.c @@ -140,7 +140,10 @@ static int __init acpi_cascade_irqdomain_init(void) if (r < 0) return r; - return 0; + if (cpu_has_avecint) + r = avecintc_acpi_init(irq_domain); + + return r; } static int __init cpuintc_acpi_init(union acpi_subtable_headers *header, diff --git a/drivers/irqchip/irq-loongson-eiointc.c b/drivers/irqchip/irq-loongson-eiointc.c index c756b7aa8055..e24db71a8783 100644 --- a/drivers/irqchip/irq-loongson-eiointc.c +++ b/drivers/irqchip/irq-loongson-eiointc.c @@ -362,6 +362,9 @@ static int __init acpi_cascade_irqdomain_init(void) if (r < 0) return r; + if (cpu_has_avecint) + return 0; + r = acpi_table_parse_madt(ACPI_MADT_TYPE_MSI_PIC, pch_msi_parse_madt, 1); if (r < 0) return r; diff --git a/drivers/irqchip/irq-loongson-pch-msi.c b/drivers/irqchip/irq-loongson-pch-msi.c index 0dc14550791d..bd337ecddb40 100644 --- a/drivers/irqchip/irq-loongson-pch-msi.c +++ b/drivers/irqchip/irq-loongson-pch-msi.c @@ -278,4 +278,18 @@ int __init pch_msi_acpi_init(struct irq_domain *parent, struct acpi_madt_msi_pic return ret; } + +int __init pch_msi_acpi_init_avec(struct irq_domain *parent) +{ + if (pch_msi_handle[0]) + return 0; + + pch_msi_handle[0] = parent->fwnode; + irq_domain_update_bus_token(parent, DOMAIN_BUS_NEXUS); + + parent->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT; + parent->msi_parent_ops = &pch_msi_parent_ops; + + return 0; +} #endif diff --git a/drivers/irqchip/irq-loongson.h b/drivers/irqchip/irq-loongson.h index b155f1258ed5..11fa138d1f44 100644 --- a/drivers/irqchip/irq-loongson.h +++ b/drivers/irqchip/irq-loongson.h @@ -12,6 +12,7 @@ int liointc_acpi_init(struct irq_domain *parent, struct acpi_madt_lio_pic *acpi_liointc); int eiointc_acpi_init(struct irq_domain *parent, struct acpi_madt_eio_pic *acpi_eiointc); +int avecintc_acpi_init(struct irq_domain *parent); int htvec_acpi_init(struct irq_domain *parent, struct acpi_madt_ht_pic *acpi_htvec); @@ -21,5 +22,6 @@ int pch_pic_acpi_init(struct irq_domain *parent, struct acpi_madt_bio_pic *acpi_pchpic); int pch_msi_acpi_init(struct irq_domain *parent, struct acpi_madt_msi_pic *acpi_pchmsi); +int pch_msi_acpi_init_avec(struct irq_domain *parent); #endif /* _DRIVERS_IRQCHIP_IRQ_LOONGSON_H */ diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index e49807f7805c..55a726d317d4 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -146,6 +146,7 @@ enum cpuhp_state { CPUHP_AP_IRQ_BCM2836_STARTING, CPUHP_AP_IRQ_MIPS_GIC_STARTING, CPUHP_AP_IRQ_EIOINTC_STARTING, + CPUHP_AP_IRQ_AVECINTC_STARTING, CPUHP_AP_IRQ_SIFIVE_PLIC_STARTING, CPUHP_AP_IRQ_RISCV_IMSIC_STARTING, CPUHP_AP_ARM_MVEBU_COHERENCY, From 17e28a9aeae40d2de3c1ea3b94819ed94bfd6392 Mon Sep 17 00:00:00 2001 From: Costa Shulyupin Date: Thu, 22 Aug 2024 15:31:58 +0300 Subject: [PATCH 128/180] genirq: Fix typo in struct comment Remove redundant "e" in "assign(e)ments". Signed-off-by: Costa Shulyupin Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240822123205.2186221-1-costa.shul@redhat.com --- include/linux/interrupt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index 694de61e0b38..457151f9f263 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -276,7 +276,7 @@ struct irq_affinity_notify { #define IRQ_AFFINITY_MAX_SETS 4 /** - * struct irq_affinity - Description for automatic irq affinity assignements + * struct irq_affinity - Description for automatic irq affinity assignments * @pre_vectors: Don't apply affinity to @pre_vectors at beginning of * the MSI(-X) vector space * @post_vectors: Don't apply affinity to @post_vectors at end of From 64b6d1d7a84538de34c22a6fc92a7dcc2b196b64 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 26 Aug 2024 09:06:18 +0100 Subject: [PATCH 129/180] genirq: Get rid of global lock in irq_do_set_affinity() Kunkun Jiang reports that for a workload involving the simultaneous startup of a large number of VMs (for a total of about 200 vcpus), a lot of CPU time gets spent on spinning on the tmp_mask_lock that exists as a static raw spinlock in irq_do_set_affinity(). This lock protects a global cpumask (tmp_mask) that is used as a temporary variable to compute the resulting affinity. While this is triggered by KVM issuing a irq_set_affinity() call each time a vcpu is about to execute, it is obvious that having a single global resource is not very scalable. Since a cpumask can be a fairly large structure on systems with a high core count, a stack allocation is not really appropriate. Instead, turn the global cpumask into a per-CPU variable, removing the need for locking altogether as the code is executed with preemption and interrupts disabled. [ tglx: Moved the per CPU variable declaration outside of the function ] Reported-by: Kunkun Jiang Suggested-by: Thomas Gleixner Signed-off-by: Marc Zyngier Signed-off-by: Thomas Gleixner Tested-by: Kunkun Jiang Link: https://lore.kernel.org/all/20240826080618.3886694-1-maz@kernel.org Link: https://lore.kernel.org/all/a7fc58e4-64c2-77fc-c1dc-f5eb78dbbb01@huawei.com --- kernel/irq/manage.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index dd53298ef1a5..f0803d6bd296 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -218,21 +218,20 @@ static void irq_validate_effective_affinity(struct irq_data *data) static inline void irq_validate_effective_affinity(struct irq_data *data) { } #endif +static DEFINE_PER_CPU(struct cpumask, __tmp_mask); + int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask, bool force) { + struct cpumask *tmp_mask = this_cpu_ptr(&__tmp_mask); struct irq_desc *desc = irq_data_to_desc(data); struct irq_chip *chip = irq_data_get_irq_chip(data); const struct cpumask *prog_mask; int ret; - static DEFINE_RAW_SPINLOCK(tmp_mask_lock); - static struct cpumask tmp_mask; - if (!chip || !chip->irq_set_affinity) return -EINVAL; - raw_spin_lock(&tmp_mask_lock); /* * If this is a managed interrupt and housekeeping is enabled on * it check whether the requested affinity mask intersects with @@ -258,11 +257,11 @@ int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask, hk_mask = housekeeping_cpumask(HK_TYPE_MANAGED_IRQ); - cpumask_and(&tmp_mask, mask, hk_mask); - if (!cpumask_intersects(&tmp_mask, cpu_online_mask)) + cpumask_and(tmp_mask, mask, hk_mask); + if (!cpumask_intersects(tmp_mask, cpu_online_mask)) prog_mask = mask; else - prog_mask = &tmp_mask; + prog_mask = tmp_mask; } else { prog_mask = mask; } @@ -272,16 +271,14 @@ int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask, * unless we are being asked to force the affinity (in which * case we do as we are told). */ - cpumask_and(&tmp_mask, prog_mask, cpu_online_mask); - if (!force && !cpumask_empty(&tmp_mask)) - ret = chip->irq_set_affinity(data, &tmp_mask, force); + cpumask_and(tmp_mask, prog_mask, cpu_online_mask); + if (!force && !cpumask_empty(tmp_mask)) + ret = chip->irq_set_affinity(data, tmp_mask, force); else if (force) ret = chip->irq_set_affinity(data, mask, force); else ret = -EINVAL; - raw_spin_unlock(&tmp_mask_lock); - switch (ret) { case IRQ_SET_MASK_OK: case IRQ_SET_MASK_OK_DONE: From 4381b895f544bb84b8cfb34ada64df67c9b2a4f0 Mon Sep 17 00:00:00 2001 From: Anna-Maria Behnsen Date: Thu, 29 Aug 2024 09:41:33 +0200 Subject: [PATCH 130/180] timers: Remove historical extra jiffie for timeout in msleep() msleep() and msleep_interruptible() add a jiffie to the requested timeout. This extra jiffie was introduced to ensure that the timeout will not happen earlier than specified. Since the rework of the timer wheel, the enqueue path already takes care of this. So the extra jiffie added by msleep*() is pointless now. Remove this extra jiffie in msleep() and msleep_interruptible(). Signed-off-by: Anna-Maria Behnsen Signed-off-by: Thomas Gleixner Acked-by: Rafael J. Wysocki Link: https://lore.kernel.org/all/20240829074133.4547-1-anna-maria@linutronix.de --- kernel/time/timer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/time/timer.c b/kernel/time/timer.c index 429232d8659a..d8eb368a5a5b 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -2732,7 +2732,7 @@ void __init init_timers(void) */ void msleep(unsigned int msecs) { - unsigned long timeout = msecs_to_jiffies(msecs) + 1; + unsigned long timeout = msecs_to_jiffies(msecs); while (timeout) timeout = schedule_timeout_uninterruptible(timeout); @@ -2746,7 +2746,7 @@ EXPORT_SYMBOL(msleep); */ unsigned long msleep_interruptible(unsigned int msecs) { - unsigned long timeout = msecs_to_jiffies(msecs) + 1; + unsigned long timeout = msecs_to_jiffies(msecs); while (timeout && !signal_pending(current)) timeout = schedule_timeout_interruptible(timeout); From c7718e5c76d49b5bb394265383ae51f766d5dd3a Mon Sep 17 00:00:00 2001 From: Jeff Xie Date: Sun, 25 Aug 2024 21:19:11 +0800 Subject: [PATCH 131/180] genirq/proc: Correctly set file permissions for affinity control files The kernel already knows at the time of interrupt allocation whether affinity of an interrupt can be controlled by userspace or not. It still creates all related procfs control files with read/write permissions. That's inconsistent and non-intuitive for system administrators and tools. Therefore set the file permissions to read-only for such interrupts. [ tglx: Massage change log, fixed UP build ] Signed-off-by: Jeff Xie Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240825131911.107119-1-jeff.xie@linux.dev --- kernel/irq/proc.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index 8cccdf40725a..9b3b12ad5dda 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c @@ -362,8 +362,13 @@ void register_irq_proc(unsigned int irq, struct irq_desc *desc) goto out_unlock; #ifdef CONFIG_SMP + umode_t umode = S_IRUGO; + + if (irq_can_set_affinity_usr(desc->irq_data.irq)) + umode |= S_IWUSR; + /* create /proc/irq//smp_affinity */ - proc_create_data("smp_affinity", 0644, desc->dir, + proc_create_data("smp_affinity", umode, desc->dir, &irq_affinity_proc_ops, irqp); /* create /proc/irq//affinity_hint */ @@ -371,7 +376,7 @@ void register_irq_proc(unsigned int irq, struct irq_desc *desc) irq_affinity_hint_proc_show, irqp); /* create /proc/irq//smp_affinity_list */ - proc_create_data("smp_affinity_list", 0644, desc->dir, + proc_create_data("smp_affinity_list", umode, desc->dir, &irq_affinity_list_proc_ops, irqp); proc_create_single_data("node", 0444, desc->dir, irq_node_proc_show, From 9012f84e1c5b653c282b7a6cca81454ecf7c5a0a Mon Sep 17 00:00:00 2001 From: Jinjie Ruan Date: Thu, 29 Aug 2024 19:15:22 +0800 Subject: [PATCH 132/180] genirq/proc: Use irq_move_pending() in show_irq_affinity() irq_move_pending() encapsulates irqd_is_setaffinity_pending() depending on CONFIG_GENERIC_PENDING_IRQ. Replace the open coded #ifdeffery with it. Signed-off-by: Jinjie Ruan Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240829111522.230595-1-ruanjinjie@huawei.com --- kernel/irq/proc.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index 9b3b12ad5dda..d98fb9c2c667 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c @@ -52,10 +52,8 @@ static int show_irq_affinity(int type, struct seq_file *m) case AFFINITY: case AFFINITY_LIST: mask = desc->irq_common_data.affinity; -#ifdef CONFIG_GENERIC_PENDING_IRQ - if (irqd_is_setaffinity_pending(&desc->irq_data)) - mask = desc->pending_mask; -#endif + if (irq_move_pending(&desc->irq_data)) + mask = irq_desc_get_pending_mask(desc); break; case EFFECTIVE: case EFFECTIVE_LIST: From eb29369fa543e7d5557c19ebecf072244bb14815 Mon Sep 17 00:00:00 2001 From: Jeff Xie Date: Mon, 26 Aug 2024 22:58:05 +0800 Subject: [PATCH 133/180] genirq/proc: Change the return value for set affinity permission error Currently, when the affinity of an irq cannot be set due to lack of permission, the write_irq_affinity() returns the error code -EIO. Change the return value to -EPERM as that reflects the cause of error correctly. Signed-off-by: Jeff Xie Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240826145805.5938-1-jeff.xie@linux.dev --- kernel/irq/proc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index d98fb9c2c667..9081ada81c3d 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c @@ -140,7 +140,7 @@ static ssize_t write_irq_affinity(int type, struct file *file, int err; if (!irq_can_set_affinity_usr(irq) || no_irq_affinity) - return -EIO; + return -EPERM; if (!zalloc_cpumask_var(&new_value, GFP_KERNEL)) return -ENOMEM; From bf1e0fb69a15fac4d6ee71d0e1c715147add986a Mon Sep 17 00:00:00 2001 From: Jinjie Ruan Date: Wed, 28 Aug 2024 15:22:19 +0800 Subject: [PATCH 134/180] genirq/msi: Use kmemdup_array() instead of kmemdup() Let kmemdup_array() take care about sizing instead of doing it open coded. Signed-off-by: Jinjie Ruan Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240828072219.1249250-1-ruanjinjie@huawei.com --- kernel/irq/msi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index 5fa0547ece0c..1c7e5159064c 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -82,7 +82,7 @@ static struct msi_desc *msi_alloc_desc(struct device *dev, int nvec, desc->dev = dev; desc->nvec_used = nvec; if (affinity) { - desc->affinity = kmemdup(affinity, nvec * sizeof(*desc->affinity), GFP_KERNEL); + desc->affinity = kmemdup_array(affinity, nvec, sizeof(*desc->affinity), GFP_KERNEL); if (!desc->affinity) { kfree(desc); return NULL; From 4609c6eab67bb1785a5337ddafb5c74c796bcd35 Mon Sep 17 00:00:00 2001 From: Hongbo Li Date: Wed, 28 Aug 2024 20:27:24 +0800 Subject: [PATCH 135/180] irqdomain: Use IS_ERR_OR_NULL() in irq_domain_trim_hierarchy() Use IS_ERR_OR_NULL() instead of open-coding a NULL and a error pointer check. Signed-off-by: Hongbo Li Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240828122724.3697447-1-lihongbo22@huawei.com --- kernel/irq/irqdomain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index 5df8780100bb..e0bff21f30e0 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -1403,7 +1403,7 @@ static int irq_domain_trim_hierarchy(unsigned int virq) tail = NULL; /* The first entry must have a valid irqchip */ - if (!irq_data->chip || IS_ERR(irq_data->chip)) + if (IS_ERR_OR_NULL(irq_data->chip)) return -EINVAL; /* From 85a147a986e4feafb9286fabaf03a302a611cd85 Mon Sep 17 00:00:00 2001 From: Jinjie Ruan Date: Tue, 3 Sep 2024 11:53:58 +0800 Subject: [PATCH 136/180] printk: Use the BITS_PER_LONG macro sizeof(unsigned long) * 8 is the number of bits in an unsigned long variable, replace it with BITS_PER_LONG macro to make it simpler. Signed-off-by: Jinjie Ruan Reviewed-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240903035358.308482-1-ruanjinjie@huawei.com Signed-off-by: Petr Mladek --- kernel/printk/printk_ringbuffer.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/printk/printk_ringbuffer.h b/kernel/printk/printk_ringbuffer.h index bd2a892deac1..8de6c495cf2b 100644 --- a/kernel/printk/printk_ringbuffer.h +++ b/kernel/printk/printk_ringbuffer.h @@ -4,6 +4,7 @@ #define _KERNEL_PRINTK_RINGBUFFER_H #include +#include #include #include #include @@ -122,7 +123,7 @@ enum desc_state { #define _DATA_SIZE(sz_bits) (1UL << (sz_bits)) #define _DESCS_COUNT(ct_bits) (1U << (ct_bits)) -#define DESC_SV_BITS (sizeof(unsigned long) * 8) +#define DESC_SV_BITS BITS_PER_LONG #define DESC_FLAGS_SHIFT (DESC_SV_BITS - 2) #define DESC_FLAGS_MASK (3UL << DESC_FLAGS_SHIFT) #define DESC_STATE(sv) (3UL & (sv >> DESC_FLAGS_SHIFT)) From 79f8b28e85f83563c86f528b91eff19c0c4d1177 Mon Sep 17 00:00:00 2001 From: Anna-Maria Behnsen Date: Thu, 29 Aug 2024 17:43:05 +0200 Subject: [PATCH 137/180] timers: Annotate possible non critical data race of next_expiry Global timers could be expired remotely when the target CPU is idle. After a remote timer expiry, the remote timer_base->next_expiry value is updated while holding the timer_base->lock. When the formerly idle CPU becomes active at the same time and checks whether timers need to expire, this check is done lockless as it is on the local CPU. This could lead to a data race, which was reported by sysbot: https://lore.kernel.org/r/000000000000916e55061f969e14@google.com When the value is read lockless but changed by the remote CPU, only two non critical scenarios could happen: 1) The already update value is read -> everything is perfect 2) The old value is read -> a superfluous timer soft interrupt is raised The same situation could happen when enqueueing a new first pinned timer by a remote CPU also with non critical scenarios: 1) The already update value is read -> everything is perfect 2) The old value is read -> when the CPU is idle, an IPI is executed nevertheless and when the CPU isn't idle, the updated value will be visible on the next tick and the timer might be late one jiffie. As this is very unlikely to happen, the overhead of doing the check under the lock is a way more effort, than a superfluous timer soft interrupt or a possible 1 jiffie delay of the timer. Document and annotate this non critical behavior in the code by using READ/WRITE_ONCE() pair when accessing timer_base->next_expiry. Reported-by: syzbot+bf285fcc0a048e028118@syzkaller.appspotmail.com Signed-off-by: Anna-Maria Behnsen Signed-off-by: Thomas Gleixner Reviewed-by: Frederic Weisbecker Link: https://lore.kernel.org/all/20240829154305.19259-1-anna-maria@linutronix.de Closes: https://lore.kernel.org/lkml/000000000000916e55061f969e14@google.com --- kernel/time/timer.c | 42 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/kernel/time/timer.c b/kernel/time/timer.c index d8eb368a5a5b..311ea459b976 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -672,7 +672,7 @@ static void enqueue_timer(struct timer_base *base, struct timer_list *timer, * Set the next expiry time and kick the CPU so it * can reevaluate the wheel: */ - base->next_expiry = bucket_expiry; + WRITE_ONCE(base->next_expiry, bucket_expiry); base->timers_pending = true; base->next_expiry_recalc = false; trigger_dyntick_cpu(base, timer); @@ -1966,7 +1966,7 @@ static void next_expiry_recalc(struct timer_base *base) clk += adj; } - base->next_expiry = next; + WRITE_ONCE(base->next_expiry, next); base->next_expiry_recalc = false; base->timers_pending = !(next == base->clk + NEXT_TIMER_MAX_DELTA); } @@ -2020,7 +2020,7 @@ static unsigned long next_timer_interrupt(struct timer_base *base, * easy comparable to find out which base holds the first pending timer. */ if (!base->timers_pending) - base->next_expiry = basej + NEXT_TIMER_MAX_DELTA; + WRITE_ONCE(base->next_expiry, basej + NEXT_TIMER_MAX_DELTA); return base->next_expiry; } @@ -2464,8 +2464,40 @@ static void run_local_timers(void) hrtimer_run_queues(); for (int i = 0; i < NR_BASES; i++, base++) { - /* Raise the softirq only if required. */ - if (time_after_eq(jiffies, base->next_expiry) || + /* + * Raise the softirq only if required. + * + * timer_base::next_expiry can be written by a remote CPU while + * holding the lock. If this write happens at the same time than + * the lockless local read, sanity checker could complain about + * data corruption. + * + * There are two possible situations where + * timer_base::next_expiry is written by a remote CPU: + * + * 1. Remote CPU expires global timers of this CPU and updates + * timer_base::next_expiry of BASE_GLOBAL afterwards in + * next_timer_interrupt() or timer_recalc_next_expiry(). The + * worst outcome is a superfluous raise of the timer softirq + * when the not yet updated value is read. + * + * 2. A new first pinned timer is enqueued by a remote CPU + * and therefore timer_base::next_expiry of BASE_LOCAL is + * updated. When this update is missed, this isn't a + * problem, as an IPI is executed nevertheless when the CPU + * was idle before. When the CPU wasn't idle but the update + * is missed, then the timer would expire one jiffie late - + * bad luck. + * + * Those unlikely corner cases where the worst outcome is only a + * one jiffie delay or a superfluous raise of the softirq are + * not that expensive as doing the check always while holding + * the lock. + * + * Possible remote writers are using WRITE_ONCE(). Local reader + * uses therefore READ_ONCE(). + */ + if (time_after_eq(jiffies, READ_ONCE(base->next_expiry)) || (i == BASE_DEF && tmigr_requires_handle_remote())) { raise_softirq(TIMER_SOFTIRQ); return; From 8db70faeab9005cbab36e05ba94383075b5cb5db Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Mon, 26 Aug 2024 00:11:53 +0200 Subject: [PATCH 138/180] cpu: Fix W=1 build kernel-doc warning Building the kernel with W=1 generates the following warning: kernel/cpu.c:2693: warning: This comment starts with '/**', but isn't a kernel-doc comment. The function topology_is_core_online() is a simple helper function and doesn't need a kernel-doc comment. Use a normal comment instead. Signed-off-by: Thorsten Blum Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240825221152.71951-2-thorsten.blum@toblux.com --- kernel/cpu.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/kernel/cpu.c b/kernel/cpu.c index c16a9b66baba..0c9c5dfc8ddd 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -2705,9 +2705,7 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval) return ret; } -/** - * Check if the core a CPU belongs to is online - */ +/* Check if the core a CPU belongs to is online */ #if !defined(topology_is_core_online) static inline bool topology_is_core_online(unsigned int cpu) { From d33d5e683b0d3b4f5fc6a49ce17583f8ca663944 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Tue, 27 Aug 2024 16:25:31 +0206 Subject: [PATCH 139/180] printk: nbcon: Use raw_cpu_ptr() instead of open coding There is no need to open code a non-migration-checking this_cpu_ptr(). That is exactly what raw_cpu_ptr() is. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/87plpum4jw.fsf@jogness.linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/nbcon.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index 92ac5c590927..cf62f675c673 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -998,8 +998,7 @@ static __ref unsigned int *nbcon_get_cpu_emergency_nesting(void) if (!printk_percpu_data_ready()) return &early_nbcon_pcpu_emergency_nesting; - /* Open code this_cpu_ptr() without checking migration. */ - return per_cpu_ptr(&nbcon_pcpu_emergency_nesting, raw_smp_processor_id()); + return raw_cpu_ptr(&nbcon_pcpu_emergency_nesting); } /** From bd07d864522e7c3e4ee364e91aee8754992f5855 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 4 Sep 2024 14:11:20 +0206 Subject: [PATCH 140/180] printk: nbcon: Add function for printers to reacquire ownership Since ownership can be lost at any time due to handover or takeover, a printing context _must_ be prepared to back out immediately and carefully. However, there are scenarios where the printing context must reacquire ownership in order to finalize or revert hardware changes. One such example is when interrupts are disabled during printing. No other context will automagically re-enable the interrupts. For this case, the disabling context _must_ reacquire nbcon ownership so that it can re-enable the interrupts. Provide nbcon_reacquire_nobuf() for exactly this purpose. It allows a printing context to reacquire ownership using the same priority as its previous ownership. Note that after a successful reacquire the printing context will have no output buffer because that has been lost. This function cannot be used to resume printing. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240904120536.115780-2-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- include/linux/console.h | 6 ++++ kernel/printk/nbcon.c | 74 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 73 insertions(+), 7 deletions(-) diff --git a/include/linux/console.h b/include/linux/console.h index 9a13f91b0c43..88050d30a9cc 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -366,6 +366,10 @@ struct console { * * The callback should allow the takeover whenever it is safe. It * increases the chance to see messages when the system is in trouble. + * If the driver must reacquire ownership in order to finalize or + * revert hardware changes, nbcon_reacquire_nobuf() can be used. + * However, on reacquire the buffer content is no longer available. A + * reacquire cannot be used to resume printing. * * The callback can be called from any context (including NMI). * Therefore it must avoid usage of any locking and instead rely @@ -558,12 +562,14 @@ extern void nbcon_cpu_emergency_exit(void); extern bool nbcon_can_proceed(struct nbcon_write_context *wctxt); extern bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt); extern bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt); +extern void nbcon_reacquire_nobuf(struct nbcon_write_context *wctxt); #else static inline void nbcon_cpu_emergency_enter(void) { } static inline void nbcon_cpu_emergency_exit(void) { } static inline bool nbcon_can_proceed(struct nbcon_write_context *wctxt) { return false; } static inline bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt) { return false; } static inline bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt) { return false; } +static inline void nbcon_reacquire_nobuf(struct nbcon_write_context *wctxt) { } #endif extern int console_set_on_cmdline; diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index cf62f675c673..8a1bf6f94bce 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -830,6 +830,19 @@ out: return nbcon_context_can_proceed(ctxt, &cur); } +static void nbcon_write_context_set_buf(struct nbcon_write_context *wctxt, + char *buf, unsigned int len) +{ + struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt); + struct console *con = ctxt->console; + struct nbcon_state cur; + + wctxt->outbuf = buf; + wctxt->len = len; + nbcon_state_read(con, &cur); + wctxt->unsafe_takeover = cur.unsafe_takeover; +} + /** * nbcon_enter_unsafe - Enter an unsafe region in the driver * @wctxt: The write context that was handed to the write function @@ -845,8 +858,12 @@ out: bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt) { struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt); + bool is_owner; - return nbcon_context_enter_unsafe(ctxt); + is_owner = nbcon_context_enter_unsafe(ctxt); + if (!is_owner) + nbcon_write_context_set_buf(wctxt, NULL, 0); + return is_owner; } EXPORT_SYMBOL_GPL(nbcon_enter_unsafe); @@ -865,11 +882,43 @@ EXPORT_SYMBOL_GPL(nbcon_enter_unsafe); bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt) { struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt); + bool ret; - return nbcon_context_exit_unsafe(ctxt); + ret = nbcon_context_exit_unsafe(ctxt); + if (!ret) + nbcon_write_context_set_buf(wctxt, NULL, 0); + return ret; } EXPORT_SYMBOL_GPL(nbcon_exit_unsafe); +/** + * nbcon_reacquire_nobuf - Reacquire a console after losing ownership + * while printing + * @wctxt: The write context that was handed to the write callback + * + * Since ownership can be lost at any time due to handover or takeover, a + * printing context _must_ be prepared to back out immediately and + * carefully. However, there are scenarios where the printing context must + * reacquire ownership in order to finalize or revert hardware changes. + * + * This function allows a printing context to reacquire ownership using the + * same priority as its previous ownership. + * + * Note that after a successful reacquire the printing context will have no + * output buffer because that has been lost. This function cannot be used to + * resume printing. + */ +void nbcon_reacquire_nobuf(struct nbcon_write_context *wctxt) +{ + struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt); + + while (!nbcon_context_try_acquire(ctxt)) + cpu_relax(); + + nbcon_write_context_set_buf(wctxt, NULL, 0); +} +EXPORT_SYMBOL_GPL(nbcon_reacquire_nobuf); + /** * nbcon_emit_next_record - Emit a record in the acquired context * @wctxt: The write context that will be handed to the write function @@ -895,7 +944,6 @@ static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt) .pbufs = ctxt->pbufs, }; unsigned long con_dropped; - struct nbcon_state cur; unsigned long dropped; /* @@ -930,10 +978,7 @@ static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt) goto update_con; /* Initialize the write context for driver callbacks. */ - wctxt->outbuf = &pmsg.pbufs->outbuf[0]; - wctxt->len = pmsg.outbuf_len; - nbcon_state_read(con, &cur); - wctxt->unsafe_takeover = cur.unsafe_takeover; + nbcon_write_context_set_buf(wctxt, &pmsg.pbufs->outbuf[0], pmsg.outbuf_len); if (con->write_atomic) { con->write_atomic(con, wctxt); @@ -947,6 +992,21 @@ static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt) return false; } + if (!wctxt->outbuf) { + /* + * Ownership was lost and reacquired by the driver. Handle it + * as if ownership was lost. + */ + nbcon_context_release(ctxt); + return false; + } + + /* + * Ownership may have been lost but _not_ reacquired by the driver. + * This case is detected and handled when entering unsafe to update + * dropped/seq values. + */ + /* * Since any dropped message was successfully output, reset the * dropped count for the console. From e37577ebbf1e875f8cc6159f25a1b559018d087f Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 4 Sep 2024 14:11:21 +0206 Subject: [PATCH 141/180] printk: Fail pr_flush() if before SYSTEM_SCHEDULING A follow-up change adds pr_flush() to console unregistration. However, with boot consoles unregistration can happen very early if there are also regular consoles registering as well. In this case the pr_flush() is not important because all consoles are flushed when checking the initial console sequence number. Allow pr_flush() to fail if @system_state has not yet reached SYSTEM_SCHEDULING. This avoids might_sleep() and msleep() explosions that would otherwise occur: [ 0.436739][ T0] printk: legacy console [ttyS0] enabled [ 0.439820][ T0] printk: legacy bootconsole [earlyser0] disabled [ 0.446822][ T0] BUG: scheduling while atomic: swapper/0/0/0x00000002 [ 0.450491][ T0] 1 lock held by swapper/0/0: [ 0.457897][ T0] #0: ffffffff82ae5f88 (console_mutex){+.+.}-{4:4}, at: console_list_lock+0x20/0x70 [ 0.463141][ T0] Modules linked in: [ 0.465307][ T0] CPU: 0 PID: 0 Comm: swapper/0 Not tainted 6.10.0-rc1+ #372 [ 0.469394][ T0] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.14.0-2 04/01/2014 [ 0.474402][ T0] Call Trace: [ 0.476246][ T0] [ 0.481473][ T0] dump_stack_lvl+0x93/0xb0 [ 0.483949][ T0] dump_stack+0x10/0x20 [ 0.486256][ T0] __schedule_bug+0x68/0x90 [ 0.488753][ T0] __schedule+0xb9b/0xd80 [ 0.491179][ T0] ? lock_release+0xb5/0x270 [ 0.493732][ T0] schedule+0x43/0x170 [ 0.495998][ T0] schedule_timeout+0xc5/0x1e0 [ 0.498634][ T0] ? __pfx_process_timeout+0x10/0x10 [ 0.501522][ T0] ? msleep+0x13/0x50 [ 0.503728][ T0] msleep+0x3c/0x50 [ 0.505847][ T0] __pr_flush.constprop.0.isra.0+0x56/0x500 [ 0.509050][ T0] ? _printk+0x58/0x80 [ 0.511332][ T0] ? lock_is_held_type+0x9c/0x110 [ 0.514106][ T0] unregister_console_locked+0xe1/0x450 [ 0.517144][ T0] register_console+0x509/0x620 [ 0.519827][ T0] ? __pfx_univ8250_console_init+0x10/0x10 [ 0.523042][ T0] univ8250_console_init+0x24/0x40 [ 0.525845][ T0] console_init+0x43/0x210 [ 0.528280][ T0] start_kernel+0x493/0x980 [ 0.530773][ T0] x86_64_start_reservations+0x18/0x30 [ 0.533755][ T0] x86_64_start_kernel+0xae/0xc0 [ 0.536473][ T0] common_startup_64+0x12c/0x138 [ 0.539210][ T0] And then the kernel goes into an infinite loop complaining about: 1. releasing a pinned lock 2. unpinning an unpinned lock 3. bad: scheduling from the idle thread! 4. goto 1 Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240904120536.115780-3-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/printk.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 6accd1704e73..acf668001096 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -3991,6 +3991,10 @@ static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre u64 diff; u64 seq; + /* Sorry, pr_flush() will not work this early. */ + if (system_state < SYSTEM_SCHEDULING) + return false; + might_sleep(); seq = prb_next_reserve_seq(prb); From 0e53e2d9f72080b7ea1a4003558041fee78cdef9 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 4 Sep 2024 14:11:22 +0206 Subject: [PATCH 142/180] printk: Flush console on unregister_console() Ensure consoles have flushed pending records before unregistering. The console should print up to at least its related "console disabled" record. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240904120536.115780-4-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/printk.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index acf668001096..c79e962b822a 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -3771,11 +3771,16 @@ static int unregister_console_locked(struct console *console) if (res > 0) return 0; + if (!console_is_registered_locked(console)) + res = -ENODEV; + else if (console_is_usable(console, console->flags)) + __pr_flush(console, 1000, true); + /* Disable it unconditionally */ console_srcu_write_flags(console, console->flags & ~CON_ENABLED); - if (!console_is_registered_locked(console)) - return -ENODEV; + if (res < 0) + return res; /* * Use the driver synchronization to ensure that the hardware is not From 6cb58cfebb293286f5ce8f6fba3e29122308e9e0 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 4 Sep 2024 14:11:23 +0206 Subject: [PATCH 143/180] printk: nbcon: Add context to usable() and emit() The nbcon consoles will have two callbacks to be used for different contexts. In order to determine if an nbcon console is usable, console_is_usable() must know if it is a context that will need to use the optional write_atomic() callback. Also, nbcon_emit_next_record() must know which callback it needs to call. Add an extra parameter @use_atomic to console_is_usable() and nbcon_emit_next_record() to specify this. Since so far only the write_atomic() callback exists, @use_atomic is set to true for all call sites. For legacy consoles, @use_atomic is not used. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240904120536.115780-5-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/internal.h | 8 +++++--- kernel/printk/nbcon.c | 34 +++++++++++++++++++--------------- kernel/printk/printk.c | 6 +++--- 3 files changed, 27 insertions(+), 21 deletions(-) diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index 8e36d8695f81..ad631824e7e5 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -97,7 +97,7 @@ bool nbcon_legacy_emit_next_record(struct console *con, bool *handover, * which can also play a role in deciding if @con can be used to print * records. */ -static inline bool console_is_usable(struct console *con, short flags) +static inline bool console_is_usable(struct console *con, short flags, bool use_atomic) { if (!(flags & CON_ENABLED)) return false; @@ -106,7 +106,8 @@ static inline bool console_is_usable(struct console *con, short flags) return false; if (flags & CON_NBCON) { - if (!con->write_atomic) + /* The write_atomic() callback is optional. */ + if (use_atomic && !con->write_atomic) return false; } else { if (!con->write) @@ -149,7 +150,8 @@ static inline void nbcon_atomic_flush_pending(void) { } static inline bool nbcon_legacy_emit_next_record(struct console *con, bool *handover, int cookie) { return false; } -static inline bool console_is_usable(struct console *con, short flags) { return false; } +static inline bool console_is_usable(struct console *con, short flags, + bool use_atomic) { return false; } #endif /* CONFIG_PRINTK */ diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index 8a1bf6f94bce..88db24f9a8de 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -922,6 +922,7 @@ EXPORT_SYMBOL_GPL(nbcon_reacquire_nobuf); /** * nbcon_emit_next_record - Emit a record in the acquired context * @wctxt: The write context that will be handed to the write function + * @use_atomic: True if the write_atomic() callback is to be used * * Return: True if this context still owns the console. False if * ownership was handed over or taken. @@ -935,7 +936,7 @@ EXPORT_SYMBOL_GPL(nbcon_reacquire_nobuf); * When true is returned, @wctxt->ctxt.backlog indicates whether there are * still records pending in the ringbuffer, */ -static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt) +static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt, bool use_atomic) { struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt); struct console *con = ctxt->console; @@ -946,6 +947,18 @@ static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt) unsigned long con_dropped; unsigned long dropped; + /* + * This function should never be called for consoles that have not + * implemented the necessary callback for writing: i.e. legacy + * consoles and, when atomic, nbcon consoles with no write_atomic(). + * Handle it as if ownership was lost and try to continue. + */ + if (WARN_ON_ONCE((use_atomic && !con->write_atomic) || + !(console_srcu_read_flags(con) & CON_NBCON))) { + nbcon_context_release(ctxt); + return false; + } + /* * The printk buffers are filled within an unsafe section. This * prevents NBCON_PRIO_NORMAL and NBCON_PRIO_EMERGENCY from @@ -980,17 +993,8 @@ static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt) /* Initialize the write context for driver callbacks. */ nbcon_write_context_set_buf(wctxt, &pmsg.pbufs->outbuf[0], pmsg.outbuf_len); - if (con->write_atomic) { + if (use_atomic) con->write_atomic(con, wctxt); - } else { - /* - * This function should never be called for legacy consoles. - * Handle it as if ownership was lost and try to continue. - */ - WARN_ON_ONCE(1); - nbcon_context_release(ctxt); - return false; - } if (!wctxt->outbuf) { /* @@ -1118,7 +1122,7 @@ static bool nbcon_atomic_emit_one(struct nbcon_write_context *wctxt) * The higher priority printing context takes over responsibility * to print the pending records. */ - if (!nbcon_emit_next_record(wctxt)) + if (!nbcon_emit_next_record(wctxt, true)) return false; nbcon_context_release(ctxt); @@ -1219,7 +1223,7 @@ static int __nbcon_atomic_flush_pending_con(struct console *con, u64 stop_seq, * handed over or taken over. In both cases the context is no * longer valid. */ - if (!nbcon_emit_next_record(&wctxt)) + if (!nbcon_emit_next_record(&wctxt, true)) return -EAGAIN; if (!ctxt->backlog) { @@ -1305,7 +1309,7 @@ static void __nbcon_atomic_flush_pending(u64 stop_seq, bool allow_unsafe_takeove if (!(flags & CON_NBCON)) continue; - if (!console_is_usable(con, flags)) + if (!console_is_usable(con, flags, true)) continue; if (nbcon_seq_read(con) >= stop_seq) @@ -1490,7 +1494,7 @@ void nbcon_device_release(struct console *con) * the console is usable throughout flushing. */ cookie = console_srcu_read_lock(); - if (console_is_usable(con, console_srcu_read_flags(con)) && + if (console_is_usable(con, console_srcu_read_flags(con), true) && prb_read_valid(prb, nbcon_seq_read(con), NULL)) { /* * If nbcon_atomic flushing is not available, fallback to diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index c79e962b822a..846306ac2663 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -3071,7 +3071,7 @@ static bool console_flush_all(bool do_cond_resched, u64 *next_seq, bool *handove u64 printk_seq; bool progress; - if (!console_is_usable(con, flags)) + if (!console_is_usable(con, flags, true)) continue; any_usable = true; @@ -3773,7 +3773,7 @@ static int unregister_console_locked(struct console *console) if (!console_is_registered_locked(console)) res = -ENODEV; - else if (console_is_usable(console, console->flags)) + else if (console_is_usable(console, console->flags, true)) __pr_flush(console, 1000, true); /* Disable it unconditionally */ @@ -4043,7 +4043,7 @@ static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre * that they make forward progress, so only increment * @diff for usable consoles. */ - if (!console_is_usable(c, flags)) + if (!console_is_usable(c, flags, true)) continue; if (flags & CON_NBCON) { From fb9fabf3d86216961d1103ecbc33e98d5f6c48f5 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 4 Sep 2024 14:11:24 +0206 Subject: [PATCH 144/180] printk: nbcon: Init @nbcon_seq to highest possible When initializing an nbcon console, have nbcon_alloc() set @nbcon_seq to the highest possible sequence number. For all practical purposes, this will guarantee that the console will have nothing to print until later when @nbcon_seq is set to the proper initial printing value. This will be particularly important once kthread printing is introduced because nbcon_alloc() can create/start the kthread before the desired initial sequence number is known. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240904120536.115780-6-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/nbcon.c | 8 +++++++- kernel/printk/printk_ringbuffer.h | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index 88db24f9a8de..bc684ff5028a 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -1397,7 +1397,13 @@ bool nbcon_alloc(struct console *con) struct nbcon_state state = { }; nbcon_state_set(con, &state); - atomic_long_set(&ACCESS_PRIVATE(con, nbcon_seq), 0); + + /* + * Initialize @nbcon_seq to the highest possible sequence number so + * that practically speaking it will have nothing to print until a + * desired initial sequence number has been set via nbcon_seq_force(). + */ + atomic_long_set(&ACCESS_PRIVATE(con, nbcon_seq), ULSEQ_MAX(prb)); if (con->flags & CON_BOOT) { /* diff --git a/kernel/printk/printk_ringbuffer.h b/kernel/printk/printk_ringbuffer.h index 8de6c495cf2b..4ef81349d9fb 100644 --- a/kernel/printk/printk_ringbuffer.h +++ b/kernel/printk/printk_ringbuffer.h @@ -404,10 +404,12 @@ u64 prb_next_reserve_seq(struct printk_ringbuffer *rb); #define __u64seq_to_ulseq(u64seq) (u64seq) #define __ulseq_to_u64seq(rb, ulseq) (ulseq) +#define ULSEQ_MAX(rb) (-1) #else /* CONFIG_64BIT */ #define __u64seq_to_ulseq(u64seq) ((u32)u64seq) +#define ULSEQ_MAX(rb) __u64seq_to_ulseq(prb_first_seq(rb) + 0x80000000UL) static inline u64 __ulseq_to_u64seq(struct printk_ringbuffer *rb, u32 ulseq) { From 76f258bf3f2aae570209319944703a92ac64e29e Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 4 Sep 2024 14:11:25 +0206 Subject: [PATCH 145/180] printk: nbcon: Introduce printer kthreads Provide the main implementation for running a printer kthread per nbcon console that is takeover/handover aware. This includes: - new mandatory write_thread() callback - kthread creation - kthread main printing loop - kthread wakeup mechanism - kthread shutdown kthread creation is a bit tricky because consoles may register before kthreads can be created. In such cases, registration will succeed, even though no kthread exists. Once kthreads can be created, an early_initcall will set @printk_kthreads_ready. If there are no registered boot consoles, the early_initcall creates the kthreads for all registered nbcon consoles. If kthread creation fails, the related console is unregistered. If there are registered boot consoles when @printk_kthreads_ready is set, no kthreads are created until the final boot console unregisters. Once kthread creation finally occurs, @printk_kthreads_running is set so that the system knows kthreads are available for all registered nbcon consoles. If @printk_kthreads_running is already set when the console is registering, the kthread is created during registration. If kthread creation fails, the registration will fail. Until @printk_kthreads_running is set, console printing occurs directly via the console_lock. kthread shutdown on system shutdown/reboot is necessary to ensure the printer kthreads finish their printing so that the system can cleanly transition back to direct printing via the console_lock in order to reliably push out the final shutdown/reboot messages. @printk_kthreads_running is cleared before shutting down the individual kthreads. The kthread uses a new mandatory write_thread() callback that is called with both device_lock() and the console context acquired. The console ownership handling is necessary for synchronization against write_atomic() which is synchronized only via the console context ownership. The device_lock() serializes acquiring the console context with NBCON_PRIO_NORMAL. It is needed in case the device_lock() does not disable preemption. It prevents the following race: CPU0 CPU1 [ task A ] nbcon_context_try_acquire() # success with NORMAL prio # .unsafe == false; // safe for takeover [ schedule: task A -> B ] WARN_ON() nbcon_atomic_flush_pending() nbcon_context_try_acquire() # success with EMERGENCY prio # flushing nbcon_context_release() # HERE: con->nbcon_state is free # to take by anyone !!! nbcon_context_try_acquire() # success with NORMAL prio [ task B ] [ schedule: task B -> A ] nbcon_enter_unsafe() nbcon_context_can_proceed() BUG: nbcon_context_can_proceed() returns "true" because the console is owned by a context on CPU0 with NBCON_PRIO_NORMAL. But it should return "false". The console is owned by a context from task B and we do the check in a context from task A. Note that with these changes, the printer kthreads do not yet take over full responsibility for nbcon printing during normal operation. These changes only focus on the lifecycle of the kthreads. Co-developed-by: John Ogness Signed-off-by: John Ogness Signed-off-by: Thomas Gleixner (Intel) Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240904120536.115780-7-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- include/linux/console.h | 40 +++++++ kernel/printk/internal.h | 27 +++++ kernel/printk/nbcon.c | 245 +++++++++++++++++++++++++++++++++++++++ kernel/printk/printk.c | 108 +++++++++++++++++ 4 files changed, 420 insertions(+) diff --git a/include/linux/console.h b/include/linux/console.h index 88050d30a9cc..788ce9c829f6 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -16,7 +16,9 @@ #include #include +#include #include +#include #include #include @@ -324,6 +326,9 @@ struct nbcon_write_context { * @nbcon_seq: Sequence number of the next record for nbcon to print * @nbcon_device_ctxt: Context available for non-printing operations * @pbufs: Pointer to nbcon private buffer + * @kthread: Printer kthread for this console + * @rcuwait: RCU-safe wait object for @kthread waking + * @irq_work: Defer @kthread waking to IRQ work context */ struct console { char name[16]; @@ -377,6 +382,37 @@ struct console { */ void (*write_atomic)(struct console *con, struct nbcon_write_context *wctxt); + /** + * @write_thread: + * + * NBCON callback to write out text in task context. + * + * This callback must be called only in task context with both + * device_lock() and the nbcon console acquired with + * NBCON_PRIO_NORMAL. + * + * The same rules for console ownership verification and unsafe + * sections handling applies as with write_atomic(). + * + * The console ownership handling is necessary for synchronization + * against write_atomic() which is synchronized only via the context. + * + * The device_lock() provides the primary serialization for operations + * on the device. It might be as relaxed (mutex)[*] or as tight + * (disabled preemption and interrupts) as needed. It allows + * the kthread to operate in the least restrictive mode[**]. + * + * [*] Standalone nbcon_context_try_acquire() is not safe with + * the preemption enabled, see nbcon_owner_matches(). But it + * can be safe when always called in the preemptive context + * under the device_lock(). + * + * [**] The device_lock() makes sure that nbcon_context_try_acquire() + * would never need to spin which is important especially with + * PREEMPT_RT. + */ + void (*write_thread)(struct console *con, struct nbcon_write_context *wctxt); + /** * @device_lock: * @@ -423,7 +459,11 @@ struct console { atomic_t __private nbcon_state; atomic_long_t __private nbcon_seq; struct nbcon_context __private nbcon_device_ctxt; + struct printk_buffers *pbufs; + struct task_struct *kthread; + struct rcuwait rcuwait; + struct irq_work irq_work; }; #ifdef CONFIG_LOCKDEP diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index ad631824e7e5..14f7fc71e20d 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -48,6 +48,7 @@ struct printk_ringbuffer; struct dev_printk_info; extern struct printk_ringbuffer *prb; +extern bool printk_kthreads_running; __printf(4, 0) int vprintk_store(int facility, int level, @@ -90,6 +91,9 @@ enum nbcon_prio nbcon_get_default_prio(void); void nbcon_atomic_flush_pending(void); bool nbcon_legacy_emit_next_record(struct console *con, bool *handover, int cookie); +bool nbcon_kthread_create(struct console *con); +void nbcon_kthread_stop(struct console *con); +void nbcon_kthreads_wake(void); /* * Check if the given console is currently capable and allowed to print @@ -125,12 +129,34 @@ static inline bool console_is_usable(struct console *con, short flags, bool use_ return true; } +/** + * nbcon_kthread_wake - Wake up a console printing thread + * @con: Console to operate on + */ +static inline void nbcon_kthread_wake(struct console *con) +{ + /* + * Guarantee any new records can be seen by tasks preparing to wait + * before this context checks if the rcuwait is empty. + * + * The full memory barrier in rcuwait_wake_up() pairs with the full + * memory barrier within set_current_state() of + * ___rcuwait_wait_event(), which is called after prepare_to_rcuwait() + * adds the waiter but before it has checked the wait condition. + * + * This pairs with nbcon_kthread_func:A. + */ + rcuwait_wake_up(&con->rcuwait); /* LMM(nbcon_kthread_wake:A) */ +} + #else #define PRINTK_PREFIX_MAX 0 #define PRINTK_MESSAGE_MAX 0 #define PRINTKRB_RECORD_MAX 0 +#define printk_kthreads_running (false) + /* * In !PRINTK builds we still export console_sem * semaphore and some of console functions (console_unlock()/etc.), so @@ -149,6 +175,7 @@ static inline enum nbcon_prio nbcon_get_default_prio(void) { return NBCON_PRIO_N static inline void nbcon_atomic_flush_pending(void) { } static inline bool nbcon_legacy_emit_next_record(struct console *con, bool *handover, int cookie) { return false; } +static inline void nbcon_kthread_wake(struct console *con) { } static inline bool console_is_usable(struct console *con, short flags, bool use_atomic) { return false; } diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index bc684ff5028a..388322c74349 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -952,6 +953,9 @@ static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt, bool use_a * implemented the necessary callback for writing: i.e. legacy * consoles and, when atomic, nbcon consoles with no write_atomic(). * Handle it as if ownership was lost and try to continue. + * + * Note that for nbcon consoles the write_thread() callback is + * mandatory and was already checked in nbcon_alloc(). */ if (WARN_ON_ONCE((use_atomic && !con->write_atomic) || !(console_srcu_read_flags(con) & CON_NBCON))) { @@ -995,6 +999,8 @@ static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt, bool use_a if (use_atomic) con->write_atomic(con, wctxt); + else + con->write_thread(con, wctxt); if (!wctxt->outbuf) { /* @@ -1036,6 +1042,228 @@ update_con: return nbcon_context_exit_unsafe(ctxt); } +/** + * nbcon_kthread_should_wakeup - Check whether a printer thread should wakeup + * @con: Console to operate on + * @ctxt: The nbcon context from nbcon_context_try_acquire() + * + * Return: True if the thread should shutdown or if the console is + * allowed to print and a record is available. False otherwise. + * + * After the thread wakes up, it must first check if it should shutdown before + * attempting any printing. + */ +static bool nbcon_kthread_should_wakeup(struct console *con, struct nbcon_context *ctxt) +{ + bool ret = false; + short flags; + int cookie; + + if (kthread_should_stop()) + return true; + + cookie = console_srcu_read_lock(); + + flags = console_srcu_read_flags(con); + if (console_is_usable(con, flags, false)) { + /* Bring the sequence in @ctxt up to date */ + ctxt->seq = nbcon_seq_read(con); + + ret = prb_read_valid(prb, ctxt->seq, NULL); + } + + console_srcu_read_unlock(cookie); + return ret; +} + +/** + * nbcon_kthread_func - The printer thread function + * @__console: Console to operate on + * + * Return: 0 + */ +static int nbcon_kthread_func(void *__console) +{ + struct console *con = __console; + struct nbcon_write_context wctxt = { + .ctxt.console = con, + .ctxt.prio = NBCON_PRIO_NORMAL, + }; + struct nbcon_context *ctxt = &ACCESS_PRIVATE(&wctxt, ctxt); + short con_flags; + bool backlog; + int cookie; + +wait_for_event: + /* + * Guarantee this task is visible on the rcuwait before + * checking the wake condition. + * + * The full memory barrier within set_current_state() of + * ___rcuwait_wait_event() pairs with the full memory + * barrier within rcuwait_has_sleeper(). + * + * This pairs with rcuwait_has_sleeper:A and nbcon_kthread_wake:A. + */ + rcuwait_wait_event(&con->rcuwait, + nbcon_kthread_should_wakeup(con, ctxt), + TASK_INTERRUPTIBLE); /* LMM(nbcon_kthread_func:A) */ + + do { + if (kthread_should_stop()) + return 0; + + backlog = false; + + /* + * Keep the srcu read lock around the entire operation so that + * synchronize_srcu() can guarantee that the kthread stopped + * or suspended printing. + */ + cookie = console_srcu_read_lock(); + + con_flags = console_srcu_read_flags(con); + + if (console_is_usable(con, con_flags, false)) { + unsigned long lock_flags; + + con->device_lock(con, &lock_flags); + + /* + * Ensure this stays on the CPU to make handover and + * takeover possible. + */ + cant_migrate(); + + if (nbcon_context_try_acquire(ctxt)) { + /* + * If the emit fails, this context is no + * longer the owner. + */ + if (nbcon_emit_next_record(&wctxt, false)) { + nbcon_context_release(ctxt); + backlog = ctxt->backlog; + } + } + + con->device_unlock(con, lock_flags); + } + + console_srcu_read_unlock(cookie); + + cond_resched(); + + } while (backlog); + + goto wait_for_event; +} + +/** + * nbcon_irq_work - irq work to wake console printer thread + * @irq_work: The irq work to operate on + */ +static void nbcon_irq_work(struct irq_work *irq_work) +{ + struct console *con = container_of(irq_work, struct console, irq_work); + + nbcon_kthread_wake(con); +} + +static inline bool rcuwait_has_sleeper(struct rcuwait *w) +{ + /* + * Guarantee any new records can be seen by tasks preparing to wait + * before this context checks if the rcuwait is empty. + * + * This full memory barrier pairs with the full memory barrier within + * set_current_state() of ___rcuwait_wait_event(), which is called + * after prepare_to_rcuwait() adds the waiter but before it has + * checked the wait condition. + * + * This pairs with nbcon_kthread_func:A. + */ + smp_mb(); /* LMM(rcuwait_has_sleeper:A) */ + return rcuwait_active(w); +} + +/** + * nbcon_kthreads_wake - Wake up printing threads using irq_work + */ +void nbcon_kthreads_wake(void) +{ + struct console *con; + int cookie; + + if (!printk_kthreads_running) + return; + + cookie = console_srcu_read_lock(); + for_each_console_srcu(con) { + if (!(console_srcu_read_flags(con) & CON_NBCON)) + continue; + + /* + * Only schedule irq_work if the printing thread is + * actively waiting. If not waiting, the thread will + * notice by itself that it has work to do. + */ + if (rcuwait_has_sleeper(&con->rcuwait)) + irq_work_queue(&con->irq_work); + } + console_srcu_read_unlock(cookie); +} + +/* + * nbcon_kthread_stop - Stop a console printer thread + * @con: Console to operate on + */ +void nbcon_kthread_stop(struct console *con) +{ + lockdep_assert_console_list_lock_held(); + + if (!con->kthread) + return; + + kthread_stop(con->kthread); + con->kthread = NULL; +} + +/** + * nbcon_kthread_create - Create a console printer thread + * @con: Console to operate on + * + * Return: True if the kthread was started or already exists. + * Otherwise false and @con must not be registered. + * + * This function is called when it will be expected that nbcon consoles are + * flushed using the kthread. The messages printed with NBCON_PRIO_NORMAL + * will be no longer flushed by the legacy loop. This is why failure must + * be fatal for console registration. + * + * If @con was already registered and this function fails, @con must be + * unregistered before the global state variable @printk_kthreads_running + * can be set. + */ +bool nbcon_kthread_create(struct console *con) +{ + struct task_struct *kt; + + lockdep_assert_console_list_lock_held(); + + if (con->kthread) + return true; + + kt = kthread_run(nbcon_kthread_func, con, "pr/%s%d", con->name, con->index); + if (WARN_ON(IS_ERR(kt))) { + con_printk(KERN_ERR, con, "failed to start printing thread\n"); + return false; + } + + con->kthread = kt; + + return true; +} + /* Track the nbcon emergency nesting per CPU. */ static DEFINE_PER_CPU(unsigned int, nbcon_pcpu_emergency_nesting); static unsigned int early_nbcon_pcpu_emergency_nesting __initdata; @@ -1396,6 +1624,12 @@ bool nbcon_alloc(struct console *con) { struct nbcon_state state = { }; + /* The write_thread() callback is mandatory. */ + if (WARN_ON(!con->write_thread)) + return false; + + rcuwait_init(&con->rcuwait); + init_irq_work(&con->irq_work, nbcon_irq_work); nbcon_state_set(con, &state); /* @@ -1418,6 +1652,14 @@ bool nbcon_alloc(struct console *con) con_printk(KERN_ERR, con, "failed to allocate printing buffer\n"); return false; } + + if (printk_kthreads_running) { + if (!nbcon_kthread_create(con)) { + kfree(con->pbufs); + con->pbufs = NULL; + return false; + } + } } return true; @@ -1431,6 +1673,9 @@ void nbcon_free(struct console *con) { struct nbcon_state state = { }; + if (printk_kthreads_running) + nbcon_kthread_stop(con); + nbcon_state_set(con, &state); /* Boot consoles share global printk buffers. */ diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 846306ac2663..f27c76c3b5cf 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -496,6 +497,9 @@ static u64 syslog_seq; static size_t syslog_partial; static bool syslog_time; +/* True when _all_ printer threads are available for printing. */ +bool printk_kthreads_running; + struct latched_seq { seqcount_latch_t latch; u64 val[2]; @@ -3027,6 +3031,8 @@ static bool console_emit_next_record(struct console *con, bool *handover, int co return false; } +static inline void printk_kthreads_check_locked(void) { } + #endif /* CONFIG_PRINTK */ /* @@ -3387,6 +3393,102 @@ void console_start(struct console *console) } EXPORT_SYMBOL(console_start); +#ifdef CONFIG_PRINTK +static int unregister_console_locked(struct console *console); + +/* True when system boot is far enough to create printer threads. */ +static bool printk_kthreads_ready __ro_after_init; + +/** + * printk_kthreads_shutdown - shutdown all threaded printers + * + * On system shutdown all threaded printers are stopped. This allows printk + * to transition back to atomic printing, thus providing a robust mechanism + * for the final shutdown/reboot messages to be output. + */ +static void printk_kthreads_shutdown(void) +{ + struct console *con; + + console_list_lock(); + if (printk_kthreads_running) { + printk_kthreads_running = false; + + for_each_console(con) { + if (con->flags & CON_NBCON) + nbcon_kthread_stop(con); + } + + /* + * The threads may have been stopped while printing a + * backlog. Flush any records left over. + */ + nbcon_atomic_flush_pending(); + } + console_list_unlock(); +} + +static struct syscore_ops printk_syscore_ops = { + .shutdown = printk_kthreads_shutdown, +}; + +/* + * If appropriate, start nbcon kthreads and set @printk_kthreads_running. + * If any kthreads fail to start, those consoles are unregistered. + * + * Must be called under console_list_lock(). + */ +static void printk_kthreads_check_locked(void) +{ + struct hlist_node *tmp; + struct console *con; + + lockdep_assert_console_list_lock_held(); + + if (!printk_kthreads_ready) + return; + + /* + * Printer threads cannot be started as long as any boot console is + * registered because there is no way to synchronize the hardware + * registers between boot console code and regular console code. + * It can only be known that there will be no new boot consoles when + * an nbcon console is registered. + */ + if (have_boot_console || !have_nbcon_console) { + /* Clear flag in case all nbcon consoles unregistered. */ + printk_kthreads_running = false; + return; + } + + if (printk_kthreads_running) + return; + + hlist_for_each_entry_safe(con, tmp, &console_list, node) { + if (!(con->flags & CON_NBCON)) + continue; + + if (!nbcon_kthread_create(con)) + unregister_console_locked(con); + } + + printk_kthreads_running = true; +} + +static int __init printk_set_kthreads_ready(void) +{ + register_syscore_ops(&printk_syscore_ops); + + console_list_lock(); + printk_kthreads_ready = true; + printk_kthreads_check_locked(); + console_list_unlock(); + + return 0; +} +early_initcall(printk_set_kthreads_ready); +#endif /* CONFIG_PRINTK */ + static int __read_mostly keep_bootcon; static int __init keep_bootcon_setup(char *str) @@ -3745,6 +3847,9 @@ void register_console(struct console *newcon) unregister_console_locked(con); } } + + /* Changed console list, may require printer threads to start/stop. */ + printk_kthreads_check_locked(); unlock: console_list_unlock(); } @@ -3841,6 +3946,9 @@ static int unregister_console_locked(struct console *console) if (!found_nbcon_con) have_nbcon_console = found_nbcon_con; + /* Changed console list, may require printer threads to start/stop. */ + printk_kthreads_check_locked(); + return res; } From 9b79a3d0d64abae92457cb830baa6c6a717778b0 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 4 Sep 2024 14:11:26 +0206 Subject: [PATCH 146/180] printk: nbcon: Relocate nbcon_atomic_emit_one() Move nbcon_atomic_emit_one() so that it can be used by nbcon_kthread_func() in a follow-up commit. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240904120536.115780-8-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/nbcon.c | 78 +++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index 388322c74349..57a0e9b542fe 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -1042,6 +1042,45 @@ update_con: return nbcon_context_exit_unsafe(ctxt); } +/* + * nbcon_atomic_emit_one - Print one record for an nbcon console using the + * write_atomic() callback + * @wctxt: An initialized write context struct to use for this context + * + * Return: True, when a record has been printed and there are still + * pending records. The caller might want to continue flushing. + * + * False, when there is no pending record, or when the console + * context cannot be acquired, or the ownership has been lost. + * The caller should give up. Either the job is done, cannot be + * done, or will be handled by the owning context. + * + * This is an internal helper to handle the locking of the console before + * calling nbcon_emit_next_record(). + */ +static bool nbcon_atomic_emit_one(struct nbcon_write_context *wctxt) +{ + struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt); + + if (!nbcon_context_try_acquire(ctxt)) + return false; + + /* + * nbcon_emit_next_record() returns false when the console was + * handed over or taken over. In both cases the context is no + * longer valid. + * + * The higher priority printing context takes over responsibility + * to print the pending records. + */ + if (!nbcon_emit_next_record(wctxt, true)) + return false; + + nbcon_context_release(ctxt); + + return ctxt->backlog; +} + /** * nbcon_kthread_should_wakeup - Check whether a printer thread should wakeup * @con: Console to operate on @@ -1319,45 +1358,6 @@ enum nbcon_prio nbcon_get_default_prio(void) return NBCON_PRIO_NORMAL; } -/* - * nbcon_atomic_emit_one - Print one record for an nbcon console using the - * write_atomic() callback - * @wctxt: An initialized write context struct to use for this context - * - * Return: True, when a record has been printed and there are still - * pending records. The caller might want to continue flushing. - * - * False, when there is no pending record, or when the console - * context cannot be acquired, or the ownership has been lost. - * The caller should give up. Either the job is done, cannot be - * done, or will be handled by the owning context. - * - * This is an internal helper to handle the locking of the console before - * calling nbcon_emit_next_record(). - */ -static bool nbcon_atomic_emit_one(struct nbcon_write_context *wctxt) -{ - struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt); - - if (!nbcon_context_try_acquire(ctxt)) - return false; - - /* - * nbcon_emit_next_record() returns false when the console was - * handed over or taken over. In both cases the context is no - * longer valid. - * - * The higher priority printing context takes over responsibility - * to print the pending records. - */ - if (!nbcon_emit_next_record(wctxt, true)) - return false; - - nbcon_context_release(ctxt); - - return ctxt->backlog; -} - /** * nbcon_legacy_emit_next_record - Print one record for an nbcon console * in legacy contexts From 5c586baa60e46d9fccd04f04e16f8a50b2fbc1af Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 4 Sep 2024 14:11:27 +0206 Subject: [PATCH 147/180] printk: nbcon: Use thread callback if in task context for legacy When printing via console_lock, the write_atomic() callback is used for nbcon consoles. However, if it is known that the current context is a task context, the write_thread() callback can be used instead. Using write_thread() instead of write_atomic() helps to reduce large disabled preemption regions when the device_lock does not disable preemption. This is mainly a preparatory change to allow avoiding write_atomic() completely during normal operation if boot consoles are registered. As a side-effect, it also allows consolidating the printing code for legacy printing and the kthread printer. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240904120536.115780-9-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/internal.h | 4 +- kernel/printk/nbcon.c | 95 +++++++++++++++++++++++----------------- kernel/printk/printk.c | 5 ++- 3 files changed, 59 insertions(+), 45 deletions(-) diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index 14f7fc71e20d..a96d4114a1db 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -90,7 +90,7 @@ void nbcon_free(struct console *con); enum nbcon_prio nbcon_get_default_prio(void); void nbcon_atomic_flush_pending(void); bool nbcon_legacy_emit_next_record(struct console *con, bool *handover, - int cookie); + int cookie, bool use_atomic); bool nbcon_kthread_create(struct console *con); void nbcon_kthread_stop(struct console *con); void nbcon_kthreads_wake(void); @@ -174,7 +174,7 @@ static inline void nbcon_free(struct console *con) { } static inline enum nbcon_prio nbcon_get_default_prio(void) { return NBCON_PRIO_NONE; } static inline void nbcon_atomic_flush_pending(void) { } static inline bool nbcon_legacy_emit_next_record(struct console *con, bool *handover, - int cookie) { return false; } + int cookie, bool use_atomic) { return false; } static inline void nbcon_kthread_wake(struct console *con) { } static inline bool console_is_usable(struct console *con, short flags, diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index 57a0e9b542fe..784e5de88abf 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -1043,9 +1043,10 @@ update_con: } /* - * nbcon_atomic_emit_one - Print one record for an nbcon console using the - * write_atomic() callback + * nbcon_emit_one - Print one record for an nbcon console using the + * specified callback * @wctxt: An initialized write context struct to use for this context + * @use_atomic: True if the write_atomic() callback is to be used * * Return: True, when a record has been printed and there are still * pending records. The caller might want to continue flushing. @@ -1058,12 +1059,25 @@ update_con: * This is an internal helper to handle the locking of the console before * calling nbcon_emit_next_record(). */ -static bool nbcon_atomic_emit_one(struct nbcon_write_context *wctxt) +static bool nbcon_emit_one(struct nbcon_write_context *wctxt, bool use_atomic) { struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt); + struct console *con = ctxt->console; + unsigned long flags; + bool ret = false; + + if (!use_atomic) { + con->device_lock(con, &flags); + + /* + * Ensure this stays on the CPU to make handover and + * takeover possible. + */ + cant_migrate(); + } if (!nbcon_context_try_acquire(ctxt)) - return false; + goto out; /* * nbcon_emit_next_record() returns false when the console was @@ -1073,12 +1087,16 @@ static bool nbcon_atomic_emit_one(struct nbcon_write_context *wctxt) * The higher priority printing context takes over responsibility * to print the pending records. */ - if (!nbcon_emit_next_record(wctxt, true)) - return false; + if (!nbcon_emit_next_record(wctxt, use_atomic)) + goto out; nbcon_context_release(ctxt); - return ctxt->backlog; + ret = ctxt->backlog; +out: + if (!use_atomic) + con->device_unlock(con, flags); + return ret; } /** @@ -1163,30 +1181,8 @@ wait_for_event: con_flags = console_srcu_read_flags(con); - if (console_is_usable(con, con_flags, false)) { - unsigned long lock_flags; - - con->device_lock(con, &lock_flags); - - /* - * Ensure this stays on the CPU to make handover and - * takeover possible. - */ - cant_migrate(); - - if (nbcon_context_try_acquire(ctxt)) { - /* - * If the emit fails, this context is no - * longer the owner. - */ - if (nbcon_emit_next_record(&wctxt, false)) { - nbcon_context_release(ctxt); - backlog = ctxt->backlog; - } - } - - con->device_unlock(con, lock_flags); - } + if (console_is_usable(con, con_flags, false)) + backlog = nbcon_emit_one(&wctxt, false); console_srcu_read_unlock(cookie); @@ -1367,6 +1363,13 @@ enum nbcon_prio nbcon_get_default_prio(void) * both the console_lock and the SRCU read lock. Otherwise it * is set to false. * @cookie: The cookie from the SRCU read lock. + * @use_atomic: Set true when called in an atomic or unknown context. + * It affects which nbcon callback will be used: write_atomic() + * or write_thread(). + * + * When false, the write_thread() callback is used and would be + * called in a preemtible context unless disabled by the + * device_lock. The legacy handover is not allowed in this mode. * * Context: Any context except NMI. * Return: True, when a record has been printed and there are still @@ -1382,26 +1385,36 @@ enum nbcon_prio nbcon_get_default_prio(void) * Essentially it is the nbcon version of console_emit_next_record(). */ bool nbcon_legacy_emit_next_record(struct console *con, bool *handover, - int cookie) + int cookie, bool use_atomic) { struct nbcon_write_context wctxt = { }; struct nbcon_context *ctxt = &ACCESS_PRIVATE(&wctxt, ctxt); unsigned long flags; bool progress; - /* Use the same procedure as console_emit_next_record(). */ - printk_safe_enter_irqsave(flags); - console_lock_spinning_enable(); - stop_critical_timings(); - ctxt->console = con; ctxt->prio = nbcon_get_default_prio(); - progress = nbcon_atomic_emit_one(&wctxt); + if (use_atomic) { + /* + * In an atomic or unknown context, use the same procedure as + * in console_emit_next_record(). It allows to handover. + */ + printk_safe_enter_irqsave(flags); + console_lock_spinning_enable(); + stop_critical_timings(); + } - start_critical_timings(); - *handover = console_lock_spinning_disable_and_check(cookie); - printk_safe_exit_irqrestore(flags); + progress = nbcon_emit_one(&wctxt, use_atomic); + + if (use_atomic) { + start_critical_timings(); + *handover = console_lock_spinning_disable_and_check(cookie); + printk_safe_exit_irqrestore(flags); + } else { + /* Non-atomic does not perform legacy spinning handovers. */ + *handover = false; + } return progress; } diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index f27c76c3b5cf..55d75db00042 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -3077,12 +3077,13 @@ static bool console_flush_all(bool do_cond_resched, u64 *next_seq, bool *handove u64 printk_seq; bool progress; - if (!console_is_usable(con, flags, true)) + if (!console_is_usable(con, flags, !do_cond_resched)) continue; any_usable = true; if (flags & CON_NBCON) { - progress = nbcon_legacy_emit_next_record(con, handover, cookie); + progress = nbcon_legacy_emit_next_record(con, handover, cookie, + !do_cond_resched); printk_seq = nbcon_seq_read(con); } else { progress = console_emit_next_record(con, handover, cookie); From 13189fa73afae2b961d29132fd36d9cc0be77ecb Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 4 Sep 2024 14:11:28 +0206 Subject: [PATCH 148/180] printk: nbcon: Rely on kthreads for normal operation Once the kthread is running and available (i.e. @printk_kthreads_running is set), the kthread becomes responsible for flushing any pending messages which are added in NBCON_PRIO_NORMAL context. Namely the legacy console_flush_all() and device_release() no longer flush the console. And nbcon_atomic_flush_pending() used by nbcon_cpu_emergency_exit() no longer flushes messages added after the emergency messages. The console context is safe when used by the kthread only when one of the following conditions are true: 1. Other caller acquires the console context with NBCON_PRIO_NORMAL with preemption disabled. It will release the context before rescheduling. 2. Other caller acquires the console context with NBCON_PRIO_NORMAL under the device_lock. 3. The kthread is the only context which acquires the console with NBCON_PRIO_NORMAL. This is satisfied for all atomic printing call sites: nbcon_legacy_emit_next_record() (#1) nbcon_atomic_flush_pending_con() (#1) nbcon_device_release() (#2) It is even double guaranteed when @printk_kthreads_running is set because then _only_ the kthread will print for NBCON_PRIO_NORMAL. (#3) Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240904120536.115780-10-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/internal.h | 26 ++++++++++++++++++++++ kernel/printk/nbcon.c | 17 +++++++++----- kernel/printk/printk.c | 48 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 84 insertions(+), 7 deletions(-) diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index a96d4114a1db..8166e24f8780 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -113,6 +113,13 @@ static inline bool console_is_usable(struct console *con, short flags, bool use_ /* The write_atomic() callback is optional. */ if (use_atomic && !con->write_atomic) return false; + + /* + * For the !use_atomic case, @printk_kthreads_running is not + * checked because the write_thread() callback is also used + * via the legacy loop when the printer threads are not + * available. + */ } else { if (!con->write) return false; @@ -176,6 +183,7 @@ static inline void nbcon_atomic_flush_pending(void) { } static inline bool nbcon_legacy_emit_next_record(struct console *con, bool *handover, int cookie, bool use_atomic) { return false; } static inline void nbcon_kthread_wake(struct console *con) { } +static inline void nbcon_kthreads_wake(void) { } static inline bool console_is_usable(struct console *con, short flags, bool use_atomic) { return false; } @@ -190,6 +198,7 @@ extern bool legacy_allow_panic_sync; /** * struct console_flush_type - Define available console flush methods * @nbcon_atomic: Flush directly using nbcon_atomic() callback + * @nbcon_offload: Offload flush to printer thread * @legacy_direct: Call the legacy loop in this context * @legacy_offload: Offload the legacy loop into IRQ * @@ -197,6 +206,7 @@ extern bool legacy_allow_panic_sync; */ struct console_flush_type { bool nbcon_atomic; + bool nbcon_offload; bool legacy_direct; bool legacy_offload; }; @@ -211,6 +221,22 @@ static inline void printk_get_console_flush_type(struct console_flush_type *ft) switch (nbcon_get_default_prio()) { case NBCON_PRIO_NORMAL: + if (have_nbcon_console && !have_boot_console) { + if (printk_kthreads_running) + ft->nbcon_offload = true; + else + ft->nbcon_atomic = true; + } + + /* Legacy consoles are flushed directly when possible. */ + if (have_legacy_console || have_boot_console) { + if (!is_printk_legacy_deferred()) + ft->legacy_direct = true; + else + ft->legacy_offload = true; + } + break; + case NBCON_PRIO_EMERGENCY: if (have_nbcon_console && !have_boot_console) ft->nbcon_atomic = true; diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index 784e5de88abf..5146ce9853a3 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -1494,6 +1494,7 @@ static int __nbcon_atomic_flush_pending_con(struct console *con, u64 stop_seq, static void nbcon_atomic_flush_pending_con(struct console *con, u64 stop_seq, bool allow_unsafe_takeover) { + struct console_flush_type ft; unsigned long flags; int err; @@ -1523,10 +1524,12 @@ again: /* * If flushing was successful but more records are available, this - * context must flush those remaining records because there is no - * other context that will do it. + * context must flush those remaining records if the printer thread + * is not available do it. */ - if (prb_read_valid(prb, nbcon_seq_read(con), NULL)) { + printk_get_console_flush_type(&ft); + if (!ft.nbcon_offload && + prb_read_valid(prb, nbcon_seq_read(con), NULL)) { stop_seq = prb_next_reserve_seq(prb); goto again; } @@ -1754,17 +1757,19 @@ void nbcon_device_release(struct console *con) /* * This context must flush any new records added while the console - * was locked. The console_srcu_read_lock must be taken to ensure - * the console is usable throughout flushing. + * was locked if the printer thread is not available to do it. The + * console_srcu_read_lock must be taken to ensure the console is + * usable throughout flushing. */ cookie = console_srcu_read_lock(); + printk_get_console_flush_type(&ft); if (console_is_usable(con, console_srcu_read_flags(con), true) && + !ft.nbcon_offload && prb_read_valid(prb, nbcon_seq_read(con), NULL)) { /* * If nbcon_atomic flushing is not available, fallback to * using the legacy loop. */ - printk_get_console_flush_type(&ft); if (ft.nbcon_atomic) { __nbcon_atomic_flush_pending_con(con, prb_next_reserve_seq(prb), false); } else if (ft.legacy_direct) { diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 55d75db00042..149c3e04c2b5 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -2384,6 +2384,9 @@ asmlinkage int vprintk_emit(int facility, int level, if (ft.nbcon_atomic) nbcon_atomic_flush_pending(); + if (ft.nbcon_offload) + nbcon_kthreads_wake(); + if (ft.legacy_direct) { /* * The caller may be holding system-critical or @@ -2732,6 +2735,7 @@ void suspend_console(void) void resume_console(void) { + struct console_flush_type ft; struct console *con; if (!console_suspend_enabled) @@ -2749,6 +2753,10 @@ void resume_console(void) */ synchronize_srcu(&console_srcu); + printk_get_console_flush_type(&ft); + if (ft.nbcon_offload) + nbcon_kthreads_wake(); + pr_flush(1000, true); } @@ -3060,6 +3068,7 @@ static inline void printk_kthreads_check_locked(void) { } */ static bool console_flush_all(bool do_cond_resched, u64 *next_seq, bool *handover) { + struct console_flush_type ft; bool any_usable = false; struct console *con; bool any_progress; @@ -3071,12 +3080,22 @@ static bool console_flush_all(bool do_cond_resched, u64 *next_seq, bool *handove do { any_progress = false; + printk_get_console_flush_type(&ft); + cookie = console_srcu_read_lock(); for_each_console_srcu(con) { short flags = console_srcu_read_flags(con); u64 printk_seq; bool progress; + /* + * console_flush_all() is only responsible for nbcon + * consoles when the nbcon consoles cannot print via + * their atomic or threaded flushing. + */ + if ((flags & CON_NBCON) && (ft.nbcon_atomic || ft.nbcon_offload)) + continue; + if (!console_is_usable(con, flags, !do_cond_resched)) continue; any_usable = true; @@ -3387,9 +3406,25 @@ EXPORT_SYMBOL(console_stop); void console_start(struct console *console) { + struct console_flush_type ft; + bool is_nbcon; + console_list_lock(); console_srcu_write_flags(console, console->flags | CON_ENABLED); + is_nbcon = console->flags & CON_NBCON; console_list_unlock(); + + /* + * Ensure that all SRCU list walks have completed. The related + * printing context must be able to see it is enabled so that + * it is guaranteed to wake up and resume printing. + */ + synchronize_srcu(&console_srcu); + + printk_get_console_flush_type(&ft); + if (is_nbcon && ft.nbcon_offload) + nbcon_kthread_wake(console); + __pr_flush(console, 1000, true); } EXPORT_SYMBOL(console_start); @@ -4115,6 +4150,8 @@ static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre /* Flush the consoles so that records up to @seq are printed. */ printk_get_console_flush_type(&ft); + if (ft.nbcon_atomic) + nbcon_atomic_flush_pending(); if (ft.legacy_direct) { console_lock(); console_unlock(); @@ -4152,8 +4189,10 @@ static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre * that they make forward progress, so only increment * @diff for usable consoles. */ - if (!console_is_usable(c, flags, true)) + if (!console_is_usable(c, flags, true) && + !console_is_usable(c, flags, false)) { continue; + } if (flags & CON_NBCON) { printk_seq = nbcon_seq_read(c); @@ -4629,8 +4668,15 @@ EXPORT_SYMBOL_GPL(kmsg_dump_rewind); */ void console_try_replay_all(void) { + struct console_flush_type ft; + + printk_get_console_flush_type(&ft); if (console_trylock()) { __console_rewind_all(); + if (ft.nbcon_atomic) + nbcon_atomic_flush_pending(); + if (ft.nbcon_offload) + nbcon_kthreads_wake(); /* Consoles are flushed as part of console_unlock(). */ console_unlock(); } From 75d430372abba5210fbc56a813d2515c001a4f45 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 4 Sep 2024 14:11:29 +0206 Subject: [PATCH 149/180] printk: Provide helper for message prepending In order to support prepending different texts to printk messages, split out the prepending code into a helper function. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240904120536.115780-11-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/printk.c | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 149c3e04c2b5..e9458569bcfd 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -2843,30 +2843,31 @@ static void __console_unlock(void) #ifdef CONFIG_PRINTK /* - * Prepend the message in @pmsg->pbufs->outbuf with a "dropped message". This - * is achieved by shifting the existing message over and inserting the dropped - * message. + * Prepend the message in @pmsg->pbufs->outbuf. This is achieved by shifting + * the existing message over and inserting the scratchbuf message. * - * @pmsg is the printk message to prepend. + * @pmsg is the original printk message. + * @fmt is the printf format of the message which will prepend the existing one. * - * @dropped is the dropped count to report in the dropped message. - * - * If the message text in @pmsg->pbufs->outbuf does not have enough space for - * the dropped message, the message text will be sufficiently truncated. + * If there is not enough space in @pmsg->pbufs->outbuf, the existing + * message text will be sufficiently truncated. * * If @pmsg->pbufs->outbuf is modified, @pmsg->outbuf_len is updated. */ -void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped) +__printf(2, 3) +static void console_prepend_message(struct printk_message *pmsg, const char *fmt, ...) { struct printk_buffers *pbufs = pmsg->pbufs; const size_t scratchbuf_sz = sizeof(pbufs->scratchbuf); const size_t outbuf_sz = sizeof(pbufs->outbuf); char *scratchbuf = &pbufs->scratchbuf[0]; char *outbuf = &pbufs->outbuf[0]; + va_list args; size_t len; - len = scnprintf(scratchbuf, scratchbuf_sz, - "** %lu printk messages dropped **\n", dropped); + va_start(args, fmt); + len = vscnprintf(scratchbuf, scratchbuf_sz, fmt, args); + va_end(args); /* * Make sure outbuf is sufficiently large before prepending. @@ -2888,6 +2889,19 @@ void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped) pmsg->outbuf_len += len; } +/* + * Prepend the message in @pmsg->pbufs->outbuf with a "dropped message". + * @pmsg->outbuf_len is updated appropriately. + * + * @pmsg is the printk message to prepend. + * + * @dropped is the dropped count to report in the dropped message. + */ +void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped) +{ + console_prepend_message(pmsg, "** %lu printk messages dropped **\n", dropped); +} + /* * Read and format the specified record (or a later record if the specified * record is not available). From 5102981d5e2a5a7a12424b5d26c7524ac2899898 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 4 Sep 2024 14:11:30 +0206 Subject: [PATCH 150/180] printk: nbcon: Show replay message on takeover An emergency or panic context can takeover console ownership while the current owner was printing a printk message. The atomic printer will re-print the message that the previous owner was printing. However, this can look confusing to the user and may even seem as though a message was lost. [3430014.1 [3430014.181123] usb 1-2: Product: USB Audio Add a new field @nbcon_prev_seq to struct console to track the sequence number to print that was assigned to the previous console owner. If this matches the sequence number to print that the current owner is assigned, then a takeover must have occurred. In this case, print an additional message to inform the user that the previous message is being printed again. [3430014.1 ** replaying previous printk message ** [3430014.181123] usb 1-2: Product: USB Audio Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240904120536.115780-12-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- include/linux/console.h | 2 ++ kernel/printk/internal.h | 1 + kernel/printk/nbcon.c | 26 ++++++++++++++++++++++++++ kernel/printk/printk.c | 11 +++++++++++ 4 files changed, 40 insertions(+) diff --git a/include/linux/console.h b/include/linux/console.h index 788ce9c829f6..eba367bf605d 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -325,6 +325,7 @@ struct nbcon_write_context { * @nbcon_state: State for nbcon consoles * @nbcon_seq: Sequence number of the next record for nbcon to print * @nbcon_device_ctxt: Context available for non-printing operations + * @nbcon_prev_seq: Seq num the previous nbcon owner was assigned to print * @pbufs: Pointer to nbcon private buffer * @kthread: Printer kthread for this console * @rcuwait: RCU-safe wait object for @kthread waking @@ -459,6 +460,7 @@ struct console { atomic_t __private nbcon_state; atomic_long_t __private nbcon_seq; struct nbcon_context __private nbcon_device_ctxt; + atomic_long_t __private nbcon_prev_seq; struct printk_buffers *pbufs; struct task_struct *kthread; diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index 8166e24f8780..c365d25b13c7 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -319,4 +319,5 @@ bool printk_get_next_message(struct printk_message *pmsg, u64 seq, #ifdef CONFIG_PRINTK void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped); +void console_prepend_replay(struct printk_message *pmsg); #endif diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index 5146ce9853a3..98440889d222 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -946,7 +946,9 @@ static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt, bool use_a .pbufs = ctxt->pbufs, }; unsigned long con_dropped; + struct nbcon_state cur; unsigned long dropped; + unsigned long ulseq; /* * This function should never be called for consoles that have not @@ -987,6 +989,29 @@ static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt, bool use_a if (dropped && !is_extended) console_prepend_dropped(&pmsg, dropped); + /* + * If the previous owner was assigned the same record, this context + * has taken over ownership and is replaying the record. Prepend a + * message to let the user know the record is replayed. + */ + ulseq = atomic_long_read(&ACCESS_PRIVATE(con, nbcon_prev_seq)); + if (__ulseq_to_u64seq(prb, ulseq) == pmsg.seq) { + console_prepend_replay(&pmsg); + } else { + /* + * Ensure this context is still the owner before trying to + * update @nbcon_prev_seq. Otherwise the value in @ulseq may + * not be from the previous owner and instead be some later + * value from the context that took over ownership. + */ + nbcon_state_read(con, &cur); + if (!nbcon_context_can_proceed(ctxt, &cur)) + return false; + + atomic_long_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_prev_seq), &ulseq, + __u64seq_to_ulseq(pmsg.seq)); + } + if (!nbcon_context_exit_unsafe(ctxt)) return false; @@ -1646,6 +1671,7 @@ bool nbcon_alloc(struct console *con) rcuwait_init(&con->rcuwait); init_irq_work(&con->irq_work, nbcon_irq_work); + atomic_long_set(&ACCESS_PRIVATE(con, nbcon_prev_seq), -1UL); nbcon_state_set(con, &state); /* diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index e9458569bcfd..c27dc54f4151 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -2902,6 +2902,17 @@ void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped) console_prepend_message(pmsg, "** %lu printk messages dropped **\n", dropped); } +/* + * Prepend the message in @pmsg->pbufs->outbuf with a "replay message". + * @pmsg->outbuf_len is updated appropriately. + * + * @pmsg is the printk message to prepend. + */ +void console_prepend_replay(struct printk_message *pmsg) +{ + console_prepend_message(pmsg, "** replaying previous printk message **\n"); +} + /* * Read and format the specified record (or a later record if the specified * record is not available). From fe6fa88d865e9c1a41b41f92ae0531470c86f0da Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 4 Sep 2024 14:11:31 +0206 Subject: [PATCH 151/180] proc: consoles: Add notation to c_start/c_stop fs/proc/consoles.c:78:13: warning: context imbalance in 'c_start' - wrong count at exit fs/proc/consoles.c:104:13: warning: context imbalance in 'c_stop' - unexpected unlock Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240904120536.115780-13-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- fs/proc/consoles.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fs/proc/consoles.c b/fs/proc/consoles.c index e0758fe7936d..7036fdfa0bec 100644 --- a/fs/proc/consoles.c +++ b/fs/proc/consoles.c @@ -68,6 +68,7 @@ static int show_console_dev(struct seq_file *m, void *v) } static void *c_start(struct seq_file *m, loff_t *pos) + __acquires(&console_mutex) { struct console *con; loff_t off = 0; @@ -94,6 +95,7 @@ static void *c_next(struct seq_file *m, void *v, loff_t *pos) } static void c_stop(struct seq_file *m, void *v) + __releases(&console_mutex) { console_list_unlock(); } From c83a20662dd099dbac9ef6df44134ef446980c56 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 4 Sep 2024 14:11:32 +0206 Subject: [PATCH 152/180] proc: Add nbcon support for /proc/consoles Update /proc/consoles output to show 'W' if an nbcon console is registered. Since the write_thread() callback is mandatory, it enough just to check if it is an nbcon console. Also update /proc/consoles output to show 'N' if it is an nbcon console. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240904120536.115780-14-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- fs/proc/consoles.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fs/proc/consoles.c b/fs/proc/consoles.c index 7036fdfa0bec..b7cab1ad990d 100644 --- a/fs/proc/consoles.c +++ b/fs/proc/consoles.c @@ -21,6 +21,7 @@ static int show_console_dev(struct seq_file *m, void *v) { CON_ENABLED, 'E' }, { CON_CONSDEV, 'C' }, { CON_BOOT, 'B' }, + { CON_NBCON, 'N' }, { CON_PRINTBUFFER, 'p' }, { CON_BRL, 'b' }, { CON_ANYTIME, 'a' }, @@ -58,8 +59,8 @@ static int show_console_dev(struct seq_file *m, void *v) seq_printf(m, "%s%d", con->name, con->index); seq_pad(m, ' '); seq_printf(m, "%c%c%c (%s)", con->read ? 'R' : '-', - con->write ? 'W' : '-', con->unblank ? 'U' : '-', - flags); + ((con->flags & CON_NBCON) || con->write) ? 'W' : '-', + con->unblank ? 'U' : '-', flags); if (dev) seq_printf(m, " %4d:%d", MAJOR(dev), MINOR(dev)); From def84b4467771d0ea0b55e6a5d0d70eb54bdf46a Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 4 Sep 2024 14:11:33 +0206 Subject: [PATCH 153/180] tty: sysfs: Add nbcon support for 'active' Allow the 'active' attribute to list nbcon consoles. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240904120536.115780-15-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- drivers/tty/tty_io.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c index 407b0d87b7c1..9140825e810f 100644 --- a/drivers/tty/tty_io.c +++ b/drivers/tty/tty_io.c @@ -3567,7 +3567,7 @@ static ssize_t show_cons_active(struct device *dev, for_each_console(c) { if (!c->device) continue; - if (!c->write) + if (!(c->flags & CON_NBCON) && !c->write) continue; if ((c->flags & CON_ENABLED) == 0) continue; From 5f53ca3ff83b4f2f3ff360bb22bba7a9a3af71a3 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 4 Sep 2024 14:11:34 +0206 Subject: [PATCH 154/180] printk: Implement legacy printer kthread for PREEMPT_RT The write() callback of legacy consoles usually makes use of spinlocks. This is not permitted with PREEMPT_RT in atomic contexts. For PREEMPT_RT, create a new kthread to handle printing of all the legacy consoles (and nbcon consoles if boot consoles are registered). This allows legacy consoles to work on PREEMPT_RT without requiring modification. (However they will not have the reliability properties guaranteed by nbcon atomic consoles.) Use the existing printk_kthreads_check_locked() to start/stop the legacy kthread as needed. Introduce the macro force_legacy_kthread() to query if the forced threading of legacy consoles is in effect. Although currently only enabled for PREEMPT_RT, this acts as a simple mechanism for the future to allow other preemption models to easily take advantage of the non-interference property provided by the legacy kthread. When force_legacy_kthread() is true, the legacy kthread fulfills the role of the console_flush_type @legacy_offload by waking the legacy kthread instead of printing via the console_lock in the irq_work. If the legacy kthread is not yet available, no legacy printing takes place (unless in panic). If for some reason the legacy kthread fails to create, any legacy consoles are unregistered. With force_legacy_kthread(), the legacy kthread is a critical component for legacy consoles. These changes only affect CONFIG_PREEMPT_RT. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240904120536.115780-16-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/internal.h | 16 +++- kernel/printk/printk.c | 157 ++++++++++++++++++++++++++++++++---- kernel/printk/printk_safe.c | 4 +- 3 files changed, 159 insertions(+), 18 deletions(-) diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index c365d25b13c7..3fcb48502adb 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -21,6 +21,19 @@ int devkmsg_sysctl_set_loglvl(const struct ctl_table *table, int write, (con->flags & CON_BOOT) ? "boot" : "", \ con->name, con->index, ##__VA_ARGS__) +/* + * Identify if legacy printing is forced in a dedicated kthread. If + * true, all printing via console lock occurs within a dedicated + * legacy printer thread. The only exception is on panic, after the + * nbcon consoles have had their chance to print the panic messages + * first. + */ +#ifdef CONFIG_PREEMPT_RT +# define force_legacy_kthread() (true) +#else +# define force_legacy_kthread() (false) +#endif + #ifdef CONFIG_PRINTK #ifdef CONFIG_PRINTK_CALLER @@ -173,6 +186,7 @@ static inline void nbcon_kthread_wake(struct console *con) #define printk_safe_exit_irqrestore(flags) local_irq_restore(flags) static inline bool printk_percpu_data_ready(void) { return false; } +static inline void defer_console_output(void) { } static inline bool is_printk_legacy_deferred(void) { return false; } static inline u64 nbcon_seq_read(struct console *con) { return 0; } static inline void nbcon_seq_force(struct console *con, u64 seq) { } @@ -200,7 +214,7 @@ extern bool legacy_allow_panic_sync; * @nbcon_atomic: Flush directly using nbcon_atomic() callback * @nbcon_offload: Offload flush to printer thread * @legacy_direct: Call the legacy loop in this context - * @legacy_offload: Offload the legacy loop into IRQ + * @legacy_offload: Offload the legacy loop into IRQ or legacy thread * * Note that the legacy loop also flushes the nbcon consoles. */ diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index c27dc54f4151..66cfe7b8f95c 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -491,6 +491,7 @@ bool legacy_allow_panic_sync; #ifdef CONFIG_PRINTK DECLARE_WAIT_QUEUE_HEAD(log_wait); +static DECLARE_WAIT_QUEUE_HEAD(legacy_wait); /* All 3 protected by @syslog_lock. */ /* the next printk record to read by syslog(READ) or /proc/kmsg */ static u64 syslog_seq; @@ -2756,6 +2757,8 @@ void resume_console(void) printk_get_console_flush_type(&ft); if (ft.nbcon_offload) nbcon_kthreads_wake(); + if (ft.legacy_offload) + defer_console_output(); pr_flush(1000, true); } @@ -3166,19 +3169,7 @@ abandon: return false; } -/** - * console_unlock - unblock the console subsystem from printing - * - * Releases the console_lock which the caller holds to block printing of - * the console subsystem. - * - * While the console_lock was held, console output may have been buffered - * by printk(). If this is the case, console_unlock(); emits - * the output prior to releasing the lock. - * - * console_unlock(); may be called from any context. - */ -void console_unlock(void) +static void __console_flush_and_unlock(void) { bool do_cond_resched; bool handover; @@ -3222,6 +3213,29 @@ void console_unlock(void) */ } while (prb_read_valid(prb, next_seq, NULL) && console_trylock()); } + +/** + * console_unlock - unblock the legacy console subsystem from printing + * + * Releases the console_lock which the caller holds to block printing of + * the legacy console subsystem. + * + * While the console_lock was held, console output may have been buffered + * by printk(). If this is the case, console_unlock() emits the output on + * legacy consoles prior to releasing the lock. + * + * console_unlock(); may be called from any context. + */ +void console_unlock(void) +{ + struct console_flush_type ft; + + printk_get_console_flush_type(&ft); + if (ft.legacy_direct) + __console_flush_and_unlock(); + else + __console_unlock(); +} EXPORT_SYMBOL(console_unlock); /** @@ -3449,6 +3463,8 @@ void console_start(struct console *console) printk_get_console_flush_type(&ft); if (is_nbcon && ft.nbcon_offload) nbcon_kthread_wake(console); + else if (ft.legacy_offload) + defer_console_output(); __pr_flush(console, 1000, true); } @@ -3460,6 +3476,88 @@ static int unregister_console_locked(struct console *console); /* True when system boot is far enough to create printer threads. */ static bool printk_kthreads_ready __ro_after_init; +static struct task_struct *printk_legacy_kthread; + +static bool legacy_kthread_should_wakeup(void) +{ + struct console_flush_type ft; + struct console *con; + bool ret = false; + int cookie; + + if (kthread_should_stop()) + return true; + + printk_get_console_flush_type(&ft); + + cookie = console_srcu_read_lock(); + for_each_console_srcu(con) { + short flags = console_srcu_read_flags(con); + u64 printk_seq; + + /* + * The legacy printer thread is only responsible for nbcon + * consoles when the nbcon consoles cannot print via their + * atomic or threaded flushing. + */ + if ((flags & CON_NBCON) && (ft.nbcon_atomic || ft.nbcon_offload)) + continue; + + if (!console_is_usable(con, flags, false)) + continue; + + if (flags & CON_NBCON) { + printk_seq = nbcon_seq_read(con); + } else { + /* + * It is safe to read @seq because only this + * thread context updates @seq. + */ + printk_seq = con->seq; + } + + if (prb_read_valid(prb, printk_seq, NULL)) { + ret = true; + break; + } + } + console_srcu_read_unlock(cookie); + + return ret; +} + +static int legacy_kthread_func(void *unused) +{ + for (;;) { + wait_event_interruptible(legacy_wait, legacy_kthread_should_wakeup()); + + if (kthread_should_stop()) + break; + + console_lock(); + __console_flush_and_unlock(); + } + + return 0; +} + +static bool legacy_kthread_create(void) +{ + struct task_struct *kt; + + lockdep_assert_console_list_lock_held(); + + kt = kthread_run(legacy_kthread_func, NULL, "pr/legacy"); + if (WARN_ON(IS_ERR(kt))) { + pr_err("failed to start legacy printing thread\n"); + return false; + } + + printk_legacy_kthread = kt; + + return true; +} + /** * printk_kthreads_shutdown - shutdown all threaded printers * @@ -3509,6 +3607,27 @@ static void printk_kthreads_check_locked(void) if (!printk_kthreads_ready) return; + if (have_legacy_console || have_boot_console) { + if (!printk_legacy_kthread && + force_legacy_kthread() && + !legacy_kthread_create()) { + /* + * All legacy consoles must be unregistered. If there + * are any nbcon consoles, they will set up their own + * kthread. + */ + hlist_for_each_entry_safe(con, tmp, &console_list, node) { + if (con->flags & CON_NBCON) + continue; + + unregister_console_locked(con); + } + } + } else if (printk_legacy_kthread) { + kthread_stop(printk_legacy_kthread); + printk_legacy_kthread = NULL; + } + /* * Printer threads cannot be started as long as any boot console is * registered because there is no way to synchronize the hardware @@ -4285,9 +4404,13 @@ static void wake_up_klogd_work_func(struct irq_work *irq_work) int pending = this_cpu_xchg(printk_pending, 0); if (pending & PRINTK_PENDING_OUTPUT) { - /* If trylock fails, someone else is doing the printing */ - if (console_trylock()) - console_unlock(); + if (force_legacy_kthread()) { + if (printk_legacy_kthread) + wake_up_interruptible(&legacy_wait); + } else { + if (console_trylock()) + console_unlock(); + } } if (pending & PRINTK_PENDING_WAKEUP) @@ -4702,6 +4825,8 @@ void console_try_replay_all(void) nbcon_atomic_flush_pending(); if (ft.nbcon_offload) nbcon_kthreads_wake(); + if (ft.legacy_offload) + defer_console_output(); /* Consoles are flushed as part of console_unlock(). */ console_unlock(); } diff --git a/kernel/printk/printk_safe.c b/kernel/printk/printk_safe.c index 86439fd20aab..2b35a9d3919d 100644 --- a/kernel/printk/printk_safe.c +++ b/kernel/printk/printk_safe.c @@ -44,7 +44,9 @@ bool is_printk_legacy_deferred(void) * The per-CPU variable @printk_context can be read safely in any * context. CPU migration is always disabled when set. */ - return (this_cpu_read(printk_context) || in_nmi()); + return (force_legacy_kthread() || + this_cpu_read(printk_context) || + in_nmi()); } asmlinkage int vprintk(const char *fmt, va_list args) From 1529bbb6e2619495f146dd519a196b67837fdfa6 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 4 Sep 2024 14:11:35 +0206 Subject: [PATCH 155/180] printk: nbcon: Assign nice -20 for printing threads It is important that console printing threads are scheduled shortly after a printk call and with generous runtime budgets. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240904120536.115780-17-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/nbcon.c | 6 ++++++ kernel/printk/printk.c | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index 98440889d222..fd12efcc4aed 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -1321,6 +1321,12 @@ bool nbcon_kthread_create(struct console *con) con->kthread = kt; + /* + * It is important that console printing threads are scheduled + * shortly after a printk call and with generous runtime budgets. + */ + sched_set_normal(con->kthread, -20); + return true; } diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 66cfe7b8f95c..afd926611f0f 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -3555,6 +3555,12 @@ static bool legacy_kthread_create(void) printk_legacy_kthread = kt; + /* + * It is important that console printing threads are scheduled + * shortly after a printk call and with generous runtime budgets. + */ + sched_set_normal(printk_legacy_kthread, -20); + return true; } From daeed1595b4ddf314bad8ee40b2662e03fd012dc Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 4 Sep 2024 14:11:36 +0206 Subject: [PATCH 156/180] printk: Avoid false positive lockdep report for legacy printing Legacy console printing from printk() caller context may invoke the console driver from atomic context. This leads to a lockdep splat because the console driver will acquire a sleeping lock and the caller may already hold a spinning lock. This is noticed by lockdep on !PREEMPT_RT configurations because it will lead to a problem on PREEMPT_RT. However, on PREEMPT_RT the printing path from atomic context is always avoided and the console driver is always invoked from a dedicated thread. Thus the lockdep splat on !PREEMPT_RT is a false positive. For !PREEMPT_RT override the lock-context before invoking the console driver to avoid the false positive. Do not override the lock-context for PREEMPT_RT in order to allow lockdep to catch any real locking context issues related to the write callback usage. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240904120536.115780-18-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/printk.c | 85 +++++++++++++++++++++++++++++++----------- 1 file changed, 64 insertions(+), 21 deletions(-) diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index afd926611f0f..578723208f14 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -2981,6 +2981,34 @@ out: return true; } +/* + * Legacy console printing from printk() caller context does not respect + * raw_spinlock/spinlock nesting. For !PREEMPT_RT the lockdep warning is a + * false positive. For PREEMPT_RT the false positive condition does not + * occur. + * + * This map is used to temporarily establish LD_WAIT_SLEEP context for the + * console write() callback when legacy printing to avoid false positive + * lockdep complaints, thus allowing lockdep to continue to function for + * real issues. + */ +#ifdef CONFIG_PREEMPT_RT +static inline void printk_legacy_allow_spinlock_enter(void) { } +static inline void printk_legacy_allow_spinlock_exit(void) { } +#else +static DEFINE_WAIT_OVERRIDE_MAP(printk_legacy_map, LD_WAIT_SLEEP); + +static inline void printk_legacy_allow_spinlock_enter(void) +{ + lock_map_acquire_try(&printk_legacy_map); +} + +static inline void printk_legacy_allow_spinlock_exit(void) +{ + lock_map_release(&printk_legacy_map); +} +#endif /* CONFIG_PREEMPT_RT */ + /* * Used as the printk buffers for non-panic, serialized console printing. * This is for legacy (!CON_NBCON) as well as all boot (CON_BOOT) consoles. @@ -3030,31 +3058,46 @@ static bool console_emit_next_record(struct console *con, bool *handover, int co con->dropped = 0; } - /* - * While actively printing out messages, if another printk() - * were to occur on another CPU, it may wait for this one to - * finish. This task can not be preempted if there is a - * waiter waiting to take over. - * - * Interrupts are disabled because the hand over to a waiter - * must not be interrupted until the hand over is completed - * (@console_waiter is cleared). - */ - printk_safe_enter_irqsave(flags); - console_lock_spinning_enable(); - - /* Do not trace print latency. */ - stop_critical_timings(); - /* Write everything out to the hardware. */ - con->write(con, outbuf, pmsg.outbuf_len); - start_critical_timings(); + if (force_legacy_kthread() && !panic_in_progress()) { + /* + * With forced threading this function is in a task context + * (either legacy kthread or get_init_console_seq()). There + * is no need for concern about printk reentrance, handovers, + * or lockdep complaints. + */ - con->seq = pmsg.seq + 1; + con->write(con, outbuf, pmsg.outbuf_len); + con->seq = pmsg.seq + 1; + } else { + /* + * While actively printing out messages, if another printk() + * were to occur on another CPU, it may wait for this one to + * finish. This task can not be preempted if there is a + * waiter waiting to take over. + * + * Interrupts are disabled because the hand over to a waiter + * must not be interrupted until the hand over is completed + * (@console_waiter is cleared). + */ + printk_safe_enter_irqsave(flags); + console_lock_spinning_enable(); - *handover = console_lock_spinning_disable_and_check(cookie); - printk_safe_exit_irqrestore(flags); + /* Do not trace print latency. */ + stop_critical_timings(); + + printk_legacy_allow_spinlock_enter(); + con->write(con, outbuf, pmsg.outbuf_len); + printk_legacy_allow_spinlock_exit(); + + start_critical_timings(); + + con->seq = pmsg.seq + 1; + + *handover = console_lock_spinning_disable_and_check(cookie); + printk_safe_exit_irqrestore(flags); + } skip: return true; } From 9e65863194ad253f1de48bb9000a586e6caa5eed Mon Sep 17 00:00:00 2001 From: Nick Chan Date: Sun, 1 Sep 2024 11:40:04 +0800 Subject: [PATCH 157/180] dt-bindings: apple,aic: Document A7-A11 compatibles Document and describe the compatibles for Apple A7-A11 SoCs. There are three feature levels: - apple,aic: No fast IPI, for A7-A10 - apple,t8015-aic: fast IPI, global only, for A11 - apple,t8103-aic: fast IPI with local and global support, for M1 Each feature level is an extension of the previous, for example, M1 will also work with the A7 feature level. All of A7-M1 gets its own SoC-specific compatible, and the "apple,aic" compatible as a fallback. Signed-off-by: Nick Chan Signed-off-by: Thomas Gleixner Reviewed-by: Sven Peter Acked-by: Krzysztof Kozlowski Link: https://lore.kernel.org/all/20240901034143.12731-2-towinchenmi@gmail.com --- .../bindings/interrupt-controller/apple,aic.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/interrupt-controller/apple,aic.yaml b/Documentation/devicetree/bindings/interrupt-controller/apple,aic.yaml index 698588e9aa86..4be9b596a790 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/apple,aic.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/apple,aic.yaml @@ -31,13 +31,25 @@ description: | This device also represents the FIQ interrupt sources on platforms using AIC, which do not go through a discrete interrupt controller. + IPIs may be performed via MMIO registers on all variants of AIC. Starting + from A11, system registers may also be used for "fast" IPIs. Starting from + M1, even faster IPIs within the same cluster may be achieved by writing to + a "local" fast IPI register as opposed to using the "global" fast IPI + register. + allOf: - $ref: /schemas/interrupt-controller.yaml# properties: compatible: items: - - const: apple,t8103-aic + - enum: + - apple,s5l8960x-aic + - apple,t7000-aic + - apple,s8000-aic + - apple,t8010-aic + - apple,t8015-aic + - apple,t8103-aic - const: apple,aic interrupt-controller: true From 5527b06c96715518bc58d1ebb29efc3653f66c5e Mon Sep 17 00:00:00 2001 From: Nick Chan Date: Sun, 1 Sep 2024 11:40:05 +0800 Subject: [PATCH 158/180] irqchip/apple-aic: Skip unnecessary enabling of use_fast_ipi use_fast_ipi is true by default and there is no need to "enable" it. Signed-off-by: Nick Chan Signed-off-by: Thomas Gleixner Reviewed-by: Sven Peter Link: https://lore.kernel.org/all/20240901034143.12731-3-towinchenmi@gmail.com --- drivers/irqchip/irq-apple-aic.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 5c534d9fd2b0..8d81d5fb3c50 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -987,9 +987,7 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p off += sizeof(u32) * (irqc->max_irq >> 5); /* MASK_CLR */ off += sizeof(u32) * (irqc->max_irq >> 5); /* HW_STATE */ - if (irqc->info.fast_ipi) - static_branch_enable(&use_fast_ipi); - else + if (!irqc->info.fast_ipi) static_branch_disable(&use_fast_ipi); irqc->info.die_stride = off - start_off; From a845342e6e5fb4937564f93cc88e00c732286fe3 Mon Sep 17 00:00:00 2001 From: Nick Chan Date: Sun, 1 Sep 2024 11:40:06 +0800 Subject: [PATCH 159/180] irqchip/apple-aic: Add a new "Global fast IPIs only" feature level Starting with the A11 (T8015) SoC, Apple began using arm64 sysregs for fast IPIs. However, on A11, there is no such things as "Local" fast IPIs, as the SYS_IMP_APL_IPI_RR_LOCAL_EL1 register does not seem to exist. Add a new feature level, used by the compatible "apple,t8015-aic", controlled by a static branch key named use_local_fast_ipi. When use_fast_ipi is true and use_local_fast_ipi is false, fast IPIs are used but all IPIs goes through the register SYS_IMP_APL_IPI_RR_GLOBAL_EL1, as "global" IPIs. Signed-off-by: Nick Chan Signed-off-by: Thomas Gleixner Reviewed-by: Sven Peter Link: https://lore.kernel.org/all/20240901034143.12731-4-towinchenmi@gmail.com --- drivers/irqchip/irq-apple-aic.c | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 8d81d5fb3c50..90126908f0ba 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -235,6 +235,8 @@ enum fiq_hwirq { }; static DEFINE_STATIC_KEY_TRUE(use_fast_ipi); +/* True if SYS_IMP_APL_IPI_RR_LOCAL_EL1 exists for local fast IPIs (M1+) */ +static DEFINE_STATIC_KEY_TRUE(use_local_fast_ipi); struct aic_info { int version; @@ -252,6 +254,7 @@ struct aic_info { /* Features */ bool fast_ipi; + bool local_fast_ipi; }; static const struct aic_info aic1_info __initconst = { @@ -270,17 +273,32 @@ static const struct aic_info aic1_fipi_info __initconst = { .fast_ipi = true, }; +static const struct aic_info aic1_local_fipi_info __initconst = { + .version = 1, + + .event = AIC_EVENT, + .target_cpu = AIC_TARGET_CPU, + + .fast_ipi = true, + .local_fast_ipi = true, +}; + static const struct aic_info aic2_info __initconst = { .version = 2, .irq_cfg = AIC2_IRQ_CFG, .fast_ipi = true, + .local_fast_ipi = true, }; static const struct of_device_id aic_info_match[] = { { .compatible = "apple,t8103-aic", + .data = &aic1_local_fipi_info, + }, + { + .compatible = "apple,t8015-aic", .data = &aic1_fipi_info, }, { @@ -750,12 +768,12 @@ static void aic_ipi_send_fast(int cpu) u64 cluster = MPIDR_CLUSTER(mpidr); u64 idx = MPIDR_CPU(mpidr); - if (MPIDR_CLUSTER(my_mpidr) == cluster) - write_sysreg_s(FIELD_PREP(IPI_RR_CPU, idx), - SYS_IMP_APL_IPI_RR_LOCAL_EL1); - else + if (static_branch_likely(&use_local_fast_ipi) && MPIDR_CLUSTER(my_mpidr) == cluster) { + write_sysreg_s(FIELD_PREP(IPI_RR_CPU, idx), SYS_IMP_APL_IPI_RR_LOCAL_EL1); + } else { write_sysreg_s(FIELD_PREP(IPI_RR_CPU, idx) | FIELD_PREP(IPI_RR_CLUSTER, cluster), SYS_IMP_APL_IPI_RR_GLOBAL_EL1); + } isb(); } @@ -990,6 +1008,9 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p if (!irqc->info.fast_ipi) static_branch_disable(&use_fast_ipi); + if (!irqc->info.local_fast_ipi) + static_branch_disable(&use_local_fast_ipi); + irqc->info.die_stride = off - start_off; irqc->hw_domain = irq_domain_create_tree(of_node_to_fwnode(node), From 59fc20ba70294d2c5f620ad6206aa661ce7718d6 Mon Sep 17 00:00:00 2001 From: Konrad Dybcio Date: Sun, 1 Sep 2024 11:40:07 +0800 Subject: [PATCH 160/180] irqchip/apple-aic: Only access system registers on SoCs which provide them Starting from the A11 (T8015) SoC, Apple introuced system registers for fast IPI and UNCORE PMC control. These sysregs do not exist on earlier A7-A10 SoCs and trying to access them results in an instant crash. Restrict sysreg access within the AIC driver to configurations where use_fast_ipi is true to allow AIC to function properly on A7-A10 SoCs. Co-developed-by: Nick Chan Signed-off-by: Konrad Dybcio Signed-off-by: Nick Chan Signed-off-by: Thomas Gleixner Reviewed-by: Sven Peter Link: https://lore.kernel.org/all/20240901034143.12731-5-towinchenmi@gmail.com --- drivers/irqchip/irq-apple-aic.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 90126908f0ba..da5250f0155c 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -234,6 +234,7 @@ enum fiq_hwirq { AIC_NR_FIQ }; +/* True if UNCORE/UNCORE2 and Sn_... IPI registers are present and used (A11+) */ static DEFINE_STATIC_KEY_TRUE(use_fast_ipi); /* True if SYS_IMP_APL_IPI_RR_LOCAL_EL1 exists for local fast IPIs (M1+) */ static DEFINE_STATIC_KEY_TRUE(use_local_fast_ipi); @@ -550,14 +551,9 @@ static void __exception_irq_entry aic_handle_fiq(struct pt_regs *regs) * we check for everything here, even things we don't support yet. */ - if (read_sysreg_s(SYS_IMP_APL_IPI_SR_EL1) & IPI_SR_PENDING) { - if (static_branch_likely(&use_fast_ipi)) { - aic_handle_ipi(regs); - } else { - pr_err_ratelimited("Fast IPI fired. Acking.\n"); - write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1); - } - } + if (static_branch_likely(&use_fast_ipi) && + (read_sysreg_s(SYS_IMP_APL_IPI_SR_EL1) & IPI_SR_PENDING)) + aic_handle_ipi(regs); if (TIMER_FIRING(read_sysreg(cntp_ctl_el0))) generic_handle_domain_irq(aic_irqc->hw_domain, @@ -592,8 +588,9 @@ static void __exception_irq_entry aic_handle_fiq(struct pt_regs *regs) AIC_FIQ_HWIRQ(irq)); } - if (FIELD_GET(UPMCR0_IMODE, read_sysreg_s(SYS_IMP_APL_UPMCR0_EL1)) == UPMCR0_IMODE_FIQ && - (read_sysreg_s(SYS_IMP_APL_UPMSR_EL1) & UPMSR_IACT)) { + if (static_branch_likely(&use_fast_ipi) && + (FIELD_GET(UPMCR0_IMODE, read_sysreg_s(SYS_IMP_APL_UPMCR0_EL1)) == UPMCR0_IMODE_FIQ) && + (read_sysreg_s(SYS_IMP_APL_UPMSR_EL1) & UPMSR_IACT)) { /* Same story with uncore PMCs */ pr_err_ratelimited("Uncore PMC FIQ fired. Masking.\n"); sysreg_clear_set_s(SYS_IMP_APL_UPMCR0_EL1, UPMCR0_IMODE, @@ -829,7 +826,8 @@ static int aic_init_cpu(unsigned int cpu) /* Mask all hard-wired per-CPU IRQ/FIQ sources */ /* Pending Fast IPI FIQs */ - write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1); + if (static_branch_likely(&use_fast_ipi)) + write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1); /* Timer FIQs */ sysreg_clear_set(cntp_ctl_el0, 0, ARCH_TIMER_CTRL_IT_MASK); @@ -850,8 +848,10 @@ static int aic_init_cpu(unsigned int cpu) FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_OFF)); /* Uncore PMC FIQ */ - sysreg_clear_set_s(SYS_IMP_APL_UPMCR0_EL1, UPMCR0_IMODE, - FIELD_PREP(UPMCR0_IMODE, UPMCR0_IMODE_OFF)); + if (static_branch_likely(&use_fast_ipi)) { + sysreg_clear_set_s(SYS_IMP_APL_UPMCR0_EL1, UPMCR0_IMODE, + FIELD_PREP(UPMCR0_IMODE, UPMCR0_IMODE_OFF)); + } /* Commit all of the above */ isb(); From 0c87282074be532eedc8d14ff4420c585c5c57fe Mon Sep 17 00:00:00 2001 From: Detlev Casanova Date: Fri, 2 Aug 2024 17:45:35 -0400 Subject: [PATCH 161/180] dt-bindings: timer: rockchip: Add rk3576 compatible Add compatible string for Rockchip RK3576 timer. Signed-off-by: Detlev Casanova Acked-by: Krzysztof Kozlowski Acked-by: Heiko Stuebner Link: https://lore.kernel.org/r/20240802214612.434179-9-detlev.casanova@collabora.com Signed-off-by: Daniel Lezcano --- Documentation/devicetree/bindings/timer/rockchip,rk-timer.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/timer/rockchip,rk-timer.yaml b/Documentation/devicetree/bindings/timer/rockchip,rk-timer.yaml index 19e56b7577a0..6d0eb0014eee 100644 --- a/Documentation/devicetree/bindings/timer/rockchip,rk-timer.yaml +++ b/Documentation/devicetree/bindings/timer/rockchip,rk-timer.yaml @@ -24,6 +24,7 @@ properties: - rockchip,rk3228-timer - rockchip,rk3229-timer - rockchip,rk3368-timer + - rockchip,rk3576-timer - rockchip,rk3588-timer - rockchip,px30-timer - const: rockchip,rk3288-timer From a7456d7d1b8e781fdaa16c10947bd2363c6fd342 Mon Sep 17 00:00:00 2001 From: Zhang Zekun Date: Wed, 7 Aug 2024 15:46:55 +0800 Subject: [PATCH 162/180] clocksource/drivers/arm_arch_timer: Using for_each_available_child_of_node_scoped() for_each_available_child_of_node_scoped() can put the device_node automatically. So, using it to make the code logic more simple and remove the device_node clean up code. Signed-off-by: Zhang Zekun Link: https://lore.kernel.org/r/20240807074655.52157-1-zhangzekun11@huawei.com Signed-off-by: Daniel Lezcano --- drivers/clocksource/arm_arch_timer.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index aeafc74181f0..03733101e231 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -1594,7 +1594,6 @@ static int __init arch_timer_mem_of_init(struct device_node *np) { struct arch_timer_mem *timer_mem; struct arch_timer_mem_frame *frame; - struct device_node *frame_node; struct resource res; int ret = -EINVAL; u32 rate; @@ -1608,33 +1607,29 @@ static int __init arch_timer_mem_of_init(struct device_node *np) timer_mem->cntctlbase = res.start; timer_mem->size = resource_size(&res); - for_each_available_child_of_node(np, frame_node) { + for_each_available_child_of_node_scoped(np, frame_node) { u32 n; struct arch_timer_mem_frame *frame; if (of_property_read_u32(frame_node, "frame-number", &n)) { pr_err(FW_BUG "Missing frame-number.\n"); - of_node_put(frame_node); goto out; } if (n >= ARCH_TIMER_MEM_MAX_FRAMES) { pr_err(FW_BUG "Wrong frame-number, only 0-%u are permitted.\n", ARCH_TIMER_MEM_MAX_FRAMES - 1); - of_node_put(frame_node); goto out; } frame = &timer_mem->frame[n]; if (frame->valid) { pr_err(FW_BUG "Duplicated frame-number.\n"); - of_node_put(frame_node); goto out; } - if (of_address_to_resource(frame_node, 0, &res)) { - of_node_put(frame_node); + if (of_address_to_resource(frame_node, 0, &res)) goto out; - } + frame->cntbase = res.start; frame->size = resource_size(&res); From 56bd72e9cd8272687aa2a8eccddabc5526cd7845 Mon Sep 17 00:00:00 2001 From: Marek Maslanka Date: Mon, 12 Aug 2024 18:41:42 +0000 Subject: [PATCH 163/180] clocksource: acpi_pm: Add external callback for suspend/resume Provides the capability to register an external callback for the ACPI PM timer, which is called during the suspend and resume processes. Signed-off-by: Marek Maslanka Reviewed-by: Hans de Goede Link: https://lore.kernel.org/r/20240812184150.1079924-1-mmaslanka@google.com Signed-off-by: Daniel Lezcano --- drivers/clocksource/acpi_pm.c | 32 ++++++++++++++++++++++++++++++++ include/linux/acpi_pmtmr.h | 13 +++++++++++++ 2 files changed, 45 insertions(+) diff --git a/drivers/clocksource/acpi_pm.c b/drivers/clocksource/acpi_pm.c index 82338773602c..b4330a01a566 100644 --- a/drivers/clocksource/acpi_pm.c +++ b/drivers/clocksource/acpi_pm.c @@ -25,6 +25,10 @@ #include #include +static void *suspend_resume_cb_data; + +static void (*suspend_resume_callback)(void *data, bool suspend); + /* * The I/O port the PMTMR resides at. * The location is detected during setup_arch(), @@ -58,6 +62,32 @@ u32 acpi_pm_read_verified(void) return v2; } +void acpi_pmtmr_register_suspend_resume_callback(void (*cb)(void *data, bool suspend), void *data) +{ + suspend_resume_callback = cb; + suspend_resume_cb_data = data; +} +EXPORT_SYMBOL_GPL(acpi_pmtmr_register_suspend_resume_callback); + +void acpi_pmtmr_unregister_suspend_resume_callback(void) +{ + suspend_resume_callback = NULL; + suspend_resume_cb_data = NULL; +} +EXPORT_SYMBOL_GPL(acpi_pmtmr_unregister_suspend_resume_callback); + +static void acpi_pm_suspend(struct clocksource *cs) +{ + if (suspend_resume_callback) + suspend_resume_callback(suspend_resume_cb_data, true); +} + +static void acpi_pm_resume(struct clocksource *cs) +{ + if (suspend_resume_callback) + suspend_resume_callback(suspend_resume_cb_data, false); +} + static u64 acpi_pm_read(struct clocksource *cs) { return (u64)read_pmtmr(); @@ -69,6 +99,8 @@ static struct clocksource clocksource_acpi_pm = { .read = acpi_pm_read, .mask = (u64)ACPI_PM_MASK, .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .suspend = acpi_pm_suspend, + .resume = acpi_pm_resume, }; diff --git a/include/linux/acpi_pmtmr.h b/include/linux/acpi_pmtmr.h index 50d88bf1498d..0ded9220d379 100644 --- a/include/linux/acpi_pmtmr.h +++ b/include/linux/acpi_pmtmr.h @@ -26,6 +26,19 @@ static inline u32 acpi_pm_read_early(void) return acpi_pm_read_verified() & ACPI_PM_MASK; } +/** + * Register callback for suspend and resume event + * + * @cb Callback triggered on suspend and resume + * @data Data passed with the callback + */ +void acpi_pmtmr_register_suspend_resume_callback(void (*cb)(void *data, bool suspend), void *data); + +/** + * Remove registered callback for suspend and resume event + */ +void acpi_pmtmr_unregister_suspend_resume_callback(void); + #else static inline u32 acpi_pm_read_early(void) From e86c8186d03a6ba018e881ed45f0962ad553e861 Mon Sep 17 00:00:00 2001 From: Marek Maslanka Date: Mon, 12 Aug 2024 18:42:00 +0000 Subject: [PATCH 164/180] platform/x86:intel/pmc: Enable the ACPI PM Timer to be turned off when suspended Allow to disable ACPI PM Timer on suspend and enable on resume. A disabled timer helps optimise power consumption when the system is suspended. On resume the timer is only reactivated if it was activated prior to suspend, so unless the ACPI PM timer is enabled in the BIOS, this won't change anything. The ACPI PM timer is used by Intel's iTCO/wdat_wdt watchdog to drive the watchdog, so it doesn't need to run during suspend. Signed-off-by: Marek Maslanka Reviewed-by: Hans de Goede Link: https://lore.kernel.org/r/20240812184208.1080710-1-mmaslanka@google.com Signed-off-by: Daniel Lezcano --- drivers/platform/x86/intel/pmc/adl.c | 2 ++ drivers/platform/x86/intel/pmc/cnp.c | 2 ++ drivers/platform/x86/intel/pmc/core.c | 45 +++++++++++++++++++++++++++ drivers/platform/x86/intel/pmc/core.h | 8 +++++ drivers/platform/x86/intel/pmc/icl.c | 2 ++ drivers/platform/x86/intel/pmc/mtl.c | 2 ++ drivers/platform/x86/intel/pmc/spt.c | 2 ++ drivers/platform/x86/intel/pmc/tgl.c | 2 ++ 8 files changed, 65 insertions(+) diff --git a/drivers/platform/x86/intel/pmc/adl.c b/drivers/platform/x86/intel/pmc/adl.c index e7878558fd90..9d9c07f44ff6 100644 --- a/drivers/platform/x86/intel/pmc/adl.c +++ b/drivers/platform/x86/intel/pmc/adl.c @@ -295,6 +295,8 @@ const struct pmc_reg_map adl_reg_map = { .ppfear_buckets = CNP_PPFEAR_NUM_ENTRIES, .pm_cfg_offset = CNP_PMC_PM_CFG_OFFSET, .pm_read_disable_bit = CNP_PMC_READ_DISABLE_BIT, + .acpi_pm_tmr_ctl_offset = SPT_PMC_ACPI_PM_TMR_CTL_OFFSET, + .acpi_pm_tmr_disable_bit = SPT_PMC_BIT_ACPI_PM_TMR_DISABLE, .ltr_ignore_max = ADL_NUM_IP_IGN_ALLOWED, .lpm_num_modes = ADL_LPM_NUM_MODES, .lpm_num_maps = ADL_LPM_NUM_MAPS, diff --git a/drivers/platform/x86/intel/pmc/cnp.c b/drivers/platform/x86/intel/pmc/cnp.c index dd72974bf71e..513c02670c5a 100644 --- a/drivers/platform/x86/intel/pmc/cnp.c +++ b/drivers/platform/x86/intel/pmc/cnp.c @@ -200,6 +200,8 @@ const struct pmc_reg_map cnp_reg_map = { .ppfear_buckets = CNP_PPFEAR_NUM_ENTRIES, .pm_cfg_offset = CNP_PMC_PM_CFG_OFFSET, .pm_read_disable_bit = CNP_PMC_READ_DISABLE_BIT, + .acpi_pm_tmr_ctl_offset = SPT_PMC_ACPI_PM_TMR_CTL_OFFSET, + .acpi_pm_tmr_disable_bit = SPT_PMC_BIT_ACPI_PM_TMR_DISABLE, .ltr_ignore_max = CNP_NUM_IP_IGN_ALLOWED, .etr3_offset = ETR3_OFFSET, }; diff --git a/drivers/platform/x86/intel/pmc/core.c b/drivers/platform/x86/intel/pmc/core.c index 01ae71c6df59..c91c753b120e 100644 --- a/drivers/platform/x86/intel/pmc/core.c +++ b/drivers/platform/x86/intel/pmc/core.c @@ -11,6 +11,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include #include #include @@ -1208,6 +1209,38 @@ static bool pmc_core_is_pson_residency_enabled(struct pmc_dev *pmcdev) return val == 1; } +/** + * Enable or disable ACPI PM Timer + * + * This function is intended to be a callback for ACPI PM suspend/resume event. + * The ACPI PM Timer is enabled on resume only if it was enabled during suspend. + */ +static void pmc_core_acpi_pm_timer_suspend_resume(void *data, bool suspend) +{ + struct pmc_dev *pmcdev = data; + struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN]; + const struct pmc_reg_map *map = pmc->map; + bool enabled; + u32 reg; + + if (!map->acpi_pm_tmr_ctl_offset) + return; + + guard(mutex)(&pmcdev->lock); + + if (!suspend && !pmcdev->enable_acpi_pm_timer_on_resume) + return; + + reg = pmc_core_reg_read(pmc, map->acpi_pm_tmr_ctl_offset); + enabled = !(reg & map->acpi_pm_tmr_disable_bit); + if (suspend) + reg |= map->acpi_pm_tmr_disable_bit; + else + reg &= ~map->acpi_pm_tmr_disable_bit; + pmc_core_reg_write(pmc, map->acpi_pm_tmr_ctl_offset, reg); + + pmcdev->enable_acpi_pm_timer_on_resume = suspend && enabled; +} static void pmc_core_dbgfs_unregister(struct pmc_dev *pmcdev) { @@ -1404,6 +1437,7 @@ static int pmc_core_probe(struct platform_device *pdev) struct pmc_dev *pmcdev; const struct x86_cpu_id *cpu_id; int (*core_init)(struct pmc_dev *pmcdev); + const struct pmc_reg_map *map; struct pmc *primary_pmc; int ret; @@ -1462,6 +1496,11 @@ static int pmc_core_probe(struct platform_device *pdev) pm_report_max_hw_sleep(FIELD_MAX(SLP_S0_RES_COUNTER_MASK) * pmc_core_adjust_slp_s0_step(primary_pmc, 1)); + map = primary_pmc->map; + if (map->acpi_pm_tmr_ctl_offset) + acpi_pmtmr_register_suspend_resume_callback(pmc_core_acpi_pm_timer_suspend_resume, + pmcdev); + device_initialized = true; dev_info(&pdev->dev, " initialized\n"); @@ -1471,6 +1510,12 @@ static int pmc_core_probe(struct platform_device *pdev) static void pmc_core_remove(struct platform_device *pdev) { struct pmc_dev *pmcdev = platform_get_drvdata(pdev); + const struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN]; + const struct pmc_reg_map *map = pmc->map; + + if (map->acpi_pm_tmr_ctl_offset) + acpi_pmtmr_unregister_suspend_resume_callback(); + pmc_core_dbgfs_unregister(pmcdev); pmc_core_clean_structure(pdev); } diff --git a/drivers/platform/x86/intel/pmc/core.h b/drivers/platform/x86/intel/pmc/core.h index ea04de7eb9e8..4d37ef7113d7 100644 --- a/drivers/platform/x86/intel/pmc/core.h +++ b/drivers/platform/x86/intel/pmc/core.h @@ -68,6 +68,8 @@ struct telem_endpoint; #define SPT_PMC_LTR_SCC 0x3A0 #define SPT_PMC_LTR_ISH 0x3A4 +#define SPT_PMC_ACPI_PM_TMR_CTL_OFFSET 0x18FC + /* Sunrise Point: PGD PFET Enable Ack Status Registers */ enum ppfear_regs { SPT_PMC_XRAM_PPFEAR0A = 0x590, @@ -148,6 +150,8 @@ enum ppfear_regs { #define SPT_PMC_VRIC1_SLPS0LVEN BIT(13) #define SPT_PMC_VRIC1_XTALSDQDIS BIT(22) +#define SPT_PMC_BIT_ACPI_PM_TMR_DISABLE BIT(1) + /* Cannonlake Power Management Controller register offsets */ #define CNP_PMC_SLPS0_DBG_OFFSET 0x10B4 #define CNP_PMC_PM_CFG_OFFSET 0x1818 @@ -351,6 +355,8 @@ struct pmc_reg_map { const u8 *lpm_reg_index; const u32 pson_residency_offset; const u32 pson_residency_counter_step; + const u32 acpi_pm_tmr_ctl_offset; + const u32 acpi_pm_tmr_disable_bit; }; /** @@ -424,6 +430,8 @@ struct pmc_dev { u32 die_c6_offset; struct telem_endpoint *punit_ep; struct pmc_info *regmap_list; + + bool enable_acpi_pm_timer_on_resume; }; enum pmc_index { diff --git a/drivers/platform/x86/intel/pmc/icl.c b/drivers/platform/x86/intel/pmc/icl.c index 71b0fd6cb7d8..cbbd44054468 100644 --- a/drivers/platform/x86/intel/pmc/icl.c +++ b/drivers/platform/x86/intel/pmc/icl.c @@ -46,6 +46,8 @@ const struct pmc_reg_map icl_reg_map = { .ppfear_buckets = ICL_PPFEAR_NUM_ENTRIES, .pm_cfg_offset = CNP_PMC_PM_CFG_OFFSET, .pm_read_disable_bit = CNP_PMC_READ_DISABLE_BIT, + .acpi_pm_tmr_ctl_offset = SPT_PMC_ACPI_PM_TMR_CTL_OFFSET, + .acpi_pm_tmr_disable_bit = SPT_PMC_BIT_ACPI_PM_TMR_DISABLE, .ltr_ignore_max = ICL_NUM_IP_IGN_ALLOWED, .etr3_offset = ETR3_OFFSET, }; diff --git a/drivers/platform/x86/intel/pmc/mtl.c b/drivers/platform/x86/intel/pmc/mtl.c index c7d15d864039..91f2fa728f5c 100644 --- a/drivers/platform/x86/intel/pmc/mtl.c +++ b/drivers/platform/x86/intel/pmc/mtl.c @@ -462,6 +462,8 @@ const struct pmc_reg_map mtl_socm_reg_map = { .ppfear_buckets = MTL_SOCM_PPFEAR_NUM_ENTRIES, .pm_cfg_offset = CNP_PMC_PM_CFG_OFFSET, .pm_read_disable_bit = CNP_PMC_READ_DISABLE_BIT, + .acpi_pm_tmr_ctl_offset = SPT_PMC_ACPI_PM_TMR_CTL_OFFSET, + .acpi_pm_tmr_disable_bit = SPT_PMC_BIT_ACPI_PM_TMR_DISABLE, .lpm_num_maps = ADL_LPM_NUM_MAPS, .ltr_ignore_max = MTL_SOCM_NUM_IP_IGN_ALLOWED, .lpm_res_counter_step_x2 = TGL_PMC_LPM_RES_COUNTER_STEP_X2, diff --git a/drivers/platform/x86/intel/pmc/spt.c b/drivers/platform/x86/intel/pmc/spt.c index ab993a69e33e..2cd2b3c68e46 100644 --- a/drivers/platform/x86/intel/pmc/spt.c +++ b/drivers/platform/x86/intel/pmc/spt.c @@ -130,6 +130,8 @@ const struct pmc_reg_map spt_reg_map = { .ppfear_buckets = SPT_PPFEAR_NUM_ENTRIES, .pm_cfg_offset = SPT_PMC_PM_CFG_OFFSET, .pm_read_disable_bit = SPT_PMC_READ_DISABLE_BIT, + .acpi_pm_tmr_ctl_offset = SPT_PMC_ACPI_PM_TMR_CTL_OFFSET, + .acpi_pm_tmr_disable_bit = SPT_PMC_BIT_ACPI_PM_TMR_DISABLE, .ltr_ignore_max = SPT_NUM_IP_IGN_ALLOWED, .pm_vric1_offset = SPT_PMC_VRIC1_OFFSET, }; diff --git a/drivers/platform/x86/intel/pmc/tgl.c b/drivers/platform/x86/intel/pmc/tgl.c index e0580de18077..371b4e30f142 100644 --- a/drivers/platform/x86/intel/pmc/tgl.c +++ b/drivers/platform/x86/intel/pmc/tgl.c @@ -197,6 +197,8 @@ const struct pmc_reg_map tgl_reg_map = { .ppfear_buckets = ICL_PPFEAR_NUM_ENTRIES, .pm_cfg_offset = CNP_PMC_PM_CFG_OFFSET, .pm_read_disable_bit = CNP_PMC_READ_DISABLE_BIT, + .acpi_pm_tmr_ctl_offset = SPT_PMC_ACPI_PM_TMR_CTL_OFFSET, + .acpi_pm_tmr_disable_bit = SPT_PMC_BIT_ACPI_PM_TMR_DISABLE, .ltr_ignore_max = TGL_NUM_IP_IGN_ALLOWED, .lpm_num_maps = TGL_LPM_NUM_MAPS, .lpm_res_counter_step_x2 = TGL_PMC_LPM_RES_COUNTER_STEP_X2, From 414b2fb4bb5aa09f39262c24ed90b831d18ff984 Mon Sep 17 00:00:00 2001 From: Huan Yang Date: Tue, 20 Aug 2024 17:46:03 +0800 Subject: [PATCH 165/180] clocksource/drivers/ingenic: Use devm_clk_get_enabled() helpers The devm_clk_get_enabled() helpers: - call devm_clk_get() - call clk_prepare_enable() and register what is needed in order to call clk_disable_unprepare() when needed, as a managed resource. This simplifies the code and avoids the calls to clk_disable_unprepare(). Signed-off-by: Huan Yang Link: https://lore.kernel.org/r/20240820094603.103598-1-link@vivo.com Signed-off-by: Daniel Lezcano --- drivers/clocksource/ingenic-ost.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/drivers/clocksource/ingenic-ost.c b/drivers/clocksource/ingenic-ost.c index 9f7c280a1336..e0ec33307c84 100644 --- a/drivers/clocksource/ingenic-ost.c +++ b/drivers/clocksource/ingenic-ost.c @@ -93,14 +93,10 @@ static int __init ingenic_ost_probe(struct platform_device *pdev) return PTR_ERR(map); } - ost->clk = devm_clk_get(dev, "ost"); + ost->clk = devm_clk_get_enabled(dev, "ost"); if (IS_ERR(ost->clk)) return PTR_ERR(ost->clk); - err = clk_prepare_enable(ost->clk); - if (err) - return err; - /* Clear counter high/low registers */ if (soc_info->is64bit) regmap_write(map, TCU_REG_OST_CNTL, 0); @@ -129,7 +125,6 @@ static int __init ingenic_ost_probe(struct platform_device *pdev) err = clocksource_register_hz(cs, rate); if (err) { dev_err(dev, "clocksource registration failed"); - clk_disable_unprepare(ost->clk); return err; } From ca140a0dc0a18acd4653b56db211fec9b2339986 Mon Sep 17 00:00:00 2001 From: Ankit Agrawal Date: Sat, 13 Jul 2024 15:27:13 +0530 Subject: [PATCH 166/180] clocksource/drivers/qcom: Add missing iounmap() on errors in msm_dt_timer_init() Add the missing iounmap() when clock frequency fails to get read by the of_property_read_u32() call, or if the call to msm_timer_init() fails. Fixes: 6e3321631ac2 ("ARM: msm: Add DT support to msm_timer") Signed-off-by: Ankit Agrawal Reviewed-by: Konrad Dybcio Link: https://lore.kernel.org/r/20240713095713.GA430091@bnew-VirtualBox Signed-off-by: Daniel Lezcano --- drivers/clocksource/timer-qcom.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/clocksource/timer-qcom.c b/drivers/clocksource/timer-qcom.c index b4afe3a67583..eac4c95c6127 100644 --- a/drivers/clocksource/timer-qcom.c +++ b/drivers/clocksource/timer-qcom.c @@ -233,6 +233,7 @@ static int __init msm_dt_timer_init(struct device_node *np) } if (of_property_read_u32(np, "clock-frequency", &freq)) { + iounmap(cpu0_base); pr_err("Unknown frequency\n"); return -EINVAL; } @@ -243,7 +244,11 @@ static int __init msm_dt_timer_init(struct device_node *np) freq /= 4; writel_relaxed(DGT_CLK_CTL_DIV_4, source_base + DGT_CLK_CTL); - return msm_timer_init(freq, 32, irq, !!percpu_offset); + ret = msm_timer_init(freq, 32, irq, !!percpu_offset); + if (ret) + iounmap(cpu0_base); + + return ret; } TIMER_OF_DECLARE(kpss_timer, "qcom,kpss-timer", msm_dt_timer_init); TIMER_OF_DECLARE(scss_timer, "qcom,scss-timer", msm_dt_timer_init); From 6cc11b65e5e0a6f1487274d1ad91088f7ac01d18 Mon Sep 17 00:00:00 2001 From: Gaosheng Cui Date: Sat, 3 Aug 2024 14:42:52 +0800 Subject: [PATCH 167/180] clocksource/drivers/asm9260: Add missing clk_disable_unprepare in asm9260_timer_init Add the missing clk_disable_unprepare() before return in asm9260_timer_init(). Signed-off-by: Gaosheng Cui Link: https://lore.kernel.org/r/20240803064253.331946-2-cuigaosheng1@huawei.com Signed-off-by: Daniel Lezcano --- drivers/clocksource/asm9260_timer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/clocksource/asm9260_timer.c b/drivers/clocksource/asm9260_timer.c index 5b39d3701fa3..8f97ab0b01ec 100644 --- a/drivers/clocksource/asm9260_timer.c +++ b/drivers/clocksource/asm9260_timer.c @@ -210,6 +210,7 @@ static int __init asm9260_timer_init(struct device_node *np) DRIVER_NAME, &event_dev); if (ret) { pr_err("Failed to setup irq!\n"); + clk_disable_unprepare(clk); return ret; } From 2e02da1d86c97dfaa8ee083d98df1d069b28a526 Mon Sep 17 00:00:00 2001 From: Gaosheng Cui Date: Sat, 3 Aug 2024 14:42:53 +0800 Subject: [PATCH 168/180] clocksource/drivers/cadence-ttc: Add missing clk_disable_unprepare in ttc_setup_clockevent Add the missing clk_disable_unprepare() before return in ttc_setup_clockevent(). Signed-off-by: Gaosheng Cui Reviewed-by: Michal Simek Link: https://lore.kernel.org/r/20240803064253.331946-3-cuigaosheng1@huawei.com Signed-off-by: Daniel Lezcano --- drivers/clocksource/timer-cadence-ttc.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/clocksource/timer-cadence-ttc.c b/drivers/clocksource/timer-cadence-ttc.c index ca7a06489c40..b8a1cf59b9d6 100644 --- a/drivers/clocksource/timer-cadence-ttc.c +++ b/drivers/clocksource/timer-cadence-ttc.c @@ -435,7 +435,7 @@ static int __init ttc_setup_clockevent(struct clk *clk, &ttcce->ttc.clk_rate_change_nb); if (err) { pr_warn("Unable to register clock notifier.\n"); - goto out_kfree; + goto out_clk_unprepare; } ttcce->ttc.freq = clk_get_rate(ttcce->ttc.clk); @@ -465,13 +465,15 @@ static int __init ttc_setup_clockevent(struct clk *clk, err = request_irq(irq, ttc_clock_event_interrupt, IRQF_TIMER, ttcce->ce.name, ttcce); if (err) - goto out_kfree; + goto out_clk_unprepare; clockevents_config_and_register(&ttcce->ce, ttcce->ttc.freq / PRESCALE, 1, 0xfffe); return 0; +out_clk_unprepare: + clk_disable_unprepare(ttcce->ttc.clk); out_kfree: kfree(ttcce); return err; From 69a9dcbd2d65217cf52f55dc57b50845dc9d8d3f Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Mon, 2 Sep 2024 12:48:04 +0200 Subject: [PATCH 169/180] clocksource/drivers/jcore: Use request_percpu_irq() Use request_percpu_irq() instead of request_irq() to solve the following sparse warning: jcore-pit.c:173:40: warning: incorrect type in argument 5 (different address spaces) jcore-pit.c:173:40: expected void *dev jcore-pit.c:173:40: got struct jcore_pit [noderef] __percpu *static [assigned] [toplevel] jcore_pit_percpu Compile tested only. Signed-off-by: Uros Bizjak Cc: Daniel Lezcano Cc: Thomas Gleixner Cc: Rich Felker Link: https://lore.kernel.org/r/20240902104810.21080-1-ubizjak@gmail.com Signed-off-by: Daniel Lezcano --- drivers/clocksource/jcore-pit.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/clocksource/jcore-pit.c b/drivers/clocksource/jcore-pit.c index a4a991101fa3..a3fe98cd3838 100644 --- a/drivers/clocksource/jcore-pit.c +++ b/drivers/clocksource/jcore-pit.c @@ -120,7 +120,7 @@ static int jcore_pit_local_init(unsigned cpu) static irqreturn_t jcore_timer_interrupt(int irq, void *dev_id) { - struct jcore_pit *pit = this_cpu_ptr(dev_id); + struct jcore_pit *pit = dev_id; if (clockevent_state_oneshot(&pit->ced)) jcore_pit_disable(pit); @@ -168,9 +168,8 @@ static int __init jcore_pit_init(struct device_node *node) return -ENOMEM; } - err = request_irq(pit_irq, jcore_timer_interrupt, - IRQF_TIMER | IRQF_PERCPU, - "jcore_pit", jcore_pit_percpu); + err = request_percpu_irq(pit_irq, jcore_timer_interrupt, + "jcore_pit", jcore_pit_percpu); if (err) { pr_err("pit irq request failed: %d\n", err); free_percpu(jcore_pit_percpu); From 2376d871f8552aadea19f5bc0b1370db54a3a5f2 Mon Sep 17 00:00:00 2001 From: Marek Maslanka Date: Wed, 4 Sep 2024 09:47:53 +0000 Subject: [PATCH 170/180] platform/x86:intel/pmc: Fix comment for the pmc_core_acpi_pm_timer_suspend_resume function Change incorrect kernel-doc comment to regular comment for the pmc_core_acpi_pm_timer_suspend_resume function. Signed-off-by: Marek Maslanka Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202409031410.a9beukFc-lkp@intel.com/ Link: https://lore.kernel.org/r/20240904094753.1615549-1-mmaslanka@google.com Signed-off-by: Daniel Lezcano --- drivers/platform/x86/intel/pmc/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/platform/x86/intel/pmc/core.c b/drivers/platform/x86/intel/pmc/core.c index c91c753b120e..e2b4c74ce7f6 100644 --- a/drivers/platform/x86/intel/pmc/core.c +++ b/drivers/platform/x86/intel/pmc/core.c @@ -1209,7 +1209,7 @@ static bool pmc_core_is_pson_residency_enabled(struct pmc_dev *pmcdev) return val == 1; } -/** +/* * Enable or disable ACPI PM Timer * * This function is intended to be a callback for ACPI PM suspend/resume event. From 87b5a153b862b7d937fc1dd499368297a1feae87 Mon Sep 17 00:00:00 2001 From: Costa Shulyupin Date: Wed, 4 Sep 2024 16:48:23 +0300 Subject: [PATCH 171/180] genirq/cpuhotplug: Use cpumask_intersects() Replace `cpumask_any_and(a, b) >= nr_cpu_ids` with the more readable `!cpumask_intersects(a, b)`. [ tglx: Massaged change log ] Signed-off-by: Costa Shulyupin Signed-off-by: Thomas Gleixner Reviewed-by: Ming Lei Link: https://lore.kernel.org/all/20240904134823.777623-2-costa.shul@redhat.com --- kernel/irq/cpuhotplug.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/irq/cpuhotplug.c b/kernel/irq/cpuhotplug.c index eb8628390156..15a7654eff68 100644 --- a/kernel/irq/cpuhotplug.c +++ b/kernel/irq/cpuhotplug.c @@ -37,7 +37,7 @@ static inline bool irq_needs_fixup(struct irq_data *d) * has been removed from the online mask already. */ if (cpumask_any_but(m, cpu) < nr_cpu_ids && - cpumask_any_and(m, cpu_online_mask) >= nr_cpu_ids) { + !cpumask_intersects(m, cpu_online_mask)) { /* * If this happens then there was a missed IRQ fixup at some * point. Warn about it and enforce fixup. @@ -110,7 +110,7 @@ static bool migrate_one_irq(struct irq_desc *desc) if (maskchip && chip->irq_mask) chip->irq_mask(d); - if (cpumask_any_and(affinity, cpu_online_mask) >= nr_cpu_ids) { + if (!cpumask_intersects(affinity, cpu_online_mask)) { /* * If the interrupt is managed, then shut it down and leave * the affinity untouched. From 1d07085402d122f223bda3f8b72bea37a46ee0c9 Mon Sep 17 00:00:00 2001 From: Bibo Mao Date: Sat, 7 Sep 2024 16:27:20 +0800 Subject: [PATCH 172/180] smp: Mark smp_prepare_boot_cpu() __init smp_prepare_boot_cpu() is only called during boot, hence mark it as __init. Signed-off-by: Bibo Mao Signed-off-by: Thomas Gleixner Reviewed-by: Huacai Chen Link: https://lore.kernel.org/all/20240907082720.452148-1-maobibo@loongson.cn --- arch/loongarch/kernel/smp.c | 2 +- arch/mips/kernel/smp.c | 2 +- arch/powerpc/kernel/smp.c | 2 +- include/linux/smp.h | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/loongarch/kernel/smp.c b/arch/loongarch/kernel/smp.c index ca405ab86aae..be2655c4c414 100644 --- a/arch/loongarch/kernel/smp.c +++ b/arch/loongarch/kernel/smp.c @@ -476,7 +476,7 @@ core_initcall(ipi_pm_init); #endif /* Preload SMP state for boot cpu */ -void smp_prepare_boot_cpu(void) +void __init smp_prepare_boot_cpu(void) { unsigned int cpu, node, rr_node; diff --git a/arch/mips/kernel/smp.c b/arch/mips/kernel/smp.c index 0362fc5df7b0..39e193cad2b9 100644 --- a/arch/mips/kernel/smp.c +++ b/arch/mips/kernel/smp.c @@ -439,7 +439,7 @@ void __init smp_prepare_cpus(unsigned int max_cpus) } /* preload SMP state for boot cpu */ -void smp_prepare_boot_cpu(void) +void __init smp_prepare_boot_cpu(void) { if (mp_ops->prepare_boot_cpu) mp_ops->prepare_boot_cpu(); diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index 46e6d2cd7a2d..4ab9b8cee77a 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -1166,7 +1166,7 @@ void __init smp_prepare_cpus(unsigned int max_cpus) cpu_smt_set_num_threads(num_threads, threads_per_core); } -void smp_prepare_boot_cpu(void) +void __init smp_prepare_boot_cpu(void) { BUG_ON(smp_processor_id() != boot_cpuid); #ifdef CONFIG_PPC64 diff --git a/include/linux/smp.h b/include/linux/smp.h index fcd61dfe2af3..6a0813c905d0 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -109,7 +109,7 @@ static inline void on_each_cpu_cond(smp_cond_func_t cond_func, * Architecture specific boot CPU setup. Defined as empty weak function in * init/main.c. Architectures can override it. */ -void smp_prepare_boot_cpu(void); +void __init smp_prepare_boot_cpu(void); #ifdef CONFIG_SMP From a6fe30d1e3657991c832702cecb44576128d7fa3 Mon Sep 17 00:00:00 2001 From: Costa Shulyupin Date: Fri, 6 Sep 2024 20:01:40 +0300 Subject: [PATCH 173/180] genirq: Use cpumask_intersects() Replace `cpumask_any_and(a, b) >= nr_cpu_ids` and `cpumask_any_and(a, b) < nr_cpu_ids` with the more readable `!cpumask_intersects(a, b)` and `cpumask_intersects(a, b)` Signed-off-by: Costa Shulyupin Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240906170142.1135207-1-costa.shul@redhat.com --- kernel/irq/chip.c | 2 +- kernel/irq/migration.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c index dc94e0bf2c94..271e9139de77 100644 --- a/kernel/irq/chip.c +++ b/kernel/irq/chip.c @@ -198,7 +198,7 @@ __irq_startup_managed(struct irq_desc *desc, const struct cpumask *aff, irqd_clr_managed_shutdown(d); - if (cpumask_any_and(aff, cpu_online_mask) >= nr_cpu_ids) { + if (!cpumask_intersects(aff, cpu_online_mask)) { /* * Catch code which fiddles with enable_irq() on a managed * and potentially shutdown IRQ. Chained interrupt diff --git a/kernel/irq/migration.c b/kernel/irq/migration.c index 61ca924ef4b4..eb150afd671f 100644 --- a/kernel/irq/migration.c +++ b/kernel/irq/migration.c @@ -26,7 +26,7 @@ bool irq_fixup_move_pending(struct irq_desc *desc, bool force_clear) * The outgoing CPU might be the last online target in a pending * interrupt move. If that's the case clear the pending move bit. */ - if (cpumask_any_and(desc->pending_mask, cpu_online_mask) >= nr_cpu_ids) { + if (!cpumask_intersects(desc->pending_mask, cpu_online_mask)) { irqd_clr_move_pending(data); return false; } @@ -74,7 +74,7 @@ void irq_move_masked_irq(struct irq_data *idata) * For correct operation this depends on the caller * masking the irqs. */ - if (cpumask_any_and(desc->pending_mask, cpu_online_mask) < nr_cpu_ids) { + if (cpumask_intersects(desc->pending_mask, cpu_online_mask)) { int ret; ret = irq_do_set_affinity(data, desc->pending_mask, false); From fe90c5ba88ad43d42acefb21b57df837be86a61a Mon Sep 17 00:00:00 2001 From: Anna-Maria Behnsen Date: Wed, 4 Sep 2024 15:04:51 +0200 Subject: [PATCH 174/180] timers: Rename next_expiry_recalc() to be unique next_expiry_recalc is the name of a function as well as the name of a struct member of struct timer_base. This might lead to confusion. Rename next_expiry_recalc() to timer_recalc_next_expiry(). No functional change. Signed-off-by: Anna-Maria Behnsen Signed-off-by: Thomas Gleixner Reviewed-by: Frederic Weisbecker Link: https://lore.kernel.org/all/20240904-devel-anna-maria-b4-timers-flseep-v1-1-e98760256370@linutronix.de --- kernel/time/timer.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/time/timer.c b/kernel/time/timer.c index 311ea459b976..5e021a2d8d61 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -1900,7 +1900,7 @@ static int next_pending_bucket(struct timer_base *base, unsigned offset, * * Store next expiry time in base->next_expiry. */ -static void next_expiry_recalc(struct timer_base *base) +static void timer_recalc_next_expiry(struct timer_base *base) { unsigned long clk, next, adj; unsigned lvl, offset = 0; @@ -2009,7 +2009,7 @@ static unsigned long next_timer_interrupt(struct timer_base *base, unsigned long basej) { if (base->next_expiry_recalc) - next_expiry_recalc(base); + timer_recalc_next_expiry(base); /* * Move next_expiry for the empty base into the future to prevent an @@ -2413,7 +2413,7 @@ static inline void __run_timers(struct timer_base *base) * jiffies to avoid endless requeuing to current jiffies. */ base->clk++; - next_expiry_recalc(base); + timer_recalc_next_expiry(base); while (levels--) expire_timers(base, heads + levels); From 662a1bfb907cd850da4de9b871640ad47a355bc0 Mon Sep 17 00:00:00 2001 From: Anna-Maria Behnsen Date: Wed, 4 Sep 2024 15:04:52 +0200 Subject: [PATCH 175/180] cpu: Use already existing usleep_range() usleep_range() is a wrapper arount usleep_range_state() which hands in TASK_UNTINTERRUPTIBLE as state argument. Use already exising wrapper usleep_range(). No functional change. Signed-off-by: Anna-Maria Behnsen Signed-off-by: Thomas Gleixner Reviewed-by: Frederic Weisbecker Link: https://lore.kernel.org/all/20240904-devel-anna-maria-b4-timers-flseep-v1-2-e98760256370@linutronix.de --- kernel/cpu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/cpu.c b/kernel/cpu.c index 1209ddaec026..031a2c15481b 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -330,7 +330,7 @@ static bool cpuhp_wait_for_sync_state(unsigned int cpu, enum cpuhp_sync_state st /* Poll for one millisecond */ arch_cpuhp_sync_state_poll(); } else { - usleep_range_state(USEC_PER_MSEC, 2 * USEC_PER_MSEC, TASK_UNINTERRUPTIBLE); + usleep_range(USEC_PER_MSEC, 2 * USEC_PER_MSEC); } sync = atomic_read(st); } From bd7c8ff9fef4b21a97f9b30a7364845ee6eaaf23 Mon Sep 17 00:00:00 2001 From: Anna-Maria Behnsen Date: Wed, 4 Sep 2024 15:04:53 +0200 Subject: [PATCH 176/180] treewide: Fix wrong singular form of jiffies in comments There are several comments all over the place, which uses a wrong singular form of jiffies. Replace 'jiffie' by 'jiffy'. No functional change. Signed-off-by: Anna-Maria Behnsen Signed-off-by: Thomas Gleixner Acked-by: Geert Uytterhoeven # m68k Link: https://lore.kernel.org/all/20240904-devel-anna-maria-b4-timers-flseep-v1-3-e98760256370@linutronix.de --- Documentation/admin-guide/media/vivid.rst | 2 +- Documentation/timers/timers-howto.rst | 2 +- .../sp_SP/scheduler/sched-design-CFS.rst | 2 +- arch/arm/mach-versatile/spc.c | 2 +- arch/m68k/q40/q40ints.c | 2 +- arch/x86/kernel/cpu/mce/dev-mcelog.c | 2 +- drivers/char/ipmi/ipmi_ssif.c | 2 +- drivers/dma-buf/st-dma-fence.c | 2 +- drivers/gpu/drm/i915/gem/i915_gem_wait.c | 2 +- drivers/gpu/drm/i915/gt/selftest_execlists.c | 4 ++-- drivers/gpu/drm/i915/i915_utils.c | 2 +- drivers/gpu/drm/v3d/v3d_bo.c | 2 +- drivers/isdn/mISDN/dsp_cmx.c | 2 +- drivers/net/ethernet/marvell/mvmdio.c | 2 +- fs/xfs/xfs_buf.h | 2 +- include/linux/jiffies.h | 2 +- include/linux/timekeeper_internal.h | 2 +- kernel/time/alarmtimer.c | 2 +- kernel/time/clockevents.c | 2 +- kernel/time/hrtimer.c | 2 +- kernel/time/posix-timers.c | 4 ++-- kernel/time/timer.c | 12 ++++++------ lib/Kconfig.debug | 2 +- net/batman-adv/types.h | 2 +- 24 files changed, 31 insertions(+), 31 deletions(-) diff --git a/Documentation/admin-guide/media/vivid.rst b/Documentation/admin-guide/media/vivid.rst index 1306f19ecb5a..c9d301ab46a3 100644 --- a/Documentation/admin-guide/media/vivid.rst +++ b/Documentation/admin-guide/media/vivid.rst @@ -328,7 +328,7 @@ and an HDMI input, one input for each input type. Those are described in more detail below. Special attention has been given to the rate at which new frames become -available. The jitter will be around 1 jiffie (that depends on the HZ +available. The jitter will be around 1 jiffy (that depends on the HZ configuration of your kernel, so usually 1/100, 1/250 or 1/1000 of a second), but the long-term behavior is exactly following the framerate. So a framerate of 59.94 Hz is really different from 60 Hz. If the framerate diff --git a/Documentation/timers/timers-howto.rst b/Documentation/timers/timers-howto.rst index 5c169e3d29a8..ef7a4652ccc9 100644 --- a/Documentation/timers/timers-howto.rst +++ b/Documentation/timers/timers-howto.rst @@ -19,7 +19,7 @@ it really need to delay in atomic context?" If so... ATOMIC CONTEXT: You must use the `*delay` family of functions. These - functions use the jiffie estimation of clock speed + functions use the jiffy estimation of clock speed and will busy wait for enough loop cycles to achieve the desired delay: diff --git a/Documentation/translations/sp_SP/scheduler/sched-design-CFS.rst b/Documentation/translations/sp_SP/scheduler/sched-design-CFS.rst index 90a153cad4e8..731c266beb1a 100644 --- a/Documentation/translations/sp_SP/scheduler/sched-design-CFS.rst +++ b/Documentation/translations/sp_SP/scheduler/sched-design-CFS.rst @@ -109,7 +109,7 @@ para que se ejecute, y la tarea en ejecución es interrumpida. ================================== CFS usa una granularidad de nanosegundos y no depende de ningún -jiffie o detalles como HZ. De este modo, el gestor de tareas CFS no tiene +jiffy o detalles como HZ. De este modo, el gestor de tareas CFS no tiene noción de "ventanas de tiempo" de la forma en que tenía el gestor de tareas previo, y tampoco tiene heurísticos. Únicamente hay un parámetro central ajustable (se ha de cambiar en CONFIG_SCHED_DEBUG): diff --git a/arch/arm/mach-versatile/spc.c b/arch/arm/mach-versatile/spc.c index 5e44170e1a9a..790092734cf6 100644 --- a/arch/arm/mach-versatile/spc.c +++ b/arch/arm/mach-versatile/spc.c @@ -73,7 +73,7 @@ /* * Even though the SPC takes max 3-5 ms to complete any OPP/COMMS - * operation, the operation could start just before jiffie is about + * operation, the operation could start just before jiffy is about * to be incremented. So setting timeout value of 20ms = 2jiffies@100Hz */ #define TIMEOUT_US 20000 diff --git a/arch/m68k/q40/q40ints.c b/arch/m68k/q40/q40ints.c index 10f1f294e91f..14b774b9d308 100644 --- a/arch/m68k/q40/q40ints.c +++ b/arch/m68k/q40/q40ints.c @@ -106,7 +106,7 @@ void __init q40_init_IRQ(void) * this stuff doesn't really belong here.. */ -int ql_ticks; /* 200Hz ticks since last jiffie */ +int ql_ticks; /* 200Hz ticks since last jiffy */ static int sound_ticks; #define SVOL 45 diff --git a/arch/x86/kernel/cpu/mce/dev-mcelog.c b/arch/x86/kernel/cpu/mce/dev-mcelog.c index a05ac0716ecf..a3aa0199222e 100644 --- a/arch/x86/kernel/cpu/mce/dev-mcelog.c +++ b/arch/x86/kernel/cpu/mce/dev-mcelog.c @@ -314,7 +314,7 @@ static ssize_t mce_chrdev_write(struct file *filp, const char __user *ubuf, /* * Need to give user space some time to set everything up, - * so do it a jiffie or two later everywhere. + * so do it a jiffy or two later everywhere. */ schedule_timeout(2); diff --git a/drivers/char/ipmi/ipmi_ssif.c b/drivers/char/ipmi/ipmi_ssif.c index 96ad571d041a..e093028391af 100644 --- a/drivers/char/ipmi/ipmi_ssif.c +++ b/drivers/char/ipmi/ipmi_ssif.c @@ -980,7 +980,7 @@ static void msg_written_handler(struct ssif_info *ssif_info, int result, ipmi_ssif_unlock_cond(ssif_info, flags); start_get(ssif_info); } else { - /* Wait a jiffie then request the next message */ + /* Wait a jiffy then request the next message */ ssif_info->waiting_alert = true; ssif_info->retries_left = SSIF_RECV_RETRIES; if (!ssif_info->stopping) diff --git a/drivers/dma-buf/st-dma-fence.c b/drivers/dma-buf/st-dma-fence.c index 6a1bfcd0cc21..cf2ce3744ce6 100644 --- a/drivers/dma-buf/st-dma-fence.c +++ b/drivers/dma-buf/st-dma-fence.c @@ -402,7 +402,7 @@ static int test_wait_timeout(void *arg) if (dma_fence_wait_timeout(wt.f, false, 2) == -ETIME) { if (timer_pending(&wt.timer)) { - pr_notice("Timer did not fire within the jiffie!\n"); + pr_notice("Timer did not fire within the jiffy!\n"); err = 0; /* not our fault! */ } else { pr_err("Wait reported incomplete after timeout\n"); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_wait.c b/drivers/gpu/drm/i915/gem/i915_gem_wait.c index d4b918fb11ce..1f55e62044a4 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_wait.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_wait.c @@ -266,7 +266,7 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns)) args->timeout_ns = 0; - /* Asked to wait beyond the jiffie/scheduler precision? */ + /* Asked to wait beyond the jiffy/scheduler precision? */ if (ret == -ETIME && args->timeout_ns) ret = -EAGAIN; } diff --git a/drivers/gpu/drm/i915/gt/selftest_execlists.c b/drivers/gpu/drm/i915/gt/selftest_execlists.c index 4202df5b8c12..222ca7c44951 100644 --- a/drivers/gpu/drm/i915/gt/selftest_execlists.c +++ b/drivers/gpu/drm/i915/gt/selftest_execlists.c @@ -93,7 +93,7 @@ static int wait_for_reset(struct intel_engine_cs *engine, return -EINVAL; } - /* Give the request a jiffie to complete after flushing the worker */ + /* Give the request a jiffy to complete after flushing the worker */ if (i915_request_wait(rq, 0, max(0l, (long)(timeout - jiffies)) + 1) < 0) { pr_err("%s: hanging request %llx:%lld did not complete\n", @@ -3426,7 +3426,7 @@ static int live_preempt_timeout(void *arg) cpu_relax(); saved_timeout = engine->props.preempt_timeout_ms; - engine->props.preempt_timeout_ms = 1; /* in ms, -> 1 jiffie */ + engine->props.preempt_timeout_ms = 1; /* in ms, -> 1 jiffy */ i915_request_get(rq); i915_request_add(rq); diff --git a/drivers/gpu/drm/i915/i915_utils.c b/drivers/gpu/drm/i915/i915_utils.c index 6f9e7b354b54..f2ba51c20e97 100644 --- a/drivers/gpu/drm/i915/i915_utils.c +++ b/drivers/gpu/drm/i915/i915_utils.c @@ -110,7 +110,7 @@ void set_timer_ms(struct timer_list *t, unsigned long timeout) * Paranoia to make sure the compiler computes the timeout before * loading 'jiffies' as jiffies is volatile and may be updated in * the background by a timer tick. All to reduce the complexity - * of the addition and reduce the risk of losing a jiffie. + * of the addition and reduce the risk of losing a jiffy. */ barrier(); diff --git a/drivers/gpu/drm/v3d/v3d_bo.c b/drivers/gpu/drm/v3d/v3d_bo.c index a165cbcdd27b..9eafe53a8f41 100644 --- a/drivers/gpu/drm/v3d/v3d_bo.c +++ b/drivers/gpu/drm/v3d/v3d_bo.c @@ -279,7 +279,7 @@ v3d_wait_bo_ioctl(struct drm_device *dev, void *data, else args->timeout_ns = 0; - /* Asked to wait beyond the jiffie/scheduler precision? */ + /* Asked to wait beyond the jiffy/scheduler precision? */ if (ret == -ETIME && args->timeout_ns) ret = -EAGAIN; diff --git a/drivers/isdn/mISDN/dsp_cmx.c b/drivers/isdn/mISDN/dsp_cmx.c index 61cb45c5d0d8..53fad9487574 100644 --- a/drivers/isdn/mISDN/dsp_cmx.c +++ b/drivers/isdn/mISDN/dsp_cmx.c @@ -82,7 +82,7 @@ * - has multiple clocks. * - has no usable clock due to jitter or packet loss (VoIP). * In this case the system's clock is used. The clock resolution depends on - * the jiffie resolution. + * the jiffy resolution. * * If a member joins a conference: * diff --git a/drivers/net/ethernet/marvell/mvmdio.c b/drivers/net/ethernet/marvell/mvmdio.c index 9190eff6c0bb..e1d003fdbc2e 100644 --- a/drivers/net/ethernet/marvell/mvmdio.c +++ b/drivers/net/ethernet/marvell/mvmdio.c @@ -104,7 +104,7 @@ static int orion_mdio_wait_ready(const struct orion_mdio_ops *ops, return 0; } else { /* wait_event_timeout does not guarantee a delay of at - * least one whole jiffie, so timeout must be no less + * least one whole jiffy, so timeout must be no less * than two. */ timeout = max(usecs_to_jiffies(MVMDIO_SMI_TIMEOUT), 2); diff --git a/fs/xfs/xfs_buf.h b/fs/xfs/xfs_buf.h index b1580644501f..209a389f2abc 100644 --- a/fs/xfs/xfs_buf.h +++ b/fs/xfs/xfs_buf.h @@ -210,7 +210,7 @@ struct xfs_buf { * success the write is considered to be failed permanently and the * iodone handler will take appropriate action. * - * For retry timeouts, we record the jiffie of the first failure. This + * For retry timeouts, we record the jiffy of the first failure. This * means that we can change the retry timeout for buffers already under * I/O and thus avoid getting stuck in a retry loop with a long timeout. * diff --git a/include/linux/jiffies.h b/include/linux/jiffies.h index d9f1435a5a13..1220f0fbe5bf 100644 --- a/include/linux/jiffies.h +++ b/include/linux/jiffies.h @@ -418,7 +418,7 @@ extern unsigned long preset_lpj; #define NSEC_CONVERSION ((unsigned long)((((u64)1 << NSEC_JIFFIE_SC) +\ TICK_NSEC -1) / (u64)TICK_NSEC)) /* - * The maximum jiffie value is (MAX_INT >> 1). Here we translate that + * The maximum jiffy value is (MAX_INT >> 1). Here we translate that * into seconds. The 64-bit case will overflow if we are not careful, * so use the messy SH_DIV macro to do it. Still all constants. */ diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h index 84ff2844df2a..902c20ef495a 100644 --- a/include/linux/timekeeper_internal.h +++ b/include/linux/timekeeper_internal.h @@ -73,7 +73,7 @@ struct tk_read_base { * @overflow_seen: Overflow warning flag (DEBUG_TIMEKEEPING) * * Note: For timespec(64) based interfaces wall_to_monotonic is what - * we need to add to xtime (or xtime corrected for sub jiffie times) + * we need to add to xtime (or xtime corrected for sub jiffy times) * to get to monotonic time. Monotonic is pegged at zero at system * boot time, so wall_to_monotonic will be negative, however, we will * ALWAYS keep the tv_nsec part positive so we can use the usual diff --git a/kernel/time/alarmtimer.c b/kernel/time/alarmtimer.c index 76bd4fda3472..8bf888641694 100644 --- a/kernel/time/alarmtimer.c +++ b/kernel/time/alarmtimer.c @@ -493,7 +493,7 @@ static u64 __alarm_forward_now(struct alarm *alarm, ktime_t interval, bool throt * promised in the context of posix_timer_fn() never * materialized, but someone should really work on it. * - * To prevent DOS fake @now to be 1 jiffie out which keeps + * To prevent DOS fake @now to be 1 jiffy out which keeps * the overrun accounting correct but creates an * inconsistency vs. timer_gettime(2). */ diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c index 60a6484831b1..78c7bd64d0dd 100644 --- a/kernel/time/clockevents.c +++ b/kernel/time/clockevents.c @@ -190,7 +190,7 @@ int clockevents_tick_resume(struct clock_event_device *dev) #ifdef CONFIG_GENERIC_CLOCKEVENTS_MIN_ADJUST -/* Limit min_delta to a jiffie */ +/* Limit min_delta to a jiffy */ #define MIN_DELTA_LIMIT (NSEC_PER_SEC / HZ) /** diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index a023946f8558..e834b2bd83df 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1177,7 +1177,7 @@ static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim, /* * CONFIG_TIME_LOW_RES indicates that the system has no way to return * granular time values. For relative timers we add hrtimer_resolution - * (i.e. one jiffie) to prevent short timeouts. + * (i.e. one jiffy) to prevent short timeouts. */ timer->is_rel = mode & HRTIMER_MODE_REL; if (timer->is_rel) diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c index 1cc830ef93a7..4576aaed13b2 100644 --- a/kernel/time/posix-timers.c +++ b/kernel/time/posix-timers.c @@ -339,14 +339,14 @@ static enum hrtimer_restart posix_timer_fn(struct hrtimer *timer) * change to the signal handling code. * * For now let timers with an interval less than a - * jiffie expire every jiffie and recheck for a + * jiffy expire every jiffy and recheck for a * valid signal handler. * * This avoids interrupt starvation in case of a * very small interval, which would expire the * timer immediately again. * - * Moving now ahead of time by one jiffie tricks + * Moving now ahead of time by one jiffy tricks * hrtimer_forward() to expire the timer later, * while it still maintains the overrun accuracy * for the price of a slight inconsistency in the diff --git a/kernel/time/timer.c b/kernel/time/timer.c index 5e021a2d8d61..2b38f3035a3e 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -365,7 +365,7 @@ static unsigned long round_jiffies_common(unsigned long j, int cpu, rem = j % HZ; /* - * If the target jiffie is just after a whole second (which can happen + * If the target jiffy is just after a whole second (which can happen * due to delays of the timer irq, long irq off times etc etc) then * we should round down to the whole second, not up. Use 1/4th second * as cutoff for this rounding as an extreme upper bound for this. @@ -1930,7 +1930,7 @@ static void timer_recalc_next_expiry(struct timer_base *base) * bits are zero, we look at the next level as is. If not we * need to advance it by one because that's going to be the * next expiring bucket in that level. base->clk is the next - * expiring jiffie. So in case of: + * expiring jiffy. So in case of: * * LVL5 LVL4 LVL3 LVL2 LVL1 LVL0 * 0 0 0 0 0 0 @@ -1995,7 +1995,7 @@ static u64 cmp_next_hrtimer_event(u64 basem, u64 expires) return basem; /* - * Round up to the next jiffie. High resolution timers are + * Round up to the next jiffy. High resolution timers are * off, so the hrtimers are expired in the tick and we need to * make sure that this tick really expires the timer to avoid * a ping pong of the nohz stop code. @@ -2254,7 +2254,7 @@ static inline u64 __get_next_timer_interrupt(unsigned long basej, u64 basem, base_global, &tevt); /* - * If the next event is only one jiffie ahead there is no need to call + * If the next event is only one jiffy ahead there is no need to call * timer migration hierarchy related functions. The value for the next * global timer in @tevt struct equals then KTIME_MAX. This is also * true, when the timer base is idle. @@ -2486,11 +2486,11 @@ static void run_local_timers(void) * updated. When this update is missed, this isn't a * problem, as an IPI is executed nevertheless when the CPU * was idle before. When the CPU wasn't idle but the update - * is missed, then the timer would expire one jiffie late - + * is missed, then the timer would expire one jiffy late - * bad luck. * * Those unlikely corner cases where the worst outcome is only a - * one jiffie delay or a superfluous raise of the softirq are + * one jiffy delay or a superfluous raise of the softirq are * not that expensive as doing the check always while holding * the lock. * diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index a30c03a66172..a40aa606cd04 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -97,7 +97,7 @@ config BOOT_PRINTK_DELAY using "boot_delay=N". It is likely that you would also need to use "lpj=M" to preset - the "loops per jiffie" value. + the "loops per jiffy" value. See a previous boot log for the "lpj" value to use for your system, and then set "lpj=M" before setting "boot_delay=N". NOTE: Using this option may adversely affect SMP systems. diff --git a/net/batman-adv/types.h b/net/batman-adv/types.h index 00840d5784fe..04f6398b3a40 100644 --- a/net/batman-adv/types.h +++ b/net/batman-adv/types.h @@ -287,7 +287,7 @@ struct batadv_frag_table_entry { /** @lock: lock to protect the list of fragments */ spinlock_t lock; - /** @timestamp: time (jiffie) of last received fragment */ + /** @timestamp: time (jiffy) of last received fragment */ unsigned long timestamp; /** @seqno: sequence number of the fragments in the list */ From e4757c710ba27a1d499cf5941fc3950e9e542731 Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Wed, 4 Sep 2024 21:39:39 +0800 Subject: [PATCH 177/180] debugobjects: Fix the compilation attributes of some global variables 1. Both debug_objects_pool_min_level and debug_objects_pool_size are read-only after initialization, change attribute '__read_mostly' to '__ro_after_init', and remove '__data_racy'. 2. Many global variables are read in the debug_stats_show() function, but didn't mask KCSAN's detection. Add '__data_racy' for them. Suggested-by: Thomas Gleixner Signed-off-by: Zhen Lei Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240904133944.2124-2-thunder.leizhen@huawei.com --- lib/debugobjects.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/debugobjects.c b/lib/debugobjects.c index 7cea91e193a8..7226fdb5e129 100644 --- a/lib/debugobjects.c +++ b/lib/debugobjects.c @@ -70,10 +70,10 @@ static HLIST_HEAD(obj_to_free); * made at debug_stats_show(). Both obj_pool_min_free and obj_pool_max_used * can be off. */ -static int obj_pool_min_free = ODEBUG_POOL_SIZE; -static int obj_pool_free = ODEBUG_POOL_SIZE; +static int __data_racy obj_pool_min_free = ODEBUG_POOL_SIZE; +static int __data_racy obj_pool_free = ODEBUG_POOL_SIZE; static int obj_pool_used; -static int obj_pool_max_used; +static int __data_racy obj_pool_max_used; static bool obj_freeing; /* The number of objs on the global free list */ static int obj_nr_tofree; @@ -84,9 +84,9 @@ static int __data_racy debug_objects_fixups __read_mostly; static int __data_racy debug_objects_warnings __read_mostly; static int __data_racy debug_objects_enabled __read_mostly = CONFIG_DEBUG_OBJECTS_ENABLE_DEFAULT; -static int __data_racy debug_objects_pool_size __read_mostly +static int debug_objects_pool_size __ro_after_init = ODEBUG_POOL_SIZE; -static int __data_racy debug_objects_pool_min_level __read_mostly +static int debug_objects_pool_min_level __ro_after_init = ODEBUG_POOL_MIN_LEVEL; static const struct debug_obj_descr *descr_test __read_mostly; @@ -95,8 +95,8 @@ static struct kmem_cache *obj_cache __ro_after_init; /* * Track numbers of kmem_cache_alloc()/free() calls done. */ -static int debug_objects_allocated; -static int debug_objects_freed; +static int __data_racy debug_objects_allocated; +static int __data_racy debug_objects_freed; static void free_obj_work(struct work_struct *work); static DECLARE_DELAYED_WORK(debug_obj_work, free_obj_work); From 684d28feb8546d1e9597aa363c3bfcf52fe250b7 Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Wed, 4 Sep 2024 21:39:40 +0800 Subject: [PATCH 178/180] debugobjects: Fix conditions in fill_pool() fill_pool() uses 'obj_pool_min_free' to decide whether objects should be handed back to the kmem cache. But 'obj_pool_min_free' records the lowest historical value of the number of objects in the object pool and not the minimum number of objects which should be kept in the pool. Use 'debug_objects_pool_min_level' instead, which holds the minimum number which was scaled to the number of CPUs at boot time. [ tglx: Massage change log ] Fixes: d26bf5056fc0 ("debugobjects: Reduce number of pool_lock acquisitions in fill_pool()") Fixes: 36c4ead6f6df ("debugobjects: Add global free list and the counter") Signed-off-by: Zhen Lei Signed-off-by: Thomas Gleixner Cc: stable@vger.kernel.org Link: https://lore.kernel.org/all/20240904133944.2124-3-thunder.leizhen@huawei.com --- lib/debugobjects.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/debugobjects.c b/lib/debugobjects.c index 7226fdb5e129..6329a86edcf1 100644 --- a/lib/debugobjects.c +++ b/lib/debugobjects.c @@ -142,13 +142,14 @@ static void fill_pool(void) * READ_ONCE()s pair with the WRITE_ONCE()s in pool_lock critical * sections. */ - while (READ_ONCE(obj_nr_tofree) && (READ_ONCE(obj_pool_free) < obj_pool_min_free)) { + while (READ_ONCE(obj_nr_tofree) && + READ_ONCE(obj_pool_free) < debug_objects_pool_min_level) { raw_spin_lock_irqsave(&pool_lock, flags); /* * Recheck with the lock held as the worker thread might have * won the race and freed the global free list already. */ - while (obj_nr_tofree && (obj_pool_free < obj_pool_min_free)) { + while (obj_nr_tofree && (obj_pool_free < debug_objects_pool_min_level)) { obj = hlist_entry(obj_to_free.first, typeof(*obj), node); hlist_del(&obj->node); WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1); From 63a4a9b52c3c7f86351710739011717a36652b72 Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Wed, 4 Sep 2024 21:39:41 +0800 Subject: [PATCH 179/180] debugobjects: Remove redundant checks in fill_pool() fill_pool() checks locklessly at the beginning whether the pool has to be refilled. After that it checks locklessly in a loop whether the free list contains objects and repeats the refill check. If both conditions are true, it acquires the pool lock and tries to move objects from the free list to the pool repeating the same checks again. There are two redundant issues with that: 1) The repeated check for the fill condition 2) The loop processing The repeated check is pointless as it was just established that fill is required. The condition has to be re-evaluated under the lock anyway. The loop processing is not required either because there is practically zero chance that a repeated attempt will succeed if the checks under the lock terminate the moving of objects. Remove the redundant check and replace the loop with a simple if condition. [ tglx: Massaged change log ] Signed-off-by: Zhen Lei Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240904133944.2124-4-thunder.leizhen@huawei.com --- lib/debugobjects.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/debugobjects.c b/lib/debugobjects.c index 6329a86edcf1..5ce473ad499b 100644 --- a/lib/debugobjects.c +++ b/lib/debugobjects.c @@ -135,15 +135,13 @@ static void fill_pool(void) return; /* - * Reuse objs from the global free list; they will be reinitialized - * when allocating. + * Reuse objs from the global obj_to_free list; they will be + * reinitialized when allocating. * - * Both obj_nr_tofree and obj_pool_free are checked locklessly; the - * READ_ONCE()s pair with the WRITE_ONCE()s in pool_lock critical - * sections. + * obj_nr_tofree is checked locklessly; the READ_ONCE() pairs with + * the WRITE_ONCE() in pool_lock critical sections. */ - while (READ_ONCE(obj_nr_tofree) && - READ_ONCE(obj_pool_free) < debug_objects_pool_min_level) { + if (READ_ONCE(obj_nr_tofree)) { raw_spin_lock_irqsave(&pool_lock, flags); /* * Recheck with the lock held as the worker thread might have From 35b603f8a78b0bd51566db277c4f7b56b3ff6bac Mon Sep 17 00:00:00 2001 From: Benjamin ROBIN Date: Sun, 8 Sep 2024 16:08:36 +0200 Subject: [PATCH 180/180] ntp: Make sure RTC is synchronized when time goes backwards sync_hw_clock() is normally called every 11 minutes when time is synchronized. This issue is that this periodic timer uses the REALTIME clock, so when time moves backwards (the NTP server jumps into the past), the timer expires late. If the timer expires late, which can be days later, the RTC will no longer be updated, which is an issue if the device is abruptly powered OFF during this period. When the device will restart (when powered ON), it will have the date prior to the ADJ_SETOFFSET call. A normal NTP server should not jump in the past like that, but it is possible... Another way of reproducing this issue is to use phc2sys to synchronize the REALTIME clock with, for example, an IRIG timecode with the source always starting at the same date (not synchronized). Also, if the time jump in the future by less than 11 minutes, the RTC may not be updated immediately (minor issue). Consider the following scenario: - Time is synchronized, and sync_hw_clock() was just called (the timer expires in 11 minutes). - A time jump is realized in the future by a couple of minutes. - The time is synchronized again. - Users may expect that RTC to be updated as soon as possible, and not after 11 minutes (for the same reason, if a power loss occurs in this period). Cancel periodic timer on any time jump (ADJ_SETOFFSET) greater than or equal to 1s. The timer will be relaunched at the end of do_adjtimex() if NTP is still considered synced. Otherwise the timer will be relaunched later when NTP is synced. This way, when the time is synchronized again, the RTC is updated after less than 2 seconds. Signed-off-by: Benjamin ROBIN Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20240908140836.203911-1-dev@benjarobin.fr --- kernel/time/ntp.c | 10 +++++++++- kernel/time/ntp_internal.h | 4 ++-- kernel/time/timekeeping.c | 4 +++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c index 8d2dd214ec68..802b336f4b8c 100644 --- a/kernel/time/ntp.c +++ b/kernel/time/ntp.c @@ -660,8 +660,16 @@ rearm: sched_sync_hw_clock(offset_nsec, res != 0); } -void ntp_notify_cmos_timer(void) +void ntp_notify_cmos_timer(bool offset_set) { + /* + * If the time jumped (using ADJ_SETOFFSET) cancels sync timer, + * which may have been running if the time was synchronized + * prior to the ADJ_SETOFFSET call. + */ + if (offset_set) + hrtimer_cancel(&sync_hrtimer); + /* * When the work is currently executed but has not yet the timer * rearmed this queues the work immediately again. No big issue, diff --git a/kernel/time/ntp_internal.h b/kernel/time/ntp_internal.h index 23d1b74c3065..5a633dce9057 100644 --- a/kernel/time/ntp_internal.h +++ b/kernel/time/ntp_internal.h @@ -14,9 +14,9 @@ extern int __do_adjtimex(struct __kernel_timex *txc, extern void __hardpps(const struct timespec64 *phase_ts, const struct timespec64 *raw_ts); #if defined(CONFIG_GENERIC_CMOS_UPDATE) || defined(CONFIG_RTC_SYSTOHC) -extern void ntp_notify_cmos_timer(void); +extern void ntp_notify_cmos_timer(bool offset_set); #else -static inline void ntp_notify_cmos_timer(void) { } +static inline void ntp_notify_cmos_timer(bool offset_set) { } #endif #endif /* _LINUX_NTP_INTERNAL_H */ diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 5391e4167d60..7e6f409bf311 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -2553,6 +2553,7 @@ int do_adjtimex(struct __kernel_timex *txc) { struct timekeeper *tk = &tk_core.timekeeper; struct audit_ntp_data ad; + bool offset_set = false; bool clock_set = false; struct timespec64 ts; unsigned long flags; @@ -2575,6 +2576,7 @@ int do_adjtimex(struct __kernel_timex *txc) if (ret) return ret; + offset_set = delta.tv_sec != 0; audit_tk_injoffset(delta); } @@ -2608,7 +2610,7 @@ int do_adjtimex(struct __kernel_timex *txc) if (clock_set) clock_was_set(CLOCK_SET_WALL); - ntp_notify_cmos_timer(); + ntp_notify_cmos_timer(offset_set); return ret; }