Tianyang Zhang
111e55db3c
commit 2468b0e3d5659dfde77f081f266e1111a981efb8 upstream.
When CONFIG_PREEMPT_COUNT is not configured (i.e. CONFIG_PREEMPT_NONE/
CONFIG_PREEMPT_VOLUNTARY), preempt_disable() / preempt_enable() merely
acts as a barrier(). However, in these cases cond_resched() can still
trigger a context switch and modify the CSR.EUEN, resulting in do_fpu()
exception being activated within the kernel-fpu critical sections, as
demonstrated in the following path:
dcn32_calculate_wm_and_dlg()
DC_FP_START()
dcn32_calculate_wm_and_dlg_fpu()
dcn32_find_dummy_latency_index_for_fw_based_mclk_switch()
dcn32_internal_validate_bw()
dcn32_enable_phantom_stream()
dc_create_stream_for_sink()
kzalloc(GFP_KERNEL)
__kmem_cache_alloc_node()
__cond_resched()
DC_FP_END()
This patch is similar to commit d02198550423a0b (x86/fpu: Improve crypto
performance by making kernel-mode FPU reliably usable in softirqs). It
uses local_bh_disable() instead of preempt_disable() for non-RT kernels
so it can avoid the cond_resched() issue, and also extend the kernel-fpu
application scenarios to the softirq context.
Cc: stable@vger.kernel.org
Signed-off-by: Tianyang Zhang <zhangtianyang@loongson.cn>
Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
109 lines
2.2 KiB
C
109 lines
2.2 KiB
C
// SPDX-License-Identifier: GPL-2.0-only
|
|
/*
|
|
* Copyright (C) 2023 Loongson Technology Corporation Limited
|
|
*/
|
|
|
|
#include <linux/cpu.h>
|
|
#include <linux/init.h>
|
|
#include <asm/fpu.h>
|
|
#include <asm/smp.h>
|
|
|
|
static unsigned int euen_mask = CSR_EUEN_FPEN;
|
|
|
|
/*
|
|
* The critical section between kernel_fpu_begin() and kernel_fpu_end()
|
|
* is non-reentrant. It is the caller's responsibility to avoid reentrance.
|
|
* See drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c as an example.
|
|
*/
|
|
static DEFINE_PER_CPU(bool, in_kernel_fpu);
|
|
static DEFINE_PER_CPU(unsigned int, euen_current);
|
|
|
|
static inline void fpregs_lock(void)
|
|
{
|
|
if (IS_ENABLED(CONFIG_PREEMPT_RT))
|
|
preempt_disable();
|
|
else
|
|
local_bh_disable();
|
|
}
|
|
|
|
static inline void fpregs_unlock(void)
|
|
{
|
|
if (IS_ENABLED(CONFIG_PREEMPT_RT))
|
|
preempt_enable();
|
|
else
|
|
local_bh_enable();
|
|
}
|
|
|
|
void kernel_fpu_begin(void)
|
|
{
|
|
unsigned int *euen_curr;
|
|
|
|
if (!irqs_disabled())
|
|
fpregs_lock();
|
|
|
|
WARN_ON(this_cpu_read(in_kernel_fpu));
|
|
|
|
this_cpu_write(in_kernel_fpu, true);
|
|
euen_curr = this_cpu_ptr(&euen_current);
|
|
|
|
*euen_curr = csr_xchg32(euen_mask, euen_mask, LOONGARCH_CSR_EUEN);
|
|
|
|
#ifdef CONFIG_CPU_HAS_LASX
|
|
if (*euen_curr & CSR_EUEN_LASXEN)
|
|
_save_lasx(¤t->thread.fpu);
|
|
else
|
|
#endif
|
|
#ifdef CONFIG_CPU_HAS_LSX
|
|
if (*euen_curr & CSR_EUEN_LSXEN)
|
|
_save_lsx(¤t->thread.fpu);
|
|
else
|
|
#endif
|
|
if (*euen_curr & CSR_EUEN_FPEN)
|
|
_save_fp(¤t->thread.fpu);
|
|
|
|
write_fcsr(LOONGARCH_FCSR0, 0);
|
|
}
|
|
EXPORT_SYMBOL_GPL(kernel_fpu_begin);
|
|
|
|
void kernel_fpu_end(void)
|
|
{
|
|
unsigned int *euen_curr;
|
|
|
|
WARN_ON(!this_cpu_read(in_kernel_fpu));
|
|
|
|
euen_curr = this_cpu_ptr(&euen_current);
|
|
|
|
#ifdef CONFIG_CPU_HAS_LASX
|
|
if (*euen_curr & CSR_EUEN_LASXEN)
|
|
_restore_lasx(¤t->thread.fpu);
|
|
else
|
|
#endif
|
|
#ifdef CONFIG_CPU_HAS_LSX
|
|
if (*euen_curr & CSR_EUEN_LSXEN)
|
|
_restore_lsx(¤t->thread.fpu);
|
|
else
|
|
#endif
|
|
if (*euen_curr & CSR_EUEN_FPEN)
|
|
_restore_fp(¤t->thread.fpu);
|
|
|
|
*euen_curr = csr_xchg32(*euen_curr, euen_mask, LOONGARCH_CSR_EUEN);
|
|
|
|
this_cpu_write(in_kernel_fpu, false);
|
|
|
|
if (!irqs_disabled())
|
|
fpregs_unlock();
|
|
}
|
|
EXPORT_SYMBOL_GPL(kernel_fpu_end);
|
|
|
|
static int __init init_euen_mask(void)
|
|
{
|
|
if (cpu_has_lsx)
|
|
euen_mask |= CSR_EUEN_LSXEN;
|
|
|
|
if (cpu_has_lasx)
|
|
euen_mask |= CSR_EUEN_LASXEN;
|
|
|
|
return 0;
|
|
}
|
|
arch_initcall(init_euen_mask);
|