From 8769177658d3559c4323200a719dd456d2f2675a Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sat, 18 Mar 2023 17:24:27 +0100 Subject: [PATCH 0001/4029] ARM: vfp: Record VFP bounces as perf emulation faults VFP 'bouncing' occurs when the VFP unit cannot complete the execution of a VFP instruction, either because it is not implemented at all, or because the values of the arguments are out of range for the hardware implementation, and the software needs to step in to complete the operation. To give some insight in how much certain programs rely on this bouncing, record the emulation of a VFP instruction in perf's emulation-faults counter. This can be used like so perf stat -e emulation-faults ./testfloat -all2 and the output will be something like Performance counter stats for './testfloat -all2': 259,277 emulation-faults:u 6.846432176 seconds time elapsed Reviewed-by: Linus Walleij Signed-off-by: Ard Biesheuvel --- arch/arm/vfp/vfpmodule.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c index 349dcb944a93..08d5dfcf7079 100644 --- a/arch/arm/vfp/vfpmodule.c +++ b/arch/arm/vfp/vfpmodule.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -313,6 +314,7 @@ static u32 vfp_emulate_instruction(u32 inst, u32 fpscr, struct pt_regs *regs) * emulate it. */ } + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->ARM_pc); return exceptions & ~VFP_NAN_FLAG; } From 4a0548c6681cd25b8d76e897e01bfb62ce93916d Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 20 Mar 2023 11:01:16 +0100 Subject: [PATCH 0002/4029] ARM: vfp: Remove workaround for Feroceon CPUs Feroceon CPUs have a non-standard implementation of VFP which reports synchronous VFP exceptions using the async VFP flag. This requires a workaround which is difficult to reconcile with other implementations, making it tricky to support both versions in a single image. Since this is a v5 CPU, it is not supported by armhf and so the likelihood that anybody is using this with recent distros/kernels and rely on the VFP at the same time is extremely low. So let's just disable VFP support on these cores, so we can remove the workaround. This will help future development to support v5 and v6 CPUs with a single kernel image. Reviewed-by: Linus Walleij Acked-by: Nicolas Pitre Acked-by: Arnd Bergmann Signed-off-by: Ard Biesheuvel --- arch/arm/mm/proc-feroceon.S | 4 ++++ arch/arm/vfp/vfphw.S | 4 ---- arch/arm/vfp/vfpmodule.c | 8 +++++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/arch/arm/mm/proc-feroceon.S b/arch/arm/mm/proc-feroceon.S index 61ce82aca6f0..072ff9b451f8 100644 --- a/arch/arm/mm/proc-feroceon.S +++ b/arch/arm/mm/proc-feroceon.S @@ -56,6 +56,10 @@ ENTRY(cpu_feroceon_proc_init) movne r2, r2, lsr #2 @ turned into # of sets sub r2, r2, #(1 << 5) stmia r1, {r2, r3} +#ifdef CONFIG_VFP + mov r1, #1 @ disable quirky VFP + str_l r1, VFP_arch_feroceon, r2 +#endif ret lr /* diff --git a/arch/arm/vfp/vfphw.S b/arch/arm/vfp/vfphw.S index a4610d0f3215..0aeb60ac3b53 100644 --- a/arch/arm/vfp/vfphw.S +++ b/arch/arm/vfp/vfphw.S @@ -110,7 +110,6 @@ ENTRY(vfp_support_entry) beq vfp_reload_hw @ then the hw state needs reloading VFPFSTMIA r4, r5 @ save the working registers VFPFMRX r5, FPSCR @ current status -#ifndef CONFIG_CPU_FEROCEON tst r1, #FPEXC_EX @ is there additional state to save? beq 1f VFPFMRX r6, FPINST @ FPINST (only if FPEXC.EX is set) @@ -118,7 +117,6 @@ ENTRY(vfp_support_entry) beq 1f VFPFMRX r8, FPINST2 @ FPINST2 if needed (and present) 1: -#endif stmia r4, {r1, r5, r6, r8} @ save FPEXC, FPSCR, FPINST, FPINST2 vfp_reload_hw: @@ -153,7 +151,6 @@ vfp_reload_hw: VFPFLDMIA r10, r5 @ reload the working registers while @ FPEXC is in a safe state ldmia r10, {r1, r5, r6, r8} @ load FPEXC, FPSCR, FPINST, FPINST2 -#ifndef CONFIG_CPU_FEROCEON tst r1, #FPEXC_EX @ is there additional state to restore? beq 1f VFPFMXR FPINST, r6 @ restore FPINST (only if FPEXC.EX is set) @@ -161,7 +158,6 @@ vfp_reload_hw: beq 1f VFPFMXR FPINST2, r8 @ FPINST2 if needed (and present) 1: -#endif VFPFMXR FPSCR, r5 @ restore status @ The context stored in the VFP hardware is up to date with this thread diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c index 08d5dfcf7079..95628e57807b 100644 --- a/arch/arm/vfp/vfpmodule.c +++ b/arch/arm/vfp/vfpmodule.c @@ -42,7 +42,11 @@ static bool have_vfp __ro_after_init; * Used in startup: set to non-zero if VFP checks fail * After startup, holds VFP architecture */ -static unsigned int __initdata VFP_arch; +static unsigned int VFP_arch; + +#ifdef CONFIG_CPU_FEROCEON +extern unsigned int VFP_arch_feroceon __alias(VFP_arch); +#endif /* * The pointer to the vfpstate structure of the thread which currently @@ -357,14 +361,12 @@ void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) } if (fpexc & FPEXC_EX) { -#ifndef CONFIG_CPU_FEROCEON /* * Asynchronous exception. The instruction is read from FPINST * and the interrupted instruction has to be restarted. */ trigger = fmrx(FPINST); regs->ARM_pc -= 4; -#endif } else if (!(fpexc & FPEXC_DEX)) { /* * Illegal combination of bits. It can be caused by an From 4708fb041346fa9cc6745dafb8c248a3e2f1075b Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 16 Mar 2023 00:51:42 +0100 Subject: [PATCH 0003/4029] ARM: vfp: Reimplement VFP exception entry in C code En/disabling softirqs from asm code turned out to be trickier than expected, so vfp_support_entry now returns by tail calling __local_enable_bh_ip() and passing the same arguments that a C call to local_bh_enable() would pass. However, this is slightly hacky, as we don't want to carry our own implementation of local_bh_enable(). So let's bite the bullet, and get rid of the asm logic in vfp_support_entry that reasons about whether or not to save and/or reload the VFP state, and about whether or not an FP exception is pending, and only keep the VFP loading logic as a function that is callable from C. Replicate the removed logic in vfp_entry(), and use the exact same reasoning as in the asm code. To emphasize the correspondence, retain some of the asm comments in the C version as well. Signed-off-by: Ard Biesheuvel Acked-by: Linus Walleij --- arch/arm/vfp/entry.S | 12 +-- arch/arm/vfp/vfp.h | 1 + arch/arm/vfp/vfphw.S | 204 +++------------------------------------ arch/arm/vfp/vfpmodule.c | 123 +++++++++++++++++++---- 4 files changed, 124 insertions(+), 216 deletions(-) diff --git a/arch/arm/vfp/entry.S b/arch/arm/vfp/entry.S index 62206ef25037..547c94c62cd3 100644 --- a/arch/arm/vfp/entry.S +++ b/arch/arm/vfp/entry.S @@ -22,10 +22,10 @@ @ IRQs enabled. @ ENTRY(do_vfp) - mov r1, r10 - str lr, [sp, #-8]! - add r3, sp, #4 - str r9, [r3] - bl vfp_entry - ldr pc, [sp], #8 + mov r1, r0 @ pass trigger opcode via R1 + mov r0, sp @ pass struct pt_regs via R0 + bl vfp_support_entry @ dispatch the VFP exception + cmp r0, #0 @ handled successfully? + reteq r9 @ then use R9 as return address + ret lr @ pass to undef handler ENDPROC(do_vfp) diff --git a/arch/arm/vfp/vfp.h b/arch/arm/vfp/vfp.h index 5cd6d5053271..e43a630f8a16 100644 --- a/arch/arm/vfp/vfp.h +++ b/arch/arm/vfp/vfp.h @@ -375,3 +375,4 @@ struct op { }; asmlinkage void vfp_save_state(void *location, u32 fpexc); +asmlinkage u32 vfp_load_state(const void *location); diff --git a/arch/arm/vfp/vfphw.S b/arch/arm/vfp/vfphw.S index 0aeb60ac3b53..d5a03f3c10c5 100644 --- a/arch/arm/vfp/vfphw.S +++ b/arch/arm/vfp/vfphw.S @@ -4,12 +4,6 @@ * * Copyright (C) 2004 ARM Limited. * Written by Deep Blue Solutions Limited. - * - * This code is called from the kernel's undefined instruction trap. - * r1 holds the thread_info pointer - * r3 holds the return address for successful handling. - * lr holds the return address for unrecognised instructions. - * sp points to a struct pt_regs (as defined in include/asm/proc/ptrace.h) */ #include #include @@ -19,20 +13,6 @@ #include #include - .macro DBGSTR, str -#ifdef DEBUG - stmfd sp!, {r0-r3, ip, lr} - ldr r0, =1f - bl _printk - ldmfd sp!, {r0-r3, ip, lr} - - .pushsection .rodata, "a" -1: .ascii KERN_DEBUG "VFP: \str\n" - .byte 0 - .previous -#endif - .endm - .macro DBGSTR1, str, arg #ifdef DEBUG stmfd sp!, {r0-r3, ip, lr} @@ -48,177 +28,25 @@ #endif .endm - .macro DBGSTR3, str, arg1, arg2, arg3 -#ifdef DEBUG - stmfd sp!, {r0-r3, ip, lr} - mov r3, \arg3 - mov r2, \arg2 - mov r1, \arg1 - ldr r0, =1f - bl _printk - ldmfd sp!, {r0-r3, ip, lr} - - .pushsection .rodata, "a" -1: .ascii KERN_DEBUG "VFP: \str\n" - .byte 0 - .previous -#endif - .endm - - -@ VFP hardware support entry point. -@ -@ r0 = instruction opcode (32-bit ARM or two 16-bit Thumb) -@ r1 = thread_info pointer -@ r2 = PC value to resume execution after successful emulation -@ r3 = normal "successful" return address -@ lr = unrecognised instruction return address -@ IRQs enabled. -ENTRY(vfp_support_entry) - ldr r11, [r1, #TI_CPU] @ CPU number - add r10, r1, #TI_VFPSTATE @ r10 = workspace - - DBGSTR3 "instr %08x pc %08x state %p", r0, r2, r10 - - .fpu vfpv2 - VFPFMRX r1, FPEXC @ Is the VFP enabled? - DBGSTR1 "fpexc %08x", r1 - tst r1, #FPEXC_EN - bne look_for_VFP_exceptions @ VFP is already enabled - - DBGSTR1 "enable %x", r10 - ldr r9, vfp_current_hw_state_address - orr r1, r1, #FPEXC_EN @ user FPEXC has the enable bit set - ldr r4, [r9, r11, lsl #2] @ vfp_current_hw_state pointer - bic r5, r1, #FPEXC_EX @ make sure exceptions are disabled - cmp r4, r10 @ this thread owns the hw context? -#ifndef CONFIG_SMP - @ For UP, checking that this thread owns the hw context is - @ sufficient to determine that the hardware state is valid. - beq vfp_hw_state_valid - - @ On UP, we lazily save the VFP context. As a different - @ thread wants ownership of the VFP hardware, save the old - @ state if there was a previous (valid) owner. - - VFPFMXR FPEXC, r5 @ enable VFP, disable any pending - @ exceptions, so we can get at the - @ rest of it - - DBGSTR1 "save old state %p", r4 - cmp r4, #0 @ if the vfp_current_hw_state is NULL - beq vfp_reload_hw @ then the hw state needs reloading - VFPFSTMIA r4, r5 @ save the working registers - VFPFMRX r5, FPSCR @ current status - tst r1, #FPEXC_EX @ is there additional state to save? - beq 1f - VFPFMRX r6, FPINST @ FPINST (only if FPEXC.EX is set) - tst r1, #FPEXC_FP2V @ is there an FPINST2 to read? - beq 1f - VFPFMRX r8, FPINST2 @ FPINST2 if needed (and present) -1: - stmia r4, {r1, r5, r6, r8} @ save FPEXC, FPSCR, FPINST, FPINST2 -vfp_reload_hw: - -#else - @ For SMP, if this thread does not own the hw context, then we - @ need to reload it. No need to save the old state as on SMP, - @ we always save the state when we switch away from a thread. - bne vfp_reload_hw - - @ This thread has ownership of the current hardware context. - @ However, it may have been migrated to another CPU, in which - @ case the saved state is newer than the hardware context. - @ Check this by looking at the CPU number which the state was - @ last loaded onto. - ldr ip, [r10, #VFP_CPU] - teq ip, r11 - beq vfp_hw_state_valid - -vfp_reload_hw: - @ We're loading this threads state into the VFP hardware. Update - @ the CPU number which contains the most up to date VFP context. - str r11, [r10, #VFP_CPU] - - VFPFMXR FPEXC, r5 @ enable VFP, disable any pending - @ exceptions, so we can get at the - @ rest of it -#endif - - DBGSTR1 "load state %p", r10 - str r10, [r9, r11, lsl #2] @ update the vfp_current_hw_state pointer +ENTRY(vfp_load_state) + @ Load the current VFP state + @ r0 - load location + @ returns FPEXC + DBGSTR1 "load VFP state %p", r0 @ Load the saved state back into the VFP - VFPFLDMIA r10, r5 @ reload the working registers while + VFPFLDMIA r0, r1 @ reload the working registers while @ FPEXC is in a safe state - ldmia r10, {r1, r5, r6, r8} @ load FPEXC, FPSCR, FPINST, FPINST2 - tst r1, #FPEXC_EX @ is there additional state to restore? + ldmia r0, {r0-r3} @ load FPEXC, FPSCR, FPINST, FPINST2 + tst r0, #FPEXC_EX @ is there additional state to restore? beq 1f - VFPFMXR FPINST, r6 @ restore FPINST (only if FPEXC.EX is set) - tst r1, #FPEXC_FP2V @ is there an FPINST2 to write? + VFPFMXR FPINST, r2 @ restore FPINST (only if FPEXC.EX is set) + tst r0, #FPEXC_FP2V @ is there an FPINST2 to write? beq 1f - VFPFMXR FPINST2, r8 @ FPINST2 if needed (and present) + VFPFMXR FPINST2, r3 @ FPINST2 if needed (and present) 1: - VFPFMXR FPSCR, r5 @ restore status - -@ The context stored in the VFP hardware is up to date with this thread -vfp_hw_state_valid: - tst r1, #FPEXC_EX - bne process_exception @ might as well handle the pending - @ exception before retrying branch - @ out before setting an FPEXC that - @ stops us reading stuff - VFPFMXR FPEXC, r1 @ Restore FPEXC last - mov sp, r3 @ we think we have handled things - pop {lr} - sub r2, r2, #4 @ Retry current instruction - if Thumb - str r2, [sp, #S_PC] @ mode it's two 16-bit instructions, - @ else it's one 32-bit instruction, so - @ always subtract 4 from the following - @ instruction address. - -local_bh_enable_and_ret: - adr r0, . - mov r1, #SOFTIRQ_DISABLE_OFFSET - b __local_bh_enable_ip @ tail call - -look_for_VFP_exceptions: - @ Check for synchronous or asynchronous exception - tst r1, #FPEXC_EX | FPEXC_DEX - bne process_exception - @ On some implementations of the VFP subarch 1, setting FPSCR.IXE - @ causes all the CDP instructions to be bounced synchronously without - @ setting the FPEXC.EX bit - VFPFMRX r5, FPSCR - tst r5, #FPSCR_IXE - bne process_exception - - tst r5, #FPSCR_LENGTH_MASK - beq skip - orr r1, r1, #FPEXC_DEX - b process_exception -skip: - - @ Fall into hand on to next handler - appropriate coproc instr - @ not recognised by VFP - - DBGSTR "not VFP" - b local_bh_enable_and_ret - -process_exception: - DBGSTR "bounce" - mov sp, r3 @ setup for a return to the user code. - pop {lr} - mov r2, sp @ nothing stacked - regdump is at TOS - - @ Now call the C code to package up the bounce to the support code - @ r0 holds the trigger instruction - @ r1 holds the FPEXC value - @ r2 pointer to register dump - b VFP_bounce @ we have handled this - the support - @ code will raise an exception if - @ required. If not, the user code will - @ retry the faulted instruction -ENDPROC(vfp_support_entry) + VFPFMXR FPSCR, r1 @ restore status + ret lr +ENDPROC(vfp_load_state) ENTRY(vfp_save_state) @ Save the current VFP state @@ -238,10 +66,6 @@ ENTRY(vfp_save_state) ret lr ENDPROC(vfp_save_state) - .align -vfp_current_hw_state_address: - .word vfp_current_hw_state - .macro tbl_branch, base, tmp, shift #ifdef CONFIG_THUMB2_KERNEL adr \tmp, 1f diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c index 95628e57807b..7572cb5b28a2 100644 --- a/arch/arm/vfp/vfpmodule.c +++ b/arch/arm/vfp/vfpmodule.c @@ -30,11 +30,6 @@ #include "vfpinstr.h" #include "vfp.h" -/* - * Our undef handlers (in entry.S) - */ -asmlinkage void vfp_support_entry(u32, void *, u32, u32); - static bool have_vfp __ro_after_init; /* @@ -325,7 +320,7 @@ static u32 vfp_emulate_instruction(u32 inst, u32 fpscr, struct pt_regs *regs) /* * Package up a bounce condition. */ -void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) +static void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) { u32 fpscr, orig_fpscr, fpsid, exceptions; @@ -374,7 +369,7 @@ void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) * on VFP subarch 1. */ vfp_raise_exceptions(VFP_EXCEPTION_ERROR, trigger, fpscr, regs); - goto exit; + return; } /* @@ -405,7 +400,7 @@ void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) * the FPEXC.FP2V bit is valid only if FPEXC.EX is 1. */ if ((fpexc & (FPEXC_EX | FPEXC_FP2V)) != (FPEXC_EX | FPEXC_FP2V)) - goto exit; + return; /* * The barrier() here prevents fpinst2 being read @@ -418,8 +413,6 @@ void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) exceptions = vfp_emulate_instruction(trigger, orig_fpscr, regs); if (exceptions) vfp_raise_exceptions(exceptions, trigger, orig_fpscr, regs); - exit: - local_bh_enable(); } static void vfp_enable(void *unused) @@ -649,22 +642,112 @@ static int vfp_starting_cpu(unsigned int unused) } /* - * Entered with: + * vfp_support_entry - Handle VFP exception from user mode * - * r0 = instruction opcode (32-bit ARM or two 16-bit Thumb) - * r1 = thread_info pointer - * r2 = PC value to resume execution after successful emulation - * r3 = normal "successful" return address - * lr = unrecognised instruction return address + * @regs: pt_regs structure holding the register state at exception entry + * @trigger: The opcode of the instruction that triggered the exception + * + * Returns 0 if the exception was handled, or an error code otherwise. */ -asmlinkage void vfp_entry(u32 trigger, struct thread_info *ti, u32 resume_pc, - u32 resume_return_address) +asmlinkage int vfp_support_entry(struct pt_regs *regs, u32 trigger) { + struct thread_info *ti = current_thread_info(); + u32 fpexc; + if (unlikely(!have_vfp)) - return; + return -ENODEV; local_bh_disable(); - vfp_support_entry(trigger, ti, resume_pc, resume_return_address); + fpexc = fmrx(FPEXC); + + /* + * If the VFP unit was not enabled yet, we have to check whether the + * VFP state in the CPU's registers is the most recent VFP state + * associated with the process. On UP systems, we don't save the VFP + * state eagerly on a context switch, so we may need to save the + * VFP state to memory first, as it may belong to another process. + */ + if (!(fpexc & FPEXC_EN)) { + /* + * Enable the VFP unit but mask the FP exception flag for the + * time being, so we can access all the registers. + */ + fpexc |= FPEXC_EN; + fmxr(FPEXC, fpexc & ~FPEXC_EX); + + /* + * Check whether or not the VFP state in the CPU's registers is + * the most recent VFP state associated with this task. On SMP, + * migration may result in multiple CPUs holding VFP states + * that belong to the same task, but only the most recent one + * is valid. + */ + if (!vfp_state_in_hw(ti->cpu, ti)) { + if (!IS_ENABLED(CONFIG_SMP) && + vfp_current_hw_state[ti->cpu] != NULL) { + /* + * This CPU is currently holding the most + * recent VFP state associated with another + * task, and we must save that to memory first. + */ + vfp_save_state(vfp_current_hw_state[ti->cpu], + fpexc); + } + + /* + * We can now proceed with loading the task's VFP state + * from memory into the CPU registers. + */ + fpexc = vfp_load_state(&ti->vfpstate); + vfp_current_hw_state[ti->cpu] = &ti->vfpstate; +#ifdef CONFIG_SMP + /* + * Record that this CPU is now the one holding the most + * recent VFP state of the task. + */ + ti->vfpstate.hard.cpu = ti->cpu; +#endif + } + + if (fpexc & FPEXC_EX) + /* + * Might as well handle the pending exception before + * retrying branch out before setting an FPEXC that + * stops us reading stuff. + */ + goto bounce; + + /* + * No FP exception is pending: just enable the VFP and + * replay the instruction that trapped. + */ + fmxr(FPEXC, fpexc); + regs->ARM_pc -= 4; + } else { + /* Check for synchronous or asynchronous exceptions */ + if (!(fpexc & (FPEXC_EX | FPEXC_DEX))) { + u32 fpscr = fmrx(FPSCR); + + /* + * On some implementations of the VFP subarch 1, + * setting FPSCR.IXE causes all the CDP instructions to + * be bounced synchronously without setting the + * FPEXC.EX bit + */ + if (!(fpscr & FPSCR_IXE)) { + if (!(fpscr & FPSCR_LENGTH_MASK)) { + pr_debug("not VFP\n"); + local_bh_enable(); + return -ENOEXEC; + } + fpexc |= FPEXC_DEX; + } + } +bounce: VFP_bounce(trigger, fpexc, regs); + } + + local_bh_enable(); + return 0; } #ifdef CONFIG_KERNEL_MODE_NEON From 6ee1e6772e1e19436f573672de5ff8aab7163be6 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 19 Mar 2023 23:55:14 +0100 Subject: [PATCH 0004/4029] ARM: kernel: Get rid of thread_info::used_cp[] array We keep track of which coprocessor triggered a fault in the used_cp[] array in thread_info, but this data is never used anywhere. So let's remove it. Linus did some digging and found out that the last user of this field was removed in commit bb1a773d5b6b ("kill unused dump_fpu() instances"). Reviewed-by: Linus Walleij Signed-off-by: Ard Biesheuvel --- arch/arm/include/asm/thread_info.h | 1 - arch/arm/kernel/asm-offsets.c | 1 - arch/arm/kernel/entry-armv.S | 6 ------ arch/arm/kernel/process.c | 1 - arch/arm/kernel/ptrace.c | 2 -- 5 files changed, 11 deletions(-) diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h index 7f092cb55a41..85c5f1e02ebf 100644 --- a/arch/arm/include/asm/thread_info.h +++ b/arch/arm/include/asm/thread_info.h @@ -66,7 +66,6 @@ struct thread_info { __u32 cpu_domain; /* cpu domain */ struct cpu_context_save cpu_context; /* cpu context */ __u32 abi_syscall; /* ABI type and syscall nr */ - __u8 used_cp[16]; /* thread used copro */ unsigned long tp_value[2]; /* TLS registers */ union fp_state fpstate __attribute__((aligned(8))); union vfp_state vfpstate; diff --git a/arch/arm/kernel/asm-offsets.c b/arch/arm/kernel/asm-offsets.c index 38121c59cbc2..f9c7111c1d65 100644 --- a/arch/arm/kernel/asm-offsets.c +++ b/arch/arm/kernel/asm-offsets.c @@ -47,7 +47,6 @@ int main(void) DEFINE(TI_CPU_DOMAIN, offsetof(struct thread_info, cpu_domain)); DEFINE(TI_CPU_SAVE, offsetof(struct thread_info, cpu_context)); DEFINE(TI_ABI_SYSCALL, offsetof(struct thread_info, abi_syscall)); - DEFINE(TI_USED_CP, offsetof(struct thread_info, used_cp)); DEFINE(TI_TP_VALUE, offsetof(struct thread_info, tp_value)); DEFINE(TI_FPSTATE, offsetof(struct thread_info, fpstate)); #ifdef CONFIG_VFP diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index c39303e5c234..ba47f6aac5ff 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -591,9 +591,6 @@ call_fpe: and r8, r0, r5 cmp r8, r7 @ NEON instruction? bne 2b - mov r7, #1 - strb r7, [r10, #TI_USED_CP + 10] @ mark CP#10 as used - strb r7, [r10, #TI_USED_CP + 11] @ mark CP#11 as used b do_vfp @ let VFP handler handle this 1: #endif @@ -601,9 +598,6 @@ call_fpe: tstne r0, #0x04000000 @ bit 26 set on both ARM and Thumb-2 reteq lr and r8, r0, #0x00000f00 @ mask out CP number - mov r7, #1 - add r6, r10, r8, lsr #8 @ add used_cp[] array offset first - strb r7, [r6, #TI_USED_CP] @ set appropriate used_cp[] #ifdef CONFIG_IWMMXT @ Test if we need to give access to iWMMXt coprocessors ldr r5, [r10, #TI_FLAGS] diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index 0e8ff85890ad..e16ed102960c 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -222,7 +222,6 @@ void flush_thread(void) flush_ptrace_hw_breakpoint(tsk); - memset(thread->used_cp, 0, sizeof(thread->used_cp)); memset(&tsk->thread.debug, 0, sizeof(struct debug_info)); memset(&thread->fpstate, 0, sizeof(union fp_state)); diff --git a/arch/arm/kernel/ptrace.c b/arch/arm/kernel/ptrace.c index 2d8e2516906b..2b945b9bd366 100644 --- a/arch/arm/kernel/ptrace.c +++ b/arch/arm/kernel/ptrace.c @@ -584,8 +584,6 @@ static int fpa_set(struct task_struct *target, { struct thread_info *thread = task_thread_info(target); - thread->used_cp[1] = thread->used_cp[2] = 1; - return user_regset_copyin(&pos, &count, &kbuf, &ubuf, &thread->fpstate, 0, sizeof(struct user_fp)); From cdd87465adfd75e4ebd11507575533c6bf7a5525 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 19 Mar 2023 00:28:35 +0100 Subject: [PATCH 0005/4029] ARM: vfp: Use undef hook for handling VFP exceptions Now that the VFP support code has been reimplemented as a C function that takes a struct pt_regs pointer and an opcode, we can use the existing undef_hook framework to deal with undef exceptions triggered by VFP instructions instead of having special handling in assembler. Reviewed-by: Linus Walleij Signed-off-by: Ard Biesheuvel --- arch/arm/kernel/entry-armv.S | 53 ----------------- arch/arm/vfp/Makefile | 2 +- arch/arm/vfp/entry.S | 31 ---------- arch/arm/vfp/vfpmodule.c | 109 +++++++++++++++++------------------ 4 files changed, 54 insertions(+), 141 deletions(-) delete mode 100644 arch/arm/vfp/entry.S diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index ba47f6aac5ff..0e40b2566f59 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -557,13 +557,6 @@ ENDPROC(__und_usr) * co-processor instructions. However, we have to watch out * for the ARM6/ARM7 SWI bug. * - * NEON is a special case that has to be handled here. Not all - * NEON instructions are co-processor instructions, so we have - * to make a special case of checking for them. Plus, there's - * five groups of them, so we have a table of mask/opcode pairs - * to check against, and if any match then we branch off into the - * NEON handler code. - * * Emulators may wish to make use of the following registers: * r0 = instruction opcode (32-bit ARM or two 16-bit Thumb) * r2 = PC value to resume execution after successful emulation @@ -575,25 +568,8 @@ ENDPROC(__und_usr) @ @ Fall-through from Thumb-2 __und_usr @ -#ifdef CONFIG_NEON - get_thread_info r10 @ get current thread - adr r6, .LCneon_thumb_opcodes - b 2f -#endif call_fpe: get_thread_info r10 @ get current thread -#ifdef CONFIG_NEON - adr r6, .LCneon_arm_opcodes -2: ldr r5, [r6], #4 @ mask value - ldr r7, [r6], #4 @ opcode bits matching in mask - cmp r5, #0 @ end mask? - beq 1f - and r8, r0, r5 - cmp r8, r7 @ NEON instruction? - bne 2b - b do_vfp @ let VFP handler handle this -1: -#endif tst r0, #0x08000000 @ only CDP/CPRT/LDC/STC have bit 27 tstne r0, #0x04000000 @ bit 26 set on both ARM and Thumb-2 reteq lr @@ -620,42 +596,13 @@ call_fpe: ret.w lr @ CP#7 ret.w lr @ CP#8 ret.w lr @ CP#9 -#ifdef CONFIG_VFP - W(b) do_vfp @ CP#10 (VFP) - W(b) do_vfp @ CP#11 (VFP) -#else ret.w lr @ CP#10 (VFP) ret.w lr @ CP#11 (VFP) -#endif ret.w lr @ CP#12 ret.w lr @ CP#13 ret.w lr @ CP#14 (Debug) ret.w lr @ CP#15 (Control) -#ifdef CONFIG_NEON - .align 6 - -.LCneon_arm_opcodes: - .word 0xfe000000 @ mask - .word 0xf2000000 @ opcode - - .word 0xff100000 @ mask - .word 0xf4000000 @ opcode - - .word 0x00000000 @ mask - .word 0x00000000 @ opcode - -.LCneon_thumb_opcodes: - .word 0xef000000 @ mask - .word 0xef000000 @ opcode - - .word 0xff100000 @ mask - .word 0xf9000000 @ opcode - - .word 0x00000000 @ mask - .word 0x00000000 @ opcode -#endif - do_fpe: add r10, r10, #TI_FPSTATE @ r10 = workspace ldr_va pc, fp_enter, tmp=r4 @ Call FP module USR entry point diff --git a/arch/arm/vfp/Makefile b/arch/arm/vfp/Makefile index 749901a72d6d..dfd64bc2b2fb 100644 --- a/arch/arm/vfp/Makefile +++ b/arch/arm/vfp/Makefile @@ -8,4 +8,4 @@ # ccflags-y := -DDEBUG # asflags-y := -DDEBUG -obj-y += vfpmodule.o entry.o vfphw.o vfpsingle.o vfpdouble.o +obj-y += vfpmodule.o vfphw.o vfpsingle.o vfpdouble.o diff --git a/arch/arm/vfp/entry.S b/arch/arm/vfp/entry.S deleted file mode 100644 index 547c94c62cd3..000000000000 --- a/arch/arm/vfp/entry.S +++ /dev/null @@ -1,31 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * linux/arch/arm/vfp/entry.S - * - * Copyright (C) 2004 ARM Limited. - * Written by Deep Blue Solutions Limited. - */ -#include -#include -#include -#include -#include -#include - -@ VFP entry point. -@ -@ r0 = instruction opcode (32-bit ARM or two 16-bit Thumb) -@ r2 = PC value to resume execution after successful emulation -@ r9 = normal "successful" return address -@ r10 = this threads thread_info structure -@ lr = unrecognised instruction return address -@ IRQs enabled. -@ -ENTRY(do_vfp) - mov r1, r0 @ pass trigger opcode via R1 - mov r0, sp @ pass struct pt_regs via R0 - bl vfp_support_entry @ dispatch the VFP exception - cmp r0, #0 @ handled successfully? - reteq r9 @ then use R9 as return address - ret lr @ pass to undef handler -ENDPROC(do_vfp) diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c index 7572cb5b28a2..58a9442add24 100644 --- a/arch/arm/vfp/vfpmodule.c +++ b/arch/arm/vfp/vfpmodule.c @@ -641,15 +641,37 @@ static int vfp_starting_cpu(unsigned int unused) return 0; } +static int vfp_kmode_exception(struct pt_regs *regs, unsigned int instr) +{ + /* + * If we reach this point, a floating point exception has been raised + * while running in kernel mode. If the NEON/VFP unit was enabled at the + * time, it means a VFP instruction has been issued that requires + * software assistance to complete, something which is not currently + * supported in kernel mode. + * If the NEON/VFP unit was disabled, and the location pointed to below + * is properly preceded by a call to kernel_neon_begin(), something has + * caused the task to be scheduled out and back in again. In this case, + * rebuilding and running with CONFIG_DEBUG_ATOMIC_SLEEP enabled should + * be helpful in localizing the problem. + */ + if (fmrx(FPEXC) & FPEXC_EN) + pr_crit("BUG: unsupported FP instruction in kernel mode\n"); + else + pr_crit("BUG: FP instruction issued in kernel mode with FP unit disabled\n"); + pr_crit("FPEXC == 0x%08x\n", fmrx(FPEXC)); + return 1; +} + /* - * vfp_support_entry - Handle VFP exception from user mode + * vfp_support_entry - Handle VFP exception * * @regs: pt_regs structure holding the register state at exception entry * @trigger: The opcode of the instruction that triggered the exception * * Returns 0 if the exception was handled, or an error code otherwise. */ -asmlinkage int vfp_support_entry(struct pt_regs *regs, u32 trigger) +static int vfp_support_entry(struct pt_regs *regs, u32 trigger) { struct thread_info *ti = current_thread_info(); u32 fpexc; @@ -657,6 +679,9 @@ asmlinkage int vfp_support_entry(struct pt_regs *regs, u32 trigger) if (unlikely(!have_vfp)) return -ENODEV; + if (!user_mode(regs)) + return vfp_kmode_exception(regs, trigger); + local_bh_disable(); fpexc = fmrx(FPEXC); @@ -722,7 +747,6 @@ asmlinkage int vfp_support_entry(struct pt_regs *regs, u32 trigger) * replay the instruction that trapped. */ fmxr(FPEXC, fpexc); - regs->ARM_pc -= 4; } else { /* Check for synchronous or asynchronous exceptions */ if (!(fpexc & (FPEXC_EX | FPEXC_DEX))) { @@ -743,78 +767,47 @@ asmlinkage int vfp_support_entry(struct pt_regs *regs, u32 trigger) fpexc |= FPEXC_DEX; } } -bounce: VFP_bounce(trigger, fpexc, regs); +bounce: regs->ARM_pc += 4; + VFP_bounce(trigger, fpexc, regs); } local_bh_enable(); return 0; } -#ifdef CONFIG_KERNEL_MODE_NEON - -static int vfp_kmode_exception(struct pt_regs *regs, unsigned int instr) -{ - /* - * If we reach this point, a floating point exception has been raised - * while running in kernel mode. If the NEON/VFP unit was enabled at the - * time, it means a VFP instruction has been issued that requires - * software assistance to complete, something which is not currently - * supported in kernel mode. - * If the NEON/VFP unit was disabled, and the location pointed to below - * is properly preceded by a call to kernel_neon_begin(), something has - * caused the task to be scheduled out and back in again. In this case, - * rebuilding and running with CONFIG_DEBUG_ATOMIC_SLEEP enabled should - * be helpful in localizing the problem. - */ - if (fmrx(FPEXC) & FPEXC_EN) - pr_crit("BUG: unsupported FP instruction in kernel mode\n"); - else - pr_crit("BUG: FP instruction issued in kernel mode with FP unit disabled\n"); - pr_crit("FPEXC == 0x%08x\n", fmrx(FPEXC)); - return 1; -} - -static struct undef_hook vfp_kmode_exception_hook[] = {{ +static struct undef_hook neon_support_hook[] = {{ .instr_mask = 0xfe000000, .instr_val = 0xf2000000, - .cpsr_mask = MODE_MASK | PSR_T_BIT, - .cpsr_val = SVC_MODE, - .fn = vfp_kmode_exception, + .cpsr_mask = PSR_T_BIT, + .cpsr_val = 0, + .fn = vfp_support_entry, }, { .instr_mask = 0xff100000, .instr_val = 0xf4000000, - .cpsr_mask = MODE_MASK | PSR_T_BIT, - .cpsr_val = SVC_MODE, - .fn = vfp_kmode_exception, + .cpsr_mask = PSR_T_BIT, + .cpsr_val = 0, + .fn = vfp_support_entry, }, { .instr_mask = 0xef000000, .instr_val = 0xef000000, - .cpsr_mask = MODE_MASK | PSR_T_BIT, - .cpsr_val = SVC_MODE | PSR_T_BIT, - .fn = vfp_kmode_exception, + .cpsr_mask = PSR_T_BIT, + .cpsr_val = PSR_T_BIT, + .fn = vfp_support_entry, }, { .instr_mask = 0xff100000, .instr_val = 0xf9000000, - .cpsr_mask = MODE_MASK | PSR_T_BIT, - .cpsr_val = SVC_MODE | PSR_T_BIT, - .fn = vfp_kmode_exception, -}, { - .instr_mask = 0x0c000e00, - .instr_val = 0x0c000a00, - .cpsr_mask = MODE_MASK, - .cpsr_val = SVC_MODE, - .fn = vfp_kmode_exception, + .cpsr_mask = PSR_T_BIT, + .cpsr_val = PSR_T_BIT, + .fn = vfp_support_entry, }}; -static int __init vfp_kmode_exception_hook_init(void) -{ - int i; +static struct undef_hook vfp_support_hook = { + .instr_mask = 0x0c000e00, + .instr_val = 0x0c000a00, + .fn = vfp_support_entry, +}; - for (i = 0; i < ARRAY_SIZE(vfp_kmode_exception_hook); i++) - register_undef_hook(&vfp_kmode_exception_hook[i]); - return 0; -} -subsys_initcall(vfp_kmode_exception_hook_init); +#ifdef CONFIG_KERNEL_MODE_NEON /* * Kernel-side NEON support functions @@ -919,8 +912,11 @@ static int __init vfp_init(void) * for NEON if the hardware has the MVFR registers. */ if (IS_ENABLED(CONFIG_NEON) && - (fmrx(MVFR1) & 0x000fff00) == 0x00011100) + (fmrx(MVFR1) & 0x000fff00) == 0x00011100) { elf_hwcap |= HWCAP_NEON; + for (int i = 0; i < ARRAY_SIZE(neon_support_hook); i++) + register_undef_hook(&neon_support_hook[i]); + } if (IS_ENABLED(CONFIG_VFPv3)) { u32 mvfr0 = fmrx(MVFR0); @@ -989,6 +985,7 @@ static int __init vfp_init(void) have_vfp = true; + register_undef_hook(&vfp_support_hook); thread_register_notifier(&vfp_notifier_block); vfp_pm_init(); From 8bcba70cb5c2204a011e06278a1fbfb1213e1df1 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 19 Mar 2023 15:18:25 +0100 Subject: [PATCH 0006/4029] ARM: entry: Disregard Thumb undef exception in coproc dispatch Now that the only remaining coprocessor instructions being handled via the dispatch in entry-armv.S are ones that only exist in a ARM (A32) encoding, we can simplify the handling of Thumb undef exceptions, and send them straight to the undefined instruction handlers in C code. This also means we can drop the code that partially decodes the instruction to decide whether it is a 16-bit or 32-bit Thumb instruction: this is all taken care of by the undef hook. Acked-by: Linus Walleij Signed-off-by: Ard Biesheuvel --- arch/arm/kernel/entry-armv.S | 121 ++++++----------------------------- 1 file changed, 18 insertions(+), 103 deletions(-) diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index 0e40b2566f59..aff6cfe58745 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -446,106 +446,32 @@ ENDPROC(__irq_usr) __und_usr: usr_entry uaccess=0 - mov r2, r4 - mov r3, r5 - - @ r2 = regs->ARM_pc, which is either 2 or 4 bytes ahead of the - @ faulting instruction depending on Thumb mode. - @ r3 = regs->ARM_cpsr - @ - @ The emulation code returns using r9 if it has emulated the - @ instruction, or the more conventional lr if we are to treat - @ this as a real undefined instruction - @ - badr r9, ret_from_exception - @ IRQs must be enabled before attempting to read the instruction from @ user space since that could cause a page/translation fault if the @ page table was modified by another CPU. enable_irq - tst r3, #PSR_T_BIT @ Thumb mode? - bne __und_usr_thumb - sub r4, r2, #4 @ ARM instr at LR - 4 -1: ldrt r0, [r4] - ARM_BE8(rev r0, r0) @ little endian instruction - + tst r5, #PSR_T_BIT @ Thumb mode? + mov r1, #2 @ set insn size to 2 for Thumb + bne 0f @ handle as Thumb undef exception + adr r9, ret_from_exception + bl call_fpe @ returns via R9 on success + mov r1, #4 @ set insn size to 4 for ARM +0: mov r0, sp uaccess_disable ip - - @ r0 = 32-bit ARM instruction which caused the exception - @ r2 = PC value for the following instruction (:= regs->ARM_pc) - @ r4 = PC value for the faulting instruction - @ lr = 32-bit undefined instruction function - badr lr, __und_usr_fault_32 - b call_fpe - -__und_usr_thumb: - @ Thumb instruction - sub r4, r2, #2 @ First half of thumb instr at LR - 2 -#if CONFIG_ARM_THUMB && __LINUX_ARM_ARCH__ >= 6 && CONFIG_CPU_V7 -/* - * Thumb-2 instruction handling. Note that because pre-v6 and >= v6 platforms - * can never be supported in a single kernel, this code is not applicable at - * all when __LINUX_ARM_ARCH__ < 6. This allows simplifying assumptions to be - * made about .arch directives. - */ -#if __LINUX_ARM_ARCH__ < 7 -/* If the target CPU may not be Thumb-2-capable, a run-time check is needed: */ - ldr_va r5, cpu_architecture - cmp r5, #CPU_ARCH_ARMv7 - blo __und_usr_fault_16 @ 16bit undefined instruction -/* - * The following code won't get run unless the running CPU really is v7, so - * coding round the lack of ldrht on older arches is pointless. Temporarily - * override the assembler target arch with the minimum required instead: - */ - .arch armv6t2 -#endif -2: ldrht r5, [r4] -ARM_BE8(rev16 r5, r5) @ little endian instruction - cmp r5, #0xe800 @ 32bit instruction if xx != 0 - blo __und_usr_fault_16_pan @ 16bit undefined instruction -3: ldrht r0, [r2] -ARM_BE8(rev16 r0, r0) @ little endian instruction - uaccess_disable ip - add r2, r2, #2 @ r2 is PC + 2, make it PC + 4 - str r2, [sp, #S_PC] @ it's a 2x16bit instr, update - orr r0, r0, r5, lsl #16 - badr lr, __und_usr_fault_32 - @ r0 = the two 16-bit Thumb instructions which caused the exception - @ r2 = PC value for the following Thumb instruction (:= regs->ARM_pc) - @ r4 = PC value for the first 16-bit Thumb instruction - @ lr = 32bit undefined instruction function - -#if __LINUX_ARM_ARCH__ < 7 -/* If the target arch was overridden, change it back: */ -#ifdef CONFIG_CPU_32v6K - .arch armv6k -#else - .arch armv6 -#endif -#endif /* __LINUX_ARM_ARCH__ < 7 */ -#else /* !(CONFIG_ARM_THUMB && __LINUX_ARM_ARCH__ >= 6 && CONFIG_CPU_V7) */ - b __und_usr_fault_16 -#endif + bl __und_fault + b ret_from_exception UNWIND(.fnend) ENDPROC(__und_usr) /* - * The out of line fixup for the ldrt instructions above. + * The out of line fixup for the ldrt instruction below. */ .pushsection .text.fixup, "ax" .align 2 4: str r4, [sp, #S_PC] @ retry current instruction ret r9 .popsection - .pushsection __ex_table,"a" - .long 1b, 4b -#if CONFIG_ARM_THUMB && __LINUX_ARM_ARCH__ >= 6 && CONFIG_CPU_V7 - .long 2b, 4b - .long 3b, 4b -#endif - .popsection /* * Check whether the instruction is a co-processor instruction. @@ -558,20 +484,22 @@ ENDPROC(__und_usr) * for the ARM6/ARM7 SWI bug. * * Emulators may wish to make use of the following registers: - * r0 = instruction opcode (32-bit ARM or two 16-bit Thumb) - * r2 = PC value to resume execution after successful emulation + * r4 = PC value to resume execution after successful emulation * r9 = normal "successful" return address * r10 = this threads thread_info structure * lr = unrecognised instruction return address * IRQs enabled, FIQs enabled. */ - @ - @ Fall-through from Thumb-2 __und_usr - @ call_fpe: + mov r2, r4 + sub r4, r4, #4 @ ARM instruction at user PC - 4 +USERL( 4b, ldrt r0, [r4]) @ load opcode from user space +ARM_BE8(rev r0, r0) @ little endian instruction + + uaccess_disable ip + get_thread_info r10 @ get current thread tst r0, #0x08000000 @ only CDP/CPRT/LDC/STC have bit 27 - tstne r0, #0x04000000 @ bit 26 set on both ARM and Thumb-2 reteq lr and r8, r0, #0x00000f00 @ mask out CP number #ifdef CONFIG_IWMMXT @@ -626,19 +554,6 @@ ENTRY(no_fp) ret lr ENDPROC(no_fp) -__und_usr_fault_32: - mov r1, #4 - b 1f -__und_usr_fault_16_pan: - uaccess_disable ip -__und_usr_fault_16: - mov r1, #2 -1: mov r0, sp - badr lr, ret_from_exception - b __und_fault -ENDPROC(__und_usr_fault_32) -ENDPROC(__und_usr_fault_16) - .align 5 __pabt_usr: usr_entry From 303d6da167dcbc3dd89adf3ca4e36c369950ed01 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 20 Mar 2023 00:07:20 +0100 Subject: [PATCH 0007/4029] ARM: iwmmxt: Use undef hook to enable coprocessor for task Define a undef hook to deal with undef exceptions triggered by iwmmxt instructions that were issued with the coprocessor disabled. This removes the dependency on the coprocessor dispatch code in entry-armv.S, which will be made NWFPE-only in a subsequent patch. Reviewed-by: Linus Walleij Signed-off-by: Ard Biesheuvel --- arch/arm/include/asm/thread_info.h | 16 ++++++++++++++++ arch/arm/kernel/entry-armv.S | 1 + arch/arm/kernel/iwmmxt.S | 18 ++++++++++++++---- arch/arm/kernel/pj4-cp0.c | 1 + arch/arm/kernel/xscale-cp0.c | 1 + 5 files changed, 33 insertions(+), 4 deletions(-) diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h index 85c5f1e02ebf..943ffcf069d2 100644 --- a/arch/arm/include/asm/thread_info.h +++ b/arch/arm/include/asm/thread_info.h @@ -40,6 +40,7 @@ struct task_struct; DECLARE_PER_CPU(struct task_struct *, __entry_task); #include +#include struct cpu_context_save { __u32 r4; @@ -104,6 +105,21 @@ extern void iwmmxt_task_restore(struct thread_info *, void *); extern void iwmmxt_task_release(struct thread_info *); extern void iwmmxt_task_switch(struct thread_info *); +extern int iwmmxt_undef_handler(struct pt_regs *, u32); + +static inline void register_iwmmxt_undef_handler(void) +{ + static struct undef_hook iwmmxt_undef_hook = { + .instr_mask = 0x0c000e00, + .instr_val = 0x0c000000, + .cpsr_mask = MODE_MASK | PSR_T_BIT, + .cpsr_val = USR_MODE, + .fn = iwmmxt_undef_handler, + }; + + register_undef_hook(&iwmmxt_undef_hook); +} + extern void vfp_sync_hwstate(struct thread_info *); extern void vfp_flush_hwstate(struct thread_info *); diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index aff6cfe58745..822b2c83bf08 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -507,6 +507,7 @@ ARM_BE8(rev r0, r0) @ little endian instruction ldr r5, [r10, #TI_FLAGS] rsbs r7, r8, #(1 << 8) @ CP 0 or 1 only movscs r7, r5, lsr #(TIF_USING_IWMMXT + 1) + movcs r0, sp @ pass struct pt_regs bcs iwmmxt_task_enable #endif ARM( add pc, pc, r8, lsr #6 ) diff --git a/arch/arm/kernel/iwmmxt.S b/arch/arm/kernel/iwmmxt.S index d2b4ac06e4ed..a0218c4867b9 100644 --- a/arch/arm/kernel/iwmmxt.S +++ b/arch/arm/kernel/iwmmxt.S @@ -58,9 +58,19 @@ .text .arm +ENTRY(iwmmxt_undef_handler) + push {r9, r10, lr} + get_thread_info r10 + mov r9, pc + b iwmmxt_task_enable + mov r0, #0 + pop {r9, r10, pc} +ENDPROC(iwmmxt_undef_handler) + /* * Lazy switching of Concan coprocessor context * + * r0 = struct pt_regs pointer * r10 = struct thread_info pointer * r9 = ret_from_exception * lr = undefined instr exit @@ -84,12 +94,12 @@ ENTRY(iwmmxt_task_enable) PJ4(mcr p15, 0, r2, c1, c0, 2) ldr r3, =concan_owner - add r0, r10, #TI_IWMMXT_STATE @ get task Concan save area - ldr r2, [sp, #60] @ current task pc value + ldr r2, [r0, #S_PC] @ current task pc value ldr r1, [r3] @ get current Concan owner - str r0, [r3] @ this task now owns Concan regs sub r2, r2, #4 @ adjust pc back - str r2, [sp, #60] + str r2, [r0, #S_PC] + add r0, r10, #TI_IWMMXT_STATE @ get task Concan save area + str r0, [r3] @ this task now owns Concan regs mrc p15, 0, r2, c2, c0, 0 mov r2, r2 @ cpwait diff --git a/arch/arm/kernel/pj4-cp0.c b/arch/arm/kernel/pj4-cp0.c index 1d1fb22f44f3..4bca8098c4ff 100644 --- a/arch/arm/kernel/pj4-cp0.c +++ b/arch/arm/kernel/pj4-cp0.c @@ -126,6 +126,7 @@ static int __init pj4_cp0_init(void) pr_info("PJ4 iWMMXt v%d coprocessor enabled.\n", vers); elf_hwcap |= HWCAP_IWMMXT; thread_register_notifier(&iwmmxt_notifier_block); + register_iwmmxt_undef_handler(); #endif return 0; diff --git a/arch/arm/kernel/xscale-cp0.c b/arch/arm/kernel/xscale-cp0.c index ed4f6e77616d..00d00d3aae97 100644 --- a/arch/arm/kernel/xscale-cp0.c +++ b/arch/arm/kernel/xscale-cp0.c @@ -166,6 +166,7 @@ static int __init xscale_cp0_init(void) pr_info("XScale iWMMXt coprocessor detected.\n"); elf_hwcap |= HWCAP_IWMMXT; thread_register_notifier(&iwmmxt_notifier_block); + register_iwmmxt_undef_handler(); #endif } else { pr_info("XScale DSP coprocessor detected.\n"); From 47ba5f39eab3c2a9a1ba878159a6050f2bbfc0e2 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 20 Mar 2023 00:25:18 +0100 Subject: [PATCH 0008/4029] ARM: entry: Make asm coproc dispatch code NWFPE only Now that we can dispatch all VFP and iWMMXT related undef exceptions using undef hooks implemented in C code, we no longer need the asm entry code that takes care of this unless we are using FPE, so we can move it into the FPE entry code. As this means it is ARM only, we can remove the Thumb2 specific decorations as well. It also means the non-standard, asm-only calling convention where returning via LR means failure and returning via R9 means success is now only used on legacy platforms that lack any kind of function return prediction, avoiding the associated performance impact. Reviewed-by: Linus Walleij Signed-off-by: Ard Biesheuvel --- arch/arm/kernel/entry-armv.S | 93 +----------------------------------- arch/arm/nwfpe/entry.S | 77 +++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 91 deletions(-) diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index 822b2c83bf08..682e92664b07 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -454,8 +454,10 @@ __und_usr: tst r5, #PSR_T_BIT @ Thumb mode? mov r1, #2 @ set insn size to 2 for Thumb bne 0f @ handle as Thumb undef exception +#ifdef CONFIG_FPE_NWFPE adr r9, ret_from_exception bl call_fpe @ returns via R9 on success +#endif mov r1, #4 @ set insn size to 4 for ARM 0: mov r0, sp uaccess_disable ip @@ -464,97 +466,6 @@ __und_usr: UNWIND(.fnend) ENDPROC(__und_usr) -/* - * The out of line fixup for the ldrt instruction below. - */ - .pushsection .text.fixup, "ax" - .align 2 -4: str r4, [sp, #S_PC] @ retry current instruction - ret r9 - .popsection - -/* - * Check whether the instruction is a co-processor instruction. - * If yes, we need to call the relevant co-processor handler. - * - * Note that we don't do a full check here for the co-processor - * instructions; all instructions with bit 27 set are well - * defined. The only instructions that should fault are the - * co-processor instructions. However, we have to watch out - * for the ARM6/ARM7 SWI bug. - * - * Emulators may wish to make use of the following registers: - * r4 = PC value to resume execution after successful emulation - * r9 = normal "successful" return address - * r10 = this threads thread_info structure - * lr = unrecognised instruction return address - * IRQs enabled, FIQs enabled. - */ -call_fpe: - mov r2, r4 - sub r4, r4, #4 @ ARM instruction at user PC - 4 -USERL( 4b, ldrt r0, [r4]) @ load opcode from user space -ARM_BE8(rev r0, r0) @ little endian instruction - - uaccess_disable ip - - get_thread_info r10 @ get current thread - tst r0, #0x08000000 @ only CDP/CPRT/LDC/STC have bit 27 - reteq lr - and r8, r0, #0x00000f00 @ mask out CP number -#ifdef CONFIG_IWMMXT - @ Test if we need to give access to iWMMXt coprocessors - ldr r5, [r10, #TI_FLAGS] - rsbs r7, r8, #(1 << 8) @ CP 0 or 1 only - movscs r7, r5, lsr #(TIF_USING_IWMMXT + 1) - movcs r0, sp @ pass struct pt_regs - bcs iwmmxt_task_enable -#endif - ARM( add pc, pc, r8, lsr #6 ) - THUMB( lsr r8, r8, #6 ) - THUMB( add pc, r8 ) - nop - - ret.w lr @ CP#0 - W(b) do_fpe @ CP#1 (FPE) - W(b) do_fpe @ CP#2 (FPE) - ret.w lr @ CP#3 - ret.w lr @ CP#4 - ret.w lr @ CP#5 - ret.w lr @ CP#6 - ret.w lr @ CP#7 - ret.w lr @ CP#8 - ret.w lr @ CP#9 - ret.w lr @ CP#10 (VFP) - ret.w lr @ CP#11 (VFP) - ret.w lr @ CP#12 - ret.w lr @ CP#13 - ret.w lr @ CP#14 (Debug) - ret.w lr @ CP#15 (Control) - -do_fpe: - add r10, r10, #TI_FPSTATE @ r10 = workspace - ldr_va pc, fp_enter, tmp=r4 @ Call FP module USR entry point - -/* - * The FP module is called with these registers set: - * r0 = instruction - * r2 = PC+4 - * r9 = normal "successful" return address - * r10 = FP workspace - * lr = unrecognised FP instruction return address - */ - - .pushsection .data - .align 2 -ENTRY(fp_enter) - .word no_fp - .popsection - -ENTRY(no_fp) - ret lr -ENDPROC(no_fp) - .align 5 __pabt_usr: usr_entry diff --git a/arch/arm/nwfpe/entry.S b/arch/arm/nwfpe/entry.S index d8f9915566e1..354d297a193b 100644 --- a/arch/arm/nwfpe/entry.S +++ b/arch/arm/nwfpe/entry.S @@ -7,6 +7,7 @@ Direct questions, comments to Scott Bambrough */ +#include #include #include @@ -104,6 +105,7 @@ next: @ plain LDR instruction. Weird, but it seems harmless. .pushsection .text.fixup,"ax" .align 2 +.Lrep: str r4, [sp, #S_PC] @ retry current instruction .Lfix: ret r9 @ let the user eat segfaults .popsection @@ -111,3 +113,78 @@ next: .align 3 .long .Lx1, .Lfix .popsection + + @ + @ Check whether the instruction is a co-processor instruction. + @ If yes, we need to call the relevant co-processor handler. + @ Only FPE instructions are dispatched here, everything else + @ is handled by undef hooks. + @ + @ Emulators may wish to make use of the following registers: + @ r4 = PC value to resume execution after successful emulation + @ r9 = normal "successful" return address + @ lr = unrecognised instruction return address + @ IRQs enabled, FIQs enabled. + @ +ENTRY(call_fpe) + mov r2, r4 + sub r4, r4, #4 @ ARM instruction at user PC - 4 +USERL( .Lrep, ldrt r0, [r4]) @ load opcode from user space +ARM_BE8(rev r0, r0) @ little endian instruction + + uaccess_disable ip + + get_thread_info r10 @ get current thread + tst r0, #0x08000000 @ only CDP/CPRT/LDC/STC have bit 27 + reteq lr + and r8, r0, #0x00000f00 @ mask out CP number +#ifdef CONFIG_IWMMXT + @ Test if we need to give access to iWMMXt coprocessors + ldr r5, [r10, #TI_FLAGS] + rsbs r7, r8, #(1 << 8) @ CP 0 or 1 only + movscs r7, r5, lsr #(TIF_USING_IWMMXT + 1) + movcs r0, sp @ pass struct pt_regs + bcs iwmmxt_task_enable +#endif + add pc, pc, r8, lsr #6 + nop + + ret lr @ CP#0 + b do_fpe @ CP#1 (FPE) + b do_fpe @ CP#2 (FPE) + ret lr @ CP#3 + ret lr @ CP#4 + ret lr @ CP#5 + ret lr @ CP#6 + ret lr @ CP#7 + ret lr @ CP#8 + ret lr @ CP#9 + ret lr @ CP#10 (VFP) + ret lr @ CP#11 (VFP) + ret lr @ CP#12 + ret lr @ CP#13 + ret lr @ CP#14 (Debug) + ret lr @ CP#15 (Control) + +do_fpe: + add r10, r10, #TI_FPSTATE @ r10 = workspace + ldr_va pc, fp_enter, tmp=r4 @ Call FP module USR entry point + + @ + @ The FP module is called with these registers set: + @ r0 = instruction + @ r2 = PC+4 + @ r9 = normal "successful" return address + @ r10 = FP workspace + @ lr = unrecognised FP instruction return address + @ + + .pushsection .data + .align 2 +ENTRY(fp_enter) + .word no_fp + .popsection + +no_fp: + ret lr +ENDPROC(no_fp) From 9930702cfebb24acf6c000b55541239095447e47 Mon Sep 17 00:00:00 2001 From: Corey Minyard Date: Wed, 17 May 2023 16:54:12 +0800 Subject: [PATCH 0009/4029] ipmi_watchdog: Fix read syscall not responding to signals during sleep Read syscall cannot response to sigals when data_to_read remains at 0 and the while loop cannot break. Check signal_pending in the loop. Reported-by: Zhen Ni Message-Id: <20230517085412.367022-1-zhen.ni@easystack.cn> Signed-off-by: Corey Minyard --- drivers/char/ipmi/ipmi_watchdog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/char/ipmi/ipmi_watchdog.c b/drivers/char/ipmi/ipmi_watchdog.c index 0d4a8dcacfd4..9a459257489f 100644 --- a/drivers/char/ipmi/ipmi_watchdog.c +++ b/drivers/char/ipmi/ipmi_watchdog.c @@ -802,7 +802,7 @@ static ssize_t ipmi_read(struct file *file, init_waitqueue_entry(&wait, current); add_wait_queue(&read_q, &wait); - while (!data_to_read) { + while (!data_to_read && !signal_pending(current)) { set_current_state(TASK_INTERRUPTIBLE); spin_unlock_irq(&ipmi_read_lock); schedule(); From e64c82b8064174a23434094d13a142254c138099 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Thu, 25 May 2023 22:40:21 +0200 Subject: [PATCH 0010/4029] ipmi: Switch i2c drivers back to use .probe() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After commit b8a1a4cd5a98 ("i2c: Provide a temporary .probe_new() call-back type"), all drivers being converted to .probe_new() and then 03c835f498b5 ("i2c: Switch .probe() to not take an id parameter") convert back to (the new) .probe() to be able to eventually drop .probe_new() from struct i2c_driver. Signed-off-by: Uwe Kleine-König Message-Id: <20230525204021.696858-1-u.kleine-koenig@pengutronix.de> Signed-off-by: Corey Minyard --- drivers/char/ipmi/ipmb_dev_int.c | 2 +- drivers/char/ipmi/ipmi_ipmb.c | 2 +- drivers/char/ipmi/ipmi_ssif.c | 2 +- drivers/char/ipmi/ssif_bmc.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/char/ipmi/ipmb_dev_int.c b/drivers/char/ipmi/ipmb_dev_int.c index a0e9e80d92ee..49100845fcb7 100644 --- a/drivers/char/ipmi/ipmb_dev_int.c +++ b/drivers/char/ipmi/ipmb_dev_int.c @@ -366,7 +366,7 @@ static struct i2c_driver ipmb_driver = { .name = "ipmb-dev", .acpi_match_table = ACPI_PTR(acpi_ipmb_id), }, - .probe_new = ipmb_probe, + .probe = ipmb_probe, .remove = ipmb_remove, .id_table = ipmb_id, }; diff --git a/drivers/char/ipmi/ipmi_ipmb.c b/drivers/char/ipmi/ipmi_ipmb.c index 3f1c9f1573e7..4e335832fc26 100644 --- a/drivers/char/ipmi/ipmi_ipmb.c +++ b/drivers/char/ipmi/ipmi_ipmb.c @@ -572,7 +572,7 @@ static struct i2c_driver ipmi_ipmb_driver = { .name = DEVICE_NAME, .of_match_table = of_ipmi_ipmb_match, }, - .probe_new = ipmi_ipmb_probe, + .probe = ipmi_ipmb_probe, .remove = ipmi_ipmb_remove, .id_table = ipmi_ipmb_id, }; diff --git a/drivers/char/ipmi/ipmi_ssif.c b/drivers/char/ipmi/ipmi_ssif.c index 3b921c78ba08..c6c9bcf6bf55 100644 --- a/drivers/char/ipmi/ipmi_ssif.c +++ b/drivers/char/ipmi/ipmi_ssif.c @@ -2054,7 +2054,7 @@ static struct i2c_driver ssif_i2c_driver = { .driver = { .name = DEVICE_NAME }, - .probe_new = ssif_probe, + .probe = ssif_probe, .remove = ssif_remove, .alert = ssif_alert, .id_table = ssif_id, diff --git a/drivers/char/ipmi/ssif_bmc.c b/drivers/char/ipmi/ssif_bmc.c index caee848261e9..56346fb32872 100644 --- a/drivers/char/ipmi/ssif_bmc.c +++ b/drivers/char/ipmi/ssif_bmc.c @@ -860,7 +860,7 @@ static struct i2c_driver ssif_bmc_driver = { .name = DEVICE_NAME, .of_match_table = ssif_bmc_match, }, - .probe_new = ssif_bmc_probe, + .probe = ssif_bmc_probe, .remove = ssif_bmc_remove, .id_table = ssif_bmc_id, }; From 1ebc9f0365efa883292caaa87bd393b3a5feafb1 Mon Sep 17 00:00:00 2001 From: Laura Nao Date: Wed, 7 Jun 2023 17:06:15 +0200 Subject: [PATCH 0011/4029] drm/panel-edp: Add AUO B116XAB01.4 edp panel entry Add a panel entry for the AUO B116XAB01.4 edp panel, found in the Acer Chromebook Spin 311 (CP311-3H) laptop. Signed-off-by: Laura Nao Reviewed-by: Douglas Anderson Signed-off-by: Douglas Anderson Link: https://patchwork.freedesktop.org/patch/msgid/20230607150615.241542-1-laura.nao@collabora.com --- drivers/gpu/drm/panel/panel-edp.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/panel/panel-edp.c b/drivers/gpu/drm/panel/panel-edp.c index fbd114b4f0be..df7e3cff004c 100644 --- a/drivers/gpu/drm/panel/panel-edp.c +++ b/drivers/gpu/drm/panel/panel-edp.c @@ -1866,6 +1866,7 @@ static const struct panel_delay delay_200_500_e200 = { */ static const struct edp_panel_entry edp_panels[] = { EDP_PANEL_ENTRY('A', 'U', 'O', 0x1062, &delay_200_500_e50, "B120XAN01.0"), + EDP_PANEL_ENTRY('A', 'U', 'O', 0x145c, &delay_200_500_e50, "B116XAB01.4"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x1e9b, &delay_200_500_e50, "B133UAN02.1"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x1ea5, &delay_200_500_e50, "B116XAK01.6"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x405c, &auo_b116xak01.delay, "B116XAK01"), From 9230d5dcb2fc00e8495dc0063be05690f9a820d8 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Mon, 29 May 2023 15:28:04 +0200 Subject: [PATCH 0012/4029] accel/ivpu: Use struct_size() Use struct_size() instead of hand-writing it. It is less verbose, more robust and more informative. Signed-off-by: Christophe JAILLET Reviewed-by: Marco Pagani Signed-off-by: Stanislaw Gruszka Link: https://patchwork.freedesktop.org/patch/msgid/0ae53be873c27c9a8740c4fe6d8e7cd1b1224994.1685366864.git.christophe.jaillet@wanadoo.fr --- drivers/accel/ivpu/ivpu_job.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/accel/ivpu/ivpu_job.c b/drivers/accel/ivpu/ivpu_job.c index 3c6f1e16cf2f..0a09bba8da24 100644 --- a/drivers/accel/ivpu/ivpu_job.c +++ b/drivers/accel/ivpu/ivpu_job.c @@ -289,15 +289,13 @@ ivpu_create_job(struct ivpu_file_priv *file_priv, u32 engine_idx, u32 bo_count) { struct ivpu_device *vdev = file_priv->vdev; struct ivpu_job *job; - size_t buf_size; int ret; ret = ivpu_rpm_get(vdev); if (ret < 0) return NULL; - buf_size = sizeof(*job) + bo_count * sizeof(struct ivpu_bo *); - job = kzalloc(buf_size, GFP_KERNEL); + job = kzalloc(struct_size(job, bos, bo_count), GFP_KERNEL); if (!job) goto err_rpm_put; From cab032239a3767dba22d33d3762474eb2a85691a Mon Sep 17 00:00:00 2001 From: Karol Wachowski Date: Thu, 18 May 2023 15:16:01 +0200 Subject: [PATCH 0013/4029] accel/ivpu: Remove configuration of MMU TBU1 and TBU3 MTL HW only uses StreamId0 and StreamId3 that map to TBU0 and TBU2. Signed-off-by: Karol Wachowski Reviewed-by: Stanislaw Gruszka Reviewed-by: Jeffrey Hugo Signed-off-by: Stanislaw Gruszka Link: https://patchwork.freedesktop.org/patch/msgid/20230518131605.650622-2-stanislaw.gruszka@linux.intel.com --- drivers/accel/ivpu/ivpu_hw_mtl.c | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/drivers/accel/ivpu/ivpu_hw_mtl.c b/drivers/accel/ivpu/ivpu_hw_mtl.c index 382ec127be8e..3210f1b4a7dd 100644 --- a/drivers/accel/ivpu/ivpu_hw_mtl.c +++ b/drivers/accel/ivpu/ivpu_hw_mtl.c @@ -537,21 +537,10 @@ static void ivpu_boot_tbu_mmu_enable(struct ivpu_device *vdev) { u32 val = REGV_RD32(MTL_VPU_HOST_IF_TBU_MMUSSIDV); - if (ivpu_is_fpga(vdev)) { - val = REG_SET_FLD(MTL_VPU_HOST_IF_TBU_MMUSSIDV, TBU0_AWMMUSSIDV, val); - val = REG_SET_FLD(MTL_VPU_HOST_IF_TBU_MMUSSIDV, TBU0_ARMMUSSIDV, val); - val = REG_SET_FLD(MTL_VPU_HOST_IF_TBU_MMUSSIDV, TBU2_AWMMUSSIDV, val); - val = REG_SET_FLD(MTL_VPU_HOST_IF_TBU_MMUSSIDV, TBU2_ARMMUSSIDV, val); - } else { - val = REG_SET_FLD(MTL_VPU_HOST_IF_TBU_MMUSSIDV, TBU0_AWMMUSSIDV, val); - val = REG_SET_FLD(MTL_VPU_HOST_IF_TBU_MMUSSIDV, TBU0_ARMMUSSIDV, val); - val = REG_SET_FLD(MTL_VPU_HOST_IF_TBU_MMUSSIDV, TBU1_AWMMUSSIDV, val); - val = REG_SET_FLD(MTL_VPU_HOST_IF_TBU_MMUSSIDV, TBU1_ARMMUSSIDV, val); - val = REG_SET_FLD(MTL_VPU_HOST_IF_TBU_MMUSSIDV, TBU2_AWMMUSSIDV, val); - val = REG_SET_FLD(MTL_VPU_HOST_IF_TBU_MMUSSIDV, TBU2_ARMMUSSIDV, val); - val = REG_SET_FLD(MTL_VPU_HOST_IF_TBU_MMUSSIDV, TBU3_AWMMUSSIDV, val); - val = REG_SET_FLD(MTL_VPU_HOST_IF_TBU_MMUSSIDV, TBU3_ARMMUSSIDV, val); - } + val = REG_SET_FLD(MTL_VPU_HOST_IF_TBU_MMUSSIDV, TBU0_AWMMUSSIDV, val); + val = REG_SET_FLD(MTL_VPU_HOST_IF_TBU_MMUSSIDV, TBU0_ARMMUSSIDV, val); + val = REG_SET_FLD(MTL_VPU_HOST_IF_TBU_MMUSSIDV, TBU2_AWMMUSSIDV, val); + val = REG_SET_FLD(MTL_VPU_HOST_IF_TBU_MMUSSIDV, TBU2_ARMMUSSIDV, val); REGV_WR32(MTL_VPU_HOST_IF_TBU_MMUSSIDV, val); } From a2fd4a6fae7d7542816452da97bbabfd49a0874c Mon Sep 17 00:00:00 2001 From: Karol Wachowski Date: Thu, 18 May 2023 15:16:02 +0200 Subject: [PATCH 0014/4029] accel/ivpu: Add MMU support for 4 level page mappings Program additional fourth level required for mappings with VA above 38bits. Co-developed-by: Raymond Tan Signed-off-by: Raymond Tan Signed-off-by: Karol Wachowski Reviewed-by: Stanislaw Gruszka Reviewed-by: Jeffrey Hugo Signed-off-by: Stanislaw Gruszka Link: https://patchwork.freedesktop.org/patch/msgid/20230518131605.650622-3-stanislaw.gruszka@linux.intel.com --- drivers/accel/ivpu/ivpu_mmu.c | 14 ++- drivers/accel/ivpu/ivpu_mmu_context.c | 162 ++++++++++++++++++-------- drivers/accel/ivpu/ivpu_mmu_context.h | 5 +- 3 files changed, 130 insertions(+), 51 deletions(-) diff --git a/drivers/accel/ivpu/ivpu_mmu.c b/drivers/accel/ivpu/ivpu_mmu.c index 694e978aba66..fa9a9ad59643 100644 --- a/drivers/accel/ivpu/ivpu_mmu.c +++ b/drivers/accel/ivpu/ivpu_mmu.c @@ -143,6 +143,16 @@ #define IVPU_MMU_CD_0_ASET BIT(47) #define IVPU_MMU_CD_0_ASID GENMASK_ULL(63, 48) +#define IVPU_MMU_T0SZ_48BIT 16 +#define IVPU_MMU_T0SZ_38BIT 26 + +#define IVPU_MMU_IPS_48BIT 5 +#define IVPU_MMU_IPS_44BIT 4 +#define IVPU_MMU_IPS_42BIT 3 +#define IVPU_MMU_IPS_40BIT 2 +#define IVPU_MMU_IPS_36BIT 1 +#define IVPU_MMU_IPS_32BIT 0 + #define IVPU_MMU_CD_1_TTB0_MASK GENMASK_ULL(51, 4) #define IVPU_MMU_STE_0_S1CDMAX GENMASK_ULL(63, 59) @@ -622,12 +632,12 @@ static int ivpu_mmu_cd_add(struct ivpu_device *vdev, u32 ssid, u64 cd_dma) entry = cdtab->base + (ssid * IVPU_MMU_CDTAB_ENT_SIZE); if (cd_dma != 0) { - cd[0] = FIELD_PREP(IVPU_MMU_CD_0_TCR_T0SZ, 26) | + cd[0] = FIELD_PREP(IVPU_MMU_CD_0_TCR_T0SZ, IVPU_MMU_T0SZ_48BIT) | FIELD_PREP(IVPU_MMU_CD_0_TCR_TG0, 0) | FIELD_PREP(IVPU_MMU_CD_0_TCR_IRGN0, 0) | FIELD_PREP(IVPU_MMU_CD_0_TCR_ORGN0, 0) | FIELD_PREP(IVPU_MMU_CD_0_TCR_SH0, 0) | - FIELD_PREP(IVPU_MMU_CD_0_TCR_IPS, 3) | + FIELD_PREP(IVPU_MMU_CD_0_TCR_IPS, IVPU_MMU_IPS_48BIT) | FIELD_PREP(IVPU_MMU_CD_0_ASID, ssid) | IVPU_MMU_CD_0_TCR_EPD1 | IVPU_MMU_CD_0_AA64 | diff --git a/drivers/accel/ivpu/ivpu_mmu_context.c b/drivers/accel/ivpu/ivpu_mmu_context.c index 8ce9b12ac356..79820ed2b8f1 100644 --- a/drivers/accel/ivpu/ivpu_mmu_context.c +++ b/drivers/accel/ivpu/ivpu_mmu_context.c @@ -11,7 +11,8 @@ #include "ivpu_mmu.h" #include "ivpu_mmu_context.h" -#define IVPU_MMU_PGD_INDEX_MASK GENMASK(38, 30) +#define IVPU_MMU_PGD_INDEX_MASK GENMASK(47, 39) +#define IVPU_MMU_PUD_INDEX_MASK GENMASK(38, 30) #define IVPU_MMU_PMD_INDEX_MASK GENMASK(29, 21) #define IVPU_MMU_PTE_INDEX_MASK GENMASK(20, 12) #define IVPU_MMU_ENTRY_FLAGS_MASK GENMASK(11, 0) @@ -25,6 +26,8 @@ #define IVPU_MMU_PAGE_SIZE SZ_4K #define IVPU_MMU_PTE_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PAGE_SIZE) #define IVPU_MMU_PMD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PTE_MAP_SIZE) +#define IVPU_MMU_PUD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PMD_MAP_SIZE) +#define IVPU_MMU_PGD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PUD_MAP_SIZE) #define IVPU_MMU_PGTABLE_SIZE (IVPU_MMU_PGTABLE_ENTRIES * sizeof(u64)) #define IVPU_MMU_DUMMY_ADDRESS 0xdeadb000 @@ -50,25 +53,38 @@ static int ivpu_mmu_pgtable_init(struct ivpu_device *vdev, struct ivpu_mmu_pgtab static void ivpu_mmu_pgtable_free(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable) { - int pgd_index, pmd_index; + int pgd_idx, pud_idx, pmd_idx; - for (pgd_index = 0; pgd_index < IVPU_MMU_PGTABLE_ENTRIES; ++pgd_index) { - u64 **pmd_entries = pgtable->pgd_cpu_entries[pgd_index]; - u64 *pmd = pgtable->pgd_entries[pgd_index]; + for (pgd_idx = 0; pgd_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pgd_idx) { + u64 **pud_entries = pgtable->pgd_cpu_entries[pgd_idx]; + u64 *pud = pgtable->pgd_entries[pgd_idx]; - if (!pmd_entries) + if (!pud_entries) continue; - for (pmd_index = 0; pmd_index < IVPU_MMU_PGTABLE_ENTRIES; ++pmd_index) { - if (pmd_entries[pmd_index]) - dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, - pmd_entries[pmd_index], - pmd[pmd_index] & ~IVPU_MMU_ENTRY_FLAGS_MASK); + for (pud_idx = 0; pud_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pud_idx) { + u64 **pmd_entries = pgtable->pgd_far_entries[pgd_idx][pud_idx]; + u64 *pmd = pgtable->pgd_cpu_entries[pgd_idx][pud_idx]; + + if (!pmd_entries) + continue; + + for (pmd_idx = 0; pmd_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pmd_idx) { + if (pmd_entries[pmd_idx]) + dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, + pmd_entries[pmd_idx], + pmd[pmd_idx] & ~IVPU_MMU_ENTRY_FLAGS_MASK); + } + + kfree(pmd_entries); + dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, + pud_entries[pud_idx], + pud[pud_idx] & ~IVPU_MMU_ENTRY_FLAGS_MASK); } - kfree(pmd_entries); - dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, pgtable->pgd_entries[pgd_index], - pgtable->pgd[pgd_index] & ~IVPU_MMU_ENTRY_FLAGS_MASK); + kfree(pud_entries); + dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, pgtable->pgd_entries[pgd_idx], + pgtable->pgd[pgd_idx] & ~IVPU_MMU_ENTRY_FLAGS_MASK); } dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, pgtable->pgd, @@ -76,14 +92,53 @@ static void ivpu_mmu_pgtable_free(struct ivpu_device *vdev, struct ivpu_mmu_pgta } static u64* -ivpu_mmu_ensure_pmd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, u64 pgd_index) +ivpu_mmu_ensure_pud(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx) +{ + u64 ***far_pud_entries; + u64 **pud_entries; + dma_addr_t pud_dma; + u64 *pud; + + if (pgtable->pgd_entries[pgd_idx]) + return pgtable->pgd_entries[pgd_idx]; + + pud = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pud_dma, GFP_KERNEL); + if (!pud) + return NULL; + + pud_entries = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL); + if (!pud_entries) + goto err_free_pud; + + far_pud_entries = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL); + if (!far_pud_entries) + goto err_free_pud_entries; + + pgtable->pgd[pgd_idx] = pud_dma | IVPU_MMU_ENTRY_VALID; + pgtable->pgd_entries[pgd_idx] = pud; + pgtable->pgd_cpu_entries[pgd_idx] = pud_entries; + pgtable->pgd_far_entries[pgd_idx] = far_pud_entries; + + return pud; + +err_free_pud_entries: + kfree(pud_entries); + +err_free_pud: + dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, pud, pud_dma); + return NULL; +} + +static u64* +ivpu_mmu_ensure_pmd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, + int pgd_idx, int pud_idx) { u64 **pmd_entries; dma_addr_t pmd_dma; u64 *pmd; - if (pgtable->pgd_entries[pgd_index]) - return pgtable->pgd_entries[pgd_index]; + if (pgtable->pgd_cpu_entries[pgd_idx][pud_idx]) + return pgtable->pgd_cpu_entries[pgd_idx][pud_idx]; pmd = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pmd_dma, GFP_KERNEL); if (!pmd) @@ -91,35 +146,35 @@ ivpu_mmu_ensure_pmd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, pmd_entries = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL); if (!pmd_entries) - goto err_free_pgd; + goto err_free_pmd; - pgtable->pgd_entries[pgd_index] = pmd; - pgtable->pgd_cpu_entries[pgd_index] = pmd_entries; - pgtable->pgd[pgd_index] = pmd_dma | IVPU_MMU_ENTRY_VALID; + pgtable->pgd_entries[pgd_idx][pud_idx] = pmd_dma | IVPU_MMU_ENTRY_VALID; + pgtable->pgd_cpu_entries[pgd_idx][pud_idx] = pmd; + pgtable->pgd_far_entries[pgd_idx][pud_idx] = pmd_entries; return pmd; -err_free_pgd: +err_free_pmd: dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, pmd, pmd_dma); return NULL; } static u64* ivpu_mmu_ensure_pte(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, - int pgd_index, int pmd_index) + int pgd_idx, int pud_idx, int pmd_idx) { dma_addr_t pte_dma; u64 *pte; - if (pgtable->pgd_cpu_entries[pgd_index][pmd_index]) - return pgtable->pgd_cpu_entries[pgd_index][pmd_index]; + if (pgtable->pgd_far_entries[pgd_idx][pud_idx][pmd_idx]) + return pgtable->pgd_far_entries[pgd_idx][pud_idx][pmd_idx]; pte = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pte_dma, GFP_KERNEL); if (!pte) return NULL; - pgtable->pgd_cpu_entries[pgd_index][pmd_index] = pte; - pgtable->pgd_entries[pgd_index][pmd_index] = pte_dma | IVPU_MMU_ENTRY_VALID; + pgtable->pgd_cpu_entries[pgd_idx][pud_idx][pmd_idx] = pte_dma | IVPU_MMU_ENTRY_VALID; + pgtable->pgd_far_entries[pgd_idx][pud_idx][pmd_idx] = pte; return pte; } @@ -129,33 +184,39 @@ ivpu_mmu_context_map_page(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx u64 vpu_addr, dma_addr_t dma_addr, int prot) { u64 *pte; - int pgd_index = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr); - int pmd_index = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr); - int pte_index = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr); + int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr); + int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr); + int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr); + int pte_idx = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr); + + /* Allocate PUD - first level page table if needed */ + if (!ivpu_mmu_ensure_pud(vdev, &ctx->pgtable, pgd_idx)) + return -ENOMEM; /* Allocate PMD - second level page table if needed */ - if (!ivpu_mmu_ensure_pmd(vdev, &ctx->pgtable, pgd_index)) + if (!ivpu_mmu_ensure_pmd(vdev, &ctx->pgtable, pgd_idx, pud_idx)) return -ENOMEM; /* Allocate PTE - third level page table if needed */ - pte = ivpu_mmu_ensure_pte(vdev, &ctx->pgtable, pgd_index, pmd_index); + pte = ivpu_mmu_ensure_pte(vdev, &ctx->pgtable, pgd_idx, pud_idx, pmd_idx); if (!pte) return -ENOMEM; /* Update PTE - third level page table with DMA address */ - pte[pte_index] = dma_addr | prot; + pte[pte_idx] = dma_addr | prot; return 0; } static void ivpu_mmu_context_unmap_page(struct ivpu_mmu_context *ctx, u64 vpu_addr) { - int pgd_index = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr); - int pmd_index = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr); - int pte_index = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr); + int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr); + int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr); + int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr); + int pte_idx = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr); /* Update PTE with dummy physical address and clear flags */ - ctx->pgtable.pgd_cpu_entries[pgd_index][pmd_index][pte_index] = IVPU_MMU_ENTRY_INVALID; + ctx->pgtable.pgd_far_entries[pgd_idx][pud_idx][pmd_idx][pte_idx] = IVPU_MMU_ENTRY_INVALID; } static void @@ -166,20 +227,27 @@ ivpu_mmu_context_flush_page_tables(struct ivpu_mmu_context *ctx, u64 vpu_addr, s /* Align to PMD entry (2 MB) */ vpu_addr &= ~(IVPU_MMU_PTE_MAP_SIZE - 1); - while (vpu_addr < end_addr) { - int pgd_index = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr); - u64 pmd_end = (pgd_index + 1) * (u64)IVPU_MMU_PMD_MAP_SIZE; - u64 *pmd = ctx->pgtable.pgd_entries[pgd_index]; + int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr); + u64 pud_end = (pgd_idx + 1) * (u64)IVPU_MMU_PUD_MAP_SIZE; + u64 *pud = ctx->pgtable.pgd_entries[pgd_idx]; - while (vpu_addr < end_addr && vpu_addr < pmd_end) { - int pmd_index = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr); - u64 *pte = ctx->pgtable.pgd_cpu_entries[pgd_index][pmd_index]; + while (vpu_addr < end_addr && vpu_addr < pud_end) { + int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr); + u64 pmd_end = (pud_idx + 1) * (u64)IVPU_MMU_PMD_MAP_SIZE; + u64 *pmd = ctx->pgtable.pgd_cpu_entries[pgd_idx][pud_idx]; - clflush_cache_range(pte, IVPU_MMU_PGTABLE_SIZE); - vpu_addr += IVPU_MMU_PTE_MAP_SIZE; + while (vpu_addr < end_addr && vpu_addr < pmd_end) { + int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr); + u64 *pte = ctx->pgtable.pgd_far_entries + [pgd_idx][pud_idx][pmd_idx]; + + clflush_cache_range(pte, IVPU_MMU_PGTABLE_SIZE); + vpu_addr += IVPU_MMU_PTE_MAP_SIZE; + } + clflush_cache_range(pmd, IVPU_MMU_PGTABLE_SIZE); } - clflush_cache_range(pmd, IVPU_MMU_PGTABLE_SIZE); + clflush_cache_range(pud, IVPU_MMU_PGTABLE_SIZE); } clflush_cache_range(pgd, IVPU_MMU_PGTABLE_SIZE); } diff --git a/drivers/accel/ivpu/ivpu_mmu_context.h b/drivers/accel/ivpu/ivpu_mmu_context.h index ddf11b95023a..29acc1151569 100644 --- a/drivers/accel/ivpu/ivpu_mmu_context.h +++ b/drivers/accel/ivpu/ivpu_mmu_context.h @@ -12,10 +12,11 @@ struct ivpu_device; struct ivpu_file_priv; struct ivpu_addr_range; -#define IVPU_MMU_PGTABLE_ENTRIES 512 +#define IVPU_MMU_PGTABLE_ENTRIES 512ull struct ivpu_mmu_pgtable { - u64 **pgd_cpu_entries[IVPU_MMU_PGTABLE_ENTRIES]; + u64 ***pgd_far_entries[IVPU_MMU_PGTABLE_ENTRIES]; + u64 **pgd_cpu_entries[IVPU_MMU_PGTABLE_ENTRIES]; u64 *pgd_entries[IVPU_MMU_PGTABLE_ENTRIES]; u64 *pgd; dma_addr_t pgd_dma; From a4172d6cf0a2b9f6fe6c5edb43e9aa32f62ac42e Mon Sep 17 00:00:00 2001 From: Karol Wachowski Date: Thu, 18 May 2023 15:16:03 +0200 Subject: [PATCH 0015/4029] accel/ivpu: Make DMA bit mask HW specific Future devices will have different dma bit mask, make it hw specific. Signed-off-by: Karol Wachowski Reviewed-by: Stanislaw Gruszka Reviewed-by: Jeffrey Hugo Signed-off-by: Stanislaw Gruszka Link: https://patchwork.freedesktop.org/patch/msgid/20230518131605.650622-4-stanislaw.gruszka@linux.intel.com --- drivers/accel/ivpu/ivpu_drv.c | 4 +++- drivers/accel/ivpu/ivpu_hw.h | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/accel/ivpu/ivpu_drv.c b/drivers/accel/ivpu/ivpu_drv.c index 8396db2b5203..21a5a0d63241 100644 --- a/drivers/accel/ivpu/ivpu_drv.c +++ b/drivers/accel/ivpu/ivpu_drv.c @@ -427,7 +427,7 @@ static int ivpu_pci_init(struct ivpu_device *vdev) return PTR_ERR(vdev->regb); } - ret = dma_set_mask_and_coherent(vdev->drm.dev, DMA_BIT_MASK(38)); + ret = dma_set_mask_and_coherent(vdev->drm.dev, DMA_BIT_MASK(vdev->hw->dma_bits)); if (ret) { ivpu_err(vdev, "Failed to set DMA mask: %d\n", ret); return ret; @@ -477,6 +477,8 @@ static int ivpu_dev_init(struct ivpu_device *vdev) return -ENOMEM; vdev->hw->ops = &ivpu_hw_mtl_ops; + vdev->hw->dma_bits = 38; + vdev->platform = IVPU_PLATFORM_INVALID; vdev->context_xa_limit.min = IVPU_USER_CONTEXT_MIN_SSID; vdev->context_xa_limit.max = IVPU_USER_CONTEXT_MAX_SSID; diff --git a/drivers/accel/ivpu/ivpu_hw.h b/drivers/accel/ivpu/ivpu_hw.h index 50a9304ab09c..0393901be492 100644 --- a/drivers/accel/ivpu/ivpu_hw.h +++ b/drivers/accel/ivpu/ivpu_hw.h @@ -57,6 +57,7 @@ struct ivpu_hw_info { u32 tile_fuse; u32 sku; u16 config; + int dma_bits; }; extern const struct ivpu_hw_ops ivpu_hw_mtl_ops; From 103d2ea13923cae3df232072d921fe0772a0a0fd Mon Sep 17 00:00:00 2001 From: Karol Wachowski Date: Thu, 18 May 2023 15:16:04 +0200 Subject: [PATCH 0016/4029] accel/ivpu: Rename and cleanup MMU600 page tables Simplify and unify naming convention in MMU600 page tables configuration. All DMA addresses in page tables directly accessed by VPU are called with _dma sufix and all CPU pointers to those page tables have _ptr sufix. Base pointers used to do a page walk on the CPU have corresponding names: pud_ptrs (pointers used to get access to PUD DMA) pmd_ptrs (pointers used to get access to PMD DMA) pte_ptrs (pointers used to get access to PTE DMA) with the following convention: u64 *pud_dma_ptr = pud_ptrs[pgd_idx]; *pud_dma_ptr = pud_dma; u64 *pmd_dma_ptr = pmd_ptrs[pgd_idx][pud_idx]; *pmd_dma_ptr = pmd_dma; u64 *pte_dma_ptr = pte_ptrs[pgd_idx][pud_idx][pmd_idx]; *pte_dma_ptr = pte_dma; On the way change to coherent dma allocation, _wc is only valid on ARM and was used by mistake. Signed-off-by: Karol Wachowski Reviewed-by: Stanislaw Gruszka Reviewed-by: Jeffrey Hugo Signed-off-by: Stanislaw Gruszka Link: https://patchwork.freedesktop.org/patch/msgid/20230518131605.650622-5-stanislaw.gruszka@linux.intel.com --- drivers/accel/ivpu/ivpu_mmu_context.c | 181 +++++++++++++------------- drivers/accel/ivpu/ivpu_mmu_context.h | 8 +- 2 files changed, 97 insertions(+), 92 deletions(-) diff --git a/drivers/accel/ivpu/ivpu_mmu_context.c b/drivers/accel/ivpu/ivpu_mmu_context.c index 79820ed2b8f1..6c7a53651075 100644 --- a/drivers/accel/ivpu/ivpu_mmu_context.c +++ b/drivers/accel/ivpu/ivpu_mmu_context.c @@ -39,123 +39,125 @@ static int ivpu_mmu_pgtable_init(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable) { dma_addr_t pgd_dma; - u64 *pgd; - pgd = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pgd_dma, GFP_KERNEL); - if (!pgd) + pgtable->pgd_dma_ptr = dma_alloc_coherent(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pgd_dma, + GFP_KERNEL); + if (!pgtable->pgd_dma_ptr) return -ENOMEM; - pgtable->pgd = pgd; pgtable->pgd_dma = pgd_dma; return 0; } -static void ivpu_mmu_pgtable_free(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable) +static void ivpu_mmu_pgtable_free(struct ivpu_device *vdev, u64 *cpu_addr, dma_addr_t dma_addr) +{ + if (cpu_addr) + dma_free_coherent(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, cpu_addr, + dma_addr & ~IVPU_MMU_ENTRY_FLAGS_MASK); +} + +static void ivpu_mmu_pgtables_free(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable) { int pgd_idx, pud_idx, pmd_idx; + dma_addr_t pud_dma, pmd_dma, pte_dma; + u64 *pud_dma_ptr, *pmd_dma_ptr, *pte_dma_ptr; for (pgd_idx = 0; pgd_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pgd_idx) { - u64 **pud_entries = pgtable->pgd_cpu_entries[pgd_idx]; - u64 *pud = pgtable->pgd_entries[pgd_idx]; + pud_dma_ptr = pgtable->pud_ptrs[pgd_idx]; + pud_dma = pgtable->pgd_dma_ptr[pgd_idx]; - if (!pud_entries) + if (!pud_dma_ptr) continue; for (pud_idx = 0; pud_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pud_idx) { - u64 **pmd_entries = pgtable->pgd_far_entries[pgd_idx][pud_idx]; - u64 *pmd = pgtable->pgd_cpu_entries[pgd_idx][pud_idx]; + pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx]; + pmd_dma = pgtable->pud_ptrs[pgd_idx][pud_idx]; - if (!pmd_entries) + if (!pmd_dma_ptr) continue; for (pmd_idx = 0; pmd_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pmd_idx) { - if (pmd_entries[pmd_idx]) - dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, - pmd_entries[pmd_idx], - pmd[pmd_idx] & ~IVPU_MMU_ENTRY_FLAGS_MASK); + pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx]; + pte_dma = pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx]; + + ivpu_mmu_pgtable_free(vdev, pte_dma_ptr, pte_dma); } - kfree(pmd_entries); - dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, - pud_entries[pud_idx], - pud[pud_idx] & ~IVPU_MMU_ENTRY_FLAGS_MASK); + kfree(pgtable->pte_ptrs[pgd_idx][pud_idx]); + ivpu_mmu_pgtable_free(vdev, pmd_dma_ptr, pmd_dma); } - kfree(pud_entries); - dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, pgtable->pgd_entries[pgd_idx], - pgtable->pgd[pgd_idx] & ~IVPU_MMU_ENTRY_FLAGS_MASK); + kfree(pgtable->pmd_ptrs[pgd_idx]); + kfree(pgtable->pte_ptrs[pgd_idx]); + ivpu_mmu_pgtable_free(vdev, pud_dma_ptr, pud_dma); } - dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, pgtable->pgd, - pgtable->pgd_dma & ~IVPU_MMU_ENTRY_FLAGS_MASK); + ivpu_mmu_pgtable_free(vdev, pgtable->pgd_dma_ptr, pgtable->pgd_dma); } static u64* ivpu_mmu_ensure_pud(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx) { - u64 ***far_pud_entries; - u64 **pud_entries; + u64 *pud_dma_ptr = pgtable->pud_ptrs[pgd_idx]; dma_addr_t pud_dma; - u64 *pud; - if (pgtable->pgd_entries[pgd_idx]) - return pgtable->pgd_entries[pgd_idx]; + if (pud_dma_ptr) + return pud_dma_ptr; - pud = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pud_dma, GFP_KERNEL); - if (!pud) + pud_dma_ptr = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pud_dma, GFP_KERNEL); + if (!pud_dma_ptr) return NULL; - pud_entries = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL); - if (!pud_entries) - goto err_free_pud; + drm_WARN_ON(&vdev->drm, pgtable->pmd_ptrs[pgd_idx]); + pgtable->pmd_ptrs[pgd_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL); + if (!pgtable->pmd_ptrs[pgd_idx]) + goto err_free_pud_dma_ptr; - far_pud_entries = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL); - if (!far_pud_entries) - goto err_free_pud_entries; + drm_WARN_ON(&vdev->drm, pgtable->pte_ptrs[pgd_idx]); + pgtable->pte_ptrs[pgd_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL); + if (!pgtable->pte_ptrs[pgd_idx]) + goto err_free_pmd_ptrs; - pgtable->pgd[pgd_idx] = pud_dma | IVPU_MMU_ENTRY_VALID; - pgtable->pgd_entries[pgd_idx] = pud; - pgtable->pgd_cpu_entries[pgd_idx] = pud_entries; - pgtable->pgd_far_entries[pgd_idx] = far_pud_entries; + pgtable->pud_ptrs[pgd_idx] = pud_dma_ptr; + pgtable->pgd_dma_ptr[pgd_idx] = pud_dma | IVPU_MMU_ENTRY_VALID; - return pud; + return pud_dma_ptr; -err_free_pud_entries: - kfree(pud_entries); +err_free_pmd_ptrs: + kfree(pgtable->pmd_ptrs[pgd_idx]); -err_free_pud: - dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, pud, pud_dma); +err_free_pud_dma_ptr: + ivpu_mmu_pgtable_free(vdev, pud_dma_ptr, pud_dma); return NULL; } static u64* -ivpu_mmu_ensure_pmd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, - int pgd_idx, int pud_idx) +ivpu_mmu_ensure_pmd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx, + int pud_idx) { - u64 **pmd_entries; + u64 *pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx]; dma_addr_t pmd_dma; - u64 *pmd; - if (pgtable->pgd_cpu_entries[pgd_idx][pud_idx]) - return pgtable->pgd_cpu_entries[pgd_idx][pud_idx]; + if (pmd_dma_ptr) + return pmd_dma_ptr; - pmd = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pmd_dma, GFP_KERNEL); - if (!pmd) + pmd_dma_ptr = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pmd_dma, GFP_KERNEL); + if (!pmd_dma_ptr) return NULL; - pmd_entries = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL); - if (!pmd_entries) - goto err_free_pmd; + drm_WARN_ON(&vdev->drm, pgtable->pte_ptrs[pgd_idx][pud_idx]); + pgtable->pte_ptrs[pgd_idx][pud_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL); + if (!pgtable->pte_ptrs[pgd_idx][pud_idx]) + goto err_free_pmd_dma_ptr; - pgtable->pgd_entries[pgd_idx][pud_idx] = pmd_dma | IVPU_MMU_ENTRY_VALID; - pgtable->pgd_cpu_entries[pgd_idx][pud_idx] = pmd; - pgtable->pgd_far_entries[pgd_idx][pud_idx] = pmd_entries; + pgtable->pmd_ptrs[pgd_idx][pud_idx] = pmd_dma_ptr; + pgtable->pud_ptrs[pgd_idx][pud_idx] = pmd_dma | IVPU_MMU_ENTRY_VALID; - return pmd; + return pmd_dma_ptr; -err_free_pmd: - dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, pmd, pmd_dma); +err_free_pmd_dma_ptr: + ivpu_mmu_pgtable_free(vdev, pmd_dma_ptr, pmd_dma); return NULL; } @@ -163,20 +165,20 @@ static u64* ivpu_mmu_ensure_pte(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx, int pud_idx, int pmd_idx) { + u64 *pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx]; dma_addr_t pte_dma; - u64 *pte; - if (pgtable->pgd_far_entries[pgd_idx][pud_idx][pmd_idx]) - return pgtable->pgd_far_entries[pgd_idx][pud_idx][pmd_idx]; + if (pte_dma_ptr) + return pte_dma_ptr; - pte = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pte_dma, GFP_KERNEL); - if (!pte) + pte_dma_ptr = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pte_dma, GFP_KERNEL); + if (!pte_dma_ptr) return NULL; - pgtable->pgd_cpu_entries[pgd_idx][pud_idx][pmd_idx] = pte_dma | IVPU_MMU_ENTRY_VALID; - pgtable->pgd_far_entries[pgd_idx][pud_idx][pmd_idx] = pte; + pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx] = pte_dma_ptr; + pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx] = pte_dma | IVPU_MMU_ENTRY_VALID; - return pte; + return pte_dma_ptr; } static int @@ -189,20 +191,20 @@ ivpu_mmu_context_map_page(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr); int pte_idx = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr); - /* Allocate PUD - first level page table if needed */ + /* Allocate PUD - second level page table if needed */ if (!ivpu_mmu_ensure_pud(vdev, &ctx->pgtable, pgd_idx)) return -ENOMEM; - /* Allocate PMD - second level page table if needed */ + /* Allocate PMD - third level page table if needed */ if (!ivpu_mmu_ensure_pmd(vdev, &ctx->pgtable, pgd_idx, pud_idx)) return -ENOMEM; - /* Allocate PTE - third level page table if needed */ + /* Allocate PTE - fourth level page table if needed */ pte = ivpu_mmu_ensure_pte(vdev, &ctx->pgtable, pgd_idx, pud_idx, pmd_idx); if (!pte) return -ENOMEM; - /* Update PTE - third level page table with DMA address */ + /* Update PTE */ pte[pte_idx] = dma_addr | prot; return 0; @@ -216,40 +218,39 @@ static void ivpu_mmu_context_unmap_page(struct ivpu_mmu_context *ctx, u64 vpu_ad int pte_idx = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr); /* Update PTE with dummy physical address and clear flags */ - ctx->pgtable.pgd_far_entries[pgd_idx][pud_idx][pmd_idx][pte_idx] = IVPU_MMU_ENTRY_INVALID; + ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] = IVPU_MMU_ENTRY_INVALID; } static void ivpu_mmu_context_flush_page_tables(struct ivpu_mmu_context *ctx, u64 vpu_addr, size_t size) { + struct ivpu_mmu_pgtable *pgtable = &ctx->pgtable; u64 end_addr = vpu_addr + size; - u64 *pgd = ctx->pgtable.pgd; /* Align to PMD entry (2 MB) */ vpu_addr &= ~(IVPU_MMU_PTE_MAP_SIZE - 1); + while (vpu_addr < end_addr) { int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr); u64 pud_end = (pgd_idx + 1) * (u64)IVPU_MMU_PUD_MAP_SIZE; - u64 *pud = ctx->pgtable.pgd_entries[pgd_idx]; while (vpu_addr < end_addr && vpu_addr < pud_end) { int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr); u64 pmd_end = (pud_idx + 1) * (u64)IVPU_MMU_PMD_MAP_SIZE; - u64 *pmd = ctx->pgtable.pgd_cpu_entries[pgd_idx][pud_idx]; while (vpu_addr < end_addr && vpu_addr < pmd_end) { int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr); - u64 *pte = ctx->pgtable.pgd_far_entries - [pgd_idx][pud_idx][pmd_idx]; - clflush_cache_range(pte, IVPU_MMU_PGTABLE_SIZE); + clflush_cache_range(pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx], + IVPU_MMU_PGTABLE_SIZE); vpu_addr += IVPU_MMU_PTE_MAP_SIZE; } - clflush_cache_range(pmd, IVPU_MMU_PGTABLE_SIZE); + clflush_cache_range(pgtable->pmd_ptrs[pgd_idx][pud_idx], + IVPU_MMU_PGTABLE_SIZE); } - clflush_cache_range(pud, IVPU_MMU_PGTABLE_SIZE); + clflush_cache_range(pgtable->pud_ptrs[pgd_idx], IVPU_MMU_PGTABLE_SIZE); } - clflush_cache_range(pgd, IVPU_MMU_PGTABLE_SIZE); + clflush_cache_range(pgtable->pgd_dma_ptr, IVPU_MMU_PGTABLE_SIZE); } static int @@ -305,7 +306,7 @@ ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, mutex_lock(&ctx->lock); for_each_sgtable_dma_sg(sgt, sg, i) { - u64 dma_addr = sg_dma_address(sg) - sg->offset; + dma_addr_t dma_addr = sg_dma_address(sg) - sg->offset; size_t size = sg_dma_len(sg) + sg->offset; ret = ivpu_mmu_context_map_pages(vdev, ctx, vpu_addr, dma_addr, size, prot); @@ -402,11 +403,15 @@ ivpu_mmu_context_init(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u3 static void ivpu_mmu_context_fini(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx) { - drm_WARN_ON(&vdev->drm, !ctx->pgtable.pgd); + if (drm_WARN_ON(&vdev->drm, !ctx->pgtable.pgd_dma_ptr)) + return; mutex_destroy(&ctx->lock); - ivpu_mmu_pgtable_free(vdev, &ctx->pgtable); + ivpu_mmu_pgtables_free(vdev, &ctx->pgtable); drm_mm_takedown(&ctx->mm); + + ctx->pgtable.pgd_dma_ptr = NULL; + ctx->pgtable.pgd_dma = 0; } int ivpu_mmu_global_context_init(struct ivpu_device *vdev) diff --git a/drivers/accel/ivpu/ivpu_mmu_context.h b/drivers/accel/ivpu/ivpu_mmu_context.h index 29acc1151569..961a0d6a6c7f 100644 --- a/drivers/accel/ivpu/ivpu_mmu_context.h +++ b/drivers/accel/ivpu/ivpu_mmu_context.h @@ -15,10 +15,10 @@ struct ivpu_addr_range; #define IVPU_MMU_PGTABLE_ENTRIES 512ull struct ivpu_mmu_pgtable { - u64 ***pgd_far_entries[IVPU_MMU_PGTABLE_ENTRIES]; - u64 **pgd_cpu_entries[IVPU_MMU_PGTABLE_ENTRIES]; - u64 *pgd_entries[IVPU_MMU_PGTABLE_ENTRIES]; - u64 *pgd; + u64 ***pte_ptrs[IVPU_MMU_PGTABLE_ENTRIES]; + u64 **pmd_ptrs[IVPU_MMU_PGTABLE_ENTRIES]; + u64 *pud_ptrs[IVPU_MMU_PGTABLE_ENTRIES]; + u64 *pgd_dma_ptr; dma_addr_t pgd_dma; }; From 95d440188d308232d0c14c1611f299eb0209878c Mon Sep 17 00:00:00 2001 From: Karol Wachowski Date: Thu, 18 May 2023 15:16:05 +0200 Subject: [PATCH 0017/4029] accel/ivpu: Mark 64 kB contiguous areas as contiguous in PTEs Whenever KMD maps region larger than 64kB that is both aligned and contiguous, set contiguous bit (52) in MMU PTE descriptor for each page in that region. This allows to treat 16 contiguous pages as one and reduce number of MMU page walks required which results in lower latency. Signed-off-by: Karol Wachowski Reviewed-by: Stanislaw Gruszka Reviewed-by: Jeffrey Hugo Signed-off-by: Stanislaw Gruszka Link: https://patchwork.freedesktop.org/patch/msgid/20230518131605.650622-6-stanislaw.gruszka@linux.intel.com --- drivers/accel/ivpu/ivpu_drv.c | 4 ++ drivers/accel/ivpu/ivpu_drv.h | 1 + drivers/accel/ivpu/ivpu_mmu_context.c | 75 +++++++++++++++++++++------ 3 files changed, 64 insertions(+), 16 deletions(-) diff --git a/drivers/accel/ivpu/ivpu_drv.c b/drivers/accel/ivpu/ivpu_drv.c index 21a5a0d63241..2df7643b843d 100644 --- a/drivers/accel/ivpu/ivpu_drv.c +++ b/drivers/accel/ivpu/ivpu_drv.c @@ -50,6 +50,10 @@ u8 ivpu_pll_max_ratio = U8_MAX; module_param_named(pll_max_ratio, ivpu_pll_max_ratio, byte, 0644); MODULE_PARM_DESC(pll_max_ratio, "Maximum PLL ratio used to set VPU frequency"); +bool ivpu_disable_mmu_cont_pages; +module_param_named(disable_mmu_cont_pages, ivpu_disable_mmu_cont_pages, bool, 0644); +MODULE_PARM_DESC(disable_mmu_cont_pages, "Disable MMU contiguous pages optimization"); + struct ivpu_file_priv *ivpu_file_priv_get(struct ivpu_file_priv *file_priv) { struct ivpu_device *vdev = file_priv->vdev; diff --git a/drivers/accel/ivpu/ivpu_drv.h b/drivers/accel/ivpu/ivpu_drv.h index d3013fbd13b3..315180dd72ce 100644 --- a/drivers/accel/ivpu/ivpu_drv.h +++ b/drivers/accel/ivpu/ivpu_drv.h @@ -131,6 +131,7 @@ struct ivpu_file_priv { extern int ivpu_dbg_mask; extern u8 ivpu_pll_min_ratio; extern u8 ivpu_pll_max_ratio; +extern bool ivpu_disable_mmu_cont_pages; #define IVPU_TEST_MODE_DISABLED 0 #define IVPU_TEST_MODE_FW_TEST 1 diff --git a/drivers/accel/ivpu/ivpu_mmu_context.c b/drivers/accel/ivpu/ivpu_mmu_context.c index 6c7a53651075..465a82298476 100644 --- a/drivers/accel/ivpu/ivpu_mmu_context.c +++ b/drivers/accel/ivpu/ivpu_mmu_context.c @@ -15,7 +15,8 @@ #define IVPU_MMU_PUD_INDEX_MASK GENMASK(38, 30) #define IVPU_MMU_PMD_INDEX_MASK GENMASK(29, 21) #define IVPU_MMU_PTE_INDEX_MASK GENMASK(20, 12) -#define IVPU_MMU_ENTRY_FLAGS_MASK GENMASK(11, 0) +#define IVPU_MMU_ENTRY_FLAGS_MASK (BIT(52) | GENMASK(11, 0)) +#define IVPU_MMU_ENTRY_FLAG_CONT BIT(52) #define IVPU_MMU_ENTRY_FLAG_NG BIT(11) #define IVPU_MMU_ENTRY_FLAG_AF BIT(10) #define IVPU_MMU_ENTRY_FLAG_USER BIT(6) @@ -23,12 +24,13 @@ #define IVPU_MMU_ENTRY_FLAG_TYPE_PAGE BIT(1) #define IVPU_MMU_ENTRY_FLAG_VALID BIT(0) -#define IVPU_MMU_PAGE_SIZE SZ_4K -#define IVPU_MMU_PTE_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PAGE_SIZE) -#define IVPU_MMU_PMD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PTE_MAP_SIZE) -#define IVPU_MMU_PUD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PMD_MAP_SIZE) -#define IVPU_MMU_PGD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PUD_MAP_SIZE) -#define IVPU_MMU_PGTABLE_SIZE (IVPU_MMU_PGTABLE_ENTRIES * sizeof(u64)) +#define IVPU_MMU_PAGE_SIZE SZ_4K +#define IVPU_MMU_CONT_PAGES_SIZE (IVPU_MMU_PAGE_SIZE * 16) +#define IVPU_MMU_PTE_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PAGE_SIZE) +#define IVPU_MMU_PMD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PTE_MAP_SIZE) +#define IVPU_MMU_PUD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PMD_MAP_SIZE) +#define IVPU_MMU_PGD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PUD_MAP_SIZE) +#define IVPU_MMU_PGTABLE_SIZE (IVPU_MMU_PGTABLE_ENTRIES * sizeof(u64)) #define IVPU_MMU_DUMMY_ADDRESS 0xdeadb000 #define IVPU_MMU_ENTRY_VALID (IVPU_MMU_ENTRY_FLAG_TYPE_PAGE | IVPU_MMU_ENTRY_FLAG_VALID) @@ -183,7 +185,7 @@ ivpu_mmu_ensure_pte(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, static int ivpu_mmu_context_map_page(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, - u64 vpu_addr, dma_addr_t dma_addr, int prot) + u64 vpu_addr, dma_addr_t dma_addr, u64 prot) { u64 *pte; int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr); @@ -210,6 +212,31 @@ ivpu_mmu_context_map_page(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx return 0; } +static int +ivpu_mmu_context_map_cont_64k(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u64 vpu_addr, + dma_addr_t dma_addr, u64 prot) +{ + size_t size = IVPU_MMU_CONT_PAGES_SIZE; + + drm_WARN_ON(&vdev->drm, !IS_ALIGNED(vpu_addr, size)); + drm_WARN_ON(&vdev->drm, !IS_ALIGNED(dma_addr, size)); + + prot |= IVPU_MMU_ENTRY_FLAG_CONT; + + while (size) { + int ret = ivpu_mmu_context_map_page(vdev, ctx, vpu_addr, dma_addr, prot); + + if (ret) + return ret; + + size -= IVPU_MMU_PAGE_SIZE; + vpu_addr += IVPU_MMU_PAGE_SIZE; + dma_addr += IVPU_MMU_PAGE_SIZE; + } + + return 0; +} + static void ivpu_mmu_context_unmap_page(struct ivpu_mmu_context *ctx, u64 vpu_addr) { int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr); @@ -255,17 +282,27 @@ ivpu_mmu_context_flush_page_tables(struct ivpu_mmu_context *ctx, u64 vpu_addr, s static int ivpu_mmu_context_map_pages(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, - u64 vpu_addr, dma_addr_t dma_addr, size_t size, int prot) + u64 vpu_addr, dma_addr_t dma_addr, size_t size, u64 prot) { + int map_size; + int ret; + while (size) { - int ret = ivpu_mmu_context_map_page(vdev, ctx, vpu_addr, dma_addr, prot); + if (!ivpu_disable_mmu_cont_pages && size >= IVPU_MMU_CONT_PAGES_SIZE && + IS_ALIGNED(vpu_addr | dma_addr, IVPU_MMU_CONT_PAGES_SIZE)) { + ret = ivpu_mmu_context_map_cont_64k(vdev, ctx, vpu_addr, dma_addr, prot); + map_size = IVPU_MMU_CONT_PAGES_SIZE; + } else { + ret = ivpu_mmu_context_map_page(vdev, ctx, vpu_addr, dma_addr, prot); + map_size = IVPU_MMU_PAGE_SIZE; + } if (ret) return ret; - vpu_addr += IVPU_MMU_PAGE_SIZE; - dma_addr += IVPU_MMU_PAGE_SIZE; - size -= IVPU_MMU_PAGE_SIZE; + vpu_addr += map_size; + dma_addr += map_size; + size -= map_size; } return 0; @@ -285,8 +322,8 @@ ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u64 vpu_addr, struct sg_table *sgt, bool llc_coherent) { struct scatterlist *sg; - int prot; int ret; + u64 prot; u64 i; if (!IS_ALIGNED(vpu_addr, IVPU_MMU_PAGE_SIZE)) @@ -362,8 +399,14 @@ ivpu_mmu_context_insert_node_locked(struct ivpu_mmu_context *ctx, { lockdep_assert_held(&ctx->lock); - return drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_PAGE_SIZE, - 0, range->start, range->end, DRM_MM_INSERT_BEST); + if (!ivpu_disable_mmu_cont_pages && size >= IVPU_MMU_CONT_PAGES_SIZE) { + if (!drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_CONT_PAGES_SIZE, 0, + range->start, range->end, DRM_MM_INSERT_BEST)) + return 0; + } + + return drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_PAGE_SIZE, 0, + range->start, range->end, DRM_MM_INSERT_BEST); } void From c5dacfe2e6c1251276e29b4cdac771f504593523 Mon Sep 17 00:00:00 2001 From: Wang Jianzheng Date: Thu, 8 Jun 2023 11:34:46 +0800 Subject: [PATCH 0018/4029] drm/panel: s6d7aa0: remove the unneeded variable in s6d7aa0_lock Remove unneeded variable and directly return 0. Signed-off-by: Wang Jianzheng Acked-by: Neil Armstrong Reviewed-by: Andi Shyti Reviewed-by: Artur Weber Signed-off-by: Neil Armstrong Link: https://patchwork.freedesktop.org/patch/msgid/20230608033446.18412-1-wangjianzheng@vivo.com --- drivers/gpu/drm/panel/panel-samsung-s6d7aa0.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/gpu/drm/panel/panel-samsung-s6d7aa0.c b/drivers/gpu/drm/panel/panel-samsung-s6d7aa0.c index 102e1fc7ee38..fec0d014fd0e 100644 --- a/drivers/gpu/drm/panel/panel-samsung-s6d7aa0.c +++ b/drivers/gpu/drm/panel/panel-samsung-s6d7aa0.c @@ -66,7 +66,6 @@ static void s6d7aa0_reset(struct s6d7aa0 *ctx) static int s6d7aa0_lock(struct s6d7aa0 *ctx, bool lock) { struct mipi_dsi_device *dsi = ctx->dsi; - int ret = 0; if (lock) { mipi_dsi_dcs_write_seq(dsi, MCS_PASSWD1, 0xa5, 0xa5); @@ -80,7 +79,7 @@ static int s6d7aa0_lock(struct s6d7aa0 *ctx, bool lock) mipi_dsi_dcs_write_seq(dsi, MCS_PASSWD3, 0xa5, 0xa5); } - return ret; + return 0; } static int s6d7aa0_on(struct s6d7aa0 *ctx) From 11c173b2b121153b8d56ef925bf898c87a7f8d2f Mon Sep 17 00:00:00 2001 From: Jessica Zhang Date: Wed, 31 May 2023 11:12:32 -0700 Subject: [PATCH 0019/4029] dt-bindings: display: panel: Add Visionox R66451 AMOLED DSI panel Document the 1080x2340 Visionox R66451 AMOLED DSI panel bindings Reviewed-by: Krzysztof Kozlowski Signed-off-by: Jessica Zhang Signed-off-by: Neil Armstrong Link: https://patchwork.freedesktop.org/patch/msgid/20230516-b4-r66451-panel-driver-v2-1-9c8d5eeef579@quicinc.com --- .../display/panel/visionox,r66451.yaml | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 Documentation/devicetree/bindings/display/panel/visionox,r66451.yaml diff --git a/Documentation/devicetree/bindings/display/panel/visionox,r66451.yaml b/Documentation/devicetree/bindings/display/panel/visionox,r66451.yaml new file mode 100644 index 000000000000..6ba323683921 --- /dev/null +++ b/Documentation/devicetree/bindings/display/panel/visionox,r66451.yaml @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: GPL-2.0-only or BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/panel/visionox,r66451.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Visionox R66451 AMOLED DSI Panel + +maintainers: + - Jessica Zhang + +allOf: + - $ref: panel-common.yaml# + +properties: + compatible: + const: visionox,r66451 + + reg: + maxItems: 1 + description: DSI virtual channel + + vddio-supply: true + vdd-supply: true + port: true + reset-gpios: true + +additionalProperties: false + +required: + - compatible + - reg + - vddio-supply + - vdd-supply + - reset-gpios + - port + +examples: + - | + #include + dsi { + #address-cells = <1>; + #size-cells = <0>; + panel@0 { + compatible = "visionox,r66451"; + reg = <0>; + vddio-supply = <&vreg_l12c_1p8>; + vdd-supply = <&vreg_l13c_3p0>; + + reset-gpios = <&tlmm 24 GPIO_ACTIVE_LOW>; + + port { + panel0_in: endpoint { + remote-endpoint = <&dsi0_out>; + }; + }; + }; + }; +... From a6dfab2738fc2a47796d85f035a06efb6559c587 Mon Sep 17 00:00:00 2001 From: Jessica Zhang Date: Wed, 31 May 2023 11:12:33 -0700 Subject: [PATCH 0020/4029] drm/panel: Add driver for Visionox r66451 panel Add support for the 1080x2340 Visionox R66451 AMOLED DSI panel that comes with the Qualcomm HDK8350 display expansion pack. The panel enables display compression (DSC v1.2) by default. Signed-off-by: Jessica Zhang Acked-by: Neil Armstrong Signed-off-by: Neil Armstrong Link: https://patchwork.freedesktop.org/patch/msgid/20230516-b4-r66451-panel-driver-v2-2-9c8d5eeef579@quicinc.com --- drivers/gpu/drm/panel/Kconfig | 9 + drivers/gpu/drm/panel/Makefile | 1 + drivers/gpu/drm/panel/panel-visionox-r66451.c | 390 ++++++++++++++++++ 3 files changed, 400 insertions(+) create mode 100644 drivers/gpu/drm/panel/panel-visionox-r66451.c diff --git a/drivers/gpu/drm/panel/Kconfig b/drivers/gpu/drm/panel/Kconfig index 203c0ef0bbfd..1a0fd0754692 100644 --- a/drivers/gpu/drm/panel/Kconfig +++ b/drivers/gpu/drm/panel/Kconfig @@ -793,6 +793,15 @@ config DRM_PANEL_VISIONOX_VTDR6130 Say Y here if you want to enable support for Visionox VTDR6130 1080x2400 AMOLED DSI panel. +config DRM_PANEL_VISIONOX_R66451 + tristate "Visionox R66451" + depends on OF + depends on DRM_MIPI_DSI + depends on BACKLIGHT_CLASS_DEVICE + help + Say Y here if you want to enable support for Visionox + R66451 1080x2340 AMOLED DSI panel. + config DRM_PANEL_WIDECHIPS_WS2401 tristate "Widechips WS2401 DPI panel driver" depends on SPI && GPIOLIB diff --git a/drivers/gpu/drm/panel/Makefile b/drivers/gpu/drm/panel/Makefile index 30cf553c8d1d..499e38244253 100644 --- a/drivers/gpu/drm/panel/Makefile +++ b/drivers/gpu/drm/panel/Makefile @@ -81,5 +81,6 @@ obj-$(CONFIG_DRM_PANEL_TPO_TPG110) += panel-tpo-tpg110.o obj-$(CONFIG_DRM_PANEL_TRULY_NT35597_WQXGA) += panel-truly-nt35597.o obj-$(CONFIG_DRM_PANEL_VISIONOX_RM69299) += panel-visionox-rm69299.o obj-$(CONFIG_DRM_PANEL_VISIONOX_VTDR6130) += panel-visionox-vtdr6130.o +obj-$(CONFIG_DRM_PANEL_VISIONOX_R66451) += panel-visionox-r66451.o obj-$(CONFIG_DRM_PANEL_WIDECHIPS_WS2401) += panel-widechips-ws2401.o obj-$(CONFIG_DRM_PANEL_XINPENG_XPP055C272) += panel-xinpeng-xpp055c272.o diff --git a/drivers/gpu/drm/panel/panel-visionox-r66451.c b/drivers/gpu/drm/panel/panel-visionox-r66451.c new file mode 100644 index 000000000000..00fc28ad3d07 --- /dev/null +++ b/drivers/gpu/drm/panel/panel-visionox-r66451.c @@ -0,0 +1,390 @@ +//SPDX-License-Identifier: GPL-2.0-only +//Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved. + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include