From ef5b6a542b1dbb718226a5f8208be09ef405983d Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 8 Nov 2023 04:40:35 -0500 Subject: [PATCH 001/336] selftests: kvm/s390x: use vm_create_barebones() This function does the same but makes it clearer why one would use the "____"-prefixed version of vm_create(). Signed-off-by: Paolo Bonzini --- tools/testing/selftests/kvm/s390x/cmma_test.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tools/testing/selftests/kvm/s390x/cmma_test.c b/tools/testing/selftests/kvm/s390x/cmma_test.c index c8e0a6495a63..626a2b8a2037 100644 --- a/tools/testing/selftests/kvm/s390x/cmma_test.c +++ b/tools/testing/selftests/kvm/s390x/cmma_test.c @@ -94,11 +94,6 @@ static void guest_dirty_test_data(void) ); } -static struct kvm_vm *create_vm(void) -{ - return ____vm_create(VM_MODE_DEFAULT); -} - static void create_main_memslot(struct kvm_vm *vm) { int i; @@ -157,7 +152,7 @@ static struct kvm_vm *create_vm_two_memslots(void) { struct kvm_vm *vm; - vm = create_vm(); + vm = vm_create_barebones(); create_memslots(vm); @@ -276,7 +271,7 @@ static void assert_exit_was_hypercall(struct kvm_vcpu *vcpu) static void test_migration_mode(void) { - struct kvm_vm *vm = create_vm(); + struct kvm_vm *vm = vm_create_barebones(); struct kvm_vcpu *vcpu; u64 orig_psw; int rc; @@ -670,7 +665,7 @@ struct testdef { */ static int machine_has_cmma(void) { - struct kvm_vm *vm = create_vm(); + struct kvm_vm *vm = vm_create_barebones(); int r; r = !__kvm_has_device_attr(vm->fd, KVM_S390_VM_MEM_CTRL, KVM_S390_VM_MEM_ENABLE_CMMA); From e97b39c5c4362dc1cbc37a563ddac313b96c84f3 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 27 Oct 2023 11:21:43 -0700 Subject: [PATCH 002/336] KVM: Tweak kvm_hva_range and hva_handler_t to allow reusing for gfn ranges Rework and rename "struct kvm_hva_range" into "kvm_mmu_notifier_range" so that the structure can be used to handle notifications that operate on gfn context, i.e. that aren't tied to a host virtual address. Rename the handler typedef too (arguably it should always have been gfn_handler_t). Practically speaking, this is a nop for 64-bit kernels as the only meaningful change is to store start+end as u64s instead of unsigned longs. Reviewed-by: Paolo Bonzini Reviewed-by: Xiaoyao Li Signed-off-by: Sean Christopherson Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Message-Id: <20231027182217.3615211-2-seanjc@google.com> Reviewed-by: Kai Huang Signed-off-by: Paolo Bonzini --- virt/kvm/kvm_main.c | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 486800a7024b..0524933856d4 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -541,18 +541,22 @@ static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn) return container_of(mn, struct kvm, mmu_notifier); } -typedef bool (*hva_handler_t)(struct kvm *kvm, struct kvm_gfn_range *range); +typedef bool (*gfn_handler_t)(struct kvm *kvm, struct kvm_gfn_range *range); typedef void (*on_lock_fn_t)(struct kvm *kvm, unsigned long start, unsigned long end); typedef void (*on_unlock_fn_t)(struct kvm *kvm); -struct kvm_hva_range { - unsigned long start; - unsigned long end; +struct kvm_mmu_notifier_range { + /* + * 64-bit addresses, as KVM notifiers can operate on host virtual + * addresses (unsigned long) and guest physical addresses (64-bit). + */ + u64 start; + u64 end; union kvm_mmu_notifier_arg arg; - hva_handler_t handler; + gfn_handler_t handler; on_lock_fn_t on_lock; on_unlock_fn_t on_unlock; bool flush_on_ret; @@ -581,7 +585,7 @@ static const union kvm_mmu_notifier_arg KVM_MMU_NOTIFIER_NO_ARG; node = interval_tree_iter_next(node, start, last)) \ static __always_inline int __kvm_handle_hva_range(struct kvm *kvm, - const struct kvm_hva_range *range) + const struct kvm_mmu_notifier_range *range) { bool ret = false, locked = false; struct kvm_gfn_range gfn_range; @@ -608,9 +612,9 @@ static __always_inline int __kvm_handle_hva_range(struct kvm *kvm, unsigned long hva_start, hva_end; slot = container_of(node, struct kvm_memory_slot, hva_node[slots->node_idx]); - hva_start = max(range->start, slot->userspace_addr); - hva_end = min(range->end, slot->userspace_addr + - (slot->npages << PAGE_SHIFT)); + hva_start = max_t(unsigned long, range->start, slot->userspace_addr); + hva_end = min_t(unsigned long, range->end, + slot->userspace_addr + (slot->npages << PAGE_SHIFT)); /* * To optimize for the likely case where the address @@ -660,10 +664,10 @@ static __always_inline int kvm_handle_hva_range(struct mmu_notifier *mn, unsigned long start, unsigned long end, union kvm_mmu_notifier_arg arg, - hva_handler_t handler) + gfn_handler_t handler) { struct kvm *kvm = mmu_notifier_to_kvm(mn); - const struct kvm_hva_range range = { + const struct kvm_mmu_notifier_range range = { .start = start, .end = end, .arg = arg, @@ -680,10 +684,10 @@ static __always_inline int kvm_handle_hva_range(struct mmu_notifier *mn, static __always_inline int kvm_handle_hva_range_no_flush(struct mmu_notifier *mn, unsigned long start, unsigned long end, - hva_handler_t handler) + gfn_handler_t handler) { struct kvm *kvm = mmu_notifier_to_kvm(mn); - const struct kvm_hva_range range = { + const struct kvm_mmu_notifier_range range = { .start = start, .end = end, .handler = handler, @@ -771,7 +775,7 @@ static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, const struct mmu_notifier_range *range) { struct kvm *kvm = mmu_notifier_to_kvm(mn); - const struct kvm_hva_range hva_range = { + const struct kvm_mmu_notifier_range hva_range = { .start = range->start, .end = range->end, .handler = kvm_unmap_gfn_range, @@ -835,7 +839,7 @@ static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, const struct mmu_notifier_range *range) { struct kvm *kvm = mmu_notifier_to_kvm(mn); - const struct kvm_hva_range hva_range = { + const struct kvm_mmu_notifier_range hva_range = { .start = range->start, .end = range->end, .handler = (void *)kvm_null_fn, From c0db19232c1ed6bd7fcb825c28b014c52732c19e Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 27 Oct 2023 11:21:44 -0700 Subject: [PATCH 003/336] KVM: Assert that mmu_invalidate_in_progress *never* goes negative Move the assertion on the in-progress invalidation count from the primary MMU's notifier path to KVM's common notification path, i.e. assert that the count doesn't go negative even when the invalidation is coming from KVM itself. Opportunistically convert the assertion to a KVM_BUG_ON(), i.e. kill only the affected VM, not the entire kernel. A corrupted count is fatal to the VM, e.g. the non-zero (negative) count will cause mmu_invalidate_retry() to block any and all attempts to install new mappings. But it's far from guaranteed that an end() without a start() is fatal or even problematic to anything other than the target VM, e.g. the underlying bug could simply be a duplicate call to end(). And it's much more likely that a missed invalidation, i.e. a potential use-after-free, would manifest as no notification whatsoever, not an end() without a start(). Signed-off-by: Sean Christopherson Reviewed-by: Paolo Bonzini Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Message-Id: <20231027182217.3615211-3-seanjc@google.com> Reviewed-by: Kai Huang Signed-off-by: Paolo Bonzini --- virt/kvm/kvm_main.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 0524933856d4..5a97e6c7d9c2 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -833,6 +833,7 @@ void kvm_mmu_invalidate_end(struct kvm *kvm, unsigned long start, * in conjunction with the smp_rmb in mmu_invalidate_retry(). */ kvm->mmu_invalidate_in_progress--; + KVM_BUG_ON(kvm->mmu_invalidate_in_progress < 0, kvm); } static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, @@ -863,8 +864,6 @@ static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, */ if (wake) rcuwait_wake_up(&kvm->mn_memslots_update_rcuwait); - - BUG_ON(kvm->mmu_invalidate_in_progress < 0); } static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn, From 8569992d64b8f750e34b7858eac5d7daaf0f80fd Mon Sep 17 00:00:00 2001 From: Chao Peng Date: Fri, 27 Oct 2023 11:21:45 -0700 Subject: [PATCH 004/336] KVM: Use gfn instead of hva for mmu_notifier_retry Currently in mmu_notifier invalidate path, hva range is recorded and then checked against by mmu_invalidate_retry_hva() in the page fault handling path. However, for the soon-to-be-introduced private memory, a page fault may not have a hva associated, checking gfn(gpa) makes more sense. For existing hva based shared memory, gfn is expected to also work. The only downside is when aliasing multiple gfns to a single hva, the current algorithm of checking multiple ranges could result in a much larger range being rejected. Such aliasing should be uncommon, so the impact is expected small. Suggested-by: Sean Christopherson Cc: Xu Yilun Signed-off-by: Chao Peng Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba [sean: convert vmx_set_apic_access_page_addr() to gfn-based API] Signed-off-by: Sean Christopherson Reviewed-by: Paolo Bonzini Reviewed-by: Xu Yilun Message-Id: <20231027182217.3615211-4-seanjc@google.com> Reviewed-by: Kai Huang Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 10 ++++++---- arch/x86/kvm/vmx/vmx.c | 11 +++++----- include/linux/kvm_host.h | 33 +++++++++++++++++++----------- virt/kvm/kvm_main.c | 43 +++++++++++++++++++++++++++++++--------- 4 files changed, 66 insertions(+), 31 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index b0f01d605617..b2d916f786ca 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3056,7 +3056,7 @@ static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep) * * There are several ways to safely use this helper: * - * - Check mmu_invalidate_retry_hva() after grabbing the mapping level, before + * - Check mmu_invalidate_retry_gfn() after grabbing the mapping level, before * consuming it. In this case, mmu_lock doesn't need to be held during the * lookup, but it does need to be held while checking the MMU notifier. * @@ -4366,7 +4366,7 @@ static bool is_page_fault_stale(struct kvm_vcpu *vcpu, return true; return fault->slot && - mmu_invalidate_retry_hva(vcpu->kvm, fault->mmu_seq, fault->hva); + mmu_invalidate_retry_gfn(vcpu->kvm, fault->mmu_seq, fault->gfn); } static int direct_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) @@ -6260,7 +6260,9 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) write_lock(&kvm->mmu_lock); - kvm_mmu_invalidate_begin(kvm, 0, -1ul); + kvm_mmu_invalidate_begin(kvm); + + kvm_mmu_invalidate_range_add(kvm, gfn_start, gfn_end); flush = kvm_rmap_zap_gfn_range(kvm, gfn_start, gfn_end); @@ -6270,7 +6272,7 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) if (flush) kvm_flush_remote_tlbs_range(kvm, gfn_start, gfn_end - gfn_start); - kvm_mmu_invalidate_end(kvm, 0, -1ul); + kvm_mmu_invalidate_end(kvm); write_unlock(&kvm->mmu_lock); } diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index be20a60047b1..40e3780d73ae 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -6757,10 +6757,10 @@ static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu) return; /* - * Grab the memslot so that the hva lookup for the mmu_notifier retry - * is guaranteed to use the same memslot as the pfn lookup, i.e. rely - * on the pfn lookup's validation of the memslot to ensure a valid hva - * is used for the retry check. + * Explicitly grab the memslot using KVM's internal slot ID to ensure + * KVM doesn't unintentionally grab a userspace memslot. It _should_ + * be impossible for userspace to create a memslot for the APIC when + * APICv is enabled, but paranoia won't hurt in this case. */ slot = id_to_memslot(slots, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT); if (!slot || slot->flags & KVM_MEMSLOT_INVALID) @@ -6785,8 +6785,7 @@ static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu) return; read_lock(&vcpu->kvm->mmu_lock); - if (mmu_invalidate_retry_hva(kvm, mmu_seq, - gfn_to_hva_memslot(slot, gfn))) { + if (mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn)) { kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); read_unlock(&vcpu->kvm->mmu_lock); goto out; diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index fb6c6109fdca..11d091688346 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -787,8 +787,8 @@ struct kvm { struct mmu_notifier mmu_notifier; unsigned long mmu_invalidate_seq; long mmu_invalidate_in_progress; - unsigned long mmu_invalidate_range_start; - unsigned long mmu_invalidate_range_end; + gfn_t mmu_invalidate_range_start; + gfn_t mmu_invalidate_range_end; #endif struct list_head devices; u64 manual_dirty_log_protect; @@ -1392,10 +1392,9 @@ void kvm_mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc); void *kvm_mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc); #endif -void kvm_mmu_invalidate_begin(struct kvm *kvm, unsigned long start, - unsigned long end); -void kvm_mmu_invalidate_end(struct kvm *kvm, unsigned long start, - unsigned long end); +void kvm_mmu_invalidate_begin(struct kvm *kvm); +void kvm_mmu_invalidate_range_add(struct kvm *kvm, gfn_t start, gfn_t end); +void kvm_mmu_invalidate_end(struct kvm *kvm); long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg); @@ -1970,9 +1969,9 @@ static inline int mmu_invalidate_retry(struct kvm *kvm, unsigned long mmu_seq) return 0; } -static inline int mmu_invalidate_retry_hva(struct kvm *kvm, +static inline int mmu_invalidate_retry_gfn(struct kvm *kvm, unsigned long mmu_seq, - unsigned long hva) + gfn_t gfn) { lockdep_assert_held(&kvm->mmu_lock); /* @@ -1981,10 +1980,20 @@ static inline int mmu_invalidate_retry_hva(struct kvm *kvm, * that might be being invalidated. Note that it may include some false * positives, due to shortcuts when handing concurrent invalidations. */ - if (unlikely(kvm->mmu_invalidate_in_progress) && - hva >= kvm->mmu_invalidate_range_start && - hva < kvm->mmu_invalidate_range_end) - return 1; + if (unlikely(kvm->mmu_invalidate_in_progress)) { + /* + * Dropping mmu_lock after bumping mmu_invalidate_in_progress + * but before updating the range is a KVM bug. + */ + if (WARN_ON_ONCE(kvm->mmu_invalidate_range_start == INVALID_GPA || + kvm->mmu_invalidate_range_end == INVALID_GPA)) + return 1; + + if (gfn >= kvm->mmu_invalidate_range_start && + gfn < kvm->mmu_invalidate_range_end) + return 1; + } + if (kvm->mmu_invalidate_seq != mmu_seq) return 1; return 0; diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 5a97e6c7d9c2..9cc57b23ec81 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -543,9 +543,7 @@ static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn) typedef bool (*gfn_handler_t)(struct kvm *kvm, struct kvm_gfn_range *range); -typedef void (*on_lock_fn_t)(struct kvm *kvm, unsigned long start, - unsigned long end); - +typedef void (*on_lock_fn_t)(struct kvm *kvm); typedef void (*on_unlock_fn_t)(struct kvm *kvm); struct kvm_mmu_notifier_range { @@ -637,7 +635,8 @@ static __always_inline int __kvm_handle_hva_range(struct kvm *kvm, locked = true; KVM_MMU_LOCK(kvm); if (!IS_KVM_NULL_FN(range->on_lock)) - range->on_lock(kvm, range->start, range->end); + range->on_lock(kvm); + if (IS_KVM_NULL_FN(range->handler)) break; } @@ -742,16 +741,29 @@ static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn, kvm_handle_hva_range(mn, address, address + 1, arg, kvm_change_spte_gfn); } -void kvm_mmu_invalidate_begin(struct kvm *kvm, unsigned long start, - unsigned long end) +void kvm_mmu_invalidate_begin(struct kvm *kvm) { + lockdep_assert_held_write(&kvm->mmu_lock); /* * The count increase must become visible at unlock time as no * spte can be established without taking the mmu_lock and * count is also read inside the mmu_lock critical section. */ kvm->mmu_invalidate_in_progress++; + if (likely(kvm->mmu_invalidate_in_progress == 1)) { + kvm->mmu_invalidate_range_start = INVALID_GPA; + kvm->mmu_invalidate_range_end = INVALID_GPA; + } +} + +void kvm_mmu_invalidate_range_add(struct kvm *kvm, gfn_t start, gfn_t end) +{ + lockdep_assert_held_write(&kvm->mmu_lock); + + WARN_ON_ONCE(!kvm->mmu_invalidate_in_progress); + + if (likely(kvm->mmu_invalidate_range_start == INVALID_GPA)) { kvm->mmu_invalidate_range_start = start; kvm->mmu_invalidate_range_end = end; } else { @@ -771,6 +783,12 @@ void kvm_mmu_invalidate_begin(struct kvm *kvm, unsigned long start, } } +static bool kvm_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) +{ + kvm_mmu_invalidate_range_add(kvm, range->start, range->end); + return kvm_unmap_gfn_range(kvm, range); +} + static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, const struct mmu_notifier_range *range) { @@ -778,7 +796,7 @@ static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, const struct kvm_mmu_notifier_range hva_range = { .start = range->start, .end = range->end, - .handler = kvm_unmap_gfn_range, + .handler = kvm_mmu_unmap_gfn_range, .on_lock = kvm_mmu_invalidate_begin, .on_unlock = kvm_arch_guest_memory_reclaimed, .flush_on_ret = true, @@ -817,9 +835,10 @@ static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, return 0; } -void kvm_mmu_invalidate_end(struct kvm *kvm, unsigned long start, - unsigned long end) +void kvm_mmu_invalidate_end(struct kvm *kvm) { + lockdep_assert_held_write(&kvm->mmu_lock); + /* * This sequence increase will notify the kvm page fault that * the page that is going to be mapped in the spte could have @@ -834,6 +853,12 @@ void kvm_mmu_invalidate_end(struct kvm *kvm, unsigned long start, */ kvm->mmu_invalidate_in_progress--; KVM_BUG_ON(kvm->mmu_invalidate_in_progress < 0, kvm); + + /* + * Assert that at least one range was added between start() and end(). + * Not adding a range isn't fatal, but it is a KVM bug. + */ + WARN_ON_ONCE(kvm->mmu_invalidate_range_start == INVALID_GPA); } static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, From d497a0fab8b8457214fcc9b1a39530920ea7e95e Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 27 Oct 2023 11:21:46 -0700 Subject: [PATCH 005/336] KVM: WARN if there are dangling MMU invalidations at VM destruction Add an assertion that there are no in-progress MMU invalidations when a VM is being destroyed, with the exception of the scenario where KVM unregisters its MMU notifier between an .invalidate_range_start() call and the corresponding .invalidate_range_end(). KVM can't detect unpaired calls from the mmu_notifier due to the above exception waiver, but the assertion can detect KVM bugs, e.g. such as the bug that *almost* escaped initial guest_memfd development. Link: https://lore.kernel.org/all/e397d30c-c6af-e68f-d18e-b4e3739c5389@linux.intel.com Signed-off-by: Sean Christopherson Reviewed-by: Paolo Bonzini Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Message-Id: <20231027182217.3615211-5-seanjc@google.com> Signed-off-by: Paolo Bonzini --- virt/kvm/kvm_main.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 9cc57b23ec81..5422ce20dcba 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1358,9 +1358,16 @@ static void kvm_destroy_vm(struct kvm *kvm) * No threads can be waiting in kvm_swap_active_memslots() as the * last reference on KVM has been dropped, but freeing * memslots would deadlock without this manual intervention. + * + * If the count isn't unbalanced, i.e. KVM did NOT unregister its MMU + * notifier between a start() and end(), then there shouldn't be any + * in-progress invalidations. */ WARN_ON(rcuwait_active(&kvm->mn_memslots_update_rcuwait)); - kvm->mn_active_invalidate_count = 0; + if (kvm->mn_active_invalidate_count) + kvm->mn_active_invalidate_count = 0; + else + WARN_ON(kvm->mmu_invalidate_in_progress); #else kvm_flush_shadow_all(kvm); #endif From 1853d7502a19b34b2cfe4ea0698bee73323fec3a Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 27 Oct 2023 11:21:47 -0700 Subject: [PATCH 006/336] KVM: PPC: Drop dead code related to KVM_ARCH_WANT_MMU_NOTIFIER Assert that both KVM_ARCH_WANT_MMU_NOTIFIER and CONFIG_MMU_NOTIFIER are defined when KVM is enabled, and return '1' unconditionally for the CONFIG_KVM_BOOK3S_HV_POSSIBLE=n path. All flavors of PPC support for KVM select MMU_NOTIFIER, and KVM_ARCH_WANT_MMU_NOTIFIER is unconditionally defined by arch/powerpc/include/asm/kvm_host.h. Effectively dropping use of KVM_ARCH_WANT_MMU_NOTIFIER will simplify a future cleanup to turn KVM_ARCH_WANT_MMU_NOTIFIER into a Kconfig, i.e. will allow combining all of the #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) checks into a single #ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER without having to worry about PPC's "bare" usage of KVM_ARCH_WANT_MMU_NOTIFIER. Signed-off-by: Sean Christopherson Reviewed-by: Paolo Bonzini Reviewed-by: Fuad Tabba Message-Id: <20231027182217.3615211-6-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/powerpc/kvm/powerpc.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index 7197c8256668..b0a512ede764 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -632,12 +632,13 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) break; #endif case KVM_CAP_SYNC_MMU: +#if !defined(CONFIG_MMU_NOTIFIER) || !defined(KVM_ARCH_WANT_MMU_NOTIFIER) + BUILD_BUG(); +#endif #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE r = hv_enabled; -#elif defined(KVM_ARCH_WANT_MMU_NOTIFIER) - r = 1; #else - r = 0; + r = 1; #endif break; #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE From 4a2e993faad3880962540ab8e3b68f22e48b18e8 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 27 Oct 2023 11:21:48 -0700 Subject: [PATCH 007/336] KVM: PPC: Return '1' unconditionally for KVM_CAP_SYNC_MMU Advertise that KVM's MMU is synchronized with the primary MMU for all flavors of PPC KVM support, i.e. advertise that the MMU is synchronized when CONFIG_KVM_BOOK3S_HV_POSSIBLE=y but the VM is not using hypervisor mode (a.k.a. PR VMs). PR VMs, via kvm_unmap_gfn_range_pr(), do the right thing for mmu_notifier invalidation events, and more tellingly, KVM returns '1' for KVM_CAP_SYNC_MMU when CONFIG_KVM_BOOK3S_HV_POSSIBLE=n and CONFIG_KVM_BOOK3S_PR_POSSIBLE=y, i.e. KVM already advertises a synchronized MMU for PR VMs, just not when CONFIG_KVM_BOOK3S_HV_POSSIBLE=y. Suggested-by: Paolo Bonzini Signed-off-by: Sean Christopherson Message-Id: <20231027182217.3615211-7-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/powerpc/kvm/powerpc.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index b0a512ede764..8d3ec483bc2b 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -635,11 +635,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) #if !defined(CONFIG_MMU_NOTIFIER) || !defined(KVM_ARCH_WANT_MMU_NOTIFIER) BUILD_BUG(); #endif -#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE - r = hv_enabled; -#else r = 1; -#endif break; #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE case KVM_CAP_PPC_HTAB_FD: From f128cf8cfbecccf95e891ae90d9c917df5117c7a Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 27 Oct 2023 11:21:49 -0700 Subject: [PATCH 008/336] KVM: Convert KVM_ARCH_WANT_MMU_NOTIFIER to CONFIG_KVM_GENERIC_MMU_NOTIFIER Convert KVM_ARCH_WANT_MMU_NOTIFIER into a Kconfig and select it where appropriate to effectively maintain existing behavior. Using a proper Kconfig will simplify building more functionality on top of KVM's mmu_notifier infrastructure. Add a forward declaration of kvm_gfn_range to kvm_types.h so that including arch/powerpc/include/asm/kvm_ppc.h's with CONFIG_KVM=n doesn't generate warnings due to kvm_gfn_range being undeclared. PPC defines hooks for PR vs. HV without guarding them via #ifdeffery, e.g. bool (*unmap_gfn_range)(struct kvm *kvm, struct kvm_gfn_range *range); bool (*age_gfn)(struct kvm *kvm, struct kvm_gfn_range *range); bool (*test_age_gfn)(struct kvm *kvm, struct kvm_gfn_range *range); bool (*set_spte_gfn)(struct kvm *kvm, struct kvm_gfn_range *range); Alternatively, PPC could forward declare kvm_gfn_range, but there's no good reason not to define it in common KVM. Acked-by: Anup Patel Signed-off-by: Sean Christopherson Reviewed-by: Paolo Bonzini Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Message-Id: <20231027182217.3615211-8-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/arm64/include/asm/kvm_host.h | 2 -- arch/arm64/kvm/Kconfig | 2 +- arch/loongarch/include/asm/kvm_host.h | 1 - arch/loongarch/kvm/Kconfig | 2 +- arch/mips/include/asm/kvm_host.h | 2 -- arch/mips/kvm/Kconfig | 2 +- arch/powerpc/include/asm/kvm_host.h | 2 -- arch/powerpc/kvm/Kconfig | 8 ++++---- arch/powerpc/kvm/powerpc.c | 4 +--- arch/riscv/include/asm/kvm_host.h | 2 -- arch/riscv/kvm/Kconfig | 2 +- arch/x86/include/asm/kvm_host.h | 2 -- arch/x86/kvm/Kconfig | 2 +- include/linux/kvm_host.h | 6 +++--- include/linux/kvm_types.h | 1 + virt/kvm/Kconfig | 4 ++++ virt/kvm/kvm_main.c | 10 +++++----- 17 files changed, 23 insertions(+), 31 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 5653d3553e3e..9029fe09f3f6 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -954,8 +954,6 @@ int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu, int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu, struct kvm_vcpu_events *events); -#define KVM_ARCH_WANT_MMU_NOTIFIER - void kvm_arm_halt_guest(struct kvm *kvm); void kvm_arm_resume_guest(struct kvm *kvm); diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig index 83c1e09be42e..1a777715199f 100644 --- a/arch/arm64/kvm/Kconfig +++ b/arch/arm64/kvm/Kconfig @@ -22,7 +22,7 @@ menuconfig KVM bool "Kernel-based Virtual Machine (KVM) support" depends on HAVE_KVM select KVM_GENERIC_HARDWARE_ENABLING - select MMU_NOTIFIER + select KVM_GENERIC_MMU_NOTIFIER select PREEMPT_NOTIFIERS select HAVE_KVM_CPU_RELAX_INTERCEPT select KVM_MMIO diff --git a/arch/loongarch/include/asm/kvm_host.h b/arch/loongarch/include/asm/kvm_host.h index 11328700d4fa..b108301c2e5a 100644 --- a/arch/loongarch/include/asm/kvm_host.h +++ b/arch/loongarch/include/asm/kvm_host.h @@ -183,7 +183,6 @@ void kvm_flush_tlb_all(void); void kvm_flush_tlb_gpa(struct kvm_vcpu *vcpu, unsigned long gpa); int kvm_handle_mm_fault(struct kvm_vcpu *vcpu, unsigned long badv, bool write); -#define KVM_ARCH_WANT_MMU_NOTIFIER void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte); int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end, bool blockable); int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end); diff --git a/arch/loongarch/kvm/Kconfig b/arch/loongarch/kvm/Kconfig index fda425babfb2..f22bae89b07d 100644 --- a/arch/loongarch/kvm/Kconfig +++ b/arch/loongarch/kvm/Kconfig @@ -26,9 +26,9 @@ config KVM select HAVE_KVM_VCPU_ASYNC_IOCTL select KVM_GENERIC_DIRTYLOG_READ_PROTECT select KVM_GENERIC_HARDWARE_ENABLING + select KVM_GENERIC_MMU_NOTIFIER select KVM_MMIO select KVM_XFER_TO_GUEST_WORK - select MMU_NOTIFIER select PREEMPT_NOTIFIERS help Support hosting virtualized guest machines using diff --git a/arch/mips/include/asm/kvm_host.h b/arch/mips/include/asm/kvm_host.h index 54a85f1d4f2c..179f320cc231 100644 --- a/arch/mips/include/asm/kvm_host.h +++ b/arch/mips/include/asm/kvm_host.h @@ -810,8 +810,6 @@ int kvm_mips_mkclean_gpa_pt(struct kvm *kvm, gfn_t start_gfn, gfn_t end_gfn); pgd_t *kvm_pgd_alloc(void); void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu); -#define KVM_ARCH_WANT_MMU_NOTIFIER - /* Emulation */ enum emulation_result update_pc(struct kvm_vcpu *vcpu, u32 cause); int kvm_get_badinstr(u32 *opc, struct kvm_vcpu *vcpu, u32 *out); diff --git a/arch/mips/kvm/Kconfig b/arch/mips/kvm/Kconfig index a8cdba75f98d..c04987d2ed2e 100644 --- a/arch/mips/kvm/Kconfig +++ b/arch/mips/kvm/Kconfig @@ -25,7 +25,7 @@ config KVM select HAVE_KVM_EVENTFD select HAVE_KVM_VCPU_ASYNC_IOCTL select KVM_MMIO - select MMU_NOTIFIER + select KVM_GENERIC_MMU_NOTIFIER select INTERVAL_TREE select KVM_GENERIC_HARDWARE_ENABLING help diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h index 14ee0dece853..4b5c3f2acf78 100644 --- a/arch/powerpc/include/asm/kvm_host.h +++ b/arch/powerpc/include/asm/kvm_host.h @@ -62,8 +62,6 @@ #include -#define KVM_ARCH_WANT_MMU_NOTIFIER - #define HPTEG_CACHE_NUM (1 << 15) #define HPTEG_HASH_BITS_PTE 13 #define HPTEG_HASH_BITS_PTE_LONG 12 diff --git a/arch/powerpc/kvm/Kconfig b/arch/powerpc/kvm/Kconfig index 902611954200..b33358ee6424 100644 --- a/arch/powerpc/kvm/Kconfig +++ b/arch/powerpc/kvm/Kconfig @@ -42,7 +42,7 @@ config KVM_BOOK3S_64_HANDLER config KVM_BOOK3S_PR_POSSIBLE bool select KVM_MMIO - select MMU_NOTIFIER + select KVM_GENERIC_MMU_NOTIFIER config KVM_BOOK3S_HV_POSSIBLE bool @@ -85,7 +85,7 @@ config KVM_BOOK3S_64_HV tristate "KVM for POWER7 and later using hypervisor mode in host" depends on KVM_BOOK3S_64 && PPC_POWERNV select KVM_BOOK3S_HV_POSSIBLE - select MMU_NOTIFIER + select KVM_GENERIC_MMU_NOTIFIER select CMA help Support running unmodified book3s_64 guest kernels in @@ -194,7 +194,7 @@ config KVM_E500V2 depends on !CONTEXT_TRACKING_USER select KVM select KVM_MMIO - select MMU_NOTIFIER + select KVM_GENERIC_MMU_NOTIFIER help Support running unmodified E500 guest kernels in virtual machines on E500v2 host processors. @@ -211,7 +211,7 @@ config KVM_E500MC select KVM select KVM_MMIO select KVM_BOOKE_HV - select MMU_NOTIFIER + select KVM_GENERIC_MMU_NOTIFIER help Support running unmodified E500MC/E5500/E6500 guest kernels in virtual machines on E500MC/E5500/E6500 host processors. diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index 8d3ec483bc2b..aac75c98a956 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -632,9 +632,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) break; #endif case KVM_CAP_SYNC_MMU: -#if !defined(CONFIG_MMU_NOTIFIER) || !defined(KVM_ARCH_WANT_MMU_NOTIFIER) - BUILD_BUG(); -#endif + BUILD_BUG_ON(!IS_ENABLED(CONFIG_KVM_GENERIC_MMU_NOTIFIER)); r = 1; break; #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h index 0eefd9c991ae..6964dd235e97 100644 --- a/arch/riscv/include/asm/kvm_host.h +++ b/arch/riscv/include/asm/kvm_host.h @@ -267,8 +267,6 @@ struct kvm_vcpu_arch { static inline void kvm_arch_sync_events(struct kvm *kvm) {} static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} -#define KVM_ARCH_WANT_MMU_NOTIFIER - #define KVM_RISCV_GSTAGE_TLB_MIN_ORDER 12 void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid, diff --git a/arch/riscv/kvm/Kconfig b/arch/riscv/kvm/Kconfig index dfc237d7875b..ae2e05f050ec 100644 --- a/arch/riscv/kvm/Kconfig +++ b/arch/riscv/kvm/Kconfig @@ -30,7 +30,7 @@ config KVM select KVM_GENERIC_HARDWARE_ENABLING select KVM_MMIO select KVM_XFER_TO_GUEST_WORK - select MMU_NOTIFIER + select KVM_GENERIC_MMU_NOTIFIER select PREEMPT_NOTIFIERS help Support hosting virtualized guest machines. diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index d7036982332e..6f559fb75e6d 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -2141,8 +2141,6 @@ enum { # define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, 0) #endif -#define KVM_ARCH_WANT_MMU_NOTIFIER - int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v); int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu); int kvm_cpu_has_extint(struct kvm_vcpu *v); diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index 950c12868d30..e61383674c75 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -24,7 +24,7 @@ config KVM depends on HIGH_RES_TIMERS depends on X86_LOCAL_APIC select PREEMPT_NOTIFIERS - select MMU_NOTIFIER + select KVM_GENERIC_MMU_NOTIFIER select HAVE_KVM_IRQCHIP select HAVE_KVM_PFNCACHE select HAVE_KVM_IRQFD diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 11d091688346..5faba69403ac 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -253,7 +253,7 @@ bool kvm_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, int kvm_async_pf_wakeup_all(struct kvm_vcpu *vcpu); #endif -#ifdef KVM_ARCH_WANT_MMU_NOTIFIER +#ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER union kvm_mmu_notifier_arg { pte_t pte; }; @@ -783,7 +783,7 @@ struct kvm { struct hlist_head irq_ack_notifier_list; #endif -#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) +#ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER struct mmu_notifier mmu_notifier; unsigned long mmu_invalidate_seq; long mmu_invalidate_in_progress; @@ -1946,7 +1946,7 @@ extern const struct _kvm_stats_desc kvm_vm_stats_desc[]; extern const struct kvm_stats_header kvm_vcpu_stats_header; extern const struct _kvm_stats_desc kvm_vcpu_stats_desc[]; -#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) +#ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER static inline int mmu_invalidate_retry(struct kvm *kvm, unsigned long mmu_seq) { if (unlikely(kvm->mmu_invalidate_in_progress)) diff --git a/include/linux/kvm_types.h b/include/linux/kvm_types.h index 6f4737d5046a..9d1f7835d8c1 100644 --- a/include/linux/kvm_types.h +++ b/include/linux/kvm_types.h @@ -6,6 +6,7 @@ struct kvm; struct kvm_async_pf; struct kvm_device_ops; +struct kvm_gfn_range; struct kvm_interrupt; struct kvm_irq_routing_table; struct kvm_memory_slot; diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig index 484d0873061c..ecae2914c97e 100644 --- a/virt/kvm/Kconfig +++ b/virt/kvm/Kconfig @@ -92,3 +92,7 @@ config HAVE_KVM_PM_NOTIFIER config KVM_GENERIC_HARDWARE_ENABLING bool + +config KVM_GENERIC_MMU_NOTIFIER + select MMU_NOTIFIER + bool diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 5422ce20dcba..dc81279ea385 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -535,7 +535,7 @@ void kvm_destroy_vcpus(struct kvm *kvm) } EXPORT_SYMBOL_GPL(kvm_destroy_vcpus); -#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) +#ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn) { return container_of(mn, struct kvm, mmu_notifier); @@ -962,14 +962,14 @@ static int kvm_init_mmu_notifier(struct kvm *kvm) return mmu_notifier_register(&kvm->mmu_notifier, current->mm); } -#else /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */ +#else /* !CONFIG_KVM_GENERIC_MMU_NOTIFIER */ static int kvm_init_mmu_notifier(struct kvm *kvm) { return 0; } -#endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */ +#endif /* CONFIG_KVM_GENERIC_MMU_NOTIFIER */ #ifdef CONFIG_HAVE_KVM_PM_NOTIFIER static int kvm_pm_notifier_call(struct notifier_block *bl, @@ -1289,7 +1289,7 @@ out_err: out_err_no_debugfs: kvm_coalesced_mmio_free(kvm); out_no_coalesced_mmio: -#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) +#ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER if (kvm->mmu_notifier.ops) mmu_notifier_unregister(&kvm->mmu_notifier, current->mm); #endif @@ -1349,7 +1349,7 @@ static void kvm_destroy_vm(struct kvm *kvm) kvm->buses[i] = NULL; } kvm_coalesced_mmio_free(kvm); -#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) +#ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm); /* * At this point, pending calls to invalidate_range_start() From bb58b90b1a8f753b582055adaf448214a8e22c31 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 27 Oct 2023 11:21:50 -0700 Subject: [PATCH 009/336] KVM: Introduce KVM_SET_USER_MEMORY_REGION2 Introduce a "version 2" of KVM_SET_USER_MEMORY_REGION so that additional information can be supplied without setting userspace up to fail. The padding in the new kvm_userspace_memory_region2 structure will be used to pass a file descriptor in addition to the userspace_addr, i.e. allow userspace to point at a file descriptor and map memory into a guest that is NOT mapped into host userspace. Alternatively, KVM could simply add "struct kvm_userspace_memory_region2" without a new ioctl(), but as Paolo pointed out, adding a new ioctl() makes detection of bad flags a bit more robust, e.g. if the new fd field is guarded only by a flag and not a new ioctl(), then a userspace bug (setting a "bad" flag) would generate out-of-bounds access instead of an -EINVAL error. Cc: Jarkko Sakkinen Reviewed-by: Paolo Bonzini Reviewed-by: Xiaoyao Li Signed-off-by: Sean Christopherson Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Message-Id: <20231027182217.3615211-9-seanjc@google.com> Acked-by: Kai Huang Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/api.rst | 22 +++++++++++++ arch/x86/kvm/x86.c | 2 +- include/linux/kvm_host.h | 4 +-- include/uapi/linux/kvm.h | 13 ++++++++ virt/kvm/kvm_main.c | 57 +++++++++++++++++++++++++++++----- 5 files changed, 87 insertions(+), 11 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 7025b3751027..9edd9e436bab 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -6192,6 +6192,28 @@ to know what fields can be changed for the system register described by ``op0, op1, crn, crm, op2``. KVM rejects ID register values that describe a superset of the features supported by the system. +4.140 KVM_SET_USER_MEMORY_REGION2 +--------------------------------- + +:Capability: KVM_CAP_USER_MEMORY2 +:Architectures: all +:Type: vm ioctl +:Parameters: struct kvm_userspace_memory_region2 (in) +:Returns: 0 on success, -1 on error + +:: + + struct kvm_userspace_memory_region2 { + __u32 slot; + __u32 flags; + __u64 guest_phys_addr; + __u64 memory_size; /* bytes */ + __u64 userspace_addr; /* start of the userspace allocated memory */ + __u64 pad[16]; + }; + +See KVM_SET_USER_MEMORY_REGION. + 5. The kvm_run structure ======================== diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 2c924075f6f1..7b389f27dffc 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -12576,7 +12576,7 @@ void __user * __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, } for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { - struct kvm_userspace_memory_region m; + struct kvm_userspace_memory_region2 m; m.slot = id | (i << 16); m.flags = 0; diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 5faba69403ac..4e741ff27af3 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1146,9 +1146,9 @@ enum kvm_mr_change { }; int kvm_set_memory_region(struct kvm *kvm, - const struct kvm_userspace_memory_region *mem); + const struct kvm_userspace_memory_region2 *mem); int __kvm_set_memory_region(struct kvm *kvm, - const struct kvm_userspace_memory_region *mem); + const struct kvm_userspace_memory_region2 *mem); void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot); void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen); int kvm_arch_prepare_memory_region(struct kvm *kvm, diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 211b86de35ac..308cc70bd6ab 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -95,6 +95,16 @@ struct kvm_userspace_memory_region { __u64 userspace_addr; /* start of the userspace allocated memory */ }; +/* for KVM_SET_USER_MEMORY_REGION2 */ +struct kvm_userspace_memory_region2 { + __u32 slot; + __u32 flags; + __u64 guest_phys_addr; + __u64 memory_size; + __u64 userspace_addr; + __u64 pad[16]; +}; + /* * The bit 0 ~ bit 15 of kvm_userspace_memory_region::flags are visible for * userspace, other bits are reserved for kvm internal use which are defined @@ -1201,6 +1211,7 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE 228 #define KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES 229 #define KVM_CAP_ARM_SUPPORTED_REG_MASK_RANGES 230 +#define KVM_CAP_USER_MEMORY2 231 #ifdef KVM_CAP_IRQ_ROUTING @@ -1483,6 +1494,8 @@ struct kvm_vfio_spapr_tce { struct kvm_userspace_memory_region) #define KVM_SET_TSS_ADDR _IO(KVMIO, 0x47) #define KVM_SET_IDENTITY_MAP_ADDR _IOW(KVMIO, 0x48, __u64) +#define KVM_SET_USER_MEMORY_REGION2 _IOW(KVMIO, 0x49, \ + struct kvm_userspace_memory_region2) /* enable ucontrol for s390 */ struct kvm_s390_ucas_mapping { diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index dc81279ea385..756b94ecd511 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1580,7 +1580,15 @@ static void kvm_replace_memslot(struct kvm *kvm, } } -static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem) +/* + * Flags that do not access any of the extra space of struct + * kvm_userspace_memory_region2. KVM_SET_USER_MEMORY_REGION_V1_FLAGS + * only allows these. + */ +#define KVM_SET_USER_MEMORY_REGION_V1_FLAGS \ + (KVM_MEM_LOG_DIRTY_PAGES | KVM_MEM_READONLY) + +static int check_memory_region_flags(const struct kvm_userspace_memory_region2 *mem) { u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES; @@ -1982,7 +1990,7 @@ static bool kvm_check_memslot_overlap(struct kvm_memslots *slots, int id, * Must be called holding kvm->slots_lock for write. */ int __kvm_set_memory_region(struct kvm *kvm, - const struct kvm_userspace_memory_region *mem) + const struct kvm_userspace_memory_region2 *mem) { struct kvm_memory_slot *old, *new; struct kvm_memslots *slots; @@ -2086,7 +2094,7 @@ int __kvm_set_memory_region(struct kvm *kvm, EXPORT_SYMBOL_GPL(__kvm_set_memory_region); int kvm_set_memory_region(struct kvm *kvm, - const struct kvm_userspace_memory_region *mem) + const struct kvm_userspace_memory_region2 *mem) { int r; @@ -2098,7 +2106,7 @@ int kvm_set_memory_region(struct kvm *kvm, EXPORT_SYMBOL_GPL(kvm_set_memory_region); static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, - struct kvm_userspace_memory_region *mem) + struct kvm_userspace_memory_region2 *mem) { if ((u16)mem->slot >= KVM_USER_MEM_SLOTS) return -EINVAL; @@ -4568,6 +4576,7 @@ static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) { switch (arg) { case KVM_CAP_USER_MEMORY: + case KVM_CAP_USER_MEMORY2: case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS: case KVM_CAP_INTERNAL_ERROR_DATA: @@ -4823,6 +4832,14 @@ static int kvm_vm_ioctl_get_stats_fd(struct kvm *kvm) return fd; } +#define SANITY_CHECK_MEM_REGION_FIELD(field) \ +do { \ + BUILD_BUG_ON(offsetof(struct kvm_userspace_memory_region, field) != \ + offsetof(struct kvm_userspace_memory_region2, field)); \ + BUILD_BUG_ON(sizeof_field(struct kvm_userspace_memory_region, field) != \ + sizeof_field(struct kvm_userspace_memory_region2, field)); \ +} while (0) + static long kvm_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { @@ -4845,15 +4862,39 @@ static long kvm_vm_ioctl(struct file *filp, r = kvm_vm_ioctl_enable_cap_generic(kvm, &cap); break; } + case KVM_SET_USER_MEMORY_REGION2: case KVM_SET_USER_MEMORY_REGION: { - struct kvm_userspace_memory_region kvm_userspace_mem; + struct kvm_userspace_memory_region2 mem; + unsigned long size; + + if (ioctl == KVM_SET_USER_MEMORY_REGION) { + /* + * Fields beyond struct kvm_userspace_memory_region shouldn't be + * accessed, but avoid leaking kernel memory in case of a bug. + */ + memset(&mem, 0, sizeof(mem)); + size = sizeof(struct kvm_userspace_memory_region); + } else { + size = sizeof(struct kvm_userspace_memory_region2); + } + + /* Ensure the common parts of the two structs are identical. */ + SANITY_CHECK_MEM_REGION_FIELD(slot); + SANITY_CHECK_MEM_REGION_FIELD(flags); + SANITY_CHECK_MEM_REGION_FIELD(guest_phys_addr); + SANITY_CHECK_MEM_REGION_FIELD(memory_size); + SANITY_CHECK_MEM_REGION_FIELD(userspace_addr); r = -EFAULT; - if (copy_from_user(&kvm_userspace_mem, argp, - sizeof(kvm_userspace_mem))) + if (copy_from_user(&mem, argp, size)) goto out; - r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem); + r = -EINVAL; + if (ioctl == KVM_SET_USER_MEMORY_REGION && + (mem.flags & ~KVM_SET_USER_MEMORY_REGION_V1_FLAGS)) + goto out; + + r = kvm_vm_ioctl_set_memory_region(kvm, &mem); break; } case KVM_GET_DIRTY_LOG: { From 16f95f3b95caded251a0440051e44a2fbe9e5f55 Mon Sep 17 00:00:00 2001 From: Chao Peng Date: Fri, 27 Oct 2023 11:21:51 -0700 Subject: [PATCH 010/336] KVM: Add KVM_EXIT_MEMORY_FAULT exit to report faults to userspace Add a new KVM exit type to allow userspace to handle memory faults that KVM cannot resolve, but that userspace *may* be able to handle (without terminating the guest). KVM will initially use KVM_EXIT_MEMORY_FAULT to report implicit conversions between private and shared memory. With guest private memory, there will be two kind of memory conversions: - explicit conversion: happens when the guest explicitly calls into KVM to map a range (as private or shared) - implicit conversion: happens when the guest attempts to access a gfn that is configured in the "wrong" state (private vs. shared) On x86 (first architecture to support guest private memory), explicit conversions will be reported via KVM_EXIT_HYPERCALL+KVM_HC_MAP_GPA_RANGE, but reporting KVM_EXIT_HYPERCALL for implicit conversions is undesriable as there is (obviously) no hypercall, and there is no guarantee that the guest actually intends to convert between private and shared, i.e. what KVM thinks is an implicit conversion "request" could actually be the result of a guest code bug. KVM_EXIT_MEMORY_FAULT will be used to report memory faults that appear to be implicit conversions. Note! To allow for future possibilities where KVM reports KVM_EXIT_MEMORY_FAULT and fills run->memory_fault on _any_ unresolved fault, KVM returns "-EFAULT" (-1 with errno == EFAULT from userspace's perspective), not '0'! Due to historical baggage within KVM, exiting to userspace with '0' from deep callstacks, e.g. in emulation paths, is infeasible as doing so would require a near-complete overhaul of KVM, whereas KVM already propagates -errno return codes to userspace even when the -errno originated in a low level helper. Report the gpa+size instead of a single gfn even though the initial usage is expected to always report single pages. It's entirely possible, likely even, that KVM will someday support sub-page granularity faults, e.g. Intel's sub-page protection feature allows for additional protections at 128-byte granularity. Link: https://lore.kernel.org/all/20230908222905.1321305-5-amoorthy@google.com Link: https://lore.kernel.org/all/ZQ3AmLO2SYv3DszH@google.com Cc: Anish Moorthy Cc: David Matlack Suggested-by: Sean Christopherson Co-developed-by: Yu Zhang Signed-off-by: Yu Zhang Signed-off-by: Chao Peng Co-developed-by: Sean Christopherson Signed-off-by: Sean Christopherson Reviewed-by: Paolo Bonzini Message-Id: <20231027182217.3615211-10-seanjc@google.com> Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Reviewed-by: Xiaoyao Li Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/api.rst | 41 ++++++++++++++++++++++++++++++++++ arch/x86/kvm/x86.c | 1 + include/linux/kvm_host.h | 11 +++++++++ include/uapi/linux/kvm.h | 8 +++++++ 4 files changed, 61 insertions(+) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 9edd9e436bab..27d945d5b4e4 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -6846,6 +6846,26 @@ array field represents return values. The userspace should update the return values of SBI call before resuming the VCPU. For more details on RISC-V SBI spec refer, https://github.com/riscv/riscv-sbi-doc. +:: + + /* KVM_EXIT_MEMORY_FAULT */ + struct { + __u64 flags; + __u64 gpa; + __u64 size; + } memory_fault; + +KVM_EXIT_MEMORY_FAULT indicates the vCPU has encountered a memory fault that +could not be resolved by KVM. The 'gpa' and 'size' (in bytes) describe the +guest physical address range [gpa, gpa + size) of the fault. The 'flags' field +describes properties of the faulting access that are likely pertinent. +Currently, no flags are defined. + +Note! KVM_EXIT_MEMORY_FAULT is unique among all KVM exit reasons in that it +accompanies a return code of '-1', not '0'! errno will always be set to EFAULT +or EHWPOISON when KVM exits with KVM_EXIT_MEMORY_FAULT, userspace should assume +kvm_run.exit_reason is stale/undefined for all other error numbers. + :: /* KVM_EXIT_NOTIFY */ @@ -7880,6 +7900,27 @@ This capability is aimed to mitigate the threat that malicious VMs can cause CPU stuck (due to event windows don't open up) and make the CPU unavailable to host or other VMs. +7.34 KVM_CAP_MEMORY_FAULT_INFO +------------------------------ + +:Architectures: x86 +:Returns: Informational only, -EINVAL on direct KVM_ENABLE_CAP. + +The presence of this capability indicates that KVM_RUN will fill +kvm_run.memory_fault if KVM cannot resolve a guest page fault VM-Exit, e.g. if +there is a valid memslot but no backing VMA for the corresponding host virtual +address. + +The information in kvm_run.memory_fault is valid if and only if KVM_RUN returns +an error with errno=EFAULT or errno=EHWPOISON *and* kvm_run.exit_reason is set +to KVM_EXIT_MEMORY_FAULT. + +Note: Userspaces which attempt to resolve memory faults so that they can retry +KVM_RUN are encouraged to guard against repeatedly receiving the same +error/annotated fault. + +See KVM_EXIT_MEMORY_FAULT for more information. + 8. Other capabilities. ====================== diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 7b389f27dffc..8f9d8939b63b 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -4625,6 +4625,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_ENABLE_CAP: case KVM_CAP_VM_DISABLE_NX_HUGE_PAGES: case KVM_CAP_IRQFD_RESAMPLE: + case KVM_CAP_MEMORY_FAULT_INFO: r = 1; break; case KVM_CAP_EXIT_HYPERCALL: diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 4e741ff27af3..96aa930536b1 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2327,4 +2327,15 @@ static inline void kvm_account_pgtable_pages(void *virt, int nr) /* Max number of entries allowed for each kvm dirty ring */ #define KVM_DIRTY_RING_MAX_ENTRIES 65536 +static inline void kvm_prepare_memory_fault_exit(struct kvm_vcpu *vcpu, + gpa_t gpa, gpa_t size) +{ + vcpu->run->exit_reason = KVM_EXIT_MEMORY_FAULT; + vcpu->run->memory_fault.gpa = gpa; + vcpu->run->memory_fault.size = size; + + /* Flags are not (yet) defined or communicated to userspace. */ + vcpu->run->memory_fault.flags = 0; +} + #endif diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 308cc70bd6ab..59010a685007 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -275,6 +275,7 @@ struct kvm_xen_exit { #define KVM_EXIT_RISCV_CSR 36 #define KVM_EXIT_NOTIFY 37 #define KVM_EXIT_LOONGARCH_IOCSR 38 +#define KVM_EXIT_MEMORY_FAULT 39 /* For KVM_EXIT_INTERNAL_ERROR */ /* Emulate instruction failed. */ @@ -528,6 +529,12 @@ struct kvm_run { #define KVM_NOTIFY_CONTEXT_INVALID (1 << 0) __u32 flags; } notify; + /* KVM_EXIT_MEMORY_FAULT */ + struct { + __u64 flags; + __u64 gpa; + __u64 size; + } memory_fault; /* Fix the size of the union. */ char padding[256]; }; @@ -1212,6 +1219,7 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES 229 #define KVM_CAP_ARM_SUPPORTED_REG_MASK_RANGES 230 #define KVM_CAP_USER_MEMORY2 231 +#define KVM_CAP_MEMORY_FAULT_INFO 232 #ifdef KVM_CAP_IRQ_ROUTING From cec29eef0a815386d520d61c2cbe16d537931639 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 27 Oct 2023 11:21:52 -0700 Subject: [PATCH 011/336] KVM: Add a dedicated mmu_notifier flag for reclaiming freed memory Handle AMD SEV's kvm_arch_guest_memory_reclaimed() hook by having __kvm_handle_hva_range() return whether or not an overlapping memslot was found, i.e. mmu_lock was acquired. Using the .on_unlock() hook works, but kvm_arch_guest_memory_reclaimed() needs to run after dropping mmu_lock, which makes .on_lock() and .on_unlock() asymmetrical. Use a small struct to return the tuple of the notifier-specific return, plus whether or not overlap was found. Because the iteration helpers are __always_inlined, practically speaking, the struct will never actually be returned from a function call (not to mention the size of the struct will be two bytes in practice). Signed-off-by: Sean Christopherson Reviewed-by: Paolo Bonzini Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Message-Id: <20231027182217.3615211-11-seanjc@google.com> Signed-off-by: Paolo Bonzini --- virt/kvm/kvm_main.c | 53 +++++++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 756b94ecd511..e18a7f152c0b 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -561,6 +561,19 @@ struct kvm_mmu_notifier_range { bool may_block; }; +/* + * The inner-most helper returns a tuple containing the return value from the + * arch- and action-specific handler, plus a flag indicating whether or not at + * least one memslot was found, i.e. if the handler found guest memory. + * + * Note, most notifiers are averse to booleans, so even though KVM tracks the + * return from arch code as a bool, outer helpers will cast it to an int. :-( + */ +typedef struct kvm_mmu_notifier_return { + bool ret; + bool found_memslot; +} kvm_mn_ret_t; + /* * Use a dedicated stub instead of NULL to indicate that there is no callback * function/handler. The compiler technically can't guarantee that a real @@ -582,22 +595,25 @@ static const union kvm_mmu_notifier_arg KVM_MMU_NOTIFIER_NO_ARG; node; \ node = interval_tree_iter_next(node, start, last)) \ -static __always_inline int __kvm_handle_hva_range(struct kvm *kvm, - const struct kvm_mmu_notifier_range *range) +static __always_inline kvm_mn_ret_t __kvm_handle_hva_range(struct kvm *kvm, + const struct kvm_mmu_notifier_range *range) { - bool ret = false, locked = false; + struct kvm_mmu_notifier_return r = { + .ret = false, + .found_memslot = false, + }; struct kvm_gfn_range gfn_range; struct kvm_memory_slot *slot; struct kvm_memslots *slots; int i, idx; if (WARN_ON_ONCE(range->end <= range->start)) - return 0; + return r; /* A null handler is allowed if and only if on_lock() is provided. */ if (WARN_ON_ONCE(IS_KVM_NULL_FN(range->on_lock) && IS_KVM_NULL_FN(range->handler))) - return 0; + return r; idx = srcu_read_lock(&kvm->srcu); @@ -631,8 +647,8 @@ static __always_inline int __kvm_handle_hva_range(struct kvm *kvm, gfn_range.end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, slot); gfn_range.slot = slot; - if (!locked) { - locked = true; + if (!r.found_memslot) { + r.found_memslot = true; KVM_MMU_LOCK(kvm); if (!IS_KVM_NULL_FN(range->on_lock)) range->on_lock(kvm); @@ -640,14 +656,14 @@ static __always_inline int __kvm_handle_hva_range(struct kvm *kvm, if (IS_KVM_NULL_FN(range->handler)) break; } - ret |= range->handler(kvm, &gfn_range); + r.ret |= range->handler(kvm, &gfn_range); } } - if (range->flush_on_ret && ret) + if (range->flush_on_ret && r.ret) kvm_flush_remote_tlbs(kvm); - if (locked) { + if (r.found_memslot) { KVM_MMU_UNLOCK(kvm); if (!IS_KVM_NULL_FN(range->on_unlock)) range->on_unlock(kvm); @@ -655,8 +671,7 @@ static __always_inline int __kvm_handle_hva_range(struct kvm *kvm, srcu_read_unlock(&kvm->srcu, idx); - /* The notifiers are averse to booleans. :-( */ - return (int)ret; + return r; } static __always_inline int kvm_handle_hva_range(struct mmu_notifier *mn, @@ -677,7 +692,7 @@ static __always_inline int kvm_handle_hva_range(struct mmu_notifier *mn, .may_block = false, }; - return __kvm_handle_hva_range(kvm, &range); + return __kvm_handle_hva_range(kvm, &range).ret; } static __always_inline int kvm_handle_hva_range_no_flush(struct mmu_notifier *mn, @@ -696,7 +711,7 @@ static __always_inline int kvm_handle_hva_range_no_flush(struct mmu_notifier *mn .may_block = false, }; - return __kvm_handle_hva_range(kvm, &range); + return __kvm_handle_hva_range(kvm, &range).ret; } static bool kvm_change_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range) @@ -798,7 +813,7 @@ static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, .end = range->end, .handler = kvm_mmu_unmap_gfn_range, .on_lock = kvm_mmu_invalidate_begin, - .on_unlock = kvm_arch_guest_memory_reclaimed, + .on_unlock = (void *)kvm_null_fn, .flush_on_ret = true, .may_block = mmu_notifier_range_blockable(range), }; @@ -830,7 +845,13 @@ static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, gfn_to_pfn_cache_invalidate_start(kvm, range->start, range->end, hva_range.may_block); - __kvm_handle_hva_range(kvm, &hva_range); + /* + * If one or more memslots were found and thus zapped, notify arch code + * that guest memory has been reclaimed. This needs to be done *after* + * dropping mmu_lock, as x86's reclaim path is slooooow. + */ + if (__kvm_handle_hva_range(kvm, &hva_range).found_memslot) + kvm_arch_guest_memory_reclaimed(kvm); return 0; } From 193bbfaacc84f9ee9c281ec0a8dd2ec8e4821e57 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 27 Oct 2023 11:21:53 -0700 Subject: [PATCH 012/336] KVM: Drop .on_unlock() mmu_notifier hook Drop the .on_unlock() mmu_notifer hook now that it's no longer used for notifying arch code that memory has been reclaimed. Adding .on_unlock() and invoking it *after* dropping mmu_lock was a terrible idea, as doing so resulted in .on_lock() and .on_unlock() having divergent and asymmetric behavior, and set future developers up for failure, i.e. all but asked for bugs where KVM relied on using .on_unlock() to try to run a callback while holding mmu_lock. Opportunistically add a lockdep assertion in kvm_mmu_invalidate_end() to guard against future bugs of this nature. Reported-by: Isaku Yamahata Link: https://lore.kernel.org/all/20230802203119.GB2021422@ls.amr.corp.intel.com Signed-off-by: Sean Christopherson Reviewed-by: Paolo Bonzini Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Message-Id: <20231027182217.3615211-12-seanjc@google.com> Signed-off-by: Paolo Bonzini --- virt/kvm/kvm_main.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index e18a7f152c0b..7f3291dec7a6 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -544,7 +544,6 @@ static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn) typedef bool (*gfn_handler_t)(struct kvm *kvm, struct kvm_gfn_range *range); typedef void (*on_lock_fn_t)(struct kvm *kvm); -typedef void (*on_unlock_fn_t)(struct kvm *kvm); struct kvm_mmu_notifier_range { /* @@ -556,7 +555,6 @@ struct kvm_mmu_notifier_range { union kvm_mmu_notifier_arg arg; gfn_handler_t handler; on_lock_fn_t on_lock; - on_unlock_fn_t on_unlock; bool flush_on_ret; bool may_block; }; @@ -663,11 +661,8 @@ static __always_inline kvm_mn_ret_t __kvm_handle_hva_range(struct kvm *kvm, if (range->flush_on_ret && r.ret) kvm_flush_remote_tlbs(kvm); - if (r.found_memslot) { + if (r.found_memslot) KVM_MMU_UNLOCK(kvm); - if (!IS_KVM_NULL_FN(range->on_unlock)) - range->on_unlock(kvm); - } srcu_read_unlock(&kvm->srcu, idx); @@ -687,7 +682,6 @@ static __always_inline int kvm_handle_hva_range(struct mmu_notifier *mn, .arg = arg, .handler = handler, .on_lock = (void *)kvm_null_fn, - .on_unlock = (void *)kvm_null_fn, .flush_on_ret = true, .may_block = false, }; @@ -706,7 +700,6 @@ static __always_inline int kvm_handle_hva_range_no_flush(struct mmu_notifier *mn .end = end, .handler = handler, .on_lock = (void *)kvm_null_fn, - .on_unlock = (void *)kvm_null_fn, .flush_on_ret = false, .may_block = false, }; @@ -813,7 +806,6 @@ static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, .end = range->end, .handler = kvm_mmu_unmap_gfn_range, .on_lock = kvm_mmu_invalidate_begin, - .on_unlock = (void *)kvm_null_fn, .flush_on_ret = true, .may_block = mmu_notifier_range_blockable(range), }; @@ -891,7 +883,6 @@ static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, .end = range->end, .handler = (void *)kvm_null_fn, .on_lock = kvm_mmu_invalidate_end, - .on_unlock = (void *)kvm_null_fn, .flush_on_ret = false, .may_block = mmu_notifier_range_blockable(range), }; From 5a475554db1e476a14216e742ea2bdb77362d5d5 Mon Sep 17 00:00:00 2001 From: Chao Peng Date: Fri, 27 Oct 2023 11:21:55 -0700 Subject: [PATCH 013/336] KVM: Introduce per-page memory attributes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In confidential computing usages, whether a page is private or shared is necessary information for KVM to perform operations like page fault handling, page zapping etc. There are other potential use cases for per-page memory attributes, e.g. to make memory read-only (or no-exec, or exec-only, etc.) without having to modify memslots. Introduce the KVM_SET_MEMORY_ATTRIBUTES ioctl, advertised by KVM_CAP_MEMORY_ATTRIBUTES, to allow userspace to set the per-page memory attributes to a guest memory range. Use an xarray to store the per-page attributes internally, with a naive, not fully optimized implementation, i.e. prioritize correctness over performance for the initial implementation. Use bit 3 for the PRIVATE attribute so that KVM can use bits 0-2 for RWX attributes/protections in the future, e.g. to give userspace fine-grained control over read, write, and execute protections for guest memory. Provide arch hooks for handling attribute changes before and after common code sets the new attributes, e.g. x86 will use the "pre" hook to zap all relevant mappings, and the "post" hook to track whether or not hugepages can be used to map the range. To simplify the implementation wrap the entire sequence with kvm_mmu_invalidate_{begin,end}() even though the operation isn't strictly guaranteed to be an invalidation. For the initial use case, x86 *will* always invalidate memory, and preventing arch code from creating new mappings while the attributes are in flux makes it much easier to reason about the correctness of consuming attributes. It's possible that future usages may not require an invalidation, e.g. if KVM ends up supporting RWX protections and userspace grants _more_ protections, but again opt for simplicity and punt optimizations to if/when they are needed. Suggested-by: Sean Christopherson Link: https://lore.kernel.org/all/Y2WB48kD0J4VGynX@google.com Cc: Fuad Tabba Cc: Xu Yilun Cc: Mickaël Salaün Signed-off-by: Chao Peng Co-developed-by: Sean Christopherson Signed-off-by: Sean Christopherson Message-Id: <20231027182217.3615211-14-seanjc@google.com> Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/api.rst | 36 ++++++ include/linux/kvm_host.h | 19 +++ include/uapi/linux/kvm.h | 13 ++ virt/kvm/Kconfig | 4 + virt/kvm/kvm_main.c | 216 +++++++++++++++++++++++++++++++++ 5 files changed, 288 insertions(+) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 27d945d5b4e4..081ef09d3148 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -6214,6 +6214,42 @@ superset of the features supported by the system. See KVM_SET_USER_MEMORY_REGION. +4.141 KVM_SET_MEMORY_ATTRIBUTES +------------------------------- + +:Capability: KVM_CAP_MEMORY_ATTRIBUTES +:Architectures: x86 +:Type: vm ioctl +:Parameters: struct kvm_memory_attributes (in) +:Returns: 0 on success, <0 on error + +KVM_SET_MEMORY_ATTRIBUTES allows userspace to set memory attributes for a range +of guest physical memory. + +:: + + struct kvm_memory_attributes { + __u64 address; + __u64 size; + __u64 attributes; + __u64 flags; + }; + + #define KVM_MEMORY_ATTRIBUTE_PRIVATE (1ULL << 3) + +The address and size must be page aligned. The supported attributes can be +retrieved via ioctl(KVM_CHECK_EXTENSION) on KVM_CAP_MEMORY_ATTRIBUTES. If +executed on a VM, KVM_CAP_MEMORY_ATTRIBUTES precisely returns the attributes +supported by that VM. If executed at system scope, KVM_CAP_MEMORY_ATTRIBUTES +returns all attributes supported by KVM. The only attribute defined at this +time is KVM_MEMORY_ATTRIBUTE_PRIVATE, which marks the associated gfn as being +guest private memory. + +Note, there is no "get" API. Userspace is responsible for explicitly tracking +the state of a gfn/page as needed. + +The "flags" field is reserved for future extensions and must be '0'. + 5. The kvm_run structure ======================== diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 96aa930536b1..68a144cb7dbc 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -256,6 +256,7 @@ int kvm_async_pf_wakeup_all(struct kvm_vcpu *vcpu); #ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER union kvm_mmu_notifier_arg { pte_t pte; + unsigned long attributes; }; struct kvm_gfn_range { @@ -806,6 +807,10 @@ struct kvm { #ifdef CONFIG_HAVE_KVM_PM_NOTIFIER struct notifier_block pm_notifier; +#endif +#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES + /* Protected by slots_locks (for writes) and RCU (for reads) */ + struct xarray mem_attr_array; #endif char stats_id[KVM_STATS_NAME_SIZE]; }; @@ -2338,4 +2343,18 @@ static inline void kvm_prepare_memory_fault_exit(struct kvm_vcpu *vcpu, vcpu->run->memory_fault.flags = 0; } +#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +static inline unsigned long kvm_get_memory_attributes(struct kvm *kvm, gfn_t gfn) +{ + return xa_to_value(xa_load(&kvm->mem_attr_array, gfn)); +} + +bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end, + unsigned long attrs); +bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range); +bool kvm_arch_post_set_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range); +#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */ + #endif diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 59010a685007..e8d167e54980 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1220,6 +1220,7 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_ARM_SUPPORTED_REG_MASK_RANGES 230 #define KVM_CAP_USER_MEMORY2 231 #define KVM_CAP_MEMORY_FAULT_INFO 232 +#define KVM_CAP_MEMORY_ATTRIBUTES 233 #ifdef KVM_CAP_IRQ_ROUTING @@ -2288,4 +2289,16 @@ struct kvm_s390_zpci_op { /* flags for kvm_s390_zpci_op->u.reg_aen.flags */ #define KVM_S390_ZPCIOP_REGAEN_HOST (1 << 0) +/* Available with KVM_CAP_MEMORY_ATTRIBUTES */ +#define KVM_SET_MEMORY_ATTRIBUTES _IOW(KVMIO, 0xd2, struct kvm_memory_attributes) + +struct kvm_memory_attributes { + __u64 address; + __u64 size; + __u64 attributes; + __u64 flags; +}; + +#define KVM_MEMORY_ATTRIBUTE_PRIVATE (1ULL << 3) + #endif /* __LINUX_KVM_H */ diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig index ecae2914c97e..5bd7fcaf9089 100644 --- a/virt/kvm/Kconfig +++ b/virt/kvm/Kconfig @@ -96,3 +96,7 @@ config KVM_GENERIC_HARDWARE_ENABLING config KVM_GENERIC_MMU_NOTIFIER select MMU_NOTIFIER bool + +config KVM_GENERIC_MEMORY_ATTRIBUTES + select KVM_GENERIC_MMU_NOTIFIER + bool diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 7f3291dec7a6..f1a575d39b3b 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1211,6 +1211,9 @@ static struct kvm *kvm_create_vm(unsigned long type, const char *fdname) spin_lock_init(&kvm->mn_invalidate_lock); rcuwait_init(&kvm->mn_memslots_update_rcuwait); xa_init(&kvm->vcpu_array); +#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES + xa_init(&kvm->mem_attr_array); +#endif INIT_LIST_HEAD(&kvm->gpc_list); spin_lock_init(&kvm->gpc_lock); @@ -1391,6 +1394,9 @@ static void kvm_destroy_vm(struct kvm *kvm) } cleanup_srcu_struct(&kvm->irq_srcu); cleanup_srcu_struct(&kvm->srcu); +#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES + xa_destroy(&kvm->mem_attr_array); +#endif kvm_arch_free_vm(kvm); preempt_notifier_dec(); hardware_disable_all(); @@ -2397,6 +2403,200 @@ static int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, } #endif /* CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */ +#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +/* + * Returns true if _all_ gfns in the range [@start, @end) have attributes + * matching @attrs. + */ +bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end, + unsigned long attrs) +{ + XA_STATE(xas, &kvm->mem_attr_array, start); + unsigned long index; + bool has_attrs; + void *entry; + + rcu_read_lock(); + + if (!attrs) { + has_attrs = !xas_find(&xas, end - 1); + goto out; + } + + has_attrs = true; + for (index = start; index < end; index++) { + do { + entry = xas_next(&xas); + } while (xas_retry(&xas, entry)); + + if (xas.xa_index != index || xa_to_value(entry) != attrs) { + has_attrs = false; + break; + } + } + +out: + rcu_read_unlock(); + return has_attrs; +} + +static u64 kvm_supported_mem_attributes(struct kvm *kvm) +{ + if (!kvm) + return KVM_MEMORY_ATTRIBUTE_PRIVATE; + + return 0; +} + +static __always_inline void kvm_handle_gfn_range(struct kvm *kvm, + struct kvm_mmu_notifier_range *range) +{ + struct kvm_gfn_range gfn_range; + struct kvm_memory_slot *slot; + struct kvm_memslots *slots; + struct kvm_memslot_iter iter; + bool found_memslot = false; + bool ret = false; + int i; + + gfn_range.arg = range->arg; + gfn_range.may_block = range->may_block; + + for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { + slots = __kvm_memslots(kvm, i); + + kvm_for_each_memslot_in_gfn_range(&iter, slots, range->start, range->end) { + slot = iter.slot; + gfn_range.slot = slot; + + gfn_range.start = max(range->start, slot->base_gfn); + gfn_range.end = min(range->end, slot->base_gfn + slot->npages); + if (gfn_range.start >= gfn_range.end) + continue; + + if (!found_memslot) { + found_memslot = true; + KVM_MMU_LOCK(kvm); + if (!IS_KVM_NULL_FN(range->on_lock)) + range->on_lock(kvm); + } + + ret |= range->handler(kvm, &gfn_range); + } + } + + if (range->flush_on_ret && ret) + kvm_flush_remote_tlbs(kvm); + + if (found_memslot) + KVM_MMU_UNLOCK(kvm); +} + +static bool kvm_pre_set_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range) +{ + /* + * Unconditionally add the range to the invalidation set, regardless of + * whether or not the arch callback actually needs to zap SPTEs. E.g. + * if KVM supports RWX attributes in the future and the attributes are + * going from R=>RW, zapping isn't strictly necessary. Unconditionally + * adding the range allows KVM to require that MMU invalidations add at + * least one range between begin() and end(), e.g. allows KVM to detect + * bugs where the add() is missed. Relaxing the rule *might* be safe, + * but it's not obvious that allowing new mappings while the attributes + * are in flux is desirable or worth the complexity. + */ + kvm_mmu_invalidate_range_add(kvm, range->start, range->end); + + return kvm_arch_pre_set_memory_attributes(kvm, range); +} + +/* Set @attributes for the gfn range [@start, @end). */ +static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end, + unsigned long attributes) +{ + struct kvm_mmu_notifier_range pre_set_range = { + .start = start, + .end = end, + .handler = kvm_pre_set_memory_attributes, + .on_lock = kvm_mmu_invalidate_begin, + .flush_on_ret = true, + .may_block = true, + }; + struct kvm_mmu_notifier_range post_set_range = { + .start = start, + .end = end, + .arg.attributes = attributes, + .handler = kvm_arch_post_set_memory_attributes, + .on_lock = kvm_mmu_invalidate_end, + .may_block = true, + }; + unsigned long i; + void *entry; + int r = 0; + + entry = attributes ? xa_mk_value(attributes) : NULL; + + mutex_lock(&kvm->slots_lock); + + /* Nothing to do if the entire range as the desired attributes. */ + if (kvm_range_has_memory_attributes(kvm, start, end, attributes)) + goto out_unlock; + + /* + * Reserve memory ahead of time to avoid having to deal with failures + * partway through setting the new attributes. + */ + for (i = start; i < end; i++) { + r = xa_reserve(&kvm->mem_attr_array, i, GFP_KERNEL_ACCOUNT); + if (r) + goto out_unlock; + } + + kvm_handle_gfn_range(kvm, &pre_set_range); + + for (i = start; i < end; i++) { + r = xa_err(xa_store(&kvm->mem_attr_array, i, entry, + GFP_KERNEL_ACCOUNT)); + KVM_BUG_ON(r, kvm); + } + + kvm_handle_gfn_range(kvm, &post_set_range); + +out_unlock: + mutex_unlock(&kvm->slots_lock); + + return r; +} +static int kvm_vm_ioctl_set_mem_attributes(struct kvm *kvm, + struct kvm_memory_attributes *attrs) +{ + gfn_t start, end; + + /* flags is currently not used. */ + if (attrs->flags) + return -EINVAL; + if (attrs->attributes & ~kvm_supported_mem_attributes(kvm)) + return -EINVAL; + if (attrs->size == 0 || attrs->address + attrs->size < attrs->address) + return -EINVAL; + if (!PAGE_ALIGNED(attrs->address) || !PAGE_ALIGNED(attrs->size)) + return -EINVAL; + + start = attrs->address >> PAGE_SHIFT; + end = (attrs->address + attrs->size) >> PAGE_SHIFT; + + /* + * xarray tracks data using "unsigned long", and as a result so does + * KVM. For simplicity, supports generic attributes only on 64-bit + * architectures. + */ + BUILD_BUG_ON(sizeof(attrs->attributes) != sizeof(unsigned long)); + + return kvm_vm_set_mem_attributes(kvm, start, end, attrs->attributes); +} +#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */ + struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) { return __gfn_to_memslot(kvm_memslots(kvm), gfn); @@ -4641,6 +4841,10 @@ static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) case KVM_CAP_BINARY_STATS_FD: case KVM_CAP_SYSTEM_EVENT_DATA: return 1; +#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES + case KVM_CAP_MEMORY_ATTRIBUTES: + return kvm_supported_mem_attributes(kvm); +#endif default: break; } @@ -5034,6 +5238,18 @@ static long kvm_vm_ioctl(struct file *filp, break; } #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */ +#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES + case KVM_SET_MEMORY_ATTRIBUTES: { + struct kvm_memory_attributes attrs; + + r = -EFAULT; + if (copy_from_user(&attrs, argp, sizeof(attrs))) + goto out; + + r = kvm_vm_ioctl_set_mem_attributes(kvm, &attrs); + break; + } +#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */ case KVM_CREATE_DEVICE: { struct kvm_create_device cd; From 0003e2a414687fff6a75250d381e4abf345d663f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 27 Oct 2023 11:21:56 -0700 Subject: [PATCH 014/336] mm: Add AS_UNMOVABLE to mark mapping as completely unmovable Add an "unmovable" flag for mappings that cannot be migrated under any circumstance. KVM will use the flag for its upcoming GUEST_MEMFD support, which will not support compaction/migration, at least not in the foreseeable future. Test AS_UNMOVABLE under folio lock as already done for the async compaction/dirty folio case, as the mapping can be removed by truncation while compaction is running. To avoid having to lock every folio with a mapping, assume/require that unmovable mappings are also unevictable, and have mapping_set_unmovable() also set AS_UNEVICTABLE. Cc: Matthew Wilcox Co-developed-by: Vlastimil Babka Signed-off-by: Vlastimil Babka Signed-off-by: Sean Christopherson Message-Id: <20231027182217.3615211-15-seanjc@google.com> Signed-off-by: Paolo Bonzini --- include/linux/pagemap.h | 19 +++++++++++++++++- mm/compaction.c | 43 +++++++++++++++++++++++++++++------------ mm/migrate.c | 2 ++ 3 files changed, 51 insertions(+), 13 deletions(-) diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 351c3b7f93a1..82c9bf506b79 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -203,7 +203,8 @@ enum mapping_flags { /* writeback related tags are not used */ AS_NO_WRITEBACK_TAGS = 5, AS_LARGE_FOLIO_SUPPORT = 6, - AS_RELEASE_ALWAYS, /* Call ->release_folio(), even if no private data */ + AS_RELEASE_ALWAYS = 7, /* Call ->release_folio(), even if no private data */ + AS_UNMOVABLE = 8, /* The mapping cannot be moved, ever */ }; /** @@ -289,6 +290,22 @@ static inline void mapping_clear_release_always(struct address_space *mapping) clear_bit(AS_RELEASE_ALWAYS, &mapping->flags); } +static inline void mapping_set_unmovable(struct address_space *mapping) +{ + /* + * It's expected unmovable mappings are also unevictable. Compaction + * migrate scanner (isolate_migratepages_block()) relies on this to + * reduce page locking. + */ + set_bit(AS_UNEVICTABLE, &mapping->flags); + set_bit(AS_UNMOVABLE, &mapping->flags); +} + +static inline bool mapping_unmovable(struct address_space *mapping) +{ + return test_bit(AS_UNMOVABLE, &mapping->flags); +} + static inline gfp_t mapping_gfp_mask(struct address_space * mapping) { return mapping->gfp_mask; diff --git a/mm/compaction.c b/mm/compaction.c index 38c8d216c6a3..12b828aed7c8 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -883,6 +883,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, /* Time to isolate some pages for migration */ for (; low_pfn < end_pfn; low_pfn++) { + bool is_dirty, is_unevictable; if (skip_on_failure && low_pfn >= next_skip_pfn) { /* @@ -1080,8 +1081,10 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, if (!folio_test_lru(folio)) goto isolate_fail_put; + is_unevictable = folio_test_unevictable(folio); + /* Compaction might skip unevictable pages but CMA takes them */ - if (!(mode & ISOLATE_UNEVICTABLE) && folio_test_unevictable(folio)) + if (!(mode & ISOLATE_UNEVICTABLE) && is_unevictable) goto isolate_fail_put; /* @@ -1093,26 +1096,42 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, if ((mode & ISOLATE_ASYNC_MIGRATE) && folio_test_writeback(folio)) goto isolate_fail_put; - if ((mode & ISOLATE_ASYNC_MIGRATE) && folio_test_dirty(folio)) { - bool migrate_dirty; + is_dirty = folio_test_dirty(folio); + + if (((mode & ISOLATE_ASYNC_MIGRATE) && is_dirty) || + (mapping && is_unevictable)) { + bool migrate_dirty = true; + bool is_unmovable; /* * Only folios without mappings or that have - * a ->migrate_folio callback are possible to - * migrate without blocking. However, we may - * be racing with truncation, which can free - * the mapping. Truncation holds the folio lock - * until after the folio is removed from the page - * cache so holding it ourselves is sufficient. + * a ->migrate_folio callback are possible to migrate + * without blocking. + * + * Folios from unmovable mappings are not migratable. + * + * However, we can be racing with truncation, which can + * free the mapping that we need to check. Truncation + * holds the folio lock until after the folio is removed + * from the page so holding it ourselves is sufficient. + * + * To avoid locking the folio just to check unmovable, + * assume every unmovable folio is also unevictable, + * which is a cheaper test. If our assumption goes + * wrong, it's not a correctness bug, just potentially + * wasted cycles. */ if (!folio_trylock(folio)) goto isolate_fail_put; mapping = folio_mapping(folio); - migrate_dirty = !mapping || - mapping->a_ops->migrate_folio; + if ((mode & ISOLATE_ASYNC_MIGRATE) && is_dirty) { + migrate_dirty = !mapping || + mapping->a_ops->migrate_folio; + } + is_unmovable = mapping && mapping_unmovable(mapping); folio_unlock(folio); - if (!migrate_dirty) + if (!migrate_dirty || is_unmovable) goto isolate_fail_put; } diff --git a/mm/migrate.c b/mm/migrate.c index 06086dc9da28..60f2ff6b36aa 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -956,6 +956,8 @@ static int move_to_new_folio(struct folio *dst, struct folio *src, if (!mapping) rc = migrate_folio(mapping, dst, src, mode); + else if (mapping_unmovable(mapping)) + rc = -EOPNOTSUPP; else if (mapping->a_ops->migrate_folio) /* * Most folios have a mapping and most filesystems From 4f0b9194bc119a9850a99e5e824808e2f468c348 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 3 Nov 2023 06:47:51 -0400 Subject: [PATCH 015/336] fs: Rename anon_inode_getfile_secure() and anon_inode_getfd_secure() The call to the inode_init_security_anon() LSM hook is not the sole reason to use anon_inode_getfile_secure() or anon_inode_getfd_secure(). For example, the functions also allow one to create a file with non-zero size, without needing a full-blown filesystem. In this case, you don't need a "secure" version, just unique inodes; the current name of the functions is confusing and does not explain well the difference with the more "standard" anon_inode_getfile() and anon_inode_getfd(). Of course, there is another side of the coin; neither io_uring nor userfaultfd strictly speaking need distinct inodes, and it is not that clear anymore that anon_inode_create_get{file,fd}() allow the LSM to intercept and block the inode's creation. If one was so inclined, anon_inode_getfile_secure() and anon_inode_getfd_secure() could be kept, using the shared inode or a new one depending on CONFIG_SECURITY. However, this is probably overkill, and potentially a cause of bugs in different configurations. Therefore, just add a comment to io_uring and userfaultfd explaining the choice of the function. While at it, remove the export for what is now anon_inode_create_getfd(). There is no in-tree module that uses it, and the old name is gone anyway. If anybody actually needs the symbol, they can ask or they can just use anon_inode_create_getfile(), which will be exported very soon for use in KVM. Suggested-by: Christian Brauner Reviewed-by: Christian Brauner Signed-off-by: Paolo Bonzini --- fs/anon_inodes.c | 50 ++++++++++++++++++++++++------------- fs/userfaultfd.c | 5 ++-- include/linux/anon_inodes.h | 4 +-- io_uring/io_uring.c | 3 ++- 4 files changed, 40 insertions(+), 22 deletions(-) diff --git a/fs/anon_inodes.c b/fs/anon_inodes.c index 24192a7667ed..42b02dc36474 100644 --- a/fs/anon_inodes.c +++ b/fs/anon_inodes.c @@ -79,7 +79,7 @@ static struct file *__anon_inode_getfile(const char *name, const struct file_operations *fops, void *priv, int flags, const struct inode *context_inode, - bool secure) + bool make_inode) { struct inode *inode; struct file *file; @@ -87,7 +87,7 @@ static struct file *__anon_inode_getfile(const char *name, if (fops->owner && !try_module_get(fops->owner)) return ERR_PTR(-ENOENT); - if (secure) { + if (make_inode) { inode = anon_inode_make_secure_inode(name, context_inode); if (IS_ERR(inode)) { file = ERR_CAST(inode); @@ -149,13 +149,10 @@ struct file *anon_inode_getfile(const char *name, EXPORT_SYMBOL_GPL(anon_inode_getfile); /** - * anon_inode_getfile_secure - Like anon_inode_getfile(), but creates a new + * anon_inode_create_getfile - Like anon_inode_getfile(), but creates a new * !S_PRIVATE anon inode rather than reuse the * singleton anon inode and calls the - * inode_init_security_anon() LSM hook. This - * allows for both the inode to have its own - * security context and for the LSM to enforce - * policy on the inode's creation. + * inode_init_security_anon() LSM hook. * * @name: [in] name of the "class" of the new file * @fops: [in] file operations for the new file @@ -164,11 +161,21 @@ EXPORT_SYMBOL_GPL(anon_inode_getfile); * @context_inode: * [in] the logical relationship with the new inode (optional) * + * Create a new anonymous inode and file pair. This can be done for two + * reasons: + * + * - for the inode to have its own security context, so that LSMs can enforce + * policy on the inode's creation; + * + * - if the caller needs a unique inode, for example in order to customize + * the size returned by fstat() + * * The LSM may use @context_inode in inode_init_security_anon(), but a - * reference to it is not held. Returns the newly created file* or an error - * pointer. See the anon_inode_getfile() documentation for more information. + * reference to it is not held. + * + * Returns the newly created file* or an error pointer. */ -struct file *anon_inode_getfile_secure(const char *name, +struct file *anon_inode_create_getfile(const char *name, const struct file_operations *fops, void *priv, int flags, const struct inode *context_inode) @@ -181,7 +188,7 @@ static int __anon_inode_getfd(const char *name, const struct file_operations *fops, void *priv, int flags, const struct inode *context_inode, - bool secure) + bool make_inode) { int error, fd; struct file *file; @@ -192,7 +199,7 @@ static int __anon_inode_getfd(const char *name, fd = error; file = __anon_inode_getfile(name, fops, priv, flags, context_inode, - secure); + make_inode); if (IS_ERR(file)) { error = PTR_ERR(file); goto err_put_unused_fd; @@ -231,10 +238,9 @@ int anon_inode_getfd(const char *name, const struct file_operations *fops, EXPORT_SYMBOL_GPL(anon_inode_getfd); /** - * anon_inode_getfd_secure - Like anon_inode_getfd(), but creates a new + * anon_inode_create_getfd - Like anon_inode_getfd(), but creates a new * !S_PRIVATE anon inode rather than reuse the singleton anon inode, and calls - * the inode_init_security_anon() LSM hook. This allows the inode to have its - * own security context and for a LSM to reject creation of the inode. + * the inode_init_security_anon() LSM hook. * * @name: [in] name of the "class" of the new file * @fops: [in] file operations for the new file @@ -243,16 +249,26 @@ EXPORT_SYMBOL_GPL(anon_inode_getfd); * @context_inode: * [in] the logical relationship with the new inode (optional) * + * Create a new anonymous inode and file pair. This can be done for two + * reasons: + * + * - for the inode to have its own security context, so that LSMs can enforce + * policy on the inode's creation; + * + * - if the caller needs a unique inode, for example in order to customize + * the size returned by fstat() + * * The LSM may use @context_inode in inode_init_security_anon(), but a * reference to it is not held. + * + * Returns a newly created file descriptor or an error code. */ -int anon_inode_getfd_secure(const char *name, const struct file_operations *fops, +int anon_inode_create_getfd(const char *name, const struct file_operations *fops, void *priv, int flags, const struct inode *context_inode) { return __anon_inode_getfd(name, fops, priv, flags, context_inode, true); } -EXPORT_SYMBOL_GPL(anon_inode_getfd_secure); static int __init anon_inode_init(void) { diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 56eaae9dac1a..7a1cf8bab5eb 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -1033,7 +1033,7 @@ static int resolve_userfault_fork(struct userfaultfd_ctx *new, { int fd; - fd = anon_inode_getfd_secure("[userfaultfd]", &userfaultfd_fops, new, + fd = anon_inode_create_getfd("[userfaultfd]", &userfaultfd_fops, new, O_RDONLY | (new->flags & UFFD_SHARED_FCNTL_FLAGS), inode); if (fd < 0) return fd; @@ -2205,7 +2205,8 @@ static int new_userfaultfd(int flags) /* prevent the mm struct to be freed */ mmgrab(ctx->mm); - fd = anon_inode_getfd_secure("[userfaultfd]", &userfaultfd_fops, ctx, + /* Create a new inode so that the LSM can block the creation. */ + fd = anon_inode_create_getfd("[userfaultfd]", &userfaultfd_fops, ctx, O_RDONLY | (flags & UFFD_SHARED_FCNTL_FLAGS), NULL); if (fd < 0) { mmdrop(ctx->mm); diff --git a/include/linux/anon_inodes.h b/include/linux/anon_inodes.h index 5deaddbd7927..93a5f16d03f3 100644 --- a/include/linux/anon_inodes.h +++ b/include/linux/anon_inodes.h @@ -15,13 +15,13 @@ struct inode; struct file *anon_inode_getfile(const char *name, const struct file_operations *fops, void *priv, int flags); -struct file *anon_inode_getfile_secure(const char *name, +struct file *anon_inode_create_getfile(const char *name, const struct file_operations *fops, void *priv, int flags, const struct inode *context_inode); int anon_inode_getfd(const char *name, const struct file_operations *fops, void *priv, int flags); -int anon_inode_getfd_secure(const char *name, +int anon_inode_create_getfd(const char *name, const struct file_operations *fops, void *priv, int flags, const struct inode *context_inode); diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 8d1bc6cdfe71..22b98f47bb28 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -3835,7 +3835,8 @@ static struct file *io_uring_get_file(struct io_ring_ctx *ctx) return ERR_PTR(ret); #endif - file = anon_inode_getfile_secure("[io_uring]", &io_uring_fops, ctx, + /* Create a new inode so that the LSM can block the creation. */ + file = anon_inode_create_getfile("[io_uring]", &io_uring_fops, ctx, O_RDWR | O_CLOEXEC, NULL); #if defined(CONFIG_UNIX) if (IS_ERR(file)) { From a7800aa80ea4d5356b8474c2302812e9d4926fa6 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 13 Nov 2023 05:42:34 -0500 Subject: [PATCH 016/336] KVM: Add KVM_CREATE_GUEST_MEMFD ioctl() for guest-specific backing memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce an ioctl(), KVM_CREATE_GUEST_MEMFD, to allow creating file-based memory that is tied to a specific KVM virtual machine and whose primary purpose is to serve guest memory. A guest-first memory subsystem allows for optimizations and enhancements that are kludgy or outright infeasible to implement/support in a generic memory subsystem. With guest_memfd, guest protections and mapping sizes are fully decoupled from host userspace mappings. E.g. KVM currently doesn't support mapping memory as writable in the guest without it also being writable in host userspace, as KVM's ABI uses VMA protections to define the allow guest protection. Userspace can fudge this by establishing two mappings, a writable mapping for the guest and readable one for itself, but that’s suboptimal on multiple fronts. Similarly, KVM currently requires the guest mapping size to be a strict subset of the host userspace mapping size, e.g. KVM doesn’t support creating a 1GiB guest mapping unless userspace also has a 1GiB guest mapping. Decoupling the mappings sizes would allow userspace to precisely map only what is needed without impacting guest performance, e.g. to harden against unintentional accesses to guest memory. Decoupling guest and userspace mappings may also allow for a cleaner alternative to high-granularity mappings for HugeTLB, which has reached a bit of an impasse and is unlikely to ever be merged. A guest-first memory subsystem also provides clearer line of sight to things like a dedicated memory pool (for slice-of-hardware VMs) and elimination of "struct page" (for offload setups where userspace _never_ needs to mmap() guest memory). More immediately, being able to map memory into KVM guests without mapping said memory into the host is critical for Confidential VMs (CoCo VMs), the initial use case for guest_memfd. While AMD's SEV and Intel's TDX prevent untrusted software from reading guest private data by encrypting guest memory with a key that isn't usable by the untrusted host, projects such as Protected KVM (pKVM) provide confidentiality and integrity *without* relying on memory encryption. And with SEV-SNP and TDX, accessing guest private memory can be fatal to the host, i.e. KVM must be prevent host userspace from accessing guest memory irrespective of hardware behavior. Attempt #1 to support CoCo VMs was to add a VMA flag to mark memory as being mappable only by KVM (or a similarly enlightened kernel subsystem). That approach was abandoned largely due to it needing to play games with PROT_NONE to prevent userspace from accessing guest memory. Attempt #2 to was to usurp PG_hwpoison to prevent the host from mapping guest private memory into userspace, but that approach failed to meet several requirements for software-based CoCo VMs, e.g. pKVM, as the kernel wouldn't easily be able to enforce a 1:1 page:guest association, let alone a 1:1 pfn:gfn mapping. And using PG_hwpoison does not work for memory that isn't backed by 'struct page', e.g. if devices gain support for exposing encrypted memory regions to guests. Attempt #3 was to extend the memfd() syscall and wrap shmem to provide dedicated file-based guest memory. That approach made it as far as v10 before feedback from Hugh Dickins and Christian Brauner (and others) led to it demise. Hugh's objection was that piggybacking shmem made no sense for KVM's use case as KVM didn't actually *want* the features provided by shmem. I.e. KVM was using memfd() and shmem to avoid having to manage memory directly, not because memfd() and shmem were the optimal solution, e.g. things like read/write/mmap in shmem were dead weight. Christian pointed out flaws with implementing a partial overlay (wrapping only _some_ of shmem), e.g. poking at inode_operations or super_operations would show shmem stuff, but address_space_operations and file_operations would show KVM's overlay. Paraphrashing heavily, Christian suggested KVM stop being lazy and create a proper API. Link: https://lore.kernel.org/all/20201020061859.18385-1-kirill.shutemov@linux.intel.com Link: https://lore.kernel.org/all/20210416154106.23721-1-kirill.shutemov@linux.intel.com Link: https://lore.kernel.org/all/20210824005248.200037-1-seanjc@google.com Link: https://lore.kernel.org/all/20211111141352.26311-1-chao.p.peng@linux.intel.com Link: https://lore.kernel.org/all/20221202061347.1070246-1-chao.p.peng@linux.intel.com Link: https://lore.kernel.org/all/ff5c5b97-acdf-9745-ebe5-c6609dd6322e@google.com Link: https://lore.kernel.org/all/20230418-anfallen-irdisch-6993a61be10b@brauner Link: https://lore.kernel.org/all/ZEM5Zq8oo+xnApW9@google.com Link: https://lore.kernel.org/linux-mm/20230306191944.GA15773@monkey Link: https://lore.kernel.org/linux-mm/ZII1p8ZHlHaQ3dDl@casper.infradead.org Cc: Fuad Tabba Cc: Vishal Annapurve Cc: Ackerley Tng Cc: Jarkko Sakkinen Cc: Maciej Szmigiero Cc: Vlastimil Babka Cc: David Hildenbrand Cc: Quentin Perret Cc: Michael Roth Cc: Wang Cc: Liam Merwick Cc: Isaku Yamahata Co-developed-by: Kirill A. Shutemov Signed-off-by: Kirill A. Shutemov Co-developed-by: Yu Zhang Signed-off-by: Yu Zhang Co-developed-by: Chao Peng Signed-off-by: Chao Peng Co-developed-by: Ackerley Tng Signed-off-by: Ackerley Tng Co-developed-by: Isaku Yamahata Signed-off-by: Isaku Yamahata Co-developed-by: Paolo Bonzini Signed-off-by: Paolo Bonzini Co-developed-by: Michael Roth Signed-off-by: Michael Roth Signed-off-by: Sean Christopherson Message-Id: <20231027182217.3615211-17-seanjc@google.com> Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Reviewed-by: Xiaoyao Li Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/api.rst | 70 ++++- fs/anon_inodes.c | 1 + include/linux/kvm_host.h | 48 +++ include/uapi/linux/kvm.h | 15 +- virt/kvm/Kconfig | 4 + virt/kvm/Makefile.kvm | 1 + virt/kvm/guest_memfd.c | 538 +++++++++++++++++++++++++++++++++ virt/kvm/kvm_main.c | 59 +++- virt/kvm/kvm_mm.h | 26 ++ 9 files changed, 754 insertions(+), 8 deletions(-) create mode 100644 virt/kvm/guest_memfd.c diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 081ef09d3148..1e61faf02b2a 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -6201,6 +6201,15 @@ superset of the features supported by the system. :Parameters: struct kvm_userspace_memory_region2 (in) :Returns: 0 on success, -1 on error +KVM_SET_USER_MEMORY_REGION2 is an extension to KVM_SET_USER_MEMORY_REGION that +allows mapping guest_memfd memory into a guest. All fields shared with +KVM_SET_USER_MEMORY_REGION identically. Userspace can set KVM_MEM_GUEST_MEMFD +in flags to have KVM bind the memory region to a given guest_memfd range of +[guest_memfd_offset, guest_memfd_offset + memory_size]. The target guest_memfd +must point at a file created via KVM_CREATE_GUEST_MEMFD on the current VM, and +the target range must not be bound to any other memory region. All standard +bounds checks apply (use common sense). + :: struct kvm_userspace_memory_region2 { @@ -6209,10 +6218,24 @@ superset of the features supported by the system. __u64 guest_phys_addr; __u64 memory_size; /* bytes */ __u64 userspace_addr; /* start of the userspace allocated memory */ - __u64 pad[16]; + __u64 guest_memfd_offset; + __u32 guest_memfd; + __u32 pad1; + __u64 pad2[14]; }; -See KVM_SET_USER_MEMORY_REGION. +A KVM_MEM_GUEST_MEMFD region _must_ have a valid guest_memfd (private memory) and +userspace_addr (shared memory). However, "valid" for userspace_addr simply +means that the address itself must be a legal userspace address. The backing +mapping for userspace_addr is not required to be valid/populated at the time of +KVM_SET_USER_MEMORY_REGION2, e.g. shared memory can be lazily mapped/allocated +on-demand. + +When mapping a gfn into the guest, KVM selects shared vs. private, i.e consumes +userspace_addr vs. guest_memfd, based on the gfn's KVM_MEMORY_ATTRIBUTE_PRIVATE +state. At VM creation time, all memory is shared, i.e. the PRIVATE attribute +is '0' for all gfns. Userspace can control whether memory is shared/private by +toggling KVM_MEMORY_ATTRIBUTE_PRIVATE via KVM_SET_MEMORY_ATTRIBUTES as needed. 4.141 KVM_SET_MEMORY_ATTRIBUTES ------------------------------- @@ -6250,6 +6273,49 @@ the state of a gfn/page as needed. The "flags" field is reserved for future extensions and must be '0'. +4.142 KVM_CREATE_GUEST_MEMFD +---------------------------- + +:Capability: KVM_CAP_GUEST_MEMFD +:Architectures: none +:Type: vm ioctl +:Parameters: struct kvm_create_guest_memfd(in) +:Returns: 0 on success, <0 on error + +KVM_CREATE_GUEST_MEMFD creates an anonymous file and returns a file descriptor +that refers to it. guest_memfd files are roughly analogous to files created +via memfd_create(), e.g. guest_memfd files live in RAM, have volatile storage, +and are automatically released when the last reference is dropped. Unlike +"regular" memfd_create() files, guest_memfd files are bound to their owning +virtual machine (see below), cannot be mapped, read, or written by userspace, +and cannot be resized (guest_memfd files do however support PUNCH_HOLE). + +:: + + struct kvm_create_guest_memfd { + __u64 size; + __u64 flags; + __u64 reserved[6]; + }; + +Conceptually, the inode backing a guest_memfd file represents physical memory, +i.e. is coupled to the virtual machine as a thing, not to a "struct kvm". The +file itself, which is bound to a "struct kvm", is that instance's view of the +underlying memory, e.g. effectively provides the translation of guest addresses +to host memory. This allows for use cases where multiple KVM structures are +used to manage a single virtual machine, e.g. when performing intrahost +migration of a virtual machine. + +KVM currently only supports mapping guest_memfd via KVM_SET_USER_MEMORY_REGION2, +and more specifically via the guest_memfd and guest_memfd_offset fields in +"struct kvm_userspace_memory_region2", where guest_memfd_offset is the offset +into the guest_memfd instance. For a given guest_memfd file, there can be at +most one mapping per page, i.e. binding multiple memory regions to a single +guest_memfd range is not allowed (any number of memory regions can be bound to +a single guest_memfd file, but the bound ranges must not overlap). + +See KVM_SET_USER_MEMORY_REGION2 for additional details. + 5. The kvm_run structure ======================== diff --git a/fs/anon_inodes.c b/fs/anon_inodes.c index 42b02dc36474..8dd436ee985b 100644 --- a/fs/anon_inodes.c +++ b/fs/anon_inodes.c @@ -183,6 +183,7 @@ struct file *anon_inode_create_getfile(const char *name, return __anon_inode_getfile(name, fops, priv, flags, context_inode, true); } +EXPORT_SYMBOL_GPL(anon_inode_create_getfile); static int __anon_inode_getfd(const char *name, const struct file_operations *fops, diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 68a144cb7dbc..a6de526c0426 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -589,8 +589,20 @@ struct kvm_memory_slot { u32 flags; short id; u16 as_id; + +#ifdef CONFIG_KVM_PRIVATE_MEM + struct { + struct file __rcu *file; + pgoff_t pgoff; + } gmem; +#endif }; +static inline bool kvm_slot_can_be_private(const struct kvm_memory_slot *slot) +{ + return slot && (slot->flags & KVM_MEM_GUEST_MEMFD); +} + static inline bool kvm_slot_dirty_track_enabled(const struct kvm_memory_slot *slot) { return slot->flags & KVM_MEM_LOG_DIRTY_PAGES; @@ -685,6 +697,17 @@ static inline int kvm_arch_vcpu_memslots_id(struct kvm_vcpu *vcpu) } #endif +/* + * Arch code must define kvm_arch_has_private_mem if support for private memory + * is enabled. + */ +#if !defined(kvm_arch_has_private_mem) && !IS_ENABLED(CONFIG_KVM_PRIVATE_MEM) +static inline bool kvm_arch_has_private_mem(struct kvm *kvm) +{ + return false; +} +#endif + struct kvm_memslots { u64 generation; atomic_long_t last_used_slot; @@ -1400,6 +1423,7 @@ void *kvm_mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc); void kvm_mmu_invalidate_begin(struct kvm *kvm); void kvm_mmu_invalidate_range_add(struct kvm *kvm, gfn_t start, gfn_t end); void kvm_mmu_invalidate_end(struct kvm *kvm); +bool kvm_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range); long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg); @@ -2355,6 +2379,30 @@ bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, struct kvm_gfn_range *range); bool kvm_arch_post_set_memory_attributes(struct kvm *kvm, struct kvm_gfn_range *range); + +static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn) +{ + return IS_ENABLED(CONFIG_KVM_PRIVATE_MEM) && + kvm_get_memory_attributes(kvm, gfn) & KVM_MEMORY_ATTRIBUTE_PRIVATE; +} +#else +static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn) +{ + return false; +} #endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */ +#ifdef CONFIG_KVM_PRIVATE_MEM +int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, + gfn_t gfn, kvm_pfn_t *pfn, int *max_order); +#else +static inline int kvm_gmem_get_pfn(struct kvm *kvm, + struct kvm_memory_slot *slot, gfn_t gfn, + kvm_pfn_t *pfn, int *max_order) +{ + KVM_BUG_ON(1, kvm); + return -EIO; +} +#endif /* CONFIG_KVM_PRIVATE_MEM */ + #endif diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index e8d167e54980..2802d10aa88c 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -102,7 +102,10 @@ struct kvm_userspace_memory_region2 { __u64 guest_phys_addr; __u64 memory_size; __u64 userspace_addr; - __u64 pad[16]; + __u64 guest_memfd_offset; + __u32 guest_memfd; + __u32 pad1; + __u64 pad2[14]; }; /* @@ -112,6 +115,7 @@ struct kvm_userspace_memory_region2 { */ #define KVM_MEM_LOG_DIRTY_PAGES (1UL << 0) #define KVM_MEM_READONLY (1UL << 1) +#define KVM_MEM_GUEST_MEMFD (1UL << 2) /* for KVM_IRQ_LINE */ struct kvm_irq_level { @@ -1221,6 +1225,7 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_USER_MEMORY2 231 #define KVM_CAP_MEMORY_FAULT_INFO 232 #define KVM_CAP_MEMORY_ATTRIBUTES 233 +#define KVM_CAP_GUEST_MEMFD 234 #ifdef KVM_CAP_IRQ_ROUTING @@ -2301,4 +2306,12 @@ struct kvm_memory_attributes { #define KVM_MEMORY_ATTRIBUTE_PRIVATE (1ULL << 3) +#define KVM_CREATE_GUEST_MEMFD _IOWR(KVMIO, 0xd4, struct kvm_create_guest_memfd) + +struct kvm_create_guest_memfd { + __u64 size; + __u64 flags; + __u64 reserved[6]; +}; + #endif /* __LINUX_KVM_H */ diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig index 5bd7fcaf9089..08afef022db9 100644 --- a/virt/kvm/Kconfig +++ b/virt/kvm/Kconfig @@ -100,3 +100,7 @@ config KVM_GENERIC_MMU_NOTIFIER config KVM_GENERIC_MEMORY_ATTRIBUTES select KVM_GENERIC_MMU_NOTIFIER bool + +config KVM_PRIVATE_MEM + select XARRAY_MULTI + bool diff --git a/virt/kvm/Makefile.kvm b/virt/kvm/Makefile.kvm index 2c27d5d0c367..724c89af78af 100644 --- a/virt/kvm/Makefile.kvm +++ b/virt/kvm/Makefile.kvm @@ -12,3 +12,4 @@ kvm-$(CONFIG_KVM_ASYNC_PF) += $(KVM)/async_pf.o kvm-$(CONFIG_HAVE_KVM_IRQ_ROUTING) += $(KVM)/irqchip.o kvm-$(CONFIG_HAVE_KVM_DIRTY_RING) += $(KVM)/dirty_ring.o kvm-$(CONFIG_HAVE_KVM_PFNCACHE) += $(KVM)/pfncache.o +kvm-$(CONFIG_KVM_PRIVATE_MEM) += $(KVM)/guest_memfd.o diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c new file mode 100644 index 000000000000..e65f4170425c --- /dev/null +++ b/virt/kvm/guest_memfd.c @@ -0,0 +1,538 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include + +#include "kvm_mm.h" + +struct kvm_gmem { + struct kvm *kvm; + struct xarray bindings; + struct list_head entry; +}; + +static struct folio *kvm_gmem_get_folio(struct inode *inode, pgoff_t index) +{ + struct folio *folio; + + /* TODO: Support huge pages. */ + folio = filemap_grab_folio(inode->i_mapping, index); + if (IS_ERR_OR_NULL(folio)) + return NULL; + + /* + * Use the up-to-date flag to track whether or not the memory has been + * zeroed before being handed off to the guest. There is no backing + * storage for the memory, so the folio will remain up-to-date until + * it's removed. + * + * TODO: Skip clearing pages when trusted firmware will do it when + * assigning memory to the guest. + */ + if (!folio_test_uptodate(folio)) { + unsigned long nr_pages = folio_nr_pages(folio); + unsigned long i; + + for (i = 0; i < nr_pages; i++) + clear_highpage(folio_page(folio, i)); + + folio_mark_uptodate(folio); + } + + /* + * Ignore accessed, referenced, and dirty flags. The memory is + * unevictable and there is no storage to write back to. + */ + return folio; +} + +static void kvm_gmem_invalidate_begin(struct kvm_gmem *gmem, pgoff_t start, + pgoff_t end) +{ + bool flush = false, found_memslot = false; + struct kvm_memory_slot *slot; + struct kvm *kvm = gmem->kvm; + unsigned long index; + + xa_for_each_range(&gmem->bindings, index, slot, start, end - 1) { + pgoff_t pgoff = slot->gmem.pgoff; + + struct kvm_gfn_range gfn_range = { + .start = slot->base_gfn + max(pgoff, start) - pgoff, + .end = slot->base_gfn + min(pgoff + slot->npages, end) - pgoff, + .slot = slot, + .may_block = true, + }; + + if (!found_memslot) { + found_memslot = true; + + KVM_MMU_LOCK(kvm); + kvm_mmu_invalidate_begin(kvm); + } + + flush |= kvm_mmu_unmap_gfn_range(kvm, &gfn_range); + } + + if (flush) + kvm_flush_remote_tlbs(kvm); + + if (found_memslot) + KVM_MMU_UNLOCK(kvm); +} + +static void kvm_gmem_invalidate_end(struct kvm_gmem *gmem, pgoff_t start, + pgoff_t end) +{ + struct kvm *kvm = gmem->kvm; + + if (xa_find(&gmem->bindings, &start, end - 1, XA_PRESENT)) { + KVM_MMU_LOCK(kvm); + kvm_mmu_invalidate_end(kvm); + KVM_MMU_UNLOCK(kvm); + } +} + +static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len) +{ + struct list_head *gmem_list = &inode->i_mapping->private_list; + pgoff_t start = offset >> PAGE_SHIFT; + pgoff_t end = (offset + len) >> PAGE_SHIFT; + struct kvm_gmem *gmem; + + /* + * Bindings must be stable across invalidation to ensure the start+end + * are balanced. + */ + filemap_invalidate_lock(inode->i_mapping); + + list_for_each_entry(gmem, gmem_list, entry) + kvm_gmem_invalidate_begin(gmem, start, end); + + truncate_inode_pages_range(inode->i_mapping, offset, offset + len - 1); + + list_for_each_entry(gmem, gmem_list, entry) + kvm_gmem_invalidate_end(gmem, start, end); + + filemap_invalidate_unlock(inode->i_mapping); + + return 0; +} + +static long kvm_gmem_allocate(struct inode *inode, loff_t offset, loff_t len) +{ + struct address_space *mapping = inode->i_mapping; + pgoff_t start, index, end; + int r; + + /* Dedicated guest is immutable by default. */ + if (offset + len > i_size_read(inode)) + return -EINVAL; + + filemap_invalidate_lock_shared(mapping); + + start = offset >> PAGE_SHIFT; + end = (offset + len) >> PAGE_SHIFT; + + r = 0; + for (index = start; index < end; ) { + struct folio *folio; + + if (signal_pending(current)) { + r = -EINTR; + break; + } + + folio = kvm_gmem_get_folio(inode, index); + if (!folio) { + r = -ENOMEM; + break; + } + + index = folio_next_index(folio); + + folio_unlock(folio); + folio_put(folio); + + /* 64-bit only, wrapping the index should be impossible. */ + if (WARN_ON_ONCE(!index)) + break; + + cond_resched(); + } + + filemap_invalidate_unlock_shared(mapping); + + return r; +} + +static long kvm_gmem_fallocate(struct file *file, int mode, loff_t offset, + loff_t len) +{ + int ret; + + if (!(mode & FALLOC_FL_KEEP_SIZE)) + return -EOPNOTSUPP; + + if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)) + return -EOPNOTSUPP; + + if (!PAGE_ALIGNED(offset) || !PAGE_ALIGNED(len)) + return -EINVAL; + + if (mode & FALLOC_FL_PUNCH_HOLE) + ret = kvm_gmem_punch_hole(file_inode(file), offset, len); + else + ret = kvm_gmem_allocate(file_inode(file), offset, len); + + if (!ret) + file_modified(file); + return ret; +} + +static int kvm_gmem_release(struct inode *inode, struct file *file) +{ + struct kvm_gmem *gmem = file->private_data; + struct kvm_memory_slot *slot; + struct kvm *kvm = gmem->kvm; + unsigned long index; + + /* + * Prevent concurrent attempts to *unbind* a memslot. This is the last + * reference to the file and thus no new bindings can be created, but + * dereferencing the slot for existing bindings needs to be protected + * against memslot updates, specifically so that unbind doesn't race + * and free the memslot (kvm_gmem_get_file() will return NULL). + */ + mutex_lock(&kvm->slots_lock); + + filemap_invalidate_lock(inode->i_mapping); + + xa_for_each(&gmem->bindings, index, slot) + rcu_assign_pointer(slot->gmem.file, NULL); + + synchronize_rcu(); + + /* + * All in-flight operations are gone and new bindings can be created. + * Zap all SPTEs pointed at by this file. Do not free the backing + * memory, as its lifetime is associated with the inode, not the file. + */ + kvm_gmem_invalidate_begin(gmem, 0, -1ul); + kvm_gmem_invalidate_end(gmem, 0, -1ul); + + list_del(&gmem->entry); + + filemap_invalidate_unlock(inode->i_mapping); + + mutex_unlock(&kvm->slots_lock); + + xa_destroy(&gmem->bindings); + kfree(gmem); + + kvm_put_kvm(kvm); + + return 0; +} + +static struct file *kvm_gmem_get_file(struct kvm_memory_slot *slot) +{ + struct file *file; + + rcu_read_lock(); + + file = rcu_dereference(slot->gmem.file); + if (file && !get_file_rcu(file)) + file = NULL; + + rcu_read_unlock(); + + return file; +} + +static struct file_operations kvm_gmem_fops = { + .open = generic_file_open, + .release = kvm_gmem_release, + .fallocate = kvm_gmem_fallocate, +}; + +void kvm_gmem_init(struct module *module) +{ + kvm_gmem_fops.owner = module; +} + +static int kvm_gmem_migrate_folio(struct address_space *mapping, + struct folio *dst, struct folio *src, + enum migrate_mode mode) +{ + WARN_ON_ONCE(1); + return -EINVAL; +} + +static int kvm_gmem_error_page(struct address_space *mapping, struct page *page) +{ + struct list_head *gmem_list = &mapping->private_list; + struct kvm_gmem *gmem; + pgoff_t start, end; + + filemap_invalidate_lock_shared(mapping); + + start = page->index; + end = start + thp_nr_pages(page); + + list_for_each_entry(gmem, gmem_list, entry) + kvm_gmem_invalidate_begin(gmem, start, end); + + /* + * Do not truncate the range, what action is taken in response to the + * error is userspace's decision (assuming the architecture supports + * gracefully handling memory errors). If/when the guest attempts to + * access a poisoned page, kvm_gmem_get_pfn() will return -EHWPOISON, + * at which point KVM can either terminate the VM or propagate the + * error to userspace. + */ + + list_for_each_entry(gmem, gmem_list, entry) + kvm_gmem_invalidate_end(gmem, start, end); + + filemap_invalidate_unlock_shared(mapping); + + return MF_DELAYED; +} + +static const struct address_space_operations kvm_gmem_aops = { + .dirty_folio = noop_dirty_folio, +#ifdef CONFIG_MIGRATION + .migrate_folio = kvm_gmem_migrate_folio, +#endif + .error_remove_page = kvm_gmem_error_page, +}; + +static int kvm_gmem_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, + unsigned int query_flags) +{ + struct inode *inode = path->dentry->d_inode; + + generic_fillattr(idmap, request_mask, inode, stat); + return 0; +} + +static int kvm_gmem_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +{ + return -EINVAL; +} +static const struct inode_operations kvm_gmem_iops = { + .getattr = kvm_gmem_getattr, + .setattr = kvm_gmem_setattr, +}; + +static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags) +{ + const char *anon_name = "[kvm-gmem]"; + struct kvm_gmem *gmem; + struct inode *inode; + struct file *file; + int fd, err; + + fd = get_unused_fd_flags(0); + if (fd < 0) + return fd; + + gmem = kzalloc(sizeof(*gmem), GFP_KERNEL); + if (!gmem) { + err = -ENOMEM; + goto err_fd; + } + + file = anon_inode_create_getfile(anon_name, &kvm_gmem_fops, gmem, + O_RDWR, NULL); + if (IS_ERR(file)) { + err = PTR_ERR(file); + goto err_gmem; + } + + file->f_flags |= O_LARGEFILE; + + inode = file->f_inode; + WARN_ON(file->f_mapping != inode->i_mapping); + + inode->i_private = (void *)(unsigned long)flags; + inode->i_op = &kvm_gmem_iops; + inode->i_mapping->a_ops = &kvm_gmem_aops; + inode->i_mode |= S_IFREG; + inode->i_size = size; + mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER); + mapping_set_unmovable(inode->i_mapping); + /* Unmovable mappings are supposed to be marked unevictable as well. */ + WARN_ON_ONCE(!mapping_unevictable(inode->i_mapping)); + + kvm_get_kvm(kvm); + gmem->kvm = kvm; + xa_init(&gmem->bindings); + list_add(&gmem->entry, &inode->i_mapping->private_list); + + fd_install(fd, file); + return fd; + +err_gmem: + kfree(gmem); +err_fd: + put_unused_fd(fd); + return err; +} + +int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args) +{ + loff_t size = args->size; + u64 flags = args->flags; + u64 valid_flags = 0; + + if (flags & ~valid_flags) + return -EINVAL; + + if (size <= 0 || !PAGE_ALIGNED(size)) + return -EINVAL; + + return __kvm_gmem_create(kvm, size, flags); +} + +int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, + unsigned int fd, loff_t offset) +{ + loff_t size = slot->npages << PAGE_SHIFT; + unsigned long start, end; + struct kvm_gmem *gmem; + struct inode *inode; + struct file *file; + int r = -EINVAL; + + BUILD_BUG_ON(sizeof(gfn_t) != sizeof(slot->gmem.pgoff)); + + file = fget(fd); + if (!file) + return -EBADF; + + if (file->f_op != &kvm_gmem_fops) + goto err; + + gmem = file->private_data; + if (gmem->kvm != kvm) + goto err; + + inode = file_inode(file); + + if (offset < 0 || !PAGE_ALIGNED(offset) || + offset + size > i_size_read(inode)) + goto err; + + filemap_invalidate_lock(inode->i_mapping); + + start = offset >> PAGE_SHIFT; + end = start + slot->npages; + + if (!xa_empty(&gmem->bindings) && + xa_find(&gmem->bindings, &start, end - 1, XA_PRESENT)) { + filemap_invalidate_unlock(inode->i_mapping); + goto err; + } + + /* + * No synchronize_rcu() needed, any in-flight readers are guaranteed to + * be see either a NULL file or this new file, no need for them to go + * away. + */ + rcu_assign_pointer(slot->gmem.file, file); + slot->gmem.pgoff = start; + + xa_store_range(&gmem->bindings, start, end - 1, slot, GFP_KERNEL); + filemap_invalidate_unlock(inode->i_mapping); + + /* + * Drop the reference to the file, even on success. The file pins KVM, + * not the other way 'round. Active bindings are invalidated if the + * file is closed before memslots are destroyed. + */ + r = 0; +err: + fput(file); + return r; +} + +void kvm_gmem_unbind(struct kvm_memory_slot *slot) +{ + unsigned long start = slot->gmem.pgoff; + unsigned long end = start + slot->npages; + struct kvm_gmem *gmem; + struct file *file; + + /* + * Nothing to do if the underlying file was already closed (or is being + * closed right now), kvm_gmem_release() invalidates all bindings. + */ + file = kvm_gmem_get_file(slot); + if (!file) + return; + + gmem = file->private_data; + + filemap_invalidate_lock(file->f_mapping); + xa_store_range(&gmem->bindings, start, end - 1, NULL, GFP_KERNEL); + rcu_assign_pointer(slot->gmem.file, NULL); + synchronize_rcu(); + filemap_invalidate_unlock(file->f_mapping); + + fput(file); +} + +int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, + gfn_t gfn, kvm_pfn_t *pfn, int *max_order) +{ + pgoff_t index = gfn - slot->base_gfn + slot->gmem.pgoff; + struct kvm_gmem *gmem; + struct folio *folio; + struct page *page; + struct file *file; + int r; + + file = kvm_gmem_get_file(slot); + if (!file) + return -EFAULT; + + gmem = file->private_data; + + if (WARN_ON_ONCE(xa_load(&gmem->bindings, index) != slot)) { + r = -EIO; + goto out_fput; + } + + folio = kvm_gmem_get_folio(file_inode(file), index); + if (!folio) { + r = -ENOMEM; + goto out_fput; + } + + if (folio_test_hwpoison(folio)) { + r = -EHWPOISON; + goto out_unlock; + } + + page = folio_file_page(folio, index); + + *pfn = page_to_pfn(page); + if (max_order) + *max_order = 0; + + r = 0; + +out_unlock: + folio_unlock(folio); +out_fput: + fput(file); + + return r; +} +EXPORT_SYMBOL_GPL(kvm_gmem_get_pfn); diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index f1a575d39b3b..8f46d757a2c5 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -791,7 +791,7 @@ void kvm_mmu_invalidate_range_add(struct kvm *kvm, gfn_t start, gfn_t end) } } -static bool kvm_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) +bool kvm_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) { kvm_mmu_invalidate_range_add(kvm, range->start, range->end); return kvm_unmap_gfn_range(kvm, range); @@ -1027,6 +1027,9 @@ static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot) /* This does not remove the slot from struct kvm_memslots data structures */ static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) { + if (slot->flags & KVM_MEM_GUEST_MEMFD) + kvm_gmem_unbind(slot); + kvm_destroy_dirty_bitmap(slot); kvm_arch_free_memslot(kvm, slot); @@ -1606,10 +1609,18 @@ static void kvm_replace_memslot(struct kvm *kvm, #define KVM_SET_USER_MEMORY_REGION_V1_FLAGS \ (KVM_MEM_LOG_DIRTY_PAGES | KVM_MEM_READONLY) -static int check_memory_region_flags(const struct kvm_userspace_memory_region2 *mem) +static int check_memory_region_flags(struct kvm *kvm, + const struct kvm_userspace_memory_region2 *mem) { u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES; + if (kvm_arch_has_private_mem(kvm)) + valid_flags |= KVM_MEM_GUEST_MEMFD; + + /* Dirty logging private memory is not currently supported. */ + if (mem->flags & KVM_MEM_GUEST_MEMFD) + valid_flags &= ~KVM_MEM_LOG_DIRTY_PAGES; + #ifdef __KVM_HAVE_READONLY_MEM valid_flags |= KVM_MEM_READONLY; #endif @@ -2018,7 +2029,7 @@ int __kvm_set_memory_region(struct kvm *kvm, int as_id, id; int r; - r = check_memory_region_flags(mem); + r = check_memory_region_flags(kvm, mem); if (r) return r; @@ -2037,6 +2048,10 @@ int __kvm_set_memory_region(struct kvm *kvm, !access_ok((void __user *)(unsigned long)mem->userspace_addr, mem->memory_size)) return -EINVAL; + if (mem->flags & KVM_MEM_GUEST_MEMFD && + (mem->guest_memfd_offset & (PAGE_SIZE - 1) || + mem->guest_memfd_offset + mem->memory_size < mem->guest_memfd_offset)) + return -EINVAL; if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM) return -EINVAL; if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr) @@ -2075,6 +2090,9 @@ int __kvm_set_memory_region(struct kvm *kvm, if ((kvm->nr_memslot_pages + npages) < kvm->nr_memslot_pages) return -EINVAL; } else { /* Modify an existing slot. */ + /* Private memslots are immutable, they can only be deleted. */ + if (mem->flags & KVM_MEM_GUEST_MEMFD) + return -EINVAL; if ((mem->userspace_addr != old->userspace_addr) || (npages != old->npages) || ((mem->flags ^ old->flags) & KVM_MEM_READONLY)) @@ -2103,10 +2121,23 @@ int __kvm_set_memory_region(struct kvm *kvm, new->npages = npages; new->flags = mem->flags; new->userspace_addr = mem->userspace_addr; + if (mem->flags & KVM_MEM_GUEST_MEMFD) { + r = kvm_gmem_bind(kvm, new, mem->guest_memfd, mem->guest_memfd_offset); + if (r) + goto out; + } r = kvm_set_memslot(kvm, old, new, change); if (r) - kfree(new); + goto out_unbind; + + return 0; + +out_unbind: + if (mem->flags & KVM_MEM_GUEST_MEMFD) + kvm_gmem_unbind(new); +out: + kfree(new); return r; } EXPORT_SYMBOL_GPL(__kvm_set_memory_region); @@ -2442,7 +2473,7 @@ out: static u64 kvm_supported_mem_attributes(struct kvm *kvm) { - if (!kvm) + if (!kvm || kvm_arch_has_private_mem(kvm)) return KVM_MEMORY_ATTRIBUTE_PRIVATE; return 0; @@ -4844,6 +4875,10 @@ static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) #ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES case KVM_CAP_MEMORY_ATTRIBUTES: return kvm_supported_mem_attributes(kvm); +#endif +#ifdef CONFIG_KVM_PRIVATE_MEM + case KVM_CAP_GUEST_MEMFD: + return !kvm || kvm_arch_has_private_mem(kvm); #endif default: break; @@ -5277,6 +5312,18 @@ static long kvm_vm_ioctl(struct file *filp, case KVM_GET_STATS_FD: r = kvm_vm_ioctl_get_stats_fd(kvm); break; +#ifdef CONFIG_KVM_PRIVATE_MEM + case KVM_CREATE_GUEST_MEMFD: { + struct kvm_create_guest_memfd guest_memfd; + + r = -EFAULT; + if (copy_from_user(&guest_memfd, argp, sizeof(guest_memfd))) + goto out; + + r = kvm_gmem_create(kvm, &guest_memfd); + break; + } +#endif default: r = kvm_arch_vm_ioctl(filp, ioctl, arg); } @@ -6409,6 +6456,8 @@ int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module) if (WARN_ON_ONCE(r)) goto err_vfio; + kvm_gmem_init(module); + /* * Registration _must_ be the very last thing done, as this exposes * /dev/kvm to userspace, i.e. all infrastructure must be setup! diff --git a/virt/kvm/kvm_mm.h b/virt/kvm/kvm_mm.h index 180f1a09e6ba..ecefc7ec51af 100644 --- a/virt/kvm/kvm_mm.h +++ b/virt/kvm/kvm_mm.h @@ -37,4 +37,30 @@ static inline void gfn_to_pfn_cache_invalidate_start(struct kvm *kvm, } #endif /* HAVE_KVM_PFNCACHE */ +#ifdef CONFIG_KVM_PRIVATE_MEM +void kvm_gmem_init(struct module *module); +int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args); +int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, + unsigned int fd, loff_t offset); +void kvm_gmem_unbind(struct kvm_memory_slot *slot); +#else +static inline void kvm_gmem_init(struct module *module) +{ + +} + +static inline int kvm_gmem_bind(struct kvm *kvm, + struct kvm_memory_slot *slot, + unsigned int fd, loff_t offset) +{ + WARN_ON_ONCE(1); + return -EIO; +} + +static inline void kvm_gmem_unbind(struct kvm_memory_slot *slot) +{ + WARN_ON_ONCE(1); +} +#endif /* CONFIG_KVM_PRIVATE_MEM */ + #endif /* __KVM_MM_H__ */ From ee605e31563348f44d2ff0b6b74d33df69dd535c Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 27 Oct 2023 11:22:00 -0700 Subject: [PATCH 017/336] KVM: x86: "Reset" vcpu->run->exit_reason early in KVM_RUN Initialize run->exit_reason to KVM_EXIT_UNKNOWN early in KVM_RUN to reduce the probability of exiting to userspace with a stale run->exit_reason that *appears* to be valid. To support fd-based guest memory (guest memory without a corresponding userspace virtual address), KVM will exit to userspace for various memory related errors, which userspace *may* be able to resolve, instead of using e.g. BUS_MCEERR_AR. And in the more distant future, KVM will also likely utilize the same functionality to let userspace "intercept" and handle memory faults when the userspace mapping is missing, i.e. when fast gup() fails. Because many of KVM's internal APIs related to guest memory use '0' to indicate "success, continue on" and not "exit to userspace", reporting memory faults/errors to userspace will set run->exit_reason and corresponding fields in the run structure fields in conjunction with a a non-zero, negative return code, e.g. -EFAULT or -EHWPOISON. And because KVM already returns -EFAULT in many paths, there's a relatively high probability that KVM could return -EFAULT without setting run->exit_reason, in which case reporting KVM_EXIT_UNKNOWN is much better than reporting whatever exit reason happened to be in the run structure. Note, KVM must wait until after run->immediate_exit is serviced to sanitize run->exit_reason as KVM's ABI is that run->exit_reason is preserved across KVM_RUN when run->immediate_exit is true. Link: https://lore.kernel.org/all/20230908222905.1321305-1-amoorthy@google.com Link: https://lore.kernel.org/all/ZFFbwOXZ5uI%2Fgdaf@google.com Signed-off-by: Sean Christopherson Reviewed-by: Paolo Bonzini Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Message-Id: <20231027182217.3615211-19-seanjc@google.com> Reviewed-by: Xiaoyao Li Signed-off-by: Paolo Bonzini --- arch/x86/kvm/x86.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 8f9d8939b63b..f661acb01c58 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -11082,6 +11082,7 @@ static int vcpu_run(struct kvm_vcpu *vcpu) { int r; + vcpu->run->exit_reason = KVM_EXIT_UNKNOWN; vcpu->arch.l1tf_flush_l1d = true; for (;;) { From 90b4fe17981e155432c4dbc490606d0c2e9c2199 Mon Sep 17 00:00:00 2001 From: Chao Peng Date: Fri, 27 Oct 2023 11:22:01 -0700 Subject: [PATCH 018/336] KVM: x86: Disallow hugepages when memory attributes are mixed Disallow creating hugepages with mixed memory attributes, e.g. shared versus private, as mapping a hugepage in this case would allow the guest to access memory with the wrong attributes, e.g. overlaying private memory with a shared hugepage. Tracking whether or not attributes are mixed via the existing disallow_lpage field, but use the most significant bit in 'disallow_lpage' to indicate a hugepage has mixed attributes instead using the normal refcounting. Whether or not attributes are mixed is binary; either they are or they aren't. Attempting to squeeze that info into the refcount is unnecessarily complex as it would require knowing the previous state of the mixed count when updating attributes. Using a flag means KVM just needs to ensure the current status is reflected in the memslots. Signed-off-by: Chao Peng Co-developed-by: Sean Christopherson Signed-off-by: Sean Christopherson Message-Id: <20231027182217.3615211-20-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 3 + arch/x86/kvm/mmu/mmu.c | 154 +++++++++++++++++++++++++++++++- arch/x86/kvm/x86.c | 4 + 3 files changed, 159 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 6f559fb75e6d..fa0d42202405 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1848,6 +1848,9 @@ int kvm_mmu_create(struct kvm_vcpu *vcpu); void kvm_mmu_init_vm(struct kvm *kvm); void kvm_mmu_uninit_vm(struct kvm *kvm); +void kvm_mmu_init_memslot_memory_attributes(struct kvm *kvm, + struct kvm_memory_slot *slot); + void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu); void kvm_mmu_reset_context(struct kvm_vcpu *vcpu); void kvm_mmu_slot_remove_write_access(struct kvm *kvm, diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index b2d916f786ca..f5c6b0643645 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -795,16 +795,26 @@ static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn, return &slot->arch.lpage_info[level - 2][idx]; } +/* + * The most significant bit in disallow_lpage tracks whether or not memory + * attributes are mixed, i.e. not identical for all gfns at the current level. + * The lower order bits are used to refcount other cases where a hugepage is + * disallowed, e.g. if KVM has shadow a page table at the gfn. + */ +#define KVM_LPAGE_MIXED_FLAG BIT(31) + static void update_gfn_disallow_lpage_count(const struct kvm_memory_slot *slot, gfn_t gfn, int count) { struct kvm_lpage_info *linfo; - int i; + int old, i; for (i = PG_LEVEL_2M; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) { linfo = lpage_info_slot(gfn, slot, i); + + old = linfo->disallow_lpage; linfo->disallow_lpage += count; - WARN_ON_ONCE(linfo->disallow_lpage < 0); + WARN_ON_ONCE((old ^ linfo->disallow_lpage) & KVM_LPAGE_MIXED_FLAG); } } @@ -7176,3 +7186,143 @@ void kvm_mmu_pre_destroy_vm(struct kvm *kvm) if (kvm->arch.nx_huge_page_recovery_thread) kthread_stop(kvm->arch.nx_huge_page_recovery_thread); } + +#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +static bool hugepage_test_mixed(struct kvm_memory_slot *slot, gfn_t gfn, + int level) +{ + return lpage_info_slot(gfn, slot, level)->disallow_lpage & KVM_LPAGE_MIXED_FLAG; +} + +static void hugepage_clear_mixed(struct kvm_memory_slot *slot, gfn_t gfn, + int level) +{ + lpage_info_slot(gfn, slot, level)->disallow_lpage &= ~KVM_LPAGE_MIXED_FLAG; +} + +static void hugepage_set_mixed(struct kvm_memory_slot *slot, gfn_t gfn, + int level) +{ + lpage_info_slot(gfn, slot, level)->disallow_lpage |= KVM_LPAGE_MIXED_FLAG; +} + +static bool hugepage_has_attrs(struct kvm *kvm, struct kvm_memory_slot *slot, + gfn_t gfn, int level, unsigned long attrs) +{ + const unsigned long start = gfn; + const unsigned long end = start + KVM_PAGES_PER_HPAGE(level); + + if (level == PG_LEVEL_2M) + return kvm_range_has_memory_attributes(kvm, start, end, attrs); + + for (gfn = start; gfn < end; gfn += KVM_PAGES_PER_HPAGE(level - 1)) { + if (hugepage_test_mixed(slot, gfn, level - 1) || + attrs != kvm_get_memory_attributes(kvm, gfn)) + return false; + } + return true; +} + +bool kvm_arch_post_set_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range) +{ + unsigned long attrs = range->arg.attributes; + struct kvm_memory_slot *slot = range->slot; + int level; + + lockdep_assert_held_write(&kvm->mmu_lock); + lockdep_assert_held(&kvm->slots_lock); + + /* + * Calculate which ranges can be mapped with hugepages even if the slot + * can't map memory PRIVATE. KVM mustn't create a SHARED hugepage over + * a range that has PRIVATE GFNs, and conversely converting a range to + * SHARED may now allow hugepages. + */ + if (WARN_ON_ONCE(!kvm_arch_has_private_mem(kvm))) + return false; + + /* + * The sequence matters here: upper levels consume the result of lower + * level's scanning. + */ + for (level = PG_LEVEL_2M; level <= KVM_MAX_HUGEPAGE_LEVEL; level++) { + gfn_t nr_pages = KVM_PAGES_PER_HPAGE(level); + gfn_t gfn = gfn_round_for_level(range->start, level); + + /* Process the head page if it straddles the range. */ + if (gfn != range->start || gfn + nr_pages > range->end) { + /* + * Skip mixed tracking if the aligned gfn isn't covered + * by the memslot, KVM can't use a hugepage due to the + * misaligned address regardless of memory attributes. + */ + if (gfn >= slot->base_gfn) { + if (hugepage_has_attrs(kvm, slot, gfn, level, attrs)) + hugepage_clear_mixed(slot, gfn, level); + else + hugepage_set_mixed(slot, gfn, level); + } + gfn += nr_pages; + } + + /* + * Pages entirely covered by the range are guaranteed to have + * only the attributes which were just set. + */ + for ( ; gfn + nr_pages <= range->end; gfn += nr_pages) + hugepage_clear_mixed(slot, gfn, level); + + /* + * Process the last tail page if it straddles the range and is + * contained by the memslot. Like the head page, KVM can't + * create a hugepage if the slot size is misaligned. + */ + if (gfn < range->end && + (gfn + nr_pages) <= (slot->base_gfn + slot->npages)) { + if (hugepage_has_attrs(kvm, slot, gfn, level, attrs)) + hugepage_clear_mixed(slot, gfn, level); + else + hugepage_set_mixed(slot, gfn, level); + } + } + return false; +} + +void kvm_mmu_init_memslot_memory_attributes(struct kvm *kvm, + struct kvm_memory_slot *slot) +{ + int level; + + if (!kvm_arch_has_private_mem(kvm)) + return; + + for (level = PG_LEVEL_2M; level <= KVM_MAX_HUGEPAGE_LEVEL; level++) { + /* + * Don't bother tracking mixed attributes for pages that can't + * be huge due to alignment, i.e. process only pages that are + * entirely contained by the memslot. + */ + gfn_t end = gfn_round_for_level(slot->base_gfn + slot->npages, level); + gfn_t start = gfn_round_for_level(slot->base_gfn, level); + gfn_t nr_pages = KVM_PAGES_PER_HPAGE(level); + gfn_t gfn; + + if (start < slot->base_gfn) + start += nr_pages; + + /* + * Unlike setting attributes, every potential hugepage needs to + * be manually checked as the attributes may already be mixed. + */ + for (gfn = start; gfn < end; gfn += nr_pages) { + unsigned long attrs = kvm_get_memory_attributes(kvm, gfn); + + if (hugepage_has_attrs(kvm, slot, gfn, level, attrs)) + hugepage_clear_mixed(slot, gfn, level); + else + hugepage_set_mixed(slot, gfn, level); + } + } +} +#endif diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index f661acb01c58..e1aad0c81f6f 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -12728,6 +12728,10 @@ static int kvm_alloc_memslot_metadata(struct kvm *kvm, } } +#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES + kvm_mmu_init_memslot_memory_attributes(kvm, slot); +#endif + if (kvm_page_track_create_memslot(kvm, slot, npages)) goto out_free; From 8dd2eee9d526c30fccfe75da7ec5365c6476e510 Mon Sep 17 00:00:00 2001 From: Chao Peng Date: Fri, 27 Oct 2023 11:22:02 -0700 Subject: [PATCH 019/336] KVM: x86/mmu: Handle page fault for private memory Add support for resolving page faults on guest private memory for VMs that differentiate between "shared" and "private" memory. For such VMs, KVM_MEM_GUEST_MEMFD memslots can include both fd-based private memory and hva-based shared memory, and KVM needs to map in the "correct" variant, i.e. KVM needs to map the gfn shared/private as appropriate based on the current state of the gfn's KVM_MEMORY_ATTRIBUTE_PRIVATE flag. For AMD's SEV-SNP and Intel's TDX, the guest effectively gets to request shared vs. private via a bit in the guest page tables, i.e. what the guest wants may conflict with the current memory attributes. To support such "implicit" conversion requests, exit to user with KVM_EXIT_MEMORY_FAULT to forward the request to userspace. Add a new flag for memory faults, KVM_MEMORY_EXIT_FLAG_PRIVATE, to communicate whether the guest wants to map memory as shared vs. private. Like KVM_MEMORY_ATTRIBUTE_PRIVATE, use bit 3 for flagging private memory so that KVM can use bits 0-2 for capturing RWX behavior if/when userspace needs such information, e.g. a likely user of KVM_EXIT_MEMORY_FAULT is to exit on missing mappings when handling guest page fault VM-Exits. In that case, userspace will want to know RWX information in order to correctly/precisely resolve the fault. Note, private memory *must* be backed by guest_memfd, i.e. shared mappings always come from the host userspace page tables, and private mappings always come from a guest_memfd instance. Co-developed-by: Yu Zhang Signed-off-by: Yu Zhang Signed-off-by: Chao Peng Co-developed-by: Sean Christopherson Signed-off-by: Sean Christopherson Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Message-Id: <20231027182217.3615211-21-seanjc@google.com> Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/api.rst | 8 ++- arch/x86/kvm/mmu/mmu.c | 101 ++++++++++++++++++++++++++++++-- arch/x86/kvm/mmu/mmu_internal.h | 1 + include/linux/kvm_host.h | 8 ++- include/uapi/linux/kvm.h | 1 + 5 files changed, 110 insertions(+), 9 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 1e61faf02b2a..726c87c35d57 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -6952,6 +6952,7 @@ spec refer, https://github.com/riscv/riscv-sbi-doc. /* KVM_EXIT_MEMORY_FAULT */ struct { + #define KVM_MEMORY_EXIT_FLAG_PRIVATE (1ULL << 3) __u64 flags; __u64 gpa; __u64 size; @@ -6960,8 +6961,11 @@ spec refer, https://github.com/riscv/riscv-sbi-doc. KVM_EXIT_MEMORY_FAULT indicates the vCPU has encountered a memory fault that could not be resolved by KVM. The 'gpa' and 'size' (in bytes) describe the guest physical address range [gpa, gpa + size) of the fault. The 'flags' field -describes properties of the faulting access that are likely pertinent. -Currently, no flags are defined. +describes properties of the faulting access that are likely pertinent: + + - KVM_MEMORY_EXIT_FLAG_PRIVATE - When set, indicates the memory fault occurred + on a private memory access. When clear, indicates the fault occurred on a + shared access. Note! KVM_EXIT_MEMORY_FAULT is unique among all KVM exit reasons in that it accompanies a return code of '-1', not '0'! errno will always be set to EFAULT diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index f5c6b0643645..754a5aaebee5 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3147,9 +3147,9 @@ out: return level; } -int kvm_mmu_max_mapping_level(struct kvm *kvm, - const struct kvm_memory_slot *slot, gfn_t gfn, - int max_level) +static int __kvm_mmu_max_mapping_level(struct kvm *kvm, + const struct kvm_memory_slot *slot, + gfn_t gfn, int max_level, bool is_private) { struct kvm_lpage_info *linfo; int host_level; @@ -3161,6 +3161,9 @@ int kvm_mmu_max_mapping_level(struct kvm *kvm, break; } + if (is_private) + return max_level; + if (max_level == PG_LEVEL_4K) return PG_LEVEL_4K; @@ -3168,6 +3171,16 @@ int kvm_mmu_max_mapping_level(struct kvm *kvm, return min(host_level, max_level); } +int kvm_mmu_max_mapping_level(struct kvm *kvm, + const struct kvm_memory_slot *slot, gfn_t gfn, + int max_level) +{ + bool is_private = kvm_slot_can_be_private(slot) && + kvm_mem_is_private(kvm, gfn); + + return __kvm_mmu_max_mapping_level(kvm, slot, gfn, max_level, is_private); +} + void kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) { struct kvm_memory_slot *slot = fault->slot; @@ -3188,8 +3201,9 @@ void kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault * Enforce the iTLB multihit workaround after capturing the requested * level, which will be used to do precise, accurate accounting. */ - fault->req_level = kvm_mmu_max_mapping_level(vcpu->kvm, slot, - fault->gfn, fault->max_level); + fault->req_level = __kvm_mmu_max_mapping_level(vcpu->kvm, slot, + fault->gfn, fault->max_level, + fault->is_private); if (fault->req_level == PG_LEVEL_4K || fault->huge_page_disallowed) return; @@ -4269,6 +4283,55 @@ void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work) kvm_mmu_do_page_fault(vcpu, work->cr2_or_gpa, 0, true, NULL); } +static inline u8 kvm_max_level_for_order(int order) +{ + BUILD_BUG_ON(KVM_MAX_HUGEPAGE_LEVEL > PG_LEVEL_1G); + + KVM_MMU_WARN_ON(order != KVM_HPAGE_GFN_SHIFT(PG_LEVEL_1G) && + order != KVM_HPAGE_GFN_SHIFT(PG_LEVEL_2M) && + order != KVM_HPAGE_GFN_SHIFT(PG_LEVEL_4K)); + + if (order >= KVM_HPAGE_GFN_SHIFT(PG_LEVEL_1G)) + return PG_LEVEL_1G; + + if (order >= KVM_HPAGE_GFN_SHIFT(PG_LEVEL_2M)) + return PG_LEVEL_2M; + + return PG_LEVEL_4K; +} + +static void kvm_mmu_prepare_memory_fault_exit(struct kvm_vcpu *vcpu, + struct kvm_page_fault *fault) +{ + kvm_prepare_memory_fault_exit(vcpu, fault->gfn << PAGE_SHIFT, + PAGE_SIZE, fault->write, fault->exec, + fault->is_private); +} + +static int kvm_faultin_pfn_private(struct kvm_vcpu *vcpu, + struct kvm_page_fault *fault) +{ + int max_order, r; + + if (!kvm_slot_can_be_private(fault->slot)) { + kvm_mmu_prepare_memory_fault_exit(vcpu, fault); + return -EFAULT; + } + + r = kvm_gmem_get_pfn(vcpu->kvm, fault->slot, fault->gfn, &fault->pfn, + &max_order); + if (r) { + kvm_mmu_prepare_memory_fault_exit(vcpu, fault); + return r; + } + + fault->max_level = min(kvm_max_level_for_order(max_order), + fault->max_level); + fault->map_writable = !(fault->slot->flags & KVM_MEM_READONLY); + + return RET_PF_CONTINUE; +} + static int __kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) { struct kvm_memory_slot *slot = fault->slot; @@ -4301,6 +4364,14 @@ static int __kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault return RET_PF_EMULATE; } + if (fault->is_private != kvm_mem_is_private(vcpu->kvm, fault->gfn)) { + kvm_mmu_prepare_memory_fault_exit(vcpu, fault); + return -EFAULT; + } + + if (fault->is_private) + return kvm_faultin_pfn_private(vcpu, fault); + async = false; fault->pfn = __gfn_to_pfn_memslot(slot, fault->gfn, false, false, &async, fault->write, &fault->map_writable, @@ -7188,6 +7259,26 @@ void kvm_mmu_pre_destroy_vm(struct kvm *kvm) } #ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range) +{ + /* + * Zap SPTEs even if the slot can't be mapped PRIVATE. KVM x86 only + * supports KVM_MEMORY_ATTRIBUTE_PRIVATE, and so it *seems* like KVM + * can simply ignore such slots. But if userspace is making memory + * PRIVATE, then KVM must prevent the guest from accessing the memory + * as shared. And if userspace is making memory SHARED and this point + * is reached, then at least one page within the range was previously + * PRIVATE, i.e. the slot's possible hugepage ranges are changing. + * Zapping SPTEs in this case ensures KVM will reassess whether or not + * a hugepage can be used for affected ranges. + */ + if (WARN_ON_ONCE(!kvm_arch_has_private_mem(kvm))) + return false; + + return kvm_unmap_gfn_range(kvm, range); +} + static bool hugepage_test_mixed(struct kvm_memory_slot *slot, gfn_t gfn, int level) { diff --git a/arch/x86/kvm/mmu/mmu_internal.h b/arch/x86/kvm/mmu/mmu_internal.h index decc1f153669..86c7cb692786 100644 --- a/arch/x86/kvm/mmu/mmu_internal.h +++ b/arch/x86/kvm/mmu/mmu_internal.h @@ -201,6 +201,7 @@ struct kvm_page_fault { /* Derived from mmu and global state. */ const bool is_tdp; + const bool is_private; const bool nx_huge_page_workaround_enabled; /* diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index a6de526c0426..67dfd4d79529 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2357,14 +2357,18 @@ static inline void kvm_account_pgtable_pages(void *virt, int nr) #define KVM_DIRTY_RING_MAX_ENTRIES 65536 static inline void kvm_prepare_memory_fault_exit(struct kvm_vcpu *vcpu, - gpa_t gpa, gpa_t size) + gpa_t gpa, gpa_t size, + bool is_write, bool is_exec, + bool is_private) { vcpu->run->exit_reason = KVM_EXIT_MEMORY_FAULT; vcpu->run->memory_fault.gpa = gpa; vcpu->run->memory_fault.size = size; - /* Flags are not (yet) defined or communicated to userspace. */ + /* RWX flags are not (yet) defined or communicated to userspace. */ vcpu->run->memory_fault.flags = 0; + if (is_private) + vcpu->run->memory_fault.flags |= KVM_MEMORY_EXIT_FLAG_PRIVATE; } #ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 2802d10aa88c..8eb10f560c69 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -535,6 +535,7 @@ struct kvm_run { } notify; /* KVM_EXIT_MEMORY_FAULT */ struct { +#define KVM_MEMORY_EXIT_FLAG_PRIVATE (1ULL << 3) __u64 flags; __u64 gpa; __u64 size; From 2333afa17af0f4b6651214ee17cfd5ae5f47787a Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 27 Oct 2023 11:22:03 -0700 Subject: [PATCH 020/336] KVM: Drop superfluous __KVM_VCPU_MULTIPLE_ADDRESS_SPACE macro Drop __KVM_VCPU_MULTIPLE_ADDRESS_SPACE and instead check the value of KVM_ADDRESS_SPACE_NUM. No functional change intended. Reviewed-by: Paolo Bonzini Signed-off-by: Sean Christopherson Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Message-Id: <20231027182217.3615211-22-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 1 - include/linux/kvm_host.h | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index fa0d42202405..061eec231299 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -2136,7 +2136,6 @@ enum { #define HF_SMM_MASK (1 << 1) #define HF_SMM_INSIDE_NMI_MASK (1 << 2) -# define __KVM_VCPU_MULTIPLE_ADDRESS_SPACE # define KVM_ADDRESS_SPACE_NUM 2 # define kvm_arch_vcpu_memslots_id(vcpu) ((vcpu)->arch.hflags & HF_SMM_MASK ? 1 : 0) # define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, (role).smm) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 67dfd4d79529..db423ea9e3a4 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -690,7 +690,7 @@ bool kvm_arch_irqchip_in_kernel(struct kvm *kvm); #define KVM_MEM_SLOTS_NUM SHRT_MAX #define KVM_USER_MEM_SLOTS (KVM_MEM_SLOTS_NUM - KVM_INTERNAL_MEM_SLOTS) -#ifndef __KVM_VCPU_MULTIPLE_ADDRESS_SPACE +#if KVM_ADDRESS_SPACE_NUM == 1 static inline int kvm_arch_vcpu_memslots_id(struct kvm_vcpu *vcpu) { return 0; From eed52e434bc33603ddb0af62b6c4ef818948489d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 27 Oct 2023 11:22:04 -0700 Subject: [PATCH 021/336] KVM: Allow arch code to track number of memslot address spaces per VM Let x86 track the number of address spaces on a per-VM basis so that KVM can disallow SMM memslots for confidential VMs. Confidentials VMs are fundamentally incompatible with emulating SMM, which as the name suggests requires being able to read and write guest memory and register state. Disallowing SMM will simplify support for guest private memory, as KVM will not need to worry about tracking memory attributes for multiple address spaces (SMM is the only "non-default" address space across all architectures). Signed-off-by: Sean Christopherson Reviewed-by: Paolo Bonzini Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Message-Id: <20231027182217.3615211-23-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/powerpc/kvm/book3s_hv.c | 2 +- arch/x86/include/asm/kvm_host.h | 8 +++++++- arch/x86/kvm/debugfs.c | 2 +- arch/x86/kvm/mmu/mmu.c | 6 +++--- arch/x86/kvm/x86.c | 2 +- include/linux/kvm_host.h | 17 +++++++++++------ virt/kvm/dirty_ring.c | 2 +- virt/kvm/kvm_main.c | 26 ++++++++++++++------------ 8 files changed, 39 insertions(+), 26 deletions(-) diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index 130bafdb1430..9b0eaa17275a 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -6084,7 +6084,7 @@ static int kvmhv_svm_off(struct kvm *kvm) } srcu_idx = srcu_read_lock(&kvm->srcu); - for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { + for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) { struct kvm_memory_slot *memslot; struct kvm_memslots *slots = __kvm_memslots(kvm, i); int bkt; diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 061eec231299..75ab0da06e64 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -2136,9 +2136,15 @@ enum { #define HF_SMM_MASK (1 << 1) #define HF_SMM_INSIDE_NMI_MASK (1 << 2) -# define KVM_ADDRESS_SPACE_NUM 2 +# define KVM_MAX_NR_ADDRESS_SPACES 2 # define kvm_arch_vcpu_memslots_id(vcpu) ((vcpu)->arch.hflags & HF_SMM_MASK ? 1 : 0) # define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, (role).smm) + +static inline int kvm_arch_nr_memslot_as_ids(struct kvm *kvm) +{ + return KVM_MAX_NR_ADDRESS_SPACES; +} + #else # define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, 0) #endif diff --git a/arch/x86/kvm/debugfs.c b/arch/x86/kvm/debugfs.c index ee8c4c3496ed..42026b3f3ff3 100644 --- a/arch/x86/kvm/debugfs.c +++ b/arch/x86/kvm/debugfs.c @@ -111,7 +111,7 @@ static int kvm_mmu_rmaps_stat_show(struct seq_file *m, void *v) mutex_lock(&kvm->slots_lock); write_lock(&kvm->mmu_lock); - for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { + for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) { int bkt; slots = __kvm_memslots(kvm, i); diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 754a5aaebee5..4de7670d5976 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3763,7 +3763,7 @@ static int mmu_first_shadow_root_alloc(struct kvm *kvm) kvm_page_track_write_tracking_enabled(kvm)) goto out_success; - for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { + for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) { slots = __kvm_memslots(kvm, i); kvm_for_each_memslot(slot, bkt, slots) { /* @@ -6309,7 +6309,7 @@ static bool kvm_rmap_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_e if (!kvm_memslots_have_rmaps(kvm)) return flush; - for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { + for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) { slots = __kvm_memslots(kvm, i); kvm_for_each_memslot_in_gfn_range(&iter, slots, gfn_start, gfn_end) { @@ -6806,7 +6806,7 @@ void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen) * modifier prior to checking for a wrap of the MMIO generation so * that a wrap in any address space is detected. */ - gen &= ~((u64)KVM_ADDRESS_SPACE_NUM - 1); + gen &= ~((u64)kvm_arch_nr_memslot_as_ids(kvm) - 1); /* * The very rare case: if the MMIO generation number has wrapped, diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index e1aad0c81f6f..f521c97f5c64 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -12577,7 +12577,7 @@ void __user * __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, hva = slot->userspace_addr; } - for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { + for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) { struct kvm_userspace_memory_region2 m; m.slot = id | (i << 16); diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index db423ea9e3a4..3ebc6912c54a 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -80,8 +80,8 @@ /* Two fragments for cross MMIO pages. */ #define KVM_MAX_MMIO_FRAGMENTS 2 -#ifndef KVM_ADDRESS_SPACE_NUM -#define KVM_ADDRESS_SPACE_NUM 1 +#ifndef KVM_MAX_NR_ADDRESS_SPACES +#define KVM_MAX_NR_ADDRESS_SPACES 1 #endif /* @@ -690,7 +690,12 @@ bool kvm_arch_irqchip_in_kernel(struct kvm *kvm); #define KVM_MEM_SLOTS_NUM SHRT_MAX #define KVM_USER_MEM_SLOTS (KVM_MEM_SLOTS_NUM - KVM_INTERNAL_MEM_SLOTS) -#if KVM_ADDRESS_SPACE_NUM == 1 +#if KVM_MAX_NR_ADDRESS_SPACES == 1 +static inline int kvm_arch_nr_memslot_as_ids(struct kvm *kvm) +{ + return KVM_MAX_NR_ADDRESS_SPACES; +} + static inline int kvm_arch_vcpu_memslots_id(struct kvm_vcpu *vcpu) { return 0; @@ -745,9 +750,9 @@ struct kvm { struct mm_struct *mm; /* userspace tied to this vm */ unsigned long nr_memslot_pages; /* The two memslot sets - active and inactive (per address space) */ - struct kvm_memslots __memslots[KVM_ADDRESS_SPACE_NUM][2]; + struct kvm_memslots __memslots[KVM_MAX_NR_ADDRESS_SPACES][2]; /* The current active memslot set for each address space */ - struct kvm_memslots __rcu *memslots[KVM_ADDRESS_SPACE_NUM]; + struct kvm_memslots __rcu *memslots[KVM_MAX_NR_ADDRESS_SPACES]; struct xarray vcpu_array; /* * Protected by slots_lock, but can be read outside if an @@ -1017,7 +1022,7 @@ void kvm_put_kvm_no_destroy(struct kvm *kvm); static inline struct kvm_memslots *__kvm_memslots(struct kvm *kvm, int as_id) { - as_id = array_index_nospec(as_id, KVM_ADDRESS_SPACE_NUM); + as_id = array_index_nospec(as_id, KVM_MAX_NR_ADDRESS_SPACES); return srcu_dereference_check(kvm->memslots[as_id], &kvm->srcu, lockdep_is_held(&kvm->slots_lock) || !refcount_read(&kvm->users_count)); diff --git a/virt/kvm/dirty_ring.c b/virt/kvm/dirty_ring.c index c1cd7dfe4a90..86d267db87bb 100644 --- a/virt/kvm/dirty_ring.c +++ b/virt/kvm/dirty_ring.c @@ -58,7 +58,7 @@ static void kvm_reset_dirty_gfn(struct kvm *kvm, u32 slot, u64 offset, u64 mask) as_id = slot >> 16; id = (u16)slot; - if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) + if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_USER_MEM_SLOTS) return; memslot = id_to_memslot(__kvm_memslots(kvm, as_id), id); diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 8f46d757a2c5..8758cb799e18 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -615,7 +615,7 @@ static __always_inline kvm_mn_ret_t __kvm_handle_hva_range(struct kvm *kvm, idx = srcu_read_lock(&kvm->srcu); - for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { + for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) { struct interval_tree_node *node; slots = __kvm_memslots(kvm, i); @@ -1241,7 +1241,7 @@ static struct kvm *kvm_create_vm(unsigned long type, const char *fdname) goto out_err_no_irq_srcu; refcount_set(&kvm->users_count, 1); - for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { + for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) { for (j = 0; j < 2; j++) { slots = &kvm->__memslots[i][j]; @@ -1391,7 +1391,7 @@ static void kvm_destroy_vm(struct kvm *kvm) #endif kvm_arch_destroy_vm(kvm); kvm_destroy_devices(kvm); - for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { + for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) { kvm_free_memslots(kvm, &kvm->__memslots[i][0]); kvm_free_memslots(kvm, &kvm->__memslots[i][1]); } @@ -1682,7 +1682,7 @@ static void kvm_swap_active_memslots(struct kvm *kvm, int as_id) * space 0 will use generations 0, 2, 4, ... while address space 1 will * use generations 1, 3, 5, ... */ - gen += KVM_ADDRESS_SPACE_NUM; + gen += kvm_arch_nr_memslot_as_ids(kvm); kvm_arch_memslots_updated(kvm, gen); @@ -2052,7 +2052,7 @@ int __kvm_set_memory_region(struct kvm *kvm, (mem->guest_memfd_offset & (PAGE_SIZE - 1) || mem->guest_memfd_offset + mem->memory_size < mem->guest_memfd_offset)) return -EINVAL; - if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM) + if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_MEM_SLOTS_NUM) return -EINVAL; if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr) return -EINVAL; @@ -2188,7 +2188,7 @@ int kvm_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log, as_id = log->slot >> 16; id = (u16)log->slot; - if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) + if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_USER_MEM_SLOTS) return -EINVAL; slots = __kvm_memslots(kvm, as_id); @@ -2250,7 +2250,7 @@ static int kvm_get_dirty_log_protect(struct kvm *kvm, struct kvm_dirty_log *log) as_id = log->slot >> 16; id = (u16)log->slot; - if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) + if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_USER_MEM_SLOTS) return -EINVAL; slots = __kvm_memslots(kvm, as_id); @@ -2362,7 +2362,7 @@ static int kvm_clear_dirty_log_protect(struct kvm *kvm, as_id = log->slot >> 16; id = (u16)log->slot; - if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) + if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_USER_MEM_SLOTS) return -EINVAL; if (log->first_page & 63) @@ -2493,7 +2493,7 @@ static __always_inline void kvm_handle_gfn_range(struct kvm *kvm, gfn_range.arg = range->arg; gfn_range.may_block = range->may_block; - for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { + for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) { slots = __kvm_memslots(kvm, i); kvm_for_each_memslot_in_gfn_range(&iter, slots, range->start, range->end) { @@ -4848,9 +4848,11 @@ static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) case KVM_CAP_IRQ_ROUTING: return KVM_MAX_IRQ_ROUTES; #endif -#if KVM_ADDRESS_SPACE_NUM > 1 +#if KVM_MAX_NR_ADDRESS_SPACES > 1 case KVM_CAP_MULTI_ADDRESS_SPACE: - return KVM_ADDRESS_SPACE_NUM; + if (kvm) + return kvm_arch_nr_memslot_as_ids(kvm); + return KVM_MAX_NR_ADDRESS_SPACES; #endif case KVM_CAP_NR_MEMSLOTS: return KVM_USER_MEM_SLOTS; @@ -4958,7 +4960,7 @@ bool kvm_are_all_memslots_empty(struct kvm *kvm) lockdep_assert_held(&kvm->slots_lock); - for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { + for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) { if (!kvm_memslots_empty(__kvm_memslots(kvm, i))) return false; } From 89ea60c2c7b5838bf192c50062d5720cd6ab8662 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 27 Oct 2023 11:22:05 -0700 Subject: [PATCH 022/336] KVM: x86: Add support for "protected VMs" that can utilize private memory Add a new x86 VM type, KVM_X86_SW_PROTECTED_VM, to serve as a development and testing vehicle for Confidential (CoCo) VMs, and potentially to even become a "real" product in the distant future, e.g. a la pKVM. The private memory support in KVM x86 is aimed at AMD's SEV-SNP and Intel's TDX, but those technologies are extremely complex (understatement), difficult to debug, don't support running as nested guests, and require hardware that's isn't universally accessible. I.e. relying SEV-SNP or TDX for maintaining guest private memory isn't a realistic option. At the very least, KVM_X86_SW_PROTECTED_VM will enable a variety of selftests for guest_memfd and private memory support without requiring unique hardware. Signed-off-by: Sean Christopherson Reviewed-by: Paolo Bonzini Message-Id: <20231027182217.3615211-24-seanjc@google.com> Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/api.rst | 32 ++++++++++++++++++++++++++++++++ arch/x86/include/asm/kvm_host.h | 15 +++++++++------ arch/x86/include/uapi/asm/kvm.h | 3 +++ arch/x86/kvm/Kconfig | 12 ++++++++++++ arch/x86/kvm/mmu/mmu_internal.h | 1 + arch/x86/kvm/x86.c | 16 +++++++++++++++- include/uapi/linux/kvm.h | 1 + virt/kvm/Kconfig | 5 +++++ 8 files changed, 78 insertions(+), 7 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 726c87c35d57..926241e23aeb 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -147,10 +147,29 @@ described as 'basic' will be available. The new VM has no virtual cpus and no memory. You probably want to use 0 as machine type. +X86: +^^^^ + +Supported X86 VM types can be queried via KVM_CAP_VM_TYPES. + +S390: +^^^^^ + In order to create user controlled virtual machines on S390, check KVM_CAP_S390_UCONTROL and use the flag KVM_VM_S390_UCONTROL as privileged user (CAP_SYS_ADMIN). +MIPS: +^^^^^ + +To use hardware assisted virtualization on MIPS (VZ ASE) rather than +the default trap & emulate implementation (which changes the virtual +memory layout to fit in user mode), check KVM_CAP_MIPS_VZ and use the +flag KVM_VM_MIPS_VZ. + +ARM64: +^^^^^^ + On arm64, the physical address size for a VM (IPA Size limit) is limited to 40bits by default. The limit can be configured if the host supports the extension KVM_CAP_ARM_VM_IPA_SIZE. When supported, use @@ -8765,6 +8784,19 @@ block sizes is exposed in KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES as a 64-bit bitmap (each bit describing a block size). The default value is 0, to disable the eager page splitting. +8.41 KVM_CAP_VM_TYPES +--------------------- + +:Capability: KVM_CAP_MEMORY_ATTRIBUTES +:Architectures: x86 +:Type: system ioctl + +This capability returns a bitmap of support VM types. The 1-setting of bit @n +means the VM type with value @n is supported. Possible values of @n are:: + + #define KVM_X86_DEFAULT_VM 0 + #define KVM_X86_SW_PROTECTED_VM 1 + 9. Known KVM API problems ========================= diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 75ab0da06e64..a565a2e70f30 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1255,6 +1255,7 @@ enum kvm_apicv_inhibit { }; struct kvm_arch { + unsigned long vm_type; unsigned long n_used_mmu_pages; unsigned long n_requested_mmu_pages; unsigned long n_max_mmu_pages; @@ -2089,6 +2090,12 @@ void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd); void kvm_configure_mmu(bool enable_tdp, int tdp_forced_root_level, int tdp_max_root_level, int tdp_huge_page_level); +#ifdef CONFIG_KVM_PRIVATE_MEM +#define kvm_arch_has_private_mem(kvm) ((kvm)->arch.vm_type != KVM_X86_DEFAULT_VM) +#else +#define kvm_arch_has_private_mem(kvm) false +#endif + static inline u16 kvm_read_ldt(void) { u16 ldt; @@ -2137,14 +2144,10 @@ enum { #define HF_SMM_INSIDE_NMI_MASK (1 << 2) # define KVM_MAX_NR_ADDRESS_SPACES 2 +/* SMM is currently unsupported for guests with private memory. */ +# define kvm_arch_nr_memslot_as_ids(kvm) (kvm_arch_has_private_mem(kvm) ? 1 : 2) # define kvm_arch_vcpu_memslots_id(vcpu) ((vcpu)->arch.hflags & HF_SMM_MASK ? 1 : 0) # define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, (role).smm) - -static inline int kvm_arch_nr_memslot_as_ids(struct kvm *kvm) -{ - return KVM_MAX_NR_ADDRESS_SPACES; -} - #else # define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, 0) #endif diff --git a/arch/x86/include/uapi/asm/kvm.h b/arch/x86/include/uapi/asm/kvm.h index 1a6a1f987949..a448d0964fc0 100644 --- a/arch/x86/include/uapi/asm/kvm.h +++ b/arch/x86/include/uapi/asm/kvm.h @@ -562,4 +562,7 @@ struct kvm_pmu_event_filter { /* x86-specific KVM_EXIT_HYPERCALL flags. */ #define KVM_EXIT_HYPERCALL_LONG_MODE BIT(0) +#define KVM_X86_DEFAULT_VM 0 +#define KVM_X86_SW_PROTECTED_VM 1 + #endif /* _ASM_X86_KVM_H */ diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index e61383674c75..c1716e83d176 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -77,6 +77,18 @@ config KVM_WERROR If in doubt, say "N". +config KVM_SW_PROTECTED_VM + bool "Enable support for KVM software-protected VMs" + depends on EXPERT + depends on X86_64 + select KVM_GENERIC_PRIVATE_MEM + help + Enable support for KVM software-protected VMs. Currently "protected" + means the VM can be backed with memory provided by + KVM_CREATE_GUEST_MEMFD. + + If unsure, say "N". + config KVM_INTEL tristate "KVM for Intel (and compatible) processors support" depends on KVM && IA32_FEAT_CTL diff --git a/arch/x86/kvm/mmu/mmu_internal.h b/arch/x86/kvm/mmu/mmu_internal.h index 86c7cb692786..b66a7d47e0e4 100644 --- a/arch/x86/kvm/mmu/mmu_internal.h +++ b/arch/x86/kvm/mmu/mmu_internal.h @@ -297,6 +297,7 @@ static inline int kvm_mmu_do_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, .max_level = KVM_MAX_HUGEPAGE_LEVEL, .req_level = PG_LEVEL_4K, .goal_level = PG_LEVEL_4K, + .is_private = kvm_mem_is_private(vcpu->kvm, cr2_or_gpa >> PAGE_SHIFT), }; int r; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index f521c97f5c64..6d0772b47041 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -4548,6 +4548,13 @@ static int kvm_ioctl_get_supported_hv_cpuid(struct kvm_vcpu *vcpu, return 0; } +static bool kvm_is_vm_type_supported(unsigned long type) +{ + return type == KVM_X86_DEFAULT_VM || + (type == KVM_X86_SW_PROTECTED_VM && + IS_ENABLED(CONFIG_KVM_SW_PROTECTED_VM) && tdp_enabled); +} + int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) { int r = 0; @@ -4739,6 +4746,11 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_X86_NOTIFY_VMEXIT: r = kvm_caps.has_notify_vmexit; break; + case KVM_CAP_VM_TYPES: + r = BIT(KVM_X86_DEFAULT_VM); + if (kvm_is_vm_type_supported(KVM_X86_SW_PROTECTED_VM)) + r |= BIT(KVM_X86_SW_PROTECTED_VM); + break; default: break; } @@ -12436,9 +12448,11 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) int ret; unsigned long flags; - if (type) + if (!kvm_is_vm_type_supported(type)) return -EINVAL; + kvm->arch.vm_type = type; + ret = kvm_page_track_init(kvm); if (ret) goto out; diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 8eb10f560c69..e9cb2df67a1d 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1227,6 +1227,7 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_MEMORY_FAULT_INFO 232 #define KVM_CAP_MEMORY_ATTRIBUTES 233 #define KVM_CAP_GUEST_MEMFD 234 +#define KVM_CAP_VM_TYPES 235 #ifdef KVM_CAP_IRQ_ROUTING diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig index 08afef022db9..2c964586aa14 100644 --- a/virt/kvm/Kconfig +++ b/virt/kvm/Kconfig @@ -104,3 +104,8 @@ config KVM_GENERIC_MEMORY_ATTRIBUTES config KVM_PRIVATE_MEM select XARRAY_MULTI bool + +config KVM_GENERIC_PRIVATE_MEM + select KVM_GENERIC_MEMORY_ATTRIBUTES + select KVM_PRIVATE_MEM + bool From 335869c3f2b881bda6eeeb2df342841a1db3310b Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 27 Oct 2023 11:22:06 -0700 Subject: [PATCH 023/336] KVM: selftests: Drop unused kvm_userspace_memory_region_find() helper Drop kvm_userspace_memory_region_find(), it's unused and a terrible API (probably why it's unused). If anything outside of kvm_util.c needs to get at the memslot, userspace_mem_region_find() can be exposed to give others full access to all memory region/slot information. Signed-off-by: Sean Christopherson Message-Id: <20231027182217.3615211-25-seanjc@google.com> Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Signed-off-by: Paolo Bonzini --- .../selftests/kvm/include/kvm_util_base.h | 4 --- tools/testing/selftests/kvm/lib/kvm_util.c | 29 ------------------- 2 files changed, 33 deletions(-) diff --git a/tools/testing/selftests/kvm/include/kvm_util_base.h b/tools/testing/selftests/kvm/include/kvm_util_base.h index a18db6a7b3cf..967eaaeacd75 100644 --- a/tools/testing/selftests/kvm/include/kvm_util_base.h +++ b/tools/testing/selftests/kvm/include/kvm_util_base.h @@ -776,10 +776,6 @@ vm_adjust_num_guest_pages(enum vm_guest_mode mode, unsigned int num_guest_pages) return n; } -struct kvm_userspace_memory_region * -kvm_userspace_memory_region_find(struct kvm_vm *vm, uint64_t start, - uint64_t end); - #define sync_global_to_guest(vm, g) ({ \ typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g)); \ memcpy(_p, &(g), sizeof(g)); \ diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 7a8af1821f5d..f09295d56c23 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -590,35 +590,6 @@ userspace_mem_region_find(struct kvm_vm *vm, uint64_t start, uint64_t end) return NULL; } -/* - * KVM Userspace Memory Region Find - * - * Input Args: - * vm - Virtual Machine - * start - Starting VM physical address - * end - Ending VM physical address, inclusive. - * - * Output Args: None - * - * Return: - * Pointer to overlapping region, NULL if no such region. - * - * Public interface to userspace_mem_region_find. Allows tests to look up - * the memslot datastructure for a given range of guest physical memory. - */ -struct kvm_userspace_memory_region * -kvm_userspace_memory_region_find(struct kvm_vm *vm, uint64_t start, - uint64_t end) -{ - struct userspace_mem_region *region; - - region = userspace_mem_region_find(vm, start, end); - if (!region) - return NULL; - - return ®ion->region; -} - __weak void vcpu_arch_free(struct kvm_vcpu *vcpu) { From 8d99e347c097ab3f9fb93d0f88dddf20051d7c88 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 27 Oct 2023 11:22:07 -0700 Subject: [PATCH 024/336] KVM: selftests: Convert lib's mem regions to KVM_SET_USER_MEMORY_REGION2 Use KVM_SET_USER_MEMORY_REGION2 throughout KVM's selftests library so that support for guest private memory can be added without needing an entirely separate set of helpers. Note, this obviously makes selftests backwards-incompatible with older KVM versions from this point forward. Signed-off-by: Sean Christopherson Message-Id: <20231027182217.3615211-26-seanjc@google.com> Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Signed-off-by: Paolo Bonzini --- .../selftests/kvm/include/kvm_util_base.h | 2 +- tools/testing/selftests/kvm/lib/kvm_util.c | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/tools/testing/selftests/kvm/include/kvm_util_base.h b/tools/testing/selftests/kvm/include/kvm_util_base.h index 967eaaeacd75..9f144841c2ee 100644 --- a/tools/testing/selftests/kvm/include/kvm_util_base.h +++ b/tools/testing/selftests/kvm/include/kvm_util_base.h @@ -44,7 +44,7 @@ typedef uint64_t vm_paddr_t; /* Virtual Machine (Guest) physical address */ typedef uint64_t vm_vaddr_t; /* Virtual Machine (Guest) virtual address */ struct userspace_mem_region { - struct kvm_userspace_memory_region region; + struct kvm_userspace_memory_region2 region; struct sparsebit *unused_phy_pages; int fd; off_t offset; diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index f09295d56c23..3676b37bea38 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -453,8 +453,9 @@ void kvm_vm_restart(struct kvm_vm *vmp) vm_create_irqchip(vmp); hash_for_each(vmp->regions.slot_hash, ctr, region, slot_node) { - int ret = ioctl(vmp->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region); - TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n" + int ret = ioctl(vmp->fd, KVM_SET_USER_MEMORY_REGION2, ®ion->region); + + TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION2 IOCTL failed,\n" " rc: %i errno: %i\n" " slot: %u flags: 0x%x\n" " guest_phys_addr: 0x%llx size: 0x%llx", @@ -657,7 +658,7 @@ static void __vm_mem_region_delete(struct kvm_vm *vm, } region->region.memory_size = 0; - vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION, ®ion->region); + vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, ®ion->region); sparsebit_free(®ion->unused_phy_pages); ret = munmap(region->mmap_start, region->mmap_size); @@ -1014,8 +1015,8 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm, region->region.guest_phys_addr = guest_paddr; region->region.memory_size = npages * vm->page_size; region->region.userspace_addr = (uintptr_t) region->host_mem; - ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION, ®ion->region); - TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n" + ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, ®ion->region); + TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION2 IOCTL failed,\n" " rc: %i errno: %i\n" " slot: %u flags: 0x%x\n" " guest_phys_addr: 0x%lx size: 0x%lx", @@ -1097,9 +1098,9 @@ void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags) region->region.flags = flags; - ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION, ®ion->region); + ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, ®ion->region); - TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n" + TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION2 IOCTL failed,\n" " rc: %i errno: %i slot: %u flags: 0x%x", ret, errno, slot, flags); } @@ -1127,9 +1128,9 @@ void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa) region->region.guest_phys_addr = new_gpa; - ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION, ®ion->region); + ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, ®ion->region); - TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION failed\n" + TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION2 failed\n" "ret: %i errno: %i slot: %u new_gpa: 0x%lx", ret, errno, slot, new_gpa); } From bb2968ad6c33c0902dce48ea57d58c5bb4f3c617 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 27 Oct 2023 11:22:08 -0700 Subject: [PATCH 025/336] KVM: selftests: Add support for creating private memslots Add support for creating "private" memslots via KVM_CREATE_GUEST_MEMFD and KVM_SET_USER_MEMORY_REGION2. Make vm_userspace_mem_region_add() a wrapper to its effective replacement, vm_mem_add(), so that private memslots are fully opt-in, i.e. don't require update all tests that add memory regions. Pivot on the KVM_MEM_PRIVATE flag instead of the validity of the "gmem" file descriptor so that simple tests can let vm_mem_add() do the heavy lifting of creating the guest memfd, but also allow the caller to pass in an explicit fd+offset so that fancier tests can do things like back multiple memslots with a single file. If the caller passes in a fd, dup() the fd so that (a) __vm_mem_region_delete() can close the fd associated with the memory region without needing yet another flag, and (b) so that the caller can safely close its copy of the fd without having to first destroy memslots. Co-developed-by: Ackerley Tng Signed-off-by: Ackerley Tng Signed-off-by: Sean Christopherson Message-Id: <20231027182217.3615211-27-seanjc@google.com> Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Signed-off-by: Paolo Bonzini --- .../selftests/kvm/include/kvm_util_base.h | 23 ++++++ .../testing/selftests/kvm/include/test_util.h | 5 ++ tools/testing/selftests/kvm/lib/kvm_util.c | 76 +++++++++++-------- 3 files changed, 73 insertions(+), 31 deletions(-) diff --git a/tools/testing/selftests/kvm/include/kvm_util_base.h b/tools/testing/selftests/kvm/include/kvm_util_base.h index 9f144841c2ee..9f861182c02a 100644 --- a/tools/testing/selftests/kvm/include/kvm_util_base.h +++ b/tools/testing/selftests/kvm/include/kvm_util_base.h @@ -431,6 +431,26 @@ static inline uint64_t vm_get_stat(struct kvm_vm *vm, const char *stat_name) void vm_create_irqchip(struct kvm_vm *vm); +static inline int __vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size, + uint64_t flags) +{ + struct kvm_create_guest_memfd guest_memfd = { + .size = size, + .flags = flags, + }; + + return __vm_ioctl(vm, KVM_CREATE_GUEST_MEMFD, &guest_memfd); +} + +static inline int vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size, + uint64_t flags) +{ + int fd = __vm_create_guest_memfd(vm, size, flags); + + TEST_ASSERT(fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_GUEST_MEMFD, fd)); + return fd; +} + void vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags, uint64_t gpa, uint64_t size, void *hva); int __vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags, @@ -439,6 +459,9 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, uint64_t guest_paddr, uint32_t slot, uint64_t npages, uint32_t flags); +void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, + uint64_t guest_paddr, uint32_t slot, uint64_t npages, + uint32_t flags, int guest_memfd_fd, uint64_t guest_memfd_offset); void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags); void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa); diff --git a/tools/testing/selftests/kvm/include/test_util.h b/tools/testing/selftests/kvm/include/test_util.h index 7e614adc6cf4..7257f2243ab9 100644 --- a/tools/testing/selftests/kvm/include/test_util.h +++ b/tools/testing/selftests/kvm/include/test_util.h @@ -142,6 +142,11 @@ static inline bool backing_src_is_shared(enum vm_mem_backing_src_type t) return vm_mem_backing_src_alias(t)->flag & MAP_SHARED; } +static inline bool backing_src_can_be_huge(enum vm_mem_backing_src_type t) +{ + return t != VM_MEM_SRC_ANONYMOUS && t != VM_MEM_SRC_SHMEM; +} + /* Aligns x up to the next multiple of size. Size must be a power of 2. */ static inline uint64_t align_up(uint64_t x, uint64_t size) { diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 3676b37bea38..b63500fca627 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -669,6 +669,8 @@ static void __vm_mem_region_delete(struct kvm_vm *vm, TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("munmap()", ret)); close(region->fd); } + if (region->region.guest_memfd >= 0) + close(region->region.guest_memfd); free(region); } @@ -870,36 +872,15 @@ void vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags, errno, strerror(errno)); } -/* - * VM Userspace Memory Region Add - * - * Input Args: - * vm - Virtual Machine - * src_type - Storage source for this region. - * NULL to use anonymous memory. - * guest_paddr - Starting guest physical address - * slot - KVM region slot - * npages - Number of physical pages - * flags - KVM memory region flags (e.g. KVM_MEM_LOG_DIRTY_PAGES) - * - * Output Args: None - * - * Return: None - * - * Allocates a memory area of the number of pages specified by npages - * and maps it to the VM specified by vm, at a starting physical address - * given by guest_paddr. The region is created with a KVM region slot - * given by slot, which must be unique and < KVM_MEM_SLOTS_NUM. The - * region is created with the flags given by flags. - */ -void vm_userspace_mem_region_add(struct kvm_vm *vm, - enum vm_mem_backing_src_type src_type, - uint64_t guest_paddr, uint32_t slot, uint64_t npages, - uint32_t flags) +/* FIXME: This thing needs to be ripped apart and rewritten. */ +void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, + uint64_t guest_paddr, uint32_t slot, uint64_t npages, + uint32_t flags, int guest_memfd, uint64_t guest_memfd_offset) { int ret; struct userspace_mem_region *region; size_t backing_src_pagesz = get_backing_src_pagesz(src_type); + size_t mem_size = npages * vm->page_size; size_t alignment; TEST_ASSERT(vm_adjust_num_guest_pages(vm->mode, npages) == npages, @@ -952,7 +933,7 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm, /* Allocate and initialize new mem region structure. */ region = calloc(1, sizeof(*region)); TEST_ASSERT(region != NULL, "Insufficient Memory"); - region->mmap_size = npages * vm->page_size; + region->mmap_size = mem_size; #ifdef __s390x__ /* On s390x, the host address must be aligned to 1M (due to PGSTEs) */ @@ -999,14 +980,38 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm, /* As needed perform madvise */ if ((src_type == VM_MEM_SRC_ANONYMOUS || src_type == VM_MEM_SRC_ANONYMOUS_THP) && thp_configured()) { - ret = madvise(region->host_mem, npages * vm->page_size, + ret = madvise(region->host_mem, mem_size, src_type == VM_MEM_SRC_ANONYMOUS ? MADV_NOHUGEPAGE : MADV_HUGEPAGE); TEST_ASSERT(ret == 0, "madvise failed, addr: %p length: 0x%lx src_type: %s", - region->host_mem, npages * vm->page_size, + region->host_mem, mem_size, vm_mem_backing_src_alias(src_type)->name); } region->backing_src_type = src_type; + + if (flags & KVM_MEM_GUEST_MEMFD) { + if (guest_memfd < 0) { + uint32_t guest_memfd_flags = 0; + TEST_ASSERT(!guest_memfd_offset, + "Offset must be zero when creating new guest_memfd"); + guest_memfd = vm_create_guest_memfd(vm, mem_size, guest_memfd_flags); + } else { + /* + * Install a unique fd for each memslot so that the fd + * can be closed when the region is deleted without + * needing to track if the fd is owned by the framework + * or by the caller. + */ + guest_memfd = dup(guest_memfd); + TEST_ASSERT(guest_memfd >= 0, __KVM_SYSCALL_ERROR("dup()", guest_memfd)); + } + + region->region.guest_memfd = guest_memfd; + region->region.guest_memfd_offset = guest_memfd_offset; + } else { + region->region.guest_memfd = -1; + } + region->unused_phy_pages = sparsebit_alloc(); sparsebit_set_num(region->unused_phy_pages, guest_paddr >> vm->page_shift, npages); @@ -1019,9 +1024,10 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm, TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION2 IOCTL failed,\n" " rc: %i errno: %i\n" " slot: %u flags: 0x%x\n" - " guest_phys_addr: 0x%lx size: 0x%lx", + " guest_phys_addr: 0x%lx size: 0x%lx guest_memfd: %d\n", ret, errno, slot, flags, - guest_paddr, (uint64_t) region->region.memory_size); + guest_paddr, (uint64_t) region->region.memory_size, + region->region.guest_memfd); /* Add to quick lookup data structures */ vm_userspace_mem_region_gpa_insert(&vm->regions.gpa_tree, region); @@ -1042,6 +1048,14 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm, } } +void vm_userspace_mem_region_add(struct kvm_vm *vm, + enum vm_mem_backing_src_type src_type, + uint64_t guest_paddr, uint32_t slot, + uint64_t npages, uint32_t flags) +{ + vm_mem_add(vm, src_type, guest_paddr, slot, npages, flags, -1, 0); +} + /* * Memslot to region * From f7fa67495d118f734f98b406fd46888616b4a3c3 Mon Sep 17 00:00:00 2001 From: Vishal Annapurve Date: Fri, 27 Oct 2023 11:22:09 -0700 Subject: [PATCH 026/336] KVM: selftests: Add helpers to convert guest memory b/w private and shared Add helpers to convert memory between private and shared via KVM's memory attributes, as well as helpers to free/allocate guest_memfd memory via fallocate(). Userspace, i.e. tests, is NOT required to do fallocate() when converting memory, as the attributes are the single source of truth. Provide allocate() helpers so that tests can mimic a userspace that frees private memory on conversion, e.g. to prioritize memory usage over performance. Signed-off-by: Vishal Annapurve Co-developed-by: Sean Christopherson Signed-off-by: Sean Christopherson Message-Id: <20231027182217.3615211-28-seanjc@google.com> Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba --- .../selftests/kvm/include/kvm_util_base.h | 48 +++++++++++++++++++ tools/testing/selftests/kvm/lib/kvm_util.c | 28 +++++++++++ 2 files changed, 76 insertions(+) diff --git a/tools/testing/selftests/kvm/include/kvm_util_base.h b/tools/testing/selftests/kvm/include/kvm_util_base.h index 9f861182c02a..1441fca6c273 100644 --- a/tools/testing/selftests/kvm/include/kvm_util_base.h +++ b/tools/testing/selftests/kvm/include/kvm_util_base.h @@ -333,6 +333,54 @@ static inline void vm_enable_cap(struct kvm_vm *vm, uint32_t cap, uint64_t arg0) vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap); } +static inline void vm_set_memory_attributes(struct kvm_vm *vm, uint64_t gpa, + uint64_t size, uint64_t attributes) +{ + struct kvm_memory_attributes attr = { + .attributes = attributes, + .address = gpa, + .size = size, + .flags = 0, + }; + + /* + * KVM_SET_MEMORY_ATTRIBUTES overwrites _all_ attributes. These flows + * need significant enhancements to support multiple attributes. + */ + TEST_ASSERT(!attributes || attributes == KVM_MEMORY_ATTRIBUTE_PRIVATE, + "Update me to support multiple attributes!"); + + vm_ioctl(vm, KVM_SET_MEMORY_ATTRIBUTES, &attr); +} + + +static inline void vm_mem_set_private(struct kvm_vm *vm, uint64_t gpa, + uint64_t size) +{ + vm_set_memory_attributes(vm, gpa, size, KVM_MEMORY_ATTRIBUTE_PRIVATE); +} + +static inline void vm_mem_set_shared(struct kvm_vm *vm, uint64_t gpa, + uint64_t size) +{ + vm_set_memory_attributes(vm, gpa, size, 0); +} + +void vm_guest_mem_fallocate(struct kvm_vm *vm, uint64_t gpa, uint64_t size, + bool punch_hole); + +static inline void vm_guest_mem_punch_hole(struct kvm_vm *vm, uint64_t gpa, + uint64_t size) +{ + vm_guest_mem_fallocate(vm, gpa, size, true); +} + +static inline void vm_guest_mem_allocate(struct kvm_vm *vm, uint64_t gpa, + uint64_t size) +{ + vm_guest_mem_fallocate(vm, gpa, size, false); +} + void vm_enable_dirty_ring(struct kvm_vm *vm, uint32_t ring_size); const char *vm_guest_mode_string(uint32_t i); diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index b63500fca627..cc6007abc4b1 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -1167,6 +1167,34 @@ void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot) __vm_mem_region_delete(vm, memslot2region(vm, slot), true); } +void vm_guest_mem_fallocate(struct kvm_vm *vm, uint64_t base, uint64_t size, + bool punch_hole) +{ + const int mode = FALLOC_FL_KEEP_SIZE | (punch_hole ? FALLOC_FL_PUNCH_HOLE : 0); + struct userspace_mem_region *region; + uint64_t end = base + size; + uint64_t gpa, len; + off_t fd_offset; + int ret; + + for (gpa = base; gpa < end; gpa += len) { + uint64_t offset; + + region = userspace_mem_region_find(vm, gpa, gpa); + TEST_ASSERT(region && region->region.flags & KVM_MEM_GUEST_MEMFD, + "Private memory region not found for GPA 0x%lx", gpa); + + offset = gpa - region->region.guest_phys_addr; + fd_offset = region->region.guest_memfd_offset + offset; + len = min_t(uint64_t, end - gpa, region->region.memory_size - offset); + + ret = fallocate(region->region.guest_memfd, mode, fd_offset, len); + TEST_ASSERT(!ret, "fallocate() failed to %s at %lx (len = %lu), fd = %d, mode = %x, offset = %lx\n", + punch_hole ? "punch hole" : "allocate", gpa, len, + region->region.guest_memfd, mode, fd_offset); + } +} + /* Returns the size of a vCPU's kvm_run structure. */ static int vcpu_mmap_sz(void) { From 01244fce2fa22176e46609c051f6fe15cf81e188 Mon Sep 17 00:00:00 2001 From: Vishal Annapurve Date: Fri, 27 Oct 2023 11:22:10 -0700 Subject: [PATCH 027/336] KVM: selftests: Add helpers to do KVM_HC_MAP_GPA_RANGE hypercalls (x86) Add helpers for x86 guests to invoke the KVM_HC_MAP_GPA_RANGE hypercall, which KVM will forward to userspace and thus can be used by tests to coordinate private<=>shared conversions between host userspace code and guest code. Signed-off-by: Vishal Annapurve [sean: drop shared/private helpers (let tests specify flags)] Signed-off-by: Sean Christopherson Message-Id: <20231027182217.3615211-29-seanjc@google.com> Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Signed-off-by: Paolo Bonzini --- .../selftests/kvm/include/x86_64/processor.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tools/testing/selftests/kvm/include/x86_64/processor.h b/tools/testing/selftests/kvm/include/x86_64/processor.h index 25bc61dac5fb..a84863503fcb 100644 --- a/tools/testing/selftests/kvm/include/x86_64/processor.h +++ b/tools/testing/selftests/kvm/include/x86_64/processor.h @@ -15,6 +15,7 @@ #include #include +#include #include #include "../kvm_util.h" @@ -1194,6 +1195,20 @@ uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2, uint64_t __xen_hypercall(uint64_t nr, uint64_t a0, void *a1); void xen_hypercall(uint64_t nr, uint64_t a0, void *a1); +static inline uint64_t __kvm_hypercall_map_gpa_range(uint64_t gpa, + uint64_t size, uint64_t flags) +{ + return kvm_hypercall(KVM_HC_MAP_GPA_RANGE, gpa, size >> PAGE_SHIFT, flags, 0); +} + +static inline void kvm_hypercall_map_gpa_range(uint64_t gpa, uint64_t size, + uint64_t flags) +{ + uint64_t ret = __kvm_hypercall_map_gpa_range(gpa, size, flags); + + GUEST_ASSERT(!ret); +} + void __vm_xsave_require_permission(uint64_t xfeature, const char *name); #define vm_xsave_require_permission(xfeature) \ From 672eaa351015d8165c41fff644ee7d2b369cea12 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 27 Oct 2023 11:22:11 -0700 Subject: [PATCH 028/336] KVM: selftests: Introduce VM "shape" to allow tests to specify the VM type Add a "vm_shape" structure to encapsulate the selftests-defined "mode", along with the KVM-defined "type" for use when creating a new VM. "mode" tracks physical and virtual address properties, as well as the preferred backing memory type, while "type" corresponds to the VM type. Taking the VM type will allow adding tests for KVM_CREATE_GUEST_MEMFD without needing an entirely separate set of helpers. At this time, guest_memfd is effectively usable only by confidential VM types in the form of guest private memory, and it's expected that x86 will double down and require unique VM types for TDX and SNP guests. Signed-off-by: Sean Christopherson Message-Id: <20231027182217.3615211-30-seanjc@google.com> Signed-off-by: Paolo Bonzini --- .../selftests/kvm/aarch64/page_fault_test.c | 2 +- tools/testing/selftests/kvm/dirty_log_test.c | 2 +- .../selftests/kvm/include/kvm_util_base.h | 54 +++++++++++++++---- .../selftests/kvm/kvm_page_table_test.c | 2 +- tools/testing/selftests/kvm/lib/kvm_util.c | 43 +++++++-------- tools/testing/selftests/kvm/lib/memstress.c | 3 +- .../kvm/x86_64/ucna_injection_test.c | 2 +- 7 files changed, 73 insertions(+), 35 deletions(-) diff --git a/tools/testing/selftests/kvm/aarch64/page_fault_test.c b/tools/testing/selftests/kvm/aarch64/page_fault_test.c index eb4217b7c768..08a5ca5bed56 100644 --- a/tools/testing/selftests/kvm/aarch64/page_fault_test.c +++ b/tools/testing/selftests/kvm/aarch64/page_fault_test.c @@ -705,7 +705,7 @@ static void run_test(enum vm_guest_mode mode, void *arg) print_test_banner(mode, p); - vm = ____vm_create(mode); + vm = ____vm_create(VM_SHAPE(mode)); setup_memslots(vm, p); kvm_vm_elf_load(vm, program_invocation_name); setup_ucall(vm); diff --git a/tools/testing/selftests/kvm/dirty_log_test.c b/tools/testing/selftests/kvm/dirty_log_test.c index 936f3a8d1b83..6cbecf499767 100644 --- a/tools/testing/selftests/kvm/dirty_log_test.c +++ b/tools/testing/selftests/kvm/dirty_log_test.c @@ -699,7 +699,7 @@ static struct kvm_vm *create_vm(enum vm_guest_mode mode, struct kvm_vcpu **vcpu, pr_info("Testing guest mode: %s\n", vm_guest_mode_string(mode)); - vm = __vm_create(mode, 1, extra_mem_pages); + vm = __vm_create(VM_SHAPE(mode), 1, extra_mem_pages); log_mode_create_vm_done(vm); *vcpu = vm_vcpu_add(vm, 0, guest_code); diff --git a/tools/testing/selftests/kvm/include/kvm_util_base.h b/tools/testing/selftests/kvm/include/kvm_util_base.h index 1441fca6c273..157508c071f3 100644 --- a/tools/testing/selftests/kvm/include/kvm_util_base.h +++ b/tools/testing/selftests/kvm/include/kvm_util_base.h @@ -188,6 +188,23 @@ enum vm_guest_mode { NUM_VM_MODES, }; +struct vm_shape { + enum vm_guest_mode mode; + unsigned int type; +}; + +#define VM_TYPE_DEFAULT 0 + +#define VM_SHAPE(__mode) \ +({ \ + struct vm_shape shape = { \ + .mode = (__mode), \ + .type = VM_TYPE_DEFAULT \ + }; \ + \ + shape; \ +}) + #if defined(__aarch64__) extern enum vm_guest_mode vm_mode_default; @@ -220,6 +237,8 @@ extern enum vm_guest_mode vm_mode_default; #endif +#define VM_SHAPE_DEFAULT VM_SHAPE(VM_MODE_DEFAULT) + #define MIN_PAGE_SIZE (1U << MIN_PAGE_SHIFT) #define PTES_PER_MIN_PAGE ptes_per_page(MIN_PAGE_SIZE) @@ -784,21 +803,21 @@ vm_paddr_t vm_alloc_page_table(struct kvm_vm *vm); * __vm_create() does NOT create vCPUs, @nr_runnable_vcpus is used purely to * calculate the amount of memory needed for per-vCPU data, e.g. stacks. */ -struct kvm_vm *____vm_create(enum vm_guest_mode mode); -struct kvm_vm *__vm_create(enum vm_guest_mode mode, uint32_t nr_runnable_vcpus, +struct kvm_vm *____vm_create(struct vm_shape shape); +struct kvm_vm *__vm_create(struct vm_shape shape, uint32_t nr_runnable_vcpus, uint64_t nr_extra_pages); static inline struct kvm_vm *vm_create_barebones(void) { - return ____vm_create(VM_MODE_DEFAULT); + return ____vm_create(VM_SHAPE_DEFAULT); } static inline struct kvm_vm *vm_create(uint32_t nr_runnable_vcpus) { - return __vm_create(VM_MODE_DEFAULT, nr_runnable_vcpus, 0); + return __vm_create(VM_SHAPE_DEFAULT, nr_runnable_vcpus, 0); } -struct kvm_vm *__vm_create_with_vcpus(enum vm_guest_mode mode, uint32_t nr_vcpus, +struct kvm_vm *__vm_create_with_vcpus(struct vm_shape shape, uint32_t nr_vcpus, uint64_t extra_mem_pages, void *guest_code, struct kvm_vcpu *vcpus[]); @@ -806,17 +825,27 @@ static inline struct kvm_vm *vm_create_with_vcpus(uint32_t nr_vcpus, void *guest_code, struct kvm_vcpu *vcpus[]) { - return __vm_create_with_vcpus(VM_MODE_DEFAULT, nr_vcpus, 0, + return __vm_create_with_vcpus(VM_SHAPE_DEFAULT, nr_vcpus, 0, guest_code, vcpus); } + +struct kvm_vm *__vm_create_shape_with_one_vcpu(struct vm_shape shape, + struct kvm_vcpu **vcpu, + uint64_t extra_mem_pages, + void *guest_code); + /* * Create a VM with a single vCPU with reasonable defaults and @extra_mem_pages * additional pages of guest memory. Returns the VM and vCPU (via out param). */ -struct kvm_vm *__vm_create_with_one_vcpu(struct kvm_vcpu **vcpu, - uint64_t extra_mem_pages, - void *guest_code); +static inline struct kvm_vm *__vm_create_with_one_vcpu(struct kvm_vcpu **vcpu, + uint64_t extra_mem_pages, + void *guest_code) +{ + return __vm_create_shape_with_one_vcpu(VM_SHAPE_DEFAULT, vcpu, + extra_mem_pages, guest_code); +} static inline struct kvm_vm *vm_create_with_one_vcpu(struct kvm_vcpu **vcpu, void *guest_code) @@ -824,6 +853,13 @@ static inline struct kvm_vm *vm_create_with_one_vcpu(struct kvm_vcpu **vcpu, return __vm_create_with_one_vcpu(vcpu, 0, guest_code); } +static inline struct kvm_vm *vm_create_shape_with_one_vcpu(struct vm_shape shape, + struct kvm_vcpu **vcpu, + void *guest_code) +{ + return __vm_create_shape_with_one_vcpu(shape, vcpu, 0, guest_code); +} + struct kvm_vcpu *vm_recreate_with_one_vcpu(struct kvm_vm *vm); void kvm_pin_this_task_to_pcpu(uint32_t pcpu); diff --git a/tools/testing/selftests/kvm/kvm_page_table_test.c b/tools/testing/selftests/kvm/kvm_page_table_test.c index 69f26d80c821..e37dc9c21888 100644 --- a/tools/testing/selftests/kvm/kvm_page_table_test.c +++ b/tools/testing/selftests/kvm/kvm_page_table_test.c @@ -254,7 +254,7 @@ static struct kvm_vm *pre_init_before_test(enum vm_guest_mode mode, void *arg) /* Create a VM with enough guest pages */ guest_num_pages = test_mem_size / guest_page_size; - vm = __vm_create_with_vcpus(mode, nr_vcpus, guest_num_pages, + vm = __vm_create_with_vcpus(VM_SHAPE(mode), nr_vcpus, guest_num_pages, guest_code, test_args.vcpus); /* Align down GPA of the testing memslot */ diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index cc6007abc4b1..bf15635eda11 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -209,7 +209,7 @@ __weak void vm_vaddr_populate_bitmap(struct kvm_vm *vm) (1ULL << (vm->va_bits - 1)) >> vm->page_shift); } -struct kvm_vm *____vm_create(enum vm_guest_mode mode) +struct kvm_vm *____vm_create(struct vm_shape shape) { struct kvm_vm *vm; @@ -221,13 +221,13 @@ struct kvm_vm *____vm_create(enum vm_guest_mode mode) vm->regions.hva_tree = RB_ROOT; hash_init(vm->regions.slot_hash); - vm->mode = mode; - vm->type = 0; + vm->mode = shape.mode; + vm->type = shape.type; - vm->pa_bits = vm_guest_mode_params[mode].pa_bits; - vm->va_bits = vm_guest_mode_params[mode].va_bits; - vm->page_size = vm_guest_mode_params[mode].page_size; - vm->page_shift = vm_guest_mode_params[mode].page_shift; + vm->pa_bits = vm_guest_mode_params[vm->mode].pa_bits; + vm->va_bits = vm_guest_mode_params[vm->mode].va_bits; + vm->page_size = vm_guest_mode_params[vm->mode].page_size; + vm->page_shift = vm_guest_mode_params[vm->mode].page_shift; /* Setup mode specific traits. */ switch (vm->mode) { @@ -265,7 +265,7 @@ struct kvm_vm *____vm_create(enum vm_guest_mode mode) /* * Ignore KVM support for 5-level paging (vm->va_bits == 57), * it doesn't take effect unless a CR4.LA57 is set, which it - * isn't for this VM_MODE. + * isn't for this mode (48-bit virtual address space). */ TEST_ASSERT(vm->va_bits == 48 || vm->va_bits == 57, "Linear address width (%d bits) not supported", @@ -285,10 +285,11 @@ struct kvm_vm *____vm_create(enum vm_guest_mode mode) vm->pgtable_levels = 5; break; default: - TEST_FAIL("Unknown guest mode, mode: 0x%x", mode); + TEST_FAIL("Unknown guest mode: 0x%x", vm->mode); } #ifdef __aarch64__ + TEST_ASSERT(!vm->type, "ARM doesn't support test-provided types"); if (vm->pa_bits != 40) vm->type = KVM_VM_TYPE_ARM_IPA_SIZE(vm->pa_bits); #endif @@ -347,19 +348,19 @@ static uint64_t vm_nr_pages_required(enum vm_guest_mode mode, return vm_adjust_num_guest_pages(mode, nr_pages); } -struct kvm_vm *__vm_create(enum vm_guest_mode mode, uint32_t nr_runnable_vcpus, +struct kvm_vm *__vm_create(struct vm_shape shape, uint32_t nr_runnable_vcpus, uint64_t nr_extra_pages) { - uint64_t nr_pages = vm_nr_pages_required(mode, nr_runnable_vcpus, + uint64_t nr_pages = vm_nr_pages_required(shape.mode, nr_runnable_vcpus, nr_extra_pages); struct userspace_mem_region *slot0; struct kvm_vm *vm; int i; - pr_debug("%s: mode='%s' pages='%ld'\n", __func__, - vm_guest_mode_string(mode), nr_pages); + pr_debug("%s: mode='%s' type='%d', pages='%ld'\n", __func__, + vm_guest_mode_string(shape.mode), shape.type, nr_pages); - vm = ____vm_create(mode); + vm = ____vm_create(shape); vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0, 0, nr_pages, 0); for (i = 0; i < NR_MEM_REGIONS; i++) @@ -400,7 +401,7 @@ struct kvm_vm *__vm_create(enum vm_guest_mode mode, uint32_t nr_runnable_vcpus, * extra_mem_pages is only used to calculate the maximum page table size, * no real memory allocation for non-slot0 memory in this function. */ -struct kvm_vm *__vm_create_with_vcpus(enum vm_guest_mode mode, uint32_t nr_vcpus, +struct kvm_vm *__vm_create_with_vcpus(struct vm_shape shape, uint32_t nr_vcpus, uint64_t extra_mem_pages, void *guest_code, struct kvm_vcpu *vcpus[]) { @@ -409,7 +410,7 @@ struct kvm_vm *__vm_create_with_vcpus(enum vm_guest_mode mode, uint32_t nr_vcpus TEST_ASSERT(!nr_vcpus || vcpus, "Must provide vCPU array"); - vm = __vm_create(mode, nr_vcpus, extra_mem_pages); + vm = __vm_create(shape, nr_vcpus, extra_mem_pages); for (i = 0; i < nr_vcpus; ++i) vcpus[i] = vm_vcpu_add(vm, i, guest_code); @@ -417,15 +418,15 @@ struct kvm_vm *__vm_create_with_vcpus(enum vm_guest_mode mode, uint32_t nr_vcpus return vm; } -struct kvm_vm *__vm_create_with_one_vcpu(struct kvm_vcpu **vcpu, - uint64_t extra_mem_pages, - void *guest_code) +struct kvm_vm *__vm_create_shape_with_one_vcpu(struct vm_shape shape, + struct kvm_vcpu **vcpu, + uint64_t extra_mem_pages, + void *guest_code) { struct kvm_vcpu *vcpus[1]; struct kvm_vm *vm; - vm = __vm_create_with_vcpus(VM_MODE_DEFAULT, 1, extra_mem_pages, - guest_code, vcpus); + vm = __vm_create_with_vcpus(shape, 1, extra_mem_pages, guest_code, vcpus); *vcpu = vcpus[0]; return vm; diff --git a/tools/testing/selftests/kvm/lib/memstress.c b/tools/testing/selftests/kvm/lib/memstress.c index df457452d146..d05487e5a371 100644 --- a/tools/testing/selftests/kvm/lib/memstress.c +++ b/tools/testing/selftests/kvm/lib/memstress.c @@ -168,7 +168,8 @@ struct kvm_vm *memstress_create_vm(enum vm_guest_mode mode, int nr_vcpus, * The memory is also added to memslot 0, but that's a benign side * effect as KVM allows aliasing HVAs in meslots. */ - vm = __vm_create_with_vcpus(mode, nr_vcpus, slot0_pages + guest_num_pages, + vm = __vm_create_with_vcpus(VM_SHAPE(mode), nr_vcpus, + slot0_pages + guest_num_pages, memstress_guest_code, vcpus); args->vm = vm; diff --git a/tools/testing/selftests/kvm/x86_64/ucna_injection_test.c b/tools/testing/selftests/kvm/x86_64/ucna_injection_test.c index 85f34ca7e49e..0ed32ec903d0 100644 --- a/tools/testing/selftests/kvm/x86_64/ucna_injection_test.c +++ b/tools/testing/selftests/kvm/x86_64/ucna_injection_test.c @@ -271,7 +271,7 @@ int main(int argc, char *argv[]) kvm_check_cap(KVM_CAP_MCE); - vm = __vm_create(VM_MODE_DEFAULT, 3, 0); + vm = __vm_create(VM_SHAPE_DEFAULT, 3, 0); kvm_ioctl(vm->kvm_fd, KVM_X86_GET_MCE_CAP_SUPPORTED, &supported_mcg_caps); From 242331dfc4959f44e706c9b09b768f312aa5e117 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 27 Oct 2023 11:22:12 -0700 Subject: [PATCH 029/336] KVM: selftests: Add GUEST_SYNC[1-6] macros for synchronizing more data Add GUEST_SYNC[1-6]() so that tests can pass the maximum amount of information supported via ucall(), without needing to resort to shared memory. Signed-off-by: Sean Christopherson Message-Id: <20231027182217.3615211-31-seanjc@google.com> Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Signed-off-by: Paolo Bonzini --- tools/testing/selftests/kvm/include/ucall_common.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tools/testing/selftests/kvm/include/ucall_common.h b/tools/testing/selftests/kvm/include/ucall_common.h index ce33d306c2cb..0fb472a5a058 100644 --- a/tools/testing/selftests/kvm/include/ucall_common.h +++ b/tools/testing/selftests/kvm/include/ucall_common.h @@ -52,6 +52,17 @@ int ucall_nr_pages_required(uint64_t page_size); #define GUEST_SYNC_ARGS(stage, arg1, arg2, arg3, arg4) \ ucall(UCALL_SYNC, 6, "hello", stage, arg1, arg2, arg3, arg4) #define GUEST_SYNC(stage) ucall(UCALL_SYNC, 2, "hello", stage) +#define GUEST_SYNC1(arg0) ucall(UCALL_SYNC, 1, arg0) +#define GUEST_SYNC2(arg0, arg1) ucall(UCALL_SYNC, 2, arg0, arg1) +#define GUEST_SYNC3(arg0, arg1, arg2) \ + ucall(UCALL_SYNC, 3, arg0, arg1, arg2) +#define GUEST_SYNC4(arg0, arg1, arg2, arg3) \ + ucall(UCALL_SYNC, 4, arg0, arg1, arg2, arg3) +#define GUEST_SYNC5(arg0, arg1, arg2, arg3, arg4) \ + ucall(UCALL_SYNC, 5, arg0, arg1, arg2, arg3, arg4) +#define GUEST_SYNC6(arg0, arg1, arg2, arg3, arg4, arg5) \ + ucall(UCALL_SYNC, 6, arg0, arg1, arg2, arg3, arg4, arg5) + #define GUEST_PRINTF(_fmt, _args...) ucall_fmt(UCALL_PRINTF, _fmt, ##_args) #define GUEST_DONE() ucall(UCALL_DONE, 0) From 43f623f350ce1c46c53b6b77f4dbe741af8c44f3 Mon Sep 17 00:00:00 2001 From: Vishal Annapurve Date: Fri, 27 Oct 2023 11:22:13 -0700 Subject: [PATCH 030/336] KVM: selftests: Add x86-only selftest for private memory conversions Add a selftest to exercise implicit/explicit conversion functionality within KVM and verify: - Shared memory is visible to host userspace - Private memory is not visible to host userspace - Host userspace and guest can communicate over shared memory - Data in shared backing is preserved across conversions (test's host userspace doesn't free the data) - Private memory is bound to the lifetime of the VM Ideally, KVM's selftests infrastructure would be reworked to allow backing a single region of guest memory with multiple memslots for _all_ backing types and shapes, i.e. ideally the code for using a single backing fd across multiple memslots would work for "regular" memory as well. But sadly, support for KVM_CREATE_GUEST_MEMFD has languished for far too long, and overhauling selftests' memslots infrastructure would likely open a can of worms, i.e. delay things even further. In addition to the more obvious tests, verify that PUNCH_HOLE actually frees memory. Directly verifying that KVM frees memory is impractical, if it's even possible, so instead indirectly verify memory is freed by asserting that the guest reads zeroes after a PUNCH_HOLE. E.g. if KVM zaps SPTEs but doesn't actually punch a hole in the inode, the subsequent read will still see the previous value. And obviously punching a hole shouldn't cause explosions. Let the user specify the number of memslots in the private mem conversion test, i.e. don't require the number of memslots to be '1' or "nr_vcpus". Creating more memslots than vCPUs is particularly interesting, e.g. it can result in a single KVM_SET_MEMORY_ATTRIBUTES spanning multiple memslots. To keep the math reasonable, align each vCPU's chunk to at least 2MiB (the size is 2MiB+4KiB), and require the total size to be cleanly divisible by the number of memslots. The goal is to be able to validate that KVM plays nice with multiple memslots, being able to create a truly arbitrary number of memslots doesn't add meaningful value, i.e. isn't worth the cost. Intentionally don't take a requirement on KVM_CAP_GUEST_MEMFD, KVM_CAP_MEMORY_FAULT_INFO, KVM_MEMORY_ATTRIBUTE_PRIVATE, etc., as it's a KVM bug to advertise KVM_X86_SW_PROTECTED_VM without its prerequisites. Signed-off-by: Vishal Annapurve Co-developed-by: Ackerley Tng Signed-off-by: Ackerley Tng Co-developed-by: Sean Christopherson Signed-off-by: Sean Christopherson Message-Id: <20231027182217.3615211-32-seanjc@google.com> Signed-off-by: Paolo Bonzini --- tools/testing/selftests/kvm/Makefile | 1 + .../kvm/x86_64/private_mem_conversions_test.c | 482 ++++++++++++++++++ 2 files changed, 483 insertions(+) create mode 100644 tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.c diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index a5963ab9215b..ecdea5e7afa8 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -91,6 +91,7 @@ TEST_GEN_PROGS_x86_64 += x86_64/monitor_mwait_test TEST_GEN_PROGS_x86_64 += x86_64/nested_exceptions_test TEST_GEN_PROGS_x86_64 += x86_64/platform_info_test TEST_GEN_PROGS_x86_64 += x86_64/pmu_event_filter_test +TEST_GEN_PROGS_x86_64 += x86_64/private_mem_conversions_test TEST_GEN_PROGS_x86_64 += x86_64/set_boot_cpu_id TEST_GEN_PROGS_x86_64 += x86_64/set_sregs_test TEST_GEN_PROGS_x86_64 += x86_64/smaller_maxphyaddr_emulation_test diff --git a/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.c b/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.c new file mode 100644 index 000000000000..4d6a37a5d896 --- /dev/null +++ b/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.c @@ -0,0 +1,482 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022, Google LLC. + */ +#define _GNU_SOURCE /* for program_invocation_short_name */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +#define BASE_DATA_SLOT 10 +#define BASE_DATA_GPA ((uint64_t)(1ull << 32)) +#define PER_CPU_DATA_SIZE ((uint64_t)(SZ_2M + PAGE_SIZE)) + +/* Horrific macro so that the line info is captured accurately :-( */ +#define memcmp_g(gpa, pattern, size) \ +do { \ + uint8_t *mem = (uint8_t *)gpa; \ + size_t i; \ + \ + for (i = 0; i < size; i++) \ + __GUEST_ASSERT(mem[i] == pattern, \ + "Guest expected 0x%x at offset %lu (gpa 0x%llx), got 0x%x", \ + pattern, i, gpa + i, mem[i]); \ +} while (0) + +static void memcmp_h(uint8_t *mem, uint64_t gpa, uint8_t pattern, size_t size) +{ + size_t i; + + for (i = 0; i < size; i++) + TEST_ASSERT(mem[i] == pattern, + "Host expected 0x%x at gpa 0x%lx, got 0x%x", + pattern, gpa + i, mem[i]); +} + +/* + * Run memory conversion tests with explicit conversion: + * Execute KVM hypercall to map/unmap gpa range which will cause userspace exit + * to back/unback private memory. Subsequent accesses by guest to the gpa range + * will not cause exit to userspace. + * + * Test memory conversion scenarios with following steps: + * 1) Access private memory using private access and verify that memory contents + * are not visible to userspace. + * 2) Convert memory to shared using explicit conversions and ensure that + * userspace is able to access the shared regions. + * 3) Convert memory back to private using explicit conversions and ensure that + * userspace is again not able to access converted private regions. + */ + +#define GUEST_STAGE(o, s) { .offset = o, .size = s } + +enum ucall_syncs { + SYNC_SHARED, + SYNC_PRIVATE, +}; + +static void guest_sync_shared(uint64_t gpa, uint64_t size, + uint8_t current_pattern, uint8_t new_pattern) +{ + GUEST_SYNC5(SYNC_SHARED, gpa, size, current_pattern, new_pattern); +} + +static void guest_sync_private(uint64_t gpa, uint64_t size, uint8_t pattern) +{ + GUEST_SYNC4(SYNC_PRIVATE, gpa, size, pattern); +} + +/* Arbitrary values, KVM doesn't care about the attribute flags. */ +#define MAP_GPA_SET_ATTRIBUTES BIT(0) +#define MAP_GPA_SHARED BIT(1) +#define MAP_GPA_DO_FALLOCATE BIT(2) + +static void guest_map_mem(uint64_t gpa, uint64_t size, bool map_shared, + bool do_fallocate) +{ + uint64_t flags = MAP_GPA_SET_ATTRIBUTES; + + if (map_shared) + flags |= MAP_GPA_SHARED; + if (do_fallocate) + flags |= MAP_GPA_DO_FALLOCATE; + kvm_hypercall_map_gpa_range(gpa, size, flags); +} + +static void guest_map_shared(uint64_t gpa, uint64_t size, bool do_fallocate) +{ + guest_map_mem(gpa, size, true, do_fallocate); +} + +static void guest_map_private(uint64_t gpa, uint64_t size, bool do_fallocate) +{ + guest_map_mem(gpa, size, false, do_fallocate); +} + +struct { + uint64_t offset; + uint64_t size; +} static const test_ranges[] = { + GUEST_STAGE(0, PAGE_SIZE), + GUEST_STAGE(0, SZ_2M), + GUEST_STAGE(PAGE_SIZE, PAGE_SIZE), + GUEST_STAGE(PAGE_SIZE, SZ_2M), + GUEST_STAGE(SZ_2M, PAGE_SIZE), +}; + +static void guest_test_explicit_conversion(uint64_t base_gpa, bool do_fallocate) +{ + const uint8_t def_p = 0xaa; + const uint8_t init_p = 0xcc; + uint64_t j; + int i; + + /* Memory should be shared by default. */ + memset((void *)base_gpa, def_p, PER_CPU_DATA_SIZE); + memcmp_g(base_gpa, def_p, PER_CPU_DATA_SIZE); + guest_sync_shared(base_gpa, PER_CPU_DATA_SIZE, def_p, init_p); + + memcmp_g(base_gpa, init_p, PER_CPU_DATA_SIZE); + + for (i = 0; i < ARRAY_SIZE(test_ranges); i++) { + uint64_t gpa = base_gpa + test_ranges[i].offset; + uint64_t size = test_ranges[i].size; + uint8_t p1 = 0x11; + uint8_t p2 = 0x22; + uint8_t p3 = 0x33; + uint8_t p4 = 0x44; + + /* + * Set the test region to pattern one to differentiate it from + * the data range as a whole (contains the initial pattern). + */ + memset((void *)gpa, p1, size); + + /* + * Convert to private, set and verify the private data, and + * then verify that the rest of the data (map shared) still + * holds the initial pattern, and that the host always sees the + * shared memory (initial pattern). Unlike shared memory, + * punching a hole in private memory is destructive, i.e. + * previous values aren't guaranteed to be preserved. + */ + guest_map_private(gpa, size, do_fallocate); + + if (size > PAGE_SIZE) { + memset((void *)gpa, p2, PAGE_SIZE); + goto skip; + } + + memset((void *)gpa, p2, size); + guest_sync_private(gpa, size, p1); + + /* + * Verify that the private memory was set to pattern two, and + * that shared memory still holds the initial pattern. + */ + memcmp_g(gpa, p2, size); + if (gpa > base_gpa) + memcmp_g(base_gpa, init_p, gpa - base_gpa); + if (gpa + size < base_gpa + PER_CPU_DATA_SIZE) + memcmp_g(gpa + size, init_p, + (base_gpa + PER_CPU_DATA_SIZE) - (gpa + size)); + + /* + * Convert odd-number page frames back to shared to verify KVM + * also correctly handles holes in private ranges. + */ + for (j = 0; j < size; j += PAGE_SIZE) { + if ((j >> PAGE_SHIFT) & 1) { + guest_map_shared(gpa + j, PAGE_SIZE, do_fallocate); + guest_sync_shared(gpa + j, PAGE_SIZE, p1, p3); + + memcmp_g(gpa + j, p3, PAGE_SIZE); + } else { + guest_sync_private(gpa + j, PAGE_SIZE, p1); + } + } + +skip: + /* + * Convert the entire region back to shared, explicitly write + * pattern three to fill in the even-number frames before + * asking the host to verify (and write pattern four). + */ + guest_map_shared(gpa, size, do_fallocate); + memset((void *)gpa, p3, size); + guest_sync_shared(gpa, size, p3, p4); + memcmp_g(gpa, p4, size); + + /* Reset the shared memory back to the initial pattern. */ + memset((void *)gpa, init_p, size); + + /* + * Free (via PUNCH_HOLE) *all* private memory so that the next + * iteration starts from a clean slate, e.g. with respect to + * whether or not there are pages/folios in guest_mem. + */ + guest_map_shared(base_gpa, PER_CPU_DATA_SIZE, true); + } +} + +static void guest_punch_hole(uint64_t gpa, uint64_t size) +{ + /* "Mapping" memory shared via fallocate() is done via PUNCH_HOLE. */ + uint64_t flags = MAP_GPA_SHARED | MAP_GPA_DO_FALLOCATE; + + kvm_hypercall_map_gpa_range(gpa, size, flags); +} + +/* + * Test that PUNCH_HOLE actually frees memory by punching holes without doing a + * proper conversion. Freeing (PUNCH_HOLE) should zap SPTEs, and reallocating + * (subsequent fault) should zero memory. + */ +static void guest_test_punch_hole(uint64_t base_gpa, bool precise) +{ + const uint8_t init_p = 0xcc; + int i; + + /* + * Convert the entire range to private, this testcase is all about + * punching holes in guest_memfd, i.e. shared mappings aren't needed. + */ + guest_map_private(base_gpa, PER_CPU_DATA_SIZE, false); + + for (i = 0; i < ARRAY_SIZE(test_ranges); i++) { + uint64_t gpa = base_gpa + test_ranges[i].offset; + uint64_t size = test_ranges[i].size; + + /* + * Free all memory before each iteration, even for the !precise + * case where the memory will be faulted back in. Freeing and + * reallocating should obviously work, and freeing all memory + * minimizes the probability of cross-testcase influence. + */ + guest_punch_hole(base_gpa, PER_CPU_DATA_SIZE); + + /* Fault-in and initialize memory, and verify the pattern. */ + if (precise) { + memset((void *)gpa, init_p, size); + memcmp_g(gpa, init_p, size); + } else { + memset((void *)base_gpa, init_p, PER_CPU_DATA_SIZE); + memcmp_g(base_gpa, init_p, PER_CPU_DATA_SIZE); + } + + /* + * Punch a hole at the target range and verify that reads from + * the guest succeed and return zeroes. + */ + guest_punch_hole(gpa, size); + memcmp_g(gpa, 0, size); + } +} + +static void guest_code(uint64_t base_gpa) +{ + /* + * Run the conversion test twice, with and without doing fallocate() on + * the guest_memfd backing when converting between shared and private. + */ + guest_test_explicit_conversion(base_gpa, false); + guest_test_explicit_conversion(base_gpa, true); + + /* + * Run the PUNCH_HOLE test twice too, once with the entire guest_memfd + * faulted in, once with only the target range faulted in. + */ + guest_test_punch_hole(base_gpa, false); + guest_test_punch_hole(base_gpa, true); + GUEST_DONE(); +} + +static void handle_exit_hypercall(struct kvm_vcpu *vcpu) +{ + struct kvm_run *run = vcpu->run; + uint64_t gpa = run->hypercall.args[0]; + uint64_t size = run->hypercall.args[1] * PAGE_SIZE; + bool set_attributes = run->hypercall.args[2] & MAP_GPA_SET_ATTRIBUTES; + bool map_shared = run->hypercall.args[2] & MAP_GPA_SHARED; + bool do_fallocate = run->hypercall.args[2] & MAP_GPA_DO_FALLOCATE; + struct kvm_vm *vm = vcpu->vm; + + TEST_ASSERT(run->hypercall.nr == KVM_HC_MAP_GPA_RANGE, + "Wanted MAP_GPA_RANGE (%u), got '%llu'", + KVM_HC_MAP_GPA_RANGE, run->hypercall.nr); + + if (do_fallocate) + vm_guest_mem_fallocate(vm, gpa, size, map_shared); + + if (set_attributes) + vm_set_memory_attributes(vm, gpa, size, + map_shared ? 0 : KVM_MEMORY_ATTRIBUTE_PRIVATE); + run->hypercall.ret = 0; +} + +static bool run_vcpus; + +static void *__test_mem_conversions(void *__vcpu) +{ + struct kvm_vcpu *vcpu = __vcpu; + struct kvm_run *run = vcpu->run; + struct kvm_vm *vm = vcpu->vm; + struct ucall uc; + + while (!READ_ONCE(run_vcpus)) + ; + + for ( ;; ) { + vcpu_run(vcpu); + + if (run->exit_reason == KVM_EXIT_HYPERCALL) { + handle_exit_hypercall(vcpu); + continue; + } + + TEST_ASSERT(run->exit_reason == KVM_EXIT_IO, + "Wanted KVM_EXIT_IO, got exit reason: %u (%s)", + run->exit_reason, exit_reason_str(run->exit_reason)); + + switch (get_ucall(vcpu, &uc)) { + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + case UCALL_SYNC: { + uint64_t gpa = uc.args[1]; + size_t size = uc.args[2]; + size_t i; + + TEST_ASSERT(uc.args[0] == SYNC_SHARED || + uc.args[0] == SYNC_PRIVATE, + "Unknown sync command '%ld'", uc.args[0]); + + for (i = 0; i < size; i += vm->page_size) { + size_t nr_bytes = min_t(size_t, vm->page_size, size - i); + uint8_t *hva = addr_gpa2hva(vm, gpa + i); + + /* In all cases, the host should observe the shared data. */ + memcmp_h(hva, gpa + i, uc.args[3], nr_bytes); + + /* For shared, write the new pattern to guest memory. */ + if (uc.args[0] == SYNC_SHARED) + memset(hva, uc.args[4], nr_bytes); + } + break; + } + case UCALL_DONE: + return NULL; + default: + TEST_FAIL("Unknown ucall 0x%lx.", uc.cmd); + } + } +} + +static void test_mem_conversions(enum vm_mem_backing_src_type src_type, uint32_t nr_vcpus, + uint32_t nr_memslots) +{ + /* + * Allocate enough memory so that each vCPU's chunk of memory can be + * naturally aligned with respect to the size of the backing store. + */ + const size_t alignment = max_t(size_t, SZ_2M, get_backing_src_pagesz(src_type)); + const size_t per_cpu_size = align_up(PER_CPU_DATA_SIZE, alignment); + const size_t memfd_size = per_cpu_size * nr_vcpus; + const size_t slot_size = memfd_size / nr_memslots; + struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; + pthread_t threads[KVM_MAX_VCPUS]; + struct kvm_vm *vm; + int memfd, i, r; + + const struct vm_shape shape = { + .mode = VM_MODE_DEFAULT, + .type = KVM_X86_SW_PROTECTED_VM, + }; + + TEST_ASSERT(slot_size * nr_memslots == memfd_size, + "The memfd size (0x%lx) needs to be cleanly divisible by the number of memslots (%u)", + memfd_size, nr_memslots); + vm = __vm_create_with_vcpus(shape, nr_vcpus, 0, guest_code, vcpus); + + vm_enable_cap(vm, KVM_CAP_EXIT_HYPERCALL, (1 << KVM_HC_MAP_GPA_RANGE)); + + memfd = vm_create_guest_memfd(vm, memfd_size, 0); + + for (i = 0; i < nr_memslots; i++) + vm_mem_add(vm, src_type, BASE_DATA_GPA + slot_size * i, + BASE_DATA_SLOT + i, slot_size / vm->page_size, + KVM_MEM_GUEST_MEMFD, memfd, slot_size * i); + + for (i = 0; i < nr_vcpus; i++) { + uint64_t gpa = BASE_DATA_GPA + i * per_cpu_size; + + vcpu_args_set(vcpus[i], 1, gpa); + + /* + * Map only what is needed so that an out-of-bounds access + * results #PF => SHUTDOWN instead of data corruption. + */ + virt_map(vm, gpa, gpa, PER_CPU_DATA_SIZE / vm->page_size); + + pthread_create(&threads[i], NULL, __test_mem_conversions, vcpus[i]); + } + + WRITE_ONCE(run_vcpus, true); + + for (i = 0; i < nr_vcpus; i++) + pthread_join(threads[i], NULL); + + kvm_vm_free(vm); + + /* + * Allocate and free memory from the guest_memfd after closing the VM + * fd. The guest_memfd is gifted a reference to its owning VM, i.e. + * should prevent the VM from being fully destroyed until the last + * reference to the guest_memfd is also put. + */ + r = fallocate(memfd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, memfd_size); + TEST_ASSERT(!r, __KVM_SYSCALL_ERROR("fallocate()", r)); + + r = fallocate(memfd, FALLOC_FL_KEEP_SIZE, 0, memfd_size); + TEST_ASSERT(!r, __KVM_SYSCALL_ERROR("fallocate()", r)); +} + +static void usage(const char *cmd) +{ + puts(""); + printf("usage: %s [-h] [-m nr_memslots] [-s mem_type] [-n nr_vcpus]\n", cmd); + puts(""); + backing_src_help("-s"); + puts(""); + puts(" -n: specify the number of vcpus (default: 1)"); + puts(""); + puts(" -m: specify the number of memslots (default: 1)"); + puts(""); +} + +int main(int argc, char *argv[]) +{ + enum vm_mem_backing_src_type src_type = DEFAULT_VM_MEM_SRC; + uint32_t nr_memslots = 1; + uint32_t nr_vcpus = 1; + int opt; + + TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM)); + + while ((opt = getopt(argc, argv, "hm:s:n:")) != -1) { + switch (opt) { + case 's': + src_type = parse_backing_src_type(optarg); + break; + case 'n': + nr_vcpus = atoi_positive("nr_vcpus", optarg); + break; + case 'm': + nr_memslots = atoi_positive("nr_memslots", optarg); + break; + case 'h': + default: + usage(argv[0]); + exit(0); + } + } + + test_mem_conversions(src_type, nr_vcpus, nr_memslots); + + return 0; +} From e6f4f345b259cad178bad7e40a9d4b65cf195067 Mon Sep 17 00:00:00 2001 From: Chao Peng Date: Fri, 27 Oct 2023 11:22:14 -0700 Subject: [PATCH 031/336] KVM: selftests: Add KVM_SET_USER_MEMORY_REGION2 helper Add helpers to invoke KVM_SET_USER_MEMORY_REGION2 directly so that tests can validate of features that are unique to "version 2" of "set user memory region", e.g. do negative testing on gmem_fd and gmem_offset. Provide a raw version as well as an assert-success version to reduce the amount of boilerplate code need for basic usage. Signed-off-by: Chao Peng Signed-off-by: Ackerley Tng Signed-off-by: Sean Christopherson Message-Id: <20231027182217.3615211-33-seanjc@google.com> Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Signed-off-by: Paolo Bonzini --- .../selftests/kvm/include/kvm_util_base.h | 7 +++++ tools/testing/selftests/kvm/lib/kvm_util.c | 29 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/tools/testing/selftests/kvm/include/kvm_util_base.h b/tools/testing/selftests/kvm/include/kvm_util_base.h index 157508c071f3..8ec122f5fcc8 100644 --- a/tools/testing/selftests/kvm/include/kvm_util_base.h +++ b/tools/testing/selftests/kvm/include/kvm_util_base.h @@ -522,6 +522,13 @@ void vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags, uint64_t gpa, uint64_t size, void *hva); int __vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags, uint64_t gpa, uint64_t size, void *hva); +void vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags, + uint64_t gpa, uint64_t size, void *hva, + uint32_t guest_memfd, uint64_t guest_memfd_offset); +int __vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags, + uint64_t gpa, uint64_t size, void *hva, + uint32_t guest_memfd, uint64_t guest_memfd_offset); + void vm_userspace_mem_region_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, uint64_t guest_paddr, uint32_t slot, uint64_t npages, diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index bf15635eda11..9b29cbf49476 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -873,6 +873,35 @@ void vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags, errno, strerror(errno)); } +int __vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags, + uint64_t gpa, uint64_t size, void *hva, + uint32_t guest_memfd, uint64_t guest_memfd_offset) +{ + struct kvm_userspace_memory_region2 region = { + .slot = slot, + .flags = flags, + .guest_phys_addr = gpa, + .memory_size = size, + .userspace_addr = (uintptr_t)hva, + .guest_memfd = guest_memfd, + .guest_memfd_offset = guest_memfd_offset, + }; + + return ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION2, ®ion); +} + +void vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags, + uint64_t gpa, uint64_t size, void *hva, + uint32_t guest_memfd, uint64_t guest_memfd_offset) +{ + int ret = __vm_set_user_memory_region2(vm, slot, flags, gpa, size, hva, + guest_memfd, guest_memfd_offset); + + TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION2 failed, errno = %d (%s)", + errno, strerror(errno)); +} + + /* FIXME: This thing needs to be ripped apart and rewritten. */ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, uint64_t guest_paddr, uint32_t slot, uint64_t npages, From 2feabb855df8f0889146c2e951307ba477d1f37b Mon Sep 17 00:00:00 2001 From: Chao Peng Date: Fri, 27 Oct 2023 11:22:15 -0700 Subject: [PATCH 032/336] KVM: selftests: Expand set_memory_region_test to validate guest_memfd() Expand set_memory_region_test to exercise various positive and negative testcases for private memory. - Non-guest_memfd() file descriptor for private memory - guest_memfd() from different VM - Overlapping bindings - Unaligned bindings Signed-off-by: Chao Peng Co-developed-by: Ackerley Tng Signed-off-by: Ackerley Tng [sean: trim the testcases to remove duplicate coverage] Signed-off-by: Sean Christopherson Message-Id: <20231027182217.3615211-34-seanjc@google.com> Signed-off-by: Paolo Bonzini --- .../selftests/kvm/include/kvm_util_base.h | 12 ++ .../selftests/kvm/set_memory_region_test.c | 104 +++++++++++++++++- 2 files changed, 114 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/kvm/include/kvm_util_base.h b/tools/testing/selftests/kvm/include/kvm_util_base.h index 8ec122f5fcc8..1b58f943562f 100644 --- a/tools/testing/selftests/kvm/include/kvm_util_base.h +++ b/tools/testing/selftests/kvm/include/kvm_util_base.h @@ -819,6 +819,18 @@ static inline struct kvm_vm *vm_create_barebones(void) return ____vm_create(VM_SHAPE_DEFAULT); } +#ifdef __x86_64__ +static inline struct kvm_vm *vm_create_barebones_protected_vm(void) +{ + const struct vm_shape shape = { + .mode = VM_MODE_DEFAULT, + .type = KVM_X86_SW_PROTECTED_VM, + }; + + return ____vm_create(shape); +} +#endif + static inline struct kvm_vm *vm_create(uint32_t nr_runnable_vcpus) { return __vm_create(VM_SHAPE_DEFAULT, nr_runnable_vcpus, 0); diff --git a/tools/testing/selftests/kvm/set_memory_region_test.c b/tools/testing/selftests/kvm/set_memory_region_test.c index b32960189f5f..a78394faee7c 100644 --- a/tools/testing/selftests/kvm/set_memory_region_test.c +++ b/tools/testing/selftests/kvm/set_memory_region_test.c @@ -385,13 +385,105 @@ static void test_add_max_memory_regions(void) kvm_vm_free(vm); } + +#ifdef __x86_64__ +static void test_invalid_guest_memfd(struct kvm_vm *vm, int memfd, + size_t offset, const char *msg) +{ + int r = __vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD, + MEM_REGION_GPA, MEM_REGION_SIZE, + 0, memfd, offset); + TEST_ASSERT(r == -1 && errno == EINVAL, "%s", msg); +} + +static void test_add_private_memory_region(void) +{ + struct kvm_vm *vm, *vm2; + int memfd, i; + + pr_info("Testing ADD of KVM_MEM_GUEST_MEMFD memory regions\n"); + + vm = vm_create_barebones_protected_vm(); + + test_invalid_guest_memfd(vm, vm->kvm_fd, 0, "KVM fd should fail"); + test_invalid_guest_memfd(vm, vm->fd, 0, "VM's fd should fail"); + + memfd = kvm_memfd_alloc(MEM_REGION_SIZE, false); + test_invalid_guest_memfd(vm, memfd, 0, "Regular memfd() should fail"); + close(memfd); + + vm2 = vm_create_barebones_protected_vm(); + memfd = vm_create_guest_memfd(vm2, MEM_REGION_SIZE, 0); + test_invalid_guest_memfd(vm, memfd, 0, "Other VM's guest_memfd() should fail"); + + vm_set_user_memory_region2(vm2, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD, + MEM_REGION_GPA, MEM_REGION_SIZE, 0, memfd, 0); + close(memfd); + kvm_vm_free(vm2); + + memfd = vm_create_guest_memfd(vm, MEM_REGION_SIZE, 0); + for (i = 1; i < PAGE_SIZE; i++) + test_invalid_guest_memfd(vm, memfd, i, "Unaligned offset should fail"); + + vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD, + MEM_REGION_GPA, MEM_REGION_SIZE, 0, memfd, 0); + close(memfd); + + kvm_vm_free(vm); +} + +static void test_add_overlapping_private_memory_regions(void) +{ + struct kvm_vm *vm; + int memfd; + int r; + + pr_info("Testing ADD of overlapping KVM_MEM_GUEST_MEMFD memory regions\n"); + + vm = vm_create_barebones_protected_vm(); + + memfd = vm_create_guest_memfd(vm, MEM_REGION_SIZE * 4, 0); + + vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD, + MEM_REGION_GPA, MEM_REGION_SIZE * 2, 0, memfd, 0); + + vm_set_user_memory_region2(vm, MEM_REGION_SLOT + 1, KVM_MEM_GUEST_MEMFD, + MEM_REGION_GPA * 2, MEM_REGION_SIZE * 2, + 0, memfd, MEM_REGION_SIZE * 2); + + /* + * Delete the first memslot, and then attempt to recreate it except + * with a "bad" offset that results in overlap in the guest_memfd(). + */ + vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD, + MEM_REGION_GPA, 0, NULL, -1, 0); + + /* Overlap the front half of the other slot. */ + r = __vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD, + MEM_REGION_GPA * 2 - MEM_REGION_SIZE, + MEM_REGION_SIZE * 2, + 0, memfd, 0); + TEST_ASSERT(r == -1 && errno == EEXIST, "%s", + "Overlapping guest_memfd() bindings should fail with EEXIST"); + + /* And now the back half of the other slot. */ + r = __vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD, + MEM_REGION_GPA * 2 + MEM_REGION_SIZE, + MEM_REGION_SIZE * 2, + 0, memfd, 0); + TEST_ASSERT(r == -1 && errno == EEXIST, "%s", + "Overlapping guest_memfd() bindings should fail with EEXIST"); + + close(memfd); + kvm_vm_free(vm); +} +#endif + int main(int argc, char *argv[]) { #ifdef __x86_64__ int i, loops; -#endif -#ifdef __x86_64__ /* * FIXME: the zero-memslot test fails on aarch64 and s390x because * KVM_RUN fails with ENOEXEC or EFAULT. @@ -402,6 +494,14 @@ int main(int argc, char *argv[]) test_add_max_memory_regions(); #ifdef __x86_64__ + if (kvm_has_cap(KVM_CAP_GUEST_MEMFD) && + (kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM))) { + test_add_private_memory_region(); + test_add_overlapping_private_memory_regions(); + } else { + pr_info("Skipping tests for KVM_MEM_GUEST_MEMFD memory regions\n"); + } + if (argc > 1) loops = atoi_positive("Number of iterations", argv[1]); else From 8a89efd43423cb3005c5e641e846184e292c1465 Mon Sep 17 00:00:00 2001 From: Chao Peng Date: Fri, 27 Oct 2023 11:22:16 -0700 Subject: [PATCH 033/336] KVM: selftests: Add basic selftest for guest_memfd() Add a selftest to verify the basic functionality of guest_memfd(): + file descriptor created with the guest_memfd() ioctl does not allow read/write/mmap operations + file size and block size as returned from fstat are as expected + fallocate on the fd checks that offset/length on fallocate(FALLOC_FL_PUNCH_HOLE) should be page aligned + invalid inputs (misaligned size, invalid flags) are rejected + file size and inode are unique (the innocuous-sounding anon_inode_getfile() backs all files with a single inode...) Signed-off-by: Chao Peng Co-developed-by: Ackerley Tng Signed-off-by: Ackerley Tng Co-developed-by: Paolo Bonzini Signed-off-by: Paolo Bonzini Co-developed-by: Sean Christopherson Signed-off-by: Sean Christopherson Message-Id: <20231027182217.3615211-35-seanjc@google.com> Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Signed-off-by: Paolo Bonzini --- tools/testing/selftests/kvm/Makefile | 1 + .../testing/selftests/kvm/guest_memfd_test.c | 207 ++++++++++++++++++ 2 files changed, 208 insertions(+) create mode 100644 tools/testing/selftests/kvm/guest_memfd_test.c diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index ecdea5e7afa8..fd3b30a4ca7b 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -134,6 +134,7 @@ TEST_GEN_PROGS_x86_64 += access_tracking_perf_test TEST_GEN_PROGS_x86_64 += demand_paging_test TEST_GEN_PROGS_x86_64 += dirty_log_test TEST_GEN_PROGS_x86_64 += dirty_log_perf_test +TEST_GEN_PROGS_x86_64 += guest_memfd_test TEST_GEN_PROGS_x86_64 += guest_print_test TEST_GEN_PROGS_x86_64 += hardware_disable_test TEST_GEN_PROGS_x86_64 += kvm_create_max_vcpus diff --git a/tools/testing/selftests/kvm/guest_memfd_test.c b/tools/testing/selftests/kvm/guest_memfd_test.c new file mode 100644 index 000000000000..fd389663c49b --- /dev/null +++ b/tools/testing/selftests/kvm/guest_memfd_test.c @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright Intel Corporation, 2023 + * + * Author: Chao Peng + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "test_util.h" +#include "kvm_util_base.h" + +static void test_file_read_write(int fd) +{ + char buf[64]; + + TEST_ASSERT(read(fd, buf, sizeof(buf)) < 0, + "read on a guest_mem fd should fail"); + TEST_ASSERT(write(fd, buf, sizeof(buf)) < 0, + "write on a guest_mem fd should fail"); + TEST_ASSERT(pread(fd, buf, sizeof(buf), 0) < 0, + "pread on a guest_mem fd should fail"); + TEST_ASSERT(pwrite(fd, buf, sizeof(buf), 0) < 0, + "pwrite on a guest_mem fd should fail"); +} + +static void test_mmap(int fd, size_t page_size) +{ + char *mem; + + mem = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + TEST_ASSERT_EQ(mem, MAP_FAILED); +} + +static void test_file_size(int fd, size_t page_size, size_t total_size) +{ + struct stat sb; + int ret; + + ret = fstat(fd, &sb); + TEST_ASSERT(!ret, "fstat should succeed"); + TEST_ASSERT_EQ(sb.st_size, total_size); + TEST_ASSERT_EQ(sb.st_blksize, page_size); +} + +static void test_fallocate(int fd, size_t page_size, size_t total_size) +{ + int ret; + + ret = fallocate(fd, FALLOC_FL_KEEP_SIZE, 0, total_size); + TEST_ASSERT(!ret, "fallocate with aligned offset and size should succeed"); + + ret = fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, + page_size - 1, page_size); + TEST_ASSERT(ret, "fallocate with unaligned offset should fail"); + + ret = fallocate(fd, FALLOC_FL_KEEP_SIZE, total_size, page_size); + TEST_ASSERT(ret, "fallocate beginning at total_size should fail"); + + ret = fallocate(fd, FALLOC_FL_KEEP_SIZE, total_size + page_size, page_size); + TEST_ASSERT(ret, "fallocate beginning after total_size should fail"); + + ret = fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, + total_size, page_size); + TEST_ASSERT(!ret, "fallocate(PUNCH_HOLE) at total_size should succeed"); + + ret = fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, + total_size + page_size, page_size); + TEST_ASSERT(!ret, "fallocate(PUNCH_HOLE) after total_size should succeed"); + + ret = fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, + page_size, page_size - 1); + TEST_ASSERT(ret, "fallocate with unaligned size should fail"); + + ret = fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, + page_size, page_size); + TEST_ASSERT(!ret, "fallocate(PUNCH_HOLE) with aligned offset and size should succeed"); + + ret = fallocate(fd, FALLOC_FL_KEEP_SIZE, page_size, page_size); + TEST_ASSERT(!ret, "fallocate to restore punched hole should succeed"); +} + +static void test_invalid_punch_hole(int fd, size_t page_size, size_t total_size) +{ + struct { + off_t offset; + off_t len; + } testcases[] = { + {0, 1}, + {0, page_size - 1}, + {0, page_size + 1}, + + {1, 1}, + {1, page_size - 1}, + {1, page_size}, + {1, page_size + 1}, + + {page_size, 1}, + {page_size, page_size - 1}, + {page_size, page_size + 1}, + }; + int ret, i; + + for (i = 0; i < ARRAY_SIZE(testcases); i++) { + ret = fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, + testcases[i].offset, testcases[i].len); + TEST_ASSERT(ret == -1 && errno == EINVAL, + "PUNCH_HOLE with !PAGE_SIZE offset (%lx) and/or length (%lx) should fail", + testcases[i].offset, testcases[i].len); + } +} + +static void test_create_guest_memfd_invalid(struct kvm_vm *vm) +{ + size_t page_size = getpagesize(); + uint64_t flag; + size_t size; + int fd; + + for (size = 1; size < page_size; size++) { + fd = __vm_create_guest_memfd(vm, size, 0); + TEST_ASSERT(fd == -1 && errno == EINVAL, + "guest_memfd() with non-page-aligned page size '0x%lx' should fail with EINVAL", + size); + } + + for (flag = 1; flag; flag <<= 1) { + uint64_t bit; + + fd = __vm_create_guest_memfd(vm, page_size, flag); + TEST_ASSERT(fd == -1 && errno == EINVAL, + "guest_memfd() with flag '0x%lx' should fail with EINVAL", + flag); + + for_each_set_bit(bit, &valid_flags, 64) { + fd = __vm_create_guest_memfd(vm, page_size, flag | BIT_ULL(bit)); + TEST_ASSERT(fd == -1 && errno == EINVAL, + "guest_memfd() with flags '0x%llx' should fail with EINVAL", + flag | BIT_ULL(bit)); + } + } +} + +static void test_create_guest_memfd_multiple(struct kvm_vm *vm) +{ + int fd1, fd2, ret; + struct stat st1, st2; + + fd1 = __vm_create_guest_memfd(vm, 4096, 0); + TEST_ASSERT(fd1 != -1, "memfd creation should succeed"); + + ret = fstat(fd1, &st1); + TEST_ASSERT(ret != -1, "memfd fstat should succeed"); + TEST_ASSERT(st1.st_size == 4096, "memfd st_size should match requested size"); + + fd2 = __vm_create_guest_memfd(vm, 8192, 0); + TEST_ASSERT(fd2 != -1, "memfd creation should succeed"); + + ret = fstat(fd2, &st2); + TEST_ASSERT(ret != -1, "memfd fstat should succeed"); + TEST_ASSERT(st2.st_size == 8192, "second memfd st_size should match requested size"); + + ret = fstat(fd1, &st1); + TEST_ASSERT(ret != -1, "memfd fstat should succeed"); + TEST_ASSERT(st1.st_size == 4096, "first memfd st_size should still match requested size"); + TEST_ASSERT(st1.st_ino != st2.st_ino, "different memfd should have different inode numbers"); +} + +int main(int argc, char *argv[]) +{ + size_t page_size; + size_t total_size; + int fd; + struct kvm_vm *vm; + + TEST_REQUIRE(kvm_has_cap(KVM_CAP_GUEST_MEMFD)); + + page_size = getpagesize(); + total_size = page_size * 4; + + vm = vm_create_barebones(); + + test_create_guest_memfd_invalid(vm); + test_create_guest_memfd_multiple(vm); + + fd = vm_create_guest_memfd(vm, total_size, 0); + + test_file_read_write(fd); + test_mmap(fd, page_size); + test_file_size(fd, page_size, total_size); + test_fallocate(fd, page_size, total_size); + test_invalid_punch_hole(fd, page_size, total_size); + + close(fd); +} From e3577788de64139202d89224fe31613c0f02b790 Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Fri, 27 Oct 2023 11:22:17 -0700 Subject: [PATCH 034/336] KVM: selftests: Test KVM exit behavior for private memory/access "Testing private access when memslot gets deleted" tests the behavior of KVM when a private memslot gets deleted while the VM is using the private memslot. When KVM looks up the deleted (slot = NULL) memslot, KVM should exit to userspace with KVM_EXIT_MEMORY_FAULT. In the second test, upon a private access to non-private memslot, KVM should also exit to userspace with KVM_EXIT_MEMORY_FAULT. Intentionally don't take a requirement on KVM_CAP_GUEST_MEMFD, KVM_CAP_MEMORY_FAULT_INFO, KVM_MEMORY_ATTRIBUTE_PRIVATE, etc., as it's a KVM bug to advertise KVM_X86_SW_PROTECTED_VM without its prerequisites. Signed-off-by: Ackerley Tng [sean: call out the similarities with set_memory_region_test] Signed-off-by: Sean Christopherson Message-Id: <20231027182217.3615211-36-seanjc@google.com> Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Signed-off-by: Paolo Bonzini --- tools/testing/selftests/kvm/Makefile | 1 + .../kvm/x86_64/private_mem_kvm_exits_test.c | 120 ++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 tools/testing/selftests/kvm/x86_64/private_mem_kvm_exits_test.c diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index fd3b30a4ca7b..69ce8e06b3a3 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -92,6 +92,7 @@ TEST_GEN_PROGS_x86_64 += x86_64/nested_exceptions_test TEST_GEN_PROGS_x86_64 += x86_64/platform_info_test TEST_GEN_PROGS_x86_64 += x86_64/pmu_event_filter_test TEST_GEN_PROGS_x86_64 += x86_64/private_mem_conversions_test +TEST_GEN_PROGS_x86_64 += x86_64/private_mem_kvm_exits_test TEST_GEN_PROGS_x86_64 += x86_64/set_boot_cpu_id TEST_GEN_PROGS_x86_64 += x86_64/set_sregs_test TEST_GEN_PROGS_x86_64 += x86_64/smaller_maxphyaddr_emulation_test diff --git a/tools/testing/selftests/kvm/x86_64/private_mem_kvm_exits_test.c b/tools/testing/selftests/kvm/x86_64/private_mem_kvm_exits_test.c new file mode 100644 index 000000000000..13e72fcec8dd --- /dev/null +++ b/tools/testing/selftests/kvm/x86_64/private_mem_kvm_exits_test.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2023, Google LLC. + */ +#include +#include +#include + +#include "kvm_util.h" +#include "processor.h" +#include "test_util.h" + +/* Arbitrarily selected to avoid overlaps with anything else */ +#define EXITS_TEST_GVA 0xc0000000 +#define EXITS_TEST_GPA EXITS_TEST_GVA +#define EXITS_TEST_NPAGES 1 +#define EXITS_TEST_SIZE (EXITS_TEST_NPAGES * PAGE_SIZE) +#define EXITS_TEST_SLOT 10 + +static uint64_t guest_repeatedly_read(void) +{ + volatile uint64_t value; + + while (true) + value = *((uint64_t *) EXITS_TEST_GVA); + + return value; +} + +static uint32_t run_vcpu_get_exit_reason(struct kvm_vcpu *vcpu) +{ + int r; + + r = _vcpu_run(vcpu); + if (r) { + TEST_ASSERT(errno == EFAULT, KVM_IOCTL_ERROR(KVM_RUN, r)); + TEST_ASSERT_EQ(vcpu->run->exit_reason, KVM_EXIT_MEMORY_FAULT); + } + return vcpu->run->exit_reason; +} + +const struct vm_shape protected_vm_shape = { + .mode = VM_MODE_DEFAULT, + .type = KVM_X86_SW_PROTECTED_VM, +}; + +static void test_private_access_memslot_deleted(void) +{ + struct kvm_vm *vm; + struct kvm_vcpu *vcpu; + pthread_t vm_thread; + void *thread_return; + uint32_t exit_reason; + + vm = vm_create_shape_with_one_vcpu(protected_vm_shape, &vcpu, + guest_repeatedly_read); + + vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, + EXITS_TEST_GPA, EXITS_TEST_SLOT, + EXITS_TEST_NPAGES, + KVM_MEM_GUEST_MEMFD); + + virt_map(vm, EXITS_TEST_GVA, EXITS_TEST_GPA, EXITS_TEST_NPAGES); + + /* Request to access page privately */ + vm_mem_set_private(vm, EXITS_TEST_GPA, EXITS_TEST_SIZE); + + pthread_create(&vm_thread, NULL, + (void *(*)(void *))run_vcpu_get_exit_reason, + (void *)vcpu); + + vm_mem_region_delete(vm, EXITS_TEST_SLOT); + + pthread_join(vm_thread, &thread_return); + exit_reason = (uint32_t)(uint64_t)thread_return; + + TEST_ASSERT_EQ(exit_reason, KVM_EXIT_MEMORY_FAULT); + TEST_ASSERT_EQ(vcpu->run->memory_fault.flags, KVM_MEMORY_EXIT_FLAG_PRIVATE); + TEST_ASSERT_EQ(vcpu->run->memory_fault.gpa, EXITS_TEST_GPA); + TEST_ASSERT_EQ(vcpu->run->memory_fault.size, EXITS_TEST_SIZE); + + kvm_vm_free(vm); +} + +static void test_private_access_memslot_not_private(void) +{ + struct kvm_vm *vm; + struct kvm_vcpu *vcpu; + uint32_t exit_reason; + + vm = vm_create_shape_with_one_vcpu(protected_vm_shape, &vcpu, + guest_repeatedly_read); + + /* Add a non-private memslot (flags = 0) */ + vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, + EXITS_TEST_GPA, EXITS_TEST_SLOT, + EXITS_TEST_NPAGES, 0); + + virt_map(vm, EXITS_TEST_GVA, EXITS_TEST_GPA, EXITS_TEST_NPAGES); + + /* Request to access page privately */ + vm_mem_set_private(vm, EXITS_TEST_GPA, EXITS_TEST_SIZE); + + exit_reason = run_vcpu_get_exit_reason(vcpu); + + TEST_ASSERT_EQ(exit_reason, KVM_EXIT_MEMORY_FAULT); + TEST_ASSERT_EQ(vcpu->run->memory_fault.flags, KVM_MEMORY_EXIT_FLAG_PRIVATE); + TEST_ASSERT_EQ(vcpu->run->memory_fault.gpa, EXITS_TEST_GPA); + TEST_ASSERT_EQ(vcpu->run->memory_fault.size, EXITS_TEST_SIZE); + + kvm_vm_free(vm); +} + +int main(int argc, char *argv[]) +{ + TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM)); + + test_private_access_memslot_deleted(); + test_private_access_memslot_not_private(); +} From 5d74316466f4aabdd2ee1e33b45e4933c9bc3ea1 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 30 Oct 2023 17:20:49 -0700 Subject: [PATCH 035/336] KVM: selftests: Add a memory region subtest to validate invalid flags Add a subtest to set_memory_region_test to verify that KVM rejects invalid flags and combinations with -EINVAL. KVM might or might not fail with EINVAL anyways, but we can at least try. Signed-off-by: Sean Christopherson Message-Id: <20231031002049.3915752-1-seanjc@google.com> Signed-off-by: Paolo Bonzini --- .../testing/selftests/kvm/guest_memfd_test.c | 9 +--- .../selftests/kvm/set_memory_region_test.c | 49 +++++++++++++++++++ 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/tools/testing/selftests/kvm/guest_memfd_test.c b/tools/testing/selftests/kvm/guest_memfd_test.c index fd389663c49b..318ba6ba8bd3 100644 --- a/tools/testing/selftests/kvm/guest_memfd_test.c +++ b/tools/testing/selftests/kvm/guest_memfd_test.c @@ -136,20 +136,13 @@ static void test_create_guest_memfd_invalid(struct kvm_vm *vm) size); } - for (flag = 1; flag; flag <<= 1) { + for (flag = 0; flag; flag <<= 1) { uint64_t bit; fd = __vm_create_guest_memfd(vm, page_size, flag); TEST_ASSERT(fd == -1 && errno == EINVAL, "guest_memfd() with flag '0x%lx' should fail with EINVAL", flag); - - for_each_set_bit(bit, &valid_flags, 64) { - fd = __vm_create_guest_memfd(vm, page_size, flag | BIT_ULL(bit)); - TEST_ASSERT(fd == -1 && errno == EINVAL, - "guest_memfd() with flags '0x%llx' should fail with EINVAL", - flag | BIT_ULL(bit)); - } } } diff --git a/tools/testing/selftests/kvm/set_memory_region_test.c b/tools/testing/selftests/kvm/set_memory_region_test.c index a78394faee7c..1efee1cfcff0 100644 --- a/tools/testing/selftests/kvm/set_memory_region_test.c +++ b/tools/testing/selftests/kvm/set_memory_region_test.c @@ -326,6 +326,53 @@ static void test_zero_memory_regions(void) } #endif /* __x86_64__ */ +static void test_invalid_memory_region_flags(void) +{ + uint32_t supported_flags = KVM_MEM_LOG_DIRTY_PAGES; + const uint32_t v2_only_flags = KVM_MEM_GUEST_MEMFD; + struct kvm_vm *vm; + int r, i; + +#ifdef __x86_64__ + supported_flags |= KVM_MEM_READONLY; + + if (kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM)) + vm = vm_create_barebones_protected_vm(); + else +#endif + vm = vm_create_barebones(); + + if (kvm_check_cap(KVM_CAP_MEMORY_ATTRIBUTES) & KVM_MEMORY_ATTRIBUTE_PRIVATE) + supported_flags |= KVM_MEM_GUEST_MEMFD; + + for (i = 0; i < 32; i++) { + if ((supported_flags & BIT(i)) && !(v2_only_flags & BIT(i))) + continue; + + r = __vm_set_user_memory_region(vm, MEM_REGION_SLOT, BIT(i), + MEM_REGION_GPA, MEM_REGION_SIZE, NULL); + + TEST_ASSERT(r && errno == EINVAL, + "KVM_SET_USER_MEMORY_REGION should have failed on v2 only flag 0x%lx", BIT(i)); + + if (supported_flags & BIT(i)) + continue; + + r = __vm_set_user_memory_region2(vm, MEM_REGION_SLOT, BIT(i), + MEM_REGION_GPA, MEM_REGION_SIZE, NULL, 0, 0); + TEST_ASSERT(r && errno == EINVAL, + "KVM_SET_USER_MEMORY_REGION2 should have failed on unsupported flag 0x%lx", BIT(i)); + } + + if (supported_flags & KVM_MEM_GUEST_MEMFD) { + r = __vm_set_user_memory_region2(vm, MEM_REGION_SLOT, + KVM_MEM_LOG_DIRTY_PAGES | KVM_MEM_GUEST_MEMFD, + MEM_REGION_GPA, MEM_REGION_SIZE, NULL, 0, 0); + TEST_ASSERT(r && errno == EINVAL, + "KVM_SET_USER_MEMORY_REGION2 should have failed, dirty logging private memory is unsupported"); + } +} + /* * Test it can be added memory slots up to KVM_CAP_NR_MEMSLOTS, then any * tentative to add further slots should fail. @@ -491,6 +538,8 @@ int main(int argc, char *argv[]) test_zero_memory_regions(); #endif + test_invalid_memory_region_flags(); + test_add_max_memory_regions(); #ifdef __x86_64__ From a4d511aa24cab1a3175f0421f60fd81ea4721fd1 Mon Sep 17 00:00:00 2001 From: Michal Simek Date: Tue, 24 Oct 2023 15:46:46 +0200 Subject: [PATCH 036/336] microblaze: Align defconfig with latest Kconfig layout Sync up patch. Signed-off-by: Michal Simek Link: https://lore.kernel.org/r/4414109ba26433f00b4c07f600162462ab6c7a93.1698155190.git.michal.simek@amd.com --- arch/microblaze/configs/mmu_defconfig | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/arch/microblaze/configs/mmu_defconfig b/arch/microblaze/configs/mmu_defconfig index 7b2d7f6f23c0..88fce95f6401 100644 --- a/arch/microblaze/configs/mmu_defconfig +++ b/arch/microblaze/configs/mmu_defconfig @@ -3,11 +3,9 @@ CONFIG_POSIX_MQUEUE=y CONFIG_AUDIT=y CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y -CONFIG_SYSFS_DEPRECATED=y -CONFIG_SYSFS_DEPRECATED_V2=y +CONFIG_EXPERT=y # CONFIG_BASE_FULL is not set CONFIG_KALLSYMS_ALL=y -CONFIG_EXPERT=y CONFIG_XILINX_MICROBLAZE0_USE_MSR_INSTR=1 CONFIG_XILINX_MICROBLAZE0_USE_PCMP_INSTR=1 CONFIG_XILINX_MICROBLAZE0_USE_BARREL=1 @@ -20,7 +18,6 @@ CONFIG_CMDLINE_FORCE=y CONFIG_HIGHMEM=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y -# CONFIG_BLK_DEV_BSG is not set CONFIG_PARTITION_ADVANCED=y # CONFIG_EFI_PARTITION is not set CONFIG_CMA=y @@ -78,13 +75,11 @@ CONFIG_CRAMFS=y CONFIG_ROMFS_FS=y CONFIG_NFS_FS=y CONFIG_CIFS=y -CONFIG_CIFS_STATS2=y CONFIG_ENCRYPTED_KEYS=y CONFIG_DMA_CMA=y CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_KGDB=y CONFIG_KGDB_TESTS=y CONFIG_KGDB_KDB=y -CONFIG_DEBUG_SLAB=y CONFIG_DETECT_HUNG_TASK=y CONFIG_DEBUG_SPINLOCK=y From 873b074050a8028634fb3d4fc95cc9a3a70d10bb Mon Sep 17 00:00:00 2001 From: Raju Kumar Pothuraju Date: Tue, 24 Oct 2023 15:46:47 +0200 Subject: [PATCH 037/336] microblaze: Enable options to mount a rootfs via NFS Enable the options to mount a rootfs over NFS for microblaze platforms and also support for automatic configuration of IP addresses during boot as needed by NFS. Signed-off-by: Raju Kumar Pothuraju Reviewed-by: Radhey Shyam Pandey Signed-off-by: Michal Simek Link: https://lore.kernel.org/r/d7bc1c93e485e0a6ab2c234a144402b7bd5f77bb.1698155190.git.michal.simek@amd.com --- arch/microblaze/configs/mmu_defconfig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/microblaze/configs/mmu_defconfig b/arch/microblaze/configs/mmu_defconfig index 88fce95f6401..85afdb377f73 100644 --- a/arch/microblaze/configs/mmu_defconfig +++ b/arch/microblaze/configs/mmu_defconfig @@ -25,6 +25,10 @@ CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y CONFIG_INET=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_IP_PNP_BOOTP=y +CONFIG_IP_PNP_RARP=y # CONFIG_IPV6 is not set CONFIG_BRIDGE=m CONFIG_PCI=y @@ -74,6 +78,7 @@ CONFIG_TMPFS=y CONFIG_CRAMFS=y CONFIG_ROMFS_FS=y CONFIG_NFS_FS=y +CONFIG_ROOT_NFS=y CONFIG_CIFS=y CONFIG_ENCRYPTED_KEYS=y CONFIG_DMA_CMA=y From ffb0399437ef582b035089d1617bbb0ea174cd0c Mon Sep 17 00:00:00 2001 From: Radhey Shyam Pandey Date: Tue, 24 Oct 2023 15:46:48 +0200 Subject: [PATCH 038/336] microblaze: defconfig: Enable the Marvell phy driver Enable the marvell phy driver by default as it is commonly used PHY on multiple microblaze based evaluation boards. Signed-off-by: Radhey Shyam Pandey Signed-off-by: Michal Simek Link: https://lore.kernel.org/r/3912fb168671dd5b418da8947ea1c463b554fa86.1698155190.git.michal.simek@amd.com --- arch/microblaze/configs/mmu_defconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/microblaze/configs/mmu_defconfig b/arch/microblaze/configs/mmu_defconfig index 85afdb377f73..4da7bc4ac4a3 100644 --- a/arch/microblaze/configs/mmu_defconfig +++ b/arch/microblaze/configs/mmu_defconfig @@ -44,6 +44,7 @@ CONFIG_NETDEVICES=y CONFIG_XILINX_EMACLITE=y CONFIG_XILINX_AXI_EMAC=y CONFIG_XILINX_LL_TEMAC=y +CONFIG_MARVELL_PHY=y # CONFIG_INPUT is not set # CONFIG_SERIO is not set # CONFIG_VT is not set From fb49b6f65a699b3463f5b2f16815340bbd3dabeb Mon Sep 17 00:00:00 2001 From: Iuliana Prodan Date: Fri, 13 Oct 2023 18:27:30 +0300 Subject: [PATCH 039/336] remoteproc: imx_dsp_rproc: Add mandatory find_loaded_rsc_table op Add the .find_loaded_rsc_table operation for i.MX DSP. We need it for inter-process communication between DSP and main core. This callback is used to find the resource table (defined in remote processor linker script) where the address of the vrings along with the other allocated resources (carveouts etc) are stored. If this is not found, the vrings are not allocated and the IPC between cores will not work. Signed-off-by: Iuliana Prodan Reviewed-by: Daniel Baluta Link: https://lore.kernel.org/r/20231013152731.23471-2-iuliana.prodan@oss.nxp.com Signed-off-by: Mathieu Poirier --- drivers/remoteproc/imx_dsp_rproc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/remoteproc/imx_dsp_rproc.c b/drivers/remoteproc/imx_dsp_rproc.c index 8fcda9b74545..a1c62d15f16c 100644 --- a/drivers/remoteproc/imx_dsp_rproc.c +++ b/drivers/remoteproc/imx_dsp_rproc.c @@ -940,6 +940,7 @@ static const struct rproc_ops imx_dsp_rproc_ops = { .kick = imx_dsp_rproc_kick, .load = imx_dsp_rproc_elf_load_segments, .parse_fw = imx_dsp_rproc_parse_fw, + .find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table, .sanity_check = rproc_elf_sanity_check, .get_boot_addr = rproc_elf_get_boot_addr, }; From cae0e61beb7acb1c54a17e5418978f096d66def6 Mon Sep 17 00:00:00 2001 From: Iuliana Prodan Date: Fri, 13 Oct 2023 18:27:31 +0300 Subject: [PATCH 040/336] arm64: dts: imx8mp: Add reserve-memory nodes for DSP Add the reserve-memory nodes used by DSP when the rpmsg feature is enabled. Signed-off-by: Iuliana Prodan Acked-by: Alexander Stein Link: https://lore.kernel.org/r/20231013152731.23471-3-iuliana.prodan@oss.nxp.com Signed-off-by: Mathieu Poirier --- arch/arm64/boot/dts/freescale/imx8mp-evk.dts | 22 ++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/arch/arm64/boot/dts/freescale/imx8mp-evk.dts b/arch/arm64/boot/dts/freescale/imx8mp-evk.dts index cc9d468b43ab..f87fa5a948cc 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-evk.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-evk.dts @@ -137,6 +137,28 @@ }; }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + dsp_vdev0vring0: vdev0vring0@942f0000 { + reg = <0 0x942f0000 0 0x8000>; + no-map; + }; + + dsp_vdev0vring1: vdev0vring1@942f8000 { + reg = <0 0x942f8000 0 0x8000>; + no-map; + }; + + dsp_vdev0buffer: vdev0buffer@94300000 { + compatible = "shared-dma-pool"; + reg = <0 0x94300000 0 0x100000>; + no-map; + }; + }; }; &flexspi { From e9e60c82fe391d04db55a91c733df4a017c28b2f Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 21 Nov 2023 11:24:08 -0500 Subject: [PATCH 041/336] selftests/kvm: fix compilation on non-x86_64 platforms MEM_REGION_SLOT and MEM_REGION_GPA are not really needed in test_invalid_memory_region_flags; the VM never runs and there are no other slots, so it is okay to use slot 0 and place it at address zero. This fixes compilation on architectures that do not define them. Fixes: 5d74316466f4 ("KVM: selftests: Add a memory region subtest to validate invalid flags") Signed-off-by: Paolo Bonzini --- tools/testing/selftests/kvm/guest_memfd_test.c | 2 -- tools/testing/selftests/kvm/set_memory_region_test.c | 12 ++++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/tools/testing/selftests/kvm/guest_memfd_test.c b/tools/testing/selftests/kvm/guest_memfd_test.c index 318ba6ba8bd3..c78a98c1a915 100644 --- a/tools/testing/selftests/kvm/guest_memfd_test.c +++ b/tools/testing/selftests/kvm/guest_memfd_test.c @@ -137,8 +137,6 @@ static void test_create_guest_memfd_invalid(struct kvm_vm *vm) } for (flag = 0; flag; flag <<= 1) { - uint64_t bit; - fd = __vm_create_guest_memfd(vm, page_size, flag); TEST_ASSERT(fd == -1 && errno == EINVAL, "guest_memfd() with flag '0x%lx' should fail with EINVAL", diff --git a/tools/testing/selftests/kvm/set_memory_region_test.c b/tools/testing/selftests/kvm/set_memory_region_test.c index 1efee1cfcff0..6637a0845acf 100644 --- a/tools/testing/selftests/kvm/set_memory_region_test.c +++ b/tools/testing/selftests/kvm/set_memory_region_test.c @@ -349,8 +349,8 @@ static void test_invalid_memory_region_flags(void) if ((supported_flags & BIT(i)) && !(v2_only_flags & BIT(i))) continue; - r = __vm_set_user_memory_region(vm, MEM_REGION_SLOT, BIT(i), - MEM_REGION_GPA, MEM_REGION_SIZE, NULL); + r = __vm_set_user_memory_region(vm, 0, BIT(i), + 0, MEM_REGION_SIZE, NULL); TEST_ASSERT(r && errno == EINVAL, "KVM_SET_USER_MEMORY_REGION should have failed on v2 only flag 0x%lx", BIT(i)); @@ -358,16 +358,16 @@ static void test_invalid_memory_region_flags(void) if (supported_flags & BIT(i)) continue; - r = __vm_set_user_memory_region2(vm, MEM_REGION_SLOT, BIT(i), - MEM_REGION_GPA, MEM_REGION_SIZE, NULL, 0, 0); + r = __vm_set_user_memory_region2(vm, 0, BIT(i), + 0, MEM_REGION_SIZE, NULL, 0, 0); TEST_ASSERT(r && errno == EINVAL, "KVM_SET_USER_MEMORY_REGION2 should have failed on unsupported flag 0x%lx", BIT(i)); } if (supported_flags & KVM_MEM_GUEST_MEMFD) { - r = __vm_set_user_memory_region2(vm, MEM_REGION_SLOT, + r = __vm_set_user_memory_region2(vm, 0, KVM_MEM_LOG_DIRTY_PAGES | KVM_MEM_GUEST_MEMFD, - MEM_REGION_GPA, MEM_REGION_SIZE, NULL, 0, 0); + 0, MEM_REGION_SIZE, NULL, 0, 0); TEST_ASSERT(r && errno == EINVAL, "KVM_SET_USER_MEMORY_REGION2 should have failed, dirty logging private memory is unsupported"); } From 26fb87ffa9d90fb16ca1b2b262f38d93bdcee934 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Mon, 6 Nov 2023 13:49:22 +0100 Subject: [PATCH 042/336] s390/uvdevice: Report additional-data length for attestation Additional data length in the attestation request is an in/out variable. Software provides the capacity of the buffer. Upon successful request, firmware reports the actual bytes written to the additional data in that field. This information is lost, as the length field was not copied back to userspace before. Attestation might fail, if user space did not specify the exact amount of needed bytes required, as this length is part of the attestation measurement. Signed-off-by: Steffen Eiden Reviewed-by: Claudio Imbrenda Reviewed-by: Janosch Frank Link: https://lore.kernel.org/r/20231106124922.3032370-1-seiden@linux.ibm.com Message-Id: <20231106124922.3032370-1-seiden@linux.ibm.com> Signed-off-by: Janosch Frank --- drivers/s390/char/uvdevice.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/s390/char/uvdevice.c b/drivers/s390/char/uvdevice.c index 144cd2e03590..42c9f77f8da0 100644 --- a/drivers/s390/char/uvdevice.c +++ b/drivers/s390/char/uvdevice.c @@ -109,6 +109,7 @@ static int uvio_copy_attest_result_to_user(struct uv_cb_attest *uvcb_attest, struct uvio_attest *uvio_attest) { struct uvio_attest __user *user_uvio_attest = (void __user *)uv_ioctl->argument_addr; + u32 __user *user_buf_add_len = (u32 __user *)&user_uvio_attest->add_data_len; void __user *user_buf_add = (void __user *)uvio_attest->add_data_addr; void __user *user_buf_meas = (void __user *)uvio_attest->meas_addr; void __user *user_buf_uid = &user_uvio_attest->config_uid; @@ -117,6 +118,8 @@ static int uvio_copy_attest_result_to_user(struct uv_cb_attest *uvcb_attest, return -EFAULT; if (add_data && copy_to_user(user_buf_add, add_data, uvio_attest->add_data_len)) return -EFAULT; + if (put_user(uvio_attest->add_data_len, user_buf_add_len)) + return -EFAULT; if (copy_to_user(user_buf_uid, uvcb_attest->config_uid, sizeof(uvcb_attest->config_uid))) return -EFAULT; return 0; From d4fbbb26da520e00d87c8187dc3de9eacee66c1c Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Mon, 27 Nov 2023 11:17:31 +0000 Subject: [PATCH 043/336] KVM: arm64: Add new (V)TCR_EL2 field definitions for FEAT_LPA2 As per Arm ARM (0487I.a), (V)TCR_EL2.DS fields control whether 52 bit input and output addresses are supported on 4K and 16K page size configurations when FEAT_LPA2 is known to have been implemented. This adds these field definitions which will be used by KVM when FEAT_LPA2 is enabled. Acked-by: Catalin Marinas Reviewed-by: Oliver Upton Signed-off-by: Ryan Roberts Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20231127111737.1897081-7-ryan.roberts@arm.com --- arch/arm64/include/asm/kvm_arm.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index b85f46a73e21..312cbc300831 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -108,6 +108,7 @@ #define HCRX_HOST_FLAGS (HCRX_EL2_MSCEn | HCRX_EL2_TCR2En) /* TCR_EL2 Registers bits */ +#define TCR_EL2_DS (1UL << 32) #define TCR_EL2_RES1 ((1U << 31) | (1 << 23)) #define TCR_EL2_TBI (1 << 20) #define TCR_EL2_PS_SHIFT 16 @@ -122,6 +123,7 @@ TCR_EL2_ORGN0_MASK | TCR_EL2_IRGN0_MASK | TCR_EL2_T0SZ_MASK) /* VTCR_EL2 Registers bits */ +#define VTCR_EL2_DS TCR_EL2_DS #define VTCR_EL2_RES1 (1U << 31) #define VTCR_EL2_HD (1 << 22) #define VTCR_EL2_HA (1 << 21) From bd412e2a310cbc43b424198b0065086b0f462625 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Mon, 27 Nov 2023 11:17:32 +0000 Subject: [PATCH 044/336] KVM: arm64: Use LPA2 page-tables for stage2 and hyp stage1 Implement a simple policy whereby if the HW supports FEAT_LPA2 for the page size we are using, always use LPA2-style page-tables for stage 2 and hyp stage 1 (assuming an nvhe hyp), regardless of the VMM-requested IPA size or HW-implemented PA size. When in use we can now support up to 52-bit IPA and PA sizes. We use the previously created cpu feature to track whether LPA2 is supported for deciding whether to use the LPA2 or classic pte format. Note that FEAT_LPA2 brings support for bigger block mappings (512GB with 4KB, 64GB with 16KB). We explicitly don't enable these in the library because stage2_apply_range() works on batch sizes of the largest used block mapping, and increasing the size of the batch would lead to soft lockups. See commit 5994bc9e05c2 ("KVM: arm64: Limit stage2_apply_range() batch size to largest block"). With the addition of LPA2 support in the hypervisor, the PA size supported by the HW must be capped with a runtime decision, rather than simply using a compile-time decision based on PA_BITS. For example, on a system that advertises 52 bit PA but does not support FEAT_LPA2, A 4KB or 16KB kernel compiled with LPA2 support must still limit the PA size to 48 bits. Therefore, move the insertion of the PS field into TCR_EL2 out of __kvm_hyp_init assembly code and instead do it in cpu_prepare_hyp_mode() where the rest of TCR_EL2 is prepared. This allows us to figure out PS with kvm_get_parange(), which has the appropriate logic to ensure the above requirement. (and the PS field of VTCR_EL2 is already populated this way). Reviewed-by: Oliver Upton Signed-off-by: Ryan Roberts Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20231127111737.1897081-8-ryan.roberts@arm.com --- arch/arm64/include/asm/kvm_pgtable.h | 45 +++++++++++++++++++++------- arch/arm64/kvm/arm.c | 5 ++++ arch/arm64/kvm/hyp/nvhe/hyp-init.S | 4 --- arch/arm64/kvm/hyp/pgtable.c | 15 ++++++++-- 4 files changed, 52 insertions(+), 17 deletions(-) diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h index 10068500d601..69a2a87ecaf6 100644 --- a/arch/arm64/include/asm/kvm_pgtable.h +++ b/arch/arm64/include/asm/kvm_pgtable.h @@ -25,14 +25,24 @@ #define KVM_PGTABLE_MIN_BLOCK_LEVEL 2U #endif -#define kvm_lpa2_is_enabled() false +#define kvm_lpa2_is_enabled() system_supports_lpa2() + +static inline u64 kvm_get_parange_max(void) +{ + if (kvm_lpa2_is_enabled() || + (IS_ENABLED(CONFIG_ARM64_PA_BITS_52) && PAGE_SHIFT == 16)) + return ID_AA64MMFR0_EL1_PARANGE_52; + else + return ID_AA64MMFR0_EL1_PARANGE_48; +} static inline u64 kvm_get_parange(u64 mmfr0) { + u64 parange_max = kvm_get_parange_max(); u64 parange = cpuid_feature_extract_unsigned_field(mmfr0, ID_AA64MMFR0_EL1_PARANGE_SHIFT); - if (parange > ID_AA64MMFR0_EL1_PARANGE_MAX) - parange = ID_AA64MMFR0_EL1_PARANGE_MAX; + if (parange > parange_max) + parange = parange_max; return parange; } @@ -43,6 +53,8 @@ typedef u64 kvm_pte_t; #define KVM_PTE_ADDR_MASK GENMASK(47, PAGE_SHIFT) #define KVM_PTE_ADDR_51_48 GENMASK(15, 12) +#define KVM_PTE_ADDR_MASK_LPA2 GENMASK(49, PAGE_SHIFT) +#define KVM_PTE_ADDR_51_50_LPA2 GENMASK(9, 8) #define KVM_PHYS_INVALID (-1ULL) @@ -53,21 +65,34 @@ static inline bool kvm_pte_valid(kvm_pte_t pte) static inline u64 kvm_pte_to_phys(kvm_pte_t pte) { - u64 pa = pte & KVM_PTE_ADDR_MASK; + u64 pa; - if (PAGE_SHIFT == 16) - pa |= FIELD_GET(KVM_PTE_ADDR_51_48, pte) << 48; + if (kvm_lpa2_is_enabled()) { + pa = pte & KVM_PTE_ADDR_MASK_LPA2; + pa |= FIELD_GET(KVM_PTE_ADDR_51_50_LPA2, pte) << 50; + } else { + pa = pte & KVM_PTE_ADDR_MASK; + if (PAGE_SHIFT == 16) + pa |= FIELD_GET(KVM_PTE_ADDR_51_48, pte) << 48; + } return pa; } static inline kvm_pte_t kvm_phys_to_pte(u64 pa) { - kvm_pte_t pte = pa & KVM_PTE_ADDR_MASK; + kvm_pte_t pte; - if (PAGE_SHIFT == 16) { - pa &= GENMASK(51, 48); - pte |= FIELD_PREP(KVM_PTE_ADDR_51_48, pa >> 48); + if (kvm_lpa2_is_enabled()) { + pte = pa & KVM_PTE_ADDR_MASK_LPA2; + pa &= GENMASK(51, 50); + pte |= FIELD_PREP(KVM_PTE_ADDR_51_50_LPA2, pa >> 50); + } else { + pte = pa & KVM_PTE_ADDR_MASK; + if (PAGE_SHIFT == 16) { + pa &= GENMASK(51, 48); + pte |= FIELD_PREP(KVM_PTE_ADDR_51_48, pa >> 48); + } } return pte; diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index e5f75f1f1085..c4bbc224549b 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -1837,6 +1837,7 @@ static int kvm_init_vector_slots(void) static void __init cpu_prepare_hyp_mode(int cpu, u32 hyp_va_bits) { struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu); + u64 mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); unsigned long tcr; /* @@ -1859,6 +1860,10 @@ static void __init cpu_prepare_hyp_mode(int cpu, u32 hyp_va_bits) } tcr &= ~TCR_T0SZ_MASK; tcr |= TCR_T0SZ(hyp_va_bits); + tcr &= ~TCR_EL2_PS_MASK; + tcr |= FIELD_PREP(TCR_EL2_PS_MASK, kvm_get_parange(mmfr0)); + if (kvm_lpa2_is_enabled()) + tcr |= TCR_EL2_DS; params->tcr_el2 = tcr; params->pgd_pa = kvm_mmu_get_httbr(); diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S index 1cc06e6797bd..f62a7d360285 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S @@ -122,11 +122,7 @@ alternative_if ARM64_HAS_CNP alternative_else_nop_endif msr ttbr0_el2, x2 - /* - * Set the PS bits in TCR_EL2. - */ ldr x0, [x0, #NVHE_INIT_TCR_EL2] - tcr_compute_pa_size x0, #TCR_EL2_PS_SHIFT, x1, x2 msr tcr_el2, x0 isb diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index 1966fdee740e..ce9a58cb02fd 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -79,7 +79,10 @@ static bool kvm_pgtable_walk_skip_cmo(const struct kvm_pgtable_visit_ctx *ctx) static bool kvm_phys_is_valid(u64 phys) { - return phys < BIT(id_aa64mmfr0_parange_to_phys_shift(ID_AA64MMFR0_EL1_PARANGE_MAX)); + u64 parange_max = kvm_get_parange_max(); + u8 shift = id_aa64mmfr0_parange_to_phys_shift(parange_max); + + return phys < BIT(shift); } static bool kvm_block_mapping_supported(const struct kvm_pgtable_visit_ctx *ctx, u64 phys) @@ -408,7 +411,8 @@ static int hyp_set_prot_attr(enum kvm_pgtable_prot prot, kvm_pte_t *ptep) } attr |= FIELD_PREP(KVM_PTE_LEAF_ATTR_LO_S1_AP, ap); - attr |= FIELD_PREP(KVM_PTE_LEAF_ATTR_LO_S1_SH, sh); + if (!kvm_lpa2_is_enabled()) + attr |= FIELD_PREP(KVM_PTE_LEAF_ATTR_LO_S1_SH, sh); attr |= KVM_PTE_LEAF_ATTR_LO_S1_AF; attr |= prot & KVM_PTE_LEAF_ATTR_HI_SW; *ptep = attr; @@ -654,6 +658,9 @@ u64 kvm_get_vtcr(u64 mmfr0, u64 mmfr1, u32 phys_shift) vtcr |= VTCR_EL2_HA; #endif /* CONFIG_ARM64_HW_AFDBM */ + if (kvm_lpa2_is_enabled()) + vtcr |= VTCR_EL2_DS; + /* Set the vmid bits */ vtcr |= (get_vmid_bits(mmfr1) == 16) ? VTCR_EL2_VS_16BIT : @@ -711,7 +718,9 @@ static int stage2_set_prot_attr(struct kvm_pgtable *pgt, enum kvm_pgtable_prot p if (prot & KVM_PGTABLE_PROT_W) attr |= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W; - attr |= FIELD_PREP(KVM_PTE_LEAF_ATTR_LO_S2_SH, sh); + if (!kvm_lpa2_is_enabled()) + attr |= FIELD_PREP(KVM_PTE_LEAF_ATTR_LO_S2_SH, sh); + attr |= KVM_PTE_LEAF_ATTR_LO_S2_AF; attr |= prot & KVM_PTE_LEAF_ATTR_HI_SW; *ptep = attr; From 419edf48d79f6fb2cc3fa090131864e95b321d41 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Mon, 27 Nov 2023 11:17:33 +0000 Subject: [PATCH 045/336] KVM: arm64: Convert translation level parameter to s8 With the introduction of FEAT_LPA2, the Arm ARM adds a new level of translation, level -1, so levels can now be in the range [-1;3]. 3 is always the last level and the first level is determined based on the number of VA bits in use. Convert level variables to use a signed type in preparation for supporting this new level -1. Since the last level is always anchored at 3, and the first level varies to suit the number of VA/IPA bits, take the opportunity to replace KVM_PGTABLE_MAX_LEVELS with the 2 macros KVM_PGTABLE_FIRST_LEVEL and KVM_PGTABLE_LAST_LEVEL. This removes the assumption from the code that levels run from 0 to KVM_PGTABLE_MAX_LEVELS - 1, which will soon no longer be true. Reviewed-by: Oliver Upton Signed-off-by: Ryan Roberts Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20231127111737.1897081-9-ryan.roberts@arm.com --- arch/arm64/include/asm/kvm_emulate.h | 2 +- arch/arm64/include/asm/kvm_pgtable.h | 31 +++++++------ arch/arm64/include/asm/kvm_pkvm.h | 5 +- arch/arm64/kvm/hyp/nvhe/mem_protect.c | 6 +-- arch/arm64/kvm/hyp/nvhe/mm.c | 4 +- arch/arm64/kvm/hyp/nvhe/setup.c | 2 +- arch/arm64/kvm/hyp/pgtable.c | 66 +++++++++++++++------------ arch/arm64/kvm/mmu.c | 16 ++++--- 8 files changed, 71 insertions(+), 61 deletions(-) diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 78a550537b67..13fd9dbf2d1d 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -409,7 +409,7 @@ static __always_inline u8 kvm_vcpu_trap_get_fault_type(const struct kvm_vcpu *vc return kvm_vcpu_get_esr(vcpu) & ESR_ELx_FSC_TYPE; } -static __always_inline u8 kvm_vcpu_trap_get_fault_level(const struct kvm_vcpu *vcpu) +static __always_inline s8 kvm_vcpu_trap_get_fault_level(const struct kvm_vcpu *vcpu) { return kvm_vcpu_get_esr(vcpu) & ESR_ELx_FSC_LEVEL; } diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h index 69a2a87ecaf6..3253828e453d 100644 --- a/arch/arm64/include/asm/kvm_pgtable.h +++ b/arch/arm64/include/asm/kvm_pgtable.h @@ -11,7 +11,8 @@ #include #include -#define KVM_PGTABLE_MAX_LEVELS 4U +#define KVM_PGTABLE_FIRST_LEVEL 0 +#define KVM_PGTABLE_LAST_LEVEL 3 /* * The largest supported block sizes for KVM (no 52-bit PA support): @@ -20,9 +21,9 @@ * - 64K (level 2): 512MB */ #ifdef CONFIG_ARM64_4K_PAGES -#define KVM_PGTABLE_MIN_BLOCK_LEVEL 1U +#define KVM_PGTABLE_MIN_BLOCK_LEVEL 1 #else -#define KVM_PGTABLE_MIN_BLOCK_LEVEL 2U +#define KVM_PGTABLE_MIN_BLOCK_LEVEL 2 #endif #define kvm_lpa2_is_enabled() system_supports_lpa2() @@ -103,28 +104,28 @@ static inline kvm_pfn_t kvm_pte_to_pfn(kvm_pte_t pte) return __phys_to_pfn(kvm_pte_to_phys(pte)); } -static inline u64 kvm_granule_shift(u32 level) +static inline u64 kvm_granule_shift(s8 level) { - /* Assumes KVM_PGTABLE_MAX_LEVELS is 4 */ + /* Assumes KVM_PGTABLE_LAST_LEVEL is 3 */ return ARM64_HW_PGTABLE_LEVEL_SHIFT(level); } -static inline u64 kvm_granule_size(u32 level) +static inline u64 kvm_granule_size(s8 level) { return BIT(kvm_granule_shift(level)); } -static inline bool kvm_level_supports_block_mapping(u32 level) +static inline bool kvm_level_supports_block_mapping(s8 level) { return level >= KVM_PGTABLE_MIN_BLOCK_LEVEL; } static inline u32 kvm_supported_block_sizes(void) { - u32 level = KVM_PGTABLE_MIN_BLOCK_LEVEL; + s8 level = KVM_PGTABLE_MIN_BLOCK_LEVEL; u32 r = 0; - for (; level < KVM_PGTABLE_MAX_LEVELS; level++) + for (; level <= KVM_PGTABLE_LAST_LEVEL; level++) r |= BIT(kvm_granule_shift(level)); return r; @@ -169,7 +170,7 @@ struct kvm_pgtable_mm_ops { void* (*zalloc_page)(void *arg); void* (*zalloc_pages_exact)(size_t size); void (*free_pages_exact)(void *addr, size_t size); - void (*free_unlinked_table)(void *addr, u32 level); + void (*free_unlinked_table)(void *addr, s8 level); void (*get_page)(void *addr); void (*put_page)(void *addr); int (*page_count)(void *addr); @@ -265,7 +266,7 @@ struct kvm_pgtable_visit_ctx { u64 start; u64 addr; u64 end; - u32 level; + s8 level; enum kvm_pgtable_walk_flags flags; }; @@ -368,7 +369,7 @@ static inline bool kvm_pgtable_walk_lock_held(void) */ struct kvm_pgtable { u32 ia_bits; - u32 start_level; + s8 start_level; kvm_pteref_t pgd; struct kvm_pgtable_mm_ops *mm_ops; @@ -502,7 +503,7 @@ void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt); * The page-table is assumed to be unreachable by any hardware walkers prior to * freeing and therefore no TLB invalidation is performed. */ -void kvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops *mm_ops, void *pgtable, u32 level); +void kvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops *mm_ops, void *pgtable, s8 level); /** * kvm_pgtable_stage2_create_unlinked() - Create an unlinked stage-2 paging structure. @@ -526,7 +527,7 @@ void kvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops *mm_ops, void *p * an ERR_PTR(error) on failure. */ kvm_pte_t *kvm_pgtable_stage2_create_unlinked(struct kvm_pgtable *pgt, - u64 phys, u32 level, + u64 phys, s8 level, enum kvm_pgtable_prot prot, void *mc, bool force_pte); @@ -752,7 +753,7 @@ int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size, * Return: 0 on success, negative error code on failure. */ int kvm_pgtable_get_leaf(struct kvm_pgtable *pgt, u64 addr, - kvm_pte_t *ptep, u32 *level); + kvm_pte_t *ptep, s8 *level); /** * kvm_pgtable_stage2_pte_prot() - Retrieve the protection attributes of a diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index e46250a02017..ad9cfb5c1ff4 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -56,10 +56,11 @@ static inline unsigned long hyp_vm_table_pages(void) static inline unsigned long __hyp_pgtable_max_pages(unsigned long nr_pages) { - unsigned long total = 0, i; + unsigned long total = 0; + int i; /* Provision the worst case scenario */ - for (i = 0; i < KVM_PGTABLE_MAX_LEVELS; i++) { + for (i = KVM_PGTABLE_FIRST_LEVEL; i <= KVM_PGTABLE_LAST_LEVEL; i++) { nr_pages = DIV_ROUND_UP(nr_pages, PTRS_PER_PTE); total += nr_pages; } diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 8d0a5834e883..861c76021a25 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -91,7 +91,7 @@ static void host_s2_put_page(void *addr) hyp_put_page(&host_s2_pool, addr); } -static void host_s2_free_unlinked_table(void *addr, u32 level) +static void host_s2_free_unlinked_table(void *addr, s8 level) { kvm_pgtable_stage2_free_unlinked(&host_mmu.mm_ops, addr, level); } @@ -443,7 +443,7 @@ static int host_stage2_adjust_range(u64 addr, struct kvm_mem_range *range) { struct kvm_mem_range cur; kvm_pte_t pte; - u32 level; + s8 level; int ret; hyp_assert_lock_held(&host_mmu.lock); @@ -462,7 +462,7 @@ static int host_stage2_adjust_range(u64 addr, struct kvm_mem_range *range) cur.start = ALIGN_DOWN(addr, granule); cur.end = cur.start + granule; level++; - } while ((level < KVM_PGTABLE_MAX_LEVELS) && + } while ((level <= KVM_PGTABLE_LAST_LEVEL) && !(kvm_level_supports_block_mapping(level) && range_included(&cur, range))); diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c index 65a7a186d7b2..b01a3d1078a8 100644 --- a/arch/arm64/kvm/hyp/nvhe/mm.c +++ b/arch/arm64/kvm/hyp/nvhe/mm.c @@ -260,7 +260,7 @@ static void fixmap_clear_slot(struct hyp_fixmap_slot *slot) * https://lore.kernel.org/kvm/20221017115209.2099-1-will@kernel.org/T/#mf10dfbaf1eaef9274c581b81c53758918c1d0f03 */ dsb(ishst); - __tlbi_level(vale2is, __TLBI_VADDR(addr, 0), (KVM_PGTABLE_MAX_LEVELS - 1)); + __tlbi_level(vale2is, __TLBI_VADDR(addr, 0), KVM_PGTABLE_LAST_LEVEL); dsb(ish); isb(); } @@ -275,7 +275,7 @@ static int __create_fixmap_slot_cb(const struct kvm_pgtable_visit_ctx *ctx, { struct hyp_fixmap_slot *slot = per_cpu_ptr(&fixmap_slots, (u64)ctx->arg); - if (!kvm_pte_valid(ctx->old) || ctx->level != KVM_PGTABLE_MAX_LEVELS - 1) + if (!kvm_pte_valid(ctx->old) || ctx->level != KVM_PGTABLE_LAST_LEVEL) return -EINVAL; slot->addr = ctx->addr; diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c index 0d5e0a89ddce..bc58d1b515af 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -181,7 +181,7 @@ static int fix_host_ownership_walker(const struct kvm_pgtable_visit_ctx *ctx, if (!kvm_pte_valid(ctx->old)) return 0; - if (ctx->level != (KVM_PGTABLE_MAX_LEVELS - 1)) + if (ctx->level != KVM_PGTABLE_LAST_LEVEL) return -EINVAL; phys = kvm_pte_to_phys(ctx->old); diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index ce9a58cb02fd..744bded18e99 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -101,7 +101,7 @@ static bool kvm_block_mapping_supported(const struct kvm_pgtable_visit_ctx *ctx, return IS_ALIGNED(ctx->addr, granule); } -static u32 kvm_pgtable_idx(struct kvm_pgtable_walk_data *data, u32 level) +static u32 kvm_pgtable_idx(struct kvm_pgtable_walk_data *data, s8 level) { u64 shift = kvm_granule_shift(level); u64 mask = BIT(PAGE_SHIFT - 3) - 1; @@ -117,7 +117,7 @@ static u32 kvm_pgd_page_idx(struct kvm_pgtable *pgt, u64 addr) return (addr & mask) >> shift; } -static u32 kvm_pgd_pages(u32 ia_bits, u32 start_level) +static u32 kvm_pgd_pages(u32 ia_bits, s8 start_level) { struct kvm_pgtable pgt = { .ia_bits = ia_bits, @@ -127,9 +127,9 @@ static u32 kvm_pgd_pages(u32 ia_bits, u32 start_level) return kvm_pgd_page_idx(&pgt, -1ULL) + 1; } -static bool kvm_pte_table(kvm_pte_t pte, u32 level) +static bool kvm_pte_table(kvm_pte_t pte, s8 level) { - if (level == KVM_PGTABLE_MAX_LEVELS - 1) + if (level == KVM_PGTABLE_LAST_LEVEL) return false; if (!kvm_pte_valid(pte)) @@ -157,11 +157,11 @@ static kvm_pte_t kvm_init_table_pte(kvm_pte_t *childp, struct kvm_pgtable_mm_ops return pte; } -static kvm_pte_t kvm_init_valid_leaf_pte(u64 pa, kvm_pte_t attr, u32 level) +static kvm_pte_t kvm_init_valid_leaf_pte(u64 pa, kvm_pte_t attr, s8 level) { kvm_pte_t pte = kvm_phys_to_pte(pa); - u64 type = (level == KVM_PGTABLE_MAX_LEVELS - 1) ? KVM_PTE_TYPE_PAGE : - KVM_PTE_TYPE_BLOCK; + u64 type = (level == KVM_PGTABLE_LAST_LEVEL) ? KVM_PTE_TYPE_PAGE : + KVM_PTE_TYPE_BLOCK; pte |= attr & (KVM_PTE_LEAF_ATTR_LO | KVM_PTE_LEAF_ATTR_HI); pte |= FIELD_PREP(KVM_PTE_TYPE, type); @@ -206,11 +206,11 @@ static bool kvm_pgtable_walk_continue(const struct kvm_pgtable_walker *walker, } static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data, - struct kvm_pgtable_mm_ops *mm_ops, kvm_pteref_t pgtable, u32 level); + struct kvm_pgtable_mm_ops *mm_ops, kvm_pteref_t pgtable, s8 level); static inline int __kvm_pgtable_visit(struct kvm_pgtable_walk_data *data, struct kvm_pgtable_mm_ops *mm_ops, - kvm_pteref_t pteref, u32 level) + kvm_pteref_t pteref, s8 level) { enum kvm_pgtable_walk_flags flags = data->walker->flags; kvm_pte_t *ptep = kvm_dereference_pteref(data->walker, pteref); @@ -275,12 +275,13 @@ out: } static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data, - struct kvm_pgtable_mm_ops *mm_ops, kvm_pteref_t pgtable, u32 level) + struct kvm_pgtable_mm_ops *mm_ops, kvm_pteref_t pgtable, s8 level) { u32 idx; int ret = 0; - if (WARN_ON_ONCE(level >= KVM_PGTABLE_MAX_LEVELS)) + if (WARN_ON_ONCE(level < KVM_PGTABLE_FIRST_LEVEL || + level > KVM_PGTABLE_LAST_LEVEL)) return -EINVAL; for (idx = kvm_pgtable_idx(data, level); idx < PTRS_PER_PTE; ++idx) { @@ -343,7 +344,7 @@ int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size, struct leaf_walk_data { kvm_pte_t pte; - u32 level; + s8 level; }; static int leaf_walker(const struct kvm_pgtable_visit_ctx *ctx, @@ -358,7 +359,7 @@ static int leaf_walker(const struct kvm_pgtable_visit_ctx *ctx, } int kvm_pgtable_get_leaf(struct kvm_pgtable *pgt, u64 addr, - kvm_pte_t *ptep, u32 *level) + kvm_pte_t *ptep, s8 *level) { struct leaf_walk_data data; struct kvm_pgtable_walker walker = { @@ -471,7 +472,7 @@ static int hyp_map_walker(const struct kvm_pgtable_visit_ctx *ctx, if (hyp_map_walker_try_leaf(ctx, data)) return 0; - if (WARN_ON(ctx->level == KVM_PGTABLE_MAX_LEVELS - 1)) + if (WARN_ON(ctx->level == KVM_PGTABLE_LAST_LEVEL)) return -EINVAL; childp = (kvm_pte_t *)mm_ops->zalloc_page(NULL); @@ -567,14 +568,19 @@ u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size) int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits, struct kvm_pgtable_mm_ops *mm_ops) { - u64 levels = ARM64_HW_PGTABLE_LEVELS(va_bits); + s8 start_level = KVM_PGTABLE_LAST_LEVEL + 1 - + ARM64_HW_PGTABLE_LEVELS(va_bits); + + if (start_level < KVM_PGTABLE_FIRST_LEVEL || + start_level > KVM_PGTABLE_LAST_LEVEL) + return -EINVAL; pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_page(NULL); if (!pgt->pgd) return -ENOMEM; pgt->ia_bits = va_bits; - pgt->start_level = KVM_PGTABLE_MAX_LEVELS - levels; + pgt->start_level = start_level; pgt->mm_ops = mm_ops; pgt->mmu = NULL; pgt->force_pte_cb = NULL; @@ -628,7 +634,7 @@ struct stage2_map_data { u64 kvm_get_vtcr(u64 mmfr0, u64 mmfr1, u32 phys_shift) { u64 vtcr = VTCR_EL2_FLAGS; - u8 lvls; + s8 lvls; vtcr |= kvm_get_parange(mmfr0) << VTCR_EL2_PS_SHIFT; vtcr |= VTCR_EL2_T0SZ(phys_shift); @@ -911,7 +917,7 @@ static bool stage2_leaf_mapping_allowed(const struct kvm_pgtable_visit_ctx *ctx, { u64 phys = stage2_map_walker_phys_addr(ctx, data); - if (data->force_pte && (ctx->level < (KVM_PGTABLE_MAX_LEVELS - 1))) + if (data->force_pte && ctx->level < KVM_PGTABLE_LAST_LEVEL) return false; return kvm_block_mapping_supported(ctx, phys); @@ -990,7 +996,7 @@ static int stage2_map_walk_leaf(const struct kvm_pgtable_visit_ctx *ctx, if (ret != -E2BIG) return ret; - if (WARN_ON(ctx->level == KVM_PGTABLE_MAX_LEVELS - 1)) + if (WARN_ON(ctx->level == KVM_PGTABLE_LAST_LEVEL)) return -EINVAL; if (!data->memcache) @@ -1160,7 +1166,7 @@ struct stage2_attr_data { kvm_pte_t attr_set; kvm_pte_t attr_clr; kvm_pte_t pte; - u32 level; + s8 level; }; static int stage2_attr_walker(const struct kvm_pgtable_visit_ctx *ctx, @@ -1203,7 +1209,7 @@ static int stage2_attr_walker(const struct kvm_pgtable_visit_ctx *ctx, static int stage2_update_leaf_attrs(struct kvm_pgtable *pgt, u64 addr, u64 size, kvm_pte_t attr_set, kvm_pte_t attr_clr, kvm_pte_t *orig_pte, - u32 *level, enum kvm_pgtable_walk_flags flags) + s8 *level, enum kvm_pgtable_walk_flags flags) { int ret; kvm_pte_t attr_mask = KVM_PTE_LEAF_ATTR_LO | KVM_PTE_LEAF_ATTR_HI; @@ -1305,7 +1311,7 @@ int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr, enum kvm_pgtable_prot prot) { int ret; - u32 level; + s8 level; kvm_pte_t set = 0, clr = 0; if (prot & KVM_PTE_LEAF_ATTR_HI_SW) @@ -1358,7 +1364,7 @@ int kvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size) } kvm_pte_t *kvm_pgtable_stage2_create_unlinked(struct kvm_pgtable *pgt, - u64 phys, u32 level, + u64 phys, s8 level, enum kvm_pgtable_prot prot, void *mc, bool force_pte) { @@ -1416,7 +1422,7 @@ kvm_pte_t *kvm_pgtable_stage2_create_unlinked(struct kvm_pgtable *pgt, * fully populated tree up to the PTE entries. Note that @level is * interpreted as in "level @level entry". */ -static int stage2_block_get_nr_page_tables(u32 level) +static int stage2_block_get_nr_page_tables(s8 level) { switch (level) { case 1: @@ -1427,7 +1433,7 @@ static int stage2_block_get_nr_page_tables(u32 level) return 0; default: WARN_ON_ONCE(level < KVM_PGTABLE_MIN_BLOCK_LEVEL || - level >= KVM_PGTABLE_MAX_LEVELS); + level > KVM_PGTABLE_LAST_LEVEL); return -EINVAL; }; } @@ -1440,13 +1446,13 @@ static int stage2_split_walker(const struct kvm_pgtable_visit_ctx *ctx, struct kvm_s2_mmu *mmu; kvm_pte_t pte = ctx->old, new, *childp; enum kvm_pgtable_prot prot; - u32 level = ctx->level; + s8 level = ctx->level; bool force_pte; int nr_pages; u64 phys; /* No huge-pages exist at the last level */ - if (level == KVM_PGTABLE_MAX_LEVELS - 1) + if (level == KVM_PGTABLE_LAST_LEVEL) return 0; /* We only split valid block mappings */ @@ -1523,7 +1529,7 @@ int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu, u64 vtcr = mmu->vtcr; u32 ia_bits = VTCR_EL2_IPA(vtcr); u32 sl0 = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr); - u32 start_level = VTCR_EL2_TGRAN_SL0_BASE - sl0; + s8 start_level = VTCR_EL2_TGRAN_SL0_BASE - sl0; pgd_sz = kvm_pgd_pages(ia_bits, start_level) * PAGE_SIZE; pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_pages_exact(pgd_sz); @@ -1546,7 +1552,7 @@ size_t kvm_pgtable_stage2_pgd_size(u64 vtcr) { u32 ia_bits = VTCR_EL2_IPA(vtcr); u32 sl0 = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr); - u32 start_level = VTCR_EL2_TGRAN_SL0_BASE - sl0; + s8 start_level = VTCR_EL2_TGRAN_SL0_BASE - sl0; return kvm_pgd_pages(ia_bits, start_level) * PAGE_SIZE; } @@ -1582,7 +1588,7 @@ void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt) pgt->pgd = NULL; } -void kvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops *mm_ops, void *pgtable, u32 level) +void kvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops *mm_ops, void *pgtable, s8 level) { kvm_pteref_t ptep = (kvm_pteref_t)pgtable; struct kvm_pgtable_walker walker = { diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index d87c8fcc4c24..986a2e6fb900 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -223,12 +223,12 @@ static void stage2_free_unlinked_table_rcu_cb(struct rcu_head *head) { struct page *page = container_of(head, struct page, rcu_head); void *pgtable = page_to_virt(page); - u32 level = page_private(page); + s8 level = page_private(page); kvm_pgtable_stage2_free_unlinked(&kvm_s2_mm_ops, pgtable, level); } -static void stage2_free_unlinked_table(void *addr, u32 level) +static void stage2_free_unlinked_table(void *addr, s8 level) { struct page *page = virt_to_page(addr); @@ -804,13 +804,13 @@ static int get_user_mapping_size(struct kvm *kvm, u64 addr) struct kvm_pgtable pgt = { .pgd = (kvm_pteref_t)kvm->mm->pgd, .ia_bits = vabits_actual, - .start_level = (KVM_PGTABLE_MAX_LEVELS - - CONFIG_PGTABLE_LEVELS), + .start_level = (KVM_PGTABLE_LAST_LEVEL - + CONFIG_PGTABLE_LEVELS + 1), .mm_ops = &kvm_user_mm_ops, }; unsigned long flags; kvm_pte_t pte = 0; /* Keep GCC quiet... */ - u32 level = ~0; + s8 level = S8_MAX; int ret; /* @@ -829,7 +829,9 @@ static int get_user_mapping_size(struct kvm *kvm, u64 addr) * Not seeing an error, but not updating level? Something went * deeply wrong... */ - if (WARN_ON(level >= KVM_PGTABLE_MAX_LEVELS)) + if (WARN_ON(level > KVM_PGTABLE_LAST_LEVEL)) + return -EFAULT; + if (WARN_ON(level < KVM_PGTABLE_FIRST_LEVEL)) return -EFAULT; /* Oops, the userspace PTs are gone... Replay the fault */ @@ -1388,7 +1390,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, gfn_t gfn; kvm_pfn_t pfn; bool logging_active = memslot_is_logging(memslot); - unsigned long fault_level = kvm_vcpu_trap_get_fault_level(vcpu); + s8 fault_level = kvm_vcpu_trap_get_fault_level(vcpu); long vma_pagesize, fault_granule; enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R; struct kvm_pgtable *pgt; From 0abc1b11a032199bb134fd25cd7ee0cdb26b7b03 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Mon, 27 Nov 2023 11:17:34 +0000 Subject: [PATCH 046/336] KVM: arm64: Support up to 5 levels of translation in kvm_pgtable FEAT_LPA2 increases the maximum levels of translation from 4 to 5 for the 4KB page case, when IA is >48 bits. While we can still use 4 levels for stage2 translation in this case (due to stage2 allowing concatenated page tables for first level lookup), the same kvm_pgtable library is used for the hyp stage1 page tables and stage1 does not support concatenation. Therefore, modify the library to support up to 5 levels. Previous patches already laid the groundwork for this by refactoring code to work in terms of KVM_PGTABLE_FIRST_LEVEL and KVM_PGTABLE_LAST_LEVEL. So we just need to change these macros. The hardware sometimes encodes the new level differently from the others: One such place is when reading the level from the FSC field in the ESR_EL2 register. We never expect to see the lowest level (-1) here since the stage 2 page tables always use concatenated tables for first level lookup and therefore only use 4 levels of lookup. So we get away with just adding a comment to explain why we are not being careful about decoding level -1. For stage2 VTCR_EL2.SL2 is introduced to encode the new start level. However, since we always use concatenated page tables for first level look up at stage2 (and therefore we will never need the new extra level) we never touch this new field. Reviewed-by: Oliver Upton Signed-off-by: Ryan Roberts Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20231127111737.1897081-10-ryan.roberts@arm.com --- arch/arm64/include/asm/kvm_emulate.h | 10 ++++++++++ arch/arm64/include/asm/kvm_pgtable.h | 2 +- arch/arm64/kvm/hyp/pgtable.c | 9 +++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 13fd9dbf2d1d..d4f1e9cdd554 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -411,6 +411,16 @@ static __always_inline u8 kvm_vcpu_trap_get_fault_type(const struct kvm_vcpu *vc static __always_inline s8 kvm_vcpu_trap_get_fault_level(const struct kvm_vcpu *vcpu) { + /* + * Note: With the introduction of FEAT_LPA2 an extra level of + * translation (level -1) is added. This level (obviously) doesn't + * follow the previous convention of encoding the 4 levels in the 2 LSBs + * of the FSC so this function breaks if the fault is for level -1. + * + * However, stage2 tables always use concatenated tables for first level + * lookup and therefore it is guaranteed that the level will be between + * 0 and 3, and this function continues to work. + */ return kvm_vcpu_get_esr(vcpu) & ESR_ELx_FSC_LEVEL; } diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h index 3253828e453d..cfdf40f734b1 100644 --- a/arch/arm64/include/asm/kvm_pgtable.h +++ b/arch/arm64/include/asm/kvm_pgtable.h @@ -11,7 +11,7 @@ #include #include -#define KVM_PGTABLE_FIRST_LEVEL 0 +#define KVM_PGTABLE_FIRST_LEVEL -1 #define KVM_PGTABLE_LAST_LEVEL 3 /* diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index 744bded18e99..c651df904fe3 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -645,6 +645,15 @@ u64 kvm_get_vtcr(u64 mmfr0, u64 mmfr1, u32 phys_shift) lvls = stage2_pgtable_levels(phys_shift); if (lvls < 2) lvls = 2; + + /* + * When LPA2 is enabled, the HW supports an extra level of translation + * (for 5 in total) when using 4K pages. It also introduces VTCR_EL2.SL2 + * to as an addition to SL0 to enable encoding this extra start level. + * However, since we always use concatenated pages for the first level + * lookup, we will never need this extra level and therefore do not need + * to touch SL2. + */ vtcr |= VTCR_EL2_LVLS_TO_SL0(lvls); #ifdef CONFIG_ARM64_HW_AFDBM From d782ac5b2ceebee5d374e13e990655d1a140d3a6 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Mon, 27 Nov 2023 11:17:35 +0000 Subject: [PATCH 047/336] KVM: arm64: Allow guests with >48-bit IPA size on FEAT_LPA2 systems With all the page-table infrastructure in place, we can finally increase the maximum permisable IPA size to 52-bits on 4KB and 16KB page systems that have FEAT_LPA2. Reviewed-by: Oliver Upton Signed-off-by: Ryan Roberts Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20231127111737.1897081-11-ryan.roberts@arm.com --- arch/arm64/kvm/reset.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c index 5bb4de162cab..68d1d05672bd 100644 --- a/arch/arm64/kvm/reset.c +++ b/arch/arm64/kvm/reset.c @@ -280,12 +280,11 @@ int __init kvm_set_ipa_limit(void) parange = cpuid_feature_extract_unsigned_field(mmfr0, ID_AA64MMFR0_EL1_PARANGE_SHIFT); /* - * IPA size beyond 48 bits could not be supported - * on either 4K or 16K page size. Hence let's cap - * it to 48 bits, in case it's reported as larger - * on the system. + * IPA size beyond 48 bits for 4K and 16K page size is only supported + * when LPA2 is available. So if we have LPA2, enable it, else cap to 48 + * bits, in case it's reported as larger on the system. */ - if (PAGE_SIZE != SZ_64K) + if (!kvm_lpa2_is_enabled() && PAGE_SIZE != SZ_64K) parange = min(parange, (unsigned int)ID_AA64MMFR0_EL1_PARANGE_48); /* From 72324ac52ddddb168d8008e36d6a617b9b74f6c1 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Mon, 27 Nov 2023 11:17:36 +0000 Subject: [PATCH 048/336] KVM: selftests: arm64: Determine max ipa size per-page size We are about to add 52 bit PA guest modes for 4K and 16K pages when the system supports LPA2. In preparation beef up the logic that parses mmfr0 to also tell us what the maximum supported PA size is for each page size. Max PA size = 0 implies the page size is not supported at all. Reviewed-by: Oliver Upton Signed-off-by: Ryan Roberts Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20231127111737.1897081-12-ryan.roberts@arm.com --- .../selftests/kvm/include/aarch64/processor.h | 4 +- .../selftests/kvm/include/guest_modes.h | 4 +- .../selftests/kvm/lib/aarch64/processor.c | 30 ++++++++++-- tools/testing/selftests/kvm/lib/guest_modes.c | 48 ++++++++----------- 4 files changed, 50 insertions(+), 36 deletions(-) diff --git a/tools/testing/selftests/kvm/include/aarch64/processor.h b/tools/testing/selftests/kvm/include/aarch64/processor.h index c42d683102c7..cf20e44e86f2 100644 --- a/tools/testing/selftests/kvm/include/aarch64/processor.h +++ b/tools/testing/selftests/kvm/include/aarch64/processor.h @@ -119,8 +119,8 @@ enum { /* Access flag update enable/disable */ #define TCR_EL1_HA (1ULL << 39) -void aarch64_get_supported_page_sizes(uint32_t ipa, - bool *ps4k, bool *ps16k, bool *ps64k); +void aarch64_get_supported_page_sizes(uint32_t ipa, uint32_t *ipa4k, + uint32_t *ipa16k, uint32_t *ipa64k); void vm_init_descriptor_tables(struct kvm_vm *vm); void vcpu_init_descriptor_tables(struct kvm_vcpu *vcpu); diff --git a/tools/testing/selftests/kvm/include/guest_modes.h b/tools/testing/selftests/kvm/include/guest_modes.h index b691df33e64e..63f5167397cc 100644 --- a/tools/testing/selftests/kvm/include/guest_modes.h +++ b/tools/testing/selftests/kvm/include/guest_modes.h @@ -11,8 +11,8 @@ struct guest_mode { extern struct guest_mode guest_modes[NUM_VM_MODES]; -#define guest_mode_append(mode, supported, enabled) ({ \ - guest_modes[mode] = (struct guest_mode){ supported, enabled }; \ +#define guest_mode_append(mode, enabled) ({ \ + guest_modes[mode] = (struct guest_mode){ (enabled), (enabled) }; \ }) void guest_modes_append_default(void); diff --git a/tools/testing/selftests/kvm/lib/aarch64/processor.c b/tools/testing/selftests/kvm/lib/aarch64/processor.c index 6fe12e985ba5..e6ffd9037c37 100644 --- a/tools/testing/selftests/kvm/lib/aarch64/processor.c +++ b/tools/testing/selftests/kvm/lib/aarch64/processor.c @@ -492,12 +492,24 @@ uint32_t guest_get_vcpuid(void) return read_sysreg(tpidr_el1); } -void aarch64_get_supported_page_sizes(uint32_t ipa, - bool *ps4k, bool *ps16k, bool *ps64k) +static uint32_t max_ipa_for_page_size(uint32_t vm_ipa, uint32_t gran, + uint32_t not_sup_val, uint32_t ipa52_min_val) +{ + if (gran == not_sup_val) + return 0; + else if (gran >= ipa52_min_val && vm_ipa >= 52) + return 52; + else + return min(vm_ipa, 48U); +} + +void aarch64_get_supported_page_sizes(uint32_t ipa, uint32_t *ipa4k, + uint32_t *ipa16k, uint32_t *ipa64k) { struct kvm_vcpu_init preferred_init; int kvm_fd, vm_fd, vcpu_fd, err; uint64_t val; + uint32_t gran; struct kvm_one_reg reg = { .id = KVM_ARM64_SYS_REG(SYS_ID_AA64MMFR0_EL1), .addr = (uint64_t)&val, @@ -518,9 +530,17 @@ void aarch64_get_supported_page_sizes(uint32_t ipa, err = ioctl(vcpu_fd, KVM_GET_ONE_REG, ®); TEST_ASSERT(err == 0, KVM_IOCTL_ERROR(KVM_GET_ONE_REG, vcpu_fd)); - *ps4k = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_TGRAN4), val) != 0xf; - *ps64k = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_TGRAN64), val) == 0; - *ps16k = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_TGRAN16), val) != 0; + gran = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_TGRAN4), val); + *ipa4k = max_ipa_for_page_size(ipa, gran, ID_AA64MMFR0_EL1_TGRAN4_NI, + ID_AA64MMFR0_EL1_TGRAN4_52_BIT); + + gran = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_TGRAN64), val); + *ipa64k = max_ipa_for_page_size(ipa, gran, ID_AA64MMFR0_EL1_TGRAN64_NI, + ID_AA64MMFR0_EL1_TGRAN64_IMP); + + gran = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_TGRAN16), val); + *ipa16k = max_ipa_for_page_size(ipa, gran, ID_AA64MMFR0_EL1_TGRAN16_NI, + ID_AA64MMFR0_EL1_TGRAN16_52_BIT); close(vcpu_fd); close(vm_fd); diff --git a/tools/testing/selftests/kvm/lib/guest_modes.c b/tools/testing/selftests/kvm/lib/guest_modes.c index 1df3ce4b16fd..772a7dd15db4 100644 --- a/tools/testing/selftests/kvm/lib/guest_modes.c +++ b/tools/testing/selftests/kvm/lib/guest_modes.c @@ -14,37 +14,31 @@ struct guest_mode guest_modes[NUM_VM_MODES]; void guest_modes_append_default(void) { #ifndef __aarch64__ - guest_mode_append(VM_MODE_DEFAULT, true, true); + guest_mode_append(VM_MODE_DEFAULT, true); #else { unsigned int limit = kvm_check_cap(KVM_CAP_ARM_VM_IPA_SIZE); - bool ps4k, ps16k, ps64k; + uint32_t ipa4k, ipa16k, ipa64k; int i; - aarch64_get_supported_page_sizes(limit, &ps4k, &ps16k, &ps64k); + aarch64_get_supported_page_sizes(limit, &ipa4k, &ipa16k, &ipa64k); - vm_mode_default = NUM_VM_MODES; + guest_mode_append(VM_MODE_P52V48_64K, ipa64k >= 52); - if (limit >= 52) - guest_mode_append(VM_MODE_P52V48_64K, ps64k, ps64k); - if (limit >= 48) { - guest_mode_append(VM_MODE_P48V48_4K, ps4k, ps4k); - guest_mode_append(VM_MODE_P48V48_16K, ps16k, ps16k); - guest_mode_append(VM_MODE_P48V48_64K, ps64k, ps64k); - } - if (limit >= 40) { - guest_mode_append(VM_MODE_P40V48_4K, ps4k, ps4k); - guest_mode_append(VM_MODE_P40V48_16K, ps16k, ps16k); - guest_mode_append(VM_MODE_P40V48_64K, ps64k, ps64k); - if (ps4k) - vm_mode_default = VM_MODE_P40V48_4K; - } - if (limit >= 36) { - guest_mode_append(VM_MODE_P36V48_4K, ps4k, ps4k); - guest_mode_append(VM_MODE_P36V48_16K, ps16k, ps16k); - guest_mode_append(VM_MODE_P36V48_64K, ps64k, ps64k); - guest_mode_append(VM_MODE_P36V47_16K, ps16k, ps16k); - } + guest_mode_append(VM_MODE_P48V48_4K, ipa4k >= 48); + guest_mode_append(VM_MODE_P48V48_16K, ipa16k >= 48); + guest_mode_append(VM_MODE_P48V48_64K, ipa64k >= 48); + + guest_mode_append(VM_MODE_P40V48_4K, ipa4k >= 40); + guest_mode_append(VM_MODE_P40V48_16K, ipa16k >= 40); + guest_mode_append(VM_MODE_P40V48_64K, ipa64k >= 40); + + guest_mode_append(VM_MODE_P36V48_4K, ipa4k >= 36); + guest_mode_append(VM_MODE_P36V48_16K, ipa16k >= 36); + guest_mode_append(VM_MODE_P36V48_64K, ipa64k >= 36); + guest_mode_append(VM_MODE_P36V47_16K, ipa16k >= 36); + + vm_mode_default = ipa4k >= 40 ? VM_MODE_P40V48_4K : NUM_VM_MODES; /* * Pick the first supported IPA size if the default @@ -72,7 +66,7 @@ void guest_modes_append_default(void) close(kvm_fd); /* Starting with z13 we have 47bits of physical address */ if (info.ibc >= 0x30) - guest_mode_append(VM_MODE_P47V64_4K, true, true); + guest_mode_append(VM_MODE_P47V64_4K, true); } #endif #ifdef __riscv @@ -80,9 +74,9 @@ void guest_modes_append_default(void) unsigned int sz = kvm_check_cap(KVM_CAP_VM_GPA_BITS); if (sz >= 52) - guest_mode_append(VM_MODE_P52V48_4K, true, true); + guest_mode_append(VM_MODE_P52V48_4K, true); if (sz >= 48) - guest_mode_append(VM_MODE_P48V48_4K, true, true); + guest_mode_append(VM_MODE_P48V48_4K, true); } #endif } From 10a0cc3b688fcf753ff3f6518bb15e7a6809e908 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Mon, 27 Nov 2023 11:17:37 +0000 Subject: [PATCH 049/336] KVM: selftests: arm64: Support P52V48 4K and 16K guest_modes Add support for VM_MODE_P52V48_4K and VM_MODE_P52V48_16K guest modes by using the FEAT_LPA2 pte format for stage1, when FEAT_LPA2 is available. Reviewed-by: Oliver Upton Signed-off-by: Ryan Roberts Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20231127111737.1897081-13-ryan.roberts@arm.com --- .../selftests/kvm/include/kvm_util_base.h | 1 + .../selftests/kvm/lib/aarch64/processor.c | 39 ++++++++++++++----- tools/testing/selftests/kvm/lib/guest_modes.c | 2 + tools/testing/selftests/kvm/lib/kvm_util.c | 3 ++ 4 files changed, 36 insertions(+), 9 deletions(-) diff --git a/tools/testing/selftests/kvm/include/kvm_util_base.h b/tools/testing/selftests/kvm/include/kvm_util_base.h index a18db6a7b3cf..406500fb6e28 100644 --- a/tools/testing/selftests/kvm/include/kvm_util_base.h +++ b/tools/testing/selftests/kvm/include/kvm_util_base.h @@ -171,6 +171,7 @@ static inline struct userspace_mem_region *vm_get_mem_region(struct kvm_vm *vm, enum vm_guest_mode { VM_MODE_P52V48_4K, + VM_MODE_P52V48_16K, VM_MODE_P52V48_64K, VM_MODE_P48V48_4K, VM_MODE_P48V48_16K, diff --git a/tools/testing/selftests/kvm/lib/aarch64/processor.c b/tools/testing/selftests/kvm/lib/aarch64/processor.c index e6ffd9037c37..41c776b642c0 100644 --- a/tools/testing/selftests/kvm/lib/aarch64/processor.c +++ b/tools/testing/selftests/kvm/lib/aarch64/processor.c @@ -12,6 +12,7 @@ #include "kvm_util.h" #include "processor.h" #include +#include #define DEFAULT_ARM64_GUEST_STACK_VADDR_MIN 0xac0000 @@ -58,13 +59,25 @@ static uint64_t pte_index(struct kvm_vm *vm, vm_vaddr_t gva) return (gva >> vm->page_shift) & mask; } +static inline bool use_lpa2_pte_format(struct kvm_vm *vm) +{ + return (vm->page_size == SZ_4K || vm->page_size == SZ_16K) && + (vm->pa_bits > 48 || vm->va_bits > 48); +} + static uint64_t addr_pte(struct kvm_vm *vm, uint64_t pa, uint64_t attrs) { uint64_t pte; - pte = pa & GENMASK(47, vm->page_shift); - if (vm->page_shift == 16) - pte |= FIELD_GET(GENMASK(51, 48), pa) << 12; + if (use_lpa2_pte_format(vm)) { + pte = pa & GENMASK(49, vm->page_shift); + pte |= FIELD_GET(GENMASK(51, 50), pa) << 8; + attrs &= ~GENMASK(9, 8); + } else { + pte = pa & GENMASK(47, vm->page_shift); + if (vm->page_shift == 16) + pte |= FIELD_GET(GENMASK(51, 48), pa) << 12; + } pte |= attrs; return pte; @@ -74,9 +87,14 @@ static uint64_t pte_addr(struct kvm_vm *vm, uint64_t pte) { uint64_t pa; - pa = pte & GENMASK(47, vm->page_shift); - if (vm->page_shift == 16) - pa |= FIELD_GET(GENMASK(15, 12), pte) << 48; + if (use_lpa2_pte_format(vm)) { + pa = pte & GENMASK(49, vm->page_shift); + pa |= FIELD_GET(GENMASK(9, 8), pte) << 50; + } else { + pa = pte & GENMASK(47, vm->page_shift); + if (vm->page_shift == 16) + pa |= FIELD_GET(GENMASK(15, 12), pte) << 48; + } return pa; } @@ -266,9 +284,6 @@ void aarch64_vcpu_setup(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init) /* Configure base granule size */ switch (vm->mode) { - case VM_MODE_P52V48_4K: - TEST_FAIL("AArch64 does not support 4K sized pages " - "with 52-bit physical address ranges"); case VM_MODE_PXXV48_4K: TEST_FAIL("AArch64 does not support 4K sized pages " "with ANY-bit physical address ranges"); @@ -278,12 +293,14 @@ void aarch64_vcpu_setup(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init) case VM_MODE_P36V48_64K: tcr_el1 |= 1ul << 14; /* TG0 = 64KB */ break; + case VM_MODE_P52V48_16K: case VM_MODE_P48V48_16K: case VM_MODE_P40V48_16K: case VM_MODE_P36V48_16K: case VM_MODE_P36V47_16K: tcr_el1 |= 2ul << 14; /* TG0 = 16KB */ break; + case VM_MODE_P52V48_4K: case VM_MODE_P48V48_4K: case VM_MODE_P40V48_4K: case VM_MODE_P36V48_4K: @@ -297,6 +314,8 @@ void aarch64_vcpu_setup(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init) /* Configure output size */ switch (vm->mode) { + case VM_MODE_P52V48_4K: + case VM_MODE_P52V48_16K: case VM_MODE_P52V48_64K: tcr_el1 |= 6ul << 32; /* IPS = 52 bits */ ttbr0_el1 |= FIELD_GET(GENMASK(51, 48), vm->pgd) << 2; @@ -325,6 +344,8 @@ void aarch64_vcpu_setup(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init) /* TCR_EL1 |= IRGN0:WBWA | ORGN0:WBWA | SH0:Inner-Shareable */; tcr_el1 |= (1 << 8) | (1 << 10) | (3 << 12); tcr_el1 |= (64 - vm->va_bits) /* T0SZ */; + if (use_lpa2_pte_format(vm)) + tcr_el1 |= (1ul << 59) /* DS */; vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL1), sctlr_el1); vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL1), tcr_el1); diff --git a/tools/testing/selftests/kvm/lib/guest_modes.c b/tools/testing/selftests/kvm/lib/guest_modes.c index 772a7dd15db4..b04901e55138 100644 --- a/tools/testing/selftests/kvm/lib/guest_modes.c +++ b/tools/testing/selftests/kvm/lib/guest_modes.c @@ -23,6 +23,8 @@ void guest_modes_append_default(void) aarch64_get_supported_page_sizes(limit, &ipa4k, &ipa16k, &ipa64k); + guest_mode_append(VM_MODE_P52V48_4K, ipa4k >= 52); + guest_mode_append(VM_MODE_P52V48_16K, ipa16k >= 52); guest_mode_append(VM_MODE_P52V48_64K, ipa64k >= 52); guest_mode_append(VM_MODE_P48V48_4K, ipa4k >= 48); diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 7a8af1821f5d..aeba7a23105c 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -148,6 +148,7 @@ const char *vm_guest_mode_string(uint32_t i) { static const char * const strings[] = { [VM_MODE_P52V48_4K] = "PA-bits:52, VA-bits:48, 4K pages", + [VM_MODE_P52V48_16K] = "PA-bits:52, VA-bits:48, 16K pages", [VM_MODE_P52V48_64K] = "PA-bits:52, VA-bits:48, 64K pages", [VM_MODE_P48V48_4K] = "PA-bits:48, VA-bits:48, 4K pages", [VM_MODE_P48V48_16K] = "PA-bits:48, VA-bits:48, 16K pages", @@ -173,6 +174,7 @@ const char *vm_guest_mode_string(uint32_t i) const struct vm_guest_mode_params vm_guest_mode_params[] = { [VM_MODE_P52V48_4K] = { 52, 48, 0x1000, 12 }, + [VM_MODE_P52V48_16K] = { 52, 48, 0x4000, 14 }, [VM_MODE_P52V48_64K] = { 52, 48, 0x10000, 16 }, [VM_MODE_P48V48_4K] = { 48, 48, 0x1000, 12 }, [VM_MODE_P48V48_16K] = { 48, 48, 0x4000, 14 }, @@ -251,6 +253,7 @@ struct kvm_vm *____vm_create(enum vm_guest_mode mode) case VM_MODE_P36V48_64K: vm->pgtable_levels = 3; break; + case VM_MODE_P52V48_16K: case VM_MODE_P48V48_16K: case VM_MODE_P40V48_16K: case VM_MODE_P36V48_16K: From 3b99d46a1170754a06f379a83be8101c5f6bfc46 Mon Sep 17 00:00:00 2001 From: angquan yu Date: Tue, 28 Nov 2023 16:11:05 -0600 Subject: [PATCH 050/336] KVM: selftests: Actually print out magic token in NX hugepages skip message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pass MAGIC_TOKEN to __TEST_REQUIRE() when printing the help message about needing to pass a magic value to manually run the NX hugepages test, otherwise the help message will contain garbage. In file included from x86_64/nx_huge_pages_test.c:15: x86_64/nx_huge_pages_test.c: In function ‘main’: include/test_util.h:40:32: error: format ‘%d’ expects a matching ‘int’ argument [-Werror=format=] 40 | ksft_exit_skip("- " fmt "\n", ##__VA_ARGS__); \ | ^~~~ x86_64/nx_huge_pages_test.c:259:9: note: in expansion of macro ‘__TEST_REQUIRE’ 259 | __TEST_REQUIRE(token == MAGIC_TOKEN, | ^~~~~~~~~~~~~~ Signed-off-by: angquan yu Link: https://lore.kernel.org/r/20231128221105.63093-1-angquan21@gmail.com [sean: rewrite shortlog+changelog] Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/x86_64/nx_huge_pages_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/x86_64/nx_huge_pages_test.c b/tools/testing/selftests/kvm/x86_64/nx_huge_pages_test.c index 18ac5c1952a3..83e25bccc139 100644 --- a/tools/testing/selftests/kvm/x86_64/nx_huge_pages_test.c +++ b/tools/testing/selftests/kvm/x86_64/nx_huge_pages_test.c @@ -259,7 +259,7 @@ int main(int argc, char **argv) __TEST_REQUIRE(token == MAGIC_TOKEN, "This test must be run with the magic token %d.\n" "This is done by nx_huge_pages_test.sh, which\n" - "also handles environment setup for the test."); + "also handles environment setup for the test.", MAGIC_TOKEN); run_test(reclaim_period_ms, false, reboot_permissions); run_test(reclaim_period_ms, true, reboot_permissions); From 7b0dd9430cf0c1ae19645d2a6608a5fb57faffe4 Mon Sep 17 00:00:00 2001 From: Binbin Wu Date: Wed, 13 Sep 2023 20:42:12 +0800 Subject: [PATCH 051/336] KVM: x86: Consolidate flags for __linearize() Consolidate @write and @fetch of __linearize() into a set of flags so that additional flags can be added without needing more/new boolean parameters, to precisely identify the access type. No functional change intended. Signed-off-by: Binbin Wu Reviewed-by: Chao Gao Acked-by: Kai Huang Tested-by: Xuelian Guo Link: https://lore.kernel.org/r/20230913124227.12574-2-binbin.wu@linux.intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/emulate.c | 21 +++++++++++---------- arch/x86/kvm/kvm_emulate.h | 4 ++++ 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index 2673cd5c46cb..87ee1802166a 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -687,8 +687,8 @@ static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size) static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt, struct segmented_address addr, unsigned *max_size, unsigned size, - bool write, bool fetch, - enum x86emul_mode mode, ulong *linear) + enum x86emul_mode mode, ulong *linear, + unsigned int flags) { struct desc_struct desc; bool usable; @@ -717,11 +717,11 @@ static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt, if (!usable) goto bad; /* code segment in protected mode or read-only data segment */ - if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8)) - || !(desc.type & 2)) && write) + if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8)) || !(desc.type & 2)) && + (flags & X86EMUL_F_WRITE)) goto bad; /* unreadable code segment */ - if (!fetch && (desc.type & 8) && !(desc.type & 2)) + if (!(flags & X86EMUL_F_FETCH) && (desc.type & 8) && !(desc.type & 2)) goto bad; lim = desc_limit_scaled(&desc); if (!(desc.type & 8) && (desc.type & 4)) { @@ -757,8 +757,8 @@ static int linearize(struct x86_emulate_ctxt *ctxt, ulong *linear) { unsigned max_size; - return __linearize(ctxt, addr, &max_size, size, write, false, - ctxt->mode, linear); + return __linearize(ctxt, addr, &max_size, size, ctxt->mode, linear, + write ? X86EMUL_F_WRITE : 0); } static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst) @@ -771,7 +771,8 @@ static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst) if (ctxt->op_bytes != sizeof(unsigned long)) addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1); - rc = __linearize(ctxt, addr, &max_size, 1, false, true, ctxt->mode, &linear); + rc = __linearize(ctxt, addr, &max_size, 1, ctxt->mode, &linear, + X86EMUL_F_FETCH); if (rc == X86EMUL_CONTINUE) ctxt->_eip = addr.ea; return rc; @@ -907,8 +908,8 @@ static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size) * boundary check itself. Instead, we use max_size to check * against op_size. */ - rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode, - &linear); + rc = __linearize(ctxt, addr, &max_size, 0, ctxt->mode, &linear, + X86EMUL_F_FETCH); if (unlikely(rc != X86EMUL_CONTINUE)) return rc; diff --git a/arch/x86/kvm/kvm_emulate.h b/arch/x86/kvm/kvm_emulate.h index be7aeb9b8ea3..e24c8ac7b930 100644 --- a/arch/x86/kvm/kvm_emulate.h +++ b/arch/x86/kvm/kvm_emulate.h @@ -88,6 +88,10 @@ struct x86_instruction_info { #define X86EMUL_IO_NEEDED 5 /* IO is needed to complete emulation */ #define X86EMUL_INTERCEPTED 6 /* Intercepted by nested VMCB/VMCS */ +/* x86-specific emulation flags */ +#define X86EMUL_F_WRITE BIT(0) +#define X86EMUL_F_FETCH BIT(1) + struct x86_emulate_ops { void (*vm_bugged)(struct x86_emulate_ctxt *ctxt); /* From 3963c52df42231f72277cd138994ac94f1183d2b Mon Sep 17 00:00:00 2001 From: Binbin Wu Date: Wed, 13 Sep 2023 20:42:14 +0800 Subject: [PATCH 052/336] KVM: x86: Add an emulation flag for implicit system access Add an emulation flag X86EMUL_F_IMPLICIT to identify implicit system access in instruction emulation. Don't bother wiring up any usage at this point, as Linear Address Space Separation (LASS) will be the first "real" consumer of the flag and LASS support will require dedicated hooks, i.e. there aren't any existing calls where passing X86EMUL_F_IMPLICIT is meaningful. Add the IMPLICIT flag even though there's no imminent usage so that Linear Address Masking (LAM) support can reference the flag to document that addresses for implicit accesses aren't untagged. Signed-off-by: Binbin Wu Tested-by: Xuelian Guo Link: https://lore.kernel.org/r/20230913124227.12574-4-binbin.wu@linux.intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/kvm_emulate.h | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/kvm/kvm_emulate.h b/arch/x86/kvm/kvm_emulate.h index e24c8ac7b930..65fc7ef5ca3d 100644 --- a/arch/x86/kvm/kvm_emulate.h +++ b/arch/x86/kvm/kvm_emulate.h @@ -91,6 +91,7 @@ struct x86_instruction_info { /* x86-specific emulation flags */ #define X86EMUL_F_WRITE BIT(0) #define X86EMUL_F_FETCH BIT(1) +#define X86EMUL_F_IMPLICIT BIT(2) struct x86_emulate_ops { void (*vm_bugged)(struct x86_emulate_ctxt *ctxt); From 538ac9a92d669c4ccfc64739a32efab2793cea1d Mon Sep 17 00:00:00 2001 From: Binbin Wu Date: Wed, 13 Sep 2023 20:42:15 +0800 Subject: [PATCH 053/336] KVM: x86: Add X86EMUL_F_INVLPG and pass it in em_invlpg() Add an emulation flag X86EMUL_F_INVLPG, which is used to identify an instruction that does TLB invalidation without true memory access. Only invlpg & invlpga implemented in emulator belong to this kind. invlpga doesn't need additional information for emulation. Just pass the flag to em_invlpg(). Linear Address Masking (LAM) and Linear Address Space Separation (LASS) don't apply to addresses that are inputs to TLB invalidation. The flag will be consumed to support LAM/LASS virtualization. Signed-off-by: Binbin Wu Tested-by: Xuelian Guo Link: https://lore.kernel.org/r/20230913124227.12574-5-binbin.wu@linux.intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/emulate.c | 4 +++- arch/x86/kvm/kvm_emulate.h | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index 87ee1802166a..ceec8c5f9687 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -3440,8 +3440,10 @@ static int em_invlpg(struct x86_emulate_ctxt *ctxt) { int rc; ulong linear; + unsigned int max_size; - rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear); + rc = __linearize(ctxt, ctxt->src.addr.mem, &max_size, 1, ctxt->mode, + &linear, X86EMUL_F_INVLPG); if (rc == X86EMUL_CONTINUE) ctxt->ops->invlpg(ctxt, linear); /* Disable writeback. */ diff --git a/arch/x86/kvm/kvm_emulate.h b/arch/x86/kvm/kvm_emulate.h index 65fc7ef5ca3d..8bd9b23543cc 100644 --- a/arch/x86/kvm/kvm_emulate.h +++ b/arch/x86/kvm/kvm_emulate.h @@ -92,6 +92,7 @@ struct x86_instruction_info { #define X86EMUL_F_WRITE BIT(0) #define X86EMUL_F_FETCH BIT(1) #define X86EMUL_F_IMPLICIT BIT(2) +#define X86EMUL_F_INVLPG BIT(3) struct x86_emulate_ops { void (*vm_bugged)(struct x86_emulate_ctxt *ctxt); From a130066f74008858ac425b7497d231742474a0ea Mon Sep 17 00:00:00 2001 From: Binbin Wu Date: Wed, 13 Sep 2023 20:42:16 +0800 Subject: [PATCH 054/336] KVM: x86/mmu: Drop non-PA bits when getting GFN for guest's PGD Drop non-PA bits when getting GFN for guest's PGD with the maximum theoretical mask for guest MAXPHYADDR. Do it unconditionally because it's harmless for 32-bit guests, querying 64-bit mode would be more expensive, and for EPT the mask isn't tied to guest mode. Using PT_BASE_ADDR_MASK would be technically wrong (PAE paging has 64-bit elements _except_ for CR3, which has only 32 valid bits), it wouldn't matter in practice though. Opportunistically use GENMASK_ULL() to define __PT_BASE_ADDR_MASK. Signed-off-by: Binbin Wu Tested-by: Xuelian Guo Link: https://lore.kernel.org/r/20230913124227.12574-6-binbin.wu@linux.intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 2 +- arch/x86/kvm/mmu/mmu_internal.h | 1 + arch/x86/kvm/mmu/paging_tmpl.h | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 59b026b6ad2a..73070650b143 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3806,7 +3806,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) hpa_t root; root_pgd = kvm_mmu_get_guest_pgd(vcpu, mmu); - root_gfn = root_pgd >> PAGE_SHIFT; + root_gfn = (root_pgd & __PT_BASE_ADDR_MASK) >> PAGE_SHIFT; if (!kvm_vcpu_is_visible_gfn(vcpu, root_gfn)) { mmu->root.hpa = kvm_mmu_get_dummy_root(); diff --git a/arch/x86/kvm/mmu/mmu_internal.h b/arch/x86/kvm/mmu/mmu_internal.h index b66a7d47e0e4..0669a8a668ca 100644 --- a/arch/x86/kvm/mmu/mmu_internal.h +++ b/arch/x86/kvm/mmu/mmu_internal.h @@ -13,6 +13,7 @@ #endif /* Page table builder macros common to shadow (host) PTEs and guest PTEs. */ +#define __PT_BASE_ADDR_MASK GENMASK_ULL(51, 12) #define __PT_LEVEL_SHIFT(level, bits_per_level) \ (PAGE_SHIFT + ((level) - 1) * (bits_per_level)) #define __PT_INDEX(address, level, bits_per_level) \ diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index c85255073f67..4d4e98fe4f35 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -62,7 +62,7 @@ #endif /* Common logic, but per-type values. These also need to be undefined. */ -#define PT_BASE_ADDR_MASK ((pt_element_t)(((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))) +#define PT_BASE_ADDR_MASK ((pt_element_t)__PT_BASE_ADDR_MASK) #define PT_LVL_ADDR_MASK(lvl) __PT_LVL_ADDR_MASK(PT_BASE_ADDR_MASK, lvl, PT_LEVEL_BITS) #define PT_LVL_OFFSET_MASK(lvl) __PT_LVL_OFFSET_MASK(PT_BASE_ADDR_MASK, lvl, PT_LEVEL_BITS) #define PT_INDEX(addr, lvl) __PT_INDEX(addr, lvl, PT_LEVEL_BITS) From 2c49db455ee27c72a680c9e4fad1c12433902ee3 Mon Sep 17 00:00:00 2001 From: Binbin Wu Date: Wed, 13 Sep 2023 20:42:17 +0800 Subject: [PATCH 055/336] KVM: x86: Add & use kvm_vcpu_is_legal_cr3() to check CR3's legality Add and use kvm_vcpu_is_legal_cr3() to check CR3's legality to provide a clear distinction between CR3 and GPA checks. This will allow exempting bits from kvm_vcpu_is_legal_cr3() without affecting general GPA checks, e.g. for upcoming features that will use high bits in CR3 for feature enabling. No functional change intended. Signed-off-by: Binbin Wu Tested-by: Xuelian Guo Link: https://lore.kernel.org/r/20230913124227.12574-7-binbin.wu@linux.intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/cpuid.h | 5 +++++ arch/x86/kvm/svm/nested.c | 4 ++-- arch/x86/kvm/vmx/nested.c | 4 ++-- arch/x86/kvm/x86.c | 4 ++-- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/cpuid.h b/arch/x86/kvm/cpuid.h index 0b90532b6e26..5fd3f2d1ca14 100644 --- a/arch/x86/kvm/cpuid.h +++ b/arch/x86/kvm/cpuid.h @@ -279,4 +279,9 @@ static __always_inline bool guest_can_use(struct kvm_vcpu *vcpu, vcpu->arch.governed_features.enabled); } +static inline bool kvm_vcpu_is_legal_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) +{ + return kvm_vcpu_is_legal_gpa(vcpu, cr3); +} + #endif diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 3fea8c47679e..90ca9489aab6 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -311,7 +311,7 @@ static bool __nested_vmcb_check_save(struct kvm_vcpu *vcpu, if ((save->efer & EFER_LME) && (save->cr0 & X86_CR0_PG)) { if (CC(!(save->cr4 & X86_CR4_PAE)) || CC(!(save->cr0 & X86_CR0_PE)) || - CC(kvm_vcpu_is_illegal_gpa(vcpu, save->cr3))) + CC(!kvm_vcpu_is_legal_cr3(vcpu, save->cr3))) return false; } @@ -520,7 +520,7 @@ static void nested_svm_transition_tlb_flush(struct kvm_vcpu *vcpu) static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool nested_npt, bool reload_pdptrs) { - if (CC(kvm_vcpu_is_illegal_gpa(vcpu, cr3))) + if (CC(!kvm_vcpu_is_legal_cr3(vcpu, cr3))) return -EINVAL; if (reload_pdptrs && !nested_npt && is_pae_paging(vcpu) && diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index c5ec0ef51ff7..db61cf8e3128 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -1085,7 +1085,7 @@ static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool nested_ept, bool reload_pdptrs, enum vm_entry_failure_code *entry_failure_code) { - if (CC(kvm_vcpu_is_illegal_gpa(vcpu, cr3))) { + if (CC(!kvm_vcpu_is_legal_cr3(vcpu, cr3))) { *entry_failure_code = ENTRY_FAIL_DEFAULT; return -EINVAL; } @@ -2912,7 +2912,7 @@ static int nested_vmx_check_host_state(struct kvm_vcpu *vcpu, if (CC(!nested_host_cr0_valid(vcpu, vmcs12->host_cr0)) || CC(!nested_host_cr4_valid(vcpu, vmcs12->host_cr4)) || - CC(kvm_vcpu_is_illegal_gpa(vcpu, vmcs12->host_cr3))) + CC(!kvm_vcpu_is_legal_cr3(vcpu, vmcs12->host_cr3))) return -EINVAL; if (CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_esp, vcpu)) || diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 6d0772b47041..3a648b6e632d 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1284,7 +1284,7 @@ int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) * stuff CR3, e.g. for RSM emulation, and there is no guarantee that * the current vCPU mode is accurate. */ - if (kvm_vcpu_is_illegal_gpa(vcpu, cr3)) + if (!kvm_vcpu_is_legal_cr3(vcpu, cr3)) return 1; if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, cr3)) @@ -11612,7 +11612,7 @@ static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) */ if (!(sregs->cr4 & X86_CR4_PAE) || !(sregs->efer & EFER_LMA)) return false; - if (kvm_vcpu_is_illegal_gpa(vcpu, sregs->cr3)) + if (!kvm_vcpu_is_legal_cr3(vcpu, sregs->cr3)) return false; } else { /* From 9c8021d4ae85f1531230fc33653e06e9f1fdb7f1 Mon Sep 17 00:00:00 2001 From: Binbin Wu Date: Wed, 13 Sep 2023 20:42:18 +0800 Subject: [PATCH 056/336] KVM: x86: Remove kvm_vcpu_is_illegal_gpa() Remove kvm_vcpu_is_illegal_gpa() and use !kvm_vcpu_is_legal_gpa() instead. The "illegal" helper actually predates the "legal" helper, the only reason the "illegal" variant wasn't removed by commit 4bda0e97868a ("KVM: x86: Add a helper to check for a legal GPA") was to avoid code churn. Now that CR3 has a dedicated helper, there are fewer callers, and so the code churn isn't that much of a deterrent. No functional change intended. Signed-off-by: Binbin Wu Tested-by: Xuelian Guo Link: https://lore.kernel.org/r/20230913124227.12574-8-binbin.wu@linux.intel.com [sean: provide a bit of history in the changelog] Signed-off-by: Sean Christopherson --- arch/x86/kvm/cpuid.h | 5 ----- arch/x86/kvm/vmx/nested.c | 2 +- arch/x86/kvm/vmx/vmx.c | 2 +- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/arch/x86/kvm/cpuid.h b/arch/x86/kvm/cpuid.h index 5fd3f2d1ca14..fa04a00090a2 100644 --- a/arch/x86/kvm/cpuid.h +++ b/arch/x86/kvm/cpuid.h @@ -47,11 +47,6 @@ static inline bool kvm_vcpu_is_legal_gpa(struct kvm_vcpu *vcpu, gpa_t gpa) return !(gpa & vcpu->arch.reserved_gpa_bits); } -static inline bool kvm_vcpu_is_illegal_gpa(struct kvm_vcpu *vcpu, gpa_t gpa) -{ - return !kvm_vcpu_is_legal_gpa(vcpu, gpa); -} - static inline bool kvm_vcpu_is_legal_aligned_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, gpa_t alignment) { diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index db61cf8e3128..51622878d6e4 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -2717,7 +2717,7 @@ static bool nested_vmx_check_eptp(struct kvm_vcpu *vcpu, u64 new_eptp) } /* Reserved bits should not be set */ - if (CC(kvm_vcpu_is_illegal_gpa(vcpu, new_eptp) || ((new_eptp >> 7) & 0x1f))) + if (CC(!kvm_vcpu_is_legal_gpa(vcpu, new_eptp) || ((new_eptp >> 7) & 0x1f))) return false; /* AD, if set, should be supported */ diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 40e3780d73ae..69072e418c73 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -5782,7 +5782,7 @@ static int handle_ept_violation(struct kvm_vcpu *vcpu) * would also use advanced VM-exit information for EPT violations to * reconstruct the page fault error code. */ - if (unlikely(allow_smaller_maxphyaddr && kvm_vcpu_is_illegal_gpa(vcpu, gpa))) + if (unlikely(allow_smaller_maxphyaddr && !kvm_vcpu_is_legal_gpa(vcpu, gpa))) return kvm_emulate_instruction(vcpu, 0); return kvm_mmu_page_fault(vcpu, gpa, error_code, NULL, 0); From 37a41847b770c722e98ace72f3851fb49b360c08 Mon Sep 17 00:00:00 2001 From: Binbin Wu Date: Wed, 13 Sep 2023 20:42:19 +0800 Subject: [PATCH 057/336] KVM: x86: Introduce get_untagged_addr() in kvm_x86_ops and call it in emulator Introduce a new interface get_untagged_addr() to kvm_x86_ops to untag the metadata from linear address. Call the interface in linearization of instruction emulator for 64-bit mode. When enabled feature like Intel Linear Address Masking (LAM) or AMD Upper Address Ignore (UAI), linear addresses may be tagged with metadata that needs to be dropped prior to canonicality checks, i.e. the metadata is ignored. Introduce get_untagged_addr() to kvm_x86_ops to hide the vendor specific code, as sadly LAM and UAI have different semantics. Pass the emulator flags to allow vendor specific implementation to precisely identify the access type (LAM doesn't untag certain accesses). Signed-off-by: Binbin Wu Reviewed-by: Chao Gao Tested-by: Xuelian Guo Link: https://lore.kernel.org/r/20230913124227.12574-9-binbin.wu@linux.intel.com [sean: massage changelog] Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm-x86-ops.h | 1 + arch/x86/include/asm/kvm_host.h | 2 ++ arch/x86/kvm/emulate.c | 2 +- arch/x86/kvm/kvm_emulate.h | 3 +++ arch/x86/kvm/x86.c | 10 ++++++++++ 5 files changed, 17 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index 26b628d84594..756791665117 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -135,6 +135,7 @@ KVM_X86_OP(msr_filter_changed) KVM_X86_OP(complete_emulated_msr) KVM_X86_OP(vcpu_deliver_sipi_vector) KVM_X86_OP_OPTIONAL_RET0(vcpu_get_apicv_inhibit_reasons); +KVM_X86_OP_OPTIONAL(get_untagged_addr) #undef KVM_X86_OP #undef KVM_X86_OP_OPTIONAL diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index a565a2e70f30..8d042f27d644 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1762,6 +1762,8 @@ struct kvm_x86_ops { * Returns vCPU specific APICv inhibit reasons */ unsigned long (*vcpu_get_apicv_inhibit_reasons)(struct kvm_vcpu *vcpu); + + gva_t (*get_untagged_addr)(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags); }; struct kvm_x86_nested_ops { diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index ceec8c5f9687..e223043ef5b2 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -701,7 +701,7 @@ static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt, *max_size = 0; switch (mode) { case X86EMUL_MODE_PROT64: - *linear = la; + *linear = la = ctxt->ops->get_untagged_addr(ctxt, la, flags); va_bits = ctxt_virt_addr_bits(ctxt); if (!__is_canonical_address(la, va_bits)) goto bad; diff --git a/arch/x86/kvm/kvm_emulate.h b/arch/x86/kvm/kvm_emulate.h index 8bd9b23543cc..e6d149825169 100644 --- a/arch/x86/kvm/kvm_emulate.h +++ b/arch/x86/kvm/kvm_emulate.h @@ -230,6 +230,9 @@ struct x86_emulate_ops { int (*leave_smm)(struct x86_emulate_ctxt *ctxt); void (*triple_fault)(struct x86_emulate_ctxt *ctxt); int (*set_xcr)(struct x86_emulate_ctxt *ctxt, u32 index, u64 xcr); + + gva_t (*get_untagged_addr)(struct x86_emulate_ctxt *ctxt, gva_t addr, + unsigned int flags); }; /* Type, address-of, and value of an instruction's operand. */ diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 3a648b6e632d..557872c0331a 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -8445,6 +8445,15 @@ static void emulator_vm_bugged(struct x86_emulate_ctxt *ctxt) kvm_vm_bugged(kvm); } +static gva_t emulator_get_untagged_addr(struct x86_emulate_ctxt *ctxt, + gva_t addr, unsigned int flags) +{ + if (!kvm_x86_ops.get_untagged_addr) + return addr; + + return static_call(kvm_x86_get_untagged_addr)(emul_to_vcpu(ctxt), addr, flags); +} + static const struct x86_emulate_ops emulate_ops = { .vm_bugged = emulator_vm_bugged, .read_gpr = emulator_read_gpr, @@ -8489,6 +8498,7 @@ static const struct x86_emulate_ops emulate_ops = { .leave_smm = emulator_leave_smm, .triple_fault = emulator_triple_fault, .set_xcr = emulator_set_xcr, + .get_untagged_addr = emulator_get_untagged_addr, }; static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask) From b39bd520a60c667a339e315ce7a3de2f7178f6e3 Mon Sep 17 00:00:00 2001 From: Binbin Wu Date: Wed, 13 Sep 2023 20:42:20 +0800 Subject: [PATCH 058/336] KVM: x86: Untag addresses for LAM emulation where applicable Stub in vmx_get_untagged_addr() and wire up calls from the emulator (via get_untagged_addr()) and "direct" calls from various VM-Exit handlers in VMX where LAM untagging is supposed to be applied. Defer implementing the guts of vmx_get_untagged_addr() to future patches purely to make the changes easier to consume. LAM is active only for 64-bit linear addresses and several types of accesses are exempted. - Cases need to untag address (handled in get_vmx_mem_address()) Operand(s) of VMX instructions and INVPCID. Operand(s) of SGX ENCLS. - Cases LAM doesn't apply to (no change needed) Operand of INVLPG. Linear address in INVPCID descriptor. Linear address in INVVPID descriptor. BASEADDR specified in SECS of ECREATE. Note: - LAM doesn't apply to write to control registers or MSRs - LAM masking is applied before walking page tables, i.e. the faulting linear address in CR2 doesn't contain the metadata. - The guest linear address saved in VMCS doesn't contain metadata. Signed-off-by: Binbin Wu Reviewed-by: Chao Gao Tested-by: Xuelian Guo Link: https://lore.kernel.org/r/20230913124227.12574-10-binbin.wu@linux.intel.com [sean: massage changelog] Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/nested.c | 5 +++++ arch/x86/kvm/vmx/sgx.c | 1 + arch/x86/kvm/vmx/vmx.c | 7 +++++++ arch/x86/kvm/vmx/vmx.h | 2 ++ arch/x86/kvm/x86.c | 4 ++++ 5 files changed, 19 insertions(+) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 51622878d6e4..4ba46e1b29d2 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -4980,6 +4980,7 @@ int get_vmx_mem_address(struct kvm_vcpu *vcpu, unsigned long exit_qualification, else *ret = off; + *ret = vmx_get_untagged_addr(vcpu, *ret, 0); /* Long mode: #GP(0)/#SS(0) if the memory address is in a * non-canonical form. This is the only check on the memory * destination for long mode! @@ -5797,6 +5798,10 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) vpid02 = nested_get_vpid02(vcpu); switch (type) { case VMX_VPID_EXTENT_INDIVIDUAL_ADDR: + /* + * LAM doesn't apply to addresses that are inputs to TLB + * invalidation. + */ if (!operand.vpid || is_noncanonical_address(operand.gla, vcpu)) return nested_vmx_fail(vcpu, diff --git a/arch/x86/kvm/vmx/sgx.c b/arch/x86/kvm/vmx/sgx.c index 3e822e582497..6fef01e0536e 100644 --- a/arch/x86/kvm/vmx/sgx.c +++ b/arch/x86/kvm/vmx/sgx.c @@ -37,6 +37,7 @@ static int sgx_get_encls_gva(struct kvm_vcpu *vcpu, unsigned long offset, if (!IS_ALIGNED(*gva, alignment)) { fault = true; } else if (likely(is_64_bit_mode(vcpu))) { + *gva = vmx_get_untagged_addr(vcpu, *gva, 0); fault = is_noncanonical_address(*gva, vcpu); } else { *gva &= 0xffffffff; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 69072e418c73..d7ff06d77720 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -8205,6 +8205,11 @@ static void vmx_vm_destroy(struct kvm *kvm) free_pages((unsigned long)kvm_vmx->pid_table, vmx_get_pid_table_order(kvm)); } +gva_t vmx_get_untagged_addr(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags) +{ + return gva; +} + static struct kvm_x86_ops vmx_x86_ops __initdata = { .name = KBUILD_MODNAME, @@ -8345,6 +8350,8 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .complete_emulated_msr = kvm_complete_insn_gp, .vcpu_deliver_sipi_vector = kvm_vcpu_deliver_sipi_vector, + + .get_untagged_addr = vmx_get_untagged_addr, }; static unsigned int vmx_handle_intel_pt_intr(void) diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index c2130d2c8e24..45cee1a8bc0a 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -420,6 +420,8 @@ void vmx_enable_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type); u64 vmx_get_l2_tsc_offset(struct kvm_vcpu *vcpu); u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu); +gva_t vmx_get_untagged_addr(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags); + static inline void vmx_set_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type, bool value) { diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 557872c0331a..ecfe97aa35c2 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -13563,6 +13563,10 @@ int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva) switch (type) { case INVPCID_TYPE_INDIV_ADDR: + /* + * LAM doesn't apply to addresses that are inputs to TLB + * invalidation. + */ if ((!pcid_enabled && (operand.pcid != 0)) || is_noncanonical_address(operand.gla, vcpu)) { kvm_inject_gp(vcpu, 0); From 93d1c9f498a7505e0e0a0198f3b3d7f97fcc5fa6 Mon Sep 17 00:00:00 2001 From: Robert Hoo Date: Wed, 13 Sep 2023 20:42:21 +0800 Subject: [PATCH 059/336] KVM: x86: Virtualize LAM for supervisor pointer Add support to allow guests to set the new CR4 control bit for LAM and add implementation to get untagged address for supervisor pointers. LAM modifies the canonicality check applied to 64-bit linear addresses for data accesses, allowing software to use of the untranslated address bits for metadata and masks the metadata bits before using them as linear addresses to access memory. LAM uses CR4.LAM_SUP (bit 28) to configure and enable LAM for supervisor pointers. It also changes VMENTER to allow the bit to be set in VMCS's HOST_CR4 and GUEST_CR4 to support virtualization. Note CR4.LAM_SUP is allowed to be set even not in 64-bit mode, but it will not take effect since LAM only applies to 64-bit linear addresses. Move CR4.LAM_SUP out of CR4_RESERVED_BITS, its reservation depends on vcpu supporting LAM or not. Leave it intercepted to prevent guest from setting the bit if LAM is not exposed to guest as well as to avoid vmread every time when KVM fetches its value, with the expectation that guest won't toggle the bit frequently. Set CR4.LAM_SUP bit in the emulated IA32_VMX_CR4_FIXED1 MSR for guests to allow guests to enable LAM for supervisor pointers in nested VMX operation. Hardware is not required to do TLB flush when CR4.LAM_SUP toggled, KVM doesn't need to emulate TLB flush based on it. There's no other features or vmx_exec_controls connection, and no other code needed in {kvm,vmx}_set_cr4(). Skip address untag for instruction fetches (which includes branch targets), operand of INVLPG instructions, and implicit system accesses, all of which are not subject to untagging. Note, get_untagged_addr() isn't invoked for implicit system accesses as there is no reason to do so, but check the flag anyways for documentation purposes. Signed-off-by: Robert Hoo Co-developed-by: Binbin Wu Signed-off-by: Binbin Wu Reviewed-by: Chao Gao Reviewed-by: Kai Huang Tested-by: Xuelian Guo Link: https://lore.kernel.org/r/20230913124227.12574-11-binbin.wu@linux.intel.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 3 ++- arch/x86/kvm/vmx/vmx.c | 39 ++++++++++++++++++++++++++++++++- arch/x86/kvm/x86.h | 2 ++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 8d042f27d644..f96988f283d5 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -133,7 +133,8 @@ | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR | X86_CR4_PCIDE \ | X86_CR4_OSXSAVE | X86_CR4_SMEP | X86_CR4_FSGSBASE \ | X86_CR4_OSXMMEXCPT | X86_CR4_LA57 | X86_CR4_VMXE \ - | X86_CR4_SMAP | X86_CR4_PKE | X86_CR4_UMIP)) + | X86_CR4_SMAP | X86_CR4_PKE | X86_CR4_UMIP \ + | X86_CR4_LAM_SUP)) #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index d7ff06d77720..3bbe5a9362a9 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -7673,6 +7673,9 @@ static void nested_vmx_cr_fixed1_bits_update(struct kvm_vcpu *vcpu) cr4_fixed1_update(X86_CR4_UMIP, ecx, feature_bit(UMIP)); cr4_fixed1_update(X86_CR4_LA57, ecx, feature_bit(LA57)); + entry = kvm_find_cpuid_entry_index(vcpu, 0x7, 1); + cr4_fixed1_update(X86_CR4_LAM_SUP, eax, feature_bit(LAM)); + #undef cr4_fixed1_update } @@ -8205,9 +8208,43 @@ static void vmx_vm_destroy(struct kvm *kvm) free_pages((unsigned long)kvm_vmx->pid_table, vmx_get_pid_table_order(kvm)); } +/* + * Note, the SDM states that the linear address is masked *after* the modified + * canonicality check, whereas KVM masks (untags) the address and then performs + * a "normal" canonicality check. Functionally, the two methods are identical, + * and when the masking occurs relative to the canonicality check isn't visible + * to software, i.e. KVM's behavior doesn't violate the SDM. + */ gva_t vmx_get_untagged_addr(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags) { - return gva; + int lam_bit; + + if (flags & (X86EMUL_F_FETCH | X86EMUL_F_IMPLICIT | X86EMUL_F_INVLPG)) + return gva; + + if (!is_64_bit_mode(vcpu)) + return gva; + + /* + * Bit 63 determines if the address should be treated as user address + * or a supervisor address. + */ + if (!(gva & BIT_ULL(63))) { + /* KVM doesn't yet virtualize LAM_U{48,57}. */ + return gva; + } else { + if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_LAM_SUP)) + return gva; + + lam_bit = kvm_is_cr4_bit_set(vcpu, X86_CR4_LA57) ? 56 : 47; + } + + /* + * Untag the address by sign-extending the lam_bit, but NOT to bit 63. + * Bit 63 is retained from the raw virtual address so that untagging + * doesn't change a user access to a supervisor access, and vice versa. + */ + return (sign_extend64(gva, lam_bit) & ~BIT_ULL(63)) | (gva & BIT_ULL(63)); } static struct kvm_x86_ops vmx_x86_ops __initdata = { diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 5184fde1dc54..2f7e19166658 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -530,6 +530,8 @@ bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type); __reserved_bits |= X86_CR4_VMXE; \ if (!__cpu_has(__c, X86_FEATURE_PCID)) \ __reserved_bits |= X86_CR4_PCIDE; \ + if (!__cpu_has(__c, X86_FEATURE_LAM)) \ + __reserved_bits |= X86_CR4_LAM_SUP; \ __reserved_bits; \ }) From 3098e6eca88e543ea0d190d1fa72b1c047bb3e7d Mon Sep 17 00:00:00 2001 From: Robert Hoo Date: Wed, 13 Sep 2023 20:42:22 +0800 Subject: [PATCH 060/336] KVM: x86: Virtualize LAM for user pointer Add support to allow guests to set the new CR3 control bits for Linear Address Masking (LAM) and add implementation to get untagged address for user pointers. LAM modifies the canonical check for 64-bit linear addresses, allowing software to use the masked/ignored address bits for metadata. Hardware masks off the metadata bits before using the linear addresses to access memory. LAM uses two new CR3 non-address bits, LAM_U48 (bit 62) and LAM_U57 (bit 61), to configure LAM for user pointers. LAM also changes VMENTER to allow both bits to be set in VMCS's HOST_CR3 and GUEST_CR3 for virtualization. When EPT is on, CR3 is not trapped by KVM and it's up to the guest to set any of the two LAM control bits. However, when EPT is off, the actual CR3 used by the guest is generated from the shadow MMU root which is different from the CR3 that is *set* by the guest, and KVM needs to manually apply any active control bits to VMCS's GUEST_CR3 based on the cached CR3 *seen* by the guest. KVM manually checks guest's CR3 to make sure it points to a valid guest physical address (i.e. to support smaller MAXPHYSADDR in the guest). Extend this check to allow the two LAM control bits to be set. After check, LAM bits of guest CR3 will be stripped off to extract guest physical address. In case of nested, for a guest which supports LAM, both VMCS12's HOST_CR3 and GUEST_CR3 are allowed to have the new LAM control bits set, i.e. when L0 enters L1 to emulate a VMEXIT from L2 to L1 or when L0 enters L2 directly. KVM also manually checks VMCS12's HOST_CR3 and GUEST_CR3 being valid physical address. Extend such check to allow the new LAM control bits too. Note, LAM doesn't have a global control bit to turn on/off LAM completely, but purely depends on hardware's CPUID to determine it can be enabled or not. That means, when EPT is on, even when KVM doesn't expose LAM to guest, the guest can still set LAM control bits in CR3 w/o causing problem. This is an unfortunate virtualization hole. KVM could choose to intercept CR3 in this case and inject fault but this would hurt performance when running a normal VM w/o LAM support. This is undesirable. Just choose to let the guest do such illegal thing as the worst case is guest being killed when KVM eventually find out such illegal behaviour and that the guest is misbehaving. Suggested-by: Sean Christopherson Signed-off-by: Robert Hoo Co-developed-by: Binbin Wu Signed-off-by: Binbin Wu Reviewed-by: Kai Huang Reviewed-by: Chao Gao Tested-by: Xuelian Guo Link: https://lore.kernel.org/r/20230913124227.12574-12-binbin.wu@linux.intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/cpuid.h | 4 ++++ arch/x86/kvm/mmu.h | 9 +++++++++ arch/x86/kvm/vmx/vmx.c | 12 +++++++++--- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/cpuid.h b/arch/x86/kvm/cpuid.h index fa04a00090a2..e4bd59c15a2c 100644 --- a/arch/x86/kvm/cpuid.h +++ b/arch/x86/kvm/cpuid.h @@ -276,6 +276,10 @@ static __always_inline bool guest_can_use(struct kvm_vcpu *vcpu, static inline bool kvm_vcpu_is_legal_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) { + if (kvm_cpu_cap_has(X86_FEATURE_LAM) && + guest_cpuid_has(vcpu, X86_FEATURE_LAM)) + cr3 &= ~(X86_CR3_LAM_U48 | X86_CR3_LAM_U57); + return kvm_vcpu_is_legal_gpa(vcpu, cr3); } diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h index bb8c86eefac0..580e2414b88c 100644 --- a/arch/x86/kvm/mmu.h +++ b/arch/x86/kvm/mmu.h @@ -146,6 +146,15 @@ static inline unsigned long kvm_get_active_pcid(struct kvm_vcpu *vcpu) return kvm_get_pcid(vcpu, kvm_read_cr3(vcpu)); } +static inline unsigned long kvm_get_active_cr3_lam_bits(struct kvm_vcpu *vcpu) +{ + if (!kvm_cpu_cap_has(X86_FEATURE_LAM) || + !guest_cpuid_has(vcpu, X86_FEATURE_LAM)) + return 0; + + return kvm_read_cr3(vcpu) & (X86_CR3_LAM_U48 | X86_CR3_LAM_U57); +} + static inline void kvm_mmu_load_pgd(struct kvm_vcpu *vcpu) { u64 root_hpa = vcpu->arch.mmu->root.hpa; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 3bbe5a9362a9..3825ab316be9 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -3400,7 +3400,8 @@ static void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, update_guest_cr3 = false; vmx_ept_load_pdptrs(vcpu); } else { - guest_cr3 = root_hpa | kvm_get_active_pcid(vcpu); + guest_cr3 = root_hpa | kvm_get_active_pcid(vcpu) | + kvm_get_active_cr3_lam_bits(vcpu); } if (update_guest_cr3) @@ -8218,6 +8219,7 @@ static void vmx_vm_destroy(struct kvm *kvm) gva_t vmx_get_untagged_addr(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags) { int lam_bit; + unsigned long cr3_bits; if (flags & (X86EMUL_F_FETCH | X86EMUL_F_IMPLICIT | X86EMUL_F_INVLPG)) return gva; @@ -8230,8 +8232,12 @@ gva_t vmx_get_untagged_addr(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags * or a supervisor address. */ if (!(gva & BIT_ULL(63))) { - /* KVM doesn't yet virtualize LAM_U{48,57}. */ - return gva; + cr3_bits = kvm_get_active_cr3_lam_bits(vcpu); + if (!(cr3_bits & (X86_CR3_LAM_U57 | X86_CR3_LAM_U48))) + return gva; + + /* LAM_U48 is ignored if LAM_U57 is set. */ + lam_bit = cr3_bits & X86_CR3_LAM_U57 ? 56 : 47; } else { if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_LAM_SUP)) return gva; From 703d794cb8cb28c07b22c1c845f5c4d4c419aff7 Mon Sep 17 00:00:00 2001 From: Robert Hoo Date: Wed, 13 Sep 2023 20:42:23 +0800 Subject: [PATCH 061/336] KVM: x86: Advertise and enable LAM (user and supervisor) LAM is enumerated by CPUID.7.1:EAX.LAM[bit 26]. Advertise the feature to userspace and enable it as the final step after the LAM virtualization support for supervisor and user pointers. SGX LAM support is not advertised yet. SGX LAM support is enumerated in SGX's own CPUID and there's no hard requirement that it must be supported when LAM is reported in CPUID leaf 0x7. Signed-off-by: Robert Hoo Signed-off-by: Binbin Wu Reviewed-by: Jingqi Liu Reviewed-by: Chao Gao Reviewed-by: Kai Huang Tested-by: Xuelian Guo Link: https://lore.kernel.org/r/20230913124227.12574-13-binbin.wu@linux.intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/cpuid.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index dda6fc4cfae8..40d68fef748a 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -671,7 +671,7 @@ void kvm_set_cpu_caps(void) kvm_cpu_cap_mask(CPUID_7_1_EAX, F(AVX_VNNI) | F(AVX512_BF16) | F(CMPCCXADD) | F(FZRM) | F(FSRS) | F(FSRC) | - F(AMX_FP16) | F(AVX_IFMA) + F(AMX_FP16) | F(AVX_IFMA) | F(LAM) ); kvm_cpu_cap_init_kvm_defined(CPUID_7_1_EDX, From 183bdd161c2b773a62f01d1c030f5a3a5b7c33b5 Mon Sep 17 00:00:00 2001 From: Binbin Wu Date: Wed, 13 Sep 2023 20:42:24 +0800 Subject: [PATCH 062/336] KVM: x86: Use KVM-governed feature framework to track "LAM enabled" Use the governed feature framework to track if Linear Address Masking (LAM) is "enabled", i.e. if LAM can be used by the guest. Using the framework to avoid the relative expensive call guest_cpuid_has() during cr3 and vmexit handling paths for LAM. No functional change intended. Signed-off-by: Binbin Wu Tested-by: Xuelian Guo Link: https://lore.kernel.org/r/20230913124227.12574-14-binbin.wu@linux.intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/cpuid.h | 3 +-- arch/x86/kvm/governed_features.h | 1 + arch/x86/kvm/mmu.h | 3 +-- arch/x86/kvm/vmx/vmx.c | 1 + 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/cpuid.h b/arch/x86/kvm/cpuid.h index e4bd59c15a2c..856e3037e74f 100644 --- a/arch/x86/kvm/cpuid.h +++ b/arch/x86/kvm/cpuid.h @@ -276,8 +276,7 @@ static __always_inline bool guest_can_use(struct kvm_vcpu *vcpu, static inline bool kvm_vcpu_is_legal_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) { - if (kvm_cpu_cap_has(X86_FEATURE_LAM) && - guest_cpuid_has(vcpu, X86_FEATURE_LAM)) + if (guest_can_use(vcpu, X86_FEATURE_LAM)) cr3 &= ~(X86_CR3_LAM_U48 | X86_CR3_LAM_U57); return kvm_vcpu_is_legal_gpa(vcpu, cr3); diff --git a/arch/x86/kvm/governed_features.h b/arch/x86/kvm/governed_features.h index 423a73395c10..ad463b1ed4e4 100644 --- a/arch/x86/kvm/governed_features.h +++ b/arch/x86/kvm/governed_features.h @@ -16,6 +16,7 @@ KVM_GOVERNED_X86_FEATURE(PAUSEFILTER) KVM_GOVERNED_X86_FEATURE(PFTHRESHOLD) KVM_GOVERNED_X86_FEATURE(VGIF) KVM_GOVERNED_X86_FEATURE(VNMI) +KVM_GOVERNED_X86_FEATURE(LAM) #undef KVM_GOVERNED_X86_FEATURE #undef KVM_GOVERNED_FEATURE diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h index 580e2414b88c..60f21bb4c27b 100644 --- a/arch/x86/kvm/mmu.h +++ b/arch/x86/kvm/mmu.h @@ -148,8 +148,7 @@ static inline unsigned long kvm_get_active_pcid(struct kvm_vcpu *vcpu) static inline unsigned long kvm_get_active_cr3_lam_bits(struct kvm_vcpu *vcpu) { - if (!kvm_cpu_cap_has(X86_FEATURE_LAM) || - !guest_cpuid_has(vcpu, X86_FEATURE_LAM)) + if (!guest_can_use(vcpu, X86_FEATURE_LAM)) return 0; return kvm_read_cr3(vcpu) & (X86_CR3_LAM_U48 | X86_CR3_LAM_U57); diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 3825ab316be9..d30df9b3fe3e 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -7763,6 +7763,7 @@ static void vmx_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_XSAVES); kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_VMX); + kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_LAM); vmx_setup_uret_msrs(vmx); From 4ea6babbdd49d15c36665013e740f2e604d8841d Mon Sep 17 00:00:00 2001 From: Yujie Liu Date: Wed, 29 Nov 2023 17:13:37 +0800 Subject: [PATCH 063/336] xtensa: fix variants path in the Kconfig help The directory in the path should be "variants" instead of "variant". Fixes: 420ae9518404 ("xtensa: simplify addition of new core variants") Signed-off-by: Yujie Liu Message-Id: <20231129091337.2084747-1-yujie.liu@intel.com> Signed-off-by: Max Filippov --- arch/xtensa/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig index 7d792077e5fd..a9ed6ec3a75e 100644 --- a/arch/xtensa/Kconfig +++ b/arch/xtensa/Kconfig @@ -144,7 +144,7 @@ config XTENSA_VARIANT_CUSTOM_NAME depends on XTENSA_VARIANT_CUSTOM help Provide the name of a custom Xtensa processor variant. - This CORENAME selects arch/xtensa/variant/CORENAME. + This CORENAME selects arch/xtensa/variants/CORENAME. Don't forget you have to select MMU if you have one. config XTENSA_VARIANT_NAME From 6d638ab8c3df7a4674b4cd2e4cc9d4051587c729 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Mon, 27 Nov 2023 00:12:22 +0900 Subject: [PATCH 064/336] xtensa: replace with Commit ddb5cdbafaaa ("kbuild: generate KSYMTAB entries by modpost") deprecated , which is now a wrapper of . Replace #include with #include . Signed-off-by: Masahiro Yamada Message-Id: <20231126151222.1556761-1-masahiroy@kernel.org> Signed-off-by: Max Filippov --- arch/xtensa/include/asm/asmmacro.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/xtensa/include/asm/asmmacro.h b/arch/xtensa/include/asm/asmmacro.h index 01bf7d9dbb19..a52d49a16ce7 100644 --- a/arch/xtensa/include/asm/asmmacro.h +++ b/arch/xtensa/include/asm/asmmacro.h @@ -11,7 +11,7 @@ #ifndef _XTENSA_ASMMACRO_H #define _XTENSA_ASMMACRO_H -#include +#include #include /* From 791beae7335caa8d8579d201d7f9b18b4d8898e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ilpo=20J=C3=A4rvinen?= Date: Fri, 24 Nov 2023 11:09:15 +0200 Subject: [PATCH 065/336] xtensa: Use PCI_HEADER_TYPE_MFD instead of literal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace literal 0x80 with PCI_HEADER_TYPE_MFD. While at it, convert found_multi into boolean. Signed-off-by: Ilpo Järvinen Message-Id: <20231124090919.23687-3-ilpo.jarvinen@linux.intel.com> Signed-off-by: Max Filippov --- arch/xtensa/lib/pci-auto.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/arch/xtensa/lib/pci-auto.c b/arch/xtensa/lib/pci-auto.c index aa6752237985..05fc02f9e1c7 100644 --- a/arch/xtensa/lib/pci-auto.c +++ b/arch/xtensa/lib/pci-auto.c @@ -11,6 +11,7 @@ * Based on work from Matt Porter */ +#include #include #include #include @@ -222,10 +223,11 @@ pciauto_postscan_setup_bridge(struct pci_dev *dev, int current_bus, int sub_bus, int __init pciauto_bus_scan(struct pci_controller *pci_ctrl, int current_bus) { - int sub_bus, pci_devfn, pci_class, cmdstat, found_multi=0; + int sub_bus, pci_devfn, pci_class, cmdstat; unsigned short vid; unsigned char header_type; struct pci_dev *dev = &pciauto_dev; + bool found_multi = false; pciauto_dev.bus = &pciauto_bus; pciauto_dev.sysdata = pci_ctrl; @@ -261,11 +263,11 @@ int __init pciauto_bus_scan(struct pci_controller *pci_ctrl, int current_bus) continue; if (!PCI_FUNC(pci_devfn)) - found_multi = header_type & 0x80; + found_multi = FIELD_GET(PCI_HEADER_TYPE_MFD, header_type); pci_read_config_word(dev, PCI_VENDOR_ID, &vid); if (vid == 0xffff || vid == 0x0000) { - found_multi = 0; + found_multi = false; continue; } From fc6543bb55d4077c44e577c321bbf158446c8000 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Sat, 28 Oct 2023 20:34:53 +0100 Subject: [PATCH 066/336] KVM: selftests: add -MP to CFLAGS Using -MD without -MP causes build failures when a header file is deleted or moved. With -MP, the compiler will emit phony targets for the header files it lists as dependencies, and the Makefiles won't refuse to attempt to rebuild a C unit which no longer includes the deleted header. Signed-off-by: David Woodhouse Link: https://lore.kernel.org/r/9fc8b5395321abbfcaf5d78477a9a7cd350b08e4.camel@infradead.org Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index 69ce8e06b3a3..914987220a62 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -226,7 +226,7 @@ else LINUX_TOOL_ARCH_INCLUDE = $(top_srcdir)/tools/arch/$(ARCH)/include endif CFLAGS += -Wall -Wstrict-prototypes -Wuninitialized -O2 -g -std=gnu99 \ - -Wno-gnu-variable-sized-type-not-at-end -MD\ + -Wno-gnu-variable-sized-type-not-at-end -MD -MP \ -fno-builtin-memcmp -fno-builtin-memcpy -fno-builtin-memset \ -fno-builtin-strnlen \ -fno-stack-protector -fno-PIE -I$(LINUX_TOOL_INCLUDE) \ From 0277022a77a56f8251e8cf8d25e9308478e79ea5 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 18 Oct 2023 12:23:25 -0700 Subject: [PATCH 067/336] KVM: x86/mmu: Declare flush_remote_tlbs{_range}() hooks iff HYPERV!=n Declare the kvm_x86_ops hooks used to wire up paravirt TLB flushes when running under Hyper-V if and only if CONFIG_HYPERV!=n. Wrapping yet more code with IS_ENABLED(CONFIG_HYPERV) eliminates a handful of conditional branches, and makes it super obvious why the hooks *might* be valid. Cc: Vitaly Kuznetsov Reviewed-by: Vitaly Kuznetsov Link: https://lore.kernel.org/r/20231018192325.1893896-1-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm-x86-ops.h | 2 ++ arch/x86/include/asm/kvm_host.h | 12 ++++++++++++ arch/x86/kvm/mmu/mmu.c | 12 ++++-------- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index 26b628d84594..f482216bbdb8 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -55,8 +55,10 @@ KVM_X86_OP(set_rflags) KVM_X86_OP(get_if_flag) KVM_X86_OP(flush_tlb_all) KVM_X86_OP(flush_tlb_current) +#if IS_ENABLED(CONFIG_HYPERV) KVM_X86_OP_OPTIONAL(flush_remote_tlbs) KVM_X86_OP_OPTIONAL(flush_remote_tlbs_range) +#endif KVM_X86_OP(flush_tlb_gva) KVM_X86_OP(flush_tlb_guest) KVM_X86_OP(vcpu_pre_run) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index a565a2e70f30..5e5e9e0abd4a 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1614,9 +1614,11 @@ struct kvm_x86_ops { void (*flush_tlb_all)(struct kvm_vcpu *vcpu); void (*flush_tlb_current)(struct kvm_vcpu *vcpu); +#if IS_ENABLED(CONFIG_HYPERV) int (*flush_remote_tlbs)(struct kvm *kvm); int (*flush_remote_tlbs_range)(struct kvm *kvm, gfn_t gfn, gfn_t nr_pages); +#endif /* * Flush any TLB entries associated with the given GVA. @@ -1825,6 +1827,7 @@ static inline struct kvm *kvm_arch_alloc_vm(void) #define __KVM_HAVE_ARCH_VM_FREE void kvm_arch_free_vm(struct kvm *kvm); +#if IS_ENABLED(CONFIG_HYPERV) #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS static inline int kvm_arch_flush_remote_tlbs(struct kvm *kvm) { @@ -1836,6 +1839,15 @@ static inline int kvm_arch_flush_remote_tlbs(struct kvm *kvm) } #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE +static inline int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn, + u64 nr_pages) +{ + if (!kvm_x86_ops.flush_remote_tlbs_range) + return -EOPNOTSUPP; + + return static_call(kvm_x86_flush_remote_tlbs_range)(kvm, gfn, nr_pages); +} +#endif /* CONFIG_HYPERV */ #define kvm_arch_pmi_in_guest(vcpu) \ ((vcpu) && (vcpu)->arch.handling_intr_from_guest) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 59b026b6ad2a..8531480e5da4 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -271,15 +271,11 @@ static inline unsigned long kvm_mmu_get_guest_pgd(struct kvm_vcpu *vcpu, static inline bool kvm_available_flush_remote_tlbs_range(void) { +#if IS_ENABLED(CONFIG_HYPERV) return kvm_x86_ops.flush_remote_tlbs_range; -} - -int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn, u64 nr_pages) -{ - if (!kvm_x86_ops.flush_remote_tlbs_range) - return -EOPNOTSUPP; - - return static_call(kvm_x86_flush_remote_tlbs_range)(kvm, gfn, nr_pages); +#else + return false; +#endif } static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index); From 6542a00369284c951185be8fa2ed45cf423cce06 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 7 Nov 2023 17:09:52 -0800 Subject: [PATCH 068/336] KVM: selftests: Drop the single-underscore ioctl() helpers Drop _kvm_ioctl(), _vm_ioctl(), and _vcpu_ioctl(), as they are no longer used by anything other than the no-underscores variants (and may have never been used directly). The single-underscore variants were never intended to be a "feature", they were a stopgap of sorts to ease the conversion to pretty printing ioctl() names when reporting errors. Opportunistically add a comment explaining when to use __KVM_IOCTL_ERROR() versus KVM_IOCTL_ERROR(). The single-underscore macros were subtly ensuring that the name of the ioctl() was printed on error, i.e. it's all too easy to overlook the fact that using __KVM_IOCTL_ERROR() is intentional. Link: https://lore.kernel.org/r/20231108010953.560824-2-seanjc@google.com Signed-off-by: Sean Christopherson --- .../selftests/kvm/include/kvm_util_base.h | 30 ++++++++----------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/tools/testing/selftests/kvm/include/kvm_util_base.h b/tools/testing/selftests/kvm/include/kvm_util_base.h index 1b58f943562f..190f9c0ef584 100644 --- a/tools/testing/selftests/kvm/include/kvm_util_base.h +++ b/tools/testing/selftests/kvm/include/kvm_util_base.h @@ -267,6 +267,13 @@ static inline bool kvm_has_cap(long cap) #define __KVM_SYSCALL_ERROR(_name, _ret) \ "%s failed, rc: %i errno: %i (%s)", (_name), (_ret), errno, strerror(errno) +/* + * Use the "inner", double-underscore macro when reporting errors from within + * other macros so that the name of ioctl() and not its literal numeric value + * is printed on error. The "outer" macro is strongly preferred when reporting + * errors "directly", i.e. without an additional layer of macros, as it reduces + * the probability of passing in the wrong string. + */ #define __KVM_IOCTL_ERROR(_name, _ret) __KVM_SYSCALL_ERROR(_name, _ret) #define KVM_IOCTL_ERROR(_ioctl, _ret) __KVM_IOCTL_ERROR(#_ioctl, _ret) @@ -279,17 +286,13 @@ static inline bool kvm_has_cap(long cap) #define __kvm_ioctl(kvm_fd, cmd, arg) \ kvm_do_ioctl(kvm_fd, cmd, arg) - -#define _kvm_ioctl(kvm_fd, cmd, name, arg) \ +#define kvm_ioctl(kvm_fd, cmd, arg) \ ({ \ int ret = __kvm_ioctl(kvm_fd, cmd, arg); \ \ - TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(name, ret)); \ + TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(#cmd, ret)); \ }) -#define kvm_ioctl(kvm_fd, cmd, arg) \ - _kvm_ioctl(kvm_fd, cmd, #cmd, arg) - static __always_inline void static_assert_is_vm(struct kvm_vm *vm) { } #define __vm_ioctl(vm, cmd, arg) \ @@ -298,17 +301,13 @@ static __always_inline void static_assert_is_vm(struct kvm_vm *vm) { } kvm_do_ioctl((vm)->fd, cmd, arg); \ }) -#define _vm_ioctl(vm, cmd, name, arg) \ +#define vm_ioctl(vm, cmd, arg) \ ({ \ int ret = __vm_ioctl(vm, cmd, arg); \ \ - TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(name, ret)); \ + TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(#cmd, ret)); \ }) -#define vm_ioctl(vm, cmd, arg) \ - _vm_ioctl(vm, cmd, #cmd, arg) - - static __always_inline void static_assert_is_vcpu(struct kvm_vcpu *vcpu) { } #define __vcpu_ioctl(vcpu, cmd, arg) \ @@ -317,16 +316,13 @@ static __always_inline void static_assert_is_vcpu(struct kvm_vcpu *vcpu) { } kvm_do_ioctl((vcpu)->fd, cmd, arg); \ }) -#define _vcpu_ioctl(vcpu, cmd, name, arg) \ +#define vcpu_ioctl(vcpu, cmd, arg) \ ({ \ int ret = __vcpu_ioctl(vcpu, cmd, arg); \ \ - TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(name, ret)); \ + TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(#cmd, ret)); \ }) -#define vcpu_ioctl(vcpu, cmd, arg) \ - _vcpu_ioctl(vcpu, cmd, #cmd, arg) - /* * Looks up and returns the value corresponding to the capability * (KVM_CAP_*) given by cap. From 1b78d474ce4ecbf15a4a8b08994b8ed8ceec0dab Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 7 Nov 2023 17:09:53 -0800 Subject: [PATCH 069/336] KVM: selftests: Add logic to detect if ioctl() failed because VM was killed Add yet another macro to the VM/vCPU ioctl() framework to detect when an ioctl() failed because KVM killed/bugged the VM, i.e. when there was nothing wrong with the ioctl() itself. If KVM kills a VM, e.g. by way of a failed KVM_BUG_ON(), all subsequent VM and vCPU ioctl()s will fail with -EIO, which can be quite misleading and ultimately waste user/developer time. Use KVM_CHECK_EXTENSION on KVM_CAP_USER_MEMORY to detect if the VM is dead and/or bug, as KVM doesn't provide a dedicated ioctl(). Using a heuristic is obviously less than ideal, but practically speaking the logic is bulletproof barring a KVM change, and any such change would arguably break userspace, e.g. if KVM returns something other than -EIO. Without the detection, tearing down a bugged VM yields a cryptic failure when deleting memslots: ==== Test Assertion Failure ==== lib/kvm_util.c:689: !ret pid=45131 tid=45131 errno=5 - Input/output error 1 0x00000000004036c3: __vm_mem_region_delete at kvm_util.c:689 2 0x00000000004042f0: kvm_vm_free at kvm_util.c:724 (discriminator 12) 3 0x0000000000402929: race_sync_regs at sync_regs_test.c:193 4 0x0000000000401cab: main at sync_regs_test.c:334 (discriminator 6) 5 0x0000000000416f13: __libc_start_call_main at libc-start.o:? 6 0x000000000041855f: __libc_start_main_impl at ??:? 7 0x0000000000401d40: _start at ??:? KVM_SET_USER_MEMORY_REGION failed, rc: -1 errno: 5 (Input/output error) Which morphs into a more pointed error message with the detection: ==== Test Assertion Failure ==== lib/kvm_util.c:689: false pid=80347 tid=80347 errno=5 - Input/output error 1 0x00000000004039ab: __vm_mem_region_delete at kvm_util.c:689 (discriminator 5) 2 0x0000000000404660: kvm_vm_free at kvm_util.c:724 (discriminator 12) 3 0x0000000000402ac9: race_sync_regs at sync_regs_test.c:193 4 0x0000000000401cb7: main at sync_regs_test.c:334 (discriminator 6) 5 0x0000000000418263: __libc_start_call_main at libc-start.o:? 6 0x00000000004198af: __libc_start_main_impl at ??:? 7 0x0000000000401d90: _start at ??:? KVM killed/bugged the VM, check the kernel log for clues Suggested-by: Michal Luczaj Cc: Oliver Upton Cc: Colton Lewis Link: https://lore.kernel.org/r/20231108010953.560824-3-seanjc@google.com Signed-off-by: Sean Christopherson --- .../selftests/kvm/include/kvm_util_base.h | 39 ++++++++++++++++--- tools/testing/selftests/kvm/lib/kvm_util.c | 2 +- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/tools/testing/selftests/kvm/include/kvm_util_base.h b/tools/testing/selftests/kvm/include/kvm_util_base.h index 190f9c0ef584..e0da036a13ae 100644 --- a/tools/testing/selftests/kvm/include/kvm_util_base.h +++ b/tools/testing/selftests/kvm/include/kvm_util_base.h @@ -301,11 +301,40 @@ static __always_inline void static_assert_is_vm(struct kvm_vm *vm) { } kvm_do_ioctl((vm)->fd, cmd, arg); \ }) +/* + * Assert that a VM or vCPU ioctl() succeeded, with extra magic to detect if + * the ioctl() failed because KVM killed/bugged the VM. To detect a dead VM, + * probe KVM_CAP_USER_MEMORY, which (a) has been supported by KVM since before + * selftests existed and (b) should never outright fail, i.e. is supposed to + * return 0 or 1. If KVM kills a VM, KVM returns -EIO for all ioctl()s for the + * VM and its vCPUs, including KVM_CHECK_EXTENSION. + */ +#define __TEST_ASSERT_VM_VCPU_IOCTL(cond, name, ret, vm) \ +do { \ + int __errno = errno; \ + \ + static_assert_is_vm(vm); \ + \ + if (cond) \ + break; \ + \ + if (errno == EIO && \ + __vm_ioctl(vm, KVM_CHECK_EXTENSION, (void *)KVM_CAP_USER_MEMORY) < 0) { \ + TEST_ASSERT(errno == EIO, "KVM killed the VM, should return -EIO"); \ + TEST_FAIL("KVM killed/bugged the VM, check the kernel log for clues"); \ + } \ + errno = __errno; \ + TEST_ASSERT(cond, __KVM_IOCTL_ERROR(name, ret)); \ +} while (0) + +#define TEST_ASSERT_VM_VCPU_IOCTL(cond, cmd, ret, vm) \ + __TEST_ASSERT_VM_VCPU_IOCTL(cond, #cmd, ret, vm) + #define vm_ioctl(vm, cmd, arg) \ ({ \ int ret = __vm_ioctl(vm, cmd, arg); \ \ - TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(#cmd, ret)); \ + __TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, vm); \ }) static __always_inline void static_assert_is_vcpu(struct kvm_vcpu *vcpu) { } @@ -320,7 +349,7 @@ static __always_inline void static_assert_is_vcpu(struct kvm_vcpu *vcpu) { } ({ \ int ret = __vcpu_ioctl(vcpu, cmd, arg); \ \ - TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(#cmd, ret)); \ + __TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, (vcpu)->vm); \ }) /* @@ -331,7 +360,7 @@ static inline int vm_check_cap(struct kvm_vm *vm, long cap) { int ret = __vm_ioctl(vm, KVM_CHECK_EXTENSION, (void *)cap); - TEST_ASSERT(ret >= 0, KVM_IOCTL_ERROR(KVM_CHECK_EXTENSION, ret)); + TEST_ASSERT_VM_VCPU_IOCTL(ret >= 0, KVM_CHECK_EXTENSION, ret, vm); return ret; } @@ -438,7 +467,7 @@ static inline int vm_get_stats_fd(struct kvm_vm *vm) { int fd = __vm_ioctl(vm, KVM_GET_STATS_FD, NULL); - TEST_ASSERT(fd >= 0, KVM_IOCTL_ERROR(KVM_GET_STATS_FD, fd)); + TEST_ASSERT_VM_VCPU_IOCTL(fd >= 0, KVM_GET_STATS_FD, fd, vm); return fd; } @@ -680,7 +709,7 @@ static inline int vcpu_get_stats_fd(struct kvm_vcpu *vcpu) { int fd = __vcpu_ioctl(vcpu, KVM_GET_STATS_FD, NULL); - TEST_ASSERT(fd >= 0, KVM_IOCTL_ERROR(KVM_GET_STATS_FD, fd)); + TEST_ASSERT_VM_VCPU_IOCTL(fd >= 0, KVM_CHECK_EXTENSION, fd, vcpu->vm); return fd; } diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 9b29cbf49476..17a978b8a2c4 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -1271,7 +1271,7 @@ struct kvm_vcpu *__vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id) vcpu->vm = vm; vcpu->id = vcpu_id; vcpu->fd = __vm_ioctl(vm, KVM_CREATE_VCPU, (void *)(unsigned long)vcpu_id); - TEST_ASSERT(vcpu->fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_VCPU, vcpu->fd)); + TEST_ASSERT_VM_VCPU_IOCTL(vcpu->fd >= 0, KVM_CREATE_VCPU, vcpu->fd, vm); TEST_ASSERT(vcpu_mmap_sz() >= sizeof(*vcpu->run), "vcpu mmap size " "smaller than expected, vcpu_mmap_sz: %i expected_min: %zi", From e29f5d0c3c7c055b36ef55f9e92100450b2506a3 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 15 Aug 2023 15:00:30 -0700 Subject: [PATCH 070/336] KVM: selftests: Remove x86's so called "MMIO warning" test Remove x86's mmio_warning_test, as it is unnecessarily complex (there's no reason to fork, spawn threads, initialize srand(), etc..), unnecessarily restrictive (triggering triple fault is not unique to Intel CPUs without unrestricted guest), and provides no meaningful coverage beyond what basic fuzzing can achieve (running a vCPU with garbage is fuzzing's bread and butter). That the test has *all* of the above flaws is not coincidental, as the code was copy+pasted almost verbatim from the syzkaller reproducer that originally found the KVM bug (which has long since been fixed). Cc: Michal Luczaj Link: https://groups.google.com/g/syzkaller/c/lHfau8E3SOE Link: https://lore.kernel.org/r/20230815220030.560372-1-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/Makefile | 1 - .../selftests/kvm/x86_64/mmio_warning_test.c | 121 ------------------ 2 files changed, 122 deletions(-) delete mode 100644 tools/testing/selftests/kvm/x86_64/mmio_warning_test.c diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index 914987220a62..4412b42d95de 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -86,7 +86,6 @@ TEST_GEN_PROGS_x86_64 += x86_64/hyperv_svm_test TEST_GEN_PROGS_x86_64 += x86_64/hyperv_tlb_flush TEST_GEN_PROGS_x86_64 += x86_64/kvm_clock_test TEST_GEN_PROGS_x86_64 += x86_64/kvm_pv_test -TEST_GEN_PROGS_x86_64 += x86_64/mmio_warning_test TEST_GEN_PROGS_x86_64 += x86_64/monitor_mwait_test TEST_GEN_PROGS_x86_64 += x86_64/nested_exceptions_test TEST_GEN_PROGS_x86_64 += x86_64/platform_info_test diff --git a/tools/testing/selftests/kvm/x86_64/mmio_warning_test.c b/tools/testing/selftests/kvm/x86_64/mmio_warning_test.c deleted file mode 100644 index ce1ccc4c1503..000000000000 --- a/tools/testing/selftests/kvm/x86_64/mmio_warning_test.c +++ /dev/null @@ -1,121 +0,0 @@ -/* - * mmio_warning_test - * - * Copyright (C) 2019, Google LLC. - * - * This work is licensed under the terms of the GNU GPL, version 2. - * - * Test that we don't get a kernel warning when we call KVM_RUN after a - * triple fault occurs. To get the triple fault to occur we call KVM_RUN - * on a VCPU that hasn't been properly setup. - * - */ - -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define NTHREAD 4 -#define NPROCESS 5 - -struct thread_context { - int kvmcpu; - struct kvm_run *run; -}; - -void *thr(void *arg) -{ - struct thread_context *tc = (struct thread_context *)arg; - int res; - int kvmcpu = tc->kvmcpu; - struct kvm_run *run = tc->run; - - res = ioctl(kvmcpu, KVM_RUN, 0); - pr_info("ret1=%d exit_reason=%d suberror=%d\n", - res, run->exit_reason, run->internal.suberror); - - return 0; -} - -void test(void) -{ - int i, kvm, kvmvm, kvmcpu; - pthread_t th[NTHREAD]; - struct kvm_run *run; - struct thread_context tc; - - kvm = open("/dev/kvm", O_RDWR); - TEST_ASSERT(kvm != -1, "failed to open /dev/kvm"); - kvmvm = __kvm_ioctl(kvm, KVM_CREATE_VM, NULL); - TEST_ASSERT(kvmvm > 0, KVM_IOCTL_ERROR(KVM_CREATE_VM, kvmvm)); - kvmcpu = ioctl(kvmvm, KVM_CREATE_VCPU, 0); - TEST_ASSERT(kvmcpu != -1, KVM_IOCTL_ERROR(KVM_CREATE_VCPU, kvmcpu)); - run = (struct kvm_run *)mmap(0, 4096, PROT_READ|PROT_WRITE, MAP_SHARED, - kvmcpu, 0); - tc.kvmcpu = kvmcpu; - tc.run = run; - srand(getpid()); - for (i = 0; i < NTHREAD; i++) { - pthread_create(&th[i], NULL, thr, (void *)(uintptr_t)&tc); - usleep(rand() % 10000); - } - for (i = 0; i < NTHREAD; i++) - pthread_join(th[i], NULL); -} - -int get_warnings_count(void) -{ - int warnings; - FILE *f; - - f = popen("dmesg | grep \"WARNING:\" | wc -l", "r"); - if (fscanf(f, "%d", &warnings) < 1) - warnings = 0; - pclose(f); - - return warnings; -} - -int main(void) -{ - int warnings_before, warnings_after; - - TEST_REQUIRE(host_cpu_is_intel); - - TEST_REQUIRE(!vm_is_unrestricted_guest(NULL)); - - warnings_before = get_warnings_count(); - - for (int i = 0; i < NPROCESS; ++i) { - int status; - int pid = fork(); - - if (pid < 0) - exit(1); - if (pid == 0) { - test(); - exit(0); - } - while (waitpid(pid, &status, __WALL) != pid) - ; - } - - warnings_after = get_warnings_count(); - TEST_ASSERT(warnings_before == warnings_after, - "Warnings found in kernel. Run 'dmesg' to inspect them."); - - return 0; -} From 11e5ea5242e38d44fcede879473566bb6d68f954 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 28 Nov 2023 15:04:01 +0100 Subject: [PATCH 071/336] KVM: arm64: Use helpers to classify exception types reported via ESR Currently, we rely on the fact that exceptions can be trivially classified by applying a mask/value pair to the syndrome value reported via the ESR register, but this will no longer be true once we enable support for 5 level paging. So introduce a couple of helpers that encapsulate this mask/value pair matching, and wire them up in the code. No functional change intended, the actual handling of translation level -1 will be added in a subsequent patch. Cc: Catalin Marinas Cc: Will Deacon Cc: Marc Zyngier Cc: Oliver Upton Cc: Ryan Roberts Signed-off-by: Ard Biesheuvel Acked-by: Mark Rutland [maz: folded in changes suggested by Mark] Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20231128140400.3132145-2-ardb@google.com --- arch/arm64/include/asm/esr.h | 15 +++++++++++ arch/arm64/include/asm/kvm_emulate.h | 36 +++++++++++-------------- arch/arm64/kvm/hyp/include/hyp/fault.h | 2 +- arch/arm64/kvm/hyp/include/hyp/switch.h | 2 +- arch/arm64/kvm/mmu.c | 35 ++++++++++++------------ 5 files changed, 50 insertions(+), 40 deletions(-) diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h index ae35939f395b..353fe08546cf 100644 --- a/arch/arm64/include/asm/esr.h +++ b/arch/arm64/include/asm/esr.h @@ -392,6 +392,21 @@ static inline bool esr_is_data_abort(unsigned long esr) return ec == ESR_ELx_EC_DABT_LOW || ec == ESR_ELx_EC_DABT_CUR; } +static inline bool esr_fsc_is_translation_fault(unsigned long esr) +{ + return (esr & ESR_ELx_FSC_TYPE) == ESR_ELx_FSC_FAULT; +} + +static inline bool esr_fsc_is_permission_fault(unsigned long esr) +{ + return (esr & ESR_ELx_FSC_TYPE) == ESR_ELx_FSC_PERM; +} + +static inline bool esr_fsc_is_access_flag_fault(unsigned long esr) +{ + return (esr & ESR_ELx_FSC_TYPE) == ESR_ELx_FSC_ACCESS; +} + const char *esr_get_class_string(unsigned long esr); #endif /* __ASSEMBLY */ diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index d4f1e9cdd554..31f13e7d339b 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -404,24 +404,25 @@ static __always_inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu) return kvm_vcpu_get_esr(vcpu) & ESR_ELx_FSC; } -static __always_inline u8 kvm_vcpu_trap_get_fault_type(const struct kvm_vcpu *vcpu) +static inline +bool kvm_vcpu_trap_is_permission_fault(const struct kvm_vcpu *vcpu) { - return kvm_vcpu_get_esr(vcpu) & ESR_ELx_FSC_TYPE; + return esr_fsc_is_permission_fault(kvm_vcpu_get_esr(vcpu)); } -static __always_inline s8 kvm_vcpu_trap_get_fault_level(const struct kvm_vcpu *vcpu) +static inline +bool kvm_vcpu_trap_is_translation_fault(const struct kvm_vcpu *vcpu) { - /* - * Note: With the introduction of FEAT_LPA2 an extra level of - * translation (level -1) is added. This level (obviously) doesn't - * follow the previous convention of encoding the 4 levels in the 2 LSBs - * of the FSC so this function breaks if the fault is for level -1. - * - * However, stage2 tables always use concatenated tables for first level - * lookup and therefore it is guaranteed that the level will be between - * 0 and 3, and this function continues to work. - */ - return kvm_vcpu_get_esr(vcpu) & ESR_ELx_FSC_LEVEL; + return esr_fsc_is_translation_fault(kvm_vcpu_get_esr(vcpu)); +} + +static inline +u64 kvm_vcpu_trap_get_perm_fault_granule(const struct kvm_vcpu *vcpu) +{ + unsigned long esr = kvm_vcpu_get_esr(vcpu); + + BUG_ON(!esr_fsc_is_permission_fault(esr)); + return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(esr & ESR_ELx_FSC_LEVEL)); } static __always_inline bool kvm_vcpu_abt_issea(const struct kvm_vcpu *vcpu) @@ -464,12 +465,7 @@ static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu) * first), then a permission fault to allow the flags * to be set. */ - switch (kvm_vcpu_trap_get_fault_type(vcpu)) { - case ESR_ELx_FSC_PERM: - return true; - default: - return false; - } + return kvm_vcpu_trap_is_permission_fault(vcpu); } if (kvm_vcpu_trap_is_iabt(vcpu)) diff --git a/arch/arm64/kvm/hyp/include/hyp/fault.h b/arch/arm64/kvm/hyp/include/hyp/fault.h index 9ddcfe2c3e57..9e13c1bc2ad5 100644 --- a/arch/arm64/kvm/hyp/include/hyp/fault.h +++ b/arch/arm64/kvm/hyp/include/hyp/fault.h @@ -60,7 +60,7 @@ static inline bool __get_fault_info(u64 esr, struct kvm_vcpu_fault_info *fault) */ if (!(esr & ESR_ELx_S1PTW) && (cpus_have_final_cap(ARM64_WORKAROUND_834220) || - (esr & ESR_ELx_FSC_TYPE) == ESR_ELx_FSC_PERM)) { + esr_fsc_is_permission_fault(esr))) { if (!__translate_far_to_hpfar(far, &hpfar)) return false; } else { diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index f99d8af0b9af..f44fb11307fb 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -591,7 +591,7 @@ static bool kvm_hyp_handle_dabt_low(struct kvm_vcpu *vcpu, u64 *exit_code) if (static_branch_unlikely(&vgic_v2_cpuif_trap)) { bool valid; - valid = kvm_vcpu_trap_get_fault_type(vcpu) == ESR_ELx_FSC_FAULT && + valid = kvm_vcpu_trap_is_translation_fault(vcpu) && kvm_vcpu_dabt_isvalid(vcpu) && !kvm_vcpu_abt_issea(vcpu) && !kvm_vcpu_abt_iss1tw(vcpu); diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 986a2e6fb900..d14504821b79 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1376,7 +1376,7 @@ static bool kvm_vma_mte_allowed(struct vm_area_struct *vma) static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, struct kvm_memory_slot *memslot, unsigned long hva, - unsigned long fault_status) + bool fault_is_perm) { int ret = 0; bool write_fault, writable, force_pte = false; @@ -1390,17 +1390,17 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, gfn_t gfn; kvm_pfn_t pfn; bool logging_active = memslot_is_logging(memslot); - s8 fault_level = kvm_vcpu_trap_get_fault_level(vcpu); long vma_pagesize, fault_granule; enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R; struct kvm_pgtable *pgt; - fault_granule = 1UL << ARM64_HW_PGTABLE_LEVEL_SHIFT(fault_level); + if (fault_is_perm) + fault_granule = kvm_vcpu_trap_get_perm_fault_granule(vcpu); write_fault = kvm_is_write_fault(vcpu); exec_fault = kvm_vcpu_trap_is_exec_fault(vcpu); VM_BUG_ON(write_fault && exec_fault); - if (fault_status == ESR_ELx_FSC_PERM && !write_fault && !exec_fault) { + if (fault_is_perm && !write_fault && !exec_fault) { kvm_err("Unexpected L2 read permission error\n"); return -EFAULT; } @@ -1411,8 +1411,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, * only exception to this is when dirty logging is enabled at runtime * and a write fault needs to collapse a block entry into a table. */ - if (fault_status != ESR_ELx_FSC_PERM || - (logging_active && write_fault)) { + if (!fault_is_perm || (logging_active && write_fault)) { ret = kvm_mmu_topup_memory_cache(memcache, kvm_mmu_cache_min_pages(vcpu->arch.hw_mmu)); if (ret) @@ -1529,8 +1528,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, * backed by a THP and thus use block mapping if possible. */ if (vma_pagesize == PAGE_SIZE && !(force_pte || device)) { - if (fault_status == ESR_ELx_FSC_PERM && - fault_granule > PAGE_SIZE) + if (fault_is_perm && fault_granule > PAGE_SIZE) vma_pagesize = fault_granule; else vma_pagesize = transparent_hugepage_adjust(kvm, memslot, @@ -1543,7 +1541,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, } } - if (fault_status != ESR_ELx_FSC_PERM && !device && kvm_has_mte(kvm)) { + if (!fault_is_perm && !device && kvm_has_mte(kvm)) { /* Check the VMM hasn't introduced a new disallowed VMA */ if (mte_allowed) { sanitise_mte_tags(kvm, pfn, vma_pagesize); @@ -1569,7 +1567,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, * permissions only if vma_pagesize equals fault_granule. Otherwise, * kvm_pgtable_stage2_map() should be called to change block size. */ - if (fault_status == ESR_ELx_FSC_PERM && vma_pagesize == fault_granule) + if (fault_is_perm && vma_pagesize == fault_granule) ret = kvm_pgtable_stage2_relax_perms(pgt, fault_ipa, prot); else ret = kvm_pgtable_stage2_map(pgt, fault_ipa, vma_pagesize, @@ -1620,7 +1618,7 @@ static void handle_access_fault(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa) */ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) { - unsigned long fault_status; + unsigned long esr; phys_addr_t fault_ipa; struct kvm_memory_slot *memslot; unsigned long hva; @@ -1628,12 +1626,12 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) gfn_t gfn; int ret, idx; - fault_status = kvm_vcpu_trap_get_fault_type(vcpu); + esr = kvm_vcpu_get_esr(vcpu); fault_ipa = kvm_vcpu_get_fault_ipa(vcpu); is_iabt = kvm_vcpu_trap_is_iabt(vcpu); - if (fault_status == ESR_ELx_FSC_FAULT) { + if (esr_fsc_is_permission_fault(esr)) { /* Beyond sanitised PARange (which is the IPA limit) */ if (fault_ipa >= BIT_ULL(get_kvm_ipa_limit())) { kvm_inject_size_fault(vcpu); @@ -1668,9 +1666,9 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) kvm_vcpu_get_hfar(vcpu), fault_ipa); /* Check the stage-2 fault is trans. fault or write fault */ - if (fault_status != ESR_ELx_FSC_FAULT && - fault_status != ESR_ELx_FSC_PERM && - fault_status != ESR_ELx_FSC_ACCESS) { + if (!esr_fsc_is_translation_fault(esr) && + !esr_fsc_is_permission_fault(esr) && + !esr_fsc_is_access_flag_fault(esr)) { kvm_err("Unsupported FSC: EC=%#x xFSC=%#lx ESR_EL2=%#lx\n", kvm_vcpu_trap_get_class(vcpu), (unsigned long)kvm_vcpu_trap_get_fault(vcpu), @@ -1732,13 +1730,14 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) /* Userspace should not be able to register out-of-bounds IPAs */ VM_BUG_ON(fault_ipa >= kvm_phys_size(vcpu->arch.hw_mmu)); - if (fault_status == ESR_ELx_FSC_ACCESS) { + if (esr_fsc_is_access_flag_fault(esr)) { handle_access_fault(vcpu, fault_ipa); ret = 1; goto out_unlock; } - ret = user_mem_abort(vcpu, fault_ipa, memslot, hva, fault_status); + ret = user_mem_abort(vcpu, fault_ipa, memslot, hva, + esr_fsc_is_permission_fault(esr)); if (ret == 0) ret = 1; out: From bddae3e7ae70586b3805fdbce193760f13b4d73e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Thu, 23 Nov 2023 22:16:59 +0100 Subject: [PATCH 072/336] remoteproc: k3-dsp: Suppress duplicate error message in .remove() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the remove callback returns non-zero, the driver core emits an error message about the error value being ignored. As the driver already emits an error message already, return zero. This has no effect apart from suppressing the core's message. The platform device gets unbound irrespective of the return value. Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20231123211657.518181-6-u.kleine-koenig@pengutronix.de Signed-off-by: Mathieu Poirier --- drivers/remoteproc/ti_k3_dsp_remoteproc.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/remoteproc/ti_k3_dsp_remoteproc.c b/drivers/remoteproc/ti_k3_dsp_remoteproc.c index ef8415a7cd54..40a5fd8763fa 100644 --- a/drivers/remoteproc/ti_k3_dsp_remoteproc.c +++ b/drivers/remoteproc/ti_k3_dsp_remoteproc.c @@ -835,8 +835,9 @@ static int k3_dsp_rproc_remove(struct platform_device *pdev) if (rproc->state == RPROC_ATTACHED) { ret = rproc_detach(rproc); if (ret) { + /* Note this error path leaks resources */ dev_err(dev, "failed to detach proc, ret = %d\n", ret); - return ret; + return 0; } } From cfd0b5c4fd1d23d1c68f704dcd74c1dbdfd7c144 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Thu, 23 Nov 2023 22:17:00 +0100 Subject: [PATCH 073/336] remoteproc: k3-dsp: Use symbolic error codes in error messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The error message failed to send mailbox message (-EINVAL) is (for a human) more useful than failed to send mailbox message, status = -22 Adapt all error messages to use the symbolic names instead of the numeric constants. The error paths in .probe() make use of dev_err_probe() which automatically handles EPROBE_DEFER. Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20231123211657.518181-7-u.kleine-koenig@pengutronix.de> Signed-off-by: Mathieu Poirier --- drivers/remoteproc/ti_k3_dsp_remoteproc.c | 78 ++++++++++------------- 1 file changed, 34 insertions(+), 44 deletions(-) diff --git a/drivers/remoteproc/ti_k3_dsp_remoteproc.c b/drivers/remoteproc/ti_k3_dsp_remoteproc.c index 40a5fd8763fa..f048ec1bb00f 100644 --- a/drivers/remoteproc/ti_k3_dsp_remoteproc.c +++ b/drivers/remoteproc/ti_k3_dsp_remoteproc.c @@ -158,8 +158,8 @@ static void k3_dsp_rproc_kick(struct rproc *rproc, int vqid) /* send the index of the triggered virtqueue in the mailbox payload */ ret = mbox_send_message(kproc->mbox, (void *)msg); if (ret < 0) - dev_err(dev, "failed to send mailbox message, status = %d\n", - ret); + dev_err(dev, "failed to send mailbox message (%pe)\n", + ERR_PTR(ret)); } /* Put the DSP processor into reset */ @@ -170,7 +170,7 @@ static int k3_dsp_rproc_reset(struct k3_dsp_rproc *kproc) ret = reset_control_assert(kproc->reset); if (ret) { - dev_err(dev, "local-reset assert failed, ret = %d\n", ret); + dev_err(dev, "local-reset assert failed (%pe)\n", ERR_PTR(ret)); return ret; } @@ -180,7 +180,7 @@ static int k3_dsp_rproc_reset(struct k3_dsp_rproc *kproc) ret = kproc->ti_sci->ops.dev_ops.put_device(kproc->ti_sci, kproc->ti_sci_id); if (ret) { - dev_err(dev, "module-reset assert failed, ret = %d\n", ret); + dev_err(dev, "module-reset assert failed (%pe)\n", ERR_PTR(ret)); if (reset_control_deassert(kproc->reset)) dev_warn(dev, "local-reset deassert back failed\n"); } @@ -200,14 +200,14 @@ static int k3_dsp_rproc_release(struct k3_dsp_rproc *kproc) ret = kproc->ti_sci->ops.dev_ops.get_device(kproc->ti_sci, kproc->ti_sci_id); if (ret) { - dev_err(dev, "module-reset deassert failed, ret = %d\n", ret); + dev_err(dev, "module-reset deassert failed (%pe)\n", ERR_PTR(ret)); return ret; } lreset: ret = reset_control_deassert(kproc->reset); if (ret) { - dev_err(dev, "local-reset deassert failed, ret = %d\n", ret); + dev_err(dev, "local-reset deassert failed, (%pe)\n", ERR_PTR(ret)); if (kproc->ti_sci->ops.dev_ops.put_device(kproc->ti_sci, kproc->ti_sci_id)) dev_warn(dev, "module-reset assert back failed\n"); @@ -246,7 +246,7 @@ static int k3_dsp_rproc_request_mbox(struct rproc *rproc) */ ret = mbox_send_message(kproc->mbox, (void *)RP_MBOX_ECHO_REQUEST); if (ret < 0) { - dev_err(dev, "mbox_send_message failed: %d\n", ret); + dev_err(dev, "mbox_send_message failed (%pe)\n", ERR_PTR(ret)); mbox_free_channel(kproc->mbox); return ret; } @@ -272,8 +272,8 @@ static int k3_dsp_rproc_prepare(struct rproc *rproc) ret = kproc->ti_sci->ops.dev_ops.get_device(kproc->ti_sci, kproc->ti_sci_id); if (ret) - dev_err(dev, "module-reset deassert failed, cannot enable internal RAM loading, ret = %d\n", - ret); + dev_err(dev, "module-reset deassert failed, cannot enable internal RAM loading (%pe)\n", + ERR_PTR(ret)); return ret; } @@ -296,7 +296,7 @@ static int k3_dsp_rproc_unprepare(struct rproc *rproc) ret = kproc->ti_sci->ops.dev_ops.put_device(kproc->ti_sci, kproc->ti_sci_id); if (ret) - dev_err(dev, "module-reset assert failed, ret = %d\n", ret); + dev_err(dev, "module-reset assert failed (%pe)\n", ERR_PTR(ret)); return ret; } @@ -561,9 +561,9 @@ static int k3_dsp_reserved_mem_init(struct k3_dsp_rproc *kproc) num_rmems = of_property_count_elems_of_size(np, "memory-region", sizeof(phandle)); - if (num_rmems <= 0) { - dev_err(dev, "device does not reserved memory regions, ret = %d\n", - num_rmems); + if (num_rmems < 0) { + dev_err(dev, "device does not reserved memory regions (%pe)\n", + ERR_PTR(num_rmems)); return -EINVAL; } if (num_rmems < 2) { @@ -575,8 +575,8 @@ static int k3_dsp_reserved_mem_init(struct k3_dsp_rproc *kproc) /* use reserved memory region 0 for vring DMA allocations */ ret = of_reserved_mem_device_init_by_idx(dev, np, 0); if (ret) { - dev_err(dev, "device cannot initialize DMA pool, ret = %d\n", - ret); + dev_err(dev, "device cannot initialize DMA pool (%pe)\n", + ERR_PTR(ret)); return ret; } @@ -687,11 +687,8 @@ static int k3_dsp_rproc_probe(struct platform_device *pdev) return -ENODEV; ret = rproc_of_parse_firmware(dev, 0, &fw_name); - if (ret) { - dev_err(dev, "failed to parse firmware-name property, ret = %d\n", - ret); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "failed to parse firmware-name property\n"); rproc = rproc_alloc(dev, dev_name(dev), &k3_dsp_rproc_ops, fw_name, sizeof(*kproc)); @@ -711,39 +708,35 @@ static int k3_dsp_rproc_probe(struct platform_device *pdev) kproc->ti_sci = ti_sci_get_by_phandle(np, "ti,sci"); if (IS_ERR(kproc->ti_sci)) { - ret = PTR_ERR(kproc->ti_sci); - if (ret != -EPROBE_DEFER) { - dev_err(dev, "failed to get ti-sci handle, ret = %d\n", - ret); - } + ret = dev_err_probe(dev, PTR_ERR(kproc->ti_sci), + "failed to get ti-sci handle\n"); kproc->ti_sci = NULL; goto free_rproc; } ret = of_property_read_u32(np, "ti,sci-dev-id", &kproc->ti_sci_id); if (ret) { - dev_err(dev, "missing 'ti,sci-dev-id' property\n"); + dev_err_probe(dev, ret, "missing 'ti,sci-dev-id' property\n"); goto put_sci; } kproc->reset = devm_reset_control_get_exclusive(dev, NULL); if (IS_ERR(kproc->reset)) { - ret = PTR_ERR(kproc->reset); - dev_err(dev, "failed to get reset, status = %d\n", ret); + ret = dev_err_probe(dev, PTR_ERR(kproc->reset), + "failed to get reset\n"); goto put_sci; } kproc->tsp = k3_dsp_rproc_of_get_tsp(dev, kproc->ti_sci); if (IS_ERR(kproc->tsp)) { - dev_err(dev, "failed to construct ti-sci proc control, ret = %d\n", - ret); - ret = PTR_ERR(kproc->tsp); + ret = dev_err_probe(dev, PTR_ERR(kproc->tsp), + "failed to construct ti-sci proc control\n"); goto put_sci; } ret = ti_sci_proc_request(kproc->tsp); if (ret < 0) { - dev_err(dev, "ti_sci_proc_request failed, ret = %d\n", ret); + dev_err_probe(dev, ret, "ti_sci_proc_request failed\n"); goto free_tsp; } @@ -753,15 +746,14 @@ static int k3_dsp_rproc_probe(struct platform_device *pdev) ret = k3_dsp_reserved_mem_init(kproc); if (ret) { - dev_err(dev, "reserved memory init failed, ret = %d\n", ret); + dev_err_probe(dev, ret, "reserved memory init failed\n"); goto release_tsp; } ret = kproc->ti_sci->ops.dev_ops.is_on(kproc->ti_sci, kproc->ti_sci_id, NULL, &p_state); if (ret) { - dev_err(dev, "failed to get initial state, mode cannot be determined, ret = %d\n", - ret); + dev_err_probe(dev, ret, "failed to get initial state, mode cannot be determined\n"); goto release_mem; } @@ -787,8 +779,7 @@ static int k3_dsp_rproc_probe(struct platform_device *pdev) if (data->uses_lreset) { ret = reset_control_status(kproc->reset); if (ret < 0) { - dev_err(dev, "failed to get reset status, status = %d\n", - ret); + dev_err_probe(dev, ret, "failed to get reset status\n"); goto release_mem; } else if (ret == 0) { dev_warn(dev, "local reset is deasserted for device\n"); @@ -799,8 +790,7 @@ static int k3_dsp_rproc_probe(struct platform_device *pdev) ret = rproc_add(rproc); if (ret) { - dev_err(dev, "failed to add register device with remoteproc core, status = %d\n", - ret); + dev_err_probe(dev, ret, "failed to add register device with remoteproc core\n"); goto release_mem; } @@ -813,13 +803,13 @@ release_mem: release_tsp: ret1 = ti_sci_proc_release(kproc->tsp); if (ret1) - dev_err(dev, "failed to release proc, ret = %d\n", ret1); + dev_err(dev, "failed to release proc (%pe)\n", ERR_PTR(ret1)); free_tsp: kfree(kproc->tsp); put_sci: ret1 = ti_sci_put_handle(kproc->ti_sci); if (ret1) - dev_err(dev, "failed to put ti_sci handle, ret = %d\n", ret1); + dev_err(dev, "failed to put ti_sci handle (%pe)\n", ERR_PTR(ret1)); free_rproc: rproc_free(rproc); return ret; @@ -836,7 +826,7 @@ static int k3_dsp_rproc_remove(struct platform_device *pdev) ret = rproc_detach(rproc); if (ret) { /* Note this error path leaks resources */ - dev_err(dev, "failed to detach proc, ret = %d\n", ret); + dev_err(dev, "failed to detach proc (%pe)\n", ERR_PTR(ret)); return 0; } } @@ -845,13 +835,13 @@ static int k3_dsp_rproc_remove(struct platform_device *pdev) ret = ti_sci_proc_release(kproc->tsp); if (ret) - dev_err(dev, "failed to release proc, ret = %d\n", ret); + dev_err(dev, "failed to release proc (%pe)\n", ERR_PTR(ret)); kfree(kproc->tsp); ret = ti_sci_put_handle(kproc->ti_sci); if (ret) - dev_err(dev, "failed to put ti_sci handle, ret = %d\n", ret); + dev_err(dev, "failed to put ti_sci handle (%pe)\n", ERR_PTR(ret)); k3_dsp_reserved_mem_exit(kproc); rproc_free(kproc->rproc); From 3f978d9889a2531e48d97cef7f5126f4104ee69c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Thu, 23 Nov 2023 22:17:01 +0100 Subject: [PATCH 074/336] remoteproc: k3-dsp: Convert to platform remove callback returning void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The .remove() callback for a platform driver returns an int which makes many driver authors wrongly assume it's possible to do error handling by returning an error code. However the value returned is ignored (apart from emitting a warning) and this typically results in resource leaks. To improve here there is a quest to make the remove callback return void. In the first step of this quest all drivers are converted to .remove_new(), which already returns void. Eventually after all drivers are converted, .remove_new() will be renamed to .remove(). There is no change in behaviour as .remove() already returned zero unconditionally. Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20231123211657.518181-8-u.kleine-koenig@pengutronix.de Signed-off-by: Mathieu Poirier --- drivers/remoteproc/ti_k3_dsp_remoteproc.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/drivers/remoteproc/ti_k3_dsp_remoteproc.c b/drivers/remoteproc/ti_k3_dsp_remoteproc.c index f048ec1bb00f..ab882e3b7130 100644 --- a/drivers/remoteproc/ti_k3_dsp_remoteproc.c +++ b/drivers/remoteproc/ti_k3_dsp_remoteproc.c @@ -815,7 +815,7 @@ free_rproc: return ret; } -static int k3_dsp_rproc_remove(struct platform_device *pdev) +static void k3_dsp_rproc_remove(struct platform_device *pdev) { struct k3_dsp_rproc *kproc = platform_get_drvdata(pdev); struct rproc *rproc = kproc->rproc; @@ -827,7 +827,7 @@ static int k3_dsp_rproc_remove(struct platform_device *pdev) if (ret) { /* Note this error path leaks resources */ dev_err(dev, "failed to detach proc (%pe)\n", ERR_PTR(ret)); - return 0; + return; } } @@ -845,8 +845,6 @@ static int k3_dsp_rproc_remove(struct platform_device *pdev) k3_dsp_reserved_mem_exit(kproc); rproc_free(kproc->rproc); - - return 0; } static const struct k3_dsp_mem_data c66_mems[] = { @@ -897,7 +895,7 @@ MODULE_DEVICE_TABLE(of, k3_dsp_of_match); static struct platform_driver k3_dsp_rproc_driver = { .probe = k3_dsp_rproc_probe, - .remove = k3_dsp_rproc_remove, + .remove_new = k3_dsp_rproc_remove, .driver = { .name = "k3-dsp-rproc", .of_match_table = k3_dsp_of_match, From 75bedc1ee90bd54ae8c5ab8be72506f8c5959584 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 18 Oct 2023 08:19:06 -0700 Subject: [PATCH 075/336] KVM: x86: Turn off KVM_WERROR by default for all configs Don't enable KVM_WERROR by default for x86-64 builds as KVM's one-off -Werror enabling is *mostly* superseded by the kernel-wide WERROR, and enabling KVM_WERROR by default can cause problems for developers working on other subsystems. E.g. subsystems that have a "zero W=1 regressions" rule can inadvertently build KVM with -Werror and W=1, and end up with build failures that are completely uninteresting to the developer (W=1 is prone to false positives, especially on older compilers). Keep KVM_WERROR as there are combinations where enabling WERROR isn't feasible, e.g. the default FRAME_WARN=1024 on i386 builds generates a non-zero number of warnings and thus errors, and there are far too many warnings throughout the kernel to enable WERROR with W=1 (building KVM with -Werror is desirable (with a sane compiler) as W=1 does generate useful warnings). Opportunistically drop the dependency on !COMPILE_TEST as it's completely meaningless (it was copied from i195's -Werror Kconfig), as the kernel's WERROR is explicitly *enabled* for COMPILE_TEST=y kernel's, i.e. enabling -Werror is obviosly not dependent on COMPILE_TEST=n. Reported-by: Jakub Kicinski Link: https://lore.kernel.org/all/20231006205415.3501535-1-kuba@kernel.org Link: https://lore.kernel.org/r/20231018151906.1841689-1-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/Kconfig | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index c1716e83d176..4d086fe67a7c 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -65,13 +65,13 @@ config KVM config KVM_WERROR bool "Compile KVM with -Werror" - # KASAN may cause the build to fail due to larger frames - default y if X86_64 && !KASAN - # We use the dependency on !COMPILE_TEST to not be enabled - # blindly in allmodconfig or allyesconfig configurations - depends on KVM - depends on (X86_64 && !KASAN) || !COMPILE_TEST - depends on EXPERT + # Disallow KVM's -Werror if KASAN is enabled, e.g. to guard against + # randomized configs from selecting KVM_WERROR=y, which doesn't play + # nice with KASAN. KASAN builds generates warnings for the default + # FRAME_WARN, i.e. KVM_WERROR=y with KASAN=y requires special tuning. + # Building KVM with -Werror and KASAN is still doable via enabling + # the kernel-wide WERROR=y. + depends on KVM && EXPERT && !KASAN help Add -Werror to the build flags for KVM. From eefe5e6682099445f77f2d97d4c525f9ac9d9b07 Mon Sep 17 00:00:00 2001 From: Jim Mattson Date: Mon, 23 Oct 2023 17:16:35 -0700 Subject: [PATCH 076/336] KVM: x86: Advertise CPUID.(EAX=7,ECX=2):EDX[5:0] to userspace The low five bits {INTEL_PSFD, IPRED_CTRL, RRSBA_CTRL, DDPD_U, BHI_CTRL} advertise the availability of specific bits in IA32_SPEC_CTRL. Since KVM dynamically determines the legal IA32_SPEC_CTRL bits for the underlying hardware, the hard work has already been done. Just let userspace know that a guest can use these IA32_SPEC_CTRL bits. The sixth bit (MCDT_NO) states that the processor does not exhibit MXCSR Configuration Dependent Timing (MCDT) behavior. This is an inherent property of the physical processor that is inherited by the virtual CPU. Pass that information on to userspace. Signed-off-by: Jim Mattson Reviewed-by: Chao Gao Link: https://lore.kernel.org/r/20231024001636.890236-1-jmattson@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/cpuid.c | 21 ++++++++++++++++++--- arch/x86/kvm/reverse_cpuid.h | 12 ++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index dda6fc4cfae8..1811a9ddfe1d 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -679,6 +679,11 @@ void kvm_set_cpu_caps(void) F(AMX_COMPLEX) ); + kvm_cpu_cap_init_kvm_defined(CPUID_7_2_EDX, + F(INTEL_PSFD) | F(IPRED_CTRL) | F(RRSBA_CTRL) | F(DDPD_U) | + F(BHI_CTRL) | F(MCDT_NO) + ); + kvm_cpu_cap_mask(CPUID_D_1_EAX, F(XSAVEOPT) | F(XSAVEC) | F(XGETBV1) | F(XSAVES) | f_xfd ); @@ -960,13 +965,13 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function) break; /* function 7 has additional index. */ case 7: - entry->eax = min(entry->eax, 1u); + max_idx = entry->eax = min(entry->eax, 2u); cpuid_entry_override(entry, CPUID_7_0_EBX); cpuid_entry_override(entry, CPUID_7_ECX); cpuid_entry_override(entry, CPUID_7_EDX); - /* KVM only supports 0x7.0 and 0x7.1, capped above via min(). */ - if (entry->eax == 1) { + /* KVM only supports up to 0x7.2, capped above via min(). */ + if (max_idx >= 1) { entry = do_host_cpuid(array, function, 1); if (!entry) goto out; @@ -976,6 +981,16 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function) entry->ebx = 0; entry->ecx = 0; } + if (max_idx >= 2) { + entry = do_host_cpuid(array, function, 2); + if (!entry) + goto out; + + cpuid_entry_override(entry, CPUID_7_2_EDX); + entry->ecx = 0; + entry->ebx = 0; + entry->eax = 0; + } break; case 0xa: { /* Architectural Performance Monitoring */ union cpuid10_eax eax; diff --git a/arch/x86/kvm/reverse_cpuid.h b/arch/x86/kvm/reverse_cpuid.h index b81650678375..17007016d8b5 100644 --- a/arch/x86/kvm/reverse_cpuid.h +++ b/arch/x86/kvm/reverse_cpuid.h @@ -16,6 +16,7 @@ enum kvm_only_cpuid_leafs { CPUID_7_1_EDX, CPUID_8000_0007_EDX, CPUID_8000_0022_EAX, + CPUID_7_2_EDX, NR_KVM_CPU_CAPS, NKVMCAPINTS = NR_KVM_CPU_CAPS - NCAPINTS, @@ -46,6 +47,14 @@ enum kvm_only_cpuid_leafs { #define X86_FEATURE_AMX_COMPLEX KVM_X86_FEATURE(CPUID_7_1_EDX, 8) #define X86_FEATURE_PREFETCHITI KVM_X86_FEATURE(CPUID_7_1_EDX, 14) +/* Intel-defined sub-features, CPUID level 0x00000007:2 (EDX) */ +#define X86_FEATURE_INTEL_PSFD KVM_X86_FEATURE(CPUID_7_2_EDX, 0) +#define X86_FEATURE_IPRED_CTRL KVM_X86_FEATURE(CPUID_7_2_EDX, 1) +#define KVM_X86_FEATURE_RRSBA_CTRL KVM_X86_FEATURE(CPUID_7_2_EDX, 2) +#define X86_FEATURE_DDPD_U KVM_X86_FEATURE(CPUID_7_2_EDX, 3) +#define X86_FEATURE_BHI_CTRL KVM_X86_FEATURE(CPUID_7_2_EDX, 4) +#define X86_FEATURE_MCDT_NO KVM_X86_FEATURE(CPUID_7_2_EDX, 5) + /* CPUID level 0x80000007 (EDX). */ #define KVM_X86_FEATURE_CONSTANT_TSC KVM_X86_FEATURE(CPUID_8000_0007_EDX, 8) @@ -80,6 +89,7 @@ static const struct cpuid_reg reverse_cpuid[] = { [CPUID_8000_0007_EDX] = {0x80000007, 0, CPUID_EDX}, [CPUID_8000_0021_EAX] = {0x80000021, 0, CPUID_EAX}, [CPUID_8000_0022_EAX] = {0x80000022, 0, CPUID_EAX}, + [CPUID_7_2_EDX] = { 7, 2, CPUID_EDX}, }; /* @@ -116,6 +126,8 @@ static __always_inline u32 __feature_translate(int x86_feature) return KVM_X86_FEATURE_CONSTANT_TSC; else if (x86_feature == X86_FEATURE_PERFMON_V2) return KVM_X86_FEATURE_PERFMON_V2; + else if (x86_feature == X86_FEATURE_RRSBA_CTRL) + return KVM_X86_FEATURE_RRSBA_CTRL; return x86_feature; } From 80c883db87d9ffe2d685e91ba07a087b1c246c78 Mon Sep 17 00:00:00 2001 From: Jim Mattson Date: Mon, 23 Oct 2023 17:16:36 -0700 Subject: [PATCH 077/336] KVM: x86: Use a switch statement and macros in __feature_translate() Use a switch statement with macro-generated case statements to handle translating feature flags in order to reduce the probability of runtime errors due to copy+paste goofs, to make compile-time errors easier to debug, and to make the code more readable. E.g. the compiler won't directly generate an error for duplicate if statements if (x86_feature == X86_FEATURE_SGX1) return KVM_X86_FEATURE_SGX1; else if (x86_feature == X86_FEATURE_SGX2) return KVM_X86_FEATURE_SGX1; and so instead reverse_cpuid_check() will fail due to the untranslated entry pointing at a Linux-defined leaf, which provides practically no hint as to what is broken arch/x86/kvm/reverse_cpuid.h:108:2: error: call to __compiletime_assert_450 declared with 'error' attribute: BUILD_BUG_ON failed: x86_leaf == CPUID_LNX_4 BUILD_BUG_ON(x86_leaf == CPUID_LNX_4); ^ whereas duplicate case statements very explicitly point at the offending code: arch/x86/kvm/reverse_cpuid.h:125:2: error: duplicate case value '361' KVM_X86_TRANSLATE_FEATURE(SGX2); ^ arch/x86/kvm/reverse_cpuid.h:124:2: error: duplicate case value '360' KVM_X86_TRANSLATE_FEATURE(SGX1); ^ And without macros, the opposite type of copy+paste goof doesn't generate any error at compile-time, e.g. this yields no complaints: case X86_FEATURE_SGX1: return KVM_X86_FEATURE_SGX1; case X86_FEATURE_SGX2: return KVM_X86_FEATURE_SGX1; Note, __feature_translate() is forcibly inlined and the feature is known at compile-time, so the code generation between an if-elif sequence and a switch statement should be identical. Signed-off-by: Jim Mattson Link: https://lore.kernel.org/r/20231024001636.890236-2-jmattson@google.com [sean: use a macro, rewrite changelog] Signed-off-by: Sean Christopherson --- arch/x86/kvm/reverse_cpuid.h | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/arch/x86/kvm/reverse_cpuid.h b/arch/x86/kvm/reverse_cpuid.h index 17007016d8b5..aadefcaa9561 100644 --- a/arch/x86/kvm/reverse_cpuid.h +++ b/arch/x86/kvm/reverse_cpuid.h @@ -116,20 +116,19 @@ static __always_inline void reverse_cpuid_check(unsigned int x86_leaf) */ static __always_inline u32 __feature_translate(int x86_feature) { - if (x86_feature == X86_FEATURE_SGX1) - return KVM_X86_FEATURE_SGX1; - else if (x86_feature == X86_FEATURE_SGX2) - return KVM_X86_FEATURE_SGX2; - else if (x86_feature == X86_FEATURE_SGX_EDECCSSA) - return KVM_X86_FEATURE_SGX_EDECCSSA; - else if (x86_feature == X86_FEATURE_CONSTANT_TSC) - return KVM_X86_FEATURE_CONSTANT_TSC; - else if (x86_feature == X86_FEATURE_PERFMON_V2) - return KVM_X86_FEATURE_PERFMON_V2; - else if (x86_feature == X86_FEATURE_RRSBA_CTRL) - return KVM_X86_FEATURE_RRSBA_CTRL; +#define KVM_X86_TRANSLATE_FEATURE(f) \ + case X86_FEATURE_##f: return KVM_X86_FEATURE_##f - return x86_feature; + switch (x86_feature) { + KVM_X86_TRANSLATE_FEATURE(SGX1); + KVM_X86_TRANSLATE_FEATURE(SGX2); + KVM_X86_TRANSLATE_FEATURE(SGX_EDECCSSA); + KVM_X86_TRANSLATE_FEATURE(CONSTANT_TSC); + KVM_X86_TRANSLATE_FEATURE(PERFMON_V2); + KVM_X86_TRANSLATE_FEATURE(RRSBA_CTRL); + default: + return x86_feature; + } } static __always_inline u32 __feature_leaf(int x86_feature) From c52ffadc65e28ab461fd055e9991e8d8106a0056 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 18 Oct 2023 12:56:38 -0700 Subject: [PATCH 078/336] KVM: x86: Don't unnecessarily force masterclock update on vCPU hotplug Don't force a masterclock update when a vCPU synchronizes to the current TSC generation, e.g. when userspace hotplugs a pre-created vCPU into the VM. Unnecessarily updating the masterclock is undesirable as it can cause kvmclock's time to jump, which is particularly painful on systems with a stable TSC as kvmclock _should_ be fully reliable on such systems. The unexpected time jumps are due to differences in the TSC=>nanoseconds conversion algorithms between kvmclock and the host's CLOCK_MONOTONIC_RAW (the pvclock algorithm is inherently lossy). When updating the masterclock, KVM refreshes the "base", i.e. moves the elapsed time since the last update from the kvmclock/pvclock algorithm to the CLOCK_MONOTONIC_RAW algorithm. Synchronizing kvmclock with CLOCK_MONOTONIC_RAW is the lesser of evils when the TSC is unstable, but adds no real value when the TSC is stable. Prior to commit 7f187922ddf6 ("KVM: x86: update masterclock values on TSC writes"), KVM did NOT force an update when synchronizing a vCPU to the current generation. commit 7f187922ddf6b67f2999a76dcb71663097b75497 Author: Marcelo Tosatti Date: Tue Nov 4 21:30:44 2014 -0200 KVM: x86: update masterclock values on TSC writes When the guest writes to the TSC, the masterclock TSC copy must be updated as well along with the TSC_OFFSET update, otherwise a negative tsc_timestamp is calculated at kvm_guest_time_update. Once "if (!vcpus_matched && ka->use_master_clock)" is simplified to "if (ka->use_master_clock)", the corresponding "if (!ka->use_master_clock)" becomes redundant, so remove the do_request boolean and collapse everything into a single condition. Before that, KVM only re-synced the masterclock if the masterclock was enabled or disabled Note, at the time of the above commit, VMX synchronized TSC on *guest* writes to MSR_IA32_TSC: case MSR_IA32_TSC: kvm_write_tsc(vcpu, msr_info); break; which is why the changelog specifically says "guest writes", but the bug that was being fixed wasn't unique to guest write, i.e. a TSC write from the host would suffer the same problem. So even though KVM stopped synchronizing on guest writes as of commit 0c899c25d754 ("KVM: x86: do not attempt TSC synchronization on guest writes"), simply reverting commit 7f187922ddf6 is not an option. Figuring out how a negative tsc_timestamp could be computed requires a bit more sleuthing. In kvm_write_tsc() (at the time), except for KVM's "less than 1 second" hack, KVM snapshotted the vCPU's current TSC *and* the current time in nanoseconds, where kvm->arch.cur_tsc_nsec is the current host kernel time in nanoseconds: ns = get_kernel_ns(); ... if (usdiff < USEC_PER_SEC && vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) { ... } else { /* * We split periods of matched TSC writes into generations. * For each generation, we track the original measured * nanosecond time, offset, and write, so if TSCs are in * sync, we can match exact offset, and if not, we can match * exact software computation in compute_guest_tsc() * * These values are tracked in kvm->arch.cur_xxx variables. */ kvm->arch.cur_tsc_generation++; kvm->arch.cur_tsc_nsec = ns; kvm->arch.cur_tsc_write = data; kvm->arch.cur_tsc_offset = offset; matched = false; pr_debug("kvm: new tsc generation %llu, clock %llu\n", kvm->arch.cur_tsc_generation, data); } ... /* Keep track of which generation this VCPU has synchronized to */ vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation; vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec; vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write; Note that the above creates a new generation and sets "matched" to false! But because kvm_track_tsc_matching() looks for matched+1, i.e. doesn't require the vCPU that creates the new generation to match itself, KVM would immediately compute vcpus_matched as true for VMs with a single vCPU. As a result, KVM would skip the masterlock update, even though a new TSC generation was created: vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 == atomic_read(&vcpu->kvm->online_vcpus)); if (vcpus_matched && gtod->clock.vclock_mode == VCLOCK_TSC) if (!ka->use_master_clock) do_request = 1; if (!vcpus_matched && ka->use_master_clock) do_request = 1; if (do_request) kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); On hardware without TSC scaling support, vcpu->tsc_catchup is set to true if the guest TSC frequency is faster than the host TSC frequency, even if the TSC is otherwise stable. And for that mode, kvm_guest_time_update(), by way of compute_guest_tsc(), uses vcpu->arch.this_tsc_nsec, a.k.a. the kernel time at the last TSC write, to compute the guest TSC relative to kernel time: static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns) { u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec, vcpu->arch.virtual_tsc_mult, vcpu->arch.virtual_tsc_shift); tsc += vcpu->arch.this_tsc_write; return tsc; } Except the "kernel_ns" passed to compute_guest_tsc() isn't the current kernel time, it's the masterclock snapshot! spin_lock(&ka->pvclock_gtod_sync_lock); use_master_clock = ka->use_master_clock; if (use_master_clock) { host_tsc = ka->master_cycle_now; kernel_ns = ka->master_kernel_ns; } spin_unlock(&ka->pvclock_gtod_sync_lock); if (vcpu->tsc_catchup) { u64 tsc = compute_guest_tsc(v, kernel_ns); if (tsc > tsc_timestamp) { adjust_tsc_offset_guest(v, tsc - tsc_timestamp); tsc_timestamp = tsc; } } And so when KVM skips the masterclock update after a TSC write, i.e. after a new TSC generation is started, the "kernel_ns-vcpu->arch.this_tsc_nsec" is *guaranteed* to generate a negative value, because this_tsc_nsec was captured after ka->master_kernel_ns. Forcing a masterclock update essentially fudged around that problem, but in a heavy handed way that introduced undesirable side effects, i.e. unnecessarily forces a masterclock update when a new vCPU joins the party via hotplug. Note, KVM forces masterclock updates in other weird ways that are also likely unnecessary, e.g. when establishing a new Xen shared info page and when userspace creates a brand new vCPU. But the Xen thing is firmly a separate mess, and there are no known userspace VMMs that utilize kvmclock *and* create new vCPUs after the VM is up and running. I.e. the other issues are future problems. Reported-by: Dongli Zhang Closes: https://lore.kernel.org/all/20230926230649.67852-1-dongli.zhang@oracle.com Fixes: 7f187922ddf6 ("KVM: x86: update masterclock values on TSC writes") Cc: David Woodhouse Reviewed-by: Dongli Zhang Tested-by: Dongli Zhang Link: https://lore.kernel.org/r/20231018195638.1898375-1-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 6d0772b47041..99ec48203667 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -2510,26 +2510,29 @@ static inline int gtod_is_based_on_tsc(int mode) } #endif -static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu) +static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu, bool new_generation) { #ifdef CONFIG_X86_64 - bool vcpus_matched; struct kvm_arch *ka = &vcpu->kvm->arch; struct pvclock_gtod_data *gtod = &pvclock_gtod_data; - vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 == - atomic_read(&vcpu->kvm->online_vcpus)); + /* + * To use the masterclock, the host clocksource must be based on TSC + * and all vCPUs must have matching TSCs. Note, the count for matching + * vCPUs doesn't include the reference vCPU, hence "+1". + */ + bool use_master_clock = (ka->nr_vcpus_matched_tsc + 1 == + atomic_read(&vcpu->kvm->online_vcpus)) && + gtod_is_based_on_tsc(gtod->clock.vclock_mode); /* - * Once the masterclock is enabled, always perform request in - * order to update it. - * - * In order to enable masterclock, the host clocksource must be TSC - * and the vcpus need to have matched TSCs. When that happens, - * perform request to enable masterclock. + * Request a masterclock update if the masterclock needs to be toggled + * on/off, or when starting a new generation and the masterclock is + * enabled (compute_guest_tsc() requires the masterclock snapshot to be + * taken _after_ the new generation is created). */ - if (ka->use_master_clock || - (gtod_is_based_on_tsc(gtod->clock.vclock_mode) && vcpus_matched)) + if ((ka->use_master_clock && new_generation) || + (ka->use_master_clock != use_master_clock)) kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc, @@ -2706,7 +2709,7 @@ static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc, vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec; vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write; - kvm_track_tsc_matching(vcpu); + kvm_track_tsc_matching(vcpu, !matched); } static void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value) From a484755ab2526ebdbe042397cdd6e427eb4b1a68 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 18 Oct 2023 12:41:03 -0700 Subject: [PATCH 079/336] Revert "nSVM: Check for reserved encodings of TLB_CONTROL in nested VMCB" Revert KVM's made-up consistency check on SVM's TLB control. The APM says that unsupported encodings are reserved, but the APM doesn't state that VMRUN checks for a supported encoding. Unless something is called out in "Canonicalization and Consistency Checks" or listed as MBZ (Must Be Zero), AMD behavior is typically to let software shoot itself in the foot. This reverts commit 174a921b6975ef959dd82ee9e8844067a62e3ec1. Fixes: 174a921b6975 ("nSVM: Check for reserved encodings of TLB_CONTROL in nested VMCB") Reported-by: Stefan Sterz Closes: https://lkml.kernel.org/r/b9915c9c-4cf6-051a-2d91-44cc6380f455%40proxmox.com Cc: stable@vger.kernel.org Reviewed-by: Maxim Levitsky Link: https://lore.kernel.org/r/20231018194104.1896415-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/nested.c | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 3fea8c47679e..60891b9ce25f 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -247,18 +247,6 @@ static bool nested_svm_check_bitmap_pa(struct kvm_vcpu *vcpu, u64 pa, u32 size) kvm_vcpu_is_legal_gpa(vcpu, addr + size - 1); } -static bool nested_svm_check_tlb_ctl(struct kvm_vcpu *vcpu, u8 tlb_ctl) -{ - /* Nested FLUSHBYASID is not supported yet. */ - switch(tlb_ctl) { - case TLB_CONTROL_DO_NOTHING: - case TLB_CONTROL_FLUSH_ALL_ASID: - return true; - default: - return false; - } -} - static bool __nested_vmcb_check_controls(struct kvm_vcpu *vcpu, struct vmcb_ctrl_area_cached *control) { @@ -278,9 +266,6 @@ static bool __nested_vmcb_check_controls(struct kvm_vcpu *vcpu, IOPM_SIZE))) return false; - if (CC(!nested_svm_check_tlb_ctl(vcpu, control->tlb_ctl))) - return false; - if (CC((control->int_ctl & V_NMI_ENABLE_MASK) && !vmcb12_is_intercept(control, INTERCEPT_NMI))) { return false; From 176bfc5b17fee327585583a427e2857d9dfd8f68 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 18 Oct 2023 12:41:04 -0700 Subject: [PATCH 080/336] KVM: nSVM: Advertise support for flush-by-ASID Advertise support for FLUSHBYASID when nested SVM is enabled, as KVM can always emulate flushing TLB entries for a vmcb12 ASID, e.g. by running L2 with a new, fresh ASID in vmcb02. Some modern hypervisors, e.g. VMWare Workstation 17, require FLUSHBYASID support and will refuse to run if it's not present. Punt on proper support, as "Honor L1's request to flush an ASID on nested VMRUN" is one of the TODO items in the (incomplete) list of issues that need to be addressed in order for KVM to NOT do a full TLB flush on every nested SVM transition (see nested_svm_transition_tlb_flush()). Reported-by: Stefan Sterz Closes: https://lkml.kernel.org/r/b9915c9c-4cf6-051a-2d91-44cc6380f455%40proxmox.com Reviewed-by: Maxim Levitsky Link: https://lore.kernel.org/r/20231018194104.1896415-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/svm.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 712146312358..30352c42d62a 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -5076,6 +5076,13 @@ static __init void svm_set_cpu_caps(void) kvm_cpu_cap_set(X86_FEATURE_SVM); kvm_cpu_cap_set(X86_FEATURE_VMCBCLEAN); + /* + * KVM currently flushes TLBs on *every* nested SVM transition, + * and so for all intents and purposes KVM supports flushing by + * ASID, i.e. KVM is guaranteed to honor every L1 ASID flush. + */ + kvm_cpu_cap_set(X86_FEATURE_FLUSHBYASID); + if (nrips) kvm_cpu_cap_set(X86_FEATURE_NRIPS); From 770d6aa2e416fd26f0356e258c77a37574ad9b8c Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 18 Oct 2023 12:36:17 -0700 Subject: [PATCH 081/336] KVM: SVM: Explicitly require FLUSHBYASID to enable SEV support Add a sanity check that FLUSHBYASID is available if SEV is supported in hardware, as SEV (and beyond) guests are bound to a single ASID, i.e. KVM can't "flush" by assigning a new, fresh ASID to the guest. If FLUSHBYASID isn't supported for some bizarre reason, KVM would completely fail to do TLB flushes for SEV+ guests (see pre_svm_run() and pre_sev_run()). Cc: Tom Lendacky Link: https://lore.kernel.org/r/20231018193617.1895752-1-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/sev.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 4900c078045a..d0c580607f00 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -2191,10 +2191,13 @@ void __init sev_hardware_setup(void) /* * SEV must obviously be supported in hardware. Sanity check that the * CPU supports decode assists, which is mandatory for SEV guests to - * support instruction emulation. + * support instruction emulation. Ditto for flushing by ASID, as SEV + * guests are bound to a single ASID, i.e. KVM can't rotate to a new + * ASID to effect a TLB flush. */ if (!boot_cpu_has(X86_FEATURE_SEV) || - WARN_ON_ONCE(!boot_cpu_has(X86_FEATURE_DECODEASSISTS))) + WARN_ON_ONCE(!boot_cpu_has(X86_FEATURE_DECODEASSISTS)) || + WARN_ON_ONCE(!boot_cpu_has(X86_FEATURE_FLUSHBYASID))) goto out; /* Retrieve SEV CPUID information */ From 72046d0a077a8f70d4d1e5bdeed324c1a310da8c Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 18 Oct 2023 12:20:21 -0700 Subject: [PATCH 082/336] KVM: SVM: Don't intercept IRET when injecting NMI and vNMI is enabled When vNMI is enabled, rely entirely on hardware to correctly handle NMI blocking, i.e. don't intercept IRET to detect when NMIs are no longer blocked. KVM already correctly ignores svm->nmi_masked when vNMI is enabled, so the effect of the bug is essentially an unnecessary VM-Exit. KVM intercepts IRET for two reasons: - To track NMI masking to be able to know at any point of time if NMI is masked. - To track NMI windows (to inject another NMI after the guest executes IRET, i.e. unblocks NMIs) When vNMI is enabled, both cases are handled by hardware: - NMI masking state resides in int_ctl.V_NMI_BLOCKING and can be read by KVM at will. - Hardware automatically "injects" pending virtual NMIs when virtual NMIs become unblocked. However, even though pending a virtual NMI for hardware to handle is the most common way to synthesize a guest NMI, KVM may still directly inject an NMI via when KVM is handling two "simultaneous" NMIs (see comments in process_nmi() for details on KVM's simultaneous NMI handling). Per AMD's APM, hardware sets the BLOCKING flag when software directly injects an NMI as well, i.e. KVM doesn't need to manually mark vNMIs as blocked: If Event Injection is used to inject an NMI when NMI Virtualization is enabled, VMRUN sets V_NMI_MASK in the guest state. Note, it's still possible that KVM could trigger a spurious IRET VM-Exit. When running a nested guest, KVM disables vNMI for L2 and thus will enable IRET interception (in both vmcb01 and vmcb02) while running L2 reason. If a nested VM-Exit happens before L2 executes IRET, KVM can end up running L1 with vNMI enable and IRET intercepted. This is also a benign bug, and even less likely to happen, i.e. can be safely punted to a future fix. Fixes: fa4c027a7956 ("KVM: x86: Add support for SVM's Virtual NMI") Link: https://lore.kernel.org/all/ZOdnuDZUd4mevCqe@google.como Cc: Santosh Shukla Cc: Maxim Levitsky Tested-by: Santosh Shukla Link: https://lore.kernel.org/r/20231018192021.1893261-1-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/svm.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 30352c42d62a..c46f07b28230 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3560,8 +3560,15 @@ static void svm_inject_nmi(struct kvm_vcpu *vcpu) if (svm->nmi_l1_to_l2) return; - svm->nmi_masked = true; - svm_set_iret_intercept(svm); + /* + * No need to manually track NMI masking when vNMI is enabled, hardware + * automatically sets V_NMI_BLOCKING_MASK as appropriate, including the + * case where software directly injects an NMI. + */ + if (!is_vnmi_enabled(svm)) { + svm->nmi_masked = true; + svm_set_iret_intercept(svm); + } ++vcpu->stat.nmi_injections; } From 15223c4f973a6120665ece9ce1ad17aec0be0e6c Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Tue, 31 Oct 2023 08:52:40 +0100 Subject: [PATCH 083/336] KVM: SVM,VMX: Use %rip-relative addressing to access kvm_rebooting Instruction with %rip-relative address operand is one byte shorter than its absolute address counterpart and is also compatible with position independent executable (-fpie) build. No functional changes intended. Cc: Sean Christopherson Cc: Paolo Bonzini Signed-off-by: Uros Bizjak Link: https://lore.kernel.org/r/20231031075312.47525-1-ubizjak@gmail.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/vmenter.S | 10 +++++----- arch/x86/kvm/vmx/vmenter.S | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/svm/vmenter.S b/arch/x86/kvm/svm/vmenter.S index ef2ebabb059c..9499f9c6b077 100644 --- a/arch/x86/kvm/svm/vmenter.S +++ b/arch/x86/kvm/svm/vmenter.S @@ -270,16 +270,16 @@ SYM_FUNC_START(__svm_vcpu_run) RESTORE_GUEST_SPEC_CTRL_BODY RESTORE_HOST_SPEC_CTRL_BODY -10: cmpb $0, kvm_rebooting +10: cmpb $0, _ASM_RIP(kvm_rebooting) jne 2b ud2 -30: cmpb $0, kvm_rebooting +30: cmpb $0, _ASM_RIP(kvm_rebooting) jne 4b ud2 -50: cmpb $0, kvm_rebooting +50: cmpb $0, _ASM_RIP(kvm_rebooting) jne 6b ud2 -70: cmpb $0, kvm_rebooting +70: cmpb $0, _ASM_RIP(kvm_rebooting) jne 8b ud2 @@ -381,7 +381,7 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) RESTORE_GUEST_SPEC_CTRL_BODY RESTORE_HOST_SPEC_CTRL_BODY -3: cmpb $0, kvm_rebooting +3: cmpb $0, _ASM_RIP(kvm_rebooting) jne 2b ud2 diff --git a/arch/x86/kvm/vmx/vmenter.S b/arch/x86/kvm/vmx/vmenter.S index be275a0410a8..906ecd001511 100644 --- a/arch/x86/kvm/vmx/vmenter.S +++ b/arch/x86/kvm/vmx/vmenter.S @@ -289,7 +289,7 @@ SYM_INNER_LABEL_ALIGN(vmx_vmexit, SYM_L_GLOBAL) RET .Lfixup: - cmpb $0, kvm_rebooting + cmpb $0, _ASM_RIP(kvm_rebooting) jne .Lvmfail ud2 .Lvmfail: From cbb359d81a2695bb5e63ec9de06fcbef28518891 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 3 Nov 2023 16:05:36 -0700 Subject: [PATCH 084/336] KVM: x86/pmu: Move PMU reset logic to common x86 code Move the common (or at least "ignored") aspects of resetting the vPMU to common x86 code, along with the stop/release helpers that are no used only by the common pmu.c. There is no need to manually handle fixed counters as all_valid_pmc_idx tracks both fixed and general purpose counters, and resetting the vPMU is far from a hot path, i.e. the extra bit of overhead to the PMC from the index is a non-issue. Zero fixed_ctr_ctrl in common code even though it's Intel specific. Ensuring it's zero doesn't harm AMD/SVM in any way, and stopping the fixed counters via all_valid_pmc_idx, but not clearing the associated control bits, would be odd/confusing. Make the .reset() hook optional as SVM no longer needs vendor specific handling. Cc: stable@vger.kernel.org Reviewed-by: Dapeng Mi Link: https://lore.kernel.org/r/20231103230541.352265-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm-x86-pmu-ops.h | 2 +- arch/x86/kvm/pmu.c | 40 +++++++++++++++++++++++++- arch/x86/kvm/pmu.h | 18 ------------ arch/x86/kvm/svm/pmu.c | 16 ----------- arch/x86/kvm/vmx/pmu_intel.c | 20 ------------- 5 files changed, 40 insertions(+), 56 deletions(-) diff --git a/arch/x86/include/asm/kvm-x86-pmu-ops.h b/arch/x86/include/asm/kvm-x86-pmu-ops.h index 6c98f4bb4228..058bc636356a 100644 --- a/arch/x86/include/asm/kvm-x86-pmu-ops.h +++ b/arch/x86/include/asm/kvm-x86-pmu-ops.h @@ -22,7 +22,7 @@ KVM_X86_PMU_OP(get_msr) KVM_X86_PMU_OP(set_msr) KVM_X86_PMU_OP(refresh) KVM_X86_PMU_OP(init) -KVM_X86_PMU_OP(reset) +KVM_X86_PMU_OP_OPTIONAL(reset) KVM_X86_PMU_OP_OPTIONAL(deliver_pmi) KVM_X86_PMU_OP_OPTIONAL(cleanup) diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index 9ae07db6f0f6..027e9c3c2b93 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -250,6 +250,24 @@ static bool pmc_resume_counter(struct kvm_pmc *pmc) return true; } +static void pmc_release_perf_event(struct kvm_pmc *pmc) +{ + if (pmc->perf_event) { + perf_event_release_kernel(pmc->perf_event); + pmc->perf_event = NULL; + pmc->current_config = 0; + pmc_to_pmu(pmc)->event_count--; + } +} + +static void pmc_stop_counter(struct kvm_pmc *pmc) +{ + if (pmc->perf_event) { + pmc->counter = pmc_read_counter(pmc); + pmc_release_perf_event(pmc); + } +} + static int filter_cmp(const void *pa, const void *pb, u64 mask) { u64 a = *(u64 *)pa & mask; @@ -654,7 +672,27 @@ void kvm_pmu_refresh(struct kvm_vcpu *vcpu) void kvm_pmu_reset(struct kvm_vcpu *vcpu) { - static_call(kvm_x86_pmu_reset)(vcpu); + struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); + struct kvm_pmc *pmc; + int i; + + bitmap_zero(pmu->reprogram_pmi, X86_PMC_IDX_MAX); + + for_each_set_bit(i, pmu->all_valid_pmc_idx, X86_PMC_IDX_MAX) { + pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, i); + if (!pmc) + continue; + + pmc_stop_counter(pmc); + pmc->counter = 0; + + if (pmc_is_gp(pmc)) + pmc->eventsel = 0; + } + + pmu->fixed_ctr_ctrl = pmu->global_ctrl = pmu->global_status = 0; + + static_call_cond(kvm_x86_pmu_reset)(vcpu); } void kvm_pmu_init(struct kvm_vcpu *vcpu) diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h index 1d64113de488..a46aa9b25150 100644 --- a/arch/x86/kvm/pmu.h +++ b/arch/x86/kvm/pmu.h @@ -80,24 +80,6 @@ static inline void pmc_write_counter(struct kvm_pmc *pmc, u64 val) pmc->counter &= pmc_bitmask(pmc); } -static inline void pmc_release_perf_event(struct kvm_pmc *pmc) -{ - if (pmc->perf_event) { - perf_event_release_kernel(pmc->perf_event); - pmc->perf_event = NULL; - pmc->current_config = 0; - pmc_to_pmu(pmc)->event_count--; - } -} - -static inline void pmc_stop_counter(struct kvm_pmc *pmc) -{ - if (pmc->perf_event) { - pmc->counter = pmc_read_counter(pmc); - pmc_release_perf_event(pmc); - } -} - static inline bool pmc_is_gp(struct kvm_pmc *pmc) { return pmc->type == KVM_PMC_GP; diff --git a/arch/x86/kvm/svm/pmu.c b/arch/x86/kvm/svm/pmu.c index 373ff6a6687b..3fd47de14b38 100644 --- a/arch/x86/kvm/svm/pmu.c +++ b/arch/x86/kvm/svm/pmu.c @@ -233,21 +233,6 @@ static void amd_pmu_init(struct kvm_vcpu *vcpu) } } -static void amd_pmu_reset(struct kvm_vcpu *vcpu) -{ - struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); - int i; - - for (i = 0; i < KVM_AMD_PMC_MAX_GENERIC; i++) { - struct kvm_pmc *pmc = &pmu->gp_counters[i]; - - pmc_stop_counter(pmc); - pmc->counter = pmc->prev_counter = pmc->eventsel = 0; - } - - pmu->global_ctrl = pmu->global_status = 0; -} - struct kvm_pmu_ops amd_pmu_ops __initdata = { .hw_event_available = amd_hw_event_available, .pmc_idx_to_pmc = amd_pmc_idx_to_pmc, @@ -259,7 +244,6 @@ struct kvm_pmu_ops amd_pmu_ops __initdata = { .set_msr = amd_pmu_set_msr, .refresh = amd_pmu_refresh, .init = amd_pmu_init, - .reset = amd_pmu_reset, .EVENTSEL_EVENT = AMD64_EVENTSEL_EVENT, .MAX_NR_GP_COUNTERS = KVM_AMD_PMC_MAX_GENERIC, .MIN_NR_GP_COUNTERS = AMD64_NUM_COUNTERS, diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index 820d3e1f6b4f..90c1f7f07e53 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -632,26 +632,6 @@ static void intel_pmu_init(struct kvm_vcpu *vcpu) static void intel_pmu_reset(struct kvm_vcpu *vcpu) { - struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); - struct kvm_pmc *pmc = NULL; - int i; - - for (i = 0; i < KVM_INTEL_PMC_MAX_GENERIC; i++) { - pmc = &pmu->gp_counters[i]; - - pmc_stop_counter(pmc); - pmc->counter = pmc->prev_counter = pmc->eventsel = 0; - } - - for (i = 0; i < KVM_PMC_MAX_FIXED; i++) { - pmc = &pmu->fixed_counters[i]; - - pmc_stop_counter(pmc); - pmc->counter = pmc->prev_counter = 0; - } - - pmu->fixed_ctr_ctrl = pmu->global_ctrl = pmu->global_status = 0; - intel_pmu_release_guest_lbr_event(vcpu); } From 1647b52757d59131fe30cf73fa36fac834d4367f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 3 Nov 2023 16:05:37 -0700 Subject: [PATCH 085/336] KVM: x86/pmu: Reset the PMU, i.e. stop counters, before refreshing Stop all counters and release all perf events before refreshing the vPMU, i.e. before reconfiguring the vPMU to respond to changes in the vCPU model. Clear need_cleanup in kvm_pmu_reset() as well so that KVM doesn't prematurely stop counters, e.g. if KVM enters the guest and enables counters before the vCPU is scheduled out. Cc: stable@vger.kernel.org Reviewed-by: Dapeng Mi Link: https://lore.kernel.org/r/20231103230541.352265-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/pmu.c | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index 027e9c3c2b93..dc8e8e907cfb 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -657,25 +657,14 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) return 0; } -/* refresh PMU settings. This function generally is called when underlying - * settings are changed (such as changes of PMU CPUID by guest VMs), which - * should rarely happen. - */ -void kvm_pmu_refresh(struct kvm_vcpu *vcpu) -{ - if (KVM_BUG_ON(kvm_vcpu_has_run(vcpu), vcpu->kvm)) - return; - - bitmap_zero(vcpu_to_pmu(vcpu)->all_valid_pmc_idx, X86_PMC_IDX_MAX); - static_call(kvm_x86_pmu_refresh)(vcpu); -} - void kvm_pmu_reset(struct kvm_vcpu *vcpu) { struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); struct kvm_pmc *pmc; int i; + pmu->need_cleanup = false; + bitmap_zero(pmu->reprogram_pmi, X86_PMC_IDX_MAX); for_each_set_bit(i, pmu->all_valid_pmc_idx, X86_PMC_IDX_MAX) { @@ -695,6 +684,26 @@ void kvm_pmu_reset(struct kvm_vcpu *vcpu) static_call_cond(kvm_x86_pmu_reset)(vcpu); } + +/* + * Refresh the PMU configuration for the vCPU, e.g. if userspace changes CPUID + * and/or PERF_CAPABILITIES. + */ +void kvm_pmu_refresh(struct kvm_vcpu *vcpu) +{ + if (KVM_BUG_ON(kvm_vcpu_has_run(vcpu), vcpu->kvm)) + return; + + /* + * Stop/release all existing counters/events before realizing the new + * vPMU model. + */ + kvm_pmu_reset(vcpu); + + bitmap_zero(vcpu_to_pmu(vcpu)->all_valid_pmc_idx, X86_PMC_IDX_MAX); + static_call(kvm_x86_pmu_refresh)(vcpu); +} + void kvm_pmu_init(struct kvm_vcpu *vcpu) { struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); From f2f63f7ec6fd13d2d5d5c6d90ea438fbb5a36adc Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 3 Nov 2023 16:05:38 -0700 Subject: [PATCH 086/336] KVM: x86/pmu: Stop calling kvm_pmu_reset() at RESET (it's redundant) Drop kvm_vcpu_reset()'s call to kvm_pmu_reset(), the call is performed only for RESET, which is really just the same thing as vCPU creation, and kvm_arch_vcpu_create() *just* called kvm_pmu_init(), i.e. there can't possibly be any work to do. Unlike Intel, AMD's amd_pmu_refresh() does fill all_valid_pmc_idx even if guest CPUID is empty, but everything that is at all dynamic is guaranteed to be '0'/NULL, e.g. it should be impossible for KVM to have already created a perf event. Reviewed-by: Dapeng Mi Link: https://lore.kernel.org/r/20231103230541.352265-4-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/pmu.c | 2 +- arch/x86/kvm/pmu.h | 1 - arch/x86/kvm/x86.c | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index dc8e8e907cfb..458e836c6efe 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -657,7 +657,7 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) return 0; } -void kvm_pmu_reset(struct kvm_vcpu *vcpu) +static void kvm_pmu_reset(struct kvm_vcpu *vcpu) { struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); struct kvm_pmc *pmc; diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h index a46aa9b25150..db9a12c0a2ef 100644 --- a/arch/x86/kvm/pmu.h +++ b/arch/x86/kvm/pmu.h @@ -243,7 +243,6 @@ bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr); int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info); int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info); void kvm_pmu_refresh(struct kvm_vcpu *vcpu); -void kvm_pmu_reset(struct kvm_vcpu *vcpu); void kvm_pmu_init(struct kvm_vcpu *vcpu); void kvm_pmu_cleanup(struct kvm_vcpu *vcpu); void kvm_pmu_destroy(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 6d0772b47041..33c071cd95b2 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -12221,7 +12221,6 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) } if (!init_event) { - kvm_pmu_reset(vcpu); vcpu->arch.smbase = 0x30000; vcpu->arch.msr_misc_features_enables = 0; From ec61b2306dfd117ba5db93dfb54808523ea2b5e0 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 3 Nov 2023 16:05:39 -0700 Subject: [PATCH 087/336] KVM: x86/pmu: Remove manual clearing of fields in kvm_pmu_init() Remove code that unnecessarily clears event_count and need_cleanup in kvm_pmu_init(), the entire kvm_pmu is zeroed just a few lines earlier. Vendor code doesn't set event_count or need_cleanup during .init(), and if either VMX or SVM did set those fields it would be a flagrant bug. Reviewed-by: Dapeng Mi Link: https://lore.kernel.org/r/20231103230541.352265-5-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/pmu.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index 458e836c6efe..c06090196b00 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -710,8 +710,6 @@ void kvm_pmu_init(struct kvm_vcpu *vcpu) memset(pmu, 0, sizeof(*pmu)); static_call(kvm_x86_pmu_init)(vcpu); - pmu->event_count = 0; - pmu->need_cleanup = false; kvm_pmu_refresh(vcpu); } From 89acf1237b81802328beaa094b1139dbb2561883 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 3 Nov 2023 16:05:40 -0700 Subject: [PATCH 088/336] KVM: x86/pmu: Update sample period in pmc_write_counter() Update a PMC's sample period in pmc_write_counter() to deduplicate code across all callers of pmc_write_counter(). Opportunistically move pmc_write_counter() into pmc.c now that it's doing more work. WRMSR isn't such a hot path that an extra CALL+RET pair will be problematic, and the order of function definitions needs to be changed anyways, i.e. now is a convenient time to eat the churn. Reviewed-by: Dapeng Mi Link: https://lore.kernel.org/r/20231103230541.352265-6-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/pmu.c | 27 +++++++++++++++++++++++++++ arch/x86/kvm/pmu.h | 25 +------------------------ arch/x86/kvm/svm/pmu.c | 1 - arch/x86/kvm/vmx/pmu_intel.c | 2 -- 4 files changed, 28 insertions(+), 27 deletions(-) diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index c06090196b00..3725d001239d 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -161,6 +161,15 @@ static u64 pmc_get_pebs_precise_level(struct kvm_pmc *pmc) return 1; } +static u64 get_sample_period(struct kvm_pmc *pmc, u64 counter_value) +{ + u64 sample_period = (-counter_value) & pmc_bitmask(pmc); + + if (!sample_period) + sample_period = pmc_bitmask(pmc) + 1; + return sample_period; +} + static int pmc_reprogram_counter(struct kvm_pmc *pmc, u32 type, u64 config, bool exclude_user, bool exclude_kernel, bool intr) @@ -268,6 +277,24 @@ static void pmc_stop_counter(struct kvm_pmc *pmc) } } +static void pmc_update_sample_period(struct kvm_pmc *pmc) +{ + if (!pmc->perf_event || pmc->is_paused || + !is_sampling_event(pmc->perf_event)) + return; + + perf_event_period(pmc->perf_event, + get_sample_period(pmc, pmc->counter)); +} + +void pmc_write_counter(struct kvm_pmc *pmc, u64 val) +{ + pmc->counter += val - pmc_read_counter(pmc); + pmc->counter &= pmc_bitmask(pmc); + pmc_update_sample_period(pmc); +} +EXPORT_SYMBOL_GPL(pmc_write_counter); + static int filter_cmp(const void *pa, const void *pb, u64 mask) { u64 a = *(u64 *)pa & mask; diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h index db9a12c0a2ef..cae85e550f60 100644 --- a/arch/x86/kvm/pmu.h +++ b/arch/x86/kvm/pmu.h @@ -74,11 +74,7 @@ static inline u64 pmc_read_counter(struct kvm_pmc *pmc) return counter & pmc_bitmask(pmc); } -static inline void pmc_write_counter(struct kvm_pmc *pmc, u64 val) -{ - pmc->counter += val - pmc_read_counter(pmc); - pmc->counter &= pmc_bitmask(pmc); -} +void pmc_write_counter(struct kvm_pmc *pmc, u64 val); static inline bool pmc_is_gp(struct kvm_pmc *pmc) { @@ -128,25 +124,6 @@ static inline struct kvm_pmc *get_fixed_pmc(struct kvm_pmu *pmu, u32 msr) return NULL; } -static inline u64 get_sample_period(struct kvm_pmc *pmc, u64 counter_value) -{ - u64 sample_period = (-counter_value) & pmc_bitmask(pmc); - - if (!sample_period) - sample_period = pmc_bitmask(pmc) + 1; - return sample_period; -} - -static inline void pmc_update_sample_period(struct kvm_pmc *pmc) -{ - if (!pmc->perf_event || pmc->is_paused || - !is_sampling_event(pmc->perf_event)) - return; - - perf_event_period(pmc->perf_event, - get_sample_period(pmc, pmc->counter)); -} - static inline bool pmc_speculative_in_use(struct kvm_pmc *pmc) { struct kvm_pmu *pmu = pmc_to_pmu(pmc); diff --git a/arch/x86/kvm/svm/pmu.c b/arch/x86/kvm/svm/pmu.c index 3fd47de14b38..b6a7ad4d6914 100644 --- a/arch/x86/kvm/svm/pmu.c +++ b/arch/x86/kvm/svm/pmu.c @@ -161,7 +161,6 @@ static int amd_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) pmc = get_gp_pmc_amd(pmu, msr, PMU_TYPE_COUNTER); if (pmc) { pmc_write_counter(pmc, data); - pmc_update_sample_period(pmc); return 0; } /* MSR_EVNTSELn */ diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index 90c1f7f07e53..a6216c874729 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -437,11 +437,9 @@ static int intel_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) !(msr & MSR_PMC_FULL_WIDTH_BIT)) data = (s64)(s32)data; pmc_write_counter(pmc, data); - pmc_update_sample_period(pmc); break; } else if ((pmc = get_fixed_pmc(pmu, msr))) { pmc_write_counter(pmc, data); - pmc_update_sample_period(pmc); break; } else if ((pmc = get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0))) { reserved_bits = pmu->reserved_bits; From fd89499a5151d197ba30f7b801f6d8f4646cf446 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 3 Nov 2023 16:05:41 -0700 Subject: [PATCH 089/336] KVM: x86/pmu: Track emulated counter events instead of previous counter Explicitly track emulated counter events instead of using the common counter value that's shared with the hardware counter owned by perf. Bumping the common counter requires snapshotting the pre-increment value in order to detect overflow from emulation, and the snapshot approach is inherently flawed. Snapshotting the previous counter at every increment assumes that there is at most one emulated counter event per emulated instruction (or rather, between checks for KVM_REQ_PMU). That's mostly holds true today because KVM only emulates (branch) instructions retired, but the approach will fall apart if KVM ever supports event types that don't have a 1:1 relationship with instructions. And KVM already has a relevant bug, as handle_invalid_guest_state() emulates multiple instructions without checking KVM_REQ_PMU, i.e. could miss an overflow event due to clobbering pmc->prev_counter. Not checking KVM_REQ_PMU is problematic in both cases, but at least with the emulated counter approach, the resulting behavior is delayed overflow detection, as opposed to completely lost detection. Tracking the emulated count fixes another bug where the snapshot approach can signal spurious overflow due to incorporating both the emulated count and perf's count in the check, i.e. if overflow is detected by perf, then KVM's emulation will also incorrectly signal overflow. Add a comment in the related code to call out the need to process emulated events *after* pausing the perf event (big kudos to Mingwei for figuring out that particular wrinkle). Cc: Mingwei Zhang Cc: Roman Kagan Cc: Jim Mattson Cc: Dapeng Mi Cc: Like Xu Reviewed-by: Mingwei Zhang Link: https://lore.kernel.org/r/20231103230541.352265-7-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 17 +++++++++++- arch/x86/kvm/pmu.c | 48 ++++++++++++++++++++++++--------- arch/x86/kvm/pmu.h | 3 ++- 3 files changed, 53 insertions(+), 15 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index a565a2e70f30..562eaa938847 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -500,8 +500,23 @@ struct kvm_pmc { u8 idx; bool is_paused; bool intr; + /* + * Base value of the PMC counter, relative to the *consumed* count in + * the associated perf_event. This value includes counter updates from + * the perf_event and emulated_count since the last time the counter + * was reprogrammed, but it is *not* the current value as seen by the + * guest or userspace. + * + * The count is relative to the associated perf_event so that KVM + * doesn't need to reprogram the perf_event every time the guest writes + * to the counter. + */ u64 counter; - u64 prev_counter; + /* + * PMC events triggered by KVM emulation that haven't been fully + * processed, i.e. haven't undergone overflow detection. + */ + u64 emulated_counter; u64 eventsel; struct perf_event *perf_event; struct kvm_vcpu *vcpu; diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index 3725d001239d..87cc6c8809ad 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -127,9 +127,9 @@ static void kvm_perf_overflow(struct perf_event *perf_event, struct kvm_pmc *pmc = perf_event->overflow_handler_context; /* - * Ignore overflow events for counters that are scheduled to be - * reprogrammed, e.g. if a PMI for the previous event races with KVM's - * handling of a related guest WRMSR. + * Ignore asynchronous overflow events for counters that are scheduled + * to be reprogrammed, e.g. if a PMI for the previous event races with + * KVM's handling of a related guest WRMSR. */ if (test_and_set_bit(pmc->idx, pmc_to_pmu(pmc)->reprogram_pmi)) return; @@ -224,17 +224,30 @@ static int pmc_reprogram_counter(struct kvm_pmc *pmc, u32 type, u64 config, return 0; } -static void pmc_pause_counter(struct kvm_pmc *pmc) +static bool pmc_pause_counter(struct kvm_pmc *pmc) { u64 counter = pmc->counter; - - if (!pmc->perf_event || pmc->is_paused) - return; + u64 prev_counter; /* update counter, reset event value to avoid redundant accumulation */ - counter += perf_event_pause(pmc->perf_event, true); + if (pmc->perf_event && !pmc->is_paused) + counter += perf_event_pause(pmc->perf_event, true); + + /* + * Snapshot the previous counter *after* accumulating state from perf. + * If overflow already happened, hardware (via perf) is responsible for + * generating a PMI. KVM just needs to detect overflow on emulated + * counter events that haven't yet been processed. + */ + prev_counter = counter & pmc_bitmask(pmc); + + counter += pmc->emulated_counter; pmc->counter = counter & pmc_bitmask(pmc); + + pmc->emulated_counter = 0; pmc->is_paused = true; + + return pmc->counter < prev_counter; } static bool pmc_resume_counter(struct kvm_pmc *pmc) @@ -289,6 +302,15 @@ static void pmc_update_sample_period(struct kvm_pmc *pmc) void pmc_write_counter(struct kvm_pmc *pmc, u64 val) { + /* + * Drop any unconsumed accumulated counts, the WRMSR is a write, not a + * read-modify-write. Adjust the counter value so that its value is + * relative to the current count, as reading the current count from + * perf is faster than pausing and repgrogramming the event in order to + * reset it to '0'. Note, this very sneakily offsets the accumulated + * emulated count too, by using pmc_read_counter()! + */ + pmc->emulated_counter = 0; pmc->counter += val - pmc_read_counter(pmc); pmc->counter &= pmc_bitmask(pmc); pmc_update_sample_period(pmc); @@ -428,14 +450,15 @@ static void reprogram_counter(struct kvm_pmc *pmc) struct kvm_pmu *pmu = pmc_to_pmu(pmc); u64 eventsel = pmc->eventsel; u64 new_config = eventsel; + bool emulate_overflow; u8 fixed_ctr_ctrl; - pmc_pause_counter(pmc); + emulate_overflow = pmc_pause_counter(pmc); if (!pmc_event_is_allowed(pmc)) goto reprogram_complete; - if (pmc->counter < pmc->prev_counter) + if (emulate_overflow) __kvm_perf_overflow(pmc, false); if (eventsel & ARCH_PERFMON_EVENTSEL_PIN_CONTROL) @@ -475,7 +498,6 @@ static void reprogram_counter(struct kvm_pmc *pmc) reprogram_complete: clear_bit(pmc->idx, (unsigned long *)&pmc_to_pmu(pmc)->reprogram_pmi); - pmc->prev_counter = 0; } void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) @@ -701,6 +723,7 @@ static void kvm_pmu_reset(struct kvm_vcpu *vcpu) pmc_stop_counter(pmc); pmc->counter = 0; + pmc->emulated_counter = 0; if (pmc_is_gp(pmc)) pmc->eventsel = 0; @@ -772,8 +795,7 @@ void kvm_pmu_destroy(struct kvm_vcpu *vcpu) static void kvm_pmu_incr_counter(struct kvm_pmc *pmc) { - pmc->prev_counter = pmc->counter; - pmc->counter = (pmc->counter + 1) & pmc_bitmask(pmc); + pmc->emulated_counter++; kvm_pmu_request_counter_reprogram(pmc); } diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h index cae85e550f60..7caeb3d8d4fd 100644 --- a/arch/x86/kvm/pmu.h +++ b/arch/x86/kvm/pmu.h @@ -66,7 +66,8 @@ static inline u64 pmc_read_counter(struct kvm_pmc *pmc) { u64 counter, enabled, running; - counter = pmc->counter; + counter = pmc->counter + pmc->emulated_counter; + if (pmc->perf_event && !pmc->is_paused) counter += perf_event_read_value(pmc->perf_event, &enabled, &running); From 63912245c19d3a4179da44beefd017eb9270f207 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Wed, 15 Mar 2023 18:16:06 +0800 Subject: [PATCH 090/336] KVM: move KVM_CAP_DEVICE_CTRL to the generic check KVM_CAP_DEVICE_CTRL allows userspace to check if the kvm_device framework (e.g. KVM_CREATE_DEVICE) is supported by KVM. Move KVM_CAP_DEVICE_CTRL to the generic check for the two reasons: 1) it already supports arch agnostic usages (i.e. KVM_DEV_TYPE_VFIO). For example, userspace VFIO implementation may needs to create KVM_DEV_TYPE_VFIO on x86, riscv, or arm etc. It is simpler to have it checked at the generic code than at each arch's code. 2) KVM_CREATE_DEVICE has been added to the generic code. Link: https://lore.kernel.org/all/20221215115207.14784-1-wei.w.wang@intel.com Signed-off-by: Wei Wang Reviewed-by: Sean Christopherson Acked-by: Anup Patel (riscv) Reviewed-by: Oliver Upton Acked-by: Michael Ellerman (powerpc) Link: https://lore.kernel.org/r/20230315101606.10636-1-wei.w.wang@intel.com Signed-off-by: Sean Christopherson --- arch/arm64/kvm/arm.c | 1 - arch/powerpc/kvm/powerpc.c | 1 - arch/riscv/kvm/vm.c | 1 - arch/s390/kvm/kvm-s390.c | 1 - virt/kvm/kvm_main.c | 1 + 5 files changed, 1 insertion(+), 4 deletions(-) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index e5f75f1f1085..9725783745b4 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -221,7 +221,6 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = vgic_present; break; case KVM_CAP_IOEVENTFD: - case KVM_CAP_DEVICE_CTRL: case KVM_CAP_USER_MEMORY: case KVM_CAP_SYNC_MMU: case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index c39945a7fce3..13d8309c7ba9 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -528,7 +528,6 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_ENABLE_CAP: case KVM_CAP_ONE_REG: case KVM_CAP_IOEVENTFD: - case KVM_CAP_DEVICE_CTRL: case KVM_CAP_IMMEDIATE_EXIT: case KVM_CAP_SET_GUEST_DEBUG: r = 1; diff --git a/arch/riscv/kvm/vm.c b/arch/riscv/kvm/vm.c index 7e2b50c692c1..ce58bc48e5b8 100644 --- a/arch/riscv/kvm/vm.c +++ b/arch/riscv/kvm/vm.c @@ -179,7 +179,6 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = kvm_riscv_aia_available(); break; case KVM_CAP_IOEVENTFD: - case KVM_CAP_DEVICE_CTRL: case KVM_CAP_USER_MEMORY: case KVM_CAP_SYNC_MMU: case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 7aa0e668488f..39463d0e4a1c 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -563,7 +563,6 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_ENABLE_CAP: case KVM_CAP_S390_CSS_SUPPORT: case KVM_CAP_IOEVENTFD: - case KVM_CAP_DEVICE_CTRL: case KVM_CAP_S390_IRQCHIP: case KVM_CAP_VM_ATTRIBUTES: case KVM_CAP_MP_STATE: diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 8758cb799e18..b744eddecbc0 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -4873,6 +4873,7 @@ static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) #endif case KVM_CAP_BINARY_STATS_FD: case KVM_CAP_SYSTEM_EVENT_DATA: + case KVM_CAP_DEVICE_CTRL: return 1; #ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES case KVM_CAP_MEMORY_ATTRIBUTES: From 573cc0e5cf142d9992d2de3502800890fc717bc0 Mon Sep 17 00:00:00 2001 From: Philipp Stanner Date: Thu, 2 Nov 2023 19:15:24 +0100 Subject: [PATCH 091/336] KVM: x86: Harden copying of userspace-array against overflow cpuid.c utilizes vmemdup_user() and array_size() to copy two userspace arrays. This, currently, does not check for an overflow. Use the new wrapper vmemdup_array_user() to copy the arrays more safely, as vmemdup_user() doesn't check for overflow. Note, KVM explicitly checks the number of entries before duplicating the array, i.e. adding the overflow check should be a glorified nop. Suggested-by: Dave Airlie Signed-off-by: Philipp Stanner Link: https://lore.kernel.org/r/20231102181526.43279-2-pstanner@redhat.com [sean: call out that KVM pre-checks the number of entries] Signed-off-by: Sean Christopherson --- arch/x86/kvm/cpuid.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index dda6fc4cfae8..ad441f0d2917 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -469,7 +469,7 @@ int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu, return -E2BIG; if (cpuid->nent) { - e = vmemdup_user(entries, array_size(sizeof(*e), cpuid->nent)); + e = vmemdup_array_user(entries, cpuid->nent, sizeof(*e)); if (IS_ERR(e)) return PTR_ERR(e); @@ -513,7 +513,7 @@ int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *vcpu, return -E2BIG; if (cpuid->nent) { - e2 = vmemdup_user(entries, array_size(sizeof(*e2), cpuid->nent)); + e2 = vmemdup_array_user(entries, cpuid->nent, sizeof(*e2)); if (IS_ERR(e2)) return PTR_ERR(e2); } From 1aa4bb916808503bf6fedd00f50f2077f91cebaa Mon Sep 17 00:00:00 2001 From: David Matlack Date: Fri, 27 Oct 2023 10:26:38 -0700 Subject: [PATCH 092/336] KVM: x86/mmu: Fix off-by-1 when splitting huge pages during CLEAR Fix an off-by-1 error when passing in the range of pages to kvm_mmu_try_split_huge_pages() during CLEAR_DIRTY_LOG. Specifically, end is the last page that needs to be split (inclusive) so pass in `end + 1` since kvm_mmu_try_split_huge_pages() expects the `end` to be non-inclusive. At worst this will cause a huge page to be write-protected instead of eagerly split, which is purely a performance issue, not a correctness issue. But even that is unlikely as it would require userspace pass in a bitmap where the last page is the only 4K page on a huge page that needs to be split. Reported-by: Vipin Sharma Fixes: cb00a70bd4b7 ("KVM: x86/mmu: Split huge pages mapped by the TDP MMU during KVM_CLEAR_DIRTY_LOG") Signed-off-by: David Matlack Link: https://lore.kernel.org/r/20231027172640.2335197-2-dmatlack@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 59b026b6ad2a..29ac130fcb84 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -1392,7 +1392,7 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, gfn_t end = slot->base_gfn + gfn_offset + __fls(mask); if (READ_ONCE(eager_page_split)) - kvm_mmu_try_split_huge_pages(kvm, slot, start, end, PG_LEVEL_4K); + kvm_mmu_try_split_huge_pages(kvm, slot, start, end + 1, PG_LEVEL_4K); kvm_mmu_slot_gfn_write_protect(kvm, slot, start, PG_LEVEL_2M); From 45a61ebb221117748d3567a86908618f431ac824 Mon Sep 17 00:00:00 2001 From: David Matlack Date: Fri, 27 Oct 2023 10:26:39 -0700 Subject: [PATCH 093/336] KVM: x86/mmu: Check for leaf SPTE when clearing dirty bit in the TDP MMU Re-check that the given SPTE is still a leaf and present SPTE after a failed cmpxchg in clear_dirty_gfn_range(). clear_dirty_gfn_range() intends to only operate on present leaf SPTEs, but that could change after a failed cmpxchg. A check for present was added in commit 3354ef5a592d ("KVM: x86/mmu: Check for present SPTE when clearing dirty bit in TDP MMU") but the check for leaf is still buried in tdp_root_for_each_leaf_pte() and does not get rechecked on retry. Fixes: a6a0b05da9f3 ("kvm: x86/mmu: Support dirty logging for the TDP MMU") Signed-off-by: David Matlack Link: https://lore.kernel.org/r/20231027172640.2335197-3-dmatlack@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/tdp_mmu.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index 6cd4dd631a2f..038983b13574 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -1522,12 +1522,13 @@ static bool clear_dirty_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root, rcu_read_lock(); - tdp_root_for_each_leaf_pte(iter, root, start, end) { + tdp_root_for_each_pte(iter, root, start, end) { retry: - if (tdp_mmu_iter_cond_resched(kvm, &iter, false, true)) + if (!is_shadow_present_pte(iter.old_spte) || + !is_last_spte(iter.old_spte, iter.level)) continue; - if (!is_shadow_present_pte(iter.old_spte)) + if (tdp_mmu_iter_cond_resched(kvm, &iter, false, true)) continue; KVM_MMU_WARN_ON(kvm_ad_enabled() && From 5f3c8c9187b6fa8675951f9fad5b99b11fed21f6 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Sat, 25 Nov 2023 03:33:57 -0500 Subject: [PATCH 094/336] KVM: x86/mmu: remove unnecessary "bool shared" argument from functions Neither tdp_mmu_next_root nor kvm_tdp_mmu_put_root need to know if the lock is taken for read or write. Either way, protection is achieved via RCU and tdp_mmu_pages_lock. Remove the argument and just assert that the lock is taken. Reviewed-by: Maxim Levitsky Signed-off-by: Paolo Bonzini Link: https://lore.kernel.org/r/20231125083400.1399197-2-pbonzini@redhat.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 2 +- arch/x86/kvm/mmu/tdp_mmu.c | 34 +++++++++++++++++++++------------- arch/x86/kvm/mmu/tdp_mmu.h | 3 +-- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 29ac130fcb84..ace9f7c13132 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3580,7 +3580,7 @@ static void mmu_free_root_page(struct kvm *kvm, hpa_t *root_hpa, return; if (is_tdp_mmu_page(sp)) - kvm_tdp_mmu_put_root(kvm, sp, false); + kvm_tdp_mmu_put_root(kvm, sp); else if (!--sp->root_count && sp->role.invalid) kvm_mmu_prepare_zap_page(kvm, sp, invalid_list); diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index 038983b13574..8cd805fa1516 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -73,10 +73,13 @@ static void tdp_mmu_free_sp_rcu_callback(struct rcu_head *head) tdp_mmu_free_sp(sp); } -void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root, - bool shared) +void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root) { - kvm_lockdep_assert_mmu_lock_held(kvm, shared); + /* + * Either read or write is okay, but mmu_lock must be held because + * writers are not required to take tdp_mmu_pages_lock. + */ + lockdep_assert_held(&kvm->mmu_lock); if (!refcount_dec_and_test(&root->tdp_mmu_root_count)) return; @@ -106,10 +109,16 @@ void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root, */ static struct kvm_mmu_page *tdp_mmu_next_root(struct kvm *kvm, struct kvm_mmu_page *prev_root, - bool shared, bool only_valid) + bool only_valid) { struct kvm_mmu_page *next_root; + /* + * While the roots themselves are RCU-protected, fields such as + * role.invalid are protected by mmu_lock. + */ + lockdep_assert_held(&kvm->mmu_lock); + rcu_read_lock(); if (prev_root) @@ -132,7 +141,7 @@ static struct kvm_mmu_page *tdp_mmu_next_root(struct kvm *kvm, rcu_read_unlock(); if (prev_root) - kvm_tdp_mmu_put_root(kvm, prev_root, shared); + kvm_tdp_mmu_put_root(kvm, prev_root); return next_root; } @@ -144,13 +153,12 @@ static struct kvm_mmu_page *tdp_mmu_next_root(struct kvm *kvm, * recent root. (Unless keeping a live reference is desirable.) * * If shared is set, this function is operating under the MMU lock in read - * mode. In the unlikely event that this thread must free a root, the lock - * will be temporarily dropped and reacquired in write mode. + * mode. */ #define __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared, _only_valid)\ - for (_root = tdp_mmu_next_root(_kvm, NULL, _shared, _only_valid); \ + for (_root = tdp_mmu_next_root(_kvm, NULL, _only_valid); \ _root; \ - _root = tdp_mmu_next_root(_kvm, _root, _shared, _only_valid)) \ + _root = tdp_mmu_next_root(_kvm, _root, _only_valid)) \ if (kvm_lockdep_assert_mmu_lock_held(_kvm, _shared) && \ kvm_mmu_page_as_id(_root) != _as_id) { \ } else @@ -159,9 +167,9 @@ static struct kvm_mmu_page *tdp_mmu_next_root(struct kvm *kvm, __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared, true) #define for_each_tdp_mmu_root_yield_safe(_kvm, _root, _shared) \ - for (_root = tdp_mmu_next_root(_kvm, NULL, _shared, false); \ + for (_root = tdp_mmu_next_root(_kvm, NULL, false); \ _root; \ - _root = tdp_mmu_next_root(_kvm, _root, _shared, false)) \ + _root = tdp_mmu_next_root(_kvm, _root, false)) \ if (!kvm_lockdep_assert_mmu_lock_held(_kvm, _shared)) { \ } else @@ -891,7 +899,7 @@ void kvm_tdp_mmu_zap_invalidated_roots(struct kvm *kvm) * the root must be reachable by mmu_notifiers while it's being * zapped */ - kvm_tdp_mmu_put_root(kvm, root, true); + kvm_tdp_mmu_put_root(kvm, root); } read_unlock(&kvm->mmu_lock); @@ -1500,7 +1508,7 @@ void kvm_tdp_mmu_try_split_huge_pages(struct kvm *kvm, for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, shared) { r = tdp_mmu_split_huge_pages_root(kvm, root, start, end, target_level, shared); if (r) { - kvm_tdp_mmu_put_root(kvm, root, shared); + kvm_tdp_mmu_put_root(kvm, root); break; } } diff --git a/arch/x86/kvm/mmu/tdp_mmu.h b/arch/x86/kvm/mmu/tdp_mmu.h index 733a3aef3a96..20d97aa46c49 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.h +++ b/arch/x86/kvm/mmu/tdp_mmu.h @@ -17,8 +17,7 @@ __must_check static inline bool kvm_tdp_mmu_get_root(struct kvm_mmu_page *root) return refcount_inc_not_zero(&root->tdp_mmu_root_count); } -void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root, - bool shared); +void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root); bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, gfn_t start, gfn_t end, bool flush); bool kvm_tdp_mmu_zap_sp(struct kvm *kvm, struct kvm_mmu_page *sp); From 484dd27c0602e01cb49db362ad42b95e70912d43 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Sat, 25 Nov 2023 03:33:58 -0500 Subject: [PATCH 095/336] KVM: x86/mmu: remove unnecessary "bool shared" argument from iterators The "bool shared" argument is more or less unnecessary in the for_each_*_tdp_mmu_root_yield_safe() macros. Many users check for the lock before calling it; all of them either call small functions that do the check, or end up calling tdp_mmu_set_spte_atomic() and tdp_mmu_iter_set_spte(). Add a few assertions to make up for the lost check in for_each_*_tdp_mmu_root_yield_safe(), but even this is probably overkill and mostly for documentation reasons. Signed-off-by: Paolo Bonzini Link: https://lore.kernel.org/r/20231125083400.1399197-3-pbonzini@redhat.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/tdp_mmu.c | 48 ++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index 8cd805fa1516..c6dc09acea12 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -155,23 +155,20 @@ static struct kvm_mmu_page *tdp_mmu_next_root(struct kvm *kvm, * If shared is set, this function is operating under the MMU lock in read * mode. */ -#define __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared, _only_valid)\ - for (_root = tdp_mmu_next_root(_kvm, NULL, _only_valid); \ - _root; \ - _root = tdp_mmu_next_root(_kvm, _root, _only_valid)) \ - if (kvm_lockdep_assert_mmu_lock_held(_kvm, _shared) && \ - kvm_mmu_page_as_id(_root) != _as_id) { \ +#define __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _only_valid)\ + for (_root = tdp_mmu_next_root(_kvm, NULL, _only_valid); \ + ({ lockdep_assert_held(&(_kvm)->mmu_lock); }), _root; \ + _root = tdp_mmu_next_root(_kvm, _root, _only_valid)) \ + if (kvm_mmu_page_as_id(_root) != _as_id) { \ } else -#define for_each_valid_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared) \ - __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared, true) +#define for_each_valid_tdp_mmu_root_yield_safe(_kvm, _root, _as_id) \ + __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, true) -#define for_each_tdp_mmu_root_yield_safe(_kvm, _root, _shared) \ - for (_root = tdp_mmu_next_root(_kvm, NULL, false); \ - _root; \ - _root = tdp_mmu_next_root(_kvm, _root, false)) \ - if (!kvm_lockdep_assert_mmu_lock_held(_kvm, _shared)) { \ - } else +#define for_each_tdp_mmu_root_yield_safe(_kvm, _root) \ + for (_root = tdp_mmu_next_root(_kvm, NULL, false); \ + ({ lockdep_assert_held(&(_kvm)->mmu_lock); }), _root; \ + _root = tdp_mmu_next_root(_kvm, _root, false)) /* * Iterate over all TDP MMU roots. Requires that mmu_lock be held for write, @@ -840,7 +837,8 @@ bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, gfn_t start, gfn_t end, bool flush) { struct kvm_mmu_page *root; - for_each_tdp_mmu_root_yield_safe(kvm, root, false) + lockdep_assert_held_write(&kvm->mmu_lock); + for_each_tdp_mmu_root_yield_safe(kvm, root) flush = tdp_mmu_zap_leafs(kvm, root, start, end, true, flush); return flush; @@ -862,7 +860,8 @@ void kvm_tdp_mmu_zap_all(struct kvm *kvm) * is being destroyed or the userspace VMM has exited. In both cases, * KVM_RUN is unreachable, i.e. no vCPUs will ever service the request. */ - for_each_tdp_mmu_root_yield_safe(kvm, root, false) + lockdep_assert_held_write(&kvm->mmu_lock); + for_each_tdp_mmu_root_yield_safe(kvm, root) tdp_mmu_zap_root(kvm, root, false); } @@ -876,7 +875,7 @@ void kvm_tdp_mmu_zap_invalidated_roots(struct kvm *kvm) read_lock(&kvm->mmu_lock); - for_each_tdp_mmu_root_yield_safe(kvm, root, true) { + for_each_tdp_mmu_root_yield_safe(kvm, root) { if (!root->tdp_mmu_scheduled_root_to_zap) continue; @@ -1133,7 +1132,7 @@ bool kvm_tdp_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range, { struct kvm_mmu_page *root; - __for_each_tdp_mmu_root_yield_safe(kvm, root, range->slot->as_id, false, false) + __for_each_tdp_mmu_root_yield_safe(kvm, root, range->slot->as_id, false) flush = tdp_mmu_zap_leafs(kvm, root, range->start, range->end, range->may_block, flush); @@ -1322,7 +1321,7 @@ bool kvm_tdp_mmu_wrprot_slot(struct kvm *kvm, lockdep_assert_held_read(&kvm->mmu_lock); - for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, true) + for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id) spte_set |= wrprot_gfn_range(kvm, root, slot->base_gfn, slot->base_gfn + slot->npages, min_level); @@ -1354,6 +1353,8 @@ static struct kvm_mmu_page *tdp_mmu_alloc_sp_for_split(struct kvm *kvm, { struct kvm_mmu_page *sp; + kvm_lockdep_assert_mmu_lock_held(kvm, shared); + /* * Since we are allocating while under the MMU lock we have to be * careful about GFP flags. Use GFP_NOWAIT to avoid blocking on direct @@ -1504,8 +1505,7 @@ void kvm_tdp_mmu_try_split_huge_pages(struct kvm *kvm, int r = 0; kvm_lockdep_assert_mmu_lock_held(kvm, shared); - - for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, shared) { + for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id) { r = tdp_mmu_split_huge_pages_root(kvm, root, start, end, target_level, shared); if (r) { kvm_tdp_mmu_put_root(kvm, root); @@ -1569,8 +1569,7 @@ bool kvm_tdp_mmu_clear_dirty_slot(struct kvm *kvm, bool spte_set = false; lockdep_assert_held_read(&kvm->mmu_lock); - - for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, true) + for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id) spte_set |= clear_dirty_gfn_range(kvm, root, slot->base_gfn, slot->base_gfn + slot->npages); @@ -1704,8 +1703,7 @@ void kvm_tdp_mmu_zap_collapsible_sptes(struct kvm *kvm, struct kvm_mmu_page *root; lockdep_assert_held_read(&kvm->mmu_lock); - - for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, true) + for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id) zap_collapsible_spte_range(kvm, root, slot); } From 250ce1b4d21a94f910c3df5141ff6434ea92524e Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Sat, 25 Nov 2023 03:33:59 -0500 Subject: [PATCH 096/336] KVM: x86/mmu: always take tdp_mmu_pages_lock It is cheap to take tdp_mmu_pages_lock in all write-side critical sections. We already do it all the time when zapping with read_lock(), so it is not a problem to do it from the kvm_tdp_mmu_zap_all() path (aka kvm_arch_flush_shadow_all(), aka VM destruction and MMU notifier release). Signed-off-by: Paolo Bonzini Link: https://lore.kernel.org/r/20231125083400.1399197-4-pbonzini@redhat.com Signed-off-by: Sean Christopherson --- Documentation/virt/kvm/locking.rst | 7 +++---- arch/x86/include/asm/kvm_host.h | 11 ++++++----- arch/x86/kvm/mmu/tdp_mmu.c | 24 ++++-------------------- 3 files changed, 13 insertions(+), 29 deletions(-) diff --git a/Documentation/virt/kvm/locking.rst b/Documentation/virt/kvm/locking.rst index 3a034db5e55f..02880d5552d5 100644 --- a/Documentation/virt/kvm/locking.rst +++ b/Documentation/virt/kvm/locking.rst @@ -43,10 +43,9 @@ On x86: - vcpu->mutex is taken outside kvm->arch.hyperv.hv_lock and kvm->arch.xen.xen_lock -- kvm->arch.mmu_lock is an rwlock. kvm->arch.tdp_mmu_pages_lock and - kvm->arch.mmu_unsync_pages_lock are taken inside kvm->arch.mmu_lock, and - cannot be taken without already holding kvm->arch.mmu_lock (typically with - ``read_lock`` for the TDP MMU, thus the need for additional spinlocks). +- kvm->arch.mmu_lock is an rwlock; critical sections for + kvm->arch.tdp_mmu_pages_lock and kvm->arch.mmu_unsync_pages_lock must + also take kvm->arch.mmu_lock Everything else is a leaf: no other lock is taken inside the critical sections. diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index a565a2e70f30..414691169a04 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1407,9 +1407,8 @@ struct kvm_arch { * the MMU lock in read mode + RCU or * the MMU lock in write mode * - * For writes, this list is protected by: - * the MMU lock in read mode + the tdp_mmu_pages_lock or - * the MMU lock in write mode + * For writes, this list is protected by tdp_mmu_pages_lock; see + * below for the details. * * Roots will remain in the list until their tdp_mmu_root_count * drops to zero, at which point the thread that decremented the @@ -1426,8 +1425,10 @@ struct kvm_arch { * - possible_nx_huge_pages; * - the possible_nx_huge_page_link field of kvm_mmu_page structs used * by the TDP MMU - * It is acceptable, but not necessary, to acquire this lock when - * the thread holds the MMU lock in write mode. + * Because the lock is only taken within the MMU lock, strictly + * speaking it is redundant to acquire this lock when the thread + * holds the MMU lock in write mode. However it often simplifies + * the code to do so. */ spinlock_t tdp_mmu_pages_lock; #endif /* CONFIG_X86_64 */ diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index c6dc09acea12..6ae19b4ee5b1 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -75,12 +75,6 @@ static void tdp_mmu_free_sp_rcu_callback(struct rcu_head *head) void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root) { - /* - * Either read or write is okay, but mmu_lock must be held because - * writers are not required to take tdp_mmu_pages_lock. - */ - lockdep_assert_held(&kvm->mmu_lock); - if (!refcount_dec_and_test(&root->tdp_mmu_root_count)) return; @@ -281,28 +275,18 @@ static void tdp_unaccount_mmu_page(struct kvm *kvm, struct kvm_mmu_page *sp) * * @kvm: kvm instance * @sp: the page to be removed - * @shared: This operation may not be running under the exclusive use of - * the MMU lock and the operation must synchronize with other - * threads that might be adding or removing pages. */ -static void tdp_mmu_unlink_sp(struct kvm *kvm, struct kvm_mmu_page *sp, - bool shared) +static void tdp_mmu_unlink_sp(struct kvm *kvm, struct kvm_mmu_page *sp) { tdp_unaccount_mmu_page(kvm, sp); if (!sp->nx_huge_page_disallowed) return; - if (shared) - spin_lock(&kvm->arch.tdp_mmu_pages_lock); - else - lockdep_assert_held_write(&kvm->mmu_lock); - + spin_lock(&kvm->arch.tdp_mmu_pages_lock); sp->nx_huge_page_disallowed = false; untrack_possible_nx_huge_page(kvm, sp); - - if (shared) - spin_unlock(&kvm->arch.tdp_mmu_pages_lock); + spin_unlock(&kvm->arch.tdp_mmu_pages_lock); } /** @@ -331,7 +315,7 @@ static void handle_removed_pt(struct kvm *kvm, tdp_ptep_t pt, bool shared) trace_kvm_mmu_prepare_zap_page(sp); - tdp_mmu_unlink_sp(kvm, sp, shared); + tdp_mmu_unlink_sp(kvm, sp); for (i = 0; i < SPTE_ENT_PER_PAGE; i++) { tdp_ptep_t sptep = pt + i; From e59f75de4e501e87de7743fec29dd247a6ae6cd3 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Sat, 25 Nov 2023 03:34:00 -0500 Subject: [PATCH 097/336] KVM: x86/mmu: fix comment about mmu_unsync_pages_lock Fix the comment about what can and cannot happen when mmu_unsync_pages_lock is not help. The comment correctly mentions "clearing sp->unsync", but then it talks about unsync going from 0 to 1. Signed-off-by: Paolo Bonzini Link: https://lore.kernel.org/r/20231125083400.1399197-5-pbonzini@redhat.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index ace9f7c13132..3fec3f6cdd53 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -2850,9 +2850,9 @@ int mmu_try_to_unsync_pages(struct kvm *kvm, const struct kvm_memory_slot *slot, /* * Recheck after taking the spinlock, a different vCPU * may have since marked the page unsync. A false - * positive on the unprotected check above is not + * negative on the unprotected check above is not * possible as clearing sp->unsync _must_ hold mmu_lock - * for write, i.e. unsync cannot transition from 0->1 + * for write, i.e. unsync cannot transition from 1->0 * while this CPU holds mmu_lock for read (or write). */ if (READ_ONCE(sp->unsync)) From 8c4976772d9b5858b8b456e84783e089c6cfa66e Mon Sep 17 00:00:00 2001 From: Philipp Stanner Date: Thu, 2 Nov 2023 19:15:25 +0100 Subject: [PATCH 098/336] KVM: s390: Harden copying of userspace-array against overflow guestdbg.c utilizes memdup_user() to copy a userspace array. This, currently, does not check for an overflow. Use the new wrapper memdup_array_user() to copy the array more safely. Note, KVM explicitly checks the number of entries before duplicating the array, i.e. adding the overflow check should be a glorified nop. Suggested-by: Dave Airlie Signed-off-by: Philipp Stanner Acked-by: Claudio Imbrenda Acked-by: Christian Borntraeger Link: https://lore.kernel.org/r/20231102181526.43279-3-pstanner@redhat.com [sean: call out that KVM pre-checks the number of entries] Signed-off-by: Sean Christopherson --- arch/s390/kvm/guestdbg.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/s390/kvm/guestdbg.c b/arch/s390/kvm/guestdbg.c index 3765c4223bf9..80879fc73c90 100644 --- a/arch/s390/kvm/guestdbg.c +++ b/arch/s390/kvm/guestdbg.c @@ -213,8 +213,8 @@ int kvm_s390_import_bp_data(struct kvm_vcpu *vcpu, else if (dbg->arch.nr_hw_bp > MAX_BP_COUNT) return -EINVAL; - bp_data = memdup_user(dbg->arch.hw_bp, - sizeof(*bp_data) * dbg->arch.nr_hw_bp); + bp_data = memdup_array_user(dbg->arch.hw_bp, dbg->arch.nr_hw_bp, + sizeof(*bp_data)); if (IS_ERR(bp_data)) return PTR_ERR(bp_data); From 1f829359c8c37f77a340575957686ca8c4bca317 Mon Sep 17 00:00:00 2001 From: Philipp Stanner Date: Thu, 2 Nov 2023 19:15:26 +0100 Subject: [PATCH 099/336] KVM: Harden copying of userspace-array against overflow kvm_main.c utilizes vmemdup_user() and array_size() to copy a userspace array. Currently, this does not check for an overflow. Use the new wrapper vmemdup_array_user() to copy the array more safely. Note, KVM explicitly checks the number of entries before duplicating the array, i.e. adding the overflow check should be a glorified nop. Suggested-by: Dave Airlie Signed-off-by: Philipp Stanner Link: https://lore.kernel.org/r/20231102181526.43279-4-pstanner@redhat.com [sean: call out that KVM pre-checks the number of entries] Signed-off-by: Sean Christopherson --- virt/kvm/kvm_main.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index b744eddecbc0..e76d49daa470 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -5262,9 +5262,8 @@ static long kvm_vm_ioctl(struct file *filp, goto out; if (routing.nr) { urouting = argp; - entries = vmemdup_user(urouting->entries, - array_size(sizeof(*entries), - routing.nr)); + entries = vmemdup_array_user(urouting->entries, + routing.nr, sizeof(*entries)); if (IS_ERR(entries)) { r = PTR_ERR(entries); goto out; From 1af3bf2befc07c7198100949dd1bece02a7dbded Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 29 Nov 2023 14:49:13 -0800 Subject: [PATCH 100/336] KVM: selftests: Fix MWAIT error message when guest assertion fails Print out the test and vector as intended when a guest assert fails an assertion regarding MONITOR/MWAIT faulting. Unfortunately, the guest printf support doesn't detect such issues at compile-time, so the bug manifests as a confusing error message, e.g. in the most confusing case, the test complains that it got vector "0" instead of expected vector "0". Fixes: 0f52e4aaa614 ("KVM: selftests: Convert the MONITOR/MWAIT test to use printf guest asserts") Reviewed-by: Maxim Levitsky Link: https://lore.kernel.org/r/20231107182159.404770-1-seanjc@google.com Link: https://lore.kernel.org/r/20231129224916.532431-2-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/x86_64/monitor_mwait_test.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/kvm/x86_64/monitor_mwait_test.c b/tools/testing/selftests/kvm/x86_64/monitor_mwait_test.c index 80aa3d8b18f8..853802641e1e 100644 --- a/tools/testing/selftests/kvm/x86_64/monitor_mwait_test.c +++ b/tools/testing/selftests/kvm/x86_64/monitor_mwait_test.c @@ -27,10 +27,12 @@ do { \ \ if (fault_wanted) \ __GUEST_ASSERT((vector) == UD_VECTOR, \ - "Expected #UD on " insn " for testcase '0x%x', got '0x%x'", vector); \ + "Expected #UD on " insn " for testcase '0x%x', got '0x%x'", \ + testcase, vector); \ else \ __GUEST_ASSERT(!(vector), \ - "Expected success on " insn " for testcase '0x%x', got '0x%x'", vector); \ + "Expected success on " insn " for testcase '0x%x', got '0x%x'", \ + testcase, vector); \ } while (0) static void guest_monitor_wait(int testcase) From 4d53dcc5d0bc2c445e29cb14df6d2cf93091731f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 29 Nov 2023 14:49:14 -0800 Subject: [PATCH 101/336] KVM: selftests: Fix benign %llx vs. %lx issues in guest asserts Convert %llx to %lx as appropriate in guest asserts. The guest printf implementation treats them the same as KVM selftests are 64-bit only, but strictly adhering to the correct format will allow annotating the underlying helpers with __printf() without introducing new warnings in the build. Link: https://lore.kernel.org/r/20231129224916.532431-3-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/set_memory_region_test.c | 6 +++--- tools/testing/selftests/kvm/x86_64/hyperv_features.c | 2 +- .../selftests/kvm/x86_64/private_mem_conversions_test.c | 2 +- .../selftests/kvm/x86_64/svm_nested_soft_inject_test.c | 4 ++-- tools/testing/selftests/kvm/x86_64/vmx_pmu_caps_test.c | 2 +- tools/testing/selftests/kvm/x86_64/xcr0_cpuid_test.c | 8 ++++---- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tools/testing/selftests/kvm/set_memory_region_test.c b/tools/testing/selftests/kvm/set_memory_region_test.c index 6637a0845acf..03ec7efd19aa 100644 --- a/tools/testing/selftests/kvm/set_memory_region_test.c +++ b/tools/testing/selftests/kvm/set_memory_region_test.c @@ -157,17 +157,17 @@ static void guest_code_move_memory_region(void) */ val = guest_spin_on_val(0); __GUEST_ASSERT(val == 1 || val == MMIO_VAL, - "Expected '1' or MMIO ('%llx'), got '%llx'", MMIO_VAL, val); + "Expected '1' or MMIO ('%lx'), got '%lx'", MMIO_VAL, val); /* Spin until the misaligning memory region move completes. */ val = guest_spin_on_val(MMIO_VAL); __GUEST_ASSERT(val == 1 || val == 0, - "Expected '0' or '1' (no MMIO), got '%llx'", val); + "Expected '0' or '1' (no MMIO), got '%lx'", val); /* Spin until the memory region starts to get re-aligned. */ val = guest_spin_on_val(0); __GUEST_ASSERT(val == 1 || val == MMIO_VAL, - "Expected '1' or MMIO ('%llx'), got '%llx'", MMIO_VAL, val); + "Expected '1' or MMIO ('%lx'), got '%lx'", MMIO_VAL, val); /* Spin until the re-aligning memory region move completes. */ val = guest_spin_on_val(MMIO_VAL); diff --git a/tools/testing/selftests/kvm/x86_64/hyperv_features.c b/tools/testing/selftests/kvm/x86_64/hyperv_features.c index 9f28aa276c4e..4bb63b6ee4a0 100644 --- a/tools/testing/selftests/kvm/x86_64/hyperv_features.c +++ b/tools/testing/selftests/kvm/x86_64/hyperv_features.c @@ -66,7 +66,7 @@ static void guest_msr(struct msr_data *msr) if (msr->write) __GUEST_ASSERT(!vector, - "WRMSR(0x%x) to '0x%llx', RDMSR read '0x%llx'", + "WRMSR(0x%x) to '0x%lx', RDMSR read '0x%lx'", msr->idx, msr->write_val, msr_val); /* Invariant TSC bit appears when TSC invariant control MSR is written to */ diff --git a/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.c b/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.c index 4d6a37a5d896..65ad38b6be1f 100644 --- a/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.c +++ b/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.c @@ -35,7 +35,7 @@ do { \ \ for (i = 0; i < size; i++) \ __GUEST_ASSERT(mem[i] == pattern, \ - "Guest expected 0x%x at offset %lu (gpa 0x%llx), got 0x%x", \ + "Guest expected 0x%x at offset %lu (gpa 0x%lx), got 0x%x", \ pattern, i, gpa + i, mem[i]); \ } while (0) diff --git a/tools/testing/selftests/kvm/x86_64/svm_nested_soft_inject_test.c b/tools/testing/selftests/kvm/x86_64/svm_nested_soft_inject_test.c index 7ee44496cf97..0c7ce3d4e83a 100644 --- a/tools/testing/selftests/kvm/x86_64/svm_nested_soft_inject_test.c +++ b/tools/testing/selftests/kvm/x86_64/svm_nested_soft_inject_test.c @@ -103,7 +103,7 @@ static void l1_guest_code(struct svm_test_data *svm, uint64_t is_nmi, uint64_t i run_guest(vmcb, svm->vmcb_gpa); __GUEST_ASSERT(vmcb->control.exit_code == SVM_EXIT_VMMCALL, - "Expected VMMCAL #VMEXIT, got '0x%x', info1 = '0x%llx, info2 = '0x%llx'", + "Expected VMMCAL #VMEXIT, got '0x%x', info1 = '0x%lx, info2 = '0x%lx'", vmcb->control.exit_code, vmcb->control.exit_info_1, vmcb->control.exit_info_2); @@ -133,7 +133,7 @@ static void l1_guest_code(struct svm_test_data *svm, uint64_t is_nmi, uint64_t i run_guest(vmcb, svm->vmcb_gpa); __GUEST_ASSERT(vmcb->control.exit_code == SVM_EXIT_HLT, - "Expected HLT #VMEXIT, got '0x%x', info1 = '0x%llx, info2 = '0x%llx'", + "Expected HLT #VMEXIT, got '0x%x', info1 = '0x%lx, info2 = '0x%lx'", vmcb->control.exit_code, vmcb->control.exit_info_1, vmcb->control.exit_info_2); diff --git a/tools/testing/selftests/kvm/x86_64/vmx_pmu_caps_test.c b/tools/testing/selftests/kvm/x86_64/vmx_pmu_caps_test.c index ebbcb0a3f743..2a8d4ac2f020 100644 --- a/tools/testing/selftests/kvm/x86_64/vmx_pmu_caps_test.c +++ b/tools/testing/selftests/kvm/x86_64/vmx_pmu_caps_test.c @@ -56,7 +56,7 @@ static void guest_test_perf_capabilities_gp(uint64_t val) uint8_t vector = wrmsr_safe(MSR_IA32_PERF_CAPABILITIES, val); __GUEST_ASSERT(vector == GP_VECTOR, - "Expected #GP for value '0x%llx', got vector '0x%x'", + "Expected #GP for value '0x%lx', got vector '0x%x'", val, vector); } diff --git a/tools/testing/selftests/kvm/x86_64/xcr0_cpuid_test.c b/tools/testing/selftests/kvm/x86_64/xcr0_cpuid_test.c index 77d04a7bdadd..dc6217440db3 100644 --- a/tools/testing/selftests/kvm/x86_64/xcr0_cpuid_test.c +++ b/tools/testing/selftests/kvm/x86_64/xcr0_cpuid_test.c @@ -25,7 +25,7 @@ do { \ \ __GUEST_ASSERT((__supported & (xfeatures)) != (xfeatures) || \ __supported == ((xfeatures) | (dependencies)), \ - "supported = 0x%llx, xfeatures = 0x%llx, dependencies = 0x%llx", \ + "supported = 0x%lx, xfeatures = 0x%llx, dependencies = 0x%llx", \ __supported, (xfeatures), (dependencies)); \ } while (0) @@ -42,7 +42,7 @@ do { \ uint64_t __supported = (supported_xcr0) & (xfeatures); \ \ __GUEST_ASSERT(!__supported || __supported == (xfeatures), \ - "supported = 0x%llx, xfeatures = 0x%llx", \ + "supported = 0x%lx, xfeatures = 0x%llx", \ __supported, (xfeatures)); \ } while (0) @@ -81,7 +81,7 @@ static void guest_code(void) vector = xsetbv_safe(0, supported_xcr0); __GUEST_ASSERT(!vector, - "Expected success on XSETBV(0x%llx), got vector '0x%x'", + "Expected success on XSETBV(0x%lx), got vector '0x%x'", supported_xcr0, vector); for (i = 0; i < 64; i++) { @@ -90,7 +90,7 @@ static void guest_code(void) vector = xsetbv_safe(0, supported_xcr0 | BIT_ULL(i)); __GUEST_ASSERT(vector == GP_VECTOR, - "Expected #GP on XSETBV(0x%llx), supported XCR0 = %llx, got vector '0x%x'", + "Expected #GP on XSETBV(0x%llx), supported XCR0 = %lx, got vector '0x%x'", BIT_ULL(i), supported_xcr0, vector); } From f813e6d41baf2bbdd1624f9f01ac4f365eb78891 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 29 Nov 2023 14:49:15 -0800 Subject: [PATCH 102/336] KVM: selftests: Fix broken assert messages in Hyper-V features test Swap the ordering of parameters to guest asserts related to {RD,WR}MSR success/failure in the Hyper-V features test. As is, the output will be mangled and broken due to passing an integer as a string and vice versa. Opportunistically fix a benign %u vs. %lu issue as well. Link: https://lore.kernel.org/r/20231129224916.532431-4-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/x86_64/hyperv_features.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/kvm/x86_64/hyperv_features.c b/tools/testing/selftests/kvm/x86_64/hyperv_features.c index 4bb63b6ee4a0..29f6bdbce817 100644 --- a/tools/testing/selftests/kvm/x86_64/hyperv_features.c +++ b/tools/testing/selftests/kvm/x86_64/hyperv_features.c @@ -55,11 +55,11 @@ static void guest_msr(struct msr_data *msr) if (msr->fault_expected) __GUEST_ASSERT(vector == GP_VECTOR, "Expected #GP on %sMSR(0x%x), got vector '0x%x'", - msr->idx, msr->write ? "WR" : "RD", vector); + msr->write ? "WR" : "RD", msr->idx, vector); else __GUEST_ASSERT(!vector, "Expected success on %sMSR(0x%x), got vector '0x%x'", - msr->idx, msr->write ? "WR" : "RD", vector); + msr->write ? "WR" : "RD", msr->idx, vector); if (vector || is_write_only_msr(msr->idx)) goto done; @@ -102,11 +102,11 @@ static void guest_hcall(vm_vaddr_t pgs_gpa, struct hcall_data *hcall) vector = __hyperv_hypercall(hcall->control, input, output, &res); if (hcall->ud_expected) { __GUEST_ASSERT(vector == UD_VECTOR, - "Expected #UD for control '%u', got vector '0x%x'", + "Expected #UD for control '%lu', got vector '0x%x'", hcall->control, vector); } else { __GUEST_ASSERT(!vector, - "Expected no exception for control '%u', got vector '0x%x'", + "Expected no exception for control '%lu', got vector '0x%x'", hcall->control, vector); GUEST_ASSERT_EQ(res, hcall->expect); } From 1b2658e4c709135fe1910423d3216632f641baf9 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 29 Nov 2023 14:49:16 -0800 Subject: [PATCH 103/336] KVM: selftests: Annotate guest ucall, printf, and assert helpers with __printf() Annotate guest printf helpers with __printf() so that the compiler will warn about incorrect formatting at compile time (see git log for how easy it is to screw up with the formatting). Suggested-by: Maxim Levitsky Link: https://lore.kernel.org/r/20231129224916.532431-5-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/test_util.h | 2 +- tools/testing/selftests/kvm/include/ucall_common.h | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/kvm/include/test_util.h b/tools/testing/selftests/kvm/include/test_util.h index a0c7dd3a5b30..71a41fa924b7 100644 --- a/tools/testing/selftests/kvm/include/test_util.h +++ b/tools/testing/selftests/kvm/include/test_util.h @@ -191,7 +191,7 @@ static inline uint32_t atoi_non_negative(const char *name, const char *num_str) } int guest_vsnprintf(char *buf, int n, const char *fmt, va_list args); -int guest_snprintf(char *buf, int n, const char *fmt, ...); +__printf(3, 4) int guest_snprintf(char *buf, int n, const char *fmt, ...); char *strdup_printf(const char *fmt, ...) __attribute__((format(printf, 1, 2), nonnull(1))); diff --git a/tools/testing/selftests/kvm/include/ucall_common.h b/tools/testing/selftests/kvm/include/ucall_common.h index 0fb472a5a058..d9d6581b8d4f 100644 --- a/tools/testing/selftests/kvm/include/ucall_common.h +++ b/tools/testing/selftests/kvm/include/ucall_common.h @@ -34,9 +34,10 @@ void ucall_arch_do_ucall(vm_vaddr_t uc); void *ucall_arch_get_ucall(struct kvm_vcpu *vcpu); void ucall(uint64_t cmd, int nargs, ...); -void ucall_fmt(uint64_t cmd, const char *fmt, ...); -void ucall_assert(uint64_t cmd, const char *exp, const char *file, - unsigned int line, const char *fmt, ...); +__printf(2, 3) void ucall_fmt(uint64_t cmd, const char *fmt, ...); +__printf(5, 6) void ucall_assert(uint64_t cmd, const char *exp, + const char *file, unsigned int line, + const char *fmt, ...); uint64_t get_ucall(struct kvm_vcpu *vcpu, struct ucall *uc); void ucall_init(struct kvm_vm *vm, vm_paddr_t mmio_gpa); int ucall_nr_pages_required(uint64_t page_size); From fd7c3c3767c38dbbf2f5c993c6fd42a71846e7e0 Mon Sep 17 00:00:00 2001 From: Mark-PK Tsai Date: Thu, 19 Oct 2023 09:35:22 +0100 Subject: [PATCH 104/336] ARM: 9327/1: vfp: Add missing VFP instructions to neon_support_hook Add the missing "Unconditional Advanced SIMD and floating-point instructions" in [1] to the VFP undef hook. This commit addresses the issue reported in [2], where executing the vudot instruction on a platform with FEAT_DotProd support resulted in an undefined instruction error. Link: https://developer.arm.com/documentation/ddi0597/2023-06/?lang=en [1] Link: https://lore.kernel.org/lkml/20230920083907.30479-1-mark-pk.tsai@mediatek.com/ [2] Signed-off-by: Mark-PK Tsai Tested-by: Xuewen Yan Signed-off-by: Russell King (Oracle) --- arch/arm/vfp/vfpmodule.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c index 7e8773a2d99d..b68efe643a12 100644 --- a/arch/arm/vfp/vfpmodule.c +++ b/arch/arm/vfp/vfpmodule.c @@ -800,6 +800,24 @@ static struct undef_hook neon_support_hook[] = {{ .cpsr_mask = PSR_T_BIT, .cpsr_val = PSR_T_BIT, .fn = vfp_support_entry, +}, { + .instr_mask = 0xff000800, + .instr_val = 0xfc000800, + .cpsr_mask = 0, + .cpsr_val = 0, + .fn = vfp_support_entry, +}, { + .instr_mask = 0xff000800, + .instr_val = 0xfd000800, + .cpsr_mask = 0, + .cpsr_val = 0, + .fn = vfp_support_entry, +}, { + .instr_mask = 0xff000800, + .instr_val = 0xfe000800, + .cpsr_mask = 0, + .cpsr_val = 0, + .fn = vfp_support_entry, }}; static struct undef_hook vfp_support_hook = { From f54e8634d1366926c807e2af6125b33cff555fa7 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Tue, 7 Nov 2023 01:36:03 +0100 Subject: [PATCH 105/336] ARM: 9330/1: davinci: also select PINCTRL kconfig warns when PINCTRL_SINGLE is selected but PINCTRL is not set, so also set PINCTRL for ARCH_DAVINCI. This prevents a kconfig/build warning: WARNING: unmet direct dependencies detected for PINCTRL_SINGLE Depends on [n]: PINCTRL [=n] && OF [=y] && HAS_IOMEM [=y] Selected by [y]: - ARCH_DAVINCI [=y] && ARCH_MULTI_V5 [=y] Closes: lore.kernel.org/r/202311070548.0f6XfBrh-lkp@intel.com Fixes: f962396ce292 ("ARM: davinci: support multiplatform build for ARM v5") Signed-off-by: Randy Dunlap Reported-by: kernel test robot Cc: Bartosz Golaszewski Cc: Arnd Bergmann Cc: linux-arm-kernel@lists.infradead.org Cc: patches@armlinux.org.uk Signed-off-by: Russell King (Oracle) --- arch/arm/mach-davinci/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm/mach-davinci/Kconfig b/arch/arm/mach-davinci/Kconfig index 4316e1370627..1c8bf56982e5 100644 --- a/arch/arm/mach-davinci/Kconfig +++ b/arch/arm/mach-davinci/Kconfig @@ -10,6 +10,7 @@ menuconfig ARCH_DAVINCI select PM_GENERIC_DOMAINS_OF if PM && OF select REGMAP_MMIO select RESET_CONTROLLER + select PINCTRL select PINCTRL_SINGLE if ARCH_DAVINCI From c16af1212479570454752671a170a1756e11fdfb Mon Sep 17 00:00:00 2001 From: Wang Kefeng Date: Thu, 19 Oct 2023 12:21:35 +0100 Subject: [PATCH 106/336] ARM: 9328/1: mm: try VMA lock-based page fault handling first Attempt VMA lock-based page fault handling first, and fall back to the existing mmap_lock-based handling if that fails, the ebizzy benchmark shows 25% improvement on qemu with 2 cpus. Signed-off-by: Kefeng Wang Signed-off-by: Russell King (Oracle) --- arch/arm/Kconfig | 1 + arch/arm/mm/fault.c | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index f8567e95f98b..8f47d6762ea4 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -35,6 +35,7 @@ config ARM select ARCH_OPTIONAL_KERNEL_RWX_DEFAULT if CPU_V7 select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_SUPPORTS_HUGETLBFS if ARM_LPAE + select ARCH_SUPPORTS_PER_VMA_LOCK select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_CMPXCHG_LOCKREF select ARCH_USE_MEMTEST diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index fef62e4a9edd..e96fb40b9cc3 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -278,6 +278,35 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); + if (!(flags & FAULT_FLAG_USER)) + goto lock_mmap; + + vma = lock_vma_under_rcu(mm, addr); + if (!vma) + goto lock_mmap; + + if (!(vma->vm_flags & vm_flags)) { + vma_end_read(vma); + goto lock_mmap; + } + fault = handle_mm_fault(vma, addr, flags | FAULT_FLAG_VMA_LOCK, regs); + if (!(fault & (VM_FAULT_RETRY | VM_FAULT_COMPLETED))) + vma_end_read(vma); + + if (!(fault & VM_FAULT_RETRY)) { + count_vm_vma_lock_event(VMA_LOCK_SUCCESS); + goto done; + } + count_vm_vma_lock_event(VMA_LOCK_RETRY); + + /* Quick path to respond to signals */ + if (fault_signal_pending(fault, regs)) { + if (!user_mode(regs)) + goto no_context; + return 0; + } +lock_mmap: + retry: vma = lock_mm_and_find_vma(mm, addr, regs); if (unlikely(!vma)) { @@ -316,6 +345,7 @@ retry: } mmap_read_unlock(mm); +done: /* * Handle the "normal" case first - VM_FAULT_MAJOR From 89320c9785e8429155f8dfa44a183b509866e852 Mon Sep 17 00:00:00 2001 From: Mark-PK Tsai Date: Thu, 19 Oct 2023 14:38:16 +0100 Subject: [PATCH 107/336] ARM: 9329/1: kasan: Use memblock_alloc_try_nid_raw for shadow page kasan_pte_populate fill KASAN_SHADOW_INIT in the newly allocated shadow page, so it's unnecessary to use memblock_alloc_try_nid, which always zero the new allocated memory. Use memblock_alloc_try_nid_raw instead of memblock_alloc_try_nid like arm64 does which can make kasan init faster. Signed-off-by: Mark-PK Tsai Signed-off-by: Russell King (Oracle) --- arch/arm/mm/kasan_init.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/arch/arm/mm/kasan_init.c b/arch/arm/mm/kasan_init.c index 24d71b5db62d..111d4f703136 100644 --- a/arch/arm/mm/kasan_init.c +++ b/arch/arm/mm/kasan_init.c @@ -28,6 +28,12 @@ static pgd_t tmp_pgd_table[PTRS_PER_PGD] __initdata __aligned(PGD_SIZE); pmd_t tmp_pmd_table[PTRS_PER_PMD] __page_aligned_bss; +static __init void *kasan_alloc_block_raw(size_t size) +{ + return memblock_alloc_try_nid_raw(size, size, __pa(MAX_DMA_ADDRESS), + MEMBLOCK_ALLOC_NOLEAKTRACE, NUMA_NO_NODE); +} + static __init void *kasan_alloc_block(size_t size) { return memblock_alloc_try_nid(size, size, __pa(MAX_DMA_ADDRESS), @@ -50,7 +56,7 @@ static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr, if (!pte_none(READ_ONCE(*ptep))) continue; - p = kasan_alloc_block(PAGE_SIZE); + p = kasan_alloc_block_raw(PAGE_SIZE); if (!p) { panic("%s failed to allocate shadow page for address 0x%lx\n", __func__, addr); From 87562052c965ba7de6dc490434e53691fe46c898 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Tue, 5 Dec 2023 11:36:15 +0100 Subject: [PATCH 108/336] KVM: x86/xen: Remove unneeded xen context from kvm_arch when !CONFIG_KVM_XEN Saving a few bytes of memory per KVM VM is certainly great but what's more important is the ability to see where the code accesses Xen emulation context while CONFIG_KVM_XEN is not enabled. Currently, kvm_cpu_get_extint() is the only such place and it is harmless: kvm_xen_has_interrupt() always returns '0' when !CONFIG_KVM_XEN. No functional change intended. Reviewed-by: Maxim Levitsky Tested-by: Jeremi Piotrowski Signed-off-by: Vitaly Kuznetsov Link: https://lore.kernel.org/r/20231205103630.1391318-2-vkuznets@redhat.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 5 +++++ arch/x86/kvm/irq.c | 2 ++ 2 files changed, 7 insertions(+) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 5e5e9e0abd4a..b0ca65632f2c 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1136,6 +1136,7 @@ struct msr_bitmap_range { unsigned long *bitmap; }; +#ifdef CONFIG_KVM_XEN /* Xen emulation context */ struct kvm_xen { struct mutex xen_lock; @@ -1147,6 +1148,7 @@ struct kvm_xen { struct idr evtchn_ports; unsigned long poll_mask[BITS_TO_LONGS(KVM_MAX_VCPUS)]; }; +#endif enum kvm_irqchip_mode { KVM_IRQCHIP_NONE, @@ -1349,7 +1351,10 @@ struct kvm_arch { struct hlist_head mask_notifier_list; struct kvm_hv hyperv; + +#ifdef CONFIG_KVM_XEN struct kvm_xen xen; +#endif bool backwards_tsc_observed; bool boot_vcpu_runs_old_kvmclock; diff --git a/arch/x86/kvm/irq.c b/arch/x86/kvm/irq.c index b2c397dd2bc6..ad9ca8a60144 100644 --- a/arch/x86/kvm/irq.c +++ b/arch/x86/kvm/irq.c @@ -118,8 +118,10 @@ static int kvm_cpu_get_extint(struct kvm_vcpu *v) if (!lapic_in_kernel(v)) return v->arch.interrupt.nr; +#ifdef CONFIG_KVM_XEN if (kvm_xen_has_interrupt(v)) return v->kvm->arch.xen.upcall_vector; +#endif if (irqchip_split(v->kvm)) { int vector = v->arch.pending_external_vector; From cfef5af3cb0e57501dcac2816ab11a20c074866d Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Tue, 5 Dec 2023 11:36:16 +0100 Subject: [PATCH 109/336] KVM: x86: Move Hyper-V partition assist page out of Hyper-V emulation context Hyper-V partition assist page is used when KVM runs on top of Hyper-V and is not used for Windows/Hyper-V guests on KVM, this means that 'hv_pa_pg' placement in 'struct kvm_hv' is unfortunate. As a preparation to making Hyper-V emulation optional, move 'hv_pa_pg' to 'struct kvm_arch' and put it under CONFIG_HYPERV. While on it, introduce hv_get_partition_assist_page() helper to allocate partition assist page. Move the comment explaining why we use a single page for all vCPUs from VMX and expand it a bit. No functional change intended. Reviewed-by: Maxim Levitsky Tested-by: Jeremi Piotrowski Signed-off-by: Vitaly Kuznetsov Link: https://lore.kernel.org/r/20231205103630.1391318-3-vkuznets@redhat.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/kvm_onhyperv.h | 20 ++++++++++++++++++++ arch/x86/kvm/svm/svm_onhyperv.c | 10 +++------- arch/x86/kvm/vmx/vmx.c | 14 +++----------- arch/x86/kvm/x86.c | 4 +++- 5 files changed, 30 insertions(+), 20 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index b0ca65632f2c..86069b985d22 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1125,7 +1125,6 @@ struct kvm_hv { */ unsigned int synic_auto_eoi_used; - struct hv_partition_assist_pg *hv_pa_pg; struct kvm_hv_syndbg hv_syndbg; }; @@ -1447,6 +1446,7 @@ struct kvm_arch { #if IS_ENABLED(CONFIG_HYPERV) hpa_t hv_root_tdp; spinlock_t hv_root_tdp_lock; + struct hv_partition_assist_pg *hv_pa_pg; #endif /* * VM-scope maximum vCPU ID. Used to determine the size of structures diff --git a/arch/x86/kvm/kvm_onhyperv.h b/arch/x86/kvm/kvm_onhyperv.h index f9ca3e7432b2..eefab3dc8498 100644 --- a/arch/x86/kvm/kvm_onhyperv.h +++ b/arch/x86/kvm/kvm_onhyperv.h @@ -10,6 +10,26 @@ int hv_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn, gfn_t nr_pages); int hv_flush_remote_tlbs(struct kvm *kvm); void hv_track_root_tdp(struct kvm_vcpu *vcpu, hpa_t root_tdp); +static inline hpa_t hv_get_partition_assist_page(struct kvm_vcpu *vcpu) +{ + /* + * Partition assist page is something which Hyper-V running in L0 + * requires from KVM running in L1 before direct TLB flush for L2 + * guests can be enabled. KVM doesn't currently use the page but to + * comply with TLFS it still needs to be allocated. For now, this + * is a single page shared among all vCPUs. + */ + struct hv_partition_assist_pg **p_hv_pa_pg = + &vcpu->kvm->arch.hv_pa_pg; + + if (!*p_hv_pa_pg) + *p_hv_pa_pg = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); + + if (!*p_hv_pa_pg) + return INVALID_PAGE; + + return __pa(*p_hv_pa_pg); +} #else /* !CONFIG_HYPERV */ static inline int hv_flush_remote_tlbs(struct kvm *kvm) { diff --git a/arch/x86/kvm/svm/svm_onhyperv.c b/arch/x86/kvm/svm/svm_onhyperv.c index 7af8422d3382..3971b3ea5d04 100644 --- a/arch/x86/kvm/svm/svm_onhyperv.c +++ b/arch/x86/kvm/svm/svm_onhyperv.c @@ -18,18 +18,14 @@ int svm_hv_enable_l2_tlb_flush(struct kvm_vcpu *vcpu) { struct hv_vmcb_enlightenments *hve; - struct hv_partition_assist_pg **p_hv_pa_pg = - &to_kvm_hv(vcpu->kvm)->hv_pa_pg; + hpa_t partition_assist_page = hv_get_partition_assist_page(vcpu); - if (!*p_hv_pa_pg) - *p_hv_pa_pg = kzalloc(PAGE_SIZE, GFP_KERNEL); - - if (!*p_hv_pa_pg) + if (partition_assist_page == INVALID_PAGE) return -ENOMEM; hve = &to_svm(vcpu)->vmcb->control.hv_enlightenments; - hve->partition_assist_page = __pa(*p_hv_pa_pg); + hve->partition_assist_page = partition_assist_page; hve->hv_vm_id = (unsigned long)vcpu->kvm; if (!hve->hv_enlightenments_control.nested_flush_hypercall) { hve->hv_enlightenments_control.nested_flush_hypercall = 1; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 40e3780d73ae..cf19a3346639 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -523,22 +523,14 @@ module_param(enlightened_vmcs, bool, 0444); static int hv_enable_l2_tlb_flush(struct kvm_vcpu *vcpu) { struct hv_enlightened_vmcs *evmcs; - struct hv_partition_assist_pg **p_hv_pa_pg = - &to_kvm_hv(vcpu->kvm)->hv_pa_pg; - /* - * Synthetic VM-Exit is not enabled in current code and so All - * evmcs in singe VM shares same assist page. - */ - if (!*p_hv_pa_pg) - *p_hv_pa_pg = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); + hpa_t partition_assist_page = hv_get_partition_assist_page(vcpu); - if (!*p_hv_pa_pg) + if (partition_assist_page == INVALID_PAGE) return -ENOMEM; evmcs = (struct hv_enlightened_vmcs *)to_vmx(vcpu)->loaded_vmcs->vmcs; - evmcs->partition_assist_page = - __pa(*p_hv_pa_pg); + evmcs->partition_assist_page = partition_assist_page; evmcs->hv_vm_id = (unsigned long)vcpu->kvm; evmcs->hv_enlightenments_control.nested_flush_hypercall = 1; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 6d0772b47041..81224b9676d9 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -12438,7 +12438,9 @@ void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) void kvm_arch_free_vm(struct kvm *kvm) { - kfree(to_kvm_hv(kvm)->hv_pa_pg); +#if IS_ENABLED(CONFIG_HYPERV) + kfree(kvm->arch.hv_pa_pg); +#endif __kvm_arch_free_vm(kvm); } From 50a82b0eb88c108d1ebc73a97f5b81df0d5918e0 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Tue, 5 Dec 2023 11:36:17 +0100 Subject: [PATCH 110/336] KVM: VMX: Split off vmx_onhyperv.{ch} from hyperv.{ch} hyperv.{ch} is currently a mix of stuff which is needed by both Hyper-V on KVM and KVM on Hyper-V. As a preparation to making Hyper-V emulation optional, put KVM-on-Hyper-V specific code into dedicated files. No functional change intended. Reviewed-by: Maxim Levitsky Tested-by: Jeremi Piotrowski Signed-off-by: Vitaly Kuznetsov Link: https://lore.kernel.org/r/20231205103630.1391318-4-vkuznets@redhat.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/Makefile | 4 + arch/x86/kvm/vmx/hyperv.c | 139 -------------------- arch/x86/kvm/vmx/hyperv.h | 217 ++++++++++++++++---------------- arch/x86/kvm/vmx/vmx.c | 1 + arch/x86/kvm/vmx/vmx_onhyperv.c | 36 ++++++ arch/x86/kvm/vmx/vmx_onhyperv.h | 124 ++++++++++++++++++ arch/x86/kvm/vmx/vmx_ops.h | 2 +- 7 files changed, 271 insertions(+), 252 deletions(-) create mode 100644 arch/x86/kvm/vmx/vmx_onhyperv.c create mode 100644 arch/x86/kvm/vmx/vmx_onhyperv.h diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index 80e3fe184d17..a99ffc3f3a3f 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -26,6 +26,10 @@ kvm-intel-y += vmx/vmx.o vmx/vmenter.o vmx/pmu_intel.o vmx/vmcs12.o \ vmx/hyperv.o vmx/nested.o vmx/posted_intr.o kvm-intel-$(CONFIG_X86_SGX_KVM) += vmx/sgx.o +ifdef CONFIG_HYPERV +kvm-intel-y += vmx/vmx_onhyperv.o +endif + kvm-amd-y += svm/svm.o svm/vmenter.o svm/pmu.o svm/nested.o svm/avic.o \ svm/sev.o svm/hyperv.o diff --git a/arch/x86/kvm/vmx/hyperv.c b/arch/x86/kvm/vmx/hyperv.c index 313b8bb5b8a7..de13dc14fe1d 100644 --- a/arch/x86/kvm/vmx/hyperv.c +++ b/arch/x86/kvm/vmx/hyperv.c @@ -13,111 +13,6 @@ #define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK -/* - * Enlightened VMCSv1 doesn't support these: - * - * POSTED_INTR_NV = 0x00000002, - * GUEST_INTR_STATUS = 0x00000810, - * APIC_ACCESS_ADDR = 0x00002014, - * POSTED_INTR_DESC_ADDR = 0x00002016, - * EOI_EXIT_BITMAP0 = 0x0000201c, - * EOI_EXIT_BITMAP1 = 0x0000201e, - * EOI_EXIT_BITMAP2 = 0x00002020, - * EOI_EXIT_BITMAP3 = 0x00002022, - * GUEST_PML_INDEX = 0x00000812, - * PML_ADDRESS = 0x0000200e, - * VM_FUNCTION_CONTROL = 0x00002018, - * EPTP_LIST_ADDRESS = 0x00002024, - * VMREAD_BITMAP = 0x00002026, - * VMWRITE_BITMAP = 0x00002028, - * - * TSC_MULTIPLIER = 0x00002032, - * PLE_GAP = 0x00004020, - * PLE_WINDOW = 0x00004022, - * VMX_PREEMPTION_TIMER_VALUE = 0x0000482E, - * - * Currently unsupported in KVM: - * GUEST_IA32_RTIT_CTL = 0x00002814, - */ -#define EVMCS1_SUPPORTED_PINCTRL \ - (PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR | \ - PIN_BASED_EXT_INTR_MASK | \ - PIN_BASED_NMI_EXITING | \ - PIN_BASED_VIRTUAL_NMIS) - -#define EVMCS1_SUPPORTED_EXEC_CTRL \ - (CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR | \ - CPU_BASED_HLT_EXITING | \ - CPU_BASED_CR3_LOAD_EXITING | \ - CPU_BASED_CR3_STORE_EXITING | \ - CPU_BASED_UNCOND_IO_EXITING | \ - CPU_BASED_MOV_DR_EXITING | \ - CPU_BASED_USE_TSC_OFFSETTING | \ - CPU_BASED_MWAIT_EXITING | \ - CPU_BASED_MONITOR_EXITING | \ - CPU_BASED_INVLPG_EXITING | \ - CPU_BASED_RDPMC_EXITING | \ - CPU_BASED_INTR_WINDOW_EXITING | \ - CPU_BASED_CR8_LOAD_EXITING | \ - CPU_BASED_CR8_STORE_EXITING | \ - CPU_BASED_RDTSC_EXITING | \ - CPU_BASED_TPR_SHADOW | \ - CPU_BASED_USE_IO_BITMAPS | \ - CPU_BASED_MONITOR_TRAP_FLAG | \ - CPU_BASED_USE_MSR_BITMAPS | \ - CPU_BASED_NMI_WINDOW_EXITING | \ - CPU_BASED_PAUSE_EXITING | \ - CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) - -#define EVMCS1_SUPPORTED_2NDEXEC \ - (SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | \ - SECONDARY_EXEC_WBINVD_EXITING | \ - SECONDARY_EXEC_ENABLE_VPID | \ - SECONDARY_EXEC_ENABLE_EPT | \ - SECONDARY_EXEC_UNRESTRICTED_GUEST | \ - SECONDARY_EXEC_DESC | \ - SECONDARY_EXEC_ENABLE_RDTSCP | \ - SECONDARY_EXEC_ENABLE_INVPCID | \ - SECONDARY_EXEC_ENABLE_XSAVES | \ - SECONDARY_EXEC_RDSEED_EXITING | \ - SECONDARY_EXEC_RDRAND_EXITING | \ - SECONDARY_EXEC_TSC_SCALING | \ - SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE | \ - SECONDARY_EXEC_PT_USE_GPA | \ - SECONDARY_EXEC_PT_CONCEAL_VMX | \ - SECONDARY_EXEC_BUS_LOCK_DETECTION | \ - SECONDARY_EXEC_NOTIFY_VM_EXITING | \ - SECONDARY_EXEC_ENCLS_EXITING) - -#define EVMCS1_SUPPORTED_3RDEXEC (0ULL) - -#define EVMCS1_SUPPORTED_VMEXIT_CTRL \ - (VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR | \ - VM_EXIT_SAVE_DEBUG_CONTROLS | \ - VM_EXIT_ACK_INTR_ON_EXIT | \ - VM_EXIT_HOST_ADDR_SPACE_SIZE | \ - VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | \ - VM_EXIT_SAVE_IA32_PAT | \ - VM_EXIT_LOAD_IA32_PAT | \ - VM_EXIT_SAVE_IA32_EFER | \ - VM_EXIT_LOAD_IA32_EFER | \ - VM_EXIT_CLEAR_BNDCFGS | \ - VM_EXIT_PT_CONCEAL_PIP | \ - VM_EXIT_CLEAR_IA32_RTIT_CTL) - -#define EVMCS1_SUPPORTED_VMENTRY_CTRL \ - (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | \ - VM_ENTRY_LOAD_DEBUG_CONTROLS | \ - VM_ENTRY_IA32E_MODE | \ - VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL | \ - VM_ENTRY_LOAD_IA32_PAT | \ - VM_ENTRY_LOAD_IA32_EFER | \ - VM_ENTRY_LOAD_BNDCFGS | \ - VM_ENTRY_PT_CONCEAL_PIP | \ - VM_ENTRY_LOAD_IA32_RTIT_CTL) - -#define EVMCS1_SUPPORTED_VMFUNC (0) - #define EVMCS1_OFFSET(x) offsetof(struct hv_enlightened_vmcs, x) #define EVMCS1_FIELD(number, name, clean_field)[ROL16(number, 6)] = \ {EVMCS1_OFFSET(name), clean_field} @@ -608,40 +503,6 @@ int nested_evmcs_check_controls(struct vmcs12 *vmcs12) return 0; } -#if IS_ENABLED(CONFIG_HYPERV) -DEFINE_STATIC_KEY_FALSE(__kvm_is_using_evmcs); - -/* - * KVM on Hyper-V always uses the latest known eVMCSv1 revision, the assumption - * is: in case a feature has corresponding fields in eVMCS described and it was - * exposed in VMX feature MSRs, KVM is free to use it. Warn if KVM meets a - * feature which has no corresponding eVMCS field, this likely means that KVM - * needs to be updated. - */ -#define evmcs_check_vmcs_conf(field, ctrl) \ - do { \ - typeof(vmcs_conf->field) unsupported; \ - \ - unsupported = vmcs_conf->field & ~EVMCS1_SUPPORTED_ ## ctrl; \ - if (unsupported) { \ - pr_warn_once(#field " unsupported with eVMCS: 0x%llx\n",\ - (u64)unsupported); \ - vmcs_conf->field &= EVMCS1_SUPPORTED_ ## ctrl; \ - } \ - } \ - while (0) - -void evmcs_sanitize_exec_ctrls(struct vmcs_config *vmcs_conf) -{ - evmcs_check_vmcs_conf(cpu_based_exec_ctrl, EXEC_CTRL); - evmcs_check_vmcs_conf(pin_based_exec_ctrl, PINCTRL); - evmcs_check_vmcs_conf(cpu_based_2nd_exec_ctrl, 2NDEXEC); - evmcs_check_vmcs_conf(cpu_based_3rd_exec_ctrl, 3RDEXEC); - evmcs_check_vmcs_conf(vmentry_ctrl, VMENTRY_CTRL); - evmcs_check_vmcs_conf(vmexit_ctrl, VMEXIT_CTRL); -} -#endif - int nested_enable_evmcs(struct kvm_vcpu *vcpu, uint16_t *vmcs_version) { diff --git a/arch/x86/kvm/vmx/hyperv.h b/arch/x86/kvm/vmx/hyperv.h index 9623fe1651c4..9401dbfaea7c 100644 --- a/arch/x86/kvm/vmx/hyperv.h +++ b/arch/x86/kvm/vmx/hyperv.h @@ -14,12 +14,113 @@ #include "vmcs.h" #include "vmcs12.h" -struct vmcs_config; - -#define current_evmcs ((struct hv_enlightened_vmcs *)this_cpu_read(current_vmcs)) - #define KVM_EVMCS_VERSION 1 +/* + * Enlightened VMCSv1 doesn't support these: + * + * POSTED_INTR_NV = 0x00000002, + * GUEST_INTR_STATUS = 0x00000810, + * APIC_ACCESS_ADDR = 0x00002014, + * POSTED_INTR_DESC_ADDR = 0x00002016, + * EOI_EXIT_BITMAP0 = 0x0000201c, + * EOI_EXIT_BITMAP1 = 0x0000201e, + * EOI_EXIT_BITMAP2 = 0x00002020, + * EOI_EXIT_BITMAP3 = 0x00002022, + * GUEST_PML_INDEX = 0x00000812, + * PML_ADDRESS = 0x0000200e, + * VM_FUNCTION_CONTROL = 0x00002018, + * EPTP_LIST_ADDRESS = 0x00002024, + * VMREAD_BITMAP = 0x00002026, + * VMWRITE_BITMAP = 0x00002028, + * + * TSC_MULTIPLIER = 0x00002032, + * PLE_GAP = 0x00004020, + * PLE_WINDOW = 0x00004022, + * VMX_PREEMPTION_TIMER_VALUE = 0x0000482E, + * + * Currently unsupported in KVM: + * GUEST_IA32_RTIT_CTL = 0x00002814, + */ +#define EVMCS1_SUPPORTED_PINCTRL \ + (PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR | \ + PIN_BASED_EXT_INTR_MASK | \ + PIN_BASED_NMI_EXITING | \ + PIN_BASED_VIRTUAL_NMIS) + +#define EVMCS1_SUPPORTED_EXEC_CTRL \ + (CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR | \ + CPU_BASED_HLT_EXITING | \ + CPU_BASED_CR3_LOAD_EXITING | \ + CPU_BASED_CR3_STORE_EXITING | \ + CPU_BASED_UNCOND_IO_EXITING | \ + CPU_BASED_MOV_DR_EXITING | \ + CPU_BASED_USE_TSC_OFFSETTING | \ + CPU_BASED_MWAIT_EXITING | \ + CPU_BASED_MONITOR_EXITING | \ + CPU_BASED_INVLPG_EXITING | \ + CPU_BASED_RDPMC_EXITING | \ + CPU_BASED_INTR_WINDOW_EXITING | \ + CPU_BASED_CR8_LOAD_EXITING | \ + CPU_BASED_CR8_STORE_EXITING | \ + CPU_BASED_RDTSC_EXITING | \ + CPU_BASED_TPR_SHADOW | \ + CPU_BASED_USE_IO_BITMAPS | \ + CPU_BASED_MONITOR_TRAP_FLAG | \ + CPU_BASED_USE_MSR_BITMAPS | \ + CPU_BASED_NMI_WINDOW_EXITING | \ + CPU_BASED_PAUSE_EXITING | \ + CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) + +#define EVMCS1_SUPPORTED_2NDEXEC \ + (SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | \ + SECONDARY_EXEC_WBINVD_EXITING | \ + SECONDARY_EXEC_ENABLE_VPID | \ + SECONDARY_EXEC_ENABLE_EPT | \ + SECONDARY_EXEC_UNRESTRICTED_GUEST | \ + SECONDARY_EXEC_DESC | \ + SECONDARY_EXEC_ENABLE_RDTSCP | \ + SECONDARY_EXEC_ENABLE_INVPCID | \ + SECONDARY_EXEC_ENABLE_XSAVES | \ + SECONDARY_EXEC_RDSEED_EXITING | \ + SECONDARY_EXEC_RDRAND_EXITING | \ + SECONDARY_EXEC_TSC_SCALING | \ + SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE | \ + SECONDARY_EXEC_PT_USE_GPA | \ + SECONDARY_EXEC_PT_CONCEAL_VMX | \ + SECONDARY_EXEC_BUS_LOCK_DETECTION | \ + SECONDARY_EXEC_NOTIFY_VM_EXITING | \ + SECONDARY_EXEC_ENCLS_EXITING) + +#define EVMCS1_SUPPORTED_3RDEXEC (0ULL) + +#define EVMCS1_SUPPORTED_VMEXIT_CTRL \ + (VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR | \ + VM_EXIT_SAVE_DEBUG_CONTROLS | \ + VM_EXIT_ACK_INTR_ON_EXIT | \ + VM_EXIT_HOST_ADDR_SPACE_SIZE | \ + VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | \ + VM_EXIT_SAVE_IA32_PAT | \ + VM_EXIT_LOAD_IA32_PAT | \ + VM_EXIT_SAVE_IA32_EFER | \ + VM_EXIT_LOAD_IA32_EFER | \ + VM_EXIT_CLEAR_BNDCFGS | \ + VM_EXIT_PT_CONCEAL_PIP | \ + VM_EXIT_CLEAR_IA32_RTIT_CTL) + +#define EVMCS1_SUPPORTED_VMENTRY_CTRL \ + (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | \ + VM_ENTRY_LOAD_DEBUG_CONTROLS | \ + VM_ENTRY_IA32E_MODE | \ + VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL | \ + VM_ENTRY_LOAD_IA32_PAT | \ + VM_ENTRY_LOAD_IA32_EFER | \ + VM_ENTRY_LOAD_BNDCFGS | \ + VM_ENTRY_PT_CONCEAL_PIP | \ + VM_ENTRY_LOAD_IA32_RTIT_CTL) + +#define EVMCS1_SUPPORTED_VMFUNC (0) + struct evmcs_field { u16 offset; u16 clean_field; @@ -65,114 +166,6 @@ static inline u64 evmcs_read_any(struct hv_enlightened_vmcs *evmcs, return vmcs12_read_any((void *)evmcs, field, offset); } -#if IS_ENABLED(CONFIG_HYPERV) - -DECLARE_STATIC_KEY_FALSE(__kvm_is_using_evmcs); - -static __always_inline bool kvm_is_using_evmcs(void) -{ - return static_branch_unlikely(&__kvm_is_using_evmcs); -} - -static __always_inline int get_evmcs_offset(unsigned long field, - u16 *clean_field) -{ - int offset = evmcs_field_offset(field, clean_field); - - WARN_ONCE(offset < 0, "accessing unsupported EVMCS field %lx\n", field); - return offset; -} - -static __always_inline void evmcs_write64(unsigned long field, u64 value) -{ - u16 clean_field; - int offset = get_evmcs_offset(field, &clean_field); - - if (offset < 0) - return; - - *(u64 *)((char *)current_evmcs + offset) = value; - - current_evmcs->hv_clean_fields &= ~clean_field; -} - -static __always_inline void evmcs_write32(unsigned long field, u32 value) -{ - u16 clean_field; - int offset = get_evmcs_offset(field, &clean_field); - - if (offset < 0) - return; - - *(u32 *)((char *)current_evmcs + offset) = value; - current_evmcs->hv_clean_fields &= ~clean_field; -} - -static __always_inline void evmcs_write16(unsigned long field, u16 value) -{ - u16 clean_field; - int offset = get_evmcs_offset(field, &clean_field); - - if (offset < 0) - return; - - *(u16 *)((char *)current_evmcs + offset) = value; - current_evmcs->hv_clean_fields &= ~clean_field; -} - -static __always_inline u64 evmcs_read64(unsigned long field) -{ - int offset = get_evmcs_offset(field, NULL); - - if (offset < 0) - return 0; - - return *(u64 *)((char *)current_evmcs + offset); -} - -static __always_inline u32 evmcs_read32(unsigned long field) -{ - int offset = get_evmcs_offset(field, NULL); - - if (offset < 0) - return 0; - - return *(u32 *)((char *)current_evmcs + offset); -} - -static __always_inline u16 evmcs_read16(unsigned long field) -{ - int offset = get_evmcs_offset(field, NULL); - - if (offset < 0) - return 0; - - return *(u16 *)((char *)current_evmcs + offset); -} - -static inline void evmcs_load(u64 phys_addr) -{ - struct hv_vp_assist_page *vp_ap = - hv_get_vp_assist_page(smp_processor_id()); - - if (current_evmcs->hv_enlightenments_control.nested_flush_hypercall) - vp_ap->nested_control.features.directhypercall = 1; - vp_ap->current_nested_vmcs = phys_addr; - vp_ap->enlighten_vmentry = 1; -} - -void evmcs_sanitize_exec_ctrls(struct vmcs_config *vmcs_conf); -#else /* !IS_ENABLED(CONFIG_HYPERV) */ -static __always_inline bool kvm_is_using_evmcs(void) { return false; } -static __always_inline void evmcs_write64(unsigned long field, u64 value) {} -static __always_inline void evmcs_write32(unsigned long field, u32 value) {} -static __always_inline void evmcs_write16(unsigned long field, u16 value) {} -static __always_inline u64 evmcs_read64(unsigned long field) { return 0; } -static __always_inline u32 evmcs_read32(unsigned long field) { return 0; } -static __always_inline u16 evmcs_read16(unsigned long field) { return 0; } -static inline void evmcs_load(u64 phys_addr) {} -#endif /* IS_ENABLED(CONFIG_HYPERV) */ - #define EVMPTR_INVALID (-1ULL) #define EVMPTR_MAP_PENDING (-2ULL) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index cf19a3346639..552593a2ac14 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -66,6 +66,7 @@ #include "vmx.h" #include "x86.h" #include "smm.h" +#include "vmx_onhyperv.h" MODULE_AUTHOR("Qumranet"); MODULE_LICENSE("GPL"); diff --git a/arch/x86/kvm/vmx/vmx_onhyperv.c b/arch/x86/kvm/vmx/vmx_onhyperv.c new file mode 100644 index 000000000000..b9a8b91166d0 --- /dev/null +++ b/arch/x86/kvm/vmx/vmx_onhyperv.c @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include "capabilities.h" +#include "vmx_onhyperv.h" + +DEFINE_STATIC_KEY_FALSE(__kvm_is_using_evmcs); + +/* + * KVM on Hyper-V always uses the latest known eVMCSv1 revision, the assumption + * is: in case a feature has corresponding fields in eVMCS described and it was + * exposed in VMX feature MSRs, KVM is free to use it. Warn if KVM meets a + * feature which has no corresponding eVMCS field, this likely means that KVM + * needs to be updated. + */ +#define evmcs_check_vmcs_conf(field, ctrl) \ + do { \ + typeof(vmcs_conf->field) unsupported; \ + \ + unsupported = vmcs_conf->field & ~EVMCS1_SUPPORTED_ ## ctrl; \ + if (unsupported) { \ + pr_warn_once(#field " unsupported with eVMCS: 0x%llx\n",\ + (u64)unsupported); \ + vmcs_conf->field &= EVMCS1_SUPPORTED_ ## ctrl; \ + } \ + } \ + while (0) + +void evmcs_sanitize_exec_ctrls(struct vmcs_config *vmcs_conf) +{ + evmcs_check_vmcs_conf(cpu_based_exec_ctrl, EXEC_CTRL); + evmcs_check_vmcs_conf(pin_based_exec_ctrl, PINCTRL); + evmcs_check_vmcs_conf(cpu_based_2nd_exec_ctrl, 2NDEXEC); + evmcs_check_vmcs_conf(cpu_based_3rd_exec_ctrl, 3RDEXEC); + evmcs_check_vmcs_conf(vmentry_ctrl, VMENTRY_CTRL); + evmcs_check_vmcs_conf(vmexit_ctrl, VMEXIT_CTRL); +} diff --git a/arch/x86/kvm/vmx/vmx_onhyperv.h b/arch/x86/kvm/vmx/vmx_onhyperv.h new file mode 100644 index 000000000000..11541d272dbd --- /dev/null +++ b/arch/x86/kvm/vmx/vmx_onhyperv.h @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ARCH_X86_KVM_VMX_ONHYPERV_H__ +#define __ARCH_X86_KVM_VMX_ONHYPERV_H__ + +#include + +#include + +#include "capabilities.h" +#include "hyperv.h" +#include "vmcs12.h" + +#define current_evmcs ((struct hv_enlightened_vmcs *)this_cpu_read(current_vmcs)) + +#if IS_ENABLED(CONFIG_HYPERV) + +DECLARE_STATIC_KEY_FALSE(__kvm_is_using_evmcs); + +static __always_inline bool kvm_is_using_evmcs(void) +{ + return static_branch_unlikely(&__kvm_is_using_evmcs); +} + +static __always_inline int get_evmcs_offset(unsigned long field, + u16 *clean_field) +{ + int offset = evmcs_field_offset(field, clean_field); + + WARN_ONCE(offset < 0, "accessing unsupported EVMCS field %lx\n", field); + return offset; +} + +static __always_inline void evmcs_write64(unsigned long field, u64 value) +{ + u16 clean_field; + int offset = get_evmcs_offset(field, &clean_field); + + if (offset < 0) + return; + + *(u64 *)((char *)current_evmcs + offset) = value; + + current_evmcs->hv_clean_fields &= ~clean_field; +} + +static __always_inline void evmcs_write32(unsigned long field, u32 value) +{ + u16 clean_field; + int offset = get_evmcs_offset(field, &clean_field); + + if (offset < 0) + return; + + *(u32 *)((char *)current_evmcs + offset) = value; + current_evmcs->hv_clean_fields &= ~clean_field; +} + +static __always_inline void evmcs_write16(unsigned long field, u16 value) +{ + u16 clean_field; + int offset = get_evmcs_offset(field, &clean_field); + + if (offset < 0) + return; + + *(u16 *)((char *)current_evmcs + offset) = value; + current_evmcs->hv_clean_fields &= ~clean_field; +} + +static __always_inline u64 evmcs_read64(unsigned long field) +{ + int offset = get_evmcs_offset(field, NULL); + + if (offset < 0) + return 0; + + return *(u64 *)((char *)current_evmcs + offset); +} + +static __always_inline u32 evmcs_read32(unsigned long field) +{ + int offset = get_evmcs_offset(field, NULL); + + if (offset < 0) + return 0; + + return *(u32 *)((char *)current_evmcs + offset); +} + +static __always_inline u16 evmcs_read16(unsigned long field) +{ + int offset = get_evmcs_offset(field, NULL); + + if (offset < 0) + return 0; + + return *(u16 *)((char *)current_evmcs + offset); +} + +static inline void evmcs_load(u64 phys_addr) +{ + struct hv_vp_assist_page *vp_ap = + hv_get_vp_assist_page(smp_processor_id()); + + if (current_evmcs->hv_enlightenments_control.nested_flush_hypercall) + vp_ap->nested_control.features.directhypercall = 1; + vp_ap->current_nested_vmcs = phys_addr; + vp_ap->enlighten_vmentry = 1; +} + +void evmcs_sanitize_exec_ctrls(struct vmcs_config *vmcs_conf); +#else /* !IS_ENABLED(CONFIG_HYPERV) */ +static __always_inline bool kvm_is_using_evmcs(void) { return false; } +static __always_inline void evmcs_write64(unsigned long field, u64 value) {} +static __always_inline void evmcs_write32(unsigned long field, u32 value) {} +static __always_inline void evmcs_write16(unsigned long field, u16 value) {} +static __always_inline u64 evmcs_read64(unsigned long field) { return 0; } +static __always_inline u32 evmcs_read32(unsigned long field) { return 0; } +static __always_inline u16 evmcs_read16(unsigned long field) { return 0; } +static inline void evmcs_load(u64 phys_addr) {} +#endif /* IS_ENABLED(CONFIG_HYPERV) */ + +#endif /* __ARCH_X86_KVM_VMX_ONHYPERV_H__ */ diff --git a/arch/x86/kvm/vmx/vmx_ops.h b/arch/x86/kvm/vmx/vmx_ops.h index 33af7b4c6eb4..f41ce3c24123 100644 --- a/arch/x86/kvm/vmx/vmx_ops.h +++ b/arch/x86/kvm/vmx/vmx_ops.h @@ -6,7 +6,7 @@ #include -#include "hyperv.h" +#include "vmx_onhyperv.h" #include "vmcs.h" #include "../x86.h" From 16e880bfa6377871da233c846ecdf23db2bf1d97 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Tue, 5 Dec 2023 11:36:18 +0100 Subject: [PATCH 111/336] KVM: x86: Introduce helper to check if auto-EOI is set in Hyper-V SynIC As a preparation to making Hyper-V emulation optional, create a dedicated kvm_hv_synic_auto_eoi_set() helper to avoid extra ifdefs in lapic.c No functional change intended. Reviewed-by: Maxim Levitsky Tested-by: Jeremi Piotrowski Signed-off-by: Vitaly Kuznetsov Link: https://lore.kernel.org/r/20231205103630.1391318-5-vkuznets@redhat.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/hyperv.h | 6 ++++++ arch/x86/kvm/lapic.c | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/hyperv.h b/arch/x86/kvm/hyperv.h index f83b8db72b11..286956fe1a2e 100644 --- a/arch/x86/kvm/hyperv.h +++ b/arch/x86/kvm/hyperv.h @@ -105,6 +105,12 @@ int kvm_hv_synic_set_irq(struct kvm *kvm, u32 vcpu_id, u32 sint); void kvm_hv_synic_send_eoi(struct kvm_vcpu *vcpu, int vector); int kvm_hv_activate_synic(struct kvm_vcpu *vcpu, bool dont_zero_synic_pages); +static inline bool kvm_hv_synic_auto_eoi_set(struct kvm_vcpu *vcpu, int vector) +{ + return to_hv_vcpu(vcpu) && + test_bit(vector, to_hv_synic(vcpu)->auto_eoi_bitmap); +} + void kvm_hv_vcpu_uninit(struct kvm_vcpu *vcpu); bool kvm_hv_assist_page_enabled(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 245b20973cae..f7abc1008cad 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -2905,7 +2905,7 @@ int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu) */ apic_clear_irr(vector, apic); - if (to_hv_vcpu(vcpu) && test_bit(vector, to_hv_synic(vcpu)->auto_eoi_bitmap)) { + if (kvm_hv_synic_auto_eoi_set(vcpu, vector)) { /* * For auto-EOI interrupts, there might be another pending * interrupt above PPR, so check whether to raise another From 0659262a2625ca3e4061796cd4f4935091220056 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Tue, 5 Dec 2023 11:36:19 +0100 Subject: [PATCH 112/336] KVM: x86: Introduce helper to check if vector is set in Hyper-V SynIC As a preparation to making Hyper-V emulation optional, create a dedicated kvm_hv_synic_has_vector() helper to avoid extra ifdefs in lapic.c. No functional change intended. Reviewed-by: Maxim Levitsky Tested-by: Jeremi Piotrowski Signed-off-by: Vitaly Kuznetsov Link: https://lore.kernel.org/r/20231205103630.1391318-6-vkuznets@redhat.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/hyperv.h | 5 +++++ arch/x86/kvm/lapic.c | 3 +-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/hyperv.h b/arch/x86/kvm/hyperv.h index 286956fe1a2e..9d8fa6ba6341 100644 --- a/arch/x86/kvm/hyperv.h +++ b/arch/x86/kvm/hyperv.h @@ -105,6 +105,11 @@ int kvm_hv_synic_set_irq(struct kvm *kvm, u32 vcpu_id, u32 sint); void kvm_hv_synic_send_eoi(struct kvm_vcpu *vcpu, int vector); int kvm_hv_activate_synic(struct kvm_vcpu *vcpu, bool dont_zero_synic_pages); +static inline bool kvm_hv_synic_has_vector(struct kvm_vcpu *vcpu, int vector) +{ + return to_hv_vcpu(vcpu) && test_bit(vector, to_hv_synic(vcpu)->vec_bitmap); +} + static inline bool kvm_hv_synic_auto_eoi_set(struct kvm_vcpu *vcpu, int vector) { return to_hv_vcpu(vcpu) && diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index f7abc1008cad..3242f3da2457 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -1475,8 +1475,7 @@ static int apic_set_eoi(struct kvm_lapic *apic) apic_clear_isr(vector, apic); apic_update_ppr(apic); - if (to_hv_vcpu(apic->vcpu) && - test_bit(vector, to_hv_synic(apic->vcpu)->vec_bitmap)) + if (kvm_hv_synic_has_vector(apic->vcpu, vector)) kvm_hv_synic_send_eoi(apic->vcpu, vector); kvm_ioapic_send_eoi(apic, vector); From e7ad84db4d718e18c7a133e941ba4c7d4c6d4cbf Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Tue, 5 Dec 2023 11:36:20 +0100 Subject: [PATCH 113/336] KVM: VMX: Split off hyperv_evmcs.{ch} Some Enlightened VMCS related code is needed both by Hyper-V on KVM and KVM on Hyper-V. As a preparation to making Hyper-V emulation optional, create dedicated 'hyperv_evmcs.{ch}' files which are used by both. No functional change intended. Reviewed-by: Maxim Levitsky Tested-by: Jeremi Piotrowski Signed-off-by: Vitaly Kuznetsov Link: https://lore.kernel.org/r/20231205103630.1391318-7-vkuznets@redhat.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/Makefile | 2 +- arch/x86/kvm/vmx/hyperv.c | 308 ------------------------------- arch/x86/kvm/vmx/hyperv.h | 163 +---------------- arch/x86/kvm/vmx/hyperv_evmcs.c | 315 ++++++++++++++++++++++++++++++++ arch/x86/kvm/vmx/hyperv_evmcs.h | 166 +++++++++++++++++ arch/x86/kvm/vmx/nested.h | 1 + arch/x86/kvm/vmx/vmx_onhyperv.h | 3 +- 7 files changed, 486 insertions(+), 472 deletions(-) create mode 100644 arch/x86/kvm/vmx/hyperv_evmcs.c create mode 100644 arch/x86/kvm/vmx/hyperv_evmcs.h diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index a99ffc3f3a3f..8ea872401cd6 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -23,7 +23,7 @@ kvm-$(CONFIG_KVM_XEN) += xen.o kvm-$(CONFIG_KVM_SMM) += smm.o kvm-intel-y += vmx/vmx.o vmx/vmenter.o vmx/pmu_intel.o vmx/vmcs12.o \ - vmx/hyperv.o vmx/nested.o vmx/posted_intr.o + vmx/hyperv.o vmx/hyperv_evmcs.o vmx/nested.o vmx/posted_intr.o kvm-intel-$(CONFIG_X86_SGX_KVM) += vmx/sgx.o ifdef CONFIG_HYPERV diff --git a/arch/x86/kvm/vmx/hyperv.c b/arch/x86/kvm/vmx/hyperv.c index de13dc14fe1d..fab6a1ad98dc 100644 --- a/arch/x86/kvm/vmx/hyperv.c +++ b/arch/x86/kvm/vmx/hyperv.c @@ -13,314 +13,6 @@ #define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK -#define EVMCS1_OFFSET(x) offsetof(struct hv_enlightened_vmcs, x) -#define EVMCS1_FIELD(number, name, clean_field)[ROL16(number, 6)] = \ - {EVMCS1_OFFSET(name), clean_field} - -const struct evmcs_field vmcs_field_to_evmcs_1[] = { - /* 64 bit rw */ - EVMCS1_FIELD(GUEST_RIP, guest_rip, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), - EVMCS1_FIELD(GUEST_RSP, guest_rsp, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC), - EVMCS1_FIELD(GUEST_RFLAGS, guest_rflags, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC), - EVMCS1_FIELD(HOST_IA32_PAT, host_ia32_pat, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), - EVMCS1_FIELD(HOST_IA32_EFER, host_ia32_efer, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), - EVMCS1_FIELD(HOST_IA32_PERF_GLOBAL_CTRL, host_ia32_perf_global_ctrl, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), - EVMCS1_FIELD(HOST_CR0, host_cr0, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), - EVMCS1_FIELD(HOST_CR3, host_cr3, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), - EVMCS1_FIELD(HOST_CR4, host_cr4, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), - EVMCS1_FIELD(HOST_IA32_SYSENTER_ESP, host_ia32_sysenter_esp, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), - EVMCS1_FIELD(HOST_IA32_SYSENTER_EIP, host_ia32_sysenter_eip, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), - EVMCS1_FIELD(HOST_RIP, host_rip, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), - EVMCS1_FIELD(IO_BITMAP_A, io_bitmap_a, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP), - EVMCS1_FIELD(IO_BITMAP_B, io_bitmap_b, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP), - EVMCS1_FIELD(MSR_BITMAP, msr_bitmap, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP), - EVMCS1_FIELD(GUEST_ES_BASE, guest_es_base, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_CS_BASE, guest_cs_base, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_SS_BASE, guest_ss_base, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_DS_BASE, guest_ds_base, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_FS_BASE, guest_fs_base, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_GS_BASE, guest_gs_base, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_LDTR_BASE, guest_ldtr_base, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_TR_BASE, guest_tr_base, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_GDTR_BASE, guest_gdtr_base, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_IDTR_BASE, guest_idtr_base, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(TSC_OFFSET, tsc_offset, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2), - EVMCS1_FIELD(VIRTUAL_APIC_PAGE_ADDR, virtual_apic_page_addr, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2), - EVMCS1_FIELD(VMCS_LINK_POINTER, vmcs_link_pointer, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), - EVMCS1_FIELD(GUEST_IA32_DEBUGCTL, guest_ia32_debugctl, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), - EVMCS1_FIELD(GUEST_IA32_PAT, guest_ia32_pat, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), - EVMCS1_FIELD(GUEST_IA32_EFER, guest_ia32_efer, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), - EVMCS1_FIELD(GUEST_IA32_PERF_GLOBAL_CTRL, guest_ia32_perf_global_ctrl, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), - EVMCS1_FIELD(GUEST_PDPTR0, guest_pdptr0, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), - EVMCS1_FIELD(GUEST_PDPTR1, guest_pdptr1, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), - EVMCS1_FIELD(GUEST_PDPTR2, guest_pdptr2, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), - EVMCS1_FIELD(GUEST_PDPTR3, guest_pdptr3, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), - EVMCS1_FIELD(GUEST_PENDING_DBG_EXCEPTIONS, guest_pending_dbg_exceptions, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), - EVMCS1_FIELD(GUEST_SYSENTER_ESP, guest_sysenter_esp, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), - EVMCS1_FIELD(GUEST_SYSENTER_EIP, guest_sysenter_eip, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), - EVMCS1_FIELD(CR0_GUEST_HOST_MASK, cr0_guest_host_mask, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), - EVMCS1_FIELD(CR4_GUEST_HOST_MASK, cr4_guest_host_mask, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), - EVMCS1_FIELD(CR0_READ_SHADOW, cr0_read_shadow, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), - EVMCS1_FIELD(CR4_READ_SHADOW, cr4_read_shadow, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), - EVMCS1_FIELD(GUEST_CR0, guest_cr0, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), - EVMCS1_FIELD(GUEST_CR3, guest_cr3, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), - EVMCS1_FIELD(GUEST_CR4, guest_cr4, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), - EVMCS1_FIELD(GUEST_DR7, guest_dr7, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), - EVMCS1_FIELD(HOST_FS_BASE, host_fs_base, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER), - EVMCS1_FIELD(HOST_GS_BASE, host_gs_base, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER), - EVMCS1_FIELD(HOST_TR_BASE, host_tr_base, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER), - EVMCS1_FIELD(HOST_GDTR_BASE, host_gdtr_base, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER), - EVMCS1_FIELD(HOST_IDTR_BASE, host_idtr_base, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER), - EVMCS1_FIELD(HOST_RSP, host_rsp, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER), - EVMCS1_FIELD(EPT_POINTER, ept_pointer, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT), - EVMCS1_FIELD(GUEST_BNDCFGS, guest_bndcfgs, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), - EVMCS1_FIELD(XSS_EXIT_BITMAP, xss_exit_bitmap, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2), - EVMCS1_FIELD(ENCLS_EXITING_BITMAP, encls_exiting_bitmap, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2), - EVMCS1_FIELD(TSC_MULTIPLIER, tsc_multiplier, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2), - /* - * Not used by KVM: - * - * EVMCS1_FIELD(0x00006828, guest_ia32_s_cet, - * HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), - * EVMCS1_FIELD(0x0000682A, guest_ssp, - * HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC), - * EVMCS1_FIELD(0x0000682C, guest_ia32_int_ssp_table_addr, - * HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), - * EVMCS1_FIELD(0x00002816, guest_ia32_lbr_ctl, - * HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), - * EVMCS1_FIELD(0x00006C18, host_ia32_s_cet, - * HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), - * EVMCS1_FIELD(0x00006C1A, host_ssp, - * HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), - * EVMCS1_FIELD(0x00006C1C, host_ia32_int_ssp_table_addr, - * HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), - */ - - /* 64 bit read only */ - EVMCS1_FIELD(GUEST_PHYSICAL_ADDRESS, guest_physical_address, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), - EVMCS1_FIELD(EXIT_QUALIFICATION, exit_qualification, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), - /* - * Not defined in KVM: - * - * EVMCS1_FIELD(0x00006402, exit_io_instruction_ecx, - * HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE); - * EVMCS1_FIELD(0x00006404, exit_io_instruction_esi, - * HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE); - * EVMCS1_FIELD(0x00006406, exit_io_instruction_esi, - * HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE); - * EVMCS1_FIELD(0x00006408, exit_io_instruction_eip, - * HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE); - */ - EVMCS1_FIELD(GUEST_LINEAR_ADDRESS, guest_linear_address, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), - - /* - * No mask defined in the spec as Hyper-V doesn't currently support - * these. Future proof by resetting the whole clean field mask on - * access. - */ - EVMCS1_FIELD(VM_EXIT_MSR_STORE_ADDR, vm_exit_msr_store_addr, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), - EVMCS1_FIELD(VM_EXIT_MSR_LOAD_ADDR, vm_exit_msr_load_addr, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), - EVMCS1_FIELD(VM_ENTRY_MSR_LOAD_ADDR, vm_entry_msr_load_addr, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), - - /* 32 bit rw */ - EVMCS1_FIELD(TPR_THRESHOLD, tpr_threshold, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), - EVMCS1_FIELD(GUEST_INTERRUPTIBILITY_INFO, guest_interruptibility_info, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC), - EVMCS1_FIELD(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC), - EVMCS1_FIELD(EXCEPTION_BITMAP, exception_bitmap, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EXCPN), - EVMCS1_FIELD(VM_ENTRY_CONTROLS, vm_entry_controls, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY), - EVMCS1_FIELD(VM_ENTRY_INTR_INFO_FIELD, vm_entry_intr_info_field, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT), - EVMCS1_FIELD(VM_ENTRY_EXCEPTION_ERROR_CODE, - vm_entry_exception_error_code, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT), - EVMCS1_FIELD(VM_ENTRY_INSTRUCTION_LEN, vm_entry_instruction_len, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT), - EVMCS1_FIELD(HOST_IA32_SYSENTER_CS, host_ia32_sysenter_cs, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), - EVMCS1_FIELD(PIN_BASED_VM_EXEC_CONTROL, pin_based_vm_exec_control, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1), - EVMCS1_FIELD(VM_EXIT_CONTROLS, vm_exit_controls, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1), - EVMCS1_FIELD(SECONDARY_VM_EXEC_CONTROL, secondary_vm_exec_control, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1), - EVMCS1_FIELD(GUEST_ES_LIMIT, guest_es_limit, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_CS_LIMIT, guest_cs_limit, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_SS_LIMIT, guest_ss_limit, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_DS_LIMIT, guest_ds_limit, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_FS_LIMIT, guest_fs_limit, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_GS_LIMIT, guest_gs_limit, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_LDTR_LIMIT, guest_ldtr_limit, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_TR_LIMIT, guest_tr_limit, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_GDTR_LIMIT, guest_gdtr_limit, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_IDTR_LIMIT, guest_idtr_limit, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_ES_AR_BYTES, guest_es_ar_bytes, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_CS_AR_BYTES, guest_cs_ar_bytes, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_SS_AR_BYTES, guest_ss_ar_bytes, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_DS_AR_BYTES, guest_ds_ar_bytes, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_FS_AR_BYTES, guest_fs_ar_bytes, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_GS_AR_BYTES, guest_gs_ar_bytes, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_LDTR_AR_BYTES, guest_ldtr_ar_bytes, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_TR_AR_BYTES, guest_tr_ar_bytes, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_ACTIVITY_STATE, guest_activity_state, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), - EVMCS1_FIELD(GUEST_SYSENTER_CS, guest_sysenter_cs, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), - - /* 32 bit read only */ - EVMCS1_FIELD(VM_INSTRUCTION_ERROR, vm_instruction_error, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), - EVMCS1_FIELD(VM_EXIT_REASON, vm_exit_reason, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), - EVMCS1_FIELD(VM_EXIT_INTR_INFO, vm_exit_intr_info, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), - EVMCS1_FIELD(VM_EXIT_INTR_ERROR_CODE, vm_exit_intr_error_code, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), - EVMCS1_FIELD(IDT_VECTORING_INFO_FIELD, idt_vectoring_info_field, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), - EVMCS1_FIELD(IDT_VECTORING_ERROR_CODE, idt_vectoring_error_code, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), - EVMCS1_FIELD(VM_EXIT_INSTRUCTION_LEN, vm_exit_instruction_len, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), - EVMCS1_FIELD(VMX_INSTRUCTION_INFO, vmx_instruction_info, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), - - /* No mask defined in the spec (not used) */ - EVMCS1_FIELD(PAGE_FAULT_ERROR_CODE_MASK, page_fault_error_code_mask, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), - EVMCS1_FIELD(PAGE_FAULT_ERROR_CODE_MATCH, page_fault_error_code_match, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), - EVMCS1_FIELD(CR3_TARGET_COUNT, cr3_target_count, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), - EVMCS1_FIELD(VM_EXIT_MSR_STORE_COUNT, vm_exit_msr_store_count, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), - EVMCS1_FIELD(VM_EXIT_MSR_LOAD_COUNT, vm_exit_msr_load_count, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), - EVMCS1_FIELD(VM_ENTRY_MSR_LOAD_COUNT, vm_entry_msr_load_count, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), - - /* 16 bit rw */ - EVMCS1_FIELD(HOST_ES_SELECTOR, host_es_selector, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), - EVMCS1_FIELD(HOST_CS_SELECTOR, host_cs_selector, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), - EVMCS1_FIELD(HOST_SS_SELECTOR, host_ss_selector, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), - EVMCS1_FIELD(HOST_DS_SELECTOR, host_ds_selector, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), - EVMCS1_FIELD(HOST_FS_SELECTOR, host_fs_selector, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), - EVMCS1_FIELD(HOST_GS_SELECTOR, host_gs_selector, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), - EVMCS1_FIELD(HOST_TR_SELECTOR, host_tr_selector, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), - EVMCS1_FIELD(GUEST_ES_SELECTOR, guest_es_selector, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_CS_SELECTOR, guest_cs_selector, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_SS_SELECTOR, guest_ss_selector, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_DS_SELECTOR, guest_ds_selector, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_FS_SELECTOR, guest_fs_selector, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_GS_SELECTOR, guest_gs_selector, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_LDTR_SELECTOR, guest_ldtr_selector, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(GUEST_TR_SELECTOR, guest_tr_selector, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), - EVMCS1_FIELD(VIRTUAL_PROCESSOR_ID, virtual_processor_id, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT), -}; -const unsigned int nr_evmcs_1_fields = ARRAY_SIZE(vmcs_field_to_evmcs_1); - u64 nested_get_evmptr(struct kvm_vcpu *vcpu) { struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); diff --git a/arch/x86/kvm/vmx/hyperv.h b/arch/x86/kvm/vmx/hyperv.h index 9401dbfaea7c..d4ed99008518 100644 --- a/arch/x86/kvm/vmx/hyperv.h +++ b/arch/x86/kvm/vmx/hyperv.h @@ -2,170 +2,9 @@ #ifndef __KVM_X86_VMX_HYPERV_H #define __KVM_X86_VMX_HYPERV_H -#include - -#include -#include -#include - -#include "../hyperv.h" - -#include "capabilities.h" -#include "vmcs.h" +#include #include "vmcs12.h" -#define KVM_EVMCS_VERSION 1 - -/* - * Enlightened VMCSv1 doesn't support these: - * - * POSTED_INTR_NV = 0x00000002, - * GUEST_INTR_STATUS = 0x00000810, - * APIC_ACCESS_ADDR = 0x00002014, - * POSTED_INTR_DESC_ADDR = 0x00002016, - * EOI_EXIT_BITMAP0 = 0x0000201c, - * EOI_EXIT_BITMAP1 = 0x0000201e, - * EOI_EXIT_BITMAP2 = 0x00002020, - * EOI_EXIT_BITMAP3 = 0x00002022, - * GUEST_PML_INDEX = 0x00000812, - * PML_ADDRESS = 0x0000200e, - * VM_FUNCTION_CONTROL = 0x00002018, - * EPTP_LIST_ADDRESS = 0x00002024, - * VMREAD_BITMAP = 0x00002026, - * VMWRITE_BITMAP = 0x00002028, - * - * TSC_MULTIPLIER = 0x00002032, - * PLE_GAP = 0x00004020, - * PLE_WINDOW = 0x00004022, - * VMX_PREEMPTION_TIMER_VALUE = 0x0000482E, - * - * Currently unsupported in KVM: - * GUEST_IA32_RTIT_CTL = 0x00002814, - */ -#define EVMCS1_SUPPORTED_PINCTRL \ - (PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR | \ - PIN_BASED_EXT_INTR_MASK | \ - PIN_BASED_NMI_EXITING | \ - PIN_BASED_VIRTUAL_NMIS) - -#define EVMCS1_SUPPORTED_EXEC_CTRL \ - (CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR | \ - CPU_BASED_HLT_EXITING | \ - CPU_BASED_CR3_LOAD_EXITING | \ - CPU_BASED_CR3_STORE_EXITING | \ - CPU_BASED_UNCOND_IO_EXITING | \ - CPU_BASED_MOV_DR_EXITING | \ - CPU_BASED_USE_TSC_OFFSETTING | \ - CPU_BASED_MWAIT_EXITING | \ - CPU_BASED_MONITOR_EXITING | \ - CPU_BASED_INVLPG_EXITING | \ - CPU_BASED_RDPMC_EXITING | \ - CPU_BASED_INTR_WINDOW_EXITING | \ - CPU_BASED_CR8_LOAD_EXITING | \ - CPU_BASED_CR8_STORE_EXITING | \ - CPU_BASED_RDTSC_EXITING | \ - CPU_BASED_TPR_SHADOW | \ - CPU_BASED_USE_IO_BITMAPS | \ - CPU_BASED_MONITOR_TRAP_FLAG | \ - CPU_BASED_USE_MSR_BITMAPS | \ - CPU_BASED_NMI_WINDOW_EXITING | \ - CPU_BASED_PAUSE_EXITING | \ - CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) - -#define EVMCS1_SUPPORTED_2NDEXEC \ - (SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | \ - SECONDARY_EXEC_WBINVD_EXITING | \ - SECONDARY_EXEC_ENABLE_VPID | \ - SECONDARY_EXEC_ENABLE_EPT | \ - SECONDARY_EXEC_UNRESTRICTED_GUEST | \ - SECONDARY_EXEC_DESC | \ - SECONDARY_EXEC_ENABLE_RDTSCP | \ - SECONDARY_EXEC_ENABLE_INVPCID | \ - SECONDARY_EXEC_ENABLE_XSAVES | \ - SECONDARY_EXEC_RDSEED_EXITING | \ - SECONDARY_EXEC_RDRAND_EXITING | \ - SECONDARY_EXEC_TSC_SCALING | \ - SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE | \ - SECONDARY_EXEC_PT_USE_GPA | \ - SECONDARY_EXEC_PT_CONCEAL_VMX | \ - SECONDARY_EXEC_BUS_LOCK_DETECTION | \ - SECONDARY_EXEC_NOTIFY_VM_EXITING | \ - SECONDARY_EXEC_ENCLS_EXITING) - -#define EVMCS1_SUPPORTED_3RDEXEC (0ULL) - -#define EVMCS1_SUPPORTED_VMEXIT_CTRL \ - (VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR | \ - VM_EXIT_SAVE_DEBUG_CONTROLS | \ - VM_EXIT_ACK_INTR_ON_EXIT | \ - VM_EXIT_HOST_ADDR_SPACE_SIZE | \ - VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | \ - VM_EXIT_SAVE_IA32_PAT | \ - VM_EXIT_LOAD_IA32_PAT | \ - VM_EXIT_SAVE_IA32_EFER | \ - VM_EXIT_LOAD_IA32_EFER | \ - VM_EXIT_CLEAR_BNDCFGS | \ - VM_EXIT_PT_CONCEAL_PIP | \ - VM_EXIT_CLEAR_IA32_RTIT_CTL) - -#define EVMCS1_SUPPORTED_VMENTRY_CTRL \ - (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | \ - VM_ENTRY_LOAD_DEBUG_CONTROLS | \ - VM_ENTRY_IA32E_MODE | \ - VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL | \ - VM_ENTRY_LOAD_IA32_PAT | \ - VM_ENTRY_LOAD_IA32_EFER | \ - VM_ENTRY_LOAD_BNDCFGS | \ - VM_ENTRY_PT_CONCEAL_PIP | \ - VM_ENTRY_LOAD_IA32_RTIT_CTL) - -#define EVMCS1_SUPPORTED_VMFUNC (0) - -struct evmcs_field { - u16 offset; - u16 clean_field; -}; - -extern const struct evmcs_field vmcs_field_to_evmcs_1[]; -extern const unsigned int nr_evmcs_1_fields; - -static __always_inline int evmcs_field_offset(unsigned long field, - u16 *clean_field) -{ - unsigned int index = ROL16(field, 6); - const struct evmcs_field *evmcs_field; - - if (unlikely(index >= nr_evmcs_1_fields)) - return -ENOENT; - - evmcs_field = &vmcs_field_to_evmcs_1[index]; - - /* - * Use offset=0 to detect holes in eVMCS. This offset belongs to - * 'revision_id' but this field has no encoding and is supposed to - * be accessed directly. - */ - if (unlikely(!evmcs_field->offset)) - return -ENOENT; - - if (clean_field) - *clean_field = evmcs_field->clean_field; - - return evmcs_field->offset; -} - -static inline u64 evmcs_read_any(struct hv_enlightened_vmcs *evmcs, - unsigned long field, u16 offset) -{ - /* - * vmcs12_read_any() doesn't care whether the supplied structure - * is 'struct vmcs12' or 'struct hv_enlightened_vmcs' as it takes - * the exact offset of the required field, use it for convenience - * here. - */ - return vmcs12_read_any((void *)evmcs, field, offset); -} - #define EVMPTR_INVALID (-1ULL) #define EVMPTR_MAP_PENDING (-2ULL) diff --git a/arch/x86/kvm/vmx/hyperv_evmcs.c b/arch/x86/kvm/vmx/hyperv_evmcs.c new file mode 100644 index 000000000000..904bfcd1519b --- /dev/null +++ b/arch/x86/kvm/vmx/hyperv_evmcs.c @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * This file contains common code for working with Enlightened VMCS which is + * used both by Hyper-V on KVM and KVM on Hyper-V. + */ + +#include "hyperv_evmcs.h" + +#define EVMCS1_OFFSET(x) offsetof(struct hv_enlightened_vmcs, x) +#define EVMCS1_FIELD(number, name, clean_field)[ROL16(number, 6)] = \ + {EVMCS1_OFFSET(name), clean_field} + +const struct evmcs_field vmcs_field_to_evmcs_1[] = { + /* 64 bit rw */ + EVMCS1_FIELD(GUEST_RIP, guest_rip, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), + EVMCS1_FIELD(GUEST_RSP, guest_rsp, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC), + EVMCS1_FIELD(GUEST_RFLAGS, guest_rflags, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC), + EVMCS1_FIELD(HOST_IA32_PAT, host_ia32_pat, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), + EVMCS1_FIELD(HOST_IA32_EFER, host_ia32_efer, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), + EVMCS1_FIELD(HOST_IA32_PERF_GLOBAL_CTRL, host_ia32_perf_global_ctrl, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), + EVMCS1_FIELD(HOST_CR0, host_cr0, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), + EVMCS1_FIELD(HOST_CR3, host_cr3, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), + EVMCS1_FIELD(HOST_CR4, host_cr4, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), + EVMCS1_FIELD(HOST_IA32_SYSENTER_ESP, host_ia32_sysenter_esp, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), + EVMCS1_FIELD(HOST_IA32_SYSENTER_EIP, host_ia32_sysenter_eip, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), + EVMCS1_FIELD(HOST_RIP, host_rip, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), + EVMCS1_FIELD(IO_BITMAP_A, io_bitmap_a, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP), + EVMCS1_FIELD(IO_BITMAP_B, io_bitmap_b, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP), + EVMCS1_FIELD(MSR_BITMAP, msr_bitmap, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP), + EVMCS1_FIELD(GUEST_ES_BASE, guest_es_base, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_CS_BASE, guest_cs_base, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_SS_BASE, guest_ss_base, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_DS_BASE, guest_ds_base, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_FS_BASE, guest_fs_base, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_GS_BASE, guest_gs_base, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_LDTR_BASE, guest_ldtr_base, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_TR_BASE, guest_tr_base, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_GDTR_BASE, guest_gdtr_base, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_IDTR_BASE, guest_idtr_base, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(TSC_OFFSET, tsc_offset, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2), + EVMCS1_FIELD(VIRTUAL_APIC_PAGE_ADDR, virtual_apic_page_addr, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2), + EVMCS1_FIELD(VMCS_LINK_POINTER, vmcs_link_pointer, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), + EVMCS1_FIELD(GUEST_IA32_DEBUGCTL, guest_ia32_debugctl, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), + EVMCS1_FIELD(GUEST_IA32_PAT, guest_ia32_pat, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), + EVMCS1_FIELD(GUEST_IA32_EFER, guest_ia32_efer, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), + EVMCS1_FIELD(GUEST_IA32_PERF_GLOBAL_CTRL, guest_ia32_perf_global_ctrl, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), + EVMCS1_FIELD(GUEST_PDPTR0, guest_pdptr0, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), + EVMCS1_FIELD(GUEST_PDPTR1, guest_pdptr1, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), + EVMCS1_FIELD(GUEST_PDPTR2, guest_pdptr2, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), + EVMCS1_FIELD(GUEST_PDPTR3, guest_pdptr3, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), + EVMCS1_FIELD(GUEST_PENDING_DBG_EXCEPTIONS, guest_pending_dbg_exceptions, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), + EVMCS1_FIELD(GUEST_SYSENTER_ESP, guest_sysenter_esp, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), + EVMCS1_FIELD(GUEST_SYSENTER_EIP, guest_sysenter_eip, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), + EVMCS1_FIELD(CR0_GUEST_HOST_MASK, cr0_guest_host_mask, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), + EVMCS1_FIELD(CR4_GUEST_HOST_MASK, cr4_guest_host_mask, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), + EVMCS1_FIELD(CR0_READ_SHADOW, cr0_read_shadow, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), + EVMCS1_FIELD(CR4_READ_SHADOW, cr4_read_shadow, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), + EVMCS1_FIELD(GUEST_CR0, guest_cr0, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), + EVMCS1_FIELD(GUEST_CR3, guest_cr3, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), + EVMCS1_FIELD(GUEST_CR4, guest_cr4, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), + EVMCS1_FIELD(GUEST_DR7, guest_dr7, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR), + EVMCS1_FIELD(HOST_FS_BASE, host_fs_base, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER), + EVMCS1_FIELD(HOST_GS_BASE, host_gs_base, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER), + EVMCS1_FIELD(HOST_TR_BASE, host_tr_base, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER), + EVMCS1_FIELD(HOST_GDTR_BASE, host_gdtr_base, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER), + EVMCS1_FIELD(HOST_IDTR_BASE, host_idtr_base, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER), + EVMCS1_FIELD(HOST_RSP, host_rsp, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER), + EVMCS1_FIELD(EPT_POINTER, ept_pointer, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT), + EVMCS1_FIELD(GUEST_BNDCFGS, guest_bndcfgs, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), + EVMCS1_FIELD(XSS_EXIT_BITMAP, xss_exit_bitmap, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2), + EVMCS1_FIELD(ENCLS_EXITING_BITMAP, encls_exiting_bitmap, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2), + EVMCS1_FIELD(TSC_MULTIPLIER, tsc_multiplier, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2), + /* + * Not used by KVM: + * + * EVMCS1_FIELD(0x00006828, guest_ia32_s_cet, + * HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), + * EVMCS1_FIELD(0x0000682A, guest_ssp, + * HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC), + * EVMCS1_FIELD(0x0000682C, guest_ia32_int_ssp_table_addr, + * HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), + * EVMCS1_FIELD(0x00002816, guest_ia32_lbr_ctl, + * HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), + * EVMCS1_FIELD(0x00006C18, host_ia32_s_cet, + * HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), + * EVMCS1_FIELD(0x00006C1A, host_ssp, + * HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), + * EVMCS1_FIELD(0x00006C1C, host_ia32_int_ssp_table_addr, + * HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), + */ + + /* 64 bit read only */ + EVMCS1_FIELD(GUEST_PHYSICAL_ADDRESS, guest_physical_address, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), + EVMCS1_FIELD(EXIT_QUALIFICATION, exit_qualification, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), + /* + * Not defined in KVM: + * + * EVMCS1_FIELD(0x00006402, exit_io_instruction_ecx, + * HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE); + * EVMCS1_FIELD(0x00006404, exit_io_instruction_esi, + * HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE); + * EVMCS1_FIELD(0x00006406, exit_io_instruction_esi, + * HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE); + * EVMCS1_FIELD(0x00006408, exit_io_instruction_eip, + * HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE); + */ + EVMCS1_FIELD(GUEST_LINEAR_ADDRESS, guest_linear_address, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), + + /* + * No mask defined in the spec as Hyper-V doesn't currently support + * these. Future proof by resetting the whole clean field mask on + * access. + */ + EVMCS1_FIELD(VM_EXIT_MSR_STORE_ADDR, vm_exit_msr_store_addr, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), + EVMCS1_FIELD(VM_EXIT_MSR_LOAD_ADDR, vm_exit_msr_load_addr, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), + EVMCS1_FIELD(VM_ENTRY_MSR_LOAD_ADDR, vm_entry_msr_load_addr, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), + + /* 32 bit rw */ + EVMCS1_FIELD(TPR_THRESHOLD, tpr_threshold, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), + EVMCS1_FIELD(GUEST_INTERRUPTIBILITY_INFO, guest_interruptibility_info, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC), + EVMCS1_FIELD(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC), + EVMCS1_FIELD(EXCEPTION_BITMAP, exception_bitmap, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EXCPN), + EVMCS1_FIELD(VM_ENTRY_CONTROLS, vm_entry_controls, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY), + EVMCS1_FIELD(VM_ENTRY_INTR_INFO_FIELD, vm_entry_intr_info_field, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT), + EVMCS1_FIELD(VM_ENTRY_EXCEPTION_ERROR_CODE, + vm_entry_exception_error_code, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT), + EVMCS1_FIELD(VM_ENTRY_INSTRUCTION_LEN, vm_entry_instruction_len, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT), + EVMCS1_FIELD(HOST_IA32_SYSENTER_CS, host_ia32_sysenter_cs, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), + EVMCS1_FIELD(PIN_BASED_VM_EXEC_CONTROL, pin_based_vm_exec_control, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1), + EVMCS1_FIELD(VM_EXIT_CONTROLS, vm_exit_controls, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1), + EVMCS1_FIELD(SECONDARY_VM_EXEC_CONTROL, secondary_vm_exec_control, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1), + EVMCS1_FIELD(GUEST_ES_LIMIT, guest_es_limit, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_CS_LIMIT, guest_cs_limit, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_SS_LIMIT, guest_ss_limit, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_DS_LIMIT, guest_ds_limit, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_FS_LIMIT, guest_fs_limit, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_GS_LIMIT, guest_gs_limit, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_LDTR_LIMIT, guest_ldtr_limit, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_TR_LIMIT, guest_tr_limit, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_GDTR_LIMIT, guest_gdtr_limit, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_IDTR_LIMIT, guest_idtr_limit, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_ES_AR_BYTES, guest_es_ar_bytes, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_CS_AR_BYTES, guest_cs_ar_bytes, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_SS_AR_BYTES, guest_ss_ar_bytes, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_DS_AR_BYTES, guest_ds_ar_bytes, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_FS_AR_BYTES, guest_fs_ar_bytes, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_GS_AR_BYTES, guest_gs_ar_bytes, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_LDTR_AR_BYTES, guest_ldtr_ar_bytes, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_TR_AR_BYTES, guest_tr_ar_bytes, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_ACTIVITY_STATE, guest_activity_state, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), + EVMCS1_FIELD(GUEST_SYSENTER_CS, guest_sysenter_cs, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), + + /* 32 bit read only */ + EVMCS1_FIELD(VM_INSTRUCTION_ERROR, vm_instruction_error, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), + EVMCS1_FIELD(VM_EXIT_REASON, vm_exit_reason, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), + EVMCS1_FIELD(VM_EXIT_INTR_INFO, vm_exit_intr_info, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), + EVMCS1_FIELD(VM_EXIT_INTR_ERROR_CODE, vm_exit_intr_error_code, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), + EVMCS1_FIELD(IDT_VECTORING_INFO_FIELD, idt_vectoring_info_field, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), + EVMCS1_FIELD(IDT_VECTORING_ERROR_CODE, idt_vectoring_error_code, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), + EVMCS1_FIELD(VM_EXIT_INSTRUCTION_LEN, vm_exit_instruction_len, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), + EVMCS1_FIELD(VMX_INSTRUCTION_INFO, vmx_instruction_info, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE), + + /* No mask defined in the spec (not used) */ + EVMCS1_FIELD(PAGE_FAULT_ERROR_CODE_MASK, page_fault_error_code_mask, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), + EVMCS1_FIELD(PAGE_FAULT_ERROR_CODE_MATCH, page_fault_error_code_match, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), + EVMCS1_FIELD(CR3_TARGET_COUNT, cr3_target_count, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), + EVMCS1_FIELD(VM_EXIT_MSR_STORE_COUNT, vm_exit_msr_store_count, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), + EVMCS1_FIELD(VM_EXIT_MSR_LOAD_COUNT, vm_exit_msr_load_count, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), + EVMCS1_FIELD(VM_ENTRY_MSR_LOAD_COUNT, vm_entry_msr_load_count, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), + + /* 16 bit rw */ + EVMCS1_FIELD(HOST_ES_SELECTOR, host_es_selector, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), + EVMCS1_FIELD(HOST_CS_SELECTOR, host_cs_selector, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), + EVMCS1_FIELD(HOST_SS_SELECTOR, host_ss_selector, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), + EVMCS1_FIELD(HOST_DS_SELECTOR, host_ds_selector, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), + EVMCS1_FIELD(HOST_FS_SELECTOR, host_fs_selector, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), + EVMCS1_FIELD(HOST_GS_SELECTOR, host_gs_selector, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), + EVMCS1_FIELD(HOST_TR_SELECTOR, host_tr_selector, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), + EVMCS1_FIELD(GUEST_ES_SELECTOR, guest_es_selector, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_CS_SELECTOR, guest_cs_selector, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_SS_SELECTOR, guest_ss_selector, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_DS_SELECTOR, guest_ds_selector, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_FS_SELECTOR, guest_fs_selector, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_GS_SELECTOR, guest_gs_selector, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_LDTR_SELECTOR, guest_ldtr_selector, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(GUEST_TR_SELECTOR, guest_tr_selector, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2), + EVMCS1_FIELD(VIRTUAL_PROCESSOR_ID, virtual_processor_id, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT), +}; +const unsigned int nr_evmcs_1_fields = ARRAY_SIZE(vmcs_field_to_evmcs_1); diff --git a/arch/x86/kvm/vmx/hyperv_evmcs.h b/arch/x86/kvm/vmx/hyperv_evmcs.h new file mode 100644 index 000000000000..a543fccfc574 --- /dev/null +++ b/arch/x86/kvm/vmx/hyperv_evmcs.h @@ -0,0 +1,166 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file contains common definitions for working with Enlightened VMCS which + * are used both by Hyper-V on KVM and KVM on Hyper-V. + */ +#ifndef __KVM_X86_VMX_HYPERV_EVMCS_H +#define __KVM_X86_VMX_HYPERV_EVMCS_H + +#include + +#include "capabilities.h" +#include "vmcs12.h" + +#define KVM_EVMCS_VERSION 1 + +/* + * Enlightened VMCSv1 doesn't support these: + * + * POSTED_INTR_NV = 0x00000002, + * GUEST_INTR_STATUS = 0x00000810, + * APIC_ACCESS_ADDR = 0x00002014, + * POSTED_INTR_DESC_ADDR = 0x00002016, + * EOI_EXIT_BITMAP0 = 0x0000201c, + * EOI_EXIT_BITMAP1 = 0x0000201e, + * EOI_EXIT_BITMAP2 = 0x00002020, + * EOI_EXIT_BITMAP3 = 0x00002022, + * GUEST_PML_INDEX = 0x00000812, + * PML_ADDRESS = 0x0000200e, + * VM_FUNCTION_CONTROL = 0x00002018, + * EPTP_LIST_ADDRESS = 0x00002024, + * VMREAD_BITMAP = 0x00002026, + * VMWRITE_BITMAP = 0x00002028, + * + * TSC_MULTIPLIER = 0x00002032, + * PLE_GAP = 0x00004020, + * PLE_WINDOW = 0x00004022, + * VMX_PREEMPTION_TIMER_VALUE = 0x0000482E, + * + * Currently unsupported in KVM: + * GUEST_IA32_RTIT_CTL = 0x00002814, + */ +#define EVMCS1_SUPPORTED_PINCTRL \ + (PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR | \ + PIN_BASED_EXT_INTR_MASK | \ + PIN_BASED_NMI_EXITING | \ + PIN_BASED_VIRTUAL_NMIS) + +#define EVMCS1_SUPPORTED_EXEC_CTRL \ + (CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR | \ + CPU_BASED_HLT_EXITING | \ + CPU_BASED_CR3_LOAD_EXITING | \ + CPU_BASED_CR3_STORE_EXITING | \ + CPU_BASED_UNCOND_IO_EXITING | \ + CPU_BASED_MOV_DR_EXITING | \ + CPU_BASED_USE_TSC_OFFSETTING | \ + CPU_BASED_MWAIT_EXITING | \ + CPU_BASED_MONITOR_EXITING | \ + CPU_BASED_INVLPG_EXITING | \ + CPU_BASED_RDPMC_EXITING | \ + CPU_BASED_INTR_WINDOW_EXITING | \ + CPU_BASED_CR8_LOAD_EXITING | \ + CPU_BASED_CR8_STORE_EXITING | \ + CPU_BASED_RDTSC_EXITING | \ + CPU_BASED_TPR_SHADOW | \ + CPU_BASED_USE_IO_BITMAPS | \ + CPU_BASED_MONITOR_TRAP_FLAG | \ + CPU_BASED_USE_MSR_BITMAPS | \ + CPU_BASED_NMI_WINDOW_EXITING | \ + CPU_BASED_PAUSE_EXITING | \ + CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) + +#define EVMCS1_SUPPORTED_2NDEXEC \ + (SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | \ + SECONDARY_EXEC_WBINVD_EXITING | \ + SECONDARY_EXEC_ENABLE_VPID | \ + SECONDARY_EXEC_ENABLE_EPT | \ + SECONDARY_EXEC_UNRESTRICTED_GUEST | \ + SECONDARY_EXEC_DESC | \ + SECONDARY_EXEC_ENABLE_RDTSCP | \ + SECONDARY_EXEC_ENABLE_INVPCID | \ + SECONDARY_EXEC_ENABLE_XSAVES | \ + SECONDARY_EXEC_RDSEED_EXITING | \ + SECONDARY_EXEC_RDRAND_EXITING | \ + SECONDARY_EXEC_TSC_SCALING | \ + SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE | \ + SECONDARY_EXEC_PT_USE_GPA | \ + SECONDARY_EXEC_PT_CONCEAL_VMX | \ + SECONDARY_EXEC_BUS_LOCK_DETECTION | \ + SECONDARY_EXEC_NOTIFY_VM_EXITING | \ + SECONDARY_EXEC_ENCLS_EXITING) + +#define EVMCS1_SUPPORTED_3RDEXEC (0ULL) + +#define EVMCS1_SUPPORTED_VMEXIT_CTRL \ + (VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR | \ + VM_EXIT_SAVE_DEBUG_CONTROLS | \ + VM_EXIT_ACK_INTR_ON_EXIT | \ + VM_EXIT_HOST_ADDR_SPACE_SIZE | \ + VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | \ + VM_EXIT_SAVE_IA32_PAT | \ + VM_EXIT_LOAD_IA32_PAT | \ + VM_EXIT_SAVE_IA32_EFER | \ + VM_EXIT_LOAD_IA32_EFER | \ + VM_EXIT_CLEAR_BNDCFGS | \ + VM_EXIT_PT_CONCEAL_PIP | \ + VM_EXIT_CLEAR_IA32_RTIT_CTL) + +#define EVMCS1_SUPPORTED_VMENTRY_CTRL \ + (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | \ + VM_ENTRY_LOAD_DEBUG_CONTROLS | \ + VM_ENTRY_IA32E_MODE | \ + VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL | \ + VM_ENTRY_LOAD_IA32_PAT | \ + VM_ENTRY_LOAD_IA32_EFER | \ + VM_ENTRY_LOAD_BNDCFGS | \ + VM_ENTRY_PT_CONCEAL_PIP | \ + VM_ENTRY_LOAD_IA32_RTIT_CTL) + +#define EVMCS1_SUPPORTED_VMFUNC (0) + +struct evmcs_field { + u16 offset; + u16 clean_field; +}; + +extern const struct evmcs_field vmcs_field_to_evmcs_1[]; +extern const unsigned int nr_evmcs_1_fields; + +static __always_inline int evmcs_field_offset(unsigned long field, + u16 *clean_field) +{ + const struct evmcs_field *evmcs_field; + unsigned int index = ROL16(field, 6); + + if (unlikely(index >= nr_evmcs_1_fields)) + return -ENOENT; + + evmcs_field = &vmcs_field_to_evmcs_1[index]; + + /* + * Use offset=0 to detect holes in eVMCS. This offset belongs to + * 'revision_id' but this field has no encoding and is supposed to + * be accessed directly. + */ + if (unlikely(!evmcs_field->offset)) + return -ENOENT; + + if (clean_field) + *clean_field = evmcs_field->clean_field; + + return evmcs_field->offset; +} + +static inline u64 evmcs_read_any(struct hv_enlightened_vmcs *evmcs, + unsigned long field, u16 offset) +{ + /* + * vmcs12_read_any() doesn't care whether the supplied structure + * is 'struct vmcs12' or 'struct hv_enlightened_vmcs' as it takes + * the exact offset of the required field, use it for convenience + * here. + */ + return vmcs12_read_any((void *)evmcs, field, offset); +} + +#endif /* __KVM_X86_VMX_HYPERV_H */ diff --git a/arch/x86/kvm/vmx/nested.h b/arch/x86/kvm/vmx/nested.h index b4b9d51438c6..b0f2e26c1aea 100644 --- a/arch/x86/kvm/vmx/nested.h +++ b/arch/x86/kvm/vmx/nested.h @@ -3,6 +3,7 @@ #define __KVM_X86_VMX_NESTED_H #include "kvm_cache_regs.h" +#include "hyperv.h" #include "vmcs12.h" #include "vmx.h" diff --git a/arch/x86/kvm/vmx/vmx_onhyperv.h b/arch/x86/kvm/vmx/vmx_onhyperv.h index 11541d272dbd..eb48153bfd73 100644 --- a/arch/x86/kvm/vmx/vmx_onhyperv.h +++ b/arch/x86/kvm/vmx/vmx_onhyperv.h @@ -4,11 +4,12 @@ #define __ARCH_X86_KVM_VMX_ONHYPERV_H__ #include +#include #include #include "capabilities.h" -#include "hyperv.h" +#include "hyperv_evmcs.h" #include "vmcs12.h" #define current_evmcs ((struct hv_enlightened_vmcs *)this_cpu_read(current_vmcs)) From af9d544a452114eb54638015544b884e1befd0fb Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Tue, 5 Dec 2023 11:36:21 +0100 Subject: [PATCH 114/336] KVM: x86: Introduce helper to handle Hyper-V paravirt TLB flush requests As a preparation to making Hyper-V emulation optional, introduce a helper to handle pending KVM_REQ_HV_TLB_FLUSH requests. No functional change intended. Reviewed-by: Maxim Levitsky Tested-by: Jeremi Piotrowski Signed-off-by: Vitaly Kuznetsov Link: https://lore.kernel.org/r/20231205103630.1391318-8-vkuznets@redhat.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/hyperv.h | 13 +++++++++++++ arch/x86/kvm/svm/nested.c | 10 ++-------- arch/x86/kvm/vmx/nested.c | 10 ++-------- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/arch/x86/kvm/hyperv.h b/arch/x86/kvm/hyperv.h index 9d8fa6ba6341..77f6549aa5de 100644 --- a/arch/x86/kvm/hyperv.h +++ b/arch/x86/kvm/hyperv.h @@ -247,6 +247,19 @@ static inline int kvm_hv_verify_vp_assist(struct kvm_vcpu *vcpu) return kvm_hv_get_assist_page(vcpu); } +static inline void kvm_hv_nested_transtion_tlb_flush(struct kvm_vcpu *vcpu, + bool tdp_enabled) +{ + /* + * KVM_REQ_HV_TLB_FLUSH flushes entries from either L1's VP_ID or + * L2's VP_ID upon request from the guest. Make sure we check for + * pending entries in the right FIFO upon L1/L2 transition as these + * requests are put by other vCPUs asynchronously. + */ + if (to_hv_vcpu(vcpu) && tdp_enabled) + kvm_make_request(KVM_REQ_HV_TLB_FLUSH, vcpu); +} + int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu); #endif diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 3fea8c47679e..74c04102ef01 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -487,14 +487,8 @@ static void nested_save_pending_event_to_vmcb12(struct vcpu_svm *svm, static void nested_svm_transition_tlb_flush(struct kvm_vcpu *vcpu) { - /* - * KVM_REQ_HV_TLB_FLUSH flushes entries from either L1's VP_ID or - * L2's VP_ID upon request from the guest. Make sure we check for - * pending entries in the right FIFO upon L1/L2 transition as these - * requests are put by other vCPUs asynchronously. - */ - if (to_hv_vcpu(vcpu) && npt_enabled) - kvm_make_request(KVM_REQ_HV_TLB_FLUSH, vcpu); + /* Handle pending Hyper-V TLB flush requests */ + kvm_hv_nested_transtion_tlb_flush(vcpu, npt_enabled); /* * TODO: optimize unconditional TLB flush/MMU sync. A partial list of diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index c5ec0ef51ff7..382c0746d069 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -1139,14 +1139,8 @@ static void nested_vmx_transition_tlb_flush(struct kvm_vcpu *vcpu, { struct vcpu_vmx *vmx = to_vmx(vcpu); - /* - * KVM_REQ_HV_TLB_FLUSH flushes entries from either L1's VP_ID or - * L2's VP_ID upon request from the guest. Make sure we check for - * pending entries in the right FIFO upon L1/L2 transition as these - * requests are put by other vCPUs asynchronously. - */ - if (to_hv_vcpu(vcpu) && enable_ept) - kvm_make_request(KVM_REQ_HV_TLB_FLUSH, vcpu); + /* Handle pending Hyper-V TLB flush requests */ + kvm_hv_nested_transtion_tlb_flush(vcpu, enable_ept); /* * If vmcs12 doesn't use VPID, L1 expects linear and combined mappings From b2e02f82b7f76234305c5a7fba4dbebc47ce4cb5 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Tue, 5 Dec 2023 11:36:22 +0100 Subject: [PATCH 115/336] KVM: nVMX: Split off helper for emulating VMCLEAR on Hyper-V eVMCS To avoid overloading handle_vmclear() with Hyper-V specific details and to prepare the code to making Hyper-V emulation optional, create a dedicated nested_evmcs_handle_vmclear() helper. No functional change intended. Suggested-by: Sean Christopherson Tested-by: Jeremi Piotrowski Signed-off-by: Vitaly Kuznetsov Reviewed-by: Maxim Levitsky Link: https://lore.kernel.org/r/20231205103630.1391318-9-vkuznets@redhat.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/nested.c | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 382c0746d069..903b6f9ea2bd 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -243,6 +243,29 @@ static inline void nested_release_evmcs(struct kvm_vcpu *vcpu) } } +static bool nested_evmcs_handle_vmclear(struct kvm_vcpu *vcpu, gpa_t vmptr) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + /* + * When Enlightened VMEntry is enabled on the calling CPU we treat + * memory area pointer by vmptr as Enlightened VMCS (as there's no good + * way to distinguish it from VMCS12) and we must not corrupt it by + * writing to the non-existent 'launch_state' field. The area doesn't + * have to be the currently active EVMCS on the calling CPU and there's + * nothing KVM has to do to transition it from 'active' to 'non-active' + * state. It is possible that the area will stay mapped as + * vmx->nested.hv_evmcs but this shouldn't be a problem. + */ + if (!guest_cpuid_has_evmcs(vcpu) || + !evmptr_is_valid(nested_get_evmptr(vcpu))) + return false; + + if (vmx->nested.hv_evmcs && vmptr == vmx->nested.hv_evmcs_vmptr) + nested_release_evmcs(vcpu); + + return true; +} + static void vmx_sync_vmcs_host_state(struct vcpu_vmx *vmx, struct loaded_vmcs *prev) { @@ -5286,18 +5309,7 @@ static int handle_vmclear(struct kvm_vcpu *vcpu) if (vmptr == vmx->nested.vmxon_ptr) return nested_vmx_fail(vcpu, VMXERR_VMCLEAR_VMXON_POINTER); - /* - * When Enlightened VMEntry is enabled on the calling CPU we treat - * memory area pointer by vmptr as Enlightened VMCS (as there's no good - * way to distinguish it from VMCS12) and we must not corrupt it by - * writing to the non-existent 'launch_state' field. The area doesn't - * have to be the currently active EVMCS on the calling CPU and there's - * nothing KVM has to do to transition it from 'active' to 'non-active' - * state. It is possible that the area will stay mapped as - * vmx->nested.hv_evmcs but this shouldn't be a problem. - */ - if (likely(!guest_cpuid_has_evmcs(vcpu) || - !evmptr_is_valid(nested_get_evmptr(vcpu)))) { + if (likely(!nested_evmcs_handle_vmclear(vcpu, vmptr))) { if (vmptr == vmx->nested.current_vmptr) nested_release_vmcs12(vcpu); @@ -5314,8 +5326,6 @@ static int handle_vmclear(struct kvm_vcpu *vcpu) vmptr + offsetof(struct vmcs12, launch_state), &zero, sizeof(zero)); - } else if (vmx->nested.hv_evmcs && vmptr == vmx->nested.hv_evmcs_vmptr) { - nested_release_evmcs(vcpu); } return nested_vmx_succeed(vcpu); From 6dac1195181cb561a1ac32b92f58c92e87a91c70 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Tue, 5 Dec 2023 11:36:23 +0100 Subject: [PATCH 116/336] KVM: selftests: Make Hyper-V tests explicitly require KVM Hyper-V support In preparation for conditional Hyper-V emulation enablement in KVM, make Hyper-V specific tests skip gracefully instead of failing when KVM support for emulating Hyper-V is not there. Reviewed-by: Maxim Levitsky Tested-by: Jeremi Piotrowski Signed-off-by: Vitaly Kuznetsov Link: https://lore.kernel.org/r/20231205103630.1391318-10-vkuznets@redhat.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/x86_64/hyperv_clock.c | 2 ++ tools/testing/selftests/kvm/x86_64/hyperv_evmcs.c | 5 +++-- .../selftests/kvm/x86_64/hyperv_extended_hypercalls.c | 2 ++ tools/testing/selftests/kvm/x86_64/hyperv_features.c | 2 ++ tools/testing/selftests/kvm/x86_64/hyperv_ipi.c | 2 ++ tools/testing/selftests/kvm/x86_64/hyperv_svm_test.c | 1 + tools/testing/selftests/kvm/x86_64/hyperv_tlb_flush.c | 2 ++ 7 files changed, 14 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/kvm/x86_64/hyperv_clock.c b/tools/testing/selftests/kvm/x86_64/hyperv_clock.c index f25749eaa6a8..f5e1e98f04f9 100644 --- a/tools/testing/selftests/kvm/x86_64/hyperv_clock.c +++ b/tools/testing/selftests/kvm/x86_64/hyperv_clock.c @@ -211,6 +211,8 @@ int main(void) vm_vaddr_t tsc_page_gva; int stage; + TEST_REQUIRE(kvm_has_cap(KVM_CAP_HYPERV_TIME)); + vm = vm_create_with_one_vcpu(&vcpu, guest_main); vcpu_set_hv_cpuid(vcpu); diff --git a/tools/testing/selftests/kvm/x86_64/hyperv_evmcs.c b/tools/testing/selftests/kvm/x86_64/hyperv_evmcs.c index 7bde0c4dfdbd..4c7257ecd2a6 100644 --- a/tools/testing/selftests/kvm/x86_64/hyperv_evmcs.c +++ b/tools/testing/selftests/kvm/x86_64/hyperv_evmcs.c @@ -240,11 +240,12 @@ int main(int argc, char *argv[]) struct ucall uc; int stage; - vm = vm_create_with_one_vcpu(&vcpu, guest_code); - TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX)); TEST_REQUIRE(kvm_has_cap(KVM_CAP_NESTED_STATE)); TEST_REQUIRE(kvm_has_cap(KVM_CAP_HYPERV_ENLIGHTENED_VMCS)); + TEST_REQUIRE(kvm_has_cap(KVM_CAP_HYPERV_DIRECT_TLBFLUSH)); + + vm = vm_create_with_one_vcpu(&vcpu, guest_code); hcall_page = vm_vaddr_alloc_pages(vm, 1); memset(addr_gva2hva(vm, hcall_page), 0x0, getpagesize()); diff --git a/tools/testing/selftests/kvm/x86_64/hyperv_extended_hypercalls.c b/tools/testing/selftests/kvm/x86_64/hyperv_extended_hypercalls.c index e036db1f32b9..949e08e98f31 100644 --- a/tools/testing/selftests/kvm/x86_64/hyperv_extended_hypercalls.c +++ b/tools/testing/selftests/kvm/x86_64/hyperv_extended_hypercalls.c @@ -43,6 +43,8 @@ int main(void) uint64_t *outval; struct ucall uc; + TEST_REQUIRE(kvm_has_cap(KVM_CAP_HYPERV_CPUID)); + /* Verify if extended hypercalls are supported */ if (!kvm_cpuid_has(kvm_get_supported_hv_cpuid(), HV_ENABLE_EXTENDED_HYPERCALLS)) { diff --git a/tools/testing/selftests/kvm/x86_64/hyperv_features.c b/tools/testing/selftests/kvm/x86_64/hyperv_features.c index 9f28aa276c4e..387c605a3077 100644 --- a/tools/testing/selftests/kvm/x86_64/hyperv_features.c +++ b/tools/testing/selftests/kvm/x86_64/hyperv_features.c @@ -690,6 +690,8 @@ static void guest_test_hcalls_access(void) int main(void) { + TEST_REQUIRE(kvm_has_cap(KVM_CAP_HYPERV_ENFORCE_CPUID)); + pr_info("Testing access to Hyper-V specific MSRs\n"); guest_test_msrs_access(); diff --git a/tools/testing/selftests/kvm/x86_64/hyperv_ipi.c b/tools/testing/selftests/kvm/x86_64/hyperv_ipi.c index 6feb5ddb031d..65e5f4c05068 100644 --- a/tools/testing/selftests/kvm/x86_64/hyperv_ipi.c +++ b/tools/testing/selftests/kvm/x86_64/hyperv_ipi.c @@ -248,6 +248,8 @@ int main(int argc, char *argv[]) int stage = 1, r; struct ucall uc; + TEST_REQUIRE(kvm_has_cap(KVM_CAP_HYPERV_SEND_IPI)); + vm = vm_create_with_one_vcpu(&vcpu[0], sender_guest_code); /* Hypercall input/output */ diff --git a/tools/testing/selftests/kvm/x86_64/hyperv_svm_test.c b/tools/testing/selftests/kvm/x86_64/hyperv_svm_test.c index 6c1278562090..c9b18707edc0 100644 --- a/tools/testing/selftests/kvm/x86_64/hyperv_svm_test.c +++ b/tools/testing/selftests/kvm/x86_64/hyperv_svm_test.c @@ -158,6 +158,7 @@ int main(int argc, char *argv[]) int stage; TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SVM)); + TEST_REQUIRE(kvm_has_cap(KVM_CAP_HYPERV_DIRECT_TLBFLUSH)); /* Create VM */ vm = vm_create_with_one_vcpu(&vcpu, guest_code); diff --git a/tools/testing/selftests/kvm/x86_64/hyperv_tlb_flush.c b/tools/testing/selftests/kvm/x86_64/hyperv_tlb_flush.c index 4758b6ef5618..c4443f71f8dd 100644 --- a/tools/testing/selftests/kvm/x86_64/hyperv_tlb_flush.c +++ b/tools/testing/selftests/kvm/x86_64/hyperv_tlb_flush.c @@ -590,6 +590,8 @@ int main(int argc, char *argv[]) struct ucall uc; int stage = 1, r, i; + TEST_REQUIRE(kvm_has_cap(KVM_CAP_HYPERV_TLBFLUSH)); + vm = vm_create_with_one_vcpu(&vcpu[0], sender_guest_code); /* Test data page */ From 225b7c1117b2f6dadbdb1d40538d37b9685e8b18 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Tue, 5 Dec 2023 11:36:24 +0100 Subject: [PATCH 117/336] KVM: selftests: Fix vmxon_pa == vmcs12_pa == -1ull nVMX testcase for !eVMCS The "vmxon_pa == vmcs12_pa == -1ull" test happens to work by accident: as Enlightened VMCS is always supported, set_default_vmx_state() adds 'KVM_STATE_NESTED_EVMCS' to 'flags' and the following branch of vmx_set_nested_state() is executed: if ((kvm_state->flags & KVM_STATE_NESTED_EVMCS) && (!guest_can_use(vcpu, X86_FEATURE_VMX) || !vmx->nested.enlightened_vmcs_enabled)) return -EINVAL; as 'enlightened_vmcs_enabled' is false. In fact, "vmxon_pa == vmcs12_pa == -1ull" is a valid state when not tainted by wrong flags so the test should aim for this branch: if (kvm_state->hdr.vmx.vmxon_pa == INVALID_GPA) return 0; Test all this properly: - Without KVM_STATE_NESTED_EVMCS in the flags, the expected return value is '0'. - With KVM_STATE_NESTED_EVMCS flag (when supported) set, the expected return value is '-EINVAL' prior to enabling eVMCS and '0' after. Reviewed-by: Maxim Levitsky Tested-by: Jeremi Piotrowski Signed-off-by: Vitaly Kuznetsov Link: https://lore.kernel.org/r/20231205103630.1391318-11-vkuznets@redhat.com Signed-off-by: Sean Christopherson --- .../kvm/x86_64/vmx_set_nested_state_test.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tools/testing/selftests/kvm/x86_64/vmx_set_nested_state_test.c b/tools/testing/selftests/kvm/x86_64/vmx_set_nested_state_test.c index 41ea7028a1f8..67a62a5a8895 100644 --- a/tools/testing/selftests/kvm/x86_64/vmx_set_nested_state_test.c +++ b/tools/testing/selftests/kvm/x86_64/vmx_set_nested_state_test.c @@ -125,21 +125,25 @@ void test_vmx_nested_state(struct kvm_vcpu *vcpu) /* * Setting vmxon_pa == -1ull and vmcs_pa == -1ull exits early without - * setting the nested state but flags other than eVMCS must be clear. - * The eVMCS flag can be set if the enlightened VMCS capability has - * been enabled. + * setting the nested state. When the eVMCS flag is not set, the + * expected return value is '0'. */ set_default_vmx_state(state, state_sz); + state->flags = 0; state->hdr.vmx.vmxon_pa = -1ull; state->hdr.vmx.vmcs12_pa = -1ull; - test_nested_state_expect_einval(vcpu, state); + test_nested_state(vcpu, state); - state->flags &= KVM_STATE_NESTED_EVMCS; + /* + * When eVMCS is supported, the eVMCS flag can only be set if the + * enlightened VMCS capability has been enabled. + */ if (have_evmcs) { + state->flags = KVM_STATE_NESTED_EVMCS; test_nested_state_expect_einval(vcpu, state); vcpu_enable_evmcs(vcpu); + test_nested_state(vcpu, state); } - test_nested_state(vcpu, state); /* It is invalid to have vmxon_pa == -1ull and SMM flags non-zero. */ state->hdr.vmx.smm.flags = 1; From f97314626734deaef49564a429c6f8eee3846bd3 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Tue, 5 Dec 2023 11:36:25 +0100 Subject: [PATCH 118/336] KVM: nVMX: Move guest_cpuid_has_evmcs() to hyperv.h In preparation for making Hyper-V emulation optional, move Hyper-V specific guest_cpuid_has_evmcs() to hyperv.h. No functional change intended. Suggested-by: Sean Christopherson Signed-off-by: Vitaly Kuznetsov Reviewed-by: Maxim Levitsky Tested-by: Jeremi Piotrowski Link: https://lore.kernel.org/r/20231205103630.1391318-12-vkuznets@redhat.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/hyperv.h | 11 +++++++++++ arch/x86/kvm/vmx/vmx.h | 10 ---------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/arch/x86/kvm/vmx/hyperv.h b/arch/x86/kvm/vmx/hyperv.h index d4ed99008518..6e1ee951e360 100644 --- a/arch/x86/kvm/vmx/hyperv.h +++ b/arch/x86/kvm/vmx/hyperv.h @@ -4,6 +4,7 @@ #include #include "vmcs12.h" +#include "vmx.h" #define EVMPTR_INVALID (-1ULL) #define EVMPTR_MAP_PENDING (-2ULL) @@ -20,6 +21,16 @@ enum nested_evmptrld_status { EVMPTRLD_ERROR, }; +static inline bool guest_cpuid_has_evmcs(struct kvm_vcpu *vcpu) +{ + /* + * eVMCS is exposed to the guest if Hyper-V is enabled in CPUID and + * eVMCS has been explicitly enabled by userspace. + */ + return vcpu->arch.hyperv_enabled && + to_vmx(vcpu)->nested.enlightened_vmcs_enabled; +} + u64 nested_get_evmptr(struct kvm_vcpu *vcpu); uint16_t nested_get_evmcs_version(struct kvm_vcpu *vcpu); int nested_enable_evmcs(struct kvm_vcpu *vcpu, diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index c2130d2c8e24..959c6d94287f 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -745,14 +745,4 @@ static inline bool vmx_can_use_ipiv(struct kvm_vcpu *vcpu) return lapic_in_kernel(vcpu) && enable_ipiv; } -static inline bool guest_cpuid_has_evmcs(struct kvm_vcpu *vcpu) -{ - /* - * eVMCS is exposed to the guest if Hyper-V is enabled in CPUID and - * eVMCS has been explicitly enabled by userspace. - */ - return vcpu->arch.hyperv_enabled && - to_vmx(vcpu)->nested.enlightened_vmcs_enabled; -} - #endif /* __KVM_X86_VMX_H */ From b4f69df0f65e97fec439130a0d0a8b9c7cc02df2 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Tue, 5 Dec 2023 11:36:26 +0100 Subject: [PATCH 119/336] KVM: x86: Make Hyper-V emulation optional Hyper-V emulation in KVM is a fairly big chunk and in some cases it may be desirable to not compile it in to reduce module sizes as well as the attack surface. Introduce CONFIG_KVM_HYPERV option to make it possible. Note, there's room for further nVMX/nSVM code optimizations when !CONFIG_KVM_HYPERV, this will be done in follow-up patches. Reorganize Makefile a bit so all CONFIG_HYPERV and CONFIG_KVM_HYPERV files are grouped together. Signed-off-by: Vitaly Kuznetsov Reviewed-by: Maxim Levitsky Tested-by: Jeremi Piotrowski Link: https://lore.kernel.org/r/20231205103630.1391318-13-vkuznets@redhat.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 6 ++++ arch/x86/kvm/Kconfig | 14 ++++++++ arch/x86/kvm/Makefile | 20 +++++------ arch/x86/kvm/cpuid.c | 6 ++++ arch/x86/kvm/hyperv.h | 61 +++++++++++++++++++++++++++++++- arch/x86/kvm/irq_comm.c | 9 ++++- arch/x86/kvm/svm/hyperv.h | 9 +++++ arch/x86/kvm/vmx/hyperv.h | 17 ++++++--- arch/x86/kvm/vmx/nested.c | 30 ++++++++++++++++ arch/x86/kvm/vmx/vmx.c | 2 ++ arch/x86/kvm/x86.c | 62 ++++++++++++++++++++++++--------- 11 files changed, 201 insertions(+), 35 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 86069b985d22..b093c2191cd3 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -937,8 +937,10 @@ struct kvm_vcpu_arch { /* used for guest single stepping over the given code position */ unsigned long singlestep_rip; +#ifdef CONFIG_KVM_HYPERV bool hyperv_enabled; struct kvm_vcpu_hv *hyperv; +#endif #ifdef CONFIG_KVM_XEN struct kvm_vcpu_xen xen; #endif @@ -1095,6 +1097,7 @@ enum hv_tsc_page_status { HV_TSC_PAGE_BROKEN, }; +#ifdef CONFIG_KVM_HYPERV /* Hyper-V emulation context */ struct kvm_hv { struct mutex hv_lock; @@ -1127,6 +1130,7 @@ struct kvm_hv { struct kvm_hv_syndbg hv_syndbg; }; +#endif struct msr_bitmap_range { u32 flags; @@ -1349,7 +1353,9 @@ struct kvm_arch { /* reads protected by irq_srcu, writes by irq_lock */ struct hlist_head mask_notifier_list; +#ifdef CONFIG_KVM_HYPERV struct kvm_hv hyperv; +#endif #ifdef CONFIG_KVM_XEN struct kvm_xen xen; diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index c1716e83d176..34f2f47cadf2 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -141,6 +141,20 @@ config KVM_SMM If unsure, say Y. +config KVM_HYPERV + bool "Support for Microsoft Hyper-V emulation" + depends on KVM + default y + help + Provides KVM support for emulating Microsoft Hyper-V. This allows KVM + to expose a subset of the paravirtualized interfaces defined in the + Hyper-V Hypervisor Top-Level Functional Specification (TLFS): + https://docs.microsoft.com/en-us/virtualization/hyper-v-on-windows/reference/tlfs + These interfaces are required for the correct and performant functioning + of Windows and Hyper-V guests on KVM. + + If unsure, say "Y". + config KVM_XEN bool "Support for Xen hypercall interface" depends on KVM diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index 8ea872401cd6..475b5fa917a6 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -11,29 +11,27 @@ include $(srctree)/virt/kvm/Makefile.kvm kvm-y += x86.o emulate.o i8259.o irq.o lapic.o \ i8254.o ioapic.o irq_comm.o cpuid.o pmu.o mtrr.o \ - hyperv.o debugfs.o mmu/mmu.o mmu/page_track.o \ + debugfs.o mmu/mmu.o mmu/page_track.o \ mmu/spte.o -ifdef CONFIG_HYPERV -kvm-y += kvm_onhyperv.o -endif - kvm-$(CONFIG_X86_64) += mmu/tdp_iter.o mmu/tdp_mmu.o +kvm-$(CONFIG_KVM_HYPERV) += hyperv.o kvm-$(CONFIG_KVM_XEN) += xen.o kvm-$(CONFIG_KVM_SMM) += smm.o kvm-intel-y += vmx/vmx.o vmx/vmenter.o vmx/pmu_intel.o vmx/vmcs12.o \ - vmx/hyperv.o vmx/hyperv_evmcs.o vmx/nested.o vmx/posted_intr.o -kvm-intel-$(CONFIG_X86_SGX_KVM) += vmx/sgx.o + vmx/nested.o vmx/posted_intr.o -ifdef CONFIG_HYPERV -kvm-intel-y += vmx/vmx_onhyperv.o -endif +kvm-intel-$(CONFIG_X86_SGX_KVM) += vmx/sgx.o +kvm-intel-$(CONFIG_KVM_HYPERV) += vmx/hyperv.o vmx/hyperv_evmcs.o kvm-amd-y += svm/svm.o svm/vmenter.o svm/pmu.o svm/nested.o svm/avic.o \ - svm/sev.o svm/hyperv.o + svm/sev.o +kvm-amd-$(CONFIG_KVM_HYPERV) += svm/hyperv.o ifdef CONFIG_HYPERV +kvm-y += kvm_onhyperv.o +kvm-intel-y += vmx/vmx_onhyperv.o vmx/hyperv_evmcs.o kvm-amd-y += svm/svm_onhyperv.o endif diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index dda6fc4cfae8..1b278a3f0689 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -314,11 +314,15 @@ EXPORT_SYMBOL_GPL(kvm_update_cpuid_runtime); static bool kvm_cpuid_has_hyperv(struct kvm_cpuid_entry2 *entries, int nent) { +#ifdef CONFIG_KVM_HYPERV struct kvm_cpuid_entry2 *entry; entry = cpuid_entry2_find(entries, nent, HYPERV_CPUID_INTERFACE, KVM_CPUID_INDEX_NOT_SIGNIFICANT); return entry && entry->eax == HYPERV_CPUID_SIGNATURE_EAX; +#else + return false; +#endif } static void kvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) @@ -433,11 +437,13 @@ static int kvm_set_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid_entry2 *e2, return 0; } +#ifdef CONFIG_KVM_HYPERV if (kvm_cpuid_has_hyperv(e2, nent)) { r = kvm_hv_vcpu_init(vcpu); if (r) return r; } +#endif r = kvm_check_cpuid(vcpu, e2, nent); if (r) diff --git a/arch/x86/kvm/hyperv.h b/arch/x86/kvm/hyperv.h index 77f6549aa5de..1dc0b6604526 100644 --- a/arch/x86/kvm/hyperv.h +++ b/arch/x86/kvm/hyperv.h @@ -24,6 +24,8 @@ #include #include "x86.h" +#ifdef CONFIG_KVM_HYPERV + /* "Hv#1" signature */ #define HYPERV_CPUID_SIGNATURE_EAX 0x31237648 @@ -261,5 +263,62 @@ static inline void kvm_hv_nested_transtion_tlb_flush(struct kvm_vcpu *vcpu, } int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu); +#else /* CONFIG_KVM_HYPERV */ +static inline void kvm_hv_setup_tsc_page(struct kvm *kvm, + struct pvclock_vcpu_time_info *hv_clock) {} +static inline void kvm_hv_request_tsc_page_update(struct kvm *kvm) {} +static inline void kvm_hv_init_vm(struct kvm *kvm) {} +static inline void kvm_hv_destroy_vm(struct kvm *kvm) {} +static inline int kvm_hv_vcpu_init(struct kvm_vcpu *vcpu) +{ + return 0; +} +static inline void kvm_hv_vcpu_uninit(struct kvm_vcpu *vcpu) {} +static inline bool kvm_hv_hypercall_enabled(struct kvm_vcpu *vcpu) +{ + return false; +} +static inline int kvm_hv_hypercall(struct kvm_vcpu *vcpu) +{ + return HV_STATUS_ACCESS_DENIED; +} +static inline void kvm_hv_vcpu_purge_flush_tlb(struct kvm_vcpu *vcpu) {} +static inline void kvm_hv_free_pa_page(struct kvm *kvm) {} +static inline bool kvm_hv_synic_has_vector(struct kvm_vcpu *vcpu, int vector) +{ + return false; +} +static inline bool kvm_hv_synic_auto_eoi_set(struct kvm_vcpu *vcpu, int vector) +{ + return false; +} +static inline void kvm_hv_synic_send_eoi(struct kvm_vcpu *vcpu, int vector) {} +static inline bool kvm_hv_invtsc_suppressed(struct kvm_vcpu *vcpu) +{ + return false; +} +static inline void kvm_hv_set_cpuid(struct kvm_vcpu *vcpu, bool hyperv_enabled) {} +static inline bool kvm_hv_has_stimer_pending(struct kvm_vcpu *vcpu) +{ + return false; +} +static inline bool kvm_hv_is_tlb_flush_hcall(struct kvm_vcpu *vcpu) +{ + return false; +} +static inline bool guest_hv_cpuid_has_l2_tlb_flush(struct kvm_vcpu *vcpu) +{ + return false; +} +static inline int kvm_hv_verify_vp_assist(struct kvm_vcpu *vcpu) +{ + return 0; +} +static inline u32 kvm_hv_get_vpindex(struct kvm_vcpu *vcpu) +{ + return vcpu->vcpu_idx; +} +static inline void kvm_hv_nested_transtion_tlb_flush(struct kvm_vcpu *vcpu, bool tdp_enabled) {} +#endif /* CONFIG_KVM_HYPERV */ -#endif +#endif /* __ARCH_X86_KVM_HYPERV_H__ */ diff --git a/arch/x86/kvm/irq_comm.c b/arch/x86/kvm/irq_comm.c index 16d076a1b91a..68f3f6c26046 100644 --- a/arch/x86/kvm/irq_comm.c +++ b/arch/x86/kvm/irq_comm.c @@ -144,7 +144,7 @@ int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, return kvm_irq_delivery_to_apic(kvm, NULL, &irq, NULL); } - +#ifdef CONFIG_KVM_HYPERV static int kvm_hv_set_sint(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, int irq_source_id, int level, bool line_status) @@ -154,6 +154,7 @@ static int kvm_hv_set_sint(struct kvm_kernel_irq_routing_entry *e, return kvm_hv_synic_set_irq(kvm, e->hv_sint.vcpu, e->hv_sint.sint); } +#endif int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, int irq_source_id, int level, @@ -163,9 +164,11 @@ int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, int r; switch (e->type) { +#ifdef CONFIG_KVM_HYPERV case KVM_IRQ_ROUTING_HV_SINT: return kvm_hv_set_sint(e, kvm, irq_source_id, level, line_status); +#endif case KVM_IRQ_ROUTING_MSI: if (kvm_msi_route_invalid(kvm, e)) @@ -314,11 +317,13 @@ int kvm_set_routing_entry(struct kvm *kvm, if (kvm_msi_route_invalid(kvm, e)) return -EINVAL; break; +#ifdef CONFIG_KVM_HYPERV case KVM_IRQ_ROUTING_HV_SINT: e->set = kvm_hv_set_sint; e->hv_sint.vcpu = ue->u.hv_sint.vcpu; e->hv_sint.sint = ue->u.hv_sint.sint; break; +#endif #ifdef CONFIG_KVM_XEN case KVM_IRQ_ROUTING_XEN_EVTCHN: return kvm_xen_setup_evtchn(kvm, e, ue); @@ -438,5 +443,7 @@ void kvm_scan_ioapic_routes(struct kvm_vcpu *vcpu, void kvm_arch_irq_routing_update(struct kvm *kvm) { +#ifdef CONFIG_KVM_HYPERV kvm_hv_irq_routing_update(kvm); +#endif } diff --git a/arch/x86/kvm/svm/hyperv.h b/arch/x86/kvm/svm/hyperv.h index 02f4784b5d44..d3f8bfc05832 100644 --- a/arch/x86/kvm/svm/hyperv.h +++ b/arch/x86/kvm/svm/hyperv.h @@ -11,6 +11,7 @@ #include "../hyperv.h" #include "svm.h" +#ifdef CONFIG_KVM_HYPERV static inline void nested_svm_hv_update_vm_vp_ids(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); @@ -41,5 +42,13 @@ static inline bool nested_svm_l2_tlb_flush_enabled(struct kvm_vcpu *vcpu) } void svm_hv_inject_synthetic_vmexit_post_tlb_flush(struct kvm_vcpu *vcpu); +#else /* CONFIG_KVM_HYPERV */ +static inline void nested_svm_hv_update_vm_vp_ids(struct kvm_vcpu *vcpu) {} +static inline bool nested_svm_l2_tlb_flush_enabled(struct kvm_vcpu *vcpu) +{ + return false; +} +static inline void svm_hv_inject_synthetic_vmexit_post_tlb_flush(struct kvm_vcpu *vcpu) {} +#endif /* CONFIG_KVM_HYPERV */ #endif /* __ARCH_X86_KVM_SVM_HYPERV_H__ */ diff --git a/arch/x86/kvm/vmx/hyperv.h b/arch/x86/kvm/vmx/hyperv.h index 6e1ee951e360..0e90ef4efe34 100644 --- a/arch/x86/kvm/vmx/hyperv.h +++ b/arch/x86/kvm/vmx/hyperv.h @@ -9,11 +9,6 @@ #define EVMPTR_INVALID (-1ULL) #define EVMPTR_MAP_PENDING (-2ULL) -static inline bool evmptr_is_valid(u64 evmptr) -{ - return evmptr != EVMPTR_INVALID && evmptr != EVMPTR_MAP_PENDING; -} - enum nested_evmptrld_status { EVMPTRLD_DISABLED, EVMPTRLD_SUCCEEDED, @@ -21,6 +16,12 @@ enum nested_evmptrld_status { EVMPTRLD_ERROR, }; +#ifdef CONFIG_KVM_HYPERV +static inline bool evmptr_is_valid(u64 evmptr) +{ + return evmptr != EVMPTR_INVALID && evmptr != EVMPTR_MAP_PENDING; +} + static inline bool guest_cpuid_has_evmcs(struct kvm_vcpu *vcpu) { /* @@ -39,5 +40,11 @@ void nested_evmcs_filter_control_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 * int nested_evmcs_check_controls(struct vmcs12 *vmcs12); bool nested_evmcs_l2_tlb_flush_enabled(struct kvm_vcpu *vcpu); void vmx_hv_inject_synthetic_vmexit_post_tlb_flush(struct kvm_vcpu *vcpu); +#else +static inline bool evmptr_is_valid(u64 evmptr) +{ + return false; +} +#endif #endif /* __KVM_X86_VMX_HYPERV_H */ diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 903b6f9ea2bd..01a94d290c12 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -226,6 +226,7 @@ static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx) static inline void nested_release_evmcs(struct kvm_vcpu *vcpu) { +#ifdef CONFIG_KVM_HYPERV struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); struct vcpu_vmx *vmx = to_vmx(vcpu); @@ -241,10 +242,12 @@ static inline void nested_release_evmcs(struct kvm_vcpu *vcpu) hv_vcpu->nested.vm_id = 0; hv_vcpu->nested.vp_id = 0; } +#endif } static bool nested_evmcs_handle_vmclear(struct kvm_vcpu *vcpu, gpa_t vmptr) { +#ifdef CONFIG_KVM_HYPERV struct vcpu_vmx *vmx = to_vmx(vcpu); /* * When Enlightened VMEntry is enabled on the calling CPU we treat @@ -264,6 +267,9 @@ static bool nested_evmcs_handle_vmclear(struct kvm_vcpu *vcpu, gpa_t vmptr) nested_release_evmcs(vcpu); return true; +#else + return false; +#endif } static void vmx_sync_vmcs_host_state(struct vcpu_vmx *vmx, @@ -1595,6 +1601,7 @@ static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx) static void copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx, u32 hv_clean_fields) { +#ifdef CONFIG_KVM_HYPERV struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(&vmx->vcpu); @@ -1835,10 +1842,14 @@ static void copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx, u32 hv_clean_fields */ return; +#else /* CONFIG_KVM_HYPERV */ + KVM_BUG_ON(1, vmx->vcpu.kvm); +#endif /* CONFIG_KVM_HYPERV */ } static void copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx) { +#ifdef CONFIG_KVM_HYPERV struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; @@ -2009,6 +2020,9 @@ static void copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx) evmcs->guest_bndcfgs = vmcs12->guest_bndcfgs; return; +#else /* CONFIG_KVM_HYPERV */ + KVM_BUG_ON(1, vmx->vcpu.kvm); +#endif /* CONFIG_KVM_HYPERV */ } /* @@ -2018,6 +2032,7 @@ static void copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx) static enum nested_evmptrld_status nested_vmx_handle_enlightened_vmptrld( struct kvm_vcpu *vcpu, bool from_launch) { +#ifdef CONFIG_KVM_HYPERV struct vcpu_vmx *vmx = to_vmx(vcpu); bool evmcs_gpa_changed = false; u64 evmcs_gpa; @@ -2099,6 +2114,9 @@ static enum nested_evmptrld_status nested_vmx_handle_enlightened_vmptrld( } return EVMPTRLD_SUCCEEDED; +#else + return EVMPTRLD_DISABLED; +#endif } void nested_sync_vmcs12_to_shadow(struct kvm_vcpu *vcpu) @@ -2905,8 +2923,10 @@ static int nested_vmx_check_controls(struct kvm_vcpu *vcpu, nested_check_vm_entry_controls(vcpu, vmcs12)) return -EINVAL; +#ifdef CONFIG_KVM_HYPERV if (guest_cpuid_has_evmcs(vcpu)) return nested_evmcs_check_controls(vmcs12); +#endif return 0; } @@ -3178,6 +3198,7 @@ static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu) return 0; } +#ifdef CONFIG_KVM_HYPERV static bool nested_get_evmcs_page(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); @@ -3205,6 +3226,7 @@ static bool nested_get_evmcs_page(struct kvm_vcpu *vcpu) return true; } +#endif static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu) { @@ -3296,6 +3318,7 @@ static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu) static bool vmx_get_nested_state_pages(struct kvm_vcpu *vcpu) { +#ifdef CONFIG_KVM_HYPERV /* * Note: nested_get_evmcs_page() also updates 'vp_assist_page' copy * in 'struct kvm_vcpu_hv' in case eVMCS is in use, this is mandatory @@ -3312,6 +3335,7 @@ static bool vmx_get_nested_state_pages(struct kvm_vcpu *vcpu) return false; } +#endif if (is_guest_mode(vcpu) && !nested_get_vmcs12_pages(vcpu)) return false; @@ -4749,6 +4773,7 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, /* trying to cancel vmlaunch/vmresume is a bug */ WARN_ON_ONCE(vmx->nested.nested_run_pending); +#ifdef CONFIG_KVM_HYPERV if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) { /* * KVM_REQ_GET_NESTED_STATE_PAGES is also used to map @@ -4758,6 +4783,7 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, */ (void)nested_get_evmcs_page(vcpu); } +#endif /* Service pending TLB flush requests for L2 before switching to L1. */ kvm_service_local_tlb_flush_requests(vcpu); @@ -6212,11 +6238,13 @@ static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu, * Handle L2's bus locks in L0 directly. */ return true; +#ifdef CONFIG_KVM_HYPERV case EXIT_REASON_VMCALL: /* Hyper-V L2 TLB flush hypercall is handled by L0 */ return guest_hv_cpuid_has_l2_tlb_flush(vcpu) && nested_evmcs_l2_tlb_flush_enabled(vcpu) && kvm_hv_is_tlb_flush_hcall(vcpu); +#endif default: break; } @@ -7100,7 +7128,9 @@ struct kvm_x86_nested_ops vmx_nested_ops = { .set_state = vmx_set_nested_state, .get_nested_state_pages = vmx_get_nested_state_pages, .write_log_dirty = nested_vmx_write_pml_buffer, +#ifdef CONFIG_KVM_HYPERV .enable_evmcs = nested_enable_evmcs, .get_evmcs_version = nested_get_evmcs_version, .hv_inject_synthetic_vmexit_post_tlb_flush = vmx_hv_inject_synthetic_vmexit_post_tlb_flush, +#endif }; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 552593a2ac14..3ff5c44dff9d 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2048,6 +2048,7 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) if (vmx_get_vmx_msr(&vmx->nested.msrs, msr_info->index, &msr_info->data)) return 1; +#ifdef CONFIG_KVM_HYPERV /* * Enlightened VMCS v1 doesn't have certain VMCS fields but * instead of just ignoring the features, different Hyper-V @@ -2058,6 +2059,7 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) if (!msr_info->host_initiated && guest_cpuid_has_evmcs(vcpu)) nested_evmcs_filter_control_msr(vcpu, msr_info->index, &msr_info->data); +#endif break; case MSR_IA32_RTIT_CTL: if (!vmx_pt_mode_is_host_guest()) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 81224b9676d9..598b057611e0 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1504,6 +1504,8 @@ static unsigned num_msrs_to_save; static const u32 emulated_msrs_all[] = { MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK, MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW, + +#ifdef CONFIG_KVM_HYPERV HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL, HV_X64_MSR_TIME_REF_COUNT, HV_X64_MSR_REFERENCE_TSC, HV_X64_MSR_TSC_FREQUENCY, HV_X64_MSR_APIC_FREQUENCY, @@ -1521,6 +1523,7 @@ static const u32 emulated_msrs_all[] = { HV_X64_MSR_SYNDBG_CONTROL, HV_X64_MSR_SYNDBG_STATUS, HV_X64_MSR_SYNDBG_SEND_BUFFER, HV_X64_MSR_SYNDBG_RECV_BUFFER, HV_X64_MSR_SYNDBG_PENDING_BUFFER, +#endif MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME, MSR_KVM_PV_EOI_EN, MSR_KVM_ASYNC_PF_INT, MSR_KVM_ASYNC_PF_ACK, @@ -4020,6 +4023,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) * the need to ignore the workaround. */ break; +#ifdef CONFIG_KVM_HYPERV case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15: case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: case HV_X64_MSR_SYNDBG_OPTIONS: @@ -4032,6 +4036,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) case HV_X64_MSR_TSC_INVARIANT_CONTROL: return kvm_hv_set_msr_common(vcpu, msr, data, msr_info->host_initiated); +#endif case MSR_IA32_BBL_CR_CTL3: /* Drop writes to this legacy MSR -- see rdmsr * counterpart for further detail. @@ -4377,6 +4382,7 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) */ msr_info->data = 0x20000000; break; +#ifdef CONFIG_KVM_HYPERV case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15: case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: case HV_X64_MSR_SYNDBG_OPTIONS: @@ -4390,6 +4396,7 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) return kvm_hv_get_msr_common(vcpu, msr_info->index, &msr_info->data, msr_info->host_initiated); +#endif case MSR_IA32_BBL_CR_CTL3: /* This legacy MSR exists but isn't fully documented in current * silicon. It is however accessed by winxp in very narrow @@ -4527,6 +4534,7 @@ static inline bool kvm_can_mwait_in_guest(void) boot_cpu_has(X86_FEATURE_ARAT); } +#ifdef CONFIG_KVM_HYPERV static int kvm_ioctl_get_supported_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 __user *cpuid_arg) { @@ -4547,6 +4555,7 @@ static int kvm_ioctl_get_supported_hv_cpuid(struct kvm_vcpu *vcpu, return 0; } +#endif static bool kvm_is_vm_type_supported(unsigned long type) { @@ -4580,9 +4589,11 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_PIT_STATE2: case KVM_CAP_SET_IDENTITY_MAP_ADDR: case KVM_CAP_VCPU_EVENTS: +#ifdef CONFIG_KVM_HYPERV case KVM_CAP_HYPERV: case KVM_CAP_HYPERV_VAPIC: case KVM_CAP_HYPERV_SPIN: + case KVM_CAP_HYPERV_TIME: case KVM_CAP_HYPERV_SYNIC: case KVM_CAP_HYPERV_SYNIC2: case KVM_CAP_HYPERV_VP_INDEX: @@ -4592,6 +4603,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_HYPERV_CPUID: case KVM_CAP_HYPERV_ENFORCE_CPUID: case KVM_CAP_SYS_HYPERV_CPUID: +#endif case KVM_CAP_PCI_SEGMENT: case KVM_CAP_DEBUGREGS: case KVM_CAP_X86_ROBUST_SINGLESTEP: @@ -4601,7 +4613,6 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_GET_TSC_KHZ: case KVM_CAP_KVMCLOCK_CTRL: case KVM_CAP_READONLY_MEM: - case KVM_CAP_HYPERV_TIME: case KVM_CAP_IOAPIC_POLARITY_IGNORED: case KVM_CAP_TSC_DEADLINE_TIMER: case KVM_CAP_DISABLE_QUIRKS: @@ -4712,12 +4723,14 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = kvm_x86_ops.nested_ops->get_state ? kvm_x86_ops.nested_ops->get_state(NULL, NULL, 0) : 0; break; +#ifdef CONFIG_KVM_HYPERV case KVM_CAP_HYPERV_DIRECT_TLBFLUSH: r = kvm_x86_ops.enable_l2_tlb_flush != NULL; break; case KVM_CAP_HYPERV_ENLIGHTENED_VMCS: r = kvm_x86_ops.nested_ops->enable_evmcs != NULL; break; +#endif case KVM_CAP_SMALLER_MAXPHYADDR: r = (int) allow_smaller_maxphyaddr; break; @@ -4884,9 +4897,11 @@ long kvm_arch_dev_ioctl(struct file *filp, case KVM_GET_MSRS: r = msr_io(NULL, argp, do_get_msr_feature, 1); break; +#ifdef CONFIG_KVM_HYPERV case KVM_GET_SUPPORTED_HV_CPUID: r = kvm_ioctl_get_supported_hv_cpuid(NULL, argp); break; +#endif case KVM_GET_DEVICE_ATTR: { struct kvm_device_attr attr; r = -EFAULT; @@ -5712,14 +5727,11 @@ static int kvm_vcpu_ioctl_device_attr(struct kvm_vcpu *vcpu, static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, struct kvm_enable_cap *cap) { - int r; - uint16_t vmcs_version; - void __user *user_ptr; - if (cap->flags) return -EINVAL; switch (cap->cap) { +#ifdef CONFIG_KVM_HYPERV case KVM_CAP_HYPERV_SYNIC2: if (cap->args[0]) return -EINVAL; @@ -5731,16 +5743,22 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, return kvm_hv_activate_synic(vcpu, cap->cap == KVM_CAP_HYPERV_SYNIC2); case KVM_CAP_HYPERV_ENLIGHTENED_VMCS: - if (!kvm_x86_ops.nested_ops->enable_evmcs) - return -ENOTTY; - r = kvm_x86_ops.nested_ops->enable_evmcs(vcpu, &vmcs_version); - if (!r) { - user_ptr = (void __user *)(uintptr_t)cap->args[0]; - if (copy_to_user(user_ptr, &vmcs_version, - sizeof(vmcs_version))) - r = -EFAULT; + { + int r; + uint16_t vmcs_version; + void __user *user_ptr; + + if (!kvm_x86_ops.nested_ops->enable_evmcs) + return -ENOTTY; + r = kvm_x86_ops.nested_ops->enable_evmcs(vcpu, &vmcs_version); + if (!r) { + user_ptr = (void __user *)(uintptr_t)cap->args[0]; + if (copy_to_user(user_ptr, &vmcs_version, + sizeof(vmcs_version))) + r = -EFAULT; + } + return r; } - return r; case KVM_CAP_HYPERV_DIRECT_TLBFLUSH: if (!kvm_x86_ops.enable_l2_tlb_flush) return -ENOTTY; @@ -5749,6 +5767,7 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, case KVM_CAP_HYPERV_ENFORCE_CPUID: return kvm_hv_set_enforce_cpuid(vcpu, cap->args[0]); +#endif case KVM_CAP_ENFORCE_PV_FEATURE_CPUID: vcpu->arch.pv_cpuid.enforce = cap->args[0]; @@ -6141,9 +6160,11 @@ long kvm_arch_vcpu_ioctl(struct file *filp, srcu_read_unlock(&vcpu->kvm->srcu, idx); break; } +#ifdef CONFIG_KVM_HYPERV case KVM_GET_SUPPORTED_HV_CPUID: r = kvm_ioctl_get_supported_hv_cpuid(vcpu, argp); break; +#endif #ifdef CONFIG_KVM_XEN case KVM_XEN_VCPU_GET_ATTR: { struct kvm_xen_vcpu_attr xva; @@ -7201,6 +7222,7 @@ set_pit2_out: r = static_call(kvm_x86_mem_enc_unregister_region)(kvm, ®ion); break; } +#ifdef CONFIG_KVM_HYPERV case KVM_HYPERV_EVENTFD: { struct kvm_hyperv_eventfd hvevfd; @@ -7210,6 +7232,7 @@ set_pit2_out: r = kvm_vm_ioctl_hv_eventfd(kvm, &hvevfd); break; } +#endif case KVM_SET_PMU_EVENT_FILTER: r = kvm_vm_ioctl_set_pmu_event_filter(kvm, argp); break; @@ -10588,19 +10611,20 @@ static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu) static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu) { - u64 eoi_exit_bitmap[4]; - if (!kvm_apic_hw_enabled(vcpu->arch.apic)) return; +#ifdef CONFIG_KVM_HYPERV if (to_hv_vcpu(vcpu)) { + u64 eoi_exit_bitmap[4]; + bitmap_or((ulong *)eoi_exit_bitmap, vcpu->arch.ioapic_handled_vectors, to_hv_synic(vcpu)->vec_bitmap, 256); static_call_cond(kvm_x86_load_eoi_exitmap)(vcpu, eoi_exit_bitmap); return; } - +#endif static_call_cond(kvm_x86_load_eoi_exitmap)( vcpu, (u64 *)vcpu->arch.ioapic_handled_vectors); } @@ -10691,9 +10715,11 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) * the flushes are considered "remote" and not "local" because * the requests can be initiated from other vCPUs. */ +#ifdef CONFIG_KVM_HYPERV if (kvm_check_request(KVM_REQ_HV_TLB_FLUSH, vcpu) && kvm_hv_vcpu_flush_tlb(vcpu)) kvm_vcpu_flush_tlb_guest(vcpu); +#endif if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) { vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS; @@ -10746,6 +10772,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) vcpu_load_eoi_exitmap(vcpu); if (kvm_check_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu)) kvm_vcpu_reload_apic_access_page(vcpu); +#ifdef CONFIG_KVM_HYPERV if (kvm_check_request(KVM_REQ_HV_CRASH, vcpu)) { vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH; @@ -10776,6 +10803,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) */ if (kvm_check_request(KVM_REQ_HV_STIMER, vcpu)) kvm_hv_process_stimers(vcpu); +#endif if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu)) kvm_vcpu_update_apicv(vcpu); if (kvm_check_request(KVM_REQ_APF_READY, vcpu)) From 453e42b0557148cc7092c72eff6677a5436e3c7c Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Tue, 5 Dec 2023 11:36:27 +0100 Subject: [PATCH 120/336] KVM: nVMX: Introduce helpers to check if Hyper-V evmptr12 is valid/set In order to get rid of raw 'vmx->nested.hv_evmcs_vmptr' accesses when !CONFIG_KVM_HYPERV, introduce a pair of helpers: nested_vmx_is_evmptr12_valid() to check that eVMPTR points to a valid address. nested_vmx_is_evmptr12_valid() to check that eVMPTR either points to a valid address or is in 'pending' port-migration state (meaning it is supposed to be valid but the exact address wasn't acquired from guest's memory yet). No functional change intended. Signed-off-by: Vitaly Kuznetsov Reviewed-by: Maxim Levitsky Tested-by: Jeremi Piotrowski Link: https://lore.kernel.org/r/20231205103630.1391318-14-vkuznets@redhat.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/hyperv.h | 30 ++++++++++++++++++++++++++++++ arch/x86/kvm/vmx/nested.c | 38 +++++++++++++++++++------------------- arch/x86/kvm/vmx/nested.h | 2 +- 3 files changed, 50 insertions(+), 20 deletions(-) diff --git a/arch/x86/kvm/vmx/hyperv.h b/arch/x86/kvm/vmx/hyperv.h index 0e90ef4efe34..71e90a16f183 100644 --- a/arch/x86/kvm/vmx/hyperv.h +++ b/arch/x86/kvm/vmx/hyperv.h @@ -22,6 +22,21 @@ static inline bool evmptr_is_valid(u64 evmptr) return evmptr != EVMPTR_INVALID && evmptr != EVMPTR_MAP_PENDING; } +static inline bool nested_vmx_is_evmptr12_valid(struct vcpu_vmx *vmx) +{ + return evmptr_is_valid(vmx->nested.hv_evmcs_vmptr); +} + +static inline bool evmptr_is_set(u64 evmptr) +{ + return evmptr != EVMPTR_INVALID; +} + +static inline bool nested_vmx_is_evmptr12_set(struct vcpu_vmx *vmx) +{ + return evmptr_is_set(vmx->nested.hv_evmcs_vmptr); +} + static inline bool guest_cpuid_has_evmcs(struct kvm_vcpu *vcpu) { /* @@ -45,6 +60,21 @@ static inline bool evmptr_is_valid(u64 evmptr) { return false; } + +static inline bool nested_vmx_is_evmptr12_valid(struct vcpu_vmx *vmx) +{ + return false; +} + +static inline bool evmptr_is_set(u64 evmptr) +{ + return false; +} + +static inline bool nested_vmx_is_evmptr12_set(struct vcpu_vmx *vmx) +{ + return false; +} #endif #endif /* __KVM_X86_VMX_HYPERV_H */ diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 01a94d290c12..0507174750e0 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -179,7 +179,7 @@ static int nested_vmx_failValid(struct kvm_vcpu *vcpu, * VM_INSTRUCTION_ERROR is not shadowed. Enlightened VMCS 'shadows' all * fields and thus must be synced. */ - if (to_vmx(vcpu)->nested.hv_evmcs_vmptr != EVMPTR_INVALID) + if (nested_vmx_is_evmptr12_set(to_vmx(vcpu))) to_vmx(vcpu)->nested.need_vmcs12_to_shadow_sync = true; return kvm_skip_emulated_instruction(vcpu); @@ -194,7 +194,7 @@ static int nested_vmx_fail(struct kvm_vcpu *vcpu, u32 vm_instruction_error) * can't be done if there isn't a current VMCS. */ if (vmx->nested.current_vmptr == INVALID_GPA && - !evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) + !nested_vmx_is_evmptr12_valid(vmx)) return nested_vmx_failInvalid(vcpu); return nested_vmx_failValid(vcpu, vm_instruction_error); @@ -230,7 +230,7 @@ static inline void nested_release_evmcs(struct kvm_vcpu *vcpu) struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); struct vcpu_vmx *vmx = to_vmx(vcpu); - if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) { + if (nested_vmx_is_evmptr12_valid(vmx)) { kvm_vcpu_unmap(vcpu, &vmx->nested.hv_evmcs_map, true); vmx->nested.hv_evmcs = NULL; } @@ -2123,7 +2123,7 @@ void nested_sync_vmcs12_to_shadow(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); - if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) + if (nested_vmx_is_evmptr12_valid(vmx)) copy_vmcs12_to_enlightened(vmx); else copy_vmcs12_to_shadow(vmx); @@ -2277,7 +2277,7 @@ static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct loaded_vmcs *vmcs0 u32 exec_control; u64 guest_efer = nested_vmx_calc_efer(vmx, vmcs12); - if (vmx->nested.dirty_vmcs12 || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) + if (vmx->nested.dirty_vmcs12 || nested_vmx_is_evmptr12_valid(vmx)) prepare_vmcs02_early_rare(vmx, vmcs12); /* @@ -2572,11 +2572,11 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, struct vcpu_vmx *vmx = to_vmx(vcpu); bool load_guest_pdptrs_vmcs12 = false; - if (vmx->nested.dirty_vmcs12 || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) { + if (vmx->nested.dirty_vmcs12 || nested_vmx_is_evmptr12_valid(vmx)) { prepare_vmcs02_rare(vmx, vmcs12); vmx->nested.dirty_vmcs12 = false; - load_guest_pdptrs_vmcs12 = !evmptr_is_valid(vmx->nested.hv_evmcs_vmptr) || + load_guest_pdptrs_vmcs12 = !nested_vmx_is_evmptr12_valid(vmx) || !(vmx->nested.hv_evmcs->hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1); } @@ -2699,7 +2699,7 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, * bits when it changes a field in eVMCS. Mark all fields as clean * here. */ - if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) + if (nested_vmx_is_evmptr12_valid(vmx)) vmx->nested.hv_evmcs->hv_clean_fields |= HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; @@ -3579,7 +3579,7 @@ vmentry_fail_vmexit: load_vmcs12_host_state(vcpu, vmcs12); vmcs12->vm_exit_reason = exit_reason.full; - if (enable_shadow_vmcs || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) + if (enable_shadow_vmcs || nested_vmx_is_evmptr12_valid(vmx)) vmx->nested.need_vmcs12_to_shadow_sync = true; return NVMX_VMENTRY_VMEXIT; } @@ -3610,7 +3610,7 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) if (CC(evmptrld_status == EVMPTRLD_VMFAIL)) return nested_vmx_failInvalid(vcpu); - if (CC(!evmptr_is_valid(vmx->nested.hv_evmcs_vmptr) && + if (CC(!nested_vmx_is_evmptr12_valid(vmx) && vmx->nested.current_vmptr == INVALID_GPA)) return nested_vmx_failInvalid(vcpu); @@ -3625,7 +3625,7 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) if (CC(vmcs12->hdr.shadow_vmcs)) return nested_vmx_failInvalid(vcpu); - if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) { + if (nested_vmx_is_evmptr12_valid(vmx)) { copy_enlightened_to_vmcs12(vmx, vmx->nested.hv_evmcs->hv_clean_fields); /* Enlightened VMCS doesn't have launch state */ vmcs12->launch_state = !launch; @@ -4370,11 +4370,11 @@ static void sync_vmcs02_to_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { struct vcpu_vmx *vmx = to_vmx(vcpu); - if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) + if (nested_vmx_is_evmptr12_valid(vmx)) sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12); vmx->nested.need_sync_vmcs02_to_vmcs12_rare = - !evmptr_is_valid(vmx->nested.hv_evmcs_vmptr); + !nested_vmx_is_evmptr12_valid(vmx); vmcs12->guest_cr0 = vmcs12_guest_cr0(vcpu, vmcs12); vmcs12->guest_cr4 = vmcs12_guest_cr4(vcpu, vmcs12); @@ -4897,7 +4897,7 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, } if ((vm_exit_reason != -1) && - (enable_shadow_vmcs || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr))) + (enable_shadow_vmcs || nested_vmx_is_evmptr12_valid(vmx))) vmx->nested.need_vmcs12_to_shadow_sync = true; /* in case we halted in L2 */ @@ -5390,7 +5390,7 @@ static int handle_vmread(struct kvm_vcpu *vcpu) /* Decode instruction info and find the field to read */ field = kvm_register_read(vcpu, (((instr_info) >> 28) & 0xf)); - if (!evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) { + if (!nested_vmx_is_evmptr12_valid(vmx)) { /* * In VMX non-root operation, when the VMCS-link pointer is INVALID_GPA, * any VMREAD sets the ALU flags for VMfailInvalid. @@ -5616,7 +5616,7 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu) return nested_vmx_fail(vcpu, VMXERR_VMPTRLD_VMXON_POINTER); /* Forbid normal VMPTRLD if Enlightened version was used */ - if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) + if (nested_vmx_is_evmptr12_valid(vmx)) return 1; if (vmx->nested.current_vmptr != vmptr) { @@ -5679,7 +5679,7 @@ static int handle_vmptrst(struct kvm_vcpu *vcpu) if (!nested_vmx_check_permission(vcpu)) return 1; - if (unlikely(evmptr_is_valid(to_vmx(vcpu)->nested.hv_evmcs_vmptr))) + if (unlikely(nested_vmx_is_evmptr12_valid(to_vmx(vcpu)))) return 1; if (get_vmx_mem_address(vcpu, exit_qual, instr_info, @@ -6467,7 +6467,7 @@ static int vmx_get_nested_state(struct kvm_vcpu *vcpu, kvm_state.size += sizeof(user_vmx_nested_state->vmcs12); /* 'hv_evmcs_vmptr' can also be EVMPTR_MAP_PENDING here */ - if (vmx->nested.hv_evmcs_vmptr != EVMPTR_INVALID) + if (nested_vmx_is_evmptr12_set(vmx)) kvm_state.flags |= KVM_STATE_NESTED_EVMCS; if (is_guest_mode(vcpu) && @@ -6523,7 +6523,7 @@ static int vmx_get_nested_state(struct kvm_vcpu *vcpu, } else { copy_vmcs02_to_vmcs12_rare(vcpu, get_vmcs12(vcpu)); if (!vmx->nested.need_vmcs12_to_shadow_sync) { - if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) + if (nested_vmx_is_evmptr12_valid(vmx)) /* * L1 hypervisor is not obliged to keep eVMCS * clean fields data always up-to-date while diff --git a/arch/x86/kvm/vmx/nested.h b/arch/x86/kvm/vmx/nested.h index b0f2e26c1aea..cce4e2aa30fb 100644 --- a/arch/x86/kvm/vmx/nested.h +++ b/arch/x86/kvm/vmx/nested.h @@ -58,7 +58,7 @@ static inline int vmx_has_valid_vmcs12(struct kvm_vcpu *vcpu) /* 'hv_evmcs_vmptr' can also be EVMPTR_MAP_PENDING here */ return vmx->nested.current_vmptr != -1ull || - vmx->nested.hv_evmcs_vmptr != EVMPTR_INVALID; + nested_vmx_is_evmptr12_set(vmx); } static inline u16 nested_get_vpid02(struct kvm_vcpu *vcpu) From c98842b26c233318bb18c77cc6e25859fe76c80e Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Tue, 5 Dec 2023 11:36:28 +0100 Subject: [PATCH 121/336] KVM: nVMX: Introduce accessor to get Hyper-V eVMCS pointer There's a number of 'vmx->nested.hv_evmcs' accesses in nested.c, introduce 'nested_vmx_evmcs()' accessor to hide them all in !CONFIG_KVM_HYPERV case. No functional change intended. Reviewed-by: Maxim Levitsky Tested-by: Jeremi Piotrowski Signed-off-by: Vitaly Kuznetsov Link: https://lore.kernel.org/r/20231205103630.1391318-15-vkuznets@redhat.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/hyperv.h | 10 ++++++++++ arch/x86/kvm/vmx/nested.c | 33 ++++++++++++++++++--------------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/arch/x86/kvm/vmx/hyperv.h b/arch/x86/kvm/vmx/hyperv.h index 71e90a16f183..a87407412615 100644 --- a/arch/x86/kvm/vmx/hyperv.h +++ b/arch/x86/kvm/vmx/hyperv.h @@ -37,6 +37,11 @@ static inline bool nested_vmx_is_evmptr12_set(struct vcpu_vmx *vmx) return evmptr_is_set(vmx->nested.hv_evmcs_vmptr); } +static inline struct hv_enlightened_vmcs *nested_vmx_evmcs(struct vcpu_vmx *vmx) +{ + return vmx->nested.hv_evmcs; +} + static inline bool guest_cpuid_has_evmcs(struct kvm_vcpu *vcpu) { /* @@ -75,6 +80,11 @@ static inline bool nested_vmx_is_evmptr12_set(struct vcpu_vmx *vmx) { return false; } + +static inline struct hv_enlightened_vmcs *nested_vmx_evmcs(struct vcpu_vmx *vmx) +{ + return NULL; +} #endif #endif /* __KVM_X86_VMX_HYPERV_H */ diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 0507174750e0..4e872863a0c9 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -263,7 +263,7 @@ static bool nested_evmcs_handle_vmclear(struct kvm_vcpu *vcpu, gpa_t vmptr) !evmptr_is_valid(nested_get_evmptr(vcpu))) return false; - if (vmx->nested.hv_evmcs && vmptr == vmx->nested.hv_evmcs_vmptr) + if (nested_vmx_evmcs(vmx) && vmptr == vmx->nested.hv_evmcs_vmptr) nested_release_evmcs(vcpu); return true; @@ -601,7 +601,6 @@ static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, int msr; unsigned long *msr_bitmap_l1; unsigned long *msr_bitmap_l0 = vmx->nested.vmcs02.msr_bitmap; - struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; struct kvm_host_map *map = &vmx->nested.msr_bitmap_map; /* Nothing to do if the MSR bitmap is not in use. */ @@ -617,10 +616,13 @@ static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, * - Nested hypervisor (L1) has enabled 'Enlightened MSR Bitmap' feature * and tells KVM (L0) there were no changes in MSR bitmap for L2. */ - if (!vmx->nested.force_msr_bitmap_recalc && evmcs && - evmcs->hv_enlightenments_control.msr_bitmap && - evmcs->hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP) - return true; + if (!vmx->nested.force_msr_bitmap_recalc) { + struct hv_enlightened_vmcs *evmcs = nested_vmx_evmcs(vmx); + + if (evmcs && evmcs->hv_enlightenments_control.msr_bitmap && + evmcs->hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP) + return true; + } if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->msr_bitmap), map)) return false; @@ -1603,7 +1605,7 @@ static void copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx, u32 hv_clean_fields { #ifdef CONFIG_KVM_HYPERV struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; - struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; + struct hv_enlightened_vmcs *evmcs = nested_vmx_evmcs(vmx); struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(&vmx->vcpu); /* HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE */ @@ -1851,7 +1853,7 @@ static void copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx) { #ifdef CONFIG_KVM_HYPERV struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; - struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; + struct hv_enlightened_vmcs *evmcs = nested_vmx_evmcs(vmx); /* * Should not be changed by KVM: @@ -2438,7 +2440,7 @@ static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct loaded_vmcs *vmcs0 static void prepare_vmcs02_rare(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) { - struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs; + struct hv_enlightened_vmcs *hv_evmcs = nested_vmx_evmcs(vmx); if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2)) { @@ -2570,6 +2572,7 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, enum vm_entry_failure_code *entry_failure_code) { struct vcpu_vmx *vmx = to_vmx(vcpu); + struct hv_enlightened_vmcs *evmcs = nested_vmx_evmcs(vmx); bool load_guest_pdptrs_vmcs12 = false; if (vmx->nested.dirty_vmcs12 || nested_vmx_is_evmptr12_valid(vmx)) { @@ -2577,8 +2580,7 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, vmx->nested.dirty_vmcs12 = false; load_guest_pdptrs_vmcs12 = !nested_vmx_is_evmptr12_valid(vmx) || - !(vmx->nested.hv_evmcs->hv_clean_fields & - HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1); + !(evmcs->hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1); } if (vmx->nested.nested_run_pending && @@ -2700,8 +2702,7 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, * here. */ if (nested_vmx_is_evmptr12_valid(vmx)) - vmx->nested.hv_evmcs->hv_clean_fields |= - HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; + evmcs->hv_clean_fields |= HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; return 0; } @@ -3626,7 +3627,9 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) return nested_vmx_failInvalid(vcpu); if (nested_vmx_is_evmptr12_valid(vmx)) { - copy_enlightened_to_vmcs12(vmx, vmx->nested.hv_evmcs->hv_clean_fields); + struct hv_enlightened_vmcs *evmcs = nested_vmx_evmcs(vmx); + + copy_enlightened_to_vmcs12(vmx, evmcs->hv_clean_fields); /* Enlightened VMCS doesn't have launch state */ vmcs12->launch_state = !launch; } else if (enable_shadow_vmcs) { @@ -5428,7 +5431,7 @@ static int handle_vmread(struct kvm_vcpu *vcpu) return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT); /* Read the field, zero-extended to a u64 value */ - value = evmcs_read_any(vmx->nested.hv_evmcs, field, offset); + value = evmcs_read_any(nested_vmx_evmcs(vmx), field, offset); } /* From 5a30f97683af802c0fa24d1cfc339f87cdb6791b Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Tue, 5 Dec 2023 11:36:29 +0100 Subject: [PATCH 122/336] KVM: nVMX: Hide more stuff under CONFIG_KVM_HYPERV 'hv_evmcs_vmptr'/'hv_evmcs_map'/'hv_evmcs' fields in 'struct nested_vmx' should not be used when !CONFIG_KVM_HYPERV, hide them when !CONFIG_KVM_HYPERV. No functional change intended. Reviewed-by: Maxim Levitsky Tested-by: Jeremi Piotrowski Signed-off-by: Vitaly Kuznetsov Link: https://lore.kernel.org/r/20231205103630.1391318-16-vkuznets@redhat.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/nested.c | 2 ++ arch/x86/kvm/vmx/vmx.c | 3 +++ arch/x86/kvm/vmx/vmx.h | 2 ++ 3 files changed, 7 insertions(+) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 4e872863a0c9..cf47b8b7f40f 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -6667,6 +6667,7 @@ static int vmx_set_nested_state(struct kvm_vcpu *vcpu, return -EINVAL; set_current_vmptr(vmx, kvm_state->hdr.vmx.vmcs12_pa); +#ifdef CONFIG_KVM_HYPERV } else if (kvm_state->flags & KVM_STATE_NESTED_EVMCS) { /* * nested_vmx_handle_enlightened_vmptrld() cannot be called @@ -6676,6 +6677,7 @@ static int vmx_set_nested_state(struct kvm_vcpu *vcpu, */ vmx->nested.hv_evmcs_vmptr = EVMPTR_MAP_PENDING; kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); +#endif } else { return -EINVAL; } diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 3ff5c44dff9d..a26603ddc968 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -4828,7 +4828,10 @@ static void __vmx_vcpu_reset(struct kvm_vcpu *vcpu) vmx->nested.posted_intr_nv = -1; vmx->nested.vmxon_ptr = INVALID_GPA; vmx->nested.current_vmptr = INVALID_GPA; + +#ifdef CONFIG_KVM_HYPERV vmx->nested.hv_evmcs_vmptr = EVMPTR_INVALID; +#endif vcpu->arch.microcode_version = 0x100000000ULL; vmx->msr_ia32_feature_control_valid_bits = FEAT_CTL_LOCKED; diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 959c6d94287f..8fe6eb2b4a34 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -241,9 +241,11 @@ struct nested_vmx { bool guest_mode; } smm; +#ifdef CONFIG_KVM_HYPERV gpa_t hv_evmcs_vmptr; struct kvm_host_map hv_evmcs_map; struct hv_enlightened_vmcs *hv_evmcs; +#endif }; struct vcpu_vmx { From 017a99a966f1183e611f0b0fa6bec40160c81813 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Tue, 5 Dec 2023 11:36:30 +0100 Subject: [PATCH 123/336] KVM: nSVM: Hide more stuff under CONFIG_KVM_HYPERV/CONFIG_HYPERV 'struct hv_vmcb_enlightenments' in VMCB only make sense when either CONFIG_KVM_HYPERV or CONFIG_HYPERV is enabled. No functional change intended. Reviewed-by: Maxim Levitsky Tested-by: Jeremi Piotrowski Signed-off-by: Vitaly Kuznetsov Link: https://lore.kernel.org/r/20231205103630.1391318-17-vkuznets@redhat.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/nested.c | 20 ++++++++++++++------ arch/x86/kvm/svm/svm.h | 2 ++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 74c04102ef01..20212aac050b 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -187,7 +187,6 @@ void recalc_intercepts(struct vcpu_svm *svm) */ static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm) { - struct hv_vmcb_enlightenments *hve = &svm->nested.ctl.hv_enlightenments; int i; /* @@ -198,11 +197,16 @@ static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm) * - Nested hypervisor (L1) is using Hyper-V emulation interface and * tells KVM (L0) there were no changes in MSR bitmap for L2. */ - if (!svm->nested.force_msr_bitmap_recalc && - kvm_hv_hypercall_enabled(&svm->vcpu) && - hve->hv_enlightenments_control.msr_bitmap && - (svm->nested.ctl.clean & BIT(HV_VMCB_NESTED_ENLIGHTENMENTS))) - goto set_msrpm_base_pa; +#ifdef CONFIG_KVM_HYPERV + if (!svm->nested.force_msr_bitmap_recalc) { + struct hv_vmcb_enlightenments *hve = &svm->nested.ctl.hv_enlightenments; + + if (kvm_hv_hypercall_enabled(&svm->vcpu) && + hve->hv_enlightenments_control.msr_bitmap && + (svm->nested.ctl.clean & BIT(HV_VMCB_NESTED_ENLIGHTENMENTS))) + goto set_msrpm_base_pa; + } +#endif if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT))) return true; @@ -230,7 +234,9 @@ static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm) svm->nested.force_msr_bitmap_recalc = false; +#ifdef CONFIG_KVM_HYPERV set_msrpm_base_pa: +#endif svm->vmcb->control.msrpm_base_pa = __sme_set(__pa(svm->nested.msrpm)); return true; @@ -378,12 +384,14 @@ void __nested_copy_vmcb_control_to_cache(struct kvm_vcpu *vcpu, to->msrpm_base_pa &= ~0x0fffULL; to->iopm_base_pa &= ~0x0fffULL; +#ifdef CONFIG_KVM_HYPERV /* Hyper-V extensions (Enlightened VMCB) */ if (kvm_hv_hypercall_enabled(vcpu)) { to->clean = from->clean; memcpy(&to->hv_enlightenments, &from->hv_enlightenments, sizeof(to->hv_enlightenments)); } +#endif } void nested_copy_vmcb_control_to_cache(struct vcpu_svm *svm, diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index be67ab7fdd10..59adff7bbf55 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -148,7 +148,9 @@ struct vmcb_ctrl_area_cached { u64 virt_ext; u32 clean; union { +#if IS_ENABLED(CONFIG_HYPERV) || IS_ENABLED(CONFIG_KVM_HYPERV) struct hv_vmcb_enlightenments hv_enlightenments; +#endif u8 reserved_sw[32]; }; }; From 6d72283526090850274d065cd5d60af732cc5fc8 Mon Sep 17 00:00:00 2001 From: Paul Durrant Date: Thu, 2 Nov 2023 16:21:28 +0000 Subject: [PATCH 124/336] KVM x86/xen: add an override for PVCLOCK_TSC_STABLE_BIT Unless explicitly told to do so (by passing 'clocksource=tsc' and 'tsc=stable:socket', and then jumping through some hoops concerning potential CPU hotplug) Xen will never use TSC as its clocksource. Hence, by default, a Xen guest will not see PVCLOCK_TSC_STABLE_BIT set in either the primary or secondary pvclock memory areas. This has led to bugs in some guest kernels which only become evident if PVCLOCK_TSC_STABLE_BIT *is* set in the pvclocks. Hence, to support such guests, give the VMM a new Xen HVM config flag to tell KVM to forcibly clear the bit in the Xen pvclocks. Signed-off-by: Paul Durrant Reviewed-by: David Woodhouse Link: https://lore.kernel.org/r/20231102162128.2353459-1-paul@xen.org Signed-off-by: Sean Christopherson --- Documentation/virt/kvm/api.rst | 6 ++++++ arch/x86/kvm/x86.c | 28 +++++++++++++++++++++++----- arch/x86/kvm/xen.c | 9 ++++++++- include/uapi/linux/kvm.h | 1 + 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 926241e23aeb..dca83c65d97f 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -8562,6 +8562,7 @@ PVHVM guests. Valid flags are:: #define KVM_XEN_HVM_CONFIG_EVTCHN_2LEVEL (1 << 4) #define KVM_XEN_HVM_CONFIG_EVTCHN_SEND (1 << 5) #define KVM_XEN_HVM_CONFIG_RUNSTATE_UPDATE_FLAG (1 << 6) + #define KVM_XEN_HVM_CONFIG_PVCLOCK_TSC_UNSTABLE (1 << 7) The KVM_XEN_HVM_CONFIG_HYPERCALL_MSR flag indicates that the KVM_XEN_HVM_CONFIG ioctl is available, for the guest to set its hypercall page. @@ -8605,6 +8606,11 @@ behave more correctly, not using the XEN_RUNSTATE_UPDATE flag until/unless specifically enabled (by the guest making the hypercall, causing the VMM to enable the KVM_XEN_ATTR_TYPE_RUNSTATE_UPDATE_FLAG attribute). +The KVM_XEN_HVM_CONFIG_PVCLOCK_TSC_UNSTABLE flag indicates that KVM supports +clearing the PVCLOCK_TSC_STABLE_BIT flag in Xen pvclock sources. This will be +done when the KVM_CAP_XEN_HVM ioctl sets the +KVM_XEN_HVM_CONFIG_PVCLOCK_TSC_UNSTABLE flag. + 8.31 KVM_CAP_PPC_MULTITCE ------------------------- diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 6d0772b47041..aa7cea9600b0 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -3104,7 +3104,8 @@ u64 get_kvmclock_ns(struct kvm *kvm) static void kvm_setup_guest_pvclock(struct kvm_vcpu *v, struct gfn_to_pfn_cache *gpc, - unsigned int offset) + unsigned int offset, + bool force_tsc_unstable) { struct kvm_vcpu_arch *vcpu = &v->arch; struct pvclock_vcpu_time_info *guest_hv_clock; @@ -3141,6 +3142,10 @@ static void kvm_setup_guest_pvclock(struct kvm_vcpu *v, } memcpy(guest_hv_clock, &vcpu->hv_clock, sizeof(*guest_hv_clock)); + + if (force_tsc_unstable) + guest_hv_clock->flags &= ~PVCLOCK_TSC_STABLE_BIT; + smp_wmb(); guest_hv_clock->version = ++vcpu->hv_clock.version; @@ -3161,6 +3166,16 @@ static int kvm_guest_time_update(struct kvm_vcpu *v) u64 tsc_timestamp, host_tsc; u8 pvclock_flags; bool use_master_clock; +#ifdef CONFIG_KVM_XEN + /* + * For Xen guests we may need to override PVCLOCK_TSC_STABLE_BIT as unless + * explicitly told to use TSC as its clocksource Xen will not set this bit. + * This default behaviour led to bugs in some guest kernels which cause + * problems if they observe PVCLOCK_TSC_STABLE_BIT in the pvclock flags. + */ + bool xen_pvclock_tsc_unstable = + ka->xen_hvm_config.flags & KVM_XEN_HVM_CONFIG_PVCLOCK_TSC_UNSTABLE; +#endif kernel_ns = 0; host_tsc = 0; @@ -3239,13 +3254,15 @@ static int kvm_guest_time_update(struct kvm_vcpu *v) vcpu->hv_clock.flags = pvclock_flags; if (vcpu->pv_time.active) - kvm_setup_guest_pvclock(v, &vcpu->pv_time, 0); + kvm_setup_guest_pvclock(v, &vcpu->pv_time, 0, false); #ifdef CONFIG_KVM_XEN if (vcpu->xen.vcpu_info_cache.active) kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_info_cache, - offsetof(struct compat_vcpu_info, time)); + offsetof(struct compat_vcpu_info, time), + xen_pvclock_tsc_unstable); if (vcpu->xen.vcpu_time_info_cache.active) - kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_time_info_cache, 0); + kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_time_info_cache, 0, + xen_pvclock_tsc_unstable); #endif kvm_hv_setup_tsc_page(v->kvm, &vcpu->hv_clock); return 0; @@ -4646,7 +4663,8 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL | KVM_XEN_HVM_CONFIG_SHARED_INFO | KVM_XEN_HVM_CONFIG_EVTCHN_2LEVEL | - KVM_XEN_HVM_CONFIG_EVTCHN_SEND; + KVM_XEN_HVM_CONFIG_EVTCHN_SEND | + KVM_XEN_HVM_CONFIG_PVCLOCK_TSC_UNSTABLE; if (sched_info_on()) r |= KVM_XEN_HVM_CONFIG_RUNSTATE | KVM_XEN_HVM_CONFIG_RUNSTATE_UPDATE_FLAG; diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index e53fad915a62..e43948b87f94 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -1162,7 +1162,9 @@ int kvm_xen_hvm_config(struct kvm *kvm, struct kvm_xen_hvm_config *xhc) { /* Only some feature flags need to be *enabled* by userspace */ u32 permitted_flags = KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL | - KVM_XEN_HVM_CONFIG_EVTCHN_SEND; + KVM_XEN_HVM_CONFIG_EVTCHN_SEND | + KVM_XEN_HVM_CONFIG_PVCLOCK_TSC_UNSTABLE; + u32 old_flags; if (xhc->flags & ~permitted_flags) return -EINVAL; @@ -1183,9 +1185,14 @@ int kvm_xen_hvm_config(struct kvm *kvm, struct kvm_xen_hvm_config *xhc) else if (!xhc->msr && kvm->arch.xen_hvm_config.msr) static_branch_slow_dec_deferred(&kvm_xen_enabled); + old_flags = kvm->arch.xen_hvm_config.flags; memcpy(&kvm->arch.xen_hvm_config, xhc, sizeof(*xhc)); mutex_unlock(&kvm->arch.xen.xen_lock); + + if ((old_flags ^ xhc->flags) & KVM_XEN_HVM_CONFIG_PVCLOCK_TSC_UNSTABLE) + kvm_make_all_cpus_request(kvm, KVM_REQ_CLOCK_UPDATE); + return 0; } diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index e9cb2df67a1d..175420b26e36 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1318,6 +1318,7 @@ struct kvm_x86_mce { #define KVM_XEN_HVM_CONFIG_EVTCHN_2LEVEL (1 << 4) #define KVM_XEN_HVM_CONFIG_EVTCHN_SEND (1 << 5) #define KVM_XEN_HVM_CONFIG_RUNSTATE_UPDATE_FLAG (1 << 6) +#define KVM_XEN_HVM_CONFIG_PVCLOCK_TSC_UNSTABLE (1 << 7) struct kvm_xen_hvm_config { __u32 flags; From 849c1816436fe359e85587fba5b69ddd3a957b31 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 8 Dec 2023 13:46:22 -0500 Subject: [PATCH 125/336] KVM: selftests: fix supported_flags for aarch64 KVM/Arm supports readonly memslots; fix the calculation of supported_flags in set_memory_region_test.c, otherwise the test fails. Signed-off-by: Paolo Bonzini --- tools/testing/selftests/kvm/set_memory_region_test.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/set_memory_region_test.c b/tools/testing/selftests/kvm/set_memory_region_test.c index 6637a0845acf..dfd1d1e22da3 100644 --- a/tools/testing/selftests/kvm/set_memory_region_test.c +++ b/tools/testing/selftests/kvm/set_memory_region_test.c @@ -333,9 +333,11 @@ static void test_invalid_memory_region_flags(void) struct kvm_vm *vm; int r, i; -#ifdef __x86_64__ +#if defined __aarch64__ || defined __x86_64__ supported_flags |= KVM_MEM_READONLY; +#endif +#ifdef __x86_64__ if (kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM)) vm = vm_create_barebones_protected_vm(); else From 80583d0cfd8ff44d60a5fa76a6bf3c08eb67c328 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 8 Dec 2023 13:46:48 -0500 Subject: [PATCH 126/336] KVM: guest-memfd: fix unused-function warning With migration disabled, one function becomes unused: virt/kvm/guest_memfd.c:262:12: error: 'kvm_gmem_migrate_folio' defined but not used [-Werror=unused-function] 262 | static int kvm_gmem_migrate_folio(struct address_space *mapping, | ^~~~~~~~~~~~~~~~~~~~~~ Remove the #ifdef around the reference so that fallback_migrate_folio() is never used. The gmem implementation of the hook is trivial; since the gmem mapping is unmovable, the pages should not be migrated anyway. Fixes: a7800aa80ea4 ("KVM: Add KVM_CREATE_GUEST_MEMFD ioctl() for guest-specific backing memory") Reported-by: Arnd Bergmann Suggested-by: Sean Christopherson Signed-off-by: Paolo Bonzini --- virt/kvm/guest_memfd.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index b99272396119..c2e2371720a9 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -300,9 +300,7 @@ static int kvm_gmem_error_page(struct address_space *mapping, struct page *page) static const struct address_space_operations kvm_gmem_aops = { .dirty_folio = noop_dirty_folio, -#ifdef CONFIG_MIGRATION .migrate_folio = kvm_gmem_migrate_folio, -#endif .error_remove_page = kvm_gmem_error_page, }; From 75d6872907cc8827248f4d5f803c03e1175c3158 Mon Sep 17 00:00:00 2001 From: Adrien Leravat Date: Sun, 3 Dec 2023 12:06:04 -0800 Subject: [PATCH 127/336] doc: rmpsg: Update with rpmsg_endpoint It seems the documentation was not updated when `rpmsg_sendto` and related switched from `rpmsg_channel` to `rpmsg_endpoint`. This change updates the proper calls, text, and the sample. Signed-off-by: Adrien Leravat Link: https://lore.kernel.org/r/20231203200606.255302-1-adrien.leravat@gmail.com Signed-off-by: Mathieu Poirier --- Documentation/staging/rpmsg.rst | 50 ++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/Documentation/staging/rpmsg.rst b/Documentation/staging/rpmsg.rst index 1ce353cb232a..dba3e5f65612 100644 --- a/Documentation/staging/rpmsg.rst +++ b/Documentation/staging/rpmsg.rst @@ -68,13 +68,14 @@ User API :: - int rpmsg_send(struct rpmsg_channel *rpdev, void *data, int len); + int rpmsg_send(struct rpmsg_endpoint *ept, void *data, int len); -sends a message across to the remote processor on a given channel. -The caller should specify the channel, the data it wants to send, +sends a message across to the remote processor from the given endpoint. +The caller should specify the endpoint, the data it wants to send, and its length (in bytes). The message will be sent on the specified -channel, i.e. its source and destination address fields will be -set to the channel's src and dst addresses. +endpoint's channel, i.e. its source and destination address fields will be +respectively set to the endpoint's src address and its parent channel +dst addresses. In case there are no TX buffers available, the function will block until one becomes available (i.e. until the remote processor consumes @@ -87,17 +88,18 @@ Returns 0 on success and an appropriate error value on failure. :: - int rpmsg_sendto(struct rpmsg_channel *rpdev, void *data, int len, u32 dst); + int rpmsg_sendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst); -sends a message across to the remote processor on a given channel, +sends a message across to the remote processor from a given endpoint, to a destination address provided by the caller. -The caller should specify the channel, the data it wants to send, +The caller should specify the endpoint, the data it wants to send, its length (in bytes), and an explicit destination address. The message will then be sent to the remote processor to which the -channel belongs, using the channel's src address, and the user-provided -dst address (thus the channel's dst address will be ignored). +endpoints's channel belongs, using the endpoints's src address, +and the user-provided dst address (thus the channel's dst address +will be ignored). In case there are no TX buffers available, the function will block until one becomes available (i.e. until the remote processor consumes @@ -110,18 +112,19 @@ Returns 0 on success and an appropriate error value on failure. :: - int rpmsg_send_offchannel(struct rpmsg_channel *rpdev, u32 src, u32 dst, + int rpmsg_send_offchannel(struct rpmsg_endpoint *ept, u32 src, u32 dst, void *data, int len); sends a message across to the remote processor, using the src and dst addresses provided by the user. -The caller should specify the channel, the data it wants to send, +The caller should specify the endpoint, the data it wants to send, its length (in bytes), and explicit source and destination addresses. The message will then be sent to the remote processor to which the -channel belongs, but the channel's src and dst addresses will be -ignored (and the user-provided addresses will be used instead). +endpoint's channel belongs, but the endpoint's src and channel dst +addresses will be ignored (and the user-provided addresses will +be used instead). In case there are no TX buffers available, the function will block until one becomes available (i.e. until the remote processor consumes @@ -134,13 +137,14 @@ Returns 0 on success and an appropriate error value on failure. :: - int rpmsg_trysend(struct rpmsg_channel *rpdev, void *data, int len); + int rpmsg_trysend(struct rpmsg_endpoint *ept, void *data, int len); -sends a message across to the remote processor on a given channel. -The caller should specify the channel, the data it wants to send, +sends a message across to the remote processor from a given endpoint. +The caller should specify the endpoint, the data it wants to send, and its length (in bytes). The message will be sent on the specified -channel, i.e. its source and destination address fields will be -set to the channel's src and dst addresses. +endpoint's channel, i.e. its source and destination address fields will be +respectively set to the endpoint's src address and its parent channel +dst addresses. In case there are no TX buffers available, the function will immediately return -ENOMEM without waiting until one becomes available. @@ -150,10 +154,10 @@ Returns 0 on success and an appropriate error value on failure. :: - int rpmsg_trysendto(struct rpmsg_channel *rpdev, void *data, int len, u32 dst) + int rpmsg_trysendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst) -sends a message across to the remote processor on a given channel, +sends a message across to the remote processor from a given endoint, to a destination address provided by the user. The user should specify the channel, the data it wants to send, @@ -171,7 +175,7 @@ Returns 0 on success and an appropriate error value on failure. :: - int rpmsg_trysend_offchannel(struct rpmsg_channel *rpdev, u32 src, u32 dst, + int rpmsg_trysend_offchannel(struct rpmsg_endpoint *ept, u32 src, u32 dst, void *data, int len); @@ -284,7 +288,7 @@ content to the console. dev_info(&rpdev->dev, "chnl: 0x%x -> 0x%x\n", rpdev->src, rpdev->dst); /* send a message on our channel */ - err = rpmsg_send(rpdev, "hello!", 6); + err = rpmsg_send(rpdev->ept, "hello!", 6); if (err) { pr_err("rpmsg_send failed: %d\n", err); return err; From 8132d887a7023b212f242a51ae89281c69fde996 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 18 Oct 2023 12:11:56 -0400 Subject: [PATCH 128/336] KVM: remove CONFIG_HAVE_KVM_EVENTFD virt/kvm/eventfd.c is compiled unconditionally, meaning that the ioeventfds member of struct kvm is accessed unconditionally. CONFIG_HAVE_KVM_EVENTFD therefore must be defined for KVM common code to compile successfully, remove it. Signed-off-by: Paolo Bonzini --- arch/arm64/kvm/Kconfig | 1 - arch/loongarch/kvm/Kconfig | 1 - arch/mips/kvm/Kconfig | 1 - arch/powerpc/kvm/Kconfig | 1 - arch/riscv/kvm/Kconfig | 1 - arch/s390/kvm/Kconfig | 1 - arch/x86/kvm/Kconfig | 1 - include/linux/kvm_host.h | 30 +----------------------------- virt/kvm/Kconfig | 5 +---- 9 files changed, 2 insertions(+), 40 deletions(-) diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig index 1a777715199f..87bd96c1254f 100644 --- a/arch/arm64/kvm/Kconfig +++ b/arch/arm64/kvm/Kconfig @@ -29,7 +29,6 @@ menuconfig KVM select KVM_GENERIC_DIRTYLOG_READ_PROTECT select KVM_XFER_TO_GUEST_WORK select KVM_VFIO - select HAVE_KVM_EVENTFD select HAVE_KVM_IRQFD select HAVE_KVM_DIRTY_RING_ACQ_REL select NEED_KVM_DIRTY_RING_WITH_BITMAP diff --git a/arch/loongarch/kvm/Kconfig b/arch/loongarch/kvm/Kconfig index f22bae89b07d..daba4cd5e87d 100644 --- a/arch/loongarch/kvm/Kconfig +++ b/arch/loongarch/kvm/Kconfig @@ -22,7 +22,6 @@ config KVM depends on AS_HAS_LVZ_EXTENSION depends on HAVE_KVM select HAVE_KVM_DIRTY_RING_ACQ_REL - select HAVE_KVM_EVENTFD select HAVE_KVM_VCPU_ASYNC_IOCTL select KVM_GENERIC_DIRTYLOG_READ_PROTECT select KVM_GENERIC_HARDWARE_ENABLING diff --git a/arch/mips/kvm/Kconfig b/arch/mips/kvm/Kconfig index c04987d2ed2e..428141b0b48f 100644 --- a/arch/mips/kvm/Kconfig +++ b/arch/mips/kvm/Kconfig @@ -22,7 +22,6 @@ config KVM select EXPORT_UASM select PREEMPT_NOTIFIERS select KVM_GENERIC_DIRTYLOG_READ_PROTECT - select HAVE_KVM_EVENTFD select HAVE_KVM_VCPU_ASYNC_IOCTL select KVM_MMIO select KVM_GENERIC_MMU_NOTIFIER diff --git a/arch/powerpc/kvm/Kconfig b/arch/powerpc/kvm/Kconfig index b33358ee6424..0f4e6e7ba35d 100644 --- a/arch/powerpc/kvm/Kconfig +++ b/arch/powerpc/kvm/Kconfig @@ -20,7 +20,6 @@ if VIRTUALIZATION config KVM bool select PREEMPT_NOTIFIERS - select HAVE_KVM_EVENTFD select HAVE_KVM_VCPU_ASYNC_IOCTL select KVM_VFIO select IRQ_BYPASS_MANAGER diff --git a/arch/riscv/kvm/Kconfig b/arch/riscv/kvm/Kconfig index ae2e05f050ec..be264b78487d 100644 --- a/arch/riscv/kvm/Kconfig +++ b/arch/riscv/kvm/Kconfig @@ -20,7 +20,6 @@ if VIRTUALIZATION config KVM tristate "Kernel-based Virtual Machine (KVM) support (EXPERIMENTAL)" depends on RISCV_SBI && MMU - select HAVE_KVM_EVENTFD select HAVE_KVM_IRQCHIP select HAVE_KVM_IRQFD select HAVE_KVM_IRQ_ROUTING diff --git a/arch/s390/kvm/Kconfig b/arch/s390/kvm/Kconfig index 45fdf2a9b2e3..ed567b858535 100644 --- a/arch/s390/kvm/Kconfig +++ b/arch/s390/kvm/Kconfig @@ -23,7 +23,6 @@ config KVM select PREEMPT_NOTIFIERS select HAVE_KVM_CPU_RELAX_INTERCEPT select HAVE_KVM_VCPU_ASYNC_IOCTL - select HAVE_KVM_EVENTFD select KVM_ASYNC_PF select KVM_ASYNC_PF_SYNC select HAVE_KVM_IRQCHIP diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index c1716e83d176..088c66bee98f 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -33,7 +33,6 @@ config KVM select IRQ_BYPASS_MANAGER select HAVE_KVM_IRQ_BYPASS select HAVE_KVM_IRQ_ROUTING - select HAVE_KVM_EVENTFD select KVM_ASYNC_PF select USER_RETURN_NOTIFIER select KVM_MMIO diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index ea1523a7b83a..3fe5a6be7768 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -782,7 +782,6 @@ struct kvm { struct list_head vm_list; struct mutex lock; struct kvm_io_bus __rcu *buses[KVM_NR_BUSES]; -#ifdef CONFIG_HAVE_KVM_EVENTFD struct { spinlock_t lock; struct list_head items; @@ -791,7 +790,6 @@ struct kvm { struct mutex resampler_lock; } irqfds; struct list_head ioeventfds; -#endif struct kvm_vm_stat stat; struct kvm_arch arch; refcount_t users_count; @@ -2056,8 +2054,6 @@ static inline void kvm_free_irq_routing(struct kvm *kvm) {} int kvm_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi); -#ifdef CONFIG_HAVE_KVM_EVENTFD - void kvm_eventfd_init(struct kvm *kvm); int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args); @@ -2082,31 +2078,7 @@ static inline bool kvm_notify_irqfd_resampler(struct kvm *kvm, { return false; } -#endif - -#else - -static inline void kvm_eventfd_init(struct kvm *kvm) {} - -static inline int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args) -{ - return -EINVAL; -} - -static inline void kvm_irqfd_release(struct kvm *kvm) {} - -#ifdef CONFIG_HAVE_KVM_IRQCHIP -static inline void kvm_irq_routing_update(struct kvm *kvm) -{ -} -#endif - -static inline int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args) -{ - return -ENOSYS; -} - -#endif /* CONFIG_HAVE_KVM_EVENTFD */ +#endif /* CONFIG_HAVE_KVM_IRQFD */ void kvm_arch_irq_routing_update(struct kvm *kvm); diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig index 2c964586aa14..72e0cac864f9 100644 --- a/virt/kvm/Kconfig +++ b/virt/kvm/Kconfig @@ -3,6 +3,7 @@ config HAVE_KVM bool + select EVENTFD config HAVE_KVM_PFNCACHE bool @@ -39,10 +40,6 @@ config NEED_KVM_DIRTY_RING_WITH_BITMAP bool depends on HAVE_KVM_DIRTY_RING -config HAVE_KVM_EVENTFD - bool - select EVENTFD - config KVM_MMIO bool From c5b31cc2371728ddefe9baf1d036aeb630a25d96 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 18 Oct 2023 12:07:32 -0400 Subject: [PATCH 129/336] KVM: remove CONFIG_HAVE_KVM_IRQFD All platforms with a kernel irqchip have support for irqfd. Unify the two configuration items so that userspace can expect to use irqfd to inject interrupts into the irqchip. Signed-off-by: Paolo Bonzini --- arch/arm64/kvm/Kconfig | 1 - arch/powerpc/kvm/Kconfig | 2 -- arch/powerpc/kvm/powerpc.c | 2 +- arch/riscv/kvm/Kconfig | 1 - arch/s390/kvm/Kconfig | 1 - arch/x86/kvm/Kconfig | 1 - include/linux/kvm_host.h | 9 ++++----- include/trace/events/kvm.h | 8 ++++---- virt/kvm/Kconfig | 3 --- virt/kvm/eventfd.c | 6 +++--- virt/kvm/kvm_main.c | 4 ++-- 11 files changed, 14 insertions(+), 24 deletions(-) diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig index 87bd96c1254f..b07c60c9737d 100644 --- a/arch/arm64/kvm/Kconfig +++ b/arch/arm64/kvm/Kconfig @@ -29,7 +29,6 @@ menuconfig KVM select KVM_GENERIC_DIRTYLOG_READ_PROTECT select KVM_XFER_TO_GUEST_WORK select KVM_VFIO - select HAVE_KVM_IRQFD select HAVE_KVM_DIRTY_RING_ACQ_REL select NEED_KVM_DIRTY_RING_WITH_BITMAP select HAVE_KVM_MSI diff --git a/arch/powerpc/kvm/Kconfig b/arch/powerpc/kvm/Kconfig index 0f4e6e7ba35d..b47196085a42 100644 --- a/arch/powerpc/kvm/Kconfig +++ b/arch/powerpc/kvm/Kconfig @@ -224,7 +224,6 @@ config KVM_MPIC bool "KVM in-kernel MPIC emulation" depends on KVM && PPC_E500 select HAVE_KVM_IRQCHIP - select HAVE_KVM_IRQFD select HAVE_KVM_IRQ_ROUTING select HAVE_KVM_MSI help @@ -237,7 +236,6 @@ config KVM_XICS bool "KVM in-kernel XICS emulation" depends on KVM_BOOK3S_64 && !KVM_MPIC select HAVE_KVM_IRQCHIP - select HAVE_KVM_IRQFD default y help Include support for the XICS (eXternal Interrupt Controller diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index c39945a7fce3..c3f82e238b70 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -578,7 +578,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) break; #endif -#ifdef CONFIG_HAVE_KVM_IRQFD +#ifdef CONFIG_HAVE_KVM_IRQCHIP case KVM_CAP_IRQFD_RESAMPLE: r = !xive_enabled(); break; diff --git a/arch/riscv/kvm/Kconfig b/arch/riscv/kvm/Kconfig index be264b78487d..2b0b51035302 100644 --- a/arch/riscv/kvm/Kconfig +++ b/arch/riscv/kvm/Kconfig @@ -21,7 +21,6 @@ config KVM tristate "Kernel-based Virtual Machine (KVM) support (EXPERIMENTAL)" depends on RISCV_SBI && MMU select HAVE_KVM_IRQCHIP - select HAVE_KVM_IRQFD select HAVE_KVM_IRQ_ROUTING select HAVE_KVM_MSI select HAVE_KVM_VCPU_ASYNC_IOCTL diff --git a/arch/s390/kvm/Kconfig b/arch/s390/kvm/Kconfig index ed567b858535..bb6d90351119 100644 --- a/arch/s390/kvm/Kconfig +++ b/arch/s390/kvm/Kconfig @@ -26,7 +26,6 @@ config KVM select KVM_ASYNC_PF select KVM_ASYNC_PF_SYNC select HAVE_KVM_IRQCHIP - select HAVE_KVM_IRQFD select HAVE_KVM_IRQ_ROUTING select HAVE_KVM_INVALID_WAKEUPS select HAVE_KVM_NO_POLL diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index 088c66bee98f..b07247b0b958 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -27,7 +27,6 @@ config KVM select KVM_GENERIC_MMU_NOTIFIER select HAVE_KVM_IRQCHIP select HAVE_KVM_PFNCACHE - select HAVE_KVM_IRQFD select HAVE_KVM_DIRTY_RING_TSO select HAVE_KVM_DIRTY_RING_ACQ_REL select IRQ_BYPASS_MANAGER diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 3fe5a6be7768..1bba24a13ec9 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -805,8 +805,7 @@ struct kvm { * Update side is protected by irq_lock. */ struct kvm_irq_routing_table __rcu *irq_routing; -#endif -#ifdef CONFIG_HAVE_KVM_IRQFD + struct hlist_head irq_ack_notifier_list; #endif @@ -996,7 +995,7 @@ static inline void kvm_arch_post_irq_routing_update(struct kvm *kvm) } #endif -#ifdef CONFIG_HAVE_KVM_IRQFD +#ifdef CONFIG_HAVE_KVM_IRQCHIP int kvm_irqfd_init(void); void kvm_irqfd_exit(void); #else @@ -2057,7 +2056,7 @@ int kvm_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi); void kvm_eventfd_init(struct kvm *kvm); int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args); -#ifdef CONFIG_HAVE_KVM_IRQFD +#ifdef CONFIG_HAVE_KVM_IRQCHIP int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args); void kvm_irqfd_release(struct kvm *kvm); bool kvm_notify_irqfd_resampler(struct kvm *kvm, @@ -2078,7 +2077,7 @@ static inline bool kvm_notify_irqfd_resampler(struct kvm *kvm, { return false; } -#endif /* CONFIG_HAVE_KVM_IRQFD */ +#endif /* CONFIG_HAVE_KVM_IRQCHIP */ void kvm_arch_irq_routing_update(struct kvm *kvm); diff --git a/include/trace/events/kvm.h b/include/trace/events/kvm.h index 3bd31ea23fee..011fba6b5552 100644 --- a/include/trace/events/kvm.h +++ b/include/trace/events/kvm.h @@ -62,7 +62,7 @@ TRACE_EVENT(kvm_vcpu_wakeup, __entry->valid ? "valid" : "invalid") ); -#if defined(CONFIG_HAVE_KVM_IRQFD) +#if defined(CONFIG_HAVE_KVM_IRQCHIP) TRACE_EVENT(kvm_set_irq, TP_PROTO(unsigned int gsi, int level, int irq_source_id), TP_ARGS(gsi, level, irq_source_id), @@ -82,7 +82,7 @@ TRACE_EVENT(kvm_set_irq, TP_printk("gsi %u level %d source %d", __entry->gsi, __entry->level, __entry->irq_source_id) ); -#endif /* defined(CONFIG_HAVE_KVM_IRQFD) */ +#endif /* defined(CONFIG_HAVE_KVM_IRQCHIP) */ #if defined(__KVM_HAVE_IOAPIC) #define kvm_deliver_mode \ @@ -170,7 +170,7 @@ TRACE_EVENT(kvm_msi_set_irq, #endif /* defined(__KVM_HAVE_IOAPIC) */ -#if defined(CONFIG_HAVE_KVM_IRQFD) +#if defined(CONFIG_HAVE_KVM_IRQCHIP) #ifdef kvm_irqchips #define kvm_ack_irq_string "irqchip %s pin %u" @@ -197,7 +197,7 @@ TRACE_EVENT(kvm_ack_irq, TP_printk(kvm_ack_irq_string, kvm_ack_irq_parm) ); -#endif /* defined(CONFIG_HAVE_KVM_IRQFD) */ +#endif /* defined(CONFIG_HAVE_KVM_IRQCHIP) */ diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig index 72e0cac864f9..6793211a0b64 100644 --- a/virt/kvm/Kconfig +++ b/virt/kvm/Kconfig @@ -11,9 +11,6 @@ config HAVE_KVM_PFNCACHE config HAVE_KVM_IRQCHIP bool -config HAVE_KVM_IRQFD - bool - config HAVE_KVM_IRQ_ROUTING bool diff --git a/virt/kvm/eventfd.c b/virt/kvm/eventfd.c index 89912a17f5d5..19534156d48c 100644 --- a/virt/kvm/eventfd.c +++ b/virt/kvm/eventfd.c @@ -28,7 +28,7 @@ #include -#ifdef CONFIG_HAVE_KVM_IRQFD +#ifdef CONFIG_HAVE_KVM_IRQCHIP static struct workqueue_struct *irqfd_cleanup_wq; @@ -531,7 +531,7 @@ void kvm_unregister_irq_ack_notifier(struct kvm *kvm, void kvm_eventfd_init(struct kvm *kvm) { -#ifdef CONFIG_HAVE_KVM_IRQFD +#ifdef CONFIG_HAVE_KVM_IRQCHIP spin_lock_init(&kvm->irqfds.lock); INIT_LIST_HEAD(&kvm->irqfds.items); INIT_LIST_HEAD(&kvm->irqfds.resampler_list); @@ -540,7 +540,7 @@ kvm_eventfd_init(struct kvm *kvm) INIT_LIST_HEAD(&kvm->ioeventfds); } -#ifdef CONFIG_HAVE_KVM_IRQFD +#ifdef CONFIG_HAVE_KVM_IRQCHIP /* * shutdown any irqfd's that match fd+gsi */ diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 8758cb799e18..a20cf1f9ad29 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1273,7 +1273,7 @@ static struct kvm *kvm_create_vm(unsigned long type, const char *fdname) if (r) goto out_err_no_disable; -#ifdef CONFIG_HAVE_KVM_IRQFD +#ifdef CONFIG_HAVE_KVM_IRQCHIP INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list); #endif @@ -4826,7 +4826,7 @@ static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) #ifdef CONFIG_HAVE_KVM_MSI case KVM_CAP_SIGNAL_MSI: #endif -#ifdef CONFIG_HAVE_KVM_IRQFD +#ifdef CONFIG_HAVE_KVM_IRQCHIP case KVM_CAP_IRQFD: #endif case KVM_CAP_IOEVENTFD_ANY_LENGTH: From a5d3df8ae13fada772fbce952e9ee7b3433dba16 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 8 Nov 2023 10:34:03 +0100 Subject: [PATCH 130/336] KVM: remove deprecated UAPIs The deprecated interfaces were removed 15 years ago. KVM's device assignment was deprecated in 4.2 and removed 6.5 years ago; the only interest might be in compiling ancient versions of QEMU, but QEMU has been using its own imported copy of the kernel headers since June 2011. So again we go into archaeology territory; just remove the cruft. Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/api.rst | 12 ----- include/uapi/linux/kvm.h | 90 ---------------------------------- virt/kvm/kvm_main.c | 5 -- 3 files changed, 107 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 926241e23aeb..9326af2a4869 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -627,18 +627,6 @@ interrupt number dequeues the interrupt. This is an asynchronous vcpu ioctl and can be invoked from any thread. -4.17 KVM_DEBUG_GUEST --------------------- - -:Capability: basic -:Architectures: none -:Type: vcpu ioctl -:Parameters: none) -:Returns: -1 on error - -Support for this has been removed. Use KVM_SET_GUEST_DEBUG instead. - - 4.18 KVM_GET_MSRS ----------------- diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index e9cb2df67a1d..b1f92a0edc35 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -16,76 +16,6 @@ #define KVM_API_VERSION 12 -/* *** Deprecated interfaces *** */ - -#define KVM_TRC_SHIFT 16 - -#define KVM_TRC_ENTRYEXIT (1 << KVM_TRC_SHIFT) -#define KVM_TRC_HANDLER (1 << (KVM_TRC_SHIFT + 1)) - -#define KVM_TRC_VMENTRY (KVM_TRC_ENTRYEXIT + 0x01) -#define KVM_TRC_VMEXIT (KVM_TRC_ENTRYEXIT + 0x02) -#define KVM_TRC_PAGE_FAULT (KVM_TRC_HANDLER + 0x01) - -#define KVM_TRC_HEAD_SIZE 12 -#define KVM_TRC_CYCLE_SIZE 8 -#define KVM_TRC_EXTRA_MAX 7 - -#define KVM_TRC_INJ_VIRQ (KVM_TRC_HANDLER + 0x02) -#define KVM_TRC_REDELIVER_EVT (KVM_TRC_HANDLER + 0x03) -#define KVM_TRC_PEND_INTR (KVM_TRC_HANDLER + 0x04) -#define KVM_TRC_IO_READ (KVM_TRC_HANDLER + 0x05) -#define KVM_TRC_IO_WRITE (KVM_TRC_HANDLER + 0x06) -#define KVM_TRC_CR_READ (KVM_TRC_HANDLER + 0x07) -#define KVM_TRC_CR_WRITE (KVM_TRC_HANDLER + 0x08) -#define KVM_TRC_DR_READ (KVM_TRC_HANDLER + 0x09) -#define KVM_TRC_DR_WRITE (KVM_TRC_HANDLER + 0x0A) -#define KVM_TRC_MSR_READ (KVM_TRC_HANDLER + 0x0B) -#define KVM_TRC_MSR_WRITE (KVM_TRC_HANDLER + 0x0C) -#define KVM_TRC_CPUID (KVM_TRC_HANDLER + 0x0D) -#define KVM_TRC_INTR (KVM_TRC_HANDLER + 0x0E) -#define KVM_TRC_NMI (KVM_TRC_HANDLER + 0x0F) -#define KVM_TRC_VMMCALL (KVM_TRC_HANDLER + 0x10) -#define KVM_TRC_HLT (KVM_TRC_HANDLER + 0x11) -#define KVM_TRC_CLTS (KVM_TRC_HANDLER + 0x12) -#define KVM_TRC_LMSW (KVM_TRC_HANDLER + 0x13) -#define KVM_TRC_APIC_ACCESS (KVM_TRC_HANDLER + 0x14) -#define KVM_TRC_TDP_FAULT (KVM_TRC_HANDLER + 0x15) -#define KVM_TRC_GTLB_WRITE (KVM_TRC_HANDLER + 0x16) -#define KVM_TRC_STLB_WRITE (KVM_TRC_HANDLER + 0x17) -#define KVM_TRC_STLB_INVAL (KVM_TRC_HANDLER + 0x18) -#define KVM_TRC_PPC_INSTR (KVM_TRC_HANDLER + 0x19) - -struct kvm_user_trace_setup { - __u32 buf_size; - __u32 buf_nr; -}; - -#define __KVM_DEPRECATED_MAIN_W_0x06 \ - _IOW(KVMIO, 0x06, struct kvm_user_trace_setup) -#define __KVM_DEPRECATED_MAIN_0x07 _IO(KVMIO, 0x07) -#define __KVM_DEPRECATED_MAIN_0x08 _IO(KVMIO, 0x08) - -#define __KVM_DEPRECATED_VM_R_0x70 _IOR(KVMIO, 0x70, struct kvm_assigned_irq) - -struct kvm_breakpoint { - __u32 enabled; - __u32 padding; - __u64 address; -}; - -struct kvm_debug_guest { - __u32 enabled; - __u32 pad; - struct kvm_breakpoint breakpoints[4]; - __u32 singlestep; -}; - -#define __KVM_DEPRECATED_VCPU_W_0x87 _IOW(KVMIO, 0x87, struct kvm_debug_guest) - -/* *** End of deprecated interfaces *** */ - - /* for KVM_SET_USER_MEMORY_REGION */ struct kvm_userspace_memory_region { __u32 slot; @@ -967,9 +897,6 @@ struct kvm_ppc_resize_hpt { */ #define KVM_GET_VCPU_MMAP_SIZE _IO(KVMIO, 0x04) /* in bytes */ #define KVM_GET_SUPPORTED_CPUID _IOWR(KVMIO, 0x05, struct kvm_cpuid2) -#define KVM_TRACE_ENABLE __KVM_DEPRECATED_MAIN_W_0x06 -#define KVM_TRACE_PAUSE __KVM_DEPRECATED_MAIN_0x07 -#define KVM_TRACE_DISABLE __KVM_DEPRECATED_MAIN_0x08 #define KVM_GET_EMULATED_CPUID _IOWR(KVMIO, 0x09, struct kvm_cpuid2) #define KVM_GET_MSR_FEATURE_INDEX_LIST _IOWR(KVMIO, 0x0a, struct kvm_msr_list) @@ -1536,20 +1463,8 @@ struct kvm_s390_ucas_mapping { _IOW(KVMIO, 0x67, struct kvm_coalesced_mmio_zone) #define KVM_UNREGISTER_COALESCED_MMIO \ _IOW(KVMIO, 0x68, struct kvm_coalesced_mmio_zone) -#define KVM_ASSIGN_PCI_DEVICE _IOR(KVMIO, 0x69, \ - struct kvm_assigned_pci_dev) #define KVM_SET_GSI_ROUTING _IOW(KVMIO, 0x6a, struct kvm_irq_routing) -/* deprecated, replaced by KVM_ASSIGN_DEV_IRQ */ -#define KVM_ASSIGN_IRQ __KVM_DEPRECATED_VM_R_0x70 -#define KVM_ASSIGN_DEV_IRQ _IOW(KVMIO, 0x70, struct kvm_assigned_irq) #define KVM_REINJECT_CONTROL _IO(KVMIO, 0x71) -#define KVM_DEASSIGN_PCI_DEVICE _IOW(KVMIO, 0x72, \ - struct kvm_assigned_pci_dev) -#define KVM_ASSIGN_SET_MSIX_NR _IOW(KVMIO, 0x73, \ - struct kvm_assigned_msix_nr) -#define KVM_ASSIGN_SET_MSIX_ENTRY _IOW(KVMIO, 0x74, \ - struct kvm_assigned_msix_entry) -#define KVM_DEASSIGN_DEV_IRQ _IOW(KVMIO, 0x75, struct kvm_assigned_irq) #define KVM_IRQFD _IOW(KVMIO, 0x76, struct kvm_irqfd) #define KVM_CREATE_PIT2 _IOW(KVMIO, 0x77, struct kvm_pit_config) #define KVM_SET_BOOT_CPU_ID _IO(KVMIO, 0x78) @@ -1566,9 +1481,6 @@ struct kvm_s390_ucas_mapping { * KVM_CAP_VM_TSC_CONTROL to set defaults for a VM */ #define KVM_SET_TSC_KHZ _IO(KVMIO, 0xa2) #define KVM_GET_TSC_KHZ _IO(KVMIO, 0xa3) -/* Available with KVM_CAP_PCI_2_3 */ -#define KVM_ASSIGN_SET_INTX_MASK _IOW(KVMIO, 0xa4, \ - struct kvm_assigned_pci_dev) /* Available with KVM_CAP_SIGNAL_MSI */ #define KVM_SIGNAL_MSI _IOW(KVMIO, 0xa5, struct kvm_msi) /* Available with KVM_CAP_PPC_GET_SMMU_INFO */ @@ -1621,8 +1533,6 @@ struct kvm_s390_ucas_mapping { #define KVM_SET_SREGS _IOW(KVMIO, 0x84, struct kvm_sregs) #define KVM_TRANSLATE _IOWR(KVMIO, 0x85, struct kvm_translation) #define KVM_INTERRUPT _IOW(KVMIO, 0x86, struct kvm_interrupt) -/* KVM_DEBUG_GUEST is no longer supported, use KVM_SET_GUEST_DEBUG instead */ -#define KVM_DEBUG_GUEST __KVM_DEPRECATED_VCPU_W_0x87 #define KVM_GET_MSRS _IOWR(KVMIO, 0x88, struct kvm_msrs) #define KVM_SET_MSRS _IOW(KVMIO, 0x89, struct kvm_msrs) #define KVM_SET_CPUID _IOW(KVMIO, 0x8a, struct kvm_cpuid) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index a20cf1f9ad29..acd67fb40183 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -5497,11 +5497,6 @@ static long kvm_dev_ioctl(struct file *filp, r += PAGE_SIZE; /* coalesced mmio ring page */ #endif break; - case KVM_TRACE_ENABLE: - case KVM_TRACE_PAUSE: - case KVM_TRACE_DISABLE: - r = -EOPNOTSUPP; - break; default: return kvm_arch_dev_ioctl(filp, ioctl, arg); } From 8ed26ab8d59111c2f7b86d200d1eb97d2a458fd1 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 18 Oct 2023 12:18:00 -0400 Subject: [PATCH 131/336] KVM: clean up directives to compile out irqfds Keep all #ifdef CONFIG_HAVE_KVM_IRQCHIP parts of eventfd.c together, and compile out the irqfds field of struct kvm if the symbol is not defined. Signed-off-by: Paolo Bonzini --- include/linux/kvm_host.h | 2 ++ virt/kvm/eventfd.c | 26 ++++++++++++-------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 1bba24a13ec9..7e7fd25b09b3 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -782,6 +782,7 @@ struct kvm { struct list_head vm_list; struct mutex lock; struct kvm_io_bus __rcu *buses[KVM_NR_BUSES]; +#ifdef CONFIG_HAVE_KVM_IRQCHIP struct { spinlock_t lock; struct list_head items; @@ -789,6 +790,7 @@ struct kvm { struct list_head resampler_list; struct mutex resampler_lock; } irqfds; +#endif struct list_head ioeventfds; struct kvm_vm_stat stat; struct kvm_arch arch; diff --git a/virt/kvm/eventfd.c b/virt/kvm/eventfd.c index 19534156d48c..d516e6d33f56 100644 --- a/virt/kvm/eventfd.c +++ b/virt/kvm/eventfd.c @@ -526,21 +526,7 @@ void kvm_unregister_irq_ack_notifier(struct kvm *kvm, synchronize_srcu(&kvm->irq_srcu); kvm_arch_post_irq_ack_notifier_list_update(kvm); } -#endif -void -kvm_eventfd_init(struct kvm *kvm) -{ -#ifdef CONFIG_HAVE_KVM_IRQCHIP - spin_lock_init(&kvm->irqfds.lock); - INIT_LIST_HEAD(&kvm->irqfds.items); - INIT_LIST_HEAD(&kvm->irqfds.resampler_list); - mutex_init(&kvm->irqfds.resampler_lock); -#endif - INIT_LIST_HEAD(&kvm->ioeventfds); -} - -#ifdef CONFIG_HAVE_KVM_IRQCHIP /* * shutdown any irqfd's that match fd+gsi */ @@ -1012,3 +998,15 @@ kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args) return kvm_assign_ioeventfd(kvm, args); } + +void +kvm_eventfd_init(struct kvm *kvm) +{ +#ifdef CONFIG_HAVE_KVM_IRQCHIP + spin_lock_init(&kvm->irqfds.lock); + INIT_LIST_HEAD(&kvm->irqfds.items); + INIT_LIST_HEAD(&kvm->irqfds.resampler_list); + mutex_init(&kvm->irqfds.resampler_lock); +#endif + INIT_LIST_HEAD(&kvm->ioeventfds); +} From 06dc6a869597ec68b2e4ebcde8147f4a52965e96 Mon Sep 17 00:00:00 2001 From: Sergio Paracuellos Date: Sun, 22 Oct 2023 11:06:33 +0200 Subject: [PATCH 132/336] MAINTAINERS: Add myself as maintainer of the Ralink architecture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Its been a while since I am making contributions to this architecture. Hence add myself as maintainer. Signed-off-by: Sergio Paracuellos Acked-by: John Crispin Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Thomas Bogendoerfer --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index e2c6187a3ac8..b3444c8c6512 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -18051,6 +18051,7 @@ F: drivers/media/cec/usb/rainshadow/ RALINK MIPS ARCHITECTURE M: John Crispin +M: Sergio Paracuellos L: linux-mips@vger.kernel.org S: Maintained F: arch/mips/ralink From 01940cd4a6b9c47995a0cdaaafdd459b0d2221a2 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Fri, 10 Nov 2023 19:02:13 -0800 Subject: [PATCH 133/336] MIPS: SGI-IP27: hubio: fix nasid kernel-doc warning ip27-hubio.c:32: warning: Function parameter or member 'nasid' not described in 'hub_pio_map' ip27-hubio.c:32: warning: Excess function parameter 'hub' description in 'hub_pio_map' Fixes: 4bf841ebf17a ("MIPS: SGI-IP27: get rid of compact node ids") Signed-off-by: Randy Dunlap Reported-by: kernel test robot Closes: lore.kernel.org/r/202311101336.BUL1JuvU-lkp@intel.com Cc: Thomas Bogendoerfer Cc: Paul Burton Cc: Ralf Baechle Cc: Jiri Slaby Cc: linux-mips@vger.kernel.org Signed-off-by: Thomas Bogendoerfer --- arch/mips/sgi-ip27/ip27-hubio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/mips/sgi-ip27/ip27-hubio.c b/arch/mips/sgi-ip27/ip27-hubio.c index 8352eb6403b4..c57f0d8f3218 100644 --- a/arch/mips/sgi-ip27/ip27-hubio.c +++ b/arch/mips/sgi-ip27/ip27-hubio.c @@ -21,7 +21,7 @@ static int force_fire_and_forget = 1; /** * hub_pio_map - establish a HUB PIO mapping * - * @hub: hub to perform PIO mapping on + * @nasid: nasid to perform PIO mapping on * @widget: widget ID to perform PIO mapping for * @xtalk_addr: xtalk_address that needs to be mapped * @size: size of the PIO mapping From e45f463a9b01aabd03c49390fc5249eeba0abc5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Tue, 14 Nov 2023 09:12:37 -0500 Subject: [PATCH 134/336] riscv: add ISA extension parsing for Zbc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Zbc was documented in the dt-bindings but actually not supported in ISA string parsing. Add it. Signed-off-by: Clément Léger Link: https://lore.kernel.org/r/20231114141256.126749-2-cleger@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/hwcap.h | 1 + arch/riscv/kernel/cpufeature.c | 1 + 2 files changed, 2 insertions(+) diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index 06d30526ef3b..afa9abc1a0b0 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -57,6 +57,7 @@ #define RISCV_ISA_EXT_ZIHPM 42 #define RISCV_ISA_EXT_SMSTATEEN 43 #define RISCV_ISA_EXT_ZICOND 44 +#define RISCV_ISA_EXT_ZBC 45 #define RISCV_ISA_EXT_MAX 64 diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index b3785ffc1570..eae14daa5a75 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -176,6 +176,7 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM), __RISCV_ISA_EXT_DATA(zba, RISCV_ISA_EXT_ZBA), __RISCV_ISA_EXT_DATA(zbb, RISCV_ISA_EXT_ZBB), + __RISCV_ISA_EXT_DATA(zbc, RISCV_ISA_EXT_ZBC), __RISCV_ISA_EXT_DATA(zbs, RISCV_ISA_EXT_ZBS), __RISCV_ISA_EXT_DATA(smaia, RISCV_ISA_EXT_SMAIA), __RISCV_ISA_EXT_DATA(smstateen, RISCV_ISA_EXT_SMSTATEEN), From be6bef2acb75ca980671de19d406c0310d646d2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Tue, 14 Nov 2023 09:12:38 -0500 Subject: [PATCH 135/336] riscv: hwprobe: export missing Zbc ISA extension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While Zba and Zbb were exported through hwprobe, Zbc was not. Export it. Signed-off-by: Clément Léger Link: https://lore.kernel.org/r/20231114141256.126749-3-cleger@rivosinc.com Signed-off-by: Palmer Dabbelt --- Documentation/arch/riscv/hwprobe.rst | 3 +++ arch/riscv/include/uapi/asm/hwprobe.h | 1 + arch/riscv/kernel/sys_riscv.c | 1 + 3 files changed, 5 insertions(+) diff --git a/Documentation/arch/riscv/hwprobe.rst b/Documentation/arch/riscv/hwprobe.rst index 7b2384de471f..3f71da935a65 100644 --- a/Documentation/arch/riscv/hwprobe.rst +++ b/Documentation/arch/riscv/hwprobe.rst @@ -80,6 +80,9 @@ The following keys are defined: * :c:macro:`RISCV_HWPROBE_EXT_ZICBOZ`: The Zicboz extension is supported, as ratified in commit 3dd606f ("Create cmobase-v1.0.pdf") of riscv-CMOs. + * :c:macro:`RISCV_HWPROBE_EXT_ZBC` The Zbc extension is supported, as defined + in version 1.0 of the Bit-Manipulation ISA extensions. + * :c:macro:`RISCV_HWPROBE_KEY_CPUPERF_0`: A bitmask that contains performance information about the selected set of processors. diff --git a/arch/riscv/include/uapi/asm/hwprobe.h b/arch/riscv/include/uapi/asm/hwprobe.h index b659ffcfcdb4..aca5abc7ebee 100644 --- a/arch/riscv/include/uapi/asm/hwprobe.h +++ b/arch/riscv/include/uapi/asm/hwprobe.h @@ -30,6 +30,7 @@ struct riscv_hwprobe { #define RISCV_HWPROBE_EXT_ZBB (1 << 4) #define RISCV_HWPROBE_EXT_ZBS (1 << 5) #define RISCV_HWPROBE_EXT_ZICBOZ (1 << 6) +#define RISCV_HWPROBE_EXT_ZBC (1 << 7) #define RISCV_HWPROBE_KEY_CPUPERF_0 5 #define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0) #define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0) diff --git a/arch/riscv/kernel/sys_riscv.c b/arch/riscv/kernel/sys_riscv.c index c712037dbe10..7a40145e9628 100644 --- a/arch/riscv/kernel/sys_riscv.c +++ b/arch/riscv/kernel/sys_riscv.c @@ -162,6 +162,7 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, EXT_KEY(ZBB); EXT_KEY(ZBS); EXT_KEY(ZICBOZ); + EXT_KEY(ZBC); #undef EXT_KEY } From 0d8295ed975b6df487f0b4018770a60b070fc76d Mon Sep 17 00:00:00 2001 From: Evan Green Date: Tue, 14 Nov 2023 09:12:39 -0500 Subject: [PATCH 136/336] riscv: add ISA extension parsing for scalar crypto MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Scalar Crypto specification defines Zk as a shorthand for the Zkn, Zkr and Zkt extensions. The same follows for both Zkn, Zks and Zbk, which are all shorthands for various other extensions. The detailed breakdown can be found in their dt-binding entries. Since Zkn also implies the Zbkb, Zbkc and Zbkx extensions, simply passing "zk" through a DT should enable all of Zbkb, Zbkc, Zbkx, Zkn, Zkr and Zkt. For example, setting the "riscv,isa" DT property to "rv64imafdc_zk" should generate the following cpuinfo output: "rv64imafdc_zicntr_zicsr_zifencei_zihpm_zbkb_zbkc_zbkx_zknd_zkne_zknh_zkr_zkt" riscv_isa_ext_data grows a pair of new members, to permit setting the relevant bits for "bundled" extensions, both while parsing the ISA string and the new dedicated extension properties. Co-developed-by: Conor Dooley Signed-off-by: Conor Dooley Signed-off-by: Evan Green Signed-off-by: Clément Léger Acked-by: Conor Dooley Link: https://lore.kernel.org/r/20231114141256.126749-4-cleger@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/cpufeature.h | 4 +- arch/riscv/include/asm/hwcap.h | 11 +++ arch/riscv/kernel/cpufeature.c | 118 ++++++++++++++++++++++------ 3 files changed, 109 insertions(+), 24 deletions(-) diff --git a/arch/riscv/include/asm/cpufeature.h b/arch/riscv/include/asm/cpufeature.h index a418c3112cd6..fbdde8b8a47e 100644 --- a/arch/riscv/include/asm/cpufeature.h +++ b/arch/riscv/include/asm/cpufeature.h @@ -59,6 +59,8 @@ struct riscv_isa_ext_data { const unsigned int id; const char *name; const char *property; + const unsigned int *subset_ext_ids; + const unsigned int subset_ext_size; }; extern const struct riscv_isa_ext_data riscv_isa_ext[]; @@ -67,7 +69,7 @@ extern bool riscv_isa_fallback; unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap); -bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, int bit); +bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, unsigned int bit); #define riscv_isa_extension_available(isa_bitmap, ext) \ __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_##ext) diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index afa9abc1a0b0..b0857c64bf6e 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -58,8 +58,19 @@ #define RISCV_ISA_EXT_SMSTATEEN 43 #define RISCV_ISA_EXT_ZICOND 44 #define RISCV_ISA_EXT_ZBC 45 +#define RISCV_ISA_EXT_ZBKB 46 +#define RISCV_ISA_EXT_ZBKC 47 +#define RISCV_ISA_EXT_ZBKX 48 +#define RISCV_ISA_EXT_ZKND 49 +#define RISCV_ISA_EXT_ZKNE 50 +#define RISCV_ISA_EXT_ZKNH 51 +#define RISCV_ISA_EXT_ZKR 52 +#define RISCV_ISA_EXT_ZKSED 53 +#define RISCV_ISA_EXT_ZKSH 54 +#define RISCV_ISA_EXT_ZKT 55 #define RISCV_ISA_EXT_MAX 64 +#define RISCV_ISA_EXT_INVALID U32_MAX #ifdef CONFIG_RISCV_M_MODE #define RISCV_ISA_EXT_SxAIA RISCV_ISA_EXT_SMAIA diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index eae14daa5a75..a2871bceaad9 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -70,7 +70,7 @@ EXPORT_SYMBOL_GPL(riscv_isa_extension_base); * * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. */ -bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, int bit) +bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, unsigned int bit) { const unsigned long *bmap = (isa_bitmap) ? isa_bitmap : riscv_isa; @@ -102,17 +102,53 @@ static bool riscv_isa_extension_check(int id) return false; } return true; + case RISCV_ISA_EXT_INVALID: + return false; } return true; } -#define __RISCV_ISA_EXT_DATA(_name, _id) { \ - .name = #_name, \ - .property = #_name, \ - .id = _id, \ +#define _RISCV_ISA_EXT_DATA(_name, _id, _subset_exts, _subset_exts_size) { \ + .name = #_name, \ + .property = #_name, \ + .id = _id, \ + .subset_ext_ids = _subset_exts, \ + .subset_ext_size = _subset_exts_size \ } +#define __RISCV_ISA_EXT_DATA(_name, _id) _RISCV_ISA_EXT_DATA(_name, _id, NULL, 0) + +/* Used to declare pure "lasso" extension (Zk for instance) */ +#define __RISCV_ISA_EXT_BUNDLE(_name, _bundled_exts) \ + _RISCV_ISA_EXT_DATA(_name, RISCV_ISA_EXT_INVALID, _bundled_exts, ARRAY_SIZE(_bundled_exts)) + +static const unsigned int riscv_zk_bundled_exts[] = { + RISCV_ISA_EXT_ZBKB, + RISCV_ISA_EXT_ZBKC, + RISCV_ISA_EXT_ZBKX, + RISCV_ISA_EXT_ZKND, + RISCV_ISA_EXT_ZKNE, + RISCV_ISA_EXT_ZKR, + RISCV_ISA_EXT_ZKT, +}; + +static const unsigned int riscv_zkn_bundled_exts[] = { + RISCV_ISA_EXT_ZBKB, + RISCV_ISA_EXT_ZBKC, + RISCV_ISA_EXT_ZBKX, + RISCV_ISA_EXT_ZKND, + RISCV_ISA_EXT_ZKNE, + RISCV_ISA_EXT_ZKNH, +}; + +static const unsigned int riscv_zks_bundled_exts[] = { + RISCV_ISA_EXT_ZBKB, + RISCV_ISA_EXT_ZBKC, + RISCV_ISA_EXT_ZKSED, + RISCV_ISA_EXT_ZKSH +}; + /* * The canonical order of ISA extension names in the ISA string is defined in * chapter 27 of the unprivileged specification. @@ -177,7 +213,20 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(zba, RISCV_ISA_EXT_ZBA), __RISCV_ISA_EXT_DATA(zbb, RISCV_ISA_EXT_ZBB), __RISCV_ISA_EXT_DATA(zbc, RISCV_ISA_EXT_ZBC), + __RISCV_ISA_EXT_DATA(zbkb, RISCV_ISA_EXT_ZBKB), + __RISCV_ISA_EXT_DATA(zbkc, RISCV_ISA_EXT_ZBKC), + __RISCV_ISA_EXT_DATA(zbkx, RISCV_ISA_EXT_ZBKX), __RISCV_ISA_EXT_DATA(zbs, RISCV_ISA_EXT_ZBS), + __RISCV_ISA_EXT_BUNDLE(zk, riscv_zk_bundled_exts), + __RISCV_ISA_EXT_BUNDLE(zkn, riscv_zkn_bundled_exts), + __RISCV_ISA_EXT_DATA(zknd, RISCV_ISA_EXT_ZKND), + __RISCV_ISA_EXT_DATA(zkne, RISCV_ISA_EXT_ZKNE), + __RISCV_ISA_EXT_DATA(zknh, RISCV_ISA_EXT_ZKNH), + __RISCV_ISA_EXT_DATA(zkr, RISCV_ISA_EXT_ZKR), + __RISCV_ISA_EXT_BUNDLE(zks, riscv_zks_bundled_exts), + __RISCV_ISA_EXT_DATA(zkt, RISCV_ISA_EXT_ZKT), + __RISCV_ISA_EXT_DATA(zksed, RISCV_ISA_EXT_ZKSED), + __RISCV_ISA_EXT_DATA(zksh, RISCV_ISA_EXT_ZKSH), __RISCV_ISA_EXT_DATA(smaia, RISCV_ISA_EXT_SMAIA), __RISCV_ISA_EXT_DATA(smstateen, RISCV_ISA_EXT_SMSTATEEN), __RISCV_ISA_EXT_DATA(ssaia, RISCV_ISA_EXT_SSAIA), @@ -190,6 +239,31 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { const size_t riscv_isa_ext_count = ARRAY_SIZE(riscv_isa_ext); +static void __init match_isa_ext(const struct riscv_isa_ext_data *ext, const char *name, + const char *name_end, struct riscv_isainfo *isainfo) +{ + if ((name_end - name == strlen(ext->name)) && + !strncasecmp(name, ext->name, name_end - name)) { + /* + * If this is a bundle, enable all the ISA extensions that + * comprise the bundle. + */ + if (ext->subset_ext_size) { + for (int i = 0; i < ext->subset_ext_size; i++) { + if (riscv_isa_extension_check(ext->subset_ext_ids[i])) + set_bit(ext->subset_ext_ids[i], isainfo->isa); + } + } + + /* + * This is valid even for bundle extensions which uses the RISCV_ISA_EXT_INVALID id + * (rejected by riscv_isa_extension_check()). + */ + if (riscv_isa_extension_check(ext->id)) + set_bit(ext->id, isainfo->isa); + } +} + static void __init riscv_parse_isa_string(unsigned long *this_hwcap, struct riscv_isainfo *isainfo, unsigned long *isa2hwcap, const char *isa) { @@ -322,14 +396,6 @@ static void __init riscv_parse_isa_string(unsigned long *this_hwcap, struct risc if (*isa == '_') ++isa; -#define SET_ISA_EXT_MAP(name, bit) \ - do { \ - if ((ext_end - ext == strlen(name)) && \ - !strncasecmp(ext, name, strlen(name)) && \ - riscv_isa_extension_check(bit)) \ - set_bit(bit, isainfo->isa); \ - } while (false) \ - if (unlikely(ext_err)) continue; if (!ext_long) { @@ -341,10 +407,8 @@ static void __init riscv_parse_isa_string(unsigned long *this_hwcap, struct risc } } else { for (int i = 0; i < riscv_isa_ext_count; i++) - SET_ISA_EXT_MAP(riscv_isa_ext[i].name, - riscv_isa_ext[i].id); + match_isa_ext(&riscv_isa_ext[i], ext, ext_end, isainfo); } -#undef SET_ISA_EXT_MAP } } @@ -443,18 +507,26 @@ static int __init riscv_fill_hwcap_from_ext_list(unsigned long *isa2hwcap) } for (int i = 0; i < riscv_isa_ext_count; i++) { + const struct riscv_isa_ext_data *ext = &riscv_isa_ext[i]; + if (of_property_match_string(cpu_node, "riscv,isa-extensions", - riscv_isa_ext[i].property) < 0) + ext->property) < 0) continue; - if (!riscv_isa_extension_check(riscv_isa_ext[i].id)) - continue; + if (ext->subset_ext_size) { + for (int j = 0; j < ext->subset_ext_size; j++) { + if (riscv_isa_extension_check(ext->subset_ext_ids[i])) + set_bit(ext->subset_ext_ids[j], isainfo->isa); + } + } - /* Only single letter extensions get set in hwcap */ - if (strnlen(riscv_isa_ext[i].name, 2) == 1) - this_hwcap |= isa2hwcap[riscv_isa_ext[i].id]; + if (riscv_isa_extension_check(ext->id)) { + set_bit(ext->id, isainfo->isa); - set_bit(riscv_isa_ext[i].id, isainfo->isa); + /* Only single letter extensions get set in hwcap */ + if (strnlen(riscv_isa_ext[i].name, 2) == 1) + this_hwcap |= isa2hwcap[riscv_isa_ext[i].id]; + } } of_node_put(cpu_node); From 794983f292cd71b27106f1226360b2cf0f4a881b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Tue, 14 Nov 2023 09:12:40 -0500 Subject: [PATCH 137/336] riscv: hwprobe: add support for scalar crypto ISA extensions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Export the following scalar crypto extensions through hwprobe: - Zbkb - Zbkc - Zbkx - Zknd - Zkne - Zknh - Zksed - Zksh - Zkt Signed-off-by: Clément Léger Link: https://lore.kernel.org/r/20231114141256.126749-5-cleger@rivosinc.com Signed-off-by: Palmer Dabbelt --- Documentation/arch/riscv/hwprobe.rst | 27 +++++++++++++++++++++++++++ arch/riscv/include/uapi/asm/hwprobe.h | 9 +++++++++ arch/riscv/kernel/sys_riscv.c | 10 ++++++++++ 3 files changed, 46 insertions(+) diff --git a/Documentation/arch/riscv/hwprobe.rst b/Documentation/arch/riscv/hwprobe.rst index 3f71da935a65..3a18a31e32c3 100644 --- a/Documentation/arch/riscv/hwprobe.rst +++ b/Documentation/arch/riscv/hwprobe.rst @@ -83,6 +83,33 @@ The following keys are defined: * :c:macro:`RISCV_HWPROBE_EXT_ZBC` The Zbc extension is supported, as defined in version 1.0 of the Bit-Manipulation ISA extensions. + * :c:macro:`RISCV_HWPROBE_EXT_ZBKB` The Zbkb extension is supported, as + defined in version 1.0 of the Scalar Crypto ISA extensions. + + * :c:macro:`RISCV_HWPROBE_EXT_ZBKC` The Zbkc extension is supported, as + defined in version 1.0 of the Scalar Crypto ISA extensions. + + * :c:macro:`RISCV_HWPROBE_EXT_ZBKX` The Zbkx extension is supported, as + defined in version 1.0 of the Scalar Crypto ISA extensions. + + * :c:macro:`RISCV_HWPROBE_EXT_ZKND` The Zknd extension is supported, as + defined in version 1.0 of the Scalar Crypto ISA extensions. + + * :c:macro:`RISCV_HWPROBE_EXT_ZKNE` The Zkne extension is supported, as + defined in version 1.0 of the Scalar Crypto ISA extensions. + + * :c:macro:`RISCV_HWPROBE_EXT_ZKNH` The Zknh extension is supported, as + defined in version 1.0 of the Scalar Crypto ISA extensions. + + * :c:macro:`RISCV_HWPROBE_EXT_ZKSED` The Zksed extension is supported, as + defined in version 1.0 of the Scalar Crypto ISA extensions. + + * :c:macro:`RISCV_HWPROBE_EXT_ZKSH` The Zksh extension is supported, as + defined in version 1.0 of the Scalar Crypto ISA extensions. + + * :c:macro:`RISCV_HWPROBE_EXT_ZKT` The Zkt extension is supported, as defined + in version 1.0 of the Scalar Crypto ISA extensions. + * :c:macro:`RISCV_HWPROBE_KEY_CPUPERF_0`: A bitmask that contains performance information about the selected set of processors. diff --git a/arch/riscv/include/uapi/asm/hwprobe.h b/arch/riscv/include/uapi/asm/hwprobe.h index aca5abc7ebee..624abd5cde29 100644 --- a/arch/riscv/include/uapi/asm/hwprobe.h +++ b/arch/riscv/include/uapi/asm/hwprobe.h @@ -31,6 +31,15 @@ struct riscv_hwprobe { #define RISCV_HWPROBE_EXT_ZBS (1 << 5) #define RISCV_HWPROBE_EXT_ZICBOZ (1 << 6) #define RISCV_HWPROBE_EXT_ZBC (1 << 7) +#define RISCV_HWPROBE_EXT_ZBKB (1 << 8) +#define RISCV_HWPROBE_EXT_ZBKC (1 << 9) +#define RISCV_HWPROBE_EXT_ZBKX (1 << 10) +#define RISCV_HWPROBE_EXT_ZKND (1 << 11) +#define RISCV_HWPROBE_EXT_ZKNE (1 << 12) +#define RISCV_HWPROBE_EXT_ZKNH (1 << 13) +#define RISCV_HWPROBE_EXT_ZKSED (1 << 14) +#define RISCV_HWPROBE_EXT_ZKSH (1 << 15) +#define RISCV_HWPROBE_EXT_ZKT (1 << 16) #define RISCV_HWPROBE_KEY_CPUPERF_0 5 #define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0) #define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0) diff --git a/arch/riscv/kernel/sys_riscv.c b/arch/riscv/kernel/sys_riscv.c index 7a40145e9628..43aa66e71418 100644 --- a/arch/riscv/kernel/sys_riscv.c +++ b/arch/riscv/kernel/sys_riscv.c @@ -163,6 +163,16 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, EXT_KEY(ZBS); EXT_KEY(ZICBOZ); EXT_KEY(ZBC); + + EXT_KEY(ZBKB); + EXT_KEY(ZBKC); + EXT_KEY(ZBKX); + EXT_KEY(ZKND); + EXT_KEY(ZKNE); + EXT_KEY(ZKNH); + EXT_KEY(ZKSED); + EXT_KEY(ZKSH); + EXT_KEY(ZKT); #undef EXT_KEY } From 9376396251c8a927018d465b4cc396af4b25b8d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Tue, 14 Nov 2023 09:12:41 -0500 Subject: [PATCH 138/336] dt-bindings: riscv: add scalar crypto ISA extensions description MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add description for scalar crypto ISA extensions: - Zbkb - Zbkc - Zbkx - Zknd - Zkne - Zknh - Zkr - Zksed - Zksh - Zkt Signed-off-by: Clément Léger Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20231114141256.126749-6-cleger@rivosinc.com Signed-off-by: Palmer Dabbelt --- .../devicetree/bindings/riscv/extensions.yaml | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/Documentation/devicetree/bindings/riscv/extensions.yaml b/Documentation/devicetree/bindings/riscv/extensions.yaml index c91ab0e46648..a3803b22cf4f 100644 --- a/Documentation/devicetree/bindings/riscv/extensions.yaml +++ b/Documentation/devicetree/bindings/riscv/extensions.yaml @@ -190,12 +190,92 @@ properties: multiplication as ratified at commit 6d33919 ("Merge pull request #158 from hirooih/clmul-fix-loop-end-condition") of riscv-bitmanip. + - const: zbkb + description: + The standard Zbkb bitmanip instructions for cryptography as ratified + in version 1.0 of RISC-V Cryptography Extensions Volume I + specification. + + - const: zbkc + description: + The standard Zbkc carry-less multiply instructions as ratified + in version 1.0 of RISC-V Cryptography Extensions Volume I + specification. + + - const: zbkx + description: + The standard Zbkx crossbar permutation instructions as ratified + in version 1.0 of RISC-V Cryptography Extensions Volume I + specification. + - const: zbs description: | The standard Zbs bit-manipulation extension for single-bit instructions as ratified at commit 6d33919 ("Merge pull request #158 from hirooih/clmul-fix-loop-end-condition") of riscv-bitmanip. + - const: zk + description: + The standard Zk Standard Scalar cryptography extension as ratified + in version 1.0 of RISC-V Cryptography Extensions Volume I + specification. + + - const: zkn + description: + The standard Zkn NIST algorithm suite extensions as ratified in + version 1.0 of RISC-V Cryptography Extensions Volume I + specification. + + - const: zknd + description: | + The standard Zknd for NIST suite: AES decryption instructions as + ratified in version 1.0 of RISC-V Cryptography Extensions Volume I + specification. + + - const: zkne + description: | + The standard Zkne for NIST suite: AES encryption instructions as + ratified in version 1.0 of RISC-V Cryptography Extensions Volume I + specification. + + - const: zknh + description: | + The standard Zknh for NIST suite: hash function instructions as + ratified in version 1.0 of RISC-V Cryptography Extensions Volume I + specification. + + - const: zkr + description: + The standard Zkr entropy source extension as ratified in version + 1.0 of RISC-V Cryptography Extensions Volume I specification. + This string being present means that the CSR associated to this + extension is accessible at the privilege level to which that + device-tree has been provided. + + - const: zks + description: + The standard Zks ShangMi algorithm suite extensions as ratified in + version 1.0 of RISC-V Cryptography Extensions Volume I + specification. + + - const: zksed + description: | + The standard Zksed for ShangMi suite: SM4 block cipher instructions + as ratified in version 1.0 of RISC-V Cryptography Extensions + Volume I specification. + + - const: zksh + description: | + The standard Zksh for ShangMi suite: SM3 hash function instructions + as ratified in version 1.0 of RISC-V Cryptography Extensions + Volume I specification. + + - const: zkt + description: + The standard Zkt for data independent execution latency as ratified + in version 1.0 of RISC-V Cryptography Extensions Volume I + specification. + - const: zicbom description: The standard Zicbom extension for base cache management operations as From aec3353963b8de889c3f1ab7cc8ba11e99626606 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Tue, 14 Nov 2023 09:12:42 -0500 Subject: [PATCH 139/336] riscv: add ISA extension parsing for vector crypto MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add parsing of some Zv* vector crypto ISA extensions that are mentioned in "RISC-V Cryptography Extensions Volume II" [1]. These ISA extensions are the following: - Zvbb: Vector Basic Bit-manipulation - Zvbc: Vector Carryless Multiplication - Zvkb: Vector Cryptography Bit-manipulation - Zvkg: Vector GCM/GMAC. - Zvkned: NIST Suite: Vector AES Block Cipher - Zvknh[ab]: NIST Suite: Vector SHA-2 Secure Hash - Zvksed: ShangMi Suite: SM4 Block Cipher - Zvksh: ShangMi Suite: SM3 Secure Hash - Zvkn: NIST Algorithm Suite - Zvknc: NIST Algorithm Suite with carryless multiply - Zvkng: NIST Algorithm Suite with GCM. - Zvks: ShangMi Algorithm Suite - Zvksc: ShangMi Algorithm Suite with carryless multiplication - Zvksg: ShangMi Algorithm Suite with GCM. - Zvkt: Vector Data-Independent Execution Latency. Link: https://drive.google.com/file/d/1gb9OLH-DhbCgWp7VwpPOVrrY6f3oSJLL/view [1] Signed-off-by: Clément Léger Link: https://lore.kernel.org/r/20231114141256.126749-7-cleger@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/hwcap.h | 12 ++++++- arch/riscv/kernel/cpufeature.c | 64 ++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index b0857c64bf6e..477254668d82 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -68,8 +68,18 @@ #define RISCV_ISA_EXT_ZKSED 53 #define RISCV_ISA_EXT_ZKSH 54 #define RISCV_ISA_EXT_ZKT 55 +#define RISCV_ISA_EXT_ZVBB 56 +#define RISCV_ISA_EXT_ZVBC 57 +#define RISCV_ISA_EXT_ZVKB 58 +#define RISCV_ISA_EXT_ZVKG 59 +#define RISCV_ISA_EXT_ZVKNED 60 +#define RISCV_ISA_EXT_ZVKNHA 61 +#define RISCV_ISA_EXT_ZVKNHB 62 +#define RISCV_ISA_EXT_ZVKSED 63 +#define RISCV_ISA_EXT_ZVKSH 64 +#define RISCV_ISA_EXT_ZVKT 65 -#define RISCV_ISA_EXT_MAX 64 +#define RISCV_ISA_EXT_MAX 128 #define RISCV_ISA_EXT_INVALID U32_MAX #ifdef CONFIG_RISCV_M_MODE diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index a2871bceaad9..c4d0f16c29b9 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -123,6 +123,10 @@ static bool riscv_isa_extension_check(int id) #define __RISCV_ISA_EXT_BUNDLE(_name, _bundled_exts) \ _RISCV_ISA_EXT_DATA(_name, RISCV_ISA_EXT_INVALID, _bundled_exts, ARRAY_SIZE(_bundled_exts)) +/* Used to declare extensions that are a superset of other extensions (Zvbb for instance) */ +#define __RISCV_ISA_EXT_SUPERSET(_name, _id, _sub_exts) \ + _RISCV_ISA_EXT_DATA(_name, _id, _sub_exts, ARRAY_SIZE(_sub_exts)) + static const unsigned int riscv_zk_bundled_exts[] = { RISCV_ISA_EXT_ZBKB, RISCV_ISA_EXT_ZBKC, @@ -149,6 +153,50 @@ static const unsigned int riscv_zks_bundled_exts[] = { RISCV_ISA_EXT_ZKSH }; +#define RISCV_ISA_EXT_ZVKN \ + RISCV_ISA_EXT_ZVKNED, \ + RISCV_ISA_EXT_ZVKNHB, \ + RISCV_ISA_EXT_ZVKB, \ + RISCV_ISA_EXT_ZVKT + +static const unsigned int riscv_zvkn_bundled_exts[] = { + RISCV_ISA_EXT_ZVKN +}; + +static const unsigned int riscv_zvknc_bundled_exts[] = { + RISCV_ISA_EXT_ZVKN, + RISCV_ISA_EXT_ZVBC +}; + +static const unsigned int riscv_zvkng_bundled_exts[] = { + RISCV_ISA_EXT_ZVKN, + RISCV_ISA_EXT_ZVKG +}; + +#define RISCV_ISA_EXT_ZVKS \ + RISCV_ISA_EXT_ZVKSED, \ + RISCV_ISA_EXT_ZVKSH, \ + RISCV_ISA_EXT_ZVKB, \ + RISCV_ISA_EXT_ZVKT + +static const unsigned int riscv_zvks_bundled_exts[] = { + RISCV_ISA_EXT_ZVKS +}; + +static const unsigned int riscv_zvksc_bundled_exts[] = { + RISCV_ISA_EXT_ZVKS, + RISCV_ISA_EXT_ZVBC +}; + +static const unsigned int riscv_zvksg_bundled_exts[] = { + RISCV_ISA_EXT_ZVKS, + RISCV_ISA_EXT_ZVKG +}; + +static const unsigned int riscv_zvbb_exts[] = { + RISCV_ISA_EXT_ZVKB +}; + /* * The canonical order of ISA extension names in the ISA string is defined in * chapter 27 of the unprivileged specification. @@ -227,6 +275,22 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(zkt, RISCV_ISA_EXT_ZKT), __RISCV_ISA_EXT_DATA(zksed, RISCV_ISA_EXT_ZKSED), __RISCV_ISA_EXT_DATA(zksh, RISCV_ISA_EXT_ZKSH), + __RISCV_ISA_EXT_SUPERSET(zvbb, RISCV_ISA_EXT_ZVBB, riscv_zvbb_exts), + __RISCV_ISA_EXT_DATA(zvbc, RISCV_ISA_EXT_ZVBC), + __RISCV_ISA_EXT_DATA(zvkb, RISCV_ISA_EXT_ZVKB), + __RISCV_ISA_EXT_DATA(zvkg, RISCV_ISA_EXT_ZVKG), + __RISCV_ISA_EXT_BUNDLE(zvkn, riscv_zvkn_bundled_exts), + __RISCV_ISA_EXT_BUNDLE(zvknc, riscv_zvknc_bundled_exts), + __RISCV_ISA_EXT_DATA(zvkned, RISCV_ISA_EXT_ZVKNED), + __RISCV_ISA_EXT_BUNDLE(zvkng, riscv_zvkng_bundled_exts), + __RISCV_ISA_EXT_DATA(zvknha, RISCV_ISA_EXT_ZVKNHA), + __RISCV_ISA_EXT_DATA(zvknhb, RISCV_ISA_EXT_ZVKNHB), + __RISCV_ISA_EXT_BUNDLE(zvks, riscv_zvks_bundled_exts), + __RISCV_ISA_EXT_BUNDLE(zvksc, riscv_zvksc_bundled_exts), + __RISCV_ISA_EXT_DATA(zvksed, RISCV_ISA_EXT_ZVKSED), + __RISCV_ISA_EXT_DATA(zvksh, RISCV_ISA_EXT_ZVKSH), + __RISCV_ISA_EXT_BUNDLE(zvksg, riscv_zvksg_bundled_exts), + __RISCV_ISA_EXT_DATA(zvkt, RISCV_ISA_EXT_ZVKT), __RISCV_ISA_EXT_DATA(smaia, RISCV_ISA_EXT_SMAIA), __RISCV_ISA_EXT_DATA(smstateen, RISCV_ISA_EXT_SMSTATEEN), __RISCV_ISA_EXT_DATA(ssaia, RISCV_ISA_EXT_SSAIA), From ca35b5b115856973620f425955fd0ce2505a51c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Tue, 14 Nov 2023 09:12:43 -0500 Subject: [PATCH 140/336] riscv: hwprobe: export vector crypto ISA extensions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Export Zv* vector crypto ISA extensions that were added in "RISC-V Cryptography Extensions Volume II" specification[1] through hwprobe. This adds support for the following instructions: - Zvbb: Vector Basic Bit-manipulation - Zvbc: Vector Carryless Multiplication - Zvkb: Vector Cryptography Bit-manipulation - Zvkg: Vector GCM/GMAC. - Zvkned: NIST Suite: Vector AES Block Cipher - Zvknh[ab]: NIST Suite: Vector SHA-2 Secure Hash - Zvksed: ShangMi Suite: SM4 Block Cipher - Zvksh: ShangMi Suite: SM3 Secure Hash - Zvknc: NIST Algorithm Suite with carryless multiply - Zvkng: NIST Algorithm Suite with GCM. - Zvksc: ShangMi Algorithm Suite with carryless multiplication - Zvksg: ShangMi Algorithm Suite with GCM. - Zvkt: Vector Data-Independent Execution Latency. Zvkn and Zvks are ommited since they are a superset of other extensions. Link: https://drive.google.com/file/d/1gb9OLH-DhbCgWp7VwpPOVrrY6f3oSJLL/view [1] Signed-off-by: Clément Léger Reviewed-by: Evan Green Link: https://lore.kernel.org/r/20231114141256.126749-8-cleger@rivosinc.com Signed-off-by: Palmer Dabbelt --- Documentation/arch/riscv/hwprobe.rst | 30 +++++++++++++++++++++++++++ arch/riscv/include/uapi/asm/hwprobe.h | 10 +++++++++ arch/riscv/kernel/sys_riscv.c | 13 ++++++++++++ 3 files changed, 53 insertions(+) diff --git a/Documentation/arch/riscv/hwprobe.rst b/Documentation/arch/riscv/hwprobe.rst index 3a18a31e32c3..a08fcd899b6d 100644 --- a/Documentation/arch/riscv/hwprobe.rst +++ b/Documentation/arch/riscv/hwprobe.rst @@ -110,6 +110,36 @@ The following keys are defined: * :c:macro:`RISCV_HWPROBE_EXT_ZKT` The Zkt extension is supported, as defined in version 1.0 of the Scalar Crypto ISA extensions. + * :c:macro:`RISCV_HWPROBE_EXT_ZVBB`: The Zvbb extension is supported as + defined in version 1.0 of the RISC-V Cryptography Extensions Volume II. + + * :c:macro:`RISCV_HWPROBE_EXT_ZVBC`: The Zvbc extension is supported as + defined in version 1.0 of the RISC-V Cryptography Extensions Volume II. + + * :c:macro:`RISCV_HWPROBE_EXT_ZVKB`: The Zvkb extension is supported as + defined in version 1.0 of the RISC-V Cryptography Extensions Volume II. + + * :c:macro:`RISCV_HWPROBE_EXT_ZVKG`: The Zvkg extension is supported as + defined in version 1.0 of the RISC-V Cryptography Extensions Volume II. + + * :c:macro:`RISCV_HWPROBE_EXT_ZVKNED`: The Zvkned extension is supported as + defined in version 1.0 of the RISC-V Cryptography Extensions Volume II. + + * :c:macro:`RISCV_HWPROBE_EXT_ZVKNHA`: The Zvknha extension is supported as + defined in version 1.0 of the RISC-V Cryptography Extensions Volume II. + + * :c:macro:`RISCV_HWPROBE_EXT_ZVKNHB`: The Zvknhb extension is supported as + defined in version 1.0 of the RISC-V Cryptography Extensions Volume II. + + * :c:macro:`RISCV_HWPROBE_EXT_ZVKSED`: The Zvksed extension is supported as + defined in version 1.0 of the RISC-V Cryptography Extensions Volume II. + + * :c:macro:`RISCV_HWPROBE_EXT_ZVKSH`: The Zvksh extension is supported as + defined in version 1.0 of the RISC-V Cryptography Extensions Volume II. + + * :c:macro:`RISCV_HWPROBE_EXT_ZVKT`: The Zvkt extension is supported as + defined in version 1.0 of the RISC-V Cryptography Extensions Volume II. + * :c:macro:`RISCV_HWPROBE_KEY_CPUPERF_0`: A bitmask that contains performance information about the selected set of processors. diff --git a/arch/riscv/include/uapi/asm/hwprobe.h b/arch/riscv/include/uapi/asm/hwprobe.h index 624abd5cde29..89628a76ca04 100644 --- a/arch/riscv/include/uapi/asm/hwprobe.h +++ b/arch/riscv/include/uapi/asm/hwprobe.h @@ -40,6 +40,16 @@ struct riscv_hwprobe { #define RISCV_HWPROBE_EXT_ZKSED (1 << 14) #define RISCV_HWPROBE_EXT_ZKSH (1 << 15) #define RISCV_HWPROBE_EXT_ZKT (1 << 16) +#define RISCV_HWPROBE_EXT_ZVBB (1 << 17) +#define RISCV_HWPROBE_EXT_ZVBC (1 << 18) +#define RISCV_HWPROBE_EXT_ZVKB (1 << 19) +#define RISCV_HWPROBE_EXT_ZVKG (1 << 20) +#define RISCV_HWPROBE_EXT_ZVKNED (1 << 21) +#define RISCV_HWPROBE_EXT_ZVKNHA (1 << 22) +#define RISCV_HWPROBE_EXT_ZVKNHB (1 << 23) +#define RISCV_HWPROBE_EXT_ZVKSED (1 << 24) +#define RISCV_HWPROBE_EXT_ZVKSH (1 << 25) +#define RISCV_HWPROBE_EXT_ZVKT (1 << 26) #define RISCV_HWPROBE_KEY_CPUPERF_0 5 #define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0) #define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0) diff --git a/arch/riscv/kernel/sys_riscv.c b/arch/riscv/kernel/sys_riscv.c index 43aa66e71418..9bbcd7334d96 100644 --- a/arch/riscv/kernel/sys_riscv.c +++ b/arch/riscv/kernel/sys_riscv.c @@ -173,6 +173,19 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, EXT_KEY(ZKSED); EXT_KEY(ZKSH); EXT_KEY(ZKT); + + if (has_vector()) { + EXT_KEY(ZVBB); + EXT_KEY(ZVBC); + EXT_KEY(ZVKB); + EXT_KEY(ZVKG); + EXT_KEY(ZVKNED); + EXT_KEY(ZVKNHA); + EXT_KEY(ZVKNHB); + EXT_KEY(ZVKSED); + EXT_KEY(ZVKSH); + EXT_KEY(ZVKT); + } #undef EXT_KEY } From 10815531c513f449ce477fb97e3f6e6962c14eaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Tue, 14 Nov 2023 09:12:44 -0500 Subject: [PATCH 141/336] dt-bindings: riscv: add vector crypto ISA extensions description MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add Zv* vector crypto extensions that were added in "RISC-V Cryptography Extensions Volume II" specificationi[1]: - Zvbb: Vector Basic Bit-manipulation - Zvbc: Vector Carryless Multiplication - Zvkb: Vector Cryptography Bit-manipulation - Zvkg: Vector GCM/GMAC. - Zvkned: NIST Suite: Vector AES Block Cipher - Zvknh[ab]: NIST Suite: Vector SHA-2 Secure Hash - Zvksed: ShangMi Suite: SM4 Block Cipher - Zvksh: ShangMi Suite: SM3 Secure Hash - Zvkn: NIST Algorithm Suite - Zvknc: NIST Algorithm Suite with carryless multiply - Zvkng: NIST Algorithm Suite with GCM. - Zvks: ShangMi Algorithm Suite - Zvksc: ShangMi Algorithm Suite with carryless multiplication - Zvksg: ShangMi Algorithm Suite with GCM. - Zvkt: Vector Data-Independent Execution Latency. Link: https://drive.google.com/file/d/1gb9OLH-DhbCgWp7VwpPOVrrY6f3oSJLL/view [1] Signed-off-by: Clément Léger Acked-by: Conor Dooley Link: https://lore.kernel.org/r/20231114141256.126749-9-cleger@rivosinc.com Signed-off-by: Palmer Dabbelt --- .../devicetree/bindings/riscv/extensions.yaml | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/Documentation/devicetree/bindings/riscv/extensions.yaml b/Documentation/devicetree/bindings/riscv/extensions.yaml index a3803b22cf4f..e845e461b6e1 100644 --- a/Documentation/devicetree/bindings/riscv/extensions.yaml +++ b/Documentation/devicetree/bindings/riscv/extensions.yaml @@ -338,5 +338,101 @@ properties: in commit 2e5236 ("Ztso is now ratified.") of the riscv-isa-manual. + - const: zvbb + description: + The standard Zvbb extension for vectored basic bit-manipulation + instructions, as ratified in commit 56ed795 ("Update + riscv-crypto-spec-vector.adoc") of riscv-crypto. + + - const: zvbc + description: + The standard Zvbc extension for vectored carryless multiplication + instructions, as ratified in commit 56ed795 ("Update + riscv-crypto-spec-vector.adoc") of riscv-crypto. + + - const: zvkb + description: + The standard Zvkb extension for vector cryptography bit-manipulation + instructions, as ratified in commit 56ed795 ("Update + riscv-crypto-spec-vector.adoc") of riscv-crypto. + + - const: zvkg + description: + The standard Zvkg extension for vector GCM/GMAC instructions, as + ratified in commit 56ed795 ("Update riscv-crypto-spec-vector.adoc") + of riscv-crypto. + + - const: zvkn + description: + The standard Zvkn extension for NIST algorithm suite instructions, as + ratified in commit 56ed795 ("Update riscv-crypto-spec-vector.adoc") + of riscv-crypto. + + - const: zvknc + description: + The standard Zvknc extension for NIST algorithm suite with carryless + multiply instructions, as ratified in commit 56ed795 ("Update + riscv-crypto-spec-vector.adoc") of riscv-crypto. + + - const: zvkned + description: + The standard Zvkned extension for Vector AES block cipher + instructions, as ratified in commit 56ed795 ("Update + riscv-crypto-spec-vector.adoc") of riscv-crypto. + + - const: zvkng + description: + The standard Zvkng extension for NIST algorithm suite with GCM + instructions, as ratified in commit 56ed795 ("Update + riscv-crypto-spec-vector.adoc") of riscv-crypto. + + - const: zvknha + description: | + The standard Zvknha extension for NIST suite: vector SHA-2 secure, + hash (SHA-256 only) instructions, as ratified in commit + 56ed795 ("Update riscv-crypto-spec-vector.adoc") of riscv-crypto. + + - const: zvknhb + description: | + The standard Zvknhb extension for NIST suite: vector SHA-2 secure, + hash (SHA-256 and SHA-512) instructions, as ratified in commit + 56ed795 ("Update riscv-crypto-spec-vector.adoc") of riscv-crypto. + + - const: zvks + description: + The standard Zvks extension for ShangMi algorithm suite + instructions, as ratified in commit 56ed795 ("Update + riscv-crypto-spec-vector.adoc") of riscv-crypto. + + - const: zvksc + description: + The standard Zvksc extension for ShangMi algorithm suite with + carryless multiplication instructions, as ratified in commit 56ed795 + ("Update riscv-crypto-spec-vector.adoc") of riscv-crypto. + + - const: zvksed + description: | + The standard Zvksed extension for ShangMi suite: SM4 block cipher + instructions, as ratified in commit 56ed795 ("Update + riscv-crypto-spec-vector.adoc") of riscv-crypto. + + - const: zvksh + description: | + The standard Zvksh extension for ShangMi suite: SM3 secure hash + instructions, as ratified in commit 56ed795 ("Update + riscv-crypto-spec-vector.adoc") of riscv-crypto. + + - const: zvksg + description: + The standard Zvksg extension for ShangMi algorithm suite with GCM + instructions, as ratified in commit 56ed795 ("Update + riscv-crypto-spec-vector.adoc") of riscv-crypto. + + - const: zvkt + description: + The standard Zvkt extension for vector data-independent execution + latency, as ratified in commit 56ed795 ("Update + riscv-crypto-spec-vector.adoc") of riscv-crypto. + additionalProperties: true ... From 11e8e1ee2c223585f1776e5c5d21bdae7184ca34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Tue, 14 Nov 2023 09:12:45 -0500 Subject: [PATCH 142/336] riscv: add ISA extension parsing for Zfh/Zfh[min] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add parsing for Zfh[min] ISA extensions[1]. Link: https://drive.google.com/file/d/1z3tQQLm5ALsAD77PM0l0CHnapxWCeVzP/view [1] Signed-off-by: Clément Léger Reviewed-by: Evan Green Link: https://lore.kernel.org/r/20231114141256.126749-10-cleger@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/hwcap.h | 2 ++ arch/riscv/kernel/cpufeature.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index 477254668d82..6a6ee93a3c9a 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -78,6 +78,8 @@ #define RISCV_ISA_EXT_ZVKSED 63 #define RISCV_ISA_EXT_ZVKSH 64 #define RISCV_ISA_EXT_ZVKT 65 +#define RISCV_ISA_EXT_ZFH 66 +#define RISCV_ISA_EXT_ZFHMIN 67 #define RISCV_ISA_EXT_MAX 128 #define RISCV_ISA_EXT_INVALID U32_MAX diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index c4d0f16c29b9..7182cf278b1c 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -258,6 +258,8 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(zifencei, RISCV_ISA_EXT_ZIFENCEI), __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE), __RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM), + __RISCV_ISA_EXT_DATA(zfh, RISCV_ISA_EXT_ZFH), + __RISCV_ISA_EXT_DATA(zfhmin, RISCV_ISA_EXT_ZFHMIN), __RISCV_ISA_EXT_DATA(zba, RISCV_ISA_EXT_ZBA), __RISCV_ISA_EXT_DATA(zbb, RISCV_ISA_EXT_ZBB), __RISCV_ISA_EXT_DATA(zbc, RISCV_ISA_EXT_ZBC), From bf4cd84111c6139313504310ff934df901a5ed3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Tue, 14 Nov 2023 09:12:46 -0500 Subject: [PATCH 143/336] riscv: hwprobe: export Zfh[min] ISA extensions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Export Zfh[min] ISA extensions[1] through hwprobe only if FPU support is available. Link: https://drive.google.com/file/d/1z3tQQLm5ALsAD77PM0l0CHnapxWCeVzP/view [1] Signed-off-by: Clément Léger Reviewed-by: Evan Green Link: https://lore.kernel.org/r/20231114141256.126749-11-cleger@rivosinc.com Signed-off-by: Palmer Dabbelt --- Documentation/arch/riscv/hwprobe.rst | 6 ++++++ arch/riscv/include/uapi/asm/hwprobe.h | 2 ++ arch/riscv/kernel/sys_riscv.c | 5 +++++ 3 files changed, 13 insertions(+) diff --git a/Documentation/arch/riscv/hwprobe.rst b/Documentation/arch/riscv/hwprobe.rst index a08fcd899b6d..397d53195f49 100644 --- a/Documentation/arch/riscv/hwprobe.rst +++ b/Documentation/arch/riscv/hwprobe.rst @@ -140,6 +140,12 @@ The following keys are defined: * :c:macro:`RISCV_HWPROBE_EXT_ZVKT`: The Zvkt extension is supported as defined in version 1.0 of the RISC-V Cryptography Extensions Volume II. + * :c:macro:`RISCV_HWPROBE_EXT_ZFH`: The Zfh extension version 1.0 is supported + as defined in the RISC-V ISA manual. + + * :c:macro:`RISCV_HWPROBE_EXT_ZFHMIN`: The Zfhmin extension version 1.0 is + supported as defined in the RISC-V ISA manual. + * :c:macro:`RISCV_HWPROBE_KEY_CPUPERF_0`: A bitmask that contains performance information about the selected set of processors. diff --git a/arch/riscv/include/uapi/asm/hwprobe.h b/arch/riscv/include/uapi/asm/hwprobe.h index 89628a76ca04..2d960777ea43 100644 --- a/arch/riscv/include/uapi/asm/hwprobe.h +++ b/arch/riscv/include/uapi/asm/hwprobe.h @@ -50,6 +50,8 @@ struct riscv_hwprobe { #define RISCV_HWPROBE_EXT_ZVKSED (1 << 24) #define RISCV_HWPROBE_EXT_ZVKSH (1 << 25) #define RISCV_HWPROBE_EXT_ZVKT (1 << 26) +#define RISCV_HWPROBE_EXT_ZFH (1 << 27) +#define RISCV_HWPROBE_EXT_ZFHMIN (1 << 28) #define RISCV_HWPROBE_KEY_CPUPERF_0 5 #define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0) #define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0) diff --git a/arch/riscv/kernel/sys_riscv.c b/arch/riscv/kernel/sys_riscv.c index 9bbcd7334d96..d776c6c39fcd 100644 --- a/arch/riscv/kernel/sys_riscv.c +++ b/arch/riscv/kernel/sys_riscv.c @@ -186,6 +186,11 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, EXT_KEY(ZVKSH); EXT_KEY(ZVKT); } + + if (has_fpu()) { + EXT_KEY(ZFH); + EXT_KEY(ZFHMIN); + } #undef EXT_KEY } From c44714c35ff861d69db9c8bb4ae1347373f817f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Tue, 14 Nov 2023 09:12:47 -0500 Subject: [PATCH 144/336] dt-bindings: riscv: add Zfh[min] ISA extensions description MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add description of Zfh[min] ISA extensions[1]. Link: https://drive.google.com/file/d/1z3tQQLm5ALsAD77PM0l0CHnapxWCeVzP/view [1] Signed-off-by: Clément Léger Acked-by: Conor Dooley Link: https://lore.kernel.org/r/20231114141256.126749-12-cleger@rivosinc.com Signed-off-by: Palmer Dabbelt --- .../devicetree/bindings/riscv/extensions.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Documentation/devicetree/bindings/riscv/extensions.yaml b/Documentation/devicetree/bindings/riscv/extensions.yaml index e845e461b6e1..f3c99e69619b 100644 --- a/Documentation/devicetree/bindings/riscv/extensions.yaml +++ b/Documentation/devicetree/bindings/riscv/extensions.yaml @@ -214,6 +214,19 @@ properties: instructions as ratified at commit 6d33919 ("Merge pull request #158 from hirooih/clmul-fix-loop-end-condition") of riscv-bitmanip. + - const: zfh + description: + The standard Zfh extension for 16-bit half-precision binary + floating-point instructions, as ratified in commit 64074bc ("Update + version numbers for Zfh/Zfinx") of riscv-isa-manual. + + - const: zfhmin + description: + The standard Zfhmin extension which provides minimal support for + 16-bit half-precision binary floating-point instructions, as ratified + in commit 64074bc ("Update version numbers for Zfh/Zfinx") of + riscv-isa-manual. + - const: zk description: The standard Zk Standard Scalar cryptography extension as ratified From eddbfa0d849fa5a315840e8c501962252b48484d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Tue, 14 Nov 2023 09:12:48 -0500 Subject: [PATCH 145/336] riscv: add ISA extension parsing for Zihintntl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add parsing for Zihintntl ISA extension[1] that was ratified in commit 0dc91f5 ("Zihintntl is ratified") of riscv-isa-manual[2]. Link: https://drive.google.com/file/d/13_wsN8YmRfH8YWysFyTX-DjTkCnBd9hj/view [1] Link: https://github.com/riscv/riscv-isa-manual/commit/0dc91f505e6d [2] Signed-off-by: Clément Léger Reviewed-by: Evan Green Link: https://lore.kernel.org/r/20231114141256.126749-13-cleger@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/hwcap.h | 1 + arch/riscv/kernel/cpufeature.c | 1 + 2 files changed, 2 insertions(+) diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index 6a6ee93a3c9a..97d106fa0f54 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -80,6 +80,7 @@ #define RISCV_ISA_EXT_ZVKT 65 #define RISCV_ISA_EXT_ZFH 66 #define RISCV_ISA_EXT_ZFHMIN 67 +#define RISCV_ISA_EXT_ZIHINTNTL 68 #define RISCV_ISA_EXT_MAX 128 #define RISCV_ISA_EXT_INVALID U32_MAX diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 7182cf278b1c..e73ee4cfd84a 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -256,6 +256,7 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(zicond, RISCV_ISA_EXT_ZICOND), __RISCV_ISA_EXT_DATA(zicsr, RISCV_ISA_EXT_ZICSR), __RISCV_ISA_EXT_DATA(zifencei, RISCV_ISA_EXT_ZIFENCEI), + __RISCV_ISA_EXT_DATA(zihintntl, RISCV_ISA_EXT_ZIHINTNTL), __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE), __RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM), __RISCV_ISA_EXT_DATA(zfh, RISCV_ISA_EXT_ZFH), From 74ba42b250a7339c72e5803490b1ea42c3556f26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Tue, 14 Nov 2023 09:12:49 -0500 Subject: [PATCH 146/336] riscv: hwprobe: export Zhintntl ISA extension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Export Zihintntl extension[1] through hwprobe. Link: https://drive.google.com/file/d/13_wsN8YmRfH8YWysFyTX-DjTkCnBd9hj/view [1] Signed-off-by: Clément Léger Reviewed-by: Evan Green Link: https://lore.kernel.org/r/20231114141256.126749-14-cleger@rivosinc.com Signed-off-by: Palmer Dabbelt --- Documentation/arch/riscv/hwprobe.rst | 3 +++ arch/riscv/include/uapi/asm/hwprobe.h | 1 + arch/riscv/kernel/sys_riscv.c | 1 + 3 files changed, 5 insertions(+) diff --git a/Documentation/arch/riscv/hwprobe.rst b/Documentation/arch/riscv/hwprobe.rst index 397d53195f49..aa8ebeeddfe6 100644 --- a/Documentation/arch/riscv/hwprobe.rst +++ b/Documentation/arch/riscv/hwprobe.rst @@ -146,6 +146,9 @@ The following keys are defined: * :c:macro:`RISCV_HWPROBE_EXT_ZFHMIN`: The Zfhmin extension version 1.0 is supported as defined in the RISC-V ISA manual. + * :c:macro:`RISCV_HWPROBE_EXT_ZIHINTNTL`: The Zihintntl extension version 1.0 + is supported as defined in the RISC-V ISA manual. + * :c:macro:`RISCV_HWPROBE_KEY_CPUPERF_0`: A bitmask that contains performance information about the selected set of processors. diff --git a/arch/riscv/include/uapi/asm/hwprobe.h b/arch/riscv/include/uapi/asm/hwprobe.h index 2d960777ea43..d72c69ea0740 100644 --- a/arch/riscv/include/uapi/asm/hwprobe.h +++ b/arch/riscv/include/uapi/asm/hwprobe.h @@ -52,6 +52,7 @@ struct riscv_hwprobe { #define RISCV_HWPROBE_EXT_ZVKT (1 << 26) #define RISCV_HWPROBE_EXT_ZFH (1 << 27) #define RISCV_HWPROBE_EXT_ZFHMIN (1 << 28) +#define RISCV_HWPROBE_EXT_ZIHINTNTL (1 << 29) #define RISCV_HWPROBE_KEY_CPUPERF_0 5 #define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0) #define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0) diff --git a/arch/riscv/kernel/sys_riscv.c b/arch/riscv/kernel/sys_riscv.c index d776c6c39fcd..a46e4f6821dd 100644 --- a/arch/riscv/kernel/sys_riscv.c +++ b/arch/riscv/kernel/sys_riscv.c @@ -173,6 +173,7 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, EXT_KEY(ZKSED); EXT_KEY(ZKSH); EXT_KEY(ZKT); + EXT_KEY(ZIHINTNTL); if (has_vector()) { EXT_KEY(ZVBB); From 892f10c8d6ca4b3c12d149085243ad8cd75f09b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Tue, 14 Nov 2023 09:12:50 -0500 Subject: [PATCH 147/336] dt-bindings: riscv: add Zihintntl ISA extension description MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add description for Zihintntl ISA extension[1]. Link: https://drive.google.com/file/d/13_wsN8YmRfH8YWysFyTX-DjTkCnBd9hj/view [1] Signed-off-by: Clément Léger Acked-by: Conor Dooley Link: https://lore.kernel.org/r/20231114141256.126749-15-cleger@rivosinc.com Signed-off-by: Palmer Dabbelt --- Documentation/devicetree/bindings/riscv/extensions.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Documentation/devicetree/bindings/riscv/extensions.yaml b/Documentation/devicetree/bindings/riscv/extensions.yaml index f3c99e69619b..f953c49be90d 100644 --- a/Documentation/devicetree/bindings/riscv/extensions.yaml +++ b/Documentation/devicetree/bindings/riscv/extensions.yaml @@ -339,6 +339,12 @@ properties: The standard Zihintpause extension for pause hints, as ratified in commit d8ab5c7 ("Zihintpause is ratified") of the riscv-isa-manual. + - const: zihintntl + description: + The standard Zihintntl extension for non-temporal locality hints, as + ratified in commit 0dc91f5 ("Zihintntl is ratified") of the + riscv-isa-manual. + - const: zihpm description: The standard Zihpm extension for hardware performance counters, as From f4961b78c37b64f48cc5c376f8aef5e1823201c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Tue, 14 Nov 2023 09:12:51 -0500 Subject: [PATCH 148/336] riscv: add ISA extension parsing for Zvfh[min] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add parsing for Zvfh[min] ISA extension[1] which were ratified in june 2023 around commit e2ccd0548d6c ("Remove draft warnings from Zvfh[min]") in riscv-v-spec[2]. Link: https://drive.google.com/file/d/1_Yt60HGAf1r1hx7JnsIptw0sqkBd9BQ8/view [1] Link: https://github.com/riscv/riscv-v-spec/commits/e2ccd0548d6c [2] Signed-off-by: Clément Léger Reviewed-by: Evan Green Link: https://lore.kernel.org/r/20231114141256.126749-16-cleger@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/hwcap.h | 2 ++ arch/riscv/kernel/cpufeature.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index 97d106fa0f54..103d4f0aaf15 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -81,6 +81,8 @@ #define RISCV_ISA_EXT_ZFH 66 #define RISCV_ISA_EXT_ZFHMIN 67 #define RISCV_ISA_EXT_ZIHINTNTL 68 +#define RISCV_ISA_EXT_ZVFH 69 +#define RISCV_ISA_EXT_ZVFHMIN 70 #define RISCV_ISA_EXT_MAX 128 #define RISCV_ISA_EXT_INVALID U32_MAX diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index e73ee4cfd84a..623a5fa48cf4 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -280,6 +280,8 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(zksh, RISCV_ISA_EXT_ZKSH), __RISCV_ISA_EXT_SUPERSET(zvbb, RISCV_ISA_EXT_ZVBB, riscv_zvbb_exts), __RISCV_ISA_EXT_DATA(zvbc, RISCV_ISA_EXT_ZVBC), + __RISCV_ISA_EXT_DATA(zvfh, RISCV_ISA_EXT_ZVFH), + __RISCV_ISA_EXT_DATA(zvfhmin, RISCV_ISA_EXT_ZVFHMIN), __RISCV_ISA_EXT_DATA(zvkb, RISCV_ISA_EXT_ZVKB), __RISCV_ISA_EXT_DATA(zvkg, RISCV_ISA_EXT_ZVKG), __RISCV_ISA_EXT_BUNDLE(zvkn, riscv_zvkn_bundled_exts), From 5dadda5e6a59a5b4f547a1b14d9518e4074c6966 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Tue, 14 Nov 2023 09:12:52 -0500 Subject: [PATCH 149/336] riscv: hwprobe: export Zvfh[min] ISA extensions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Export Zvfh[min] ISA extension[1] through hwprobe. Link: https://drive.google.com/file/d/1_Yt60HGAf1r1hx7JnsIptw0sqkBd9BQ8/view [1] Signed-off-by: Clément Léger Reviewed-by: Evan Green Link: https://lore.kernel.org/r/20231114141256.126749-17-cleger@rivosinc.com Signed-off-by: Palmer Dabbelt --- Documentation/arch/riscv/hwprobe.rst | 8 ++++++++ arch/riscv/include/uapi/asm/hwprobe.h | 2 ++ arch/riscv/kernel/sys_riscv.c | 2 ++ 3 files changed, 12 insertions(+) diff --git a/Documentation/arch/riscv/hwprobe.rst b/Documentation/arch/riscv/hwprobe.rst index aa8ebeeddfe6..896ecfbbe5f4 100644 --- a/Documentation/arch/riscv/hwprobe.rst +++ b/Documentation/arch/riscv/hwprobe.rst @@ -149,6 +149,14 @@ The following keys are defined: * :c:macro:`RISCV_HWPROBE_EXT_ZIHINTNTL`: The Zihintntl extension version 1.0 is supported as defined in the RISC-V ISA manual. + * :c:macro:`RISCV_HWPROBE_EXT_ZVFH`: The Zvfh extension is supported as + defined in the RISC-V Vector manual starting from commit e2ccd0548d6c + ("Remove draft warnings from Zvfh[min]"). + + * :c:macro:`RISCV_HWPROBE_EXT_ZVFHMIN`: The Zvfhmin extension is supported as + defined in the RISC-V Vector manual starting from commit e2ccd0548d6c + ("Remove draft warnings from Zvfh[min]"). + * :c:macro:`RISCV_HWPROBE_KEY_CPUPERF_0`: A bitmask that contains performance information about the selected set of processors. diff --git a/arch/riscv/include/uapi/asm/hwprobe.h b/arch/riscv/include/uapi/asm/hwprobe.h index d72c69ea0740..fffc69d9f6ba 100644 --- a/arch/riscv/include/uapi/asm/hwprobe.h +++ b/arch/riscv/include/uapi/asm/hwprobe.h @@ -53,6 +53,8 @@ struct riscv_hwprobe { #define RISCV_HWPROBE_EXT_ZFH (1 << 27) #define RISCV_HWPROBE_EXT_ZFHMIN (1 << 28) #define RISCV_HWPROBE_EXT_ZIHINTNTL (1 << 29) +#define RISCV_HWPROBE_EXT_ZVFH (1 << 30) +#define RISCV_HWPROBE_EXT_ZVFHMIN (1 << 31) #define RISCV_HWPROBE_KEY_CPUPERF_0 5 #define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0) #define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0) diff --git a/arch/riscv/kernel/sys_riscv.c b/arch/riscv/kernel/sys_riscv.c index a46e4f6821dd..e90537593f5f 100644 --- a/arch/riscv/kernel/sys_riscv.c +++ b/arch/riscv/kernel/sys_riscv.c @@ -186,6 +186,8 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, EXT_KEY(ZVKSED); EXT_KEY(ZVKSH); EXT_KEY(ZVKT); + EXT_KEY(ZVFH); + EXT_KEY(ZVFHMIN); } if (has_fpu()) { From e11880b4be3a8db558147409b9ed0d1855526910 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Tue, 14 Nov 2023 09:12:53 -0500 Subject: [PATCH 150/336] dt-bindings: riscv: add Zvfh[min] ISA extension description MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add description for Zvfh[min] ISA extension[1]. Link: https://drive.google.com/file/d/1_Yt60HGAf1r1hx7JnsIptw0sqkBd9BQ8/view [1] Signed-off-by: Clément Léger Acked-by: Conor Dooley Link: https://lore.kernel.org/r/20231114141256.126749-18-cleger@rivosinc.com Signed-off-by: Palmer Dabbelt --- .../devicetree/bindings/riscv/extensions.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Documentation/devicetree/bindings/riscv/extensions.yaml b/Documentation/devicetree/bindings/riscv/extensions.yaml index f953c49be90d..b91d49b7c3a0 100644 --- a/Documentation/devicetree/bindings/riscv/extensions.yaml +++ b/Documentation/devicetree/bindings/riscv/extensions.yaml @@ -369,6 +369,18 @@ properties: instructions, as ratified in commit 56ed795 ("Update riscv-crypto-spec-vector.adoc") of riscv-crypto. + - const: zvfh + description: + The standard Zvfh extension for vectored half-precision + floating-point instructions, as ratified in commit e2ccd05 + ("Remove draft warnings from Zvfh[min]") of riscv-v-spec. + + - const: zvfhmin + description: + The standard Zvfhmin extension for vectored minimal half-precision + floating-point instructions, as ratified in commit e2ccd05 + ("Remove draft warnings from Zvfh[min]") of riscv-v-spec. + - const: zvkb description: The standard Zvkb extension for vector cryptography bit-manipulation From fe987e84b0120e2b41db69ed4ede166797aa8d2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Tue, 14 Nov 2023 09:12:54 -0500 Subject: [PATCH 151/336] riscv: add ISA extension parsing for Zfa MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add parsing for Zfa ISA extension [1] which were ratified in commit 056b6ff467c7 ("Zfa is ratified") of riscv-isa-manual[2]. Link: https://drive.google.com/file/d/1VT6QIggpb59-8QRV266dEE4T8FZTxGq4/view [1] Link: https://github.com/riscv/riscv-isa-manual/commits/056b6ff467c7 [2] Signed-off-by: Clément Léger Reviewed-by: Evan Green Link: https://lore.kernel.org/r/20231114141256.126749-19-cleger@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/hwcap.h | 1 + arch/riscv/kernel/cpufeature.c | 1 + 2 files changed, 2 insertions(+) diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index 103d4f0aaf15..2438d4685da6 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -83,6 +83,7 @@ #define RISCV_ISA_EXT_ZIHINTNTL 68 #define RISCV_ISA_EXT_ZVFH 69 #define RISCV_ISA_EXT_ZVFHMIN 70 +#define RISCV_ISA_EXT_ZFA 71 #define RISCV_ISA_EXT_MAX 128 #define RISCV_ISA_EXT_INVALID U32_MAX diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 623a5fa48cf4..dc0ab3e97cd2 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -259,6 +259,7 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(zihintntl, RISCV_ISA_EXT_ZIHINTNTL), __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE), __RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM), + __RISCV_ISA_EXT_DATA(zfa, RISCV_ISA_EXT_ZFA), __RISCV_ISA_EXT_DATA(zfh, RISCV_ISA_EXT_ZFH), __RISCV_ISA_EXT_DATA(zfhmin, RISCV_ISA_EXT_ZFHMIN), __RISCV_ISA_EXT_DATA(zba, RISCV_ISA_EXT_ZBA), From dc6ccb21f42ce5b3e4df6f52e14edab721e1b26e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Tue, 14 Nov 2023 09:12:55 -0500 Subject: [PATCH 152/336] riscv: hwprobe: export Zfa ISA extension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Export Zfa ISA extension[1] through hwprobe. Link: https://drive.google.com/file/d/1VT6QIggpb59-8QRV266dEE4T8FZTxGq4/view [1] Signed-off-by: Clément Léger Reviewed-by: Evan Green Link: https://lore.kernel.org/r/20231114141256.126749-20-cleger@rivosinc.com Signed-off-by: Palmer Dabbelt --- Documentation/arch/riscv/hwprobe.rst | 4 ++++ arch/riscv/include/uapi/asm/hwprobe.h | 1 + arch/riscv/kernel/sys_riscv.c | 1 + 3 files changed, 6 insertions(+) diff --git a/Documentation/arch/riscv/hwprobe.rst b/Documentation/arch/riscv/hwprobe.rst index 896ecfbbe5f4..41463b932268 100644 --- a/Documentation/arch/riscv/hwprobe.rst +++ b/Documentation/arch/riscv/hwprobe.rst @@ -157,6 +157,10 @@ The following keys are defined: defined in the RISC-V Vector manual starting from commit e2ccd0548d6c ("Remove draft warnings from Zvfh[min]"). + * :c:macro:`RISCV_HWPROBE_EXT_ZFA`: The Zfa extension is supported as + defined in the RISC-V ISA manual starting from commit 056b6ff467c7 + ("Zfa is ratified"). + * :c:macro:`RISCV_HWPROBE_KEY_CPUPERF_0`: A bitmask that contains performance information about the selected set of processors. diff --git a/arch/riscv/include/uapi/asm/hwprobe.h b/arch/riscv/include/uapi/asm/hwprobe.h index fffc69d9f6ba..91fbe1a7f2e2 100644 --- a/arch/riscv/include/uapi/asm/hwprobe.h +++ b/arch/riscv/include/uapi/asm/hwprobe.h @@ -55,6 +55,7 @@ struct riscv_hwprobe { #define RISCV_HWPROBE_EXT_ZIHINTNTL (1 << 29) #define RISCV_HWPROBE_EXT_ZVFH (1 << 30) #define RISCV_HWPROBE_EXT_ZVFHMIN (1 << 31) +#define RISCV_HWPROBE_EXT_ZFA (1ULL << 32) #define RISCV_HWPROBE_KEY_CPUPERF_0 5 #define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0) #define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0) diff --git a/arch/riscv/kernel/sys_riscv.c b/arch/riscv/kernel/sys_riscv.c index e90537593f5f..f0bd7b480b7f 100644 --- a/arch/riscv/kernel/sys_riscv.c +++ b/arch/riscv/kernel/sys_riscv.c @@ -193,6 +193,7 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, if (has_fpu()) { EXT_KEY(ZFH); EXT_KEY(ZFHMIN); + EXT_KEY(ZFA); } #undef EXT_KEY } From 9726acfdfa3bbf324b305e0b32fc028c278f6d43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Tue, 14 Nov 2023 09:12:56 -0500 Subject: [PATCH 153/336] dt-bindings: riscv: add Zfa ISA extension description MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add description for the Zfa ISA extension[1]. Link: https://drive.google.com/file/d/1VT6QIggpb59-8QRV266dEE4T8FZTxGq4/view [1] Signed-off-by: Clément Léger Acked-by: Conor Dooley Link: https://lore.kernel.org/r/20231114141256.126749-21-cleger@rivosinc.com Signed-off-by: Palmer Dabbelt --- Documentation/devicetree/bindings/riscv/extensions.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Documentation/devicetree/bindings/riscv/extensions.yaml b/Documentation/devicetree/bindings/riscv/extensions.yaml index b91d49b7c3a0..3574a0b70be4 100644 --- a/Documentation/devicetree/bindings/riscv/extensions.yaml +++ b/Documentation/devicetree/bindings/riscv/extensions.yaml @@ -214,6 +214,12 @@ properties: instructions as ratified at commit 6d33919 ("Merge pull request #158 from hirooih/clmul-fix-loop-end-condition") of riscv-bitmanip. + - const: zfa + description: + The standard Zfa extension for additional floating point + instructions, as ratified in commit 056b6ff ("Zfa is ratified") of + riscv-isa-manual. + - const: zfh description: The standard Zfh extension for 16-bit half-precision binary From 9d598fab9731055638c6e9333c4f21aa0d174a48 Mon Sep 17 00:00:00 2001 From: Luca Weiss Date: Fri, 8 Dec 2023 16:07:57 +0100 Subject: [PATCH 154/336] dt-bindings: remoteproc: qcom: sc7180-pas: Fix SC7280 MPSS PD-names The power domains for MPSS on SC7280 are actually named CX and MSS, and not CX and MX. Adjust the name which also aligns the bindings with the dts and fixes validation. Fixes: 8bb92d6fd0b3 ("dt-bindings: remoteproc: qcom,sc7180-pas: split into separate file") Acked-by: Krzysztof Kozlowski Signed-off-by: Luca Weiss Link: https://lore.kernel.org/r/20231208-sc7280-remoteprocs-v3-1-6aa394d33edf@fairphone.com Signed-off-by: Bjorn Andersson --- .../devicetree/bindings/remoteproc/qcom,sc7180-pas.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sc7180-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sc7180-pas.yaml index f10f329677d8..6f0bd6fa5d26 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,sc7180-pas.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,sc7180-pas.yaml @@ -114,11 +114,11 @@ allOf: power-domains: items: - description: CX power domain - - description: MX power domain + - description: MSS power domain power-domain-names: items: - const: cx - - const: mx + - const: mss unevaluatedProperties: false From 11eff1020440060c53d2261531432927c9fb4ee3 Mon Sep 17 00:00:00 2001 From: Luca Weiss Date: Fri, 8 Dec 2023 16:08:01 +0100 Subject: [PATCH 155/336] dt-bindings: remoteproc: qcom: sc7180-pas: Add SC7280 compatibles Add the compatibles and constraints for the ADSP, CDSP and WPSS found on the SC7280 SoC. Reviewed-by: Krzysztof Kozlowski Signed-off-by: Luca Weiss Link: https://lore.kernel.org/r/20231208-sc7280-remoteprocs-v3-5-6aa394d33edf@fairphone.com Signed-off-by: Bjorn Andersson --- .../bindings/remoteproc/qcom,sc7180-pas.yaml | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sc7180-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sc7180-pas.yaml index 6f0bd6fa5d26..c054b84fdcd5 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,sc7180-pas.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,sc7180-pas.yaml @@ -18,7 +18,10 @@ properties: enum: - qcom,sc7180-adsp-pas - qcom,sc7180-mpss-pas + - qcom,sc7280-adsp-pas + - qcom,sc7280-cdsp-pas - qcom,sc7280-mpss-pas + - qcom,sc7280-wpss-pas reg: maxItems: 1 @@ -75,6 +78,7 @@ allOf: compatible: enum: - qcom,sc7180-adsp-pas + - qcom,sc7280-adsp-pas then: properties: power-domains: @@ -120,6 +124,23 @@ allOf: - const: cx - const: mss + - if: + properties: + compatible: + enum: + - qcom,sc7280-cdsp-pas + - qcom,sc7280-wpss-pas + then: + properties: + power-domains: + items: + - description: CX power domain + - description: MX power domain + power-domain-names: + items: + - const: cx + - const: mx + unevaluatedProperties: false examples: From 300ed425dfa99f6926299ec196a1eedf05f47b21 Mon Sep 17 00:00:00 2001 From: Luca Weiss Date: Fri, 8 Dec 2023 16:08:02 +0100 Subject: [PATCH 156/336] remoteproc: qcom_q6v5_pas: Add SC7280 ADSP, CDSP & WPSS Add support for the ADSP, CDSP and WPSS remoteprocs found on the SC7280 SoC using the q6v5-pas driver. This driver can be used on regular LA ("Linux Android") based releases, however the SC7280 ChromeOS devices need different driver support due to firmware differences. Reviewed-by: Konrad Dybcio Signed-off-by: Luca Weiss Link: https://lore.kernel.org/r/20231208-sc7280-remoteprocs-v3-6-6aa394d33edf@fairphone.com Signed-off-by: Bjorn Andersson --- drivers/remoteproc/qcom_q6v5_pas.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/drivers/remoteproc/qcom_q6v5_pas.c b/drivers/remoteproc/qcom_q6v5_pas.c index 913a5d2068e8..a9dd58608052 100644 --- a/drivers/remoteproc/qcom_q6v5_pas.c +++ b/drivers/remoteproc/qcom_q6v5_pas.c @@ -1165,6 +1165,22 @@ static const struct adsp_data sm8550_mpss_resource = { .region_assign_idx = 2, }; +static const struct adsp_data sc7280_wpss_resource = { + .crash_reason_smem = 626, + .firmware_name = "wpss.mdt", + .pas_id = 6, + .auto_boot = true, + .proxy_pd_names = (char*[]){ + "cx", + "mx", + NULL + }, + .load_state = "wpss", + .ssr_name = "wpss", + .sysmon_name = "wpss", + .ssctl_id = 0x19, +}; + static const struct of_device_id adsp_of_match[] = { { .compatible = "qcom,msm8226-adsp-pil", .data = &adsp_resource_init}, { .compatible = "qcom,msm8953-adsp-pil", .data = &msm8996_adsp_resource}, @@ -1178,7 +1194,10 @@ static const struct of_device_id adsp_of_match[] = { { .compatible = "qcom,qcs404-wcss-pas", .data = &wcss_resource_init }, { .compatible = "qcom,sc7180-adsp-pas", .data = &sm8250_adsp_resource}, { .compatible = "qcom,sc7180-mpss-pas", .data = &mpss_resource_init}, + { .compatible = "qcom,sc7280-adsp-pas", .data = &sm8350_adsp_resource}, + { .compatible = "qcom,sc7280-cdsp-pas", .data = &sm6350_cdsp_resource}, { .compatible = "qcom,sc7280-mpss-pas", .data = &mpss_resource_init}, + { .compatible = "qcom,sc7280-wpss-pas", .data = &sc7280_wpss_resource}, { .compatible = "qcom,sc8180x-adsp-pas", .data = &sm8150_adsp_resource}, { .compatible = "qcom,sc8180x-cdsp-pas", .data = &sm8150_cdsp_resource}, { .compatible = "qcom,sc8180x-mpss-pas", .data = &sc8180x_mpss_resource}, From 1565c881c3df053447309ff69ec7fd5dee2085e4 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 14 Dec 2023 10:01:45 +0000 Subject: [PATCH 157/336] KVM: arm64: Explicitly trap unsupported HFGxTR_EL2 features Do not rely on the value of __HFGRTR_EL2_nMASK to trap unsupported features, since the nMASK can (and will) change as new traps are added and as its value is updated. Instead, explicitly specify the trap bits. Suggested-by: Joey Gouly Signed-off-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20231214100158.2305400-6-tabba@google.com --- arch/arm64/kvm/hyp/include/hyp/switch.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index f99d8af0b9af..7b4909dfd1f5 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -110,12 +110,15 @@ static inline void __activate_traps_hfgxtr(struct kvm_vcpu *vcpu) compute_clr_set(vcpu, HFGWTR_EL2, w_clr, w_set); } - /* The default is not to trap anything but ACCDATA_EL1 */ - r_val = __HFGRTR_EL2_nMASK & ~HFGxTR_EL2_nACCDATA_EL1; + /* The default to trap everything not handled or supported in KVM. */ + tmp = HFGxTR_EL2_nAMAIR2_EL1 | HFGxTR_EL2_nMAIR2_EL1 | HFGxTR_EL2_nS2POR_EL1 | + HFGxTR_EL2_nPOR_EL1 | HFGxTR_EL2_nPOR_EL0 | HFGxTR_EL2_nACCDATA_EL1; + + r_val = __HFGRTR_EL2_nMASK & ~tmp; r_val |= r_set; r_val &= ~r_clr; - w_val = __HFGWTR_EL2_nMASK & ~HFGxTR_EL2_nACCDATA_EL1; + w_val = __HFGWTR_EL2_nMASK & ~tmp; w_val |= w_set; w_val &= ~w_clr; From 9d400eb722bd1be712b007149ff1d8fb2d6470db Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 14 Dec 2023 10:01:46 +0000 Subject: [PATCH 158/336] KVM: arm64: Add missing HFGxTR_EL2 FGT entries to nested virt Add the missing nested virt FGT table entries HFGxTR_EL2. Based on DDI0601 2023-09. Signed-off-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20231214100158.2305400-7-tabba@google.com --- arch/arm64/kvm/emulate-nested.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c index 06185216a297..8b473a1bbc11 100644 --- a/arch/arm64/kvm/emulate-nested.c +++ b/arch/arm64/kvm/emulate-nested.c @@ -1042,10 +1042,20 @@ enum fg_filter_id { static const struct encoding_to_trap_config encoding_to_fgt[] __initconst = { /* HFGRTR_EL2, HFGWTR_EL2 */ + SR_FGT(SYS_AMAIR2_EL1, HFGxTR, nAMAIR2_EL1, 0), + SR_FGT(SYS_MAIR2_EL1, HFGxTR, nMAIR2_EL1, 0), + SR_FGT(SYS_S2POR_EL1, HFGxTR, nS2POR_EL1, 0), + SR_FGT(SYS_POR_EL1, HFGxTR, nPOR_EL1, 0), + SR_FGT(SYS_POR_EL0, HFGxTR, nPOR_EL0, 0), SR_FGT(SYS_PIR_EL1, HFGxTR, nPIR_EL1, 0), SR_FGT(SYS_PIRE0_EL1, HFGxTR, nPIRE0_EL1, 0), + SR_FGT(SYS_RCWMASK_EL1, HFGxTR, nRCWMASK_EL1, 0), SR_FGT(SYS_TPIDR2_EL0, HFGxTR, nTPIDR2_EL0, 0), SR_FGT(SYS_SMPRI_EL1, HFGxTR, nSMPRI_EL1, 0), + SR_FGT(SYS_GCSCR_EL1, HFGxTR, nGCS_EL1, 0), + SR_FGT(SYS_GCSPR_EL1, HFGxTR, nGCS_EL1, 0), + SR_FGT(SYS_GCSCRE0_EL1, HFGxTR, nGCS_EL0, 0), + SR_FGT(SYS_GCSPR_EL0, HFGxTR, nGCS_EL0, 0), SR_FGT(SYS_ACCDATA_EL1, HFGxTR, nACCDATA_EL1, 0), SR_FGT(SYS_ERXADDR_EL1, HFGxTR, ERXADDR_EL1, 1), SR_FGT(SYS_ERXPFGCDN_EL1, HFGxTR, ERXPFGCDN_EL1, 1), From 863ac38984a822ff9f4337d70853d771dcf7aae5 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 14 Dec 2023 10:01:47 +0000 Subject: [PATCH 159/336] KVM: arm64: Add missing HFGITR_EL2 FGT entries to nested virt Add the missing nested virt FGT table entries HFGITR_EL2. Based on DDI0601 and DDI0602 2023-09. Signed-off-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20231214100158.2305400-8-tabba@google.com --- arch/arm64/kvm/emulate-nested.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c index 8b473a1bbc11..89901550db34 100644 --- a/arch/arm64/kvm/emulate-nested.c +++ b/arch/arm64/kvm/emulate-nested.c @@ -1117,6 +1117,11 @@ static const struct encoding_to_trap_config encoding_to_fgt[] __initconst = { SR_FGT(SYS_AFSR1_EL1, HFGxTR, AFSR1_EL1, 1), SR_FGT(SYS_AFSR0_EL1, HFGxTR, AFSR0_EL1, 1), /* HFGITR_EL2 */ + SR_FGT(OP_AT_S1E1A, HFGITR, ATS1E1A, 1), + SR_FGT(OP_COSP_RCTX, HFGITR, COSPRCTX, 1), + SR_FGT(OP_GCSPUSHX, HFGITR, nGCSEPP, 0), + SR_FGT(OP_GCSPOPX, HFGITR, nGCSEPP, 0), + SR_FGT(OP_GCSPUSHM, HFGITR, nGCSPUSHM_EL1, 0), SR_FGT(OP_BRB_IALL, HFGITR, nBRBIALL, 0), SR_FGT(OP_BRB_INJ, HFGITR, nBRBINJ, 0), SR_FGT(SYS_DC_CVAC, HFGITR, DCCVAC, 1), From f9d6ed0213021ea00af30efbfa33e9a06c0610f2 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 14 Dec 2023 10:01:48 +0000 Subject: [PATCH 160/336] KVM: arm64: Add bit masks for HAFGRTR_EL2 To support HAFGRTR_EL2 supported in nested virt in the following patch, first add its bitmask definitions based on DDI0601 2023-09. Reviewed-by: Mark Brown Signed-off-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20231214100158.2305400-9-tabba@google.com --- arch/arm64/include/asm/kvm_arm.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index b85f46a73e21..7de0a7062625 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -370,6 +370,10 @@ #define __HDFGWTR_EL2_MASK ~__HDFGWTR_EL2_nMASK #define __HDFGWTR_EL2_nMASK GENMASK(62, 60) +#define __HAFGRTR_EL2_RES0 (GENMASK(63, 50) | GENMASK(16, 5)) +#define __HAFGRTR_EL2_MASK (GENMASK(49, 17) | GENMASK(4, 0)) +#define __HAFGRTR_EL2_nMASK 0UL + /* Similar definitions for HCRX_EL2 */ #define __HCRX_EL2_RES0 (GENMASK(63, 16) | GENMASK(13, 12)) #define __HCRX_EL2_MASK (0) From 676f482354886caa9b0cfa9236f5d20ac78f8c6a Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 14 Dec 2023 10:01:49 +0000 Subject: [PATCH 161/336] KVM: arm64: Handle HAFGRTR_EL2 trapping in nested virt Add the encodings to fine grain trapping fields for HAFGRTR_EL2 and add the associated handling code in nested virt. Based on DDI0601 2023-09. Add the missing field definitions as well, both to generate the correct RES0 mask and to be able to toggle their FGT bits. Also add the code for handling FGT trapping, reading of the register, to nested virt. Reviewed-by: Mark Brown Signed-off-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20231214100158.2305400-10-tabba@google.com --- arch/arm64/include/asm/kvm_host.h | 1 + arch/arm64/kvm/emulate-nested.c | 48 +++++++++++++++++++++++++ arch/arm64/kvm/hyp/include/hyp/switch.h | 25 +++++++++++++ arch/arm64/kvm/sys_regs.c | 1 + 4 files changed, 75 insertions(+) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 824f29f04916..ba14648e2de2 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -443,6 +443,7 @@ enum vcpu_sysreg { HFGITR_EL2, HDFGRTR_EL2, HDFGWTR_EL2, + HAFGRTR_EL2, CNTHP_CTL_EL2, CNTHP_CVAL_EL2, CNTHV_CTL_EL2, diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c index 89901550db34..431fd429932d 100644 --- a/arch/arm64/kvm/emulate-nested.c +++ b/arch/arm64/kvm/emulate-nested.c @@ -1012,6 +1012,7 @@ enum fgt_group_id { HDFGRTR_GROUP, HDFGWTR_GROUP, HFGITR_GROUP, + HAFGRTR_GROUP, /* Must be last */ __NR_FGT_GROUP_IDS__ @@ -1689,6 +1690,49 @@ static const struct encoding_to_trap_config encoding_to_fgt[] __initconst = { SR_FGT(SYS_PMCR_EL0, HDFGWTR, PMCR_EL0, 1), SR_FGT(SYS_PMSWINC_EL0, HDFGWTR, PMSWINC_EL0, 1), SR_FGT(SYS_OSLAR_EL1, HDFGWTR, OSLAR_EL1, 1), + /* + * HAFGRTR_EL2 + */ + SR_FGT(SYS_AMEVTYPER1_EL0(15), HAFGRTR, AMEVTYPER115_EL0, 1), + SR_FGT(SYS_AMEVTYPER1_EL0(14), HAFGRTR, AMEVTYPER114_EL0, 1), + SR_FGT(SYS_AMEVTYPER1_EL0(13), HAFGRTR, AMEVTYPER113_EL0, 1), + SR_FGT(SYS_AMEVTYPER1_EL0(12), HAFGRTR, AMEVTYPER112_EL0, 1), + SR_FGT(SYS_AMEVTYPER1_EL0(11), HAFGRTR, AMEVTYPER111_EL0, 1), + SR_FGT(SYS_AMEVTYPER1_EL0(10), HAFGRTR, AMEVTYPER110_EL0, 1), + SR_FGT(SYS_AMEVTYPER1_EL0(9), HAFGRTR, AMEVTYPER19_EL0, 1), + SR_FGT(SYS_AMEVTYPER1_EL0(8), HAFGRTR, AMEVTYPER18_EL0, 1), + SR_FGT(SYS_AMEVTYPER1_EL0(7), HAFGRTR, AMEVTYPER17_EL0, 1), + SR_FGT(SYS_AMEVTYPER1_EL0(6), HAFGRTR, AMEVTYPER16_EL0, 1), + SR_FGT(SYS_AMEVTYPER1_EL0(5), HAFGRTR, AMEVTYPER15_EL0, 1), + SR_FGT(SYS_AMEVTYPER1_EL0(4), HAFGRTR, AMEVTYPER14_EL0, 1), + SR_FGT(SYS_AMEVTYPER1_EL0(3), HAFGRTR, AMEVTYPER13_EL0, 1), + SR_FGT(SYS_AMEVTYPER1_EL0(2), HAFGRTR, AMEVTYPER12_EL0, 1), + SR_FGT(SYS_AMEVTYPER1_EL0(1), HAFGRTR, AMEVTYPER11_EL0, 1), + SR_FGT(SYS_AMEVTYPER1_EL0(0), HAFGRTR, AMEVTYPER10_EL0, 1), + SR_FGT(SYS_AMEVCNTR1_EL0(15), HAFGRTR, AMEVCNTR115_EL0, 1), + SR_FGT(SYS_AMEVCNTR1_EL0(14), HAFGRTR, AMEVCNTR114_EL0, 1), + SR_FGT(SYS_AMEVCNTR1_EL0(13), HAFGRTR, AMEVCNTR113_EL0, 1), + SR_FGT(SYS_AMEVCNTR1_EL0(12), HAFGRTR, AMEVCNTR112_EL0, 1), + SR_FGT(SYS_AMEVCNTR1_EL0(11), HAFGRTR, AMEVCNTR111_EL0, 1), + SR_FGT(SYS_AMEVCNTR1_EL0(10), HAFGRTR, AMEVCNTR110_EL0, 1), + SR_FGT(SYS_AMEVCNTR1_EL0(9), HAFGRTR, AMEVCNTR19_EL0, 1), + SR_FGT(SYS_AMEVCNTR1_EL0(8), HAFGRTR, AMEVCNTR18_EL0, 1), + SR_FGT(SYS_AMEVCNTR1_EL0(7), HAFGRTR, AMEVCNTR17_EL0, 1), + SR_FGT(SYS_AMEVCNTR1_EL0(6), HAFGRTR, AMEVCNTR16_EL0, 1), + SR_FGT(SYS_AMEVCNTR1_EL0(5), HAFGRTR, AMEVCNTR15_EL0, 1), + SR_FGT(SYS_AMEVCNTR1_EL0(4), HAFGRTR, AMEVCNTR14_EL0, 1), + SR_FGT(SYS_AMEVCNTR1_EL0(3), HAFGRTR, AMEVCNTR13_EL0, 1), + SR_FGT(SYS_AMEVCNTR1_EL0(2), HAFGRTR, AMEVCNTR12_EL0, 1), + SR_FGT(SYS_AMEVCNTR1_EL0(1), HAFGRTR, AMEVCNTR11_EL0, 1), + SR_FGT(SYS_AMEVCNTR1_EL0(0), HAFGRTR, AMEVCNTR10_EL0, 1), + SR_FGT(SYS_AMCNTENCLR1_EL0, HAFGRTR, AMCNTEN1, 1), + SR_FGT(SYS_AMCNTENSET1_EL0, HAFGRTR, AMCNTEN1, 1), + SR_FGT(SYS_AMCNTENCLR0_EL0, HAFGRTR, AMCNTEN0, 1), + SR_FGT(SYS_AMCNTENSET0_EL0, HAFGRTR, AMCNTEN0, 1), + SR_FGT(SYS_AMEVCNTR0_EL0(3), HAFGRTR, AMEVCNTR03_EL0, 1), + SR_FGT(SYS_AMEVCNTR0_EL0(2), HAFGRTR, AMEVCNTR02_EL0, 1), + SR_FGT(SYS_AMEVCNTR0_EL0(1), HAFGRTR, AMEVCNTR01_EL0, 1), + SR_FGT(SYS_AMEVCNTR0_EL0(0), HAFGRTR, AMEVCNTR00_EL0, 1), }; static union trap_config get_trap_config(u32 sysreg) @@ -1909,6 +1953,10 @@ bool __check_nv_sr_forward(struct kvm_vcpu *vcpu) val = sanitised_sys_reg(vcpu, HDFGWTR_EL2); break; + case HAFGRTR_GROUP: + val = sanitised_sys_reg(vcpu, HAFGRTR_EL2); + break; + case HFGITR_GROUP: val = sanitised_sys_reg(vcpu, HFGITR_EL2); switch (tc.fgf) { diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index 7b4909dfd1f5..0436f0da98a4 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -80,6 +80,14 @@ static inline void __activate_traps_fpsimd32(struct kvm_vcpu *vcpu) } while(0) +static inline bool cpu_has_amu(void) +{ + u64 pfr0 = read_sysreg_s(SYS_ID_AA64PFR0_EL1); + + return cpuid_feature_extract_unsigned_field(pfr0, + ID_AA64PFR0_EL1_AMU_SHIFT); +} + static inline void __activate_traps_hfgxtr(struct kvm_vcpu *vcpu) { struct kvm_cpu_context *hctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt; @@ -156,6 +164,20 @@ static inline void __activate_traps_hfgxtr(struct kvm_vcpu *vcpu) write_sysreg_s(r_val, SYS_HDFGRTR_EL2); write_sysreg_s(w_val, SYS_HDFGWTR_EL2); + + if (!cpu_has_amu()) + return; + + ctxt_sys_reg(hctxt, HAFGRTR_EL2) = read_sysreg_s(SYS_HAFGRTR_EL2); + + r_clr = r_set = 0; + compute_clr_set(vcpu, HAFGRTR_EL2, r_clr, r_set); + + r_val = __HAFGRTR_EL2_nMASK; + r_val |= r_set; + r_val &= ~r_clr; + + write_sysreg_s(r_val, SYS_HAFGRTR_EL2); } static inline void __deactivate_traps_hfgxtr(struct kvm_vcpu *vcpu) @@ -174,6 +196,9 @@ static inline void __deactivate_traps_hfgxtr(struct kvm_vcpu *vcpu) write_sysreg_s(ctxt_sys_reg(hctxt, HFGITR_EL2), SYS_HFGITR_EL2); write_sysreg_s(ctxt_sys_reg(hctxt, HDFGRTR_EL2), SYS_HDFGRTR_EL2); write_sysreg_s(ctxt_sys_reg(hctxt, HDFGWTR_EL2), SYS_HDFGWTR_EL2); + + if (vcpu_has_amu()) + write_sysreg_s(ctxt_sys_reg(hctxt, HAFGRTR_EL2), SYS_HAFGRTR_EL2); } static inline void __activate_traps_common(struct kvm_vcpu *vcpu) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 4735e1b37fb3..8bb297a2df38 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -2532,6 +2532,7 @@ static const struct sys_reg_desc sys_reg_descs[] = { { SYS_DESC(SYS_DACR32_EL2), trap_undef, reset_unknown, DACR32_EL2 }, EL2_REG(HDFGRTR_EL2, access_rw, reset_val, 0), EL2_REG(HDFGWTR_EL2, access_rw, reset_val, 0), + EL2_REG(HAFGRTR_EL2, access_rw, reset_val, 0), EL2_REG(SPSR_EL2, access_rw, reset_val, 0), EL2_REG(ELR_EL2, access_rw, reset_val, 0), { SYS_DESC(SYS_SP_EL1), access_sp_el1}, From fc04838f9c00fcbc90a8926bbd46928d6fb36477 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 14 Dec 2023 10:01:50 +0000 Subject: [PATCH 162/336] KVM: arm64: Update and fix FGT register masks New trap bits have been defined since the latest update to this patch. Moreover, the existing definitions of some of the mask and the RES0 bits overlap, which could be wrong, confusing, or both. Update the bits based on DDI0601 2023-09, and ensure that the existing bits are consistent. Subsequent patches will use the generated RES0 fields instead of specifying them manually. This patch keeps the manual encoding of the bits to make it easier to review the series. Fixes: 0fd76865006d ("KVM: arm64: Add nPIR{E0}_EL1 to HFG traps") Signed-off-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20231214100158.2305400-11-tabba@google.com --- arch/arm64/include/asm/kvm_arm.h | 39 ++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 7de0a7062625..b0dc3249d5cd 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -344,30 +344,39 @@ * Once we get to a point where the two describe the same thing, we'll * merge the definitions. One day. */ -#define __HFGRTR_EL2_RES0 (GENMASK(63, 56) | GENMASK(53, 51)) +#define __HFGRTR_EL2_RES0 BIT(51) #define __HFGRTR_EL2_MASK GENMASK(49, 0) -#define __HFGRTR_EL2_nMASK (GENMASK(58, 57) | GENMASK(55, 54) | BIT(50)) +#define __HFGRTR_EL2_nMASK (GENMASK(63, 52) | BIT(50)) -#define __HFGWTR_EL2_RES0 (GENMASK(63, 56) | GENMASK(53, 51) | \ - BIT(46) | BIT(42) | BIT(40) | BIT(28) | \ - GENMASK(26, 25) | BIT(21) | BIT(18) | \ +#define __HFGWTR_EL2_RES0 (BIT(51) | BIT(46) | BIT(42) | BIT(40) | \ + BIT(28) | GENMASK(26, 25) | BIT(21) | BIT(18) | \ GENMASK(15, 14) | GENMASK(10, 9) | BIT(2)) -#define __HFGWTR_EL2_MASK GENMASK(49, 0) -#define __HFGWTR_EL2_nMASK (GENMASK(58, 57) | GENMASK(55, 54) | BIT(50)) +#define __HFGWTR_EL2_MASK (GENMASK(49, 47) | GENMASK(45, 43) | \ + BIT(41) | GENMASK(39, 29) | BIT(27) | \ + GENMASK(24, 22) | GENMASK(20, 19) | \ + GENMASK(17, 16) | GENMASK(13, 11) | \ + GENMASK(8, 3) | GENMASK(1, 0)) +#define __HFGWTR_EL2_nMASK (GENMASK(63, 52) | BIT(50)) -#define __HFGITR_EL2_RES0 GENMASK(63, 57) -#define __HFGITR_EL2_MASK GENMASK(54, 0) -#define __HFGITR_EL2_nMASK GENMASK(56, 55) +#define __HFGITR_EL2_RES0 (BIT(63) | BIT(61)) +#define __HFGITR_EL2_MASK (BIT(62) | BIT(60) | GENMASK(54, 0)) +#define __HFGITR_EL2_nMASK GENMASK(59, 55) #define __HDFGRTR_EL2_RES0 (BIT(49) | BIT(42) | GENMASK(39, 38) | \ GENMASK(21, 20) | BIT(8)) -#define __HDFGRTR_EL2_MASK ~__HDFGRTR_EL2_nMASK +#define __HDFGRTR_EL2_MASK (BIT(63) | GENMASK(58, 50) | GENMASK(48, 43) | \ + GENMASK(41, 40) | GENMASK(37, 22) | \ + GENMASK(19, 9) | GENMASK(7, 0)) #define __HDFGRTR_EL2_nMASK GENMASK(62, 59) #define __HDFGWTR_EL2_RES0 (BIT(63) | GENMASK(59, 58) | BIT(51) | BIT(47) | \ BIT(43) | GENMASK(40, 38) | BIT(34) | BIT(30) | \ BIT(22) | BIT(9) | BIT(6)) -#define __HDFGWTR_EL2_MASK ~__HDFGWTR_EL2_nMASK +#define __HDFGWTR_EL2_MASK (GENMASK(57, 52) | GENMASK(50, 48) | \ + GENMASK(46, 44) | GENMASK(42, 41) | \ + GENMASK(37, 35) | GENMASK(33, 31) | \ + GENMASK(29, 23) | GENMASK(21, 10) | \ + GENMASK(8, 7) | GENMASK(5, 0)) #define __HDFGWTR_EL2_nMASK GENMASK(62, 60) #define __HAFGRTR_EL2_RES0 (GENMASK(63, 50) | GENMASK(16, 5)) @@ -375,9 +384,9 @@ #define __HAFGRTR_EL2_nMASK 0UL /* Similar definitions for HCRX_EL2 */ -#define __HCRX_EL2_RES0 (GENMASK(63, 16) | GENMASK(13, 12)) -#define __HCRX_EL2_MASK (0) -#define __HCRX_EL2_nMASK (GENMASK(15, 14) | GENMASK(4, 0)) +#define __HCRX_EL2_RES0 (GENMASK(63, 25) | GENMASK(13, 12)) +#define __HCRX_EL2_MASK (BIT(6)) +#define __HCRX_EL2_nMASK (GENMASK(24, 14) | GENMASK(11, 7) | GENMASK(5, 0)) /* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */ #define HPFAR_MASK (~UL(0xf)) From 6c4abbea6d9c09df448b43624074a208c38e68e0 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 14 Dec 2023 10:01:51 +0000 Subject: [PATCH 163/336] KVM: arm64: Add build validation for FGT trap mask values These checks help ensure that all the bits are accounted for, that there hasn't been a transcribing error from the spec nor from the generated mask values, which will be used in subsequent patches. Signed-off-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20231214100158.2305400-12-tabba@google.com --- arch/arm64/kvm/hyp/include/hyp/switch.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index 0436f0da98a4..ecccf99619e6 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -79,6 +79,16 @@ static inline void __activate_traps_fpsimd32(struct kvm_vcpu *vcpu) clr |= ~hfg & __ ## reg ## _nMASK; \ } while(0) +/* + * Validate the fine grain trap masks. + * Check that the masks do not overlap and that all bits are accounted for. + */ +#define CHECK_FGT_MASKS(reg) \ + do { \ + BUILD_BUG_ON((__ ## reg ## _MASK) & (__ ## reg ## _nMASK)); \ + BUILD_BUG_ON(~((__ ## reg ## _RES0) ^ (__ ## reg ## _MASK) ^ \ + (__ ## reg ## _nMASK))); \ + } while(0) static inline bool cpu_has_amu(void) { @@ -94,6 +104,14 @@ static inline void __activate_traps_hfgxtr(struct kvm_vcpu *vcpu) u64 r_clr = 0, w_clr = 0, r_set = 0, w_set = 0, tmp; u64 r_val, w_val; + CHECK_FGT_MASKS(HFGRTR_EL2); + CHECK_FGT_MASKS(HFGWTR_EL2); + CHECK_FGT_MASKS(HFGITR_EL2); + CHECK_FGT_MASKS(HDFGRTR_EL2); + CHECK_FGT_MASKS(HDFGWTR_EL2); + CHECK_FGT_MASKS(HAFGRTR_EL2); + CHECK_FGT_MASKS(HCRX_EL2); + if (!cpus_have_final_cap(ARM64_HAS_FGT)) return; From 9ff67dd26a9eed9d73dc23aa63e87b16b3382184 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 14 Dec 2023 10:01:52 +0000 Subject: [PATCH 164/336] KVM: arm64: Use generated FGT RES0 bits instead of specifying them Now that all FGT fields are accounted for and represented, use the generated value instead of manually specifying them. For __HFGWTR_EL2_RES0, however, there is no generated value. Its fields are subset of HFGRTR_EL2, with the remaining being RES0. Therefore, add a mask that represents the HFGRTR_EL2 only bits and define __HFGWTR_EL2_* using those and the __HFGRTR_EL2_* fields. No functional change intended. Signed-off-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20231214100158.2305400-13-tabba@google.com --- arch/arm64/include/asm/kvm_arm.h | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index b0dc3249d5cd..bd20d27f1b33 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -344,34 +344,32 @@ * Once we get to a point where the two describe the same thing, we'll * merge the definitions. One day. */ -#define __HFGRTR_EL2_RES0 BIT(51) +#define __HFGRTR_EL2_RES0 HFGxTR_EL2_RES0 #define __HFGRTR_EL2_MASK GENMASK(49, 0) #define __HFGRTR_EL2_nMASK (GENMASK(63, 52) | BIT(50)) -#define __HFGWTR_EL2_RES0 (BIT(51) | BIT(46) | BIT(42) | BIT(40) | \ - BIT(28) | GENMASK(26, 25) | BIT(21) | BIT(18) | \ +/* + * The HFGWTR bits are a subset of HFGRTR bits. To ensure we don't miss any + * future additions, define __HFGWTR* macros relative to __HFGRTR* ones. + */ +#define __HFGRTR_ONLY_MASK (BIT(46) | BIT(42) | BIT(40) | BIT(28) | \ + GENMASK(26, 25) | BIT(21) | BIT(18) | \ GENMASK(15, 14) | GENMASK(10, 9) | BIT(2)) -#define __HFGWTR_EL2_MASK (GENMASK(49, 47) | GENMASK(45, 43) | \ - BIT(41) | GENMASK(39, 29) | BIT(27) | \ - GENMASK(24, 22) | GENMASK(20, 19) | \ - GENMASK(17, 16) | GENMASK(13, 11) | \ - GENMASK(8, 3) | GENMASK(1, 0)) -#define __HFGWTR_EL2_nMASK (GENMASK(63, 52) | BIT(50)) +#define __HFGWTR_EL2_RES0 (__HFGRTR_EL2_RES0 | __HFGRTR_ONLY_MASK) +#define __HFGWTR_EL2_MASK (__HFGRTR_EL2_MASK & ~__HFGRTR_ONLY_MASK) +#define __HFGWTR_EL2_nMASK (__HFGRTR_EL2_nMASK & ~__HFGRTR_ONLY_MASK) -#define __HFGITR_EL2_RES0 (BIT(63) | BIT(61)) +#define __HFGITR_EL2_RES0 HFGITR_EL2_RES0 #define __HFGITR_EL2_MASK (BIT(62) | BIT(60) | GENMASK(54, 0)) #define __HFGITR_EL2_nMASK GENMASK(59, 55) -#define __HDFGRTR_EL2_RES0 (BIT(49) | BIT(42) | GENMASK(39, 38) | \ - GENMASK(21, 20) | BIT(8)) +#define __HDFGRTR_EL2_RES0 HDFGRTR_EL2_RES0 #define __HDFGRTR_EL2_MASK (BIT(63) | GENMASK(58, 50) | GENMASK(48, 43) | \ GENMASK(41, 40) | GENMASK(37, 22) | \ GENMASK(19, 9) | GENMASK(7, 0)) #define __HDFGRTR_EL2_nMASK GENMASK(62, 59) -#define __HDFGWTR_EL2_RES0 (BIT(63) | GENMASK(59, 58) | BIT(51) | BIT(47) | \ - BIT(43) | GENMASK(40, 38) | BIT(34) | BIT(30) | \ - BIT(22) | BIT(9) | BIT(6)) +#define __HDFGWTR_EL2_RES0 HDFGWTR_EL2_RES0 #define __HDFGWTR_EL2_MASK (GENMASK(57, 52) | GENMASK(50, 48) | \ GENMASK(46, 44) | GENMASK(42, 41) | \ GENMASK(37, 35) | GENMASK(33, 31) | \ @@ -379,12 +377,12 @@ GENMASK(8, 7) | GENMASK(5, 0)) #define __HDFGWTR_EL2_nMASK GENMASK(62, 60) -#define __HAFGRTR_EL2_RES0 (GENMASK(63, 50) | GENMASK(16, 5)) +#define __HAFGRTR_EL2_RES0 HAFGRTR_EL2_RES0 #define __HAFGRTR_EL2_MASK (GENMASK(49, 17) | GENMASK(4, 0)) #define __HAFGRTR_EL2_nMASK 0UL /* Similar definitions for HCRX_EL2 */ -#define __HCRX_EL2_RES0 (GENMASK(63, 25) | GENMASK(13, 12)) +#define __HCRX_EL2_RES0 HCRX_EL2_RES0 #define __HCRX_EL2_MASK (BIT(6)) #define __HCRX_EL2_nMASK (GENMASK(24, 14) | GENMASK(11, 7) | GENMASK(5, 0)) From 5f6bd3f3daaaab8559ad7d2266ba38345231b7ae Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 14 Dec 2023 10:01:53 +0000 Subject: [PATCH 165/336] KVM: arm64: Define FGT nMASK bits relative to other fields Now that RES0 and MASK have full coverage, no need to manually encode nMASK. Calculate it relative to the other fields. No functional change intended. Signed-off-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20231214100158.2305400-14-tabba@google.com --- arch/arm64/include/asm/kvm_arm.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index bd20d27f1b33..b7a9fe36bb59 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -346,7 +346,7 @@ */ #define __HFGRTR_EL2_RES0 HFGxTR_EL2_RES0 #define __HFGRTR_EL2_MASK GENMASK(49, 0) -#define __HFGRTR_EL2_nMASK (GENMASK(63, 52) | BIT(50)) +#define __HFGRTR_EL2_nMASK ~(__HFGRTR_EL2_RES0 | __HFGRTR_EL2_MASK) /* * The HFGWTR bits are a subset of HFGRTR bits. To ensure we don't miss any @@ -357,17 +357,17 @@ GENMASK(15, 14) | GENMASK(10, 9) | BIT(2)) #define __HFGWTR_EL2_RES0 (__HFGRTR_EL2_RES0 | __HFGRTR_ONLY_MASK) #define __HFGWTR_EL2_MASK (__HFGRTR_EL2_MASK & ~__HFGRTR_ONLY_MASK) -#define __HFGWTR_EL2_nMASK (__HFGRTR_EL2_nMASK & ~__HFGRTR_ONLY_MASK) +#define __HFGWTR_EL2_nMASK ~(__HFGWTR_EL2_RES0 | __HFGWTR_EL2_MASK) #define __HFGITR_EL2_RES0 HFGITR_EL2_RES0 #define __HFGITR_EL2_MASK (BIT(62) | BIT(60) | GENMASK(54, 0)) -#define __HFGITR_EL2_nMASK GENMASK(59, 55) +#define __HFGITR_EL2_nMASK ~(__HFGITR_EL2_RES0 | __HFGITR_EL2_MASK) #define __HDFGRTR_EL2_RES0 HDFGRTR_EL2_RES0 #define __HDFGRTR_EL2_MASK (BIT(63) | GENMASK(58, 50) | GENMASK(48, 43) | \ GENMASK(41, 40) | GENMASK(37, 22) | \ GENMASK(19, 9) | GENMASK(7, 0)) -#define __HDFGRTR_EL2_nMASK GENMASK(62, 59) +#define __HDFGRTR_EL2_nMASK ~(__HDFGRTR_EL2_RES0 | __HDFGRTR_EL2_MASK) #define __HDFGWTR_EL2_RES0 HDFGWTR_EL2_RES0 #define __HDFGWTR_EL2_MASK (GENMASK(57, 52) | GENMASK(50, 48) | \ @@ -375,16 +375,16 @@ GENMASK(37, 35) | GENMASK(33, 31) | \ GENMASK(29, 23) | GENMASK(21, 10) | \ GENMASK(8, 7) | GENMASK(5, 0)) -#define __HDFGWTR_EL2_nMASK GENMASK(62, 60) +#define __HDFGWTR_EL2_nMASK ~(__HDFGWTR_EL2_RES0 | __HDFGWTR_EL2_MASK) #define __HAFGRTR_EL2_RES0 HAFGRTR_EL2_RES0 #define __HAFGRTR_EL2_MASK (GENMASK(49, 17) | GENMASK(4, 0)) -#define __HAFGRTR_EL2_nMASK 0UL +#define __HAFGRTR_EL2_nMASK ~(__HAFGRTR_EL2_RES0 | __HAFGRTR_EL2_MASK) /* Similar definitions for HCRX_EL2 */ #define __HCRX_EL2_RES0 HCRX_EL2_RES0 #define __HCRX_EL2_MASK (BIT(6)) -#define __HCRX_EL2_nMASK (GENMASK(24, 14) | GENMASK(11, 7) | GENMASK(5, 0)) +#define __HCRX_EL2_nMASK ~(__HCRX_EL2_RES0 | __HCRX_EL2_MASK) /* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */ #define HPFAR_MASK (~UL(0xf)) From 0ccd901da1886cf9dc53ab36ad8f1160b65e41f1 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 14 Dec 2023 10:01:54 +0000 Subject: [PATCH 166/336] KVM: arm64: Macros for setting/clearing FGT bits There's a lot of boilerplate code for setting and clearing FGT bits when activating guest traps. Refactor it into macros. These macros will also be used in future patch series. No functional change intended. Signed-off-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20231214100158.2305400-15-tabba@google.com --- arch/arm64/kvm/hyp/include/hyp/switch.h | 69 ++++++++++--------------- 1 file changed, 27 insertions(+), 42 deletions(-) diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index ecccf99619e6..d56fef44dc31 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -79,6 +79,27 @@ static inline void __activate_traps_fpsimd32(struct kvm_vcpu *vcpu) clr |= ~hfg & __ ## reg ## _nMASK; \ } while(0) +#define update_fgt_traps_cs(vcpu, reg, clr, set) \ + do { \ + struct kvm_cpu_context *hctxt = \ + &this_cpu_ptr(&kvm_host_data)->host_ctxt; \ + u64 c = 0, s = 0; \ + \ + ctxt_sys_reg(hctxt, reg) = read_sysreg_s(SYS_ ## reg); \ + compute_clr_set(vcpu, reg, c, s); \ + s |= set; \ + c |= clr; \ + if (c || s) { \ + u64 val = __ ## reg ## _nMASK; \ + val |= s; \ + val &= ~c; \ + write_sysreg_s(val, SYS_ ## reg); \ + } \ + } while(0) + +#define update_fgt_traps(vcpu, reg) \ + update_fgt_traps_cs(vcpu, reg, 0, 0) + /* * Validate the fine grain trap masks. * Check that the masks do not overlap and that all bits are accounted for. @@ -154,48 +175,12 @@ static inline void __activate_traps_hfgxtr(struct kvm_vcpu *vcpu) if (!vcpu_has_nv(vcpu) || is_hyp_ctxt(vcpu)) return; - ctxt_sys_reg(hctxt, HFGITR_EL2) = read_sysreg_s(SYS_HFGITR_EL2); + update_fgt_traps(vcpu, HFGITR_EL2); + update_fgt_traps(vcpu, HDFGRTR_EL2); + update_fgt_traps(vcpu, HDFGWTR_EL2); - r_set = r_clr = 0; - compute_clr_set(vcpu, HFGITR_EL2, r_clr, r_set); - r_val = __HFGITR_EL2_nMASK; - r_val |= r_set; - r_val &= ~r_clr; - - write_sysreg_s(r_val, SYS_HFGITR_EL2); - - ctxt_sys_reg(hctxt, HDFGRTR_EL2) = read_sysreg_s(SYS_HDFGRTR_EL2); - ctxt_sys_reg(hctxt, HDFGWTR_EL2) = read_sysreg_s(SYS_HDFGWTR_EL2); - - r_clr = r_set = w_clr = w_set = 0; - - compute_clr_set(vcpu, HDFGRTR_EL2, r_clr, r_set); - compute_clr_set(vcpu, HDFGWTR_EL2, w_clr, w_set); - - r_val = __HDFGRTR_EL2_nMASK; - r_val |= r_set; - r_val &= ~r_clr; - - w_val = __HDFGWTR_EL2_nMASK; - w_val |= w_set; - w_val &= ~w_clr; - - write_sysreg_s(r_val, SYS_HDFGRTR_EL2); - write_sysreg_s(w_val, SYS_HDFGWTR_EL2); - - if (!cpu_has_amu()) - return; - - ctxt_sys_reg(hctxt, HAFGRTR_EL2) = read_sysreg_s(SYS_HAFGRTR_EL2); - - r_clr = r_set = 0; - compute_clr_set(vcpu, HAFGRTR_EL2, r_clr, r_set); - - r_val = __HAFGRTR_EL2_nMASK; - r_val |= r_set; - r_val &= ~r_clr; - - write_sysreg_s(r_val, SYS_HAFGRTR_EL2); + if (cpu_has_amu()) + update_fgt_traps(vcpu, HAFGRTR_EL2); } static inline void __deactivate_traps_hfgxtr(struct kvm_vcpu *vcpu) @@ -215,7 +200,7 @@ static inline void __deactivate_traps_hfgxtr(struct kvm_vcpu *vcpu) write_sysreg_s(ctxt_sys_reg(hctxt, HDFGRTR_EL2), SYS_HDFGRTR_EL2); write_sysreg_s(ctxt_sys_reg(hctxt, HDFGWTR_EL2), SYS_HDFGWTR_EL2); - if (vcpu_has_amu()) + if (cpu_has_amu()) write_sysreg_s(ctxt_sys_reg(hctxt, HAFGRTR_EL2), SYS_HAFGRTR_EL2); } From 73e3ce3f4a0e561e24ca71b20de00f03b427981e Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 14 Dec 2023 10:01:55 +0000 Subject: [PATCH 167/336] KVM: arm64: Fix which features are marked as allowed for protected VMs Cache maintenance operations are not trapped for protected VMs, and shouldn't be. Mark them as allowed. Moreover, features advertised by ID_AA64PFR2 and ID_AA64MMFR3 are (already) not allowed, mark them as such. Signed-off-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20231214100158.2305400-16-tabba@google.com --- arch/arm64/kvm/hyp/include/nvhe/fixed_config.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h b/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h index e91922daa8ca..8d97dff4bb7b 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h +++ b/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h @@ -69,6 +69,8 @@ ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_SSBS) \ ) +#define PVM_ID_AA64PFR2_ALLOW 0ULL + /* * Allow for protected VMs: * - Mixed-endian @@ -101,6 +103,7 @@ * - Privileged Access Never * - SError interrupt exceptions from speculative reads * - Enhanced Translation Synchronization + * - Control for cache maintenance permission */ #define PVM_ID_AA64MMFR1_ALLOW (\ ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_HAFDBS) | \ @@ -108,7 +111,8 @@ ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_HPDS) | \ ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_PAN) | \ ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_SpecSEI) | \ - ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_ETS) \ + ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_ETS) | \ + ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_CMOW) \ ) /* @@ -133,6 +137,8 @@ ARM64_FEATURE_MASK(ID_AA64MMFR2_EL1_E0PD) \ ) +#define PVM_ID_AA64MMFR3_ALLOW (0ULL) + /* * No support for Scalable Vectors for protected VMs: * Requires additional support from KVM, e.g., context-switching and From 21de26dbc5170dde8e4dfbfa1ecb77804ed6a377 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 14 Dec 2023 10:01:56 +0000 Subject: [PATCH 168/336] KVM: arm64: Mark PAuth as a restricted feature for protected VMs Protected VMs will only support basic PAuth (FEAT_PAuth). Mark it as restricted to ensure that later versions aren't supported for protected guests. Signed-off-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20231214100158.2305400-17-tabba@google.com --- arch/arm64/kvm/hyp/include/nvhe/fixed_config.h | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h b/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h index 8d97dff4bb7b..51f043649146 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h +++ b/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h @@ -184,10 +184,18 @@ ARM64_FEATURE_MASK(ID_AA64ISAR0_EL1_RNDR) \ ) +/* Restrict pointer authentication to the basic version. */ +#define PVM_ID_AA64ISAR1_RESTRICT_UNSIGNED (\ + FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_APA), ID_AA64ISAR1_EL1_APA_PAuth) | \ + FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_API), ID_AA64ISAR1_EL1_API_PAuth) \ + ) + +#define PVM_ID_AA64ISAR2_RESTRICT_UNSIGNED (\ + FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_APA3), ID_AA64ISAR2_EL1_APA3_PAuth) \ + ) + #define PVM_ID_AA64ISAR1_ALLOW (\ ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_DPB) | \ - ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_APA) | \ - ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_API) | \ ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_JSCVT) | \ ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_FCMA) | \ ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_LRCPC) | \ @@ -202,8 +210,8 @@ ) #define PVM_ID_AA64ISAR2_ALLOW (\ + ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_ATS1A)| \ ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_GPA3) | \ - ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_APA3) | \ ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_MOPS) \ ) From 9d52612690985fc0ee1ae1fbad61530a4f6bbb53 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 14 Dec 2023 10:01:57 +0000 Subject: [PATCH 169/336] KVM: arm64: Trap external trace for protected VMs pKVM does not support external trace for protected VMs. Trap external trace, and add the ExtTrcBuff to make it possible to check for the feature. Signed-off-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20231214100158.2305400-18-tabba@google.com --- arch/arm64/kvm/hyp/nvhe/pkvm.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 9d23a51d7f75..84b5c3f387d8 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -136,6 +136,10 @@ static void pvm_init_traps_aa64dfr0(struct kvm_vcpu *vcpu) cptr_set |= CPTR_EL2_TTA; } + /* Trap External Trace */ + if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_ExtTrcBuff), feature_ids)) + mdcr_clear |= MDCR_EL2_E2TB_MASK << MDCR_EL2_E2TB_SHIFT; + vcpu->arch.mdcr_el2 |= mdcr_set; vcpu->arch.mdcr_el2 &= ~mdcr_clear; vcpu->arch.cptr_el2 |= cptr_set; From d5362c37e1f8a40096452fc201c30e705750e687 Mon Sep 17 00:00:00 2001 From: Xiaolei Wang Date: Fri, 15 Dec 2023 10:00:49 +0800 Subject: [PATCH 170/336] rpmsg: virtio: Free driver_override when rpmsg_remove() Free driver_override when rpmsg_remove(), otherwise the following memory leak will occur: unreferenced object 0xffff0000d55d7080 (size 128): comm "kworker/u8:2", pid 56, jiffies 4294893188 (age 214.272s) hex dump (first 32 bytes): 72 70 6d 73 67 5f 6e 73 00 00 00 00 00 00 00 00 rpmsg_ns........ 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ backtrace: [<000000009c94c9c1>] __kmem_cache_alloc_node+0x1f8/0x320 [<000000002300d89b>] __kmalloc_node_track_caller+0x44/0x70 [<00000000228a60c3>] kstrndup+0x4c/0x90 [<0000000077158695>] driver_set_override+0xd0/0x164 [<000000003e9c4ea5>] rpmsg_register_device_override+0x98/0x170 [<000000001c0c89a8>] rpmsg_ns_register_device+0x24/0x30 [<000000008bbf8fa2>] rpmsg_probe+0x2e0/0x3ec [<00000000e65a68df>] virtio_dev_probe+0x1c0/0x280 [<00000000443331cc>] really_probe+0xbc/0x2dc [<00000000391064b1>] __driver_probe_device+0x78/0xe0 [<00000000a41c9a5b>] driver_probe_device+0xd8/0x160 [<000000009c3bd5df>] __device_attach_driver+0xb8/0x140 [<0000000043cd7614>] bus_for_each_drv+0x7c/0xd4 [<000000003b929a36>] __device_attach+0x9c/0x19c [<00000000a94e0ba8>] device_initial_probe+0x14/0x20 [<000000003c999637>] bus_probe_device+0xa0/0xac Signed-off-by: Xiaolei Wang Fixes: b0b03b811963 ("rpmsg: Release rpmsg devices in backends") Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20231215020049.78750-1-xiaolei.wang@windriver.com Signed-off-by: Mathieu Poirier --- drivers/rpmsg/virtio_rpmsg_bus.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/rpmsg/virtio_rpmsg_bus.c b/drivers/rpmsg/virtio_rpmsg_bus.c index dc87965f8164..1062939c3264 100644 --- a/drivers/rpmsg/virtio_rpmsg_bus.c +++ b/drivers/rpmsg/virtio_rpmsg_bus.c @@ -378,6 +378,7 @@ static void virtio_rpmsg_release_device(struct device *dev) struct rpmsg_device *rpdev = to_rpmsg_device(dev); struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev); + kfree(rpdev->driver_override); kfree(vch); } From 7ab6fb505b2a7447c4a7237a12c59e3ad0c7298c Mon Sep 17 00:00:00 2001 From: Bibo Mao Date: Tue, 19 Dec 2023 10:48:27 +0800 Subject: [PATCH 171/336] LoongArch: KVM: Optimization for memslot hugepage checking During shadow mmu page fault, there is checking for huge page for specified memslot. Page fault is hot path, check logic can be done when memslot is created. Here two flags are added for huge page checking, KVM_MEM_HUGEPAGE_CAPABLE and KVM_MEM_HUGEPAGE_INCAPABLE. Indeed for an optimized qemu, memslot for DRAM is always huge page aligned. The flag is firstly checked during hot page fault path. Now only huge page flag is supported, there is a long way for super page support in LoongArch system. Since super page size is 64G for 16K pagesize and 1G for 4K pagesize, 64G physical address is rarely used and LoongArch kernel needs support super page for 4K. Also memory layout of LoongArch qemu VM should be 1G aligned. Signed-off-by: Bibo Mao Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/kvm_host.h | 3 + arch/loongarch/kvm/mmu.c | 124 +++++++++++++++++--------- 2 files changed, 86 insertions(+), 41 deletions(-) diff --git a/arch/loongarch/include/asm/kvm_host.h b/arch/loongarch/include/asm/kvm_host.h index 11328700d4fa..0e89db020481 100644 --- a/arch/loongarch/include/asm/kvm_host.h +++ b/arch/loongarch/include/asm/kvm_host.h @@ -45,7 +45,10 @@ struct kvm_vcpu_stat { u64 signal_exits; }; +#define KVM_MEM_HUGEPAGE_CAPABLE (1UL << 0) +#define KVM_MEM_HUGEPAGE_INCAPABLE (1UL << 1) struct kvm_arch_memory_slot { + unsigned long flags; }; struct kvm_context { diff --git a/arch/loongarch/kvm/mmu.c b/arch/loongarch/kvm/mmu.c index 80480df5f550..915f17527893 100644 --- a/arch/loongarch/kvm/mmu.c +++ b/arch/loongarch/kvm/mmu.c @@ -13,6 +13,16 @@ #include #include +static inline bool kvm_hugepage_capable(struct kvm_memory_slot *slot) +{ + return slot->arch.flags & KVM_MEM_HUGEPAGE_CAPABLE; +} + +static inline bool kvm_hugepage_incapable(struct kvm_memory_slot *slot) +{ + return slot->arch.flags & KVM_MEM_HUGEPAGE_INCAPABLE; +} + static inline void kvm_ptw_prepare(struct kvm *kvm, kvm_ptw_ctx *ctx) { ctx->level = kvm->arch.root_level; @@ -365,6 +375,69 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, kvm_ptw_top(kvm->arch.pgd, start << PAGE_SHIFT, end << PAGE_SHIFT, &ctx); } +int kvm_arch_prepare_memory_region(struct kvm *kvm, const struct kvm_memory_slot *old, + struct kvm_memory_slot *new, enum kvm_mr_change change) +{ + gpa_t gpa_start; + hva_t hva_start; + size_t size, gpa_offset, hva_offset; + + if ((change != KVM_MR_MOVE) && (change != KVM_MR_CREATE)) + return 0; + /* + * Prevent userspace from creating a memory region outside of the + * VM GPA address space + */ + if ((new->base_gfn + new->npages) > (kvm->arch.gpa_size >> PAGE_SHIFT)) + return -ENOMEM; + + new->arch.flags = 0; + size = new->npages * PAGE_SIZE; + gpa_start = new->base_gfn << PAGE_SHIFT; + hva_start = new->userspace_addr; + if (IS_ALIGNED(size, PMD_SIZE) && IS_ALIGNED(gpa_start, PMD_SIZE) + && IS_ALIGNED(hva_start, PMD_SIZE)) + new->arch.flags |= KVM_MEM_HUGEPAGE_CAPABLE; + else { + /* + * Pages belonging to memslots that don't have the same + * alignment within a PMD for userspace and GPA cannot be + * mapped with PMD entries, because we'll end up mapping + * the wrong pages. + * + * Consider a layout like the following: + * + * memslot->userspace_addr: + * +-----+--------------------+--------------------+---+ + * |abcde|fgh Stage-1 block | Stage-1 block tv|xyz| + * +-----+--------------------+--------------------+---+ + * + * memslot->base_gfn << PAGE_SIZE: + * +---+--------------------+--------------------+-----+ + * |abc|def Stage-2 block | Stage-2 block |tvxyz| + * +---+--------------------+--------------------+-----+ + * + * If we create those stage-2 blocks, we'll end up with this + * incorrect mapping: + * d -> f + * e -> g + * f -> h + */ + gpa_offset = gpa_start & (PMD_SIZE - 1); + hva_offset = hva_start & (PMD_SIZE - 1); + if (gpa_offset != hva_offset) { + new->arch.flags |= KVM_MEM_HUGEPAGE_INCAPABLE; + } else { + if (gpa_offset == 0) + gpa_offset = PMD_SIZE; + if ((size + gpa_offset) < (PMD_SIZE * 2)) + new->arch.flags |= KVM_MEM_HUGEPAGE_INCAPABLE; + } + } + + return 0; +} + void kvm_arch_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *old, const struct kvm_memory_slot *new, @@ -562,47 +635,23 @@ out: } static bool fault_supports_huge_mapping(struct kvm_memory_slot *memslot, - unsigned long hva, unsigned long map_size, bool write) + unsigned long hva, bool write) { - size_t size; - gpa_t gpa_start; - hva_t uaddr_start, uaddr_end; + hva_t start, end; /* Disable dirty logging on HugePages */ if (kvm_slot_dirty_track_enabled(memslot) && write) return false; - size = memslot->npages * PAGE_SIZE; - gpa_start = memslot->base_gfn << PAGE_SHIFT; - uaddr_start = memslot->userspace_addr; - uaddr_end = uaddr_start + size; + if (kvm_hugepage_capable(memslot)) + return true; - /* - * Pages belonging to memslots that don't have the same alignment - * within a PMD for userspace and GPA cannot be mapped with stage-2 - * PMD entries, because we'll end up mapping the wrong pages. - * - * Consider a layout like the following: - * - * memslot->userspace_addr: - * +-----+--------------------+--------------------+---+ - * |abcde|fgh Stage-1 block | Stage-1 block tv|xyz| - * +-----+--------------------+--------------------+---+ - * - * memslot->base_gfn << PAGE_SIZE: - * +---+--------------------+--------------------+-----+ - * |abc|def Stage-2 block | Stage-2 block |tvxyz| - * +---+--------------------+--------------------+-----+ - * - * If we create those stage-2 blocks, we'll end up with this incorrect - * mapping: - * d -> f - * e -> g - * f -> h - */ - if ((gpa_start & (map_size - 1)) != (uaddr_start & (map_size - 1))) + if (kvm_hugepage_incapable(memslot)) return false; + start = memslot->userspace_addr; + end = start + memslot->npages * PAGE_SIZE; + /* * Next, let's make sure we're not trying to map anything not covered * by the memslot. This means we have to prohibit block size mappings @@ -615,8 +664,7 @@ static bool fault_supports_huge_mapping(struct kvm_memory_slot *memslot, * userspace_addr or the base_gfn, as both are equally aligned (per * the check above) and equally sized. */ - return (hva & ~(map_size - 1)) >= uaddr_start && - (hva & ~(map_size - 1)) + map_size <= uaddr_end; + return (hva >= ALIGN(start, PMD_SIZE)) && (hva < ALIGN_DOWN(end, PMD_SIZE)); } /* @@ -842,7 +890,7 @@ retry: /* Disable dirty logging on HugePages */ level = 0; - if (!fault_supports_huge_mapping(memslot, hva, PMD_SIZE, write)) { + if (!fault_supports_huge_mapping(memslot, hva, write)) { level = 0; } else { level = host_pfn_mapping_level(kvm, gfn, memslot); @@ -901,12 +949,6 @@ void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) { } -int kvm_arch_prepare_memory_region(struct kvm *kvm, const struct kvm_memory_slot *old, - struct kvm_memory_slot *new, enum kvm_mr_change change) -{ - return 0; -} - void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, const struct kvm_memory_slot *memslot) { From 161267320158920a601e40d83fdac60bcaa2acb5 Mon Sep 17 00:00:00 2001 From: Bibo Mao Date: Tue, 19 Dec 2023 10:48:27 +0800 Subject: [PATCH 172/336] LoongArch: KVM: Remove SW timer switch when vcpu is halt polling With halt-polling supported, there is checking for pending events or interrupts when vcpu executes idle instruction. Pending interrupts include injected SW interrupts and passthrough HW interrupts, such as HW timer interrupts, since HW timer works still even if vcpu exists from VM mode. Since HW timer pending interrupt can be set directly with CSR status register, and pending HW timer interrupt checking is used in vcpu block checking function, it is not necessary to switch to SW timer during halt-polling. This patch adds preemption disabling in function kvm_cpu_has_pending_timer(), and removes SW timer switching in idle instruction emulation function. Signed-off-by: Bibo Mao Signed-off-by: Huacai Chen --- arch/loongarch/kvm/exit.c | 13 ++----------- arch/loongarch/kvm/timer.c | 12 +++++++++--- arch/loongarch/kvm/vcpu.c | 9 ++++++++- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/arch/loongarch/kvm/exit.c b/arch/loongarch/kvm/exit.c index ce8de3fa472c..e708a1786d6b 100644 --- a/arch/loongarch/kvm/exit.c +++ b/arch/loongarch/kvm/exit.c @@ -200,17 +200,8 @@ int kvm_emu_idle(struct kvm_vcpu *vcpu) ++vcpu->stat.idle_exits; trace_kvm_exit_idle(vcpu, KVM_TRACE_EXIT_IDLE); - if (!kvm_arch_vcpu_runnable(vcpu)) { - /* - * Switch to the software timer before halt-polling/blocking as - * the guest's timer may be a break event for the vCPU, and the - * hypervisor timer runs only when the CPU is in guest mode. - * Switch before halt-polling so that KVM recognizes an expired - * timer before blocking. - */ - kvm_save_timer(vcpu); - kvm_vcpu_block(vcpu); - } + if (!kvm_arch_vcpu_runnable(vcpu)) + kvm_vcpu_halt(vcpu); return EMULATE_DONE; } diff --git a/arch/loongarch/kvm/timer.c b/arch/loongarch/kvm/timer.c index 284bf553fefe..12d58040122d 100644 --- a/arch/loongarch/kvm/timer.c +++ b/arch/loongarch/kvm/timer.c @@ -155,11 +155,17 @@ static void _kvm_save_timer(struct kvm_vcpu *vcpu) */ hrtimer_cancel(&vcpu->arch.swtimer); hrtimer_start(&vcpu->arch.swtimer, expire, HRTIMER_MODE_ABS_PINNED); - } else + } else if (vcpu->stat.generic.blocking) { /* - * Inject timer interrupt so that hall polling can dectect and exit + * Inject timer interrupt so that halt polling can dectect and exit. + * VCPU is scheduled out already and sleeps in rcuwait queue and + * will not poll pending events again. kvm_queue_irq() is not enough, + * hrtimer swtimer should be used here. */ - kvm_queue_irq(vcpu, INT_TI); + expire = ktime_add_ns(ktime_get(), 10); + vcpu->arch.expire = expire; + hrtimer_start(&vcpu->arch.swtimer, expire, HRTIMER_MODE_ABS_PINNED); + } } /* diff --git a/arch/loongarch/kvm/vcpu.c b/arch/loongarch/kvm/vcpu.c index 73d0c2b9c1a5..54f544b30f32 100644 --- a/arch/loongarch/kvm/vcpu.c +++ b/arch/loongarch/kvm/vcpu.c @@ -187,8 +187,15 @@ int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) { - return kvm_pending_timer(vcpu) || + int ret; + + /* Protect from TOD sync and vcpu_load/put() */ + preempt_disable(); + ret = kvm_pending_timer(vcpu) || kvm_read_hw_gcsr(LOONGARCH_CSR_ESTAT) & (1 << INT_TI); + preempt_enable(); + + return ret; } int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu) From 0d2abe67029644741bf7400b0d00c2faa3e1c455 Mon Sep 17 00:00:00 2001 From: Bibo Mao Date: Tue, 19 Dec 2023 10:48:27 +0800 Subject: [PATCH 173/336] LoongArch: KVM: Allow to access HW timer CSR registers always Currently HW timer CSR registers are allowed to access before entering to vm and disabled if switch to SW timer in host mode, instead it is not necessary to do so. HW timer CSR registers can be accessed always, it is nothing to do with whether it is in vm mode or host mode. This patch removes the limitation. Signed-off-by: Bibo Mao Signed-off-by: Huacai Chen --- arch/loongarch/kvm/main.c | 1 - arch/loongarch/kvm/timer.c | 25 +++++-------------------- 2 files changed, 5 insertions(+), 21 deletions(-) diff --git a/arch/loongarch/kvm/main.c b/arch/loongarch/kvm/main.c index 1c1d5199500e..86a2f2d0cb27 100644 --- a/arch/loongarch/kvm/main.c +++ b/arch/loongarch/kvm/main.c @@ -287,7 +287,6 @@ int kvm_arch_hardware_enable(void) if (env & CSR_GCFG_MATC_ROOT) gcfg |= CSR_GCFG_MATC_ROOT; - gcfg |= CSR_GCFG_TIT; write_csr_gcfg(gcfg); kvm_flush_tlb_all(); diff --git a/arch/loongarch/kvm/timer.c b/arch/loongarch/kvm/timer.c index 12d58040122d..d6d5bcea349b 100644 --- a/arch/loongarch/kvm/timer.c +++ b/arch/loongarch/kvm/timer.c @@ -70,15 +70,6 @@ void kvm_init_timer(struct kvm_vcpu *vcpu, unsigned long timer_hz) */ void kvm_acquire_timer(struct kvm_vcpu *vcpu) { - unsigned long cfg; - - cfg = read_csr_gcfg(); - if (!(cfg & CSR_GCFG_TIT)) - return; - - /* Enable guest access to hard timer */ - write_csr_gcfg(cfg & ~CSR_GCFG_TIT); - /* * Freeze the soft-timer and sync the guest stable timer with it. We do * this with interrupts disabled to avoid latency. @@ -174,21 +165,15 @@ static void _kvm_save_timer(struct kvm_vcpu *vcpu) */ void kvm_save_timer(struct kvm_vcpu *vcpu) { - unsigned long cfg; struct loongarch_csrs *csr = vcpu->arch.csr; preempt_disable(); - cfg = read_csr_gcfg(); - if (!(cfg & CSR_GCFG_TIT)) { - /* Disable guest use of hard timer */ - write_csr_gcfg(cfg | CSR_GCFG_TIT); - /* Save hard timer state */ - kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TCFG); - kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TVAL); - if (kvm_read_sw_gcsr(csr, LOONGARCH_CSR_TCFG) & CSR_TCFG_EN) - _kvm_save_timer(vcpu); - } + /* Save hard timer state */ + kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TCFG); + kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TVAL); + if (kvm_read_sw_gcsr(csr, LOONGARCH_CSR_TCFG) & CSR_TCFG_EN) + _kvm_save_timer(vcpu); /* Save timer-related state to vCPU context */ kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ESTAT); From 1ab9c6099495f79bfbcd6058d02d7556034a89b0 Mon Sep 17 00:00:00 2001 From: Bibo Mao Date: Tue, 19 Dec 2023 10:48:28 +0800 Subject: [PATCH 174/336] LoongArch: KVM: Remove kvm_acquire_timer() before entering guest Timer emulation method in VM is switch to SW timer, there are two places where timer emulation is needed. One is during vcpu thread context switch, the other is halt-polling with idle instruction emulation. SW timer switching is removed during halt-polling mode, so it is not necessary to disable SW timer before entering to guest. This patch removes SW timer handling before entering guest mode, and put it in HW timer restoring flow when vcpu thread is sched-in. With this patch, vm timer emulation is simpler, there is SW/HW timer switch only in vcpu thread context switch scenario. Signed-off-by: Bibo Mao Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/kvm_vcpu.h | 1 - arch/loongarch/kvm/timer.c | 22 ++++++-------------- arch/loongarch/kvm/vcpu.c | 29 --------------------------- 3 files changed, 6 insertions(+), 46 deletions(-) diff --git a/arch/loongarch/include/asm/kvm_vcpu.h b/arch/loongarch/include/asm/kvm_vcpu.h index 553cfa2b2b1c..0e87652f780a 100644 --- a/arch/loongarch/include/asm/kvm_vcpu.h +++ b/arch/loongarch/include/asm/kvm_vcpu.h @@ -55,7 +55,6 @@ void kvm_save_fpu(struct loongarch_fpu *fpu); void kvm_restore_fpu(struct loongarch_fpu *fpu); void kvm_restore_fcsr(struct loongarch_fpu *fpu); -void kvm_acquire_timer(struct kvm_vcpu *vcpu); void kvm_init_timer(struct kvm_vcpu *vcpu, unsigned long hz); void kvm_reset_timer(struct kvm_vcpu *vcpu); void kvm_save_timer(struct kvm_vcpu *vcpu); diff --git a/arch/loongarch/kvm/timer.c b/arch/loongarch/kvm/timer.c index d6d5bcea349b..d362d87a54aa 100644 --- a/arch/loongarch/kvm/timer.c +++ b/arch/loongarch/kvm/timer.c @@ -64,19 +64,6 @@ void kvm_init_timer(struct kvm_vcpu *vcpu, unsigned long timer_hz) kvm_write_sw_gcsr(vcpu->arch.csr, LOONGARCH_CSR_TVAL, 0); } -/* - * Restore hard timer state and enable guest to access timer registers - * without trap, should be called with irq disabled - */ -void kvm_acquire_timer(struct kvm_vcpu *vcpu) -{ - /* - * Freeze the soft-timer and sync the guest stable timer with it. We do - * this with interrupts disabled to avoid latency. - */ - hrtimer_cancel(&vcpu->arch.swtimer); -} - /* * Restore soft timer state from saved context. */ @@ -98,6 +85,11 @@ void kvm_restore_timer(struct kvm_vcpu *vcpu) return; } + /* + * Freeze the soft-timer and sync the guest stable timer with it. + */ + hrtimer_cancel(&vcpu->arch.swtimer); + /* * Set remainder tick value if not expired */ @@ -115,7 +107,7 @@ void kvm_restore_timer(struct kvm_vcpu *vcpu) /* * Inject timer here though sw timer should inject timer * interrupt async already, since sw timer may be cancelled - * during injecting intr async in function kvm_acquire_timer + * during injecting intr async */ kvm_queue_irq(vcpu, INT_TI); } @@ -140,11 +132,9 @@ static void _kvm_save_timer(struct kvm_vcpu *vcpu) vcpu->arch.expire = expire; if (ticks) { /* - * Update hrtimer to use new timeout * HRTIMER_MODE_PINNED is suggested since vcpu may run in * the same physical cpu in next time */ - hrtimer_cancel(&vcpu->arch.swtimer); hrtimer_start(&vcpu->arch.swtimer, expire, HRTIMER_MODE_ABS_PINNED); } else if (vcpu->stat.generic.blocking) { /* diff --git a/arch/loongarch/kvm/vcpu.c b/arch/loongarch/kvm/vcpu.c index 54f544b30f32..53fcef8b24a1 100644 --- a/arch/loongarch/kvm/vcpu.c +++ b/arch/loongarch/kvm/vcpu.c @@ -95,7 +95,6 @@ static int kvm_pre_enter_guest(struct kvm_vcpu *vcpu) * check vmid before vcpu enter guest */ local_irq_disable(); - kvm_acquire_timer(vcpu); kvm_deliver_intr(vcpu); kvm_deliver_exception(vcpu); /* Make sure the vcpu mode has been written */ @@ -251,23 +250,6 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, return -EINVAL; } -/** - * kvm_migrate_count() - Migrate timer. - * @vcpu: Virtual CPU. - * - * Migrate hrtimer to the current CPU by cancelling and restarting it - * if the hrtimer is active. - * - * Must be called when the vCPU is migrated to a different CPU, so that - * the timer can interrupt the guest at the new CPU, and the timer irq can - * be delivered to the vCPU. - */ -static void kvm_migrate_count(struct kvm_vcpu *vcpu) -{ - if (hrtimer_cancel(&vcpu->arch.swtimer)) - hrtimer_restart(&vcpu->arch.swtimer); -} - static int _kvm_getcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 *val) { unsigned long gintc; @@ -796,17 +778,6 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) unsigned long flags; local_irq_save(flags); - if (vcpu->arch.last_sched_cpu != cpu) { - kvm_debug("[%d->%d]KVM vCPU[%d] switch\n", - vcpu->arch.last_sched_cpu, cpu, vcpu->vcpu_id); - /* - * Migrate the timer interrupt to the current CPU so that it - * always interrupts the guest and synchronously triggers a - * guest timer interrupt. - */ - kvm_migrate_count(vcpu); - } - /* Restore guest state to registers */ _kvm_vcpu_load(vcpu, cpu); local_irq_restore(flags); From 5b3d524993ff1fb36089be850ccb121ac3296bcf Mon Sep 17 00:00:00 2001 From: Bibo Mao Date: Tue, 19 Dec 2023 10:48:28 +0800 Subject: [PATCH 175/336] LoongArch: KVM: Fix timer emulation with oneshot mode When timer is fired in oneshot mode, CSR TVAL will be -1 rather than 0. There needs special handing for this situation. There are two scenarios when oneshot timer is fired. One scenario is that time is fired after exiting to host, CSR TVAL is set with 0 in order to inject hw interrupt, and -1 will assigned to CSR TVAL soon. The other situation is that timer is fired in VM and guest kernel is hanlding timer IRQ, IRQ is acked and is ready to set next expired timer value, then vm exits to host. Timer interrupt should not be inject at this point, else there will be spurious timer interrupt. Here hw timer irq status in CSR ESTAT is used to judge these two scenarios. If CSR TVAL is -1, the oneshot timer is fired; and if timer hw irq is on in CSR ESTAT register, it happens after exiting to host; else if timer hw irq is off, we think that it happens in vm and timer IRQ handler has already acked IRQ. With this patch, runltp with version ltp20230516 passes to run in vm. Signed-off-by: Bibo Mao Signed-off-by: Huacai Chen --- arch/loongarch/kvm/timer.c | 68 ++++++++++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 13 deletions(-) diff --git a/arch/loongarch/kvm/timer.c b/arch/loongarch/kvm/timer.c index d362d87a54aa..111328f60872 100644 --- a/arch/loongarch/kvm/timer.c +++ b/arch/loongarch/kvm/timer.c @@ -69,14 +69,19 @@ void kvm_init_timer(struct kvm_vcpu *vcpu, unsigned long timer_hz) */ void kvm_restore_timer(struct kvm_vcpu *vcpu) { - unsigned long cfg, delta, period; + unsigned long cfg, estat; + unsigned long ticks, delta, period; ktime_t expire, now; struct loongarch_csrs *csr = vcpu->arch.csr; /* * Set guest stable timer cfg csr + * Disable timer before restore estat CSR register, avoid to + * get invalid timer interrupt for old timer cfg */ cfg = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_TCFG); + + write_gcsr_timercfg(0); kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ESTAT); kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TCFG); if (!(cfg & CSR_TCFG_EN)) { @@ -90,20 +95,47 @@ void kvm_restore_timer(struct kvm_vcpu *vcpu) */ hrtimer_cancel(&vcpu->arch.swtimer); + /* + * From LoongArch Reference Manual Volume 1 Chapter 7.6.2 + * If oneshot timer is fired, CSR TVAL will be -1, there are two + * conditions: + * 1) timer is fired during exiting to host + * 2) timer is fired and vm is doing timer irq, and then exiting to + * host. Host should not inject timer irq to avoid spurious + * timer interrupt again + */ + ticks = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_TVAL); + estat = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_ESTAT); + if (!(cfg & CSR_TCFG_PERIOD) && (ticks > cfg)) { + /* + * Writing 0 to LOONGARCH_CSR_TVAL will inject timer irq + * and set CSR TVAL with -1 + */ + write_gcsr_timertick(0); + + /* + * Writing CSR_TINTCLR_TI to LOONGARCH_CSR_TINTCLR will clear + * timer interrupt, and CSR TVAL keeps unchanged with -1, it + * avoids spurious timer interrupt + */ + if (!(estat & CPU_TIMER)) + gcsr_write(CSR_TINTCLR_TI, LOONGARCH_CSR_TINTCLR); + return; + } + /* * Set remainder tick value if not expired */ + delta = 0; now = ktime_get(); expire = vcpu->arch.expire; if (ktime_before(now, expire)) delta = ktime_to_tick(vcpu, ktime_sub(expire, now)); - else { - if (cfg & CSR_TCFG_PERIOD) { - period = cfg & CSR_TCFG_VAL; - delta = ktime_to_tick(vcpu, ktime_sub(now, expire)); - delta = period - (delta % period); - } else - delta = 0; + else if (cfg & CSR_TCFG_PERIOD) { + period = cfg & CSR_TCFG_VAL; + delta = ktime_to_tick(vcpu, ktime_sub(now, expire)); + delta = period - (delta % period); + /* * Inject timer here though sw timer should inject timer * interrupt async already, since sw timer may be cancelled @@ -122,15 +154,25 @@ void kvm_restore_timer(struct kvm_vcpu *vcpu) */ static void _kvm_save_timer(struct kvm_vcpu *vcpu) { - unsigned long ticks, delta; + unsigned long ticks, delta, cfg; ktime_t expire; struct loongarch_csrs *csr = vcpu->arch.csr; + cfg = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_TCFG); ticks = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_TVAL); - delta = tick_to_ns(vcpu, ticks); - expire = ktime_add_ns(ktime_get(), delta); - vcpu->arch.expire = expire; - if (ticks) { + + /* + * From LoongArch Reference Manual Volume 1 Chapter 7.6.2 + * If period timer is fired, CSR TVAL will be reloaded from CSR TCFG + * If oneshot timer is fired, CSR TVAL will be -1 + * Here judge one-shot timer fired by checking whether TVAL is larger + * than TCFG + */ + if (ticks < cfg) { + delta = tick_to_ns(vcpu, ticks); + expire = ktime_add_ns(ktime_get(), delta); + vcpu->arch.expire = expire; + /* * HRTIMER_MODE_PINNED is suggested since vcpu may run in * the same physical cpu in next time From db1ecca22edf27c5a3dd66af406c88b5b5ac7cc1 Mon Sep 17 00:00:00 2001 From: Tianrui Zhao Date: Tue, 19 Dec 2023 10:48:28 +0800 Subject: [PATCH 176/336] LoongArch: KVM: Add LSX (128bit SIMD) support This patch adds LSX (128bit SIMD) support for LoongArch KVM. There will be LSX exception in KVM when guest use the LSX instructions. KVM will enable LSX and restore the vector registers for guest and then return to guest to continue running. Signed-off-by: Tianrui Zhao Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/kvm_host.h | 15 +- arch/loongarch/include/asm/kvm_vcpu.h | 10 ++ arch/loongarch/include/uapi/asm/kvm.h | 1 + arch/loongarch/kernel/fpu.S | 1 + arch/loongarch/kvm/exit.c | 21 +++ arch/loongarch/kvm/switch.S | 16 ++ arch/loongarch/kvm/trace.h | 4 +- arch/loongarch/kvm/vcpu.c | 220 +++++++++++++++++++++++++- 8 files changed, 280 insertions(+), 8 deletions(-) diff --git a/arch/loongarch/include/asm/kvm_host.h b/arch/loongarch/include/asm/kvm_host.h index 0e89db020481..b0c5cdd8014c 100644 --- a/arch/loongarch/include/asm/kvm_host.h +++ b/arch/loongarch/include/asm/kvm_host.h @@ -95,8 +95,9 @@ enum emulation_result { }; #define KVM_LARCH_FPU (0x1 << 0) -#define KVM_LARCH_SWCSR_LATEST (0x1 << 1) -#define KVM_LARCH_HWCSR_USABLE (0x1 << 2) +#define KVM_LARCH_LSX (0x1 << 1) +#define KVM_LARCH_SWCSR_LATEST (0x1 << 2) +#define KVM_LARCH_HWCSR_USABLE (0x1 << 3) struct kvm_vcpu_arch { /* @@ -178,6 +179,16 @@ static inline void writel_sw_gcsr(struct loongarch_csrs *csr, int reg, unsigned csr->csrs[reg] = val; } +static inline bool kvm_guest_has_fpu(struct kvm_vcpu_arch *arch) +{ + return arch->cpucfg[2] & CPUCFG2_FP; +} + +static inline bool kvm_guest_has_lsx(struct kvm_vcpu_arch *arch) +{ + return arch->cpucfg[2] & CPUCFG2_LSX; +} + /* Debug: dump vcpu state */ int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu); diff --git a/arch/loongarch/include/asm/kvm_vcpu.h b/arch/loongarch/include/asm/kvm_vcpu.h index 0e87652f780a..db08dd46b525 100644 --- a/arch/loongarch/include/asm/kvm_vcpu.h +++ b/arch/loongarch/include/asm/kvm_vcpu.h @@ -55,6 +55,16 @@ void kvm_save_fpu(struct loongarch_fpu *fpu); void kvm_restore_fpu(struct loongarch_fpu *fpu); void kvm_restore_fcsr(struct loongarch_fpu *fpu); +#ifdef CONFIG_CPU_HAS_LSX +int kvm_own_lsx(struct kvm_vcpu *vcpu); +void kvm_save_lsx(struct loongarch_fpu *fpu); +void kvm_restore_lsx(struct loongarch_fpu *fpu); +#else +static inline int kvm_own_lsx(struct kvm_vcpu *vcpu) { } +static inline void kvm_save_lsx(struct loongarch_fpu *fpu) { } +static inline void kvm_restore_lsx(struct loongarch_fpu *fpu) { } +#endif + void kvm_init_timer(struct kvm_vcpu *vcpu, unsigned long hz); void kvm_reset_timer(struct kvm_vcpu *vcpu); void kvm_save_timer(struct kvm_vcpu *vcpu); diff --git a/arch/loongarch/include/uapi/asm/kvm.h b/arch/loongarch/include/uapi/asm/kvm.h index c6ad2ee6106c..923d0bd38294 100644 --- a/arch/loongarch/include/uapi/asm/kvm.h +++ b/arch/loongarch/include/uapi/asm/kvm.h @@ -79,6 +79,7 @@ struct kvm_fpu { #define LOONGARCH_REG_64(TYPE, REG) (TYPE | KVM_REG_SIZE_U64 | (REG << LOONGARCH_REG_SHIFT)) #define KVM_IOC_CSRID(REG) LOONGARCH_REG_64(KVM_REG_LOONGARCH_CSR, REG) #define KVM_IOC_CPUCFG(REG) LOONGARCH_REG_64(KVM_REG_LOONGARCH_CPUCFG, REG) +#define KVM_LOONGARCH_VCPU_CPUCFG 0 struct kvm_debug_exit_arch { }; diff --git a/arch/loongarch/kernel/fpu.S b/arch/loongarch/kernel/fpu.S index d53ab10f4644..a400924c0348 100644 --- a/arch/loongarch/kernel/fpu.S +++ b/arch/loongarch/kernel/fpu.S @@ -349,6 +349,7 @@ SYM_FUNC_START(_restore_lsx_upper) lsx_restore_all_upper a0 t0 t1 jr ra SYM_FUNC_END(_restore_lsx_upper) +EXPORT_SYMBOL(_restore_lsx_upper) SYM_FUNC_START(_init_lsx_upper) lsx_init_all_upper t1 diff --git a/arch/loongarch/kvm/exit.c b/arch/loongarch/kvm/exit.c index e708a1786d6b..676f7a3a335c 100644 --- a/arch/loongarch/kvm/exit.c +++ b/arch/loongarch/kvm/exit.c @@ -634,6 +634,11 @@ static int kvm_handle_fpu_disabled(struct kvm_vcpu *vcpu) { struct kvm_run *run = vcpu->run; + if (!kvm_guest_has_fpu(&vcpu->arch)) { + kvm_queue_exception(vcpu, EXCCODE_INE, 0); + return RESUME_GUEST; + } + /* * If guest FPU not present, the FPU operation should have been * treated as a reserved instruction! @@ -650,6 +655,21 @@ static int kvm_handle_fpu_disabled(struct kvm_vcpu *vcpu) return RESUME_GUEST; } +/* + * kvm_handle_lsx_disabled() - Guest used LSX while disabled in root. + * @vcpu: Virtual CPU context. + * + * Handle when the guest attempts to use LSX when it is disabled in the root + * context. + */ +static int kvm_handle_lsx_disabled(struct kvm_vcpu *vcpu) +{ + if (kvm_own_lsx(vcpu)) + kvm_queue_exception(vcpu, EXCCODE_INE, 0); + + return RESUME_GUEST; +} + /* * LoongArch KVM callback handling for unimplemented guest exiting */ @@ -678,6 +698,7 @@ static exit_handle_fn kvm_fault_tables[EXCCODE_INT_START] = { [EXCCODE_TLBS] = kvm_handle_write_fault, [EXCCODE_TLBM] = kvm_handle_write_fault, [EXCCODE_FPDIS] = kvm_handle_fpu_disabled, + [EXCCODE_LSXDIS] = kvm_handle_lsx_disabled, [EXCCODE_GSPR] = kvm_handle_gspr, }; diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S index 0ed9040307b7..00fbf772d16f 100644 --- a/arch/loongarch/kvm/switch.S +++ b/arch/loongarch/kvm/switch.S @@ -245,6 +245,22 @@ SYM_FUNC_START(kvm_restore_fpu) jr ra SYM_FUNC_END(kvm_restore_fpu) +#ifdef CONFIG_CPU_HAS_LSX +SYM_FUNC_START(kvm_save_lsx) + fpu_save_csr a0 t1 + fpu_save_cc a0 t1 t2 + lsx_save_data a0 t1 + jr ra +SYM_FUNC_END(kvm_save_lsx) + +SYM_FUNC_START(kvm_restore_lsx) + lsx_restore_data a0 t1 + fpu_restore_cc a0 t1 t2 + fpu_restore_csr a0 t1 t2 + jr ra +SYM_FUNC_END(kvm_restore_lsx) +#endif + .section ".rodata" SYM_DATA(kvm_exception_size, .quad kvm_exc_entry_end - kvm_exc_entry) SYM_DATA(kvm_enter_guest_size, .quad kvm_enter_guest_end - kvm_enter_guest) diff --git a/arch/loongarch/kvm/trace.h b/arch/loongarch/kvm/trace.h index a1e35d655418..7da4e230e896 100644 --- a/arch/loongarch/kvm/trace.h +++ b/arch/loongarch/kvm/trace.h @@ -102,6 +102,7 @@ TRACE_EVENT(kvm_exit_gspr, #define KVM_TRACE_AUX_DISCARD 4 #define KVM_TRACE_AUX_FPU 1 +#define KVM_TRACE_AUX_LSX 2 #define kvm_trace_symbol_aux_op \ { KVM_TRACE_AUX_SAVE, "save" }, \ @@ -111,7 +112,8 @@ TRACE_EVENT(kvm_exit_gspr, { KVM_TRACE_AUX_DISCARD, "discard" } #define kvm_trace_symbol_aux_state \ - { KVM_TRACE_AUX_FPU, "FPU" } + { KVM_TRACE_AUX_FPU, "FPU" }, \ + { KVM_TRACE_AUX_LSX, "LSX" } TRACE_EVENT(kvm_aux, TP_PROTO(struct kvm_vcpu *vcpu, unsigned int op, diff --git a/arch/loongarch/kvm/vcpu.c b/arch/loongarch/kvm/vcpu.c index 53fcef8b24a1..80487d177ca4 100644 --- a/arch/loongarch/kvm/vcpu.c +++ b/arch/loongarch/kvm/vcpu.c @@ -298,6 +298,69 @@ static int _kvm_setcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 val) return ret; } +static int _kvm_get_cpucfg(int id, u64 *v) +{ + int ret = 0; + + if (id < 0 && id >= KVM_MAX_CPUCFG_REGS) + return -EINVAL; + + switch (id) { + case 2: + /* Return CPUCFG2 features which have been supported by KVM */ + *v = CPUCFG2_FP | CPUCFG2_FPSP | CPUCFG2_FPDP | + CPUCFG2_FPVERS | CPUCFG2_LLFTP | CPUCFG2_LLFTPREV | + CPUCFG2_LAM; + /* + * If LSX is supported by CPU, it is also supported by KVM, + * as we implement it. + */ + if (cpu_has_lsx) + *v |= CPUCFG2_LSX; + break; + default: + ret = -EINVAL; + break; + } + return ret; +} + +static int kvm_check_cpucfg(int id, u64 val) +{ + u64 mask; + int ret = 0; + + if (id < 0 && id >= KVM_MAX_CPUCFG_REGS) + return -EINVAL; + + if (_kvm_get_cpucfg(id, &mask)) + return ret; + + switch (id) { + case 2: + /* CPUCFG2 features checking */ + if (val & ~mask) + /* The unsupported features should not be set */ + ret = -EINVAL; + else if (!(val & CPUCFG2_LLFTP)) + /* The LLFTP must be set, as guest must has a constant timer */ + ret = -EINVAL; + else if ((val & CPUCFG2_FP) && (!(val & CPUCFG2_FPSP) || !(val & CPUCFG2_FPDP))) + /* Single and double float point must both be set when enable FP */ + ret = -EINVAL; + else if ((val & CPUCFG2_LSX) && !(val & CPUCFG2_FP)) + /* FP should be set when enable LSX */ + ret = -EINVAL; + else if ((val & CPUCFG2_LASX) && !(val & CPUCFG2_LSX)) + /* LSX, FP should be set when enable LASX, and FP has been checked before. */ + ret = -EINVAL; + break; + default: + break; + } + return ret; +} + static int kvm_get_one_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg, u64 *v) { @@ -367,10 +430,10 @@ static int kvm_set_one_reg(struct kvm_vcpu *vcpu, break; case KVM_REG_LOONGARCH_CPUCFG: id = KVM_GET_IOC_CPUCFG_IDX(reg->id); - if (id >= 0 && id < KVM_MAX_CPUCFG_REGS) - vcpu->arch.cpucfg[id] = (u32)v; - else - ret = -EINVAL; + ret = kvm_check_cpucfg(id, v); + if (ret) + break; + vcpu->arch.cpucfg[id] = (u32)v; break; case KVM_REG_LOONGARCH_KVM: switch (reg->id) { @@ -460,10 +523,94 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, return -EINVAL; } +static int kvm_loongarch_cpucfg_has_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + switch (attr->attr) { + case 2: + return 0; + default: + return -ENXIO; + } + + return -ENXIO; +} + +static int kvm_loongarch_vcpu_has_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + int ret = -ENXIO; + + switch (attr->group) { + case KVM_LOONGARCH_VCPU_CPUCFG: + ret = kvm_loongarch_cpucfg_has_attr(vcpu, attr); + break; + default: + break; + } + + return ret; +} + +static int kvm_loongarch_get_cpucfg_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + int ret = 0; + uint64_t val; + uint64_t __user *uaddr = (uint64_t __user *)attr->addr; + + ret = _kvm_get_cpucfg(attr->attr, &val); + if (ret) + return ret; + + put_user(val, uaddr); + + return ret; +} + +static int kvm_loongarch_vcpu_get_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + int ret = -ENXIO; + + switch (attr->group) { + case KVM_LOONGARCH_VCPU_CPUCFG: + ret = kvm_loongarch_get_cpucfg_attr(vcpu, attr); + break; + default: + break; + } + + return ret; +} + +static int kvm_loongarch_cpucfg_set_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + return -ENXIO; +} + +static int kvm_loongarch_vcpu_set_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + int ret = -ENXIO; + + switch (attr->group) { + case KVM_LOONGARCH_VCPU_CPUCFG: + ret = kvm_loongarch_cpucfg_set_attr(vcpu, attr); + break; + default: + break; + } + + return ret; +} + long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { long r; + struct kvm_device_attr attr; void __user *argp = (void __user *)arg; struct kvm_vcpu *vcpu = filp->private_data; @@ -503,6 +650,27 @@ long kvm_arch_vcpu_ioctl(struct file *filp, r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap); break; } + case KVM_HAS_DEVICE_ATTR: { + r = -EFAULT; + if (copy_from_user(&attr, argp, sizeof(attr))) + break; + r = kvm_loongarch_vcpu_has_attr(vcpu, &attr); + break; + } + case KVM_GET_DEVICE_ATTR: { + r = -EFAULT; + if (copy_from_user(&attr, argp, sizeof(attr))) + break; + r = kvm_loongarch_vcpu_get_attr(vcpu, &attr); + break; + } + case KVM_SET_DEVICE_ATTR: { + r = -EFAULT; + if (copy_from_user(&attr, argp, sizeof(attr))) + break; + r = kvm_loongarch_vcpu_set_attr(vcpu, &attr); + break; + } default: r = -ENOIOCTLCMD; break; @@ -550,12 +718,54 @@ void kvm_own_fpu(struct kvm_vcpu *vcpu) preempt_enable(); } +#ifdef CONFIG_CPU_HAS_LSX +/* Enable LSX and restore context */ +int kvm_own_lsx(struct kvm_vcpu *vcpu) +{ + if (!kvm_guest_has_fpu(&vcpu->arch) || !kvm_guest_has_lsx(&vcpu->arch)) + return -EINVAL; + + preempt_disable(); + + /* Enable LSX for guest */ + set_csr_euen(CSR_EUEN_LSXEN | CSR_EUEN_FPEN); + switch (vcpu->arch.aux_inuse & KVM_LARCH_FPU) { + case KVM_LARCH_FPU: + /* + * Guest FPU state already loaded, + * only restore upper LSX state + */ + _restore_lsx_upper(&vcpu->arch.fpu); + break; + default: + /* Neither FP or LSX already active, + * restore full LSX state + */ + kvm_restore_lsx(&vcpu->arch.fpu); + break; + } + + trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_LSX); + vcpu->arch.aux_inuse |= KVM_LARCH_LSX | KVM_LARCH_FPU; + preempt_enable(); + + return 0; +} +#endif + /* Save context and disable FPU */ void kvm_lose_fpu(struct kvm_vcpu *vcpu) { preempt_disable(); - if (vcpu->arch.aux_inuse & KVM_LARCH_FPU) { + if (vcpu->arch.aux_inuse & KVM_LARCH_LSX) { + kvm_save_lsx(&vcpu->arch.fpu); + vcpu->arch.aux_inuse &= ~(KVM_LARCH_LSX | KVM_LARCH_FPU); + trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_LSX); + + /* Disable LSX & FPU */ + clear_csr_euen(CSR_EUEN_FPEN | CSR_EUEN_LSXEN); + } else if (vcpu->arch.aux_inuse & KVM_LARCH_FPU) { kvm_save_fpu(&vcpu->arch.fpu); vcpu->arch.aux_inuse &= ~KVM_LARCH_FPU; trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_FPU); From 118e10cd893d57df55b3302dfd188a981b6e6d1c Mon Sep 17 00:00:00 2001 From: Tianrui Zhao Date: Tue, 19 Dec 2023 10:48:28 +0800 Subject: [PATCH 177/336] LoongArch: KVM: Add LASX (256bit SIMD) support This patch adds LASX (256bit SIMD) support for LoongArch KVM. There will be LASX exception in KVM when guest use the LASX instructions. KVM will enable LASX and restore the vector registers for guest and then return to guest to continue running. Reviewed-by: Bibo Mao Signed-off-by: Tianrui Zhao Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/kvm_host.h | 10 ++++-- arch/loongarch/include/asm/kvm_vcpu.h | 10 ++++++ arch/loongarch/kernel/fpu.S | 1 + arch/loongarch/kvm/exit.c | 16 +++++++++ arch/loongarch/kvm/switch.S | 15 ++++++++ arch/loongarch/kvm/trace.h | 4 ++- arch/loongarch/kvm/vcpu.c | 51 ++++++++++++++++++++++++++- 7 files changed, 103 insertions(+), 4 deletions(-) diff --git a/arch/loongarch/include/asm/kvm_host.h b/arch/loongarch/include/asm/kvm_host.h index b0c5cdd8014c..5bdb34b2c5d6 100644 --- a/arch/loongarch/include/asm/kvm_host.h +++ b/arch/loongarch/include/asm/kvm_host.h @@ -96,8 +96,9 @@ enum emulation_result { #define KVM_LARCH_FPU (0x1 << 0) #define KVM_LARCH_LSX (0x1 << 1) -#define KVM_LARCH_SWCSR_LATEST (0x1 << 2) -#define KVM_LARCH_HWCSR_USABLE (0x1 << 3) +#define KVM_LARCH_LASX (0x1 << 2) +#define KVM_LARCH_SWCSR_LATEST (0x1 << 3) +#define KVM_LARCH_HWCSR_USABLE (0x1 << 4) struct kvm_vcpu_arch { /* @@ -189,6 +190,11 @@ static inline bool kvm_guest_has_lsx(struct kvm_vcpu_arch *arch) return arch->cpucfg[2] & CPUCFG2_LSX; } +static inline bool kvm_guest_has_lasx(struct kvm_vcpu_arch *arch) +{ + return arch->cpucfg[2] & CPUCFG2_LASX; +} + /* Debug: dump vcpu state */ int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu); diff --git a/arch/loongarch/include/asm/kvm_vcpu.h b/arch/loongarch/include/asm/kvm_vcpu.h index db08dd46b525..e71ceb88f29e 100644 --- a/arch/loongarch/include/asm/kvm_vcpu.h +++ b/arch/loongarch/include/asm/kvm_vcpu.h @@ -65,6 +65,16 @@ static inline void kvm_save_lsx(struct loongarch_fpu *fpu) { } static inline void kvm_restore_lsx(struct loongarch_fpu *fpu) { } #endif +#ifdef CONFIG_CPU_HAS_LASX +int kvm_own_lasx(struct kvm_vcpu *vcpu); +void kvm_save_lasx(struct loongarch_fpu *fpu); +void kvm_restore_lasx(struct loongarch_fpu *fpu); +#else +static inline int kvm_own_lasx(struct kvm_vcpu *vcpu) { } +static inline void kvm_save_lasx(struct loongarch_fpu *fpu) { } +static inline void kvm_restore_lasx(struct loongarch_fpu *fpu) { } +#endif + void kvm_init_timer(struct kvm_vcpu *vcpu, unsigned long hz); void kvm_reset_timer(struct kvm_vcpu *vcpu); void kvm_save_timer(struct kvm_vcpu *vcpu); diff --git a/arch/loongarch/kernel/fpu.S b/arch/loongarch/kernel/fpu.S index a400924c0348..4382e36ae3d4 100644 --- a/arch/loongarch/kernel/fpu.S +++ b/arch/loongarch/kernel/fpu.S @@ -385,6 +385,7 @@ SYM_FUNC_START(_restore_lasx_upper) lasx_restore_all_upper a0 t0 t1 jr ra SYM_FUNC_END(_restore_lasx_upper) +EXPORT_SYMBOL(_restore_lasx_upper) SYM_FUNC_START(_init_lasx_upper) lasx_init_all_upper t1 diff --git a/arch/loongarch/kvm/exit.c b/arch/loongarch/kvm/exit.c index 676f7a3a335c..ed1d89d53e2e 100644 --- a/arch/loongarch/kvm/exit.c +++ b/arch/loongarch/kvm/exit.c @@ -670,6 +670,21 @@ static int kvm_handle_lsx_disabled(struct kvm_vcpu *vcpu) return RESUME_GUEST; } +/* + * kvm_handle_lasx_disabled() - Guest used LASX while disabled in root. + * @vcpu: Virtual CPU context. + * + * Handle when the guest attempts to use LASX when it is disabled in the root + * context. + */ +static int kvm_handle_lasx_disabled(struct kvm_vcpu *vcpu) +{ + if (kvm_own_lasx(vcpu)) + kvm_queue_exception(vcpu, EXCCODE_INE, 0); + + return RESUME_GUEST; +} + /* * LoongArch KVM callback handling for unimplemented guest exiting */ @@ -699,6 +714,7 @@ static exit_handle_fn kvm_fault_tables[EXCCODE_INT_START] = { [EXCCODE_TLBM] = kvm_handle_write_fault, [EXCCODE_FPDIS] = kvm_handle_fpu_disabled, [EXCCODE_LSXDIS] = kvm_handle_lsx_disabled, + [EXCCODE_LASXDIS] = kvm_handle_lasx_disabled, [EXCCODE_GSPR] = kvm_handle_gspr, }; diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S index 00fbf772d16f..ba976509bfe8 100644 --- a/arch/loongarch/kvm/switch.S +++ b/arch/loongarch/kvm/switch.S @@ -261,6 +261,21 @@ SYM_FUNC_START(kvm_restore_lsx) SYM_FUNC_END(kvm_restore_lsx) #endif +#ifdef CONFIG_CPU_HAS_LASX +SYM_FUNC_START(kvm_save_lasx) + fpu_save_csr a0 t1 + fpu_save_cc a0 t1 t2 + lasx_save_data a0 t1 + jr ra +SYM_FUNC_END(kvm_save_lasx) + +SYM_FUNC_START(kvm_restore_lasx) + lasx_restore_data a0 t1 + fpu_restore_cc a0 t1 t2 + fpu_restore_csr a0 t1 t2 + jr ra +SYM_FUNC_END(kvm_restore_lasx) +#endif .section ".rodata" SYM_DATA(kvm_exception_size, .quad kvm_exc_entry_end - kvm_exc_entry) SYM_DATA(kvm_enter_guest_size, .quad kvm_enter_guest_end - kvm_enter_guest) diff --git a/arch/loongarch/kvm/trace.h b/arch/loongarch/kvm/trace.h index 7da4e230e896..c2484ad4cffa 100644 --- a/arch/loongarch/kvm/trace.h +++ b/arch/loongarch/kvm/trace.h @@ -103,6 +103,7 @@ TRACE_EVENT(kvm_exit_gspr, #define KVM_TRACE_AUX_FPU 1 #define KVM_TRACE_AUX_LSX 2 +#define KVM_TRACE_AUX_LASX 3 #define kvm_trace_symbol_aux_op \ { KVM_TRACE_AUX_SAVE, "save" }, \ @@ -113,7 +114,8 @@ TRACE_EVENT(kvm_exit_gspr, #define kvm_trace_symbol_aux_state \ { KVM_TRACE_AUX_FPU, "FPU" }, \ - { KVM_TRACE_AUX_LSX, "LSX" } + { KVM_TRACE_AUX_LSX, "LSX" }, \ + { KVM_TRACE_AUX_LASX, "LASX" } TRACE_EVENT(kvm_aux, TP_PROTO(struct kvm_vcpu *vcpu, unsigned int op, diff --git a/arch/loongarch/kvm/vcpu.c b/arch/loongarch/kvm/vcpu.c index 80487d177ca4..27701991886d 100644 --- a/arch/loongarch/kvm/vcpu.c +++ b/arch/loongarch/kvm/vcpu.c @@ -317,6 +317,13 @@ static int _kvm_get_cpucfg(int id, u64 *v) */ if (cpu_has_lsx) *v |= CPUCFG2_LSX; + /* + * if LASX is supported by CPU, it is also supported by KVM, + * as we implement it. + */ + if (cpu_has_lasx) + *v |= CPUCFG2_LASX; + break; default: ret = -EINVAL; @@ -753,12 +760,54 @@ int kvm_own_lsx(struct kvm_vcpu *vcpu) } #endif +#ifdef CONFIG_CPU_HAS_LASX +/* Enable LASX and restore context */ +int kvm_own_lasx(struct kvm_vcpu *vcpu) +{ + if (!kvm_guest_has_fpu(&vcpu->arch) || !kvm_guest_has_lsx(&vcpu->arch) || !kvm_guest_has_lasx(&vcpu->arch)) + return -EINVAL; + + preempt_disable(); + + set_csr_euen(CSR_EUEN_FPEN | CSR_EUEN_LSXEN | CSR_EUEN_LASXEN); + switch (vcpu->arch.aux_inuse & (KVM_LARCH_FPU | KVM_LARCH_LSX)) { + case KVM_LARCH_LSX: + case KVM_LARCH_LSX | KVM_LARCH_FPU: + /* Guest LSX state already loaded, only restore upper LASX state */ + _restore_lasx_upper(&vcpu->arch.fpu); + break; + case KVM_LARCH_FPU: + /* Guest FP state already loaded, only restore upper LSX & LASX state */ + _restore_lsx_upper(&vcpu->arch.fpu); + _restore_lasx_upper(&vcpu->arch.fpu); + break; + default: + /* Neither FP or LSX already active, restore full LASX state */ + kvm_restore_lasx(&vcpu->arch.fpu); + break; + } + + trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_LASX); + vcpu->arch.aux_inuse |= KVM_LARCH_LASX | KVM_LARCH_LSX | KVM_LARCH_FPU; + preempt_enable(); + + return 0; +} +#endif + /* Save context and disable FPU */ void kvm_lose_fpu(struct kvm_vcpu *vcpu) { preempt_disable(); - if (vcpu->arch.aux_inuse & KVM_LARCH_LSX) { + if (vcpu->arch.aux_inuse & KVM_LARCH_LASX) { + kvm_save_lasx(&vcpu->arch.fpu); + vcpu->arch.aux_inuse &= ~(KVM_LARCH_LSX | KVM_LARCH_FPU | KVM_LARCH_LASX); + trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_LASX); + + /* Disable LASX & LSX & FPU */ + clear_csr_euen(CSR_EUEN_FPEN | CSR_EUEN_LSXEN | CSR_EUEN_LASXEN); + } else if (vcpu->arch.aux_inuse & KVM_LARCH_LSX) { kvm_save_lsx(&vcpu->arch.fpu); vcpu->arch.aux_inuse &= ~(KVM_LARCH_LSX | KVM_LARCH_FPU); trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_LSX); From 2bfc654b89c4dd1c372bb2cbba6b5a0eb578d214 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 9 Nov 2023 15:47:49 +0000 Subject: [PATCH 178/336] arm64: cpufeatures: Restrict NV support to FEAT_NV2 To anyone who has played with FEAT_NV, it is obvious that the level of performance is rather low due to the trap amplification that it imposes on the host hypervisor. FEAT_NV2 solves a number of the problems that FEAT_NV had. It also turns out that all the existing hardware that has FEAT_NV also has FEAT_NV2. Finally, it is now allowed by the architecture to build FEAT_NV2 *only* (as denoted by ID_AA64MMFR4_EL1.NV_frac), which effectively seals the fate of FEAT_NV. Restrict the NV support to NV2, and be done with it. Nobody will cry over the old crap. NV_frac will eventually be supported once the intrastructure is ready. Reviewed-by: Oliver Upton Signed-off-by: Marc Zyngier --- arch/arm64/kernel/cpufeature.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 646591c67e7a..1329e974d187 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2339,7 +2339,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .capability = ARM64_HAS_NESTED_VIRT, .type = ARM64_CPUCAP_SYSTEM_FEATURE, .matches = has_nested_virt_support, - ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, NV, IMP) + ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, NV, NV2) }, { .capability = ARM64_HAS_32BIT_EL0_DO_NOT_USE, From 111903d1f5b9334d1100e1c6ee08e740fa374d91 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 13 Nov 2023 14:16:02 +0000 Subject: [PATCH 179/336] KVM: arm64: nv: Hoist vcpu_has_nv() into is_hyp_ctxt() A rather common idiom when writing NV code as part of KVM is to have things such has: if (vcpu_has_nv(vcpu) && is_hyp_ctxt(vcpu)) { [...] } to check that we are in a hyp-related context. The second part of the conjunction would be enough, but the first one contains a static key that allows the rest of the checkis to be elided when in a non-NV environment. Rewrite is_hyp_ctxt() to directly use vcpu_has_nv(). The result is the same, and the code easier to read. The one occurence of this that is already merged is rewritten in the process. In order to avoid nasty cirtular dependencies between kvm_emulate.h and kvm_nested.h, vcpu_has_feature() is itself hoisted into kvm_host.h, at the cost of some #deferry... Reviewed-by: Oliver Upton Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_emulate.h | 8 ++------ arch/arm64/include/asm/kvm_host.h | 7 +++++++ arch/arm64/kvm/arch_timer.c | 3 +-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 78a550537b67..84829b7e6f1d 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -54,11 +55,6 @@ void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu); int kvm_inject_nested_sync(struct kvm_vcpu *vcpu, u64 esr_el2); int kvm_inject_nested_irq(struct kvm_vcpu *vcpu); -static inline bool vcpu_has_feature(const struct kvm_vcpu *vcpu, int feature) -{ - return test_bit(feature, vcpu->kvm->arch.vcpu_features); -} - #if defined(__KVM_VHE_HYPERVISOR__) || defined(__KVM_NVHE_HYPERVISOR__) static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu) { @@ -248,7 +244,7 @@ static inline bool __is_hyp_ctxt(const struct kvm_cpu_context *ctxt) static inline bool is_hyp_ctxt(const struct kvm_vcpu *vcpu) { - return __is_hyp_ctxt(&vcpu->arch.ctxt); + return vcpu_has_nv(vcpu) && __is_hyp_ctxt(&vcpu->arch.ctxt); } /* diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 824f29f04916..4103a12ecaaf 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -1177,6 +1177,13 @@ bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu); #define kvm_vm_has_ran_once(kvm) \ (test_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &(kvm)->arch.flags)) +static inline bool __vcpu_has_feature(const struct kvm_arch *ka, int feature) +{ + return test_bit(feature, ka->vcpu_features); +} + +#define vcpu_has_feature(v, f) __vcpu_has_feature(&(v)->kvm->arch, (f)) + int kvm_trng_call(struct kvm_vcpu *vcpu); #ifdef CONFIG_KVM extern phys_addr_t hyp_mem_base; diff --git a/arch/arm64/kvm/arch_timer.c b/arch/arm64/kvm/arch_timer.c index 13ba691b848f..9dec8c419bf4 100644 --- a/arch/arm64/kvm/arch_timer.c +++ b/arch/arm64/kvm/arch_timer.c @@ -295,8 +295,7 @@ static u64 wfit_delay_ns(struct kvm_vcpu *vcpu) u64 val = vcpu_get_reg(vcpu, kvm_vcpu_sys_get_rt(vcpu)); struct arch_timer_context *ctx; - ctx = (vcpu_has_nv(vcpu) && is_hyp_ctxt(vcpu)) ? vcpu_hvtimer(vcpu) - : vcpu_vtimer(vcpu); + ctx = is_hyp_ctxt(vcpu) ? vcpu_hvtimer(vcpu) : vcpu_vtimer(vcpu); return kvm_counter_compute_delta(ctx, val); } From 3ed0b5123cd5a2a4f1fe4e594e7bf319e9eaf1da Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sun, 12 Nov 2023 21:05:14 +0000 Subject: [PATCH 180/336] KVM: arm64: nv: Compute NV view of idregs as a one-off Now that we have a full copy of the idregs for each VM, there is no point in repainting the sysregs on each access. Instead, we can simply perform the transmation as a one-off and be done with it. Reviewed-by: Oliver Upton Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 1 + arch/arm64/include/asm/kvm_nested.h | 6 +----- arch/arm64/kvm/arm.c | 6 ++++++ arch/arm64/kvm/nested.c | 22 +++++++++++++++------- arch/arm64/kvm/sys_regs.c | 2 -- 5 files changed, 23 insertions(+), 14 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 4103a12ecaaf..fce2e5f583a7 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -306,6 +306,7 @@ struct kvm_arch { * Atomic access to multiple idregs are guarded by kvm_arch.config_lock. */ #define IDREG_IDX(id) (((sys_reg_CRm(id) - 1) << 3) | sys_reg_Op2(id)) +#define IDX_IDREG(idx) sys_reg(3, 0, 0, ((idx) >> 3) + 1, (idx) & Op2_mask) #define IDREG(kvm, id) ((kvm)->arch.id_regs[IDREG_IDX(id)]) #define KVM_ARM_ID_REG_NUM (IDREG_IDX(sys_reg(3, 0, 0, 7, 7)) + 1) u64 id_regs[KVM_ARM_ID_REG_NUM]; diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h index 6cec8e9c6c91..249b03fc2cce 100644 --- a/arch/arm64/include/asm/kvm_nested.h +++ b/arch/arm64/include/asm/kvm_nested.h @@ -14,10 +14,6 @@ static inline bool vcpu_has_nv(const struct kvm_vcpu *vcpu) extern bool __check_nv_sr_forward(struct kvm_vcpu *vcpu); -struct sys_reg_params; -struct sys_reg_desc; - -void access_nested_id_reg(struct kvm_vcpu *v, struct sys_reg_params *p, - const struct sys_reg_desc *r); +int kvm_init_nv_sysregs(struct kvm *kvm); #endif /* __ARM64_KVM_NESTED_H */ diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index e5f75f1f1085..b65df612b41b 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -669,6 +669,12 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) return ret; } + if (vcpu_has_nv(vcpu)) { + ret = kvm_init_nv_sysregs(vcpu->kvm); + if (ret) + return ret; + } + ret = kvm_timer_enable(vcpu); if (ret) return ret; diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 042695a210ce..ba95d044bc98 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -23,13 +23,9 @@ * This list should get updated as new features get added to the NV * support, and new extension to the architecture. */ -void access_nested_id_reg(struct kvm_vcpu *v, struct sys_reg_params *p, - const struct sys_reg_desc *r) +static u64 limit_nv_id_reg(u32 id, u64 val) { - u32 id = reg_to_encoding(r); - u64 val, tmp; - - val = p->regval; + u64 tmp; switch (id) { case SYS_ID_AA64ISAR0_EL1: @@ -158,5 +154,17 @@ void access_nested_id_reg(struct kvm_vcpu *v, struct sys_reg_params *p, break; } - p->regval = val; + return val; +} +int kvm_init_nv_sysregs(struct kvm *kvm) +{ + mutex_lock(&kvm->arch.config_lock); + + for (int i = 0; i < KVM_ARM_ID_REG_NUM; i++) + kvm->arch.id_regs[i] = limit_nv_id_reg(IDX_IDREG(i), + kvm->arch.id_regs[i]); + + mutex_unlock(&kvm->arch.config_lock); + + return 0; } diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 4735e1b37fb3..3eae84195d48 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1505,8 +1505,6 @@ static bool access_id_reg(struct kvm_vcpu *vcpu, return write_to_read_only(vcpu, p, r); p->regval = read_id_reg(vcpu, r); - if (vcpu_has_nv(vcpu)) - access_nested_id_reg(vcpu, p, r); return true; } From 4d4f52052ba8357f1591cb9bc3086541070711af Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 8 Nov 2023 19:10:12 +0000 Subject: [PATCH 181/336] KVM: arm64: nv: Drop EL12 register traps that are redirected to VNCR With FEAT_NV2, a bunch of system register writes are turned into memory writes. This is specially the fate of the EL12 registers that the guest hypervisor manipulates out of context. Remove the trap descriptors for those, as they are never going to be used again. Reviewed-by: Oliver Upton Signed-off-by: Marc Zyngier --- arch/arm64/kvm/sys_regs.c | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 3eae84195d48..cdfc79ccc7a1 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -2566,21 +2566,6 @@ static const struct sys_reg_desc sys_reg_descs[] = { EL2_REG(CNTVOFF_EL2, access_rw, reset_val, 0), EL2_REG(CNTHCTL_EL2, access_rw, reset_val, 0), - EL12_REG(SCTLR, access_vm_reg, reset_val, 0x00C50078), - EL12_REG(CPACR, access_rw, reset_val, 0), - EL12_REG(TTBR0, access_vm_reg, reset_unknown, 0), - EL12_REG(TTBR1, access_vm_reg, reset_unknown, 0), - EL12_REG(TCR, access_vm_reg, reset_val, 0), - { SYS_DESC(SYS_SPSR_EL12), access_spsr}, - { SYS_DESC(SYS_ELR_EL12), access_elr}, - EL12_REG(AFSR0, access_vm_reg, reset_unknown, 0), - EL12_REG(AFSR1, access_vm_reg, reset_unknown, 0), - EL12_REG(ESR, access_vm_reg, reset_unknown, 0), - EL12_REG(FAR, access_vm_reg, reset_unknown, 0), - EL12_REG(MAIR, access_vm_reg, reset_unknown, 0), - EL12_REG(AMAIR, access_vm_reg, reset_amair_el1, 0), - EL12_REG(VBAR, access_rw, reset_val, 0), - EL12_REG(CONTEXTIDR, access_vm_reg, reset_val, 0), EL12_REG(CNTKCTL, access_rw, reset_val, 0), EL2_REG(SP_EL2, NULL, reset_unknown, 0), From 3606e0b2e462164bced151dbb54ccfe42ac6c35b Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sun, 25 Dec 2016 10:49:48 -0500 Subject: [PATCH 182/336] KVM: arm64: nv: Add non-VHE-EL2->EL1 translation helpers Some EL2 system registers immediately affect the current execution of the system, so we need to use their respective EL1 counterparts. For this we need to define a mapping between the two. In general, this only affects non-VHE guest hypervisors, as VHE system registers are compatible with the EL1 counterparts. These helpers will get used in subsequent patches. Reviewed-by: Oliver Upton Co-developed-by: Andre Przywara Signed-off-by: Andre Przywara Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_nested.h | 50 ++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h index 249b03fc2cce..4882905357f4 100644 --- a/arch/arm64/include/asm/kvm_nested.h +++ b/arch/arm64/include/asm/kvm_nested.h @@ -2,8 +2,9 @@ #ifndef __ARM64_KVM_NESTED_H #define __ARM64_KVM_NESTED_H -#include +#include #include +#include static inline bool vcpu_has_nv(const struct kvm_vcpu *vcpu) { @@ -12,6 +13,53 @@ static inline bool vcpu_has_nv(const struct kvm_vcpu *vcpu) vcpu_has_feature(vcpu, KVM_ARM_VCPU_HAS_EL2)); } +/* Translation helpers from non-VHE EL2 to EL1 */ +static inline u64 tcr_el2_ps_to_tcr_el1_ips(u64 tcr_el2) +{ + return (u64)FIELD_GET(TCR_EL2_PS_MASK, tcr_el2) << TCR_IPS_SHIFT; +} + +static inline u64 translate_tcr_el2_to_tcr_el1(u64 tcr) +{ + return TCR_EPD1_MASK | /* disable TTBR1_EL1 */ + ((tcr & TCR_EL2_TBI) ? TCR_TBI0 : 0) | + tcr_el2_ps_to_tcr_el1_ips(tcr) | + (tcr & TCR_EL2_TG0_MASK) | + (tcr & TCR_EL2_ORGN0_MASK) | + (tcr & TCR_EL2_IRGN0_MASK) | + (tcr & TCR_EL2_T0SZ_MASK); +} + +static inline u64 translate_cptr_el2_to_cpacr_el1(u64 cptr_el2) +{ + u64 cpacr_el1 = 0; + + if (cptr_el2 & CPTR_EL2_TTA) + cpacr_el1 |= CPACR_ELx_TTA; + if (!(cptr_el2 & CPTR_EL2_TFP)) + cpacr_el1 |= CPACR_ELx_FPEN; + if (!(cptr_el2 & CPTR_EL2_TZ)) + cpacr_el1 |= CPACR_ELx_ZEN; + + return cpacr_el1; +} + +static inline u64 translate_sctlr_el2_to_sctlr_el1(u64 val) +{ + /* Only preserve the minimal set of bits we support */ + val &= (SCTLR_ELx_M | SCTLR_ELx_A | SCTLR_ELx_C | SCTLR_ELx_SA | + SCTLR_ELx_I | SCTLR_ELx_IESB | SCTLR_ELx_WXN | SCTLR_ELx_EE); + val |= SCTLR_EL1_RES1; + + return val; +} + +static inline u64 translate_ttbr0_el2_to_ttbr0_el1(u64 ttbr0) +{ + /* Clear the ASID field */ + return ttbr0 & ~GENMASK_ULL(63, 48); +} + extern bool __check_nv_sr_forward(struct kvm_vcpu *vcpu); int kvm_init_nv_sysregs(struct kvm *kvm); From 60ce16cc122aad999129d23061fa35f63d5b1e9b Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 21 Jun 2019 13:54:37 +0100 Subject: [PATCH 183/336] KVM: arm64: nv: Add include containing the VNCR_EL2 offsets VNCR_EL2 points to a page containing a number of system registers accessed by a guest hypervisor when ARMv8.4-NV is enabled. Let's document the offsets in that page, as we are going to use this layout. Reviewed-by: Oliver Upton Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/vncr_mapping.h | 103 ++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 arch/arm64/include/asm/vncr_mapping.h diff --git a/arch/arm64/include/asm/vncr_mapping.h b/arch/arm64/include/asm/vncr_mapping.h new file mode 100644 index 000000000000..df2c47c55972 --- /dev/null +++ b/arch/arm64/include/asm/vncr_mapping.h @@ -0,0 +1,103 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * System register offsets in the VNCR page + * All offsets are *byte* displacements! + */ + +#ifndef __ARM64_VNCR_MAPPING_H__ +#define __ARM64_VNCR_MAPPING_H__ + +#define VNCR_VTTBR_EL2 0x020 +#define VNCR_VTCR_EL2 0x040 +#define VNCR_VMPIDR_EL2 0x050 +#define VNCR_CNTVOFF_EL2 0x060 +#define VNCR_HCR_EL2 0x078 +#define VNCR_HSTR_EL2 0x080 +#define VNCR_VPIDR_EL2 0x088 +#define VNCR_TPIDR_EL2 0x090 +#define VNCR_HCRX_EL2 0x0A0 +#define VNCR_VNCR_EL2 0x0B0 +#define VNCR_CPACR_EL1 0x100 +#define VNCR_CONTEXTIDR_EL1 0x108 +#define VNCR_SCTLR_EL1 0x110 +#define VNCR_ACTLR_EL1 0x118 +#define VNCR_TCR_EL1 0x120 +#define VNCR_AFSR0_EL1 0x128 +#define VNCR_AFSR1_EL1 0x130 +#define VNCR_ESR_EL1 0x138 +#define VNCR_MAIR_EL1 0x140 +#define VNCR_AMAIR_EL1 0x148 +#define VNCR_MDSCR_EL1 0x158 +#define VNCR_SPSR_EL1 0x160 +#define VNCR_CNTV_CVAL_EL0 0x168 +#define VNCR_CNTV_CTL_EL0 0x170 +#define VNCR_CNTP_CVAL_EL0 0x178 +#define VNCR_CNTP_CTL_EL0 0x180 +#define VNCR_SCXTNUM_EL1 0x188 +#define VNCR_TFSR_EL1 0x190 +#define VNCR_HFGRTR_EL2 0x1B8 +#define VNCR_HFGWTR_EL2 0x1C0 +#define VNCR_HFGITR_EL2 0x1C8 +#define VNCR_HDFGRTR_EL2 0x1D0 +#define VNCR_HDFGWTR_EL2 0x1D8 +#define VNCR_ZCR_EL1 0x1E0 +#define VNCR_HAFGRTR_EL2 0x1E8 +#define VNCR_TTBR0_EL1 0x200 +#define VNCR_TTBR1_EL1 0x210 +#define VNCR_FAR_EL1 0x220 +#define VNCR_ELR_EL1 0x230 +#define VNCR_SP_EL1 0x240 +#define VNCR_VBAR_EL1 0x250 +#define VNCR_TCR2_EL1 0x270 +#define VNCR_PIRE0_EL1 0x290 +#define VNCR_PIRE0_EL2 0x298 +#define VNCR_PIR_EL1 0x2A0 +#define VNCR_ICH_LR0_EL2 0x400 +#define VNCR_ICH_LR1_EL2 0x408 +#define VNCR_ICH_LR2_EL2 0x410 +#define VNCR_ICH_LR3_EL2 0x418 +#define VNCR_ICH_LR4_EL2 0x420 +#define VNCR_ICH_LR5_EL2 0x428 +#define VNCR_ICH_LR6_EL2 0x430 +#define VNCR_ICH_LR7_EL2 0x438 +#define VNCR_ICH_LR8_EL2 0x440 +#define VNCR_ICH_LR9_EL2 0x448 +#define VNCR_ICH_LR10_EL2 0x450 +#define VNCR_ICH_LR11_EL2 0x458 +#define VNCR_ICH_LR12_EL2 0x460 +#define VNCR_ICH_LR13_EL2 0x468 +#define VNCR_ICH_LR14_EL2 0x470 +#define VNCR_ICH_LR15_EL2 0x478 +#define VNCR_ICH_AP0R0_EL2 0x480 +#define VNCR_ICH_AP0R1_EL2 0x488 +#define VNCR_ICH_AP0R2_EL2 0x490 +#define VNCR_ICH_AP0R3_EL2 0x498 +#define VNCR_ICH_AP1R0_EL2 0x4A0 +#define VNCR_ICH_AP1R1_EL2 0x4A8 +#define VNCR_ICH_AP1R2_EL2 0x4B0 +#define VNCR_ICH_AP1R3_EL2 0x4B8 +#define VNCR_ICH_HCR_EL2 0x4C0 +#define VNCR_ICH_VMCR_EL2 0x4C8 +#define VNCR_VDISR_EL2 0x500 +#define VNCR_PMBLIMITR_EL1 0x800 +#define VNCR_PMBPTR_EL1 0x810 +#define VNCR_PMBSR_EL1 0x820 +#define VNCR_PMSCR_EL1 0x828 +#define VNCR_PMSEVFR_EL1 0x830 +#define VNCR_PMSICR_EL1 0x838 +#define VNCR_PMSIRR_EL1 0x840 +#define VNCR_PMSLATFR_EL1 0x848 +#define VNCR_TRFCR_EL1 0x880 +#define VNCR_MPAM1_EL1 0x900 +#define VNCR_MPAMHCR_EL2 0x930 +#define VNCR_MPAMVPMV_EL2 0x938 +#define VNCR_MPAMVPM0_EL2 0x940 +#define VNCR_MPAMVPM1_EL2 0x948 +#define VNCR_MPAMVPM2_EL2 0x950 +#define VNCR_MPAMVPM3_EL2 0x958 +#define VNCR_MPAMVPM4_EL2 0x960 +#define VNCR_MPAMVPM5_EL2 0x968 +#define VNCR_MPAMVPM6_EL2 0x970 +#define VNCR_MPAMVPM7_EL2 0x978 + +#endif /* __ARM64_VNCR_MAPPING_H__ */ From 2733dd10701abc6ab23d65a732f58fbeb80bd203 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 6 Nov 2023 16:42:13 +0000 Subject: [PATCH 184/336] KVM: arm64: Introduce a bad_trap() primitive for unexpected trap handling In order to ease the debugging of NV, it is helpful to have the kernel shout at you when an unexpected trap is handled. We already have this in a couple of cases. Make this a more generic infrastructure that we will make use of very shortly. Reviewed-by: Oliver Upton Signed-off-by: Marc Zyngier --- arch/arm64/kvm/sys_regs.c | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index cdfc79ccc7a1..3709c35666a2 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -45,24 +45,31 @@ static u64 sys_reg_to_index(const struct sys_reg_desc *reg); static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, u64 val); +static bool bad_trap(struct kvm_vcpu *vcpu, + struct sys_reg_params *params, + const struct sys_reg_desc *r, + const char *msg) +{ + WARN_ONCE(1, "Unexpected %s\n", msg); + print_sys_reg_instr(params); + kvm_inject_undefined(vcpu); + return false; +} + static bool read_from_write_only(struct kvm_vcpu *vcpu, struct sys_reg_params *params, const struct sys_reg_desc *r) { - WARN_ONCE(1, "Unexpected sys_reg read to write-only register\n"); - print_sys_reg_instr(params); - kvm_inject_undefined(vcpu); - return false; + return bad_trap(vcpu, params, r, + "sys_reg read to write-only register"); } static bool write_to_read_only(struct kvm_vcpu *vcpu, struct sys_reg_params *params, const struct sys_reg_desc *r) { - WARN_ONCE(1, "Unexpected sys_reg write to read-only register\n"); - print_sys_reg_instr(params); - kvm_inject_undefined(vcpu); - return false; + return bad_trap(vcpu, params, r, + "sys_reg write to read-only register"); } u64 vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg) From 9b9cce60be85e6807bdb0eaa2f520e78dbab0659 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 7 Nov 2023 09:02:10 +0000 Subject: [PATCH 185/336] KVM: arm64: nv: Add EL2_REG_VNCR()/EL2_REG_REDIR() sysreg helpers Add two helpers to deal with EL2 registers are are either redirected to the VNCR page, or that are redirected to their EL1 counterpart. In either cases, no trap is expected. THe relevant register descriptors are repainted accordingly. Reviewed-by: Oliver Upton Signed-off-by: Marc Zyngier --- arch/arm64/kvm/sys_regs.c | 65 ++++++++++++++++++++++++++++----------- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 3709c35666a2..7fef170ec67b 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1890,6 +1890,32 @@ static unsigned int el2_visibility(const struct kvm_vcpu *vcpu, return REG_HIDDEN; } +static bool bad_vncr_trap(struct kvm_vcpu *vcpu, + struct sys_reg_params *p, + const struct sys_reg_desc *r) +{ + /* + * We really shouldn't be here, and this is likely the result + * of a misconfigured trap, as this register should target the + * VNCR page, and nothing else. + */ + return bad_trap(vcpu, p, r, + "trap of VNCR-backed register"); +} + +static bool bad_redir_trap(struct kvm_vcpu *vcpu, + struct sys_reg_params *p, + const struct sys_reg_desc *r) +{ + /* + * We really shouldn't be here, and this is likely the result + * of a misconfigured trap, as this register should target the + * corresponding EL1, and nothing else. + */ + return bad_trap(vcpu, p, r, + "trap of EL2 register redirected to EL1"); +} + #define EL2_REG(name, acc, rst, v) { \ SYS_DESC(SYS_##name), \ .access = acc, \ @@ -1899,6 +1925,9 @@ static unsigned int el2_visibility(const struct kvm_vcpu *vcpu, .val = v, \ } +#define EL2_REG_VNCR(name, rst, v) EL2_REG(name, bad_vncr_trap, rst, v) +#define EL2_REG_REDIR(name, rst, v) EL2_REG(name, bad_redir_trap, rst, v) + /* * EL{0,1}2 registers are the EL2 view on an EL0 or EL1 register when * HCR_EL2.E2H==1, and only in the sysreg table for convenience of @@ -2513,32 +2542,32 @@ static const struct sys_reg_desc sys_reg_descs[] = { { PMU_SYS_REG(PMCCFILTR_EL0), .access = access_pmu_evtyper, .reset = reset_val, .reg = PMCCFILTR_EL0, .val = 0 }, - EL2_REG(VPIDR_EL2, access_rw, reset_unknown, 0), - EL2_REG(VMPIDR_EL2, access_rw, reset_unknown, 0), + EL2_REG_VNCR(VPIDR_EL2, reset_unknown, 0), + EL2_REG_VNCR(VMPIDR_EL2, reset_unknown, 0), EL2_REG(SCTLR_EL2, access_rw, reset_val, SCTLR_EL2_RES1), EL2_REG(ACTLR_EL2, access_rw, reset_val, 0), - EL2_REG(HCR_EL2, access_rw, reset_val, 0), + EL2_REG_VNCR(HCR_EL2, reset_val, 0), EL2_REG(MDCR_EL2, access_rw, reset_val, 0), EL2_REG(CPTR_EL2, access_rw, reset_val, CPTR_NVHE_EL2_RES1), - EL2_REG(HSTR_EL2, access_rw, reset_val, 0), - EL2_REG(HFGRTR_EL2, access_rw, reset_val, 0), - EL2_REG(HFGWTR_EL2, access_rw, reset_val, 0), - EL2_REG(HFGITR_EL2, access_rw, reset_val, 0), - EL2_REG(HACR_EL2, access_rw, reset_val, 0), + EL2_REG_VNCR(HSTR_EL2, reset_val, 0), + EL2_REG_VNCR(HFGRTR_EL2, reset_val, 0), + EL2_REG_VNCR(HFGWTR_EL2, reset_val, 0), + EL2_REG_VNCR(HFGITR_EL2, reset_val, 0), + EL2_REG_VNCR(HACR_EL2, reset_val, 0), - EL2_REG(HCRX_EL2, access_rw, reset_val, 0), + EL2_REG_VNCR(HCRX_EL2, reset_val, 0), EL2_REG(TTBR0_EL2, access_rw, reset_val, 0), EL2_REG(TTBR1_EL2, access_rw, reset_val, 0), EL2_REG(TCR_EL2, access_rw, reset_val, TCR_EL2_RES1), - EL2_REG(VTTBR_EL2, access_rw, reset_val, 0), - EL2_REG(VTCR_EL2, access_rw, reset_val, 0), + EL2_REG_VNCR(VTTBR_EL2, reset_val, 0), + EL2_REG_VNCR(VTCR_EL2, reset_val, 0), { SYS_DESC(SYS_DACR32_EL2), trap_undef, reset_unknown, DACR32_EL2 }, - EL2_REG(HDFGRTR_EL2, access_rw, reset_val, 0), - EL2_REG(HDFGWTR_EL2, access_rw, reset_val, 0), - EL2_REG(SPSR_EL2, access_rw, reset_val, 0), - EL2_REG(ELR_EL2, access_rw, reset_val, 0), + EL2_REG_VNCR(HDFGRTR_EL2, reset_val, 0), + EL2_REG_VNCR(HDFGWTR_EL2, reset_val, 0), + EL2_REG_REDIR(SPSR_EL2, reset_val, 0), + EL2_REG_REDIR(ELR_EL2, reset_val, 0), { SYS_DESC(SYS_SP_EL1), access_sp_el1}, /* AArch32 SPSR_* are RES0 if trapped from a NV guest */ @@ -2554,10 +2583,10 @@ static const struct sys_reg_desc sys_reg_descs[] = { { SYS_DESC(SYS_IFSR32_EL2), trap_undef, reset_unknown, IFSR32_EL2 }, EL2_REG(AFSR0_EL2, access_rw, reset_val, 0), EL2_REG(AFSR1_EL2, access_rw, reset_val, 0), - EL2_REG(ESR_EL2, access_rw, reset_val, 0), + EL2_REG_REDIR(ESR_EL2, reset_val, 0), { SYS_DESC(SYS_FPEXC32_EL2), trap_undef, reset_val, FPEXC32_EL2, 0x700 }, - EL2_REG(FAR_EL2, access_rw, reset_val, 0), + EL2_REG_REDIR(FAR_EL2, reset_val, 0), EL2_REG(HPFAR_EL2, access_rw, reset_val, 0), EL2_REG(MAIR_EL2, access_rw, reset_val, 0), @@ -2570,7 +2599,7 @@ static const struct sys_reg_desc sys_reg_descs[] = { EL2_REG(CONTEXTIDR_EL2, access_rw, reset_val, 0), EL2_REG(TPIDR_EL2, access_rw, reset_val, 0), - EL2_REG(CNTVOFF_EL2, access_rw, reset_val, 0), + EL2_REG_VNCR(CNTVOFF_EL2, reset_val, 0), EL2_REG(CNTHCTL_EL2, access_rw, reset_val, 0), EL12_REG(CNTKCTL, access_rw, reset_val, 0), From d8bd48e3f0ee9e1fdba2a2e453155a5354e48a8d Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 26 Jun 2019 19:59:56 +0100 Subject: [PATCH 186/336] KVM: arm64: nv: Map VNCR-capable registers to a separate page With ARMv8.4-NV, registers that can be directly accessed in memory by the guest have to live at architected offsets in a special page. Let's annotate the sysreg enum to reflect the offset at which they are in this page, whith a little twist: If running on HW that doesn't have the ARMv8.4-NV feature, or even a VM that doesn't use NV, we store all the system registers in the usual sys_regs array. The only difference with the pre-8.4 situation is that VNCR-capable registers are at a "similar" offset as in the VNCR page (we can compute the actual offset at compile time), and that the sys_regs array is both bigger and sparse. Reviewed-by: Oliver Upton Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 127 +++++++++++++++++++----------- 1 file changed, 81 insertions(+), 46 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index fce2e5f583a7..9e8cd2bb95c3 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -27,6 +27,7 @@ #include #include #include +#include #define __KVM_HAVE_ARCH_INTC_INITIALIZED @@ -325,33 +326,33 @@ struct kvm_vcpu_fault_info { u64 disr_el1; /* Deferred [SError] Status Register */ }; +/* + * VNCR() just places the VNCR_capable registers in the enum after + * __VNCR_START__, and the value (after correction) to be an 8-byte offset + * from the VNCR base. As we don't require the enum to be otherwise ordered, + * we need the terrible hack below to ensure that we correctly size the + * sys_regs array, no matter what. + * + * The __MAX__ macro has been lifted from Sean Eron Anderson's wonderful + * treasure trove of bit hacks: + * https://graphics.stanford.edu/~seander/bithacks.html#IntegerMinOrMax + */ +#define __MAX__(x,y) ((x) ^ (((x) ^ (y)) & -((x) < (y)))) +#define VNCR(r) \ + __before_##r, \ + r = __VNCR_START__ + ((VNCR_ ## r) / 8), \ + __after_##r = __MAX__(__before_##r - 1, r) + enum vcpu_sysreg { __INVALID_SYSREG__, /* 0 is reserved as an invalid value */ MPIDR_EL1, /* MultiProcessor Affinity Register */ CLIDR_EL1, /* Cache Level ID Register */ CSSELR_EL1, /* Cache Size Selection Register */ - SCTLR_EL1, /* System Control Register */ - ACTLR_EL1, /* Auxiliary Control Register */ - CPACR_EL1, /* Coprocessor Access Control */ - ZCR_EL1, /* SVE Control */ - TTBR0_EL1, /* Translation Table Base Register 0 */ - TTBR1_EL1, /* Translation Table Base Register 1 */ - TCR_EL1, /* Translation Control Register */ - TCR2_EL1, /* Extended Translation Control Register */ - ESR_EL1, /* Exception Syndrome Register */ - AFSR0_EL1, /* Auxiliary Fault Status Register 0 */ - AFSR1_EL1, /* Auxiliary Fault Status Register 1 */ - FAR_EL1, /* Fault Address Register */ - MAIR_EL1, /* Memory Attribute Indirection Register */ - VBAR_EL1, /* Vector Base Address Register */ - CONTEXTIDR_EL1, /* Context ID Register */ TPIDR_EL0, /* Thread ID, User R/W */ TPIDRRO_EL0, /* Thread ID, User R/O */ TPIDR_EL1, /* Thread ID, Privileged */ - AMAIR_EL1, /* Aux Memory Attribute Indirection Register */ CNTKCTL_EL1, /* Timer Control Register (EL1) */ PAR_EL1, /* Physical Address Register */ - MDSCR_EL1, /* Monitor Debug System Control Register */ MDCCINT_EL1, /* Monitor Debug Comms Channel Interrupt Enable Reg */ OSLSR_EL1, /* OS Lock Status Register */ DISR_EL1, /* Deferred Interrupt Status Register */ @@ -382,26 +383,11 @@ enum vcpu_sysreg { APGAKEYLO_EL1, APGAKEYHI_EL1, - ELR_EL1, - SP_EL1, - SPSR_EL1, - - CNTVOFF_EL2, - CNTV_CVAL_EL0, - CNTV_CTL_EL0, - CNTP_CVAL_EL0, - CNTP_CTL_EL0, - /* Memory Tagging Extension registers */ RGSR_EL1, /* Random Allocation Tag Seed Register */ GCR_EL1, /* Tag Control Register */ - TFSR_EL1, /* Tag Fault Status Register (EL1) */ TFSRE0_EL1, /* Tag Fault Status Register (EL0) */ - /* Permission Indirection Extension registers */ - PIR_EL1, /* Permission Indirection Register 1 (EL1) */ - PIRE0_EL1, /* Permission Indirection Register 0 (EL1) */ - /* 32bit specific registers. */ DACR32_EL2, /* Domain Access Control Register */ IFSR32_EL2, /* Instruction Fault Status Register */ @@ -409,21 +395,14 @@ enum vcpu_sysreg { DBGVCR32_EL2, /* Debug Vector Catch Register */ /* EL2 registers */ - VPIDR_EL2, /* Virtualization Processor ID Register */ - VMPIDR_EL2, /* Virtualization Multiprocessor ID Register */ SCTLR_EL2, /* System Control Register (EL2) */ ACTLR_EL2, /* Auxiliary Control Register (EL2) */ - HCR_EL2, /* Hypervisor Configuration Register */ MDCR_EL2, /* Monitor Debug Configuration Register (EL2) */ CPTR_EL2, /* Architectural Feature Trap Register (EL2) */ - HSTR_EL2, /* Hypervisor System Trap Register */ HACR_EL2, /* Hypervisor Auxiliary Control Register */ - HCRX_EL2, /* Extended Hypervisor Configuration Register */ TTBR0_EL2, /* Translation Table Base Register 0 (EL2) */ TTBR1_EL2, /* Translation Table Base Register 1 (EL2) */ TCR_EL2, /* Translation Control Register (EL2) */ - VTTBR_EL2, /* Virtualization Translation Table Base Register */ - VTCR_EL2, /* Virtualization Translation Control Register */ SPSR_EL2, /* EL2 saved program status register */ ELR_EL2, /* EL2 exception link register */ AFSR0_EL2, /* Auxiliary Fault Status Register 0 (EL2) */ @@ -436,19 +415,61 @@ enum vcpu_sysreg { VBAR_EL2, /* Vector Base Address Register (EL2) */ RVBAR_EL2, /* Reset Vector Base Address Register */ CONTEXTIDR_EL2, /* Context ID Register (EL2) */ - TPIDR_EL2, /* EL2 Software Thread ID Register */ CNTHCTL_EL2, /* Counter-timer Hypervisor Control register */ SP_EL2, /* EL2 Stack Pointer */ - HFGRTR_EL2, - HFGWTR_EL2, - HFGITR_EL2, - HDFGRTR_EL2, - HDFGWTR_EL2, CNTHP_CTL_EL2, CNTHP_CVAL_EL2, CNTHV_CTL_EL2, CNTHV_CVAL_EL2, + __VNCR_START__, /* Any VNCR-capable reg goes after this point */ + + VNCR(SCTLR_EL1),/* System Control Register */ + VNCR(ACTLR_EL1),/* Auxiliary Control Register */ + VNCR(CPACR_EL1),/* Coprocessor Access Control */ + VNCR(ZCR_EL1), /* SVE Control */ + VNCR(TTBR0_EL1),/* Translation Table Base Register 0 */ + VNCR(TTBR1_EL1),/* Translation Table Base Register 1 */ + VNCR(TCR_EL1), /* Translation Control Register */ + VNCR(TCR2_EL1), /* Extended Translation Control Register */ + VNCR(ESR_EL1), /* Exception Syndrome Register */ + VNCR(AFSR0_EL1),/* Auxiliary Fault Status Register 0 */ + VNCR(AFSR1_EL1),/* Auxiliary Fault Status Register 1 */ + VNCR(FAR_EL1), /* Fault Address Register */ + VNCR(MAIR_EL1), /* Memory Attribute Indirection Register */ + VNCR(VBAR_EL1), /* Vector Base Address Register */ + VNCR(CONTEXTIDR_EL1), /* Context ID Register */ + VNCR(AMAIR_EL1),/* Aux Memory Attribute Indirection Register */ + VNCR(MDSCR_EL1),/* Monitor Debug System Control Register */ + VNCR(ELR_EL1), + VNCR(SP_EL1), + VNCR(SPSR_EL1), + VNCR(TFSR_EL1), /* Tag Fault Status Register (EL1) */ + VNCR(VPIDR_EL2),/* Virtualization Processor ID Register */ + VNCR(VMPIDR_EL2),/* Virtualization Multiprocessor ID Register */ + VNCR(HCR_EL2), /* Hypervisor Configuration Register */ + VNCR(HSTR_EL2), /* Hypervisor System Trap Register */ + VNCR(VTTBR_EL2),/* Virtualization Translation Table Base Register */ + VNCR(VTCR_EL2), /* Virtualization Translation Control Register */ + VNCR(TPIDR_EL2),/* EL2 Software Thread ID Register */ + VNCR(HCRX_EL2), /* Extended Hypervisor Configuration Register */ + + /* Permission Indirection Extension registers */ + VNCR(PIR_EL1), /* Permission Indirection Register 1 (EL1) */ + VNCR(PIRE0_EL1), /* Permission Indirection Register 0 (EL1) */ + + VNCR(HFGRTR_EL2), + VNCR(HFGWTR_EL2), + VNCR(HFGITR_EL2), + VNCR(HDFGRTR_EL2), + VNCR(HDFGWTR_EL2), + + VNCR(CNTVOFF_EL2), + VNCR(CNTV_CVAL_EL0), + VNCR(CNTV_CTL_EL0), + VNCR(CNTP_CVAL_EL0), + VNCR(CNTP_CTL_EL0), + NR_SYS_REGS /* Nothing after this line! */ }; @@ -465,6 +486,9 @@ struct kvm_cpu_context { u64 sys_regs[NR_SYS_REGS]; struct kvm_vcpu *__hyp_running_vcpu; + + /* This pointer has to be 4kB aligned. */ + u64 *vncr_array; }; struct kvm_host_data { @@ -827,8 +851,19 @@ struct kvm_vcpu_arch { * accessed by a running VCPU. For example, for userspace access or * for system registers that are never context switched, but only * emulated. + * + * Don't bother with VNCR-based accesses in the nVHE code, it has no + * business dealing with NV. */ -#define __ctxt_sys_reg(c,r) (&(c)->sys_regs[(r)]) +static inline u64 *__ctxt_sys_reg(const struct kvm_cpu_context *ctxt, int r) +{ +#if !defined (__KVM_NVHE_HYPERVISOR__) + if (unlikely(cpus_have_final_cap(ARM64_HAS_NESTED_VIRT) && + r >= __VNCR_START__ && ctxt->vncr_array)) + return &ctxt->vncr_array[r - __VNCR_START__]; +#endif + return (u64 *)&ctxt->sys_regs[r]; +} #define ctxt_sys_reg(c,r) (*__ctxt_sys_reg(c,r)) From fedc612314acfebf506e071bf3a941076aa56d10 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 17 Dec 2022 13:28:40 +0000 Subject: [PATCH 187/336] KVM: arm64: nv: Handle virtual EL2 registers in vcpu_read/write_sys_reg() KVM internally uses accessor functions when reading or writing the guest's system registers. This takes care of accessing either the stored copy or using the "live" EL1 system registers when the host uses VHE. With the introduction of virtual EL2 we add a bunch of EL2 system registers, which now must also be taken care of: - If the guest is running in vEL2, and we access an EL1 sysreg, we must revert to the stored version of that, and not use the CPU's copy. - If the guest is running in vEL1, and we access an EL2 sysreg, we must also use the stored version, since the CPU carries the EL1 copy. - Some EL2 system registers are supposed to affect the current execution of the system, so we need to put them into their respective EL1 counterparts. For this we need to define a mapping between the two. - Some EL2 system registers have a different format than their EL1 counterpart, so we need to translate them before writing them to the CPU. This is done using an (optional) translate function in the map. All of these cases are now wrapped into the existing accessor functions, so KVM users wouldn't need to care whether they access EL2 or EL1 registers and also which state the guest is in. Reviewed-by: Ganapatrao Kulkarni Reviewed-by: Alexandru Elisei Reviewed-by: Russell King (Oracle) Reviewed-by: Oliver Upton Co-developed-by: Andre Przywara Signed-off-by: Andre Przywara Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 2 + arch/arm64/kvm/sys_regs.c | 129 ++++++++++++++++++++++++++++-- 2 files changed, 126 insertions(+), 5 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 9e8cd2bb95c3..f17fb7c42973 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -907,6 +907,7 @@ static inline bool __vcpu_read_sys_reg_from_cpu(int reg, u64 *val) case AMAIR_EL1: *val = read_sysreg_s(SYS_AMAIR_EL12); break; case CNTKCTL_EL1: *val = read_sysreg_s(SYS_CNTKCTL_EL12); break; case ELR_EL1: *val = read_sysreg_s(SYS_ELR_EL12); break; + case SPSR_EL1: *val = read_sysreg_s(SYS_SPSR_EL12); break; case PAR_EL1: *val = read_sysreg_par(); break; case DACR32_EL2: *val = read_sysreg_s(SYS_DACR32_EL2); break; case IFSR32_EL2: *val = read_sysreg_s(SYS_IFSR32_EL2); break; @@ -951,6 +952,7 @@ static inline bool __vcpu_write_sys_reg_to_cpu(u64 val, int reg) case AMAIR_EL1: write_sysreg_s(val, SYS_AMAIR_EL12); break; case CNTKCTL_EL1: write_sysreg_s(val, SYS_CNTKCTL_EL12); break; case ELR_EL1: write_sysreg_s(val, SYS_ELR_EL12); break; + case SPSR_EL1: write_sysreg_s(val, SYS_SPSR_EL12); break; case PAR_EL1: write_sysreg_s(val, SYS_PAR_EL1); break; case DACR32_EL2: write_sysreg_s(val, SYS_DACR32_EL2); break; case IFSR32_EL2: write_sysreg_s(val, SYS_IFSR32_EL2); break; diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 7fef170ec67b..3198542dcded 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -72,24 +72,143 @@ static bool write_to_read_only(struct kvm_vcpu *vcpu, "sys_reg write to read-only register"); } +#define PURE_EL2_SYSREG(el2) \ + case el2: { \ + *el1r = el2; \ + return true; \ + } + +#define MAPPED_EL2_SYSREG(el2, el1, fn) \ + case el2: { \ + *xlate = fn; \ + *el1r = el1; \ + return true; \ + } + +static bool get_el2_to_el1_mapping(unsigned int reg, + unsigned int *el1r, u64 (**xlate)(u64)) +{ + switch (reg) { + PURE_EL2_SYSREG( VPIDR_EL2 ); + PURE_EL2_SYSREG( VMPIDR_EL2 ); + PURE_EL2_SYSREG( ACTLR_EL2 ); + PURE_EL2_SYSREG( HCR_EL2 ); + PURE_EL2_SYSREG( MDCR_EL2 ); + PURE_EL2_SYSREG( HSTR_EL2 ); + PURE_EL2_SYSREG( HACR_EL2 ); + PURE_EL2_SYSREG( VTTBR_EL2 ); + PURE_EL2_SYSREG( VTCR_EL2 ); + PURE_EL2_SYSREG( RVBAR_EL2 ); + PURE_EL2_SYSREG( TPIDR_EL2 ); + PURE_EL2_SYSREG( HPFAR_EL2 ); + PURE_EL2_SYSREG( CNTHCTL_EL2 ); + MAPPED_EL2_SYSREG(SCTLR_EL2, SCTLR_EL1, + translate_sctlr_el2_to_sctlr_el1 ); + MAPPED_EL2_SYSREG(CPTR_EL2, CPACR_EL1, + translate_cptr_el2_to_cpacr_el1 ); + MAPPED_EL2_SYSREG(TTBR0_EL2, TTBR0_EL1, + translate_ttbr0_el2_to_ttbr0_el1 ); + MAPPED_EL2_SYSREG(TTBR1_EL2, TTBR1_EL1, NULL ); + MAPPED_EL2_SYSREG(TCR_EL2, TCR_EL1, + translate_tcr_el2_to_tcr_el1 ); + MAPPED_EL2_SYSREG(VBAR_EL2, VBAR_EL1, NULL ); + MAPPED_EL2_SYSREG(AFSR0_EL2, AFSR0_EL1, NULL ); + MAPPED_EL2_SYSREG(AFSR1_EL2, AFSR1_EL1, NULL ); + MAPPED_EL2_SYSREG(ESR_EL2, ESR_EL1, NULL ); + MAPPED_EL2_SYSREG(FAR_EL2, FAR_EL1, NULL ); + MAPPED_EL2_SYSREG(MAIR_EL2, MAIR_EL1, NULL ); + MAPPED_EL2_SYSREG(AMAIR_EL2, AMAIR_EL1, NULL ); + MAPPED_EL2_SYSREG(ELR_EL2, ELR_EL1, NULL ); + MAPPED_EL2_SYSREG(SPSR_EL2, SPSR_EL1, NULL ); + default: + return false; + } +} + u64 vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg) { u64 val = 0x8badf00d8badf00d; + u64 (*xlate)(u64) = NULL; + unsigned int el1r; - if (vcpu_get_flag(vcpu, SYSREGS_ON_CPU) && - __vcpu_read_sys_reg_from_cpu(reg, &val)) + if (!vcpu_get_flag(vcpu, SYSREGS_ON_CPU)) + goto memory_read; + + if (unlikely(get_el2_to_el1_mapping(reg, &el1r, &xlate))) { + if (!is_hyp_ctxt(vcpu)) + goto memory_read; + + /* + * If this register does not have an EL1 counterpart, + * then read the stored EL2 version. + */ + if (reg == el1r) + goto memory_read; + + /* + * If we have a non-VHE guest and that the sysreg + * requires translation to be used at EL1, use the + * in-memory copy instead. + */ + if (!vcpu_el2_e2h_is_set(vcpu) && xlate) + goto memory_read; + + /* Get the current version of the EL1 counterpart. */ + WARN_ON(!__vcpu_read_sys_reg_from_cpu(el1r, &val)); + return val; + } + + /* EL1 register can't be on the CPU if the guest is in vEL2. */ + if (unlikely(is_hyp_ctxt(vcpu))) + goto memory_read; + + if (__vcpu_read_sys_reg_from_cpu(reg, &val)) return val; +memory_read: return __vcpu_sys_reg(vcpu, reg); } void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg) { - if (vcpu_get_flag(vcpu, SYSREGS_ON_CPU) && - __vcpu_write_sys_reg_to_cpu(val, reg)) + u64 (*xlate)(u64) = NULL; + unsigned int el1r; + + if (!vcpu_get_flag(vcpu, SYSREGS_ON_CPU)) + goto memory_write; + + if (unlikely(get_el2_to_el1_mapping(reg, &el1r, &xlate))) { + if (!is_hyp_ctxt(vcpu)) + goto memory_write; + + /* + * Always store a copy of the write to memory to avoid having + * to reverse-translate virtual EL2 system registers for a + * non-VHE guest hypervisor. + */ + __vcpu_sys_reg(vcpu, reg) = val; + + /* No EL1 counterpart? We're done here.? */ + if (reg == el1r) + return; + + if (!vcpu_el2_e2h_is_set(vcpu) && xlate) + val = xlate(val); + + /* Redirect this to the EL1 version of the register. */ + WARN_ON(!__vcpu_write_sys_reg_to_cpu(val, el1r)); + return; + } + + /* EL1 register can't be on the CPU if the guest is in vEL2. */ + if (unlikely(is_hyp_ctxt(vcpu))) + goto memory_write; + + if (__vcpu_write_sys_reg_to_cpu(val, reg)) return; - __vcpu_sys_reg(vcpu, reg) = val; +memory_write: + __vcpu_sys_reg(vcpu, reg) = val; } /* CSSELR values; used to index KVM_REG_ARM_DEMUX_ID_CCSIDR */ From 4a6b93f5629668d1dc8fa5945657fdd124629c55 Mon Sep 17 00:00:00 2001 From: Michal Simek Date: Mon, 6 Nov 2023 12:37:47 +0100 Subject: [PATCH 188/336] dt-bindings: riscv: cpus: Add AMD MicroBlaze V compatible MicroBlaze V is new AMD/Xilinx soft-core 32bit RISC-V processor IP. It is hardware compatible with classic MicroBlaze processor. Signed-off-by: Michal Simek Acked-by: Rob Herring Acked-by: Conor Dooley Link: https://lore.kernel.org/r/d442d916204d26f82c1c3a924a4cdfb117960e1b.1699270661.git.michal.simek@amd.com Signed-off-by: Palmer Dabbelt --- Documentation/devicetree/bindings/riscv/cpus.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/riscv/cpus.yaml b/Documentation/devicetree/bindings/riscv/cpus.yaml index f392e367d673..23646b684ea2 100644 --- a/Documentation/devicetree/bindings/riscv/cpus.yaml +++ b/Documentation/devicetree/bindings/riscv/cpus.yaml @@ -32,6 +32,7 @@ properties: oneOf: - items: - enum: + - amd,mbv32 - andestech,ax45mp - canaan,k210 - sifive,bullet0 From c30fa83b49897e708a52e122dd10616a52a4c82b Mon Sep 17 00:00:00 2001 From: Alexandre Ghiti Date: Wed, 13 Dec 2023 21:29:58 +0100 Subject: [PATCH 189/336] riscv: Use WRITE_ONCE() when setting page table entries To avoid any compiler "weirdness" when accessing page table entries which are concurrently modified by the HW, let's use WRITE_ONCE() macro (commit 20a004e7b017 ("arm64: mm: Use READ_ONCE/WRITE_ONCE when accessing page tables") gives a great explanation with more details). Signed-off-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20231213203001.179237-2-alexghiti@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/pgtable-64.h | 6 +++--- arch/riscv/include/asm/pgtable.h | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/riscv/include/asm/pgtable-64.h b/arch/riscv/include/asm/pgtable-64.h index 9a2c780a11e9..5d8431a390dd 100644 --- a/arch/riscv/include/asm/pgtable-64.h +++ b/arch/riscv/include/asm/pgtable-64.h @@ -202,7 +202,7 @@ static inline int pud_user(pud_t pud) static inline void set_pud(pud_t *pudp, pud_t pud) { - *pudp = pud; + WRITE_ONCE(*pudp, pud); } static inline void pud_clear(pud_t *pudp) @@ -278,7 +278,7 @@ static inline unsigned long _pmd_pfn(pmd_t pmd) static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) { if (pgtable_l4_enabled) - *p4dp = p4d; + WRITE_ONCE(*p4dp, p4d); else set_pud((pud_t *)p4dp, (pud_t){ p4d_val(p4d) }); } @@ -351,7 +351,7 @@ static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) { if (pgtable_l5_enabled) - *pgdp = pgd; + WRITE_ONCE(*pgdp, pgd); else set_p4d((p4d_t *)pgdp, (p4d_t){ pgd_val(pgd) }); } diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 294044429e8e..c9f4b250b4ee 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -248,7 +248,7 @@ static inline int pmd_leaf(pmd_t pmd) static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) { - *pmdp = pmd; + WRITE_ONCE(*pmdp, pmd); } static inline void pmd_clear(pmd_t *pmdp) @@ -510,7 +510,7 @@ static inline int pte_same(pte_t pte_a, pte_t pte_b) */ static inline void set_pte(pte_t *ptep, pte_t pteval) { - *ptep = pteval; + WRITE_ONCE(*ptep, pteval); } void flush_icache_pte(pte_t pte); From eba2591d99d1f14a04c8a8a845ab0795b93f5646 Mon Sep 17 00:00:00 2001 From: Alexandre Ghiti Date: Wed, 13 Dec 2023 21:29:59 +0100 Subject: [PATCH 190/336] mm: Introduce pudp/p4dp/pgdp_get() functions Instead of directly dereferencing page tables entries, which can cause issues (see commit 20a004e7b017 ("arm64: mm: Use READ_ONCE/WRITE_ONCE when accessing page tables"), let's introduce new functions to get the pud/p4d/pgd entries (the pte and pmd versions already exist). Note that arm pgd_t is actually an array so pgdp_get() is defined as a macro to avoid a build error. Those new functions will be used in subsequent commits by the riscv architecture. Signed-off-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20231213203001.179237-3-alexghiti@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/arm/include/asm/pgtable.h | 2 ++ include/linux/pgtable.h | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h index 16b02f44c7d3..d657b84b6bf7 100644 --- a/arch/arm/include/asm/pgtable.h +++ b/arch/arm/include/asm/pgtable.h @@ -151,6 +151,8 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; +#define pgdp_get(pgpd) READ_ONCE(*pgdp) + #define pud_page(pud) pmd_page(__pmd(pud_val(pud))) #define pud_write(pud) pmd_write(__pmd(pud_val(pud))) diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index af7639c3b0a3..8b7daccd11be 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -292,6 +292,27 @@ static inline pmd_t pmdp_get(pmd_t *pmdp) } #endif +#ifndef pudp_get +static inline pud_t pudp_get(pud_t *pudp) +{ + return READ_ONCE(*pudp); +} +#endif + +#ifndef p4dp_get +static inline p4d_t p4dp_get(p4d_t *p4dp) +{ + return READ_ONCE(*p4dp); +} +#endif + +#ifndef pgdp_get +static inline pgd_t pgdp_get(pgd_t *pgdp) +{ + return READ_ONCE(*pgdp); +} +#endif + #ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long address, From d6508999d1882ddd0db8b3b4bd7967d83e9909fa Mon Sep 17 00:00:00 2001 From: Alexandre Ghiti Date: Wed, 13 Dec 2023 21:30:00 +0100 Subject: [PATCH 191/336] riscv: mm: Only compile pgtable.c if MMU All functions defined in there depend on MMU, so no need to compile it for !MMU configs. Signed-off-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20231213203001.179237-4-alexghiti@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/mm/Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/riscv/mm/Makefile b/arch/riscv/mm/Makefile index 3a4dfc8babcf..2c869f8026a8 100644 --- a/arch/riscv/mm/Makefile +++ b/arch/riscv/mm/Makefile @@ -13,10 +13,9 @@ endif KCOV_INSTRUMENT_init.o := n obj-y += init.o -obj-$(CONFIG_MMU) += extable.o fault.o pageattr.o +obj-$(CONFIG_MMU) += extable.o fault.o pageattr.o pgtable.o obj-y += cacheflush.o obj-y += context.o -obj-y += pgtable.o obj-y += pmem.o ifeq ($(CONFIG_MMU),y) From edf955647269422e387732870d04fc15933a25ea Mon Sep 17 00:00:00 2001 From: Alexandre Ghiti Date: Wed, 13 Dec 2023 21:30:01 +0100 Subject: [PATCH 192/336] riscv: Use accessors to page table entries instead of direct dereference As very well explained in commit 20a004e7b017 ("arm64: mm: Use READ_ONCE/WRITE_ONCE when accessing page tables"), an architecture whose page table walker can modify the PTE in parallel must use READ_ONCE()/WRITE_ONCE() macro to avoid any compiler transformation. So apply that to riscv which is such architecture. Signed-off-by: Alexandre Ghiti Acked-by: Anup Patel Link: https://lore.kernel.org/r/20231213203001.179237-5-alexghiti@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/kfence.h | 4 +-- arch/riscv/include/asm/pgtable-64.h | 16 ++------- arch/riscv/include/asm/pgtable.h | 29 ++++------------ arch/riscv/kernel/efi.c | 2 +- arch/riscv/kvm/mmu.c | 22 ++++++------- arch/riscv/mm/fault.c | 16 ++++----- arch/riscv/mm/hugetlbpage.c | 12 +++---- arch/riscv/mm/kasan_init.c | 45 +++++++++++++------------ arch/riscv/mm/pageattr.c | 44 ++++++++++++------------- arch/riscv/mm/pgtable.c | 51 ++++++++++++++++++++++++++--- 10 files changed, 128 insertions(+), 113 deletions(-) diff --git a/arch/riscv/include/asm/kfence.h b/arch/riscv/include/asm/kfence.h index 0bbffd528096..7388edd88986 100644 --- a/arch/riscv/include/asm/kfence.h +++ b/arch/riscv/include/asm/kfence.h @@ -18,9 +18,9 @@ static inline bool kfence_protect_page(unsigned long addr, bool protect) pte_t *pte = virt_to_kpte(addr); if (protect) - set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_PRESENT)); + set_pte(pte, __pte(pte_val(ptep_get(pte)) & ~_PAGE_PRESENT)); else - set_pte(pte, __pte(pte_val(*pte) | _PAGE_PRESENT)); + set_pte(pte, __pte(pte_val(ptep_get(pte)) | _PAGE_PRESENT)); flush_tlb_kernel_range(addr, addr + PAGE_SIZE); diff --git a/arch/riscv/include/asm/pgtable-64.h b/arch/riscv/include/asm/pgtable-64.h index 5d8431a390dd..b42017d76924 100644 --- a/arch/riscv/include/asm/pgtable-64.h +++ b/arch/riscv/include/asm/pgtable-64.h @@ -340,13 +340,7 @@ static inline struct page *p4d_page(p4d_t p4d) #define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)) #define pud_offset pud_offset -static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) -{ - if (pgtable_l4_enabled) - return p4d_pgtable(*p4d) + pud_index(address); - - return (pud_t *)p4d; -} +pud_t *pud_offset(p4d_t *p4d, unsigned long address); static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) { @@ -404,12 +398,6 @@ static inline struct page *pgd_page(pgd_t pgd) #define p4d_index(addr) (((addr) >> P4D_SHIFT) & (PTRS_PER_P4D - 1)) #define p4d_offset p4d_offset -static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) -{ - if (pgtable_l5_enabled) - return pgd_pgtable(*pgd) + p4d_index(address); - - return (p4d_t *)pgd; -} +p4d_t *p4d_offset(pgd_t *pgd, unsigned long address); #endif /* _ASM_RISCV_PGTABLE_64_H */ diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index c9f4b250b4ee..3773f454f0fa 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -544,19 +544,12 @@ static inline void pte_clear(struct mm_struct *mm, __set_pte_at(ptep, __pte(0)); } -#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS -static inline int ptep_set_access_flags(struct vm_area_struct *vma, - unsigned long address, pte_t *ptep, - pte_t entry, int dirty) -{ - if (!pte_same(*ptep, entry)) - __set_pte_at(ptep, entry); - /* - * update_mmu_cache will unconditionally execute, handling both - * the case that the PTE changed and the spurious fault case. - */ - return true; -} +#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS /* defined in mm/pgtable.c */ +extern int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, + pte_t *ptep, pte_t entry, int dirty); +#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG /* defined in mm/pgtable.c */ +extern int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long address, + pte_t *ptep); #define __HAVE_ARCH_PTEP_GET_AND_CLEAR static inline pte_t ptep_get_and_clear(struct mm_struct *mm, @@ -569,16 +562,6 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, return pte; } -#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG -static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, - unsigned long address, - pte_t *ptep) -{ - if (!pte_young(*ptep)) - return 0; - return test_and_clear_bit(_PAGE_ACCESSED_OFFSET, &pte_val(*ptep)); -} - #define __HAVE_ARCH_PTEP_SET_WRPROTECT static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep) diff --git a/arch/riscv/kernel/efi.c b/arch/riscv/kernel/efi.c index aa6209a74c83..b64bf1624a05 100644 --- a/arch/riscv/kernel/efi.c +++ b/arch/riscv/kernel/efi.c @@ -60,7 +60,7 @@ int __init efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md) static int __init set_permissions(pte_t *ptep, unsigned long addr, void *data) { efi_memory_desc_t *md = data; - pte_t pte = READ_ONCE(*ptep); + pte_t pte = ptep_get(ptep); unsigned long val; if (md->attribute & EFI_MEMORY_RO) { diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c index 068c74593871..a9e2fd7245e1 100644 --- a/arch/riscv/kvm/mmu.c +++ b/arch/riscv/kvm/mmu.c @@ -103,7 +103,7 @@ static bool gstage_get_leaf_entry(struct kvm *kvm, gpa_t addr, *ptep_level = current_level; ptep = (pte_t *)kvm->arch.pgd; ptep = &ptep[gstage_pte_index(addr, current_level)]; - while (ptep && pte_val(*ptep)) { + while (ptep && pte_val(ptep_get(ptep))) { if (gstage_pte_leaf(ptep)) { *ptep_level = current_level; *ptepp = ptep; @@ -113,7 +113,7 @@ static bool gstage_get_leaf_entry(struct kvm *kvm, gpa_t addr, if (current_level) { current_level--; *ptep_level = current_level; - ptep = (pte_t *)gstage_pte_page_vaddr(*ptep); + ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep)); ptep = &ptep[gstage_pte_index(addr, current_level)]; } else { ptep = NULL; @@ -149,25 +149,25 @@ static int gstage_set_pte(struct kvm *kvm, u32 level, if (gstage_pte_leaf(ptep)) return -EEXIST; - if (!pte_val(*ptep)) { + if (!pte_val(ptep_get(ptep))) { if (!pcache) return -ENOMEM; next_ptep = kvm_mmu_memory_cache_alloc(pcache); if (!next_ptep) return -ENOMEM; - *ptep = pfn_pte(PFN_DOWN(__pa(next_ptep)), - __pgprot(_PAGE_TABLE)); + set_pte(ptep, pfn_pte(PFN_DOWN(__pa(next_ptep)), + __pgprot(_PAGE_TABLE))); } else { if (gstage_pte_leaf(ptep)) return -EEXIST; - next_ptep = (pte_t *)gstage_pte_page_vaddr(*ptep); + next_ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep)); } current_level--; ptep = &next_ptep[gstage_pte_index(addr, current_level)]; } - *ptep = *new_pte; + set_pte(ptep, *new_pte); if (gstage_pte_leaf(ptep)) gstage_remote_tlb_flush(kvm, current_level, addr); @@ -239,11 +239,11 @@ static void gstage_op_pte(struct kvm *kvm, gpa_t addr, BUG_ON(addr & (page_size - 1)); - if (!pte_val(*ptep)) + if (!pte_val(ptep_get(ptep))) return; if (ptep_level && !gstage_pte_leaf(ptep)) { - next_ptep = (pte_t *)gstage_pte_page_vaddr(*ptep); + next_ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep)); next_ptep_level = ptep_level - 1; ret = gstage_level_to_page_size(next_ptep_level, &next_page_size); @@ -261,7 +261,7 @@ static void gstage_op_pte(struct kvm *kvm, gpa_t addr, if (op == GSTAGE_OP_CLEAR) set_pte(ptep, __pte(0)); else if (op == GSTAGE_OP_WP) - set_pte(ptep, __pte(pte_val(*ptep) & ~_PAGE_WRITE)); + set_pte(ptep, __pte(pte_val(ptep_get(ptep)) & ~_PAGE_WRITE)); gstage_remote_tlb_flush(kvm, ptep_level, addr); } } @@ -603,7 +603,7 @@ bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) &ptep, &ptep_level)) return false; - return pte_young(*ptep); + return pte_young(ptep_get(ptep)); } int kvm_riscv_gstage_map(struct kvm_vcpu *vcpu, diff --git a/arch/riscv/mm/fault.c b/arch/riscv/mm/fault.c index 90d4ba36d1d0..76f1df709a21 100644 --- a/arch/riscv/mm/fault.c +++ b/arch/riscv/mm/fault.c @@ -136,24 +136,24 @@ static inline void vmalloc_fault(struct pt_regs *regs, int code, unsigned long a pgd = (pgd_t *)pfn_to_virt(pfn) + index; pgd_k = init_mm.pgd + index; - if (!pgd_present(*pgd_k)) { + if (!pgd_present(pgdp_get(pgd_k))) { no_context(regs, addr); return; } - set_pgd(pgd, *pgd_k); + set_pgd(pgd, pgdp_get(pgd_k)); p4d_k = p4d_offset(pgd_k, addr); - if (!p4d_present(*p4d_k)) { + if (!p4d_present(p4dp_get(p4d_k))) { no_context(regs, addr); return; } pud_k = pud_offset(p4d_k, addr); - if (!pud_present(*pud_k)) { + if (!pud_present(pudp_get(pud_k))) { no_context(regs, addr); return; } - if (pud_leaf(*pud_k)) + if (pud_leaf(pudp_get(pud_k))) goto flush_tlb; /* @@ -161,11 +161,11 @@ static inline void vmalloc_fault(struct pt_regs *regs, int code, unsigned long a * to copy individual PTEs */ pmd_k = pmd_offset(pud_k, addr); - if (!pmd_present(*pmd_k)) { + if (!pmd_present(pmdp_get(pmd_k))) { no_context(regs, addr); return; } - if (pmd_leaf(*pmd_k)) + if (pmd_leaf(pmdp_get(pmd_k))) goto flush_tlb; /* @@ -175,7 +175,7 @@ static inline void vmalloc_fault(struct pt_regs *regs, int code, unsigned long a * silently loop forever. */ pte_k = pte_offset_kernel(pmd_k, addr); - if (!pte_present(*pte_k)) { + if (!pte_present(ptep_get(pte_k))) { no_context(regs, addr); return; } diff --git a/arch/riscv/mm/hugetlbpage.c b/arch/riscv/mm/hugetlbpage.c index b52f0210481f..431596c0e20e 100644 --- a/arch/riscv/mm/hugetlbpage.c +++ b/arch/riscv/mm/hugetlbpage.c @@ -54,7 +54,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, } if (sz == PMD_SIZE) { - if (want_pmd_share(vma, addr) && pud_none(*pud)) + if (want_pmd_share(vma, addr) && pud_none(pudp_get(pud))) pte = huge_pmd_share(mm, vma, addr, pud); else pte = (pte_t *)pmd_alloc(mm, pud, addr); @@ -93,11 +93,11 @@ pte_t *huge_pte_offset(struct mm_struct *mm, pmd_t *pmd; pgd = pgd_offset(mm, addr); - if (!pgd_present(*pgd)) + if (!pgd_present(pgdp_get(pgd))) return NULL; p4d = p4d_offset(pgd, addr); - if (!p4d_present(*p4d)) + if (!p4d_present(p4dp_get(p4d))) return NULL; pud = pud_offset(p4d, addr); @@ -105,7 +105,7 @@ pte_t *huge_pte_offset(struct mm_struct *mm, /* must be pud huge, non-present or none */ return (pte_t *)pud; - if (!pud_present(*pud)) + if (!pud_present(pudp_get(pud))) return NULL; pmd = pmd_offset(pud, addr); @@ -113,7 +113,7 @@ pte_t *huge_pte_offset(struct mm_struct *mm, /* must be pmd huge, non-present or none */ return (pte_t *)pmd; - if (!pmd_present(*pmd)) + if (!pmd_present(pmdp_get(pmd))) return NULL; for_each_napot_order(order) { @@ -293,7 +293,7 @@ void huge_pte_clear(struct mm_struct *mm, pte_t *ptep, unsigned long sz) { - pte_t pte = READ_ONCE(*ptep); + pte_t pte = ptep_get(ptep); int i, pte_num; if (!pte_napot(pte)) { diff --git a/arch/riscv/mm/kasan_init.c b/arch/riscv/mm/kasan_init.c index 5e39dcf23fdb..e96251853037 100644 --- a/arch/riscv/mm/kasan_init.c +++ b/arch/riscv/mm/kasan_init.c @@ -31,7 +31,7 @@ static void __init kasan_populate_pte(pmd_t *pmd, unsigned long vaddr, unsigned phys_addr_t phys_addr; pte_t *ptep, *p; - if (pmd_none(*pmd)) { + if (pmd_none(pmdp_get(pmd))) { p = memblock_alloc(PTRS_PER_PTE * sizeof(pte_t), PAGE_SIZE); set_pmd(pmd, pfn_pmd(PFN_DOWN(__pa(p)), PAGE_TABLE)); } @@ -39,7 +39,7 @@ static void __init kasan_populate_pte(pmd_t *pmd, unsigned long vaddr, unsigned ptep = pte_offset_kernel(pmd, vaddr); do { - if (pte_none(*ptep)) { + if (pte_none(ptep_get(ptep))) { phys_addr = memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); set_pte(ptep, pfn_pte(PFN_DOWN(phys_addr), PAGE_KERNEL)); memset(__va(phys_addr), KASAN_SHADOW_INIT, PAGE_SIZE); @@ -53,7 +53,7 @@ static void __init kasan_populate_pmd(pud_t *pud, unsigned long vaddr, unsigned pmd_t *pmdp, *p; unsigned long next; - if (pud_none(*pud)) { + if (pud_none(pudp_get(pud))) { p = memblock_alloc(PTRS_PER_PMD * sizeof(pmd_t), PAGE_SIZE); set_pud(pud, pfn_pud(PFN_DOWN(__pa(p)), PAGE_TABLE)); } @@ -63,7 +63,8 @@ static void __init kasan_populate_pmd(pud_t *pud, unsigned long vaddr, unsigned do { next = pmd_addr_end(vaddr, end); - if (pmd_none(*pmdp) && IS_ALIGNED(vaddr, PMD_SIZE) && (next - vaddr) >= PMD_SIZE) { + if (pmd_none(pmdp_get(pmdp)) && IS_ALIGNED(vaddr, PMD_SIZE) && + (next - vaddr) >= PMD_SIZE) { phys_addr = memblock_phys_alloc(PMD_SIZE, PMD_SIZE); if (phys_addr) { set_pmd(pmdp, pfn_pmd(PFN_DOWN(phys_addr), PAGE_KERNEL)); @@ -83,7 +84,7 @@ static void __init kasan_populate_pud(p4d_t *p4d, pud_t *pudp, *p; unsigned long next; - if (p4d_none(*p4d)) { + if (p4d_none(p4dp_get(p4d))) { p = memblock_alloc(PTRS_PER_PUD * sizeof(pud_t), PAGE_SIZE); set_p4d(p4d, pfn_p4d(PFN_DOWN(__pa(p)), PAGE_TABLE)); } @@ -93,7 +94,8 @@ static void __init kasan_populate_pud(p4d_t *p4d, do { next = pud_addr_end(vaddr, end); - if (pud_none(*pudp) && IS_ALIGNED(vaddr, PUD_SIZE) && (next - vaddr) >= PUD_SIZE) { + if (pud_none(pudp_get(pudp)) && IS_ALIGNED(vaddr, PUD_SIZE) && + (next - vaddr) >= PUD_SIZE) { phys_addr = memblock_phys_alloc(PUD_SIZE, PUD_SIZE); if (phys_addr) { set_pud(pudp, pfn_pud(PFN_DOWN(phys_addr), PAGE_KERNEL)); @@ -113,7 +115,7 @@ static void __init kasan_populate_p4d(pgd_t *pgd, p4d_t *p4dp, *p; unsigned long next; - if (pgd_none(*pgd)) { + if (pgd_none(pgdp_get(pgd))) { p = memblock_alloc(PTRS_PER_P4D * sizeof(p4d_t), PAGE_SIZE); set_pgd(pgd, pfn_pgd(PFN_DOWN(__pa(p)), PAGE_TABLE)); } @@ -123,7 +125,8 @@ static void __init kasan_populate_p4d(pgd_t *pgd, do { next = p4d_addr_end(vaddr, end); - if (p4d_none(*p4dp) && IS_ALIGNED(vaddr, P4D_SIZE) && (next - vaddr) >= P4D_SIZE) { + if (p4d_none(p4dp_get(p4dp)) && IS_ALIGNED(vaddr, P4D_SIZE) && + (next - vaddr) >= P4D_SIZE) { phys_addr = memblock_phys_alloc(P4D_SIZE, P4D_SIZE); if (phys_addr) { set_p4d(p4dp, pfn_p4d(PFN_DOWN(phys_addr), PAGE_KERNEL)); @@ -145,7 +148,7 @@ static void __init kasan_populate_pgd(pgd_t *pgdp, do { next = pgd_addr_end(vaddr, end); - if (pgd_none(*pgdp) && IS_ALIGNED(vaddr, PGDIR_SIZE) && + if (pgd_none(pgdp_get(pgdp)) && IS_ALIGNED(vaddr, PGDIR_SIZE) && (next - vaddr) >= PGDIR_SIZE) { phys_addr = memblock_phys_alloc(PGDIR_SIZE, PGDIR_SIZE); if (phys_addr) { @@ -168,7 +171,7 @@ static void __init kasan_early_clear_pud(p4d_t *p4dp, if (!pgtable_l4_enabled) { pudp = (pud_t *)p4dp; } else { - base_pud = pt_ops.get_pud_virt(pfn_to_phys(_p4d_pfn(*p4dp))); + base_pud = pt_ops.get_pud_virt(pfn_to_phys(_p4d_pfn(p4dp_get(p4dp)))); pudp = base_pud + pud_index(vaddr); } @@ -193,7 +196,7 @@ static void __init kasan_early_clear_p4d(pgd_t *pgdp, if (!pgtable_l5_enabled) { p4dp = (p4d_t *)pgdp; } else { - base_p4d = pt_ops.get_p4d_virt(pfn_to_phys(_pgd_pfn(*pgdp))); + base_p4d = pt_ops.get_p4d_virt(pfn_to_phys(_pgd_pfn(pgdp_get(pgdp)))); p4dp = base_p4d + p4d_index(vaddr); } @@ -239,14 +242,14 @@ static void __init kasan_early_populate_pud(p4d_t *p4dp, if (!pgtable_l4_enabled) { pudp = (pud_t *)p4dp; } else { - base_pud = pt_ops.get_pud_virt(pfn_to_phys(_p4d_pfn(*p4dp))); + base_pud = pt_ops.get_pud_virt(pfn_to_phys(_p4d_pfn(p4dp_get(p4dp)))); pudp = base_pud + pud_index(vaddr); } do { next = pud_addr_end(vaddr, end); - if (pud_none(*pudp) && IS_ALIGNED(vaddr, PUD_SIZE) && + if (pud_none(pudp_get(pudp)) && IS_ALIGNED(vaddr, PUD_SIZE) && (next - vaddr) >= PUD_SIZE) { phys_addr = __pa((uintptr_t)kasan_early_shadow_pmd); set_pud(pudp, pfn_pud(PFN_DOWN(phys_addr), PAGE_TABLE)); @@ -277,14 +280,14 @@ static void __init kasan_early_populate_p4d(pgd_t *pgdp, if (!pgtable_l5_enabled) { p4dp = (p4d_t *)pgdp; } else { - base_p4d = pt_ops.get_p4d_virt(pfn_to_phys(_pgd_pfn(*pgdp))); + base_p4d = pt_ops.get_p4d_virt(pfn_to_phys(_pgd_pfn(pgdp_get(pgdp)))); p4dp = base_p4d + p4d_index(vaddr); } do { next = p4d_addr_end(vaddr, end); - if (p4d_none(*p4dp) && IS_ALIGNED(vaddr, P4D_SIZE) && + if (p4d_none(p4dp_get(p4dp)) && IS_ALIGNED(vaddr, P4D_SIZE) && (next - vaddr) >= P4D_SIZE) { phys_addr = __pa((uintptr_t)kasan_early_shadow_pud); set_p4d(p4dp, pfn_p4d(PFN_DOWN(phys_addr), PAGE_TABLE)); @@ -305,7 +308,7 @@ static void __init kasan_early_populate_pgd(pgd_t *pgdp, do { next = pgd_addr_end(vaddr, end); - if (pgd_none(*pgdp) && IS_ALIGNED(vaddr, PGDIR_SIZE) && + if (pgd_none(pgdp_get(pgdp)) && IS_ALIGNED(vaddr, PGDIR_SIZE) && (next - vaddr) >= PGDIR_SIZE) { phys_addr = __pa((uintptr_t)kasan_early_shadow_p4d); set_pgd(pgdp, pfn_pgd(PFN_DOWN(phys_addr), PAGE_TABLE)); @@ -381,7 +384,7 @@ static void __init kasan_shallow_populate_pud(p4d_t *p4d, do { next = pud_addr_end(vaddr, end); - if (pud_none(*pud_k)) { + if (pud_none(pudp_get(pud_k))) { p = memblock_alloc(PAGE_SIZE, PAGE_SIZE); set_pud(pud_k, pfn_pud(PFN_DOWN(__pa(p)), PAGE_TABLE)); continue; @@ -401,7 +404,7 @@ static void __init kasan_shallow_populate_p4d(pgd_t *pgd, do { next = p4d_addr_end(vaddr, end); - if (p4d_none(*p4d_k)) { + if (p4d_none(p4dp_get(p4d_k))) { p = memblock_alloc(PAGE_SIZE, PAGE_SIZE); set_p4d(p4d_k, pfn_p4d(PFN_DOWN(__pa(p)), PAGE_TABLE)); continue; @@ -420,7 +423,7 @@ static void __init kasan_shallow_populate_pgd(unsigned long vaddr, unsigned long do { next = pgd_addr_end(vaddr, end); - if (pgd_none(*pgd_k)) { + if (pgd_none(pgdp_get(pgd_k))) { p = memblock_alloc(PAGE_SIZE, PAGE_SIZE); set_pgd(pgd_k, pfn_pgd(PFN_DOWN(__pa(p)), PAGE_TABLE)); continue; @@ -451,7 +454,7 @@ static void __init create_tmp_mapping(void) /* Copy the last p4d since it is shared with the kernel mapping. */ if (pgtable_l5_enabled) { - ptr = (p4d_t *)pgd_page_vaddr(*pgd_offset_k(KASAN_SHADOW_END)); + ptr = (p4d_t *)pgd_page_vaddr(pgdp_get(pgd_offset_k(KASAN_SHADOW_END))); memcpy(tmp_p4d, ptr, sizeof(p4d_t) * PTRS_PER_P4D); set_pgd(&tmp_pg_dir[pgd_index(KASAN_SHADOW_END)], pfn_pgd(PFN_DOWN(__pa(tmp_p4d)), PAGE_TABLE)); @@ -462,7 +465,7 @@ static void __init create_tmp_mapping(void) /* Copy the last pud since it is shared with the kernel mapping. */ if (pgtable_l4_enabled) { - ptr = (pud_t *)p4d_page_vaddr(*(base_p4d + p4d_index(KASAN_SHADOW_END))); + ptr = (pud_t *)p4d_page_vaddr(p4dp_get(base_p4d + p4d_index(KASAN_SHADOW_END))); memcpy(tmp_pud, ptr, sizeof(pud_t) * PTRS_PER_PUD); set_p4d(&base_p4d[p4d_index(KASAN_SHADOW_END)], pfn_p4d(PFN_DOWN(__pa(tmp_pud)), PAGE_TABLE)); diff --git a/arch/riscv/mm/pageattr.c b/arch/riscv/mm/pageattr.c index fc5fc4f785c4..0b5e38e018c8 100644 --- a/arch/riscv/mm/pageattr.c +++ b/arch/riscv/mm/pageattr.c @@ -29,7 +29,7 @@ static unsigned long set_pageattr_masks(unsigned long val, struct mm_walk *walk) static int pageattr_p4d_entry(p4d_t *p4d, unsigned long addr, unsigned long next, struct mm_walk *walk) { - p4d_t val = READ_ONCE(*p4d); + p4d_t val = p4dp_get(p4d); if (p4d_leaf(val)) { val = __p4d(set_pageattr_masks(p4d_val(val), walk)); @@ -42,7 +42,7 @@ static int pageattr_p4d_entry(p4d_t *p4d, unsigned long addr, static int pageattr_pud_entry(pud_t *pud, unsigned long addr, unsigned long next, struct mm_walk *walk) { - pud_t val = READ_ONCE(*pud); + pud_t val = pudp_get(pud); if (pud_leaf(val)) { val = __pud(set_pageattr_masks(pud_val(val), walk)); @@ -55,7 +55,7 @@ static int pageattr_pud_entry(pud_t *pud, unsigned long addr, static int pageattr_pmd_entry(pmd_t *pmd, unsigned long addr, unsigned long next, struct mm_walk *walk) { - pmd_t val = READ_ONCE(*pmd); + pmd_t val = pmdp_get(pmd); if (pmd_leaf(val)) { val = __pmd(set_pageattr_masks(pmd_val(val), walk)); @@ -68,7 +68,7 @@ static int pageattr_pmd_entry(pmd_t *pmd, unsigned long addr, static int pageattr_pte_entry(pte_t *pte, unsigned long addr, unsigned long next, struct mm_walk *walk) { - pte_t val = READ_ONCE(*pte); + pte_t val = ptep_get(pte); val = __pte(set_pageattr_masks(pte_val(val), walk)); set_pte(pte, val); @@ -108,10 +108,10 @@ static int __split_linear_mapping_pmd(pud_t *pudp, vaddr <= (vaddr & PMD_MASK) && end >= next) continue; - if (pmd_leaf(*pmdp)) { + if (pmd_leaf(pmdp_get(pmdp))) { struct page *pte_page; - unsigned long pfn = _pmd_pfn(*pmdp); - pgprot_t prot = __pgprot(pmd_val(*pmdp) & ~_PAGE_PFN_MASK); + unsigned long pfn = _pmd_pfn(pmdp_get(pmdp)); + pgprot_t prot = __pgprot(pmd_val(pmdp_get(pmdp)) & ~_PAGE_PFN_MASK); pte_t *ptep_new; int i; @@ -148,10 +148,10 @@ static int __split_linear_mapping_pud(p4d_t *p4dp, vaddr <= (vaddr & PUD_MASK) && end >= next) continue; - if (pud_leaf(*pudp)) { + if (pud_leaf(pudp_get(pudp))) { struct page *pmd_page; - unsigned long pfn = _pud_pfn(*pudp); - pgprot_t prot = __pgprot(pud_val(*pudp) & ~_PAGE_PFN_MASK); + unsigned long pfn = _pud_pfn(pudp_get(pudp)); + pgprot_t prot = __pgprot(pud_val(pudp_get(pudp)) & ~_PAGE_PFN_MASK); pmd_t *pmdp_new; int i; @@ -197,10 +197,10 @@ static int __split_linear_mapping_p4d(pgd_t *pgdp, vaddr <= (vaddr & P4D_MASK) && end >= next) continue; - if (p4d_leaf(*p4dp)) { + if (p4d_leaf(p4dp_get(p4dp))) { struct page *pud_page; - unsigned long pfn = _p4d_pfn(*p4dp); - pgprot_t prot = __pgprot(p4d_val(*p4dp) & ~_PAGE_PFN_MASK); + unsigned long pfn = _p4d_pfn(p4dp_get(p4dp)); + pgprot_t prot = __pgprot(p4d_val(p4dp_get(p4dp)) & ~_PAGE_PFN_MASK); pud_t *pudp_new; int i; @@ -406,29 +406,29 @@ bool kernel_page_present(struct page *page) pte_t *pte; pgd = pgd_offset_k(addr); - if (!pgd_present(*pgd)) + if (!pgd_present(pgdp_get(pgd))) return false; - if (pgd_leaf(*pgd)) + if (pgd_leaf(pgdp_get(pgd))) return true; p4d = p4d_offset(pgd, addr); - if (!p4d_present(*p4d)) + if (!p4d_present(p4dp_get(p4d))) return false; - if (p4d_leaf(*p4d)) + if (p4d_leaf(p4dp_get(p4d))) return true; pud = pud_offset(p4d, addr); - if (!pud_present(*pud)) + if (!pud_present(pudp_get(pud))) return false; - if (pud_leaf(*pud)) + if (pud_leaf(pudp_get(pud))) return true; pmd = pmd_offset(pud, addr); - if (!pmd_present(*pmd)) + if (!pmd_present(pmdp_get(pmd))) return false; - if (pmd_leaf(*pmd)) + if (pmd_leaf(pmdp_get(pmd))) return true; pte = pte_offset_kernel(pmd, addr); - return pte_present(*pte); + return pte_present(ptep_get(pte)); } diff --git a/arch/riscv/mm/pgtable.c b/arch/riscv/mm/pgtable.c index fef4e7328e49..ef887efcb679 100644 --- a/arch/riscv/mm/pgtable.c +++ b/arch/riscv/mm/pgtable.c @@ -5,6 +5,47 @@ #include #include +int ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep, + pte_t entry, int dirty) +{ + if (!pte_same(ptep_get(ptep), entry)) + __set_pte_at(ptep, entry); + /* + * update_mmu_cache will unconditionally execute, handling both + * the case that the PTE changed and the spurious fault case. + */ + return true; +} + +int ptep_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, + pte_t *ptep) +{ + if (!pte_young(ptep_get(ptep))) + return 0; + return test_and_clear_bit(_PAGE_ACCESSED_OFFSET, &pte_val(*ptep)); +} +EXPORT_SYMBOL_GPL(ptep_test_and_clear_young); + +#ifdef CONFIG_64BIT +pud_t *pud_offset(p4d_t *p4d, unsigned long address) +{ + if (pgtable_l4_enabled) + return p4d_pgtable(p4dp_get(p4d)) + pud_index(address); + + return (pud_t *)p4d; +} + +p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) +{ + if (pgtable_l5_enabled) + return pgd_pgtable(pgdp_get(pgd)) + p4d_index(address); + + return (p4d_t *)pgd; +} +#endif + #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot) { @@ -25,7 +66,7 @@ int pud_set_huge(pud_t *pud, phys_addr_t phys, pgprot_t prot) int pud_clear_huge(pud_t *pud) { - if (!pud_leaf(READ_ONCE(*pud))) + if (!pud_leaf(pudp_get(pud))) return 0; pud_clear(pud); return 1; @@ -33,7 +74,7 @@ int pud_clear_huge(pud_t *pud) int pud_free_pmd_page(pud_t *pud, unsigned long addr) { - pmd_t *pmd = pud_pgtable(*pud); + pmd_t *pmd = pud_pgtable(pudp_get(pud)); int i; pud_clear(pud); @@ -63,7 +104,7 @@ int pmd_set_huge(pmd_t *pmd, phys_addr_t phys, pgprot_t prot) int pmd_clear_huge(pmd_t *pmd) { - if (!pmd_leaf(READ_ONCE(*pmd))) + if (!pmd_leaf(pmdp_get(pmd))) return 0; pmd_clear(pmd); return 1; @@ -71,7 +112,7 @@ int pmd_clear_huge(pmd_t *pmd) int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) { - pte_t *pte = (pte_t *)pmd_page_vaddr(*pmd); + pte_t *pte = (pte_t *)pmd_page_vaddr(pmdp_get(pmd)); pmd_clear(pmd); @@ -88,7 +129,7 @@ pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, pmd_t pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp); VM_BUG_ON(address & ~HPAGE_PMD_MASK); - VM_BUG_ON(pmd_trans_huge(*pmdp)); + VM_BUG_ON(pmd_trans_huge(pmdp_get(pmdp))); /* * When leaf PTE entries (regular pages) are collapsed into a leaf * PMD entry (huge page), a valid non-leaf PTE is converted into a From f99c37d562250cbceed262723f91944c981eeb7b Mon Sep 17 00:00:00 2001 From: Gregory CLEMENT Date: Tue, 12 Dec 2023 17:34:33 +0100 Subject: [PATCH 193/336] MIPS: compressed: Use correct instruction for 64 bit code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The code clearing BSS already use macro or use correct instruction depending if the CPU is 32 bits or 64 bits. However, a few instructions remained 32 bits only. By using the accurate MACRO, it is now possible to deal with memory address beyond 32 bits. As a side effect, when using 64bits processor, it also divides the loop number needed to clear the BSS by 2. Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Florian Fainelli Signed-off-by: Gregory CLEMENT Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/compressed/head.S | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/mips/boot/compressed/head.S b/arch/mips/boot/compressed/head.S index 5795d0af1e1b..d237a834b85e 100644 --- a/arch/mips/boot/compressed/head.S +++ b/arch/mips/boot/compressed/head.S @@ -25,8 +25,8 @@ /* Clear BSS */ PTR_LA a0, _edata PTR_LA a2, _end -1: sw zero, 0(a0) - addiu a0, a0, 4 +1: PTR_S zero, 0(a0) + PTR_ADDIU a0, a0, PTRSIZE bne a2, a0, 1b PTR_LA a0, (.heap) /* heap address */ From 0d0a3748a2cb38f9da1f08d357688ebd982eb788 Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Sat, 2 Dec 2023 14:14:18 +0300 Subject: [PATCH 194/336] mips: dmi: Fix early remap on MIPS32 dmi_early_remap() has been defined as ioremap_cache() which on MIPS32 gets to be converted to the VM-based mapping. DMI early remapping is performed at the setup_arch() stage with no VM available. So calling the dmi_early_remap() for MIPS32 causes the system to crash at the early boot time. Fix that by converting dmi_early_remap() to the uncached remapping which is always available on both 32 and 64-bits MIPS systems. Note this change shall not cause any regressions on the current DMI support implementation because on the early boot-up stage neither MIPS32 nor MIPS64 has the cacheable ioremapping support anyway. Fixes: be8fa1cb444c ("MIPS: Add support for Desktop Management Interface (DMI)") Signed-off-by: Serge Semin Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/dmi.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/mips/include/asm/dmi.h b/arch/mips/include/asm/dmi.h index 27415a288adf..dc397f630c66 100644 --- a/arch/mips/include/asm/dmi.h +++ b/arch/mips/include/asm/dmi.h @@ -5,7 +5,7 @@ #include #include -#define dmi_early_remap(x, l) ioremap_cache(x, l) +#define dmi_early_remap(x, l) ioremap(x, l) #define dmi_early_unmap(x, l) iounmap(x) #define dmi_remap(x, l) ioremap_cache(x, l) #define dmi_unmap(x) iounmap(x) From 0f5cc249ff73552d3bd864e62f85841dafaa107d Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Sat, 2 Dec 2023 14:14:19 +0300 Subject: [PATCH 195/336] mips: Fix incorrect max_low_pfn adjustment max_low_pfn variable is incorrectly adjusted if the kernel is built with high memory support and the later is detected in a running system, so the memory which actually can be directly mapped is getting into the highmem zone. See the ZONE_NORMAL range on my MIPS32r5 system: > Zone ranges: > DMA [mem 0x0000000000000000-0x0000000000ffffff] > Normal [mem 0x0000000001000000-0x0000000007ffffff] > HighMem [mem 0x0000000008000000-0x000000020fffffff] while the zones are supposed to look as follows: > Zone ranges: > DMA [mem 0x0000000000000000-0x0000000000ffffff] > Normal [mem 0x0000000001000000-0x000000001fffffff] > HighMem [mem 0x0000000020000000-0x000000020fffffff] Even though the physical memory within the range [0x08000000;0x20000000] belongs to MMIO on our system, we don't really want it to be considered as high memory since on MIPS32 that range still can be directly mapped. Note there might be other problems caused by the max_low_pfn variable misconfiguration. For instance high_memory variable is initialize with virtual address corresponding to the max_low_pfn PFN, and by design it must define the upper bound on direct map memory, then end of the normal zone. That in its turn potentially may cause problems in accessing the memory by means of the /dev/mem and /dev/kmem devices. Let's fix the discovered misconfiguration then. It turns out the commit a94e4f24ec83 ("MIPS: init: Drop boot_mem_map") didn't introduce the max_low_pfn adjustment quite correct. If the kernel is built with high memory support and the system is equipped with high memory, the max_low_pfn variable will need to be initialized with PFN of the most upper directly reachable memory address so the zone normal would be correctly setup. On MIPS that PFN corresponds to PFN_DOWN(HIGHMEM_START). If the system is built with no high memory support and one is detected in the running system, we'll just need to adjust the max_pfn variable to discard the found high memory from the system and leave the max_low_pfn as is, since the later will be less than PFN_DOWN(HIGHMEM_START) anyway by design of the for_each_memblock() loop performed a bit early in the bootmem_init() method. Fixes: a94e4f24ec83 ("MIPS: init: Drop boot_mem_map") Signed-off-by: Serge Semin Signed-off-by: Thomas Bogendoerfer --- arch/mips/kernel/setup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c index 2d2ca024bd47..0461ab49e8f1 100644 --- a/arch/mips/kernel/setup.c +++ b/arch/mips/kernel/setup.c @@ -321,11 +321,11 @@ static void __init bootmem_init(void) panic("Incorrect memory mapping !!!"); if (max_pfn > PFN_DOWN(HIGHMEM_START)) { + max_low_pfn = PFN_DOWN(HIGHMEM_START); #ifdef CONFIG_HIGHMEM - highstart_pfn = PFN_DOWN(HIGHMEM_START); + highstart_pfn = max_low_pfn; highend_pfn = max_pfn; #else - max_low_pfn = PFN_DOWN(HIGHMEM_START); max_pfn = max_low_pfn; #endif } From e1a9ae45736989c972a8d1c151bc390678ae6205 Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Sat, 2 Dec 2023 14:14:20 +0300 Subject: [PATCH 196/336] mips: Fix max_mapnr being uninitialized on early stages max_mapnr variable is utilized in the pfn_valid() method in order to determine the upper PFN space boundary. Having it uninitialized effectively makes any PFN passed to that method invalid. That in its turn causes the kernel mm-subsystem occasion malfunctions even after the max_mapnr variable is actually properly updated. For instance, pfn_valid() is called in the init_unavailable_range() method in the framework of the calls-chain on MIPS: setup_arch() +-> paging_init() +-> free_area_init() +-> memmap_init() +-> memmap_init_zone_range() +-> init_unavailable_range() Since pfn_valid() always returns "false" value before max_mapnr is initialized in the mem_init() method, any flatmem page-holes will be left in the poisoned/uninitialized state including the IO-memory pages. Thus any further attempts to map/remap the IO-memory by using MMU may fail. In particular it happened in my case on attempt to map the SRAM region. The kernel bootup procedure just crashed on the unhandled unaligned access bug raised in the __update_cache() method: > Unhandled kernel unaligned access[#1]: > CPU: 0 PID: 1 Comm: swapper/0 Not tainted 6.7.0-rc1-XXX-dirty #2056 > ... > Call Trace: > [<8011ef9c>] __update_cache+0x88/0x1bc > [<80385944>] ioremap_page_range+0x110/0x2a4 > [<80126948>] ioremap_prot+0x17c/0x1f4 > [<80711b80>] __devm_ioremap+0x8c/0x120 > [<80711e0c>] __devm_ioremap_resource+0xf4/0x218 > [<808bf244>] sram_probe+0x4f4/0x930 > [<80889d20>] platform_probe+0x68/0xec > ... Let's fix the problem by initializing the max_mapnr variable as soon as the required data is available. In particular it can be done right in the paging_init() method before free_area_init() is called since all the PFN zone boundaries have already been calculated by that time. Cc: stable@vger.kernel.org Signed-off-by: Serge Semin Signed-off-by: Thomas Bogendoerfer --- arch/mips/mm/init.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c index 5dcb525a8995..6e368a4658b5 100644 --- a/arch/mips/mm/init.c +++ b/arch/mips/mm/init.c @@ -422,7 +422,12 @@ void __init paging_init(void) (highend_pfn - max_low_pfn) << (PAGE_SHIFT - 10)); max_zone_pfns[ZONE_HIGHMEM] = max_low_pfn; } + + max_mapnr = highend_pfn ? highend_pfn : max_low_pfn; +#else + max_mapnr = max_low_pfn; #endif + high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT); free_area_init(max_zone_pfns); } @@ -458,13 +463,6 @@ void __init mem_init(void) */ BUILD_BUG_ON(IS_ENABLED(CONFIG_32BIT) && (PFN_PTE_SHIFT > PAGE_SHIFT)); -#ifdef CONFIG_HIGHMEM - max_mapnr = highend_pfn ? highend_pfn : max_low_pfn; -#else - max_mapnr = max_low_pfn; -#endif - high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT); - maar_init(); memblock_free_all(); setup_zero_pages(); /* Setup zeroed pages. */ From 1c0150229f6a658687c245dfc4dcfa3fae69df49 Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Sat, 2 Dec 2023 14:14:21 +0300 Subject: [PATCH 197/336] mips: Optimize max_mapnr init procedure max_mapnr defines the upper boundary of the pages space in the system. Currently in case if HIGHMEM is available it's calculated based on the upper high memory PFN limit value. Seeing there is a case when it isn't fully correct let's optimize out the max_mapnr variable initialization procedure to cover all the handled in the paging_init() method cases: 1. If CPU has DC-aliases, then high memory is unavailable so the PFNs upper boundary is determined by max_low_pfn. 2. Otherwise if high memory is available, use highend_pfn value representing the upper high memory PFNs limit. 3. Otherwise no high memory is available so set max_mapnr with the low-memory upper limit. Signed-off-by: Serge Semin Signed-off-by: Thomas Bogendoerfer --- arch/mips/mm/init.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c index 6e368a4658b5..b2dce07116e8 100644 --- a/arch/mips/mm/init.c +++ b/arch/mips/mm/init.c @@ -421,9 +421,13 @@ void __init paging_init(void) " %ldk highmem ignored\n", (highend_pfn - max_low_pfn) << (PAGE_SHIFT - 10)); max_zone_pfns[ZONE_HIGHMEM] = max_low_pfn; - } - max_mapnr = highend_pfn ? highend_pfn : max_low_pfn; + max_mapnr = max_low_pfn; + } else if (highend_pfn) { + max_mapnr = highend_pfn; + } else { + max_mapnr = max_low_pfn; + } #else max_mapnr = max_low_pfn; #endif From e540b8c5da04c66ff610d3bf84a7566d9f6bffcf Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Sat, 2 Dec 2023 14:14:22 +0300 Subject: [PATCH 198/336] mips: mm: add slab availability checking in ioremap_prot Recent commit a5f616483110 ("mm/ioremap: add slab availability checking in ioremap_prot") added the slab availability check to the generic ioremap_prot() implementation. It is reasonable to be done for the MIPS32-specific method too since it also relies on the get_vm_area_caller() function (by means of get_vm_area()) which requires the slab allocator being up and running before being called. Signed-off-by: Serge Semin Signed-off-by: Thomas Bogendoerfer --- arch/mips/mm/ioremap.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/mips/mm/ioremap.c b/arch/mips/mm/ioremap.c index b6dad2fd5575..d8243d61ef32 100644 --- a/arch/mips/mm/ioremap.c +++ b/arch/mips/mm/ioremap.c @@ -72,6 +72,10 @@ void __iomem *ioremap_prot(phys_addr_t phys_addr, unsigned long size, flags == _CACHE_UNCACHED) return (void __iomem *) CKSEG1ADDR(phys_addr); + /* Early remaps should use the unmapped regions til' VM is available */ + if (WARN_ON_ONCE(!slab_is_available())) + return NULL; + /* * Don't allow anybody to remap RAM that may be allocated by the page * allocator, since that could lead to races & data clobbering. From efe8ee1a8b9a89835dcbbec8cebc9d5a27428914 Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Sat, 2 Dec 2023 14:14:23 +0300 Subject: [PATCH 199/336] mips: Set dump-stack arch description In the framework of the MIPS architecture the mips_set_machine_name() method is defined to set the machine name. The name currently is only used in the /proc/cpuinfo file content generation. Let's have it utilized to mach-personalize the dump-stack data too in a way it's done on ARM, ARM64, RISC-V, etc. Signed-off-by: Serge Semin Signed-off-by: Thomas Bogendoerfer --- arch/mips/kernel/prom.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/mips/kernel/prom.c b/arch/mips/kernel/prom.c index f88ce78e13e3..6062e6fa589a 100644 --- a/arch/mips/kernel/prom.c +++ b/arch/mips/kernel/prom.c @@ -28,6 +28,8 @@ __init void mips_set_machine_name(const char *name) strscpy(mips_machine_name, name, sizeof(mips_machine_name)); pr_info("MIPS: machine is %s\n", mips_get_machine_name()); + + dump_stack_set_arch_desc(name); } char *mips_get_machine_name(void) From 7b95382f965133ef61ce44aaabc518c16eb46909 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sun, 17 Dec 2023 11:15:09 +0000 Subject: [PATCH 200/336] KVM: arm64: vgic-v4: Restore pending state on host userspace write When the VMM writes to ISPENDR0 to set the state pending state of an SGI, we fail to convey this to the HW if this SGI is already backed by a GICv4.1 vSGI. This is a bit of a corner case, as this would only occur if the vgic state is changed on an already running VM, but this can apparently happen across a guest reset driven by the VMM. Fix this by always writing out the pending_latch value to the HW, and reseting it to false. Reported-by: Kunkun Jiang Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Cc: stable@vger.kernel.org # 5.10+ Link: https://lore.kernel.org/r/7e7f2c0c-448b-10a9-8929-4b8f4f6e2a32@huawei.com --- arch/arm64/kvm/vgic/vgic-mmio-v3.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-mmio-v3.c b/arch/arm64/kvm/vgic/vgic-mmio-v3.c index 89117ba2528a..111bd7f42729 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio-v3.c +++ b/arch/arm64/kvm/vgic/vgic-mmio-v3.c @@ -365,19 +365,26 @@ static int vgic_v3_uaccess_write_pending(struct kvm_vcpu *vcpu, struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); raw_spin_lock_irqsave(&irq->irq_lock, flags); - if (test_bit(i, &val)) { - /* - * pending_latch is set irrespective of irq type - * (level or edge) to avoid dependency that VM should - * restore irq config before pending info. - */ - irq->pending_latch = true; - vgic_queue_irq_unlock(vcpu->kvm, irq, flags); - } else { + + /* + * pending_latch is set irrespective of irq type + * (level or edge) to avoid dependency that VM should + * restore irq config before pending info. + */ + irq->pending_latch = test_bit(i, &val); + + if (irq->hw && vgic_irq_is_sgi(irq->intid)) { + irq_set_irqchip_state(irq->host_irq, + IRQCHIP_STATE_PENDING, + irq->pending_latch); irq->pending_latch = false; - raw_spin_unlock_irqrestore(&irq->irq_lock, flags); } + if (irq->pending_latch) + vgic_queue_irq_unlock(vcpu->kvm, irq, flags); + else + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); + vgic_put_irq(vcpu->kvm, irq); } From 13886f34444596e6eca124677cd8362a941b585b Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Tue, 19 Dec 2023 06:58:53 +0000 Subject: [PATCH 201/336] KVM: arm64: vgic: Use common accessor for writes to ISPENDR Perhaps unsurprisingly, there is a considerable amount of duplicate code between the MMIO and user accessors for ISPENDR. At the same time there are some important differences between user and guest MMIO, like how SGIs can only be made pending from userspace. Fold user and MMIO accessors into a common helper, maintaining the distinction between the two. Signed-off-by: Oliver Upton Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20231219065855.1019608-2-oliver.upton@linux.dev --- arch/arm64/kvm/vgic/vgic-mmio.c | 50 ++++++++++++++------------------- 1 file changed, 21 insertions(+), 29 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-mmio.c b/arch/arm64/kvm/vgic/vgic-mmio.c index ff558c05e990..273912083056 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio.c +++ b/arch/arm64/kvm/vgic/vgic-mmio.c @@ -301,9 +301,8 @@ static bool is_vgic_v2_sgi(struct kvm_vcpu *vcpu, struct vgic_irq *irq) vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2); } -void vgic_mmio_write_spending(struct kvm_vcpu *vcpu, - gpa_t addr, unsigned int len, - unsigned long val) +static void __set_pending(struct kvm_vcpu *vcpu, gpa_t addr, unsigned int len, + unsigned long val, bool is_user) { u32 intid = VGIC_ADDR_TO_INTID(addr, 1); int i; @@ -312,14 +311,22 @@ void vgic_mmio_write_spending(struct kvm_vcpu *vcpu, for_each_set_bit(i, &val, len * 8) { struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); - /* GICD_ISPENDR0 SGI bits are WI */ - if (is_vgic_v2_sgi(vcpu, irq)) { + /* GICD_ISPENDR0 SGI bits are WI when written from the guest. */ + if (is_vgic_v2_sgi(vcpu, irq) && !is_user) { vgic_put_irq(vcpu->kvm, irq); continue; } raw_spin_lock_irqsave(&irq->irq_lock, flags); + /* + * GICv2 SGIs are terribly broken. We can't restore + * the source of the interrupt, so just pick the vcpu + * itself as the source... + */ + if (is_vgic_v2_sgi(vcpu, irq)) + irq->source |= BIT(vcpu->vcpu_id); + if (irq->hw && vgic_irq_is_sgi(irq->intid)) { /* HW SGI? Ask the GIC to inject it */ int err; @@ -335,7 +342,7 @@ void vgic_mmio_write_spending(struct kvm_vcpu *vcpu, } irq->pending_latch = true; - if (irq->hw) + if (irq->hw && !is_user) vgic_irq_set_phys_active(irq, true); vgic_queue_irq_unlock(vcpu->kvm, irq, flags); @@ -343,33 +350,18 @@ void vgic_mmio_write_spending(struct kvm_vcpu *vcpu, } } +void vgic_mmio_write_spending(struct kvm_vcpu *vcpu, + gpa_t addr, unsigned int len, + unsigned long val) +{ + __set_pending(vcpu, addr, len, val, false); +} + int vgic_uaccess_write_spending(struct kvm_vcpu *vcpu, gpa_t addr, unsigned int len, unsigned long val) { - u32 intid = VGIC_ADDR_TO_INTID(addr, 1); - int i; - unsigned long flags; - - for_each_set_bit(i, &val, len * 8) { - struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); - - raw_spin_lock_irqsave(&irq->irq_lock, flags); - irq->pending_latch = true; - - /* - * GICv2 SGIs are terribly broken. We can't restore - * the source of the interrupt, so just pick the vcpu - * itself as the source... - */ - if (is_vgic_v2_sgi(vcpu, irq)) - irq->source |= BIT(vcpu->vcpu_id); - - vgic_queue_irq_unlock(vcpu->kvm, irq, flags); - - vgic_put_irq(vcpu->kvm, irq); - } - + __set_pending(vcpu, addr, len, val, true); return 0; } From 561851424d93e91083df4071781b68dc4ba1fc5a Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Tue, 19 Dec 2023 06:58:54 +0000 Subject: [PATCH 202/336] KVM: arm64: vgic: Use common accessor for writes to ICPENDR Fold MMIO and user accessors into a common helper while maintaining the distinction between the two. Signed-off-by: Oliver Upton Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20231219065855.1019608-3-oliver.upton@linux.dev --- arch/arm64/kvm/vgic/vgic-mmio.c | 51 ++++++++++++++------------------- 1 file changed, 22 insertions(+), 29 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-mmio.c b/arch/arm64/kvm/vgic/vgic-mmio.c index 273912083056..cf76523a2194 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio.c +++ b/arch/arm64/kvm/vgic/vgic-mmio.c @@ -386,9 +386,9 @@ static void vgic_hw_irq_cpending(struct kvm_vcpu *vcpu, struct vgic_irq *irq) vgic_irq_set_phys_active(irq, false); } -void vgic_mmio_write_cpending(struct kvm_vcpu *vcpu, - gpa_t addr, unsigned int len, - unsigned long val) +static void __clear_pending(struct kvm_vcpu *vcpu, + gpa_t addr, unsigned int len, + unsigned long val, bool is_user) { u32 intid = VGIC_ADDR_TO_INTID(addr, 1); int i; @@ -397,14 +397,22 @@ void vgic_mmio_write_cpending(struct kvm_vcpu *vcpu, for_each_set_bit(i, &val, len * 8) { struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); - /* GICD_ICPENDR0 SGI bits are WI */ - if (is_vgic_v2_sgi(vcpu, irq)) { + /* GICD_ICPENDR0 SGI bits are WI when written from the guest. */ + if (is_vgic_v2_sgi(vcpu, irq) && !is_user) { vgic_put_irq(vcpu->kvm, irq); continue; } raw_spin_lock_irqsave(&irq->irq_lock, flags); + /* + * More fun with GICv2 SGIs! If we're clearing one of them + * from userspace, which source vcpu to clear? Let's not + * even think of it, and blow the whole set. + */ + if (is_vgic_v2_sgi(vcpu, irq)) + irq->source = 0; + if (irq->hw && vgic_irq_is_sgi(irq->intid)) { /* HW SGI? Ask the GIC to clear its pending bit */ int err; @@ -419,7 +427,7 @@ void vgic_mmio_write_cpending(struct kvm_vcpu *vcpu, continue; } - if (irq->hw) + if (irq->hw && !is_user) vgic_hw_irq_cpending(vcpu, irq); else irq->pending_latch = false; @@ -429,33 +437,18 @@ void vgic_mmio_write_cpending(struct kvm_vcpu *vcpu, } } +void vgic_mmio_write_cpending(struct kvm_vcpu *vcpu, + gpa_t addr, unsigned int len, + unsigned long val) +{ + __clear_pending(vcpu, addr, len, val, false); +} + int vgic_uaccess_write_cpending(struct kvm_vcpu *vcpu, gpa_t addr, unsigned int len, unsigned long val) { - u32 intid = VGIC_ADDR_TO_INTID(addr, 1); - int i; - unsigned long flags; - - for_each_set_bit(i, &val, len * 8) { - struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); - - raw_spin_lock_irqsave(&irq->irq_lock, flags); - /* - * More fun with GICv2 SGIs! If we're clearing one of them - * from userspace, which source vcpu to clear? Let's not - * even think of it, and blow the whole set. - */ - if (is_vgic_v2_sgi(vcpu, irq)) - irq->source = 0; - - irq->pending_latch = false; - - raw_spin_unlock_irqrestore(&irq->irq_lock, flags); - - vgic_put_irq(vcpu->kvm, irq); - } - + __clear_pending(vcpu, addr, len, val, true); return 0; } From 39084ba8d0fceb477a264e2bb8dfd3553876b84c Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Tue, 19 Dec 2023 06:58:55 +0000 Subject: [PATCH 203/336] KVM: arm64: vgic-v3: Reinterpret user ISPENDR writes as I{C,S}PENDR User writes to ISPENDR for GICv3 are treated specially, as zeroes actually clear the pending state for interrupts (unlike HW). Reimplement it using the ISPENDR and ICPENDR user accessors. Signed-off-by: Oliver Upton Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20231219065855.1019608-4-oliver.upton@linux.dev --- arch/arm64/kvm/vgic/vgic-mmio-v3.c | 35 +++++------------------------- 1 file changed, 5 insertions(+), 30 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-mmio-v3.c b/arch/arm64/kvm/vgic/vgic-mmio-v3.c index 111bd7f42729..2962ccd8013a 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio-v3.c +++ b/arch/arm64/kvm/vgic/vgic-mmio-v3.c @@ -357,38 +357,13 @@ static int vgic_v3_uaccess_write_pending(struct kvm_vcpu *vcpu, gpa_t addr, unsigned int len, unsigned long val) { - u32 intid = VGIC_ADDR_TO_INTID(addr, 1); - int i; - unsigned long flags; + int ret; - for (i = 0; i < len * 8; i++) { - struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); + ret = vgic_uaccess_write_spending(vcpu, addr, len, val); + if (ret) + return ret; - raw_spin_lock_irqsave(&irq->irq_lock, flags); - - /* - * pending_latch is set irrespective of irq type - * (level or edge) to avoid dependency that VM should - * restore irq config before pending info. - */ - irq->pending_latch = test_bit(i, &val); - - if (irq->hw && vgic_irq_is_sgi(irq->intid)) { - irq_set_irqchip_state(irq->host_irq, - IRQCHIP_STATE_PENDING, - irq->pending_latch); - irq->pending_latch = false; - } - - if (irq->pending_latch) - vgic_queue_irq_unlock(vcpu->kvm, irq, flags); - else - raw_spin_unlock_irqrestore(&irq->irq_lock, flags); - - vgic_put_irq(vcpu->kvm, irq); - } - - return 0; + return vgic_uaccess_write_cpending(vcpu, addr, len, ~val); } /* We want to avoid outer shareable. */ From 2731d605d5478052a10ac5a7c80f7aa7e1788cc5 Mon Sep 17 00:00:00 2001 From: Nina Schoetterl-Glausch Date: Tue, 19 Dec 2023 15:08:50 +0100 Subject: [PATCH 204/336] KVM: s390: vsie: Fix STFLE interpretive execution identification STFLE can be interpretively executed. This occurs when the facility list designation is unequal to zero. Perform the check before applying the address mask instead of after. Fixes: 66b630d5b7f2 ("KVM: s390: vsie: support STFLE interpretation") Reviewed-by: Claudio Imbrenda Acked-by: David Hildenbrand Signed-off-by: Nina Schoetterl-Glausch Reviewed-by: Christian Borntraeger Reviewed-by: Janosch Frank Link: https://lore.kernel.org/r/20231219140854.1042599-2-nsg@linux.ibm.com Signed-off-by: Janosch Frank Message-ID: <20231219140854.1042599-2-nsg@linux.ibm.com> --- arch/s390/kvm/vsie.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c index 02dcbe82a8e5..3cf95bc0401d 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/vsie.c @@ -988,10 +988,15 @@ static void retry_vsie_icpt(struct vsie_page *vsie_page) static int handle_stfle(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) { struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; - __u32 fac = READ_ONCE(vsie_page->scb_o->fac) & 0x7ffffff8U; + __u32 fac = READ_ONCE(vsie_page->scb_o->fac); if (fac && test_kvm_facility(vcpu->kvm, 7)) { retry_vsie_icpt(vsie_page); + /* + * The facility list origin (FLO) is in bits 1 - 28 of the FLD + * so we need to mask here before reading. + */ + fac = fac & 0x7ffffff8U; if (read_guest_real(vcpu, fac, &vsie_page->fac, sizeof(vsie_page->fac))) return set_validity_icpt(scb_s, 0x1090U); From 682dbf430d27bc0e23d8d6921116b4f77f5dc9c6 Mon Sep 17 00:00:00 2001 From: Nina Schoetterl-Glausch Date: Tue, 19 Dec 2023 15:08:51 +0100 Subject: [PATCH 205/336] KVM: s390: vsie: Fix length of facility list shadowed The length of the facility list accessed when interpretively executing STFLE is the same as the hosts facility list (in case of format-0) The memory following the facility list doesn't need to be accessible. The current VSIE implementation accesses a fixed length that exceeds the guest/host facility list length and can therefore wrongly inject a validity intercept. Instead, find out the host facility list length by running STFLE and copy only as much as necessary when shadowing. Acked-by: David Hildenbrand Reviewed-by: Claudio Imbrenda Acked-by: Heiko Carstens Signed-off-by: Nina Schoetterl-Glausch Reviewed-by: Janosch Frank Link: https://lore.kernel.org/r/20231219140854.1042599-3-nsg@linux.ibm.com Signed-off-by: Janosch Frank Message-ID: <20231219140854.1042599-3-nsg@linux.ibm.com> --- arch/s390/include/asm/facility.h | 6 ++++++ arch/s390/kernel/Makefile | 2 +- arch/s390/kernel/facility.c | 21 +++++++++++++++++++++ arch/s390/kvm/vsie.c | 12 +++++++++++- 4 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 arch/s390/kernel/facility.c diff --git a/arch/s390/include/asm/facility.h b/arch/s390/include/asm/facility.h index 94b6919026df..796007125dff 100644 --- a/arch/s390/include/asm/facility.h +++ b/arch/s390/include/asm/facility.h @@ -111,4 +111,10 @@ static inline void stfle(u64 *stfle_fac_list, int size) preempt_enable(); } +/** + * stfle_size - Actual size of the facility list as specified by stfle + * (number of double words) + */ +unsigned int stfle_size(void); + #endif /* __ASM_FACILITY_H */ diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 353def93973b..7a562b4199c8 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -41,7 +41,7 @@ obj-y += sysinfo.o lgr.o os_info.o ctlreg.o obj-y += runtime_instr.o cache.o fpu.o dumpstack.o guarded_storage.o sthyi.o obj-y += entry.o reipl.o kdebugfs.o alternative.o obj-y += nospec-branch.o ipl_vmparm.o machine_kexec_reloc.o unwind_bc.o -obj-y += smp.o text_amode31.o stacktrace.o abs_lowcore.o +obj-y += smp.o text_amode31.o stacktrace.o abs_lowcore.o facility.o extra-y += vmlinux.lds diff --git a/arch/s390/kernel/facility.c b/arch/s390/kernel/facility.c new file mode 100644 index 000000000000..f02127219a27 --- /dev/null +++ b/arch/s390/kernel/facility.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright IBM Corp. 2023 + */ + +#include + +unsigned int stfle_size(void) +{ + static unsigned int size; + unsigned int r; + u64 dummy; + + r = READ_ONCE(size); + if (!r) { + r = __stfle_asm(&dummy, 1) + 1; + WRITE_ONCE(size, r); + } + return r; +} +EXPORT_SYMBOL(stfle_size); diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c index 3cf95bc0401d..aa8f4ab11e33 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/vsie.c @@ -19,6 +19,7 @@ #include #include #include +#include #include "kvm-s390.h" #include "gaccess.h" @@ -990,6 +991,10 @@ static int handle_stfle(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; __u32 fac = READ_ONCE(vsie_page->scb_o->fac); + /* + * Alternate-STFLE-Interpretive-Execution facilities are not supported + * -> format-0 flcb + */ if (fac && test_kvm_facility(vcpu->kvm, 7)) { retry_vsie_icpt(vsie_page); /* @@ -997,8 +1002,13 @@ static int handle_stfle(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) * so we need to mask here before reading. */ fac = fac & 0x7ffffff8U; + /* + * format-0 -> size of nested guest's facility list == guest's size + * guest's size == host's size, since STFLE is interpretatively executed + * using a format-0 for the guest, too. + */ if (read_guest_real(vcpu, fac, &vsie_page->fac, - sizeof(vsie_page->fac))) + stfle_size() * sizeof(u64))) return set_validity_icpt(scb_s, 0x1090U); scb_s->fac = (__u32)(__u64) &vsie_page->fac; } From 10f7b1dcdfe05efcd26e90e337daf1bfd8f4a6da Mon Sep 17 00:00:00 2001 From: Nina Schoetterl-Glausch Date: Tue, 19 Dec 2023 15:08:52 +0100 Subject: [PATCH 206/336] KVM: s390: cpu model: Use proper define for facility mask size Use the previously unused S390_ARCH_FAC_MASK_SIZE_U64 instead of S390_ARCH_FAC_LIST_SIZE_U64 for defining the fac_mask array. Note that both values are the same, there is no functional change. Reviewed-by: Claudio Imbrenda Reviewed-by: David Hildenbrand Reviewed-by: Janosch Frank Signed-off-by: Nina Schoetterl-Glausch Link: https://lore.kernel.org/r/20231219140854.1042599-4-nsg@linux.ibm.com Signed-off-by: Janosch Frank Message-ID: <20231219140854.1042599-4-nsg@linux.ibm.com> --- arch/s390/include/asm/kvm_host.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index 67a298b6cf6e..52664105a473 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -818,7 +818,7 @@ struct s390_io_adapter { struct kvm_s390_cpu_model { /* facility mask supported by kvm & hosting machine */ - __u64 fac_mask[S390_ARCH_FAC_LIST_SIZE_U64]; + __u64 fac_mask[S390_ARCH_FAC_MASK_SIZE_U64]; struct kvm_s390_vm_cpu_subfunc subfuncs; /* facility list requested by guest (in dma page) */ __u64 *fac_list; From 683c5bbbf6aea247bc95a7eb9fdfba4fcc8c909a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Tue, 24 Oct 2023 15:26:54 +0200 Subject: [PATCH 207/336] riscv: kvm: Use SYM_*() assembly macros instead of deprecated ones MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ENTRY()/END()/WEAK() macros are deprecated and we should make use of the new SYM_*() macros [1] for better annotation of symbols. Replace the deprecated ones with the new ones and fix wrong usage of END()/ENDPROC() to correctly describe the symbols. [1] https://docs.kernel.org/core-api/asm-annotations.html Signed-off-by: Clément Léger Reviewed-by: Andrew Jones Acked-by: Palmer Dabbelt Signed-off-by: Anup Patel --- arch/riscv/kvm/vcpu_switch.S | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/arch/riscv/kvm/vcpu_switch.S b/arch/riscv/kvm/vcpu_switch.S index d74df8eb4d71..8b18473780ac 100644 --- a/arch/riscv/kvm/vcpu_switch.S +++ b/arch/riscv/kvm/vcpu_switch.S @@ -15,7 +15,7 @@ .altmacro .option norelax -ENTRY(__kvm_riscv_switch_to) +SYM_FUNC_START(__kvm_riscv_switch_to) /* Save Host GPRs (except A0 and T0-T6) */ REG_S ra, (KVM_ARCH_HOST_RA)(a0) REG_S sp, (KVM_ARCH_HOST_SP)(a0) @@ -208,9 +208,9 @@ __kvm_switch_return: /* Return to C code */ ret -ENDPROC(__kvm_riscv_switch_to) +SYM_FUNC_END(__kvm_riscv_switch_to) -ENTRY(__kvm_riscv_unpriv_trap) +SYM_CODE_START(__kvm_riscv_unpriv_trap) /* * We assume that faulting unpriv load/store instruction is * 4-byte long and blindly increment SEPC by 4. @@ -231,12 +231,10 @@ ENTRY(__kvm_riscv_unpriv_trap) csrr a1, CSR_HTINST REG_S a1, (KVM_ARCH_TRAP_HTINST)(a0) sret -ENDPROC(__kvm_riscv_unpriv_trap) +SYM_CODE_END(__kvm_riscv_unpriv_trap) #ifdef CONFIG_FPU - .align 3 - .global __kvm_riscv_fp_f_save -__kvm_riscv_fp_f_save: +SYM_FUNC_START(__kvm_riscv_fp_f_save) csrr t2, CSR_SSTATUS li t1, SR_FS csrs CSR_SSTATUS, t1 @@ -276,10 +274,9 @@ __kvm_riscv_fp_f_save: sw t0, KVM_ARCH_FP_F_FCSR(a0) csrw CSR_SSTATUS, t2 ret +SYM_FUNC_END(__kvm_riscv_fp_f_save) - .align 3 - .global __kvm_riscv_fp_d_save -__kvm_riscv_fp_d_save: +SYM_FUNC_START(__kvm_riscv_fp_d_save) csrr t2, CSR_SSTATUS li t1, SR_FS csrs CSR_SSTATUS, t1 @@ -319,10 +316,9 @@ __kvm_riscv_fp_d_save: sw t0, KVM_ARCH_FP_D_FCSR(a0) csrw CSR_SSTATUS, t2 ret +SYM_FUNC_END(__kvm_riscv_fp_d_save) - .align 3 - .global __kvm_riscv_fp_f_restore -__kvm_riscv_fp_f_restore: +SYM_FUNC_START(__kvm_riscv_fp_f_restore) csrr t2, CSR_SSTATUS li t1, SR_FS lw t0, KVM_ARCH_FP_F_FCSR(a0) @@ -362,10 +358,9 @@ __kvm_riscv_fp_f_restore: fscsr t0 csrw CSR_SSTATUS, t2 ret +SYM_FUNC_END(__kvm_riscv_fp_f_restore) - .align 3 - .global __kvm_riscv_fp_d_restore -__kvm_riscv_fp_d_restore: +SYM_FUNC_START(__kvm_riscv_fp_d_restore) csrr t2, CSR_SSTATUS li t1, SR_FS lw t0, KVM_ARCH_FP_D_FCSR(a0) @@ -405,4 +400,5 @@ __kvm_riscv_fp_d_restore: fscsr t0 csrw CSR_SSTATUS, t2 ret +SYM_FUNC_END(__kvm_riscv_fp_d_restore) #endif From e5ff012743cbc3cf13d2243aaaf032a2ca4d0791 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Tue, 24 Oct 2023 15:26:55 +0200 Subject: [PATCH 208/336] riscv: kvm: use ".L" local labels in assembly when applicable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For the sake of coherency, use local labels in assembly when applicable. This also avoid kprobes being confused when applying a kprobe since the size of function is computed by checking where the next visible symbol is located. This might end up in computing some function size to be way shorter than expected and thus failing to apply kprobes to the specified offset. Signed-off-by: Clément Léger Reviewed-by: Andrew Jones Acked-by: Palmer Dabbelt Signed-off-by: Anup Patel --- arch/riscv/kvm/vcpu_switch.S | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/riscv/kvm/vcpu_switch.S b/arch/riscv/kvm/vcpu_switch.S index 8b18473780ac..0c26189aa01c 100644 --- a/arch/riscv/kvm/vcpu_switch.S +++ b/arch/riscv/kvm/vcpu_switch.S @@ -45,7 +45,7 @@ SYM_FUNC_START(__kvm_riscv_switch_to) REG_L t0, (KVM_ARCH_GUEST_SSTATUS)(a0) REG_L t1, (KVM_ARCH_GUEST_HSTATUS)(a0) REG_L t2, (KVM_ARCH_GUEST_SCOUNTEREN)(a0) - la t4, __kvm_switch_return + la t4, .Lkvm_switch_return REG_L t5, (KVM_ARCH_GUEST_SEPC)(a0) /* Save Host and Restore Guest SSTATUS */ @@ -113,7 +113,7 @@ SYM_FUNC_START(__kvm_riscv_switch_to) /* Back to Host */ .align 2 -__kvm_switch_return: +.Lkvm_switch_return: /* Swap Guest A0 with SSCRATCH */ csrrw a0, CSR_SSCRATCH, a0 From bcd08e9bae57b5585e438b7fa58aba4b145a59cf Mon Sep 17 00:00:00 2001 From: Chao Du Date: Mon, 11 Dec 2023 09:40:14 +0000 Subject: [PATCH 209/336] RISC-V: KVM: remove a redundant condition in kvm_arch_vcpu_ioctl_run() The latest ret value is updated by kvm_riscv_vcpu_aia_update(), the loop will continue if the ret is less than or equal to zero. So the later condition will never hit. Thus remove it. Signed-off-by: Chao Du Reviewed-by: Daniel Henrique Barboza Signed-off-by: Anup Patel --- arch/riscv/kvm/vcpu.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index e087c809073c..bf3952d1a621 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -757,8 +757,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) /* Update HVIP CSR for current CPU */ kvm_riscv_update_hvip(vcpu); - if (ret <= 0 || - kvm_riscv_gstage_vmid_ver_changed(&vcpu->kvm->arch.vmid) || + if (kvm_riscv_gstage_vmid_ver_changed(&vcpu->kvm->arch.vmid) || kvm_request_pending(vcpu) || xfer_to_guest_mode_work_pending()) { vcpu->mode = OUTSIDE_GUEST_MODE; From c19829ba1e4d119e69b1ac9a96d5a0b86f7233e9 Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Tue, 28 Nov 2023 20:23:43 +0530 Subject: [PATCH 210/336] KVM: riscv: selftests: Generate ISA extension reg_list using macros Various ISA extension reg_list have common pattern so let us generate these using macros. We define two macros for the above purpose: 1) KVM_ISA_EXT_SIMPLE_CONFIG - Macro to generate reg_list for ISA extension without any additional ONE_REG registers 2) KVM_ISA_EXT_SUBLIST_CONFIG - Macro to generate reg_list for ISA extension with additional ONE_REG registers This patch also adds the missing config for svnapot. Signed-off-by: Anup Patel Reviewed-by: Andrew Jones Signed-off-by: Anup Patel --- .../selftests/kvm/riscv/get-reg-list.c | 325 ++++-------------- 1 file changed, 73 insertions(+), 252 deletions(-) diff --git a/tools/testing/selftests/kvm/riscv/get-reg-list.c b/tools/testing/selftests/kvm/riscv/get-reg-list.c index 25de4b8bc347..5d86c761784e 100644 --- a/tools/testing/selftests/kvm/riscv/get-reg-list.c +++ b/tools/testing/selftests/kvm/riscv/get-reg-list.c @@ -583,10 +583,6 @@ static __u64 base_skips_set[] = { KVM_REG_RISCV | KVM_REG_SIZE_U64 | KVM_REG_RISCV_TIMER | KVM_REG_RISCV_TIMER_REG(state), }; -static __u64 h_regs[] = { - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_H, -}; - static __u64 zicbom_regs[] = { KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_CONFIG | KVM_REG_RISCV_CONFIG_REG(zicbom_block_size), KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZICBOM, @@ -597,54 +593,6 @@ static __u64 zicboz_regs[] = { KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZICBOZ, }; -static __u64 svpbmt_regs[] = { - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_SVPBMT, -}; - -static __u64 sstc_regs[] = { - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_SSTC, -}; - -static __u64 svinval_regs[] = { - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_SVINVAL, -}; - -static __u64 zihintpause_regs[] = { - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZIHINTPAUSE, -}; - -static __u64 zba_regs[] = { - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZBA, -}; - -static __u64 zbb_regs[] = { - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZBB, -}; - -static __u64 zbs_regs[] = { - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZBS, -}; - -static __u64 zicntr_regs[] = { - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZICNTR, -}; - -static __u64 zicond_regs[] = { - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZICOND, -}; - -static __u64 zicsr_regs[] = { - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZICSR, -}; - -static __u64 zifencei_regs[] = { - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZIFENCEI, -}; - -static __u64 zihpm_regs[] = { - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZIHPM, -}; - static __u64 aia_regs[] = { KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_CSR | KVM_REG_RISCV_CSR_AIA | KVM_REG_RISCV_CSR_AIA_REG(siselect), KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_CSR | KVM_REG_RISCV_CSR_AIA | KVM_REG_RISCV_CSR_AIA_REG(iprio1), @@ -735,221 +683,94 @@ static __u64 fp_d_regs[] = { KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_D, }; -#define BASE_SUBLIST \ +#define SUBLIST_BASE \ {"base", .regs = base_regs, .regs_n = ARRAY_SIZE(base_regs), \ .skips_set = base_skips_set, .skips_set_n = ARRAY_SIZE(base_skips_set),} -#define H_REGS_SUBLIST \ - {"h", .feature = KVM_RISCV_ISA_EXT_H, .regs = h_regs, .regs_n = ARRAY_SIZE(h_regs),} -#define ZICBOM_REGS_SUBLIST \ +#define SUBLIST_ZICBOM \ {"zicbom", .feature = KVM_RISCV_ISA_EXT_ZICBOM, .regs = zicbom_regs, .regs_n = ARRAY_SIZE(zicbom_regs),} -#define ZICBOZ_REGS_SUBLIST \ +#define SUBLIST_ZICBOZ \ {"zicboz", .feature = KVM_RISCV_ISA_EXT_ZICBOZ, .regs = zicboz_regs, .regs_n = ARRAY_SIZE(zicboz_regs),} -#define SVPBMT_REGS_SUBLIST \ - {"svpbmt", .feature = KVM_RISCV_ISA_EXT_SVPBMT, .regs = svpbmt_regs, .regs_n = ARRAY_SIZE(svpbmt_regs),} -#define SSTC_REGS_SUBLIST \ - {"sstc", .feature = KVM_RISCV_ISA_EXT_SSTC, .regs = sstc_regs, .regs_n = ARRAY_SIZE(sstc_regs),} -#define SVINVAL_REGS_SUBLIST \ - {"svinval", .feature = KVM_RISCV_ISA_EXT_SVINVAL, .regs = svinval_regs, .regs_n = ARRAY_SIZE(svinval_regs),} -#define ZIHINTPAUSE_REGS_SUBLIST \ - {"zihintpause", .feature = KVM_RISCV_ISA_EXT_ZIHINTPAUSE, .regs = zihintpause_regs, .regs_n = ARRAY_SIZE(zihintpause_regs),} -#define ZBA_REGS_SUBLIST \ - {"zba", .feature = KVM_RISCV_ISA_EXT_ZBA, .regs = zba_regs, .regs_n = ARRAY_SIZE(zba_regs),} -#define ZBB_REGS_SUBLIST \ - {"zbb", .feature = KVM_RISCV_ISA_EXT_ZBB, .regs = zbb_regs, .regs_n = ARRAY_SIZE(zbb_regs),} -#define ZBS_REGS_SUBLIST \ - {"zbs", .feature = KVM_RISCV_ISA_EXT_ZBS, .regs = zbs_regs, .regs_n = ARRAY_SIZE(zbs_regs),} -#define ZICNTR_REGS_SUBLIST \ - {"zicntr", .feature = KVM_RISCV_ISA_EXT_ZICNTR, .regs = zicntr_regs, .regs_n = ARRAY_SIZE(zicntr_regs),} -#define ZICOND_REGS_SUBLIST \ - {"zicond", .feature = KVM_RISCV_ISA_EXT_ZICOND, .regs = zicond_regs, .regs_n = ARRAY_SIZE(zicond_regs),} -#define ZICSR_REGS_SUBLIST \ - {"zicsr", .feature = KVM_RISCV_ISA_EXT_ZICSR, .regs = zicsr_regs, .regs_n = ARRAY_SIZE(zicsr_regs),} -#define ZIFENCEI_REGS_SUBLIST \ - {"zifencei", .feature = KVM_RISCV_ISA_EXT_ZIFENCEI, .regs = zifencei_regs, .regs_n = ARRAY_SIZE(zifencei_regs),} -#define ZIHPM_REGS_SUBLIST \ - {"zihpm", .feature = KVM_RISCV_ISA_EXT_ZIHPM, .regs = zihpm_regs, .regs_n = ARRAY_SIZE(zihpm_regs),} -#define AIA_REGS_SUBLIST \ +#define SUBLIST_AIA \ {"aia", .feature = KVM_RISCV_ISA_EXT_SSAIA, .regs = aia_regs, .regs_n = ARRAY_SIZE(aia_regs),} -#define SMSTATEEN_REGS_SUBLIST \ +#define SUBLIST_SMSTATEEN \ {"smstateen", .feature = KVM_RISCV_ISA_EXT_SMSTATEEN, .regs = smstateen_regs, .regs_n = ARRAY_SIZE(smstateen_regs),} -#define FP_F_REGS_SUBLIST \ +#define SUBLIST_FP_F \ {"fp_f", .feature = KVM_RISCV_ISA_EXT_F, .regs = fp_f_regs, \ .regs_n = ARRAY_SIZE(fp_f_regs),} -#define FP_D_REGS_SUBLIST \ +#define SUBLIST_FP_D \ {"fp_d", .feature = KVM_RISCV_ISA_EXT_D, .regs = fp_d_regs, \ .regs_n = ARRAY_SIZE(fp_d_regs),} -static struct vcpu_reg_list h_config = { - .sublists = { - BASE_SUBLIST, - H_REGS_SUBLIST, - {0}, - }, -}; +#define KVM_ISA_EXT_SIMPLE_CONFIG(ext, extu) \ +static __u64 regs_##ext[] = { \ + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | \ + KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_##extu, \ +}; \ +static struct vcpu_reg_list config_##ext = { \ + .sublists = { \ + SUBLIST_BASE, \ + { \ + .name = #ext, \ + .feature = KVM_RISCV_ISA_EXT_##extu, \ + .regs = regs_##ext, \ + .regs_n = ARRAY_SIZE(regs_##ext), \ + }, \ + {0}, \ + }, \ +} \ -static struct vcpu_reg_list zicbom_config = { - .sublists = { - BASE_SUBLIST, - ZICBOM_REGS_SUBLIST, - {0}, - }, -}; +#define KVM_ISA_EXT_SUBLIST_CONFIG(ext, extu) \ +static struct vcpu_reg_list config_##ext = { \ + .sublists = { \ + SUBLIST_BASE, \ + SUBLIST_##extu, \ + {0}, \ + }, \ +} \ -static struct vcpu_reg_list zicboz_config = { - .sublists = { - BASE_SUBLIST, - ZICBOZ_REGS_SUBLIST, - {0}, - }, -}; +/* Note: The below list is alphabetically sorted. */ -static struct vcpu_reg_list svpbmt_config = { - .sublists = { - BASE_SUBLIST, - SVPBMT_REGS_SUBLIST, - {0}, - }, -}; - -static struct vcpu_reg_list sstc_config = { - .sublists = { - BASE_SUBLIST, - SSTC_REGS_SUBLIST, - {0}, - }, -}; - -static struct vcpu_reg_list svinval_config = { - .sublists = { - BASE_SUBLIST, - SVINVAL_REGS_SUBLIST, - {0}, - }, -}; - -static struct vcpu_reg_list zihintpause_config = { - .sublists = { - BASE_SUBLIST, - ZIHINTPAUSE_REGS_SUBLIST, - {0}, - }, -}; - -static struct vcpu_reg_list zba_config = { - .sublists = { - BASE_SUBLIST, - ZBA_REGS_SUBLIST, - {0}, - }, -}; - -static struct vcpu_reg_list zbb_config = { - .sublists = { - BASE_SUBLIST, - ZBB_REGS_SUBLIST, - {0}, - }, -}; - -static struct vcpu_reg_list zbs_config = { - .sublists = { - BASE_SUBLIST, - ZBS_REGS_SUBLIST, - {0}, - }, -}; - -static struct vcpu_reg_list zicntr_config = { - .sublists = { - BASE_SUBLIST, - ZICNTR_REGS_SUBLIST, - {0}, - }, -}; - -static struct vcpu_reg_list zicond_config = { - .sublists = { - BASE_SUBLIST, - ZICOND_REGS_SUBLIST, - {0}, - }, -}; - -static struct vcpu_reg_list zicsr_config = { - .sublists = { - BASE_SUBLIST, - ZICSR_REGS_SUBLIST, - {0}, - }, -}; - -static struct vcpu_reg_list zifencei_config = { - .sublists = { - BASE_SUBLIST, - ZIFENCEI_REGS_SUBLIST, - {0}, - }, -}; - -static struct vcpu_reg_list zihpm_config = { - .sublists = { - BASE_SUBLIST, - ZIHPM_REGS_SUBLIST, - {0}, - }, -}; - -static struct vcpu_reg_list aia_config = { - .sublists = { - BASE_SUBLIST, - AIA_REGS_SUBLIST, - {0}, - }, -}; - -static struct vcpu_reg_list smstateen_config = { - .sublists = { - BASE_SUBLIST, - SMSTATEEN_REGS_SUBLIST, - {0}, - }, -}; - -static struct vcpu_reg_list fp_f_config = { - .sublists = { - BASE_SUBLIST, - FP_F_REGS_SUBLIST, - {0}, - }, -}; - -static struct vcpu_reg_list fp_d_config = { - .sublists = { - BASE_SUBLIST, - FP_D_REGS_SUBLIST, - {0}, - }, -}; +KVM_ISA_EXT_SUBLIST_CONFIG(aia, AIA); +KVM_ISA_EXT_SUBLIST_CONFIG(fp_f, FP_F); +KVM_ISA_EXT_SUBLIST_CONFIG(fp_d, FP_D); +KVM_ISA_EXT_SIMPLE_CONFIG(h, H); +KVM_ISA_EXT_SUBLIST_CONFIG(smstateen, SMSTATEEN); +KVM_ISA_EXT_SIMPLE_CONFIG(sstc, SSTC); +KVM_ISA_EXT_SIMPLE_CONFIG(svinval, SVINVAL); +KVM_ISA_EXT_SIMPLE_CONFIG(svnapot, SVNAPOT); +KVM_ISA_EXT_SIMPLE_CONFIG(svpbmt, SVPBMT); +KVM_ISA_EXT_SIMPLE_CONFIG(zba, ZBA); +KVM_ISA_EXT_SIMPLE_CONFIG(zbb, ZBB); +KVM_ISA_EXT_SIMPLE_CONFIG(zbs, ZBS); +KVM_ISA_EXT_SUBLIST_CONFIG(zicbom, ZICBOM); +KVM_ISA_EXT_SUBLIST_CONFIG(zicboz, ZICBOZ); +KVM_ISA_EXT_SIMPLE_CONFIG(zicntr, ZICNTR); +KVM_ISA_EXT_SIMPLE_CONFIG(zicond, ZICOND); +KVM_ISA_EXT_SIMPLE_CONFIG(zicsr, ZICSR); +KVM_ISA_EXT_SIMPLE_CONFIG(zifencei, ZIFENCEI); +KVM_ISA_EXT_SIMPLE_CONFIG(zihintpause, ZIHINTPAUSE); +KVM_ISA_EXT_SIMPLE_CONFIG(zihpm, ZIHPM); struct vcpu_reg_list *vcpu_configs[] = { - &h_config, - &zicbom_config, - &zicboz_config, - &svpbmt_config, - &sstc_config, - &svinval_config, - &zihintpause_config, - &zba_config, - &zbb_config, - &zbs_config, - &zicntr_config, - &zicond_config, - &zicsr_config, - &zifencei_config, - &zihpm_config, - &aia_config, - &smstateen_config, - &fp_f_config, - &fp_d_config, + &config_aia, + &config_fp_f, + &config_fp_d, + &config_h, + &config_smstateen, + &config_sstc, + &config_svinval, + &config_svnapot, + &config_svpbmt, + &config_zba, + &config_zbb, + &config_zbs, + &config_zicbom, + &config_zicboz, + &config_zicntr, + &config_zicond, + &config_zicsr, + &config_zifencei, + &config_zihintpause, + &config_zihpm, }; int vcpu_configs_n = ARRAY_SIZE(vcpu_configs); From 7f58de96aa5e871dd553499e2c84fc801658eab6 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Wed, 13 Dec 2023 18:09:53 +0100 Subject: [PATCH 211/336] RISC-V: KVM: Don't add SBI multi regs in get-reg-list The multi regs are derived from the single registers. Only list the single registers in get-reg-list. This also makes the SBI extension register listing consistent with the ISA extension register listing. Signed-off-by: Andrew Jones Reviewed-by: Haibo Xu Reviewed-by: Anup Patel Signed-off-by: Anup Patel --- arch/riscv/kvm/vcpu_onereg.c | 36 ++---------------------------------- 1 file changed, 2 insertions(+), 34 deletions(-) diff --git a/arch/riscv/kvm/vcpu_onereg.c b/arch/riscv/kvm/vcpu_onereg.c index f8c9fa0c03c5..f9bfa8a5db21 100644 --- a/arch/riscv/kvm/vcpu_onereg.c +++ b/arch/riscv/kvm/vcpu_onereg.c @@ -933,20 +933,12 @@ static inline unsigned long num_isa_ext_regs(const struct kvm_vcpu *vcpu) static inline unsigned long num_sbi_ext_regs(void) { - /* - * number of KVM_REG_RISCV_SBI_SINGLE + - * 2 x (number of KVM_REG_RISCV_SBI_MULTI) - */ - return KVM_RISCV_SBI_EXT_MAX + 2*(KVM_REG_RISCV_SBI_MULTI_REG_LAST+1); + return KVM_RISCV_SBI_EXT_MAX; } static int copy_sbi_ext_reg_indices(u64 __user *uindices) { - int n; - - /* copy KVM_REG_RISCV_SBI_SINGLE */ - n = KVM_RISCV_SBI_EXT_MAX; - for (int i = 0; i < n; i++) { + for (int i = 0; i < KVM_RISCV_SBI_EXT_MAX; i++) { u64 size = IS_ENABLED(CONFIG_32BIT) ? KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_SBI_EXT | @@ -959,30 +951,6 @@ static int copy_sbi_ext_reg_indices(u64 __user *uindices) } } - /* copy KVM_REG_RISCV_SBI_MULTI */ - n = KVM_REG_RISCV_SBI_MULTI_REG_LAST + 1; - for (int i = 0; i < n; i++) { - u64 size = IS_ENABLED(CONFIG_32BIT) ? - KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; - u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_SBI_EXT | - KVM_REG_RISCV_SBI_MULTI_EN | i; - - if (uindices) { - if (put_user(reg, uindices)) - return -EFAULT; - uindices++; - } - - reg = KVM_REG_RISCV | size | KVM_REG_RISCV_SBI_EXT | - KVM_REG_RISCV_SBI_MULTI_DIS | i; - - if (uindices) { - if (put_user(reg, uindices)) - return -EFAULT; - uindices++; - } - } - return num_sbi_ext_regs(); } From 7602730d7f18ad9738d8fc5e5fd7f52a11fee399 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Wed, 13 Dec 2023 18:09:54 +0100 Subject: [PATCH 212/336] KVM: riscv: selftests: Drop SBI multi registers These registers are no longer getting added to get-reg-list. We keep sbi_ext_multi_id_to_str() for printing, even though we don't expect it to normally be used, because it may be useful for debug. Signed-off-by: Andrew Jones Reviewed-by: Anup Patel Signed-off-by: Anup Patel --- tools/testing/selftests/kvm/riscv/get-reg-list.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/testing/selftests/kvm/riscv/get-reg-list.c b/tools/testing/selftests/kvm/riscv/get-reg-list.c index 5d86c761784e..27d07a32a1ef 100644 --- a/tools/testing/selftests/kvm/riscv/get-reg-list.c +++ b/tools/testing/selftests/kvm/riscv/get-reg-list.c @@ -571,8 +571,6 @@ static __u64 base_regs[] = { KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_EXPERIMENTAL, KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_VENDOR, KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_DBCN, - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_MULTI_EN | 0, - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_MULTI_DIS | 0, }; /* From 23e1dc45022eb65529aa30b1851a8d21a639c8f5 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Wed, 13 Dec 2023 18:09:55 +0100 Subject: [PATCH 213/336] RISC-V: KVM: Make SBI uapi consistent with ISA uapi When an SBI extension cannot be enabled, that's a distinct state vs. enabled and disabled. Modify enum kvm_riscv_sbi_ext_status to accommodate it, which allows KVM userspace to tell the difference in state too, as the SBI extension register will disappear when it cannot be enabled, i.e. accesses to it return ENOENT. get-reg-list is updated as well to only add SBI extension registers to the list which may be enabled. Returning ENOENT for SBI extension registers which cannot be enabled makes them consistent with ISA extension registers. Any SBI extensions which were enabled by default are still enabled by default, if they can be enabled at all. Signed-off-by: Andrew Jones Reviewed-by: Anup Patel Signed-off-by: Anup Patel --- arch/riscv/include/asm/kvm_vcpu_sbi.h | 10 ++-- arch/riscv/kvm/vcpu_onereg.c | 23 +++++--- arch/riscv/kvm/vcpu_sbi.c | 75 +++++++++++++++------------ arch/riscv/kvm/vcpu_sbi_replace.c | 2 +- 4 files changed, 65 insertions(+), 45 deletions(-) diff --git a/arch/riscv/include/asm/kvm_vcpu_sbi.h b/arch/riscv/include/asm/kvm_vcpu_sbi.h index 6a453f7f8b56..bffda0ac59b6 100644 --- a/arch/riscv/include/asm/kvm_vcpu_sbi.h +++ b/arch/riscv/include/asm/kvm_vcpu_sbi.h @@ -15,9 +15,10 @@ #define KVM_SBI_VERSION_MINOR 0 enum kvm_riscv_sbi_ext_status { - KVM_RISCV_SBI_EXT_UNINITIALIZED, - KVM_RISCV_SBI_EXT_AVAILABLE, - KVM_RISCV_SBI_EXT_UNAVAILABLE, + KVM_RISCV_SBI_EXT_STATUS_UNINITIALIZED, + KVM_RISCV_SBI_EXT_STATUS_UNAVAILABLE, + KVM_RISCV_SBI_EXT_STATUS_ENABLED, + KVM_RISCV_SBI_EXT_STATUS_DISABLED, }; struct kvm_vcpu_sbi_context { @@ -36,7 +37,7 @@ struct kvm_vcpu_sbi_extension { unsigned long extid_start; unsigned long extid_end; - bool default_unavail; + bool default_disabled; /** * SBI extension handler. It can be defined for a given extension or group of @@ -61,6 +62,7 @@ int kvm_riscv_vcpu_get_reg_sbi_ext(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); const struct kvm_vcpu_sbi_extension *kvm_vcpu_sbi_find_ext( struct kvm_vcpu *vcpu, unsigned long extid); +bool riscv_vcpu_supports_sbi_ext(struct kvm_vcpu *vcpu, int idx); int kvm_riscv_vcpu_sbi_ecall(struct kvm_vcpu *vcpu, struct kvm_run *run); void kvm_riscv_vcpu_sbi_init(struct kvm_vcpu *vcpu); diff --git a/arch/riscv/kvm/vcpu_onereg.c b/arch/riscv/kvm/vcpu_onereg.c index f9bfa8a5db21..48262be73aa0 100644 --- a/arch/riscv/kvm/vcpu_onereg.c +++ b/arch/riscv/kvm/vcpu_onereg.c @@ -931,27 +931,34 @@ static inline unsigned long num_isa_ext_regs(const struct kvm_vcpu *vcpu) return copy_isa_ext_reg_indices(vcpu, NULL);; } -static inline unsigned long num_sbi_ext_regs(void) +static int copy_sbi_ext_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) { - return KVM_RISCV_SBI_EXT_MAX; -} + unsigned int n = 0; -static int copy_sbi_ext_reg_indices(u64 __user *uindices) -{ for (int i = 0; i < KVM_RISCV_SBI_EXT_MAX; i++) { u64 size = IS_ENABLED(CONFIG_32BIT) ? KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | i; + if (!riscv_vcpu_supports_sbi_ext(vcpu, i)) + continue; + if (uindices) { if (put_user(reg, uindices)) return -EFAULT; uindices++; } + + n++; } - return num_sbi_ext_regs(); + return n; +} + +static unsigned long num_sbi_ext_regs(struct kvm_vcpu *vcpu) +{ + return copy_sbi_ext_reg_indices(vcpu, NULL); } /* @@ -970,7 +977,7 @@ unsigned long kvm_riscv_vcpu_num_regs(struct kvm_vcpu *vcpu) res += num_fp_f_regs(vcpu); res += num_fp_d_regs(vcpu); res += num_isa_ext_regs(vcpu); - res += num_sbi_ext_regs(); + res += num_sbi_ext_regs(vcpu); return res; } @@ -1018,7 +1025,7 @@ int kvm_riscv_vcpu_copy_reg_indices(struct kvm_vcpu *vcpu, return ret; uindices += ret; - ret = copy_sbi_ext_reg_indices(uindices); + ret = copy_sbi_ext_reg_indices(vcpu, uindices); if (ret < 0) return ret; diff --git a/arch/riscv/kvm/vcpu_sbi.c b/arch/riscv/kvm/vcpu_sbi.c index a04ff98085d9..dcdff4458190 100644 --- a/arch/riscv/kvm/vcpu_sbi.c +++ b/arch/riscv/kvm/vcpu_sbi.c @@ -80,6 +80,34 @@ static const struct kvm_riscv_sbi_extension_entry sbi_ext[] = { }, }; +static const struct kvm_riscv_sbi_extension_entry * +riscv_vcpu_get_sbi_ext(struct kvm_vcpu *vcpu, unsigned long idx) +{ + const struct kvm_riscv_sbi_extension_entry *sext = NULL; + + if (idx >= KVM_RISCV_SBI_EXT_MAX) + return NULL; + + for (int i = 0; i < ARRAY_SIZE(sbi_ext); i++) { + if (sbi_ext[i].ext_idx == idx) { + sext = &sbi_ext[i]; + break; + } + } + + return sext; +} + +bool riscv_vcpu_supports_sbi_ext(struct kvm_vcpu *vcpu, int idx) +{ + struct kvm_vcpu_sbi_context *scontext = &vcpu->arch.sbi_context; + const struct kvm_riscv_sbi_extension_entry *sext; + + sext = riscv_vcpu_get_sbi_ext(vcpu, idx); + + return sext && scontext->ext_status[sext->ext_idx] != KVM_RISCV_SBI_EXT_STATUS_UNAVAILABLE; +} + void kvm_riscv_vcpu_sbi_forward(struct kvm_vcpu *vcpu, struct kvm_run *run) { struct kvm_cpu_context *cp = &vcpu->arch.guest_context; @@ -140,28 +168,19 @@ static int riscv_vcpu_set_sbi_ext_single(struct kvm_vcpu *vcpu, unsigned long reg_num, unsigned long reg_val) { - unsigned long i; - const struct kvm_riscv_sbi_extension_entry *sext = NULL; struct kvm_vcpu_sbi_context *scontext = &vcpu->arch.sbi_context; - - if (reg_num >= KVM_RISCV_SBI_EXT_MAX) - return -ENOENT; + const struct kvm_riscv_sbi_extension_entry *sext; if (reg_val != 1 && reg_val != 0) return -EINVAL; - for (i = 0; i < ARRAY_SIZE(sbi_ext); i++) { - if (sbi_ext[i].ext_idx == reg_num) { - sext = &sbi_ext[i]; - break; - } - } - if (!sext) + sext = riscv_vcpu_get_sbi_ext(vcpu, reg_num); + if (!sext || scontext->ext_status[sext->ext_idx] == KVM_RISCV_SBI_EXT_STATUS_UNAVAILABLE) return -ENOENT; scontext->ext_status[sext->ext_idx] = (reg_val) ? - KVM_RISCV_SBI_EXT_AVAILABLE : - KVM_RISCV_SBI_EXT_UNAVAILABLE; + KVM_RISCV_SBI_EXT_STATUS_ENABLED : + KVM_RISCV_SBI_EXT_STATUS_DISABLED; return 0; } @@ -170,24 +189,16 @@ static int riscv_vcpu_get_sbi_ext_single(struct kvm_vcpu *vcpu, unsigned long reg_num, unsigned long *reg_val) { - unsigned long i; - const struct kvm_riscv_sbi_extension_entry *sext = NULL; struct kvm_vcpu_sbi_context *scontext = &vcpu->arch.sbi_context; + const struct kvm_riscv_sbi_extension_entry *sext; - if (reg_num >= KVM_RISCV_SBI_EXT_MAX) - return -ENOENT; - - for (i = 0; i < ARRAY_SIZE(sbi_ext); i++) { - if (sbi_ext[i].ext_idx == reg_num) { - sext = &sbi_ext[i]; - break; - } - } - if (!sext) + sext = riscv_vcpu_get_sbi_ext(vcpu, reg_num); + if (!sext || scontext->ext_status[sext->ext_idx] == KVM_RISCV_SBI_EXT_STATUS_UNAVAILABLE) return -ENOENT; *reg_val = scontext->ext_status[sext->ext_idx] == - KVM_RISCV_SBI_EXT_AVAILABLE; + KVM_RISCV_SBI_EXT_STATUS_ENABLED; + return 0; } @@ -325,7 +336,7 @@ const struct kvm_vcpu_sbi_extension *kvm_vcpu_sbi_find_ext( if (ext->extid_start <= extid && ext->extid_end >= extid) { if (entry->ext_idx >= KVM_RISCV_SBI_EXT_MAX || scontext->ext_status[entry->ext_idx] == - KVM_RISCV_SBI_EXT_AVAILABLE) + KVM_RISCV_SBI_EXT_STATUS_ENABLED) return ext; return NULL; @@ -413,12 +424,12 @@ void kvm_riscv_vcpu_sbi_init(struct kvm_vcpu *vcpu) if (ext->probe && !ext->probe(vcpu)) { scontext->ext_status[entry->ext_idx] = - KVM_RISCV_SBI_EXT_UNAVAILABLE; + KVM_RISCV_SBI_EXT_STATUS_UNAVAILABLE; continue; } - scontext->ext_status[entry->ext_idx] = ext->default_unavail ? - KVM_RISCV_SBI_EXT_UNAVAILABLE : - KVM_RISCV_SBI_EXT_AVAILABLE; + scontext->ext_status[entry->ext_idx] = ext->default_disabled ? + KVM_RISCV_SBI_EXT_STATUS_DISABLED : + KVM_RISCV_SBI_EXT_STATUS_ENABLED; } } diff --git a/arch/riscv/kvm/vcpu_sbi_replace.c b/arch/riscv/kvm/vcpu_sbi_replace.c index 23b57c931b15..9c2ab3dfa93a 100644 --- a/arch/riscv/kvm/vcpu_sbi_replace.c +++ b/arch/riscv/kvm/vcpu_sbi_replace.c @@ -204,6 +204,6 @@ static int kvm_sbi_ext_dbcn_handler(struct kvm_vcpu *vcpu, const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_dbcn = { .extid_start = SBI_EXT_DBCN, .extid_end = SBI_EXT_DBCN, - .default_unavail = true, + .default_disabled = true, .handler = kvm_sbi_ext_dbcn_handler, }; From 6ccf119a4cc886678099a3526f37db98b67024d7 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Wed, 13 Dec 2023 18:09:56 +0100 Subject: [PATCH 214/336] KVM: riscv: selftests: Add RISCV_SBI_EXT_REG While adding RISCV_SBI_EXT_REG(), acknowledge that some registers have subtypes and extend __kvm_reg_id() to take a subtype field. Then, update all macros to set the new field appropriately. The general CSR macro gets renamed to include "GENERAL", but the other macros, like the new RISCV_SBI_EXT_REG, just use the SINGLE subtype. Signed-off-by: Andrew Jones Reviewed-by: Anup Patel Signed-off-by: Anup Patel --- .../selftests/kvm/include/riscv/processor.h | 40 +++++++++++-------- .../selftests/kvm/lib/riscv/processor.c | 4 +- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/tools/testing/selftests/kvm/include/riscv/processor.h b/tools/testing/selftests/kvm/include/riscv/processor.h index 5b62a3d2aa9b..e70ccda2011b 100644 --- a/tools/testing/selftests/kvm/include/riscv/processor.h +++ b/tools/testing/selftests/kvm/include/riscv/processor.h @@ -10,10 +10,10 @@ #include "kvm_util.h" #include -static inline uint64_t __kvm_reg_id(uint64_t type, uint64_t idx, - uint64_t size) +static inline uint64_t __kvm_reg_id(uint64_t type, uint64_t subtype, + uint64_t idx, uint64_t size) { - return KVM_REG_RISCV | type | idx | size; + return KVM_REG_RISCV | type | subtype | idx | size; } #if __riscv_xlen == 64 @@ -22,24 +22,30 @@ static inline uint64_t __kvm_reg_id(uint64_t type, uint64_t idx, #define KVM_REG_SIZE_ULONG KVM_REG_SIZE_U32 #endif -#define RISCV_CONFIG_REG(name) __kvm_reg_id(KVM_REG_RISCV_CONFIG, \ - KVM_REG_RISCV_CONFIG_REG(name), \ - KVM_REG_SIZE_ULONG) +#define RISCV_CONFIG_REG(name) __kvm_reg_id(KVM_REG_RISCV_CONFIG, 0, \ + KVM_REG_RISCV_CONFIG_REG(name), \ + KVM_REG_SIZE_ULONG) -#define RISCV_CORE_REG(name) __kvm_reg_id(KVM_REG_RISCV_CORE, \ - KVM_REG_RISCV_CORE_REG(name), \ - KVM_REG_SIZE_ULONG) +#define RISCV_CORE_REG(name) __kvm_reg_id(KVM_REG_RISCV_CORE, 0, \ + KVM_REG_RISCV_CORE_REG(name), \ + KVM_REG_SIZE_ULONG) -#define RISCV_CSR_REG(name) __kvm_reg_id(KVM_REG_RISCV_CSR, \ - KVM_REG_RISCV_CSR_REG(name), \ - KVM_REG_SIZE_ULONG) +#define RISCV_GENERAL_CSR_REG(name) __kvm_reg_id(KVM_REG_RISCV_CSR, \ + KVM_REG_RISCV_CSR_GENERAL, \ + KVM_REG_RISCV_CSR_REG(name), \ + KVM_REG_SIZE_ULONG) -#define RISCV_TIMER_REG(name) __kvm_reg_id(KVM_REG_RISCV_TIMER, \ - KVM_REG_RISCV_TIMER_REG(name), \ - KVM_REG_SIZE_U64) +#define RISCV_TIMER_REG(name) __kvm_reg_id(KVM_REG_RISCV_TIMER, 0, \ + KVM_REG_RISCV_TIMER_REG(name), \ + KVM_REG_SIZE_U64) -#define RISCV_ISA_EXT_REG(idx) __kvm_reg_id(KVM_REG_RISCV_ISA_EXT, \ - idx, KVM_REG_SIZE_ULONG) +#define RISCV_ISA_EXT_REG(idx) __kvm_reg_id(KVM_REG_RISCV_ISA_EXT, \ + KVM_REG_RISCV_ISA_SINGLE, \ + idx, KVM_REG_SIZE_ULONG) + +#define RISCV_SBI_EXT_REG(idx) __kvm_reg_id(KVM_REG_RISCV_SBI_EXT, \ + KVM_REG_RISCV_SBI_SINGLE, \ + idx, KVM_REG_SIZE_ULONG) /* L3 index Bit[47:39] */ #define PGTBL_L3_INDEX_MASK 0x0000FF8000000000ULL diff --git a/tools/testing/selftests/kvm/lib/riscv/processor.c b/tools/testing/selftests/kvm/lib/riscv/processor.c index d146ca71e0c0..6c25f7843ef4 100644 --- a/tools/testing/selftests/kvm/lib/riscv/processor.c +++ b/tools/testing/selftests/kvm/lib/riscv/processor.c @@ -201,7 +201,7 @@ void riscv_vcpu_mmu_setup(struct kvm_vcpu *vcpu) satp = (vm->pgd >> PGTBL_PAGE_SIZE_SHIFT) & SATP_PPN; satp |= SATP_MODE_48; - vcpu_set_reg(vcpu, RISCV_CSR_REG(satp), satp); + vcpu_set_reg(vcpu, RISCV_GENERAL_CSR_REG(satp), satp); } void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, uint8_t indent) @@ -315,7 +315,7 @@ struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id, vcpu_set_reg(vcpu, RISCV_CORE_REG(regs.pc), (unsigned long)guest_code); /* Setup default exception vector of guest */ - vcpu_set_reg(vcpu, RISCV_CSR_REG(stvec), (unsigned long)guest_unexp_trap); + vcpu_set_reg(vcpu, RISCV_GENERAL_CSR_REG(stvec), (unsigned long)guest_unexp_trap); return vcpu; } From b26e70d72d12dc9ddb276898a78b1c35c7ab9b95 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Wed, 13 Dec 2023 18:09:57 +0100 Subject: [PATCH 215/336] KVM: riscv: selftests: Use register subtypes Always use register subtypes in the get-reg-list test when registers have them. The only registers neglecting to do so were ISA extension registers. While we don't really need to use KVM_REG_RISCV_ISA_SINGLE (since it's zero), the main purpose is to avoid confusion and to self-document the tests. Also add print support for the multi registers like SBI extensions have, even though they're only used for debugging. Signed-off-by: Andrew Jones Reviewed-by: Haibo Xu Reviewed-by: Anup Patel Signed-off-by: Anup Patel --- .../selftests/kvm/riscv/get-reg-list.c | 113 +++++++++++------- 1 file changed, 73 insertions(+), 40 deletions(-) diff --git a/tools/testing/selftests/kvm/riscv/get-reg-list.c b/tools/testing/selftests/kvm/riscv/get-reg-list.c index 27d07a32a1ef..4bcc597d34b9 100644 --- a/tools/testing/selftests/kvm/riscv/get-reg-list.c +++ b/tools/testing/selftests/kvm/riscv/get-reg-list.c @@ -28,31 +28,31 @@ bool filter_reg(__u64 reg) * * Note: The below list is alphabetically sorted. */ - case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_A: - case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_C: - case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_D: - case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_F: - case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_H: - case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_I: - case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_M: - case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_V: - case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_SMSTATEEN: - case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_SSAIA: - case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_SSTC: - case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_SVINVAL: - case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_SVNAPOT: - case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_SVPBMT: - case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZBA: - case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZBB: - case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZBS: - case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZICBOM: - case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZICBOZ: - case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZICNTR: - case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZICOND: - case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZICSR: - case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZIFENCEI: - case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZIHINTPAUSE: - case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZIHPM: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_A: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_C: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_D: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_F: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_H: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_I: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_M: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_V: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SMSTATEEN: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SSAIA: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SSTC: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SVINVAL: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SVNAPOT: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SVPBMT: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZBA: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZBB: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZBS: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICBOM: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICBOZ: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICNTR: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICOND: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICSR: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZIFENCEI: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZIHINTPAUSE: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZIHPM: return true; /* AIA registers are always available when Ssaia can't be disabled */ case KVM_REG_RISCV_CSR | KVM_REG_RISCV_CSR_AIA | KVM_REG_RISCV_CSR_AIA_REG(siselect): @@ -335,15 +335,10 @@ static const char *fp_d_id_to_str(const char *prefix, __u64 id) } #define KVM_ISA_EXT_ARR(ext) \ -[KVM_RISCV_ISA_EXT_##ext] = "KVM_RISCV_ISA_EXT_" #ext +[KVM_RISCV_ISA_EXT_##ext] = "KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_" #ext -static const char *isa_ext_id_to_str(const char *prefix, __u64 id) +static const char *isa_ext_single_id_to_str(__u64 reg_off) { - /* reg_off is the offset into unsigned long kvm_isa_ext_arr[] */ - __u64 reg_off = id & ~(REG_MASK | KVM_REG_RISCV_ISA_EXT); - - assert((id & KVM_REG_RISCV_TYPE_MASK) == KVM_REG_RISCV_ISA_EXT); - static const char * const kvm_isa_ext_reg_name[] = { KVM_ISA_EXT_ARR(A), KVM_ISA_EXT_ARR(C), @@ -373,11 +368,48 @@ static const char *isa_ext_id_to_str(const char *prefix, __u64 id) }; if (reg_off >= ARRAY_SIZE(kvm_isa_ext_reg_name)) - return strdup_printf("%lld /* UNKNOWN */", reg_off); + return strdup_printf("KVM_REG_RISCV_ISA_SINGLE | %lld /* UNKNOWN */", reg_off); return kvm_isa_ext_reg_name[reg_off]; } +static const char *isa_ext_multi_id_to_str(__u64 reg_subtype, __u64 reg_off) +{ + const char *unknown = ""; + + if (reg_off > KVM_REG_RISCV_ISA_MULTI_REG_LAST) + unknown = " /* UNKNOWN */"; + + switch (reg_subtype) { + case KVM_REG_RISCV_ISA_MULTI_EN: + return strdup_printf("KVM_REG_RISCV_ISA_MULTI_EN | %lld%s", reg_off, unknown); + case KVM_REG_RISCV_ISA_MULTI_DIS: + return strdup_printf("KVM_REG_RISCV_ISA_MULTI_DIS | %lld%s", reg_off, unknown); + } + + return strdup_printf("%lld | %lld /* UNKNOWN */", reg_subtype, reg_off); +} + +static const char *isa_ext_id_to_str(const char *prefix, __u64 id) +{ + __u64 reg_off = id & ~(REG_MASK | KVM_REG_RISCV_ISA_EXT); + __u64 reg_subtype = reg_off & KVM_REG_RISCV_SUBTYPE_MASK; + + assert((id & KVM_REG_RISCV_TYPE_MASK) == KVM_REG_RISCV_ISA_EXT); + + reg_off &= ~KVM_REG_RISCV_SUBTYPE_MASK; + + switch (reg_subtype) { + case KVM_REG_RISCV_ISA_SINGLE: + return isa_ext_single_id_to_str(reg_off); + case KVM_REG_RISCV_ISA_MULTI_EN: + case KVM_REG_RISCV_ISA_MULTI_DIS: + return isa_ext_multi_id_to_str(reg_subtype, reg_off); + } + + return strdup_printf("%lld | %lld /* UNKNOWN */", reg_subtype, reg_off); +} + #define KVM_SBI_EXT_ARR(ext) \ [ext] = "KVM_REG_RISCV_SBI_SINGLE | " #ext @@ -583,12 +615,12 @@ static __u64 base_skips_set[] = { static __u64 zicbom_regs[] = { KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_CONFIG | KVM_REG_RISCV_CONFIG_REG(zicbom_block_size), - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZICBOM, + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICBOM, }; static __u64 zicboz_regs[] = { KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_CONFIG | KVM_REG_RISCV_CONFIG_REG(zicboz_block_size), - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZICBOZ, + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICBOZ, }; static __u64 aia_regs[] = { @@ -599,12 +631,12 @@ static __u64 aia_regs[] = { KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_CSR | KVM_REG_RISCV_CSR_AIA | KVM_REG_RISCV_CSR_AIA_REG(siph), KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_CSR | KVM_REG_RISCV_CSR_AIA | KVM_REG_RISCV_CSR_AIA_REG(iprio1h), KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_CSR | KVM_REG_RISCV_CSR_AIA | KVM_REG_RISCV_CSR_AIA_REG(iprio2h), - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_SSAIA, + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SSAIA, }; static __u64 smstateen_regs[] = { KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_CSR | KVM_REG_RISCV_CSR_SMSTATEEN | KVM_REG_RISCV_CSR_SMSTATEEN_REG(sstateen0), - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_SMSTATEEN, + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SMSTATEEN, }; static __u64 fp_f_regs[] = { @@ -641,7 +673,7 @@ static __u64 fp_f_regs[] = { KVM_REG_RISCV | KVM_REG_SIZE_U32 | KVM_REG_RISCV_FP_F | KVM_REG_RISCV_FP_F_REG(f[30]), KVM_REG_RISCV | KVM_REG_SIZE_U32 | KVM_REG_RISCV_FP_F | KVM_REG_RISCV_FP_F_REG(f[31]), KVM_REG_RISCV | KVM_REG_SIZE_U32 | KVM_REG_RISCV_FP_F | KVM_REG_RISCV_FP_F_REG(fcsr), - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_F, + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_F, }; static __u64 fp_d_regs[] = { @@ -678,7 +710,7 @@ static __u64 fp_d_regs[] = { KVM_REG_RISCV | KVM_REG_SIZE_U64 | KVM_REG_RISCV_FP_D | KVM_REG_RISCV_FP_D_REG(f[30]), KVM_REG_RISCV | KVM_REG_SIZE_U64 | KVM_REG_RISCV_FP_D | KVM_REG_RISCV_FP_D_REG(f[31]), KVM_REG_RISCV | KVM_REG_SIZE_U32 | KVM_REG_RISCV_FP_D | KVM_REG_RISCV_FP_D_REG(fcsr), - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_D, + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_D, }; #define SUBLIST_BASE \ @@ -702,7 +734,8 @@ static __u64 fp_d_regs[] = { #define KVM_ISA_EXT_SIMPLE_CONFIG(ext, extu) \ static __u64 regs_##ext[] = { \ KVM_REG_RISCV | KVM_REG_SIZE_ULONG | \ - KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_##extu, \ + KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | \ + KVM_RISCV_ISA_EXT_##extu, \ }; \ static struct vcpu_reg_list config_##ext = { \ .sublists = { \ From bdf6aa328f137e184b0fce607fd585354c3742f1 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Wed, 13 Dec 2023 18:09:58 +0100 Subject: [PATCH 216/336] RISC-V: KVM: selftests: Treat SBI ext regs like ISA ext regs SBI extension registers may not be present and indeed when running on a platform without sscofpmf the PMU SBI extension is not. Move the SBI extension registers from the base set of registers to the filter list. Individual configs should test for any that may or may not be present separately. Since the PMU extension may disappear and the DBCN extension is only present in later kernels, separate them from the rest into their own configs. The rest are lumped together into the same config. Signed-off-by: Andrew Jones Reviewed-by: Anup Patel Signed-off-by: Anup Patel --- .../selftests/kvm/include/kvm_util_base.h | 1 + .../selftests/kvm/riscv/get-reg-list.c | 105 +++++++++++++++--- 2 files changed, 92 insertions(+), 14 deletions(-) diff --git a/tools/testing/selftests/kvm/include/kvm_util_base.h b/tools/testing/selftests/kvm/include/kvm_util_base.h index a18db6a7b3cf..e112ee30867f 100644 --- a/tools/testing/selftests/kvm/include/kvm_util_base.h +++ b/tools/testing/selftests/kvm/include/kvm_util_base.h @@ -129,6 +129,7 @@ struct vcpu_reg_sublist { const char *name; long capability; int feature; + int feature_type; bool finalize; __u64 *regs; __u64 regs_n; diff --git a/tools/testing/selftests/kvm/riscv/get-reg-list.c b/tools/testing/selftests/kvm/riscv/get-reg-list.c index 4bcc597d34b9..b8da2e86bf9c 100644 --- a/tools/testing/selftests/kvm/riscv/get-reg-list.c +++ b/tools/testing/selftests/kvm/riscv/get-reg-list.c @@ -12,6 +12,11 @@ #define REG_MASK (KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK) +enum { + VCPU_FEATURE_ISA_EXT = 0, + VCPU_FEATURE_SBI_EXT, +}; + static bool isa_ext_cant_disable[KVM_RISCV_ISA_EXT_MAX]; bool filter_reg(__u64 reg) @@ -53,6 +58,21 @@ bool filter_reg(__u64 reg) case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZIFENCEI: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZIHINTPAUSE: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZIHPM: + /* + * Like ISA_EXT registers, SBI_EXT registers are only visible when the + * host supports them and disabling them does not affect the visibility + * of the SBI_EXT register itself. + */ + case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_V01: + case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_TIME: + case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_IPI: + case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_RFENCE: + case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_SRST: + case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_HSM: + case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_PMU: + case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_DBCN: + case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_EXPERIMENTAL: + case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_VENDOR: return true; /* AIA registers are always available when Ssaia can't be disabled */ case KVM_REG_RISCV_CSR | KVM_REG_RISCV_CSR_AIA | KVM_REG_RISCV_CSR_AIA_REG(siselect): @@ -75,12 +95,12 @@ bool check_reject_set(int err) return err == EINVAL; } -static inline bool vcpu_has_ext(struct kvm_vcpu *vcpu, int ext) +static bool vcpu_has_ext(struct kvm_vcpu *vcpu, uint64_t ext_id) { int ret; unsigned long value; - ret = __vcpu_get_reg(vcpu, RISCV_ISA_EXT_REG(ext), &value); + ret = __vcpu_get_reg(vcpu, ext_id, &value); return (ret) ? false : !!value; } @@ -88,6 +108,7 @@ void finalize_vcpu(struct kvm_vcpu *vcpu, struct vcpu_reg_list *c) { unsigned long isa_ext_state[KVM_RISCV_ISA_EXT_MAX] = { 0 }; struct vcpu_reg_sublist *s; + uint64_t feature; int rc; for (int i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++) @@ -103,15 +124,31 @@ void finalize_vcpu(struct kvm_vcpu *vcpu, struct vcpu_reg_list *c) isa_ext_cant_disable[i] = true; } + for (int i = 0; i < KVM_RISCV_SBI_EXT_MAX; i++) { + rc = __vcpu_set_reg(vcpu, RISCV_SBI_EXT_REG(i), 0); + TEST_ASSERT(!rc || (rc == -1 && errno == ENOENT), "Unexpected error"); + } + for_each_sublist(c, s) { if (!s->feature) continue; + switch (s->feature_type) { + case VCPU_FEATURE_ISA_EXT: + feature = RISCV_ISA_EXT_REG(s->feature); + break; + case VCPU_FEATURE_SBI_EXT: + feature = RISCV_SBI_EXT_REG(s->feature); + break; + default: + TEST_FAIL("Unknown feature type"); + } + /* Try to enable the desired extension */ - __vcpu_set_reg(vcpu, RISCV_ISA_EXT_REG(s->feature), 1); + __vcpu_set_reg(vcpu, feature, 1); /* Double check whether the desired extension was enabled */ - __TEST_REQUIRE(vcpu_has_ext(vcpu, s->feature), + __TEST_REQUIRE(vcpu_has_ext(vcpu, feature), "%s not available, skipping tests\n", s->name); } } @@ -593,16 +630,6 @@ static __u64 base_regs[] = { KVM_REG_RISCV | KVM_REG_SIZE_U64 | KVM_REG_RISCV_TIMER | KVM_REG_RISCV_TIMER_REG(time), KVM_REG_RISCV | KVM_REG_SIZE_U64 | KVM_REG_RISCV_TIMER | KVM_REG_RISCV_TIMER_REG(compare), KVM_REG_RISCV | KVM_REG_SIZE_U64 | KVM_REG_RISCV_TIMER | KVM_REG_RISCV_TIMER_REG(state), - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_V01, - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_TIME, - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_IPI, - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_RFENCE, - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_SRST, - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_HSM, - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_PMU, - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_EXPERIMENTAL, - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_VENDOR, - KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_DBCN, }; /* @@ -613,6 +640,17 @@ static __u64 base_skips_set[] = { KVM_REG_RISCV | KVM_REG_SIZE_U64 | KVM_REG_RISCV_TIMER | KVM_REG_RISCV_TIMER_REG(state), }; +static __u64 sbi_base_regs[] = { + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_V01, + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_TIME, + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_IPI, + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_RFENCE, + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_SRST, + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_HSM, + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_EXPERIMENTAL, + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_VENDOR, +}; + static __u64 zicbom_regs[] = { KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_CONFIG | KVM_REG_RISCV_CONFIG_REG(zicbom_block_size), KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICBOM, @@ -716,6 +754,9 @@ static __u64 fp_d_regs[] = { #define SUBLIST_BASE \ {"base", .regs = base_regs, .regs_n = ARRAY_SIZE(base_regs), \ .skips_set = base_skips_set, .skips_set_n = ARRAY_SIZE(base_skips_set),} +#define SUBLIST_SBI_BASE \ + {"sbi-base", .feature_type = VCPU_FEATURE_SBI_EXT, .feature = KVM_RISCV_SBI_EXT_V01, \ + .regs = sbi_base_regs, .regs_n = ARRAY_SIZE(sbi_base_regs),} #define SUBLIST_ZICBOM \ {"zicbom", .feature = KVM_RISCV_ISA_EXT_ZICBOM, .regs = zicbom_regs, .regs_n = ARRAY_SIZE(zicbom_regs),} #define SUBLIST_ZICBOZ \ @@ -750,6 +791,26 @@ static struct vcpu_reg_list config_##ext = { \ }, \ } \ +#define KVM_SBI_EXT_SIMPLE_CONFIG(ext, extu) \ +static __u64 regs_sbi_##ext[] = { \ + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | \ + KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | \ + KVM_RISCV_SBI_EXT_##extu, \ +}; \ +static struct vcpu_reg_list config_sbi_##ext = { \ + .sublists = { \ + SUBLIST_BASE, \ + { \ + .name = "sbi-"#ext, \ + .feature_type = VCPU_FEATURE_SBI_EXT, \ + .feature = KVM_RISCV_SBI_EXT_##extu, \ + .regs = regs_sbi_##ext, \ + .regs_n = ARRAY_SIZE(regs_sbi_##ext), \ + }, \ + {0}, \ + }, \ +} \ + #define KVM_ISA_EXT_SUBLIST_CONFIG(ext, extu) \ static struct vcpu_reg_list config_##ext = { \ .sublists = { \ @@ -759,8 +820,21 @@ static struct vcpu_reg_list config_##ext = { \ }, \ } \ +#define KVM_SBI_EXT_SUBLIST_CONFIG(ext, extu) \ +static struct vcpu_reg_list config_sbi_##ext = { \ + .sublists = { \ + SUBLIST_BASE, \ + SUBLIST_SBI_##extu, \ + {0}, \ + }, \ +} \ + /* Note: The below list is alphabetically sorted. */ +KVM_SBI_EXT_SUBLIST_CONFIG(base, BASE); +KVM_SBI_EXT_SIMPLE_CONFIG(pmu, PMU); +KVM_SBI_EXT_SIMPLE_CONFIG(dbcn, DBCN); + KVM_ISA_EXT_SUBLIST_CONFIG(aia, AIA); KVM_ISA_EXT_SUBLIST_CONFIG(fp_f, FP_F); KVM_ISA_EXT_SUBLIST_CONFIG(fp_d, FP_D); @@ -783,6 +857,9 @@ KVM_ISA_EXT_SIMPLE_CONFIG(zihintpause, ZIHINTPAUSE); KVM_ISA_EXT_SIMPLE_CONFIG(zihpm, ZIHPM); struct vcpu_reg_list *vcpu_configs[] = { + &config_sbi_base, + &config_sbi_pmu, + &config_sbi_dbcn, &config_aia, &config_fp_f, &config_fp_d, From 197bd237b67268651ac544e8fedbe1fd275d41e0 Mon Sep 17 00:00:00 2001 From: Daniel Henrique Barboza Date: Tue, 5 Dec 2023 14:45:07 -0300 Subject: [PATCH 217/336] RISC-V: KVM: set 'vlenb' in kvm_riscv_vcpu_alloc_vector_context() 'vlenb', added to riscv_v_ext_state by commit c35f3aa34509 ("RISC-V: vector: export VLENB csr in __sc_riscv_v_state"), isn't being initialized in guest_context. If we export 'vlenb' as a KVM CSR, something we want to do in the next patch, it'll always return 0. Set 'vlenb' to riscv_v_size/32. Signed-off-by: Daniel Henrique Barboza Reviewed-by: Anup Patel Signed-off-by: Anup Patel --- arch/riscv/kvm/vcpu_vector.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/riscv/kvm/vcpu_vector.c b/arch/riscv/kvm/vcpu_vector.c index b339a2682f25..530e49c588d6 100644 --- a/arch/riscv/kvm/vcpu_vector.c +++ b/arch/riscv/kvm/vcpu_vector.c @@ -76,6 +76,7 @@ int kvm_riscv_vcpu_alloc_vector_context(struct kvm_vcpu *vcpu, cntx->vector.datap = kmalloc(riscv_v_vsize, GFP_KERNEL); if (!cntx->vector.datap) return -ENOMEM; + cntx->vector.vlenb = riscv_v_vsize / 32; vcpu->arch.host_context.vector.datap = kzalloc(riscv_v_vsize, GFP_KERNEL); if (!vcpu->arch.host_context.vector.datap) From 2fa290372dfe7dd248b1c16f943f273a3e674f22 Mon Sep 17 00:00:00 2001 From: Daniel Henrique Barboza Date: Tue, 5 Dec 2023 14:45:08 -0300 Subject: [PATCH 218/336] RISC-V: KVM: add 'vlenb' Vector CSR Userspace requires 'vlenb' to be able to encode it in reg ID. Otherwise it is not possible to retrieve any vector reg since we're returning EINVAL if reg_size isn't vlenb (see kvm_riscv_vcpu_vreg_addr()). Signed-off-by: Daniel Henrique Barboza Reviewed-by: Anup Patel Signed-off-by: Anup Patel --- arch/riscv/kvm/vcpu_vector.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/arch/riscv/kvm/vcpu_vector.c b/arch/riscv/kvm/vcpu_vector.c index 530e49c588d6..d92d1348045c 100644 --- a/arch/riscv/kvm/vcpu_vector.c +++ b/arch/riscv/kvm/vcpu_vector.c @@ -116,6 +116,9 @@ static int kvm_riscv_vcpu_vreg_addr(struct kvm_vcpu *vcpu, case KVM_REG_RISCV_VECTOR_CSR_REG(vcsr): *reg_addr = &cntx->vector.vcsr; break; + case KVM_REG_RISCV_VECTOR_CSR_REG(vlenb): + *reg_addr = &cntx->vector.vlenb; + break; case KVM_REG_RISCV_VECTOR_CSR_REG(datap): default: return -ENOENT; @@ -174,6 +177,18 @@ int kvm_riscv_vcpu_set_reg_vector(struct kvm_vcpu *vcpu, if (!riscv_isa_extension_available(isa, v)) return -ENOENT; + if (reg_num == KVM_REG_RISCV_VECTOR_CSR_REG(vlenb)) { + struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; + unsigned long reg_val; + + if (copy_from_user(®_val, uaddr, reg_size)) + return -EFAULT; + if (reg_val != cntx->vector.vlenb) + return -EINVAL; + + return 0; + } + rc = kvm_riscv_vcpu_vreg_addr(vcpu, reg_num, reg_size, ®_addr); if (rc) return rc; From 3975525e554559117bbf569239c8b41f2c2fa5cf Mon Sep 17 00:00:00 2001 From: Daniel Henrique Barboza Date: Tue, 5 Dec 2023 14:45:09 -0300 Subject: [PATCH 219/336] RISC-V: KVM: add vector registers and CSRs in KVM_GET_REG_LIST Add all vector registers and CSRs (vstart, vl, vtype, vcsr, vlenb) in get-reg-list. Signed-off-by: Daniel Henrique Barboza Reviewed-by: Anup Patel Signed-off-by: Anup Patel --- arch/riscv/kvm/vcpu_onereg.c | 55 ++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/arch/riscv/kvm/vcpu_onereg.c b/arch/riscv/kvm/vcpu_onereg.c index 48262be73aa0..11cdbf844291 100644 --- a/arch/riscv/kvm/vcpu_onereg.c +++ b/arch/riscv/kvm/vcpu_onereg.c @@ -961,6 +961,55 @@ static unsigned long num_sbi_ext_regs(struct kvm_vcpu *vcpu) return copy_sbi_ext_reg_indices(vcpu, NULL); } +static inline unsigned long num_vector_regs(const struct kvm_vcpu *vcpu) +{ + if (!riscv_isa_extension_available(vcpu->arch.isa, v)) + return 0; + + /* vstart, vl, vtype, vcsr, vlenb and 32 vector regs */ + return 37; +} + +static int copy_vector_reg_indices(const struct kvm_vcpu *vcpu, + u64 __user *uindices) +{ + const struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; + int n = num_vector_regs(vcpu); + u64 reg, size; + int i; + + if (n == 0) + return 0; + + /* copy vstart, vl, vtype, vcsr and vlenb */ + size = IS_ENABLED(CONFIG_32BIT) ? KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; + for (i = 0; i < 5; i++) { + reg = KVM_REG_RISCV | size | KVM_REG_RISCV_VECTOR | i; + + if (uindices) { + if (put_user(reg, uindices)) + return -EFAULT; + uindices++; + } + } + + /* vector_regs have a variable 'vlenb' size */ + size = __builtin_ctzl(cntx->vector.vlenb); + size <<= KVM_REG_SIZE_SHIFT; + for (i = 0; i < 32; i++) { + reg = KVM_REG_RISCV | KVM_REG_RISCV_VECTOR | size | + KVM_REG_RISCV_VECTOR_REG(i); + + if (uindices) { + if (put_user(reg, uindices)) + return -EFAULT; + uindices++; + } + } + + return n; +} + /* * kvm_riscv_vcpu_num_regs - how many registers do we present via KVM_GET/SET_ONE_REG * @@ -976,6 +1025,7 @@ unsigned long kvm_riscv_vcpu_num_regs(struct kvm_vcpu *vcpu) res += num_timer_regs(); res += num_fp_f_regs(vcpu); res += num_fp_d_regs(vcpu); + res += num_vector_regs(vcpu); res += num_isa_ext_regs(vcpu); res += num_sbi_ext_regs(vcpu); @@ -1020,6 +1070,11 @@ int kvm_riscv_vcpu_copy_reg_indices(struct kvm_vcpu *vcpu, return ret; uindices += ret; + ret = copy_vector_reg_indices(vcpu, uindices); + if (ret < 0) + return ret; + uindices += ret; + ret = copy_isa_ext_reg_indices(vcpu, uindices); if (ret < 0) return ret; From 4c460eb369514d53383a7c6ba1aefbca4914c68b Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Sun, 24 Dec 2023 14:04:02 +0530 Subject: [PATCH 220/336] RISC-V: KVM: Fix indentation in kvm_riscv_vcpu_set_reg_csr() The indentation of "break" in kvm_riscv_vcpu_set_reg_csr() is inconsistent hence let us fix it. Fixes: c04913f2b54e ("RISCV: KVM: Add sstateen0 to ONE_REG") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202312190719.kBuYl6oJ-lkp@intel.com/ Signed-off-by: Anup Patel Signed-off-by: Anup Patel --- arch/riscv/kvm/vcpu_onereg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/kvm/vcpu_onereg.c b/arch/riscv/kvm/vcpu_onereg.c index 11cdbf844291..c0bad1aed9f0 100644 --- a/arch/riscv/kvm/vcpu_onereg.c +++ b/arch/riscv/kvm/vcpu_onereg.c @@ -485,7 +485,7 @@ static int kvm_riscv_vcpu_set_reg_csr(struct kvm_vcpu *vcpu, if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) rc = kvm_riscv_vcpu_smstateen_set_csr(vcpu, reg_num, reg_val); -break; + break; default: rc = -ENOENT; break; From 323925ed6dbb0ed877047b28fae4152527cc63db Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Wed, 20 Dec 2023 17:00:14 +0100 Subject: [PATCH 221/336] RISC-V: paravirt: Add skeleton for pv-time support Add the files and functions needed to support paravirt time on RISC-V. Also include the common code needed for the first application of pv-time, which is steal-time. In the next patches we'll complete the functions to fully enable steal-time support. Acked-by: Palmer Dabbelt Reviewed-by: Anup Patel Reviewed-by: Atish Patra Signed-off-by: Andrew Jones Signed-off-by: Anup Patel --- .../admin-guide/kernel-parameters.txt | 6 +- arch/riscv/include/asm/paravirt.h | 28 +++++++ arch/riscv/include/asm/paravirt_api_clock.h | 1 + arch/riscv/kernel/Makefile | 1 + arch/riscv/kernel/paravirt.c | 79 +++++++++++++++++++ arch/riscv/kernel/time.c | 3 + 6 files changed, 115 insertions(+), 3 deletions(-) create mode 100644 arch/riscv/include/asm/paravirt.h create mode 100644 arch/riscv/include/asm/paravirt_api_clock.h create mode 100644 arch/riscv/kernel/paravirt.c diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 65731b060e3f..a0d9259e4857 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3985,9 +3985,9 @@ vulnerability. System may allow data leaks with this option. - no-steal-acc [X86,PV_OPS,ARM64,PPC/PSERIES] Disable paravirtualized - steal time accounting. steal time is computed, but - won't influence scheduler behaviour + no-steal-acc [X86,PV_OPS,ARM64,PPC/PSERIES,RISCV] Disable + paravirtualized steal time accounting. steal time is + computed, but won't influence scheduler behaviour nosync [HW,M68K] Disables sync negotiation for all devices. diff --git a/arch/riscv/include/asm/paravirt.h b/arch/riscv/include/asm/paravirt.h new file mode 100644 index 000000000000..c0abde70fc2c --- /dev/null +++ b/arch/riscv/include/asm/paravirt.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_RISCV_PARAVIRT_H +#define _ASM_RISCV_PARAVIRT_H + +#ifdef CONFIG_PARAVIRT +#include + +struct static_key; +extern struct static_key paravirt_steal_enabled; +extern struct static_key paravirt_steal_rq_enabled; + +u64 dummy_steal_clock(int cpu); + +DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); + +static inline u64 paravirt_steal_clock(int cpu) +{ + return static_call(pv_steal_clock)(cpu); +} + +int __init pv_time_init(void); + +#else + +#define pv_time_init() do {} while (0) + +#endif /* CONFIG_PARAVIRT */ +#endif /* _ASM_RISCV_PARAVIRT_H */ diff --git a/arch/riscv/include/asm/paravirt_api_clock.h b/arch/riscv/include/asm/paravirt_api_clock.h new file mode 100644 index 000000000000..65ac7cee0dad --- /dev/null +++ b/arch/riscv/include/asm/paravirt_api_clock.h @@ -0,0 +1 @@ +#include diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index fee22a3d1b53..807c2bde1f83 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -85,6 +85,7 @@ obj-$(CONFIG_SMP) += sbi-ipi.o obj-$(CONFIG_SMP) += cpu_ops_sbi.o endif obj-$(CONFIG_HOTPLUG_CPU) += cpu-hotplug.o +obj-$(CONFIG_PARAVIRT) += paravirt.o obj-$(CONFIG_KGDB) += kgdb.o obj-$(CONFIG_KEXEC_CORE) += kexec_relocate.o crash_save_regs.o machine_kexec.o obj-$(CONFIG_KEXEC_FILE) += elf_kexec.o machine_kexec_file.o diff --git a/arch/riscv/kernel/paravirt.c b/arch/riscv/kernel/paravirt.c new file mode 100644 index 000000000000..48d802df7739 --- /dev/null +++ b/arch/riscv/kernel/paravirt.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2023 Ventana Micro Systems Inc. + */ + +#define pr_fmt(fmt) "riscv-pv: " fmt + +#include +#include +#include +#include +#include +#include + +#include + +struct static_key paravirt_steal_enabled; +struct static_key paravirt_steal_rq_enabled; + +static u64 native_steal_clock(int cpu) +{ + return 0; +} + +DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); + +static bool steal_acc = true; +static int __init parse_no_stealacc(char *arg) +{ + steal_acc = false; + return 0; +} + +early_param("no-steal-acc", parse_no_stealacc); + +static bool __init has_pv_steal_clock(void) +{ + return false; +} + +static int pv_time_cpu_online(unsigned int cpu) +{ + return 0; +} + +static int pv_time_cpu_down_prepare(unsigned int cpu) +{ + return 0; +} + +static u64 pv_time_steal_clock(int cpu) +{ + return 0; +} + +int __init pv_time_init(void) +{ + int ret; + + if (!has_pv_steal_clock()) + return 0; + + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, + "riscv/pv_time:online", + pv_time_cpu_online, + pv_time_cpu_down_prepare); + if (ret < 0) + return ret; + + static_call_update(pv_steal_clock, pv_time_steal_clock); + + static_key_slow_inc(¶virt_steal_enabled); + if (steal_acc) + static_key_slow_inc(¶virt_steal_rq_enabled); + + pr_info("Computing paravirt steal-time\n"); + + return 0; +} diff --git a/arch/riscv/kernel/time.c b/arch/riscv/kernel/time.c index 23641e82a9df..ba3477197789 100644 --- a/arch/riscv/kernel/time.c +++ b/arch/riscv/kernel/time.c @@ -12,6 +12,7 @@ #include #include #include +#include unsigned long riscv_timebase __ro_after_init; EXPORT_SYMBOL_GPL(riscv_timebase); @@ -45,4 +46,6 @@ void __init time_init(void) timer_probe(); tick_setup_hrtimer_broadcast(); + + pv_time_init(); } From 6cfc624576a64145b1d6d3d48de7161a7505f403 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Wed, 20 Dec 2023 17:00:15 +0100 Subject: [PATCH 222/336] RISC-V: Add SBI STA extension definitions The SBI STA extension enables steal-time accounting. Add the definitions it specifies. Acked-by: Palmer Dabbelt Reviewed-by: Conor Dooley Reviewed-by: Anup Patel Reviewed-by: Atish Patra Signed-off-by: Andrew Jones Signed-off-by: Anup Patel --- arch/riscv/include/asm/sbi.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/arch/riscv/include/asm/sbi.h b/arch/riscv/include/asm/sbi.h index 0892f4421bc4..b6f898c56940 100644 --- a/arch/riscv/include/asm/sbi.h +++ b/arch/riscv/include/asm/sbi.h @@ -31,6 +31,7 @@ enum sbi_ext_id { SBI_EXT_SRST = 0x53525354, SBI_EXT_PMU = 0x504D55, SBI_EXT_DBCN = 0x4442434E, + SBI_EXT_STA = 0x535441, /* Experimentals extensions must lie within this range */ SBI_EXT_EXPERIMENTAL_START = 0x08000000, @@ -243,6 +244,22 @@ enum sbi_ext_dbcn_fid { SBI_EXT_DBCN_CONSOLE_WRITE_BYTE = 2, }; +/* SBI STA (steal-time accounting) extension */ +enum sbi_ext_sta_fid { + SBI_EXT_STA_STEAL_TIME_SET_SHMEM = 0, +}; + +struct sbi_sta_struct { + __le32 sequence; + __le32 flags; + __le64 steal; + u8 preempted; + u8 pad[47]; +} __packed; + +#define SBI_STA_SHMEM_DISABLE -1 + +/* SBI spec version fields */ #define SBI_SPEC_VERSION_DEFAULT 0x1 #define SBI_SPEC_VERSION_MAJOR_SHIFT 24 #define SBI_SPEC_VERSION_MAJOR_MASK 0x7f From fdf68acccfc6af9497c34ee411d89af13b6516ed Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Wed, 20 Dec 2023 17:00:16 +0100 Subject: [PATCH 223/336] RISC-V: paravirt: Implement steal-time support When the SBI STA extension exists we can use it to implement paravirt steal-time support. Fill in the empty pv-time functions with an SBI STA implementation and add the Kconfig knobs allowing it to be enabled. Acked-by: Palmer Dabbelt Reviewed-by: Atish Patra Reviewed-by: Anup Patel Signed-off-by: Andrew Jones Signed-off-by: Anup Patel --- arch/riscv/Kconfig | 19 +++++++++++ arch/riscv/kernel/paravirt.c | 62 ++++++++++++++++++++++++++++++++++-- 2 files changed, 78 insertions(+), 3 deletions(-) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 24c1799e2ec4..ef8b7b012a0b 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -724,6 +724,25 @@ config COMPAT If you want to execute 32-bit userspace applications, say Y. +config PARAVIRT + bool "Enable paravirtualization code" + depends on RISCV_SBI + help + This changes the kernel so it can modify itself when it is run + under a hypervisor, potentially improving performance significantly + over full virtualization. + +config PARAVIRT_TIME_ACCOUNTING + bool "Paravirtual steal time accounting" + depends on PARAVIRT + help + Select this option to enable fine granularity task steal time + accounting. Time spent executing other tasks in parallel with + the current vCPU is discounted from the vCPU power. To account for + that, there can be a small performance impact. + + If in doubt, say N here. + config RELOCATABLE bool "Build a relocatable kernel" depends on MMU && 64BIT && !XIP_KERNEL diff --git a/arch/riscv/kernel/paravirt.c b/arch/riscv/kernel/paravirt.c index 48d802df7739..8e114f5930ce 100644 --- a/arch/riscv/kernel/paravirt.c +++ b/arch/riscv/kernel/paravirt.c @@ -6,13 +6,21 @@ #define pr_fmt(fmt) "riscv-pv: " fmt #include +#include +#include #include #include +#include +#include +#include #include #include #include +#include +#include #include +#include struct static_key paravirt_steal_enabled; struct static_key paravirt_steal_rq_enabled; @@ -33,24 +41,72 @@ static int __init parse_no_stealacc(char *arg) early_param("no-steal-acc", parse_no_stealacc); +DEFINE_PER_CPU(struct sbi_sta_struct, steal_time) __aligned(64); + static bool __init has_pv_steal_clock(void) { + if (sbi_spec_version >= sbi_mk_version(2, 0) && + sbi_probe_extension(SBI_EXT_STA) > 0) { + pr_info("SBI STA extension detected\n"); + return true; + } + return false; } +static int sbi_sta_steal_time_set_shmem(unsigned long lo, unsigned long hi, + unsigned long flags) +{ + struct sbiret ret; + + ret = sbi_ecall(SBI_EXT_STA, SBI_EXT_STA_STEAL_TIME_SET_SHMEM, + lo, hi, flags, 0, 0, 0); + if (ret.error) { + if (lo == SBI_STA_SHMEM_DISABLE && hi == SBI_STA_SHMEM_DISABLE) + pr_warn("Failed to disable steal-time shmem"); + else + pr_warn("Failed to set steal-time shmem"); + return sbi_err_map_linux_errno(ret.error); + } + + return 0; +} + static int pv_time_cpu_online(unsigned int cpu) { - return 0; + struct sbi_sta_struct *st = this_cpu_ptr(&steal_time); + phys_addr_t pa = __pa(st); + unsigned long lo = (unsigned long)pa; + unsigned long hi = IS_ENABLED(CONFIG_32BIT) ? upper_32_bits((u64)pa) : 0; + + return sbi_sta_steal_time_set_shmem(lo, hi, 0); } static int pv_time_cpu_down_prepare(unsigned int cpu) { - return 0; + return sbi_sta_steal_time_set_shmem(SBI_STA_SHMEM_DISABLE, + SBI_STA_SHMEM_DISABLE, 0); } static u64 pv_time_steal_clock(int cpu) { - return 0; + struct sbi_sta_struct *st = per_cpu_ptr(&steal_time, cpu); + u32 sequence; + u64 steal; + + /* + * Check the sequence field before and after reading the steal + * field. Repeat the read if it is different or odd. + */ + do { + sequence = READ_ONCE(st->sequence); + virt_rmb(); + steal = READ_ONCE(st->steal); + virt_rmb(); + } while ((le32_to_cpu(sequence) & 1) || + sequence != READ_ONCE(st->sequence)); + + return le64_to_cpu(steal); } int __init pv_time_init(void) From 5fed84a800e6048656c17be6e921787db2b5c6c0 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Wed, 20 Dec 2023 17:00:17 +0100 Subject: [PATCH 224/336] RISC-V: KVM: Add SBI STA extension skeleton Add the files and functions needed to support the SBI STA (steal-time accounting) extension. In the next patches we'll complete the functions to fully enable SBI STA support. Reviewed-by: Anup Patel Reviewed-by: Atish Patra Signed-off-by: Andrew Jones Signed-off-by: Anup Patel --- arch/riscv/include/asm/kvm_vcpu_sbi.h | 1 + arch/riscv/include/uapi/asm/kvm.h | 1 + arch/riscv/kvm/Makefile | 1 + arch/riscv/kvm/vcpu_sbi.c | 4 +++ arch/riscv/kvm/vcpu_sbi_sta.c | 47 +++++++++++++++++++++++++++ 5 files changed, 54 insertions(+) create mode 100644 arch/riscv/kvm/vcpu_sbi_sta.c diff --git a/arch/riscv/include/asm/kvm_vcpu_sbi.h b/arch/riscv/include/asm/kvm_vcpu_sbi.h index bffda0ac59b6..99c23bb37a37 100644 --- a/arch/riscv/include/asm/kvm_vcpu_sbi.h +++ b/arch/riscv/include/asm/kvm_vcpu_sbi.h @@ -76,6 +76,7 @@ extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_rfence; extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_srst; extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_hsm; extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_dbcn; +extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_sta; extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_experimental; extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_vendor; diff --git a/arch/riscv/include/uapi/asm/kvm.h b/arch/riscv/include/uapi/asm/kvm.h index 60d3b21dead7..e961d79622fb 100644 --- a/arch/riscv/include/uapi/asm/kvm.h +++ b/arch/riscv/include/uapi/asm/kvm.h @@ -157,6 +157,7 @@ enum KVM_RISCV_SBI_EXT_ID { KVM_RISCV_SBI_EXT_EXPERIMENTAL, KVM_RISCV_SBI_EXT_VENDOR, KVM_RISCV_SBI_EXT_DBCN, + KVM_RISCV_SBI_EXT_STA, KVM_RISCV_SBI_EXT_MAX, }; diff --git a/arch/riscv/kvm/Makefile b/arch/riscv/kvm/Makefile index 4c2067fc59fc..c9646521f113 100644 --- a/arch/riscv/kvm/Makefile +++ b/arch/riscv/kvm/Makefile @@ -26,6 +26,7 @@ kvm-$(CONFIG_RISCV_SBI_V01) += vcpu_sbi_v01.o kvm-y += vcpu_sbi_base.o kvm-y += vcpu_sbi_replace.o kvm-y += vcpu_sbi_hsm.o +kvm-y += vcpu_sbi_sta.o kvm-y += vcpu_timer.o kvm-$(CONFIG_RISCV_PMU_SBI) += vcpu_pmu.o vcpu_sbi_pmu.o kvm-y += aia.o diff --git a/arch/riscv/kvm/vcpu_sbi.c b/arch/riscv/kvm/vcpu_sbi.c index dcdff4458190..088daaa23dd8 100644 --- a/arch/riscv/kvm/vcpu_sbi.c +++ b/arch/riscv/kvm/vcpu_sbi.c @@ -70,6 +70,10 @@ static const struct kvm_riscv_sbi_extension_entry sbi_ext[] = { .ext_idx = KVM_RISCV_SBI_EXT_DBCN, .ext_ptr = &vcpu_sbi_ext_dbcn, }, + { + .ext_idx = KVM_RISCV_SBI_EXT_STA, + .ext_ptr = &vcpu_sbi_ext_sta, + }, { .ext_idx = KVM_RISCV_SBI_EXT_EXPERIMENTAL, .ext_ptr = &vcpu_sbi_ext_experimental, diff --git a/arch/riscv/kvm/vcpu_sbi_sta.c b/arch/riscv/kvm/vcpu_sbi_sta.c new file mode 100644 index 000000000000..839911dcd837 --- /dev/null +++ b/arch/riscv/kvm/vcpu_sbi_sta.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2023 Ventana Micro Systems Inc. + */ + +#include + +#include +#include + +static int kvm_sbi_sta_steal_time_set_shmem(struct kvm_vcpu *vcpu) +{ + return SBI_ERR_FAILURE; +} + +static int kvm_sbi_ext_sta_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, + struct kvm_vcpu_sbi_return *retdata) +{ + struct kvm_cpu_context *cp = &vcpu->arch.guest_context; + unsigned long funcid = cp->a6; + int ret; + + switch (funcid) { + case SBI_EXT_STA_STEAL_TIME_SET_SHMEM: + ret = kvm_sbi_sta_steal_time_set_shmem(vcpu); + break; + default: + ret = SBI_ERR_NOT_SUPPORTED; + break; + } + + retdata->err_val = ret; + + return 0; +} + +static unsigned long kvm_sbi_ext_sta_probe(struct kvm_vcpu *vcpu) +{ + return 0; +} + +const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_sta = { + .extid_start = SBI_EXT_STA, + .extid_end = SBI_EXT_STA, + .handler = kvm_sbi_ext_sta_handler, + .probe = kvm_sbi_ext_sta_probe, +}; From 2a1f6bf079700f0f9d8045ab77b302aeb4d12c06 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Wed, 20 Dec 2023 17:00:18 +0100 Subject: [PATCH 225/336] RISC-V: KVM: Add steal-update vcpu request Add a new vcpu request to inform a vcpu that it should record its steal-time information. The request is made each time it has been detected that the vcpu task was not assigned a cpu for some time, which is easy to do by making the request from vcpu-load. The record function is just a stub for now and will be filled in with the rest of the steal-time support functions in following patches. Reviewed-by: Anup Patel Reviewed-by: Atish Patra Signed-off-by: Andrew Jones Signed-off-by: Anup Patel --- arch/riscv/include/asm/kvm_host.h | 3 +++ arch/riscv/kvm/vcpu.c | 5 +++++ arch/riscv/kvm/vcpu_sbi_sta.c | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h index 0eefd9c991ae..230b82c3118d 100644 --- a/arch/riscv/include/asm/kvm_host.h +++ b/arch/riscv/include/asm/kvm_host.h @@ -41,6 +41,7 @@ KVM_ARCH_REQ_FLAGS(4, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_HFENCE \ KVM_ARCH_REQ_FLAGS(5, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) +#define KVM_REQ_STEAL_UPDATE KVM_ARCH_REQ(6) enum kvm_riscv_hfence_type { KVM_RISCV_HFENCE_UNKNOWN = 0, @@ -372,4 +373,6 @@ bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, u64 mask); void kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu); void kvm_riscv_vcpu_power_on(struct kvm_vcpu *vcpu); +void kvm_riscv_vcpu_record_steal_time(struct kvm_vcpu *vcpu); + #endif /* __RISCV_KVM_HOST_H__ */ diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index bf3952d1a621..6995b8b641e4 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -541,6 +541,8 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) kvm_riscv_vcpu_aia_load(vcpu, cpu); + kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); + vcpu->cpu = cpu; } @@ -614,6 +616,9 @@ static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu) if (kvm_check_request(KVM_REQ_HFENCE, vcpu)) kvm_riscv_hfence_process(vcpu); + + if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu)) + kvm_riscv_vcpu_record_steal_time(vcpu); } } diff --git a/arch/riscv/kvm/vcpu_sbi_sta.c b/arch/riscv/kvm/vcpu_sbi_sta.c index 839911dcd837..e28351c9488b 100644 --- a/arch/riscv/kvm/vcpu_sbi_sta.c +++ b/arch/riscv/kvm/vcpu_sbi_sta.c @@ -8,6 +8,10 @@ #include #include +void kvm_riscv_vcpu_record_steal_time(struct kvm_vcpu *vcpu) +{ +} + static int kvm_sbi_sta_steal_time_set_shmem(struct kvm_vcpu *vcpu) { return SBI_ERR_FAILURE; From 38b3390ee4880140b6245fe3273fe9ce53f65bde Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Wed, 20 Dec 2023 17:00:19 +0100 Subject: [PATCH 226/336] RISC-V: KVM: Add SBI STA info to vcpu_arch KVM's implementation of SBI STA needs to track the address of each VCPU's steal-time shared memory region as well as the amount of stolen time. Add a structure to vcpu_arch to contain this state and make sure that the address is always set to INVALID_GPA on vcpu reset. And, of course, ensure KVM won't try to update steal- time when the shared memory address is invalid. Reviewed-by: Anup Patel Reviewed-by: Atish Patra Signed-off-by: Andrew Jones Signed-off-by: Anup Patel --- arch/riscv/include/asm/kvm_host.h | 7 +++++++ arch/riscv/kvm/vcpu.c | 2 ++ arch/riscv/kvm/vcpu_sbi_sta.c | 10 ++++++++++ 3 files changed, 19 insertions(+) diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h index 230b82c3118d..525cba63e0c5 100644 --- a/arch/riscv/include/asm/kvm_host.h +++ b/arch/riscv/include/asm/kvm_host.h @@ -263,6 +263,12 @@ struct kvm_vcpu_arch { /* 'static' configurations which are set only once */ struct kvm_vcpu_config cfg; + + /* SBI steal-time accounting */ + struct { + gpa_t shmem; + u64 last_steal; + } sta; }; static inline void kvm_arch_sync_events(struct kvm *kvm) {} @@ -373,6 +379,7 @@ bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, u64 mask); void kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu); void kvm_riscv_vcpu_power_on(struct kvm_vcpu *vcpu); +void kvm_riscv_vcpu_sbi_sta_reset(struct kvm_vcpu *vcpu); void kvm_riscv_vcpu_record_steal_time(struct kvm_vcpu *vcpu); #endif /* __RISCV_KVM_HOST_H__ */ diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index 6995b8b641e4..b5ca9f2e98ac 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -83,6 +83,8 @@ static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu) vcpu->arch.hfence_tail = 0; memset(vcpu->arch.hfence_queue, 0, sizeof(vcpu->arch.hfence_queue)); + kvm_riscv_vcpu_sbi_sta_reset(vcpu); + /* Reset the guest CSRs for hotplug usecase */ if (loaded) kvm_arch_vcpu_load(vcpu, smp_processor_id()); diff --git a/arch/riscv/kvm/vcpu_sbi_sta.c b/arch/riscv/kvm/vcpu_sbi_sta.c index e28351c9488b..6592d287fc4e 100644 --- a/arch/riscv/kvm/vcpu_sbi_sta.c +++ b/arch/riscv/kvm/vcpu_sbi_sta.c @@ -8,8 +8,18 @@ #include #include +void kvm_riscv_vcpu_sbi_sta_reset(struct kvm_vcpu *vcpu) +{ + vcpu->arch.sta.shmem = INVALID_GPA; + vcpu->arch.sta.last_steal = 0; +} + void kvm_riscv_vcpu_record_steal_time(struct kvm_vcpu *vcpu) { + gpa_t shmem = vcpu->arch.sta.shmem; + + if (shmem == INVALID_GPA) + return; } static int kvm_sbi_sta_steal_time_set_shmem(struct kvm_vcpu *vcpu) From 5b9e41321ba919dd051c68d2a1d2c753aa61634c Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Wed, 20 Dec 2023 17:00:20 +0100 Subject: [PATCH 227/336] RISC-V: KVM: Add support for SBI extension registers Some SBI extensions have state that needs to be saved / restored when migrating the VM. Provide a get/set-one-reg register type for SBI extension registers. Each SBI extension that uses this type will have its own subtype. There are currently no subtypes defined. The next patch introduces the first one. Reviewed-by: Anup Patel Reviewed-by: Atish Patra Signed-off-by: Andrew Jones Signed-off-by: Anup Patel --- arch/riscv/include/asm/kvm_vcpu_sbi.h | 4 ++ arch/riscv/include/uapi/asm/kvm.h | 3 ++ arch/riscv/kvm/vcpu_onereg.c | 42 +++++++++++++++++-- arch/riscv/kvm/vcpu_sbi.c | 58 +++++++++++++++++++++++++++ 4 files changed, 103 insertions(+), 4 deletions(-) diff --git a/arch/riscv/include/asm/kvm_vcpu_sbi.h b/arch/riscv/include/asm/kvm_vcpu_sbi.h index 99c23bb37a37..dd60f73b5c36 100644 --- a/arch/riscv/include/asm/kvm_vcpu_sbi.h +++ b/arch/riscv/include/asm/kvm_vcpu_sbi.h @@ -60,6 +60,10 @@ int kvm_riscv_vcpu_set_reg_sbi_ext(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); int kvm_riscv_vcpu_get_reg_sbi_ext(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); +int kvm_riscv_vcpu_set_reg_sbi(struct kvm_vcpu *vcpu, + const struct kvm_one_reg *reg); +int kvm_riscv_vcpu_get_reg_sbi(struct kvm_vcpu *vcpu, + const struct kvm_one_reg *reg); const struct kvm_vcpu_sbi_extension *kvm_vcpu_sbi_find_ext( struct kvm_vcpu *vcpu, unsigned long extid); bool riscv_vcpu_supports_sbi_ext(struct kvm_vcpu *vcpu, int idx); diff --git a/arch/riscv/include/uapi/asm/kvm.h b/arch/riscv/include/uapi/asm/kvm.h index e961d79622fb..3471b1e48d18 100644 --- a/arch/riscv/include/uapi/asm/kvm.h +++ b/arch/riscv/include/uapi/asm/kvm.h @@ -242,6 +242,9 @@ enum KVM_RISCV_SBI_EXT_ID { #define KVM_REG_RISCV_VECTOR_REG(n) \ ((n) + sizeof(struct __riscv_v_ext_state) / sizeof(unsigned long)) +/* Registers for specific SBI extensions are mapped as type 10 */ +#define KVM_REG_RISCV_SBI_STATE (0x0a << KVM_REG_RISCV_TYPE_SHIFT) + /* Device Control API: RISC-V AIA */ #define KVM_DEV_RISCV_APLIC_ALIGN 0x1000 #define KVM_DEV_RISCV_APLIC_SIZE 0x4000 diff --git a/arch/riscv/kvm/vcpu_onereg.c b/arch/riscv/kvm/vcpu_onereg.c index c0bad1aed9f0..143d0edd7f63 100644 --- a/arch/riscv/kvm/vcpu_onereg.c +++ b/arch/riscv/kvm/vcpu_onereg.c @@ -961,6 +961,29 @@ static unsigned long num_sbi_ext_regs(struct kvm_vcpu *vcpu) return copy_sbi_ext_reg_indices(vcpu, NULL); } +static inline unsigned long num_sbi_regs(struct kvm_vcpu *vcpu) +{ + return 0; +} + +static int copy_sbi_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) +{ + int n = num_sbi_regs(vcpu); + + for (int i = 0; i < n; i++) { + u64 reg = KVM_REG_RISCV | KVM_REG_SIZE_U64 | + KVM_REG_RISCV_SBI_STATE | i; + + if (uindices) { + if (put_user(reg, uindices)) + return -EFAULT; + uindices++; + } + } + + return n; +} + static inline unsigned long num_vector_regs(const struct kvm_vcpu *vcpu) { if (!riscv_isa_extension_available(vcpu->arch.isa, v)) @@ -1028,6 +1051,7 @@ unsigned long kvm_riscv_vcpu_num_regs(struct kvm_vcpu *vcpu) res += num_vector_regs(vcpu); res += num_isa_ext_regs(vcpu); res += num_sbi_ext_regs(vcpu); + res += num_sbi_regs(vcpu); return res; } @@ -1083,6 +1107,12 @@ int kvm_riscv_vcpu_copy_reg_indices(struct kvm_vcpu *vcpu, ret = copy_sbi_ext_reg_indices(vcpu, uindices); if (ret < 0) return ret; + uindices += ret; + + ret = copy_sbi_reg_indices(vcpu, uindices); + if (ret < 0) + return ret; + uindices += ret; return 0; } @@ -1105,12 +1135,14 @@ int kvm_riscv_vcpu_set_reg(struct kvm_vcpu *vcpu, case KVM_REG_RISCV_FP_D: return kvm_riscv_vcpu_set_reg_fp(vcpu, reg, KVM_REG_RISCV_FP_D); + case KVM_REG_RISCV_VECTOR: + return kvm_riscv_vcpu_set_reg_vector(vcpu, reg); case KVM_REG_RISCV_ISA_EXT: return kvm_riscv_vcpu_set_reg_isa_ext(vcpu, reg); case KVM_REG_RISCV_SBI_EXT: return kvm_riscv_vcpu_set_reg_sbi_ext(vcpu, reg); - case KVM_REG_RISCV_VECTOR: - return kvm_riscv_vcpu_set_reg_vector(vcpu, reg); + case KVM_REG_RISCV_SBI_STATE: + return kvm_riscv_vcpu_set_reg_sbi(vcpu, reg); default: break; } @@ -1136,12 +1168,14 @@ int kvm_riscv_vcpu_get_reg(struct kvm_vcpu *vcpu, case KVM_REG_RISCV_FP_D: return kvm_riscv_vcpu_get_reg_fp(vcpu, reg, KVM_REG_RISCV_FP_D); + case KVM_REG_RISCV_VECTOR: + return kvm_riscv_vcpu_get_reg_vector(vcpu, reg); case KVM_REG_RISCV_ISA_EXT: return kvm_riscv_vcpu_get_reg_isa_ext(vcpu, reg); case KVM_REG_RISCV_SBI_EXT: return kvm_riscv_vcpu_get_reg_sbi_ext(vcpu, reg); - case KVM_REG_RISCV_VECTOR: - return kvm_riscv_vcpu_get_reg_vector(vcpu, reg); + case KVM_REG_RISCV_SBI_STATE: + return kvm_riscv_vcpu_get_reg_sbi(vcpu, reg); default: break; } diff --git a/arch/riscv/kvm/vcpu_sbi.c b/arch/riscv/kvm/vcpu_sbi.c index 088daaa23dd8..a1997c39dfde 100644 --- a/arch/riscv/kvm/vcpu_sbi.c +++ b/arch/riscv/kvm/vcpu_sbi.c @@ -325,6 +325,64 @@ int kvm_riscv_vcpu_get_reg_sbi_ext(struct kvm_vcpu *vcpu, return 0; } +int kvm_riscv_vcpu_set_reg_sbi(struct kvm_vcpu *vcpu, + const struct kvm_one_reg *reg) +{ + unsigned long __user *uaddr = + (unsigned long __user *)(unsigned long)reg->addr; + unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | + KVM_REG_SIZE_MASK | + KVM_REG_RISCV_SBI_STATE); + unsigned long reg_subtype, reg_val; + + if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) + return -EINVAL; + + if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) + return -EFAULT; + + reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; + reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; + + switch (reg_subtype) { + default: + return -EINVAL; + } + + return 0; +} + +int kvm_riscv_vcpu_get_reg_sbi(struct kvm_vcpu *vcpu, + const struct kvm_one_reg *reg) +{ + unsigned long __user *uaddr = + (unsigned long __user *)(unsigned long)reg->addr; + unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | + KVM_REG_SIZE_MASK | + KVM_REG_RISCV_SBI_STATE); + unsigned long reg_subtype, reg_val; + int ret; + + if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) + return -EINVAL; + + reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; + reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; + + switch (reg_subtype) { + default: + return -EINVAL; + } + + if (ret) + return ret; + + if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) + return -EFAULT; + + return 0; +} + const struct kvm_vcpu_sbi_extension *kvm_vcpu_sbi_find_ext( struct kvm_vcpu *vcpu, unsigned long extid) { From f61ce890b1f0742f17b3a5d1f8c72574a33ffeb2 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Wed, 20 Dec 2023 17:00:21 +0100 Subject: [PATCH 228/336] RISC-V: KVM: Add support for SBI STA registers KVM userspace needs to be able to save and restore the steal-time shared memory address. Provide the address through the get/set-one-reg interface with two ulong-sized SBI STA extension registers (lo and hi). 64-bit KVM userspace must not set the hi register to anything other than zero and is allowed to completely neglect saving/restoring it. Reviewed-by: Anup Patel Signed-off-by: Andrew Jones Signed-off-by: Anup Patel --- arch/riscv/include/asm/kvm_vcpu_sbi.h | 5 +++ arch/riscv/include/uapi/asm/kvm.h | 9 +++++ arch/riscv/kvm/vcpu_onereg.c | 37 +++++++++++------- arch/riscv/kvm/vcpu_sbi.c | 5 +++ arch/riscv/kvm/vcpu_sbi_sta.c | 55 +++++++++++++++++++++++++++ 5 files changed, 97 insertions(+), 14 deletions(-) diff --git a/arch/riscv/include/asm/kvm_vcpu_sbi.h b/arch/riscv/include/asm/kvm_vcpu_sbi.h index dd60f73b5c36..b96705258cf9 100644 --- a/arch/riscv/include/asm/kvm_vcpu_sbi.h +++ b/arch/riscv/include/asm/kvm_vcpu_sbi.h @@ -70,6 +70,11 @@ bool riscv_vcpu_supports_sbi_ext(struct kvm_vcpu *vcpu, int idx); int kvm_riscv_vcpu_sbi_ecall(struct kvm_vcpu *vcpu, struct kvm_run *run); void kvm_riscv_vcpu_sbi_init(struct kvm_vcpu *vcpu); +int kvm_riscv_vcpu_get_reg_sbi_sta(struct kvm_vcpu *vcpu, unsigned long reg_num, + unsigned long *reg_val); +int kvm_riscv_vcpu_set_reg_sbi_sta(struct kvm_vcpu *vcpu, unsigned long reg_num, + unsigned long reg_val); + #ifdef CONFIG_RISCV_SBI_V01 extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_v01; #endif diff --git a/arch/riscv/include/uapi/asm/kvm.h b/arch/riscv/include/uapi/asm/kvm.h index 3471b1e48d18..d6b7a5b95874 100644 --- a/arch/riscv/include/uapi/asm/kvm.h +++ b/arch/riscv/include/uapi/asm/kvm.h @@ -161,6 +161,12 @@ enum KVM_RISCV_SBI_EXT_ID { KVM_RISCV_SBI_EXT_MAX, }; +/* SBI STA extension registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */ +struct kvm_riscv_sbi_sta { + unsigned long shmem_lo; + unsigned long shmem_hi; +}; + /* Possible states for kvm_riscv_timer */ #define KVM_RISCV_TIMER_STATE_OFF 0 #define KVM_RISCV_TIMER_STATE_ON 1 @@ -244,6 +250,9 @@ enum KVM_RISCV_SBI_EXT_ID { /* Registers for specific SBI extensions are mapped as type 10 */ #define KVM_REG_RISCV_SBI_STATE (0x0a << KVM_REG_RISCV_TYPE_SHIFT) +#define KVM_REG_RISCV_SBI_STA (0x0 << KVM_REG_RISCV_SUBTYPE_SHIFT) +#define KVM_REG_RISCV_SBI_STA_REG(name) \ + (offsetof(struct kvm_riscv_sbi_sta, name) / sizeof(unsigned long)) /* Device Control API: RISC-V AIA */ #define KVM_DEV_RISCV_APLIC_ALIGN 0x1000 diff --git a/arch/riscv/kvm/vcpu_onereg.c b/arch/riscv/kvm/vcpu_onereg.c index 143d0edd7f63..fc34557f5356 100644 --- a/arch/riscv/kvm/vcpu_onereg.c +++ b/arch/riscv/kvm/vcpu_onereg.c @@ -961,27 +961,36 @@ static unsigned long num_sbi_ext_regs(struct kvm_vcpu *vcpu) return copy_sbi_ext_reg_indices(vcpu, NULL); } -static inline unsigned long num_sbi_regs(struct kvm_vcpu *vcpu) -{ - return 0; -} - static int copy_sbi_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) { - int n = num_sbi_regs(vcpu); + struct kvm_vcpu_sbi_context *scontext = &vcpu->arch.sbi_context; + int total = 0; - for (int i = 0; i < n; i++) { - u64 reg = KVM_REG_RISCV | KVM_REG_SIZE_U64 | - KVM_REG_RISCV_SBI_STATE | i; + if (scontext->ext_status[KVM_RISCV_SBI_EXT_STA] == KVM_RISCV_SBI_EXT_STATUS_ENABLED) { + u64 size = IS_ENABLED(CONFIG_32BIT) ? KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; + int n = sizeof(struct kvm_riscv_sbi_sta) / sizeof(unsigned long); - if (uindices) { - if (put_user(reg, uindices)) - return -EFAULT; - uindices++; + for (int i = 0; i < n; i++) { + u64 reg = KVM_REG_RISCV | size | + KVM_REG_RISCV_SBI_STATE | + KVM_REG_RISCV_SBI_STA | i; + + if (uindices) { + if (put_user(reg, uindices)) + return -EFAULT; + uindices++; + } } + + total += n; } - return n; + return total; +} + +static inline unsigned long num_sbi_regs(struct kvm_vcpu *vcpu) +{ + return copy_sbi_reg_indices(vcpu, NULL); } static inline unsigned long num_vector_regs(const struct kvm_vcpu *vcpu) diff --git a/arch/riscv/kvm/vcpu_sbi.c b/arch/riscv/kvm/vcpu_sbi.c index a1997c39dfde..72a2ffb8dcd1 100644 --- a/arch/riscv/kvm/vcpu_sbi.c +++ b/arch/riscv/kvm/vcpu_sbi.c @@ -345,6 +345,8 @@ int kvm_riscv_vcpu_set_reg_sbi(struct kvm_vcpu *vcpu, reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; switch (reg_subtype) { + case KVM_REG_RISCV_SBI_STA: + return kvm_riscv_vcpu_set_reg_sbi_sta(vcpu, reg_num, reg_val); default: return -EINVAL; } @@ -370,6 +372,9 @@ int kvm_riscv_vcpu_get_reg_sbi(struct kvm_vcpu *vcpu, reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; switch (reg_subtype) { + case KVM_REG_RISCV_SBI_STA: + ret = kvm_riscv_vcpu_get_reg_sbi_sta(vcpu, reg_num, ®_val); + break; default: return -EINVAL; } diff --git a/arch/riscv/kvm/vcpu_sbi_sta.c b/arch/riscv/kvm/vcpu_sbi_sta.c index 6592d287fc4e..87bf1a5f05ce 100644 --- a/arch/riscv/kvm/vcpu_sbi_sta.c +++ b/arch/riscv/kvm/vcpu_sbi_sta.c @@ -3,6 +3,8 @@ * Copyright (c) 2023 Ventana Micro Systems Inc. */ +#include +#include #include #include @@ -59,3 +61,56 @@ const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_sta = { .handler = kvm_sbi_ext_sta_handler, .probe = kvm_sbi_ext_sta_probe, }; + +int kvm_riscv_vcpu_get_reg_sbi_sta(struct kvm_vcpu *vcpu, + unsigned long reg_num, + unsigned long *reg_val) +{ + switch (reg_num) { + case KVM_REG_RISCV_SBI_STA_REG(shmem_lo): + *reg_val = (unsigned long)vcpu->arch.sta.shmem; + break; + case KVM_REG_RISCV_SBI_STA_REG(shmem_hi): + if (IS_ENABLED(CONFIG_32BIT)) + *reg_val = upper_32_bits(vcpu->arch.sta.shmem); + else + *reg_val = 0; + break; + default: + return -EINVAL; + } + + return 0; +} + +int kvm_riscv_vcpu_set_reg_sbi_sta(struct kvm_vcpu *vcpu, + unsigned long reg_num, + unsigned long reg_val) +{ + switch (reg_num) { + case KVM_REG_RISCV_SBI_STA_REG(shmem_lo): + if (IS_ENABLED(CONFIG_32BIT)) { + gpa_t hi = upper_32_bits(vcpu->arch.sta.shmem); + + vcpu->arch.sta.shmem = reg_val; + vcpu->arch.sta.shmem |= hi << 32; + } else { + vcpu->arch.sta.shmem = reg_val; + } + break; + case KVM_REG_RISCV_SBI_STA_REG(shmem_hi): + if (IS_ENABLED(CONFIG_32BIT)) { + gpa_t lo = lower_32_bits(vcpu->arch.sta.shmem); + + vcpu->arch.sta.shmem = ((gpa_t)reg_val << 32); + vcpu->arch.sta.shmem |= lo; + } else if (reg_val != 0) { + return -EINVAL; + } + break; + default: + return -EINVAL; + } + + return 0; +} From e9f12b5fff8ad0eefd0340273767d329ef65fd69 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Wed, 20 Dec 2023 17:00:22 +0100 Subject: [PATCH 229/336] RISC-V: KVM: Implement SBI STA extension Add a select SCHED_INFO to the KVM config in order to get run_delay info. Then implement SBI STA's set-steal-time-shmem function and kvm_riscv_vcpu_record_steal_time() to provide the steal-time info to guests. Reviewed-by: Anup Patel Reviewed-by: Atish Patra Signed-off-by: Andrew Jones Signed-off-by: Anup Patel --- arch/riscv/kvm/Kconfig | 1 + arch/riscv/kvm/vcpu_sbi_sta.c | 96 ++++++++++++++++++++++++++++++++++- 2 files changed, 95 insertions(+), 2 deletions(-) diff --git a/arch/riscv/kvm/Kconfig b/arch/riscv/kvm/Kconfig index dfc237d7875b..148e52b516cf 100644 --- a/arch/riscv/kvm/Kconfig +++ b/arch/riscv/kvm/Kconfig @@ -32,6 +32,7 @@ config KVM select KVM_XFER_TO_GUEST_WORK select MMU_NOTIFIER select PREEMPT_NOTIFIERS + select SCHED_INFO help Support hosting virtualized guest machines. diff --git a/arch/riscv/kvm/vcpu_sbi_sta.c b/arch/riscv/kvm/vcpu_sbi_sta.c index 87bf1a5f05ce..01f09fe8c3b0 100644 --- a/arch/riscv/kvm/vcpu_sbi_sta.c +++ b/arch/riscv/kvm/vcpu_sbi_sta.c @@ -6,9 +6,15 @@ #include #include #include +#include +#include +#include +#include #include +#include #include +#include void kvm_riscv_vcpu_sbi_sta_reset(struct kvm_vcpu *vcpu) { @@ -19,14 +25,100 @@ void kvm_riscv_vcpu_sbi_sta_reset(struct kvm_vcpu *vcpu) void kvm_riscv_vcpu_record_steal_time(struct kvm_vcpu *vcpu) { gpa_t shmem = vcpu->arch.sta.shmem; + u64 last_steal = vcpu->arch.sta.last_steal; + u32 *sequence_ptr, sequence; + u64 *steal_ptr, steal; + unsigned long hva; + gfn_t gfn; if (shmem == INVALID_GPA) return; + + /* + * shmem is 64-byte aligned (see the enforcement in + * kvm_sbi_sta_steal_time_set_shmem()) and the size of sbi_sta_struct + * is 64 bytes, so we know all its offsets are in the same page. + */ + gfn = shmem >> PAGE_SHIFT; + hva = kvm_vcpu_gfn_to_hva(vcpu, gfn); + + if (WARN_ON(kvm_is_error_hva(hva))) { + vcpu->arch.sta.shmem = INVALID_GPA; + return; + } + + sequence_ptr = (u32 *)(hva + offset_in_page(shmem) + + offsetof(struct sbi_sta_struct, sequence)); + steal_ptr = (u64 *)(hva + offset_in_page(shmem) + + offsetof(struct sbi_sta_struct, steal)); + + if (WARN_ON(get_user(sequence, sequence_ptr))) + return; + + sequence = le32_to_cpu(sequence); + sequence += 1; + + if (WARN_ON(put_user(cpu_to_le32(sequence), sequence_ptr))) + return; + + if (!WARN_ON(get_user(steal, steal_ptr))) { + steal = le64_to_cpu(steal); + vcpu->arch.sta.last_steal = READ_ONCE(current->sched_info.run_delay); + steal += vcpu->arch.sta.last_steal - last_steal; + WARN_ON(put_user(cpu_to_le64(steal), steal_ptr)); + } + + sequence += 1; + WARN_ON(put_user(cpu_to_le32(sequence), sequence_ptr)); + + kvm_vcpu_mark_page_dirty(vcpu, gfn); } static int kvm_sbi_sta_steal_time_set_shmem(struct kvm_vcpu *vcpu) { - return SBI_ERR_FAILURE; + struct kvm_cpu_context *cp = &vcpu->arch.guest_context; + unsigned long shmem_phys_lo = cp->a0; + unsigned long shmem_phys_hi = cp->a1; + u32 flags = cp->a2; + struct sbi_sta_struct zero_sta = {0}; + unsigned long hva; + bool writable; + gpa_t shmem; + int ret; + + if (flags != 0) + return SBI_ERR_INVALID_PARAM; + + if (shmem_phys_lo == SBI_STA_SHMEM_DISABLE && + shmem_phys_hi == SBI_STA_SHMEM_DISABLE) { + vcpu->arch.sta.shmem = INVALID_GPA; + return 0; + } + + if (shmem_phys_lo & (SZ_64 - 1)) + return SBI_ERR_INVALID_PARAM; + + shmem = shmem_phys_lo; + + if (shmem_phys_hi != 0) { + if (IS_ENABLED(CONFIG_32BIT)) + shmem |= ((gpa_t)shmem_phys_hi << 32); + else + return SBI_ERR_INVALID_ADDRESS; + } + + hva = kvm_vcpu_gfn_to_hva_prot(vcpu, shmem >> PAGE_SHIFT, &writable); + if (kvm_is_error_hva(hva) || !writable) + return SBI_ERR_INVALID_ADDRESS; + + ret = kvm_vcpu_write_guest(vcpu, shmem, &zero_sta, sizeof(zero_sta)); + if (ret) + return SBI_ERR_FAILURE; + + vcpu->arch.sta.shmem = shmem; + vcpu->arch.sta.last_steal = current->sched_info.run_delay; + + return 0; } static int kvm_sbi_ext_sta_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, @@ -52,7 +144,7 @@ static int kvm_sbi_ext_sta_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, static unsigned long kvm_sbi_ext_sta_probe(struct kvm_vcpu *vcpu) { - return 0; + return !!sched_info_on(); } const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_sta = { From 0dcab5c4762ac166aa7e635ae4b6d649e15717e2 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Wed, 20 Dec 2023 17:00:23 +0100 Subject: [PATCH 230/336] RISC-V: KVM: selftests: Move sbi_ecall to processor.c sbi_ecall() isn't ucall specific and its prototype is already in processor.h. Move its implementation to processor.c. Reviewed-by: Anup Patel Signed-off-by: Andrew Jones Signed-off-by: Anup Patel --- .../selftests/kvm/lib/riscv/processor.c | 26 +++++++++++++++++++ tools/testing/selftests/kvm/lib/riscv/ucall.c | 26 ------------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/tools/testing/selftests/kvm/lib/riscv/processor.c b/tools/testing/selftests/kvm/lib/riscv/processor.c index 6c25f7843ef4..6905a4348380 100644 --- a/tools/testing/selftests/kvm/lib/riscv/processor.c +++ b/tools/testing/selftests/kvm/lib/riscv/processor.c @@ -367,3 +367,29 @@ void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...) void assert_on_unhandled_exception(struct kvm_vcpu *vcpu) { } + +struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0, + unsigned long arg1, unsigned long arg2, + unsigned long arg3, unsigned long arg4, + unsigned long arg5) +{ + register uintptr_t a0 asm ("a0") = (uintptr_t)(arg0); + register uintptr_t a1 asm ("a1") = (uintptr_t)(arg1); + register uintptr_t a2 asm ("a2") = (uintptr_t)(arg2); + register uintptr_t a3 asm ("a3") = (uintptr_t)(arg3); + register uintptr_t a4 asm ("a4") = (uintptr_t)(arg4); + register uintptr_t a5 asm ("a5") = (uintptr_t)(arg5); + register uintptr_t a6 asm ("a6") = (uintptr_t)(fid); + register uintptr_t a7 asm ("a7") = (uintptr_t)(ext); + struct sbiret ret; + + asm volatile ( + "ecall" + : "+r" (a0), "+r" (a1) + : "r" (a2), "r" (a3), "r" (a4), "r" (a5), "r" (a6), "r" (a7) + : "memory"); + ret.error = a0; + ret.value = a1; + + return ret; +} diff --git a/tools/testing/selftests/kvm/lib/riscv/ucall.c b/tools/testing/selftests/kvm/lib/riscv/ucall.c index fe6d1004f018..14ee17151a59 100644 --- a/tools/testing/selftests/kvm/lib/riscv/ucall.c +++ b/tools/testing/selftests/kvm/lib/riscv/ucall.c @@ -10,32 +10,6 @@ #include "kvm_util.h" #include "processor.h" -struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0, - unsigned long arg1, unsigned long arg2, - unsigned long arg3, unsigned long arg4, - unsigned long arg5) -{ - register uintptr_t a0 asm ("a0") = (uintptr_t)(arg0); - register uintptr_t a1 asm ("a1") = (uintptr_t)(arg1); - register uintptr_t a2 asm ("a2") = (uintptr_t)(arg2); - register uintptr_t a3 asm ("a3") = (uintptr_t)(arg3); - register uintptr_t a4 asm ("a4") = (uintptr_t)(arg4); - register uintptr_t a5 asm ("a5") = (uintptr_t)(arg5); - register uintptr_t a6 asm ("a6") = (uintptr_t)(fid); - register uintptr_t a7 asm ("a7") = (uintptr_t)(ext); - struct sbiret ret; - - asm volatile ( - "ecall" - : "+r" (a0), "+r" (a1) - : "r" (a2), "r" (a3), "r" (a4), "r" (a5), "r" (a6), "r" (a7) - : "memory"); - ret.error = a0; - ret.value = a1; - - return ret; -} - void *ucall_arch_get_ucall(struct kvm_vcpu *vcpu) { struct kvm_run *run = vcpu->run; From 945d880d6be0fd19bbc77d80d113bd2ca74c74f8 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Wed, 20 Dec 2023 17:00:24 +0100 Subject: [PATCH 231/336] RISC-V: KVM: selftests: Add guest_sbi_probe_extension Add guest_sbi_probe_extension(), allowing guest code to probe for SBI extensions. As guest_sbi_probe_extension() needs SBI_ERR_NOT_SUPPORTED, take the opportunity to bring in all SBI error codes. We don't bring in all current extension IDs or base extension function IDs though, even though we need one of each, because we'd prefer to bring those in as necessary. Reviewed-by: Anup Patel Reviewed-by: Atish Patra Signed-off-by: Andrew Jones Signed-off-by: Anup Patel --- .../selftests/kvm/include/riscv/processor.h | 21 +++++++++++++++++++ .../selftests/kvm/lib/riscv/processor.c | 19 +++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/tools/testing/selftests/kvm/include/riscv/processor.h b/tools/testing/selftests/kvm/include/riscv/processor.h index e70ccda2011b..dc50ad62e150 100644 --- a/tools/testing/selftests/kvm/include/riscv/processor.h +++ b/tools/testing/selftests/kvm/include/riscv/processor.h @@ -108,6 +108,17 @@ static inline uint64_t __kvm_reg_id(uint64_t type, uint64_t subtype, #define SATP_ASID_SHIFT 44 #define SATP_ASID_MASK _AC(0xFFFF, UL) +/* SBI return error codes */ +#define SBI_SUCCESS 0 +#define SBI_ERR_FAILURE -1 +#define SBI_ERR_NOT_SUPPORTED -2 +#define SBI_ERR_INVALID_PARAM -3 +#define SBI_ERR_DENIED -4 +#define SBI_ERR_INVALID_ADDRESS -5 +#define SBI_ERR_ALREADY_AVAILABLE -6 +#define SBI_ERR_ALREADY_STARTED -7 +#define SBI_ERR_ALREADY_STOPPED -8 + #define SBI_EXT_EXPERIMENTAL_START 0x08000000 #define SBI_EXT_EXPERIMENTAL_END 0x08FFFFFF @@ -115,6 +126,14 @@ static inline uint64_t __kvm_reg_id(uint64_t type, uint64_t subtype, #define KVM_RISCV_SELFTESTS_SBI_UCALL 0 #define KVM_RISCV_SELFTESTS_SBI_UNEXP 1 +enum sbi_ext_id { + SBI_EXT_BASE = 0x10, +}; + +enum sbi_ext_base_fid { + SBI_EXT_BASE_PROBE_EXT = 3, +}; + struct sbiret { long error; long value; @@ -125,4 +144,6 @@ struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0, unsigned long arg3, unsigned long arg4, unsigned long arg5); +bool guest_sbi_probe_extension(int extid, long *out_val); + #endif /* SELFTEST_KVM_PROCESSOR_H */ diff --git a/tools/testing/selftests/kvm/lib/riscv/processor.c b/tools/testing/selftests/kvm/lib/riscv/processor.c index 6905a4348380..7ca736fb4194 100644 --- a/tools/testing/selftests/kvm/lib/riscv/processor.c +++ b/tools/testing/selftests/kvm/lib/riscv/processor.c @@ -393,3 +393,22 @@ struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0, return ret; } + +bool guest_sbi_probe_extension(int extid, long *out_val) +{ + struct sbiret ret; + + ret = sbi_ecall(SBI_EXT_BASE, SBI_EXT_BASE_PROBE_EXT, extid, + 0, 0, 0, 0, 0); + + __GUEST_ASSERT(!ret.error || ret.error == SBI_ERR_NOT_SUPPORTED, + "ret.error=%ld, ret.value=%ld\n", ret.error, ret.value); + + if (ret.error == SBI_ERR_NOT_SUPPORTED) + return false; + + if (out_val) + *out_val = ret.value; + + return true; +} From 60b6e31c499643b25d4b3ccb4cc8e365dfdb8863 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Wed, 20 Dec 2023 17:00:25 +0100 Subject: [PATCH 232/336] RISC-V: KVM: selftests: Add steal_time test support With the introduction of steal-time accounting support for RISC-V KVM we can add RISC-V support to the steal_time test. Reviewed-by: Anup Patel Reviewed-by: Atish Patra Signed-off-by: Andrew Jones Signed-off-by: Anup Patel --- tools/testing/selftests/kvm/Makefile | 5 +- .../selftests/kvm/include/riscv/processor.h | 1 + tools/testing/selftests/kvm/steal_time.c | 99 +++++++++++++++++++ 3 files changed, 103 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index 3e0c36b8ddd5..017efabcf502 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -184,12 +184,13 @@ TEST_GEN_PROGS_s390x += kvm_binary_stats_test TEST_GEN_PROGS_riscv += demand_paging_test TEST_GEN_PROGS_riscv += dirty_log_test -TEST_GEN_PROGS_riscv += guest_print_test TEST_GEN_PROGS_riscv += get-reg-list +TEST_GEN_PROGS_riscv += guest_print_test +TEST_GEN_PROGS_riscv += kvm_binary_stats_test TEST_GEN_PROGS_riscv += kvm_create_max_vcpus TEST_GEN_PROGS_riscv += kvm_page_table_test TEST_GEN_PROGS_riscv += set_memory_region_test -TEST_GEN_PROGS_riscv += kvm_binary_stats_test +TEST_GEN_PROGS_riscv += steal_time SPLIT_TESTS += get-reg-list diff --git a/tools/testing/selftests/kvm/include/riscv/processor.h b/tools/testing/selftests/kvm/include/riscv/processor.h index dc50ad62e150..a0f9efe5a2a8 100644 --- a/tools/testing/selftests/kvm/include/riscv/processor.h +++ b/tools/testing/selftests/kvm/include/riscv/processor.h @@ -128,6 +128,7 @@ static inline uint64_t __kvm_reg_id(uint64_t type, uint64_t subtype, enum sbi_ext_id { SBI_EXT_BASE = 0x10, + SBI_EXT_STA = 0x535441, }; enum sbi_ext_base_fid { diff --git a/tools/testing/selftests/kvm/steal_time.c b/tools/testing/selftests/kvm/steal_time.c index 171adfb2a6cb..bae0c5026f82 100644 --- a/tools/testing/selftests/kvm/steal_time.c +++ b/tools/testing/selftests/kvm/steal_time.c @@ -11,7 +11,9 @@ #include #include #include +#ifndef __riscv #include +#endif #include "test_util.h" #include "kvm_util.h" @@ -203,6 +205,103 @@ static void steal_time_dump(struct kvm_vm *vm, uint32_t vcpu_idx) pr_info(" st_time: %ld\n", st->st_time); } +#elif defined(__riscv) + +/* SBI STA shmem must have 64-byte alignment */ +#define STEAL_TIME_SIZE ((sizeof(struct sta_struct) + 63) & ~63) + +static vm_paddr_t st_gpa[NR_VCPUS]; + +struct sta_struct { + uint32_t sequence; + uint32_t flags; + uint64_t steal; + uint8_t preempted; + uint8_t pad[47]; +} __packed; + +static void sta_set_shmem(vm_paddr_t gpa, unsigned long flags) +{ + unsigned long lo = (unsigned long)gpa; +#if __riscv_xlen == 32 + unsigned long hi = (unsigned long)(gpa >> 32); +#else + unsigned long hi = gpa == -1 ? -1 : 0; +#endif + struct sbiret ret = sbi_ecall(SBI_EXT_STA, 0, lo, hi, flags, 0, 0, 0); + + GUEST_ASSERT(ret.value == 0 && ret.error == 0); +} + +static void check_status(struct sta_struct *st) +{ + GUEST_ASSERT(!(READ_ONCE(st->sequence) & 1)); + GUEST_ASSERT(READ_ONCE(st->flags) == 0); + GUEST_ASSERT(READ_ONCE(st->preempted) == 0); +} + +static void guest_code(int cpu) +{ + struct sta_struct *st = st_gva[cpu]; + uint32_t sequence; + long out_val = 0; + bool probe; + + probe = guest_sbi_probe_extension(SBI_EXT_STA, &out_val); + GUEST_ASSERT(probe && out_val == 1); + + sta_set_shmem(st_gpa[cpu], 0); + GUEST_SYNC(0); + + check_status(st); + WRITE_ONCE(guest_stolen_time[cpu], st->steal); + sequence = READ_ONCE(st->sequence); + check_status(st); + GUEST_SYNC(1); + + check_status(st); + GUEST_ASSERT(sequence < READ_ONCE(st->sequence)); + WRITE_ONCE(guest_stolen_time[cpu], st->steal); + check_status(st); + GUEST_DONE(); +} + +static bool is_steal_time_supported(struct kvm_vcpu *vcpu) +{ + uint64_t id = RISCV_SBI_EXT_REG(KVM_RISCV_SBI_EXT_STA); + unsigned long enabled; + + vcpu_get_reg(vcpu, id, &enabled); + TEST_ASSERT(enabled == 0 || enabled == 1, "Expected boolean result"); + + return enabled; +} + +static void steal_time_init(struct kvm_vcpu *vcpu, uint32_t i) +{ + /* ST_GPA_BASE is identity mapped */ + st_gva[i] = (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE); + st_gpa[i] = addr_gva2gpa(vcpu->vm, (vm_vaddr_t)st_gva[i]); + sync_global_to_guest(vcpu->vm, st_gva[i]); + sync_global_to_guest(vcpu->vm, st_gpa[i]); +} + +static void steal_time_dump(struct kvm_vm *vm, uint32_t vcpu_idx) +{ + struct sta_struct *st = addr_gva2hva(vm, (ulong)st_gva[vcpu_idx]); + int i; + + pr_info("VCPU%d:\n", vcpu_idx); + pr_info(" sequence: %d\n", st->sequence); + pr_info(" flags: %d\n", st->flags); + pr_info(" steal: %"PRIu64"\n", st->steal); + pr_info(" preempted: %d\n", st->preempted); + pr_info(" pad: "); + for (i = 0; i < 47; ++i) + pr_info("%d", st->pad[i]); + pr_info("\n"); +} + #endif static void *do_steal_time(void *arg) From aad86da229bc9d0390dc2c02eb0db9ab1f50d059 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Wed, 20 Dec 2023 17:00:26 +0100 Subject: [PATCH 233/336] RISC-V: KVM: selftests: Add get-reg-list test for STA registers Add SBI STA and its two registers to the get-reg-list test. Reviewed-by: Anup Patel Reviewed-by: Atish Patra Signed-off-by: Andrew Jones Signed-off-by: Anup Patel --- .../selftests/kvm/riscv/get-reg-list.c | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tools/testing/selftests/kvm/riscv/get-reg-list.c b/tools/testing/selftests/kvm/riscv/get-reg-list.c index b8da2e86bf9c..6652108816db 100644 --- a/tools/testing/selftests/kvm/riscv/get-reg-list.c +++ b/tools/testing/selftests/kvm/riscv/get-reg-list.c @@ -71,6 +71,7 @@ bool filter_reg(__u64 reg) case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_HSM: case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_PMU: case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_DBCN: + case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_STA: case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_EXPERIMENTAL: case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_VENDOR: return true; @@ -461,6 +462,7 @@ static const char *sbi_ext_single_id_to_str(__u64 reg_off) KVM_SBI_EXT_ARR(KVM_RISCV_SBI_EXT_SRST), KVM_SBI_EXT_ARR(KVM_RISCV_SBI_EXT_HSM), KVM_SBI_EXT_ARR(KVM_RISCV_SBI_EXT_PMU), + KVM_SBI_EXT_ARR(KVM_RISCV_SBI_EXT_STA), KVM_SBI_EXT_ARR(KVM_RISCV_SBI_EXT_EXPERIMENTAL), KVM_SBI_EXT_ARR(KVM_RISCV_SBI_EXT_VENDOR), KVM_SBI_EXT_ARR(KVM_RISCV_SBI_EXT_DBCN), @@ -509,6 +511,32 @@ static const char *sbi_ext_id_to_str(const char *prefix, __u64 id) return strdup_printf("%lld | %lld /* UNKNOWN */", reg_subtype, reg_off); } +static const char *sbi_sta_id_to_str(__u64 reg_off) +{ + switch (reg_off) { + case 0: return "KVM_REG_RISCV_SBI_STA | KVM_REG_RISCV_SBI_STA_REG(shmem_lo)"; + case 1: return "KVM_REG_RISCV_SBI_STA | KVM_REG_RISCV_SBI_STA_REG(shmem_hi)"; + } + return strdup_printf("KVM_REG_RISCV_SBI_STA | %lld /* UNKNOWN */", reg_off); +} + +static const char *sbi_id_to_str(const char *prefix, __u64 id) +{ + __u64 reg_off = id & ~(REG_MASK | KVM_REG_RISCV_SBI_STATE); + __u64 reg_subtype = reg_off & KVM_REG_RISCV_SUBTYPE_MASK; + + assert((id & KVM_REG_RISCV_TYPE_MASK) == KVM_REG_RISCV_SBI_STATE); + + reg_off &= ~KVM_REG_RISCV_SUBTYPE_MASK; + + switch (reg_subtype) { + case KVM_REG_RISCV_SBI_STA: + return sbi_sta_id_to_str(reg_off); + } + + return strdup_printf("%lld | %lld /* UNKNOWN */", reg_subtype, reg_off); +} + void print_reg(const char *prefix, __u64 id) { const char *reg_size = NULL; @@ -565,6 +593,10 @@ void print_reg(const char *prefix, __u64 id) printf("\tKVM_REG_RISCV | %s | KVM_REG_RISCV_SBI_EXT | %s,\n", reg_size, sbi_ext_id_to_str(prefix, id)); break; + case KVM_REG_RISCV_SBI_STATE: + printf("\tKVM_REG_RISCV | %s | KVM_REG_RISCV_SBI_STATE | %s,\n", + reg_size, sbi_id_to_str(prefix, id)); + break; default: printf("\tKVM_REG_RISCV | %s | 0x%llx /* UNKNOWN */,\n", reg_size, id & ~REG_MASK); @@ -651,6 +683,12 @@ static __u64 sbi_base_regs[] = { KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_VENDOR, }; +static __u64 sbi_sta_regs[] = { + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_STA, + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_STATE | KVM_REG_RISCV_SBI_STA | KVM_REG_RISCV_SBI_STA_REG(shmem_lo), + KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_STATE | KVM_REG_RISCV_SBI_STA | KVM_REG_RISCV_SBI_STA_REG(shmem_hi), +}; + static __u64 zicbom_regs[] = { KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_CONFIG | KVM_REG_RISCV_CONFIG_REG(zicbom_block_size), KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICBOM, @@ -757,6 +795,9 @@ static __u64 fp_d_regs[] = { #define SUBLIST_SBI_BASE \ {"sbi-base", .feature_type = VCPU_FEATURE_SBI_EXT, .feature = KVM_RISCV_SBI_EXT_V01, \ .regs = sbi_base_regs, .regs_n = ARRAY_SIZE(sbi_base_regs),} +#define SUBLIST_SBI_STA \ + {"sbi-sta", .feature_type = VCPU_FEATURE_SBI_EXT, .feature = KVM_RISCV_SBI_EXT_STA, \ + .regs = sbi_sta_regs, .regs_n = ARRAY_SIZE(sbi_sta_regs),} #define SUBLIST_ZICBOM \ {"zicbom", .feature = KVM_RISCV_ISA_EXT_ZICBOM, .regs = zicbom_regs, .regs_n = ARRAY_SIZE(zicbom_regs),} #define SUBLIST_ZICBOZ \ @@ -832,6 +873,7 @@ static struct vcpu_reg_list config_sbi_##ext = { \ /* Note: The below list is alphabetically sorted. */ KVM_SBI_EXT_SUBLIST_CONFIG(base, BASE); +KVM_SBI_EXT_SUBLIST_CONFIG(sta, STA); KVM_SBI_EXT_SIMPLE_CONFIG(pmu, PMU); KVM_SBI_EXT_SIMPLE_CONFIG(dbcn, DBCN); @@ -858,6 +900,7 @@ KVM_ISA_EXT_SIMPLE_CONFIG(zihpm, ZIHPM); struct vcpu_reg_list *vcpu_configs[] = { &config_sbi_base, + &config_sbi_sta, &config_sbi_pmu, &config_sbi_dbcn, &config_aia, From 682fb5be353117b7321f1dfb65b7bb98cbfe59ab Mon Sep 17 00:00:00 2001 From: Thomas Bogendoerfer Date: Thu, 21 Dec 2023 13:54:03 +0100 Subject: [PATCH 234/336] MIPS: Allow vectored interrupt handler to reside everywhere for 64bit Setting up vector interrupts worked only with handlers, which resided in CKSEG0 space. This limits the kernel placement for 64bit platforms. By patching in the offset into vi_handlers[] instead of the full handler address, the vectored exception handler can load the address by itself and jump to it. Signed-off-by: Thomas Bogendoerfer Reviewed-by: Jiaxun Yang --- arch/mips/kernel/genex.S | 8 ++++---- arch/mips/kernel/traps.c | 9 +++------ 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/arch/mips/kernel/genex.S b/arch/mips/kernel/genex.S index b6de8e88c1bd..a572ce36a24f 100644 --- a/arch/mips/kernel/genex.S +++ b/arch/mips/kernel/genex.S @@ -272,18 +272,17 @@ NESTED(except_vec_vi, 0, sp) .set push .set noreorder PTR_LA v1, except_vec_vi_handler -FEXPORT(except_vec_vi_lui) - lui v0, 0 /* Patched */ jr v1 FEXPORT(except_vec_vi_ori) - ori v0, 0 /* Patched */ + ori v0, zero, 0 /* Offset in vi_handlers[] */ .set pop END(except_vec_vi) EXPORT(except_vec_vi_end) /* * Common Vectored Interrupt code - * Complete the register saves and invoke the handler which is passed in $v0 + * Complete the register saves and invoke the handler, $v0 holds + * offset into vi_handlers[] */ NESTED(except_vec_vi_handler, 0, sp) SAVE_TEMP @@ -331,6 +330,7 @@ NESTED(except_vec_vi_handler, 0, sp) /* Save task's sp on IRQ stack so that unwinding can follow it */ LONG_S s1, 0(sp) 2: + PTR_L v0, vi_handlers(v0) jalr v0 /* Restore sp */ diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index 246c6a6b0261..d90b18908692 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -2091,16 +2091,14 @@ static void *set_vi_srs_handler(int n, vi_handler_t addr, int srs) * If no shadow set is selected then use the default handler * that does normal register saving and standard interrupt exit */ - extern const u8 except_vec_vi[], except_vec_vi_lui[]; + extern const u8 except_vec_vi[]; extern const u8 except_vec_vi_ori[], except_vec_vi_end[]; extern const u8 rollback_except_vec_vi[]; const u8 *vec_start = using_rollback_handler() ? rollback_except_vec_vi : except_vec_vi; #if defined(CONFIG_CPU_MICROMIPS) || defined(CONFIG_CPU_BIG_ENDIAN) - const int lui_offset = except_vec_vi_lui - vec_start + 2; const int ori_offset = except_vec_vi_ori - vec_start + 2; #else - const int lui_offset = except_vec_vi_lui - vec_start; const int ori_offset = except_vec_vi_ori - vec_start; #endif const int handler_len = except_vec_vi_end - vec_start; @@ -2119,10 +2117,9 @@ static void *set_vi_srs_handler(int n, vi_handler_t addr, int srs) #else handler_len); #endif - h = (u16 *)(b + lui_offset); - *h = (handler >> 16) & 0xffff; + /* insert offset into vi_handlers[] */ h = (u16 *)(b + ori_offset); - *h = (handler & 0xffff); + *h = n * sizeof(handler); local_flush_icache_range((unsigned long)b, (unsigned long)(b+handler_len)); } From 8e1803900ef1b61ed33e6963d9e6a95028b41110 Mon Sep 17 00:00:00 2001 From: Thomas Bogendoerfer Date: Thu, 21 Dec 2023 13:54:04 +0100 Subject: [PATCH 235/336] MIPS: Remove unused shadow GPR support from vector irq setup Using shadow GPRs for vectored interrupts has never been used, time to remove it. Signed-off-by: Thomas Bogendoerfer --- arch/mips/kernel/traps.c | 94 +++++++++++++--------------------------- 1 file changed, 30 insertions(+), 64 deletions(-) diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index d90b18908692..c1b2b18b0505 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -2055,105 +2055,71 @@ static void do_default_vi(void) panic("Caught unexpected vectored interrupt."); } -static void *set_vi_srs_handler(int n, vi_handler_t addr, int srs) +void *set_vi_handler(int n, vi_handler_t addr) { + extern const u8 except_vec_vi[]; + extern const u8 except_vec_vi_ori[], except_vec_vi_end[]; + extern const u8 rollback_except_vec_vi[]; unsigned long handler; unsigned long old_handler = vi_handlers[n]; int srssets = current_cpu_data.srsets; u16 *h; unsigned char *b; + const u8 *vec_start; + int ori_offset; + int handler_len; BUG_ON(!cpu_has_veic && !cpu_has_vint); if (addr == NULL) { handler = (unsigned long) do_default_vi; - srs = 0; } else handler = (unsigned long) addr; vi_handlers[n] = handler; b = (unsigned char *)(ebase + 0x200 + n*VECTORSPACING); - if (srs >= srssets) - panic("Shadow register set %d not supported", srs); - if (cpu_has_veic) { if (board_bind_eic_interrupt) - board_bind_eic_interrupt(n, srs); + board_bind_eic_interrupt(n, 0); } else if (cpu_has_vint) { /* SRSMap is only defined if shadow sets are implemented */ if (srssets > 1) - change_c0_srsmap(0xf << n*4, srs << n*4); + change_c0_srsmap(0xf << n*4, 0 << n*4); } - if (srs == 0) { - /* - * If no shadow set is selected then use the default handler - * that does normal register saving and standard interrupt exit - */ - extern const u8 except_vec_vi[]; - extern const u8 except_vec_vi_ori[], except_vec_vi_end[]; - extern const u8 rollback_except_vec_vi[]; - const u8 *vec_start = using_rollback_handler() ? - rollback_except_vec_vi : except_vec_vi; + vec_start = using_rollback_handler() ? rollback_except_vec_vi : + except_vec_vi; #if defined(CONFIG_CPU_MICROMIPS) || defined(CONFIG_CPU_BIG_ENDIAN) - const int ori_offset = except_vec_vi_ori - vec_start + 2; + ori_offset = except_vec_vi_ori - vec_start + 2; #else - const int ori_offset = except_vec_vi_ori - vec_start; + ori_offset = except_vec_vi_ori - vec_start; #endif - const int handler_len = except_vec_vi_end - vec_start; + handler_len = except_vec_vi_end - vec_start; - if (handler_len > VECTORSPACING) { - /* - * Sigh... panicing won't help as the console - * is probably not configured :( - */ - panic("VECTORSPACING too small"); - } - - set_handler(((unsigned long)b - ebase), vec_start, -#ifdef CONFIG_CPU_MICROMIPS - (handler_len - 1)); -#else - handler_len); -#endif - /* insert offset into vi_handlers[] */ - h = (u16 *)(b + ori_offset); - *h = n * sizeof(handler); - local_flush_icache_range((unsigned long)b, - (unsigned long)(b+handler_len)); - } - else { + if (handler_len > VECTORSPACING) { /* - * In other cases jump directly to the interrupt handler. It - * is the handler's responsibility to save registers if required - * (eg hi/lo) and return from the exception using "eret". + * Sigh... panicing won't help as the console + * is probably not configured :( */ - u32 insn; - - h = (u16 *)b; - /* j handler */ -#ifdef CONFIG_CPU_MICROMIPS - insn = 0xd4000000 | (((u32)handler & 0x07ffffff) >> 1); -#else - insn = 0x08000000 | (((u32)handler & 0x0fffffff) >> 2); -#endif - h[0] = (insn >> 16) & 0xffff; - h[1] = insn & 0xffff; - h[2] = 0; - h[3] = 0; - local_flush_icache_range((unsigned long)b, - (unsigned long)(b+8)); + panic("VECTORSPACING too small"); } + set_handler(((unsigned long)b - ebase), vec_start, +#ifdef CONFIG_CPU_MICROMIPS + (handler_len - 1)); +#else + handler_len); +#endif + /* insert offset into vi_handlers[] */ + h = (u16 *)(b + ori_offset); + *h = n * sizeof(handler); + local_flush_icache_range((unsigned long)b, + (unsigned long)(b+handler_len)); + return (void *)old_handler; } -void *set_vi_handler(int n, vi_handler_t addr) -{ - return set_vi_srs_handler(n, addr, 0); -} - extern void tlb_init(void); /* From cbc911392c05762d27b96a376d4be90c5f21f99d Mon Sep 17 00:00:00 2001 From: Palmer Dabbelt Date: Fri, 10 Nov 2023 09:59:03 -0800 Subject: [PATCH 236/336] RISC-V: Remove the removed single-letter extensions There were a few single-letter extensions that we had references to floating around in the kernel, but that never ended up as actual ISA specs and have mostly been replaced by multi-letter extensions. This removes the references to those extensions. Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20231110175903.2631-1-palmer@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/hwcap.h | 6 ------ arch/riscv/kernel/cpufeature.c | 4 ---- 2 files changed, 10 deletions(-) diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index 2438d4685da6..e1a0bd4f394a 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -11,19 +11,13 @@ #include #define RISCV_ISA_EXT_a ('a' - 'a') -#define RISCV_ISA_EXT_b ('b' - 'a') #define RISCV_ISA_EXT_c ('c' - 'a') #define RISCV_ISA_EXT_d ('d' - 'a') #define RISCV_ISA_EXT_f ('f' - 'a') #define RISCV_ISA_EXT_h ('h' - 'a') #define RISCV_ISA_EXT_i ('i' - 'a') -#define RISCV_ISA_EXT_j ('j' - 'a') -#define RISCV_ISA_EXT_k ('k' - 'a') #define RISCV_ISA_EXT_m ('m' - 'a') -#define RISCV_ISA_EXT_p ('p' - 'a') #define RISCV_ISA_EXT_q ('q' - 'a') -#define RISCV_ISA_EXT_s ('s' - 'a') -#define RISCV_ISA_EXT_u ('u' - 'a') #define RISCV_ISA_EXT_v ('v' - 'a') /* diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index dc0ab3e97cd2..b06c8830a194 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -244,10 +244,6 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(d, RISCV_ISA_EXT_d), __RISCV_ISA_EXT_DATA(q, RISCV_ISA_EXT_q), __RISCV_ISA_EXT_DATA(c, RISCV_ISA_EXT_c), - __RISCV_ISA_EXT_DATA(b, RISCV_ISA_EXT_b), - __RISCV_ISA_EXT_DATA(k, RISCV_ISA_EXT_k), - __RISCV_ISA_EXT_DATA(j, RISCV_ISA_EXT_j), - __RISCV_ISA_EXT_DATA(p, RISCV_ISA_EXT_p), __RISCV_ISA_EXT_DATA(v, RISCV_ISA_EXT_v), __RISCV_ISA_EXT_DATA(h, RISCV_ISA_EXT_h), __RISCV_ISA_EXT_DATA(zicbom, RISCV_ISA_EXT_ZICBOM), From 36d842d654beba970e74ff0f6016c93623b82d37 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Wed, 22 Nov 2023 17:47:02 +0100 Subject: [PATCH 237/336] RISC-V: hwprobe: Clarify cpus size parameter The "count" parameter associated with the 'cpus' parameter of the hwprobe syscall is the size in bytes of 'cpus'. Naming it 'cpu_count' may mislead users (it did me) to think it's the number of CPUs that are or can be represented by 'cpus' instead. This is particularly easy (IMO) to get wrong since 'cpus' is documented to be defined by CPU_SET(3) and CPU_SET(3) also documents a CPU_COUNT() (the number of CPUs in set) macro. CPU_SET(3) refers to the size of cpu sets with 'setsize'. Adopt 'cpusetsize' for the hwprobe parameter and specifically state it is in bytes in Documentation/riscv/hwprobe.rst to clarify. Reviewed-by: Palmer Dabbelt Reviewed-by: Conor Dooley Signed-off-by: Andrew Jones Link: https://lore.kernel.org/r/20231122164700.127954-7-ajones@ventanamicro.com Signed-off-by: Palmer Dabbelt --- Documentation/arch/riscv/hwprobe.rst | 15 ++++++++------- arch/riscv/kernel/sys_riscv.c | 14 +++++++------- arch/riscv/kernel/vdso/hwprobe.c | 10 +++++----- tools/testing/selftests/riscv/hwprobe/hwprobe.c | 2 +- tools/testing/selftests/riscv/hwprobe/hwprobe.h | 2 +- .../testing/selftests/riscv/vector/vstate_prctl.c | 10 +--------- 6 files changed, 23 insertions(+), 30 deletions(-) diff --git a/Documentation/arch/riscv/hwprobe.rst b/Documentation/arch/riscv/hwprobe.rst index 7b2384de471f..132e9acaa8f4 100644 --- a/Documentation/arch/riscv/hwprobe.rst +++ b/Documentation/arch/riscv/hwprobe.rst @@ -12,7 +12,7 @@ is defined in :: }; long sys_riscv_hwprobe(struct riscv_hwprobe *pairs, size_t pair_count, - size_t cpu_count, cpu_set_t *cpus, + size_t cpusetsize, cpu_set_t *cpus, unsigned int flags); The arguments are split into three groups: an array of key-value pairs, a CPU @@ -20,12 +20,13 @@ set, and some flags. The key-value pairs are supplied with a count. Userspace must prepopulate the key field for each element, and the kernel will fill in the value if the key is recognized. If a key is unknown to the kernel, its key field will be cleared to -1, and its value set to 0. The CPU set is defined by -CPU_SET(3). For value-like keys (eg. vendor/arch/impl), the returned value will -be only be valid if all CPUs in the given set have the same value. Otherwise -1 -will be returned. For boolean-like keys, the value returned will be a logical -AND of the values for the specified CPUs. Usermode can supply NULL for cpus and -0 for cpu_count as a shortcut for all online CPUs. There are currently no flags, -this value must be zero for future compatibility. +CPU_SET(3) with size ``cpusetsize`` bytes. For value-like keys (eg. vendor, +arch, impl), the returned value will only be valid if all CPUs in the given set +have the same value. Otherwise -1 will be returned. For boolean-like keys, the +value returned will be a logical AND of the values for the specified CPUs. +Usermode can supply NULL for ``cpus`` and 0 for ``cpusetsize`` as a shortcut for +all online CPUs. There are currently no flags, this value must be zero for +future compatibility. On success 0 is returned, on failure a negative error code is returned. diff --git a/arch/riscv/kernel/sys_riscv.c b/arch/riscv/kernel/sys_riscv.c index c712037dbe10..5e4deb1308b8 100644 --- a/arch/riscv/kernel/sys_riscv.c +++ b/arch/riscv/kernel/sys_riscv.c @@ -246,7 +246,7 @@ static void hwprobe_one_pair(struct riscv_hwprobe *pair, } static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs, - size_t pair_count, size_t cpu_count, + size_t pair_count, size_t cpusetsize, unsigned long __user *cpus_user, unsigned int flags) { @@ -264,13 +264,13 @@ static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs, * 0 as a shortcut to all online CPUs. */ cpumask_clear(&cpus); - if (!cpu_count && !cpus_user) { + if (!cpusetsize && !cpus_user) { cpumask_copy(&cpus, cpu_online_mask); } else { - if (cpu_count > cpumask_size()) - cpu_count = cpumask_size(); + if (cpusetsize > cpumask_size()) + cpusetsize = cpumask_size(); - ret = copy_from_user(&cpus, cpus_user, cpu_count); + ret = copy_from_user(&cpus, cpus_user, cpusetsize); if (ret) return -EFAULT; @@ -347,10 +347,10 @@ arch_initcall_sync(init_hwprobe_vdso_data); #endif /* CONFIG_MMU */ SYSCALL_DEFINE5(riscv_hwprobe, struct riscv_hwprobe __user *, pairs, - size_t, pair_count, size_t, cpu_count, unsigned long __user *, + size_t, pair_count, size_t, cpusetsize, unsigned long __user *, cpus, unsigned int, flags) { - return do_riscv_hwprobe(pairs, pair_count, cpu_count, + return do_riscv_hwprobe(pairs, pair_count, cpusetsize, cpus, flags); } diff --git a/arch/riscv/kernel/vdso/hwprobe.c b/arch/riscv/kernel/vdso/hwprobe.c index cadf725ef798..026b7645c5ab 100644 --- a/arch/riscv/kernel/vdso/hwprobe.c +++ b/arch/riscv/kernel/vdso/hwprobe.c @@ -8,21 +8,21 @@ #include extern int riscv_hwprobe(struct riscv_hwprobe *pairs, size_t pair_count, - size_t cpu_count, unsigned long *cpus, + size_t cpusetsize, unsigned long *cpus, unsigned int flags); /* Add a prototype to avoid -Wmissing-prototypes warning. */ int __vdso_riscv_hwprobe(struct riscv_hwprobe *pairs, size_t pair_count, - size_t cpu_count, unsigned long *cpus, + size_t cpusetsize, unsigned long *cpus, unsigned int flags); int __vdso_riscv_hwprobe(struct riscv_hwprobe *pairs, size_t pair_count, - size_t cpu_count, unsigned long *cpus, + size_t cpusetsize, unsigned long *cpus, unsigned int flags) { const struct vdso_data *vd = __arch_get_vdso_data(); const struct arch_vdso_data *avd = &vd->arch_data; - bool all_cpus = !cpu_count && !cpus; + bool all_cpus = !cpusetsize && !cpus; struct riscv_hwprobe *p = pairs; struct riscv_hwprobe *end = pairs + pair_count; @@ -33,7 +33,7 @@ int __vdso_riscv_hwprobe(struct riscv_hwprobe *pairs, size_t pair_count, * masks. */ if ((flags != 0) || (!all_cpus && !avd->homogeneous_cpus)) - return riscv_hwprobe(pairs, pair_count, cpu_count, cpus, flags); + return riscv_hwprobe(pairs, pair_count, cpusetsize, cpus, flags); /* This is something we can handle, fill out the pairs. */ while (p < end) { diff --git a/tools/testing/selftests/riscv/hwprobe/hwprobe.c b/tools/testing/selftests/riscv/hwprobe/hwprobe.c index c474891df307..d53e0889b59e 100644 --- a/tools/testing/selftests/riscv/hwprobe/hwprobe.c +++ b/tools/testing/selftests/riscv/hwprobe/hwprobe.c @@ -47,7 +47,7 @@ int main(int argc, char **argv) ksft_test_result(out != 0, "Bad CPU set\n"); out = riscv_hwprobe(pairs, 8, 1, 0, 0); - ksft_test_result(out != 0, "NULL CPU set with non-zero count\n"); + ksft_test_result(out != 0, "NULL CPU set with non-zero size\n"); pairs[0].key = RISCV_HWPROBE_KEY_BASE_BEHAVIOR; out = riscv_hwprobe(pairs, 1, 1, &cpus, 0); diff --git a/tools/testing/selftests/riscv/hwprobe/hwprobe.h b/tools/testing/selftests/riscv/hwprobe/hwprobe.h index 721b0ce73a56..e3fccb390c4d 100644 --- a/tools/testing/selftests/riscv/hwprobe/hwprobe.h +++ b/tools/testing/selftests/riscv/hwprobe/hwprobe.h @@ -10,6 +10,6 @@ * contain the call. */ long riscv_hwprobe(struct riscv_hwprobe *pairs, size_t pair_count, - size_t cpu_count, unsigned long *cpus, unsigned int flags); + size_t cpusetsize, unsigned long *cpus, unsigned int flags); #endif diff --git a/tools/testing/selftests/riscv/vector/vstate_prctl.c b/tools/testing/selftests/riscv/vector/vstate_prctl.c index b348b475be57..8dcd399ef7fc 100644 --- a/tools/testing/selftests/riscv/vector/vstate_prctl.c +++ b/tools/testing/selftests/riscv/vector/vstate_prctl.c @@ -1,20 +1,12 @@ // SPDX-License-Identifier: GPL-2.0-only #include #include -#include #include #include +#include "../hwprobe/hwprobe.h" #include "../../kselftest.h" -/* - * Rather than relying on having a new enough libc to define this, just do it - * ourselves. This way we don't need to be coupled to a new-enough libc to - * contain the call. - */ -long riscv_hwprobe(struct riscv_hwprobe *pairs, size_t pair_count, - size_t cpu_count, unsigned long *cpus, unsigned int flags); - #define NEXT_PROGRAM "./vstate_exec_nolibc" static int launch_test(int test_inherit) { From 53b2b22850e1ff9e2729ce8efe2d846ed7d3bff4 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Wed, 22 Nov 2023 17:47:03 +0100 Subject: [PATCH 238/336] RISC-V: Move the hwprobe syscall to its own file As Palmer says, hwprobe is "sort of its own thing now, and it's only going to get bigger..." Suggested-by: Palmer Dabbelt Signed-off-by: Andrew Jones Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20231122164700.127954-8-ajones@ventanamicro.com Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/Makefile | 1 + arch/riscv/kernel/sys_hwprobe.c | 289 ++++++++++++++++++++++++++++++++ arch/riscv/kernel/sys_riscv.c | 285 ------------------------------- 3 files changed, 290 insertions(+), 285 deletions(-) create mode 100644 arch/riscv/kernel/sys_hwprobe.c diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index fee22a3d1b53..a1f5dc145574 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -50,6 +50,7 @@ obj-y += setup.o obj-y += signal.o obj-y += syscall_table.o obj-y += sys_riscv.o +obj-y += sys_hwprobe.o obj-y += time.o obj-y += traps.o obj-y += riscv_ksyms.o diff --git a/arch/riscv/kernel/sys_hwprobe.c b/arch/riscv/kernel/sys_hwprobe.c new file mode 100644 index 000000000000..c0ccf1f45aaa --- /dev/null +++ b/arch/riscv/kernel/sys_hwprobe.c @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * The hwprobe interface, for allowing userspace to probe to see which features + * are supported by the hardware. See Documentation/arch/riscv/hwprobe.rst for + * more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +static void hwprobe_arch_id(struct riscv_hwprobe *pair, + const struct cpumask *cpus) +{ + u64 id = -1ULL; + bool first = true; + int cpu; + + for_each_cpu(cpu, cpus) { + u64 cpu_id; + + switch (pair->key) { + case RISCV_HWPROBE_KEY_MVENDORID: + cpu_id = riscv_cached_mvendorid(cpu); + break; + case RISCV_HWPROBE_KEY_MIMPID: + cpu_id = riscv_cached_mimpid(cpu); + break; + case RISCV_HWPROBE_KEY_MARCHID: + cpu_id = riscv_cached_marchid(cpu); + break; + } + + if (first) { + id = cpu_id; + first = false; + } + + /* + * If there's a mismatch for the given set, return -1 in the + * value. + */ + if (id != cpu_id) { + id = -1ULL; + break; + } + } + + pair->value = id; +} + +static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, + const struct cpumask *cpus) +{ + int cpu; + u64 missing = 0; + + pair->value = 0; + if (has_fpu()) + pair->value |= RISCV_HWPROBE_IMA_FD; + + if (riscv_isa_extension_available(NULL, c)) + pair->value |= RISCV_HWPROBE_IMA_C; + + if (has_vector()) + pair->value |= RISCV_HWPROBE_IMA_V; + + /* + * Loop through and record extensions that 1) anyone has, and 2) anyone + * doesn't have. + */ + for_each_cpu(cpu, cpus) { + struct riscv_isainfo *isainfo = &hart_isa[cpu]; + +#define EXT_KEY(ext) \ + do { \ + if (__riscv_isa_extension_available(isainfo->isa, RISCV_ISA_EXT_##ext)) \ + pair->value |= RISCV_HWPROBE_EXT_##ext; \ + else \ + missing |= RISCV_HWPROBE_EXT_##ext; \ + } while (false) + + /* + * Only use EXT_KEY() for extensions which can be exposed to userspace, + * regardless of the kernel's configuration, as no other checks, besides + * presence in the hart_isa bitmap, are made. + */ + EXT_KEY(ZBA); + EXT_KEY(ZBB); + EXT_KEY(ZBS); + EXT_KEY(ZICBOZ); +#undef EXT_KEY + } + + /* Now turn off reporting features if any CPU is missing it. */ + pair->value &= ~missing; +} + +static bool hwprobe_ext0_has(const struct cpumask *cpus, unsigned long ext) +{ + struct riscv_hwprobe pair; + + hwprobe_isa_ext0(&pair, cpus); + return (pair.value & ext); +} + +static u64 hwprobe_misaligned(const struct cpumask *cpus) +{ + int cpu; + u64 perf = -1ULL; + + for_each_cpu(cpu, cpus) { + int this_perf = per_cpu(misaligned_access_speed, cpu); + + if (perf == -1ULL) + perf = this_perf; + + if (perf != this_perf) { + perf = RISCV_HWPROBE_MISALIGNED_UNKNOWN; + break; + } + } + + if (perf == -1ULL) + return RISCV_HWPROBE_MISALIGNED_UNKNOWN; + + return perf; +} + +static void hwprobe_one_pair(struct riscv_hwprobe *pair, + const struct cpumask *cpus) +{ + switch (pair->key) { + case RISCV_HWPROBE_KEY_MVENDORID: + case RISCV_HWPROBE_KEY_MARCHID: + case RISCV_HWPROBE_KEY_MIMPID: + hwprobe_arch_id(pair, cpus); + break; + /* + * The kernel already assumes that the base single-letter ISA + * extensions are supported on all harts, and only supports the + * IMA base, so just cheat a bit here and tell that to + * userspace. + */ + case RISCV_HWPROBE_KEY_BASE_BEHAVIOR: + pair->value = RISCV_HWPROBE_BASE_BEHAVIOR_IMA; + break; + + case RISCV_HWPROBE_KEY_IMA_EXT_0: + hwprobe_isa_ext0(pair, cpus); + break; + + case RISCV_HWPROBE_KEY_CPUPERF_0: + pair->value = hwprobe_misaligned(cpus); + break; + + case RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE: + pair->value = 0; + if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ)) + pair->value = riscv_cboz_block_size; + break; + + /* + * For forward compatibility, unknown keys don't fail the whole + * call, but get their element key set to -1 and value set to 0 + * indicating they're unrecognized. + */ + default: + pair->key = -1; + pair->value = 0; + break; + } +} + +static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs, + size_t pair_count, size_t cpusetsize, + unsigned long __user *cpus_user, + unsigned int flags) +{ + size_t out; + int ret; + cpumask_t cpus; + + /* Check the reserved flags. */ + if (flags != 0) + return -EINVAL; + + /* + * The interface supports taking in a CPU mask, and returns values that + * are consistent across that mask. Allow userspace to specify NULL and + * 0 as a shortcut to all online CPUs. + */ + cpumask_clear(&cpus); + if (!cpusetsize && !cpus_user) { + cpumask_copy(&cpus, cpu_online_mask); + } else { + if (cpusetsize > cpumask_size()) + cpusetsize = cpumask_size(); + + ret = copy_from_user(&cpus, cpus_user, cpusetsize); + if (ret) + return -EFAULT; + + /* + * Userspace must provide at least one online CPU, without that + * there's no way to define what is supported. + */ + cpumask_and(&cpus, &cpus, cpu_online_mask); + if (cpumask_empty(&cpus)) + return -EINVAL; + } + + for (out = 0; out < pair_count; out++, pairs++) { + struct riscv_hwprobe pair; + + if (get_user(pair.key, &pairs->key)) + return -EFAULT; + + pair.value = 0; + hwprobe_one_pair(&pair, &cpus); + ret = put_user(pair.key, &pairs->key); + if (ret == 0) + ret = put_user(pair.value, &pairs->value); + + if (ret) + return -EFAULT; + } + + return 0; +} + +#ifdef CONFIG_MMU + +static int __init init_hwprobe_vdso_data(void) +{ + struct vdso_data *vd = __arch_get_k_vdso_data(); + struct arch_vdso_data *avd = &vd->arch_data; + u64 id_bitsmash = 0; + struct riscv_hwprobe pair; + int key; + + /* + * Initialize vDSO data with the answers for the "all CPUs" case, to + * save a syscall in the common case. + */ + for (key = 0; key <= RISCV_HWPROBE_MAX_KEY; key++) { + pair.key = key; + hwprobe_one_pair(&pair, cpu_online_mask); + + WARN_ON_ONCE(pair.key < 0); + + avd->all_cpu_hwprobe_values[key] = pair.value; + /* + * Smash together the vendor, arch, and impl IDs to see if + * they're all 0 or any negative. + */ + if (key <= RISCV_HWPROBE_KEY_MIMPID) + id_bitsmash |= pair.value; + } + + /* + * If the arch, vendor, and implementation ID are all the same across + * all harts, then assume all CPUs are the same, and allow the vDSO to + * answer queries for arbitrary masks. However if all values are 0 (not + * populated) or any value returns -1 (varies across CPUs), then the + * vDSO should defer to the kernel for exotic cpu masks. + */ + avd->homogeneous_cpus = id_bitsmash != 0 && id_bitsmash != -1; + return 0; +} + +arch_initcall_sync(init_hwprobe_vdso_data); + +#endif /* CONFIG_MMU */ + +SYSCALL_DEFINE5(riscv_hwprobe, struct riscv_hwprobe __user *, pairs, + size_t, pair_count, size_t, cpusetsize, unsigned long __user *, + cpus, unsigned int, flags) +{ + return do_riscv_hwprobe(pairs, pair_count, cpusetsize, + cpus, flags); +} diff --git a/arch/riscv/kernel/sys_riscv.c b/arch/riscv/kernel/sys_riscv.c index 5e4deb1308b8..f1c1416a9f1e 100644 --- a/arch/riscv/kernel/sys_riscv.c +++ b/arch/riscv/kernel/sys_riscv.c @@ -7,15 +7,7 @@ #include #include -#include -#include -#include -#include -#include -#include -#include #include -#include static long riscv_sys_mmap(unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, @@ -77,283 +69,6 @@ SYSCALL_DEFINE3(riscv_flush_icache, uintptr_t, start, uintptr_t, end, return 0; } -/* - * The hwprobe interface, for allowing userspace to probe to see which features - * are supported by the hardware. See Documentation/arch/riscv/hwprobe.rst for more - * details. - */ -static void hwprobe_arch_id(struct riscv_hwprobe *pair, - const struct cpumask *cpus) -{ - u64 id = -1ULL; - bool first = true; - int cpu; - - for_each_cpu(cpu, cpus) { - u64 cpu_id; - - switch (pair->key) { - case RISCV_HWPROBE_KEY_MVENDORID: - cpu_id = riscv_cached_mvendorid(cpu); - break; - case RISCV_HWPROBE_KEY_MIMPID: - cpu_id = riscv_cached_mimpid(cpu); - break; - case RISCV_HWPROBE_KEY_MARCHID: - cpu_id = riscv_cached_marchid(cpu); - break; - } - - if (first) { - id = cpu_id; - first = false; - } - - /* - * If there's a mismatch for the given set, return -1 in the - * value. - */ - if (id != cpu_id) { - id = -1ULL; - break; - } - } - - pair->value = id; -} - -static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, - const struct cpumask *cpus) -{ - int cpu; - u64 missing = 0; - - pair->value = 0; - if (has_fpu()) - pair->value |= RISCV_HWPROBE_IMA_FD; - - if (riscv_isa_extension_available(NULL, c)) - pair->value |= RISCV_HWPROBE_IMA_C; - - if (has_vector()) - pair->value |= RISCV_HWPROBE_IMA_V; - - /* - * Loop through and record extensions that 1) anyone has, and 2) anyone - * doesn't have. - */ - for_each_cpu(cpu, cpus) { - struct riscv_isainfo *isainfo = &hart_isa[cpu]; - -#define EXT_KEY(ext) \ - do { \ - if (__riscv_isa_extension_available(isainfo->isa, RISCV_ISA_EXT_##ext)) \ - pair->value |= RISCV_HWPROBE_EXT_##ext; \ - else \ - missing |= RISCV_HWPROBE_EXT_##ext; \ - } while (false) - - /* - * Only use EXT_KEY() for extensions which can be exposed to userspace, - * regardless of the kernel's configuration, as no other checks, besides - * presence in the hart_isa bitmap, are made. - */ - EXT_KEY(ZBA); - EXT_KEY(ZBB); - EXT_KEY(ZBS); - EXT_KEY(ZICBOZ); -#undef EXT_KEY - } - - /* Now turn off reporting features if any CPU is missing it. */ - pair->value &= ~missing; -} - -static bool hwprobe_ext0_has(const struct cpumask *cpus, unsigned long ext) -{ - struct riscv_hwprobe pair; - - hwprobe_isa_ext0(&pair, cpus); - return (pair.value & ext); -} - -static u64 hwprobe_misaligned(const struct cpumask *cpus) -{ - int cpu; - u64 perf = -1ULL; - - for_each_cpu(cpu, cpus) { - int this_perf = per_cpu(misaligned_access_speed, cpu); - - if (perf == -1ULL) - perf = this_perf; - - if (perf != this_perf) { - perf = RISCV_HWPROBE_MISALIGNED_UNKNOWN; - break; - } - } - - if (perf == -1ULL) - return RISCV_HWPROBE_MISALIGNED_UNKNOWN; - - return perf; -} - -static void hwprobe_one_pair(struct riscv_hwprobe *pair, - const struct cpumask *cpus) -{ - switch (pair->key) { - case RISCV_HWPROBE_KEY_MVENDORID: - case RISCV_HWPROBE_KEY_MARCHID: - case RISCV_HWPROBE_KEY_MIMPID: - hwprobe_arch_id(pair, cpus); - break; - /* - * The kernel already assumes that the base single-letter ISA - * extensions are supported on all harts, and only supports the - * IMA base, so just cheat a bit here and tell that to - * userspace. - */ - case RISCV_HWPROBE_KEY_BASE_BEHAVIOR: - pair->value = RISCV_HWPROBE_BASE_BEHAVIOR_IMA; - break; - - case RISCV_HWPROBE_KEY_IMA_EXT_0: - hwprobe_isa_ext0(pair, cpus); - break; - - case RISCV_HWPROBE_KEY_CPUPERF_0: - pair->value = hwprobe_misaligned(cpus); - break; - - case RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE: - pair->value = 0; - if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ)) - pair->value = riscv_cboz_block_size; - break; - - /* - * For forward compatibility, unknown keys don't fail the whole - * call, but get their element key set to -1 and value set to 0 - * indicating they're unrecognized. - */ - default: - pair->key = -1; - pair->value = 0; - break; - } -} - -static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs, - size_t pair_count, size_t cpusetsize, - unsigned long __user *cpus_user, - unsigned int flags) -{ - size_t out; - int ret; - cpumask_t cpus; - - /* Check the reserved flags. */ - if (flags != 0) - return -EINVAL; - - /* - * The interface supports taking in a CPU mask, and returns values that - * are consistent across that mask. Allow userspace to specify NULL and - * 0 as a shortcut to all online CPUs. - */ - cpumask_clear(&cpus); - if (!cpusetsize && !cpus_user) { - cpumask_copy(&cpus, cpu_online_mask); - } else { - if (cpusetsize > cpumask_size()) - cpusetsize = cpumask_size(); - - ret = copy_from_user(&cpus, cpus_user, cpusetsize); - if (ret) - return -EFAULT; - - /* - * Userspace must provide at least one online CPU, without that - * there's no way to define what is supported. - */ - cpumask_and(&cpus, &cpus, cpu_online_mask); - if (cpumask_empty(&cpus)) - return -EINVAL; - } - - for (out = 0; out < pair_count; out++, pairs++) { - struct riscv_hwprobe pair; - - if (get_user(pair.key, &pairs->key)) - return -EFAULT; - - pair.value = 0; - hwprobe_one_pair(&pair, &cpus); - ret = put_user(pair.key, &pairs->key); - if (ret == 0) - ret = put_user(pair.value, &pairs->value); - - if (ret) - return -EFAULT; - } - - return 0; -} - -#ifdef CONFIG_MMU - -static int __init init_hwprobe_vdso_data(void) -{ - struct vdso_data *vd = __arch_get_k_vdso_data(); - struct arch_vdso_data *avd = &vd->arch_data; - u64 id_bitsmash = 0; - struct riscv_hwprobe pair; - int key; - - /* - * Initialize vDSO data with the answers for the "all CPUs" case, to - * save a syscall in the common case. - */ - for (key = 0; key <= RISCV_HWPROBE_MAX_KEY; key++) { - pair.key = key; - hwprobe_one_pair(&pair, cpu_online_mask); - - WARN_ON_ONCE(pair.key < 0); - - avd->all_cpu_hwprobe_values[key] = pair.value; - /* - * Smash together the vendor, arch, and impl IDs to see if - * they're all 0 or any negative. - */ - if (key <= RISCV_HWPROBE_KEY_MIMPID) - id_bitsmash |= pair.value; - } - - /* - * If the arch, vendor, and implementation ID are all the same across - * all harts, then assume all CPUs are the same, and allow the vDSO to - * answer queries for arbitrary masks. However if all values are 0 (not - * populated) or any value returns -1 (varies across CPUs), then the - * vDSO should defer to the kernel for exotic cpu masks. - */ - avd->homogeneous_cpus = id_bitsmash != 0 && id_bitsmash != -1; - return 0; -} - -arch_initcall_sync(init_hwprobe_vdso_data); - -#endif /* CONFIG_MMU */ - -SYSCALL_DEFINE5(riscv_hwprobe, struct riscv_hwprobe __user *, pairs, - size_t, pair_count, size_t, cpusetsize, unsigned long __user *, - cpus, unsigned int, flags) -{ - return do_riscv_hwprobe(pairs, pair_count, cpusetsize, - cpus, flags); -} - /* Not defined using SYSCALL_DEFINE0 to avoid error injection */ asmlinkage long __riscv_sys_ni_syscall(const struct pt_regs *__unused) { From e178bf146e4b8c774a7b00aa2419e400f4f7894f Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Wed, 22 Nov 2023 17:47:04 +0100 Subject: [PATCH 239/336] RISC-V: hwprobe: Introduce which-cpus flag Introduce the first flag for the hwprobe syscall. The flag basically reverses its behavior, i.e. instead of populating the values of keys for a given set of cpus, the set of cpus after the call is the result of finding a set which supports the values of the keys. In order to do this, we implement a pair compare function which takes the type of value (a single value vs. a bitmask of booleans) into consideration. We also implement vdso support for the new flag. Signed-off-by: Andrew Jones Reviewed-by: Evan Green Link: https://lore.kernel.org/r/20231122164700.127954-9-ajones@ventanamicro.com Signed-off-by: Palmer Dabbelt --- Documentation/arch/riscv/hwprobe.rst | 17 ++++- arch/riscv/include/asm/hwprobe.h | 24 +++++++ arch/riscv/include/uapi/asm/hwprobe.h | 3 + arch/riscv/kernel/sys_hwprobe.c | 94 +++++++++++++++++++++++++-- arch/riscv/kernel/vdso/hwprobe.c | 80 ++++++++++++++++++++--- 5 files changed, 204 insertions(+), 14 deletions(-) diff --git a/Documentation/arch/riscv/hwprobe.rst b/Documentation/arch/riscv/hwprobe.rst index 132e9acaa8f4..12f9b0a043ac 100644 --- a/Documentation/arch/riscv/hwprobe.rst +++ b/Documentation/arch/riscv/hwprobe.rst @@ -25,8 +25,21 @@ arch, impl), the returned value will only be valid if all CPUs in the given set have the same value. Otherwise -1 will be returned. For boolean-like keys, the value returned will be a logical AND of the values for the specified CPUs. Usermode can supply NULL for ``cpus`` and 0 for ``cpusetsize`` as a shortcut for -all online CPUs. There are currently no flags, this value must be zero for -future compatibility. +all online CPUs. The currently supported flags are: + +* :c:macro:`RISCV_HWPROBE_WHICH_CPUS`: This flag basically reverses the behavior + of sys_riscv_hwprobe(). Instead of populating the values of keys for a given + set of CPUs, the values of each key are given and the set of CPUs is reduced + by sys_riscv_hwprobe() to only those which match each of the key-value pairs. + How matching is done depends on the key type. For value-like keys, matching + means to be the exact same as the value. For boolean-like keys, matching + means the result of a logical AND of the pair's value with the CPU's value is + exactly the same as the pair's value. Additionally, when ``cpus`` is an empty + set, then it is initialized to all online CPUs which fit within it, i.e. the + CPU set returned is the reduction of all the online CPUs which can be + represented with a CPU set of size ``cpusetsize``. + +All other flags are reserved for future compatibility and must be zero. On success 0 is returned, on failure a negative error code is returned. diff --git a/arch/riscv/include/asm/hwprobe.h b/arch/riscv/include/asm/hwprobe.h index 5c48f48e79a6..630507dff5ea 100644 --- a/arch/riscv/include/asm/hwprobe.h +++ b/arch/riscv/include/asm/hwprobe.h @@ -15,4 +15,28 @@ static inline bool riscv_hwprobe_key_is_valid(__s64 key) return key >= 0 && key <= RISCV_HWPROBE_MAX_KEY; } +static inline bool hwprobe_key_is_bitmask(__s64 key) +{ + switch (key) { + case RISCV_HWPROBE_KEY_BASE_BEHAVIOR: + case RISCV_HWPROBE_KEY_IMA_EXT_0: + case RISCV_HWPROBE_KEY_CPUPERF_0: + return true; + } + + return false; +} + +static inline bool riscv_hwprobe_pair_cmp(struct riscv_hwprobe *pair, + struct riscv_hwprobe *other_pair) +{ + if (pair->key != other_pair->key) + return false; + + if (hwprobe_key_is_bitmask(pair->key)) + return (pair->value & other_pair->value) == other_pair->value; + + return pair->value == other_pair->value; +} + #endif diff --git a/arch/riscv/include/uapi/asm/hwprobe.h b/arch/riscv/include/uapi/asm/hwprobe.h index b659ffcfcdb4..7aa7d5c71e79 100644 --- a/arch/riscv/include/uapi/asm/hwprobe.h +++ b/arch/riscv/include/uapi/asm/hwprobe.h @@ -40,4 +40,7 @@ struct riscv_hwprobe { #define RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE 6 /* Increase RISCV_HWPROBE_MAX_KEY when adding items. */ +/* Flags */ +#define RISCV_HWPROBE_WHICH_CPUS (1 << 0) + #endif diff --git a/arch/riscv/kernel/sys_hwprobe.c b/arch/riscv/kernel/sys_hwprobe.c index c0ccf1f45aaa..b7cfb26ce31c 100644 --- a/arch/riscv/kernel/sys_hwprobe.c +++ b/arch/riscv/kernel/sys_hwprobe.c @@ -179,10 +179,10 @@ static void hwprobe_one_pair(struct riscv_hwprobe *pair, } } -static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs, - size_t pair_count, size_t cpusetsize, - unsigned long __user *cpus_user, - unsigned int flags) +static int hwprobe_get_values(struct riscv_hwprobe __user *pairs, + size_t pair_count, size_t cpusetsize, + unsigned long __user *cpus_user, + unsigned int flags) { size_t out; int ret; @@ -236,6 +236,92 @@ static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs, return 0; } +static int hwprobe_get_cpus(struct riscv_hwprobe __user *pairs, + size_t pair_count, size_t cpusetsize, + unsigned long __user *cpus_user, + unsigned int flags) +{ + cpumask_t cpus, one_cpu; + bool clear_all = false; + size_t i; + int ret; + + if (flags != RISCV_HWPROBE_WHICH_CPUS) + return -EINVAL; + + if (!cpusetsize || !cpus_user) + return -EINVAL; + + if (cpusetsize > cpumask_size()) + cpusetsize = cpumask_size(); + + ret = copy_from_user(&cpus, cpus_user, cpusetsize); + if (ret) + return -EFAULT; + + if (cpumask_empty(&cpus)) + cpumask_copy(&cpus, cpu_online_mask); + + cpumask_and(&cpus, &cpus, cpu_online_mask); + + cpumask_clear(&one_cpu); + + for (i = 0; i < pair_count; i++) { + struct riscv_hwprobe pair, tmp; + int cpu; + + ret = copy_from_user(&pair, &pairs[i], sizeof(pair)); + if (ret) + return -EFAULT; + + if (!riscv_hwprobe_key_is_valid(pair.key)) { + clear_all = true; + pair = (struct riscv_hwprobe){ .key = -1, }; + ret = copy_to_user(&pairs[i], &pair, sizeof(pair)); + if (ret) + return -EFAULT; + } + + if (clear_all) + continue; + + tmp = (struct riscv_hwprobe){ .key = pair.key, }; + + for_each_cpu(cpu, &cpus) { + cpumask_set_cpu(cpu, &one_cpu); + + hwprobe_one_pair(&tmp, &one_cpu); + + if (!riscv_hwprobe_pair_cmp(&tmp, &pair)) + cpumask_clear_cpu(cpu, &cpus); + + cpumask_clear_cpu(cpu, &one_cpu); + } + } + + if (clear_all) + cpumask_clear(&cpus); + + ret = copy_to_user(cpus_user, &cpus, cpusetsize); + if (ret) + return -EFAULT; + + return 0; +} + +static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs, + size_t pair_count, size_t cpusetsize, + unsigned long __user *cpus_user, + unsigned int flags) +{ + if (flags & RISCV_HWPROBE_WHICH_CPUS) + return hwprobe_get_cpus(pairs, pair_count, cpusetsize, + cpus_user, flags); + + return hwprobe_get_values(pairs, pair_count, cpusetsize, + cpus_user, flags); +} + #ifdef CONFIG_MMU static int __init init_hwprobe_vdso_data(void) diff --git a/arch/riscv/kernel/vdso/hwprobe.c b/arch/riscv/kernel/vdso/hwprobe.c index 026b7645c5ab..1e926e4b5881 100644 --- a/arch/riscv/kernel/vdso/hwprobe.c +++ b/arch/riscv/kernel/vdso/hwprobe.c @@ -3,6 +3,7 @@ * Copyright 2023 Rivos, Inc */ +#include #include #include #include @@ -11,14 +12,9 @@ extern int riscv_hwprobe(struct riscv_hwprobe *pairs, size_t pair_count, size_t cpusetsize, unsigned long *cpus, unsigned int flags); -/* Add a prototype to avoid -Wmissing-prototypes warning. */ -int __vdso_riscv_hwprobe(struct riscv_hwprobe *pairs, size_t pair_count, - size_t cpusetsize, unsigned long *cpus, - unsigned int flags); - -int __vdso_riscv_hwprobe(struct riscv_hwprobe *pairs, size_t pair_count, - size_t cpusetsize, unsigned long *cpus, - unsigned int flags) +static int riscv_vdso_get_values(struct riscv_hwprobe *pairs, size_t pair_count, + size_t cpusetsize, unsigned long *cpus, + unsigned int flags) { const struct vdso_data *vd = __arch_get_vdso_data(); const struct arch_vdso_data *avd = &vd->arch_data; @@ -50,3 +46,71 @@ int __vdso_riscv_hwprobe(struct riscv_hwprobe *pairs, size_t pair_count, return 0; } + +static int riscv_vdso_get_cpus(struct riscv_hwprobe *pairs, size_t pair_count, + size_t cpusetsize, unsigned long *cpus, + unsigned int flags) +{ + const struct vdso_data *vd = __arch_get_vdso_data(); + const struct arch_vdso_data *avd = &vd->arch_data; + struct riscv_hwprobe *p = pairs; + struct riscv_hwprobe *end = pairs + pair_count; + unsigned char *c = (unsigned char *)cpus; + bool empty_cpus = true; + bool clear_all = false; + int i; + + if (!cpusetsize || !cpus) + return -EINVAL; + + for (i = 0; i < cpusetsize; i++) { + if (c[i]) { + empty_cpus = false; + break; + } + } + + if (empty_cpus || flags != RISCV_HWPROBE_WHICH_CPUS || !avd->homogeneous_cpus) + return riscv_hwprobe(pairs, pair_count, cpusetsize, cpus, flags); + + while (p < end) { + if (riscv_hwprobe_key_is_valid(p->key)) { + struct riscv_hwprobe t = { + .key = p->key, + .value = avd->all_cpu_hwprobe_values[p->key], + }; + + if (!riscv_hwprobe_pair_cmp(&t, p)) + clear_all = true; + } else { + clear_all = true; + p->key = -1; + p->value = 0; + } + p++; + } + + if (clear_all) { + for (i = 0; i < cpusetsize; i++) + c[i] = 0; + } + + return 0; +} + +/* Add a prototype to avoid -Wmissing-prototypes warning. */ +int __vdso_riscv_hwprobe(struct riscv_hwprobe *pairs, size_t pair_count, + size_t cpusetsize, unsigned long *cpus, + unsigned int flags); + +int __vdso_riscv_hwprobe(struct riscv_hwprobe *pairs, size_t pair_count, + size_t cpusetsize, unsigned long *cpus, + unsigned int flags) +{ + if (flags & RISCV_HWPROBE_WHICH_CPUS) + return riscv_vdso_get_cpus(pairs, pair_count, cpusetsize, + cpus, flags); + + return riscv_vdso_get_values(pairs, pair_count, cpusetsize, + cpus, flags); +} From ef7d6abb2cf57a18d34b332f0865c4d03bd810a3 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Wed, 22 Nov 2023 17:47:05 +0100 Subject: [PATCH 240/336] RISC-V: selftests: Add which-cpus hwprobe test Test the RISCV_HWPROBE_WHICH_CPUS flag of hwprobe. The test also has a command line interface in order to get the cpu list for arbitrary hwprobe pairs. Reviewed-by: Palmer Dabbelt Signed-off-by: Andrew Jones Link: https://lore.kernel.org/r/20231122164700.127954-10-ajones@ventanamicro.com Signed-off-by: Palmer Dabbelt --- .../testing/selftests/riscv/hwprobe/Makefile | 5 +- .../selftests/riscv/hwprobe/which-cpus.c | 154 ++++++++++++++++++ 2 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/riscv/hwprobe/which-cpus.c diff --git a/tools/testing/selftests/riscv/hwprobe/Makefile b/tools/testing/selftests/riscv/hwprobe/Makefile index f224b84591fb..cec81610a5f2 100644 --- a/tools/testing/selftests/riscv/hwprobe/Makefile +++ b/tools/testing/selftests/riscv/hwprobe/Makefile @@ -4,7 +4,7 @@ CFLAGS += -I$(top_srcdir)/tools/include -TEST_GEN_PROGS := hwprobe cbo +TEST_GEN_PROGS := hwprobe cbo which-cpus include ../../lib.mk @@ -13,3 +13,6 @@ $(OUTPUT)/hwprobe: hwprobe.c sys_hwprobe.S $(OUTPUT)/cbo: cbo.c sys_hwprobe.S $(CC) -static -o$@ $(CFLAGS) $(LDFLAGS) $^ + +$(OUTPUT)/which-cpus: which-cpus.c sys_hwprobe.S + $(CC) -static -o$@ $(CFLAGS) $(LDFLAGS) $^ diff --git a/tools/testing/selftests/riscv/hwprobe/which-cpus.c b/tools/testing/selftests/riscv/hwprobe/which-cpus.c new file mode 100644 index 000000000000..82c121412dfc --- /dev/null +++ b/tools/testing/selftests/riscv/hwprobe/which-cpus.c @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2023 Ventana Micro Systems Inc. + * + * Test the RISCV_HWPROBE_WHICH_CPUS flag of hwprobe. Also provides a command + * line interface to get the cpu list for arbitrary hwprobe pairs. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +#include "hwprobe.h" +#include "../../kselftest.h" + +static void help(void) +{ + printf("\n" + "which-cpus: [-h] [ [ ...]]\n\n" + " Without parameters, tests the RISCV_HWPROBE_WHICH_CPUS flag of hwprobe.\n" + " With parameters, where each parameter is a hwprobe pair written as\n" + " , outputs the cpulist for cpus which all match the given set\n" + " of pairs. 'key' and 'value' should be in numeric form, e.g. 4=0x3b\n"); +} + +static void print_cpulist(cpu_set_t *cpus) +{ + int start = 0, end = 0; + + if (!CPU_COUNT(cpus)) { + printf("cpus: None\n"); + return; + } + + printf("cpus:"); + for (int i = 0, c = 0; i < CPU_COUNT(cpus); i++, c++) { + if (start != end && !CPU_ISSET(c, cpus)) + printf("-%d", end); + + while (!CPU_ISSET(c, cpus)) + ++c; + + if (i != 0 && c == end + 1) { + end = c; + continue; + } + + printf("%c%d", i == 0 ? ' ' : ',', c); + start = end = c; + } + if (start != end) + printf("-%d", end); + printf("\n"); +} + +static void do_which_cpus(int argc, char **argv, cpu_set_t *cpus) +{ + struct riscv_hwprobe *pairs; + int nr_pairs = argc - 1; + char *start, *end; + int rc; + + pairs = malloc(nr_pairs * sizeof(struct riscv_hwprobe)); + assert(pairs); + + for (int i = 0; i < nr_pairs; i++) { + start = argv[i + 1]; + pairs[i].key = strtol(start, &end, 0); + assert(end != start && *end == '='); + start = end + 1; + pairs[i].value = strtoul(start, &end, 0); + assert(end != start && *end == '\0'); + } + + rc = riscv_hwprobe(pairs, nr_pairs, sizeof(cpu_set_t), (unsigned long *)cpus, RISCV_HWPROBE_WHICH_CPUS); + assert(rc == 0); + print_cpulist(cpus); + free(pairs); +} + +int main(int argc, char **argv) +{ + struct riscv_hwprobe pairs[2]; + cpu_set_t cpus_aff, cpus; + __u64 ext0_all; + long rc; + + rc = sched_getaffinity(0, sizeof(cpu_set_t), &cpus_aff); + assert(rc == 0); + + if (argc > 1) { + if (!strcmp(argv[1], "-h")) + help(); + else + do_which_cpus(argc, argv, &cpus_aff); + return 0; + } + + ksft_print_header(); + ksft_set_plan(7); + + pairs[0] = (struct riscv_hwprobe){ .key = RISCV_HWPROBE_KEY_BASE_BEHAVIOR, }; + rc = riscv_hwprobe(pairs, 1, 0, NULL, 0); + assert(rc == 0 && pairs[0].key == RISCV_HWPROBE_KEY_BASE_BEHAVIOR && + pairs[0].value == RISCV_HWPROBE_BASE_BEHAVIOR_IMA); + + pairs[0] = (struct riscv_hwprobe){ .key = RISCV_HWPROBE_KEY_IMA_EXT_0, }; + rc = riscv_hwprobe(pairs, 1, 0, NULL, 0); + assert(rc == 0 && pairs[0].key == RISCV_HWPROBE_KEY_IMA_EXT_0); + ext0_all = pairs[0].value; + + pairs[0] = (struct riscv_hwprobe){ .key = RISCV_HWPROBE_KEY_BASE_BEHAVIOR, .value = RISCV_HWPROBE_BASE_BEHAVIOR_IMA, }; + CPU_ZERO(&cpus); + rc = riscv_hwprobe(pairs, 1, 0, (unsigned long *)&cpus, RISCV_HWPROBE_WHICH_CPUS); + ksft_test_result(rc == -EINVAL, "no cpusetsize\n"); + + pairs[0] = (struct riscv_hwprobe){ .key = RISCV_HWPROBE_KEY_BASE_BEHAVIOR, .value = RISCV_HWPROBE_BASE_BEHAVIOR_IMA, }; + rc = riscv_hwprobe(pairs, 1, sizeof(cpu_set_t), NULL, RISCV_HWPROBE_WHICH_CPUS); + ksft_test_result(rc == -EINVAL, "NULL cpus\n"); + + pairs[0] = (struct riscv_hwprobe){ .key = 0xbadc0de, }; + CPU_ZERO(&cpus); + rc = riscv_hwprobe(pairs, 1, sizeof(cpu_set_t), (unsigned long *)&cpus, RISCV_HWPROBE_WHICH_CPUS); + ksft_test_result(rc == 0 && CPU_COUNT(&cpus) == 0, "unknown key\n"); + + pairs[0] = (struct riscv_hwprobe){ .key = RISCV_HWPROBE_KEY_BASE_BEHAVIOR, .value = RISCV_HWPROBE_BASE_BEHAVIOR_IMA, }; + pairs[1] = (struct riscv_hwprobe){ .key = RISCV_HWPROBE_KEY_BASE_BEHAVIOR, .value = RISCV_HWPROBE_BASE_BEHAVIOR_IMA, }; + CPU_ZERO(&cpus); + rc = riscv_hwprobe(pairs, 2, sizeof(cpu_set_t), (unsigned long *)&cpus, RISCV_HWPROBE_WHICH_CPUS); + ksft_test_result(rc == 0, "duplicate keys\n"); + + pairs[0] = (struct riscv_hwprobe){ .key = RISCV_HWPROBE_KEY_BASE_BEHAVIOR, .value = RISCV_HWPROBE_BASE_BEHAVIOR_IMA, }; + pairs[1] = (struct riscv_hwprobe){ .key = RISCV_HWPROBE_KEY_IMA_EXT_0, .value = ext0_all, }; + CPU_ZERO(&cpus); + rc = riscv_hwprobe(pairs, 2, sizeof(cpu_set_t), (unsigned long *)&cpus, RISCV_HWPROBE_WHICH_CPUS); + ksft_test_result(rc == 0 && CPU_COUNT(&cpus) == sysconf(_SC_NPROCESSORS_ONLN), "set all cpus\n"); + + pairs[0] = (struct riscv_hwprobe){ .key = RISCV_HWPROBE_KEY_BASE_BEHAVIOR, .value = RISCV_HWPROBE_BASE_BEHAVIOR_IMA, }; + pairs[1] = (struct riscv_hwprobe){ .key = RISCV_HWPROBE_KEY_IMA_EXT_0, .value = ext0_all, }; + memcpy(&cpus, &cpus_aff, sizeof(cpu_set_t)); + rc = riscv_hwprobe(pairs, 2, sizeof(cpu_set_t), (unsigned long *)&cpus, RISCV_HWPROBE_WHICH_CPUS); + ksft_test_result(rc == 0 && CPU_EQUAL(&cpus, &cpus_aff), "set all affinity cpus\n"); + + pairs[0] = (struct riscv_hwprobe){ .key = RISCV_HWPROBE_KEY_BASE_BEHAVIOR, .value = RISCV_HWPROBE_BASE_BEHAVIOR_IMA, }; + pairs[1] = (struct riscv_hwprobe){ .key = RISCV_HWPROBE_KEY_IMA_EXT_0, .value = ~ext0_all, }; + memcpy(&cpus, &cpus_aff, sizeof(cpu_set_t)); + rc = riscv_hwprobe(pairs, 2, sizeof(cpu_set_t), (unsigned long *)&cpus, RISCV_HWPROBE_WHICH_CPUS); + ksft_test_result(rc == 0 && CPU_COUNT(&cpus) == 0, "clear all cpus\n"); + + ksft_finished(); +} From 38709af26c33e398c3292e96837ccfde41fd9e6b Mon Sep 17 00:00:00 2001 From: Cristian Ciocaltea Date: Thu, 4 Jan 2024 16:39:49 +0200 Subject: [PATCH 241/336] drm/rockchip: vop2: Drop superfluous include The rockchip_drm_fb.h header contains just a single function which is not directly used by the VOP2 driver. Drop the unnecessary include. Signed-off-by: Cristian Ciocaltea Signed-off-by: Heiko Stuebner Link: https://patchwork.freedesktop.org/patch/msgid/20240104143951.85219-1-cristian.ciocaltea@collabora.com --- drivers/gpu/drm/rockchip/rockchip_drm_vop2.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c index 574103fc79f9..a3a03e9cfe13 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c +++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c @@ -35,7 +35,6 @@ #include "rockchip_drm_drv.h" #include "rockchip_drm_gem.h" -#include "rockchip_drm_fb.h" #include "rockchip_drm_vop2.h" #include "rockchip_rgb.h" From ad362fe07fecf0aba839ff2cc59a3617bd42c33f Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Thu, 4 Jan 2024 18:32:32 +0000 Subject: [PATCH 242/336] KVM: arm64: vgic-its: Avoid potential UAF in LPI translation cache There is a potential UAF scenario in the case of an LPI translation cache hit racing with an operation that invalidates the cache, such as a DISCARD ITS command. The root of the problem is that vgic_its_check_cache() does not elevate the refcount on the vgic_irq before dropping the lock that serializes refcount changes. Have vgic_its_check_cache() raise the refcount on the returned vgic_irq and add the corresponding decrement after queueing the interrupt. Cc: stable@vger.kernel.org Signed-off-by: Oliver Upton Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20240104183233.3560639-1-oliver.upton@linux.dev --- arch/arm64/kvm/vgic/vgic-its.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 2dad2d095160..e2764d0ffa9f 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -590,7 +590,11 @@ static struct vgic_irq *vgic_its_check_cache(struct kvm *kvm, phys_addr_t db, unsigned long flags; raw_spin_lock_irqsave(&dist->lpi_list_lock, flags); + irq = __vgic_its_check_cache(dist, db, devid, eventid); + if (irq) + vgic_get_irq_kref(irq); + raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags); return irq; @@ -769,6 +773,7 @@ int vgic_its_inject_cached_translation(struct kvm *kvm, struct kvm_msi *msi) raw_spin_lock_irqsave(&irq->irq_lock, flags); irq->pending_latch = true; vgic_queue_irq_unlock(kvm, irq, flags); + vgic_put_irq(kvm, irq); return 0; } From 040113fa32f27096f531c377001936e0d7964597 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 4 Jan 2024 16:42:20 +0000 Subject: [PATCH 243/336] KVM: arm64: Add missing memory barriers when switching to pKVM's hyp pgd In commit f320bc742bc23 ("KVM: arm64: Prepare the creation of s1 mappings at EL2"), pKVM switches from a temporary host-provided page-table to its own page-table at EL2. Since there is only a single TTBR for the nVHE hypervisor, this involves disabling and re-enabling the MMU in __pkvm_init_switch_pgd(). Unfortunately, the memory barriers here are not quite correct. Specifically: - A DSB is required to complete the TLB invalidation executed while the MMU is disabled. - An ISB is required to make the new TTBR value visible to the page-table walker before the MMU is enabled in the SCTLR. An earlier version of the patch actually got this correct: https://lore.kernel.org/lkml/20210304184717.GB21795@willie-the-truck/ but thanks to some badly worded review comments from yours truly, these were dropped for the version that was eventually merged. Bring back the barriers and fix the potential issue (but note that this was found by code inspection). Cc: Quentin Perret Fixes: f320bc742bc23 ("KVM: arm64: Prepare the creation of s1 mappings at EL2") Signed-off-by: Will Deacon Reviewed-by: Oliver Upton Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20240104164220.7968-1-will@kernel.org --- arch/arm64/kvm/hyp/nvhe/hyp-init.S | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S index f62a7d360285..2994878d68ea 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S @@ -288,6 +288,8 @@ alternative_else_nop_endif mov sp, x0 /* And turn the MMU back on! */ + dsb nsh + isb set_sctlr_el2 x2 ret x1 SYM_FUNC_END(__pkvm_init_switch_pgd) From 32253f00ac8a8073bf6db4bfe9d6511cc93c4aef Mon Sep 17 00:00:00 2001 From: Vincent Whitchurch Date: Fri, 1 Sep 2023 15:35:43 +0200 Subject: [PATCH 244/336] um: virt-pci: fix platform map offset The offset is currently always zero so the backend can't distinguish between accesses to different ioremapped areas. Fixes: 522c532c4fe7 ("virt-pci: add platform bus support") Signed-off-by: Vincent Whitchurch Signed-off-by: Richard Weinberger --- arch/um/drivers/virt-pci.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/um/drivers/virt-pci.c b/arch/um/drivers/virt-pci.c index ffe2ee8a0246..97a37c062997 100644 --- a/arch/um/drivers/virt-pci.c +++ b/arch/um/drivers/virt-pci.c @@ -971,7 +971,7 @@ static long um_pci_map_platform(unsigned long offset, size_t size, *ops = &um_pci_device_bar_ops; *priv = &um_pci_platform_device->resptr[0]; - return 0; + return offset; } static const struct logic_iomem_region_ops um_pci_platform_ops = { From 541d4e4d435c8b9bfd29f70a1da4a2db97794e0a Mon Sep 17 00:00:00 2001 From: Anton Ivanov Date: Thu, 21 Sep 2023 15:34:44 +0100 Subject: [PATCH 245/336] um: Fix naming clash between UML and scheduler __cant_sleep was already used and exported by the scheduler. The name had to be changed to a UML specific one. Signed-off-by: Anton Ivanov Reviewed-by: Peter Lafreniere Signed-off-by: Richard Weinberger --- arch/um/include/shared/kern_util.h | 2 +- arch/um/kernel/process.c | 2 +- arch/um/os-Linux/helper.c | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/um/include/shared/kern_util.h b/arch/um/include/shared/kern_util.h index d8b8b4f07e42..444bae755b16 100644 --- a/arch/um/include/shared/kern_util.h +++ b/arch/um/include/shared/kern_util.h @@ -50,7 +50,7 @@ extern void do_uml_exitcalls(void); * Are we disallowed to sleep? Used to choose between GFP_KERNEL and * GFP_ATOMIC. */ -extern int __cant_sleep(void); +extern int __uml_cant_sleep(void); extern int get_current_pid(void); extern int copy_from_user_proc(void *to, void *from, int size); extern char *uml_strdup(const char *string); diff --git a/arch/um/kernel/process.c b/arch/um/kernel/process.c index 106b7da2f8d6..6daffb9d8a8d 100644 --- a/arch/um/kernel/process.c +++ b/arch/um/kernel/process.c @@ -220,7 +220,7 @@ void arch_cpu_idle(void) um_idle_sleep(); } -int __cant_sleep(void) { +int __uml_cant_sleep(void) { return in_atomic() || irqs_disabled() || in_interrupt(); /* Is in_interrupt() really needed? */ } diff --git a/arch/um/os-Linux/helper.c b/arch/um/os-Linux/helper.c index b459745f52e2..3cb8ac63be6e 100644 --- a/arch/um/os-Linux/helper.c +++ b/arch/um/os-Linux/helper.c @@ -46,7 +46,7 @@ int run_helper(void (*pre_exec)(void *), void *pre_data, char **argv) unsigned long stack, sp; int pid, fds[2], ret, n; - stack = alloc_stack(0, __cant_sleep()); + stack = alloc_stack(0, __uml_cant_sleep()); if (stack == 0) return -ENOMEM; @@ -70,7 +70,7 @@ int run_helper(void (*pre_exec)(void *), void *pre_data, char **argv) data.pre_data = pre_data; data.argv = argv; data.fd = fds[1]; - data.buf = __cant_sleep() ? uml_kmalloc(PATH_MAX, UM_GFP_ATOMIC) : + data.buf = __uml_cant_sleep() ? uml_kmalloc(PATH_MAX, UM_GFP_ATOMIC) : uml_kmalloc(PATH_MAX, UM_GFP_KERNEL); pid = clone(helper_child, (void *) sp, CLONE_VM, &data); if (pid < 0) { @@ -121,7 +121,7 @@ int run_helper_thread(int (*proc)(void *), void *arg, unsigned int flags, unsigned long stack, sp; int pid, status, err; - stack = alloc_stack(0, __cant_sleep()); + stack = alloc_stack(0, __uml_cant_sleep()); if (stack == 0) return -ENOMEM; From 085bc003baab8223b87649ee56aa0db05c33b5b8 Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Mon, 25 Sep 2023 21:29:48 +0200 Subject: [PATCH 246/336] um: mmu: remove stub_pages I removed all the users of this some time ago, but evidently forgot the pointers. Remove them from the data structure too. Fixes: bfc58e2b98e9 ("um: remove process stub VMA") Signed-off-by: Johannes Berg Signed-off-by: Richard Weinberger --- arch/um/include/asm/mmu.h | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/um/include/asm/mmu.h b/arch/um/include/asm/mmu.h index 5b072aba5b65..a7555e43ed14 100644 --- a/arch/um/include/asm/mmu.h +++ b/arch/um/include/asm/mmu.h @@ -12,7 +12,6 @@ typedef struct mm_context { struct mm_id id; struct uml_arch_mm_context arch; - struct page *stub_pages[2]; } mm_context_t; extern void __switch_mm(struct mm_id * mm_idp); From a8e75902f4d7d342350ea3f79e3e65f2bbfa4c8d Mon Sep 17 00:00:00 2001 From: Anton Ivanov Date: Tue, 17 Oct 2023 09:32:01 +0100 Subject: [PATCH 247/336] um: document arch_futex_atomic_op_inuser arch_futex_atomic_op_inuser was not documented correctly resulting in build time warnings. Signed-off-by: Anton Ivanov Signed-off-by: Richard Weinberger --- arch/um/kernel/skas/uaccess.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/um/kernel/skas/uaccess.c b/arch/um/kernel/skas/uaccess.c index aaee96f07172..198269e384c4 100644 --- a/arch/um/kernel/skas/uaccess.c +++ b/arch/um/kernel/skas/uaccess.c @@ -236,7 +236,9 @@ EXPORT_SYMBOL(strnlen_user); * argument and comparison of the previous * futex value with another constant. * - * @encoded_op: encoded operation to execute + * @op: operation to execute + * @oparg: argument to operation + * @oval: old value at uaddr * @uaddr: pointer to user space address * * Return: From a55719847da0a780baa84d0baee745358f144c39 Mon Sep 17 00:00:00 2001 From: Benjamin Berg Date: Fri, 10 Nov 2023 12:03:38 +0100 Subject: [PATCH 248/336] um: Drop support for hosts without SYSEMU_SINGLESTEP support These features have existed since Linux 2.6.14 and can be considered widely available at this point. Also drop the backward compatibility code for PTRACE_SETOPTIONS. Signed-off-by: Benjamin Berg ---- v2: * Continue to define PTRACE_SYSEMU_SINGLESTEP as glibc only added it in version 2.27. Signed-off-by: Richard Weinberger --- arch/um/include/asm/processor-generic.h | 1 - arch/um/include/shared/kern_util.h | 3 +- arch/um/include/shared/ptrace_user.h | 41 --------------- arch/um/kernel/process.c | 12 +---- arch/um/kernel/ptrace.c | 2 - arch/um/kernel/signal.c | 12 ----- arch/um/os-Linux/skas/process.c | 60 ++++----------------- arch/um/os-Linux/start_up.c | 70 +++---------------------- arch/x86/um/ptrace_32.c | 24 --------- arch/x86/um/ptrace_64.c | 26 --------- arch/x86/um/shared/sysdep/ptrace_32.h | 4 -- arch/x86/um/shared/sysdep/ptrace_user.h | 12 ++--- 12 files changed, 24 insertions(+), 243 deletions(-) diff --git a/arch/um/include/asm/processor-generic.h b/arch/um/include/asm/processor-generic.h index 7414154b8e9a..6c3779541845 100644 --- a/arch/um/include/asm/processor-generic.h +++ b/arch/um/include/asm/processor-generic.h @@ -22,7 +22,6 @@ struct mm_struct; struct thread_struct { struct pt_regs regs; struct pt_regs *segv_regs; - int singlestep_syscall; void *fault_addr; jmp_buf *fault_catcher; struct task_struct *prev_sched; diff --git a/arch/um/include/shared/kern_util.h b/arch/um/include/shared/kern_util.h index 444bae755b16..789b83013f35 100644 --- a/arch/um/include/shared/kern_util.h +++ b/arch/um/include/shared/kern_util.h @@ -34,7 +34,6 @@ extern int handle_page_fault(unsigned long address, unsigned long ip, extern unsigned int do_IRQ(int irq, struct uml_pt_regs *regs); extern void initial_thread_cb(void (*proc)(void *), void *arg); -extern int is_syscall(unsigned long addr); extern void timer_handler(int sig, struct siginfo *unused_si, struct uml_pt_regs *regs); @@ -58,7 +57,7 @@ extern char *uml_strdup(const char *string); extern unsigned long to_irq_stack(unsigned long *mask_out); extern unsigned long from_irq_stack(int nested); -extern int singlestepping(void *t); +extern int singlestepping(void); extern void segv_handler(int sig, struct siginfo *unused_si, struct uml_pt_regs *regs); extern void bus_handler(int sig, struct siginfo *si, struct uml_pt_regs *regs); diff --git a/arch/um/include/shared/ptrace_user.h b/arch/um/include/shared/ptrace_user.h index 95455e8996e7..8a705d8f96ce 100644 --- a/arch/um/include/shared/ptrace_user.h +++ b/arch/um/include/shared/ptrace_user.h @@ -12,45 +12,4 @@ extern int ptrace_getregs(long pid, unsigned long *regs_out); extern int ptrace_setregs(long pid, unsigned long *regs_in); -/* syscall emulation path in ptrace */ - -#ifndef PTRACE_SYSEMU -#define PTRACE_SYSEMU 31 -#endif -#ifndef PTRACE_SYSEMU_SINGLESTEP -#define PTRACE_SYSEMU_SINGLESTEP 32 -#endif - -/* On architectures, that started to support PTRACE_O_TRACESYSGOOD - * in linux 2.4, there are two different definitions of - * PTRACE_SETOPTIONS: linux 2.4 uses 21 while linux 2.6 uses 0x4200. - * For binary compatibility, 2.6 also supports the old "21", named - * PTRACE_OLDSETOPTION. On these architectures, UML always must use - * "21", to ensure the kernel runs on 2.4 and 2.6 host without - * recompilation. So, we use PTRACE_OLDSETOPTIONS in UML. - * We also want to be able to build the kernel on 2.4, which doesn't - * have PTRACE_OLDSETOPTIONS. So, if it is missing, we declare - * PTRACE_OLDSETOPTIONS to be the same as PTRACE_SETOPTIONS. - * - * On architectures, that start to support PTRACE_O_TRACESYSGOOD on - * linux 2.6, PTRACE_OLDSETOPTIONS never is defined, and also isn't - * supported by the host kernel. In that case, our trick lets us use - * the new 0x4200 with the name PTRACE_OLDSETOPTIONS. - */ -#ifndef PTRACE_OLDSETOPTIONS -#define PTRACE_OLDSETOPTIONS PTRACE_SETOPTIONS -#endif - -void set_using_sysemu(int value); -int get_using_sysemu(void); -extern int sysemu_supported; - -#define SELECT_PTRACE_OPERATION(sysemu_mode, singlestep_mode) \ - (((int[3][3] ) { \ - { PTRACE_SYSCALL, PTRACE_SYSCALL, PTRACE_SINGLESTEP }, \ - { PTRACE_SYSEMU, PTRACE_SYSEMU, PTRACE_SINGLESTEP }, \ - { PTRACE_SYSEMU, PTRACE_SYSEMU_SINGLESTEP, \ - PTRACE_SYSEMU_SINGLESTEP } }) \ - [sysemu_mode][singlestep_mode]) - #endif diff --git a/arch/um/kernel/process.c b/arch/um/kernel/process.c index 6daffb9d8a8d..ab95648e93e1 100644 --- a/arch/um/kernel/process.c +++ b/arch/um/kernel/process.c @@ -332,17 +332,9 @@ int __init make_proc_sysemu(void) late_initcall(make_proc_sysemu); -int singlestepping(void * t) +int singlestepping(void) { - struct task_struct *task = t ? t : current; - - if (!test_thread_flag(TIF_SINGLESTEP)) - return 0; - - if (task->thread.singlestep_syscall) - return 1; - - return 2; + return test_thread_flag(TIF_SINGLESTEP); } /* diff --git a/arch/um/kernel/ptrace.c b/arch/um/kernel/ptrace.c index 5154b27de580..6600a2782796 100644 --- a/arch/um/kernel/ptrace.c +++ b/arch/um/kernel/ptrace.c @@ -12,7 +12,6 @@ void user_enable_single_step(struct task_struct *child) { set_tsk_thread_flag(child, TIF_SINGLESTEP); - child->thread.singlestep_syscall = 0; #ifdef SUBARCH_SET_SINGLESTEPPING SUBARCH_SET_SINGLESTEPPING(child, 1); @@ -22,7 +21,6 @@ void user_enable_single_step(struct task_struct *child) void user_disable_single_step(struct task_struct *child) { clear_tsk_thread_flag(child, TIF_SINGLESTEP); - child->thread.singlestep_syscall = 0; #ifdef SUBARCH_SET_SINGLESTEPPING SUBARCH_SET_SINGLESTEPPING(child, 0); diff --git a/arch/um/kernel/signal.c b/arch/um/kernel/signal.c index ae4658f576ab..a56b44522766 100644 --- a/arch/um/kernel/signal.c +++ b/arch/um/kernel/signal.c @@ -120,18 +120,6 @@ void do_signal(struct pt_regs *regs) } } - /* - * This closes a way to execute a system call on the host. If - * you set a breakpoint on a system call instruction and singlestep - * from it, the tracing thread used to PTRACE_SINGLESTEP the process - * rather than PTRACE_SYSCALL it, allowing the system call to execute - * on the host. The tracing thread will check this flag and - * PTRACE_SYSCALL if necessary. - */ - if (test_thread_flag(TIF_SINGLESTEP)) - current->thread.singlestep_syscall = - is_syscall(PT_REGS_IP(¤t->thread.regs)); - /* * if there's no signal to deliver, we just put the saved sigmask * back diff --git a/arch/um/os-Linux/skas/process.c b/arch/um/os-Linux/skas/process.c index 9464833e741a..43168c51850c 100644 --- a/arch/um/os-Linux/skas/process.c +++ b/arch/um/os-Linux/skas/process.c @@ -177,48 +177,11 @@ static void handle_segv(int pid, struct uml_pt_regs *regs, unsigned long *aux_fp segv(regs->faultinfo, 0, 1, NULL); } -/* - * To use the same value of using_sysemu as the caller, ask it that value - * (in local_using_sysemu - */ -static void handle_trap(int pid, struct uml_pt_regs *regs, - int local_using_sysemu) +static void handle_trap(int pid, struct uml_pt_regs *regs) { - int err, status; - if ((UPT_IP(regs) >= STUB_START) && (UPT_IP(regs) < STUB_END)) fatal_sigsegv(); - if (!local_using_sysemu) - { - err = ptrace(PTRACE_POKEUSER, pid, PT_SYSCALL_NR_OFFSET, - __NR_getpid); - if (err < 0) { - printk(UM_KERN_ERR "%s - nullifying syscall failed, errno = %d\n", - __func__, errno); - fatal_sigsegv(); - } - - err = ptrace(PTRACE_SYSCALL, pid, 0, 0); - if (err < 0) { - printk(UM_KERN_ERR "%s - continuing to end of syscall failed, errno = %d\n", - __func__, errno); - fatal_sigsegv(); - } - - CATCH_EINTR(err = waitpid(pid, &status, WUNTRACED | __WALL)); - if ((err < 0) || !WIFSTOPPED(status) || - (WSTOPSIG(status) != SIGTRAP + 0x80)) { - err = ptrace_dump_regs(pid); - if (err) - printk(UM_KERN_ERR "Failed to get registers from process, errno = %d\n", - -err); - printk(UM_KERN_ERR "%s - failed to wait at end of syscall, errno = %d, status = %d\n", - __func__, errno, status); - fatal_sigsegv(); - } - } - handle_syscall(regs); } @@ -355,10 +318,10 @@ int start_userspace(unsigned long stub_stack) goto out_kill; } - if (ptrace(PTRACE_OLDSETOPTIONS, pid, NULL, + if (ptrace(PTRACE_SETOPTIONS, pid, NULL, (void *) PTRACE_O_TRACESYSGOOD) < 0) { err = -errno; - printk(UM_KERN_ERR "%s : PTRACE_OLDSETOPTIONS failed, errno = %d\n", + printk(UM_KERN_ERR "%s : PTRACE_SETOPTIONS failed, errno = %d\n", __func__, errno); goto out_kill; } @@ -380,8 +343,6 @@ int start_userspace(unsigned long stub_stack) void userspace(struct uml_pt_regs *regs, unsigned long *aux_fp_regs) { int err, status, op, pid = userspace_pid[0]; - /* To prevent races if using_sysemu changes under us.*/ - int local_using_sysemu; siginfo_t si; /* Handle any immediate reschedules or signals */ @@ -411,11 +372,10 @@ void userspace(struct uml_pt_regs *regs, unsigned long *aux_fp_regs) fatal_sigsegv(); } - /* Now we set local_using_sysemu to be used for one loop */ - local_using_sysemu = get_using_sysemu(); - - op = SELECT_PTRACE_OPERATION(local_using_sysemu, - singlestepping(NULL)); + if (singlestepping()) + op = PTRACE_SYSEMU_SINGLESTEP; + else + op = PTRACE_SYSEMU; if (ptrace(op, pid, 0, 0)) { printk(UM_KERN_ERR "%s - ptrace continue failed, op = %d, errno = %d\n", @@ -474,7 +434,7 @@ void userspace(struct uml_pt_regs *regs, unsigned long *aux_fp_regs) else handle_segv(pid, regs, aux_fp_regs); break; case SIGTRAP + 0x80: - handle_trap(pid, regs, local_using_sysemu); + handle_trap(pid, regs); break; case SIGTRAP: relay_signal(SIGTRAP, (struct siginfo *)&si, regs); @@ -597,10 +557,10 @@ int copy_context_skas0(unsigned long new_stack, int pid) goto out_kill; } - if (ptrace(PTRACE_OLDSETOPTIONS, pid, NULL, + if (ptrace(PTRACE_SETOPTIONS, pid, NULL, (void *)PTRACE_O_TRACESYSGOOD) < 0) { err = -errno; - printk(UM_KERN_ERR "%s : PTRACE_OLDSETOPTIONS failed, errno = %d\n", + printk(UM_KERN_ERR "%s : PTRACE_SETOPTIONS failed, errno = %d\n", __func__, errno); goto out_kill; } diff --git a/arch/um/os-Linux/start_up.c b/arch/um/os-Linux/start_up.c index e3ee4db58b40..561c08e0cc9c 100644 --- a/arch/um/os-Linux/start_up.c +++ b/arch/um/os-Linux/start_up.c @@ -143,71 +143,16 @@ static int stop_ptraced_child(int pid, int exitcode, int mustexit) return ret; } -/* Changed only during early boot */ -static int force_sysemu_disabled = 0; - -static int __init nosysemu_cmd_param(char *str, int* add) -{ - force_sysemu_disabled = 1; - return 0; -} - -__uml_setup("nosysemu", nosysemu_cmd_param, -"nosysemu\n" -" Turns off syscall emulation patch for ptrace (SYSEMU).\n" -" SYSEMU is a performance-patch introduced by Laurent Vivier. It changes\n" -" behaviour of ptrace() and helps reduce host context switch rates.\n" -" To make it work, you need a kernel patch for your host, too.\n" -" See http://perso.wanadoo.fr/laurent.vivier/UML/ for further \n" -" information.\n\n"); - static void __init check_sysemu(void) { - unsigned long regs[MAX_REG_NR]; int pid, n, status, count=0; - os_info("Checking syscall emulation patch for ptrace..."); - sysemu_supported = 0; + os_info("Checking syscall emulation for ptrace..."); pid = start_ptraced_child(); - if (ptrace(PTRACE_SYSEMU, pid, 0, 0) < 0) - goto fail; - - CATCH_EINTR(n = waitpid(pid, &status, WUNTRACED)); - if (n < 0) - fatal_perror("check_sysemu : wait failed"); - if (!WIFSTOPPED(status) || (WSTOPSIG(status) != SIGTRAP)) - fatal("check_sysemu : expected SIGTRAP, got status = %d\n", - status); - - if (ptrace(PTRACE_GETREGS, pid, 0, regs) < 0) - fatal_perror("check_sysemu : PTRACE_GETREGS failed"); - if (PT_SYSCALL_NR(regs) != __NR_getpid) { - non_fatal("check_sysemu got system call number %d, " - "expected %d...", PT_SYSCALL_NR(regs), __NR_getpid); - goto fail; - } - - n = ptrace(PTRACE_POKEUSER, pid, PT_SYSCALL_RET_OFFSET, os_getpid()); - if (n < 0) { - non_fatal("check_sysemu : failed to modify system call " - "return"); - goto fail; - } - - if (stop_ptraced_child(pid, 0, 0) < 0) - goto fail_stopped; - - sysemu_supported = 1; - os_info("OK\n"); - set_using_sysemu(!force_sysemu_disabled); - - os_info("Checking advanced syscall emulation patch for ptrace..."); - pid = start_ptraced_child(); - - if ((ptrace(PTRACE_OLDSETOPTIONS, pid, 0, + if ((ptrace(PTRACE_SETOPTIONS, pid, 0, (void *) PTRACE_O_TRACESYSGOOD) < 0)) - fatal_perror("check_sysemu: PTRACE_OLDSETOPTIONS failed"); + fatal_perror("check_sysemu: PTRACE_SETOPTIONS failed"); while (1) { count++; @@ -243,17 +188,14 @@ static void __init check_sysemu(void) if (stop_ptraced_child(pid, 0, 0) < 0) goto fail_stopped; - sysemu_supported = 2; os_info("OK\n"); - if (!force_sysemu_disabled) - set_using_sysemu(sysemu_supported); return; fail: stop_ptraced_child(pid, 1, 0); fail_stopped: - non_fatal("missing\n"); + fatal("missing\n"); } static void __init check_ptrace(void) @@ -263,9 +205,9 @@ static void __init check_ptrace(void) os_info("Checking that ptrace can change system call numbers..."); pid = start_ptraced_child(); - if ((ptrace(PTRACE_OLDSETOPTIONS, pid, 0, + if ((ptrace(PTRACE_SETOPTIONS, pid, 0, (void *) PTRACE_O_TRACESYSGOOD) < 0)) - fatal_perror("check_ptrace: PTRACE_OLDSETOPTIONS failed"); + fatal_perror("check_ptrace: PTRACE_SETOPTIONS failed"); while (1) { if (ptrace(PTRACE_SYSCALL, pid, 0, 0) < 0) diff --git a/arch/x86/um/ptrace_32.c b/arch/x86/um/ptrace_32.c index 0bc4b73a9cde..7f1abde2c84b 100644 --- a/arch/x86/um/ptrace_32.c +++ b/arch/x86/um/ptrace_32.c @@ -25,30 +25,6 @@ void arch_switch_to(struct task_struct *to) printk(KERN_WARNING "arch_switch_tls failed, errno = EINVAL\n"); } -int is_syscall(unsigned long addr) -{ - unsigned short instr; - int n; - - n = copy_from_user(&instr, (void __user *) addr, sizeof(instr)); - if (n) { - /* access_process_vm() grants access to vsyscall and stub, - * while copy_from_user doesn't. Maybe access_process_vm is - * slow, but that doesn't matter, since it will be called only - * in case of singlestepping, if copy_from_user failed. - */ - n = access_process_vm(current, addr, &instr, sizeof(instr), - FOLL_FORCE); - if (n != sizeof(instr)) { - printk(KERN_ERR "is_syscall : failed to read " - "instruction from 0x%lx\n", addr); - return 1; - } - } - /* int 0x80 or sysenter */ - return (instr == 0x80cd) || (instr == 0x340f); -} - /* determines which flags the user has access to. */ /* 1 = access 0 = no access */ #define FLAG_MASK 0x00044dd5 diff --git a/arch/x86/um/ptrace_64.c b/arch/x86/um/ptrace_64.c index 289d0159b041..aa68d83d3f44 100644 --- a/arch/x86/um/ptrace_64.c +++ b/arch/x86/um/ptrace_64.c @@ -188,32 +188,6 @@ int peek_user(struct task_struct *child, long addr, long data) return put_user(tmp, (unsigned long *) data); } -/* XXX Mostly copied from sys-i386 */ -int is_syscall(unsigned long addr) -{ - unsigned short instr; - int n; - - n = copy_from_user(&instr, (void __user *) addr, sizeof(instr)); - if (n) { - /* - * access_process_vm() grants access to vsyscall and stub, - * while copy_from_user doesn't. Maybe access_process_vm is - * slow, but that doesn't matter, since it will be called only - * in case of singlestepping, if copy_from_user failed. - */ - n = access_process_vm(current, addr, &instr, sizeof(instr), - FOLL_FORCE); - if (n != sizeof(instr)) { - printk("is_syscall : failed to read instruction from " - "0x%lx\n", addr); - return 1; - } - } - /* sysenter */ - return instr == 0x050f; -} - static int get_fpregs(struct user_i387_struct __user *buf, struct task_struct *child) { int err, n, cpu = ((struct thread_info *) child->stack)->cpu; diff --git a/arch/x86/um/shared/sysdep/ptrace_32.h b/arch/x86/um/shared/sysdep/ptrace_32.h index db8478a83a09..0c4989842fbe 100644 --- a/arch/x86/um/shared/sysdep/ptrace_32.h +++ b/arch/x86/um/shared/sysdep/ptrace_32.h @@ -8,10 +8,6 @@ #define MAX_FP_NR HOST_FPX_SIZE -void set_using_sysemu(int value); -int get_using_sysemu(void); -extern int sysemu_supported; - #define UPT_SYSCALL_ARG1(r) UPT_BX(r) #define UPT_SYSCALL_ARG2(r) UPT_CX(r) #define UPT_SYSCALL_ARG3(r) UPT_DX(r) diff --git a/arch/x86/um/shared/sysdep/ptrace_user.h b/arch/x86/um/shared/sysdep/ptrace_user.h index 44782bbad41e..1d1a824fa652 100644 --- a/arch/x86/um/shared/sysdep/ptrace_user.h +++ b/arch/x86/um/shared/sysdep/ptrace_user.h @@ -15,14 +15,12 @@ #define FP_SIZE ((HOST_FPX_SIZE > HOST_FP_SIZE) ? HOST_FPX_SIZE : HOST_FP_SIZE) #else #define FP_SIZE HOST_FP_SIZE +#endif /* - * x86_64 FC3 doesn't define this in /usr/include/linux/ptrace.h even though - * it's defined in the kernel's include/linux/ptrace.h. Additionally, use the - * 2.4 name and value for 2.4 host compatibility. + * glibc before 2.27 does not include PTRACE_SYSEMU_SINGLESTEP in its enum, + * ensure we have a definition by (re-)defining it here. */ -#ifndef PTRACE_OLDSETOPTIONS -#define PTRACE_OLDSETOPTIONS 21 -#endif - +#ifndef PTRACE_SYSEMU_SINGLESTEP +#define PTRACE_SYSEMU_SINGLESTEP 32 #endif From 571353379470f1d0aaad3cce4ad4db4b6c8f9ada Mon Sep 17 00:00:00 2001 From: Benjamin Berg Date: Fri, 10 Nov 2023 12:03:39 +0100 Subject: [PATCH 249/336] um: Drop NULL check from start_userspace start_userspace is only called from exactly one location, and the passed pointer for the userspace process stack cannot be NULL. Remove the check, without changing the control flow. Signed-off-by: Benjamin Berg Signed-off-by: Richard Weinberger --- arch/um/os-Linux/skas/process.c | 51 +++++++++++++++------------------ 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/arch/um/os-Linux/skas/process.c b/arch/um/os-Linux/skas/process.c index 43168c51850c..ed20cefb2963 100644 --- a/arch/um/os-Linux/skas/process.c +++ b/arch/um/os-Linux/skas/process.c @@ -189,7 +189,7 @@ extern char __syscall_stub_start[]; /** * userspace_tramp() - userspace trampoline - * @stack: pointer to the new userspace stack page, can be NULL, if? FIXME: + * @stack: pointer to the new userspace stack page * * The userspace trampoline is used to setup a new userspace process in start_userspace() after it was clone()'ed. * This function will run on a temporary stack page. @@ -204,9 +204,13 @@ extern char __syscall_stub_start[]; */ static int userspace_tramp(void *stack) { + struct sigaction sa; void *addr; int fd; unsigned long long offset; + unsigned long segv_handler = STUB_CODE + + (unsigned long) stub_segv_handler - + (unsigned long) __syscall_stub_start; ptrace(PTRACE_TRACEME, 0, 0, 0); @@ -222,34 +226,25 @@ static int userspace_tramp(void *stack) exit(1); } - if (stack != NULL) { - fd = phys_mapping(uml_to_phys(stack), &offset); - addr = mmap((void *) STUB_DATA, - STUB_DATA_PAGES * UM_KERN_PAGE_SIZE, PROT_READ | PROT_WRITE, - MAP_FIXED | MAP_SHARED, fd, offset); - if (addr == MAP_FAILED) { - printk(UM_KERN_ERR "mapping segfault stack at 0x%lx failed, errno = %d\n", - STUB_DATA, errno); - exit(1); - } + fd = phys_mapping(uml_to_phys(stack), &offset); + addr = mmap((void *) STUB_DATA, + STUB_DATA_PAGES * UM_KERN_PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_SHARED, fd, offset); + if (addr == MAP_FAILED) { + printk(UM_KERN_ERR "mapping segfault stack at 0x%lx failed, errno = %d\n", + STUB_DATA, errno); + exit(1); } - if (stack != NULL) { - struct sigaction sa; - unsigned long v = STUB_CODE + - (unsigned long) stub_segv_handler - - (unsigned long) __syscall_stub_start; - - set_sigstack((void *) STUB_DATA, STUB_DATA_PAGES * UM_KERN_PAGE_SIZE); - sigemptyset(&sa.sa_mask); - sa.sa_flags = SA_ONSTACK | SA_NODEFER | SA_SIGINFO; - sa.sa_sigaction = (void *) v; - sa.sa_restorer = NULL; - if (sigaction(SIGSEGV, &sa, NULL) < 0) { - printk(UM_KERN_ERR "%s - setting SIGSEGV handler failed - errno = %d\n", - __func__, errno); - exit(1); - } + set_sigstack((void *) STUB_DATA, STUB_DATA_PAGES * UM_KERN_PAGE_SIZE); + sigemptyset(&sa.sa_mask); + sa.sa_flags = SA_ONSTACK | SA_NODEFER | SA_SIGINFO; + sa.sa_sigaction = (void *) segv_handler; + sa.sa_restorer = NULL; + if (sigaction(SIGSEGV, &sa, NULL) < 0) { + printk(UM_KERN_ERR "%s - setting SIGSEGV handler failed - errno = %d\n", + __func__, errno); + exit(1); } kill(os_getpid(), SIGSTOP); @@ -261,7 +256,7 @@ int kill_userspace_mm[NR_CPUS]; /** * start_userspace() - prepare a new userspace process - * @stub_stack: pointer to the stub stack. Can be NULL, if? FIXME: + * @stub_stack: pointer to the stub stack. * * Setups a new temporary stack page that is used while userspace_tramp() runs * Clones the kernel process into a new userspace process, with FDs only. From 9e16fb933fd1f2132c0d137f3666ebf20f93e33a Mon Sep 17 00:00:00 2001 From: Benjamin Berg Date: Fri, 10 Nov 2023 12:03:40 +0100 Subject: [PATCH 250/336] um: Make errors to stop ptraced child fatal during startup For the detection code to check whether SYSEMU_SINGLESTEP works correctly we needed some error cases while stopping to be non-fatal. However, at this point stop_ptraced_child must always succeed, and we can therefore simplify it slightly to exit immediately on error. Signed-off-by: Benjamin Berg Signed-off-by: Richard Weinberger --- arch/um/os-Linux/start_up.c | 41 +++++++++++-------------------------- 1 file changed, 12 insertions(+), 29 deletions(-) diff --git a/arch/um/os-Linux/start_up.c b/arch/um/os-Linux/start_up.c index 561c08e0cc9c..8b0e98ab842c 100644 --- a/arch/um/os-Linux/start_up.c +++ b/arch/um/os-Linux/start_up.c @@ -112,35 +112,20 @@ static int start_ptraced_child(void) return pid; } -/* When testing for SYSEMU support, if it is one of the broken versions, we - * must just avoid using sysemu, not panic, but only if SYSEMU features are - * broken. - * So only for SYSEMU features we test mustpanic, while normal host features - * must work anyway! - */ -static int stop_ptraced_child(int pid, int exitcode, int mustexit) +static void stop_ptraced_child(int pid, int exitcode) { - int status, n, ret = 0; + int status, n; + + if (ptrace(PTRACE_CONT, pid, 0, 0) < 0) + fatal_perror("stop_ptraced_child : ptrace failed"); - if (ptrace(PTRACE_CONT, pid, 0, 0) < 0) { - perror("stop_ptraced_child : ptrace failed"); - return -1; - } CATCH_EINTR(n = waitpid(pid, &status, 0)); if (!WIFEXITED(status) || (WEXITSTATUS(status) != exitcode)) { int exit_with = WEXITSTATUS(status); - if (exit_with == 2) - non_fatal("check_ptrace : child exited with status 2. " - "\nDisabling SYSEMU support.\n"); - non_fatal("check_ptrace : child exited with exitcode %d, while " - "expecting %d; status 0x%x\n", exit_with, - exitcode, status); - if (mustexit) - exit(1); - ret = -1; + fatal("stop_ptraced_child : child exited with exitcode %d, " + "while expecting %d; status 0x%x\n", exit_with, + exitcode, status); } - - return ret; } static void __init check_sysemu(void) @@ -185,16 +170,14 @@ static void __init check_sysemu(void) goto fail; } } - if (stop_ptraced_child(pid, 0, 0) < 0) - goto fail_stopped; + stop_ptraced_child(pid, 0); os_info("OK\n"); return; fail: - stop_ptraced_child(pid, 1, 0); -fail_stopped: + stop_ptraced_child(pid, 1); fatal("missing\n"); } @@ -233,7 +216,7 @@ static void __init check_ptrace(void) break; } } - stop_ptraced_child(pid, 0, 1); + stop_ptraced_child(pid, 0); os_info("OK\n"); check_sysemu(); } @@ -312,7 +295,7 @@ void __init os_early_checks(void) pid = start_ptraced_child(); if (init_pid_registers(pid)) fatal("Failed to initialize default registers"); - stop_ptraced_child(pid, 1, 1); + stop_ptraced_child(pid, 1); } int __init parse_iomem(char *str, int *add) From 236f9fe39b02c15fa5530b53e9cca48354394389 Mon Sep 17 00:00:00 2001 From: Benjamin Berg Date: Fri, 10 Nov 2023 12:03:41 +0100 Subject: [PATCH 251/336] um: Don't use vfprintf() for os_info() The threads allocated inside the kernel have only a single page of stack. Unfortunately, the vfprintf function in standard glibc may use too much stack-space, overflowing it. To make os_info safe to be used by helper threads, use the kernel vscnprintf function into a smallish buffer and write out the information to stderr. Signed-off-by: Benjamin Berg Signed-off-by: Richard Weinberger --- arch/um/os-Linux/util.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/arch/um/os-Linux/util.c b/arch/um/os-Linux/util.c index fc0f2a9dee5a..1dca4ffbd572 100644 --- a/arch/um/os-Linux/util.c +++ b/arch/um/os-Linux/util.c @@ -173,23 +173,38 @@ __uml_setup("quiet", quiet_cmd_param, "quiet\n" " Turns off information messages during boot.\n\n"); +/* + * The os_info/os_warn functions will be called by helper threads. These + * have a very limited stack size and using the libc formatting functions + * may overflow the stack. + * So pull in the kernel vscnprintf and use that instead with a fixed + * on-stack buffer. + */ +int vscnprintf(char *buf, size_t size, const char *fmt, va_list args); + void os_info(const char *fmt, ...) { + char buf[256]; va_list list; + int len; if (quiet_info) return; va_start(list, fmt); - vfprintf(stderr, fmt, list); + len = vscnprintf(buf, sizeof(buf), fmt, list); + fwrite(buf, len, 1, stderr); va_end(list); } void os_warn(const char *fmt, ...) { + char buf[256]; va_list list; + int len; va_start(list, fmt); - vfprintf(stderr, fmt, list); + len = vscnprintf(buf, sizeof(buf), fmt, list); + fwrite(buf, len, 1, stderr); va_end(list); } From 1818b8406678a78b823dad44c91580f859403ced Mon Sep 17 00:00:00 2001 From: Benjamin Berg Date: Fri, 10 Nov 2023 12:03:42 +0100 Subject: [PATCH 252/336] um: Do not use printk in SIGWINCH helper thread The thread is running outside of the UML kernel scope and is a helper. As such, printk cannot work and os_info must be used instead. Signed-off-by: Benjamin Berg Signed-off-by: Richard Weinberger --- arch/um/drivers/chan_user.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/arch/um/drivers/chan_user.c b/arch/um/drivers/chan_user.c index 25727ed648b7..c2b83cb99aae 100644 --- a/arch/um/drivers/chan_user.c +++ b/arch/um/drivers/chan_user.c @@ -153,8 +153,8 @@ static int winch_thread(void *arg) pipe_fd = data->pipe_fd; count = write(pipe_fd, &c, sizeof(c)); if (count != sizeof(c)) - printk(UM_KERN_ERR "winch_thread : failed to write " - "synchronization byte, err = %d\n", -count); + os_info("winch_thread : failed to write synchronization byte, err = %d\n", + -count); /* * We are not using SIG_IGN on purpose, so don't fix it as I thought to @@ -166,28 +166,28 @@ static int winch_thread(void *arg) sigfillset(&sigs); /* Block all signals possible. */ if (sigprocmask(SIG_SETMASK, &sigs, NULL) < 0) { - printk(UM_KERN_ERR "winch_thread : sigprocmask failed, " - "errno = %d\n", errno); + os_info("winch_thread : sigprocmask failed, errno = %d\n", + errno); exit(1); } /* In sigsuspend(), block anything else than SIGWINCH. */ sigdelset(&sigs, SIGWINCH); if (setsid() < 0) { - printk(UM_KERN_ERR "winch_thread : setsid failed, errno = %d\n", + os_info("winch_thread : setsid failed, errno = %d\n", errno); exit(1); } if (ioctl(pty_fd, TIOCSCTTY, 0) < 0) { - printk(UM_KERN_ERR "winch_thread : TIOCSCTTY failed on " - "fd %d err = %d\n", pty_fd, errno); + os_info("winch_thread : TIOCSCTTY failed on " + "fd %d err = %d\n", pty_fd, errno); exit(1); } if (tcsetpgrp(pty_fd, os_getpid()) < 0) { - printk(UM_KERN_ERR "winch_thread : tcsetpgrp failed on " - "fd %d err = %d\n", pty_fd, errno); + os_info("winch_thread : tcsetpgrp failed on fd %d err = %d\n", + pty_fd, errno); exit(1); } @@ -199,8 +199,8 @@ static int winch_thread(void *arg) */ count = read(pipe_fd, &c, sizeof(c)); if (count != sizeof(c)) - printk(UM_KERN_ERR "winch_thread : failed to read " - "synchronization byte, err = %d\n", errno); + os_info("winch_thread : failed to read synchronization byte, err = %d\n", + errno); while(1) { /* @@ -211,8 +211,8 @@ static int winch_thread(void *arg) count = write(pipe_fd, &c, sizeof(c)); if (count != sizeof(c)) - printk(UM_KERN_ERR "winch_thread : write failed, " - "err = %d\n", errno); + os_info("winch_thread : write failed, err = %d\n", + errno); } } From 139e6e8ef6ee9a56fc1737240ecd8402fbcadd82 Mon Sep 17 00:00:00 2001 From: Benjamin Berg Date: Fri, 10 Nov 2023 12:03:43 +0100 Subject: [PATCH 253/336] um: Reap winch thread if it fails When the winch thread runs into an error condition, it would exit(1) and never be reaped until shutdown time. Change this to write a command byte which causes the driver to kill it, therefore reaping the child. Signed-off-by: Benjamin Berg Signed-off-by: Richard Weinberger --- arch/um/drivers/chan_user.c | 16 +++++++++++----- arch/um/drivers/line.c | 13 ++++++++----- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/arch/um/drivers/chan_user.c b/arch/um/drivers/chan_user.c index c2b83cb99aae..ed7cc830b3e7 100644 --- a/arch/um/drivers/chan_user.c +++ b/arch/um/drivers/chan_user.c @@ -141,7 +141,7 @@ struct winch_data { int pipe_fd; }; -static int winch_thread(void *arg) +static __noreturn int winch_thread(void *arg) { struct winch_data *data = arg; sigset_t sigs; @@ -168,7 +168,7 @@ static int winch_thread(void *arg) if (sigprocmask(SIG_SETMASK, &sigs, NULL) < 0) { os_info("winch_thread : sigprocmask failed, errno = %d\n", errno); - exit(1); + goto wait_kill; } /* In sigsuspend(), block anything else than SIGWINCH. */ sigdelset(&sigs, SIGWINCH); @@ -176,19 +176,19 @@ static int winch_thread(void *arg) if (setsid() < 0) { os_info("winch_thread : setsid failed, errno = %d\n", errno); - exit(1); + goto wait_kill; } if (ioctl(pty_fd, TIOCSCTTY, 0) < 0) { os_info("winch_thread : TIOCSCTTY failed on " "fd %d err = %d\n", pty_fd, errno); - exit(1); + goto wait_kill; } if (tcsetpgrp(pty_fd, os_getpid()) < 0) { os_info("winch_thread : tcsetpgrp failed on fd %d err = %d\n", pty_fd, errno); - exit(1); + goto wait_kill; } /* @@ -214,6 +214,12 @@ static int winch_thread(void *arg) os_info("winch_thread : write failed, err = %d\n", errno); } + +wait_kill: + c = 2; + count = write(pipe_fd, &c, sizeof(c)); + while (1) + pause(); } static int winch_tramp(int fd, struct tty_port *port, int *fd_out, diff --git a/arch/um/drivers/line.c b/arch/um/drivers/line.c index b98545f3edb5..449d320c3f55 100644 --- a/arch/um/drivers/line.c +++ b/arch/um/drivers/line.c @@ -629,15 +629,18 @@ static irqreturn_t winch_interrupt(int irq, void *data) if (fd != -1) { err = generic_read(fd, &c, NULL); - if (err < 0) { + /* A read of 2 means the winch thread failed and has warned */ + if (err < 0 || (err == 1 && c == 2)) { if (err != -EAGAIN) { winch->fd = -1; list_del(&winch->list); os_close_file(fd); - printk(KERN_ERR "winch_interrupt : " - "read failed, errno = %d\n", -err); - printk(KERN_ERR "fd %d is losing SIGWINCH " - "support\n", winch->tty_fd); + if (err < 0) { + printk(KERN_ERR "winch_interrupt : read failed, errno = %d\n", + -err); + printk(KERN_ERR "fd %d is losing SIGWINCH support\n", + winch->tty_fd); + } INIT_WORK(&winch->work, __free_winch); schedule_work(&winch->work); return IRQ_HANDLED; From 6d64095ea8698e3cb1698ec4e81acb2aa1500322 Mon Sep 17 00:00:00 2001 From: Benjamin Berg Date: Fri, 10 Nov 2023 12:03:44 +0100 Subject: [PATCH 254/336] um: Do not use printk in userspace trampoline The trampoline is running in a cloned process. It is not safe to use printk for error printing there. Signed-off-by: Benjamin Berg Signed-off-by: Richard Weinberger --- arch/um/os-Linux/skas/process.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/um/os-Linux/skas/process.c b/arch/um/os-Linux/skas/process.c index ed20cefb2963..1f5c3f2523d1 100644 --- a/arch/um/os-Linux/skas/process.c +++ b/arch/um/os-Linux/skas/process.c @@ -221,8 +221,8 @@ static int userspace_tramp(void *stack) addr = mmap64((void *) STUB_CODE, UM_KERN_PAGE_SIZE, PROT_EXEC, MAP_FIXED | MAP_PRIVATE, fd, offset); if (addr == MAP_FAILED) { - printk(UM_KERN_ERR "mapping mmap stub at 0x%lx failed, errno = %d\n", - STUB_CODE, errno); + os_info("mapping mmap stub at 0x%lx failed, errno = %d\n", + STUB_CODE, errno); exit(1); } @@ -231,8 +231,8 @@ static int userspace_tramp(void *stack) STUB_DATA_PAGES * UM_KERN_PAGE_SIZE, PROT_READ | PROT_WRITE, MAP_FIXED | MAP_SHARED, fd, offset); if (addr == MAP_FAILED) { - printk(UM_KERN_ERR "mapping segfault stack at 0x%lx failed, errno = %d\n", - STUB_DATA, errno); + os_info("mapping segfault stack at 0x%lx failed, errno = %d\n", + STUB_DATA, errno); exit(1); } @@ -242,8 +242,8 @@ static int userspace_tramp(void *stack) sa.sa_sigaction = (void *) segv_handler; sa.sa_restorer = NULL; if (sigaction(SIGSEGV, &sa, NULL) < 0) { - printk(UM_KERN_ERR "%s - setting SIGSEGV handler failed - errno = %d\n", - __func__, errno); + os_info("%s - setting SIGSEGV handler failed - errno = %d\n", + __func__, errno); exit(1); } From 7b84543cbd8861c83a2ec7c8848e936ce214bc01 Mon Sep 17 00:00:00 2001 From: Benjamin Berg Date: Fri, 10 Nov 2023 12:03:45 +0100 Subject: [PATCH 255/336] um: Always inline stub functions The stub executable page is remapped to a different location in the userland process. As these functions may be used by the stub, they really need to be always inlined rather than permitting the compiler to emit a function. Signed-off-by: Benjamin Berg Signed-off-by: Richard Weinberger --- arch/x86/um/shared/sysdep/stub_32.h | 21 +++++++++++---------- arch/x86/um/shared/sysdep/stub_64.h | 17 +++++++++-------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/arch/x86/um/shared/sysdep/stub_32.h b/arch/x86/um/shared/sysdep/stub_32.h index 38fa894b65d0..4e763f8b380f 100644 --- a/arch/x86/um/shared/sysdep/stub_32.h +++ b/arch/x86/um/shared/sysdep/stub_32.h @@ -12,7 +12,7 @@ #define STUB_MMAP_NR __NR_mmap2 #define MMAP_OFFSET(o) ((o) >> UM_KERN_PAGE_SHIFT) -static inline long stub_syscall0(long syscall) +static __always_inline long stub_syscall0(long syscall) { long ret; @@ -21,7 +21,7 @@ static inline long stub_syscall0(long syscall) return ret; } -static inline long stub_syscall1(long syscall, long arg1) +static __always_inline long stub_syscall1(long syscall, long arg1) { long ret; @@ -30,7 +30,7 @@ static inline long stub_syscall1(long syscall, long arg1) return ret; } -static inline long stub_syscall2(long syscall, long arg1, long arg2) +static __always_inline long stub_syscall2(long syscall, long arg1, long arg2) { long ret; @@ -40,7 +40,8 @@ static inline long stub_syscall2(long syscall, long arg1, long arg2) return ret; } -static inline long stub_syscall3(long syscall, long arg1, long arg2, long arg3) +static __always_inline long stub_syscall3(long syscall, long arg1, long arg2, + long arg3) { long ret; @@ -50,8 +51,8 @@ static inline long stub_syscall3(long syscall, long arg1, long arg2, long arg3) return ret; } -static inline long stub_syscall4(long syscall, long arg1, long arg2, long arg3, - long arg4) +static __always_inline long stub_syscall4(long syscall, long arg1, long arg2, + long arg3, long arg4) { long ret; @@ -61,8 +62,8 @@ static inline long stub_syscall4(long syscall, long arg1, long arg2, long arg3, return ret; } -static inline long stub_syscall5(long syscall, long arg1, long arg2, long arg3, - long arg4, long arg5) +static __always_inline long stub_syscall5(long syscall, long arg1, long arg2, + long arg3, long arg4, long arg5) { long ret; @@ -72,12 +73,12 @@ static inline long stub_syscall5(long syscall, long arg1, long arg2, long arg3, return ret; } -static inline void trap_myself(void) +static __always_inline void trap_myself(void) { __asm("int3"); } -static inline void remap_stack_and_trap(void) +static __always_inline void remap_stack_and_trap(void) { __asm__ volatile ( "movl %%esp,%%ebx ;" diff --git a/arch/x86/um/shared/sysdep/stub_64.h b/arch/x86/um/shared/sysdep/stub_64.h index 2de1c8f88173..b24168ef0ac4 100644 --- a/arch/x86/um/shared/sysdep/stub_64.h +++ b/arch/x86/um/shared/sysdep/stub_64.h @@ -16,7 +16,7 @@ #define __syscall_clobber "r11","rcx","memory" #define __syscall "syscall" -static inline long stub_syscall0(long syscall) +static __always_inline long stub_syscall0(long syscall) { long ret; @@ -27,7 +27,7 @@ static inline long stub_syscall0(long syscall) return ret; } -static inline long stub_syscall2(long syscall, long arg1, long arg2) +static __always_inline long stub_syscall2(long syscall, long arg1, long arg2) { long ret; @@ -38,7 +38,8 @@ static inline long stub_syscall2(long syscall, long arg1, long arg2) return ret; } -static inline long stub_syscall3(long syscall, long arg1, long arg2, long arg3) +static __always_inline long stub_syscall3(long syscall, long arg1, long arg2, + long arg3) { long ret; @@ -50,7 +51,7 @@ static inline long stub_syscall3(long syscall, long arg1, long arg2, long arg3) return ret; } -static inline long stub_syscall4(long syscall, long arg1, long arg2, long arg3, +static __always_inline long stub_syscall4(long syscall, long arg1, long arg2, long arg3, long arg4) { long ret; @@ -64,8 +65,8 @@ static inline long stub_syscall4(long syscall, long arg1, long arg2, long arg3, return ret; } -static inline long stub_syscall5(long syscall, long arg1, long arg2, long arg3, - long arg4, long arg5) +static __always_inline long stub_syscall5(long syscall, long arg1, long arg2, + long arg3, long arg4, long arg5) { long ret; @@ -78,12 +79,12 @@ static inline long stub_syscall5(long syscall, long arg1, long arg2, long arg3, return ret; } -static inline void trap_myself(void) +static __always_inline void trap_myself(void) { __asm("int3"); } -static inline void remap_stack_and_trap(void) +static __always_inline void remap_stack_and_trap(void) { __asm__ volatile ( "movq %0,%%rax ;" From a4166aec11309d24e650e74cf8191a6d25a97fcf Mon Sep 17 00:00:00 2001 From: Samuel Holland Date: Tue, 21 Nov 2023 15:47:24 -0800 Subject: [PATCH 256/336] riscv: Deduplicate code in setup_smp() Both the ACPI and DT implementations contain some of the same code. Move it to the calling function so it is not duplicated. Signed-off-by: Samuel Holland Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20231121234736.3489608-2-samuel.holland@sifive.com Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/smpboot.c | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c index d162bf339beb..1c68e61fb852 100644 --- a/arch/riscv/kernel/smpboot.c +++ b/arch/riscv/kernel/smpboot.c @@ -125,18 +125,7 @@ static int __init acpi_parse_rintc(union acpi_subtable_headers *header, const un static void __init acpi_parse_and_init_cpus(void) { - int cpuid; - - cpu_set_ops(0); - acpi_table_parse_madt(ACPI_MADT_TYPE_RINTC, acpi_parse_rintc, 0); - - for (cpuid = 1; cpuid < nr_cpu_ids; cpuid++) { - if (cpuid_to_hartid_map(cpuid) != INVALID_HARTID) { - cpu_set_ops(cpuid); - set_cpu_possible(cpuid, true); - } - } } #else #define acpi_parse_and_init_cpus(...) do { } while (0) @@ -150,8 +139,6 @@ static void __init of_parse_and_init_cpus(void) int cpuid = 1; int rc; - cpu_set_ops(0); - for_each_of_cpu_node(dn) { rc = riscv_early_of_processor_hartid(dn, &hart); if (rc < 0) @@ -179,6 +166,18 @@ static void __init of_parse_and_init_cpus(void) if (cpuid > nr_cpu_ids) pr_warn("Total number of cpus [%d] is greater than nr_cpus option value [%d]\n", cpuid, nr_cpu_ids); +} + +void __init setup_smp(void) +{ + int cpuid; + + cpu_set_ops(0); + + if (acpi_disabled) + of_parse_and_init_cpus(); + else + acpi_parse_and_init_cpus(); for (cpuid = 1; cpuid < nr_cpu_ids; cpuid++) { if (cpuid_to_hartid_map(cpuid) != INVALID_HARTID) { @@ -188,14 +187,6 @@ static void __init of_parse_and_init_cpus(void) } } -void __init setup_smp(void) -{ - if (acpi_disabled) - of_parse_and_init_cpus(); - else - acpi_parse_and_init_cpus(); -} - static int start_secondary_cpu(int cpu, struct task_struct *tidle) { if (cpu_ops[cpu]->cpu_start) From 79093f3ec39c90edf4bd1a532d922ee6163441ec Mon Sep 17 00:00:00 2001 From: Samuel Holland Date: Tue, 21 Nov 2023 15:47:25 -0800 Subject: [PATCH 257/336] riscv: Remove unused members from struct cpu_operations name is not used anywhere at all. cpu_prepare and cpu_disable do nothing and always return 0 if implemented. Signed-off-by: Samuel Holland Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20231121234736.3489608-3-samuel.holland@sifive.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/cpu_ops.h | 10 ---------- arch/riscv/kernel/cpu-hotplug.c | 9 +-------- arch/riscv/kernel/cpu_ops.c | 2 -- arch/riscv/kernel/cpu_ops_sbi.c | 19 ------------------- arch/riscv/kernel/cpu_ops_spinwait.c | 11 ----------- arch/riscv/kernel/head.S | 1 - arch/riscv/kernel/setup.c | 1 - arch/riscv/kernel/smpboot.c | 6 ------ 8 files changed, 1 insertion(+), 58 deletions(-) diff --git a/arch/riscv/include/asm/cpu_ops.h b/arch/riscv/include/asm/cpu_ops.h index aa128466c4d4..18af75e6873c 100644 --- a/arch/riscv/include/asm/cpu_ops.h +++ b/arch/riscv/include/asm/cpu_ops.h @@ -13,26 +13,16 @@ /** * struct cpu_operations - Callback operations for hotplugging CPUs. * - * @name: Name of the boot protocol. - * @cpu_prepare: Early one-time preparation step for a cpu. If there - * is a mechanism for doing so, tests whether it is - * possible to boot the given HART. * @cpu_start: Boots a cpu into the kernel. - * @cpu_disable: Prepares a cpu to die. May fail for some - * mechanism-specific reason, which will cause the hot - * unplug to be aborted. Called from the cpu to be killed. * @cpu_stop: Makes a cpu leave the kernel. Must not fail. Called from * the cpu being stopped. * @cpu_is_stopped: Ensures a cpu has left the kernel. Called from another * cpu. */ struct cpu_operations { - const char *name; - int (*cpu_prepare)(unsigned int cpu); int (*cpu_start)(unsigned int cpu, struct task_struct *tidle); #ifdef CONFIG_HOTPLUG_CPU - int (*cpu_disable)(unsigned int cpu); void (*cpu_stop)(void); int (*cpu_is_stopped)(unsigned int cpu); #endif diff --git a/arch/riscv/kernel/cpu-hotplug.c b/arch/riscv/kernel/cpu-hotplug.c index 457a18efcb11..934eb64da0d0 100644 --- a/arch/riscv/kernel/cpu-hotplug.c +++ b/arch/riscv/kernel/cpu-hotplug.c @@ -29,25 +29,18 @@ bool cpu_has_hotplug(unsigned int cpu) */ int __cpu_disable(void) { - int ret = 0; unsigned int cpu = smp_processor_id(); if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_stop) return -EOPNOTSUPP; - if (cpu_ops[cpu]->cpu_disable) - ret = cpu_ops[cpu]->cpu_disable(cpu); - - if (ret) - return ret; - remove_cpu_topology(cpu); numa_remove_cpu(cpu); set_cpu_online(cpu, false); riscv_ipi_disable(); irq_migrate_all_off_this_cpu(); - return ret; + return 0; } #ifdef CONFIG_HOTPLUG_CPU diff --git a/arch/riscv/kernel/cpu_ops.c b/arch/riscv/kernel/cpu_ops.c index eb479a88a954..5540e2880abb 100644 --- a/arch/riscv/kernel/cpu_ops.c +++ b/arch/riscv/kernel/cpu_ops.c @@ -18,8 +18,6 @@ const struct cpu_operations *cpu_ops[NR_CPUS] __ro_after_init; extern const struct cpu_operations cpu_ops_sbi; #ifndef CONFIG_RISCV_BOOT_SPINWAIT const struct cpu_operations cpu_ops_spinwait = { - .name = "", - .cpu_prepare = NULL, .cpu_start = NULL, }; #endif diff --git a/arch/riscv/kernel/cpu_ops_sbi.c b/arch/riscv/kernel/cpu_ops_sbi.c index efa0f0816634..1cc7df740edd 100644 --- a/arch/riscv/kernel/cpu_ops_sbi.c +++ b/arch/riscv/kernel/cpu_ops_sbi.c @@ -79,23 +79,7 @@ static int sbi_cpu_start(unsigned int cpuid, struct task_struct *tidle) return sbi_hsm_hart_start(hartid, boot_addr, hsm_data); } -static int sbi_cpu_prepare(unsigned int cpuid) -{ - if (!cpu_ops_sbi.cpu_start) { - pr_err("cpu start method not defined for CPU [%d]\n", cpuid); - return -ENODEV; - } - return 0; -} - #ifdef CONFIG_HOTPLUG_CPU -static int sbi_cpu_disable(unsigned int cpuid) -{ - if (!cpu_ops_sbi.cpu_stop) - return -EOPNOTSUPP; - return 0; -} - static void sbi_cpu_stop(void) { int ret; @@ -118,11 +102,8 @@ static int sbi_cpu_is_stopped(unsigned int cpuid) #endif const struct cpu_operations cpu_ops_sbi = { - .name = "sbi", - .cpu_prepare = sbi_cpu_prepare, .cpu_start = sbi_cpu_start, #ifdef CONFIG_HOTPLUG_CPU - .cpu_disable = sbi_cpu_disable, .cpu_stop = sbi_cpu_stop, .cpu_is_stopped = sbi_cpu_is_stopped, #endif diff --git a/arch/riscv/kernel/cpu_ops_spinwait.c b/arch/riscv/kernel/cpu_ops_spinwait.c index d98d19226b5f..613872b0a21a 100644 --- a/arch/riscv/kernel/cpu_ops_spinwait.c +++ b/arch/riscv/kernel/cpu_ops_spinwait.c @@ -39,15 +39,6 @@ static void cpu_update_secondary_bootdata(unsigned int cpuid, WRITE_ONCE(__cpu_spinwait_task_pointer[hartid], tidle); } -static int spinwait_cpu_prepare(unsigned int cpuid) -{ - if (!cpu_ops_spinwait.cpu_start) { - pr_err("cpu start method not defined for CPU [%d]\n", cpuid); - return -ENODEV; - } - return 0; -} - static int spinwait_cpu_start(unsigned int cpuid, struct task_struct *tidle) { /* @@ -64,7 +55,5 @@ static int spinwait_cpu_start(unsigned int cpuid, struct task_struct *tidle) } const struct cpu_operations cpu_ops_spinwait = { - .name = "spinwait", - .cpu_prepare = spinwait_cpu_prepare, .cpu_start = spinwait_cpu_start, }; diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S index b77397432403..0349e5cdfe1d 100644 --- a/arch/riscv/kernel/head.S +++ b/arch/riscv/kernel/head.S @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index 535a837de55d..2bf882804624 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -26,7 +26,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c index 1c68e61fb852..5551945255cd 100644 --- a/arch/riscv/kernel/smpboot.c +++ b/arch/riscv/kernel/smpboot.c @@ -49,7 +49,6 @@ void __init smp_prepare_boot_cpu(void) void __init smp_prepare_cpus(unsigned int max_cpus) { int cpuid; - int ret; unsigned int curr_cpuid; init_cpu_topology(); @@ -66,11 +65,6 @@ void __init smp_prepare_cpus(unsigned int max_cpus) for_each_possible_cpu(cpuid) { if (cpuid == curr_cpuid) continue; - if (cpu_ops[cpuid]->cpu_prepare) { - ret = cpu_ops[cpuid]->cpu_prepare(cpuid); - if (ret) - continue; - } set_cpu_present(cpuid, true); numa_store_cpu_info(cpuid); } From 62ff262227a45bf917fe198885ab7aa19be5a01f Mon Sep 17 00:00:00 2001 From: Samuel Holland Date: Tue, 21 Nov 2023 15:47:26 -0800 Subject: [PATCH 258/336] riscv: Use the same CPU operations for all CPUs RISC-V provides no binding (ACPI or DT) to describe per-cpu start/stop operations, so cpu_set_ops() will always detect the same operations for every CPU. Replace the cpu_ops array with a single pointer to save space and reduce boot time. Signed-off-by: Samuel Holland Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20231121234736.3489608-4-samuel.holland@sifive.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/cpu_ops.h | 4 ++-- arch/riscv/kernel/cpu-hotplug.c | 10 +++++----- arch/riscv/kernel/cpu_ops.c | 12 +++++------- arch/riscv/kernel/smp.c | 2 +- arch/riscv/kernel/smpboot.c | 13 +++++-------- 5 files changed, 18 insertions(+), 23 deletions(-) diff --git a/arch/riscv/include/asm/cpu_ops.h b/arch/riscv/include/asm/cpu_ops.h index 18af75e6873c..176b570ef982 100644 --- a/arch/riscv/include/asm/cpu_ops.h +++ b/arch/riscv/include/asm/cpu_ops.h @@ -29,7 +29,7 @@ struct cpu_operations { }; extern const struct cpu_operations cpu_ops_spinwait; -extern const struct cpu_operations *cpu_ops[NR_CPUS]; -void __init cpu_set_ops(int cpu); +extern const struct cpu_operations *cpu_ops; +void __init cpu_set_ops(void); #endif /* ifndef __ASM_CPU_OPS_H */ diff --git a/arch/riscv/kernel/cpu-hotplug.c b/arch/riscv/kernel/cpu-hotplug.c index 934eb64da0d0..28b58fc5ad19 100644 --- a/arch/riscv/kernel/cpu-hotplug.c +++ b/arch/riscv/kernel/cpu-hotplug.c @@ -18,7 +18,7 @@ bool cpu_has_hotplug(unsigned int cpu) { - if (cpu_ops[cpu]->cpu_stop) + if (cpu_ops->cpu_stop) return true; return false; @@ -31,7 +31,7 @@ int __cpu_disable(void) { unsigned int cpu = smp_processor_id(); - if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_stop) + if (!cpu_ops->cpu_stop) return -EOPNOTSUPP; remove_cpu_topology(cpu); @@ -55,8 +55,8 @@ void arch_cpuhp_cleanup_dead_cpu(unsigned int cpu) pr_notice("CPU%u: off\n", cpu); /* Verify from the firmware if the cpu is really stopped*/ - if (cpu_ops[cpu]->cpu_is_stopped) - ret = cpu_ops[cpu]->cpu_is_stopped(cpu); + if (cpu_ops->cpu_is_stopped) + ret = cpu_ops->cpu_is_stopped(cpu); if (ret) pr_warn("CPU%d may not have stopped: %d\n", cpu, ret); } @@ -70,7 +70,7 @@ void __noreturn arch_cpu_idle_dead(void) cpuhp_ap_report_dead(); - cpu_ops[smp_processor_id()]->cpu_stop(); + cpu_ops->cpu_stop(); /* It should never reach here */ BUG(); } diff --git a/arch/riscv/kernel/cpu_ops.c b/arch/riscv/kernel/cpu_ops.c index 5540e2880abb..6a8bd8f4db07 100644 --- a/arch/riscv/kernel/cpu_ops.c +++ b/arch/riscv/kernel/cpu_ops.c @@ -13,7 +13,7 @@ #include #include -const struct cpu_operations *cpu_ops[NR_CPUS] __ro_after_init; +const struct cpu_operations *cpu_ops __ro_after_init = &cpu_ops_spinwait; extern const struct cpu_operations cpu_ops_sbi; #ifndef CONFIG_RISCV_BOOT_SPINWAIT @@ -22,14 +22,12 @@ const struct cpu_operations cpu_ops_spinwait = { }; #endif -void __init cpu_set_ops(int cpuid) +void __init cpu_set_ops(void) { #if IS_ENABLED(CONFIG_RISCV_SBI) if (sbi_probe_extension(SBI_EXT_HSM)) { - if (!cpuid) - pr_info("SBI HSM extension detected\n"); - cpu_ops[cpuid] = &cpu_ops_sbi; - } else + pr_info("SBI HSM extension detected\n"); + cpu_ops = &cpu_ops_sbi; + } #endif - cpu_ops[cpuid] = &cpu_ops_spinwait; } diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c index 40420afbb1a0..45dd4035416e 100644 --- a/arch/riscv/kernel/smp.c +++ b/arch/riscv/kernel/smp.c @@ -81,7 +81,7 @@ static inline void ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs) #ifdef CONFIG_HOTPLUG_CPU if (cpu_has_hotplug(cpu)) - cpu_ops[cpu]->cpu_stop(); + cpu_ops->cpu_stop(); #endif for(;;) diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c index 5551945255cd..519b6bd946e5 100644 --- a/arch/riscv/kernel/smpboot.c +++ b/arch/riscv/kernel/smpboot.c @@ -166,25 +166,22 @@ void __init setup_smp(void) { int cpuid; - cpu_set_ops(0); + cpu_set_ops(); if (acpi_disabled) of_parse_and_init_cpus(); else acpi_parse_and_init_cpus(); - for (cpuid = 1; cpuid < nr_cpu_ids; cpuid++) { - if (cpuid_to_hartid_map(cpuid) != INVALID_HARTID) { - cpu_set_ops(cpuid); + for (cpuid = 1; cpuid < nr_cpu_ids; cpuid++) + if (cpuid_to_hartid_map(cpuid) != INVALID_HARTID) set_cpu_possible(cpuid, true); - } - } } static int start_secondary_cpu(int cpu, struct task_struct *tidle) { - if (cpu_ops[cpu]->cpu_start) - return cpu_ops[cpu]->cpu_start(cpu, tidle); + if (cpu_ops->cpu_start) + return cpu_ops->cpu_start(cpu, tidle); return -EOPNOTSUPP; } From 7d748f60a4b82b50bf25fad1bd42d33f049f76aa Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 6 Dec 2023 09:49:46 -0700 Subject: [PATCH 259/336] um: net: Fix return type of uml_net_start_xmit() With clang's kernel control flow integrity (kCFI, CONFIG_CFI_CLANG), indirect call targets are validated against the expected function pointer prototype to make sure the call target is valid to help mitigate ROP attacks. If they are not identical, there is a failure at run time, which manifests as either a kernel panic or thread getting killed. A warning in clang aims to catch these at compile time, which reveals: arch/um/drivers/net_kern.c:353:21: warning: incompatible function pointer types initializing 'netdev_tx_t (*)(struct sk_buff *, struct net_device *)' (aka 'enum netdev_tx (*)(struct sk_buff *, struct net_device *)') with an expression of type 'int (struct sk_buff *, struct net_device *)' [-Wincompatible-function-pointer-types-strict] 353 | .ndo_start_xmit = uml_net_start_xmit, | ^~~~~~~~~~~~~~~~~~ 1 warning generated. ->ndo_start_xmit() in 'struct net_device_ops' expects a return type of 'netdev_tx_t', not 'int'. Adjust the return type of uml_net_start_xmit() to match the prototype's to resolve the warning. While UML does not currently implement support for kCFI, it could in the future, which means this warning becomes a fatal CFI failure at run time. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202310031340.v1vPh207-lkp@intel.com/ Acked-by: Anton Ivanov Signed-off-by: Nathan Chancellor Signed-off-by: Richard Weinberger --- arch/um/drivers/net_kern.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/um/drivers/net_kern.c b/arch/um/drivers/net_kern.c index 3d7836c46507..cabcc501b448 100644 --- a/arch/um/drivers/net_kern.c +++ b/arch/um/drivers/net_kern.c @@ -204,7 +204,7 @@ static int uml_net_close(struct net_device *dev) return 0; } -static int uml_net_start_xmit(struct sk_buff *skb, struct net_device *dev) +static netdev_tx_t uml_net_start_xmit(struct sk_buff *skb, struct net_device *dev) { struct uml_net_private *lp = netdev_priv(dev); unsigned long flags; From abe4eaa8618bb36c2b33e9cdde0499296a23448c Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Wed, 25 Oct 2023 22:45:05 +0200 Subject: [PATCH 260/336] um: time-travel: fix time corruption In 'basic' time-travel mode (without =inf-cpu or =ext), we still get timer interrupts. These can happen at arbitrary points in time, i.e. while in timer_read(), which pushes time forward just a little bit. Then, if we happen to get the interrupt after calculating the new time to push to, but before actually finishing that, the interrupt will set the time to a value that's incompatible with the forward, and we'll crash because time goes backwards when we do the forwarding. Fix this by reading the time_travel_time, calculating the adjustment, and doing the adjustment all with interrupts disabled. Reported-by: Vincent Whitchurch Signed-off-by: Johannes Berg Signed-off-by: Richard Weinberger --- arch/um/kernel/time.c | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/arch/um/kernel/time.c b/arch/um/kernel/time.c index fddd1dec27e6..3e270da6b6f6 100644 --- a/arch/um/kernel/time.c +++ b/arch/um/kernel/time.c @@ -432,9 +432,29 @@ static void time_travel_update_time(unsigned long long next, bool idle) time_travel_del_event(&ne); } +static void time_travel_update_time_rel(unsigned long long offs) +{ + unsigned long flags; + + /* + * Disable interrupts before calculating the new time so + * that a real timer interrupt (signal) can't happen at + * a bad time e.g. after we read time_travel_time but + * before we've completed updating the time. + */ + local_irq_save(flags); + time_travel_update_time(time_travel_time + offs, false); + local_irq_restore(flags); +} + void time_travel_ndelay(unsigned long nsec) { - time_travel_update_time(time_travel_time + nsec, false); + /* + * Not strictly needed to use _rel() version since this is + * only used in INFCPU/EXT modes, but it doesn't hurt and + * is more readable too. + */ + time_travel_update_time_rel(nsec); } EXPORT_SYMBOL(time_travel_ndelay); @@ -568,7 +588,11 @@ static void time_travel_set_start(void) #define time_travel_time 0 #define time_travel_ext_waiting 0 -static inline void time_travel_update_time(unsigned long long ns, bool retearly) +static inline void time_travel_update_time(unsigned long long ns, bool idle) +{ +} + +static inline void time_travel_update_time_rel(unsigned long long offs) { } @@ -720,9 +744,7 @@ static u64 timer_read(struct clocksource *cs) */ if (!irqs_disabled() && !in_interrupt() && !in_softirq() && !time_travel_ext_waiting) - time_travel_update_time(time_travel_time + - TIMER_MULTIPLIER, - false); + time_travel_update_time_rel(TIMER_MULTIPLIER); return time_travel_time / TIMER_MULTIPLIER; } From 32a84cfc6caf830431375c5182391284986d4066 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Winiarski?= Date: Fri, 20 Oct 2023 11:21:58 +0200 Subject: [PATCH 261/336] arch: um: Add Clang coverage support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clang uses a different set of command line arguments for enabling coverage. Signed-off-by: Michał Winiarski Tested-by: David Gow Signed-off-by: Richard Weinberger --- arch/um/Makefile-skas | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/um/Makefile-skas b/arch/um/Makefile-skas index ac35de5316a6..67323b028999 100644 --- a/arch/um/Makefile-skas +++ b/arch/um/Makefile-skas @@ -4,7 +4,12 @@ # GPROF_OPT += -pg + +ifdef CONFIG_CC_IS_CLANG +GCOV_OPT += -fprofile-instr-generate -fcoverage-mapping +else GCOV_OPT += -fprofile-arcs -ftest-coverage +endif CFLAGS-$(CONFIG_GCOV) += $(GCOV_OPT) CFLAGS-$(CONFIG_GPROF) += $(GPROF_OPT) From c17d8847c3bef9e4fe4aef34edecc29cee3cd06f Mon Sep 17 00:00:00 2001 From: Chen Haonan Date: Fri, 15 Dec 2023 11:04:40 +0100 Subject: [PATCH 262/336] ARM: 9331/1: ARM/dma-mapping: replace kzalloc() and vzalloc() with kvzalloc() using kvzalloc() simplifies the code by avoiding the use of different memory allocation functions for different situations, making the code more uniform and readable. Signed-off-by: Chen Haonan Signed-off-by: Russell King (Oracle) --- arch/arm/mm/dma-mapping.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 5409225b4abc..d688eac6dbc1 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -859,10 +859,7 @@ static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, int i = 0; int order_idx = 0; - if (array_size <= PAGE_SIZE) - pages = kzalloc(array_size, GFP_KERNEL); - else - pages = vzalloc(array_size); + pages = kvzalloc(array_size, GFP_KERNEL); if (!pages) return NULL; From 21822553a5f424892c12d1664b3c1235b095c6c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Winiarski?= Date: Fri, 20 Oct 2023 11:21:59 +0200 Subject: [PATCH 263/336] Documentation: kunit: Add clang UML coverage example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LLVM-based toolchain is using a different set of tools for coverage. Add an example that produces output in lcov format. Signed-off-by: Michał Winiarski Reviewed-by: David Gow [rw: Added spelling fixes from David Gow] Signed-off-by: Richard Weinberger --- Documentation/dev-tools/kunit/running_tips.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Documentation/dev-tools/kunit/running_tips.rst b/Documentation/dev-tools/kunit/running_tips.rst index 766f9cdea0fa..221c730b9cb2 100644 --- a/Documentation/dev-tools/kunit/running_tips.rst +++ b/Documentation/dev-tools/kunit/running_tips.rst @@ -139,6 +139,17 @@ If your installed version of gcc doesn't work, you can tweak the steps: $ ./tools/testing/kunit/kunit.py run --make_options=CC=/usr/bin/gcc-6 $ lcov -t "my_kunit_tests" -o coverage.info -c -d .kunit/ --gcov-tool=/usr/bin/gcov-6 +Alternatively, LLVM-based toolchains can also be used: + +.. code-block:: bash + + # Build with LLVM and append coverage options to the current config + $ ./tools/testing/kunit/kunit.py run --make_options LLVM=1 --kunitconfig=.kunit/ --kunitconfig=tools/testing/kunit/configs/coverage_uml.config + $ llvm-profdata merge -sparse default.profraw -o default.profdata + $ llvm-cov export --format=lcov .kunit/vmlinux -instr-profile default.profdata > coverage.info + # The coverage.info file is in lcov-compatible format and it can be used to e.g. generate HTML report + $ genhtml -o /tmp/coverage_html coverage.info + Running tests manually ====================== From 1ca1443570e4085c180ecc657d319c21b22a76f6 Mon Sep 17 00:00:00 2001 From: Benjamin Berg Date: Fri, 10 Nov 2023 12:03:46 +0100 Subject: [PATCH 264/336] um: Rely on PTRACE_SETREGSET to set FS/GS base registers These registers are saved/restored together with the other general registers using ptrace. In arch_set_tls we then just need to set the register and it will be synced back normally. Most of this logic was introduced in commit f355559cf7845 ("[PATCH] uml: x86_64 thread fixes"). However, at least today we can rely on ptrace to restore the base registers for us. As such, only the part of the patch that tracks the FS register for use as thread local storage is actually needed. Signed-off-by: Benjamin Berg Signed-off-by: Richard Weinberger --- arch/um/include/shared/os.h | 3 -- arch/x86/um/asm/elf.h | 4 +-- arch/x86/um/asm/processor_64.h | 3 -- arch/x86/um/os-Linux/Makefile | 1 - arch/x86/um/os-Linux/prctl.c | 12 ------- arch/x86/um/syscalls_64.c | 62 +++++++--------------------------- arch/x86/um/tls_64.c | 2 +- 7 files changed, 16 insertions(+), 71 deletions(-) delete mode 100644 arch/x86/um/os-Linux/prctl.c diff --git a/arch/um/include/shared/os.h b/arch/um/include/shared/os.h index 0df646c6651e..aff8906304ea 100644 --- a/arch/um/include/shared/os.h +++ b/arch/um/include/shared/os.h @@ -323,9 +323,6 @@ extern void sigio_broken(int fd); extern int __add_sigio_fd(int fd); extern int __ignore_sigio_fd(int fd); -/* prctl.c */ -extern int os_arch_prctl(int pid, int option, unsigned long *arg2); - /* tty.c */ extern int get_pty(void); diff --git a/arch/x86/um/asm/elf.h b/arch/x86/um/asm/elf.h index 6523eb7c3bd1..6052200fe925 100644 --- a/arch/x86/um/asm/elf.h +++ b/arch/x86/um/asm/elf.h @@ -168,8 +168,8 @@ do { \ (pr_reg)[18] = (_regs)->regs.gp[18]; \ (pr_reg)[19] = (_regs)->regs.gp[19]; \ (pr_reg)[20] = (_regs)->regs.gp[20]; \ - (pr_reg)[21] = current->thread.arch.fs; \ - (pr_reg)[22] = 0; \ + (pr_reg)[21] = (_regs)->regs.gp[21]; \ + (pr_reg)[22] = (_regs)->regs.gp[22]; \ (pr_reg)[23] = 0; \ (pr_reg)[24] = 0; \ (pr_reg)[25] = 0; \ diff --git a/arch/x86/um/asm/processor_64.h b/arch/x86/um/asm/processor_64.h index 1ef9c21877bc..f90159508936 100644 --- a/arch/x86/um/asm/processor_64.h +++ b/arch/x86/um/asm/processor_64.h @@ -10,13 +10,11 @@ struct arch_thread { unsigned long debugregs[8]; int debugregs_seq; - unsigned long fs; struct faultinfo faultinfo; }; #define INIT_ARCH_THREAD { .debugregs = { [ 0 ... 7 ] = 0 }, \ .debugregs_seq = 0, \ - .fs = 0, \ .faultinfo = { 0, 0, 0 } } #define STACKSLOTS_PER_LINE 4 @@ -28,7 +26,6 @@ static inline void arch_flush_thread(struct arch_thread *thread) static inline void arch_copy_thread(struct arch_thread *from, struct arch_thread *to) { - to->fs = from->fs; } #define current_sp() ({ void *sp; __asm__("movq %%rsp, %0" : "=r" (sp) : ); sp; }) diff --git a/arch/x86/um/os-Linux/Makefile b/arch/x86/um/os-Linux/Makefile index ae169125d03f..5249bbc30dcd 100644 --- a/arch/x86/um/os-Linux/Makefile +++ b/arch/x86/um/os-Linux/Makefile @@ -6,7 +6,6 @@ obj-y = registers.o task_size.o mcontext.o obj-$(CONFIG_X86_32) += tls.o -obj-$(CONFIG_64BIT) += prctl.o USER_OBJS := $(obj-y) diff --git a/arch/x86/um/os-Linux/prctl.c b/arch/x86/um/os-Linux/prctl.c deleted file mode 100644 index 8431e87ac333..000000000000 --- a/arch/x86/um/os-Linux/prctl.c +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright (C) 2007 Jeff Dike (jdike@{addtoit.com,linux.intel.com}) - * Licensed under the GPL - */ - -#include -#include - -int os_arch_prctl(int pid, int option, unsigned long *arg2) -{ - return ptrace(PTRACE_ARCH_PRCTL, pid, (unsigned long) arg2, option); -} diff --git a/arch/x86/um/syscalls_64.c b/arch/x86/um/syscalls_64.c index 27b29ae6c471..6a00a28c9cca 100644 --- a/arch/x86/um/syscalls_64.c +++ b/arch/x86/um/syscalls_64.c @@ -16,60 +16,24 @@ long arch_prctl(struct task_struct *task, int option, unsigned long __user *arg2) { - unsigned long *ptr = arg2, tmp; - long ret; - int pid = task->mm->context.id.u.pid; - - /* - * With ARCH_SET_FS (and ARCH_SET_GS is treated similarly to - * be safe), we need to call arch_prctl on the host because - * setting %fs may result in something else happening (like a - * GDT or thread.fs being set instead). So, we let the host - * fiddle the registers and thread struct and restore the - * registers afterwards. - * - * So, the saved registers are stored to the process (this - * needed because a stub may have been the last thing to run), - * arch_prctl is run on the host, then the registers are read - * back. - */ - switch (option) { - case ARCH_SET_FS: - case ARCH_SET_GS: - ret = restore_pid_registers(pid, ¤t->thread.regs.regs); - if (ret) - return ret; - break; - case ARCH_GET_FS: - case ARCH_GET_GS: - /* - * With these two, we read to a local pointer and - * put_user it to the userspace pointer that we were - * given. If addr isn't valid (because it hasn't been - * faulted in or is just bogus), we want put_user to - * fault it in (or return -EFAULT) instead of having - * the host return -EFAULT. - */ - ptr = &tmp; - } - - ret = os_arch_prctl(pid, option, ptr); - if (ret) - return ret; + long ret = -EINVAL; switch (option) { case ARCH_SET_FS: - current->thread.arch.fs = (unsigned long) ptr; - ret = save_registers(pid, ¤t->thread.regs.regs); + current->thread.regs.regs.gp[FS_BASE / sizeof(unsigned long)] = + (unsigned long) arg2; + ret = 0; break; case ARCH_SET_GS: - ret = save_registers(pid, ¤t->thread.regs.regs); + current->thread.regs.regs.gp[GS_BASE / sizeof(unsigned long)] = + (unsigned long) arg2; + ret = 0; break; case ARCH_GET_FS: - ret = put_user(tmp, arg2); + ret = put_user(current->thread.regs.regs.gp[FS_BASE / sizeof(unsigned long)], arg2); break; case ARCH_GET_GS: - ret = put_user(tmp, arg2); + ret = put_user(current->thread.regs.regs.gp[GS_BASE / sizeof(unsigned long)], arg2); break; } @@ -83,10 +47,10 @@ SYSCALL_DEFINE2(arch_prctl, int, option, unsigned long, arg2) void arch_switch_to(struct task_struct *to) { - if ((to->thread.arch.fs == 0) || (to->mm == NULL)) - return; - - arch_prctl(to, ARCH_SET_FS, (void __user *) to->thread.arch.fs); + /* + * Nothing needs to be done on x86_64. + * The FS_BASE/GS_BASE registers are saved in the ptrace register set. + */ } SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len, diff --git a/arch/x86/um/tls_64.c b/arch/x86/um/tls_64.c index ebd3855d9b13..c51a613f6f5c 100644 --- a/arch/x86/um/tls_64.c +++ b/arch/x86/um/tls_64.c @@ -12,7 +12,7 @@ int arch_set_tls(struct task_struct *t, unsigned long tls) * If CLONE_SETTLS is set, we need to save the thread id * so it can be set during context switches. */ - t->thread.arch.fs = tls; + t->thread.regs.regs.gp[FS_BASE / sizeof(unsigned long)] = tls; return 0; } From 1e41c415e21ff03ddf5c3725b608d0e4f0c239e3 Mon Sep 17 00:00:00 2001 From: Benjamin Berg Date: Fri, 10 Nov 2023 12:03:47 +0100 Subject: [PATCH 265/336] um: Remove unused register save/restore functions These functions were only used when calling PTRACE_ARCH_PRCTL, but this code has been removed. Signed-off-by: Benjamin Berg Signed-off-by: Richard Weinberger --- arch/um/include/shared/registers.h | 2 -- arch/um/os-Linux/registers.c | 20 -------------------- 2 files changed, 22 deletions(-) diff --git a/arch/um/include/shared/registers.h b/arch/um/include/shared/registers.h index 2f9c3ce5b45e..a0450326521c 100644 --- a/arch/um/include/shared/registers.h +++ b/arch/um/include/shared/registers.h @@ -14,8 +14,6 @@ extern int save_fp_registers(int pid, unsigned long *fp_regs); extern int restore_fp_registers(int pid, unsigned long *fp_regs); extern int save_fpx_registers(int pid, unsigned long *fp_regs); extern int restore_fpx_registers(int pid, unsigned long *fp_regs); -extern int save_registers(int pid, struct uml_pt_regs *regs); -extern int restore_pid_registers(int pid, struct uml_pt_regs *regs); extern int init_pid_registers(int pid); extern void get_safe_registers(unsigned long *regs, unsigned long *fp_regs); extern int get_fp_registers(int pid, unsigned long *regs); diff --git a/arch/um/os-Linux/registers.c b/arch/um/os-Linux/registers.c index b123955be7ac..bd80b921add0 100644 --- a/arch/um/os-Linux/registers.c +++ b/arch/um/os-Linux/registers.c @@ -11,26 +11,6 @@ #include #include -int save_registers(int pid, struct uml_pt_regs *regs) -{ - int err; - - err = ptrace(PTRACE_GETREGS, pid, 0, regs->gp); - if (err < 0) - return -errno; - return 0; -} - -int restore_pid_registers(int pid, struct uml_pt_regs *regs) -{ - int err; - - err = ptrace(PTRACE_SETREGS, pid, 0, regs->gp); - if (err < 0) - return -errno; - return 0; -} - /* This is set once at boot time and not changed thereafter */ static unsigned long exec_regs[MAX_REG_NR]; From 83aec96c631e0fa75cfe6d6a1b113a32151aaa88 Mon Sep 17 00:00:00 2001 From: Benjamin Berg Date: Fri, 10 Nov 2023 12:03:48 +0100 Subject: [PATCH 266/336] um: Mark 32bit syscall helpers as clobbering memory The 64bit helper are marked to clobber the memory, but the 32bit ones are not. Add the appropriate clobber to the 32bit helper routines so that the compiler cannot do invalid optimizations. Signed-off-by: Benjamin Berg Signed-off-by: Richard Weinberger --- arch/x86/um/shared/sysdep/stub_32.h | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/arch/x86/um/shared/sysdep/stub_32.h b/arch/x86/um/shared/sysdep/stub_32.h index 4e763f8b380f..ea8b5a2d67af 100644 --- a/arch/x86/um/shared/sysdep/stub_32.h +++ b/arch/x86/um/shared/sysdep/stub_32.h @@ -16,7 +16,8 @@ static __always_inline long stub_syscall0(long syscall) { long ret; - __asm__ volatile ("int $0x80" : "=a" (ret) : "0" (syscall)); + __asm__ volatile ("int $0x80" : "=a" (ret) : "0" (syscall) + : "memory"); return ret; } @@ -25,7 +26,8 @@ static __always_inline long stub_syscall1(long syscall, long arg1) { long ret; - __asm__ volatile ("int $0x80" : "=a" (ret) : "0" (syscall), "b" (arg1)); + __asm__ volatile ("int $0x80" : "=a" (ret) : "0" (syscall), "b" (arg1) + : "memory"); return ret; } @@ -35,7 +37,8 @@ static __always_inline long stub_syscall2(long syscall, long arg1, long arg2) long ret; __asm__ volatile ("int $0x80" : "=a" (ret) : "0" (syscall), "b" (arg1), - "c" (arg2)); + "c" (arg2) + : "memory"); return ret; } @@ -46,7 +49,8 @@ static __always_inline long stub_syscall3(long syscall, long arg1, long arg2, long ret; __asm__ volatile ("int $0x80" : "=a" (ret) : "0" (syscall), "b" (arg1), - "c" (arg2), "d" (arg3)); + "c" (arg2), "d" (arg3) + : "memory"); return ret; } @@ -57,7 +61,8 @@ static __always_inline long stub_syscall4(long syscall, long arg1, long arg2, long ret; __asm__ volatile ("int $0x80" : "=a" (ret) : "0" (syscall), "b" (arg1), - "c" (arg2), "d" (arg3), "S" (arg4)); + "c" (arg2), "d" (arg3), "S" (arg4) + : "memory"); return ret; } @@ -68,7 +73,8 @@ static __always_inline long stub_syscall5(long syscall, long arg1, long arg2, long ret; __asm__ volatile ("int $0x80" : "=a" (ret) : "0" (syscall), "b" (arg1), - "c" (arg2), "d" (arg3), "S" (arg4), "D" (arg5)); + "c" (arg2), "d" (arg3), "S" (arg4), "D" (arg5) + : "memory"); return ret; } From 196da3f3f76a46905f7daab29c56974f1aba9a7a Mon Sep 17 00:00:00 2001 From: Cristian Ciocaltea Date: Fri, 5 Jan 2024 19:40:06 +0200 Subject: [PATCH 267/336] drm/rockchip: vop2: Drop unused if_dclk_rate variable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit 5a028e8f062f ("drm/rockchip: vop2: Add support for rk3588") introduced a variable which ended up being unused: rockchip_drm_vop2.c:1688:23: warning: variable ‘if_dclk_rate’ set but not used [-Wunused-but-set-variable] This has been initially used as part of a formula to compute the clock dividers, but eventually it has been replaced by static values. Drop the variable declaration and move its assignment to the comment block, to serve as documentation of how the constants have been generated. Signed-off-by: Cristian Ciocaltea Signed-off-by: Heiko Stuebner Link: https://patchwork.freedesktop.org/patch/msgid/20240105174007.98054-1-cristian.ciocaltea@collabora.com --- drivers/gpu/drm/rockchip/rockchip_drm_vop2.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c index a3a03e9cfe13..85b3b4871a1d 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c +++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c @@ -1680,7 +1680,6 @@ static unsigned long rk3588_calc_cru_cfg(struct vop2_video_port *vp, int id, unsigned long dclk_core_rate = v_pixclk >> 2; unsigned long dclk_rate = v_pixclk; unsigned long dclk_out_rate; - unsigned long if_dclk_rate; unsigned long if_pixclk_rate; int K = 1; @@ -1695,8 +1694,8 @@ static unsigned long rk3588_calc_cru_cfg(struct vop2_video_port *vp, int id, } if_pixclk_rate = (dclk_core_rate << 1) / K; - if_dclk_rate = dclk_core_rate / K; /* + * if_dclk_rate = dclk_core_rate / K; * *if_pixclk_div = dclk_rate / if_pixclk_rate; * *if_dclk_div = dclk_rate / if_dclk_rate; */ From 735ae74f73e55c191d48689bd11ff4a06ea0508f Mon Sep 17 00:00:00 2001 From: Helge Deller Date: Wed, 3 Jan 2024 21:02:16 +0100 Subject: [PATCH 268/336] parisc/firmware: Fix F-extend for PDC addresses When running with narrow firmware (64-bit kernel using a 32-bit firmware), extend PDC addresses into the 0xfffffff0.00000000 region instead of the 0xf0f0f0f0.00000000 region. This fixes the power button on the C3700 machine in qemu (64-bit CPU with 32-bit firmware), and my assumption is that the previous code was really never used (because most 64-bit machines have a 64-bit firmware), or it just worked on very old machines because they may only decode 40-bit of virtual addresses. Cc: stable@vger.kernel.org Signed-off-by: Helge Deller --- arch/parisc/kernel/firmware.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/parisc/kernel/firmware.c b/arch/parisc/kernel/firmware.c index 904ca3b9e7a7..c69f6d5946e9 100644 --- a/arch/parisc/kernel/firmware.c +++ b/arch/parisc/kernel/firmware.c @@ -123,10 +123,10 @@ static unsigned long f_extend(unsigned long address) #ifdef CONFIG_64BIT if(unlikely(parisc_narrow_firmware)) { if((address & 0xff000000) == 0xf0000000) - return 0xf0f0f0f000000000UL | (u32)address; + return (0xfffffff0UL << 32) | (u32)address; if((address & 0xf0000000) == 0xf0000000) - return 0xffffffff00000000UL | (u32)address; + return (0xffffffffUL << 32) | (u32)address; } #endif return address; From 6472036581f947109b20664121db1d143e916f0b Mon Sep 17 00:00:00 2001 From: Helge Deller Date: Wed, 3 Jan 2024 21:17:23 +0100 Subject: [PATCH 269/336] parisc/power: Fix power soft-off button emulation on qemu Make sure to start the kthread to check the power button on qemu as well if the power button address was provided. This fixes the qemu built-in system_powerdown runtime command. Fixes: d0c219472980 ("parisc/power: Add power soft-off when running on qemu") Signed-off-by: Helge Deller Cc: stable@vger.kernel.org # v6.0+ --- drivers/parisc/power.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/parisc/power.c b/drivers/parisc/power.c index bb0d92461b08..7a6a3e7f2825 100644 --- a/drivers/parisc/power.c +++ b/drivers/parisc/power.c @@ -213,7 +213,7 @@ static int __init power_init(void) if (running_on_qemu && soft_power_reg) register_sys_off_handler(SYS_OFF_MODE_POWER_OFF, SYS_OFF_PRIO_DEFAULT, qemu_power_off, (void *)soft_power_reg); - else + if (!running_on_qemu || soft_power_reg) power_task = kthread_run(kpowerswd, (void*)soft_power_reg, KTHREAD_NAME); if (IS_ERR(power_task)) { From 2f9060b1db4aa2c21c248e34476d8936a2b69cf6 Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Wed, 3 Jan 2024 17:16:03 -0600 Subject: [PATCH 270/336] MIPS: Fix typos Fix typos, most reported by "codespell arch/mips". Only touches comments, no code changes. Signed-off-by: Bjorn Helgaas Cc: linux-mips@vger.kernel.org Reviewed-by: Randy Dunlap Signed-off-by: Thomas Bogendoerfer --- arch/mips/bcm47xx/buttons.c | 6 +++--- arch/mips/bcm63xx/clk.c | 4 ++-- arch/mips/boot/compressed/dbg.c | 2 +- arch/mips/boot/elf2ecoff.c | 2 +- arch/mips/cavium-octeon/csrc-octeon.c | 2 +- arch/mips/cavium-octeon/executive/cvmx-boot-vector.c | 2 +- arch/mips/cavium-octeon/executive/cvmx-bootmem.c | 2 +- arch/mips/cavium-octeon/executive/cvmx-cmd-queue.c | 4 ++-- arch/mips/cavium-octeon/executive/cvmx-helper-jtag.c | 2 +- arch/mips/cavium-octeon/executive/cvmx-pko.c | 2 +- arch/mips/cavium-octeon/octeon-platform.c | 2 +- arch/mips/fw/arc/promlib.c | 6 +++--- arch/mips/include/asm/debug.h | 2 +- arch/mips/include/asm/io.h | 4 ++-- arch/mips/include/asm/mach-au1x00/au1000_dma.h | 2 +- arch/mips/include/asm/mach-au1x00/gpio-au1000.h | 2 +- arch/mips/include/asm/mach-lantiq/falcon/lantiq_soc.h | 2 +- arch/mips/include/asm/mach-loongson64/loongson_hwmon.h | 2 +- arch/mips/include/asm/mach-loongson64/loongson_regs.h | 2 +- arch/mips/include/asm/mach-malta/spaces.h | 4 ++-- arch/mips/include/asm/mips-boards/bonito64.h | 2 +- arch/mips/include/asm/mips-cpc.h | 2 +- arch/mips/include/asm/mipsregs.h | 4 ++-- arch/mips/include/asm/octeon/cvmx-bootinfo.h | 2 +- arch/mips/include/asm/octeon/cvmx-cmd-queue.h | 6 +++--- arch/mips/include/asm/octeon/cvmx-pko.h | 2 +- arch/mips/include/asm/octeon/cvmx-pow.h | 4 ++-- arch/mips/include/asm/octeon/octeon-model.h | 4 ++-- arch/mips/include/asm/page.h | 2 +- arch/mips/include/asm/pci.h | 2 +- arch/mips/include/asm/pgtable-bits.h | 2 +- arch/mips/include/asm/sgi/mc.h | 2 +- arch/mips/include/asm/sn/klconfig.h | 2 +- arch/mips/include/asm/sync.h | 2 +- arch/mips/include/asm/thread_info.h | 2 +- arch/mips/include/asm/timex.h | 2 +- arch/mips/include/asm/vdso/vdso.h | 2 +- arch/mips/include/uapi/asm/mman.h | 2 +- arch/mips/include/uapi/asm/msgbuf.h | 2 +- arch/mips/kernel/cpu-probe.c | 2 +- arch/mips/kernel/kprobes.c | 2 +- arch/mips/kernel/relocate.c | 2 +- arch/mips/kernel/relocate_kernel.S | 2 +- arch/mips/kernel/setup.c | 2 +- arch/mips/kernel/signal.c | 2 +- arch/mips/kernel/traps.c | 2 +- arch/mips/kernel/vpe.c | 4 ++-- arch/mips/kvm/emulate.c | 2 +- arch/mips/loongson2ef/common/platform.c | 2 +- arch/mips/loongson64/smp.c | 2 +- arch/mips/mm/c-r4k.c | 2 +- arch/mips/mm/cex-gen.S | 2 +- arch/mips/mm/tlb-r3k.c | 2 +- arch/mips/mm/tlb-r4k.c | 2 +- arch/mips/mm/tlbex.c | 4 ++-- arch/mips/net/bpf_jit_comp32.c | 2 +- arch/mips/pci/ops-loongson2.c | 2 +- arch/mips/pci/pci-alchemy.c | 2 +- arch/mips/pci/pci-ar2315.c | 2 +- arch/mips/pci/pci-lantiq.c | 2 +- arch/mips/pci/pci-octeon.c | 2 +- arch/mips/pci/pci-xtalk-bridge.c | 2 +- arch/mips/pci/pcie-octeon.c | 2 +- arch/mips/ralink/mt7621.c | 2 +- arch/mips/txx9/generic/pci.c | 2 +- 65 files changed, 80 insertions(+), 80 deletions(-) diff --git a/arch/mips/bcm47xx/buttons.c b/arch/mips/bcm47xx/buttons.c index 437a737c01dd..46994f9bb821 100644 --- a/arch/mips/bcm47xx/buttons.c +++ b/arch/mips/bcm47xx/buttons.c @@ -147,21 +147,21 @@ static const struct gpio_keys_button bcm47xx_buttons_buffalo_whr_g125[] __initconst = { BCM47XX_GPIO_KEY(0, KEY_WPS_BUTTON), BCM47XX_GPIO_KEY(4, KEY_RESTART), - BCM47XX_GPIO_KEY(5, BTN_0), /* Router / AP mode swtich */ + BCM47XX_GPIO_KEY(5, BTN_0), /* Router / AP mode switch */ }; static const struct gpio_keys_button bcm47xx_buttons_buffalo_whr_g54s[] __initconst = { BCM47XX_GPIO_KEY(0, KEY_WPS_BUTTON), BCM47XX_GPIO_KEY_H(4, KEY_RESTART), - BCM47XX_GPIO_KEY(5, BTN_0), /* Router / AP mode swtich */ + BCM47XX_GPIO_KEY(5, BTN_0), /* Router / AP mode switch */ }; static const struct gpio_keys_button bcm47xx_buttons_buffalo_whr_hp_g54[] __initconst = { BCM47XX_GPIO_KEY(0, KEY_WPS_BUTTON), BCM47XX_GPIO_KEY(4, KEY_RESTART), - BCM47XX_GPIO_KEY(5, BTN_0), /* Router / AP mode swtich */ + BCM47XX_GPIO_KEY(5, BTN_0), /* Router / AP mode switch */ }; static const struct gpio_keys_button diff --git a/arch/mips/bcm63xx/clk.c b/arch/mips/bcm63xx/clk.c index 86a6e2590866..3144965fb7dc 100644 --- a/arch/mips/bcm63xx/clk.c +++ b/arch/mips/bcm63xx/clk.c @@ -174,7 +174,7 @@ static void enetsw_set(struct clk *clk, int enable) } if (enable) { - /* reset switch core afer clock change */ + /* reset switch core after clock change */ bcm63xx_core_set_reset(BCM63XX_RESET_ENETSW, 1); msleep(10); bcm63xx_core_set_reset(BCM63XX_RESET_ENETSW, 0); @@ -304,7 +304,7 @@ static void xtm_set(struct clk *clk, int enable) bcm_hwclock_set(CKCTL_6368_SAR_EN, enable); if (enable) { - /* reset sar core afer clock change */ + /* reset sar core after clock change */ bcm63xx_core_set_reset(BCM63XX_RESET_SAR, 1); mdelay(1); bcm63xx_core_set_reset(BCM63XX_RESET_SAR, 0); diff --git a/arch/mips/boot/compressed/dbg.c b/arch/mips/boot/compressed/dbg.c index f6728a8fd1c3..98dfb2e59dfe 100644 --- a/arch/mips/boot/compressed/dbg.c +++ b/arch/mips/boot/compressed/dbg.c @@ -3,7 +3,7 @@ * MIPS-specific debug support for pre-boot environment * * NOTE: putc() is board specific, if your board have a 16550 compatible uart, - * please select SYS_SUPPORTS_ZBOOT_UART16550 for your machine. othewise, you + * please select SYS_SUPPORTS_ZBOOT_UART16550 for your machine. otherwise, you * need to implement your own putc(). */ #include diff --git a/arch/mips/boot/elf2ecoff.c b/arch/mips/boot/elf2ecoff.c index 6972b97235da..549c5d6ef6d7 100644 --- a/arch/mips/boot/elf2ecoff.c +++ b/arch/mips/boot/elf2ecoff.c @@ -443,7 +443,7 @@ int main(int argc, char *argv[]) efh.f_symptr = 0; efh.f_nsyms = 0; efh.f_opthdr = sizeof eah; - efh.f_flags = 0x100f; /* Stripped, not sharable. */ + efh.f_flags = 0x100f; /* Stripped, not shareable. */ memset(esecs, 0, sizeof esecs); strcpy(esecs[0].s_name, ".text"); diff --git a/arch/mips/cavium-octeon/csrc-octeon.c b/arch/mips/cavium-octeon/csrc-octeon.c index 124817609ce0..af62a210a40b 100644 --- a/arch/mips/cavium-octeon/csrc-octeon.c +++ b/arch/mips/cavium-octeon/csrc-octeon.c @@ -113,7 +113,7 @@ static struct clocksource clocksource_mips = { unsigned long long notrace sched_clock(void) { - /* 64-bit arithmatic can overflow, so use 128-bit. */ + /* 64-bit arithmetic can overflow, so use 128-bit. */ u64 t1, t2, t3; unsigned long long rv; u64 mult = clocksource_mips.mult; diff --git a/arch/mips/cavium-octeon/executive/cvmx-boot-vector.c b/arch/mips/cavium-octeon/executive/cvmx-boot-vector.c index b7019d21808e..76446db66def 100644 --- a/arch/mips/cavium-octeon/executive/cvmx-boot-vector.c +++ b/arch/mips/cavium-octeon/executive/cvmx-boot-vector.c @@ -143,7 +143,7 @@ static void cvmx_boot_vector_init(void *mem) uint64_t v = _cvmx_bootvector_data[i]; if (OCTEON_IS_OCTEON1PLUS() && (i == 0 || i == 7)) - v &= 0xffffffff00000000ull; /* KScratch not availble. */ + v &= 0xffffffff00000000ull; /* KScratch not available */ cvmx_write_csr(CVMX_MIO_BOOT_LOC_ADR, i * 8); cvmx_write_csr(CVMX_MIO_BOOT_LOC_DAT, v); } diff --git a/arch/mips/cavium-octeon/executive/cvmx-bootmem.c b/arch/mips/cavium-octeon/executive/cvmx-bootmem.c index 334bf8e577e5..628ebdf4b9c5 100644 --- a/arch/mips/cavium-octeon/executive/cvmx-bootmem.c +++ b/arch/mips/cavium-octeon/executive/cvmx-bootmem.c @@ -264,7 +264,7 @@ int64_t cvmx_bootmem_phy_alloc(uint64_t req_size, uint64_t address_min, * Convert !0 address_min and 0 address_max to special case of * range that specifies an exact memory block to allocate. Do * this before other checks and adjustments so that this - * tranformation will be validated. + * transformation will be validated. */ if (address_min && !address_max) address_max = address_min + req_size; diff --git a/arch/mips/cavium-octeon/executive/cvmx-cmd-queue.c b/arch/mips/cavium-octeon/executive/cvmx-cmd-queue.c index aa7bbf8d0df5..042a6bc44b5c 100644 --- a/arch/mips/cavium-octeon/executive/cvmx-cmd-queue.c +++ b/arch/mips/cavium-octeon/executive/cvmx-cmd-queue.c @@ -192,7 +192,7 @@ cvmx_cmd_queue_result_t cvmx_cmd_queue_initialize(cvmx_cmd_queue_id_t queue_id, } /* - * Shutdown a queue a free it's command buffers to the FPA. The + * Shutdown a queue and free its command buffers to the FPA. The * hardware connected to the queue must be stopped before this * function is called. * @@ -285,7 +285,7 @@ int cvmx_cmd_queue_length(cvmx_cmd_queue_id_t queue_id) /* * Return the command buffer to be written to. The purpose of this - * function is to allow CVMX routine access t othe low level buffer + * function is to allow CVMX routine access to the low level buffer * for initial hardware setup. User applications should not call this * function directly. * diff --git a/arch/mips/cavium-octeon/executive/cvmx-helper-jtag.c b/arch/mips/cavium-octeon/executive/cvmx-helper-jtag.c index 607b4e659579..1fceb7fd2c94 100644 --- a/arch/mips/cavium-octeon/executive/cvmx-helper-jtag.c +++ b/arch/mips/cavium-octeon/executive/cvmx-helper-jtag.c @@ -103,7 +103,7 @@ uint32_t cvmx_helper_qlm_jtag_shift(int qlm, int bits, uint32_t data) /** * Shift long sequences of zeros into the QLM JTAG chain. It is * common to need to shift more than 32 bits of zeros into the - * chain. This function is a convience wrapper around + * chain. This function is a convenience wrapper around * cvmx_helper_qlm_jtag_shift() to shift more than 32 bits of * zeros at a time. * diff --git a/arch/mips/cavium-octeon/executive/cvmx-pko.c b/arch/mips/cavium-octeon/executive/cvmx-pko.c index 15faca494c80..6e70b859a0ac 100644 --- a/arch/mips/cavium-octeon/executive/cvmx-pko.c +++ b/arch/mips/cavium-octeon/executive/cvmx-pko.c @@ -615,7 +615,7 @@ int cvmx_pko_rate_limit_bits(int port, uint64_t bits_s, int burst) /* * Each packet has a 12 bytes of interframe gap, an 8 byte * preamble, and a 4 byte CRC. These are not included in the - * per word count. Multiply by 8 to covert to bits and divide + * per word count. Multiply by 8 to convert to bits and divide * by 256 for limit granularity. */ pko_mem_port_rate0.s.rate_pkt = (12 + 8 + 4) * 8 * tokens_per_bit / 256; diff --git a/arch/mips/cavium-octeon/octeon-platform.c b/arch/mips/cavium-octeon/octeon-platform.c index f76783c24338..5e1dd4e6e82f 100644 --- a/arch/mips/cavium-octeon/octeon-platform.c +++ b/arch/mips/cavium-octeon/octeon-platform.c @@ -973,7 +973,7 @@ int __init octeon_prune_device_tree(void) * zero. */ - /* Asume that CS1 immediately follows. */ + /* Assume that CS1 immediately follows. */ mio_boot_reg_cfg.u64 = cvmx_read_csr(CVMX_MIO_BOOT_REG_CFGX(cs + 1)); region1_base = mio_boot_reg_cfg.s.base << 16; diff --git a/arch/mips/fw/arc/promlib.c b/arch/mips/fw/arc/promlib.c index 5e9e840a9314..93e1e70393ee 100644 --- a/arch/mips/fw/arc/promlib.c +++ b/arch/mips/fw/arc/promlib.c @@ -15,11 +15,11 @@ /* * For 64bit kernels working with a 32bit ARC PROM pointer arguments * for ARC calls need to reside in CKEG0/1. But as soon as the kernel - * switches to it's first kernel thread stack is set to an address in + * switches to its first kernel thread stack is set to an address in * XKPHYS, so anything on stack can't be used anymore. This is solved - * by using a * static declartion variables are put into BSS, which is + * by using a * static declaration variables are put into BSS, which is * linked to a CKSEG0 address. Since this is only used on UP platforms - * there is not spinlock needed + * there is no spinlock needed */ #define O32_STATIC static #else diff --git a/arch/mips/include/asm/debug.h b/arch/mips/include/asm/debug.h index c7013e1cb53f..e70392429246 100644 --- a/arch/mips/include/asm/debug.h +++ b/arch/mips/include/asm/debug.h @@ -10,7 +10,7 @@ /* * mips_debugfs_dir corresponds to the "mips" directory at the top level - * of the DebugFS hierarchy. MIPS-specific DebugFS entires should be + * of the DebugFS hierarchy. MIPS-specific DebugFS entries should be * placed beneath this directory. */ extern struct dentry *mips_debugfs_dir; diff --git a/arch/mips/include/asm/io.h b/arch/mips/include/asm/io.h index 062dd4e6b954..bba327dc1226 100644 --- a/arch/mips/include/asm/io.h +++ b/arch/mips/include/asm/io.h @@ -179,7 +179,7 @@ void iounmap(const volatile void __iomem *addr); * address is not guaranteed to be usable directly as a virtual * address. * - * This version of ioremap ensures that the memory is marked cachable by + * This version of ioremap ensures that the memory is marked cacheable by * the CPU. Also enables full write-combining. Useful for some * memory-like regions on I/O busses. */ @@ -197,7 +197,7 @@ void iounmap(const volatile void __iomem *addr); * address is not guaranteed to be usable directly as a virtual * address. * - * This version of ioremap ensures that the memory is marked uncachable + * This version of ioremap ensures that the memory is marked uncacheable * but accelerated by means of write-combining feature. It is specifically * useful for PCIe prefetchable windows, which may vastly improve a * communications performance. If it was determined on boot stage, what diff --git a/arch/mips/include/asm/mach-au1x00/au1000_dma.h b/arch/mips/include/asm/mach-au1x00/au1000_dma.h index 0a0cd4270c6f..b82e513c8523 100644 --- a/arch/mips/include/asm/mach-au1x00/au1000_dma.h +++ b/arch/mips/include/asm/mach-au1x00/au1000_dma.h @@ -259,7 +259,7 @@ static inline void set_dma_mode(unsigned int dmanr, unsigned int mode) if (!chan) return; /* - * set_dma_mode is only allowed to change endianess, direction, + * set_dma_mode is only allowed to change endianness, direction, * transfer size, device FIFO width, and coherency settings. * Make sure anything else is masked off. */ diff --git a/arch/mips/include/asm/mach-au1x00/gpio-au1000.h b/arch/mips/include/asm/mach-au1x00/gpio-au1000.h index 82bc2766e2ec..d820b481ac56 100644 --- a/arch/mips/include/asm/mach-au1x00/gpio-au1000.h +++ b/arch/mips/include/asm/mach-au1x00/gpio-au1000.h @@ -435,7 +435,7 @@ static inline void alchemy_gpio2_disable_int(int gpio2) /** * alchemy_gpio2_enable - Activate GPIO2 block. * - * The GPIO2 block must be enabled excplicitly to work. On systems + * The GPIO2 block must be enabled explicitly to work. On systems * where this isn't done by the bootloader, this macro can be used. */ static inline void alchemy_gpio2_enable(void) diff --git a/arch/mips/include/asm/mach-lantiq/falcon/lantiq_soc.h b/arch/mips/include/asm/mach-lantiq/falcon/lantiq_soc.h index 5855ba1bd1ec..40eaa72e54d0 100644 --- a/arch/mips/include/asm/mach-lantiq/falcon/lantiq_soc.h +++ b/arch/mips/include/asm/mach-lantiq/falcon/lantiq_soc.h @@ -55,7 +55,7 @@ extern __iomem void *ltq_sys1_membase; #define ltq_sys1_w32_mask(clear, set, reg) \ ltq_sys1_w32((ltq_sys1_r32(reg) & ~(clear)) | (set), reg) -/* allow the gpio and pinctrl drivers to talk to eachother */ +/* allow the gpio and pinctrl drivers to talk to each other */ extern int pinctrl_falcon_get_range_size(int id); extern void pinctrl_falcon_add_gpio_range(struct pinctrl_gpio_range *range); diff --git a/arch/mips/include/asm/mach-loongson64/loongson_hwmon.h b/arch/mips/include/asm/mach-loongson64/loongson_hwmon.h index 545f91f2ae16..721eafc4644e 100644 --- a/arch/mips/include/asm/mach-loongson64/loongson_hwmon.h +++ b/arch/mips/include/asm/mach-loongson64/loongson_hwmon.h @@ -42,7 +42,7 @@ struct loongson_fan_policy { /* period between two check. (Unit: S) */ u8 adjust_period; - /* fan adjust usually depend on a temprature input */ + /* fan adjust usually depend on a temperature input */ get_temp_fun depend_temp; /* up_step/down_step used when type is STEP_SPEED_POLICY */ diff --git a/arch/mips/include/asm/mach-loongson64/loongson_regs.h b/arch/mips/include/asm/mach-loongson64/loongson_regs.h index b5be7511f6cd..fec767507604 100644 --- a/arch/mips/include/asm/mach-loongson64/loongson_regs.h +++ b/arch/mips/include/asm/mach-loongson64/loongson_regs.h @@ -227,7 +227,7 @@ static inline void csr_writeq(u64 val, u32 reg) #define LOONGSON_CSR_NODECNT 0x408 #define LOONGSON_CSR_CPUTEMP 0x428 -/* PerCore CSR, only accessable by local cores */ +/* PerCore CSR, only accessible by local cores */ #define LOONGSON_CSR_IPI_STATUS 0x1000 #define LOONGSON_CSR_IPI_EN 0x1004 #define LOONGSON_CSR_IPI_SET 0x1008 diff --git a/arch/mips/include/asm/mach-malta/spaces.h b/arch/mips/include/asm/mach-malta/spaces.h index d7e54971ec66..1ce4ba97852f 100644 --- a/arch/mips/include/asm/mach-malta/spaces.h +++ b/arch/mips/include/asm/mach-malta/spaces.h @@ -23,13 +23,13 @@ * The kernel is still located in 0x80000000(kseg0). However, * the physical mask has been shifted to 0x80000000 which exploits the alias * on the Malta board. As a result of which, we override the __pa_symbol - * to peform direct mapping from virtual to physical addresses. In other + * to perform direct mapping from virtual to physical addresses. In other * words, the 0x80000000 virtual address maps to 0x80000000 physical address * which in turn aliases to 0x0. We do this in order to be able to use a flat * 2GB of memory (0x80000000 - 0xffffffff) so we can avoid the I/O hole in * 0x10000000 - 0x1fffffff. * The last 64KB of physical memory are reserved for correct HIGHMEM - * macros arithmetics. + * macros arithmetic. * */ diff --git a/arch/mips/include/asm/mips-boards/bonito64.h b/arch/mips/include/asm/mips-boards/bonito64.h index 5368891d424b..31a31fe78d77 100644 --- a/arch/mips/include/asm/mips-boards/bonito64.h +++ b/arch/mips/include/asm/mips-boards/bonito64.h @@ -16,7 +16,7 @@ */ /* Revision 1.48 autogenerated on 08/17/99 15:20:01 */ -/* This bonito64 version editted from bonito.h Revision 1.48 on 11/09/00 */ +/* This bonito64 version edited from bonito.h Revision 1.48 on 11/09/00 */ #ifndef _ASM_MIPS_BOARDS_BONITO64_H #define _ASM_MIPS_BOARDS_BONITO64_H diff --git a/arch/mips/include/asm/mips-cpc.h b/arch/mips/include/asm/mips-cpc.h index b54453f1648c..5f3a7a9f42bf 100644 --- a/arch/mips/include/asm/mips-cpc.h +++ b/arch/mips/include/asm/mips-cpc.h @@ -22,7 +22,7 @@ extern void __iomem *mips_cpc_base; * the CPC * * Returns the default physical base address of the Cluster Power Controller - * memory mapped registers. This is platform dependant & must therefore be + * memory mapped registers. This is platform dependent & must therefore be * implemented per-platform. */ extern phys_addr_t mips_cpc_default_phys_base(void); diff --git a/arch/mips/include/asm/mipsregs.h b/arch/mips/include/asm/mipsregs.h index 2d53704d9f24..ec58cb76d076 100644 --- a/arch/mips/include/asm/mipsregs.h +++ b/arch/mips/include/asm/mipsregs.h @@ -98,7 +98,7 @@ /* * R4640/R4650 cp0 register names. These registers are listed - * here only for completeness; without MMU these CPUs are not useable + * here only for completeness; without MMU these CPUs are not usable * by Linux. A future ELKS port might take make Linux run on them * though ... */ @@ -461,7 +461,7 @@ #define EXCCODE_THREAD 25 /* Thread exceptions (MT) */ #define EXCCODE_DSPDIS 26 /* DSP disabled exception */ #define EXCCODE_GE 27 /* Virtualized guest exception (VZ) */ -#define EXCCODE_CACHEERR 30 /* Parity/ECC occured on a core */ +#define EXCCODE_CACHEERR 30 /* Parity/ECC occurred on a core */ /* Implementation specific trap codes used by MIPS cores */ #define MIPS_EXCCODE_TLBPAR 16 /* TLB parity error exception */ diff --git a/arch/mips/include/asm/octeon/cvmx-bootinfo.h b/arch/mips/include/asm/octeon/cvmx-bootinfo.h index c1c0b3230e0a..028bf1d6daee 100644 --- a/arch/mips/include/asm/octeon/cvmx-bootinfo.h +++ b/arch/mips/include/asm/octeon/cvmx-bootinfo.h @@ -114,7 +114,7 @@ struct cvmx_bootinfo { /* * flags indicating various configuration options. These - * flags supercede the 'flags' variable and should be used + * flags supersede the 'flags' variable and should be used * instead if available. */ uint32_t config_flags; diff --git a/arch/mips/include/asm/octeon/cvmx-cmd-queue.h b/arch/mips/include/asm/octeon/cvmx-cmd-queue.h index a07a36f7d814..67e1b2162b19 100644 --- a/arch/mips/include/asm/octeon/cvmx-cmd-queue.h +++ b/arch/mips/include/asm/octeon/cvmx-cmd-queue.h @@ -145,7 +145,7 @@ typedef struct { /** * This structure contains the global state of all command queues. * It is stored in a bootmem named block and shared by all - * applications running on Octeon. Tickets are stored in a differnet + * applications running on Octeon. Tickets are stored in a different * cache line that queue information to reduce the contention on the * ll/sc used to get a ticket. If this is not the case, the update * of queue state causes the ll/sc to fail quite often. @@ -172,7 +172,7 @@ cvmx_cmd_queue_result_t cvmx_cmd_queue_initialize(cvmx_cmd_queue_id_t queue_id, int pool_size); /** - * Shutdown a queue a free it's command buffers to the FPA. The + * Shutdown a queue and free its command buffers to the FPA. The * hardware connected to the queue must be stopped before this * function is called. * @@ -194,7 +194,7 @@ int cvmx_cmd_queue_length(cvmx_cmd_queue_id_t queue_id); /** * Return the command buffer to be written to. The purpose of this - * function is to allow CVMX routine access t othe low level buffer + * function is to allow CVMX routine access to the low level buffer * for initial hardware setup. User applications should not call this * function directly. * diff --git a/arch/mips/include/asm/octeon/cvmx-pko.h b/arch/mips/include/asm/octeon/cvmx-pko.h index 5fec8476e421..f18a7f24daf8 100644 --- a/arch/mips/include/asm/octeon/cvmx-pko.h +++ b/arch/mips/include/asm/octeon/cvmx-pko.h @@ -91,7 +91,7 @@ typedef enum { } cvmx_pko_status_t; /** - * This enumeration represents the differnet locking modes supported by PKO. + * This enumeration represents the different locking modes supported by PKO. */ typedef enum { /* diff --git a/arch/mips/include/asm/octeon/cvmx-pow.h b/arch/mips/include/asm/octeon/cvmx-pow.h index a3b23811e0c3..21b4378244fa 100644 --- a/arch/mips/include/asm/octeon/cvmx-pow.h +++ b/arch/mips/include/asm/octeon/cvmx-pow.h @@ -1342,7 +1342,7 @@ static inline void cvmx_pow_tag_sw_wait(void) * This function does NOT wait for previous tag switches to complete, * so the caller must ensure that there is not a pending tag switch. * - * @wait: When set, call stalls until work becomes avaiable, or times out. + * @wait: When set, call stalls until work becomes available, or times out. * If not set, returns immediately. * * Returns: the WQE pointer from POW. Returns NULL if no work @@ -1376,7 +1376,7 @@ static inline struct cvmx_wqe *cvmx_pow_work_request_sync_nocheck(cvmx_pow_wait_ * This function waits for any previous tag switch to complete before * requesting the new work. * - * @wait: When set, call stalls until work becomes avaiable, or times out. + * @wait: When set, call stalls until work becomes available, or times out. * If not set, returns immediately. * * Returns: the WQE pointer from POW. Returns NULL if no work diff --git a/arch/mips/include/asm/octeon/octeon-model.h b/arch/mips/include/asm/octeon/octeon-model.h index 6c68517c2770..e53b61a8e32f 100644 --- a/arch/mips/include/asm/octeon/octeon-model.h +++ b/arch/mips/include/asm/octeon/octeon-model.h @@ -54,7 +54,7 @@ #define OM_CHECK_SUBMODEL 0x02000000 /* Match all models previous than the one specified */ #define OM_MATCH_PREVIOUS_MODELS 0x04000000 -/* Ignores the minor revison on newer parts */ +/* Ignores the minor revision on newer parts */ #define OM_IGNORE_MINOR_REVISION 0x08000000 #define OM_FLAG_MASK 0xff000000 @@ -226,7 +226,7 @@ #define OCTEON_CN52XX_PASS2 OCTEON_CN52XX_PASS2_X /* - * CN3XXX models with old revision enconding + * CN3XXX models with old revision encoding */ #define OCTEON_CN38XX_PASS1 0x000d0000 #define OCTEON_CN38XX_PASS2 0x000d0001 diff --git a/arch/mips/include/asm/page.h b/arch/mips/include/asm/page.h index 5978a8dfb917..ef9585d96f6b 100644 --- a/arch/mips/include/asm/page.h +++ b/arch/mips/include/asm/page.h @@ -173,7 +173,7 @@ static inline unsigned long ___pa(unsigned long x) if (IS_ENABLED(CONFIG_64BIT)) { /* * For MIPS64 the virtual address may either be in one of - * the compatibility segements ckseg0 or ckseg1, or it may + * the compatibility segments ckseg0 or ckseg1, or it may * be in xkphys. */ return x < CKSEG0 ? XPHYSADDR(x) : CPHYSADDR(x); diff --git a/arch/mips/include/asm/pci.h b/arch/mips/include/asm/pci.h index 3fd6e22c108b..d993df6302dc 100644 --- a/arch/mips/include/asm/pci.h +++ b/arch/mips/include/asm/pci.h @@ -23,7 +23,7 @@ #ifdef CONFIG_PCI_DRIVERS_LEGACY /* - * Each pci channel is a top-level PCI bus seem by CPU. A machine with + * Each PCI channel is a top-level PCI bus seem by CPU. A machine with * multiple PCI channels may have multiple PCI host controllers or a * single controller supporting multiple channels. */ diff --git a/arch/mips/include/asm/pgtable-bits.h b/arch/mips/include/asm/pgtable-bits.h index 421e78c30253..088623ba7b8b 100644 --- a/arch/mips/include/asm/pgtable-bits.h +++ b/arch/mips/include/asm/pgtable-bits.h @@ -201,7 +201,7 @@ enum pgtable_bits { * The final layouts of the PTE bits are: * * 64-bit, R1 or earlier: CCC D V G [S H] M A W R P - * 32-bit, R1 or earler: CCC D V G M A W R P + * 32-bit, R1 or earlier: CCC D V G M A W R P * 64-bit, R2 or later: CCC D V G RI/R XI [S H] M A W P * 32-bit, R2 or later: CCC D V G RI/R XI M A W P */ diff --git a/arch/mips/include/asm/sgi/mc.h b/arch/mips/include/asm/sgi/mc.h index 3a070cec97e7..5e96f9d32624 100644 --- a/arch/mips/include/asm/sgi/mc.h +++ b/arch/mips/include/asm/sgi/mc.h @@ -96,7 +96,7 @@ struct sgimc_regs { volatile u32 lbursttp; /* Time period for long bursts */ /* MC chip can drive up to 4 bank 4 SIMMs each. All SIMMs in bank must - * be the same size. The size encoding for supported SIMMs is bellow */ + * be the same size. The size encoding for supported SIMMs is below */ u32 _unused11[9]; volatile u32 mconfig0; /* Memory config register zero */ u32 _unused12; diff --git a/arch/mips/include/asm/sn/klconfig.h b/arch/mips/include/asm/sn/klconfig.h index 117f85e4bef5..3d1670b3e052 100644 --- a/arch/mips/include/asm/sn/klconfig.h +++ b/arch/mips/include/asm/sn/klconfig.h @@ -851,7 +851,7 @@ typedef union kldev_s { /* for device structure allocation */ /* * TBD - Allocation issues. * - * Do we need to Mark off sepatate heaps for lboard_t, rboard_t, component, + * Do we need to Mark off separate heaps for lboard_t, rboard_t, component, * errinfo and allocate from them, or have a single heap and allocate all * structures from it. Debug is easier in the former method since we can * dump all similar structs in one command, but there will be lots of holes, diff --git a/arch/mips/include/asm/sync.h b/arch/mips/include/asm/sync.h index aabd097933fe..44c04a82d0b7 100644 --- a/arch/mips/include/asm/sync.h +++ b/arch/mips/include/asm/sync.h @@ -19,7 +19,7 @@ * * Ordering barriers can be more efficient than completion barriers, since: * - * a) Ordering barriers only require memory access instructions which preceed + * a) Ordering barriers only require memory access instructions which precede * them in program order (older instructions) to reach a point in the * load/store datapath beyond which reordering is not possible before * allowing memory access instructions which follow them (younger diff --git a/arch/mips/include/asm/thread_info.h b/arch/mips/include/asm/thread_info.h index ecae7470faa4..b9d76e8ac5a2 100644 --- a/arch/mips/include/asm/thread_info.h +++ b/arch/mips/include/asm/thread_info.h @@ -27,7 +27,7 @@ struct thread_info { unsigned long flags; /* low level flags */ unsigned long tp_value; /* thread pointer */ __u32 cpu; /* current CPU */ - int preempt_count; /* 0 => preemptable, <0 => BUG */ + int preempt_count; /* 0 => preemptible, <0 => BUG */ struct pt_regs *regs; long syscall; /* syscall number */ }; diff --git a/arch/mips/include/asm/timex.h b/arch/mips/include/asm/timex.h index 2e107886f97a..7ef06dcdc46e 100644 --- a/arch/mips/include/asm/timex.h +++ b/arch/mips/include/asm/timex.h @@ -46,7 +46,7 @@ typedef unsigned int cycles_t; * * There is a suggested workaround and also the erratum can't strike if * the compare interrupt isn't being used as the clock source device. - * However for now the implementaton of this function doesn't get these + * However for now the implementation of this function doesn't get these * fine details right. */ static inline int can_use_mips_counter(unsigned int prid) diff --git a/arch/mips/include/asm/vdso/vdso.h b/arch/mips/include/asm/vdso/vdso.h index a327ca21270e..6cd88191fefa 100644 --- a/arch/mips/include/asm/vdso/vdso.h +++ b/arch/mips/include/asm/vdso/vdso.h @@ -32,7 +32,7 @@ static inline unsigned long get_vdso_base(void) #else /* * Get the base load address of the VDSO. We have to avoid generating - * relocations and references to the GOT because ld.so does not peform + * relocations and references to the GOT because ld.so does not perform * relocations on the VDSO. We use the current offset from the VDSO base * and perform a PC-relative branch which gives the absolute address in * ra, and take the difference. The assembler chokes on diff --git a/arch/mips/include/uapi/asm/mman.h b/arch/mips/include/uapi/asm/mman.h index c6e1fc77c996..9c48d9a21aa0 100644 --- a/arch/mips/include/uapi/asm/mman.h +++ b/arch/mips/include/uapi/asm/mman.h @@ -88,7 +88,7 @@ #define MADV_HUGEPAGE 14 /* Worth backing with hugepages */ #define MADV_NOHUGEPAGE 15 /* Not worth backing with hugepages */ -#define MADV_DONTDUMP 16 /* Explicity exclude from the core dump, +#define MADV_DONTDUMP 16 /* Explicitly exclude from core dump, overrides the coredump filter bits */ #define MADV_DODUMP 17 /* Clear the MADV_NODUMP flag */ diff --git a/arch/mips/include/uapi/asm/msgbuf.h b/arch/mips/include/uapi/asm/msgbuf.h index 128af72f2dfe..d546642fc67e 100644 --- a/arch/mips/include/uapi/asm/msgbuf.h +++ b/arch/mips/include/uapi/asm/msgbuf.h @@ -62,7 +62,7 @@ struct msqid64_ds { unsigned long __unused5; }; #else -#warning no endianess set +#warning no endianness set #endif #endif /* _ASM_MSGBUF_H */ diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c index b406d8bfb15a..2f4fafdc5fcc 100644 --- a/arch/mips/kernel/cpu-probe.c +++ b/arch/mips/kernel/cpu-probe.c @@ -1139,7 +1139,7 @@ static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu) * This processor doesn't have an MMU, so it's not * "real easy" to run Linux on it. It is left purely * for documentation. Commented out because it shares - * it's c0_prid id number with the TX3900. + * its c0_prid id number with the TX3900. */ c->cputype = CPU_R4650; __cpu_name[cpu] = "R4650"; diff --git a/arch/mips/kernel/kprobes.c b/arch/mips/kernel/kprobes.c index 316b27d0d2fb..dc39f5b3fb83 100644 --- a/arch/mips/kernel/kprobes.c +++ b/arch/mips/kernel/kprobes.c @@ -55,7 +55,7 @@ NOKPROBE_SYMBOL(insn_has_delayslot); * one; putting breakpoint on top of atomic ll/sc pair is bad idea; * so we need to prevent it and refuse kprobes insertion for such * instructions; cannot do much about breakpoint in the middle of - * ll/sc pair; it is upto user to avoid those places + * ll/sc pair; it is up to user to avoid those places */ static int insn_has_ll_or_sc(union mips_instruction insn) { diff --git a/arch/mips/kernel/relocate.c b/arch/mips/kernel/relocate.c index 58fc8d089402..7eeeaf1ff95d 100644 --- a/arch/mips/kernel/relocate.c +++ b/arch/mips/kernel/relocate.c @@ -380,7 +380,7 @@ void *__init relocate_kernel(void) } #endif /* CONFIG_USE_OF */ - /* Copy the kernel to it's new location */ + /* Copy the kernel to its new location */ memcpy(loc_new, &_text, kernel_length); /* Perform relocations on the new kernel */ diff --git a/arch/mips/kernel/relocate_kernel.S b/arch/mips/kernel/relocate_kernel.S index 8f0a7263a9d6..de894a0211d7 100644 --- a/arch/mips/kernel/relocate_kernel.S +++ b/arch/mips/kernel/relocate_kernel.S @@ -70,7 +70,7 @@ copy_word: done: #ifdef CONFIG_SMP /* kexec_flag reset is signal to other CPUs what kernel - was moved to it's location. Note - we need relocated address + was moved to its location. Note - we need relocated address of kexec_flag. */ bal 1f diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c index 0461ab49e8f1..844307a86b15 100644 --- a/arch/mips/kernel/setup.c +++ b/arch/mips/kernel/setup.c @@ -146,7 +146,7 @@ static unsigned long __init init_initrd(void) /* * Board specific code or command line parser should have * already set up initrd_start and initrd_end. In these cases - * perfom sanity checks and use them if all looks good. + * perform sanity checks and use them if all looks good. */ if (!initrd_start || initrd_end <= initrd_start) goto disable; diff --git a/arch/mips/kernel/signal.c b/arch/mips/kernel/signal.c index 479999b7f2de..a8e20fdc7afd 100644 --- a/arch/mips/kernel/signal.c +++ b/arch/mips/kernel/signal.c @@ -569,7 +569,7 @@ void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, return (void __user __force *)(-1UL); /* - * FPU emulator may have it's own trampoline active just + * FPU emulator may have its own trampoline active just * above the user stack, 16-bytes before the next lowest * 16 byte boundary. Try to avoid trashing it. */ diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index c1b2b18b0505..0e3f2f2fa732 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -2381,7 +2381,7 @@ void __init trap_init(void) set_except_vector(i, handle_reserved); /* - * Copy the EJTAG debug exception vector handler code to it's final + * Copy the EJTAG debug exception vector handler code to its final * destination. */ if (cpu_has_ejtag && board_ejtag_handler_setup) diff --git a/arch/mips/kernel/vpe.c b/arch/mips/kernel/vpe.c index e9a0cfd02ae2..737d0d4fdcd3 100644 --- a/arch/mips/kernel/vpe.c +++ b/arch/mips/kernel/vpe.c @@ -6,9 +6,9 @@ * Copyright (C) 2004, 2005 MIPS Technologies, Inc. All rights reserved. * Copyright (C) 2013 Imagination Technologies Ltd. * - * VPE spport module for loading a MIPS SP program into VPE1. The SP + * VPE support module for loading a MIPS SP program into VPE1. The SP * environment is rather simple since there are no TLBs. It needs - * to be relocatable (or partiall linked). Initialize your stack in + * to be relocatable (or partially linked). Initialize your stack in * the startup-code. The loader looks for the symbol __start and sets * up the execution to resume from there. To load and run, simply do * a cat SP 'binary' to the /dev/vpe1 device. diff --git a/arch/mips/kvm/emulate.c b/arch/mips/kvm/emulate.c index e64372b8f66a..0feec52222fb 100644 --- a/arch/mips/kvm/emulate.c +++ b/arch/mips/kvm/emulate.c @@ -531,7 +531,7 @@ static void kvm_mips_resume_hrtimer(struct kvm_vcpu *vcpu, * to be used for a period of time, but the exact ktime corresponding to the * final Count that must be restored is not known. * - * It is gauranteed that a timer interrupt immediately after restore will be + * It is guaranteed that a timer interrupt immediately after restore will be * handled, but not if CP0_Compare is exactly at @count. That case should * already be handled when the hardware timer state is saved. * diff --git a/arch/mips/loongson2ef/common/platform.c b/arch/mips/loongson2ef/common/platform.c index 0084820cffaa..b10300a527af 100644 --- a/arch/mips/loongson2ef/common/platform.c +++ b/arch/mips/loongson2ef/common/platform.c @@ -17,7 +17,7 @@ static int __init loongson2_cpufreq_init(void) { struct cpuinfo_mips *c = ¤t_cpu_data; - /* Only 2F revision and it's successors support CPUFreq */ + /* Only 2F revision and its successors support CPUFreq */ if ((c->processor_id & PRID_REV_MASK) >= PRID_REV_LOONGSON2F) return platform_device_register(&loongson2_cpufreq_device); diff --git a/arch/mips/loongson64/smp.c b/arch/mips/loongson64/smp.c index e015a26a40f7..4f6c714430da 100644 --- a/arch/mips/loongson64/smp.c +++ b/arch/mips/loongson64/smp.c @@ -516,7 +516,7 @@ static void __init loongson3_prepare_cpus(unsigned int max_cpus) } /* - * Setup the PC, SP, and GP of a secondary processor and start it runing! + * Setup the PC, SP, and GP of a secondary processor and start it running! */ static int loongson3_boot_secondary(int cpu, struct task_struct *idle) { diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c index 187d1c16361c..20d37773b162 100644 --- a/arch/mips/mm/c-r4k.c +++ b/arch/mips/mm/c-r4k.c @@ -1654,7 +1654,7 @@ static void coherency_setup(void) /* * c0_status.cu=0 specifies that updates by the sc instruction use - * the coherency mode specified by the TLB; 1 means cachable + * the coherency mode specified by the TLB; 1 means cacheable * coherent update on write will be used. Not all processors have * this bit and; some wire it to zero, others like Toshiba had the * silly idea of putting something else there ... diff --git a/arch/mips/mm/cex-gen.S b/arch/mips/mm/cex-gen.S index 45dff5cd4b8e..e528583d1331 100644 --- a/arch/mips/mm/cex-gen.S +++ b/arch/mips/mm/cex-gen.S @@ -25,7 +25,7 @@ * This is a very bad place to be. Our cache error * detection has triggered. If we have write-back data * in the cache, we may not be able to recover. As a - * first-order desperate measure, turn off KSEG0 cacheing. + * first-order desperate measure, turn off KSEG0 caching. */ mfc0 k0,CP0_CONFIG li k1,~CONF_CM_CMASK diff --git a/arch/mips/mm/tlb-r3k.c b/arch/mips/mm/tlb-r3k.c index 53dfa2b9316b..22a8f488ae1d 100644 --- a/arch/mips/mm/tlb-r3k.c +++ b/arch/mips/mm/tlb-r3k.c @@ -183,7 +183,7 @@ void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte) int idx, pid; /* - * Handle debugger faulting in for debugee. + * Handle debugger faulting in for debuggee. */ if (current->active_mm != vma->vm_mm) return; diff --git a/arch/mips/mm/tlb-r4k.c b/arch/mips/mm/tlb-r4k.c index 93c2d695588a..128a0a5ec2b7 100644 --- a/arch/mips/mm/tlb-r4k.c +++ b/arch/mips/mm/tlb-r4k.c @@ -301,7 +301,7 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte) int idx, pid; /* - * Handle debugger faulting in for debugee. + * Handle debugger faulting in for debuggee. */ if (current->active_mm != vma->vm_mm) return; diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c index b4e1c783e617..4017fa0e2f68 100644 --- a/arch/mips/mm/tlbex.c +++ b/arch/mips/mm/tlbex.c @@ -789,7 +789,7 @@ void build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, if (check_for_high_segbits) { /* - * The kernel currently implicitely assumes that the + * The kernel currently implicitly assumes that the * MIPS SEGBITS parameter for the processor is * (PGDIR_SHIFT+PGDIR_BITS) or less, and will never * allocate virtual addresses outside the maximum @@ -1715,7 +1715,7 @@ iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, /* * Check if PTE is present, if not then jump to LABEL. PTR points to * the page table where this PTE is located, PTE will be re-loaded - * with it's original value. + * with its original value. */ static void build_pte_present(u32 **p, struct uasm_reloc **r, diff --git a/arch/mips/net/bpf_jit_comp32.c b/arch/mips/net/bpf_jit_comp32.c index ace5db3fbd17..40a878b672f5 100644 --- a/arch/mips/net/bpf_jit_comp32.c +++ b/arch/mips/net/bpf_jit_comp32.c @@ -95,7 +95,7 @@ /* * Mapping of 64-bit eBPF registers to 32-bit native MIPS registers. * - * 1) Native register pairs are ordered according to CPU endiannes, following + * 1) Native register pairs are ordered according to CPU endianness, following * the MIPS convention for passing 64-bit arguments and return values. * 2) The eBPF return value, arguments and callee-saved registers are mapped * to their native MIPS equivalents. diff --git a/arch/mips/pci/ops-loongson2.c b/arch/mips/pci/ops-loongson2.c index 0d1b36ba1c21..068113f5c49d 100644 --- a/arch/mips/pci/ops-loongson2.c +++ b/arch/mips/pci/ops-loongson2.c @@ -49,7 +49,7 @@ static int loongson_pcibios_config_access(unsigned char access_type, */ #ifdef CONFIG_CS5536 /* cs5536_pci_conf_read4/write4() will call _rdmsr/_wrmsr() to - * access the regsters PCI_MSR_ADDR, PCI_MSR_DATA_LO, + * access the registers PCI_MSR_ADDR, PCI_MSR_DATA_LO, * PCI_MSR_DATA_HI, which is bigger than PCI_MSR_CTRL, so, it * will not go this branch, but the others. so, no calling dead * loop here. diff --git a/arch/mips/pci/pci-alchemy.c b/arch/mips/pci/pci-alchemy.c index 1c722dd0c130..58625d1b6465 100644 --- a/arch/mips/pci/pci-alchemy.c +++ b/arch/mips/pci/pci-alchemy.c @@ -453,7 +453,7 @@ static int alchemy_pci_probe(struct platform_device *pdev) /* we can't ioremap the entire pci config space because it's too large, * nor can we dynamically ioremap it because some drivers use the - * PCI config routines from within atomic contex and that becomes a + * PCI config routines from within atomic context and that becomes a * problem in get_vm_area(). Instead we use one wired TLB entry to * handle all config accesses for all busses. */ diff --git a/arch/mips/pci/pci-ar2315.c b/arch/mips/pci/pci-ar2315.c index e17d862cfa4c..a925842ee125 100644 --- a/arch/mips/pci/pci-ar2315.c +++ b/arch/mips/pci/pci-ar2315.c @@ -16,7 +16,7 @@ * the CFG_SEL bit in the PCI_MISC_CONFIG register. * * Devices on the bus can perform DMA requests via chip BAR1. PCI host - * controller BARs are programmend as if an external device is programmed. + * controller BARs are programmed as if an external device is programmed. * Which means that during configuration, IDSEL pin of the chip should be * asserted. * diff --git a/arch/mips/pci/pci-lantiq.c b/arch/mips/pci/pci-lantiq.c index 80f7293166bb..68a8cefed420 100644 --- a/arch/mips/pci/pci-lantiq.c +++ b/arch/mips/pci/pci-lantiq.c @@ -152,7 +152,7 @@ static int ltq_pci_startup(struct platform_device *pdev) temp_buffer &= ~0xf0000; /* enable internal arbiter */ temp_buffer |= (1 << INTERNAL_ARB_ENABLE_BIT); - /* enable internal PCI master reqest */ + /* enable internal PCI master request */ temp_buffer &= (~(3 << PCI_MASTER0_REQ_MASK_2BITS)); /* enable EBU request */ diff --git a/arch/mips/pci/pci-octeon.c b/arch/mips/pci/pci-octeon.c index d19d9d456309..36d12cea3512 100644 --- a/arch/mips/pci/pci-octeon.c +++ b/arch/mips/pci/pci-octeon.c @@ -376,7 +376,7 @@ static void octeon_pci_initialize(void) ctl_status.s.timer = 1; cvmx_write_csr(CVMX_NPI_CTL_STATUS, ctl_status.u64); - /* Deassert PCI reset and advertize PCX Host Mode Device Capability + /* Deassert PCI reset and advertise PCX Host Mode Device Capability (64b) */ cvmx_write_csr(CVMX_CIU_SOFT_PRST, 0x4); cvmx_read_csr(CVMX_CIU_SOFT_PRST); diff --git a/arch/mips/pci/pci-xtalk-bridge.c b/arch/mips/pci/pci-xtalk-bridge.c index 68d5211afea8..45ddbaa6c123 100644 --- a/arch/mips/pci/pci-xtalk-bridge.c +++ b/arch/mips/pci/pci-xtalk-bridge.c @@ -114,7 +114,7 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_SGI, PCI_DEVICE_ID_SGI_IOC3, * * The function is complicated by the ultimate brokenness of the IOC3 chip * which is used in SGI systems. The IOC3 can only handle 32-bit PCI - * accesses and does only decode parts of it's address space. + * accesses and does only decode parts of its address space. */ static int pci_conf0_read_config(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *value) diff --git a/arch/mips/pci/pcie-octeon.c b/arch/mips/pci/pcie-octeon.c index c9edd3fb380d..2583e318e8c6 100644 --- a/arch/mips/pci/pcie-octeon.c +++ b/arch/mips/pci/pcie-octeon.c @@ -1037,7 +1037,7 @@ retry: in_fif_p_count = dbg_data.s.data & 0xff; } while (in_fif_p_count != ((old_in_fif_p_count+1) & 0xff)); - /* Update in_fif_p_count for it's offset with respect to out_p_count */ + /* Update in_fif_p_count for its offset with respect to out_p_count */ in_fif_p_count = (in_fif_p_count + in_p_offset) & 0xff; /* Read the OUT_P_COUNT from the debug select */ diff --git a/arch/mips/ralink/mt7621.c b/arch/mips/ralink/mt7621.c index 137781d0bd0a..5a9fd3fe41d7 100644 --- a/arch/mips/ralink/mt7621.c +++ b/arch/mips/ralink/mt7621.c @@ -175,7 +175,7 @@ void __init prom_soc_init(struct ralink_soc_info *soc_info) * mips_cm_probe() wipes out bootloader * config for CM regions and we have to configure them * again. This SoC cannot talk to pamlbus devices - * witout proper iocu region set up. + * without proper iocu region set up. * * FIXME: it would be better to do this with values * from DT, but we need this very early because diff --git a/arch/mips/txx9/generic/pci.c b/arch/mips/txx9/generic/pci.c index 5ae30b78d38d..d9249f5a632e 100644 --- a/arch/mips/txx9/generic/pci.c +++ b/arch/mips/txx9/generic/pci.c @@ -348,7 +348,7 @@ static void final_fixup(struct pci_dev *dev) unsigned char bist; int ret; - /* Do build-in self test */ + /* Do built-in self test */ ret = pci_read_config_byte(dev, PCI_BIST, &bist); if ((ret != PCIBIOS_SUCCESSFUL) || !(bist & PCI_BIST_CAPABLE)) return; From caadf876bb7449bf25ef817afe7fb881df8198a2 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 4 Jan 2024 11:15:07 -0500 Subject: [PATCH 271/336] KVM: introduce CONFIG_KVM_COMMON CONFIG_HAVE_KVM is currently used by some architectures to either enabled the KVM config proper, or to enable host-side code that is not part of the KVM module. However, CONFIG_KVM's "select" statement in virt/kvm/Kconfig corresponds to a third meaning, namely to enable common Kconfigs required by all architectures that support KVM. These three meanings can be replaced respectively by an architecture-specific Kconfig, by IS_ENABLED(CONFIG_KVM), or by a new Kconfig symbol that is in turn selected by the architecture-specific "config KVM". Start by introducing such a new Kconfig symbol, CONFIG_KVM_COMMON. Unlike CONFIG_HAVE_KVM, it is selected by CONFIG_KVM, not by architecture code, and it brings in all dependencies of common KVM code. In particular, INTERVAL_TREE was missing in loongarch and riscv, so that is another thing that is fixed. Fixes: 8132d887a702 ("KVM: remove CONFIG_HAVE_KVM_EVENTFD", 2023-12-08) Reported-by: Randy Dunlap Closes: https://lore.kernel.org/all/44907c6b-c5bd-4e4a-a921-e4d3825539d8@infradead.org/ Reviewed-by: Andrew Jones Signed-off-by: Paolo Bonzini --- arch/arm64/kvm/Kconfig | 3 +-- arch/loongarch/kvm/Kconfig | 2 +- arch/mips/kvm/Kconfig | 3 +-- arch/powerpc/kvm/Kconfig | 3 +-- arch/riscv/kvm/Kconfig | 2 +- arch/s390/kvm/Kconfig | 3 +-- arch/x86/kvm/Kconfig | 3 +-- virt/kvm/Kconfig | 5 +++++ 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig index b07c60c9737d..6c3c8ca73e7f 100644 --- a/arch/arm64/kvm/Kconfig +++ b/arch/arm64/kvm/Kconfig @@ -21,9 +21,9 @@ if VIRTUALIZATION menuconfig KVM bool "Kernel-based Virtual Machine (KVM) support" depends on HAVE_KVM + select KVM_COMMON select KVM_GENERIC_HARDWARE_ENABLING select KVM_GENERIC_MMU_NOTIFIER - select PREEMPT_NOTIFIERS select HAVE_KVM_CPU_RELAX_INTERCEPT select KVM_MMIO select KVM_GENERIC_DIRTYLOG_READ_PROTECT @@ -39,7 +39,6 @@ menuconfig KVM select HAVE_KVM_VCPU_RUN_PID_CHANGE select SCHED_INFO select GUEST_PERF_EVENTS if PERF_EVENTS - select INTERVAL_TREE select XARRAY_MULTI help Support hosting virtualized guest machines. diff --git a/arch/loongarch/kvm/Kconfig b/arch/loongarch/kvm/Kconfig index daba4cd5e87d..61f7e33b1f95 100644 --- a/arch/loongarch/kvm/Kconfig +++ b/arch/loongarch/kvm/Kconfig @@ -23,12 +23,12 @@ config KVM depends on HAVE_KVM select HAVE_KVM_DIRTY_RING_ACQ_REL select HAVE_KVM_VCPU_ASYNC_IOCTL + select KVM_COMMON select KVM_GENERIC_DIRTYLOG_READ_PROTECT select KVM_GENERIC_HARDWARE_ENABLING select KVM_GENERIC_MMU_NOTIFIER select KVM_MMIO select KVM_XFER_TO_GUEST_WORK - select PREEMPT_NOTIFIERS help Support hosting virtualized guest machines using hardware virtualization extensions. You will need diff --git a/arch/mips/kvm/Kconfig b/arch/mips/kvm/Kconfig index 428141b0b48f..18e7a17d5115 100644 --- a/arch/mips/kvm/Kconfig +++ b/arch/mips/kvm/Kconfig @@ -20,12 +20,11 @@ config KVM depends on HAVE_KVM depends on MIPS_FP_SUPPORT select EXPORT_UASM - select PREEMPT_NOTIFIERS + select KVM_COMMON select KVM_GENERIC_DIRTYLOG_READ_PROTECT select HAVE_KVM_VCPU_ASYNC_IOCTL select KVM_MMIO select KVM_GENERIC_MMU_NOTIFIER - select INTERVAL_TREE select KVM_GENERIC_HARDWARE_ENABLING help Support for hosting Guest kernels. diff --git a/arch/powerpc/kvm/Kconfig b/arch/powerpc/kvm/Kconfig index b47196085a42..074263429faf 100644 --- a/arch/powerpc/kvm/Kconfig +++ b/arch/powerpc/kvm/Kconfig @@ -19,12 +19,11 @@ if VIRTUALIZATION config KVM bool - select PREEMPT_NOTIFIERS + select KVM_COMMON select HAVE_KVM_VCPU_ASYNC_IOCTL select KVM_VFIO select IRQ_BYPASS_MANAGER select HAVE_KVM_IRQ_BYPASS - select INTERVAL_TREE config KVM_BOOK3S_HANDLER bool diff --git a/arch/riscv/kvm/Kconfig b/arch/riscv/kvm/Kconfig index 1fd76aee3b71..d490db943858 100644 --- a/arch/riscv/kvm/Kconfig +++ b/arch/riscv/kvm/Kconfig @@ -24,12 +24,12 @@ config KVM select HAVE_KVM_IRQ_ROUTING select HAVE_KVM_MSI select HAVE_KVM_VCPU_ASYNC_IOCTL + select KVM_COMMON select KVM_GENERIC_DIRTYLOG_READ_PROTECT select KVM_GENERIC_HARDWARE_ENABLING select KVM_MMIO select KVM_XFER_TO_GUEST_WORK select KVM_GENERIC_MMU_NOTIFIER - select PREEMPT_NOTIFIERS select SCHED_INFO help Support hosting virtualized guest machines. diff --git a/arch/s390/kvm/Kconfig b/arch/s390/kvm/Kconfig index bb6d90351119..72e9b7dcdf7d 100644 --- a/arch/s390/kvm/Kconfig +++ b/arch/s390/kvm/Kconfig @@ -20,17 +20,16 @@ config KVM def_tristate y prompt "Kernel-based Virtual Machine (KVM) support" depends on HAVE_KVM - select PREEMPT_NOTIFIERS select HAVE_KVM_CPU_RELAX_INTERCEPT select HAVE_KVM_VCPU_ASYNC_IOCTL select KVM_ASYNC_PF select KVM_ASYNC_PF_SYNC + select KVM_COMMON select HAVE_KVM_IRQCHIP select HAVE_KVM_IRQ_ROUTING select HAVE_KVM_INVALID_WAKEUPS select HAVE_KVM_NO_POLL select KVM_VFIO - select INTERVAL_TREE select MMU_NOTIFIER help Support hosting paravirtualized guest machines using the SIE diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index b07247b0b958..cce3dea27920 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -23,7 +23,7 @@ config KVM depends on HAVE_KVM depends on HIGH_RES_TIMERS depends on X86_LOCAL_APIC - select PREEMPT_NOTIFIERS + select KVM_COMMON select KVM_GENERIC_MMU_NOTIFIER select HAVE_KVM_IRQCHIP select HAVE_KVM_PFNCACHE @@ -44,7 +44,6 @@ config KVM select KVM_XFER_TO_GUEST_WORK select KVM_GENERIC_DIRTYLOG_READ_PROTECT select KVM_VFIO - select INTERVAL_TREE select HAVE_KVM_PM_NOTIFIER if PM select KVM_GENERIC_HARDWARE_ENABLING help diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig index 6793211a0b64..ace72be98fb2 100644 --- a/virt/kvm/Kconfig +++ b/virt/kvm/Kconfig @@ -3,7 +3,12 @@ config HAVE_KVM bool + +config KVM_COMMON + bool select EVENTFD + select INTERVAL_TREE + select PREEMPT_NOTIFIERS config HAVE_KVM_PFNCACHE bool From 3a373e027d8b0ed14963cc84b48a11e69e4506b6 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Sat, 6 Jan 2024 02:24:00 -0500 Subject: [PATCH 272/336] KVM: fix direction of dependency on MMU notifiers KVM_GENERIC_MEMORY_ATTRIBUTES requires the generic MMU notifier code, because it uses kvm_mmu_invalidate_begin/end. However, it would not work with a bespoke implementation of MMU notifiers that does not use KVM_GENERIC_MMU_NOTIFIER, because most likely it would not synchronize correctly on invalidation. So the right thing to do is to note the problematic configuration if the architecture does not select itself KVM_GENERIC_MMU_NOTIFIER; not to enable it blindly. Signed-off-by: Paolo Bonzini --- virt/kvm/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig index ace72be98fb2..184dab4ee871 100644 --- a/virt/kvm/Kconfig +++ b/virt/kvm/Kconfig @@ -97,7 +97,7 @@ config KVM_GENERIC_MMU_NOTIFIER bool config KVM_GENERIC_MEMORY_ATTRIBUTES - select KVM_GENERIC_MMU_NOTIFIER + depends on KVM_GENERIC_MMU_NOTIFIER bool config KVM_PRIVATE_MEM From 783288010035e4c250a0b6491a4642cdb8d30548 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 8 Jan 2024 07:51:26 -0500 Subject: [PATCH 273/336] KVM: x86: add missing "depends on KVM" Support for KVM software-protected VMs should not be configurable, if KVM is not available at all. Fixes: 89ea60c2c7b5 ("KVM: x86: Add support for "protected VMs" that can utilize private memory") Signed-off-by: Paolo Bonzini --- arch/x86/kvm/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index cce3dea27920..10c56603cc06 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -77,7 +77,7 @@ config KVM_WERROR config KVM_SW_PROTECTED_VM bool "Enable support for KVM software-protected VMs" depends on EXPERT - depends on X86_64 + depends on KVM && X86_64 select KVM_GENERIC_PRIVATE_MEM help Enable support for KVM software-protected VMs. Currently "protected" From 1c6d984f523f67ecfad1083bb04c55d91977bb15 Mon Sep 17 00:00:00 2001 From: "Kirill A. Shutemov" Date: Tue, 5 Dec 2023 03:45:01 +0300 Subject: [PATCH 274/336] x86/kvm: Do not try to disable kvmclock if it was not enabled kvm_guest_cpu_offline() tries to disable kvmclock regardless if it is present in the VM. It leads to write to a MSR that doesn't exist on some configurations, namely in TDX guest: unchecked MSR access error: WRMSR to 0x12 (tried to write 0x0000000000000000) at rIP: 0xffffffff8110687c (kvmclock_disable+0x1c/0x30) kvmclock enabling is gated by CLOCKSOURCE and CLOCKSOURCE2 KVM paravirt features. Do not disable kvmclock if it was not enabled. Signed-off-by: Kirill A. Shutemov Fixes: c02027b5742b ("x86/kvm: Disable kvmclock on all CPUs on shutdown") Reviewed-by: Sean Christopherson Reviewed-by: Vitaly Kuznetsov Cc: Paolo Bonzini Cc: Wanpeng Li Cc: stable@vger.kernel.org Message-Id: <20231205004510.27164-6-kirill.shutemov@linux.intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kernel/kvmclock.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c index fb8f52149be9..f2fff625576d 100644 --- a/arch/x86/kernel/kvmclock.c +++ b/arch/x86/kernel/kvmclock.c @@ -24,8 +24,8 @@ static int kvmclock __initdata = 1; static int kvmclock_vsyscall __initdata = 1; -static int msr_kvm_system_time __ro_after_init = MSR_KVM_SYSTEM_TIME; -static int msr_kvm_wall_clock __ro_after_init = MSR_KVM_WALL_CLOCK; +static int msr_kvm_system_time __ro_after_init; +static int msr_kvm_wall_clock __ro_after_init; static u64 kvm_sched_clock_offset __ro_after_init; static int __init parse_no_kvmclock(char *arg) @@ -195,7 +195,8 @@ static void kvm_setup_secondary_clock(void) void kvmclock_disable(void) { - native_write_msr(msr_kvm_system_time, 0, 0); + if (msr_kvm_system_time) + native_write_msr(msr_kvm_system_time, 0, 0); } static void __init kvmclock_init_mem(void) @@ -294,7 +295,10 @@ void __init kvmclock_init(void) if (kvm_para_has_feature(KVM_FEATURE_CLOCKSOURCE2)) { msr_kvm_system_time = MSR_KVM_SYSTEM_TIME_NEW; msr_kvm_wall_clock = MSR_KVM_WALL_CLOCK_NEW; - } else if (!kvm_para_has_feature(KVM_FEATURE_CLOCKSOURCE)) { + } else if (kvm_para_has_feature(KVM_FEATURE_CLOCKSOURCE)) { + msr_kvm_system_time = MSR_KVM_SYSTEM_TIME; + msr_kvm_wall_clock = MSR_KVM_WALL_CLOCK; + } else { return; } From 125c0a646a257fd58de223f2c3e1fe8a99085644 Mon Sep 17 00:00:00 2001 From: Juergen Gross Date: Tue, 5 Dec 2023 12:51:21 +0100 Subject: [PATCH 275/336] xen: update PV-device interface headers Update the Xen PV-device interface headers in order to avoid undefined behavior with flexible arrays being defined with one array element. Reported-by: Pry Mar Signed-off-by: Juergen Gross Acked-by: Stefano Stabellini Link: https://lore.kernel.org/r/20231205115121.11627-1-jgross@suse.com Signed-off-by: Juergen Gross --- include/xen/interface/io/displif.h | 2 +- include/xen/interface/io/ring.h | 2 +- include/xen/interface/io/sndif.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/xen/interface/io/displif.h b/include/xen/interface/io/displif.h index 18417b017869..60e42d3b760e 100644 --- a/include/xen/interface/io/displif.h +++ b/include/xen/interface/io/displif.h @@ -537,7 +537,7 @@ struct xendispl_dbuf_create_req { struct xendispl_page_directory { grant_ref_t gref_dir_next_page; - grant_ref_t gref[1]; /* Variable length */ + grant_ref_t gref[]; }; /* diff --git a/include/xen/interface/io/ring.h b/include/xen/interface/io/ring.h index ba4c4274b714..4fef1efcdcab 100644 --- a/include/xen/interface/io/ring.h +++ b/include/xen/interface/io/ring.h @@ -95,7 +95,7 @@ struct __name##_sring { \ RING_IDX req_prod, req_event; \ RING_IDX rsp_prod, rsp_event; \ uint8_t __pad[48]; \ - union __name##_sring_entry ring[1]; /* variable-length */ \ + union __name##_sring_entry ring[]; \ }; \ \ /* "Front" end's private variables */ \ diff --git a/include/xen/interface/io/sndif.h b/include/xen/interface/io/sndif.h index 445657cdb1de..b818517588b5 100644 --- a/include/xen/interface/io/sndif.h +++ b/include/xen/interface/io/sndif.h @@ -659,7 +659,7 @@ struct xensnd_open_req { struct xensnd_page_directory { grant_ref_t gref_dir_next_page; - grant_ref_t gref[1]; /* Variable length */ + grant_ref_t gref[]; }; /* From f1479f0a4f53ef4fc12108a59caee61c39bc6843 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Wed, 6 Dec 2023 10:17:24 -0800 Subject: [PATCH 276/336] xen/xenbus: client: fix kernel-doc comments Correct function kernel-doc notation to prevent warnings from scripts/kernel-doc. xenbus_client.c:134: warning: No description found for return value of 'xenbus_watch_path' xenbus_client.c:177: warning: No description found for return value of 'xenbus_watch_pathfmt' xenbus_client.c:258: warning: missing initial short description on line: * xenbus_switch_state xenbus_client.c:267: warning: No description found for return value of 'xenbus_switch_state' xenbus_client.c:308: warning: missing initial short description on line: * xenbus_dev_error xenbus_client.c:327: warning: missing initial short description on line: * xenbus_dev_fatal xenbus_client.c:350: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst * Equivalent to xenbus_dev_fatal(dev, err, fmt, args), but helps xenbus_client.c:457: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst * Allocate an event channel for the given xenbus_device, assigning the newly xenbus_client.c:486: warning: expecting prototype for Free an existing event channel. Returns 0 on success or(). Prototype was for xenbus_free_evtchn() instead xenbus_client.c:502: warning: missing initial short description on line: * xenbus_map_ring_valloc xenbus_client.c:517: warning: No description found for return value of 'xenbus_map_ring_valloc' xenbus_client.c:602: warning: missing initial short description on line: * xenbus_unmap_ring xenbus_client.c:614: warning: No description found for return value of 'xenbus_unmap_ring' xenbus_client.c:715: warning: missing initial short description on line: * xenbus_unmap_ring_vfree xenbus_client.c:727: warning: No description found for return value of 'xenbus_unmap_ring_vfree' xenbus_client.c:919: warning: missing initial short description on line: * xenbus_read_driver_state xenbus_client.c:926: warning: No description found for return value of 'xenbus_read_driver_state' Signed-off-by: Randy Dunlap Cc: Juergen Gross Cc: Stefano Stabellini Cc: Oleksandr Tyshchenko Cc: xen-devel@lists.xenproject.org Reviewed-by: Juergen Gross Link: https://lore.kernel.org/r/20231206181724.27767-1-rdunlap@infradead.org Signed-off-by: Juergen Gross --- drivers/xen/xenbus/xenbus_client.c | 59 +++++++++++++++++------------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/drivers/xen/xenbus/xenbus_client.c b/drivers/xen/xenbus/xenbus_client.c index d4b251925796..32835b4b9bc5 100644 --- a/drivers/xen/xenbus/xenbus_client.c +++ b/drivers/xen/xenbus/xenbus_client.c @@ -119,11 +119,13 @@ EXPORT_SYMBOL_GPL(xenbus_strstate); * @callback: callback to register * * Register a @watch on the given path, using the given xenbus_watch structure - * for storage, and the given @callback function as the callback. Return 0 on - * success, or -errno on error. On success, the given @path will be saved as - * @watch->node, and remains the caller's to free. On error, @watch->node will + * for storage, and the given @callback function as the callback. On success, + * the given @path will be saved as @watch->node, and remains the + * caller's to free. On error, @watch->node will * be NULL, the device will switch to %XenbusStateClosing, and the error will * be saved in the store. + * + * Returns: %0 on success or -errno on error */ int xenbus_watch_path(struct xenbus_device *dev, const char *path, struct xenbus_watch *watch, @@ -160,12 +162,14 @@ EXPORT_SYMBOL_GPL(xenbus_watch_path); * @pathfmt: format of path to watch * * Register a watch on the given @path, using the given xenbus_watch - * structure for storage, and the given @callback function as the callback. - * Return 0 on success, or -errno on error. On success, the watched path - * (@path/@path2) will be saved as @watch->node, and becomes the caller's to - * kfree(). On error, watch->node will be NULL, so the caller has nothing to + * structure for storage, and the given @callback function as the + * callback. On success, the watched path (@path/@path2) will be saved + * as @watch->node, and becomes the caller's to kfree(). + * On error, watch->node will be NULL, so the caller has nothing to * free, the device will switch to %XenbusStateClosing, and the error will be * saved in the store. + * + * Returns: %0 on success or -errno on error */ int xenbus_watch_pathfmt(struct xenbus_device *dev, struct xenbus_watch *watch, @@ -255,13 +259,15 @@ abort: } /** - * xenbus_switch_state + * xenbus_switch_state - save the new state of a driver * @dev: xenbus device * @state: new state * * Advertise in the store a change of the given driver to the given new_state. - * Return 0 on success, or -errno on error. On error, the device will switch - * to XenbusStateClosing, and the error will be saved in the store. + * On error, the device will switch to XenbusStateClosing, and the error + * will be saved in the store. + * + * Returns: %0 on success or -errno on error */ int xenbus_switch_state(struct xenbus_device *dev, enum xenbus_state state) { @@ -305,7 +311,7 @@ static void xenbus_va_dev_error(struct xenbus_device *dev, int err, } /** - * xenbus_dev_error + * xenbus_dev_error - place an error message into the store * @dev: xenbus device * @err: error to report * @fmt: error message format @@ -324,7 +330,7 @@ void xenbus_dev_error(struct xenbus_device *dev, int err, const char *fmt, ...) EXPORT_SYMBOL_GPL(xenbus_dev_error); /** - * xenbus_dev_fatal + * xenbus_dev_fatal - put an error messages into the store and then shutdown * @dev: xenbus device * @err: error to report * @fmt: error message format @@ -346,7 +352,7 @@ void xenbus_dev_fatal(struct xenbus_device *dev, int err, const char *fmt, ...) } EXPORT_SYMBOL_GPL(xenbus_dev_fatal); -/** +/* * Equivalent to xenbus_dev_fatal(dev, err, fmt, args), but helps * avoiding recursion within xenbus_switch_state. */ @@ -453,7 +459,7 @@ void xenbus_teardown_ring(void **vaddr, unsigned int nr_pages, } EXPORT_SYMBOL_GPL(xenbus_teardown_ring); -/** +/* * Allocate an event channel for the given xenbus_device, assigning the newly * created local port to *port. Return 0 on success, or -errno on error. On * error, the device will switch to XenbusStateClosing, and the error will be @@ -479,7 +485,7 @@ int xenbus_alloc_evtchn(struct xenbus_device *dev, evtchn_port_t *port) EXPORT_SYMBOL_GPL(xenbus_alloc_evtchn); -/** +/* * Free an existing event channel. Returns 0 on success or -errno on error. */ int xenbus_free_evtchn(struct xenbus_device *dev, evtchn_port_t port) @@ -499,7 +505,7 @@ EXPORT_SYMBOL_GPL(xenbus_free_evtchn); /** - * xenbus_map_ring_valloc + * xenbus_map_ring_valloc - allocate & map pages of VA space * @dev: xenbus device * @gnt_refs: grant reference array * @nr_grefs: number of grant references @@ -507,10 +513,11 @@ EXPORT_SYMBOL_GPL(xenbus_free_evtchn); * * Map @nr_grefs pages of memory into this domain from another * domain's grant table. xenbus_map_ring_valloc allocates @nr_grefs - * pages of virtual address space, maps the pages to that address, and - * sets *vaddr to that address. Returns 0 on success, and -errno on - * error. If an error is returned, device will switch to + * pages of virtual address space, maps the pages to that address, and sets + * *vaddr to that address. If an error is returned, device will switch to * XenbusStateClosing and the error message will be saved in XenStore. + * + * Returns: %0 on success or -errno on error */ int xenbus_map_ring_valloc(struct xenbus_device *dev, grant_ref_t *gnt_refs, unsigned int nr_grefs, void **vaddr) @@ -599,14 +606,15 @@ static int __xenbus_map_ring(struct xenbus_device *dev, } /** - * xenbus_unmap_ring + * xenbus_unmap_ring - unmap memory from another domain * @dev: xenbus device * @handles: grant handle array * @nr_handles: number of handles in the array * @vaddrs: addresses to unmap * * Unmap memory in this domain that was imported from another domain. - * Returns 0 on success and returns GNTST_* on error + * + * Returns: %0 on success or GNTST_* on error * (see xen/include/interface/grant_table.h). */ static int xenbus_unmap_ring(struct xenbus_device *dev, grant_handle_t *handles, @@ -712,7 +720,7 @@ static int xenbus_map_ring_hvm(struct xenbus_device *dev, } /** - * xenbus_unmap_ring_vfree + * xenbus_unmap_ring_vfree - unmap a page of memory from another domain * @dev: xenbus device * @vaddr: addr to unmap * @@ -720,7 +728,8 @@ static int xenbus_map_ring_hvm(struct xenbus_device *dev, * Unmap a page of memory in this domain that was imported from another domain. * Use xenbus_unmap_ring_vfree if you mapped in your memory with * xenbus_map_ring_valloc (it will free the virtual address space). - * Returns 0 on success and returns GNTST_* on error + * + * Returns: %0 on success or GNTST_* on error * (see xen/include/interface/grant_table.h). */ int xenbus_unmap_ring_vfree(struct xenbus_device *dev, void *vaddr) @@ -916,10 +925,10 @@ static int xenbus_unmap_ring_hvm(struct xenbus_device *dev, void *vaddr) } /** - * xenbus_read_driver_state + * xenbus_read_driver_state - read state from a store path * @path: path for driver * - * Return the state of the driver rooted at the given store path, or + * Returns: the state of the driver rooted at the given store path, or * XenbusStateUnknown if no state can be read. */ enum xenbus_state xenbus_read_driver_state(const char *path) From 7e72fc41d4243800206ff76615cfebb15d632027 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Wed, 3 Jan 2024 12:49:57 +0100 Subject: [PATCH 277/336] thermal: netlink: Pass pointers to thermal_notify_tz_trip_change() Instead of requiring the caller of thermal_notify_tz_trip_change() to provide specific values needed to populate struct param in it, make it extract those values from objects passed to it by the caller via const pointers. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Lukasz Luba Reviewed-by: Daniel Lezcano --- drivers/thermal/thermal_netlink.c | 12 +++++++----- drivers/thermal/thermal_netlink.h | 11 +++++++---- drivers/thermal/thermal_trip.c | 8 ++------ 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/drivers/thermal/thermal_netlink.c b/drivers/thermal/thermal_netlink.c index 332052e24a86..4a2c299c0462 100644 --- a/drivers/thermal/thermal_netlink.c +++ b/drivers/thermal/thermal_netlink.c @@ -377,12 +377,14 @@ int thermal_notify_tz_trip_delete(int tz_id, int trip_id) return thermal_genl_send_event(THERMAL_GENL_EVENT_TZ_TRIP_DELETE, &p); } -int thermal_notify_tz_trip_change(int tz_id, int trip_id, int trip_type, - int trip_temp, int trip_hyst) +int thermal_notify_tz_trip_change(const struct thermal_zone_device *tz, + const struct thermal_trip *trip) { - struct param p = { .tz_id = tz_id, .trip_id = trip_id, - .trip_type = trip_type, .trip_temp = trip_temp, - .trip_hyst = trip_hyst }; + struct param p = { .tz_id = tz->id, + .trip_id = thermal_zone_trip_id(tz, trip), + .trip_type = trip->type, + .trip_temp = trip->temperature, + .trip_hyst = trip->hysteresis }; return thermal_genl_send_event(THERMAL_GENL_EVENT_TZ_TRIP_CHANGE, &p); } diff --git a/drivers/thermal/thermal_netlink.h b/drivers/thermal/thermal_netlink.h index 0a9987c3bc57..bb4c47d685e3 100644 --- a/drivers/thermal/thermal_netlink.h +++ b/drivers/thermal/thermal_netlink.h @@ -10,6 +10,9 @@ struct thermal_genl_cpu_caps { int efficiency; }; +struct thermal_zone_device; +struct thermal_trip; + /* Netlink notification function */ #ifdef CONFIG_THERMAL_NETLINK int __init thermal_netlink_init(void); @@ -23,8 +26,8 @@ int thermal_notify_tz_trip_up(int tz_id, int id, int temp); int thermal_notify_tz_trip_delete(int tz_id, int id); int thermal_notify_tz_trip_add(int tz_id, int id, int type, int temp, int hyst); -int thermal_notify_tz_trip_change(int tz_id, int id, int type, - int temp, int hyst); +int thermal_notify_tz_trip_change(const struct thermal_zone_device *tz, + const struct thermal_trip *trip); int thermal_notify_cdev_state_update(int cdev_id, int state); int thermal_notify_cdev_add(int cdev_id, const char *name, int max_state); int thermal_notify_cdev_delete(int cdev_id); @@ -79,8 +82,8 @@ static inline int thermal_notify_tz_trip_add(int tz_id, int id, int type, return 0; } -static inline int thermal_notify_tz_trip_change(int tz_id, int id, int type, - int temp, int hyst) +static inline int thermal_notify_tz_trip_change(const struct thermal_zone_device *tz, + const struct thermal_trip *trip) { return 0; } diff --git a/drivers/thermal/thermal_trip.c b/drivers/thermal/thermal_trip.c index 8bffa1e5e206..c875a26d5adf 100644 --- a/drivers/thermal/thermal_trip.c +++ b/drivers/thermal/thermal_trip.c @@ -155,9 +155,7 @@ int thermal_zone_trip_id(const struct thermal_zone_device *tz, void thermal_zone_trip_updated(struct thermal_zone_device *tz, const struct thermal_trip *trip) { - thermal_notify_tz_trip_change(tz->id, thermal_zone_trip_id(tz, trip), - trip->type, trip->temperature, - trip->hysteresis); + thermal_notify_tz_trip_change(tz, trip); __thermal_zone_device_update(tz, THERMAL_TRIP_CHANGED); } @@ -168,8 +166,6 @@ void thermal_zone_set_trip_temp(struct thermal_zone_device *tz, return; trip->temperature = temp; - thermal_notify_tz_trip_change(tz->id, thermal_zone_trip_id(tz, trip), - trip->type, trip->temperature, - trip->hysteresis); + thermal_notify_tz_trip_change(tz, trip); } EXPORT_SYMBOL_GPL(thermal_zone_set_trip_temp); From f52557edf0648b471e2006f9377ea0ba4f73f9b2 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 15 Dec 2023 20:57:50 +0100 Subject: [PATCH 278/336] thermal: netlink: Pass pointers to thermal_notify_tz_trip_up/down() Instead of requiring the callers of thermal_notify_tz_trip_up/down() to provide specific values needed to populate struct param in them, make them extract those values from objects passed by the callers via const pointers. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Lukasz Luba Reviewed-by: Daniel Lezcano --- drivers/thermal/thermal_core.c | 8 ++------ drivers/thermal/thermal_netlink.c | 14 ++++++++++---- drivers/thermal/thermal_netlink.h | 12 ++++++++---- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c index fa88d8707241..a7e2008d34c9 100644 --- a/drivers/thermal/thermal_core.c +++ b/drivers/thermal/thermal_core.c @@ -381,9 +381,7 @@ static void handle_thermal_trip(struct thermal_zone_device *tz, * the threshold and the trip temperature will be equal. */ if (tz->temperature >= trip->temperature) { - thermal_notify_tz_trip_up(tz->id, - thermal_zone_trip_id(tz, trip), - tz->temperature); + thermal_notify_tz_trip_up(tz, trip); trip->threshold = trip->temperature - trip->hysteresis; } else { trip->threshold = trip->temperature; @@ -400,9 +398,7 @@ static void handle_thermal_trip(struct thermal_zone_device *tz, * the trip. */ if (tz->temperature < trip->temperature - trip->hysteresis) { - thermal_notify_tz_trip_down(tz->id, - thermal_zone_trip_id(tz, trip), - tz->temperature); + thermal_notify_tz_trip_down(tz, trip); trip->threshold = trip->temperature; } else { trip->threshold = trip->temperature - trip->hysteresis; diff --git a/drivers/thermal/thermal_netlink.c b/drivers/thermal/thermal_netlink.c index 4a2c299c0462..46e4ea45d67d 100644 --- a/drivers/thermal/thermal_netlink.c +++ b/drivers/thermal/thermal_netlink.c @@ -346,16 +346,22 @@ int thermal_notify_tz_disable(int tz_id) return thermal_genl_send_event(THERMAL_GENL_EVENT_TZ_DISABLE, &p); } -int thermal_notify_tz_trip_down(int tz_id, int trip_id, int temp) +int thermal_notify_tz_trip_down(const struct thermal_zone_device *tz, + const struct thermal_trip *trip) { - struct param p = { .tz_id = tz_id, .trip_id = trip_id, .temp = temp }; + struct param p = { .tz_id = tz->id, + .trip_id = thermal_zone_trip_id(tz, trip), + .temp = tz->temperature }; return thermal_genl_send_event(THERMAL_GENL_EVENT_TZ_TRIP_DOWN, &p); } -int thermal_notify_tz_trip_up(int tz_id, int trip_id, int temp) +int thermal_notify_tz_trip_up(const struct thermal_zone_device *tz, + const struct thermal_trip *trip) { - struct param p = { .tz_id = tz_id, .trip_id = trip_id, .temp = temp }; + struct param p = { .tz_id = tz->id, + .trip_id = thermal_zone_trip_id(tz, trip), + .temp = tz->temperature }; return thermal_genl_send_event(THERMAL_GENL_EVENT_TZ_TRIP_UP, &p); } diff --git a/drivers/thermal/thermal_netlink.h b/drivers/thermal/thermal_netlink.h index bb4c47d685e3..907dc88d3317 100644 --- a/drivers/thermal/thermal_netlink.h +++ b/drivers/thermal/thermal_netlink.h @@ -21,8 +21,10 @@ int thermal_notify_tz_create(int tz_id, const char *name); int thermal_notify_tz_delete(int tz_id); int thermal_notify_tz_enable(int tz_id); int thermal_notify_tz_disable(int tz_id); -int thermal_notify_tz_trip_down(int tz_id, int id, int temp); -int thermal_notify_tz_trip_up(int tz_id, int id, int temp); +int thermal_notify_tz_trip_down(const struct thermal_zone_device *tz, + const struct thermal_trip *trip); +int thermal_notify_tz_trip_up(const struct thermal_zone_device *tz, + const struct thermal_trip *trip); int thermal_notify_tz_trip_delete(int tz_id, int id); int thermal_notify_tz_trip_add(int tz_id, int id, int type, int temp, int hyst); @@ -61,12 +63,14 @@ static inline int thermal_notify_tz_disable(int tz_id) return 0; } -static inline int thermal_notify_tz_trip_down(int tz_id, int id, int temp) +static inline int thermal_notify_tz_trip_down(const struct thermal_zone_device *tz, + const struct thermal_trip *trip) { return 0; } -static inline int thermal_notify_tz_trip_up(int tz_id, int id, int temp) +static inline int thermal_notify_tz_trip_up(const struct thermal_zone_device *tz, + const struct thermal_trip *trip) { return 0; } From 4ae535f37d0e3c5731f90c385e883c0bada59fc9 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 15 Dec 2023 20:59:08 +0100 Subject: [PATCH 279/336] thermal: netlink: Drop thermal_notify_tz_trip_add/delete() Because thermal_notify_tz_trip_add/delete() are never used, drop them entirely along with the related code. The addition or removal of trip points is not supported by the thermal core and is unlikely to be supported in the future, so it is also unlikely that these functions will ever be needed. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Lukasz Luba Acked-by: Daniel Lezcano --- drivers/thermal/thermal_netlink.c | 33 +------------------------------ drivers/thermal/thermal_netlink.h | 14 ------------- 2 files changed, 1 insertion(+), 46 deletions(-) diff --git a/drivers/thermal/thermal_netlink.c b/drivers/thermal/thermal_netlink.c index 46e4ea45d67d..91992181e845 100644 --- a/drivers/thermal/thermal_netlink.c +++ b/drivers/thermal/thermal_netlink.c @@ -148,7 +148,7 @@ static int thermal_genl_event_tz_trip_up(struct param *p) return 0; } -static int thermal_genl_event_tz_trip_add(struct param *p) +static int thermal_genl_event_tz_trip_change(struct param *p) { if (nla_put_u32(p->msg, THERMAL_GENL_ATTR_TZ_ID, p->tz_id) || nla_put_u32(p->msg, THERMAL_GENL_ATTR_TZ_TRIP_ID, p->trip_id) || @@ -160,15 +160,6 @@ static int thermal_genl_event_tz_trip_add(struct param *p) return 0; } -static int thermal_genl_event_tz_trip_delete(struct param *p) -{ - if (nla_put_u32(p->msg, THERMAL_GENL_ATTR_TZ_ID, p->tz_id) || - nla_put_u32(p->msg, THERMAL_GENL_ATTR_TZ_TRIP_ID, p->trip_id)) - return -EMSGSIZE; - - return 0; -} - static int thermal_genl_event_cdev_add(struct param *p) { if (nla_put_string(p->msg, THERMAL_GENL_ATTR_CDEV_NAME, @@ -258,9 +249,6 @@ int thermal_genl_event_tz_disable(struct param *p) int thermal_genl_event_tz_trip_down(struct param *p) __attribute__((alias("thermal_genl_event_tz_trip_up"))); -int thermal_genl_event_tz_trip_change(struct param *p) - __attribute__((alias("thermal_genl_event_tz_trip_add"))); - static cb_t event_cb[] = { [THERMAL_GENL_EVENT_TZ_CREATE] = thermal_genl_event_tz_create, [THERMAL_GENL_EVENT_TZ_DELETE] = thermal_genl_event_tz_delete, @@ -269,8 +257,6 @@ static cb_t event_cb[] = { [THERMAL_GENL_EVENT_TZ_TRIP_UP] = thermal_genl_event_tz_trip_up, [THERMAL_GENL_EVENT_TZ_TRIP_DOWN] = thermal_genl_event_tz_trip_down, [THERMAL_GENL_EVENT_TZ_TRIP_CHANGE] = thermal_genl_event_tz_trip_change, - [THERMAL_GENL_EVENT_TZ_TRIP_ADD] = thermal_genl_event_tz_trip_add, - [THERMAL_GENL_EVENT_TZ_TRIP_DELETE] = thermal_genl_event_tz_trip_delete, [THERMAL_GENL_EVENT_CDEV_ADD] = thermal_genl_event_cdev_add, [THERMAL_GENL_EVENT_CDEV_DELETE] = thermal_genl_event_cdev_delete, [THERMAL_GENL_EVENT_CDEV_STATE_UPDATE] = thermal_genl_event_cdev_state_update, @@ -366,23 +352,6 @@ int thermal_notify_tz_trip_up(const struct thermal_zone_device *tz, return thermal_genl_send_event(THERMAL_GENL_EVENT_TZ_TRIP_UP, &p); } -int thermal_notify_tz_trip_add(int tz_id, int trip_id, int trip_type, - int trip_temp, int trip_hyst) -{ - struct param p = { .tz_id = tz_id, .trip_id = trip_id, - .trip_type = trip_type, .trip_temp = trip_temp, - .trip_hyst = trip_hyst }; - - return thermal_genl_send_event(THERMAL_GENL_EVENT_TZ_TRIP_ADD, &p); -} - -int thermal_notify_tz_trip_delete(int tz_id, int trip_id) -{ - struct param p = { .tz_id = tz_id, .trip_id = trip_id }; - - return thermal_genl_send_event(THERMAL_GENL_EVENT_TZ_TRIP_DELETE, &p); -} - int thermal_notify_tz_trip_change(const struct thermal_zone_device *tz, const struct thermal_trip *trip) { diff --git a/drivers/thermal/thermal_netlink.h b/drivers/thermal/thermal_netlink.h index 907dc88d3317..e780ce3f7db6 100644 --- a/drivers/thermal/thermal_netlink.h +++ b/drivers/thermal/thermal_netlink.h @@ -25,9 +25,6 @@ int thermal_notify_tz_trip_down(const struct thermal_zone_device *tz, const struct thermal_trip *trip); int thermal_notify_tz_trip_up(const struct thermal_zone_device *tz, const struct thermal_trip *trip); -int thermal_notify_tz_trip_delete(int tz_id, int id); -int thermal_notify_tz_trip_add(int tz_id, int id, int type, - int temp, int hyst); int thermal_notify_tz_trip_change(const struct thermal_zone_device *tz, const struct thermal_trip *trip); int thermal_notify_cdev_state_update(int cdev_id, int state); @@ -75,17 +72,6 @@ static inline int thermal_notify_tz_trip_up(const struct thermal_zone_device *tz return 0; } -static inline int thermal_notify_tz_trip_delete(int tz_id, int id) -{ - return 0; -} - -static inline int thermal_notify_tz_trip_add(int tz_id, int id, int type, - int temp, int hyst) -{ - return 0; -} - static inline int thermal_notify_tz_trip_change(const struct thermal_zone_device *tz, const struct thermal_trip *trip) { From 2f521890aa5b8a5619d31a95caec0b08d4cb9e1a Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Wed, 3 Jan 2024 12:59:10 +0100 Subject: [PATCH 280/336] thermal: netlink: Pass thermal zone pointer to notify routines There are several rountines in the thermal netlink API that take a thermal zone ID or a thermal zone type as their arguments, but from their callers perspective it would be more convenient to pass a thermal zone pointer to them and let them extract the necessary data from the given thermal zone object by themselves. Modify the code accordingly. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Lukasz Luba Acked-by: Daniel Lezcano --- drivers/thermal/thermal_core.c | 13 +++++-------- drivers/thermal/thermal_netlink.c | 21 +++++++++++---------- drivers/thermal/thermal_netlink.h | 22 ++++++++++++---------- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c index a7e2008d34c9..c0b012e075f3 100644 --- a/drivers/thermal/thermal_core.c +++ b/drivers/thermal/thermal_core.c @@ -211,7 +211,7 @@ exit: mutex_unlock(&tz->lock); mutex_unlock(&thermal_governor_lock); - thermal_notify_tz_gov_change(tz->id, policy); + thermal_notify_tz_gov_change(tz, policy); return ret; } @@ -501,9 +501,9 @@ static int thermal_zone_device_set_mode(struct thermal_zone_device *tz, mutex_unlock(&tz->lock); if (mode == THERMAL_DEVICE_ENABLED) - thermal_notify_tz_enable(tz->id); + thermal_notify_tz_enable(tz); else - thermal_notify_tz_disable(tz->id); + thermal_notify_tz_disable(tz); return ret; } @@ -1407,7 +1407,7 @@ thermal_zone_device_register_with_trips(const char *type, struct thermal_trip *t if (atomic_cmpxchg(&tz->need_update, 1, 0)) thermal_zone_device_update(tz, THERMAL_EVENT_UNSPECIFIED); - thermal_notify_tz_create(tz->id, tz->type); + thermal_notify_tz_create(tz); return tz; @@ -1466,15 +1466,12 @@ EXPORT_SYMBOL_GPL(thermal_zone_device); */ void thermal_zone_device_unregister(struct thermal_zone_device *tz) { - int tz_id; struct thermal_cooling_device *cdev; struct thermal_zone_device *pos = NULL; if (!tz) return; - tz_id = tz->id; - mutex_lock(&thermal_list_lock); list_for_each_entry(pos, &thermal_tz_list, node) if (pos == tz) @@ -1510,7 +1507,7 @@ void thermal_zone_device_unregister(struct thermal_zone_device *tz) put_device(&tz->device); - thermal_notify_tz_delete(tz_id); + thermal_notify_tz_delete(tz); wait_for_completion(&tz->removal); kfree(tz); diff --git a/drivers/thermal/thermal_netlink.c b/drivers/thermal/thermal_netlink.c index 91992181e845..46b0156a6319 100644 --- a/drivers/thermal/thermal_netlink.c +++ b/drivers/thermal/thermal_netlink.c @@ -304,30 +304,30 @@ out_free_msg: return ret; } -int thermal_notify_tz_create(int tz_id, const char *name) +int thermal_notify_tz_create(const struct thermal_zone_device *tz) { - struct param p = { .tz_id = tz_id, .name = name }; + struct param p = { .tz_id = tz->id, .name = tz->type }; return thermal_genl_send_event(THERMAL_GENL_EVENT_TZ_CREATE, &p); } -int thermal_notify_tz_delete(int tz_id) +int thermal_notify_tz_delete(const struct thermal_zone_device *tz) { - struct param p = { .tz_id = tz_id }; + struct param p = { .tz_id = tz->id }; return thermal_genl_send_event(THERMAL_GENL_EVENT_TZ_DELETE, &p); } -int thermal_notify_tz_enable(int tz_id) +int thermal_notify_tz_enable(const struct thermal_zone_device *tz) { - struct param p = { .tz_id = tz_id }; + struct param p = { .tz_id = tz->id }; return thermal_genl_send_event(THERMAL_GENL_EVENT_TZ_ENABLE, &p); } -int thermal_notify_tz_disable(int tz_id) +int thermal_notify_tz_disable(const struct thermal_zone_device *tz) { - struct param p = { .tz_id = tz_id }; + struct param p = { .tz_id = tz->id }; return thermal_genl_send_event(THERMAL_GENL_EVENT_TZ_DISABLE, &p); } @@ -386,9 +386,10 @@ int thermal_notify_cdev_delete(int cdev_id) return thermal_genl_send_event(THERMAL_GENL_EVENT_CDEV_DELETE, &p); } -int thermal_notify_tz_gov_change(int tz_id, const char *name) +int thermal_notify_tz_gov_change(const struct thermal_zone_device *tz, + const char *name) { - struct param p = { .tz_id = tz_id, .name = name }; + struct param p = { .tz_id = tz->id, .name = name }; return thermal_genl_send_event(THERMAL_GENL_EVENT_TZ_GOV_CHANGE, &p); } diff --git a/drivers/thermal/thermal_netlink.h b/drivers/thermal/thermal_netlink.h index e780ce3f7db6..13ea2ad23d3a 100644 --- a/drivers/thermal/thermal_netlink.h +++ b/drivers/thermal/thermal_netlink.h @@ -17,10 +17,10 @@ struct thermal_trip; #ifdef CONFIG_THERMAL_NETLINK int __init thermal_netlink_init(void); void __init thermal_netlink_exit(void); -int thermal_notify_tz_create(int tz_id, const char *name); -int thermal_notify_tz_delete(int tz_id); -int thermal_notify_tz_enable(int tz_id); -int thermal_notify_tz_disable(int tz_id); +int thermal_notify_tz_create(const struct thermal_zone_device *tz); +int thermal_notify_tz_delete(const struct thermal_zone_device *tz); +int thermal_notify_tz_enable(const struct thermal_zone_device *tz); +int thermal_notify_tz_disable(const struct thermal_zone_device *tz); int thermal_notify_tz_trip_down(const struct thermal_zone_device *tz, const struct thermal_trip *trip); int thermal_notify_tz_trip_up(const struct thermal_zone_device *tz, @@ -30,7 +30,8 @@ int thermal_notify_tz_trip_change(const struct thermal_zone_device *tz, int thermal_notify_cdev_state_update(int cdev_id, int state); int thermal_notify_cdev_add(int cdev_id, const char *name, int max_state); int thermal_notify_cdev_delete(int cdev_id); -int thermal_notify_tz_gov_change(int tz_id, const char *name); +int thermal_notify_tz_gov_change(const struct thermal_zone_device *tz, + const char *name); int thermal_genl_sampling_temp(int id, int temp); int thermal_genl_cpu_capability_event(int count, struct thermal_genl_cpu_caps *caps); @@ -40,22 +41,22 @@ static inline int thermal_netlink_init(void) return 0; } -static inline int thermal_notify_tz_create(int tz_id, const char *name) +static inline int thermal_notify_tz_create(const struct thermal_zone_device *tz) { return 0; } -static inline int thermal_notify_tz_delete(int tz_id) +static inline int thermal_notify_tz_delete(const struct thermal_zone_device *tz) { return 0; } -static inline int thermal_notify_tz_enable(int tz_id) +static inline int thermal_notify_tz_enable(const struct thermal_zone_device *tz) { return 0; } -static inline int thermal_notify_tz_disable(int tz_id) +static inline int thermal_notify_tz_disable(const struct thermal_zone_device *tz) { return 0; } @@ -94,7 +95,8 @@ static inline int thermal_notify_cdev_delete(int cdev_id) return 0; } -static inline int thermal_notify_tz_gov_change(int tz_id, const char *name) +static inline int thermal_notify_tz_gov_change(const struct thermal_zone_device *tz, + const char *name) { return 0; } From 022e6f5184ff73eaaf6aa7a89b7fafc7c2bd8c9c Mon Sep 17 00:00:00 2001 From: Erick Archer Date: Fri, 5 Jan 2024 18:11:04 +0000 Subject: [PATCH 281/336] PM: QoS: Use kcalloc() instead of kzalloc() Use 2-factor multiplication argument form kcalloc() instead of kzalloc(). Link: https://github.com/KSPP/linux/issues/162 Signed-off-by: Erick Archer Reviewed-by: Gustavo A. R. Silva Signed-off-by: Rafael J. Wysocki --- drivers/base/power/qos.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/base/power/qos.c b/drivers/base/power/qos.c index 8e93167f1783..bd77f6734f14 100644 --- a/drivers/base/power/qos.c +++ b/drivers/base/power/qos.c @@ -201,7 +201,7 @@ static int dev_pm_qos_constraints_allocate(struct device *dev) if (!qos) return -ENOMEM; - n = kzalloc(3 * sizeof(*n), GFP_KERNEL); + n = kcalloc(3, sizeof(*n), GFP_KERNEL); if (!n) { kfree(qos); return -ENOMEM; From 2fb7e4dd35c52933b18ff127bf92d703c8e2e897 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 5 Jan 2024 13:51:21 +0100 Subject: [PATCH 282/336] PNP: make pnp_bus_type const Now that the driver core can properly handle constant struct bus_type, move the pnp_bus_type variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Signed-off-by: Greg Kroah-Hartman Signed-off-by: Rafael J. Wysocki --- drivers/pnp/driver.c | 2 +- include/linux/pnp.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pnp/driver.c b/drivers/pnp/driver.c index 46c534f6b1c9..0a5d0d8befa8 100644 --- a/drivers/pnp/driver.c +++ b/drivers/pnp/driver.c @@ -256,7 +256,7 @@ static const struct dev_pm_ops pnp_bus_dev_pm_ops = { .restore = pnp_bus_resume, }; -struct bus_type pnp_bus_type = { +const struct bus_type pnp_bus_type = { .name = "pnp", .match = pnp_bus_match, .probe = pnp_device_probe, diff --git a/include/linux/pnp.h b/include/linux/pnp.h index 267fb8a4fb6e..ddbe7c3ca4ce 100644 --- a/include/linux/pnp.h +++ b/include/linux/pnp.h @@ -435,7 +435,7 @@ struct pnp_protocol { #define protocol_for_each_dev(protocol, dev) \ list_for_each_entry(dev, &(protocol)->devices, protocol_list) -extern struct bus_type pnp_bus_type; +extern const struct bus_type pnp_bus_type; #if defined(CONFIG_PNP) From d2aaf19965045f70bb2ece514399cdc6fcce2e73 Mon Sep 17 00:00:00 2001 From: Ben Mayo Date: Sat, 6 Jan 2024 21:13:23 -0500 Subject: [PATCH 283/336] ACPI: resource: Add DMI quirks for ASUS Vivobook E1504GA and E1504GAB Asus Vivobook E1504GA and E1504GAB notebooks are affected by bug #216158 (DSDT specifies the kbd IRQ as level active-low and using the override changes this to rising edge, stopping the keyboard from working). Users of these notebooks do not have a working keyboard unless they add their DMI information to the struct irq1_level_low_skip_override array and compile a custom kernel. Add support for these computers to the Linux kernel without requiring the end-user to recompile the kernel. Link: https://bugzilla.kernel.org/show_bug.cgi?id=216158 Signed-off-by: Ben Mayo [ rjw: Link tag, subject and changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/resource.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/drivers/acpi/resource.c b/drivers/acpi/resource.c index 0e2c397b1399..b59bcfdbe48a 100644 --- a/drivers/acpi/resource.c +++ b/drivers/acpi/resource.c @@ -482,6 +482,20 @@ static const struct dmi_system_id irq1_level_low_skip_override[] = { DMI_MATCH(DMI_BOARD_NAME, "B2502CBA"), }, }, + { + /* Asus Vivobook E1504GA */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_BOARD_NAME, "E1504GA"), + }, + }, + { + /* Asus Vivobook E1504GAB */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_BOARD_NAME, "E1504GAB"), + }, + }, { /* LG Electronics 17U70P */ .matches = { From e315e8692f7922cd1b2a26bd7a1741cc8ce77085 Mon Sep 17 00:00:00 2001 From: Michael Maltsev Date: Sun, 7 Jan 2024 13:53:07 +0500 Subject: [PATCH 284/336] ACPI: resource: Skip IRQ override on ASUS ExpertBook B1502CGA Like the ASUS ExpertBook B1502CBA and various ASUS laptops, the ASUS ExpertBook B1502CGA has an ACPI DSDT table that describes IRQ 1 as ActiveLow while the kernel overrides it to Edge_High. $ sudo dmesg | grep DMI [ 0.000000] DMI: ASUSTeK COMPUTER INC. ASUS EXPERTBOOK B1502CGA_B1502CGA/B1502CGA, BIOS B1502CGA.303 06/05/2023 $ grep -A 40 PS2K dsdt.dsl | grep IRQ -A 1 IRQ (Level, ActiveLow, Exclusive, ) {1} This prevents the keyboard from working. To fix this issue, add this laptop to the skip_override_table so that the kernel does not override IRQ 1. Signed-off-by: Michael Maltsev [ rjw: rebase, replace .ident field with a comment ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/resource.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/acpi/resource.c b/drivers/acpi/resource.c index b59bcfdbe48a..dacad1d846c0 100644 --- a/drivers/acpi/resource.c +++ b/drivers/acpi/resource.c @@ -461,6 +461,13 @@ static const struct dmi_system_id irq1_level_low_skip_override[] = { DMI_MATCH(DMI_BOARD_NAME, "B1502CBA"), }, }, + { + /* Asus ExpertBook B1502CGA */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_BOARD_NAME, "B1502CGA"), + }, + }, { /* Asus ExpertBook B2402CBA */ .matches = { From cb1210a7c03c55f9fb7496bd44155c6a024fe458 Mon Sep 17 00:00:00 2001 From: Lorenzo Pieralisi Date: Wed, 27 Dec 2023 12:00:36 +0100 Subject: [PATCH 285/336] ACPICA: MADT: Add GICC online capable bit handling ACPICA commit 16f0befdeddf25756f317907798192bbaa417e5e Implement code to handle the GICC online capable bit management added into ACPI v6.5. Link: https://github.com/acpica/acpica/commit/16f0befd Signed-off-by: Lorenzo Pieralisi Signed-off-by: Rafael J. Wysocki --- include/acpi/actbl2.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/acpi/actbl2.h b/include/acpi/actbl2.h index 3751ae69432f..2b4dd2c3348f 100644 --- a/include/acpi/actbl2.h +++ b/include/acpi/actbl2.h @@ -1046,6 +1046,7 @@ struct acpi_madt_generic_interrupt { /* ACPI_MADT_ENABLED (1) Processor is usable if set */ #define ACPI_MADT_PERFORMANCE_IRQ_MODE (1<<1) /* 01: Performance Interrupt Mode */ #define ACPI_MADT_VGIC_IRQ_MODE (1<<2) /* 02: VGIC Maintenance Interrupt mode */ +#define ACPI_MADT_GICC_ONLINE_CAPABLE (1<<3) /* 03: Processor is online capable */ /* 12: Generic Distributor (ACPI 5.0 + ACPI 6.0 changes) */ From 3be8fb1bc6fc10b6c8e79052126fa5a423698fd1 Mon Sep 17 00:00:00 2001 From: Lorenzo Pieralisi Date: Wed, 27 Dec 2023 12:00:37 +0100 Subject: [PATCH 286/336] ACPICA: MADT: Add new MADT GICC/GICR/ITS non-coherent flags handling ACPICA commit c5d2010744b1bf7efba0bd04a8a9c200ef8fb610 Add new flags and related fields to the MADT GICC/GICR/ITS structures according to the code first ECR: https://bugzilla.tianocore.org/show_bug.cgi?id=4557 Update the MADT template to the latest MADT revision. Link: https://github.com/acpica/acpica/commit/c5d20107 Signed-off-by: Lorenzo Pieralisi Signed-off-by: Rafael J. Wysocki --- include/acpi/actbl2.h | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/include/acpi/actbl2.h b/include/acpi/actbl2.h index 2b4dd2c3348f..9775384d61c6 100644 --- a/include/acpi/actbl2.h +++ b/include/acpi/actbl2.h @@ -1047,6 +1047,7 @@ struct acpi_madt_generic_interrupt { #define ACPI_MADT_PERFORMANCE_IRQ_MODE (1<<1) /* 01: Performance Interrupt Mode */ #define ACPI_MADT_VGIC_IRQ_MODE (1<<2) /* 02: VGIC Maintenance Interrupt mode */ #define ACPI_MADT_GICC_ONLINE_CAPABLE (1<<3) /* 03: Processor is online capable */ +#define ACPI_MADT_GICC_NON_COHERENT (1<<4) /* 04: GIC redistributor is not coherent */ /* 12: Generic Distributor (ACPI 5.0 + ACPI 6.0 changes) */ @@ -1091,21 +1092,27 @@ struct acpi_madt_generic_msi_frame { struct acpi_madt_generic_redistributor { struct acpi_subtable_header header; - u16 reserved; /* reserved - must be zero */ + u8 flags; + u8 reserved; /* reserved - must be zero */ u64 base_address; u32 length; }; +#define ACPI_MADT_GICR_NON_COHERENT (1) + /* 15: Generic Translator (ACPI 6.0) */ struct acpi_madt_generic_translator { struct acpi_subtable_header header; - u16 reserved; /* reserved - must be zero */ + u8 flags; + u8 reserved; /* reserved - must be zero */ u32 translation_id; u64 base_address; u32 reserved2; }; +#define ACPI_MADT_ITS_NON_COHERENT (1) + /* 16: Multiprocessor wakeup (ACPI 6.4) */ struct acpi_madt_multiproc_wakeup { From 89fe46019a62bc1d0cb49c9615cb3520096c4bc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ADra=20Canal?= Date: Tue, 9 Jan 2024 08:30:40 -0300 Subject: [PATCH 287/336] drm/v3d: Fix support for register debugging on the RPi 4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RPi 4 uses V3D 4.2, which is currently not supported by the register definition stated at `v3d_core_reg_defs`. We should be able to support V3D 4.2, therefore, change the maximum version of the register definition to 42, not 41. Fixes: 0ad5bc1ce463 ("drm/v3d: fix up register addresses for V3D 7.x") Signed-off-by: Maíra Canal Reviewed-by: Iago Toral Quiroga Link: https://patchwork.freedesktop.org/patch/msgid/20240109113126.929446-1-mcanal@igalia.com --- drivers/gpu/drm/v3d/v3d_debugfs.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/v3d/v3d_debugfs.c b/drivers/gpu/drm/v3d/v3d_debugfs.c index f843a50d5dce..94eafcecc65b 100644 --- a/drivers/gpu/drm/v3d/v3d_debugfs.c +++ b/drivers/gpu/drm/v3d/v3d_debugfs.c @@ -62,9 +62,9 @@ static const struct v3d_reg_def v3d_core_reg_defs[] = { REGDEF(33, 71, V3D_PTB_BPCA), REGDEF(33, 71, V3D_PTB_BPCS), - REGDEF(33, 41, V3D_GMP_STATUS(33)), - REGDEF(33, 41, V3D_GMP_CFG(33)), - REGDEF(33, 41, V3D_GMP_VIO_ADDR(33)), + REGDEF(33, 42, V3D_GMP_STATUS(33)), + REGDEF(33, 42, V3D_GMP_CFG(33)), + REGDEF(33, 42, V3D_GMP_VIO_ADDR(33)), REGDEF(33, 71, V3D_ERR_FDBGO), REGDEF(33, 71, V3D_ERR_FDBGB), @@ -74,13 +74,13 @@ static const struct v3d_reg_def v3d_core_reg_defs[] = { static const struct v3d_reg_def v3d_csd_reg_defs[] = { REGDEF(41, 71, V3D_CSD_STATUS), - REGDEF(41, 41, V3D_CSD_CURRENT_CFG0(41)), - REGDEF(41, 41, V3D_CSD_CURRENT_CFG1(41)), - REGDEF(41, 41, V3D_CSD_CURRENT_CFG2(41)), - REGDEF(41, 41, V3D_CSD_CURRENT_CFG3(41)), - REGDEF(41, 41, V3D_CSD_CURRENT_CFG4(41)), - REGDEF(41, 41, V3D_CSD_CURRENT_CFG5(41)), - REGDEF(41, 41, V3D_CSD_CURRENT_CFG6(41)), + REGDEF(41, 42, V3D_CSD_CURRENT_CFG0(41)), + REGDEF(41, 42, V3D_CSD_CURRENT_CFG1(41)), + REGDEF(41, 42, V3D_CSD_CURRENT_CFG2(41)), + REGDEF(41, 42, V3D_CSD_CURRENT_CFG3(41)), + REGDEF(41, 42, V3D_CSD_CURRENT_CFG4(41)), + REGDEF(41, 42, V3D_CSD_CURRENT_CFG5(41)), + REGDEF(41, 42, V3D_CSD_CURRENT_CFG6(41)), REGDEF(71, 71, V3D_CSD_CURRENT_CFG0(71)), REGDEF(71, 71, V3D_CSD_CURRENT_CFG1(71)), REGDEF(71, 71, V3D_CSD_CURRENT_CFG2(71)), From 420370f3ae3d3b883813fd3051a38805160b2b9f Mon Sep 17 00:00:00 2001 From: Alexandre Ghiti Date: Thu, 14 Dec 2023 10:19:26 +0100 Subject: [PATCH 288/336] riscv: Check if the code to patch lies in the exit section Otherwise we fall through to vmalloc_to_page() which panics since the address does not lie in the vmalloc region. Fixes: 043cb41a85de ("riscv: introduce interfaces to patch kernel code") Signed-off-by: Alexandre Ghiti Reviewed-by: Charlie Jenkins Link: https://lore.kernel.org/r/20231214091926.203439-1-alexghiti@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/sections.h | 1 + arch/riscv/kernel/patch.c | 11 ++++++++++- arch/riscv/kernel/vmlinux-xip.lds.S | 2 ++ arch/riscv/kernel/vmlinux.lds.S | 2 ++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/arch/riscv/include/asm/sections.h b/arch/riscv/include/asm/sections.h index 32336e8a17cb..a393d5035c54 100644 --- a/arch/riscv/include/asm/sections.h +++ b/arch/riscv/include/asm/sections.h @@ -13,6 +13,7 @@ extern char _start_kernel[]; extern char __init_data_begin[], __init_data_end[]; extern char __init_text_begin[], __init_text_end[]; extern char __alt_start[], __alt_end[]; +extern char __exittext_begin[], __exittext_end[]; static inline bool is_va_kernel_text(uintptr_t va) { diff --git a/arch/riscv/kernel/patch.c b/arch/riscv/kernel/patch.c index 13ee7bf589a1..37e87fdcf6a0 100644 --- a/arch/riscv/kernel/patch.c +++ b/arch/riscv/kernel/patch.c @@ -14,6 +14,7 @@ #include #include #include +#include struct patch_insn { void *addr; @@ -25,6 +26,14 @@ struct patch_insn { int riscv_patch_in_stop_machine = false; #ifdef CONFIG_MMU + +static inline bool is_kernel_exittext(uintptr_t addr) +{ + return system_state < SYSTEM_RUNNING && + addr >= (uintptr_t)__exittext_begin && + addr < (uintptr_t)__exittext_end; +} + /* * The fix_to_virt(, idx) needs a const value (not a dynamic variable of * reg-a0) or BUILD_BUG_ON failed with "idx >= __end_of_fixed_addresses". @@ -35,7 +44,7 @@ static __always_inline void *patch_map(void *addr, const unsigned int fixmap) uintptr_t uintaddr = (uintptr_t) addr; struct page *page; - if (core_kernel_text(uintaddr)) + if (core_kernel_text(uintaddr) || is_kernel_exittext(uintaddr)) page = phys_to_page(__pa_symbol(addr)); else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) page = vmalloc_to_page(addr); diff --git a/arch/riscv/kernel/vmlinux-xip.lds.S b/arch/riscv/kernel/vmlinux-xip.lds.S index 50767647fbc6..8c3daa1b0531 100644 --- a/arch/riscv/kernel/vmlinux-xip.lds.S +++ b/arch/riscv/kernel/vmlinux-xip.lds.S @@ -29,10 +29,12 @@ SECTIONS HEAD_TEXT_SECTION INIT_TEXT_SECTION(PAGE_SIZE) /* we have to discard exit text and such at runtime, not link time */ + __exittext_begin = .; .exit.text : { EXIT_TEXT } + __exittext_end = .; .text : { _text = .; diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S index 492dd4b8f3d6..002ca58dd998 100644 --- a/arch/riscv/kernel/vmlinux.lds.S +++ b/arch/riscv/kernel/vmlinux.lds.S @@ -69,10 +69,12 @@ SECTIONS __soc_builtin_dtb_table_end = .; } /* we have to discard exit text and such at runtime, not link time */ + __exittext_begin = .; .exit.text : { EXIT_TEXT } + __exittext_end = .; __init_text_end = .; . = ALIGN(SECTION_ALIGN); From c29fc621e1a49949a14c7fa031dd4760087bfb29 Mon Sep 17 00:00:00 2001 From: Alexandre Ghiti Date: Tue, 12 Dec 2023 20:54:00 +0100 Subject: [PATCH 289/336] riscv: Fix wrong usage of lm_alias() when splitting a huge linear mapping lm_alias() can only be used on kernel mappings since it explicitly uses __pa_symbol(), so simply fix this by checking where the address belongs to before. Fixes: 311cd2f6e253 ("riscv: Fix set_memory_XX() and set_direct_map_XX() by splitting huge linear mappings") Reported-by: syzbot+afb726d49f84c8d95ee1@syzkaller.appspotmail.com Closes: https://lore.kernel.org/linux-riscv/000000000000620dd0060c02c5e1@google.com/ Signed-off-by: Alexandre Ghiti Reviewed-by: Charlie Jenkins Link: https://lore.kernel.org/r/20231212195400.128457-1-alexghiti@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/mm/pageattr.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/arch/riscv/mm/pageattr.c b/arch/riscv/mm/pageattr.c index fc5fc4f785c4..96cbda683936 100644 --- a/arch/riscv/mm/pageattr.c +++ b/arch/riscv/mm/pageattr.c @@ -305,8 +305,13 @@ static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask, goto unlock; } } else if (is_kernel_mapping(start) || is_linear_mapping(start)) { - lm_start = (unsigned long)lm_alias(start); - lm_end = (unsigned long)lm_alias(end); + if (is_kernel_mapping(start)) { + lm_start = (unsigned long)lm_alias(start); + lm_end = (unsigned long)lm_alias(end); + } else { + lm_start = start; + lm_end = end; + } ret = split_linear_mapping(lm_start, lm_end); if (ret) From 749b94b08005929bbc636df21a23322733166e35 Mon Sep 17 00:00:00 2001 From: Alexandre Ghiti Date: Wed, 13 Dec 2023 14:40:26 +0100 Subject: [PATCH 290/336] riscv: Fix module_alloc() that did not reset the linear mapping permissions After unloading a module, we must reset the linear mapping permissions, see the example below: Before unloading a module: 0xffffaf809d65d000-0xffffaf809d6dc000 0x000000011d65d000 508K PTE . .. .. D A G . . W R V 0xffffaf809d6dc000-0xffffaf809d6dd000 0x000000011d6dc000 4K PTE . .. .. D A G . . . R V 0xffffaf809d6dd000-0xffffaf809d6e1000 0x000000011d6dd000 16K PTE . .. .. D A G . . W R V 0xffffaf809d6e1000-0xffffaf809d6e7000 0x000000011d6e1000 24K PTE . .. .. D A G . X . R V After unloading a module: 0xffffaf809d65d000-0xffffaf809d6e1000 0x000000011d65d000 528K PTE . .. .. D A G . . W R V 0xffffaf809d6e1000-0xffffaf809d6e7000 0x000000011d6e1000 24K PTE . .. .. D A G . X W R V The last mapping is not reset and we end up with WX mappings in the linear mapping. So add VM_FLUSH_RESET_PERMS to our module_alloc() definition. Fixes: 0cff8bff7af8 ("riscv: avoid the PIC offset of static percpu data in module beyond 2G limits") Signed-off-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20231213134027.155327-2-alexghiti@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/module.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/riscv/kernel/module.c b/arch/riscv/kernel/module.c index aac019ed63b1..862834bb1d64 100644 --- a/arch/riscv/kernel/module.c +++ b/arch/riscv/kernel/module.c @@ -894,7 +894,8 @@ void *module_alloc(unsigned long size) { return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, GFP_KERNEL, - PAGE_KERNEL, 0, NUMA_NO_NODE, + PAGE_KERNEL, VM_FLUSH_RESET_PERMS, + NUMA_NO_NODE, __builtin_return_address(0)); } #endif From b8b2711336f03ece539de61479d6ffc44fb603d3 Mon Sep 17 00:00:00 2001 From: Alexandre Ghiti Date: Wed, 13 Dec 2023 14:40:27 +0100 Subject: [PATCH 291/336] riscv: Fix set_direct_map_default_noflush() to reset _PAGE_EXEC When resetting the linear mapping permissions, we must make sure that we clear the X bit so that do not end up with WX mappings (since we set PAGE_KERNEL). Fixes: 395a21ff859c ("riscv: add ARCH_HAS_SET_DIRECT_MAP support") Signed-off-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20231213134027.155327-3-alexghiti@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/mm/pageattr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/mm/pageattr.c b/arch/riscv/mm/pageattr.c index 96cbda683936..01398fee5cf8 100644 --- a/arch/riscv/mm/pageattr.c +++ b/arch/riscv/mm/pageattr.c @@ -383,7 +383,7 @@ int set_direct_map_invalid_noflush(struct page *page) int set_direct_map_default_noflush(struct page *page) { return __set_memory((unsigned long)page_address(page), 1, - PAGE_KERNEL, __pgprot(0)); + PAGE_KERNEL, __pgprot(_PAGE_EXEC)); } #ifdef CONFIG_DEBUG_PAGEALLOC From 66f1e68093979816a23412a3fad066f5bcbc0360 Mon Sep 17 00:00:00 2001 From: Frederik Haxel Date: Tue, 12 Dec 2023 14:01:12 +0100 Subject: [PATCH 292/336] riscv: Make XIP bootable again Currently, the XIP kernel seems to fail to boot due to missing XIP_FIXUP and a wrong page_offset value. A superfluous XIP_FIXUP has also been removed. Signed-off-by: Frederik Haxel Link: https://lore.kernel.org/r/20231212130116.848530-2-haxel@fzi.de Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/head.S | 1 + arch/riscv/mm/init.c | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S index b77397432403..a2e2f0dd3899 100644 --- a/arch/riscv/kernel/head.S +++ b/arch/riscv/kernel/head.S @@ -89,6 +89,7 @@ relocate_enable_mmu: /* Compute satp for kernel page tables, but don't load it yet */ srl a2, a0, PAGE_SHIFT la a1, satp_mode + XIP_FIXUP_OFFSET a1 REG_L a1, 0(a1) or a2, a2, a1 diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index 2e011cbddf3a..a65937336cdc 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -174,6 +174,9 @@ void __init mem_init(void) /* Limit the memory size via mem. */ static phys_addr_t memory_limit; +#ifdef CONFIG_XIP_KERNEL +#define memory_limit (*(phys_addr_t *)XIP_FIXUP(&memory_limit)) +#endif /* CONFIG_XIP_KERNEL */ static int __init early_mem(char *p) { @@ -952,7 +955,7 @@ static void __init create_fdt_early_page_table(uintptr_t fix_fdt_va, * setup_vm_final installs the linear mapping. For 32-bit kernel, as the * kernel is mapped in the linear mapping, that makes no difference. */ - dtb_early_va = kernel_mapping_pa_to_va(XIP_FIXUP(dtb_pa)); + dtb_early_va = kernel_mapping_pa_to_va(dtb_pa); #endif dtb_early_pa = dtb_pa; @@ -1055,9 +1058,9 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) #endif kernel_map.virt_addr = KERNEL_LINK_ADDR + kernel_map.virt_offset; - kernel_map.page_offset = _AC(CONFIG_PAGE_OFFSET, UL); #ifdef CONFIG_XIP_KERNEL + kernel_map.page_offset = PAGE_OFFSET_L3; kernel_map.xiprom = (uintptr_t)CONFIG_XIP_PHYS_ADDR; kernel_map.xiprom_sz = (uintptr_t)(&_exiprom) - (uintptr_t)(&_xiprom); @@ -1067,6 +1070,7 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) kernel_map.va_kernel_xip_pa_offset = kernel_map.virt_addr - kernel_map.xiprom; #else + kernel_map.page_offset = _AC(CONFIG_PAGE_OFFSET, UL); kernel_map.phys_addr = (uintptr_t)(&_start); kernel_map.size = (uintptr_t)(&_end) - kernel_map.phys_addr; #endif From 5daa3726410288075ba73c336bb2e80d6b06aa4d Mon Sep 17 00:00:00 2001 From: Frederik Haxel Date: Tue, 12 Dec 2023 14:01:13 +0100 Subject: [PATCH 293/336] riscv: Fixed wrong register in XIP_FIXUP_FLASH_OFFSET macro During the refactoring, a bug was introduced in the rarly used XIP_FIXUP_FLASH_OFFSET macro. Fixes: bee7fbc38579 ("RISC-V CPU Idle Support") Fixes: e7681beba992 ("RISC-V: Split out the XIP fixups into their own file") Signed-off-by: Frederik Haxel Link: https://lore.kernel.org/r/20231212130116.848530-3-haxel@fzi.de Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/xip_fixup.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/include/asm/xip_fixup.h b/arch/riscv/include/asm/xip_fixup.h index d4ffc3c37649..b65bf6306f69 100644 --- a/arch/riscv/include/asm/xip_fixup.h +++ b/arch/riscv/include/asm/xip_fixup.h @@ -13,7 +13,7 @@ add \reg, \reg, t0 .endm .macro XIP_FIXUP_FLASH_OFFSET reg - la t1, __data_loc + la t0, __data_loc REG_L t1, _xip_phys_offset sub \reg, \reg, t1 add \reg, \reg, t0 From 6c4a2f6329f0925161a80d2fd90aa8438c1ca82f Mon Sep 17 00:00:00 2001 From: Frederik Haxel Date: Tue, 12 Dec 2023 14:01:14 +0100 Subject: [PATCH 294/336] riscv: Allow disabling of BUILTIN_DTB for XIP This enables, among other things, testing with the QEMU virt machine. To build an XIP kernel for the QEMU virt machine, configure the the kernel as desired and apply the following configuration ``` CONFIG_NONPORTABLE=y CONFIG_XIP_KERNEL=y CONFIG_XIP_PHYS_ADDR=0x20000000 CONFIG_PHYS_RAM_BASE=0x80200000 CONFIG_BUILTIN_DTB=n ``` Since the QEMU virt flash memory expects a 32 MB file, the built image must be padded. For example, with `truncate -s 32M arch/riscv/boot/xipImage` The kernel can be started using the following command in QEMU (v8+) ``` qemu-system-riscv64 -M virt,pflash0=pflash0 \ -blockdev node-name=pflash0,driver=file,read-only=on,\ filename=arch/riscv/boot/xipImage ``` Signed-off-by: Frederik Haxel Link: https://lore.kernel.org/r/20231212130116.848530-4-haxel@fzi.de Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig | 6 +++--- arch/riscv/kernel/head.S | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 95a2a06acc6a..72bc31b6eeb9 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -904,13 +904,13 @@ config RISCV_ISA_FALLBACK on the replacement properties, "riscv,isa-base" and "riscv,isa-extensions". -endmenu # "Boot options" - config BUILTIN_DTB - bool + bool "Built-in device tree" depends on OF && NONPORTABLE default y if XIP_KERNEL +endmenu # "Boot options" + config PORTABLE bool default !NONPORTABLE diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S index a2e2f0dd3899..a8939558702c 100644 --- a/arch/riscv/kernel/head.S +++ b/arch/riscv/kernel/head.S @@ -266,10 +266,12 @@ SYM_CODE_START(_start_kernel) la sp, _end + THREAD_SIZE XIP_FIXUP_OFFSET sp mv s0, a0 + mv s1, a1 call __copy_data - /* Restore a0 copy */ + /* Restore a0 & a1 copy */ mv a0, s0 + mv a1, s1 #endif #ifndef CONFIG_XIP_KERNEL From b7b4e4d79fc72e2b565cb3da38897b0e4c891e79 Mon Sep 17 00:00:00 2001 From: Samuel Holland Date: Tue, 21 Nov 2023 14:53:18 -0800 Subject: [PATCH 295/336] riscv: Remove obsolete rv32_defconfig file This file is not used since commit 72f045d19f25 ("riscv: Fixup difference with defconfig"), where it was replaced by the 32-bit.config fragment. Delete the old file to avoid any confusion. Signed-off-by: Samuel Holland Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20231121225320.3430550-1-samuel.holland@sifive.com Signed-off-by: Palmer Dabbelt --- arch/riscv/configs/rv32_defconfig | 139 ------------------------------ 1 file changed, 139 deletions(-) delete mode 100644 arch/riscv/configs/rv32_defconfig diff --git a/arch/riscv/configs/rv32_defconfig b/arch/riscv/configs/rv32_defconfig deleted file mode 100644 index 89b601e253a6..000000000000 --- a/arch/riscv/configs/rv32_defconfig +++ /dev/null @@ -1,139 +0,0 @@ -CONFIG_SYSVIPC=y -CONFIG_POSIX_MQUEUE=y -CONFIG_NO_HZ_IDLE=y -CONFIG_HIGH_RES_TIMERS=y -CONFIG_BPF_SYSCALL=y -CONFIG_IKCONFIG=y -CONFIG_IKCONFIG_PROC=y -CONFIG_CGROUPS=y -CONFIG_CGROUP_SCHED=y -CONFIG_CFS_BANDWIDTH=y -CONFIG_CGROUP_BPF=y -CONFIG_NAMESPACES=y -CONFIG_USER_NS=y -CONFIG_CHECKPOINT_RESTORE=y -CONFIG_BLK_DEV_INITRD=y -CONFIG_EXPERT=y -# CONFIG_SYSFS_SYSCALL is not set -CONFIG_PROFILING=y -CONFIG_SOC_SIFIVE=y -CONFIG_SOC_VIRT=y -CONFIG_NONPORTABLE=y -CONFIG_ARCH_RV32I=y -CONFIG_SMP=y -CONFIG_HOTPLUG_CPU=y -CONFIG_PM=y -CONFIG_CPU_IDLE=y -CONFIG_VIRTUALIZATION=y -CONFIG_KVM=m -CONFIG_JUMP_LABEL=y -CONFIG_MODULES=y -CONFIG_MODULE_UNLOAD=y -CONFIG_NET=y -CONFIG_PACKET=y -CONFIG_UNIX=y -CONFIG_INET=y -CONFIG_IP_MULTICAST=y -CONFIG_IP_ADVANCED_ROUTER=y -CONFIG_IP_PNP=y -CONFIG_IP_PNP_DHCP=y -CONFIG_IP_PNP_BOOTP=y -CONFIG_IP_PNP_RARP=y -CONFIG_NETLINK_DIAG=y -CONFIG_NET_9P=y -CONFIG_NET_9P_VIRTIO=y -CONFIG_PCI=y -CONFIG_PCIEPORTBUS=y -CONFIG_PCI_HOST_GENERIC=y -CONFIG_PCIE_XILINX=y -CONFIG_DEVTMPFS=y -CONFIG_DEVTMPFS_MOUNT=y -CONFIG_BLK_DEV_LOOP=y -CONFIG_VIRTIO_BLK=y -CONFIG_BLK_DEV_SD=y -CONFIG_BLK_DEV_SR=y -CONFIG_SCSI_VIRTIO=y -CONFIG_ATA=y -CONFIG_SATA_AHCI=y -CONFIG_SATA_AHCI_PLATFORM=y -CONFIG_NETDEVICES=y -CONFIG_VIRTIO_NET=y -CONFIG_MACB=y -CONFIG_E1000E=y -CONFIG_R8169=y -CONFIG_MICROSEMI_PHY=y -CONFIG_INPUT_MOUSEDEV=y -CONFIG_SERIAL_8250=y -CONFIG_SERIAL_8250_CONSOLE=y -CONFIG_SERIAL_OF_PLATFORM=y -CONFIG_VIRTIO_CONSOLE=y -CONFIG_HW_RANDOM=y -CONFIG_HW_RANDOM_VIRTIO=y -CONFIG_SPI=y -CONFIG_SPI_SIFIVE=y -# CONFIG_PTP_1588_CLOCK is not set -CONFIG_DRM=y -CONFIG_DRM_RADEON=y -CONFIG_DRM_VIRTIO_GPU=y -CONFIG_FB=y -CONFIG_FRAMEBUFFER_CONSOLE=y -CONFIG_USB=y -CONFIG_USB_XHCI_HCD=y -CONFIG_USB_XHCI_PLATFORM=y -CONFIG_USB_EHCI_HCD=y -CONFIG_USB_EHCI_HCD_PLATFORM=y -CONFIG_USB_OHCI_HCD=y -CONFIG_USB_OHCI_HCD_PLATFORM=y -CONFIG_USB_STORAGE=y -CONFIG_USB_UAS=y -CONFIG_MMC=y -CONFIG_MMC_SPI=y -CONFIG_RTC_CLASS=y -CONFIG_VIRTIO_PCI=y -CONFIG_VIRTIO_BALLOON=y -CONFIG_VIRTIO_INPUT=y -CONFIG_VIRTIO_MMIO=y -CONFIG_RPMSG_CHAR=y -CONFIG_RPMSG_CTRL=y -CONFIG_RPMSG_VIRTIO=y -CONFIG_EXT4_FS=y -CONFIG_EXT4_FS_POSIX_ACL=y -CONFIG_AUTOFS_FS=y -CONFIG_MSDOS_FS=y -CONFIG_VFAT_FS=y -CONFIG_TMPFS=y -CONFIG_TMPFS_POSIX_ACL=y -CONFIG_HUGETLBFS=y -CONFIG_NFS_FS=y -CONFIG_NFS_V4=y -CONFIG_NFS_V4_1=y -CONFIG_NFS_V4_2=y -CONFIG_ROOT_NFS=y -CONFIG_9P_FS=y -CONFIG_CRYPTO_USER_API_HASH=y -CONFIG_CRYPTO_DEV_VIRTIO=y -CONFIG_PRINTK_TIME=y -CONFIG_DEBUG_FS=y -CONFIG_DEBUG_PAGEALLOC=y -CONFIG_SCHED_STACK_END_CHECK=y -CONFIG_DEBUG_VM=y -CONFIG_DEBUG_VM_PGFLAGS=y -CONFIG_DEBUG_MEMORY_INIT=y -CONFIG_DEBUG_PER_CPU_MAPS=y -CONFIG_SOFTLOCKUP_DETECTOR=y -CONFIG_WQ_WATCHDOG=y -CONFIG_DEBUG_TIMEKEEPING=y -CONFIG_DEBUG_RT_MUTEXES=y -CONFIG_DEBUG_SPINLOCK=y -CONFIG_DEBUG_MUTEXES=y -CONFIG_DEBUG_RWSEMS=y -CONFIG_DEBUG_ATOMIC_SLEEP=y -CONFIG_STACKTRACE=y -CONFIG_DEBUG_LIST=y -CONFIG_DEBUG_PLIST=y -CONFIG_DEBUG_SG=y -# CONFIG_RCU_TRACE is not set -CONFIG_RCU_EQS_DEBUG=y -# CONFIG_FTRACE is not set -# CONFIG_RUNTIME_TESTING_MENU is not set -CONFIG_MEMTEST=y From cfbc4f81c9d0ea7d6b6726d55a93e859f51ef196 Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Thu, 23 Nov 2023 22:22:23 +0800 Subject: [PATCH 296/336] riscv: Select ARCH_WANTS_NO_INSTR As said in the help of ARCH_WANTS_NO_INSTR entry in arch/Kconfig: "An architecture should select this if the noinstr macro is being used on functions to denote that the toolchain should avoid instrumenting such functions and is required for correctness." Select ARCH_WANTS_NO_INSTR for correctness. PS: The reason we didn't find any issue so far is that the CC_HAS_NO_PROFILE_FN_ATTR is true. Signed-off-by: Jisheng Zhang Link: https://lore.kernel.org/r/20231123142223.1787-1-jszhang@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 95a2a06acc6a..5e12582f66d4 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -59,6 +59,7 @@ config RISCV select ARCH_WANT_HUGE_PMD_SHARE if 64BIT select ARCH_WANT_LD_ORPHAN_WARN if !XIP_KERNEL select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP + select ARCH_WANTS_NO_INSTR select ARCH_WANTS_THP_SWAP if HAVE_ARCH_TRANSPARENT_HUGEPAGE select BINFMT_FLAT_NO_DATA_START_OFFSET if !MMU select BUILDTIME_TABLE_SORT if MMU From ca0e433b41a6aa5115f752644eb39470f9a12a99 Mon Sep 17 00:00:00 2001 From: Ben Dooks Date: Thu, 23 Nov 2023 14:16:17 +0000 Subject: [PATCH 297/336] riscv: fix __user annotation in traps_misaligned.c The instruction reading code can read from either user or kernel addresses and thus the use of __user on pointers to instructions depends on which context. Fix a few sparse warnings by using __user for user-accesses and remove it when not. Fixes: arch/riscv/kernel/traps_misaligned.c:361:21: warning: dereference of noderef expression arch/riscv/kernel/traps_misaligned.c:373:21: warning: dereference of noderef expression arch/riscv/kernel/traps_misaligned.c:381:21: warning: dereference of noderef expression arch/riscv/kernel/traps_misaligned.c:322:24: warning: incorrect type in initializer (different address spaces) arch/riscv/kernel/traps_misaligned.c:322:24: expected unsigned char const [noderef] __user *__gu_ptr arch/riscv/kernel/traps_misaligned.c:322:24: got unsigned char const [usertype] *addr arch/riscv/kernel/traps_misaligned.c:361:21: warning: dereference of noderef expression arch/riscv/kernel/traps_misaligned.c:373:21: warning: dereference of noderef expression arch/riscv/kernel/traps_misaligned.c:381:21: warning: dereference of noderef expression arch/riscv/kernel/traps_misaligned.c:332:24: warning: incorrect type in initializer (different address spaces) arch/riscv/kernel/traps_misaligned.c:332:24: expected unsigned char [noderef] __user *__gu_ptr arch/riscv/kernel/traps_misaligned.c:332:24: got unsigned char [usertype] *addr Fixes: 7c83232161f60 ("riscv: add support for misaligned trap handling in S-mode") Signed-off-by: Ben Dooks Link: https://lore.kernel.org/r/20231123141617.259591-1-ben.dooks@codethink.co.uk Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/traps_misaligned.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/riscv/kernel/traps_misaligned.c b/arch/riscv/kernel/traps_misaligned.c index 5eba37147caa..446e3d4eeea9 100644 --- a/arch/riscv/kernel/traps_misaligned.c +++ b/arch/riscv/kernel/traps_misaligned.c @@ -319,7 +319,7 @@ static inline int get_insn(struct pt_regs *regs, ulong mepc, ulong *r_insn) static inline int load_u8(struct pt_regs *regs, const u8 *addr, u8 *r_val) { if (user_mode(regs)) { - return __get_user(*r_val, addr); + return __get_user(*r_val, (u8 __user *)addr); } else { *r_val = *addr; return 0; @@ -329,7 +329,7 @@ static inline int load_u8(struct pt_regs *regs, const u8 *addr, u8 *r_val) static inline int store_u8(struct pt_regs *regs, u8 *addr, u8 val) { if (user_mode(regs)) { - return __put_user(val, addr); + return __put_user(val, (u8 __user *)addr); } else { *addr = val; return 0; @@ -343,7 +343,7 @@ static inline int store_u8(struct pt_regs *regs, u8 *addr, u8 val) if (user_mode(regs)) { \ __ret = __get_user(insn, insn_addr); \ } else { \ - insn = *insn_addr; \ + insn = *(__force u16 *)insn_addr; \ __ret = 0; \ } \ \ From 869436dae72acf1629b41437e9d08d31a7360fdb Mon Sep 17 00:00:00 2001 From: Ben Dooks Date: Thu, 23 Nov 2023 14:27:08 +0000 Subject: [PATCH 298/336] riscv; fix __user annotation in save_v_state() The save_v_state() is technically sending a __user pointer through __put_user() and thus is generating a sparse warning so force the value to be "void *" to fix: arch/riscv/kernel/signal.c:94:16: warning: incorrect type in initializer (different address spaces) arch/riscv/kernel/signal.c:94:16: expected void *__val arch/riscv/kernel/signal.c:94:16: got void [noderef] __user *[assigned] datap Fixes: 8ee0b41898fa26f66e32 ("riscv: signal: Add sigcontext save/restore for vector") Signed-off-by: Ben Dooks Link: https://lore.kernel.org/r/20231123142708.261733-1-ben.dooks@codethink.co.uk Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/signal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/kernel/signal.c b/arch/riscv/kernel/signal.c index 88b6220b2608..33dfb5078301 100644 --- a/arch/riscv/kernel/signal.c +++ b/arch/riscv/kernel/signal.c @@ -91,7 +91,7 @@ static long save_v_state(struct pt_regs *regs, void __user **sc_vec) err = __copy_to_user(&state->v_state, ¤t->thread.vstate, offsetof(struct __riscv_v_ext_state, datap)); /* Copy the pointer datap itself. */ - err |= __put_user(datap, &state->v_state.datap); + err |= __put_user((__force void *)datap, &state->v_state.datap); /* Copy the whole vector content to user space datap. */ err |= __copy_to_user(datap, current->thread.vstate.datap, riscv_v_vsize); /* Copy magic to the user space after saving all vector conetext */ From a7565f4d068b2e60f95c3223c3167c40b8fe83ae Mon Sep 17 00:00:00 2001 From: Song Shuai Date: Mon, 11 Dec 2023 19:03:31 +0800 Subject: [PATCH 299/336] riscv: Remove SHADOW_OVERFLOW_STACK_SIZE macro The commit be97d0db5f44 ("riscv: VMAP_STACK overflow detection thread-safe") got rid of `shadow_stack`, so SHADOW_OVERFLOW_STACK_SIZE should be removed too. Fixes: be97d0db5f44 ("riscv: VMAP_STACK overflow detection thread-safe") Signed-off-by: Song Shuai Reviewed-by: Sami Tolvanen Link: https://lore.kernel.org/r/20231211110331.359534-1-songshuaishuai@tinylab.org Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/thread_info.h | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h index 574779900bfb..4856697c5f25 100644 --- a/arch/riscv/include/asm/thread_info.h +++ b/arch/riscv/include/asm/thread_info.h @@ -28,7 +28,6 @@ #define THREAD_SHIFT (PAGE_SHIFT + THREAD_SIZE_ORDER) #define OVERFLOW_STACK_SIZE SZ_4K -#define SHADOW_OVERFLOW_STACK_SIZE (1024) #define IRQ_STACK_SIZE THREAD_SIZE From 62694797f56bf72e4c598bc0e319c7b828c1b187 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Thu, 14 Dec 2023 19:19:22 +0000 Subject: [PATCH 300/336] use linux/export.h rather than asm-generic/export.h asm-generic/export.h is a wrapper for linux/export.h, with explicit request to use linux/export.h directly. Signed-off-by: Al Viro Link: https://lore.kernel.org/r/20231214191922.GQ1674809@ZenIV Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/mcount-dyn.S | 2 +- arch/riscv/kernel/mcount.S | 2 +- arch/riscv/lib/clear_page.S | 2 +- arch/riscv/lib/tishift.S | 2 +- arch/riscv/lib/uaccess.S | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/riscv/kernel/mcount-dyn.S b/arch/riscv/kernel/mcount-dyn.S index 58dd96a2a153..79dc81223238 100644 --- a/arch/riscv/kernel/mcount-dyn.S +++ b/arch/riscv/kernel/mcount-dyn.S @@ -3,12 +3,12 @@ #include #include +#include #include #include #include #include #include -#include #include .text diff --git a/arch/riscv/kernel/mcount.S b/arch/riscv/kernel/mcount.S index b4dd9ed6849e..d7ec69ac6910 100644 --- a/arch/riscv/kernel/mcount.S +++ b/arch/riscv/kernel/mcount.S @@ -4,12 +4,12 @@ #include #include #include +#include #include #include #include #include #include -#include #include .text diff --git a/arch/riscv/lib/clear_page.S b/arch/riscv/lib/clear_page.S index b22de1231144..20ff03f5b0f2 100644 --- a/arch/riscv/lib/clear_page.S +++ b/arch/riscv/lib/clear_page.S @@ -4,9 +4,9 @@ */ #include +#include #include #include -#include #include #include #include diff --git a/arch/riscv/lib/tishift.S b/arch/riscv/lib/tishift.S index ef90075c4b0a..c8294bf72c06 100644 --- a/arch/riscv/lib/tishift.S +++ b/arch/riscv/lib/tishift.S @@ -4,7 +4,7 @@ */ #include -#include +#include SYM_FUNC_START(__lshrti3) beqz a2, .L1 diff --git a/arch/riscv/lib/uaccess.S b/arch/riscv/lib/uaccess.S index 3ab438f30d13..a9d356d6c03c 100644 --- a/arch/riscv/lib/uaccess.S +++ b/arch/riscv/lib/uaccess.S @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include From 1ec9f381e84877085ed4ce891e89172f8494ddb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Wed, 20 Dec 2023 16:57:17 +0100 Subject: [PATCH 301/336] riscv: add ISA extension parsing for Ztso MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support to parse the Ztso string in the riscv,isa string. The bindings already supports it but not the ISA parsing code. Signed-off-by: Clément Léger Link: https://lore.kernel.org/r/20231220155723.684081-2-cleger@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/hwcap.h | 1 + arch/riscv/kernel/cpufeature.c | 1 + 2 files changed, 2 insertions(+) diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index 2438d4685da6..3b31efe2f716 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -84,6 +84,7 @@ #define RISCV_ISA_EXT_ZVFH 69 #define RISCV_ISA_EXT_ZVFHMIN 70 #define RISCV_ISA_EXT_ZFA 71 +#define RISCV_ISA_EXT_ZTSO 72 #define RISCV_ISA_EXT_MAX 128 #define RISCV_ISA_EXT_INVALID U32_MAX diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index dc0ab3e97cd2..3eb48a0eecb3 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -279,6 +279,7 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(zkt, RISCV_ISA_EXT_ZKT), __RISCV_ISA_EXT_DATA(zksed, RISCV_ISA_EXT_ZKSED), __RISCV_ISA_EXT_DATA(zksh, RISCV_ISA_EXT_ZKSH), + __RISCV_ISA_EXT_DATA(ztso, RISCV_ISA_EXT_ZTSO), __RISCV_ISA_EXT_SUPERSET(zvbb, RISCV_ISA_EXT_ZVBB, riscv_zvbb_exts), __RISCV_ISA_EXT_DATA(zvbc, RISCV_ISA_EXT_ZVBC), __RISCV_ISA_EXT_DATA(zvfh, RISCV_ISA_EXT_ZVFH), From 5b4d64a819c05e7e96e7ab3f00f4f0967246905f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Wed, 20 Dec 2023 16:57:18 +0100 Subject: [PATCH 302/336] riscv: hwprobe: export Ztso ISA extension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Export the Ztso extension to userspace. Signed-off-by: Clément Léger Link: https://lore.kernel.org/r/20231220155723.684081-3-cleger@rivosinc.com Signed-off-by: Palmer Dabbelt --- Documentation/arch/riscv/hwprobe.rst | 4 ++++ arch/riscv/include/uapi/asm/hwprobe.h | 1 + arch/riscv/kernel/sys_riscv.c | 1 + 3 files changed, 6 insertions(+) diff --git a/Documentation/arch/riscv/hwprobe.rst b/Documentation/arch/riscv/hwprobe.rst index 41463b932268..10bd7b170118 100644 --- a/Documentation/arch/riscv/hwprobe.rst +++ b/Documentation/arch/riscv/hwprobe.rst @@ -161,6 +161,10 @@ The following keys are defined: defined in the RISC-V ISA manual starting from commit 056b6ff467c7 ("Zfa is ratified"). + * :c:macro:`RISCV_HWPROBE_EXT_ZTSO`: The Ztso extension is supported as + defined in the RISC-V ISA manual starting from commit 5618fb5a216b + ("Ztso is now ratified.") + * :c:macro:`RISCV_HWPROBE_KEY_CPUPERF_0`: A bitmask that contains performance information about the selected set of processors. diff --git a/arch/riscv/include/uapi/asm/hwprobe.h b/arch/riscv/include/uapi/asm/hwprobe.h index 91fbe1a7f2e2..01ac3dc196e5 100644 --- a/arch/riscv/include/uapi/asm/hwprobe.h +++ b/arch/riscv/include/uapi/asm/hwprobe.h @@ -56,6 +56,7 @@ struct riscv_hwprobe { #define RISCV_HWPROBE_EXT_ZVFH (1 << 30) #define RISCV_HWPROBE_EXT_ZVFHMIN (1 << 31) #define RISCV_HWPROBE_EXT_ZFA (1ULL << 32) +#define RISCV_HWPROBE_EXT_ZTSO (1ULL << 33) #define RISCV_HWPROBE_KEY_CPUPERF_0 5 #define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0) #define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0) diff --git a/arch/riscv/kernel/sys_riscv.c b/arch/riscv/kernel/sys_riscv.c index f0bd7b480b7f..6564fa9e7a7f 100644 --- a/arch/riscv/kernel/sys_riscv.c +++ b/arch/riscv/kernel/sys_riscv.c @@ -174,6 +174,7 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, EXT_KEY(ZKSH); EXT_KEY(ZKT); EXT_KEY(ZIHINTNTL); + EXT_KEY(ZTSO); if (has_vector()) { EXT_KEY(ZVBB); From cd7be4d02f418d5d62bcaf26e5c44ceb4ce00029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Wed, 20 Dec 2023 16:57:19 +0100 Subject: [PATCH 303/336] dt-bindings: riscv: add Zacas ISA extension description MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add description for the Zacas ISA extension which was ratified recently. Signed-off-by: Clément Léger Acked-by: Conor Dooley Link: https://lore.kernel.org/r/20231220155723.684081-4-cleger@rivosinc.com Signed-off-by: Palmer Dabbelt --- Documentation/devicetree/bindings/riscv/extensions.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Documentation/devicetree/bindings/riscv/extensions.yaml b/Documentation/devicetree/bindings/riscv/extensions.yaml index 3574a0b70be4..27beedb98198 100644 --- a/Documentation/devicetree/bindings/riscv/extensions.yaml +++ b/Documentation/devicetree/bindings/riscv/extensions.yaml @@ -171,6 +171,12 @@ properties: memory types as ratified in the 20191213 version of the privileged ISA specification. + - const: zacas + description: | + The Zacas extension for Atomic Compare-and-Swap (CAS) instructions + is supported as ratified at commit 5059e0ca641c ("update to + ratified") of the riscv-zacas. + - const: zba description: | The standard Zba bit-manipulation extension for address generation From 188a2122c8274b64a739544b1bcfd30e30aed5dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Wed, 20 Dec 2023 16:57:20 +0100 Subject: [PATCH 304/336] riscv: add ISA extension parsing for Zacas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add parsing for Zacas ISA extension which was ratified recently in the riscv-zacas manual. Signed-off-by: Clément Léger Link: https://lore.kernel.org/r/20231220155723.684081-5-cleger@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/hwcap.h | 1 + arch/riscv/kernel/cpufeature.c | 1 + 2 files changed, 2 insertions(+) diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index 3b31efe2f716..34f86424d743 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -85,6 +85,7 @@ #define RISCV_ISA_EXT_ZVFHMIN 70 #define RISCV_ISA_EXT_ZFA 71 #define RISCV_ISA_EXT_ZTSO 72 +#define RISCV_ISA_EXT_ZACAS 73 #define RISCV_ISA_EXT_MAX 128 #define RISCV_ISA_EXT_INVALID U32_MAX diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 3eb48a0eecb3..9a9d915b5bb2 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -259,6 +259,7 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(zihintntl, RISCV_ISA_EXT_ZIHINTNTL), __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE), __RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM), + __RISCV_ISA_EXT_DATA(zacas, RISCV_ISA_EXT_ZACAS), __RISCV_ISA_EXT_DATA(zfa, RISCV_ISA_EXT_ZFA), __RISCV_ISA_EXT_DATA(zfh, RISCV_ISA_EXT_ZFH), __RISCV_ISA_EXT_DATA(zfhmin, RISCV_ISA_EXT_ZFHMIN), From 154a3706122978eeb34d8223d49285ed4f3c61fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Wed, 20 Dec 2023 16:57:21 +0100 Subject: [PATCH 305/336] riscv: hwprobe: export Zacas ISA extension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Export Zacas ISA extension through hwprobe. Signed-off-by: Clément Léger Link: https://lore.kernel.org/r/20231220155723.684081-6-cleger@rivosinc.com Signed-off-by: Palmer Dabbelt --- Documentation/arch/riscv/hwprobe.rst | 4 ++++ arch/riscv/include/uapi/asm/hwprobe.h | 1 + arch/riscv/kernel/sys_riscv.c | 1 + 3 files changed, 6 insertions(+) diff --git a/Documentation/arch/riscv/hwprobe.rst b/Documentation/arch/riscv/hwprobe.rst index 10bd7b170118..bff68004ad43 100644 --- a/Documentation/arch/riscv/hwprobe.rst +++ b/Documentation/arch/riscv/hwprobe.rst @@ -165,6 +165,10 @@ The following keys are defined: defined in the RISC-V ISA manual starting from commit 5618fb5a216b ("Ztso is now ratified.") + * :c:macro:`RISCV_HWPROBE_EXT_ZACAS`: The Zacas extension is supported as + defined in the Atomic Compare-and-Swap (CAS) instructions manual starting + from commit 5059e0ca641c ("update to ratified"). + * :c:macro:`RISCV_HWPROBE_KEY_CPUPERF_0`: A bitmask that contains performance information about the selected set of processors. diff --git a/arch/riscv/include/uapi/asm/hwprobe.h b/arch/riscv/include/uapi/asm/hwprobe.h index 01ac3dc196e5..ac65bb43c8e7 100644 --- a/arch/riscv/include/uapi/asm/hwprobe.h +++ b/arch/riscv/include/uapi/asm/hwprobe.h @@ -57,6 +57,7 @@ struct riscv_hwprobe { #define RISCV_HWPROBE_EXT_ZVFHMIN (1 << 31) #define RISCV_HWPROBE_EXT_ZFA (1ULL << 32) #define RISCV_HWPROBE_EXT_ZTSO (1ULL << 33) +#define RISCV_HWPROBE_EXT_ZACAS (1ULL << 34) #define RISCV_HWPROBE_KEY_CPUPERF_0 5 #define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0) #define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0) diff --git a/arch/riscv/kernel/sys_riscv.c b/arch/riscv/kernel/sys_riscv.c index 6564fa9e7a7f..6c680c75ac0d 100644 --- a/arch/riscv/kernel/sys_riscv.c +++ b/arch/riscv/kernel/sys_riscv.c @@ -175,6 +175,7 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, EXT_KEY(ZKT); EXT_KEY(ZIHINTNTL); EXT_KEY(ZTSO); + EXT_KEY(ZACAS); if (has_vector()) { EXT_KEY(ZVBB); From 3359866b40a97038405c72e68097a92a4a9caa71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Wed, 20 Dec 2023 16:57:22 +0100 Subject: [PATCH 306/336] riscv: hwprobe: export Zicond extension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Export the zicond extension to userspace using hwprobe. Signed-off-by: Clément Léger Link: https://lore.kernel.org/r/20231220155723.684081-7-cleger@rivosinc.com Signed-off-by: Palmer Dabbelt --- Documentation/arch/riscv/hwprobe.rst | 5 +++++ arch/riscv/include/uapi/asm/hwprobe.h | 1 + arch/riscv/kernel/sys_riscv.c | 1 + 3 files changed, 7 insertions(+) diff --git a/Documentation/arch/riscv/hwprobe.rst b/Documentation/arch/riscv/hwprobe.rst index bff68004ad43..ee320fe7581b 100644 --- a/Documentation/arch/riscv/hwprobe.rst +++ b/Documentation/arch/riscv/hwprobe.rst @@ -169,6 +169,11 @@ The following keys are defined: defined in the Atomic Compare-and-Swap (CAS) instructions manual starting from commit 5059e0ca641c ("update to ratified"). + * :c:macro:`RISCV_HWPROBE_EXT_ZICOND`: The Zicond extension is supported as + defined in the RISC-V Integer Conditional (Zicond) operations extension + manual starting from commit 95cf1f9 ("Add changes requested by Ved + during signoff") + * :c:macro:`RISCV_HWPROBE_KEY_CPUPERF_0`: A bitmask that contains performance information about the selected set of processors. diff --git a/arch/riscv/include/uapi/asm/hwprobe.h b/arch/riscv/include/uapi/asm/hwprobe.h index ac65bb43c8e7..fd7af0dddb12 100644 --- a/arch/riscv/include/uapi/asm/hwprobe.h +++ b/arch/riscv/include/uapi/asm/hwprobe.h @@ -58,6 +58,7 @@ struct riscv_hwprobe { #define RISCV_HWPROBE_EXT_ZFA (1ULL << 32) #define RISCV_HWPROBE_EXT_ZTSO (1ULL << 33) #define RISCV_HWPROBE_EXT_ZACAS (1ULL << 34) +#define RISCV_HWPROBE_EXT_ZICOND (1ULL << 35) #define RISCV_HWPROBE_KEY_CPUPERF_0 5 #define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0) #define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0) diff --git a/arch/riscv/kernel/sys_riscv.c b/arch/riscv/kernel/sys_riscv.c index 6c680c75ac0d..cca9b1e35647 100644 --- a/arch/riscv/kernel/sys_riscv.c +++ b/arch/riscv/kernel/sys_riscv.c @@ -176,6 +176,7 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, EXT_KEY(ZIHINTNTL); EXT_KEY(ZTSO); EXT_KEY(ZACAS); + EXT_KEY(ZICOND); if (has_vector()) { EXT_KEY(ZVBB); From 2d2db7d40254d5fb53b11ebd703cd1ed0c5de7a1 Mon Sep 17 00:00:00 2001 From: Oleksandr Tyshchenko Date: Sun, 7 Jan 2024 12:34:26 +0200 Subject: [PATCH 307/336] xen/gntdev: Fix the abuse of underlying struct page in DMA-buf import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DO NOT access the underlying struct page of an sg table exported by DMA-buf in dmabuf_imp_to_refs(), this is not allowed. Please see drivers/dma-buf/dma-buf.c:mangle_sg_table() for details. Fortunately, here (for special Xen device) we can avoid using pages and calculate gfns directly from dma addresses provided by the sg table. Suggested-by: Daniel Vetter Signed-off-by: Oleksandr Tyshchenko Acked-by: Christian König Reviewed-by: Stefano Stabellini Acked-by: Daniel Vetter Link: https://lore.kernel.org/r/20240107103426.2038075-1-olekstysh@gmail.com Signed-off-by: Juergen Gross --- drivers/xen/gntdev-dmabuf.c | 54 ++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/drivers/xen/gntdev-dmabuf.c b/drivers/xen/gntdev-dmabuf.c index 4440e626b797..42adc2c1e06b 100644 --- a/drivers/xen/gntdev-dmabuf.c +++ b/drivers/xen/gntdev-dmabuf.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -50,7 +51,7 @@ struct gntdev_dmabuf { /* Number of pages this buffer has. */ int nr_pages; - /* Pages of this buffer. */ + /* Pages of this buffer (only for dma-buf export). */ struct page **pages; }; @@ -484,7 +485,7 @@ out: /* DMA buffer import support. */ static int -dmabuf_imp_grant_foreign_access(struct page **pages, u32 *refs, +dmabuf_imp_grant_foreign_access(unsigned long *gfns, u32 *refs, int count, int domid) { grant_ref_t priv_gref_head; @@ -507,7 +508,7 @@ dmabuf_imp_grant_foreign_access(struct page **pages, u32 *refs, } gnttab_grant_foreign_access_ref(cur_ref, domid, - xen_page_to_gfn(pages[i]), 0); + gfns[i], 0); refs[i] = cur_ref; } @@ -529,7 +530,6 @@ static void dmabuf_imp_end_foreign_access(u32 *refs, int count) static void dmabuf_imp_free_storage(struct gntdev_dmabuf *gntdev_dmabuf) { - kfree(gntdev_dmabuf->pages); kfree(gntdev_dmabuf->u.imp.refs); kfree(gntdev_dmabuf); } @@ -549,12 +549,6 @@ static struct gntdev_dmabuf *dmabuf_imp_alloc_storage(int count) if (!gntdev_dmabuf->u.imp.refs) goto fail; - gntdev_dmabuf->pages = kcalloc(count, - sizeof(gntdev_dmabuf->pages[0]), - GFP_KERNEL); - if (!gntdev_dmabuf->pages) - goto fail; - gntdev_dmabuf->nr_pages = count; for (i = 0; i < count; i++) @@ -576,7 +570,8 @@ dmabuf_imp_to_refs(struct gntdev_dmabuf_priv *priv, struct device *dev, struct dma_buf *dma_buf; struct dma_buf_attachment *attach; struct sg_table *sgt; - struct sg_page_iter sg_iter; + struct sg_dma_page_iter sg_iter; + unsigned long *gfns; int i; dma_buf = dma_buf_get(fd); @@ -624,26 +619,31 @@ dmabuf_imp_to_refs(struct gntdev_dmabuf_priv *priv, struct device *dev, gntdev_dmabuf->u.imp.sgt = sgt; - /* Now convert sgt to array of pages and check for page validity. */ - i = 0; - for_each_sgtable_page(sgt, &sg_iter, 0) { - struct page *page = sg_page_iter_page(&sg_iter); - /* - * Check if page is valid: this can happen if we are given - * a page from VRAM or other resources which are not backed - * by a struct page. - */ - if (!pfn_valid(page_to_pfn(page))) { - ret = ERR_PTR(-EINVAL); - goto fail_unmap; - } - - gntdev_dmabuf->pages[i++] = page; + gfns = kcalloc(count, sizeof(*gfns), GFP_KERNEL); + if (!gfns) { + ret = ERR_PTR(-ENOMEM); + goto fail_unmap; } - ret = ERR_PTR(dmabuf_imp_grant_foreign_access(gntdev_dmabuf->pages, + /* + * Now convert sgt to array of gfns without accessing underlying pages. + * It is not allowed to access the underlying struct page of an sg table + * exported by DMA-buf, but since we deal with special Xen dma device here + * (not a normal physical one) look at the dma addresses in the sg table + * and then calculate gfns directly from them. + */ + i = 0; + for_each_sgtable_dma_page(sgt, &sg_iter, 0) { + dma_addr_t addr = sg_page_iter_dma_address(&sg_iter); + unsigned long pfn = bfn_to_pfn(XEN_PFN_DOWN(dma_to_phys(dev, addr))); + + gfns[i++] = pfn_to_gfn(pfn); + } + + ret = ERR_PTR(dmabuf_imp_grant_foreign_access(gfns, gntdev_dmabuf->u.imp.refs, count, domid)); + kfree(gfns); if (IS_ERR(ret)) goto fail_end_access; From a4a9779d7642111b4fb6e7415aae9da9783850bd Mon Sep 17 00:00:00 2001 From: Mika Kahola Date: Tue, 2 Jan 2024 13:57:39 +0200 Subject: [PATCH 308/336] drm/i915/display: Fix C20 pll selection for state verification Add pll selection check for C20 as well as clock state verification0. We have been relying on sw state to select A or B pll's. This is incorrect as the hw might see this selection differently. This patch fixes this shortcoming by reading pll selection for both sw and hw states and compares if these two selections match. Fixes: 59be90248b42 ("drm/i915/mtl: C20 state verification") v2: reword commit message and include fix to a original commit (Imre) Compare pll selection (Jani) Signed-off-by: Mika Kahola Reviewed-by: Imre Deak Link: https://patchwork.freedesktop.org/patch/msgid/20240102115741.118525-2-mika.kahola@intel.com (cherry picked from commit f4304beadd88d074333b23fdc7f35d00ee763e14) Signed-off-by: Joonas Lahtinen --- drivers/gpu/drm/i915/display/intel_cx0_phy.c | 25 ++++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_cx0_phy.c b/drivers/gpu/drm/i915/display/intel_cx0_phy.c index 884a1da36089..6b25e195232f 100644 --- a/drivers/gpu/drm/i915/display/intel_cx0_phy.c +++ b/drivers/gpu/drm/i915/display/intel_cx0_phy.c @@ -3067,24 +3067,29 @@ static void intel_c20pll_state_verify(const struct intel_crtc_state *state, { struct drm_i915_private *i915 = to_i915(crtc->base.dev); const struct intel_c20pll_state *mpll_sw_state = &state->cx0pll_state.c20; - bool use_mplla; + bool sw_use_mpllb = mpll_sw_state->tx[0] & C20_PHY_USE_MPLLB; + bool hw_use_mpllb = mpll_hw_state->tx[0] & C20_PHY_USE_MPLLB; int i; - use_mplla = intel_c20_use_mplla(mpll_hw_state->clock); - if (use_mplla) { - for (i = 0; i < ARRAY_SIZE(mpll_sw_state->mplla); i++) { - I915_STATE_WARN(i915, mpll_hw_state->mplla[i] != mpll_sw_state->mplla[i], - "[CRTC:%d:%s] mismatch in C20MPLLA: Register[%d] (expected 0x%04x, found 0x%04x)", - crtc->base.base.id, crtc->base.name, i, - mpll_sw_state->mplla[i], mpll_hw_state->mplla[i]); - } - } else { + I915_STATE_WARN(i915, sw_use_mpllb != hw_use_mpllb, + "[CRTC:%d:%s] mismatch in C20: Register MPLLB selection (expected %d, found %d)", + crtc->base.base.id, crtc->base.name, + sw_use_mpllb, hw_use_mpllb); + + if (hw_use_mpllb) { for (i = 0; i < ARRAY_SIZE(mpll_sw_state->mpllb); i++) { I915_STATE_WARN(i915, mpll_hw_state->mpllb[i] != mpll_sw_state->mpllb[i], "[CRTC:%d:%s] mismatch in C20MPLLB: Register[%d] (expected 0x%04x, found 0x%04x)", crtc->base.base.id, crtc->base.name, i, mpll_sw_state->mpllb[i], mpll_hw_state->mpllb[i]); } + } else { + for (i = 0; i < ARRAY_SIZE(mpll_sw_state->mplla); i++) { + I915_STATE_WARN(i915, mpll_hw_state->mplla[i] != mpll_sw_state->mplla[i], + "[CRTC:%d:%s] mismatch in C20MPLLA: Register[%d] (expected 0x%04x, found 0x%04x)", + crtc->base.base.id, crtc->base.name, i, + mpll_sw_state->mplla[i], mpll_hw_state->mplla[i]); + } } for (i = 0; i < ARRAY_SIZE(mpll_sw_state->tx); i++) { From ae8986e681e9c26fb6c140ae1ed41e6d74d38fc4 Mon Sep 17 00:00:00 2001 From: Imre Deak Date: Wed, 3 Jan 2024 17:26:09 +0200 Subject: [PATCH 309/336] drm/i915/dp: Fix the PSR debugfs entries wrt. MST connectors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MST connectors don't have a static attached encoder, as their encoder can change depending on the pipe they use; so the encoder for an MST connector can't be retrieved using intel_dp_attached_encoder() (which may return NULL for MST). Most of the PSR debugfs entries depend on a static connector -> encoder mapping which is only true for eDP and SST DP connectors and not for MST. These debugfs entries were enabled for MST connectors as well recently to provide PR information for them, but handling MST connectors needs more changes. Fix this by not adding for now the PSR entries on MST connectors. To make things more uniform add the entries for SST connectors on all platforms, not just on platforms supporting DP2.0. v2: - Keep adding the entries for SST connectors. (Jouni) - Add a TODO: comment for MST support. Fixes: ef75c25e8fed ("drm/i915/panelreplay: Debugfs support for panel replay") Closes: https://gitlab.freedesktop.org/drm/intel/-/issues/9850 Cc: Animesh Manna Cc: Jouni Högander Reviewed-by: Jouni Högander Signed-off-by: Imre Deak Link: https://patchwork.freedesktop.org/patch/msgid/20240103152609.2434100-1-imre.deak@intel.com (cherry picked from commit 9b0b61c5bc08e1aa55a0c1e7cda28f952b2d02cc) Signed-off-by: Joonas Lahtinen --- drivers/gpu/drm/i915/display/intel_psr.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c index b6e2e70e1290..8f702c3fc62d 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.c +++ b/drivers/gpu/drm/i915/display/intel_psr.c @@ -3319,11 +3319,11 @@ void intel_psr_connector_debugfs_add(struct intel_connector *connector) struct drm_i915_private *i915 = to_i915(connector->base.dev); struct dentry *root = connector->base.debugfs_entry; - if (connector->base.connector_type != DRM_MODE_CONNECTOR_eDP) { - if (!(HAS_DP20(i915) && - connector->base.connector_type == DRM_MODE_CONNECTOR_DisplayPort)) - return; - } + /* TODO: Add support for MST connectors as well. */ + if ((connector->base.connector_type != DRM_MODE_CONNECTOR_eDP && + connector->base.connector_type != DRM_MODE_CONNECTOR_DisplayPort) || + connector->mst_port) + return; debugfs_create_file("i915_psr_sink_status", 0444, root, connector, &i915_psr_sink_status_fops); From 11809687954ab2a073ec5a4bafd8281a42ff407a Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Thu, 4 Jan 2024 18:46:00 +0200 Subject: [PATCH 310/336] drm/i915: don't make assumptions about intel_wakeref_t type intel_wakeref_t is supposed to be a mostly opaque cookie to its users. It should only be checked for being non-zero and set to zero. Debug logging its actual value is meaningless. Switch to just debug logging whether the async_put_wakeref is non-zero. The issue dates back to much earlier than commit b49e894c3fd8 ("drm/i915: Replace custom intel runtime_pm tracker with ref_tracker library"), but this is the one that brought about a build failure due to the printf format. Reported-by: Stephen Rothwell Closes: https://lore.kernel.org/r/20240102111222.2db11208@canb.auug.org.au Fixes: b49e894c3fd8 ("drm/i915: Replace custom intel runtime_pm tracker with ref_tracker library") Cc: Andrzej Hajda Cc: Imre Deak Signed-off-by: Jani Nikula Reviewed-by: Imre Deak Reviewed-by: Andrzej Hajda Reviewed-by: Andi Shyti Link: https://patchwork.freedesktop.org/patch/msgid/20240104164600.783371-1-jani.nikula@intel.com (cherry picked from commit de06b42edc5bf05aefbb7e2f59475d6022ed57e1) Signed-off-by: Joonas Lahtinen --- drivers/gpu/drm/i915/display/intel_display_power.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_display_power.c b/drivers/gpu/drm/i915/display/intel_display_power.c index 5f091502719b..6fd4fa52253a 100644 --- a/drivers/gpu/drm/i915/display/intel_display_power.c +++ b/drivers/gpu/drm/i915/display/intel_display_power.c @@ -405,8 +405,8 @@ print_async_put_domains_state(struct i915_power_domains *power_domains) struct drm_i915_private, display.power.domains); - drm_dbg(&i915->drm, "async_put_wakeref %lu\n", - power_domains->async_put_wakeref); + drm_dbg(&i915->drm, "async_put_wakeref: %s\n", + str_yes_no(power_domains->async_put_wakeref)); print_power_domains(power_domains, "async_put_domains[0]", &power_domains->async_put_domains[0]); From 584ebbefd12296c6bad009c8a0c9e610eb8283c8 Mon Sep 17 00:00:00 2001 From: Ankit Nautiyal Date: Wed, 13 Dec 2023 14:46:29 +0530 Subject: [PATCH 311/336] drm/i915/dp: Fix the max DSC bpc supported by source Use correct helper for getting max DSC bpc supported by the source. Fixes: 1c56e9a39833 ("drm/i915/dp: Get optimal link config to have best compressed bpp") Cc: Ankit Nautiyal Cc: Stanislav Lisovskiy Cc: Jani Nikula Signed-off-by: Ankit Nautiyal Reviewed-by: Swati Sharma Link: https://patchwork.freedesktop.org/patch/msgid/20231213091632.431557-3-ankit.k.nautiyal@intel.com (cherry picked from commit cd7b0b2dd3d9fecc6057c07b40e8087db2f9f71a) Signed-off-by: Joonas Lahtinen --- drivers/gpu/drm/i915/display/intel_dp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c index 3b2482bf683f..c3b906ebe542 100644 --- a/drivers/gpu/drm/i915/display/intel_dp.c +++ b/drivers/gpu/drm/i915/display/intel_dp.c @@ -2101,7 +2101,7 @@ static int intel_dp_dsc_compute_pipe_bpp(struct intel_dp *intel_dp, } } - dsc_max_bpc = intel_dp_dsc_min_src_input_bpc(i915); + dsc_max_bpc = intel_dp_dsc_max_src_input_bpc(i915); if (!dsc_max_bpc) return -EINVAL; From 30e18a89fb1f84718a174bc02807bd9a590e2bd0 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Tue, 26 Dec 2023 11:54:29 -0800 Subject: [PATCH 312/336] drm/i915/gem: reconcile Excess struct member kernel-doc warnings Document nested struct members with full names as described in Documentation/doc-guide/kernel-doc.rst. i915_gem_context_types.h:420: warning: Excess struct member 'lock' description in 'i915_gem_context' Signed-off-by: Randy Dunlap Cc: Jani Nikula Cc: Joonas Lahtinen Cc: Rodrigo Vivi Cc: Tvrtko Ursulin Cc: intel-gfx@lists.freedesktop.org Cc: Jonathan Corbet Cc: dri-devel@lists.freedesktop.org Reviewed-by: Rodrigo Vivi Reviewed-by: Andi Shyti Signed-off-by: Andi Shyti Link: https://patchwork.freedesktop.org/patch/msgid/20231226195432.10891-1-rdunlap@infradead.org (cherry picked from commit 7353c3d7c15017140a8b984e41f94be0bf535e73) Signed-off-by: Joonas Lahtinen --- drivers/gpu/drm/i915/gem/i915_gem_context_types.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context_types.h b/drivers/gpu/drm/i915/gem/i915_gem_context_types.h index c573c067779f..03bc7f9d191b 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_context_types.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_context_types.h @@ -412,9 +412,9 @@ struct i915_gem_context { /** @stale: tracks stale engines to be destroyed */ struct { - /** @lock: guards engines */ + /** @stale.lock: guards engines */ spinlock_t lock; - /** @engines: list of stale engines */ + /** @stale.engines: list of stale engines */ struct list_head engines; } stale; }; From 53cd65a9c95109eef402db0ed7822b7c9a8ad732 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Thu, 28 Dec 2023 15:49:46 -0800 Subject: [PATCH 313/336] drm/i915/gt: reconcile Excess struct member kernel-doc warnings Document nested struct members with full names as described in Documentation/doc-guide/kernel-doc.rst. intel_gsc.h:34: warning: Excess struct member 'gem_obj' description in 'intel_gsc' Also add missing field member descriptions. Signed-off-by: Randy Dunlap Cc: Jani Nikula Cc: Joonas Lahtinen Cc: Rodrigo Vivi Cc: Tvrtko Ursulin Cc: intel-gfx@lists.freedesktop.org Cc: Jonathan Corbet Cc: dri-devel@lists.freedesktop.org Reviewed-by: Rodrigo Vivi Cc: Andi Shyti Reviewed-by: Andi Shyti Signed-off-by: Andi Shyti Link: https://patchwork.freedesktop.org/patch/msgid/20231228234946.12405-1-rdunlap@infradead.org (cherry picked from commit cd1d91115ff1929ec346d85f512ef260ddf8131e) Signed-off-by: Joonas Lahtinen --- drivers/gpu/drm/i915/gt/intel_gsc.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/i915/gt/intel_gsc.h b/drivers/gpu/drm/i915/gt/intel_gsc.h index 7ab3ca0f9f26..013c64251448 100644 --- a/drivers/gpu/drm/i915/gt/intel_gsc.h +++ b/drivers/gpu/drm/i915/gt/intel_gsc.h @@ -21,8 +21,11 @@ struct mei_aux_device; /** * struct intel_gsc - graphics security controller * - * @gem_obj: scratch memory GSC operations - * @intf : gsc interface + * @intf: gsc interface + * @intf.adev: MEI aux. device for this @intf + * @intf.gem_obj: scratch memory GSC operations + * @intf.irq: IRQ for this device (%-1 for no IRQ) + * @intf.id: this interface's id number/index */ struct intel_gsc { struct intel_gsc_intf { From af3cfcad492f2ffbef5de36c8ee1e8f8a701938f Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Tue, 26 Dec 2023 11:54:31 -0800 Subject: [PATCH 314/336] drm/i915/guc: reconcile Excess struct member kernel-doc warnings Document nested struct members with full names as described in Documentation/doc-guide/kernel-doc.rst. intel_guc.h:305: warning: Excess struct member 'lock' description in 'intel_guc' intel_guc.h:305: warning: Excess struct member 'guc_ids' description in 'intel_guc' intel_guc.h:305: warning: Excess struct member 'num_guc_ids' description in 'intel_guc' intel_guc.h:305: warning: Excess struct member 'guc_ids_bitmap' description in 'intel_guc' intel_guc.h:305: warning: Excess struct member 'guc_id_list' description in 'intel_guc' intel_guc.h:305: warning: Excess struct member 'guc_ids_in_use' description in 'intel_guc' intel_guc.h:305: warning: Excess struct member 'destroyed_contexts' description in 'intel_guc' intel_guc.h:305: warning: Excess struct member 'destroyed_worker' description in 'intel_guc' intel_guc.h:305: warning: Excess struct member 'reset_fail_worker' description in 'intel_guc' intel_guc.h:305: warning: Excess struct member 'reset_fail_mask' description in 'intel_guc' intel_guc.h:305: warning: Excess struct member 'sched_disable_delay_ms' description in 'intel_guc' intel_guc.h:305: warning: Excess struct member 'sched_disable_gucid_threshold' description in 'intel_guc' intel_guc.h:305: warning: Excess struct member 'lock' description in 'intel_guc' intel_guc.h:305: warning: Excess struct member 'gt_stamp' description in 'intel_guc' intel_guc.h:305: warning: Excess struct member 'ping_delay' description in 'intel_guc' intel_guc.h:305: warning: Excess struct member 'work' description in 'intel_guc' intel_guc.h:305: warning: Excess struct member 'shift' description in 'intel_guc' intel_guc.h:305: warning: Excess struct member 'last_stat_jiffies' description in 'intel_guc' 18 warnings as Errors Signed-off-by: Randy Dunlap Cc: Jani Nikula Cc: Joonas Lahtinen Cc: Rodrigo Vivi Cc: Tvrtko Ursulin Cc: intel-gfx@lists.freedesktop.org Cc: Jonathan Corbet Cc: dri-devel@lists.freedesktop.org Reviewed-by: Rodrigo Vivi Reviewed-by: Andi Shyti Signed-off-by: Andi Shyti Link: https://patchwork.freedesktop.org/patch/msgid/20231226195432.10891-3-rdunlap@infradead.org (cherry picked from commit e4cf1a70fad3e2107503e99cfe9cc0c9cba19dad) Signed-off-by: Joonas Lahtinen --- drivers/gpu/drm/i915/gt/uc/intel_guc.h | 75 ++++++++++++++------------ 1 file changed, 42 insertions(+), 33 deletions(-) diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.h b/drivers/gpu/drm/i915/gt/uc/intel_guc.h index e22c12ce245a..813cc888e6fa 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.h @@ -105,61 +105,67 @@ struct intel_guc { */ struct { /** - * @lock: protects everything in submission_state, - * ce->guc_id.id, and ce->guc_id.ref when transitioning in and - * out of zero + * @submission_state.lock: protects everything in + * submission_state, ce->guc_id.id, and ce->guc_id.ref + * when transitioning in and out of zero */ spinlock_t lock; /** - * @guc_ids: used to allocate new guc_ids, single-lrc + * @submission_state.guc_ids: used to allocate new + * guc_ids, single-lrc */ struct ida guc_ids; /** - * @num_guc_ids: Number of guc_ids, selftest feature to be able - * to reduce this number while testing. + * @submission_state.num_guc_ids: Number of guc_ids, selftest + * feature to be able to reduce this number while testing. */ int num_guc_ids; /** - * @guc_ids_bitmap: used to allocate new guc_ids, multi-lrc + * @submission_state.guc_ids_bitmap: used to allocate + * new guc_ids, multi-lrc */ unsigned long *guc_ids_bitmap; /** - * @guc_id_list: list of intel_context with valid guc_ids but no - * refs + * @submission_state.guc_id_list: list of intel_context + * with valid guc_ids but no refs */ struct list_head guc_id_list; /** - * @guc_ids_in_use: Number single-lrc guc_ids in use + * @submission_state.guc_ids_in_use: Number single-lrc + * guc_ids in use */ unsigned int guc_ids_in_use; /** - * @destroyed_contexts: list of contexts waiting to be destroyed - * (deregistered with the GuC) + * @submission_state.destroyed_contexts: list of contexts + * waiting to be destroyed (deregistered with the GuC) */ struct list_head destroyed_contexts; /** - * @destroyed_worker: worker to deregister contexts, need as we - * need to take a GT PM reference and can't from destroy - * function as it might be in an atomic context (no sleeping) + * @submission_state.destroyed_worker: worker to deregister + * contexts, need as we need to take a GT PM reference and + * can't from destroy function as it might be in an atomic + * context (no sleeping) */ struct work_struct destroyed_worker; /** - * @reset_fail_worker: worker to trigger a GT reset after an - * engine reset fails + * @submission_state.reset_fail_worker: worker to trigger + * a GT reset after an engine reset fails */ struct work_struct reset_fail_worker; /** - * @reset_fail_mask: mask of engines that failed to reset + * @submission_state.reset_fail_mask: mask of engines that + * failed to reset */ intel_engine_mask_t reset_fail_mask; /** - * @sched_disable_delay_ms: schedule disable delay, in ms, for - * contexts + * @submission_state.sched_disable_delay_ms: schedule + * disable delay, in ms, for contexts */ unsigned int sched_disable_delay_ms; /** - * @sched_disable_gucid_threshold: threshold of min remaining available - * guc_ids before we start bypassing the schedule disable delay + * @submission_state.sched_disable_gucid_threshold: + * threshold of min remaining available guc_ids before + * we start bypassing the schedule disable delay */ unsigned int sched_disable_gucid_threshold; } submission_state; @@ -243,37 +249,40 @@ struct intel_guc { */ struct { /** - * @lock: Lock protecting the below fields and the engine stats. + * @timestamp.lock: Lock protecting the below fields and + * the engine stats. */ spinlock_t lock; /** - * @gt_stamp: 64 bit extended value of the GT timestamp. + * @timestamp.gt_stamp: 64-bit extended value of the GT + * timestamp. */ u64 gt_stamp; /** - * @ping_delay: Period for polling the GT timestamp for - * overflow. + * @timestamp.ping_delay: Period for polling the GT + * timestamp for overflow. */ unsigned long ping_delay; /** - * @work: Periodic work to adjust GT timestamp, engine and - * context usage for overflows. + * @timestamp.work: Periodic work to adjust GT timestamp, + * engine and context usage for overflows. */ struct delayed_work work; /** - * @shift: Right shift value for the gpm timestamp + * @timestamp.shift: Right shift value for the gpm timestamp */ u32 shift; /** - * @last_stat_jiffies: jiffies at last actual stats collection time - * We use this timestamp to ensure we don't oversample the - * stats because runtime power management events can trigger - * stats collection at much higher rates than required. + * @timestamp.last_stat_jiffies: jiffies at last actual + * stats collection time. We use this timestamp to ensure + * we don't oversample the stats because runtime power + * management events can trigger stats collection at much + * higher rates than required. */ unsigned long last_stat_jiffies; } timestamp; From d505a16e00c35919fd9fe5735894645e0f70a415 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Tue, 26 Dec 2023 11:54:32 -0800 Subject: [PATCH 315/336] drm/i915/perf: reconcile Excess struct member kernel-doc warnings Document nested struct members with full names as described in Documentation/doc-guide/kernel-doc.rst. i915_perf_types.h:341: warning: Excess struct member 'ptr_lock' description in 'i915_perf_stream' i915_perf_types.h:341: warning: Excess struct member 'head' description in 'i915_perf_stream' i915_perf_types.h:341: warning: Excess struct member 'tail' description in 'i915_perf_stream' 3 warnings as Errors Signed-off-by: Randy Dunlap Cc: Jani Nikula Cc: Joonas Lahtinen Cc: Rodrigo Vivi Cc: Tvrtko Ursulin Cc: intel-gfx@lists.freedesktop.org Cc: Jonathan Corbet Cc: dri-devel@lists.freedesktop.org Reviewed-by: Rodrigo Vivi Reviewed-by: Andi Shyti Signed-off-by: Andi Shyti Link: https://patchwork.freedesktop.org/patch/msgid/20231226195432.10891-4-rdunlap@infradead.org (cherry picked from commit aa253baca534357e033bd29b074ce1eade2a9362) Signed-off-by: Joonas Lahtinen --- drivers/gpu/drm/i915/i915_perf_types.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/i915/i915_perf_types.h b/drivers/gpu/drm/i915/i915_perf_types.h index 13b1ae9b96c7..46445248d193 100644 --- a/drivers/gpu/drm/i915/i915_perf_types.h +++ b/drivers/gpu/drm/i915/i915_perf_types.h @@ -291,7 +291,8 @@ struct i915_perf_stream { int size_exponent; /** - * @ptr_lock: Locks reads and writes to all head/tail state + * @oa_buffer.ptr_lock: Locks reads and writes to all + * head/tail state * * Consider: the head and tail pointer state needs to be read * consistently from a hrtimer callback (atomic context) and @@ -313,7 +314,8 @@ struct i915_perf_stream { spinlock_t ptr_lock; /** - * @head: Although we can always read back the head pointer register, + * @oa_buffer.head: Although we can always read back + * the head pointer register, * we prefer to avoid trusting the HW state, just to avoid any * risk that some hardware condition could * somehow bump the * head pointer unpredictably and cause us to forward the wrong @@ -322,7 +324,8 @@ struct i915_perf_stream { u32 head; /** - * @tail: The last verified tail that can be read by userspace. + * @oa_buffer.tail: The last verified tail that can be + * read by userspace. */ u32 tail; } oa_buffer; From bde4f5ff8295601554601f78a523d4d97e42433e Mon Sep 17 00:00:00 2001 From: Srinivas Pandruvada Date: Tue, 9 Jan 2024 09:48:48 -0800 Subject: [PATCH 316/336] cpufreq: intel_pstate: Update hybrid scaling factor for Meteor Lake On some Meteor Lake platforms, maximum one core turbo frequency is not observed. During hybrid performance to frequency conversion, the maximum frequency is 100 MHz less. This results in requesting maximum frequency 100 MHz less. For example when the max one core turbo is 4.9 GHz: MSR HWP_CAPABILITIES shows highest performance ratio for P-core is 0x3E. With the current scaling factor of 78741 (1.27x for converting frequency to performance) results in max frequency of 4.8 GHz. This results in capping the max scaling frequency as 4.8 GHz, which is 100 MHz less than the desired. Add capability to define per CPU model specific scaling factor and define scaling factor of 80000 (1.25x for converting frequency to performance for P-cores) for Meteor Lake. Signed-off-by: Srinivas Pandruvada [ rjw: Debug message adjustment, subject edit ] Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 3c69040920b8..2ca70b0b5fdc 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -302,7 +302,10 @@ static bool hwp_forced __read_mostly; static struct cpufreq_driver *intel_pstate_driver __read_mostly; -#define HYBRID_SCALING_FACTOR 78741 +#define HYBRID_SCALING_FACTOR 78741 +#define HYBRID_SCALING_FACTOR_MTL 80000 + +static int hybrid_scaling_factor = HYBRID_SCALING_FACTOR; static inline int core_get_scaling(void) { @@ -422,7 +425,7 @@ static int intel_pstate_cppc_get_scaling(int cpu) */ if (!ret && cppc_perf.nominal_perf && cppc_perf.nominal_freq && cppc_perf.nominal_perf * 100 != cppc_perf.nominal_freq) - return HYBRID_SCALING_FACTOR; + return hybrid_scaling_factor; return core_get_scaling(); } @@ -1968,7 +1971,7 @@ static int hwp_get_cpu_scaling(int cpu) smp_call_function_single(cpu, hybrid_get_type, &cpu_type, 1); /* P-cores have a smaller perf level-to-freqency scaling factor. */ if (cpu_type == 0x40) - return HYBRID_SCALING_FACTOR; + return hybrid_scaling_factor; /* Use default core scaling for E-cores */ if (cpu_type == 0x20) @@ -3399,6 +3402,11 @@ static const struct x86_cpu_id intel_epp_balance_perf[] = { {} }; +static const struct x86_cpu_id intel_hybrid_scaling_factor[] = { + X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE_L, HYBRID_SCALING_FACTOR_MTL), + {} +}; + static int __init intel_pstate_init(void) { static struct cpudata **_all_cpu_data; @@ -3489,9 +3497,16 @@ hwp_cpu_matched: if (hwp_active) { const struct x86_cpu_id *id = x86_match_cpu(intel_epp_balance_perf); + const struct x86_cpu_id *hybrid_id = x86_match_cpu(intel_hybrid_scaling_factor); if (id) epp_values[EPP_INDEX_BALANCE_PERFORMANCE] = id->driver_data; + + if (hybrid_id) { + hybrid_scaling_factor = hybrid_id->driver_data; + pr_debug("hybrid scaling factor: %d\n", hybrid_scaling_factor); + } + } mutex_lock(&intel_pstate_driver_lock); From 03c305861c70d6db898dd2379b882e7772a5c5d0 Mon Sep 17 00:00:00 2001 From: Erwan Velu Date: Tue, 9 Jan 2024 18:57:53 +0100 Subject: [PATCH 317/336] Documentation: admin-guide: PM: Fix two typos Fix two typos in the admin-guide: - a missing e in "reference_perf" in cppc_sysfs.rst. - the amd_pstate sysfs path uses a dash instead of an underscore. Signed-off-by: Erwan Velu [ rjw: Subject and changelog edits ] Signed-off-by: Rafael J. Wysocki --- Documentation/admin-guide/acpi/cppc_sysfs.rst | 2 +- Documentation/admin-guide/pm/amd-pstate.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/acpi/cppc_sysfs.rst b/Documentation/admin-guide/acpi/cppc_sysfs.rst index e53d76365aa7..36981c667823 100644 --- a/Documentation/admin-guide/acpi/cppc_sysfs.rst +++ b/Documentation/admin-guide/acpi/cppc_sysfs.rst @@ -75,4 +75,4 @@ taking two different snapshots of feedback counters at time T1 and T2. delivered_counter_delta = fbc_t2[del] - fbc_t1[del] reference_counter_delta = fbc_t2[ref] - fbc_t1[ref] - delivered_perf = (refernce_perf x delivered_counter_delta) / reference_counter_delta + delivered_perf = (reference_perf x delivered_counter_delta) / reference_counter_delta diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst index 1cf40f69278c..9eb26014d34b 100644 --- a/Documentation/admin-guide/pm/amd-pstate.rst +++ b/Documentation/admin-guide/pm/amd-pstate.rst @@ -361,7 +361,7 @@ Global Attributes ``amd-pstate`` exposes several global attributes (files) in ``sysfs`` to control its functionality at the system level. They are located in the -``/sys/devices/system/cpu/amd-pstate/`` directory and affect all CPUs. +``/sys/devices/system/cpu/amd_pstate/`` directory and affect all CPUs. ``status`` Operation mode of the driver: "active", "passive" or "disable". From 3e999770ac1c7c31a70685dd5b88e89473509e9c Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 9 Jan 2024 17:59:22 +0100 Subject: [PATCH 318/336] PM: sleep: Restore asynchronous device resume optimization Before commit 7839d0078e0d ("PM: sleep: Fix possible deadlocks in core system-wide PM code"), the resume of devices that were allowed to resume asynchronously was scheduled before starting the resume of the other devices, so the former did not have to wait for the latter unless functional dependencies were present. Commit 7839d0078e0d removed that optimization in order to address a correctness issue, but it can be restored with the help of a new device power management flag, so do that now. Signed-off-by: Rafael J. Wysocki Reviewed-by: Stanislaw Gruszka --- drivers/base/power/main.c | 117 +++++++++++++++++++++----------------- include/linux/pm.h | 1 + 2 files changed, 65 insertions(+), 53 deletions(-) diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c index 9c5a5f4dba5a..fadcd0379dc2 100644 --- a/drivers/base/power/main.c +++ b/drivers/base/power/main.c @@ -579,7 +579,7 @@ bool dev_pm_skip_resume(struct device *dev) } /** - * __device_resume_noirq - Execute a "noirq resume" callback for given device. + * device_resume_noirq - Execute a "noirq resume" callback for given device. * @dev: Device to handle. * @state: PM transition of the system being carried out. * @async: If true, the device is being resumed asynchronously. @@ -587,7 +587,7 @@ bool dev_pm_skip_resume(struct device *dev) * The driver of @dev will not receive interrupts while this function is being * executed. */ -static void __device_resume_noirq(struct device *dev, pm_message_t state, bool async) +static void device_resume_noirq(struct device *dev, pm_message_t state, bool async) { pm_callback_t callback = NULL; const char *info = NULL; @@ -674,16 +674,22 @@ static bool dpm_async_fn(struct device *dev, async_func_t func) { reinit_completion(&dev->power.completion); - if (!is_async(dev)) - return false; + if (is_async(dev)) { + dev->power.async_in_progress = true; - get_device(dev); + get_device(dev); - if (async_schedule_dev_nocall(func, dev)) - return true; - - put_device(dev); + if (async_schedule_dev_nocall(func, dev)) + return true; + put_device(dev); + } + /* + * Because async_schedule_dev_nocall() above has returned false or it + * has not been called at all, func() is not running and it is safe to + * update the async_in_progress flag without extra synchronization. + */ + dev->power.async_in_progress = false; return false; } @@ -691,18 +697,10 @@ static void async_resume_noirq(void *data, async_cookie_t cookie) { struct device *dev = data; - __device_resume_noirq(dev, pm_transition, true); + device_resume_noirq(dev, pm_transition, true); put_device(dev); } -static void device_resume_noirq(struct device *dev) -{ - if (dpm_async_fn(dev, async_resume_noirq)) - return; - - __device_resume_noirq(dev, pm_transition, false); -} - static void dpm_noirq_resume_devices(pm_message_t state) { struct device *dev; @@ -712,18 +710,28 @@ static void dpm_noirq_resume_devices(pm_message_t state) mutex_lock(&dpm_list_mtx); pm_transition = state; + /* + * Trigger the resume of "async" devices upfront so they don't have to + * wait for the "non-async" ones they don't depend on. + */ + list_for_each_entry(dev, &dpm_noirq_list, power.entry) + dpm_async_fn(dev, async_resume_noirq); + while (!list_empty(&dpm_noirq_list)) { dev = to_device(dpm_noirq_list.next); - get_device(dev); list_move_tail(&dev->power.entry, &dpm_late_early_list); - mutex_unlock(&dpm_list_mtx); + if (!dev->power.async_in_progress) { + get_device(dev); - device_resume_noirq(dev); + mutex_unlock(&dpm_list_mtx); - put_device(dev); + device_resume_noirq(dev, state, false); - mutex_lock(&dpm_list_mtx); + put_device(dev); + + mutex_lock(&dpm_list_mtx); + } } mutex_unlock(&dpm_list_mtx); async_synchronize_full(); @@ -747,14 +755,14 @@ void dpm_resume_noirq(pm_message_t state) } /** - * __device_resume_early - Execute an "early resume" callback for given device. + * device_resume_early - Execute an "early resume" callback for given device. * @dev: Device to handle. * @state: PM transition of the system being carried out. * @async: If true, the device is being resumed asynchronously. * * Runtime PM is disabled for @dev while this function is being executed. */ -static void __device_resume_early(struct device *dev, pm_message_t state, bool async) +static void device_resume_early(struct device *dev, pm_message_t state, bool async) { pm_callback_t callback = NULL; const char *info = NULL; @@ -820,18 +828,10 @@ static void async_resume_early(void *data, async_cookie_t cookie) { struct device *dev = data; - __device_resume_early(dev, pm_transition, true); + device_resume_early(dev, pm_transition, true); put_device(dev); } -static void device_resume_early(struct device *dev) -{ - if (dpm_async_fn(dev, async_resume_early)) - return; - - __device_resume_early(dev, pm_transition, false); -} - /** * dpm_resume_early - Execute "early resume" callbacks for all devices. * @state: PM transition of the system being carried out. @@ -845,18 +845,28 @@ void dpm_resume_early(pm_message_t state) mutex_lock(&dpm_list_mtx); pm_transition = state; + /* + * Trigger the resume of "async" devices upfront so they don't have to + * wait for the "non-async" ones they don't depend on. + */ + list_for_each_entry(dev, &dpm_late_early_list, power.entry) + dpm_async_fn(dev, async_resume_early); + while (!list_empty(&dpm_late_early_list)) { dev = to_device(dpm_late_early_list.next); - get_device(dev); list_move_tail(&dev->power.entry, &dpm_suspended_list); - mutex_unlock(&dpm_list_mtx); + if (!dev->power.async_in_progress) { + get_device(dev); - device_resume_early(dev); + mutex_unlock(&dpm_list_mtx); - put_device(dev); + device_resume_early(dev, state, false); - mutex_lock(&dpm_list_mtx); + put_device(dev); + + mutex_lock(&dpm_list_mtx); + } } mutex_unlock(&dpm_list_mtx); async_synchronize_full(); @@ -876,12 +886,12 @@ void dpm_resume_start(pm_message_t state) EXPORT_SYMBOL_GPL(dpm_resume_start); /** - * __device_resume - Execute "resume" callbacks for given device. + * device_resume - Execute "resume" callbacks for given device. * @dev: Device to handle. * @state: PM transition of the system being carried out. * @async: If true, the device is being resumed asynchronously. */ -static void __device_resume(struct device *dev, pm_message_t state, bool async) +static void device_resume(struct device *dev, pm_message_t state, bool async) { pm_callback_t callback = NULL; const char *info = NULL; @@ -975,18 +985,10 @@ static void async_resume(void *data, async_cookie_t cookie) { struct device *dev = data; - __device_resume(dev, pm_transition, true); + device_resume(dev, pm_transition, true); put_device(dev); } -static void device_resume(struct device *dev) -{ - if (dpm_async_fn(dev, async_resume)) - return; - - __device_resume(dev, pm_transition, false); -} - /** * dpm_resume - Execute "resume" callbacks for non-sysdev devices. * @state: PM transition of the system being carried out. @@ -1006,16 +1008,25 @@ void dpm_resume(pm_message_t state) pm_transition = state; async_error = 0; + /* + * Trigger the resume of "async" devices upfront so they don't have to + * wait for the "non-async" ones they don't depend on. + */ + list_for_each_entry(dev, &dpm_suspended_list, power.entry) + dpm_async_fn(dev, async_resume); + while (!list_empty(&dpm_suspended_list)) { dev = to_device(dpm_suspended_list.next); get_device(dev); - mutex_unlock(&dpm_list_mtx); + if (!dev->power.async_in_progress) { + mutex_unlock(&dpm_list_mtx); - device_resume(dev); + device_resume(dev, state, false); - mutex_lock(&dpm_list_mtx); + mutex_lock(&dpm_list_mtx); + } if (!list_empty(&dev->power.entry)) list_move_tail(&dev->power.entry, &dpm_prepared_list); diff --git a/include/linux/pm.h b/include/linux/pm.h index 92a4f69de0e8..a2f3e53a8196 100644 --- a/include/linux/pm.h +++ b/include/linux/pm.h @@ -681,6 +681,7 @@ struct dev_pm_info { bool wakeup_path:1; bool syscore:1; bool no_pm_callbacks:1; /* Owned by the PM core */ + bool async_in_progress:1; /* Owned by the PM core */ unsigned int must_resume:1; /* Owned by the PM core */ unsigned int may_skip_resume:1; /* Set by subsystems */ #else From 89c4b588d11e9acf01d604de4b0c715884f59213 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Wed, 10 Jan 2024 19:07:36 +0100 Subject: [PATCH 319/336] MIPS: Alchemy: Fix an out-of-bound access in db1200_dev_setup() When calling spi_register_board_info(), we should pass the number of elements in 'db1200_spi_devs', not 'db1200_i2c_devs'. Fixes: 63323ec54a7e ("MIPS: Alchemy: Extended DB1200 board support.") Signed-off-by: Christophe JAILLET Signed-off-by: Thomas Bogendoerfer --- arch/mips/alchemy/devboards/db1200.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/mips/alchemy/devboards/db1200.c b/arch/mips/alchemy/devboards/db1200.c index f521874ebb07..67f067706af2 100644 --- a/arch/mips/alchemy/devboards/db1200.c +++ b/arch/mips/alchemy/devboards/db1200.c @@ -847,7 +847,7 @@ int __init db1200_dev_setup(void) i2c_register_board_info(0, db1200_i2c_devs, ARRAY_SIZE(db1200_i2c_devs)); spi_register_board_info(db1200_spi_devs, - ARRAY_SIZE(db1200_i2c_devs)); + ARRAY_SIZE(db1200_spi_devs)); /* SWITCHES: S6.8 I2C/SPI selector (OFF=I2C ON=SPI) * S6.7 AC97/I2S selector (OFF=AC97 ON=I2S) From 3c1e5abcda64bed0c7bffa65af2316995f269a61 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Wed, 10 Jan 2024 19:09:46 +0100 Subject: [PATCH 320/336] MIPS: Alchemy: Fix an out-of-bound access in db1550_dev_setup() When calling spi_register_board_info(), Fixes: f869d42e580f ("MIPS: Alchemy: Improved DB1550 support, with audio and serial busses.") Signed-off-by: Christophe JAILLET Signed-off-by: Thomas Bogendoerfer --- arch/mips/alchemy/devboards/db1550.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/mips/alchemy/devboards/db1550.c b/arch/mips/alchemy/devboards/db1550.c index fd91d9c9a252..6c6837181f55 100644 --- a/arch/mips/alchemy/devboards/db1550.c +++ b/arch/mips/alchemy/devboards/db1550.c @@ -589,7 +589,7 @@ int __init db1550_dev_setup(void) i2c_register_board_info(0, db1550_i2c_devs, ARRAY_SIZE(db1550_i2c_devs)); spi_register_board_info(db1550_spi_devs, - ARRAY_SIZE(db1550_i2c_devs)); + ARRAY_SIZE(db1550_spi_devs)); c = clk_get(NULL, "psc0_intclk"); if (!IS_ERR(c)) { From 1f4cac0f7465830a17a266983acbd60a2ce7ee6f Mon Sep 17 00:00:00 2001 From: Vegard Nossum Date: Wed, 10 Jan 2024 11:46:46 +0100 Subject: [PATCH 321/336] Documentation: constrain alabaster package to older versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 'alabaster' theme dropped support for Sphinx < v3.4: 0.7.14 – 2024-01-08 * Dropped support for Python 3.8 and earlier. * Dropped support for Sphinx 3.3 and earlier. [...] (Source: https://alabaster.readthedocs.io/en/latest/changelog.html) This manifests as an error when running 'make htmldocs' in a virtualenv constructed from Documentation/sphinx/requirements.txt: Sphinx version error: The alabaster extension used by this project needs at least Sphinx v3.4; it therefore cannot be built with this version. Raising the Sphinx version is not really a good option at this point, since 3.x through 6.x have horrible performance regressions (7.x still does, but not quite as bad). Instead, constrain the 'alabaster' package to versions that still support Sphinx 2.4.4. Signed-off-by: Vegard Nossum Link: https://lore.kernel.org/r/20240110104646.3647600-1-vegard.nossum@oracle.com Signed-off-by: Jonathan Corbet --- Documentation/sphinx/requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/sphinx/requirements.txt b/Documentation/sphinx/requirements.txt index 335b53df35e2..6b0a981dcb2c 100644 --- a/Documentation/sphinx/requirements.txt +++ b/Documentation/sphinx/requirements.txt @@ -1,3 +1,5 @@ # jinja2>=3.1 is not compatible with Sphinx<4.0 jinja2<3.1 +# alabaster>=0.7.14 is not compatible with Sphinx<=3.3 +alabaster<0.7.14 Sphinx==2.4.4 From c48a7c44a1d02516309015b6134c9bb982e17008 Mon Sep 17 00:00:00 2001 From: Vegard Nossum Date: Wed, 10 Jan 2024 18:47:58 +0100 Subject: [PATCH 322/336] docs: kernel_feat.py: fix potential command injection The kernel-feat directive passes its argument straight to the shell. This is unfortunate and unnecessary. Let's always use paths relative to $srctree/Documentation/ and use subprocess.check_call() instead of subprocess.Popen(shell=True). This also makes the code shorter. This is analogous to commit 3231dd586277 ("docs: kernel_abi.py: fix command injection") where we did exactly the same thing for kernel_abi.py, somehow I completely missed this one. Link: https://fosstodon.org/@jani/111676532203641247 Reported-by: Jani Nikula Signed-off-by: Vegard Nossum Cc: stable@vger.kernel.org Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20240110174758.3680506-1-vegard.nossum@oracle.com --- Documentation/admin-guide/features.rst | 2 +- Documentation/arch/arc/features.rst | 2 +- Documentation/arch/arm/features.rst | 2 +- Documentation/arch/arm64/features.rst | 2 +- Documentation/arch/loongarch/features.rst | 2 +- Documentation/arch/m68k/features.rst | 2 +- Documentation/arch/mips/features.rst | 2 +- Documentation/arch/nios2/features.rst | 2 +- Documentation/arch/openrisc/features.rst | 2 +- Documentation/arch/parisc/features.rst | 2 +- Documentation/arch/powerpc/features.rst | 2 +- Documentation/arch/riscv/features.rst | 2 +- Documentation/arch/s390/features.rst | 2 +- Documentation/arch/sh/features.rst | 2 +- Documentation/arch/sparc/features.rst | 2 +- Documentation/arch/x86/features.rst | 2 +- Documentation/arch/xtensa/features.rst | 2 +- Documentation/sphinx/kernel_feat.py | 57 ++++--------------- .../zh_CN/arch/loongarch/features.rst | 2 +- .../translations/zh_CN/arch/mips/features.rst | 2 +- .../zh_TW/arch/loongarch/features.rst | 2 +- .../translations/zh_TW/arch/mips/features.rst | 2 +- 22 files changed, 33 insertions(+), 66 deletions(-) diff --git a/Documentation/admin-guide/features.rst b/Documentation/admin-guide/features.rst index 8c167082a84f..7651eca38227 100644 --- a/Documentation/admin-guide/features.rst +++ b/Documentation/admin-guide/features.rst @@ -1,3 +1,3 @@ .. SPDX-License-Identifier: GPL-2.0 -.. kernel-feat:: $srctree/Documentation/features +.. kernel-feat:: features diff --git a/Documentation/arch/arc/features.rst b/Documentation/arch/arc/features.rst index b793583d688a..49ff446ff744 100644 --- a/Documentation/arch/arc/features.rst +++ b/Documentation/arch/arc/features.rst @@ -1,3 +1,3 @@ .. SPDX-License-Identifier: GPL-2.0 -.. kernel-feat:: $srctree/Documentation/features arc +.. kernel-feat:: features arc diff --git a/Documentation/arch/arm/features.rst b/Documentation/arch/arm/features.rst index 7414ec03dd15..0e76aaf68eca 100644 --- a/Documentation/arch/arm/features.rst +++ b/Documentation/arch/arm/features.rst @@ -1,3 +1,3 @@ .. SPDX-License-Identifier: GPL-2.0 -.. kernel-feat:: $srctree/Documentation/features arm +.. kernel-feat:: features arm diff --git a/Documentation/arch/arm64/features.rst b/Documentation/arch/arm64/features.rst index dfa4cb3cd3ef..03321f4309d0 100644 --- a/Documentation/arch/arm64/features.rst +++ b/Documentation/arch/arm64/features.rst @@ -1,3 +1,3 @@ .. SPDX-License-Identifier: GPL-2.0 -.. kernel-feat:: $srctree/Documentation/features arm64 +.. kernel-feat:: features arm64 diff --git a/Documentation/arch/loongarch/features.rst b/Documentation/arch/loongarch/features.rst index ebacade3ea45..009f44c7951f 100644 --- a/Documentation/arch/loongarch/features.rst +++ b/Documentation/arch/loongarch/features.rst @@ -1,3 +1,3 @@ .. SPDX-License-Identifier: GPL-2.0 -.. kernel-feat:: $srctree/Documentation/features loongarch +.. kernel-feat:: features loongarch diff --git a/Documentation/arch/m68k/features.rst b/Documentation/arch/m68k/features.rst index 5107a2119472..de7f0ccf7fc8 100644 --- a/Documentation/arch/m68k/features.rst +++ b/Documentation/arch/m68k/features.rst @@ -1,3 +1,3 @@ .. SPDX-License-Identifier: GPL-2.0 -.. kernel-feat:: $srctree/Documentation/features m68k +.. kernel-feat:: features m68k diff --git a/Documentation/arch/mips/features.rst b/Documentation/arch/mips/features.rst index 1973d729b29a..6e0ffe3e7354 100644 --- a/Documentation/arch/mips/features.rst +++ b/Documentation/arch/mips/features.rst @@ -1,3 +1,3 @@ .. SPDX-License-Identifier: GPL-2.0 -.. kernel-feat:: $srctree/Documentation/features mips +.. kernel-feat:: features mips diff --git a/Documentation/arch/nios2/features.rst b/Documentation/arch/nios2/features.rst index 8449e63f69b2..89913810ccb5 100644 --- a/Documentation/arch/nios2/features.rst +++ b/Documentation/arch/nios2/features.rst @@ -1,3 +1,3 @@ .. SPDX-License-Identifier: GPL-2.0 -.. kernel-feat:: $srctree/Documentation/features nios2 +.. kernel-feat:: features nios2 diff --git a/Documentation/arch/openrisc/features.rst b/Documentation/arch/openrisc/features.rst index 3f7c40d219f2..bae2e25adfd6 100644 --- a/Documentation/arch/openrisc/features.rst +++ b/Documentation/arch/openrisc/features.rst @@ -1,3 +1,3 @@ .. SPDX-License-Identifier: GPL-2.0 -.. kernel-feat:: $srctree/Documentation/features openrisc +.. kernel-feat:: features openrisc diff --git a/Documentation/arch/parisc/features.rst b/Documentation/arch/parisc/features.rst index 501d7c450037..b3aa4d243b93 100644 --- a/Documentation/arch/parisc/features.rst +++ b/Documentation/arch/parisc/features.rst @@ -1,3 +1,3 @@ .. SPDX-License-Identifier: GPL-2.0 -.. kernel-feat:: $srctree/Documentation/features parisc +.. kernel-feat:: features parisc diff --git a/Documentation/arch/powerpc/features.rst b/Documentation/arch/powerpc/features.rst index aeae73df86b0..ee4b95e04202 100644 --- a/Documentation/arch/powerpc/features.rst +++ b/Documentation/arch/powerpc/features.rst @@ -1,3 +1,3 @@ .. SPDX-License-Identifier: GPL-2.0 -.. kernel-feat:: $srctree/Documentation/features powerpc +.. kernel-feat:: features powerpc diff --git a/Documentation/arch/riscv/features.rst b/Documentation/arch/riscv/features.rst index c70ef6ac2368..36e90144adab 100644 --- a/Documentation/arch/riscv/features.rst +++ b/Documentation/arch/riscv/features.rst @@ -1,3 +1,3 @@ .. SPDX-License-Identifier: GPL-2.0 -.. kernel-feat:: $srctree/Documentation/features riscv +.. kernel-feat:: features riscv diff --git a/Documentation/arch/s390/features.rst b/Documentation/arch/s390/features.rst index 57c296a9d8f3..2883dc950681 100644 --- a/Documentation/arch/s390/features.rst +++ b/Documentation/arch/s390/features.rst @@ -1,3 +1,3 @@ .. SPDX-License-Identifier: GPL-2.0 -.. kernel-feat:: $srctree/Documentation/features s390 +.. kernel-feat:: features s390 diff --git a/Documentation/arch/sh/features.rst b/Documentation/arch/sh/features.rst index f722af3b6c99..fae48fe81e9b 100644 --- a/Documentation/arch/sh/features.rst +++ b/Documentation/arch/sh/features.rst @@ -1,3 +1,3 @@ .. SPDX-License-Identifier: GPL-2.0 -.. kernel-feat:: $srctree/Documentation/features sh +.. kernel-feat:: features sh diff --git a/Documentation/arch/sparc/features.rst b/Documentation/arch/sparc/features.rst index c0c92468b0fe..96835b6d598a 100644 --- a/Documentation/arch/sparc/features.rst +++ b/Documentation/arch/sparc/features.rst @@ -1,3 +1,3 @@ .. SPDX-License-Identifier: GPL-2.0 -.. kernel-feat:: $srctree/Documentation/features sparc +.. kernel-feat:: features sparc diff --git a/Documentation/arch/x86/features.rst b/Documentation/arch/x86/features.rst index b663f15053ce..a33616346a38 100644 --- a/Documentation/arch/x86/features.rst +++ b/Documentation/arch/x86/features.rst @@ -1,3 +1,3 @@ .. SPDX-License-Identifier: GPL-2.0 -.. kernel-feat:: $srctree/Documentation/features x86 +.. kernel-feat:: features x86 diff --git a/Documentation/arch/xtensa/features.rst b/Documentation/arch/xtensa/features.rst index 6b92c7bfa19d..28dcce1759be 100644 --- a/Documentation/arch/xtensa/features.rst +++ b/Documentation/arch/xtensa/features.rst @@ -1,3 +1,3 @@ .. SPDX-License-Identifier: GPL-2.0 -.. kernel-feat:: $srctree/Documentation/features xtensa +.. kernel-feat:: features xtensa diff --git a/Documentation/sphinx/kernel_feat.py b/Documentation/sphinx/kernel_feat.py index b5fa2f0542a5..b9df61eb4501 100644 --- a/Documentation/sphinx/kernel_feat.py +++ b/Documentation/sphinx/kernel_feat.py @@ -37,8 +37,6 @@ import re import subprocess import sys -from os import path - from docutils import nodes, statemachine from docutils.statemachine import ViewList from docutils.parsers.rst import directives, Directive @@ -76,33 +74,26 @@ class KernelFeat(Directive): self.state.document.settings.env.app.warn(message, prefix="") def run(self): - doc = self.state.document if not doc.settings.file_insertion_enabled: raise self.warning("docutils: file insertion disabled") env = doc.settings.env - cwd = path.dirname(doc.current_source) - cmd = "get_feat.pl rest --enable-fname --dir " - cmd += self.arguments[0] + + srctree = os.path.abspath(os.environ["srctree"]) + + args = [ + os.path.join(srctree, 'scripts/get_feat.pl'), + 'rest', + '--enable-fname', + '--dir', + os.path.join(srctree, 'Documentation', self.arguments[0]), + ] if len(self.arguments) > 1: - cmd += " --arch " + self.arguments[1] + args.extend(['--arch', self.arguments[1]]) - srctree = path.abspath(os.environ["srctree"]) - - fname = cmd - - # extend PATH with $(srctree)/scripts - path_env = os.pathsep.join([ - srctree + os.sep + "scripts", - os.environ["PATH"] - ]) - shell_env = os.environ.copy() - shell_env["PATH"] = path_env - shell_env["srctree"] = srctree - - lines = self.runCmd(cmd, shell=True, cwd=cwd, env=shell_env) + lines = subprocess.check_output(args, cwd=os.path.dirname(doc.current_source)).decode('utf-8') line_regex = re.compile(r"^\.\. FILE (\S+)$") @@ -121,30 +112,6 @@ class KernelFeat(Directive): nodeList = self.nestedParse(out_lines, fname) return nodeList - def runCmd(self, cmd, **kwargs): - u"""Run command ``cmd`` and return its stdout as unicode.""" - - try: - proc = subprocess.Popen( - cmd - , stdout = subprocess.PIPE - , stderr = subprocess.PIPE - , **kwargs - ) - out, err = proc.communicate() - - out, err = codecs.decode(out, 'utf-8'), codecs.decode(err, 'utf-8') - - if proc.returncode != 0: - raise self.severe( - u"command '%s' failed with return code %d" - % (cmd, proc.returncode) - ) - except OSError as exc: - raise self.severe(u"problems with '%s' directive: %s." - % (self.name, ErrorString(exc))) - return out - def nestedParse(self, lines, fname): content = ViewList() node = nodes.section() diff --git a/Documentation/translations/zh_CN/arch/loongarch/features.rst b/Documentation/translations/zh_CN/arch/loongarch/features.rst index 82bfac180bdc..cec38dda8298 100644 --- a/Documentation/translations/zh_CN/arch/loongarch/features.rst +++ b/Documentation/translations/zh_CN/arch/loongarch/features.rst @@ -5,4 +5,4 @@ :Original: Documentation/arch/loongarch/features.rst :Translator: Huacai Chen -.. kernel-feat:: $srctree/Documentation/features loongarch +.. kernel-feat:: features loongarch diff --git a/Documentation/translations/zh_CN/arch/mips/features.rst b/Documentation/translations/zh_CN/arch/mips/features.rst index da1b956e4a40..0d6df97db069 100644 --- a/Documentation/translations/zh_CN/arch/mips/features.rst +++ b/Documentation/translations/zh_CN/arch/mips/features.rst @@ -10,4 +10,4 @@ .. _cn_features: -.. kernel-feat:: $srctree/Documentation/features mips +.. kernel-feat:: features mips diff --git a/Documentation/translations/zh_TW/arch/loongarch/features.rst b/Documentation/translations/zh_TW/arch/loongarch/features.rst index b64e430f55ae..c2175fd32b54 100644 --- a/Documentation/translations/zh_TW/arch/loongarch/features.rst +++ b/Documentation/translations/zh_TW/arch/loongarch/features.rst @@ -5,5 +5,5 @@ :Original: Documentation/arch/loongarch/features.rst :Translator: Huacai Chen -.. kernel-feat:: $srctree/Documentation/features loongarch +.. kernel-feat:: features loongarch diff --git a/Documentation/translations/zh_TW/arch/mips/features.rst b/Documentation/translations/zh_TW/arch/mips/features.rst index f69410420035..3d3906c4d08e 100644 --- a/Documentation/translations/zh_TW/arch/mips/features.rst +++ b/Documentation/translations/zh_TW/arch/mips/features.rst @@ -10,5 +10,5 @@ .. _tw_features: -.. kernel-feat:: $srctree/Documentation/features mips +.. kernel-feat:: features mips From 22160b08d88898898b2bb99282663cdf3caa5c1c Mon Sep 17 00:00:00 2001 From: attreyee-muk Date: Thu, 11 Jan 2024 00:27:47 +0530 Subject: [PATCH 323/336] Documentation/core-api: fix spelling mistake in workqueue Correct to "following" from "followings" in the sentence "The followings are the read bandwidths and CPU utilizations depending on different affinity scope settings on ``kcryptd`` measured over five runs." Signed-off-by: Attreyee Mukherjee Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20240110185746.24974-1-tintinm2017@gmail.com --- Documentation/core-api/workqueue.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/core-api/workqueue.rst b/Documentation/core-api/workqueue.rst index 33c4539155d9..3599cf9267b4 100644 --- a/Documentation/core-api/workqueue.rst +++ b/Documentation/core-api/workqueue.rst @@ -446,7 +446,7 @@ The command used: :: There are 24 issuers, each issuing 64 IOs concurrently. ``--verify=sha512`` makes ``fio`` generate and read back the content each time which makes -execution locality matter between the issuer and ``kcryptd``. The followings +execution locality matter between the issuer and ``kcryptd``. The following are the read bandwidths and CPU utilizations depending on different affinity scope settings on ``kcryptd`` measured over five runs. Bandwidths are in MiBps, and CPU util in percents. From 54a2ffe9524f5aef34397bbd53366c6a95661491 Mon Sep 17 00:00:00 2001 From: Baruch Siach Date: Mon, 8 Jan 2024 15:21:45 +0200 Subject: [PATCH 324/336] docs: admin-guide: hw_random: update rng-tools website rng-tools upstream moved to github. New upstream does not appear to consider itself official website for hw_random. Drop that part. Signed-off-by: Baruch Siach Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/ef52ace5008fa934084442149f64f5f9ddbba465.1704720105.git.baruch@tkos.co.il --- Documentation/admin-guide/hw_random.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Documentation/admin-guide/hw_random.rst b/Documentation/admin-guide/hw_random.rst index d494601717f1..bfc39f1cf470 100644 --- a/Documentation/admin-guide/hw_random.rst +++ b/Documentation/admin-guide/hw_random.rst @@ -14,10 +14,9 @@ into that core. To make the most effective use of these mechanisms, you should download the support software as well. Download the -latest version of the "rng-tools" package from the -hw_random driver's official Web site: +latest version of the "rng-tools" package from: - http://sourceforge.net/projects/gkernel/ + https://github.com/nhorman/rng-tools Those tools use /dev/hwrng to fill the kernel entropy pool, which is used internally and exported by the /dev/urandom and From b65a6b44f0eaab8f217335d821c9b79aed47d9d8 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Tue, 19 Dec 2023 14:23:29 +0800 Subject: [PATCH 325/336] docs, kprobes: Update email address of Masami Hiramatsu According to the latest authorship and Signed-off-by: Masami Hiramatsu (Google) Masami Hiramatsu is working at Google, so the current email @redhat.com is out of date, it is better to use the email @kernel.org. Signed-off-by: Tiezhu Yang Acked-by: "Masami Hiramatsu (Google)" Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20231219062330.22813-2-yangtiezhu@loongson.cn --- Documentation/trace/kprobes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/trace/kprobes.rst b/Documentation/trace/kprobes.rst index f825970a1495..968ae080accd 100644 --- a/Documentation/trace/kprobes.rst +++ b/Documentation/trace/kprobes.rst @@ -4,7 +4,7 @@ Kernel Probes (Kprobes) :Author: Jim Keniston :Author: Prasanna S Panchamukhi -:Author: Masami Hiramatsu +:Author: Masami Hiramatsu .. CONTENTS From ead8467f96d6dc35bbf8c63ee9d244a357ede84a Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Tue, 19 Dec 2023 14:23:30 +0800 Subject: [PATCH 326/336] docs, kprobes: Add loongarch as supported architecture After the following three changes at the beginning of the year: commit 6d4cc40fb5f5 ("LoongArch: Add kprobes support") commit 3f5536860086 ("LoongArch: Add kretprobes support") commit 09e679c28a4d ("LoongArch: Add kprobes on ftrace support") it is appropriate to add loongarch as supported architecture in kprobes documentation. Signed-off-by: Tiezhu Yang Acked-by: "Masami Hiramatsu (Google)" Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20231219062330.22813-3-yangtiezhu@loongson.cn --- Documentation/trace/kprobes.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/trace/kprobes.rst b/Documentation/trace/kprobes.rst index 968ae080accd..e1636e579c9c 100644 --- a/Documentation/trace/kprobes.rst +++ b/Documentation/trace/kprobes.rst @@ -321,6 +321,7 @@ architectures: - mips - s390 - parisc +- loongarch Configuring Kprobes =================== From a03cd7602a090eae277d2b79d43925661e7fbe9a Mon Sep 17 00:00:00 2001 From: Max Filippov Date: Wed, 10 Jan 2024 20:20:00 -0800 Subject: [PATCH 327/336] xtensa: don't produce FDPIC output with fdpic toolchain The kernel doesn't use features of the FDPIC toolchain, disable FDPIC code generation when using such toolchain and select ordinary ELF linker emulation. Signed-off-by: Max Filippov --- arch/xtensa/Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/xtensa/Makefile b/arch/xtensa/Makefile index bfd8e433ed62..4c14a02179eb 100644 --- a/arch/xtensa/Makefile +++ b/arch/xtensa/Makefile @@ -35,15 +35,19 @@ KBUILD_CFLAGS += -ffreestanding -D__linux__ KBUILD_CFLAGS += -pipe -mlongcalls -mtext-section-literals KBUILD_CFLAGS += $(call cc-option,-mforce-no-pic,) KBUILD_CFLAGS += $(call cc-option,-mno-serialize-volatile,) +KBUILD_CFLAGS += $(call cc-option,-mno-fdpic,) ifneq ($(CONFIG_KERNEL_ABI_CALL0),) KBUILD_CFLAGS += -mabi=call0 KBUILD_AFLAGS += -mabi=call0 endif KBUILD_AFLAGS += -mlongcalls -mtext-section-literals +KBUILD_AFLAGS += $(call cc-option,-mno-fdpic,) + +KBUILD_LDFLAGS += -m elf32xtensa ifneq ($(CONFIG_LD_NO_RELAX),) -KBUILD_LDFLAGS := --no-relax +KBUILD_LDFLAGS += --no-relax endif CHECKFLAGS += -D$(if $(CONFIG_CPU_BIG_ENDIAN),__XTENSA_EB__,__XTENSA_EL__) From 755113d7678681a137c330f7997ceb680adb644e Mon Sep 17 00:00:00 2001 From: Daniel Lezcano Date: Tue, 9 Jan 2024 10:41:11 +0100 Subject: [PATCH 328/336] thermal/debugfs: Add thermal cooling device debugfs information The thermal framework does not have any debug information except a sysfs stat which is a bit controversial. This one allocates big chunks of memory for every cooling devices with a high number of states and could represent on some systems in production several megabytes of memory for just a portion of it. As the sysfs is limited to a page size, the output is not exploitable with large data array and gets truncated. The patch provides the same information than sysfs except the transitions are dynamically allocated, thus they won't show more events than the ones which actually occurred. There is no longer a size limitation and it opens the field for more debugging information where the debugfs is designed for, not sysfs. The thermal debugfs directory structure tries to stay consistent with the sysfs one but in a very simplified way: thermal/ -- cooling_devices |-- 0 | |-- clear | |-- time_in_state_ms | |-- total_trans | `-- trans_table |-- 1 | |-- clear | |-- time_in_state_ms | |-- total_trans | `-- trans_table |-- 2 | |-- clear | |-- time_in_state_ms | |-- total_trans | `-- trans_table |-- 3 | |-- clear | |-- time_in_state_ms | |-- total_trans | `-- trans_table `-- 4 |-- clear |-- time_in_state_ms |-- total_trans `-- trans_table The content of the files in the cooling devices directory is the same as the sysfs one except for the trans_table which has the following format: Transition Hits 1->0 246 0->1 246 2->1 632 1->2 632 3->2 98 2->3 98 Signed-off-by: Daniel Lezcano [ rjw: White space fixups, rebase ] Signed-off-by: Rafael J. Wysocki --- drivers/thermal/Kconfig | 7 + drivers/thermal/Makefile | 2 + drivers/thermal/thermal_core.c | 6 + drivers/thermal/thermal_core.h | 1 + drivers/thermal/thermal_debugfs.c | 446 ++++++++++++++++++++++++++++++ drivers/thermal/thermal_debugfs.h | 14 + drivers/thermal/thermal_helpers.c | 20 +- include/linux/thermal.h | 7 + 8 files changed, 497 insertions(+), 6 deletions(-) create mode 100644 drivers/thermal/thermal_debugfs.c create mode 100644 drivers/thermal/thermal_debugfs.h diff --git a/drivers/thermal/Kconfig b/drivers/thermal/Kconfig index c81a00fbca7d..3ff7add3fb7c 100644 --- a/drivers/thermal/Kconfig +++ b/drivers/thermal/Kconfig @@ -33,6 +33,13 @@ config THERMAL_STATISTICS If in doubt, say N. +config THERMAL_DEBUGFS + bool "Thermal subsystem debug support" + depends on DEBUG_FS + help + Say Y to allow the thermal subsystem to collect diagnostic + information that can be accessed via debugfs. + config THERMAL_EMERGENCY_POWEROFF_DELAY_MS int "Emergency poweroff delay in milli-seconds" default 0 diff --git a/drivers/thermal/Makefile b/drivers/thermal/Makefile index c934cab309ae..0f65ae86a9c6 100644 --- a/drivers/thermal/Makefile +++ b/drivers/thermal/Makefile @@ -10,6 +10,8 @@ thermal_sys-y += thermal_trip.o thermal_helpers.o # netlink interface to manage the thermal framework thermal_sys-$(CONFIG_THERMAL_NETLINK) += thermal_netlink.o +thermal_sys-$(CONFIG_THERMAL_DEBUGFS) += thermal_debugfs.o + # interface to/from other layers providing sensors thermal_sys-$(CONFIG_THERMAL_HWMON) += thermal_hwmon.o thermal_sys-$(CONFIG_THERMAL_OF) += thermal_of.o diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c index c0b012e075f3..5d9cc422d7ba 100644 --- a/drivers/thermal/thermal_core.c +++ b/drivers/thermal/thermal_core.c @@ -960,6 +960,8 @@ __thermal_cooling_device_register(struct device_node *np, mutex_unlock(&thermal_list_lock); + thermal_debug_cdev_add(cdev); + return cdev; out_cooling_dev: @@ -1166,6 +1168,8 @@ void thermal_cooling_device_unregister(struct thermal_cooling_device *cdev) if (!cdev) return; + thermal_debug_cdev_remove(cdev); + mutex_lock(&thermal_list_lock); if (!thermal_cooling_device_present(cdev)) { @@ -1629,6 +1633,8 @@ static int __init thermal_init(void) { int result; + thermal_debug_init(); + result = thermal_netlink_init(); if (result) goto error; diff --git a/drivers/thermal/thermal_core.h b/drivers/thermal/thermal_core.h index 4e023d54fd27..e9c099ecdd0f 100644 --- a/drivers/thermal/thermal_core.h +++ b/drivers/thermal/thermal_core.h @@ -13,6 +13,7 @@ #include #include "thermal_netlink.h" +#include "thermal_debugfs.h" /* Default Thermal Governor */ #if defined(CONFIG_THERMAL_DEFAULT_GOV_STEP_WISE) diff --git a/drivers/thermal/thermal_debugfs.c b/drivers/thermal/thermal_debugfs.c new file mode 100644 index 000000000000..b18bdde47847 --- /dev/null +++ b/drivers/thermal/thermal_debugfs.c @@ -0,0 +1,446 @@ + +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2023 Linaro Limited + * + * Author: Daniel Lezcano + * + * Thermal subsystem debug support + */ +#include +#include +#include +#include +#include +#include + +static struct dentry *d_root; +static struct dentry *d_cdev; + +/* + * Length of the string containing the thermal zone id or the cooling + * device id, including the ending nul character. We can reasonably + * assume there won't be more than 256 thermal zones as the maximum + * observed today is around 32. + */ +#define IDSLENGTH 4 + +/* + * The cooling device transition list is stored in a hash table where + * the size is CDEVSTATS_HASH_SIZE. The majority of cooling devices + * have dozen of states but some can have much more, so a hash table + * is more adequate in this case, because the cost of browsing the entire + * list when storing the transitions may not be negligible. + */ +#define CDEVSTATS_HASH_SIZE 16 + +/** + * struct cdev_debugfs - per cooling device statistics structure + * A cooling device can have a high number of states. Showing the + * transitions on a matrix based representation can be overkill given + * most of the transitions won't happen and we end up with a matrix + * filled with zero. Instead, we show the transitions which actually + * happened. + * + * Every transition updates the current_state and the timestamp. The + * transitions and the durations are stored in lists. + * + * @total: the number of transitions for this cooling device + * @current_state: the current cooling device state + * @timestamp: the state change timestamp + * @transitions: an array of lists containing the state transitions + * @durations: an array of lists containing the residencies of each state + */ +struct cdev_debugfs { + u32 total; + int current_state; + ktime_t timestamp; + struct list_head transitions[CDEVSTATS_HASH_SIZE]; + struct list_head durations[CDEVSTATS_HASH_SIZE]; +}; + +/** + * struct cdev_value - Common structure for cooling device entry + * + * The following common structure allows to store the information + * related to the transitions and to the state residencies. They are + * identified with a id which is associated to a value. It is used as + * nodes for the "transitions" and "durations" above. + * + * @node: node to insert the structure in a list + * @id: identifier of the value which can be a state or a transition + * @residency: a ktime_t representing a state residency duration + * @count: a number of occurrences + */ +struct cdev_record { + struct list_head node; + int id; + union { + ktime_t residency; + u64 count; + }; +}; + +/** + * struct thermal_debugfs - High level structure for a thermal object in debugfs + * + * The thermal_debugfs structure is the common structure used by the + * cooling device to compute the statistics. + * + * @d_top: top directory of the thermal object directory + * @lock: per object lock to protect the internals + * + * @cdev: a cooling device debug structure + */ +struct thermal_debugfs { + struct dentry *d_top; + struct mutex lock; + union { + struct cdev_debugfs cdev_dbg; + }; +}; + +void thermal_debug_init(void) +{ + d_root = debugfs_create_dir("thermal", NULL); + if (!d_root) + return; + + d_cdev = debugfs_create_dir("cooling_devices", d_root); +} + +static struct thermal_debugfs *thermal_debugfs_add_id(struct dentry *d, int id) +{ + struct thermal_debugfs *thermal_dbg; + char ids[IDSLENGTH]; + + thermal_dbg = kzalloc(sizeof(*thermal_dbg), GFP_KERNEL); + if (!thermal_dbg) + return NULL; + + mutex_init(&thermal_dbg->lock); + + snprintf(ids, IDSLENGTH, "%d", id); + + thermal_dbg->d_top = debugfs_create_dir(ids, d); + if (!thermal_dbg->d_top) { + kfree(thermal_dbg); + return NULL; + } + + return thermal_dbg; +} + +static void thermal_debugfs_remove_id(struct thermal_debugfs *thermal_dbg) +{ + if (!thermal_dbg) + return; + + debugfs_remove(thermal_dbg->d_top); + + kfree(thermal_dbg); +} + +static struct cdev_record * +thermal_debugfs_cdev_record_alloc(struct thermal_debugfs *thermal_dbg, + struct list_head *lists, int id) +{ + struct cdev_record *cdev_record; + + cdev_record = kzalloc(sizeof(*cdev_record), GFP_KERNEL); + if (!cdev_record) + return NULL; + + cdev_record->id = id; + INIT_LIST_HEAD(&cdev_record->node); + list_add_tail(&cdev_record->node, + &lists[cdev_record->id % CDEVSTATS_HASH_SIZE]); + + return cdev_record; +} + +static struct cdev_record * +thermal_debugfs_cdev_record_find(struct thermal_debugfs *thermal_dbg, + struct list_head *lists, int id) +{ + struct cdev_record *entry; + + list_for_each_entry(entry, &lists[id % CDEVSTATS_HASH_SIZE], node) + if (entry->id == id) + return entry; + + return NULL; +} + +static struct cdev_record * +thermal_debugfs_cdev_record_get(struct thermal_debugfs *thermal_dbg, + struct list_head *lists, int id) +{ + struct cdev_record *cdev_record; + + cdev_record = thermal_debugfs_cdev_record_find(thermal_dbg, lists, id); + if (cdev_record) + return cdev_record; + + return thermal_debugfs_cdev_record_alloc(thermal_dbg, lists, id); +} + +static void thermal_debugfs_cdev_clear(struct cdev_debugfs *cdev_dbg) +{ + int i; + struct cdev_record *entry, *tmp; + + for (i = 0; i < CDEVSTATS_HASH_SIZE; i++) { + + list_for_each_entry_safe(entry, tmp, + &cdev_dbg->transitions[i], node) { + list_del(&entry->node); + kfree(entry); + } + + list_for_each_entry_safe(entry, tmp, + &cdev_dbg->durations[i], node) { + list_del(&entry->node); + kfree(entry); + } + } + + cdev_dbg->total = 0; +} + +static void *cdev_seq_start(struct seq_file *s, loff_t *pos) +{ + struct thermal_debugfs *thermal_dbg = s->private; + + mutex_lock(&thermal_dbg->lock); + + return (*pos < CDEVSTATS_HASH_SIZE) ? pos : NULL; +} + +static void *cdev_seq_next(struct seq_file *s, void *v, loff_t *pos) +{ + (*pos)++; + + return (*pos < CDEVSTATS_HASH_SIZE) ? pos : NULL; +} + +static void cdev_seq_stop(struct seq_file *s, void *v) +{ + struct thermal_debugfs *thermal_dbg = s->private; + + mutex_unlock(&thermal_dbg->lock); +} + +static int cdev_tt_seq_show(struct seq_file *s, void *v) +{ + struct thermal_debugfs *thermal_dbg = s->private; + struct cdev_debugfs *cdev_dbg = &thermal_dbg->cdev_dbg; + struct list_head *transitions = cdev_dbg->transitions; + struct cdev_record *entry; + int i = *(loff_t *)v; + + if (!i) + seq_puts(s, "Transition\tOccurences\n"); + + list_for_each_entry(entry, &transitions[i], node) { + /* + * Assuming maximum cdev states is 1024, the longer + * string for a transition would be "1024->1024\0" + */ + char buffer[11]; + + snprintf(buffer, ARRAY_SIZE(buffer), "%d->%d", + entry->id >> 16, entry->id & 0xFFFF); + + seq_printf(s, "%-10s\t%-10llu\n", buffer, entry->count); + } + + return 0; +} + +static const struct seq_operations tt_sops = { + .start = cdev_seq_start, + .next = cdev_seq_next, + .stop = cdev_seq_stop, + .show = cdev_tt_seq_show, +}; + +DEFINE_SEQ_ATTRIBUTE(tt); + +static int cdev_dt_seq_show(struct seq_file *s, void *v) +{ + struct thermal_debugfs *thermal_dbg = s->private; + struct cdev_debugfs *cdev_dbg = &thermal_dbg->cdev_dbg; + struct list_head *durations = cdev_dbg->durations; + struct cdev_record *entry; + int i = *(loff_t *)v; + + if (!i) + seq_puts(s, "State\tResidency\n"); + + list_for_each_entry(entry, &durations[i], node) { + s64 duration = ktime_to_ms(entry->residency); + + if (entry->id == cdev_dbg->current_state) + duration += ktime_ms_delta(ktime_get(), + cdev_dbg->timestamp); + + seq_printf(s, "%-5d\t%-10llu\n", entry->id, duration); + } + + return 0; +} + +static const struct seq_operations dt_sops = { + .start = cdev_seq_start, + .next = cdev_seq_next, + .stop = cdev_seq_stop, + .show = cdev_dt_seq_show, +}; + +DEFINE_SEQ_ATTRIBUTE(dt); + +static int cdev_clear_set(void *data, u64 val) +{ + struct thermal_debugfs *thermal_dbg = data; + + if (!val) + return -EINVAL; + + mutex_lock(&thermal_dbg->lock); + + thermal_debugfs_cdev_clear(&thermal_dbg->cdev_dbg); + + mutex_unlock(&thermal_dbg->lock); + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(cdev_clear_fops, NULL, cdev_clear_set, "%llu\n"); + +/** + * thermal_debug_cdev_state_update - Update a cooling device state change + * + * Computes a transition and the duration of the previous state residency. + * + * @cdev : a pointer to a cooling device + * @new_state: an integer corresponding to the new cooling device state + */ +void thermal_debug_cdev_state_update(const struct thermal_cooling_device *cdev, + int new_state) +{ + struct thermal_debugfs *thermal_dbg = cdev->debugfs; + struct cdev_debugfs *cdev_dbg; + struct cdev_record *cdev_record; + int transition, old_state; + + if (!thermal_dbg || (thermal_dbg->cdev_dbg.current_state == new_state)) + return; + + mutex_lock(&thermal_dbg->lock); + + cdev_dbg = &thermal_dbg->cdev_dbg; + + old_state = cdev_dbg->current_state; + + /* + * Get the old state information in the durations list. If + * this one does not exist, a new allocated one will be + * returned. Recompute the total duration in the old state and + * get a new timestamp for the new state. + */ + cdev_record = thermal_debugfs_cdev_record_get(thermal_dbg, + cdev_dbg->durations, + old_state); + if (cdev_record) { + ktime_t now = ktime_get(); + ktime_t delta = ktime_sub(now, cdev_dbg->timestamp); + cdev_record->residency = ktime_add(cdev_record->residency, delta); + cdev_dbg->timestamp = now; + } + + cdev_dbg->current_state = new_state; + transition = (old_state << 16) | new_state; + + /* + * Get the transition in the transitions list. If this one + * does not exist, a new allocated one will be returned. + * Increment the occurrence of this transition which is stored + * in the value field. + */ + cdev_record = thermal_debugfs_cdev_record_get(thermal_dbg, + cdev_dbg->transitions, + transition); + if (cdev_record) + cdev_record->count++; + + cdev_dbg->total++; + + mutex_unlock(&thermal_dbg->lock); +} + +/** + * thermal_debug_cdev_add - Add a cooling device debugfs entry + * + * Allocates a cooling device object for debug, initializes the + * statistics and create the entries in sysfs. + * @cdev: a pointer to a cooling device + */ +void thermal_debug_cdev_add(struct thermal_cooling_device *cdev) +{ + struct thermal_debugfs *thermal_dbg; + struct cdev_debugfs *cdev_dbg; + int i; + + thermal_dbg = thermal_debugfs_add_id(d_cdev, cdev->id); + if (!thermal_dbg) + return; + + cdev_dbg = &thermal_dbg->cdev_dbg; + + for (i = 0; i < CDEVSTATS_HASH_SIZE; i++) { + INIT_LIST_HEAD(&cdev_dbg->transitions[i]); + INIT_LIST_HEAD(&cdev_dbg->durations[i]); + } + + cdev_dbg->current_state = 0; + cdev_dbg->timestamp = ktime_get(); + + debugfs_create_file("trans_table", 0400, thermal_dbg->d_top, + thermal_dbg, &tt_fops); + + debugfs_create_file("time_in_state_ms", 0400, thermal_dbg->d_top, + thermal_dbg, &dt_fops); + + debugfs_create_file("clear", 0200, thermal_dbg->d_top, + thermal_dbg, &cdev_clear_fops); + + debugfs_create_u32("total_trans", 0400, thermal_dbg->d_top, + &cdev_dbg->total); + + cdev->debugfs = thermal_dbg; +} + +/** + * thermal_debug_cdev_remove - Remove a cooling device debugfs entry + * + * Frees the statistics memory data and remove the debugfs entry + * + * @cdev: a pointer to a cooling device + */ +void thermal_debug_cdev_remove(struct thermal_cooling_device *cdev) +{ + struct thermal_debugfs *thermal_dbg = cdev->debugfs; + + if (!thermal_dbg) + return; + + mutex_lock(&thermal_dbg->lock); + + thermal_debugfs_cdev_clear(&thermal_dbg->cdev_dbg); + cdev->debugfs = NULL; + + mutex_unlock(&thermal_dbg->lock); + + thermal_debugfs_remove_id(thermal_dbg); +} diff --git a/drivers/thermal/thermal_debugfs.h b/drivers/thermal/thermal_debugfs.h new file mode 100644 index 000000000000..341499388448 --- /dev/null +++ b/drivers/thermal/thermal_debugfs.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifdef CONFIG_THERMAL_DEBUGFS +void thermal_debug_init(void); +void thermal_debug_cdev_add(struct thermal_cooling_device *cdev); +void thermal_debug_cdev_remove(struct thermal_cooling_device *cdev); +void thermal_debug_cdev_state_update(const struct thermal_cooling_device *cdev, int state); +#else +static inline void thermal_debug_init(void) {} +static inline void thermal_debug_cdev_add(struct thermal_cooling_device *cdev) {} +static inline void thermal_debug_cdev_remove(struct thermal_cooling_device *cdev) {} +static inline void thermal_debug_cdev_state_update(const struct thermal_cooling_device *cdev, + int state) {} +#endif /* CONFIG_THERMAL_DEBUGFS */ diff --git a/drivers/thermal/thermal_helpers.c b/drivers/thermal/thermal_helpers.c index c3982e0f0075..8f85581693e7 100644 --- a/drivers/thermal/thermal_helpers.c +++ b/drivers/thermal/thermal_helpers.c @@ -146,14 +146,22 @@ unlock: } EXPORT_SYMBOL_GPL(thermal_zone_get_temp); -static void thermal_cdev_set_cur_state(struct thermal_cooling_device *cdev, - int target) +static int thermal_cdev_set_cur_state(struct thermal_cooling_device *cdev, int state) { - if (cdev->ops->set_cur_state(cdev, target)) - return; + int ret; - thermal_notify_cdev_state_update(cdev->id, target); - thermal_cooling_device_stats_update(cdev, target); + /* + * No check is needed for the ops->set_cur_state as the + * registering function checked the ops are correctly set + */ + ret = cdev->ops->set_cur_state(cdev, state); + if (!ret) { + thermal_notify_cdev_state_update(cdev->id, state); + thermal_cooling_device_stats_update(cdev, state); + thermal_debug_cdev_state_update(cdev, state); + } + + return ret; } void __thermal_cdev_update(struct thermal_cooling_device *cdev) diff --git a/include/linux/thermal.h b/include/linux/thermal.h index 9d0427da32af..7defea8fa223 100644 --- a/include/linux/thermal.h +++ b/include/linux/thermal.h @@ -32,6 +32,7 @@ struct thermal_zone_device; struct thermal_cooling_device; struct thermal_instance; +struct thermal_debugfs; struct thermal_attr; enum thermal_trend { @@ -113,6 +114,9 @@ struct thermal_cooling_device { struct mutex lock; /* protect thermal_instances list */ struct list_head thermal_instances; struct list_head node; +#ifdef CONFIG_THERMAL_DEBUGFS + struct thermal_debugfs *debugfs; +#endif }; /** @@ -189,6 +193,9 @@ struct thermal_zone_device { struct list_head node; struct delayed_work poll_queue; enum thermal_notify_event notify_event; +#ifdef CONFIG_THERMAL_DEBUGFS + struct thermal_debugfs *debugfs; +#endif bool suspended; }; From 7ef01f228c9f54c6260319858be138a8a7e9e704 Mon Sep 17 00:00:00 2001 From: Daniel Lezcano Date: Tue, 9 Jan 2024 10:41:12 +0100 Subject: [PATCH 329/336] thermal/debugfs: Add thermal debugfs information for mitigation episodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mitigation episodes are recorded. A mitigation episode happens when the first trip point is crossed the way up and then the way down. During this episode other trip points can be crossed also and are accounted for this mitigation episode. The interesting information is the average temperature at the trip point, the undershot and the overshot. The standard deviation of the mitigated temperature will be added later. The thermal debugfs directory structure tries to stay consistent with the sysfs one but in a very simplified way: thermal/ `-- thermal_zones |-- 0 | `-- mitigations `-- 1 `-- mitigations The content of the mitigations file has the following format: ,-Mitigation at 349988258us, duration=130136ms | trip | type | temp(°mC) | hyst(°mC) | duration | avg(°mC) | min(°mC) | max(°mC) | | 0 | passive | 65000 | 2000 | 130136 | 68227 | 62500 | 75625 | | 1 | passive | 75000 | 2000 | 104209 | 74857 | 71666 | 77500 | ,-Mitigation at 272451637us, duration=75000ms | trip | type | temp(°mC) | hyst(°mC) | duration | avg(°mC) | min(°mC) | max(°mC) | | 0 | passive | 65000 | 2000 | 75000 | 68561 | 62500 | 75000 | | 1 | passive | 75000 | 2000 | 60714 | 74820 | 70555 | 77500 | ,-Mitigation at 238184119us, duration=27316ms | trip | type | temp(°mC) | hyst(°mC) | duration | avg(°mC) | min(°mC) | max(°mC) | | 0 | passive | 65000 | 2000 | 27316 | 73377 | 62500 | 75000 | | 1 | passive | 75000 | 2000 | 19468 | 75284 | 69444 | 77500 | ,-Mitigation at 39863713us, duration=136196ms | trip | type | temp(°mC) | hyst(°mC) | duration | avg(°mC) | min(°mC) | max(°mC) | | 0 | passive | 65000 | 2000 | 136196 | 73922 | 62500 | 75000 | | 1 | passive | 75000 | 2000 | 91721 | 74386 | 69444 | 78125 | More information for a better understanding of the thermal behavior will be added after. The idea is to give detailed statistics information about the undershots and overshots, the temperature speed, etc... As all the information in a single file is too much, the idea would be to create a directory named with the mitigation timestamp where all data could be added. Please note this code is immune against trip ordering but not against a trip temperature change while a mitigation is happening. However, this situation should be extremely rare, perhaps not happening and we might question ourselves if something should be done in the core framework for other components first. Signed-off-by: Daniel Lezcano [ rjw: White space fixups, rebase ] Signed-off-by: Rafael J. Wysocki --- drivers/thermal/thermal_core.c | 7 + drivers/thermal/thermal_debugfs.c | 400 +++++++++++++++++++++++++++++- drivers/thermal/thermal_debugfs.h | 14 ++ 3 files changed, 417 insertions(+), 4 deletions(-) diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c index 5d9cc422d7ba..abf4c2390b19 100644 --- a/drivers/thermal/thermal_core.c +++ b/drivers/thermal/thermal_core.c @@ -382,6 +382,7 @@ static void handle_thermal_trip(struct thermal_zone_device *tz, */ if (tz->temperature >= trip->temperature) { thermal_notify_tz_trip_up(tz, trip); + thermal_debug_tz_trip_up(tz, trip); trip->threshold = trip->temperature - trip->hysteresis; } else { trip->threshold = trip->temperature; @@ -399,6 +400,7 @@ static void handle_thermal_trip(struct thermal_zone_device *tz, */ if (tz->temperature < trip->temperature - trip->hysteresis) { thermal_notify_tz_trip_down(tz, trip); + thermal_debug_tz_trip_down(tz, trip); trip->threshold = trip->temperature; } else { trip->threshold = trip->temperature - trip->hysteresis; @@ -430,6 +432,7 @@ static void update_temperature(struct thermal_zone_device *tz) trace_thermal_temperature(tz); thermal_genl_sampling_temp(tz->id, temp); + thermal_debug_update_temp(tz); } static void thermal_zone_device_check(struct work_struct *work) @@ -1413,6 +1416,8 @@ thermal_zone_device_register_with_trips(const char *type, struct thermal_trip *t thermal_notify_tz_create(tz); + thermal_debug_tz_add(tz); + return tz; unregister: @@ -1476,6 +1481,8 @@ void thermal_zone_device_unregister(struct thermal_zone_device *tz) if (!tz) return; + thermal_debug_tz_remove(tz); + mutex_lock(&thermal_list_lock); list_for_each_entry(pos, &thermal_tz_list, node) if (pos == tz) diff --git a/drivers/thermal/thermal_debugfs.c b/drivers/thermal/thermal_debugfs.c index b18bdde47847..b53881e9e0b6 100644 --- a/drivers/thermal/thermal_debugfs.c +++ b/drivers/thermal/thermal_debugfs.c @@ -1,4 +1,3 @@ - // SPDX-License-Identifier: GPL-2.0 /* * Copyright 2023 Linaro Limited @@ -14,8 +13,11 @@ #include #include +#include "thermal_core.h" + static struct dentry *d_root; static struct dentry *d_cdev; +static struct dentry *d_tz; /* * Length of the string containing the thermal zone id or the cooling @@ -60,7 +62,7 @@ struct cdev_debugfs { }; /** - * struct cdev_value - Common structure for cooling device entry + * struct cdev_record - Common structure for cooling device entry * * The following common structure allows to store the information * related to the transitions and to the state residencies. They are @@ -81,22 +83,89 @@ struct cdev_record { }; }; +/** + * struct trip_stats - Thermal trip statistics + * + * The trip_stats structure has the relevant information to show the + * statistics related to temperature going above a trip point. + * + * @timestamp: the trip crossing timestamp + * @duration: total time when the zone temperature was above the trip point + * @count: the number of times the zone temperature was above the trip point + * @max: maximum recorded temperature above the trip point + * @min: minimum recorded temperature above the trip point + * @avg: average temperature above the trip point + */ +struct trip_stats { + ktime_t timestamp; + ktime_t duration; + int count; + int max; + int min; + int avg; +}; + +/** + * struct tz_episode - A mitigation episode information + * + * The tz_episode structure describes a mitigation episode. A + * mitigation episode begins the trip point with the lower temperature + * is crossed the way up and ends when it is crossed the way + * down. During this episode we can have multiple trip points crossed + * the way up and down if there are multiple trip described in the + * firmware after the lowest temperature trip point. + * + * @timestamp: first trip point crossed the way up + * @duration: total duration of the mitigation episode + * @node: a list element to be added to the list of tz events + * @trip_stats: per trip point statistics, flexible array + */ +struct tz_episode { + ktime_t timestamp; + ktime_t duration; + struct list_head node; + struct trip_stats trip_stats[]; +}; + +/** + * struct tz_debugfs - Store all mitigation episodes for a thermal zone + * + * The tz_debugfs structure contains the list of the mitigation + * episodes and has to track which trip point has been crossed in + * order to handle correctly nested trip point mitigation episodes. + * + * We keep the history of the trip point crossed in an array and as we + * can go back and forth inside this history, eg. trip 0,1,2,1,2,1,0, + * we keep track of the current position in the history array. + * + * @tz_episodes: a list of thermal mitigation episodes + * @trips_crossed: an array of trip points crossed by id + * @nr_trips: the number of trip points currently being crossed + */ +struct tz_debugfs { + struct list_head tz_episodes; + int *trips_crossed; + int nr_trips; +}; + /** * struct thermal_debugfs - High level structure for a thermal object in debugfs * * The thermal_debugfs structure is the common structure used by the - * cooling device to compute the statistics. + * cooling device or the thermal zone to store the statistics. * * @d_top: top directory of the thermal object directory * @lock: per object lock to protect the internals * - * @cdev: a cooling device debug structure + * @cdev_dbg: a cooling device debug structure + * @tz_dbg: a thermal zone debug structure */ struct thermal_debugfs { struct dentry *d_top; struct mutex lock; union { struct cdev_debugfs cdev_dbg; + struct tz_debugfs tz_dbg; }; }; @@ -107,6 +176,10 @@ void thermal_debug_init(void) return; d_cdev = debugfs_create_dir("cooling_devices", d_root); + if (!d_cdev) + return; + + d_tz = debugfs_create_dir("thermal_zones", d_root); } static struct thermal_debugfs *thermal_debugfs_add_id(struct dentry *d, int id) @@ -444,3 +517,322 @@ void thermal_debug_cdev_remove(struct thermal_cooling_device *cdev) thermal_debugfs_remove_id(thermal_dbg); } + +static struct tz_episode *thermal_debugfs_tz_event_alloc(struct thermal_zone_device *tz, + ktime_t now) +{ + struct tz_episode *tze; + int i; + + tze = kzalloc(struct_size(tze, trip_stats, tz->num_trips), GFP_KERNEL); + if (!tze) + return NULL; + + INIT_LIST_HEAD(&tze->node); + tze->timestamp = now; + + for (i = 0; i < tz->num_trips; i++) { + tze->trip_stats[i].min = INT_MAX; + tze->trip_stats[i].max = INT_MIN; + } + + return tze; +} + +void thermal_debug_tz_trip_up(struct thermal_zone_device *tz, + const struct thermal_trip *trip) +{ + struct tz_episode *tze; + struct tz_debugfs *tz_dbg; + struct thermal_debugfs *thermal_dbg = tz->debugfs; + int temperature = tz->temperature; + int trip_id = thermal_zone_trip_id(tz, trip); + ktime_t now = ktime_get(); + + if (!thermal_dbg) + return; + + mutex_lock(&thermal_dbg->lock); + + tz_dbg = &thermal_dbg->tz_dbg; + + /* + * The mitigation is starting. A mitigation can contain + * several episodes where each of them is related to a + * temperature crossing a trip point. The episodes are + * nested. That means when the temperature is crossing the + * first trip point, the duration begins to be measured. If + * the temperature continues to increase and reaches the + * second trip point, the duration of the first trip must be + * also accumulated. + * + * eg. + * + * temp + * ^ + * | -------- + * trip 2 / \ ------ + * | /| |\ /| |\ + * trip 1 / | | `---- | | \ + * | /| | | | | |\ + * trip 0 / | | | | | | \ + * | /| | | | | | | |\ + * | / | | | | | | | | `-- + * | / | | | | | | | | + * |----- | | | | | | | | + * | | | | | | | | | + * --------|-|-|--------|--------|------|-|-|------------------> time + * | | |<--t2-->| |<-t2'>| | | + * | | | | + * | |<------------t1------------>| | + * | | + * |<-------------t0--------------->| + * + */ + if (!tz_dbg->nr_trips) { + tze = thermal_debugfs_tz_event_alloc(tz, now); + if (!tze) + return; + + list_add(&tze->node, &tz_dbg->tz_episodes); + } + + /* + * Each time a trip point is crossed the way up, the trip_id + * is stored in the trip_crossed array and the nr_trips is + * incremented. A nr_trips equal to zero means we are entering + * a mitigation episode. + * + * The trip ids may not be in the ascending order but the + * result in the array trips_crossed will be in the ascending + * temperature order. The function detecting when a trip point + * is crossed the way down will handle the very rare case when + * the trip points may have been reordered during this + * mitigation episode. + */ + tz_dbg->trips_crossed[tz_dbg->nr_trips++] = trip_id; + + tze = list_first_entry(&tz_dbg->tz_episodes, struct tz_episode, node); + tze->trip_stats[trip_id].timestamp = now; + tze->trip_stats[trip_id].max = max(tze->trip_stats[trip_id].max, temperature); + tze->trip_stats[trip_id].min = min(tze->trip_stats[trip_id].min, temperature); + tze->trip_stats[trip_id].avg = tze->trip_stats[trip_id].avg + + (temperature - tze->trip_stats[trip_id].avg) / + tze->trip_stats[trip_id].count; + + mutex_unlock(&thermal_dbg->lock); +} + +void thermal_debug_tz_trip_down(struct thermal_zone_device *tz, + const struct thermal_trip *trip) +{ + struct thermal_debugfs *thermal_dbg = tz->debugfs; + struct tz_episode *tze; + struct tz_debugfs *tz_dbg; + ktime_t delta, now = ktime_get(); + int trip_id = thermal_zone_trip_id(tz, trip); + int i; + + if (!thermal_dbg) + return; + + mutex_lock(&thermal_dbg->lock); + + tz_dbg = &thermal_dbg->tz_dbg; + + /* + * The temperature crosses the way down but there was not + * mitigation detected before. That may happen when the + * temperature is greater than a trip point when registering a + * thermal zone, which is a common use case as the kernel has + * no mitigation mechanism yet at boot time. + */ + if (!tz_dbg->nr_trips) + goto out; + + for (i = tz_dbg->nr_trips - 1; i >= 0; i--) { + if (tz_dbg->trips_crossed[i] == trip_id) + break; + } + + if (i < 0) + goto out; + + tz_dbg->nr_trips--; + + if (i < tz_dbg->nr_trips) + tz_dbg->trips_crossed[i] = tz_dbg->trips_crossed[tz_dbg->nr_trips]; + + tze = list_first_entry(&tz_dbg->tz_episodes, struct tz_episode, node); + + delta = ktime_sub(now, tze->trip_stats[trip_id].timestamp); + + tze->trip_stats[trip_id].duration = + ktime_add(delta, tze->trip_stats[trip_id].duration); + + /* + * This event closes the mitigation as we are crossing the + * last trip point the way down. + */ + if (!tz_dbg->nr_trips) + tze->duration = ktime_sub(now, tze->timestamp); + +out: + mutex_unlock(&thermal_dbg->lock); +} + +void thermal_debug_update_temp(struct thermal_zone_device *tz) +{ + struct thermal_debugfs *thermal_dbg = tz->debugfs; + struct tz_episode *tze; + struct tz_debugfs *tz_dbg; + int trip_id, i; + + if (!thermal_dbg) + return; + + mutex_lock(&thermal_dbg->lock); + + tz_dbg = &thermal_dbg->tz_dbg; + + if (!tz_dbg->nr_trips) + goto out; + + for (i = 0; i < tz_dbg->nr_trips; i++) { + trip_id = tz_dbg->trips_crossed[i]; + tze = list_first_entry(&tz_dbg->tz_episodes, struct tz_episode, node); + tze->trip_stats[trip_id].count++; + tze->trip_stats[trip_id].max = max(tze->trip_stats[trip_id].max, tz->temperature); + tze->trip_stats[trip_id].min = min(tze->trip_stats[trip_id].min, tz->temperature); + tze->trip_stats[trip_id].avg = tze->trip_stats[trip_id].avg + + (tz->temperature - tze->trip_stats[trip_id].avg) / + tze->trip_stats[trip_id].count; + } +out: + mutex_unlock(&thermal_dbg->lock); +} + +static void *tze_seq_start(struct seq_file *s, loff_t *pos) +{ + struct thermal_zone_device *tz = s->private; + struct thermal_debugfs *thermal_dbg = tz->debugfs; + struct tz_debugfs *tz_dbg = &thermal_dbg->tz_dbg; + + mutex_lock(&thermal_dbg->lock); + + return seq_list_start(&tz_dbg->tz_episodes, *pos); +} + +static void *tze_seq_next(struct seq_file *s, void *v, loff_t *pos) +{ + struct thermal_zone_device *tz = s->private; + struct thermal_debugfs *thermal_dbg = tz->debugfs; + struct tz_debugfs *tz_dbg = &thermal_dbg->tz_dbg; + + return seq_list_next(v, &tz_dbg->tz_episodes, pos); +} + +static void tze_seq_stop(struct seq_file *s, void *v) +{ + struct thermal_zone_device *tz = s->private; + struct thermal_debugfs *thermal_dbg = tz->debugfs; + + mutex_unlock(&thermal_dbg->lock); +} + +static int tze_seq_show(struct seq_file *s, void *v) +{ + struct thermal_zone_device *tz = s->private; + struct thermal_trip *trip; + struct tz_episode *tze; + const char *type; + int trip_id; + + tze = list_entry((struct list_head *)v, struct tz_episode, node); + + seq_printf(s, ",-Mitigation at %lluus, duration=%llums\n", + ktime_to_us(tze->timestamp), + ktime_to_ms(tze->duration)); + + seq_printf(s, "| trip | type | temp(°mC) | hyst(°mC) | duration | avg(°mC) | min(°mC) | max(°mC) |\n"); + + for_each_trip(tz, trip) { + /* + * There is no possible mitigation happening at the + * critical trip point, so the stats will be always + * zero, skip this trip point + */ + if (trip->type == THERMAL_TRIP_CRITICAL) + continue; + + if (trip->type == THERMAL_TRIP_PASSIVE) + type = "passive"; + else if (trip->type == THERMAL_TRIP_ACTIVE) + type = "active"; + else + type = "hot"; + + trip_id = thermal_zone_trip_id(tz, trip); + + seq_printf(s, "| %*d | %*s | %*d | %*d | %*lld | %*d | %*d | %*d |\n", + 4 , trip_id, + 8, type, + 9, trip->temperature, + 9, trip->hysteresis, + 10, ktime_to_ms(tze->trip_stats[trip_id].duration), + 9, tze->trip_stats[trip_id].avg, + 9, tze->trip_stats[trip_id].min, + 9, tze->trip_stats[trip_id].max); + } + + return 0; +} + +static const struct seq_operations tze_sops = { + .start = tze_seq_start, + .next = tze_seq_next, + .stop = tze_seq_stop, + .show = tze_seq_show, +}; + +DEFINE_SEQ_ATTRIBUTE(tze); + +void thermal_debug_tz_add(struct thermal_zone_device *tz) +{ + struct thermal_debugfs *thermal_dbg; + struct tz_debugfs *tz_dbg; + + thermal_dbg = thermal_debugfs_add_id(d_tz, tz->id); + if (!thermal_dbg) + return; + + tz_dbg = &thermal_dbg->tz_dbg; + + tz_dbg->trips_crossed = kzalloc(sizeof(int) * tz->num_trips, GFP_KERNEL); + if (!tz_dbg->trips_crossed) { + thermal_debugfs_remove_id(thermal_dbg); + return; + } + + INIT_LIST_HEAD(&tz_dbg->tz_episodes); + + debugfs_create_file("mitigations", 0400, thermal_dbg->d_top, tz, &tze_fops); + + tz->debugfs = thermal_dbg; +} + +void thermal_debug_tz_remove(struct thermal_zone_device *tz) +{ + struct thermal_debugfs *thermal_dbg = tz->debugfs; + + if (!thermal_dbg) + return; + + mutex_lock(&thermal_dbg->lock); + + tz->debugfs = NULL; + + mutex_unlock(&thermal_dbg->lock); + + thermal_debugfs_remove_id(thermal_dbg); +} diff --git a/drivers/thermal/thermal_debugfs.h b/drivers/thermal/thermal_debugfs.h index 341499388448..155b9af5fe87 100644 --- a/drivers/thermal/thermal_debugfs.h +++ b/drivers/thermal/thermal_debugfs.h @@ -5,10 +5,24 @@ void thermal_debug_init(void); void thermal_debug_cdev_add(struct thermal_cooling_device *cdev); void thermal_debug_cdev_remove(struct thermal_cooling_device *cdev); void thermal_debug_cdev_state_update(const struct thermal_cooling_device *cdev, int state); +void thermal_debug_tz_add(struct thermal_zone_device *tz); +void thermal_debug_tz_remove(struct thermal_zone_device *tz); +void thermal_debug_tz_trip_up(struct thermal_zone_device *tz, + const struct thermal_trip *trip); +void thermal_debug_tz_trip_down(struct thermal_zone_device *tz, + const struct thermal_trip *trip); +void thermal_debug_update_temp(struct thermal_zone_device *tz); #else static inline void thermal_debug_init(void) {} static inline void thermal_debug_cdev_add(struct thermal_cooling_device *cdev) {} static inline void thermal_debug_cdev_remove(struct thermal_cooling_device *cdev) {} static inline void thermal_debug_cdev_state_update(const struct thermal_cooling_device *cdev, int state) {} +static inline void thermal_debug_tz_add(struct thermal_zone_device *tz) {} +static inline void thermal_debug_tz_remove(struct thermal_zone_device *tz) {} +static inline void thermal_debug_tz_trip_up(struct thermal_zone_device *tz, + const struct thermal_trip *trip) {}; +static inline void thermal_debug_tz_trip_down(struct thermal_zone_device *tz, + const struct thermal_trip *trip) {} +static inline void thermal_debug_update_temp(struct thermal_zone_device *tz) {} #endif /* CONFIG_THERMAL_DEBUGFS */ From 57a427c81c322c5f0cdfe7c46cdee553d18b1ec6 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Fri, 5 Jan 2024 14:45:11 +0100 Subject: [PATCH 330/336] thermal: core: Use kstrdup_const() during cooling device registration Some *thermal_cooling_device_register() calls pass a string literal as the 'type' parameter, so kstrdup_const() can be used instead of kstrdup() to avoid a memory allocation in such cases. Signed-off-by: Christophe JAILLET [ rjw: Subject and changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/thermal/thermal_core.c | 6 +++--- include/linux/thermal.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c index abf4c2390b19..dfaa6341694a 100644 --- a/drivers/thermal/thermal_core.c +++ b/drivers/thermal/thermal_core.c @@ -845,7 +845,7 @@ static void thermal_release(struct device *dev) sizeof("cooling_device") - 1)) { cdev = to_cooling_device(dev); thermal_cooling_device_destroy_sysfs(cdev); - kfree(cdev->type); + kfree_const(cdev->type); ida_free(&thermal_cdev_ida, cdev->id); kfree(cdev); } @@ -917,7 +917,7 @@ __thermal_cooling_device_register(struct device_node *np, cdev->id = ret; id = ret; - cdev->type = kstrdup(type ? type : "", GFP_KERNEL); + cdev->type = kstrdup_const(type ? type : "", GFP_KERNEL); if (!cdev->type) { ret = -ENOMEM; goto out_ida_remove; @@ -970,7 +970,7 @@ __thermal_cooling_device_register(struct device_node *np, out_cooling_dev: thermal_cooling_device_destroy_sysfs(cdev); out_cdev_type: - kfree(cdev->type); + kfree_const(cdev->type); out_ida_remove: ida_free(&thermal_cdev_ida, id); out_kfree_cdev: diff --git a/include/linux/thermal.h b/include/linux/thermal.h index 7defea8fa223..3227335fb447 100644 --- a/include/linux/thermal.h +++ b/include/linux/thermal.h @@ -103,7 +103,7 @@ struct thermal_cooling_device_ops { struct thermal_cooling_device { int id; - char *type; + const char *type; unsigned long max_state; struct device device; struct device_node *np; From 11fde939314836c4375b567d60407d752d987069 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 9 Jan 2024 17:42:04 +0100 Subject: [PATCH 331/336] thermal: netlink: Rework notify API for cooling devices In analogy with some previous thermal netlink API changes, redefine thermal_notify_cdev_state_update(), thermal_notify_cdev_add() and thermal_notify_cdev_delete() to take a const cdev pointer as their first argument and let them extract the requisite information from there by themselves. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Daniel Lezcano --- drivers/thermal/thermal_helpers.c | 2 +- drivers/thermal/thermal_netlink.c | 15 ++++++++------- drivers/thermal/thermal_netlink.h | 16 +++++++++------- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/drivers/thermal/thermal_helpers.c b/drivers/thermal/thermal_helpers.c index 8f85581693e7..dd72eecfc7ca 100644 --- a/drivers/thermal/thermal_helpers.c +++ b/drivers/thermal/thermal_helpers.c @@ -156,7 +156,7 @@ static int thermal_cdev_set_cur_state(struct thermal_cooling_device *cdev, int s */ ret = cdev->ops->set_cur_state(cdev, state); if (!ret) { - thermal_notify_cdev_state_update(cdev->id, state); + thermal_notify_cdev_state_update(cdev, state); thermal_cooling_device_stats_update(cdev, state); thermal_debug_cdev_state_update(cdev, state); } diff --git a/drivers/thermal/thermal_netlink.c b/drivers/thermal/thermal_netlink.c index 46b0156a6319..76a231a29654 100644 --- a/drivers/thermal/thermal_netlink.c +++ b/drivers/thermal/thermal_netlink.c @@ -364,24 +364,25 @@ int thermal_notify_tz_trip_change(const struct thermal_zone_device *tz, return thermal_genl_send_event(THERMAL_GENL_EVENT_TZ_TRIP_CHANGE, &p); } -int thermal_notify_cdev_state_update(int cdev_id, int cdev_state) +int thermal_notify_cdev_state_update(const struct thermal_cooling_device *cdev, + int state) { - struct param p = { .cdev_id = cdev_id, .cdev_state = cdev_state }; + struct param p = { .cdev_id = cdev->id, .cdev_state = state }; return thermal_genl_send_event(THERMAL_GENL_EVENT_CDEV_STATE_UPDATE, &p); } -int thermal_notify_cdev_add(int cdev_id, const char *name, int cdev_max_state) +int thermal_notify_cdev_add(const struct thermal_cooling_device *cdev) { - struct param p = { .cdev_id = cdev_id, .name = name, - .cdev_max_state = cdev_max_state }; + struct param p = { .cdev_id = cdev->id, .name = cdev->type, + .cdev_max_state = cdev->max_state }; return thermal_genl_send_event(THERMAL_GENL_EVENT_CDEV_ADD, &p); } -int thermal_notify_cdev_delete(int cdev_id) +int thermal_notify_cdev_delete(const struct thermal_cooling_device *cdev) { - struct param p = { .cdev_id = cdev_id }; + struct param p = { .cdev_id = cdev->id }; return thermal_genl_send_event(THERMAL_GENL_EVENT_CDEV_DELETE, &p); } diff --git a/drivers/thermal/thermal_netlink.h b/drivers/thermal/thermal_netlink.h index 13ea2ad23d3a..93a927e144d5 100644 --- a/drivers/thermal/thermal_netlink.h +++ b/drivers/thermal/thermal_netlink.h @@ -12,6 +12,7 @@ struct thermal_genl_cpu_caps { struct thermal_zone_device; struct thermal_trip; +struct thermal_cooling_device; /* Netlink notification function */ #ifdef CONFIG_THERMAL_NETLINK @@ -27,9 +28,10 @@ int thermal_notify_tz_trip_up(const struct thermal_zone_device *tz, const struct thermal_trip *trip); int thermal_notify_tz_trip_change(const struct thermal_zone_device *tz, const struct thermal_trip *trip); -int thermal_notify_cdev_state_update(int cdev_id, int state); -int thermal_notify_cdev_add(int cdev_id, const char *name, int max_state); -int thermal_notify_cdev_delete(int cdev_id); +int thermal_notify_cdev_state_update(const struct thermal_cooling_device *cdev, + int state); +int thermal_notify_cdev_add(const struct thermal_cooling_device *cdev); +int thermal_notify_cdev_delete(const struct thermal_cooling_device *cdev); int thermal_notify_tz_gov_change(const struct thermal_zone_device *tz, const char *name); int thermal_genl_sampling_temp(int id, int temp); @@ -79,18 +81,18 @@ static inline int thermal_notify_tz_trip_change(const struct thermal_zone_device return 0; } -static inline int thermal_notify_cdev_state_update(int cdev_id, int state) +static inline int thermal_notify_cdev_state_update(const struct thermal_cooling_device *cdev, + int state) { return 0; } -static inline int thermal_notify_cdev_add(int cdev_id, const char *name, - int max_state) +static inline int thermal_notify_cdev_add(const struct thermal_cooling_device *cdev) { return 0; } -static inline int thermal_notify_cdev_delete(int cdev_id) +static inline int thermal_notify_cdev_delete(const struct thermal_cooling_device *cdev) { return 0; } From fd881eac3af6c4e36b34ce92d69fb1d7e95f5920 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 9 Jan 2024 17:42:48 +0100 Subject: [PATCH 332/336] thermal: helpers: Rearrange thermal_cdev_set_cur_state() Change the code layout in thermal_cdev_set_cur_state() so it returns early on errors which is more consistent with what happens elsewhere. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Daniel Lezcano --- drivers/thermal/thermal_helpers.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/thermal/thermal_helpers.c b/drivers/thermal/thermal_helpers.c index dd72eecfc7ca..0329f4a71b02 100644 --- a/drivers/thermal/thermal_helpers.c +++ b/drivers/thermal/thermal_helpers.c @@ -155,13 +155,14 @@ static int thermal_cdev_set_cur_state(struct thermal_cooling_device *cdev, int s * registering function checked the ops are correctly set */ ret = cdev->ops->set_cur_state(cdev, state); - if (!ret) { - thermal_notify_cdev_state_update(cdev, state); - thermal_cooling_device_stats_update(cdev, state); - thermal_debug_cdev_state_update(cdev, state); - } + if (ret) + return ret; - return ret; + thermal_notify_cdev_state_update(cdev, state); + thermal_cooling_device_stats_update(cdev, state); + thermal_debug_cdev_state_update(cdev, state); + + return 0; } void __thermal_cdev_update(struct thermal_cooling_device *cdev) From e95fa7404716f6e25021e66067271a4ad8eb1486 Mon Sep 17 00:00:00 2001 From: Di Shen Date: Wed, 10 Jan 2024 19:55:26 +0800 Subject: [PATCH 333/336] thermal: gov_power_allocator: avoid inability to reset a cdev Commit 0952177f2a1f ("thermal/core/power_allocator: Update once cooling devices when temp is low") adds an update flag to avoid triggering a thermal event when there is no need, and the thermal cdev is updated once when the temperature is low. But when the trips are writable, and switch_on_temp is set to be a higher value, the cooling device state may not be reset to 0, because last_temperature is smaller than switch_on_temp. For example: First: switch_on_temp=70 control_temp=85; Then userspace change the trip_temp: switch_on_temp=45 control_temp=55 cur_temp=54 Then userspace reset the trip_temp: switch_on_temp=70 control_temp=85 cur_temp=57 last_temp=54 At this time, the cooling device state should be reset to 0. However, because cur_temp(57) < switch_on_temp(70) last_temp(54) < switch_on_temp(70) ----> update = false, update is false, the cooling device state can not be reset. Using the observation that tz->passive can also be regarded as the temperature status, set the update flag to the tz->passive value. When the temperature drops below switch_on for the first time, the states of cooling devices can be reset once, and tz->passive is updated to 0. In the next round, because tz->passive is 0, cdev->state will not be updated. By using the tz->passive value as the "update" flag, the issue above can be solved, and the cooling devices can be updated only once when the temperature is low. Fixes: 0952177f2a1f ("thermal/core/power_allocator: Update once cooling devices when temp is low") Cc: 5.13+ # 5.13+ Suggested-by: Wei Wang Signed-off-by: Di Shen Reviewed-by: Lukasz Luba [ rjw: Subject and changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/thermal/gov_power_allocator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/thermal/gov_power_allocator.c b/drivers/thermal/gov_power_allocator.c index 7b6aa265ff6a..81e061f183ad 100644 --- a/drivers/thermal/gov_power_allocator.c +++ b/drivers/thermal/gov_power_allocator.c @@ -762,7 +762,7 @@ static int power_allocator_throttle(struct thermal_zone_device *tz, trip = params->trip_switch_on; if (trip && tz->temperature < trip->temperature) { - update = tz->last_temperature >= trip->temperature; + update = tz->passive; tz->passive = 0; reset_pid_controller(params); allow_maximum_power(tz, update); From 97566d09fd02d2ab329774bb89a2cdf2267e86d9 Mon Sep 17 00:00:00 2001 From: Ricardo Neri Date: Tue, 9 Jan 2024 19:07:04 -0800 Subject: [PATCH 334/336] thermal: intel: hfi: Add syscore callbacks for system-wide PM The kernel allocates a memory buffer and provides its location to the hardware, which uses it to update the HFI table. This allocation occurs during boot and remains constant throughout runtime. When resuming from hibernation, the restore kernel allocates a second memory buffer and reprograms the HFI hardware with the new location as part of a normal boot. The location of the second memory buffer may differ from the one allocated by the image kernel. When the restore kernel transfers control to the image kernel, its HFI buffer becomes invalid, potentially leading to memory corruption if the hardware writes to it (the hardware continues to use the buffer from the restore kernel). It is also possible that the hardware "forgets" the address of the memory buffer when resuming from "deep" suspend. Memory corruption may also occur in such a scenario. To prevent the described memory corruption, disable HFI when preparing to suspend or hibernate. Enable it when resuming. Add syscore callbacks to handle the package of the boot CPU (packages of non-boot CPUs are handled via CPU offline). Syscore ops always run on the boot CPU. Additionally, HFI only needs to be disabled during "deep" suspend and hibernation. Syscore ops only run in these cases. Cc: 6.1+ # 6.1+ Signed-off-by: Ricardo Neri [ rjw: Comment adjustment, subject and changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/thermal/intel/intel_hfi.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/drivers/thermal/intel/intel_hfi.c b/drivers/thermal/intel/intel_hfi.c index 22445403b520..3b04c6ec4fca 100644 --- a/drivers/thermal/intel/intel_hfi.c +++ b/drivers/thermal/intel/intel_hfi.c @@ -35,7 +35,9 @@ #include #include #include +#include #include +#include #include #include @@ -571,6 +573,30 @@ static __init int hfi_parse_features(void) return 0; } +static void hfi_do_enable(void) +{ + /* This code runs only on the boot CPU. */ + struct hfi_cpu_info *info = &per_cpu(hfi_cpu_info, 0); + struct hfi_instance *hfi_instance = info->hfi_instance; + + /* No locking needed. There is no concurrency with CPU online. */ + hfi_set_hw_table(hfi_instance); + hfi_enable(); +} + +static int hfi_do_disable(void) +{ + /* No locking needed. There is no concurrency with CPU offline. */ + hfi_disable(); + + return 0; +} + +static struct syscore_ops hfi_pm_ops = { + .resume = hfi_do_enable, + .suspend = hfi_do_disable, +}; + void __init intel_hfi_init(void) { struct hfi_instance *hfi_instance; @@ -602,6 +628,8 @@ void __init intel_hfi_init(void) if (!hfi_updates_wq) goto err_nomem; + register_syscore_ops(&hfi_pm_ops); + return; err_nomem: From 6dcb35088e264306b948141971286257681ca412 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 12 Jan 2024 17:30:18 +0300 Subject: [PATCH 335/336] thermal/debugfs: Unlock on error path in thermal_debug_tz_trip_up() Add a missing mutex_unlock(&thermal_dbg->lock) to this error path. Fixes: 7ef01f228c9f ("thermal/debugfs: Add thermal debugfs information for mitigation episodes") Signed-off-by: Dan Carpenter Signed-off-by: Rafael J. Wysocki --- drivers/thermal/thermal_debugfs.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/thermal/thermal_debugfs.c b/drivers/thermal/thermal_debugfs.c index b53881e9e0b6..c617e8b9f0dd 100644 --- a/drivers/thermal/thermal_debugfs.c +++ b/drivers/thermal/thermal_debugfs.c @@ -592,7 +592,7 @@ void thermal_debug_tz_trip_up(struct thermal_zone_device *tz, if (!tz_dbg->nr_trips) { tze = thermal_debugfs_tz_event_alloc(tz, now); if (!tze) - return; + goto unlock; list_add(&tze->node, &tz_dbg->tz_episodes); } @@ -620,6 +620,7 @@ void thermal_debug_tz_trip_up(struct thermal_zone_device *tz, (temperature - tze->trip_stats[trip_id].avg) / tze->trip_stats[trip_id].count; +unlock: mutex_unlock(&thermal_dbg->lock); } From 205e18c13545ab43cc4fe4930732b4feef551198 Mon Sep 17 00:00:00 2001 From: Dave Airlie Date: Wed, 10 Jan 2024 11:14:05 +1000 Subject: [PATCH 336/336] nouveau/gsp: handle engines in runl without nonstall interrupts. It appears on TU106 GPUs (2070), that some of the nvdec engines are in the runlist but have no valid nonstall interrupt, nouveau didn't handle that too well. This should let nouveau/gsp work on those. Cc: stable@vger.kernel.org # v6.7+ Signed-off-by: Dave Airlie Link: https://lore.kernel.org/all/20240110011826.3996289-1-airlied@gmail.com/ --- drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c | 4 ++++ drivers/gpu/drm/nouveau/nvkm/engine/fifo/r535.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/gsp/base.c | 8 ++------ 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c index c8ce7ff18713..e74493a4569e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c @@ -550,6 +550,10 @@ ga100_fifo_nonstall_ctor(struct nvkm_fifo *fifo) struct nvkm_engn *engn = list_first_entry(&runl->engns, typeof(*engn), head); runl->nonstall.vector = engn->func->nonstall(engn); + + /* if no nonstall vector just keep going */ + if (runl->nonstall.vector == -1) + continue; if (runl->nonstall.vector < 0) { RUNL_ERROR(runl, "nonstall %d", runl->nonstall.vector); return runl->nonstall.vector; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/r535.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/r535.c index d088e636edc3..de2ebe8f2134 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/r535.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/r535.c @@ -350,7 +350,7 @@ r535_engn_nonstall(struct nvkm_engn *engn) int ret; ret = nvkm_gsp_intr_nonstall(subdev->device->gsp, subdev->type, subdev->inst); - WARN_ON(ret < 0); + WARN_ON(ret == -ENOENT); return ret; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/base.c index 04bceaa28a19..da1bebb896f7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/base.c @@ -25,12 +25,8 @@ int nvkm_gsp_intr_nonstall(struct nvkm_gsp *gsp, enum nvkm_subdev_type type, int inst) { for (int i = 0; i < gsp->intr_nr; i++) { - if (gsp->intr[i].type == type && gsp->intr[i].inst == inst) { - if (gsp->intr[i].nonstall != ~0) - return gsp->intr[i].nonstall; - - return -EINVAL; - } + if (gsp->intr[i].type == type && gsp->intr[i].inst == inst) + return gsp->intr[i].nonstall; } return -ENOENT;