From 9d2f5a0465c3ec7ce902f85f2238dc150418dcce Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Fri, 30 Jun 2023 17:14:17 +0100 Subject: [PATCH] ANDROID: KVM: arm64: Add atomics-based checking refcount implementation at EL2 The current nVHE refcount implementation at EL2 uses a simple spinlock to serialise accesses. Although this works, it forces serialisation in places where it is not necessary, so introduce a simple atomics-based refcount implementation instead. Bug: 357781595 Change-Id: I238861fe1cc2880c552c239cdd707dfa8a400be2 Signed-off-by: Will Deacon Signed-off-by: Fuad Tabba --- arch/arm64/kvm/hyp/include/nvhe/refcount.h | 76 ++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 arch/arm64/kvm/hyp/include/nvhe/refcount.h diff --git a/arch/arm64/kvm/hyp/include/nvhe/refcount.h b/arch/arm64/kvm/hyp/include/nvhe/refcount.h new file mode 100644 index 000000000000..bf6bd4ef89bf --- /dev/null +++ b/arch/arm64/kvm/hyp/include/nvhe/refcount.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Atomics-based checking refcount implementation. + * Copyright (C) 2023 Google LLC + * Author: Will Deacon + */ +#ifndef __ARM64_KVM_NVHE_REFCOUNT_H__ +#define __ARM64_KVM_NVHE_REFCOUNT_H__ + +#include + +static inline s16 __ll_sc_refcount_fetch_add_16(u16 *refcount, s16 addend) +{ + u16 new; + u32 flag; + + asm volatile( + " prfm pstl1strm, %[refcount]\n" + "1: ldxrh %w[new], %[refcount]\n" + " add %w[new], %w[new], %w[addend]\n" + " stxrh %w[flag], %w[new], %[refcount]\n" + " cbnz %w[flag], 1b" + : [refcount] "+Q" (*refcount), + [new] "=&r" (new), + [flag] "=&r" (flag) + : [addend] "Ir" (addend)); + + return new; +} + +#ifdef CONFIG_ARM64_LSE_ATOMICS + +static inline s16 __lse_refcount_fetch_add_16(u16 *refcount, s16 addend) +{ + s16 old; + + asm volatile(__LSE_PREAMBLE + " ldaddh %w[addend], %w[old], %[refcount]" + : [refcount] "+Q" (*refcount), + [old] "=r" (old) + : [addend] "r" (addend)); + + return old + addend; +} + +#endif /* CONFIG_ARM64_LSE_ATOMICS */ + +static inline u64 __hyp_refcount_fetch_add(void *refcount, const size_t size, + const s64 addend) +{ + s64 new; + + switch (size) { + case 2: + new = __lse_ll_sc_body(refcount_fetch_add_16, refcount, addend); + break; + default: + BUILD_BUG_ON_MSG(1, "Unsupported refcount size"); + unreachable(); + } + + BUG_ON(new < 0); + return new; +} + + +#define hyp_refcount_inc(r) __hyp_refcount_fetch_add(&(r), sizeof(r), 1) +#define hyp_refcount_dec(r) __hyp_refcount_fetch_add(&(r), sizeof(r), -1) +#define hyp_refcount_get(r) READ_ONCE(r) +#define hyp_refcount_set(r, v) do { \ + typeof(r) *__rp = &(r); \ + WARN_ON(hyp_refcount_get(*__rp)); \ + WRITE_ONCE(*__rp, v); \ +} while (0) + +#endif /* __ARM64_KVM_NVHE_REFCOUNT_H__ */