diff --git a/drivers/virt/gunyah/Kconfig b/drivers/virt/gunyah/Kconfig index 6f4c85db80b5..23ba523d25dc 100644 --- a/drivers/virt/gunyah/Kconfig +++ b/drivers/virt/gunyah/Kconfig @@ -3,6 +3,7 @@ config GUNYAH tristate "Gunyah Virtualization drivers" depends on ARM64 + select GUNYAH_PLATFORM_HOOKS help The Gunyah drivers are the helper interfaces that run in a guest VM such as basic inter-VM IPC and signaling mechanisms, and higher level @@ -10,3 +11,6 @@ config GUNYAH Say Y/M here to enable the drivers needed to interact in a Gunyah virtual environment. + +config GUNYAH_PLATFORM_HOOKS + tristate diff --git a/drivers/virt/gunyah/Makefile b/drivers/virt/gunyah/Makefile index f3c9507224ee..ffcde0e0ccfa 100644 --- a/drivers/virt/gunyah/Makefile +++ b/drivers/virt/gunyah/Makefile @@ -3,3 +3,4 @@ gunyah_rsc_mgr-y += rsc_mgr.o rsc_mgr_rpc.o vm_mgr.o vm_mgr_mem.o obj-$(CONFIG_GUNYAH) += gunyah.o gunyah_rsc_mgr.o gunyah_vcpu.o +obj-$(CONFIG_GUNYAH_PLATFORM_HOOKS) += gunyah_platform_hooks.o diff --git a/drivers/virt/gunyah/gunyah_platform_hooks.c b/drivers/virt/gunyah/gunyah_platform_hooks.c new file mode 100644 index 000000000000..b03795df0014 --- /dev/null +++ b/drivers/virt/gunyah/gunyah_platform_hooks.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2022-2024 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include +#include +#include + +#include "rsc_mgr.h" + +static const struct gunyah_rm_platform_ops *rm_platform_ops; +static DECLARE_RWSEM(rm_platform_ops_lock); + +int gunyah_rm_platform_pre_mem_share(struct gunyah_rm *rm, + struct gunyah_rm_mem_parcel *mem_parcel) +{ + int ret = 0; + + down_read(&rm_platform_ops_lock); + if (rm_platform_ops && rm_platform_ops->pre_mem_share) + ret = rm_platform_ops->pre_mem_share(rm, mem_parcel); + up_read(&rm_platform_ops_lock); + return ret; +} +EXPORT_SYMBOL_GPL(gunyah_rm_platform_pre_mem_share); + +int gunyah_rm_platform_post_mem_reclaim(struct gunyah_rm *rm, + struct gunyah_rm_mem_parcel *mem_parcel) +{ + int ret = 0; + + down_read(&rm_platform_ops_lock); + if (rm_platform_ops && rm_platform_ops->post_mem_reclaim) + ret = rm_platform_ops->post_mem_reclaim(rm, mem_parcel); + up_read(&rm_platform_ops_lock); + return ret; +} +EXPORT_SYMBOL_GPL(gunyah_rm_platform_post_mem_reclaim); + +int gunyah_rm_platform_pre_demand_page(struct gunyah_rm *rm, u16 vmid, + enum gunyah_pagetable_access access, + struct folio *folio) +{ + int ret = 0; + + down_read(&rm_platform_ops_lock); + if (rm_platform_ops && rm_platform_ops->pre_demand_page) + ret = rm_platform_ops->pre_demand_page(rm, vmid, access, folio); + up_read(&rm_platform_ops_lock); + return ret; +} +EXPORT_SYMBOL_GPL(gunyah_rm_platform_pre_demand_page); + +int gunyah_rm_platform_reclaim_demand_page(struct gunyah_rm *rm, u16 vmid, + enum gunyah_pagetable_access access, + struct folio *folio) +{ + int ret = 0; + + down_read(&rm_platform_ops_lock); + if (rm_platform_ops && rm_platform_ops->pre_demand_page) + ret = rm_platform_ops->release_demand_page(rm, vmid, access, + folio); + up_read(&rm_platform_ops_lock); + return ret; +} +EXPORT_SYMBOL_GPL(gunyah_rm_platform_reclaim_demand_page); + +int gunyah_rm_register_platform_ops( + const struct gunyah_rm_platform_ops *platform_ops) +{ + int ret = 0; + + down_write(&rm_platform_ops_lock); + if (!rm_platform_ops) + rm_platform_ops = platform_ops; + else + ret = -EEXIST; + up_write(&rm_platform_ops_lock); + return ret; +} +EXPORT_SYMBOL_GPL(gunyah_rm_register_platform_ops); + +void gunyah_rm_unregister_platform_ops( + const struct gunyah_rm_platform_ops *platform_ops) +{ + down_write(&rm_platform_ops_lock); + if (rm_platform_ops == platform_ops) + rm_platform_ops = NULL; + up_write(&rm_platform_ops_lock); +} +EXPORT_SYMBOL_GPL(gunyah_rm_unregister_platform_ops); + +static void _devm_gunyah_rm_unregister_platform_ops(void *data) +{ + gunyah_rm_unregister_platform_ops( + (const struct gunyah_rm_platform_ops *)data); +} + +int devm_gunyah_rm_register_platform_ops( + struct device *dev, const struct gunyah_rm_platform_ops *ops) +{ + int ret; + + ret = gunyah_rm_register_platform_ops(ops); + if (ret) + return ret; + + return devm_add_action(dev, _devm_gunyah_rm_unregister_platform_ops, + (void *)ops); +} +EXPORT_SYMBOL_GPL(devm_gunyah_rm_register_platform_ops); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Gunyah Platform Hooks"); diff --git a/drivers/virt/gunyah/rsc_mgr.h b/drivers/virt/gunyah/rsc_mgr.h index ec8ad8149e8e..68d08d3cff02 100644 --- a/drivers/virt/gunyah/rsc_mgr.h +++ b/drivers/virt/gunyah/rsc_mgr.h @@ -117,4 +117,14 @@ int gunyah_rm_call(struct gunyah_rm *rsc_mgr, u32 message_id, const void *req_buf, size_t req_buf_size, void **resp_buf, size_t *resp_buf_size); +int gunyah_rm_platform_pre_mem_share(struct gunyah_rm *rm, + struct gunyah_rm_mem_parcel *mem_parcel); +int gunyah_rm_platform_post_mem_reclaim( + struct gunyah_rm *rm, struct gunyah_rm_mem_parcel *mem_parcel); + +int gunyah_rm_platform_pre_demand_page(struct gunyah_rm *rm, u16 vmid, + u32 flags, struct folio *folio); +int gunyah_rm_platform_reclaim_demand_page(struct gunyah_rm *rm, u16 vmid, + u32 flags, struct folio *folio); + #endif diff --git a/drivers/virt/gunyah/rsc_mgr_rpc.c b/drivers/virt/gunyah/rsc_mgr_rpc.c index eb06940bb3c5..99e5a3f1a5bb 100644 --- a/drivers/virt/gunyah/rsc_mgr_rpc.c +++ b/drivers/virt/gunyah/rsc_mgr_rpc.c @@ -209,6 +209,12 @@ int gunyah_rm_mem_share(struct gunyah_rm *rm, struct gunyah_rm_mem_parcel *p) if (!msg) return -ENOMEM; + ret = gunyah_rm_platform_pre_mem_share(rm, p); + if (ret) { + kfree(msg); + return ret; + } + req_header = msg; acl = (void *)req_header + sizeof(*req_header); mem = (void *)acl + acl_size; @@ -234,8 +240,10 @@ int gunyah_rm_mem_share(struct gunyah_rm *rm, struct gunyah_rm_mem_parcel *p) &resp_size); kfree(msg); - if (ret) + if (ret) { + gunyah_rm_platform_post_mem_reclaim(rm, p); return ret; + } p->mem_handle = le32_to_cpu(*resp); kfree(resp); @@ -267,9 +275,15 @@ int gunyah_rm_mem_reclaim(struct gunyah_rm *rm, struct gunyah_rm_mem_release_req req = { .mem_handle = cpu_to_le32(parcel->mem_handle), }; + int ret; - return gunyah_rm_call(rm, GUNYAH_RM_RPC_MEM_RECLAIM, &req, sizeof(req), - NULL, NULL); + ret = gunyah_rm_call(rm, GUNYAH_RM_RPC_MEM_RECLAIM, &req, sizeof(req), + NULL, NULL); + /* Only call platform mem reclaim hooks if we reclaimed the memory */ + if (ret) + return ret; + + return gunyah_rm_platform_post_mem_reclaim(rm, parcel); } ALLOW_ERROR_INJECTION(gunyah_rm_mem_reclaim, ERRNO); diff --git a/drivers/virt/gunyah/vm_mgr_mem.c b/drivers/virt/gunyah/vm_mgr_mem.c index bcc84473004b..5a4ea2b1a12e 100644 --- a/drivers/virt/gunyah/vm_mgr_mem.c +++ b/drivers/virt/gunyah/vm_mgr_mem.c @@ -9,6 +9,7 @@ #include #include +#include "rsc_mgr.h" #include "vm_mgr.h" #define WRITE_TAG (1 << 0) @@ -107,7 +108,7 @@ int gunyah_vm_provide_folio(struct gunyah_vm *ghvm, struct folio *folio, size_t size = folio_size(folio); enum gunyah_error gunyah_error; unsigned long tag = 0; - int ret; + int ret, tmp; /* clang-format off */ if (share) { @@ -158,6 +159,11 @@ int gunyah_vm_provide_folio(struct gunyah_vm *ghvm, struct folio *folio, else /* !share && !write */ access = GUNYAH_PAGETABLE_ACCESS_RX; + ret = gunyah_rm_platform_pre_demand_page(ghvm->rm, ghvm->vmid, access, + folio); + if (ret) + goto reclaim_host; + gunyah_error = gunyah_hypercall_memextent_donate(donate_flags(share), host_extent->capid, guest_extent->capid, @@ -166,7 +172,7 @@ int gunyah_vm_provide_folio(struct gunyah_vm *ghvm, struct folio *folio, pr_err("Failed to donate memory for guest address 0x%016llx: %d\n", gpa, gunyah_error); ret = gunyah_error_remap(gunyah_error); - goto remove; + goto platform_release; } extent_attrs = @@ -197,6 +203,16 @@ memextent_reclaim: if (gunyah_error != GUNYAH_ERROR_OK) pr_err("Failed to reclaim memory donation for guest address 0x%016llx: %d\n", gpa, gunyah_error); +platform_release: + tmp = gunyah_rm_platform_reclaim_demand_page(ghvm->rm, ghvm->vmid, + access, folio); + if (tmp) { + pr_err("Platform failed to reclaim memory for guest address 0x%016llx: %d", + gpa, tmp); + return ret; + } +reclaim_host: + gunyah_folio_host_reclaim(folio); remove: mtree_erase(&ghvm->mm, gfn); return ret; @@ -207,6 +223,7 @@ static int __gunyah_vm_reclaim_folio_locked(struct gunyah_vm *ghvm, void *entry, { u32 map_flags = BIT(GUNYAH_ADDRSPACE_MAP_FLAG_PARTIAL); struct gunyah_resource *guest_extent, *host_extent, *addrspace; + enum gunyah_pagetable_access access; enum gunyah_error gunyah_error; struct folio *folio; bool write, share; @@ -264,6 +281,24 @@ static int __gunyah_vm_reclaim_folio_locked(struct gunyah_vm *ghvm, void *entry, goto err; } + if (share && write) + access = GUNYAH_PAGETABLE_ACCESS_RW; + else if (share && !write) + access = GUNYAH_PAGETABLE_ACCESS_R; + else if (!share && write) + access = GUNYAH_PAGETABLE_ACCESS_RWX; + else /* !share && !write */ + access = GUNYAH_PAGETABLE_ACCESS_RX; + + ret = gunyah_rm_platform_reclaim_demand_page(ghvm->rm, ghvm->vmid, + access, folio); + if (ret) { + pr_err_ratelimited( + "Platform failed to reclaim memory for guest address 0x%016llx: %d", + gunyah_gfn_to_gpa(gfn), ret); + goto err; + } + BUG_ON(mtree_erase(&ghvm->mm, gfn) != entry); if (folio_test_private(folio)) { diff --git a/include/linux/gunyah.h b/include/linux/gunyah.h index c23e1a88703e..cfb80f40d4e4 100644 --- a/include/linux/gunyah.h +++ b/include/linux/gunyah.h @@ -198,6 +198,57 @@ struct gunyah_rm_mem_parcel { u32 mem_handle; }; +enum gunyah_pagetable_access { + /* clang-format off */ + GUNYAH_PAGETABLE_ACCESS_NONE = 0, + GUNYAH_PAGETABLE_ACCESS_X = 1, + GUNYAH_PAGETABLE_ACCESS_W = 2, + GUNYAH_PAGETABLE_ACCESS_R = 4, + GUNYAH_PAGETABLE_ACCESS_RX = 5, + GUNYAH_PAGETABLE_ACCESS_RW = 6, + GUNYAH_PAGETABLE_ACCESS_RWX = 7, + /* clang-format on */ +}; + +struct gunyah_rm_platform_ops { + int (*pre_mem_share)(struct gunyah_rm *rm, + struct gunyah_rm_mem_parcel *mem_parcel); + int (*post_mem_reclaim)(struct gunyah_rm *rm, + struct gunyah_rm_mem_parcel *mem_parcel); + + int (*pre_demand_page)(struct gunyah_rm *rm, u16 vmid, + enum gunyah_pagetable_access access, + struct folio *folio); + int (*release_demand_page)(struct gunyah_rm *rm, u16 vmid, + enum gunyah_pagetable_access access, + struct folio *folio); +}; + +#if IS_ENABLED(CONFIG_GUNYAH_PLATFORM_HOOKS) +int gunyah_rm_register_platform_ops( + const struct gunyah_rm_platform_ops *platform_ops); +void gunyah_rm_unregister_platform_ops( + const struct gunyah_rm_platform_ops *platform_ops); +int devm_gunyah_rm_register_platform_ops( + struct device *dev, const struct gunyah_rm_platform_ops *ops); +#else +static inline int gunyah_rm_register_platform_ops( + const struct gunyah_rm_platform_ops *platform_ops) +{ + return 0; +} +static inline void gunyah_rm_unregister_platform_ops( + const struct gunyah_rm_platform_ops *platform_ops) +{ +} +static inline int +devm_gunyah_rm_register_platform_ops(struct device *dev, + const struct gunyah_rm_platform_ops *ops) +{ + return 0; +} +#endif + /******************************************************************************/ /* Common arch-independent definitions for Gunyah hypercalls */ #define GUNYAH_CAPID_INVAL U64_MAX @@ -318,18 +369,6 @@ enum gunyah_error gunyah_hypercall_msgq_recv(u64 capid, void *buff, size_t size, #define GUNYAH_ADDRSPACE_SELF_CAP 0 -enum gunyah_pagetable_access { - /* clang-format off */ - GUNYAH_PAGETABLE_ACCESS_NONE = 0, - GUNYAH_PAGETABLE_ACCESS_X = 1, - GUNYAH_PAGETABLE_ACCESS_W = 2, - GUNYAH_PAGETABLE_ACCESS_R = 4, - GUNYAH_PAGETABLE_ACCESS_RX = 5, - GUNYAH_PAGETABLE_ACCESS_RW = 6, - GUNYAH_PAGETABLE_ACCESS_RWX = 7, - /* clang-format on */ -}; - /* clang-format off */ #define GUNYAH_MEMEXTENT_MAPPING_USER_ACCESS GENMASK_ULL(2, 0) #define GUNYAH_MEMEXTENT_MAPPING_KERNEL_ACCESS GENMASK_ULL(6, 4)