diff --git a/arch/arm64/kvm/hyp/include/nvhe/alloc.h b/arch/arm64/kvm/hyp/include/nvhe/alloc.h new file mode 100644 index 000000000000..c704711b2265 --- /dev/null +++ b/arch/arm64/kvm/hyp/include/nvhe/alloc.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __KVM_NVHE_ALLOC__ +#define __KVM_NVHE_ALLOC__ +#include + +#include + +/** + * hyp_alloc() - Allocate memory from the heap allocator + * + * @size: Allocation size in bytes. + * + * Return: A pointer to the allocated memory on success, else NULL. + */ +void *hyp_alloc(size_t size); + +/** + * hyp_alloc_errno() - Read the errno on allocation error + * + * Get the return code from an allocation failure. + * + * Return: -ENOMEM if the allocator needs a refill from the host, -E2BIG if + * there is no VA space left else 0. + */ +int hyp_alloc_errno(void); + +/** + * hyp_free() - Free memory allocated with hyp_alloc() + * + * @addr: Address returned by the original hyp_alloc(). + * + * The use of any other address than one returned by hyp_alloc() will cause a + * hypervisor panic. + */ +void hyp_free(void *addr); + +int hyp_alloc_init(size_t size); +int hyp_alloc_refill(struct kvm_hyp_memcache *host_mc); +int hyp_alloc_reclaimable(void); +void hyp_alloc_reclaim(struct kvm_hyp_memcache *host_mc, int target); +#endif diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile index ace9f4094cc5..ec653eebcf9b 100644 --- a/arch/arm64/kvm/hyp/nvhe/Makefile +++ b/arch/arm64/kvm/hyp/nvhe/Makefile @@ -6,7 +6,7 @@ lib-objs := $(addprefix ../../../lib/, $(lib-objs)) CFLAGS_switch.nvhe.o += -Wno-override-init hyp-obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o \ - hyp-main.o hyp-smp.o psci-relay.o early_alloc.o page_alloc.o \ + hyp-main.o hyp-smp.o psci-relay.o alloc.o early_alloc.o page_alloc.o \ cache.o setup.o mm.o mem_protect.o sys_regs.o pkvm.o stacktrace.o ffa.o \ serial.o hyp-obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \ diff --git a/arch/arm64/kvm/hyp/nvhe/alloc.c b/arch/arm64/kvm/hyp/nvhe/alloc.c new file mode 100644 index 000000000000..14274a18987b --- /dev/null +++ b/arch/arm64/kvm/hyp/nvhe/alloc.c @@ -0,0 +1,785 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2023 Google LLC + * Author: Vincent Donnefort + */ + +#include +#include +#include +#include + +#include +#include +#include + +#define MIN_ALLOC 8UL + +static DEFINE_PER_CPU(int, hyp_allocator_errno); +static DEFINE_PER_CPU(struct kvm_hyp_memcache, hyp_allocator_mc); + +static struct hyp_allocator { + struct list_head chunks; + unsigned long start; + u32 size; + hyp_spinlock_t lock; +} hyp_allocator; + +struct chunk_hdr { + u32 alloc_size; + u32 mapped_size; + struct list_head node; + u32 hash; + char data __aligned(8); +}; + +static u32 chunk_hash_compute(struct chunk_hdr *chunk) +{ + size_t len = offsetof(struct chunk_hdr, hash); + u64 *data = (u64 *)chunk; + u32 hash = 0; + + BUILD_BUG_ON(!IS_ALIGNED(offsetof(struct chunk_hdr, hash), sizeof(u32))); + + while (len >= sizeof(u64)) { + hash ^= hash_64(*data, 32); + len -= sizeof(u64); + data++; + } + + if (len) + hash ^= hash_32(*(u32 *)data, 32); + + return hash; +} + +static inline void chunk_hash_update(struct chunk_hdr *chunk) +{ + if (chunk) + chunk->hash = chunk_hash_compute(chunk); +} + +static inline void chunk_hash_validate(struct chunk_hdr *chunk) +{ + if (chunk) + WARN_ON(chunk->hash != chunk_hash_compute(chunk)); +} + +#define chunk_is_used(chunk) \ + (!!(chunk)->alloc_size) + +#define chunk_hdr_size() \ + offsetof(struct chunk_hdr, data) + +#define chunk_size(size) \ + (chunk_hdr_size() + max((size_t)(size), MIN_ALLOC)) + +#define chunk_data(chunk) \ + ((void *)(&(chunk)->data)) + +#define __chunk_next(chunk, allocator) \ +({ \ + list_is_last(&(chunk)->node, &(allocator)->chunks) ? \ + NULL : list_next_entry(chunk, node); \ +}) + +#define __chunk_prev(chunk, allocator) \ +({ \ + list_is_first(&(chunk)->node, &(allocator)->chunks) ? \ + NULL : list_prev_entry(chunk, node); \ +}) + +#define chunk_get_next(chunk, allocator) \ +({ \ + struct chunk_hdr *next = __chunk_next(chunk, allocator);\ + chunk_hash_validate(next); \ + next; \ +}) + +#define chunk_get_prev(chunk, allocator) \ +({ \ + struct chunk_hdr *prev = __chunk_prev(chunk, allocator);\ + chunk_hash_validate(prev); \ + prev; \ +}) + +#define chunk_get(addr) \ +({ \ + struct chunk_hdr *chunk = (struct chunk_hdr *)addr; \ + chunk_hash_validate(chunk); \ + chunk; \ +}) + +#define chunk_unmapped_region(chunk) \ + ((unsigned long)(chunk) + chunk->mapped_size) + +#define chunk_unmapped_size(chunk, allocator) \ +({ \ + struct chunk_hdr *next = chunk_get_next(chunk, allocator); \ + unsigned long allocator_end = (allocator)->start + \ + (allocator)->size; \ + next ? (unsigned long)next - chunk_unmapped_region(chunk) : \ + allocator_end - chunk_unmapped_region(chunk); \ +}) + +static inline void chunk_list_insert(struct chunk_hdr *chunk, + struct chunk_hdr *prev, + struct hyp_allocator *allocator) +{ + list_add(&chunk->node, &prev->node); + chunk_hash_update(prev); + chunk_hash_update(__chunk_next(chunk, allocator)); + chunk_hash_update(chunk); +} + +static inline void chunk_list_del(struct chunk_hdr *chunk, + struct hyp_allocator *allocator) +{ + struct chunk_hdr *prev = __chunk_prev(chunk, allocator); + struct chunk_hdr *next = __chunk_next(chunk, allocator); + + list_del(&chunk->node); + chunk_hash_update(prev); + chunk_hash_update(next); +} + +static void hyp_allocator_unmap(struct hyp_allocator *allocator, + unsigned long va, size_t size) +{ + struct kvm_hyp_memcache *mc = this_cpu_ptr(&hyp_allocator_mc); + int nr_pages = size >> PAGE_SHIFT; + unsigned long __va = va; + + WARN_ON(!PAGE_ALIGNED(va)); + WARN_ON(!PAGE_ALIGNED(size)); + + while (nr_pages--) { + phys_addr_t pa = __pkvm_private_range_pa((void *)__va); + void *page = hyp_phys_to_virt(pa); + + push_hyp_memcache(mc, page, hyp_virt_to_phys); + __va += PAGE_SIZE; + } + + pkvm_remove_mappings((void *)va, (void *)(va + size)); +} + +static int hyp_allocator_map(struct hyp_allocator *allocator, + unsigned long va, size_t size) +{ + struct kvm_hyp_memcache *mc = this_cpu_ptr(&hyp_allocator_mc); + unsigned long va_end = va + size; + int ret, nr_pages = 0; + + if (!PAGE_ALIGNED(va) || !PAGE_ALIGNED(size)) + return -EINVAL; + + if (va_end < va || va_end > (allocator->start + allocator->size)) + return -E2BIG; + + if (mc->nr_pages < (size >> PAGE_SHIFT)) + return -ENOMEM; + + while (nr_pages < (size >> PAGE_SHIFT)) { + void *page; + + page = pop_hyp_memcache(mc, hyp_phys_to_virt); + WARN_ON(!page); + + ret = __hyp_allocator_map(va, hyp_virt_to_phys(page)); + if (ret) { + push_hyp_memcache(mc, page, hyp_virt_to_phys); + break; + } + va += PAGE_SIZE; + nr_pages++; + } + + if (ret && nr_pages) { + va -= PAGE_SIZE * nr_pages; + hyp_allocator_unmap(allocator, va, nr_pages << PAGE_SHIFT); + } + + return ret; +} + +static int chunk_install(struct chunk_hdr *chunk, size_t size, + struct chunk_hdr *prev, + struct hyp_allocator *allocator) +{ + size_t prev_mapped_size; + + /* First chunk, first allocation */ + if (!prev) { + INIT_LIST_HEAD(&chunk->node); + list_add(&chunk->node, &allocator->chunks); + chunk->mapped_size = PAGE_ALIGN(chunk_size(size)); + chunk->alloc_size = size; + + chunk_hash_update(chunk); + + return 0; + } + + if (chunk_unmapped_region(prev) < (unsigned long)chunk) + return -EINVAL; + if ((unsigned long)chunk_data(prev) + prev->alloc_size > (unsigned long)chunk) + return -EINVAL; + + prev_mapped_size = prev->mapped_size; + prev->mapped_size = (unsigned long)chunk - (unsigned long)prev; + + chunk->mapped_size = prev_mapped_size - prev->mapped_size; + chunk->alloc_size = size; + + chunk_list_insert(chunk, prev, allocator); + + return 0; +} + +static int chunk_merge(struct chunk_hdr *chunk, struct hyp_allocator *allocator) +{ + /* The caller already validates prev */ + struct chunk_hdr *prev = __chunk_prev(chunk, allocator); + + if (WARN_ON(!prev)) + return -EINVAL; + + /* Can only merge free chunks */ + if (chunk_is_used(chunk) || chunk_is_used(prev)) + return -EBUSY; + + /* Can't merge non-contiguous mapped regions */ + if (chunk_unmapped_region(prev) != (unsigned long)chunk) + return 0; + + /* mapped region inheritance */ + prev->mapped_size += chunk->mapped_size; + + chunk_list_del(chunk, allocator); + + return 0; +} + +static size_t chunk_needs_mapping(struct chunk_hdr *chunk, size_t size) +{ + size_t mapping_missing, mapping_needs = chunk_size(size); + + if (mapping_needs <= chunk->mapped_size) + return 0; + + mapping_missing = PAGE_ALIGN(mapping_needs - chunk->mapped_size); + + return mapping_missing; +} + +/* + * When a chunk spans over several pages, split it at the start of the latest + * page. This allows to punch holes in the mapping to reclaim pages. + * + * +--------------+ + * |______________| + * |______________|<- Next chunk + * |_ _ _ __ _ _ _| + * | |<- New chunk installed, page aligned + * +--------------+ + * +--------------+ + * | | + * | |<- Allow to reclaim this page + * | | + * | | + * +--------------+ + * +--------------+ + * | | + * |______________| + * |______________|<- Chunk to split at page alignment + * | | + * +--------------+ + */ +static int chunk_split_aligned(struct chunk_hdr *chunk, + struct hyp_allocator *allocator) +{ + struct chunk_hdr *next_chunk = chunk_get_next(chunk, allocator); + unsigned long delta, mapped_end = chunk_unmapped_region(chunk); + struct chunk_hdr *new_chunk; + + if (PAGE_ALIGNED(mapped_end)) + return 0; + + new_chunk = (struct chunk_hdr *)PAGE_ALIGN_DOWN(mapped_end); + if ((unsigned long)new_chunk <= (unsigned long)chunk) + return -EINVAL; + + delta = ((unsigned long)next_chunk - (unsigned long)new_chunk); + + /* + * This shouldn't happen, chunks are installed to a minimum distance + * from the page start + */ + WARN_ON(delta < chunk_size(0UL)); + + WARN_ON(chunk_install(new_chunk, 0, chunk, allocator)); + + return 0; +} + +static int chunk_inc_map(struct chunk_hdr *chunk, size_t map_size, + struct hyp_allocator *allocator) +{ + int ret; + + if (chunk_unmapped_size(chunk, allocator) < map_size) + return -EINVAL; + + ret = hyp_allocator_map(allocator, chunk_unmapped_region(chunk), + map_size); + if (ret) + return ret; + + chunk->mapped_size += map_size; + chunk_hash_update(chunk); + + return 0; +} + +static size_t chunk_dec_map(struct chunk_hdr *chunk, + struct hyp_allocator *allocator, + size_t reclaim_target) +{ + unsigned long start, end; + size_t reclaimable; + + start = PAGE_ALIGN((unsigned long)chunk + + chunk_size(chunk->alloc_size)); + end = chunk_unmapped_region(chunk); + + if (start >= end) + return 0; + + reclaimable = end - start; + if (reclaimable < PAGE_SIZE) + return 0; + + if (chunk_split_aligned(chunk, allocator)) + return 0; + + end = chunk_unmapped_region(chunk); + reclaimable = min(end - start, reclaim_target); + start = end - reclaimable; + + hyp_allocator_unmap(allocator, start, reclaimable); + + chunk->mapped_size -= reclaimable; + chunk_hash_update(chunk); + + return reclaimable; +} + +static unsigned long chunk_addr_fixup(unsigned long addr) +{ + unsigned long min_chunk_size = chunk_size(0UL); + unsigned long page = PAGE_ALIGN_DOWN(addr); + unsigned long delta = addr - page; + + if (!delta) + return addr; + + /* + * To maximize reclaim, a chunk must fit between the page start and this + * addr. + */ + if (delta < min_chunk_size) + return page + min_chunk_size; + + return addr; +} + +static bool chunk_can_split(struct chunk_hdr *chunk, unsigned long addr, + struct hyp_allocator *allocator) +{ + unsigned long chunk_end; + + /* + * There is no point splitting the last chunk, subsequent allocations + * would be able to use this space anyway. + */ + if (list_is_last(&chunk->node, &allocator->chunks)) + return false; + + chunk_end = (unsigned long)chunk + chunk->mapped_size + + chunk_unmapped_size(chunk, allocator); + + return addr + chunk_size(0UL) < chunk_end; +} + +static int chunk_recycle(struct chunk_hdr *chunk, size_t size, + struct hyp_allocator *allocator) +{ + unsigned long new_chunk_addr = (unsigned long)chunk + chunk_size(size); + size_t missing_map, expected_mapping = size; + struct chunk_hdr *new_chunk = NULL; + int ret; + + new_chunk_addr = chunk_addr_fixup(new_chunk_addr); + if (chunk_can_split(chunk, new_chunk_addr, allocator)) { + new_chunk = (struct chunk_hdr *)new_chunk_addr; + expected_mapping = new_chunk_addr + chunk_hdr_size() - + (unsigned long)chunk_data(chunk); + } + + missing_map = chunk_needs_mapping(chunk, expected_mapping); + if (missing_map) { + ret = chunk_inc_map(chunk, missing_map, allocator); + if (ret) + return ret; + } + + chunk->alloc_size = size; + chunk_hash_update(chunk); + + if (new_chunk) + WARN_ON(chunk_install(new_chunk, 0, chunk, allocator)); + + return 0; +} + +static size_t chunk_try_destroy(struct chunk_hdr *chunk, + struct hyp_allocator *allocator, + size_t reclaim_target) +{ + size_t unmapped; + + if (chunk_is_used(chunk)) + return 0; + + /* Don't kill the entire chunk if this is not necessary */ + if (chunk->mapped_size > reclaim_target) + return 0; + + if (list_is_first(&chunk->node, &allocator->chunks)) { + /* last standing chunk ? */ + if (!list_is_last(&chunk->node, &allocator->chunks)) + return 0; + + list_del(&chunk->node); + goto unmap; + } + + /* + * Resolve discontiguous unmapped zones that are the result + * of a previous chunk_dec_map(). + * + * To make sure we still keep track of that unmapped zone in our free + * list, we need either to be the last chunk or to have prev unused. Two + * contiguous chunks can be both free if they are separated by an + * unmapped zone (see chunk_recycle()). + */ + + if (!PAGE_ALIGNED((unsigned long)chunk)) + return 0; + + if (list_is_last(&chunk->node, &allocator->chunks)) + goto destroy; + + if (chunk_is_used(chunk_get_prev(chunk, allocator))) + return 0; + + if (chunk_split_aligned(chunk, allocator)) + return 0; +destroy: + chunk_list_del(chunk, allocator); +unmap: + unmapped = chunk->mapped_size; + hyp_allocator_unmap(allocator, (unsigned long)chunk, + chunk->mapped_size); + + return unmapped; +} + +static int setup_first_chunk(struct hyp_allocator *allocator, size_t size) +{ + int ret; + + ret = hyp_allocator_map(allocator, allocator->start, + PAGE_ALIGN(chunk_size(size))); + if (ret) + return ret; + + return chunk_install((struct chunk_hdr *)allocator->start, size, NULL, allocator); +} + +static struct chunk_hdr * +get_free_chunk(struct hyp_allocator *allocator, size_t size) +{ + struct chunk_hdr *chunk, *best_chunk = NULL; + size_t best_available_size = allocator->size; + + list_for_each_entry(chunk, &allocator->chunks, node) { + size_t available_size = chunk->mapped_size + + chunk_unmapped_size(chunk, allocator); + if (chunk_is_used(chunk)) + continue; + + if (chunk_size(size) > available_size) + continue; + + if (!best_chunk) { + best_chunk = chunk; + continue; + } + + if (best_available_size <= available_size) + continue; + + best_chunk = chunk; + best_available_size = available_size; + } + + return chunk_get(best_chunk); +} + +void *hyp_alloc(size_t size) +{ + struct hyp_allocator *allocator = &hyp_allocator; + struct chunk_hdr *chunk, *last_chunk; + unsigned long chunk_addr; + int missing_map, ret = 0; + + size = ALIGN(size, MIN_ALLOC); + + hyp_spin_lock(&allocator->lock); + + if (list_empty(&hyp_allocator.chunks)) { + ret = setup_first_chunk(allocator, size); + if (ret) + goto end; + + chunk = (struct chunk_hdr *)allocator->start; + goto end; + } + + chunk = get_free_chunk(allocator, size); + if (chunk) { + ret = chunk_recycle(chunk, size, allocator); + goto end; + } + + last_chunk = chunk_get(list_last_entry(&allocator->chunks, struct chunk_hdr, node)); + + chunk_addr = (unsigned long)last_chunk + chunk_size(last_chunk->alloc_size); + chunk_addr = chunk_addr_fixup(chunk_addr); + chunk = (struct chunk_hdr *)chunk_addr; + + missing_map = chunk_needs_mapping(last_chunk, + chunk_addr + chunk_size(size) - + (unsigned long)chunk_data(last_chunk)); + if (missing_map) { + ret = chunk_inc_map(last_chunk, missing_map, allocator); + if (ret) + goto end; + } + + WARN_ON(chunk_install(chunk, size, last_chunk, allocator)); +end: + hyp_spin_unlock(&allocator->lock); + + *(this_cpu_ptr(&hyp_allocator_errno)) = ret; + + /* Enforce zeroing allocated memory */ + if (!ret) + memset(chunk_data(chunk), 0, size); + + return ret ? NULL : chunk_data(chunk); +} + +void hyp_free(void *addr) +{ + struct chunk_hdr *chunk, *prev_chunk, *next_chunk; + struct hyp_allocator *allocator = &hyp_allocator; + char *chunk_data = (char *)addr; + + hyp_spin_lock(&allocator->lock); + + chunk = chunk_get(container_of(chunk_data, struct chunk_hdr, data)); + prev_chunk = chunk_get_prev(chunk, allocator); + next_chunk = chunk_get_next(chunk, allocator); + + chunk->alloc_size = 0; + chunk_hash_update(chunk); + + if (next_chunk && !chunk_is_used(next_chunk)) + WARN_ON(chunk_merge(next_chunk, allocator)); + + if (prev_chunk && !chunk_is_used(prev_chunk)) + WARN_ON(chunk_merge(chunk, allocator)); + + hyp_spin_unlock(&allocator->lock); +} + +/* + * While chunk_try_destroy() is actually destroying what can be, this function + * only help with estimating how much pages can be reclaimed. However the same + * comments apply here. + */ +static bool chunk_destroyable(struct chunk_hdr *chunk, + struct hyp_allocator *allocator) +{ + if (chunk_is_used(chunk)) + return false; + + if (!PAGE_ALIGNED(chunk)) + return false; + + if (list_is_first(&chunk->node, &allocator->chunks)) { + if (list_is_last(&chunk->node, &allocator->chunks)) + return true; + + return false; + } + + return !chunk_is_used(chunk_get_prev(chunk, allocator)); +} + +static size_t chunk_reclaimable(struct chunk_hdr *chunk, + struct hyp_allocator *allocator) +{ + unsigned long start, end = chunk_unmapped_region(chunk); + + /* + * This should not happen, chunks are installed at a minimum distance + * from the page start + */ + WARN_ON(!PAGE_ALIGNED(end) && + (end - PAGE_ALIGN_DOWN(end) < chunk_size(0UL))); + + if (chunk_destroyable(chunk, allocator)) + start = (unsigned long)chunk; + else + start = PAGE_ALIGN((unsigned long)chunk + chunk_size(chunk->alloc_size)); + + end = PAGE_ALIGN_DOWN(end); + if (start > end) + return 0; + + return end - start; +} + +int hyp_alloc_reclaimable(void) +{ + struct hyp_allocator *allocator = &hyp_allocator; + struct chunk_hdr *chunk; + int reclaimable = 0; + int cpu; + + hyp_spin_lock(&allocator->lock); + + /* + * This is slightly pessimistic: a real reclaim might be able to "fix" + * discontiguous unmapped region by deleting chunks from the top to the + * bottom. + */ + list_for_each_entry(chunk, &allocator->chunks, node) + reclaimable += chunk_reclaimable(chunk, allocator) >> PAGE_SHIFT; + + for (cpu = 0; cpu < hyp_nr_cpus; cpu++) { + struct kvm_hyp_memcache *mc = per_cpu_ptr(&hyp_allocator_mc, cpu); + + reclaimable += mc->nr_pages; + } + + hyp_spin_unlock(&allocator->lock); + + return reclaimable; +} + +void hyp_alloc_reclaim(struct kvm_hyp_memcache *mc, int target) +{ + struct hyp_allocator *allocator = &hyp_allocator; + struct kvm_hyp_memcache *alloc_mc; + struct chunk_hdr *chunk, *tmp; + int cpu; + + if (target <= 0) + return; + + hyp_spin_lock(&allocator->lock); + + /* Start emptying potential unused memcache */ + for (cpu = 0; cpu < hyp_nr_cpus; cpu++) { + alloc_mc = per_cpu_ptr(&hyp_allocator_mc, cpu); + + while (alloc_mc->nr_pages) { + void *page = pop_hyp_memcache(alloc_mc, hyp_phys_to_virt); + + push_hyp_memcache(mc, page, hyp_virt_to_phys); + WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(page), 1)); + + target--; + if (target <= 0) + goto done; + } + } + + list_for_each_entry_safe_reverse(chunk, tmp, &allocator->chunks, node) { + size_t r; + + chunk_hash_validate(chunk); + r = chunk_try_destroy(chunk, allocator, target << PAGE_SHIFT); + if (!r) + r = chunk_dec_map(chunk, allocator, target << PAGE_SHIFT); + + target -= r >> PAGE_SHIFT; + if (target <= 0) + break; + } + + alloc_mc = this_cpu_ptr(&hyp_allocator_mc); + while (alloc_mc->nr_pages) { + void *page = pop_hyp_memcache(alloc_mc, hyp_phys_to_virt); + + memset(page, 0, PAGE_SIZE); + kvm_flush_dcache_to_poc(page, PAGE_SIZE); + push_hyp_memcache(mc, page, hyp_virt_to_phys); + WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(page), 1)); + } +done: + hyp_spin_unlock(&allocator->lock); +} + +int hyp_alloc_refill(struct kvm_hyp_memcache *host_mc) +{ + struct kvm_hyp_memcache *alloc_mc = this_cpu_ptr(&hyp_allocator_mc); + + return refill_memcache(alloc_mc, host_mc->nr_pages + alloc_mc->nr_pages, + host_mc); +} + +int hyp_alloc_init(size_t size) +{ + struct hyp_allocator *allocator = &hyp_allocator; + int ret; + + size = PAGE_ALIGN(size); + + /* constrained by chunk_hdr *_size types */ + if (size > U32_MAX) + return -EINVAL; + + ret = pkvm_alloc_private_va_range(size, &allocator->start); + if (ret) + return ret; + + allocator->size = size; + INIT_LIST_HEAD(&allocator->chunks); + hyp_spin_lock_init(&allocator->lock); + + return 0; +} + +int hyp_alloc_errno(void) +{ + int *errno = this_cpu_ptr(&hyp_allocator_errno); + + return *errno; +} +