ANDROID: KVM: arm64: Add a heap allocator for the pKVM hyp
Currently, memory used by the hypervisor comes from donations that are embedded within HVCs. e.g. __pkvm_init_vm()'s hyp_vm, last_ran ... This is not flexible at all. It means the host needs to know the size of the hyp private structures, and there are no mechanism for guests HVCs to allocate memory for the hyp in a similar fashion. Also, donations must be page-aligned and use physically contiguous memory whenever more than one page is necessary which might not be possible when the memory is highly fragmented. But here's a heap allocator to save the day. It works as any regular heap allocator: * hyp_alloc(size) : Returns a pointer in the hyp VA space. * hyp_free(addr) : Free the previously allocated memory. But also integrates the management of host donations to map and reclaim dynamically the memory. * hyp_alloc_refill(host_mc) : Admit host donations. * hyp_alloc_reclaimable() : Number of pages that could be reclaimed. * hyp_alloc_reclaim(mc, target) : Teardown pages up to the target. Missing donations are signaled via the errno -ENOMEM which can be read with: * hyp_alloc_errno() The meta-data are stored in the header of each chunk of allocated memory. A hash ensures their integrity. As long as pages are not reclaimed they are kept mapped in the heap to speed-up subsequent allocations. Bug: 357781595 Bug: 273748186 Change-Id: Ifa7eca06bce8e5e8af9ff32562240ab78cbcb860 Signed-off-by: Vincent Donnefort <vdonnefort@google.com>
This commit is contained in:
Vincent Donnefort
committed by
Keir Fraser
Keir Fraser
parent
ff527a855c
commit
08477328e7
41
arch/arm64/kvm/hyp/include/nvhe/alloc.h
Normal file
41
arch/arm64/kvm/hyp/include/nvhe/alloc.h
Normal file
@@ -0,0 +1,41 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
#ifndef __KVM_NVHE_ALLOC__
|
||||
#define __KVM_NVHE_ALLOC__
|
||||
#include <linux/types.h>
|
||||
|
||||
#include <asm/kvm_host.h>
|
||||
|
||||
/**
|
||||
* hyp_alloc() - Allocate memory from the heap allocator
|
||||
*
|
||||
* @size: Allocation size in bytes.
|
||||
*
|
||||
* Return: A pointer to the allocated memory on success, else NULL.
|
||||
*/
|
||||
void *hyp_alloc(size_t size);
|
||||
|
||||
/**
|
||||
* hyp_alloc_errno() - Read the errno on allocation error
|
||||
*
|
||||
* Get the return code from an allocation failure.
|
||||
*
|
||||
* Return: -ENOMEM if the allocator needs a refill from the host, -E2BIG if
|
||||
* there is no VA space left else 0.
|
||||
*/
|
||||
int hyp_alloc_errno(void);
|
||||
|
||||
/**
|
||||
* hyp_free() - Free memory allocated with hyp_alloc()
|
||||
*
|
||||
* @addr: Address returned by the original hyp_alloc().
|
||||
*
|
||||
* The use of any other address than one returned by hyp_alloc() will cause a
|
||||
* hypervisor panic.
|
||||
*/
|
||||
void hyp_free(void *addr);
|
||||
|
||||
int hyp_alloc_init(size_t size);
|
||||
int hyp_alloc_refill(struct kvm_hyp_memcache *host_mc);
|
||||
int hyp_alloc_reclaimable(void);
|
||||
void hyp_alloc_reclaim(struct kvm_hyp_memcache *host_mc, int target);
|
||||
#endif
|
||||
@@ -6,7 +6,7 @@ lib-objs := $(addprefix ../../../lib/, $(lib-objs))
|
||||
CFLAGS_switch.nvhe.o += -Wno-override-init
|
||||
|
||||
hyp-obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o \
|
||||
hyp-main.o hyp-smp.o psci-relay.o early_alloc.o page_alloc.o \
|
||||
hyp-main.o hyp-smp.o psci-relay.o alloc.o early_alloc.o page_alloc.o \
|
||||
cache.o setup.o mm.o mem_protect.o sys_regs.o pkvm.o stacktrace.o ffa.o \
|
||||
serial.o
|
||||
hyp-obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \
|
||||
|
||||
785
arch/arm64/kvm/hyp/nvhe/alloc.c
Normal file
785
arch/arm64/kvm/hyp/nvhe/alloc.c
Normal file
@@ -0,0 +1,785 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (C) 2023 Google LLC
|
||||
* Author: Vincent Donnefort <vdonnefort@google.com>
|
||||
*/
|
||||
|
||||
#include <nvhe/alloc.h>
|
||||
#include <nvhe/mem_protect.h>
|
||||
#include <nvhe/mm.h>
|
||||
#include <nvhe/spinlock.h>
|
||||
|
||||
#include <linux/build_bug.h>
|
||||
#include <linux/hash.h>
|
||||
#include <linux/list.h>
|
||||
|
||||
#define MIN_ALLOC 8UL
|
||||
|
||||
static DEFINE_PER_CPU(int, hyp_allocator_errno);
|
||||
static DEFINE_PER_CPU(struct kvm_hyp_memcache, hyp_allocator_mc);
|
||||
|
||||
static struct hyp_allocator {
|
||||
struct list_head chunks;
|
||||
unsigned long start;
|
||||
u32 size;
|
||||
hyp_spinlock_t lock;
|
||||
} hyp_allocator;
|
||||
|
||||
struct chunk_hdr {
|
||||
u32 alloc_size;
|
||||
u32 mapped_size;
|
||||
struct list_head node;
|
||||
u32 hash;
|
||||
char data __aligned(8);
|
||||
};
|
||||
|
||||
static u32 chunk_hash_compute(struct chunk_hdr *chunk)
|
||||
{
|
||||
size_t len = offsetof(struct chunk_hdr, hash);
|
||||
u64 *data = (u64 *)chunk;
|
||||
u32 hash = 0;
|
||||
|
||||
BUILD_BUG_ON(!IS_ALIGNED(offsetof(struct chunk_hdr, hash), sizeof(u32)));
|
||||
|
||||
while (len >= sizeof(u64)) {
|
||||
hash ^= hash_64(*data, 32);
|
||||
len -= sizeof(u64);
|
||||
data++;
|
||||
}
|
||||
|
||||
if (len)
|
||||
hash ^= hash_32(*(u32 *)data, 32);
|
||||
|
||||
return hash;
|
||||
}
|
||||
|
||||
static inline void chunk_hash_update(struct chunk_hdr *chunk)
|
||||
{
|
||||
if (chunk)
|
||||
chunk->hash = chunk_hash_compute(chunk);
|
||||
}
|
||||
|
||||
static inline void chunk_hash_validate(struct chunk_hdr *chunk)
|
||||
{
|
||||
if (chunk)
|
||||
WARN_ON(chunk->hash != chunk_hash_compute(chunk));
|
||||
}
|
||||
|
||||
#define chunk_is_used(chunk) \
|
||||
(!!(chunk)->alloc_size)
|
||||
|
||||
#define chunk_hdr_size() \
|
||||
offsetof(struct chunk_hdr, data)
|
||||
|
||||
#define chunk_size(size) \
|
||||
(chunk_hdr_size() + max((size_t)(size), MIN_ALLOC))
|
||||
|
||||
#define chunk_data(chunk) \
|
||||
((void *)(&(chunk)->data))
|
||||
|
||||
#define __chunk_next(chunk, allocator) \
|
||||
({ \
|
||||
list_is_last(&(chunk)->node, &(allocator)->chunks) ? \
|
||||
NULL : list_next_entry(chunk, node); \
|
||||
})
|
||||
|
||||
#define __chunk_prev(chunk, allocator) \
|
||||
({ \
|
||||
list_is_first(&(chunk)->node, &(allocator)->chunks) ? \
|
||||
NULL : list_prev_entry(chunk, node); \
|
||||
})
|
||||
|
||||
#define chunk_get_next(chunk, allocator) \
|
||||
({ \
|
||||
struct chunk_hdr *next = __chunk_next(chunk, allocator);\
|
||||
chunk_hash_validate(next); \
|
||||
next; \
|
||||
})
|
||||
|
||||
#define chunk_get_prev(chunk, allocator) \
|
||||
({ \
|
||||
struct chunk_hdr *prev = __chunk_prev(chunk, allocator);\
|
||||
chunk_hash_validate(prev); \
|
||||
prev; \
|
||||
})
|
||||
|
||||
#define chunk_get(addr) \
|
||||
({ \
|
||||
struct chunk_hdr *chunk = (struct chunk_hdr *)addr; \
|
||||
chunk_hash_validate(chunk); \
|
||||
chunk; \
|
||||
})
|
||||
|
||||
#define chunk_unmapped_region(chunk) \
|
||||
((unsigned long)(chunk) + chunk->mapped_size)
|
||||
|
||||
#define chunk_unmapped_size(chunk, allocator) \
|
||||
({ \
|
||||
struct chunk_hdr *next = chunk_get_next(chunk, allocator); \
|
||||
unsigned long allocator_end = (allocator)->start + \
|
||||
(allocator)->size; \
|
||||
next ? (unsigned long)next - chunk_unmapped_region(chunk) : \
|
||||
allocator_end - chunk_unmapped_region(chunk); \
|
||||
})
|
||||
|
||||
static inline void chunk_list_insert(struct chunk_hdr *chunk,
|
||||
struct chunk_hdr *prev,
|
||||
struct hyp_allocator *allocator)
|
||||
{
|
||||
list_add(&chunk->node, &prev->node);
|
||||
chunk_hash_update(prev);
|
||||
chunk_hash_update(__chunk_next(chunk, allocator));
|
||||
chunk_hash_update(chunk);
|
||||
}
|
||||
|
||||
static inline void chunk_list_del(struct chunk_hdr *chunk,
|
||||
struct hyp_allocator *allocator)
|
||||
{
|
||||
struct chunk_hdr *prev = __chunk_prev(chunk, allocator);
|
||||
struct chunk_hdr *next = __chunk_next(chunk, allocator);
|
||||
|
||||
list_del(&chunk->node);
|
||||
chunk_hash_update(prev);
|
||||
chunk_hash_update(next);
|
||||
}
|
||||
|
||||
static void hyp_allocator_unmap(struct hyp_allocator *allocator,
|
||||
unsigned long va, size_t size)
|
||||
{
|
||||
struct kvm_hyp_memcache *mc = this_cpu_ptr(&hyp_allocator_mc);
|
||||
int nr_pages = size >> PAGE_SHIFT;
|
||||
unsigned long __va = va;
|
||||
|
||||
WARN_ON(!PAGE_ALIGNED(va));
|
||||
WARN_ON(!PAGE_ALIGNED(size));
|
||||
|
||||
while (nr_pages--) {
|
||||
phys_addr_t pa = __pkvm_private_range_pa((void *)__va);
|
||||
void *page = hyp_phys_to_virt(pa);
|
||||
|
||||
push_hyp_memcache(mc, page, hyp_virt_to_phys);
|
||||
__va += PAGE_SIZE;
|
||||
}
|
||||
|
||||
pkvm_remove_mappings((void *)va, (void *)(va + size));
|
||||
}
|
||||
|
||||
static int hyp_allocator_map(struct hyp_allocator *allocator,
|
||||
unsigned long va, size_t size)
|
||||
{
|
||||
struct kvm_hyp_memcache *mc = this_cpu_ptr(&hyp_allocator_mc);
|
||||
unsigned long va_end = va + size;
|
||||
int ret, nr_pages = 0;
|
||||
|
||||
if (!PAGE_ALIGNED(va) || !PAGE_ALIGNED(size))
|
||||
return -EINVAL;
|
||||
|
||||
if (va_end < va || va_end > (allocator->start + allocator->size))
|
||||
return -E2BIG;
|
||||
|
||||
if (mc->nr_pages < (size >> PAGE_SHIFT))
|
||||
return -ENOMEM;
|
||||
|
||||
while (nr_pages < (size >> PAGE_SHIFT)) {
|
||||
void *page;
|
||||
|
||||
page = pop_hyp_memcache(mc, hyp_phys_to_virt);
|
||||
WARN_ON(!page);
|
||||
|
||||
ret = __hyp_allocator_map(va, hyp_virt_to_phys(page));
|
||||
if (ret) {
|
||||
push_hyp_memcache(mc, page, hyp_virt_to_phys);
|
||||
break;
|
||||
}
|
||||
va += PAGE_SIZE;
|
||||
nr_pages++;
|
||||
}
|
||||
|
||||
if (ret && nr_pages) {
|
||||
va -= PAGE_SIZE * nr_pages;
|
||||
hyp_allocator_unmap(allocator, va, nr_pages << PAGE_SHIFT);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int chunk_install(struct chunk_hdr *chunk, size_t size,
|
||||
struct chunk_hdr *prev,
|
||||
struct hyp_allocator *allocator)
|
||||
{
|
||||
size_t prev_mapped_size;
|
||||
|
||||
/* First chunk, first allocation */
|
||||
if (!prev) {
|
||||
INIT_LIST_HEAD(&chunk->node);
|
||||
list_add(&chunk->node, &allocator->chunks);
|
||||
chunk->mapped_size = PAGE_ALIGN(chunk_size(size));
|
||||
chunk->alloc_size = size;
|
||||
|
||||
chunk_hash_update(chunk);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (chunk_unmapped_region(prev) < (unsigned long)chunk)
|
||||
return -EINVAL;
|
||||
if ((unsigned long)chunk_data(prev) + prev->alloc_size > (unsigned long)chunk)
|
||||
return -EINVAL;
|
||||
|
||||
prev_mapped_size = prev->mapped_size;
|
||||
prev->mapped_size = (unsigned long)chunk - (unsigned long)prev;
|
||||
|
||||
chunk->mapped_size = prev_mapped_size - prev->mapped_size;
|
||||
chunk->alloc_size = size;
|
||||
|
||||
chunk_list_insert(chunk, prev, allocator);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int chunk_merge(struct chunk_hdr *chunk, struct hyp_allocator *allocator)
|
||||
{
|
||||
/* The caller already validates prev */
|
||||
struct chunk_hdr *prev = __chunk_prev(chunk, allocator);
|
||||
|
||||
if (WARN_ON(!prev))
|
||||
return -EINVAL;
|
||||
|
||||
/* Can only merge free chunks */
|
||||
if (chunk_is_used(chunk) || chunk_is_used(prev))
|
||||
return -EBUSY;
|
||||
|
||||
/* Can't merge non-contiguous mapped regions */
|
||||
if (chunk_unmapped_region(prev) != (unsigned long)chunk)
|
||||
return 0;
|
||||
|
||||
/* mapped region inheritance */
|
||||
prev->mapped_size += chunk->mapped_size;
|
||||
|
||||
chunk_list_del(chunk, allocator);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static size_t chunk_needs_mapping(struct chunk_hdr *chunk, size_t size)
|
||||
{
|
||||
size_t mapping_missing, mapping_needs = chunk_size(size);
|
||||
|
||||
if (mapping_needs <= chunk->mapped_size)
|
||||
return 0;
|
||||
|
||||
mapping_missing = PAGE_ALIGN(mapping_needs - chunk->mapped_size);
|
||||
|
||||
return mapping_missing;
|
||||
}
|
||||
|
||||
/*
|
||||
* When a chunk spans over several pages, split it at the start of the latest
|
||||
* page. This allows to punch holes in the mapping to reclaim pages.
|
||||
*
|
||||
* +--------------+
|
||||
* |______________|
|
||||
* |______________|<- Next chunk
|
||||
* |_ _ _ __ _ _ _|
|
||||
* | |<- New chunk installed, page aligned
|
||||
* +--------------+
|
||||
* +--------------+
|
||||
* | |
|
||||
* | |<- Allow to reclaim this page
|
||||
* | |
|
||||
* | |
|
||||
* +--------------+
|
||||
* +--------------+
|
||||
* | |
|
||||
* |______________|
|
||||
* |______________|<- Chunk to split at page alignment
|
||||
* | |
|
||||
* +--------------+
|
||||
*/
|
||||
static int chunk_split_aligned(struct chunk_hdr *chunk,
|
||||
struct hyp_allocator *allocator)
|
||||
{
|
||||
struct chunk_hdr *next_chunk = chunk_get_next(chunk, allocator);
|
||||
unsigned long delta, mapped_end = chunk_unmapped_region(chunk);
|
||||
struct chunk_hdr *new_chunk;
|
||||
|
||||
if (PAGE_ALIGNED(mapped_end))
|
||||
return 0;
|
||||
|
||||
new_chunk = (struct chunk_hdr *)PAGE_ALIGN_DOWN(mapped_end);
|
||||
if ((unsigned long)new_chunk <= (unsigned long)chunk)
|
||||
return -EINVAL;
|
||||
|
||||
delta = ((unsigned long)next_chunk - (unsigned long)new_chunk);
|
||||
|
||||
/*
|
||||
* This shouldn't happen, chunks are installed to a minimum distance
|
||||
* from the page start
|
||||
*/
|
||||
WARN_ON(delta < chunk_size(0UL));
|
||||
|
||||
WARN_ON(chunk_install(new_chunk, 0, chunk, allocator));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int chunk_inc_map(struct chunk_hdr *chunk, size_t map_size,
|
||||
struct hyp_allocator *allocator)
|
||||
{
|
||||
int ret;
|
||||
|
||||
if (chunk_unmapped_size(chunk, allocator) < map_size)
|
||||
return -EINVAL;
|
||||
|
||||
ret = hyp_allocator_map(allocator, chunk_unmapped_region(chunk),
|
||||
map_size);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
chunk->mapped_size += map_size;
|
||||
chunk_hash_update(chunk);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static size_t chunk_dec_map(struct chunk_hdr *chunk,
|
||||
struct hyp_allocator *allocator,
|
||||
size_t reclaim_target)
|
||||
{
|
||||
unsigned long start, end;
|
||||
size_t reclaimable;
|
||||
|
||||
start = PAGE_ALIGN((unsigned long)chunk +
|
||||
chunk_size(chunk->alloc_size));
|
||||
end = chunk_unmapped_region(chunk);
|
||||
|
||||
if (start >= end)
|
||||
return 0;
|
||||
|
||||
reclaimable = end - start;
|
||||
if (reclaimable < PAGE_SIZE)
|
||||
return 0;
|
||||
|
||||
if (chunk_split_aligned(chunk, allocator))
|
||||
return 0;
|
||||
|
||||
end = chunk_unmapped_region(chunk);
|
||||
reclaimable = min(end - start, reclaim_target);
|
||||
start = end - reclaimable;
|
||||
|
||||
hyp_allocator_unmap(allocator, start, reclaimable);
|
||||
|
||||
chunk->mapped_size -= reclaimable;
|
||||
chunk_hash_update(chunk);
|
||||
|
||||
return reclaimable;
|
||||
}
|
||||
|
||||
static unsigned long chunk_addr_fixup(unsigned long addr)
|
||||
{
|
||||
unsigned long min_chunk_size = chunk_size(0UL);
|
||||
unsigned long page = PAGE_ALIGN_DOWN(addr);
|
||||
unsigned long delta = addr - page;
|
||||
|
||||
if (!delta)
|
||||
return addr;
|
||||
|
||||
/*
|
||||
* To maximize reclaim, a chunk must fit between the page start and this
|
||||
* addr.
|
||||
*/
|
||||
if (delta < min_chunk_size)
|
||||
return page + min_chunk_size;
|
||||
|
||||
return addr;
|
||||
}
|
||||
|
||||
static bool chunk_can_split(struct chunk_hdr *chunk, unsigned long addr,
|
||||
struct hyp_allocator *allocator)
|
||||
{
|
||||
unsigned long chunk_end;
|
||||
|
||||
/*
|
||||
* There is no point splitting the last chunk, subsequent allocations
|
||||
* would be able to use this space anyway.
|
||||
*/
|
||||
if (list_is_last(&chunk->node, &allocator->chunks))
|
||||
return false;
|
||||
|
||||
chunk_end = (unsigned long)chunk + chunk->mapped_size +
|
||||
chunk_unmapped_size(chunk, allocator);
|
||||
|
||||
return addr + chunk_size(0UL) < chunk_end;
|
||||
}
|
||||
|
||||
static int chunk_recycle(struct chunk_hdr *chunk, size_t size,
|
||||
struct hyp_allocator *allocator)
|
||||
{
|
||||
unsigned long new_chunk_addr = (unsigned long)chunk + chunk_size(size);
|
||||
size_t missing_map, expected_mapping = size;
|
||||
struct chunk_hdr *new_chunk = NULL;
|
||||
int ret;
|
||||
|
||||
new_chunk_addr = chunk_addr_fixup(new_chunk_addr);
|
||||
if (chunk_can_split(chunk, new_chunk_addr, allocator)) {
|
||||
new_chunk = (struct chunk_hdr *)new_chunk_addr;
|
||||
expected_mapping = new_chunk_addr + chunk_hdr_size() -
|
||||
(unsigned long)chunk_data(chunk);
|
||||
}
|
||||
|
||||
missing_map = chunk_needs_mapping(chunk, expected_mapping);
|
||||
if (missing_map) {
|
||||
ret = chunk_inc_map(chunk, missing_map, allocator);
|
||||
if (ret)
|
||||
return ret;
|
||||
}
|
||||
|
||||
chunk->alloc_size = size;
|
||||
chunk_hash_update(chunk);
|
||||
|
||||
if (new_chunk)
|
||||
WARN_ON(chunk_install(new_chunk, 0, chunk, allocator));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static size_t chunk_try_destroy(struct chunk_hdr *chunk,
|
||||
struct hyp_allocator *allocator,
|
||||
size_t reclaim_target)
|
||||
{
|
||||
size_t unmapped;
|
||||
|
||||
if (chunk_is_used(chunk))
|
||||
return 0;
|
||||
|
||||
/* Don't kill the entire chunk if this is not necessary */
|
||||
if (chunk->mapped_size > reclaim_target)
|
||||
return 0;
|
||||
|
||||
if (list_is_first(&chunk->node, &allocator->chunks)) {
|
||||
/* last standing chunk ? */
|
||||
if (!list_is_last(&chunk->node, &allocator->chunks))
|
||||
return 0;
|
||||
|
||||
list_del(&chunk->node);
|
||||
goto unmap;
|
||||
}
|
||||
|
||||
/*
|
||||
* Resolve discontiguous unmapped zones that are the result
|
||||
* of a previous chunk_dec_map().
|
||||
*
|
||||
* To make sure we still keep track of that unmapped zone in our free
|
||||
* list, we need either to be the last chunk or to have prev unused. Two
|
||||
* contiguous chunks can be both free if they are separated by an
|
||||
* unmapped zone (see chunk_recycle()).
|
||||
*/
|
||||
|
||||
if (!PAGE_ALIGNED((unsigned long)chunk))
|
||||
return 0;
|
||||
|
||||
if (list_is_last(&chunk->node, &allocator->chunks))
|
||||
goto destroy;
|
||||
|
||||
if (chunk_is_used(chunk_get_prev(chunk, allocator)))
|
||||
return 0;
|
||||
|
||||
if (chunk_split_aligned(chunk, allocator))
|
||||
return 0;
|
||||
destroy:
|
||||
chunk_list_del(chunk, allocator);
|
||||
unmap:
|
||||
unmapped = chunk->mapped_size;
|
||||
hyp_allocator_unmap(allocator, (unsigned long)chunk,
|
||||
chunk->mapped_size);
|
||||
|
||||
return unmapped;
|
||||
}
|
||||
|
||||
static int setup_first_chunk(struct hyp_allocator *allocator, size_t size)
|
||||
{
|
||||
int ret;
|
||||
|
||||
ret = hyp_allocator_map(allocator, allocator->start,
|
||||
PAGE_ALIGN(chunk_size(size)));
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
return chunk_install((struct chunk_hdr *)allocator->start, size, NULL, allocator);
|
||||
}
|
||||
|
||||
static struct chunk_hdr *
|
||||
get_free_chunk(struct hyp_allocator *allocator, size_t size)
|
||||
{
|
||||
struct chunk_hdr *chunk, *best_chunk = NULL;
|
||||
size_t best_available_size = allocator->size;
|
||||
|
||||
list_for_each_entry(chunk, &allocator->chunks, node) {
|
||||
size_t available_size = chunk->mapped_size +
|
||||
chunk_unmapped_size(chunk, allocator);
|
||||
if (chunk_is_used(chunk))
|
||||
continue;
|
||||
|
||||
if (chunk_size(size) > available_size)
|
||||
continue;
|
||||
|
||||
if (!best_chunk) {
|
||||
best_chunk = chunk;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (best_available_size <= available_size)
|
||||
continue;
|
||||
|
||||
best_chunk = chunk;
|
||||
best_available_size = available_size;
|
||||
}
|
||||
|
||||
return chunk_get(best_chunk);
|
||||
}
|
||||
|
||||
void *hyp_alloc(size_t size)
|
||||
{
|
||||
struct hyp_allocator *allocator = &hyp_allocator;
|
||||
struct chunk_hdr *chunk, *last_chunk;
|
||||
unsigned long chunk_addr;
|
||||
int missing_map, ret = 0;
|
||||
|
||||
size = ALIGN(size, MIN_ALLOC);
|
||||
|
||||
hyp_spin_lock(&allocator->lock);
|
||||
|
||||
if (list_empty(&hyp_allocator.chunks)) {
|
||||
ret = setup_first_chunk(allocator, size);
|
||||
if (ret)
|
||||
goto end;
|
||||
|
||||
chunk = (struct chunk_hdr *)allocator->start;
|
||||
goto end;
|
||||
}
|
||||
|
||||
chunk = get_free_chunk(allocator, size);
|
||||
if (chunk) {
|
||||
ret = chunk_recycle(chunk, size, allocator);
|
||||
goto end;
|
||||
}
|
||||
|
||||
last_chunk = chunk_get(list_last_entry(&allocator->chunks, struct chunk_hdr, node));
|
||||
|
||||
chunk_addr = (unsigned long)last_chunk + chunk_size(last_chunk->alloc_size);
|
||||
chunk_addr = chunk_addr_fixup(chunk_addr);
|
||||
chunk = (struct chunk_hdr *)chunk_addr;
|
||||
|
||||
missing_map = chunk_needs_mapping(last_chunk,
|
||||
chunk_addr + chunk_size(size) -
|
||||
(unsigned long)chunk_data(last_chunk));
|
||||
if (missing_map) {
|
||||
ret = chunk_inc_map(last_chunk, missing_map, allocator);
|
||||
if (ret)
|
||||
goto end;
|
||||
}
|
||||
|
||||
WARN_ON(chunk_install(chunk, size, last_chunk, allocator));
|
||||
end:
|
||||
hyp_spin_unlock(&allocator->lock);
|
||||
|
||||
*(this_cpu_ptr(&hyp_allocator_errno)) = ret;
|
||||
|
||||
/* Enforce zeroing allocated memory */
|
||||
if (!ret)
|
||||
memset(chunk_data(chunk), 0, size);
|
||||
|
||||
return ret ? NULL : chunk_data(chunk);
|
||||
}
|
||||
|
||||
void hyp_free(void *addr)
|
||||
{
|
||||
struct chunk_hdr *chunk, *prev_chunk, *next_chunk;
|
||||
struct hyp_allocator *allocator = &hyp_allocator;
|
||||
char *chunk_data = (char *)addr;
|
||||
|
||||
hyp_spin_lock(&allocator->lock);
|
||||
|
||||
chunk = chunk_get(container_of(chunk_data, struct chunk_hdr, data));
|
||||
prev_chunk = chunk_get_prev(chunk, allocator);
|
||||
next_chunk = chunk_get_next(chunk, allocator);
|
||||
|
||||
chunk->alloc_size = 0;
|
||||
chunk_hash_update(chunk);
|
||||
|
||||
if (next_chunk && !chunk_is_used(next_chunk))
|
||||
WARN_ON(chunk_merge(next_chunk, allocator));
|
||||
|
||||
if (prev_chunk && !chunk_is_used(prev_chunk))
|
||||
WARN_ON(chunk_merge(chunk, allocator));
|
||||
|
||||
hyp_spin_unlock(&allocator->lock);
|
||||
}
|
||||
|
||||
/*
|
||||
* While chunk_try_destroy() is actually destroying what can be, this function
|
||||
* only help with estimating how much pages can be reclaimed. However the same
|
||||
* comments apply here.
|
||||
*/
|
||||
static bool chunk_destroyable(struct chunk_hdr *chunk,
|
||||
struct hyp_allocator *allocator)
|
||||
{
|
||||
if (chunk_is_used(chunk))
|
||||
return false;
|
||||
|
||||
if (!PAGE_ALIGNED(chunk))
|
||||
return false;
|
||||
|
||||
if (list_is_first(&chunk->node, &allocator->chunks)) {
|
||||
if (list_is_last(&chunk->node, &allocator->chunks))
|
||||
return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
return !chunk_is_used(chunk_get_prev(chunk, allocator));
|
||||
}
|
||||
|
||||
static size_t chunk_reclaimable(struct chunk_hdr *chunk,
|
||||
struct hyp_allocator *allocator)
|
||||
{
|
||||
unsigned long start, end = chunk_unmapped_region(chunk);
|
||||
|
||||
/*
|
||||
* This should not happen, chunks are installed at a minimum distance
|
||||
* from the page start
|
||||
*/
|
||||
WARN_ON(!PAGE_ALIGNED(end) &&
|
||||
(end - PAGE_ALIGN_DOWN(end) < chunk_size(0UL)));
|
||||
|
||||
if (chunk_destroyable(chunk, allocator))
|
||||
start = (unsigned long)chunk;
|
||||
else
|
||||
start = PAGE_ALIGN((unsigned long)chunk + chunk_size(chunk->alloc_size));
|
||||
|
||||
end = PAGE_ALIGN_DOWN(end);
|
||||
if (start > end)
|
||||
return 0;
|
||||
|
||||
return end - start;
|
||||
}
|
||||
|
||||
int hyp_alloc_reclaimable(void)
|
||||
{
|
||||
struct hyp_allocator *allocator = &hyp_allocator;
|
||||
struct chunk_hdr *chunk;
|
||||
int reclaimable = 0;
|
||||
int cpu;
|
||||
|
||||
hyp_spin_lock(&allocator->lock);
|
||||
|
||||
/*
|
||||
* This is slightly pessimistic: a real reclaim might be able to "fix"
|
||||
* discontiguous unmapped region by deleting chunks from the top to the
|
||||
* bottom.
|
||||
*/
|
||||
list_for_each_entry(chunk, &allocator->chunks, node)
|
||||
reclaimable += chunk_reclaimable(chunk, allocator) >> PAGE_SHIFT;
|
||||
|
||||
for (cpu = 0; cpu < hyp_nr_cpus; cpu++) {
|
||||
struct kvm_hyp_memcache *mc = per_cpu_ptr(&hyp_allocator_mc, cpu);
|
||||
|
||||
reclaimable += mc->nr_pages;
|
||||
}
|
||||
|
||||
hyp_spin_unlock(&allocator->lock);
|
||||
|
||||
return reclaimable;
|
||||
}
|
||||
|
||||
void hyp_alloc_reclaim(struct kvm_hyp_memcache *mc, int target)
|
||||
{
|
||||
struct hyp_allocator *allocator = &hyp_allocator;
|
||||
struct kvm_hyp_memcache *alloc_mc;
|
||||
struct chunk_hdr *chunk, *tmp;
|
||||
int cpu;
|
||||
|
||||
if (target <= 0)
|
||||
return;
|
||||
|
||||
hyp_spin_lock(&allocator->lock);
|
||||
|
||||
/* Start emptying potential unused memcache */
|
||||
for (cpu = 0; cpu < hyp_nr_cpus; cpu++) {
|
||||
alloc_mc = per_cpu_ptr(&hyp_allocator_mc, cpu);
|
||||
|
||||
while (alloc_mc->nr_pages) {
|
||||
void *page = pop_hyp_memcache(alloc_mc, hyp_phys_to_virt);
|
||||
|
||||
push_hyp_memcache(mc, page, hyp_virt_to_phys);
|
||||
WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(page), 1));
|
||||
|
||||
target--;
|
||||
if (target <= 0)
|
||||
goto done;
|
||||
}
|
||||
}
|
||||
|
||||
list_for_each_entry_safe_reverse(chunk, tmp, &allocator->chunks, node) {
|
||||
size_t r;
|
||||
|
||||
chunk_hash_validate(chunk);
|
||||
r = chunk_try_destroy(chunk, allocator, target << PAGE_SHIFT);
|
||||
if (!r)
|
||||
r = chunk_dec_map(chunk, allocator, target << PAGE_SHIFT);
|
||||
|
||||
target -= r >> PAGE_SHIFT;
|
||||
if (target <= 0)
|
||||
break;
|
||||
}
|
||||
|
||||
alloc_mc = this_cpu_ptr(&hyp_allocator_mc);
|
||||
while (alloc_mc->nr_pages) {
|
||||
void *page = pop_hyp_memcache(alloc_mc, hyp_phys_to_virt);
|
||||
|
||||
memset(page, 0, PAGE_SIZE);
|
||||
kvm_flush_dcache_to_poc(page, PAGE_SIZE);
|
||||
push_hyp_memcache(mc, page, hyp_virt_to_phys);
|
||||
WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(page), 1));
|
||||
}
|
||||
done:
|
||||
hyp_spin_unlock(&allocator->lock);
|
||||
}
|
||||
|
||||
int hyp_alloc_refill(struct kvm_hyp_memcache *host_mc)
|
||||
{
|
||||
struct kvm_hyp_memcache *alloc_mc = this_cpu_ptr(&hyp_allocator_mc);
|
||||
|
||||
return refill_memcache(alloc_mc, host_mc->nr_pages + alloc_mc->nr_pages,
|
||||
host_mc);
|
||||
}
|
||||
|
||||
int hyp_alloc_init(size_t size)
|
||||
{
|
||||
struct hyp_allocator *allocator = &hyp_allocator;
|
||||
int ret;
|
||||
|
||||
size = PAGE_ALIGN(size);
|
||||
|
||||
/* constrained by chunk_hdr *_size types */
|
||||
if (size > U32_MAX)
|
||||
return -EINVAL;
|
||||
|
||||
ret = pkvm_alloc_private_va_range(size, &allocator->start);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
allocator->size = size;
|
||||
INIT_LIST_HEAD(&allocator->chunks);
|
||||
hyp_spin_lock_init(&allocator->lock);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int hyp_alloc_errno(void)
|
||||
{
|
||||
int *errno = this_cpu_ptr(&hyp_allocator_errno);
|
||||
|
||||
return *errno;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user