ANDROID: KVM: arm64: Introducing pkvm_mem_abort_range()

Previously, guest stage-2 mapping was only triggered on a guest fault.
If the hypervisor needs one (e.g to share memory between guest and
host) its only solution is to fake a fault and to return to host. Of
course being done on a per-page basis, it is quite inefficient.

Instead provide a function to map an entire range for a guest stage-2
which will later be used to satisfy hypervisor mapping requests.

Even if a range is passed, this function doesn't allow the creation of
block mappings.

Bug: 357781595
Bug: 243642516
Change-Id: I80d87a6598c3d3daf3c8b6641a37f2e9d17b55f2
Signed-off-by: Vincent Donnefort <vdonnefort@google.com>

arch/arm64/include/asm/kvm_mmu.h

@@ -185,6 +185,7 @@ int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
 			  phys_addr_t pa, unsigned long size, bool writable);
 
 int kvm_handle_guest_abort(struct kvm_vcpu *vcpu);
+int pkvm_mem_abort_range(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, size_t size);
 
 phys_addr_t kvm_mmu_get_httbr(void);
 phys_addr_t kvm_get_idmap_vector(void);

arch/arm64/kvm/mmu.c

@@ -1643,10 +1643,13 @@ static int pkvm_host_map_guest(u64 pfn, u64 gfn, enum kvm_pgtable_prot prot)
 	return (ret == -EPERM) ? -EAGAIN : ret;
 }
 
+#define node_ppage(__node) \
+	(container_of(__node, struct kvm_pinned_page, node))
+
 static int cmp_ppage_ipa(const void *key, const struct rb_node *node)
 {
 	s64 a_ipa = (s64)key;
-	s64 b_ipa = (s64)container_of(node, struct kvm_pinned_page, node)->ipa;
+	s64 b_ipa = (s64)node_ppage(node)->ipa;
 
 	if (a_ipa < b_ipa)
 		return -1;
@@ -1657,7 +1660,32 @@ static int cmp_ppage_ipa(const void *key, const struct rb_node *node)
 
 static int cmp_ppages(struct rb_node *node, const struct rb_node *parent)
 {
-	return cmp_ppage_ipa((void *)container_of(node, struct kvm_pinned_page, node)->ipa, parent);
+	return cmp_ppage_ipa((void *)(node_ppage(node))->ipa, parent);
+}
+
+static struct kvm_pinned_page *
+find_ppage_or_above(struct kvm *kvm, phys_addr_t ipa)
+{
+	struct rb_node *node = kvm->arch.pkvm.pinned_pages.rb_node;
+
+	while (node) {
+		int ret = cmp_ppage_ipa((void *)ipa, node);
+
+		if (!ret) {
+			break;
+		} else if (ret > 0) {
+			node = node->rb_right;
+		} else if (ret < 0) {
+			if (!node->rb_left)
+				break;
+			node = node->rb_left;
+		}
+	}
+
+	if (!node)
+		return NULL;
+
+	return node_ppage(node);
 }
 
 static int insert_ppage(struct kvm *kvm, struct kvm_pinned_page *ppage)
@@ -1797,6 +1825,63 @@ free_ppage:
 	return ret;
 }
 
+int pkvm_mem_abort_range(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, size_t size)
+{
+	phys_addr_t ipa_end = fault_ipa + size - 1;
+	struct kvm_pinned_page *ppage;
+	int err = 0, idx;
+
+	if (!PAGE_ALIGNED(size) || !PAGE_ALIGNED(fault_ipa))
+		return -EINVAL;
+
+	if (ipa_end >= BIT_ULL(get_kvm_ipa_limit()) ||
+	    ipa_end >= kvm_phys_size(vcpu->arch.hw_mmu) ||
+	    ipa_end <= fault_ipa)
+		return -EINVAL;
+
+	idx = srcu_read_lock(&vcpu->kvm->srcu);
+
+	read_lock(&vcpu->kvm->mmu_lock);
+	ppage = find_ppage_or_above(vcpu->kvm, fault_ipa);
+
+	while (fault_ipa < ipa_end) {
+		if (ppage && ppage->ipa == fault_ipa) {
+			ppage = node_ppage(rb_next(&ppage->node));
+		} else {
+			gfn_t gfn = gpa_to_gfn(fault_ipa);
+			struct kvm_memory_slot *memslot;
+			unsigned long hva;
+			bool writable;
+
+			memslot = gfn_to_memslot(vcpu->kvm, gfn);
+			hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable);
+			if (kvm_is_error_hva(hva) || !writable) {
+				err = -EINVAL;
+				goto end;
+			}
+
+			read_unlock(&vcpu->kvm->mmu_lock);
+			err = pkvm_mem_abort(vcpu, fault_ipa, memslot);
+			read_lock(&vcpu->kvm->mmu_lock);
+			if (err)
+				goto end;
+
+			/*
+			 * We had to release the mmu_lock so let's update the
+			 * reference.
+			 */
+			ppage = find_ppage_or_above(vcpu->kvm, fault_ipa + PAGE_SIZE);
+		}
+
+		fault_ipa += PAGE_SIZE;
+	}
+end:
+	read_unlock(&vcpu->kvm->mmu_lock);
+	srcu_read_unlock(&vcpu->kvm->srcu, idx);
+
+	return err;
+}
+
 static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
 			  struct kvm_s2_trans *nested,
 			  struct kvm_memory_slot *memslot,