ANDROID: KVM: arm64: Mandate IO guard for guest physical MMIO

The host should only be allow to install MMIO for regions that the
guest have declared as MMIO with IO guard. That is critical to avoid
the host tricking the guest to access MMIO while it thinks it's normal
memory.
Replace the donation logic to check for IO guard first and remove it.

Bug: 357781595
Bug: 348382247
Change-Id: I461a33cb9b8275a3dcfe88ce8501e11ae49dd954
Signed-off-by: Mostafa Saleh <smostafa@google.com>

arch/arm64/kvm/hyp/include/nvhe/mem_protect.h

@@ -71,8 +71,8 @@ int __pkvm_host_lazy_pte(u64 pfn, u64 nr_pages, bool enable);
 u64 __pkvm_ptdump_get_config(pkvm_handle_t handle, enum pkvm_ptdump_ops op);
 u64 __pkvm_ptdump_walk_range(pkvm_handle_t handle, struct pkvm_ptdump_log_hdr *log_hva);
 
-int pkvm_hyp_donate_guest(struct pkvm_hyp_vcpu *vcpu, u64 pfn, u64 gfn);
 int hyp_check_range_owned(u64 addr, u64 size);
+int __pkvm_install_guest_mmio(struct pkvm_hyp_vcpu *hyp_vcpu, u64 pfn, u64 gfn);
 
 bool addr_is_memory(phys_addr_t phys);
 int host_stage2_idmap_locked(phys_addr_t addr, u64 size,

arch/arm64/kvm/hyp/nvhe/device/device.c

@@ -228,7 +228,7 @@ int pkvm_host_map_guest_mmio(struct pkvm_hyp_vcpu *hyp_vcpu, u64 pfn, u64 gfn)
 	if (ret)
 		goto out_ret;
 
-	ret = pkvm_hyp_donate_guest(hyp_vcpu, pfn, gfn);
+	ret = __pkvm_install_guest_mmio(hyp_vcpu, pfn, gfn);
 
 out_ret:
 	hyp_spin_unlock(&device_spinlock);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

@@ -1411,7 +1411,7 @@ int ___pkvm_host_donate_hyp(u64 pfn, u64 nr_pages, bool accept_mmio)
 					    default_hyp_prot(hyp_pfn_to_phys(pfn)));
 }
 
-int pkvm_hyp_donate_guest(struct pkvm_hyp_vcpu *vcpu, u64 pfn, u64 gfn)
+static int pkvm_hyp_donate_guest(struct pkvm_hyp_vcpu *vcpu, u64 pfn, u64 gfn)
 {
 	struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu);
 	u64 phys = hyp_pfn_to_phys(pfn);
@@ -1421,26 +1421,20 @@ int pkvm_hyp_donate_guest(struct pkvm_hyp_vcpu *vcpu, u64 pfn, u64 gfn)
 	enum kvm_pgtable_prot prot;
 	int ret;
 
-	hyp_lock_component();
-	guest_lock_component(vm);
+	hyp_assert_lock_held(&pkvm_pgd_lock);
+	hyp_assert_lock_held(&vm->pgtable_lock);
 
 	ret = __hyp_check_page_state_range(hyp_addr, size, PKVM_PAGE_OWNED);
 	if (ret)
-		goto unlock;
+		return ret;;
 	ret = __guest_check_page_state_range(vcpu, ipa, size, PKVM_NOPAGE);
 	if (ret)
-		goto unlock;
+		return ret;
 
 	WARN_ON(kvm_pgtable_hyp_unmap(&pkvm_pgtable, hyp_addr, size) != size);
 	prot = pkvm_mkstate(default_guest_prot(addr_is_memory(phys)), PKVM_PAGE_OWNED);
-	WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, size, phys, prot,
-				       &vcpu->vcpu.arch.stage2_mc, 0));
-
-unlock:
-	guest_unlock_component(vm);
-	hyp_unlock_component();
-
-	return ret;
+	return WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, size, phys, prot,
+					      &vcpu->vcpu.arch.stage2_mc, 0));
 }
 
 int __pkvm_host_donate_hyp_locked(u64 pfn, u64 nr_pages, enum kvm_pgtable_prot prot)
@@ -2172,6 +2166,49 @@ bool __pkvm_check_ioguard_page(struct pkvm_hyp_vcpu *hyp_vcpu)
 	return ret;
 }
 
+static int __pkvm_remove_ioguard_page(struct pkvm_hyp_vm *vm, u64 ipa)
+{
+	int ret;
+	kvm_pte_t pte;
+	s8 level;
+
+	hyp_assert_lock_held(&vm->pgtable_lock);
+
+	if (!test_bit(KVM_ARCH_FLAG_MMIO_GUARD, &vm->kvm.arch.flags))
+		return -EINVAL;
+
+	if (!PAGE_ALIGNED(ipa))
+		return -EINVAL;
+
+	ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level);
+	if (ret)
+		return ret;
+
+	if (BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(level)) == PAGE_SIZE &&
+	    pte == KVM_INVALID_PTE_MMIO_NOTE)
+		return kvm_pgtable_stage2_unmap(&vm->pgt, ipa, PAGE_SIZE);
+
+	return kvm_pte_valid(pte) ? -EEXIST : -EINVAL;
+}
+
+int __pkvm_install_guest_mmio(struct pkvm_hyp_vcpu *hyp_vcpu, u64 pfn, u64 gfn)
+{
+	struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu);
+	u64 ipa = gfn << PAGE_SHIFT;
+	int ret;
+
+	hyp_lock_component();
+	guest_lock_component(vm);
+	ret = __pkvm_remove_ioguard_page(vm, ipa);
+	if (ret)
+		goto out_unlock;
+	ret = pkvm_hyp_donate_guest(hyp_vcpu, pfn, gfn);
+out_unlock:
+	guest_unlock_component(vm);
+	hyp_unlock_component();
+	return ret;
+}
+
 int host_stage2_get_leaf(phys_addr_t phys, kvm_pte_t *ptep, s8 *level)
 {
 	int ret;