tmakin/ack-tegra
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge 63eb28bb14 ("Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm") into android-mainline

Browse Source 
Steps on the way to v6.17-rc1

Change-Id: Iedefd900a09fe53153289fa19d93139f33d33867
Signed-off-by: Carlos Llamas <cmllamas@google.com>
This commit is contained in:
Carlos Llamas committed 2025-08-19 03:23:17 +00:00
2 parents a2bd81eede 63eb28bb14
commit ad6c104988
209 files changed
+12103 -4778
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Documentation/arch/arm64/booting.rst
+41
View File
@@ -223,6 +223,47 @@ Before jumping into the kernel, the following conditions must be met:
- SCR_EL3.HCE (bit 8) must be initialised to 0b1.
For systems with a GICv5 interrupt controller to be used in v5 mode:
- If the kernel is entered at EL1 and EL2 is present:
- ICH_HFGRTR_EL2.ICC_PPI_ACTIVERn_EL1 (bit 20) must be initialised to 0b1.
- ICH_HFGRTR_EL2.ICC_PPI_PRIORITYRn_EL1 (bit 19) must be initialised to 0b1.
- ICH_HFGRTR_EL2.ICC_PPI_PENDRn_EL1 (bit 18) must be initialised to 0b1.
- ICH_HFGRTR_EL2.ICC_PPI_ENABLERn_EL1 (bit 17) must be initialised to 0b1.
- ICH_HFGRTR_EL2.ICC_PPI_HMRn_EL1 (bit 16) must be initialised to 0b1.
- ICH_HFGRTR_EL2.ICC_IAFFIDR_EL1 (bit 7) must be initialised to 0b1.
- ICH_HFGRTR_EL2.ICC_ICSR_EL1 (bit 6) must be initialised to 0b1.
- ICH_HFGRTR_EL2.ICC_PCR_EL1 (bit 5) must be initialised to 0b1.
- ICH_HFGRTR_EL2.ICC_HPPIR_EL1 (bit 4) must be initialised to 0b1.
- ICH_HFGRTR_EL2.ICC_HAPR_EL1 (bit 3) must be initialised to 0b1.
- ICH_HFGRTR_EL2.ICC_CR0_EL1 (bit 2) must be initialised to 0b1.
- ICH_HFGRTR_EL2.ICC_IDRn_EL1 (bit 1) must be initialised to 0b1.
- ICH_HFGRTR_EL2.ICC_APR_EL1 (bit 0) must be initialised to 0b1.
- ICH_HFGWTR_EL2.ICC_PPI_ACTIVERn_EL1 (bit 20) must be initialised to 0b1.
- ICH_HFGWTR_EL2.ICC_PPI_PRIORITYRn_EL1 (bit 19) must be initialised to 0b1.
- ICH_HFGWTR_EL2.ICC_PPI_PENDRn_EL1 (bit 18) must be initialised to 0b1.
- ICH_HFGWTR_EL2.ICC_PPI_ENABLERn_EL1 (bit 17) must be initialised to 0b1.
- ICH_HFGWTR_EL2.ICC_ICSR_EL1 (bit 6) must be initialised to 0b1.
- ICH_HFGWTR_EL2.ICC_PCR_EL1 (bit 5) must be initialised to 0b1.
- ICH_HFGWTR_EL2.ICC_CR0_EL1 (bit 2) must be initialised to 0b1.
- ICH_HFGWTR_EL2.ICC_APR_EL1 (bit 0) must be initialised to 0b1.
- ICH_HFGITR_EL2.GICRCDNMIA (bit 10) must be initialised to 0b1.
- ICH_HFGITR_EL2.GICRCDIA (bit 9) must be initialised to 0b1.
- ICH_HFGITR_EL2.GICCDDI (bit 8) must be initialised to 0b1.
- ICH_HFGITR_EL2.GICCDEOI (bit 7) must be initialised to 0b1.
- ICH_HFGITR_EL2.GICCDHM (bit 6) must be initialised to 0b1.
- ICH_HFGITR_EL2.GICCDRCFG (bit 5) must be initialised to 0b1.
- ICH_HFGITR_EL2.GICCDPEND (bit 4) must be initialised to 0b1.
- ICH_HFGITR_EL2.GICCDAFF (bit 3) must be initialised to 0b1.
- ICH_HFGITR_EL2.GICCDPRI (bit 2) must be initialised to 0b1.
- ICH_HFGITR_EL2.GICCDDIS (bit 1) must be initialised to 0b1.
- ICH_HFGITR_EL2.GICCDEN (bit 0) must be initialised to 0b1.
- The DT or ACPI tables must describe a GICv5 interrupt controller.
For systems with a GICv3 interrupt controller to be used in v3 mode:
- If EL3 is present:
Documentation/devicetree/bindings/interrupt-controller/arm,gic-v5-iwb.yaml
+78
View File
@@ -0,0 +1,78 @@
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
%YAML 1.2
---
$id: http://devicetree.org/schemas/interrupt-controller/arm,gic-v5-iwb.yaml#
$schema: http://devicetree.org/meta-schemas/core.yaml#
title: ARM Generic Interrupt Controller, version 5 Interrupt Wire Bridge (IWB)
maintainers:
- Lorenzo Pieralisi <lpieralisi@kernel.org>
- Marc Zyngier <maz@kernel.org>
description: |
The GICv5 architecture defines the guidelines to implement GICv5
compliant interrupt controllers for AArch64 systems.
The GICv5 specification can be found at
https://developer.arm.com/documentation/aes0070
GICv5 has zero or more Interrupt Wire Bridges (IWB) that are responsible
for translating wire signals into interrupt messages to the GICv5 ITS.
allOf:
- $ref: /schemas/interrupt-controller.yaml#
properties:
compatible:
const: arm,gic-v5-iwb
reg:
items:
- description: IWB control frame
"#address-cells":
const: 0
"#interrupt-cells":
description: |
The 1st cell corresponds to the IWB wire.
The 2nd cell is the flags, encoded as follows:
bits[3:0] trigger type and level flags.
1 = low-to-high edge triggered
2 = high-to-low edge triggered
4 = active high level-sensitive
8 = active low level-sensitive
const: 2
interrupt-controller: true
msi-parent:
maxItems: 1
required:
- compatible
- reg
- "#interrupt-cells"
- interrupt-controller
- msi-parent
additionalProperties: false
examples:
- |
interrupt-controller@2f000000 {
compatible = "arm,gic-v5-iwb";
reg = <0x2f000000 0x10000>;
#address-cells = <0>;
#interrupt-cells = <2>;
interrupt-controller;
msi-parent = <&its0 64>;
};
...
Documentation/devicetree/bindings/interrupt-controller/arm,gic-v5.yaml
+267
View File
@@ -0,0 +1,267 @@
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
%YAML 1.2
---
$id: http://devicetree.org/schemas/interrupt-controller/arm,gic-v5.yaml#
$schema: http://devicetree.org/meta-schemas/core.yaml#
title: ARM Generic Interrupt Controller, version 5
maintainers:
- Lorenzo Pieralisi <lpieralisi@kernel.org>
- Marc Zyngier <maz@kernel.org>
description: |
The GICv5 architecture defines the guidelines to implement GICv5
compliant interrupt controllers for AArch64 systems.
The GICv5 specification can be found at
https://developer.arm.com/documentation/aes0070
The GICv5 architecture is composed of multiple components:
- one or more IRS (Interrupt Routing Service)
- zero or more ITS (Interrupt Translation Service)
The architecture defines:
- PE-Private Peripheral Interrupts (PPI)
- Shared Peripheral Interrupts (SPI)
- Logical Peripheral Interrupts (LPI)
allOf:
- $ref: /schemas/interrupt-controller.yaml#
properties:
compatible:
const: arm,gic-v5
"#address-cells":
enum: [ 1, 2 ]
"#size-cells":
enum: [ 1, 2 ]
ranges: true
"#interrupt-cells":
description: |
The 1st cell corresponds to the INTID.Type field in the INTID; 1 for PPI,
3 for SPI. LPI interrupts must not be described in the bindings since
they are allocated dynamically by the software component managing them.
The 2nd cell contains the interrupt INTID.ID field.
The 3rd cell is the flags, encoded as follows:
bits[3:0] trigger type and level flags.
1 = low-to-high edge triggered
2 = high-to-low edge triggered
4 = active high level-sensitive
8 = active low level-sensitive
const: 3
interrupt-controller: true
interrupts:
description:
The VGIC maintenance interrupt.
maxItems: 1
required:
- compatible
- "#address-cells"
- "#size-cells"
- ranges
- "#interrupt-cells"
- interrupt-controller
patternProperties:
"^irs@[0-9a-f]+$":
type: object
description:
GICv5 has one or more Interrupt Routing Services (IRS) that are
responsible for handling IRQ state and routing.
additionalProperties: false
properties:
compatible:
const: arm,gic-v5-irs
reg:
minItems: 1
items:
- description: IRS config frames
- description: IRS setlpi frames
reg-names:
description:
Describe config and setlpi frames that are present.
"ns-" stands for non-secure, "s-" for secure, "realm-" for realm
and "el3-" for EL3.
minItems: 1
maxItems: 8
items:
enum: [ ns-config, s-config, realm-config, el3-config, ns-setlpi,
s-setlpi, realm-setlpi, el3-setlpi ]
"#address-cells":
enum: [ 1, 2 ]
"#size-cells":
enum: [ 1, 2 ]
ranges: true
dma-noncoherent:
description:
Present if the GIC IRS permits programming shareability and
cacheability attributes but is connected to a non-coherent
downstream interconnect.
cpus:
description:
CPUs managed by the IRS.
arm,iaffids:
$ref: /schemas/types.yaml#/definitions/uint16-array
description:
Interrupt AFFinity ID (IAFFID) associated with the CPU whose
CPU node phandle is at the same index in the cpus array.
patternProperties:
"^its@[0-9a-f]+$":
type: object
description:
GICv5 has zero or more Interrupt Translation Services (ITS) that are
used to route Message Signalled Interrupts (MSI) to the CPUs. Each
ITS is connected to an IRS.
additionalProperties: false
properties:
compatible:
const: arm,gic-v5-its
reg:
items:
- description: ITS config frames
reg-names:
description:
Describe config frames that are present.
"ns-" stands for non-secure, "s-" for secure, "realm-" for realm
and "el3-" for EL3.
minItems: 1
maxItems: 4
items:
enum: [ ns-config, s-config, realm-config, el3-config ]
"#address-cells":
enum: [ 1, 2 ]
"#size-cells":
enum: [ 1, 2 ]
ranges: true
dma-noncoherent:
description:
Present if the GIC ITS permits programming shareability and
cacheability attributes but is connected to a non-coherent
downstream interconnect.
patternProperties:
"^msi-controller@[0-9a-f]+$":
type: object
description:
GICv5 ITS has one or more translate register frames.
additionalProperties: false
properties:
reg:
items:
- description: ITS translate frames
reg-names:
description:
Describe translate frames that are present.
"ns-" stands for non-secure, "s-" for secure, "realm-" for realm
and "el3-" for EL3.
minItems: 1
maxItems: 4
items:
enum: [ ns-translate, s-translate, realm-translate, el3-translate ]
"#msi-cells":
description:
The single msi-cell is the DeviceID of the device which will
generate the MSI.
const: 1
msi-controller: true
required:
- reg
- reg-names
- "#msi-cells"
- msi-controller
required:
- compatible
- reg
- reg-names
required:
- compatible
- reg
- reg-names
- cpus
- arm,iaffids
additionalProperties: false
examples:
- |
interrupt-controller {
compatible = "arm,gic-v5";
#interrupt-cells = <3>;
interrupt-controller;
#address-cells = <1>;
#size-cells = <1>;
ranges;
interrupts = <1 25 4>;
irs@2f1a0000 {
compatible = "arm,gic-v5-irs";
reg = <0x2f1a0000 0x10000>; // IRS_CONFIG_FRAME
reg-names = "ns-config";
#address-cells = <1>;
#size-cells = <1>;
ranges;
cpus = <&cpu0>, <&cpu1>, <&cpu2>, <&cpu3>, <&cpu4>, <&cpu5>, <&cpu6>, <&cpu7>;
arm,iaffids = /bits/ 16 <0 1 2 3 4 5 6 7>;
its@2f120000 {
compatible = "arm,gic-v5-its";
reg = <0x2f120000 0x10000>; // ITS_CONFIG_FRAME
reg-names = "ns-config";
#address-cells = <1>;
#size-cells = <1>;
ranges;
msi-controller@2f130000 {
reg = <0x2f130000 0x10000>; // ITS_TRANSLATE_FRAME
reg-names = "ns-translate";
#msi-cells = <1>;
msi-controller;
};
};
};
};
...
Documentation/virt/kvm/api.rst
+37 -3
View File
@@ -2006,7 +2006,7 @@ frequency is KHz.
If the KVM_CAP_VM_TSC_CONTROL capability is advertised, this can also
be used as a vm ioctl to set the initial tsc frequency of subsequently
created vCPUs.
created vCPUs. Note, the vm ioctl is only allowed prior to creating vCPUs.
For TSC protected Confidential Computing (CoCo) VMs where TSC frequency
is configured once at VM scope and remains unchanged during VM's
@@ -7851,6 +7851,7 @@ Valid bits in args[0] are::
#define KVM_X86_DISABLE_EXITS_HLT (1 << 1)
#define KVM_X86_DISABLE_EXITS_PAUSE (1 << 2)
#define KVM_X86_DISABLE_EXITS_CSTATE (1 << 3)
#define KVM_X86_DISABLE_EXITS_APERFMPERF (1 << 4)
Enabling this capability on a VM provides userspace with a way to no
longer intercept some instructions for improved latency in some
@@ -7861,6 +7862,28 @@ all such vmexits.
Do not enable KVM_FEATURE_PV_UNHALT if you disable HLT exits.
Virtualizing the ``IA32_APERF`` and ``IA32_MPERF`` MSRs requires more
than just disabling APERF/MPERF exits. While both Intel and AMD
document strict usage conditions for these MSRs--emphasizing that only
the ratio of their deltas over a time interval (T0 to T1) is
architecturally defined--simply passing through the MSRs can still
produce an incorrect ratio.
This erroneous ratio can occur if, between T0 and T1:
1. The vCPU thread migrates between logical processors.
2. Live migration or suspend/resume operations take place.
3. Another task shares the vCPU's logical processor.
4. C-states lower than C0 are emulated (e.g., via HLT interception).
5. The guest TSC frequency doesn't match the host TSC frequency.
Due to these complexities, KVM does not automatically associate this
passthrough capability with the guest CPUID bit,
``CPUID.6:ECX.APERFMPERF[bit 0]``. Userspace VMMs that deem this
mechanism adequate for virtualizing the ``IA32_APERF`` and
``IA32_MPERF`` MSRs must set the guest CPUID bit explicitly.
7.14 KVM_CAP_S390_HPAGE_1M
--------------------------
@@ -8387,7 +8410,7 @@ core crystal clock frequency, if a non-zero CPUID 0x15 is exposed to the guest.
7.36 KVM_CAP_DIRTY_LOG_RING/KVM_CAP_DIRTY_LOG_RING_ACQ_REL
----------------------------------------------------------
:Architectures: x86, arm64
:Architectures: x86, arm64, riscv
:Type: vm
:Parameters: args[0] - size of the dirty log ring
@@ -8599,7 +8622,7 @@ ENOSYS for the others.
When enabled, KVM will exit to userspace with KVM_EXIT_SYSTEM_EVENT of
type KVM_SYSTEM_EVENT_SUSPEND to process the guest suspend request.
7.37 KVM_CAP_ARM_WRITABLE_IMP_ID_REGS
7.42 KVM_CAP_ARM_WRITABLE_IMP_ID_REGS
-------------------------------------
:Architectures: arm64
@@ -8628,6 +8651,17 @@ given VM.
When this capability is enabled, KVM resets the VCPU when setting
MP_STATE_INIT_RECEIVED through IOCTL. The original MP_STATE is preserved.
7.43 KVM_CAP_ARM_CACHEABLE_PFNMAP_SUPPORTED
-------------------------------------------
:Architectures: arm64
:Target: VM
:Parameters: None
This capability indicate to the userspace whether a PFNMAP memory region
can be safely mapped as cacheable. This relies on the presence of
force write back (FWB) feature support on the hardware.
8. Other capabilities.
======================
Documentation/virt/kvm/devices/arm-vgic-v3.rst
+72 -5
View File
@@ -78,6 +78,8 @@ Groups:
-ENXIO The group or attribute is unknown/unsupported for this device
or hardware support is missing.
-EFAULT Invalid user pointer for attr->addr.
-EBUSY Attempt to write a register that is read-only after
initialization
======= =============================================================
@@ -120,6 +122,12 @@ Groups:
Note that distributor fields are not banked, but return the same value
regardless of the mpidr used to access the register.
Userspace is allowed to write the following register fields prior to
initialization of the VGIC:
* GICD_IIDR.Revision
* GICD_TYPER2.nASSGIcap
GICD_IIDR.Revision is updated when the KVM implementation is changed in a
way directly observable by the guest or userspace. Userspace should read
GICD_IIDR from KVM and write back the read value to confirm its expected
@@ -128,6 +136,12 @@ Groups:
behavior.
GICD_TYPER2.nASSGIcap allows userspace to control the support of SGIs
without an active state. At VGIC creation the field resets to the
maximum capability of the system. Userspace is expected to read the field
to determine the supported value(s) before writing to the field.
The GICD_STATUSR and GICR_STATUSR registers are architecturally defined such
that a write of a clear bit has no effect, whereas a write with a set bit
clears that value. To allow userspace to freely set the values of these two
@@ -202,16 +216,69 @@ Groups:
KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS accesses the CPU interface registers for the
CPU specified by the mpidr field.
CPU interface registers access is not implemented for AArch32 mode.
Error -ENXIO is returned when accessed in AArch32 mode.
The available registers are:
=============== ====================================================
ICC_PMR_EL1
ICC_BPR0_EL1
ICC_AP0R0_EL1
ICC_AP0R1_EL1 when the host implements at least 6 bits of priority
ICC_AP0R2_EL1 when the host implements 7 bits of priority
ICC_AP0R3_EL1 when the host implements 7 bits of priority
ICC_AP1R0_EL1
ICC_AP1R1_EL1 when the host implements at least 6 bits of priority
ICC_AP1R2_EL1 when the host implements 7 bits of priority
ICC_AP1R3_EL1 when the host implements 7 bits of priority
ICC_BPR1_EL1
ICC_CTLR_EL1
ICC_SRE_EL1
ICC_IGRPEN0_EL1
ICC_IGRPEN1_EL1
=============== ====================================================
When EL2 is available for the guest, these registers are also available:
============= ====================================================
ICH_AP0R0_EL2
ICH_AP0R1_EL2 when the host implements at least 6 bits of priority
ICH_AP0R2_EL2 when the host implements 7 bits of priority
ICH_AP0R3_EL2 when the host implements 7 bits of priority
ICH_AP1R0_EL2
ICH_AP1R1_EL2 when the host implements at least 6 bits of priority
ICH_AP1R2_EL2 when the host implements 7 bits of priority
ICH_AP1R3_EL2 when the host implements 7 bits of priority
ICH_HCR_EL2
ICC_SRE_EL2
ICH_VTR_EL2
ICH_VMCR_EL2
ICH_LR0_EL2
ICH_LR1_EL2
ICH_LR2_EL2
ICH_LR3_EL2
ICH_LR4_EL2
ICH_LR5_EL2
ICH_LR6_EL2
ICH_LR7_EL2
ICH_LR8_EL2
ICH_LR9_EL2
ICH_LR10_EL2
ICH_LR11_EL2
ICH_LR12_EL2
ICH_LR13_EL2
ICH_LR14_EL2
ICH_LR15_EL2
============= ====================================================
CPU interface registers are only described using the AArch64
encoding.
Errors:
======= =====================================================
-ENXIO Getting or setting this register is not yet supported
======= =================================================
-ENXIO Getting or setting this register is not supported
-EBUSY VCPU is running
-EINVAL Invalid mpidr or register value supplied
======= =====================================================
======= =================================================
KVM_DEV_ARM_VGIC_GRP_NR_IRQS
MAINTAINERS
+10
View File
@@ -2002,6 +2002,16 @@ F: drivers/irqchip/irq-gic*.[ch]
F: include/linux/irqchip/arm-gic*.h
F: include/linux/irqchip/arm-vgic-info.h
ARM GENERIC INTERRUPT CONTROLLER V5 DRIVERS
M: Lorenzo Pieralisi <lpieralisi@kernel.org>
M: Marc Zyngier <maz@kernel.org>
L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
S: Maintained
F: Documentation/devicetree/bindings/interrupt-controller/arm,gic-v5*.yaml
F: drivers/irqchip/irq-gic-its-msi-parent.[ch]
F: drivers/irqchip/irq-gic-v5*.[ch]
F: include/linux/irqchip/arm-gic-v5.h
ARM HDLCD DRM DRIVER
M: Liviu Dudau <liviu.dudau@arm.com>
S: Supported
arch/arm64/Kconfig
+1
View File
@@ -127,6 +127,7 @@ config ARM64
select ARM_GIC_V2M if PCI
select ARM_GIC_V3
select ARM_GIC_V3_ITS if PCI
select ARM_GIC_V5
select ARM_PSCI_FW
select BUILDTIME_TABLE_SORT
select CLONE_BACKWARDS
arch/arm64/include/asm/barrier.h
+3
View File
@@ -44,6 +44,9 @@
SB_BARRIER_INSN"nop\n", \
ARM64_HAS_SB))
#define gsb_ack() asm volatile(GSB_ACK_BARRIER_INSN : : : "memory")
#define gsb_sys() asm volatile(GSB_SYS_BARRIER_INSN : : : "memory")
#ifdef CONFIG_ARM64_PSEUDO_NMI
#define pmr_sync() \
do { \
arch/arm64/include/asm/el2_setup.h
+45
View File
@@ -165,6 +165,50 @@
.Lskip_gicv3_\@:
.endm
/* GICv5 system register access */
.macro __init_el2_gicv5
mrs_s x0, SYS_ID_AA64PFR2_EL1
ubfx x0, x0, #ID_AA64PFR2_EL1_GCIE_SHIFT, #4
cbz x0, .Lskip_gicv5_\@
mov x0, #(ICH_HFGITR_EL2_GICRCDNMIA | \
ICH_HFGITR_EL2_GICRCDIA | \
ICH_HFGITR_EL2_GICCDDI | \
ICH_HFGITR_EL2_GICCDEOI | \
ICH_HFGITR_EL2_GICCDHM | \
ICH_HFGITR_EL2_GICCDRCFG | \
ICH_HFGITR_EL2_GICCDPEND | \
ICH_HFGITR_EL2_GICCDAFF | \
ICH_HFGITR_EL2_GICCDPRI | \
ICH_HFGITR_EL2_GICCDDIS | \
ICH_HFGITR_EL2_GICCDEN)
msr_s SYS_ICH_HFGITR_EL2, x0 // Disable instruction traps
mov_q x0, (ICH_HFGRTR_EL2_ICC_PPI_ACTIVERn_EL1 | \
ICH_HFGRTR_EL2_ICC_PPI_PRIORITYRn_EL1 | \
ICH_HFGRTR_EL2_ICC_PPI_PENDRn_EL1 | \
ICH_HFGRTR_EL2_ICC_PPI_ENABLERn_EL1 | \
ICH_HFGRTR_EL2_ICC_PPI_HMRn_EL1 | \
ICH_HFGRTR_EL2_ICC_IAFFIDR_EL1 | \
ICH_HFGRTR_EL2_ICC_ICSR_EL1 | \
ICH_HFGRTR_EL2_ICC_PCR_EL1 | \
ICH_HFGRTR_EL2_ICC_HPPIR_EL1 | \
ICH_HFGRTR_EL2_ICC_HAPR_EL1 | \
ICH_HFGRTR_EL2_ICC_CR0_EL1 | \
ICH_HFGRTR_EL2_ICC_IDRn_EL1 | \
ICH_HFGRTR_EL2_ICC_APR_EL1)
msr_s SYS_ICH_HFGRTR_EL2, x0 // Disable reg read traps
mov_q x0, (ICH_HFGWTR_EL2_ICC_PPI_ACTIVERn_EL1 | \
ICH_HFGWTR_EL2_ICC_PPI_PRIORITYRn_EL1 | \
ICH_HFGWTR_EL2_ICC_PPI_PENDRn_EL1 | \
ICH_HFGWTR_EL2_ICC_PPI_ENABLERn_EL1 | \
ICH_HFGWTR_EL2_ICC_ICSR_EL1 | \
ICH_HFGWTR_EL2_ICC_PCR_EL1 | \
ICH_HFGWTR_EL2_ICC_CR0_EL1 | \
ICH_HFGWTR_EL2_ICC_APR_EL1)
msr_s SYS_ICH_HFGWTR_EL2, x0 // Disable reg write traps
.Lskip_gicv5_\@:
.endm
.macro __init_el2_hstr
msr hstr_el2, xzr // Disable CP15 traps to EL2
.endm
@@ -368,6 +412,7 @@
__init_el2_lor
__init_el2_stage2
__init_el2_gicv3
__init_el2_gicv5
__init_el2_hstr
__init_el2_nvhe_idregs
__init_el2_cptr
arch/arm64/include/asm/kvm_emulate.h
+48 -3
View File
@@ -45,16 +45,39 @@ bool kvm_condition_valid32(const struct kvm_vcpu *vcpu);
void kvm_skip_instr32(struct kvm_vcpu *vcpu);
void kvm_inject_undefined(struct kvm_vcpu *vcpu);
void kvm_inject_vabt(struct kvm_vcpu *vcpu);
void kvm_inject_dabt(struct kvm_vcpu *vcpu, unsigned long addr);
void kvm_inject_pabt(struct kvm_vcpu *vcpu, unsigned long addr);
int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr);
int kvm_inject_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr);
void kvm_inject_size_fault(struct kvm_vcpu *vcpu);
static inline int kvm_inject_sea_dabt(struct kvm_vcpu *vcpu, u64 addr)
{
return kvm_inject_sea(vcpu, false, addr);
}
static inline int kvm_inject_sea_iabt(struct kvm_vcpu *vcpu, u64 addr)
{
return kvm_inject_sea(vcpu, true, addr);
}
static inline int kvm_inject_serror(struct kvm_vcpu *vcpu)
{
/*
* ESR_ELx.ISV (later renamed to IDS) indicates whether or not
* ESR_ELx.ISS contains IMPLEMENTATION DEFINED syndrome information.
*
* Set the bit when injecting an SError w/o an ESR to indicate ISS
* does not follow the architected format.
*/
return kvm_inject_serror_esr(vcpu, ESR_ELx_ISV);
}
void kvm_vcpu_wfi(struct kvm_vcpu *vcpu);
void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu);
int kvm_inject_nested_sync(struct kvm_vcpu *vcpu, u64 esr_el2);
int kvm_inject_nested_irq(struct kvm_vcpu *vcpu);
int kvm_inject_nested_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr);
int kvm_inject_nested_serror(struct kvm_vcpu *vcpu, u64 esr);
static inline void kvm_inject_nested_sve_trap(struct kvm_vcpu *vcpu)
{
@@ -195,6 +218,11 @@ static inline bool vcpu_el2_tge_is_set(const struct kvm_vcpu *vcpu)
return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_TGE;
}
static inline bool vcpu_el2_amo_is_set(const struct kvm_vcpu *vcpu)
{
return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_AMO;
}
static inline bool is_hyp_ctxt(const struct kvm_vcpu *vcpu)
{
bool e2h, tge;
@@ -224,6 +252,20 @@ static inline bool vcpu_is_host_el0(const struct kvm_vcpu *vcpu)
return is_hyp_ctxt(vcpu) && !vcpu_is_el2(vcpu);
}
static inline bool is_nested_ctxt(struct kvm_vcpu *vcpu)
{
return vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu);
}
static inline bool vserror_state_is_nested(struct kvm_vcpu *vcpu)
{
if (!is_nested_ctxt(vcpu))
return false;
return vcpu_el2_amo_is_set(vcpu) ||
(__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TMEA);
}
/*
* The layout of SPSR for an AArch32 state is different when observed from an
* AArch64 SPSR_ELx or an AArch32 SPSR_*. This function generates the AArch32
@@ -627,6 +669,9 @@ static inline void vcpu_set_hcrx(struct kvm_vcpu *vcpu)
if (kvm_has_fpmr(kvm))
vcpu->arch.hcrx_el2 |= HCRX_EL2_EnFPM;
if (kvm_has_sctlr2(kvm))
vcpu->arch.hcrx_el2 |= HCRX_EL2_SCTLR2En;
}
}
#endif /* __ARM64_KVM_EMULATE_H__ */
arch/arm64/include/asm/kvm_host.h
+33 -3
View File
@@ -523,6 +523,7 @@ enum vcpu_sysreg {
/* Anything from this can be RES0/RES1 sanitised */
MARKER(__SANITISED_REG_START__),
TCR2_EL2, /* Extended Translation Control Register (EL2) */
SCTLR2_EL2, /* System Control Register 2 (EL2) */
MDCR_EL2, /* Monitor Debug Configuration Register (EL2) */
CNTHCTL_EL2, /* Counter-timer Hypervisor Control register */
@@ -537,6 +538,7 @@ enum vcpu_sysreg {
VNCR(TTBR1_EL1),/* Translation Table Base Register 1 */
VNCR(TCR_EL1), /* Translation Control Register */
VNCR(TCR2_EL1), /* Extended Translation Control Register */
VNCR(SCTLR2_EL1), /* System Control Register 2 */
VNCR(ESR_EL1), /* Exception Syndrome Register */
VNCR(AFSR0_EL1),/* Auxiliary Fault Status Register 0 */
VNCR(AFSR1_EL1),/* Auxiliary Fault Status Register 1 */
@@ -565,6 +567,10 @@ enum vcpu_sysreg {
VNCR(POR_EL1), /* Permission Overlay Register 1 (EL1) */
/* FEAT_RAS registers */
VNCR(VDISR_EL2),
VNCR(VSESR_EL2),
VNCR(HFGRTR_EL2),
VNCR(HFGWTR_EL2),
VNCR(HFGITR_EL2),
@@ -819,7 +825,7 @@ struct kvm_vcpu_arch {
u8 iflags;
/* State flags for kernel bookkeeping, unused by the hypervisor code */
u8 sflags;
u16 sflags;
/*
* Don't run the guest (internal implementation need).
@@ -955,9 +961,21 @@ struct kvm_vcpu_arch {
__vcpu_flags_preempt_enable(); \
} while (0)
#define __vcpu_test_and_clear_flag(v, flagset, f, m) \
({ \
typeof(v->arch.flagset) set; \
\
set = __vcpu_get_flag(v, flagset, f, m); \
__vcpu_clear_flag(v, flagset, f, m); \
\
set; \
})
#define vcpu_get_flag(v, ...) __vcpu_get_flag((v), __VA_ARGS__)
#define vcpu_set_flag(v, ...) __vcpu_set_flag((v), __VA_ARGS__)
#define vcpu_clear_flag(v, ...) __vcpu_clear_flag((v), __VA_ARGS__)
#define vcpu_test_and_clear_flag(v, ...) \
__vcpu_test_and_clear_flag((v), __VA_ARGS__)
/* KVM_ARM_VCPU_INIT completed */
#define VCPU_INITIALIZED __vcpu_single_flag(cflags, BIT(0))
@@ -1017,6 +1035,8 @@ struct kvm_vcpu_arch {
#define IN_WFI __vcpu_single_flag(sflags, BIT(6))
/* KVM is currently emulating a nested ERET */
#define IN_NESTED_ERET __vcpu_single_flag(sflags, BIT(7))
/* SError pending for nested guest */
#define NESTED_SERROR_PENDING __vcpu_single_flag(sflags, BIT(8))
/* Pointer to the vcpu's SVE FFR for sve_{save,load}_state() */
@@ -1151,6 +1171,8 @@ static inline bool __vcpu_read_sys_reg_from_cpu(int reg, u64 *val)
* System registers listed in the switch are not saved on every
* exit from the guest but are only saved on vcpu_put.
*
* SYSREGS_ON_CPU *MUST* be checked before using this helper.
*
* Note that MPIDR_EL1 for the guest is set by KVM via VMPIDR_EL2 but
* should never be listed below, because the guest cannot modify its
* own MPIDR_EL1 and MPIDR_EL1 is accessed for VCPU A from VCPU B's
@@ -1188,6 +1210,7 @@ static inline bool __vcpu_read_sys_reg_from_cpu(int reg, u64 *val)
case IFSR32_EL2: *val = read_sysreg_s(SYS_IFSR32_EL2); break;
case DBGVCR32_EL2: *val = read_sysreg_s(SYS_DBGVCR32_EL2); break;
case ZCR_EL1: *val = read_sysreg_s(SYS_ZCR_EL12); break;
case SCTLR2_EL1: *val = read_sysreg_s(SYS_SCTLR2_EL12); break;
default: return false;
}
@@ -1202,6 +1225,8 @@ static inline bool __vcpu_write_sys_reg_to_cpu(u64 val, int reg)
* System registers listed in the switch are not restored on every
* entry to the guest but are only restored on vcpu_load.
*
* SYSREGS_ON_CPU *MUST* be checked before using this helper.
*
* Note that MPIDR_EL1 for the guest is set by KVM via VMPIDR_EL2 but
* should never be listed below, because the MPIDR should only be set
* once, before running the VCPU, and never changed later.
@@ -1238,6 +1263,7 @@ static inline bool __vcpu_write_sys_reg_to_cpu(u64 val, int reg)
case IFSR32_EL2: write_sysreg_s(val, SYS_IFSR32_EL2); break;
case DBGVCR32_EL2: write_sysreg_s(val, SYS_DBGVCR32_EL2); break;
case ZCR_EL1: write_sysreg_s(val, SYS_ZCR_EL12); break;
case SCTLR2_EL1: write_sysreg_s(val, SYS_SCTLR2_EL12); break;
default: return false;
}
@@ -1389,8 +1415,6 @@ static inline bool kvm_arm_is_pvtime_enabled(struct kvm_vcpu_arch *vcpu_arch)
return (vcpu_arch->steal.base != INVALID_GPA);
}
void kvm_set_sei_esr(struct kvm_vcpu *vcpu, u64 syndrome);
struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr);
DECLARE_KVM_HYP_PER_CPU(struct kvm_host_data, kvm_host_data);
@@ -1667,6 +1691,12 @@ void kvm_set_vm_id_reg(struct kvm *kvm, u32 reg, u64 val);
#define kvm_has_s1poe(k) \
(kvm_has_feat((k), ID_AA64MMFR3_EL1, S1POE, IMP))
#define kvm_has_ras(k) \
(kvm_has_feat((k), ID_AA64PFR0_EL1, RAS, IMP))
#define kvm_has_sctlr2(k) \
(kvm_has_feat((k), ID_AA64MMFR3_EL1, SCTLRX, IMP))
static inline bool kvm_arch_has_irq_bypass(void)
{
return true;
arch/arm64/include/asm/kvm_mmu.h
+18
View File
@@ -371,6 +371,24 @@ static inline void kvm_fault_unlock(struct kvm *kvm)
read_unlock(&kvm->mmu_lock);
}
/*
* ARM64 KVM relies on a simple conversion from physaddr to a kernel
* virtual address (KVA) when it does cache maintenance as the CMO
* instructions work on virtual addresses. This is incompatible with
* VM_PFNMAP VMAs which may not have a kernel direct mapping to a
* virtual address.
*
* With S2FWB and CACHE DIC features, KVM need not do cache flushing
* and CMOs are NOP'd. This has the effect of no longer requiring a
* KVA for addresses mapped into the S2. The presence of these features
* are thus necessary to support cacheable S2 mapping of VM_PFNMAP.
*/
static inline bool kvm_supports_cacheable_pfnmap(void)
{
return cpus_have_final_cap(ARM64_HAS_STAGE2_FWB) &&
cpus_have_final_cap(ARM64_HAS_CACHE_DIC);
}
#ifdef CONFIG_PTDUMP_STAGE2_DEBUGFS
void kvm_s2_ptdump_create_debugfs(struct kvm *kvm);
#else
arch/arm64/include/asm/kvm_nested.h
+2
View File
@@ -80,6 +80,8 @@ extern void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu);
extern void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu);
extern void check_nested_vcpu_requests(struct kvm_vcpu *vcpu);
extern void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu);
extern void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu);
struct kvm_s2_trans {
phys_addr_t output;
arch/arm64/include/asm/smp.h
+23 -1
View File
@@ -50,10 +50,32 @@ struct seq_file;
*/
extern void smp_init_cpus(void);
enum ipi_msg_type {
IPI_RESCHEDULE,
IPI_CALL_FUNC,
IPI_CPU_STOP,
IPI_CPU_STOP_NMI,
IPI_TIMER,
IPI_IRQ_WORK,
NR_IPI,
/*
* Any enum >= NR_IPI and < MAX_IPI is special and not tracable
* with trace_ipi_*
*/
IPI_CPU_BACKTRACE = NR_IPI,
IPI_KGDB_ROUNDUP,
MAX_IPI
};
/*
* Register IPI interrupts with the arch SMP code
*/
extern void set_smp_ipi_range(int ipi_base, int nr_ipi);
extern void set_smp_ipi_range_percpu(int ipi_base, int nr_ipi, int ncpus);
static inline void set_smp_ipi_range(int ipi_base, int n)
{
set_smp_ipi_range_percpu(ipi_base, n, 0);
}
/*
* Called from the secondary holding pen, this is the secondary CPU entry point.
arch/arm64/include/asm/sysreg.h
+68 -3
View File
@@ -113,10 +113,14 @@
/* Register-based PAN access, for save/restore purposes */
#define SYS_PSTATE_PAN sys_reg(3, 0, 4, 2, 3)
#define __SYS_BARRIER_INSN(CRm, op2, Rt) \
__emit_inst(0xd5000000 | sys_insn(0, 3, 3, (CRm), (op2)) | ((Rt) & 0x1f))
#define __SYS_BARRIER_INSN(op0, op1, CRn, CRm, op2, Rt) \
__emit_inst(0xd5000000 | \
sys_insn((op0), (op1), (CRn), (CRm), (op2)) | \
((Rt) & 0x1f))
#define SB_BARRIER_INSN __SYS_BARRIER_INSN(0, 7, 31)
#define SB_BARRIER_INSN __SYS_BARRIER_INSN(0, 3, 3, 0, 7, 31)
#define GSB_SYS_BARRIER_INSN __SYS_BARRIER_INSN(1, 0, 12, 0, 0, 31)
#define GSB_ACK_BARRIER_INSN __SYS_BARRIER_INSN(1, 0, 12, 0, 1, 31)
/* Data cache zero operations */
#define SYS_DC_ISW sys_insn(1, 0, 7, 6, 2)
@@ -1074,6 +1078,67 @@
#define GCS_CAP(x) ((((unsigned long)x) & GCS_CAP_ADDR_MASK) | \
GCS_CAP_VALID_TOKEN)
/*
* Definitions for GICv5 instructions
*/
#define GICV5_OP_GIC_CDAFF sys_insn(1, 0, 12, 1, 3)
#define GICV5_OP_GIC_CDDI sys_insn(1, 0, 12, 2, 0)
#define GICV5_OP_GIC_CDDIS sys_insn(1, 0, 12, 1, 0)
#define GICV5_OP_GIC_CDHM sys_insn(1, 0, 12, 2, 1)
#define GICV5_OP_GIC_CDEN sys_insn(1, 0, 12, 1, 1)
#define GICV5_OP_GIC_CDEOI sys_insn(1, 0, 12, 1, 7)
#define GICV5_OP_GIC_CDPEND sys_insn(1, 0, 12, 1, 4)
#define GICV5_OP_GIC_CDPRI sys_insn(1, 0, 12, 1, 2)
#define GICV5_OP_GIC_CDRCFG sys_insn(1, 0, 12, 1, 5)
#define GICV5_OP_GICR_CDIA sys_insn(1, 0, 12, 3, 0)
/* Definitions for GIC CDAFF */
#define GICV5_GIC_CDAFF_IAFFID_MASK GENMASK_ULL(47, 32)
#define GICV5_GIC_CDAFF_TYPE_MASK GENMASK_ULL(31, 29)
#define GICV5_GIC_CDAFF_IRM_MASK BIT_ULL(28)
#define GICV5_GIC_CDAFF_ID_MASK GENMASK_ULL(23, 0)
/* Definitions for GIC CDDI */
#define GICV5_GIC_CDDI_TYPE_MASK GENMASK_ULL(31, 29)
#define GICV5_GIC_CDDI_ID_MASK GENMASK_ULL(23, 0)
/* Definitions for GIC CDDIS */
#define GICV5_GIC_CDDIS_TYPE_MASK GENMASK_ULL(31, 29)
#define GICV5_GIC_CDDIS_TYPE(r) FIELD_GET(GICV5_GIC_CDDIS_TYPE_MASK, r)
#define GICV5_GIC_CDDIS_ID_MASK GENMASK_ULL(23, 0)
#define GICV5_GIC_CDDIS_ID(r) FIELD_GET(GICV5_GIC_CDDIS_ID_MASK, r)
/* Definitions for GIC CDEN */
#define GICV5_GIC_CDEN_TYPE_MASK GENMASK_ULL(31, 29)
#define GICV5_GIC_CDEN_ID_MASK GENMASK_ULL(23, 0)
/* Definitions for GIC CDHM */
#define GICV5_GIC_CDHM_HM_MASK BIT_ULL(32)
#define GICV5_GIC_CDHM_TYPE_MASK GENMASK_ULL(31, 29)
#define GICV5_GIC_CDHM_ID_MASK GENMASK_ULL(23, 0)
/* Definitions for GIC CDPEND */
#define GICV5_GIC_CDPEND_PENDING_MASK BIT_ULL(32)
#define GICV5_GIC_CDPEND_TYPE_MASK GENMASK_ULL(31, 29)
#define GICV5_GIC_CDPEND_ID_MASK GENMASK_ULL(23, 0)
/* Definitions for GIC CDPRI */
#define GICV5_GIC_CDPRI_PRIORITY_MASK GENMASK_ULL(39, 35)
#define GICV5_GIC_CDPRI_TYPE_MASK GENMASK_ULL(31, 29)
#define GICV5_GIC_CDPRI_ID_MASK GENMASK_ULL(23, 0)
/* Definitions for GIC CDRCFG */
#define GICV5_GIC_CDRCFG_TYPE_MASK GENMASK_ULL(31, 29)
#define GICV5_GIC_CDRCFG_ID_MASK GENMASK_ULL(23, 0)
/* Definitions for GICR CDIA */
#define GICV5_GIC_CDIA_VALID_MASK BIT_ULL(32)
#define GICV5_GICR_CDIA_VALID(r) FIELD_GET(GICV5_GIC_CDIA_VALID_MASK, r)
#define GICV5_GIC_CDIA_TYPE_MASK GENMASK_ULL(31, 29)
#define GICV5_GIC_CDIA_ID_MASK GENMASK_ULL(23, 0)
#define gicr_insn(insn) read_sysreg_s(GICV5_OP_GICR_##insn)
#define gic_insn(v, insn) write_sysreg_s(v, GICV5_OP_GIC_##insn)
#define ARM64_FEATURE_FIELD_BITS 4
arch/arm64/include/asm/vncr_mapping.h
+2
View File
@@ -51,6 +51,7 @@
#define VNCR_SP_EL1 0x240
#define VNCR_VBAR_EL1 0x250
#define VNCR_TCR2_EL1 0x270
#define VNCR_SCTLR2_EL1 0x278
#define VNCR_PIRE0_EL1 0x290
#define VNCR_PIR_EL1 0x2A0
#define VNCR_POR_EL1 0x2A8
@@ -84,6 +85,7 @@
#define VNCR_ICH_HCR_EL2 0x4C0
#define VNCR_ICH_VMCR_EL2 0x4C8
#define VNCR_VDISR_EL2 0x500
#define VNCR_VSESR_EL2 0x508
#define VNCR_PMBLIMITR_EL1 0x800
#define VNCR_PMBPTR_EL1 0x810
#define VNCR_PMBSR_EL1 0x820
arch/arm64/kernel/cpufeature.c
+21 -5
View File
@@ -303,6 +303,7 @@ static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = {
};
static const struct arm64_ftr_bits ftr_id_aa64pfr1[] = {
ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_DF2_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_GCS),
FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_GCS_SHIFT, 4, 0),
S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_MTE_frac_SHIFT, 4, 0),
@@ -502,6 +503,7 @@ static const struct arm64_ftr_bits ftr_id_aa64mmfr3[] = {
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_POE),
FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_S1POE_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_S1PIE_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_SCTLRX_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_TCRX_SHIFT, 4, 0),
ARM64_FTR_END,
};
@@ -2331,11 +2333,11 @@ static bool can_use_gic_priorities(const struct arm64_cpu_capabilities *entry,
int scope)
{
/*
* ARM64_HAS_GIC_CPUIF_SYSREGS has a lower index, and is a boot CPU
* ARM64_HAS_GICV3_CPUIF has a lower index, and is a boot CPU
* feature, so will be detected earlier.
*/
BUILD_BUG_ON(ARM64_HAS_GIC_PRIO_MASKING <= ARM64_HAS_GIC_CPUIF_SYSREGS);
if (!cpus_have_cap(ARM64_HAS_GIC_CPUIF_SYSREGS))
BUILD_BUG_ON(ARM64_HAS_GIC_PRIO_MASKING <= ARM64_HAS_GICV3_CPUIF);
if (!cpus_have_cap(ARM64_HAS_GICV3_CPUIF))
return false;
return enable_pseudo_nmi;
@@ -2531,8 +2533,8 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
.matches = has_always,
},
{
.desc = "GIC system register CPU interface",
.capability = ARM64_HAS_GIC_CPUIF_SYSREGS,
.desc = "GICv3 CPU interface",
.capability = ARM64_HAS_GICV3_CPUIF,
.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
.matches = has_useable_gicv3_cpuif,
ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, GIC, IMP)
@@ -3116,6 +3118,20 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
.matches = has_pmuv3,
},
#endif
{
.desc = "SCTLR2",
.capability = ARM64_HAS_SCTLR2,
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.matches = has_cpuid_feature,
ARM64_CPUID_FIELDS(ID_AA64MMFR3_EL1, SCTLRX, IMP)
},
{
.desc = "GICv5 CPU interface",
.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
.capability = ARM64_HAS_GICV5_CPUIF,
.matches = has_cpuid_feature,
ARM64_CPUID_FIELDS(ID_AA64PFR2_EL1, GCIE, IMP)
},
{},
};
arch/arm64/kernel/smp.c
+93 -51
View File
@@ -70,26 +70,18 @@ struct secondary_data secondary_data;
/* Number of CPUs which aren't online, but looping in kernel text. */
static int cpus_stuck_in_kernel;
enum ipi_msg_type {
IPI_RESCHEDULE,
IPI_CALL_FUNC,
IPI_CPU_STOP,
IPI_CPU_STOP_NMI,
IPI_TIMER,
IPI_IRQ_WORK,
NR_IPI,
/*
* Any enum >= NR_IPI and < MAX_IPI is special and not tracable
* with trace_ipi_*
*/
IPI_CPU_BACKTRACE = NR_IPI,
IPI_KGDB_ROUNDUP,
MAX_IPI
};
static int ipi_irq_base __ro_after_init;
static int nr_ipi __ro_after_init = NR_IPI;
static struct irq_desc *ipi_desc[MAX_IPI] __ro_after_init;
struct ipi_descs {
struct irq_desc *descs[MAX_IPI];
};
static DEFINE_PER_CPU_READ_MOSTLY(struct ipi_descs, pcpu_ipi_desc);
#define get_ipi_desc(__cpu, __ipi) (per_cpu_ptr(&pcpu_ipi_desc, __cpu)->descs[__ipi])
static bool percpu_ipi_descs __ro_after_init;
static bool crash_stop;
@@ -850,7 +842,7 @@ int arch_show_interrupts(struct seq_file *p, int prec)
seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i,
prec >= 4 ? " " : "");
for_each_online_cpu(cpu)
seq_printf(p, "%10u ", irq_desc_kstat_cpu(ipi_desc[i], cpu));
seq_printf(p, "%10u ", irq_desc_kstat_cpu(get_ipi_desc(cpu, i), cpu));
seq_printf(p, " %s\n", ipi_types[i]);
}
@@ -923,9 +915,20 @@ static void __noreturn ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs
#endif
}
static void arm64_send_ipi(const cpumask_t *mask, unsigned int nr)
{
unsigned int cpu;
if (!percpu_ipi_descs)
__ipi_send_mask(get_ipi_desc(0, nr), mask);
else
for_each_cpu(cpu, mask)
__ipi_send_single(get_ipi_desc(cpu, nr), cpu);
}
static void arm64_backtrace_ipi(cpumask_t *mask)
{
__ipi_send_mask(ipi_desc[IPI_CPU_BACKTRACE], mask);
arm64_send_ipi(mask, IPI_CPU_BACKTRACE);
}
void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu)
@@ -950,7 +953,7 @@ void kgdb_roundup_cpus(void)
if (cpu == this_cpu)
continue;
__ipi_send_single(ipi_desc[IPI_KGDB_ROUNDUP], cpu);
__ipi_send_single(get_ipi_desc(cpu, IPI_KGDB_ROUNDUP), cpu);
}
}
#endif
@@ -1020,14 +1023,16 @@ static void do_handle_IPI(int ipinr)
static irqreturn_t ipi_handler(int irq, void *data)
{
do_handle_IPI(irq - ipi_irq_base);
unsigned int ipi = (irq - ipi_irq_base) % nr_ipi;
do_handle_IPI(ipi);
return IRQ_HANDLED;
}
static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
{
trace_ipi_raise(target, ipi_types[ipinr]);
__ipi_send_mask(ipi_desc[ipinr], target);
arm64_send_ipi(target, ipinr);
}
static bool ipi_should_be_nmi(enum ipi_msg_type ipi)
@@ -1053,11 +1058,15 @@ static void ipi_setup(int cpu)
return;
for (i = 0; i < nr_ipi; i++) {
if (ipi_should_be_nmi(i)) {
prepare_percpu_nmi(ipi_irq_base + i);
enable_percpu_nmi(ipi_irq_base + i, 0);
if (!percpu_ipi_descs) {
if (ipi_should_be_nmi(i)) {
prepare_percpu_nmi(ipi_irq_base + i);
enable_percpu_nmi(ipi_irq_base + i, 0);
} else {
enable_percpu_irq(ipi_irq_base + i, 0);
}
} else {
enable_percpu_irq(ipi_irq_base + i, 0);
enable_irq(irq_desc_get_irq(get_ipi_desc(cpu, i)));
}
}
}
@@ -1071,44 +1080,77 @@ static void ipi_teardown(int cpu)
return;
for (i = 0; i < nr_ipi; i++) {
if (ipi_should_be_nmi(i)) {
disable_percpu_nmi(ipi_irq_base + i);
teardown_percpu_nmi(ipi_irq_base + i);
if (!percpu_ipi_descs) {
if (ipi_should_be_nmi(i)) {
disable_percpu_nmi(ipi_irq_base + i);
teardown_percpu_nmi(ipi_irq_base + i);
} else {
disable_percpu_irq(ipi_irq_base + i);
}
} else {
disable_percpu_irq(ipi_irq_base + i);
disable_irq(irq_desc_get_irq(get_ipi_desc(cpu, i)));
}
}
}
#endif
void __init set_smp_ipi_range(int ipi_base, int n)
static void ipi_setup_sgi(int ipi)
{
int err, irq, cpu;
irq = ipi_irq_base + ipi;
if (ipi_should_be_nmi(ipi)) {
err = request_percpu_nmi(irq, ipi_handler, "IPI", &irq_stat);
WARN(err, "Could not request IRQ %d as NMI, err=%d\n", irq, err);
} else {
err = request_percpu_irq(irq, ipi_handler, "IPI", &irq_stat);
WARN(err, "Could not request IRQ %d as IRQ, err=%d\n", irq, err);
}
for_each_possible_cpu(cpu)
get_ipi_desc(cpu, ipi) = irq_to_desc(irq);
irq_set_status_flags(irq, IRQ_HIDDEN);
}
static void ipi_setup_lpi(int ipi, int ncpus)
{
for (int cpu = 0; cpu < ncpus; cpu++) {
int err, irq;
irq = ipi_irq_base + (cpu * nr_ipi) + ipi;
err = irq_force_affinity(irq, cpumask_of(cpu));
WARN(err, "Could not force affinity IRQ %d, err=%d\n", irq, err);
err = request_irq(irq, ipi_handler, IRQF_NO_AUTOEN, "IPI",
NULL);
WARN(err, "Could not request IRQ %d, err=%d\n", irq, err);
irq_set_status_flags(irq, (IRQ_HIDDEN | IRQ_NO_BALANCING_MASK));
get_ipi_desc(cpu, ipi) = irq_to_desc(irq);
}
}
void __init set_smp_ipi_range_percpu(int ipi_base, int n, int ncpus)
{
int i;
WARN_ON(n < MAX_IPI);
nr_ipi = min(n, MAX_IPI);
for (i = 0; i < nr_ipi; i++) {
int err;
if (ipi_should_be_nmi(i)) {
err = request_percpu_nmi(ipi_base + i, ipi_handler,
"IPI", &irq_stat);
WARN(err, "Could not request IPI %d as NMI, err=%d\n",
i, err);
} else {
err = request_percpu_irq(ipi_base + i, ipi_handler,
"IPI", &irq_stat);
WARN(err, "Could not request IPI %d as IRQ, err=%d\n",
i, err);
}
ipi_desc[i] = irq_to_desc(ipi_base + i);
irq_set_status_flags(ipi_base + i, IRQ_HIDDEN);
}
percpu_ipi_descs = !!ncpus;
ipi_irq_base = ipi_base;
for (i = 0; i < nr_ipi; i++) {
if (!percpu_ipi_descs)
ipi_setup_sgi(i);
else
ipi_setup_lpi(i, ncpus);
}
/* Setup the boot CPU immediately */
ipi_setup(smp_processor_id());
}
arch/arm64/kvm/Makefile
+2 -1
View File
@@ -23,7 +23,8 @@ kvm-y += arm.o mmu.o mmio.o psci.o hypercalls.o pvtime.o \
vgic/vgic-v3.o vgic/vgic-v4.o \
vgic/vgic-mmio.o vgic/vgic-mmio-v2.o \
vgic/vgic-mmio-v3.o vgic/vgic-kvm-device.o \
vgic/vgic-its.o vgic/vgic-debug.o vgic/vgic-v3-nested.o
vgic/vgic-its.o vgic/vgic-debug.o vgic/vgic-v3-nested.o \
vgic/vgic-v5.o
kvm-$(CONFIG_HW_PERF_EVENTS) += pmu-emul.o pmu.o
kvm-$(CONFIG_ARM64_PTR_AUTH) += pauth.o
arch/arm64/kvm/arch_timer.c
+1 -1
View File
@@ -830,7 +830,7 @@ static void timer_set_traps(struct kvm_vcpu *vcpu, struct timer_map *map)
* by the guest (either FEAT_VHE or FEAT_E2H0 is implemented, but
* not both). This simplifies the handling of the EL1NV* bits.
*/
if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu)) {
if (is_nested_ctxt(vcpu)) {
u64 val = __vcpu_sys_reg(vcpu, CNTHCTL_EL2);
/* Use the VHE format for mental sanity */
arch/arm64/kvm/arm.c
+22 -14
View File
@@ -408,6 +408,13 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
case KVM_CAP_ARM_SUPPORTED_REG_MASK_RANGES:
r = BIT(0);
break;
case KVM_CAP_ARM_CACHEABLE_PFNMAP_SUPPORTED:
if (!kvm)
r = -EINVAL;
else
r = kvm_supports_cacheable_pfnmap();
break;
default:
r = 0;
}
@@ -521,7 +528,7 @@ static void vcpu_set_pauth_traps(struct kvm_vcpu *vcpu)
* Either we're running an L2 guest, and the API/APK bits come
* from L1's HCR_EL2, or API/APK are both set.
*/
if (unlikely(vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu))) {
if (unlikely(is_nested_ctxt(vcpu))) {
u64 val;
val = __vcpu_sys_reg(vcpu, HCR_EL2);
@@ -740,7 +747,8 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
*/
int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
{
bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF);
bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF | HCR_VSE);
return ((irq_lines || kvm_vgic_vcpu_pending_irq(v))
&& !kvm_arm_vcpu_stopped(v) && !v->arch.pause);
}
@@ -1183,6 +1191,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
*/
preempt_disable();
kvm_nested_flush_hwstate(vcpu);
if (kvm_vcpu_has_pmu(vcpu))
kvm_pmu_flush_hwstate(vcpu);
@@ -1282,6 +1292,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
/* Exit types that need handling before we can be preempted */
handle_exit_early(vcpu, ret);
kvm_nested_sync_hwstate(vcpu);
preempt_enable();
/*
@@ -2765,19 +2777,15 @@ void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
kvm_vgic_v4_unset_forwarding(irqfd->kvm, prod->irq);
}
bool kvm_arch_irqfd_route_changed(struct kvm_kernel_irq_routing_entry *old,
struct kvm_kernel_irq_routing_entry *new)
void kvm_arch_update_irqfd_routing(struct kvm_kernel_irqfd *irqfd,
struct kvm_kernel_irq_routing_entry *old,
struct kvm_kernel_irq_routing_entry *new)
{
if (old->type != KVM_IRQ_ROUTING_MSI ||
new->type != KVM_IRQ_ROUTING_MSI)
return true;
if (old->type == KVM_IRQ_ROUTING_MSI &&
new->type == KVM_IRQ_ROUTING_MSI &&
!memcmp(&old->msi, &new->msi, sizeof(new->msi)))
return;
return memcmp(&old->msi, &new->msi, sizeof(new->msi));
}
int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq,
uint32_t guest_irq, bool set)
{
/*
* Remapping the vLPI requires taking the its_lock mutex to resolve
* the new translation. We're in spinlock land at this point, so no
@@ -2785,7 +2793,7 @@ int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq,
*
* Unmap the vLPI and fall back to software LPI injection.
*/
return kvm_vgic_v4_unset_forwarding(kvm, host_irq);
return kvm_vgic_v4_unset_forwarding(irqfd->kvm, irqfd->producer->irq);
}
void kvm_arch_irq_bypass_stop(struct irq_bypass_consumer *cons)
arch/arm64/kvm/at.c
+44 -36
View File
@@ -1047,34 +1047,51 @@ static void compute_s1_overlay_permissions(struct kvm_vcpu *vcpu,
idx = FIELD_GET(PTE_PO_IDX_MASK, wr->desc);
switch (wi->regime) {
case TR_EL10:
pov_perms = perm_idx(vcpu, POR_EL1, idx);
uov_perms = perm_idx(vcpu, POR_EL0, idx);
break;
case TR_EL20:
pov_perms = perm_idx(vcpu, POR_EL2, idx);
uov_perms = perm_idx(vcpu, POR_EL0, idx);
break;
case TR_EL2:
pov_perms = perm_idx(vcpu, POR_EL2, idx);
uov_perms = 0;
break;
}
if (wr->pov) {
switch (wi->regime) {
case TR_EL10:
pov_perms = perm_idx(vcpu, POR_EL1, idx);
break;
case TR_EL20:
pov_perms = perm_idx(vcpu, POR_EL2, idx);
break;
case TR_EL2:
pov_perms = perm_idx(vcpu, POR_EL2, idx);
break;
}
if (pov_perms & ~POE_RWX)
pov_perms = POE_NONE;
if (pov_perms & ~POE_RWX)
pov_perms = POE_NONE;
/* R_QXXPC, S1PrivOverflow enabled */
if (wr->pwxn && (pov_perms & POE_X))
pov_perms &= ~POE_W;
if (wi->poe && wr->pov) {
wr->pr &= pov_perms & POE_R;
wr->pw &= pov_perms & POE_W;
wr->px &= pov_perms & POE_X;
}
if (uov_perms & ~POE_RWX)
uov_perms = POE_NONE;
if (wr->uov) {
switch (wi->regime) {
case TR_EL10:
uov_perms = perm_idx(vcpu, POR_EL0, idx);
break;
case TR_EL20:
uov_perms = perm_idx(vcpu, POR_EL0, idx);
break;
case TR_EL2:
uov_perms = 0;
break;
}
if (uov_perms & ~POE_RWX)
uov_perms = POE_NONE;
/* R_NPBXC, S1UnprivOverlay enabled */
if (wr->uwxn && (uov_perms & POE_X))
uov_perms &= ~POE_W;
if (wi->e0poe && wr->uov) {
wr->ur &= uov_perms & POE_R;
wr->uw &= uov_perms & POE_W;
wr->ux &= uov_perms & POE_X;
@@ -1095,24 +1112,15 @@ static void compute_s1_permissions(struct kvm_vcpu *vcpu,
if (!wi->hpd)
compute_s1_hierarchical_permissions(vcpu, wi, wr);
if (wi->poe || wi->e0poe)
compute_s1_overlay_permissions(vcpu, wi, wr);
compute_s1_overlay_permissions(vcpu, wi, wr);
/* R_QXXPC */
if (wr->pwxn) {
if (!wr->pov && wr->pw)
wr->px = false;
if (wr->pov && wr->px)
wr->pw = false;
}
/* R_QXXPC, S1PrivOverlay disabled */
if (!wr->pov)
wr->px &= !(wr->pwxn && wr->pw);
/* R_NPBXC */
if (wr->uwxn) {
if (!wr->uov && wr->uw)
wr->ux = false;
if (wr->uov && wr->ux)
wr->uw = false;
}
/* R_NPBXC, S1UnprivOverlay disabled */
if (!wr->uov)
wr->ux &= !(wr->uwxn && wr->uw);
pan = wi->pan && (wr->ur || wr->uw ||
(pan3_enabled(vcpu, wi->regime) && wr->ux));
arch/arm64/kvm/config.c
+252 -3
View File
@@ -66,7 +66,6 @@ struct reg_bits_to_feat_map {
#define FEAT_BRBE ID_AA64DFR0_EL1, BRBE, IMP
#define FEAT_TRC_SR ID_AA64DFR0_EL1, TraceVer, IMP
#define FEAT_PMUv3 ID_AA64DFR0_EL1, PMUVer, IMP
#define FEAT_PMUv3p9 ID_AA64DFR0_EL1, PMUVer, V3P9
#define FEAT_TRBE ID_AA64DFR0_EL1, TraceBuffer, IMP
#define FEAT_TRBEv1p1 ID_AA64DFR0_EL1, TraceBuffer, TRBE_V1P1
#define FEAT_DoubleLock ID_AA64DFR0_EL1, DoubleLock, IMP
@@ -89,6 +88,7 @@ struct reg_bits_to_feat_map {
#define FEAT_RASv2 ID_AA64PFR0_EL1, RAS, V2
#define FEAT_GICv3 ID_AA64PFR0_EL1, GIC, IMP
#define FEAT_LOR ID_AA64MMFR1_EL1, LO, IMP
#define FEAT_SPEv1p2 ID_AA64DFR0_EL1, PMSVer, V1P2
#define FEAT_SPEv1p4 ID_AA64DFR0_EL1, PMSVer, V1P4
#define FEAT_SPEv1p5 ID_AA64DFR0_EL1, PMSVer, V1P5
#define FEAT_ATS1A ID_AA64ISAR2_EL1, ATS1A, IMP
@@ -131,6 +131,27 @@ struct reg_bits_to_feat_map {
#define FEAT_SPMU ID_AA64DFR1_EL1, SPMU, IMP
#define FEAT_SPE_nVM ID_AA64DFR2_EL1, SPE_nVM, IMP
#define FEAT_STEP2 ID_AA64DFR2_EL1, STEP, IMP
#define FEAT_SYSREG128 ID_AA64ISAR2_EL1, SYSREG_128, IMP
#define FEAT_CPA2 ID_AA64ISAR3_EL1, CPA, CPA2
#define FEAT_ASID2 ID_AA64MMFR4_EL1, ASID2, IMP
#define FEAT_MEC ID_AA64MMFR3_EL1, MEC, IMP
#define FEAT_HAFT ID_AA64MMFR1_EL1, HAFDBS, HAFT
#define FEAT_BTI ID_AA64PFR1_EL1, BT, IMP
#define FEAT_ExS ID_AA64MMFR0_EL1, EXS, IMP
#define FEAT_IESB ID_AA64MMFR2_EL1, IESB, IMP
#define FEAT_LSE2 ID_AA64MMFR2_EL1, AT, IMP
#define FEAT_LSMAOC ID_AA64MMFR2_EL1, LSM, IMP
#define FEAT_MixedEnd ID_AA64MMFR0_EL1, BIGEND, IMP
#define FEAT_MixedEndEL0 ID_AA64MMFR0_EL1, BIGENDEL0, IMP
#define FEAT_MTE2 ID_AA64PFR1_EL1, MTE, MTE2
#define FEAT_MTE_ASYNC ID_AA64PFR1_EL1, MTE_frac, ASYNC
#define FEAT_MTE_STORE_ONLY ID_AA64PFR2_EL1, MTESTOREONLY, IMP
#define FEAT_PAN ID_AA64MMFR1_EL1, PAN, IMP
#define FEAT_PAN3 ID_AA64MMFR1_EL1, PAN, PAN3
#define FEAT_SSBS ID_AA64PFR1_EL1, SSBS, IMP
#define FEAT_TIDCP1 ID_AA64MMFR1_EL1, TIDCP1, IMP
#define FEAT_FGT ID_AA64MMFR0_EL1, FGT, IMP
#define FEAT_MTPMU ID_AA64DFR0_EL1, MTPMU, IMP
static bool not_feat_aa64el3(struct kvm *kvm)
{
@@ -218,11 +239,62 @@ static bool feat_trbe_mpam(struct kvm *kvm)
(read_sysreg_s(SYS_TRBIDR_EL1) & TRBIDR_EL1_MPAM));
}
static bool feat_asid2_e2h1(struct kvm *kvm)
{
return kvm_has_feat(kvm, FEAT_ASID2) && !kvm_has_feat(kvm, FEAT_E2H0);
}
static bool feat_d128_e2h1(struct kvm *kvm)
{
return kvm_has_feat(kvm, FEAT_D128) && !kvm_has_feat(kvm, FEAT_E2H0);
}
static bool feat_mec_e2h1(struct kvm *kvm)
{
return kvm_has_feat(kvm, FEAT_MEC) && !kvm_has_feat(kvm, FEAT_E2H0);
}
static bool feat_ebep_pmuv3_ss(struct kvm *kvm)
{
return kvm_has_feat(kvm, FEAT_EBEP) || kvm_has_feat(kvm, FEAT_PMUv3_SS);
}
static bool feat_mixedendel0(struct kvm *kvm)
{
return kvm_has_feat(kvm, FEAT_MixedEnd) || kvm_has_feat(kvm, FEAT_MixedEndEL0);
}
static bool feat_mte_async(struct kvm *kvm)
{
return kvm_has_feat(kvm, FEAT_MTE2) && kvm_has_feat_enum(kvm, FEAT_MTE_ASYNC);
}
#define check_pmu_revision(k, r) \
({ \
(kvm_has_feat((k), ID_AA64DFR0_EL1, PMUVer, r) && \
!kvm_has_feat((k), ID_AA64DFR0_EL1, PMUVer, IMP_DEF)); \
})
static bool feat_pmuv3p1(struct kvm *kvm)
{
return check_pmu_revision(kvm, V3P1);
}
static bool feat_pmuv3p5(struct kvm *kvm)
{
return check_pmu_revision(kvm, V3P5);
}
static bool feat_pmuv3p7(struct kvm *kvm)
{
return check_pmu_revision(kvm, V3P7);
}
static bool feat_pmuv3p9(struct kvm *kvm)
{
return check_pmu_revision(kvm, V3P9);
}
static bool compute_hcr_rw(struct kvm *kvm, u64 *bits)
{
/* This is purely academic: AArch32 and NV are mutually exclusive */
@@ -681,7 +753,7 @@ static const struct reg_bits_to_feat_map hdfgrtr2_feat_map[] = {
NEEDS_FEAT(HDFGRTR2_EL2_nPMICFILTR_EL0 |
HDFGRTR2_EL2_nPMICNTR_EL0,
FEAT_PMUv3_ICNTR),
NEEDS_FEAT(HDFGRTR2_EL2_nPMUACR_EL1, FEAT_PMUv3p9),
NEEDS_FEAT(HDFGRTR2_EL2_nPMUACR_EL1, feat_pmuv3p9),
NEEDS_FEAT(HDFGRTR2_EL2_nPMSSCR_EL1 |
HDFGRTR2_EL2_nPMSSDATA,
FEAT_PMUv3_SS),
@@ -713,7 +785,7 @@ static const struct reg_bits_to_feat_map hdfgwtr2_feat_map[] = {
FEAT_PMUv3_ICNTR),
NEEDS_FEAT(HDFGWTR2_EL2_nPMUACR_EL1 |
HDFGWTR2_EL2_nPMZR_EL0,
FEAT_PMUv3p9),
feat_pmuv3p9),
NEEDS_FEAT(HDFGWTR2_EL2_nPMSSCR_EL1, FEAT_PMUv3_SS),
NEEDS_FEAT(HDFGWTR2_EL2_nPMIAR_EL1, FEAT_SEBEP),
NEEDS_FEAT(HDFGWTR2_EL2_nPMSDSFR_EL1, feat_spe_fds),
@@ -832,6 +904,150 @@ static const struct reg_bits_to_feat_map hcr_feat_map[] = {
NEEDS_FEAT_FIXED(HCR_EL2_E2H, compute_hcr_e2h),
};
static const struct reg_bits_to_feat_map sctlr2_feat_map[] = {
NEEDS_FEAT(SCTLR2_EL1_NMEA |
SCTLR2_EL1_EASE,
FEAT_DoubleFault2),
NEEDS_FEAT(SCTLR2_EL1_EnADERR, feat_aderr),
NEEDS_FEAT(SCTLR2_EL1_EnANERR, feat_anerr),
NEEDS_FEAT(SCTLR2_EL1_EnIDCP128, FEAT_SYSREG128),
NEEDS_FEAT(SCTLR2_EL1_EnPACM |
SCTLR2_EL1_EnPACM0,
feat_pauth_lr),
NEEDS_FEAT(SCTLR2_EL1_CPTA |
SCTLR2_EL1_CPTA0 |
SCTLR2_EL1_CPTM |
SCTLR2_EL1_CPTM0,
FEAT_CPA2),
};
static const struct reg_bits_to_feat_map tcr2_el2_feat_map[] = {
NEEDS_FEAT(TCR2_EL2_FNG1 |
TCR2_EL2_FNG0 |
TCR2_EL2_A2,
feat_asid2_e2h1),
NEEDS_FEAT(TCR2_EL2_DisCH1 |
TCR2_EL2_DisCH0 |
TCR2_EL2_D128,
feat_d128_e2h1),
NEEDS_FEAT(TCR2_EL2_AMEC1, feat_mec_e2h1),
NEEDS_FEAT(TCR2_EL2_AMEC0, FEAT_MEC),
NEEDS_FEAT(TCR2_EL2_HAFT, FEAT_HAFT),
NEEDS_FEAT(TCR2_EL2_PTTWI |
TCR2_EL2_PnCH,
FEAT_THE),
NEEDS_FEAT(TCR2_EL2_AIE, FEAT_AIE),
NEEDS_FEAT(TCR2_EL2_POE |
TCR2_EL2_E0POE,
FEAT_S1POE),
NEEDS_FEAT(TCR2_EL2_PIE, FEAT_S1PIE),
};
static const struct reg_bits_to_feat_map sctlr_el1_feat_map[] = {
NEEDS_FEAT(SCTLR_EL1_CP15BEN |
SCTLR_EL1_ITD |
SCTLR_EL1_SED,
FEAT_AA32EL0),
NEEDS_FEAT(SCTLR_EL1_BT0 |
SCTLR_EL1_BT1,
FEAT_BTI),
NEEDS_FEAT(SCTLR_EL1_CMOW, FEAT_CMOW),
NEEDS_FEAT(SCTLR_EL1_TSCXT, feat_csv2_2_csv2_1p2),
NEEDS_FEAT(SCTLR_EL1_EIS |
SCTLR_EL1_EOS,
FEAT_ExS),
NEEDS_FEAT(SCTLR_EL1_EnFPM, FEAT_FPMR),
NEEDS_FEAT(SCTLR_EL1_IESB, FEAT_IESB),
NEEDS_FEAT(SCTLR_EL1_EnALS, FEAT_LS64),
NEEDS_FEAT(SCTLR_EL1_EnAS0, FEAT_LS64_ACCDATA),
NEEDS_FEAT(SCTLR_EL1_EnASR, FEAT_LS64_V),
NEEDS_FEAT(SCTLR_EL1_nAA, FEAT_LSE2),
NEEDS_FEAT(SCTLR_EL1_LSMAOE |
SCTLR_EL1_nTLSMD,
FEAT_LSMAOC),
NEEDS_FEAT(SCTLR_EL1_EE, FEAT_MixedEnd),
NEEDS_FEAT(SCTLR_EL1_E0E, feat_mixedendel0),
NEEDS_FEAT(SCTLR_EL1_MSCEn, FEAT_MOPS),
NEEDS_FEAT(SCTLR_EL1_ATA0 |
SCTLR_EL1_ATA |
SCTLR_EL1_TCF0 |
SCTLR_EL1_TCF,
FEAT_MTE2),
NEEDS_FEAT(SCTLR_EL1_ITFSB, feat_mte_async),
NEEDS_FEAT(SCTLR_EL1_TCSO0 |
SCTLR_EL1_TCSO,
FEAT_MTE_STORE_ONLY),
NEEDS_FEAT(SCTLR_EL1_NMI |
SCTLR_EL1_SPINTMASK,
FEAT_NMI),
NEEDS_FEAT(SCTLR_EL1_SPAN, FEAT_PAN),
NEEDS_FEAT(SCTLR_EL1_EPAN, FEAT_PAN3),
NEEDS_FEAT(SCTLR_EL1_EnDA |
SCTLR_EL1_EnDB |
SCTLR_EL1_EnIA |
SCTLR_EL1_EnIB,
feat_pauth),
NEEDS_FEAT(SCTLR_EL1_EnTP2, FEAT_SME),
NEEDS_FEAT(SCTLR_EL1_EnRCTX, FEAT_SPECRES),
NEEDS_FEAT(SCTLR_EL1_DSSBS, FEAT_SSBS),
NEEDS_FEAT(SCTLR_EL1_TIDCP, FEAT_TIDCP1),
NEEDS_FEAT(SCTLR_EL1_TME0 |
SCTLR_EL1_TME |
SCTLR_EL1_TMT0 |
SCTLR_EL1_TMT,
FEAT_TME),
NEEDS_FEAT(SCTLR_EL1_TWEDEL |
SCTLR_EL1_TWEDEn,
FEAT_TWED),
NEEDS_FEAT(SCTLR_EL1_UCI |
SCTLR_EL1_EE |
SCTLR_EL1_E0E |
SCTLR_EL1_WXN |
SCTLR_EL1_nTWE |
SCTLR_EL1_nTWI |
SCTLR_EL1_UCT |
SCTLR_EL1_DZE |
SCTLR_EL1_I |
SCTLR_EL1_UMA |
SCTLR_EL1_SA0 |
SCTLR_EL1_SA |
SCTLR_EL1_C |
SCTLR_EL1_A |
SCTLR_EL1_M,
FEAT_AA64EL1),
};
static const struct reg_bits_to_feat_map mdcr_el2_feat_map[] = {
NEEDS_FEAT(MDCR_EL2_EBWE, FEAT_Debugv8p9),
NEEDS_FEAT(MDCR_EL2_TDOSA, FEAT_DoubleLock),
NEEDS_FEAT(MDCR_EL2_PMEE, FEAT_EBEP),
NEEDS_FEAT(MDCR_EL2_TDCC, FEAT_FGT),
NEEDS_FEAT(MDCR_EL2_MTPME, FEAT_MTPMU),
NEEDS_FEAT(MDCR_EL2_HPME |
MDCR_EL2_HPMN |
MDCR_EL2_TPMCR |
MDCR_EL2_TPM,
FEAT_PMUv3),
NEEDS_FEAT(MDCR_EL2_HPMD, feat_pmuv3p1),
NEEDS_FEAT(MDCR_EL2_HCCD |
MDCR_EL2_HLP,
feat_pmuv3p5),
NEEDS_FEAT(MDCR_EL2_HPMFZO, feat_pmuv3p7),
NEEDS_FEAT(MDCR_EL2_PMSSE, FEAT_PMUv3_SS),
NEEDS_FEAT(MDCR_EL2_E2PB |
MDCR_EL2_TPMS,
FEAT_SPE),
NEEDS_FEAT(MDCR_EL2_HPMFZS, FEAT_SPEv1p2),
NEEDS_FEAT(MDCR_EL2_EnSPM, FEAT_SPMU),
NEEDS_FEAT(MDCR_EL2_EnSTEPOP, FEAT_STEP2),
NEEDS_FEAT(MDCR_EL2_E2TB, FEAT_TRBE),
NEEDS_FEAT(MDCR_EL2_TTRF, FEAT_TRF),
NEEDS_FEAT(MDCR_EL2_TDA |
MDCR_EL2_TDE |
MDCR_EL2_TDRA,
FEAT_AA64EL1),
};
static void __init check_feat_map(const struct reg_bits_to_feat_map *map,
int map_size, u64 res0, const char *str)
{
@@ -863,6 +1079,14 @@ void __init check_feature_map(void)
__HCRX_EL2_RES0, "HCRX_EL2");
check_feat_map(hcr_feat_map, ARRAY_SIZE(hcr_feat_map),
HCR_EL2_RES0, "HCR_EL2");
check_feat_map(sctlr2_feat_map, ARRAY_SIZE(sctlr2_feat_map),
SCTLR2_EL1_RES0, "SCTLR2_EL1");
check_feat_map(tcr2_el2_feat_map, ARRAY_SIZE(tcr2_el2_feat_map),
TCR2_EL2_RES0, "TCR2_EL2");
check_feat_map(sctlr_el1_feat_map, ARRAY_SIZE(sctlr_el1_feat_map),
SCTLR_EL1_RES0, "SCTLR_EL1");
check_feat_map(mdcr_el2_feat_map, ARRAY_SIZE(mdcr_el2_feat_map),
MDCR_EL2_RES0, "MDCR_EL2");
}
static bool idreg_feat_match(struct kvm *kvm, const struct reg_bits_to_feat_map *map)
@@ -1077,6 +1301,31 @@ void get_reg_fixed_bits(struct kvm *kvm, enum vcpu_sysreg reg, u64 *res0, u64 *r
*res0 |= HCR_EL2_RES0 | (mask & ~fixed);
*res1 = HCR_EL2_RES1 | (mask & fixed);
break;
case SCTLR2_EL1:
case SCTLR2_EL2:
*res0 = compute_res0_bits(kvm, sctlr2_feat_map,
ARRAY_SIZE(sctlr2_feat_map), 0, 0);
*res0 |= SCTLR2_EL1_RES0;
*res1 = SCTLR2_EL1_RES1;
break;
case TCR2_EL2:
*res0 = compute_res0_bits(kvm, tcr2_el2_feat_map,
ARRAY_SIZE(tcr2_el2_feat_map), 0, 0);
*res0 |= TCR2_EL2_RES0;
*res1 = TCR2_EL2_RES1;
break;
case SCTLR_EL1:
*res0 = compute_res0_bits(kvm, sctlr_el1_feat_map,
ARRAY_SIZE(sctlr_el1_feat_map), 0, 0);
*res0 |= SCTLR_EL1_RES0;
*res1 = SCTLR_EL1_RES1;
break;
case MDCR_EL2:
*res0 = compute_res0_bits(kvm, mdcr_el2_feat_map,
ARRAY_SIZE(mdcr_el2_feat_map), 0, 0);
*res0 |= MDCR_EL2_RES0;
*res1 = MDCR_EL2_RES1;
break;
default:
WARN_ON_ONCE(1);
*res0 = *res1 = 0;
arch/arm64/kvm/emulate-nested.c
+42 -7
View File
@@ -88,6 +88,7 @@ enum cgt_group_id {
CGT_HCRX_EnFPM,
CGT_HCRX_TCR2En,
CGT_HCRX_SCTLR2En,
CGT_CNTHCTL_EL1TVT,
CGT_CNTHCTL_EL1TVCT,
@@ -108,6 +109,7 @@ enum cgt_group_id {
CGT_HCR_TTLB_TTLBOS,
CGT_HCR_TVM_TRVM,
CGT_HCR_TVM_TRVM_HCRX_TCR2En,
CGT_HCR_TVM_TRVM_HCRX_SCTLR2En,
CGT_HCR_TPU_TICAB,
CGT_HCR_TPU_TOCU,
CGT_HCR_NV1_nNV2_ENSCXT,
@@ -398,6 +400,12 @@ static const struct trap_bits coarse_trap_bits[] = {
.mask = HCRX_EL2_TCR2En,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_HCRX_SCTLR2En] = {
.index = HCRX_EL2,
.value = 0,
.mask = HCRX_EL2_SCTLR2En,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_CNTHCTL_EL1TVT] = {
.index = CNTHCTL_EL2,
.value = CNTHCTL_EL1TVT,
@@ -449,6 +457,8 @@ static const enum cgt_group_id *coarse_control_combo[] = {
MCB(CGT_HCR_TVM_TRVM, CGT_HCR_TVM, CGT_HCR_TRVM),
MCB(CGT_HCR_TVM_TRVM_HCRX_TCR2En,
CGT_HCR_TVM, CGT_HCR_TRVM, CGT_HCRX_TCR2En),
MCB(CGT_HCR_TVM_TRVM_HCRX_SCTLR2En,
CGT_HCR_TVM, CGT_HCR_TRVM, CGT_HCRX_SCTLR2En),
MCB(CGT_HCR_TPU_TICAB, CGT_HCR_TPU, CGT_HCR_TICAB),
MCB(CGT_HCR_TPU_TOCU, CGT_HCR_TPU, CGT_HCR_TOCU),
MCB(CGT_HCR_NV1_nNV2_ENSCXT, CGT_HCR_NV1_nNV2, CGT_HCR_ENSCXT),
@@ -782,6 +792,7 @@ static const struct encoding_to_trap_config encoding_to_cgt[] __initconst = {
SR_TRAP(OP_TLBI_RVALE1OSNXS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_RVAALE1OSNXS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(SYS_SCTLR_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_SCTLR2_EL1, CGT_HCR_TVM_TRVM_HCRX_SCTLR2En),
SR_TRAP(SYS_TTBR0_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_TTBR1_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_TCR_EL1, CGT_HCR_TVM_TRVM),
@@ -1354,6 +1365,7 @@ static const struct encoding_to_trap_config encoding_to_fgt[] __initconst = {
SR_FGT(SYS_SCXTNUM_EL0, HFGRTR, SCXTNUM_EL0, 1),
SR_FGT(SYS_SCXTNUM_EL1, HFGRTR, SCXTNUM_EL1, 1),
SR_FGT(SYS_SCTLR_EL1, HFGRTR, SCTLR_EL1, 1),
SR_FGT(SYS_SCTLR2_EL1, HFGRTR, SCTLR_EL1, 1),
SR_FGT(SYS_REVIDR_EL1, HFGRTR, REVIDR_EL1, 1),
SR_FGT(SYS_PAR_EL1, HFGRTR, PAR_EL1, 1),
SR_FGT(SYS_MPIDR_EL1, HFGRTR, MPIDR_EL1, 1),
@@ -2592,13 +2604,8 @@ inject:
static bool __forward_traps(struct kvm_vcpu *vcpu, unsigned int reg, u64 control_bit)
{
bool control_bit_set;
if (!vcpu_has_nv(vcpu))
return false;
control_bit_set = __vcpu_sys_reg(vcpu, reg) & control_bit;
if (!is_hyp_ctxt(vcpu) && control_bit_set) {
if (is_nested_ctxt(vcpu) &&
(__vcpu_sys_reg(vcpu, reg) & control_bit)) {
kvm_inject_nested_sync(vcpu, kvm_vcpu_get_esr(vcpu));
return true;
}
@@ -2719,6 +2726,9 @@ static void kvm_inject_el2_exception(struct kvm_vcpu *vcpu, u64 esr_el2,
case except_type_irq:
kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_IRQ);
break;
case except_type_serror:
kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SERR);
break;
default:
WARN_ONCE(1, "Unsupported EL2 exception injection %d\n", type);
}
@@ -2816,3 +2826,28 @@ int kvm_inject_nested_irq(struct kvm_vcpu *vcpu)
/* esr_el2 value doesn't matter for exits due to irqs. */
return kvm_inject_nested(vcpu, 0, except_type_irq);
}
int kvm_inject_nested_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr)
{
u64 esr = FIELD_PREP(ESR_ELx_EC_MASK,
iabt ? ESR_ELx_EC_IABT_LOW : ESR_ELx_EC_DABT_LOW);
esr |= ESR_ELx_FSC_EXTABT | ESR_ELx_IL;
vcpu_write_sys_reg(vcpu, FAR_EL2, addr);
if (__vcpu_sys_reg(vcpu, SCTLR2_EL2) & SCTLR2_EL1_EASE)
return kvm_inject_nested(vcpu, esr, except_type_serror);
return kvm_inject_nested_sync(vcpu, esr);
}
int kvm_inject_nested_serror(struct kvm_vcpu *vcpu, u64 esr)
{
/*
* Hardware sets up the EC field when propagating ESR as a result of
* vSError injection. Manually populate EC for an emulated SError
* exception.
*/
esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR);
return kvm_inject_nested(vcpu, esr, except_type_serror);
}
arch/arm64/kvm/guest.c
+48 -14
View File
@@ -818,8 +818,9 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu,
struct kvm_vcpu_events *events)
{
events->exception.serror_pending = !!(vcpu->arch.hcr_el2 & HCR_VSE);
events->exception.serror_has_esr = cpus_have_final_cap(ARM64_HAS_RAS_EXTN);
events->exception.serror_pending = (vcpu->arch.hcr_el2 & HCR_VSE) ||
vcpu_get_flag(vcpu, NESTED_SERROR_PENDING);
if (events->exception.serror_pending && events->exception.serror_has_esr)
events->exception.serror_esr = vcpu_get_vsesr(vcpu);
@@ -833,29 +834,62 @@ int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu,
return 0;
}
static void commit_pending_events(struct kvm_vcpu *vcpu)
{
if (!vcpu_get_flag(vcpu, PENDING_EXCEPTION))
return;
/*
* Reset the MMIO emulation state to avoid stepping PC after emulating
* the exception entry.
*/
vcpu->mmio_needed = false;
kvm_call_hyp(__kvm_adjust_pc, vcpu);
}
int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
struct kvm_vcpu_events *events)
{
bool serror_pending = events->exception.serror_pending;
bool has_esr = events->exception.serror_has_esr;
bool ext_dabt_pending = events->exception.ext_dabt_pending;
u64 esr = events->exception.serror_esr;
int ret = 0;
if (serror_pending && has_esr) {
if (!cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
return -EINVAL;
if (!((events->exception.serror_esr) & ~ESR_ELx_ISS_MASK))
kvm_set_sei_esr(vcpu, events->exception.serror_esr);
else
return -EINVAL;
} else if (serror_pending) {
kvm_inject_vabt(vcpu);
/*
* Immediately commit the pending SEA to the vCPU's architectural
* state which is necessary since we do not return a pending SEA
* to userspace via KVM_GET_VCPU_EVENTS.
*/
if (ext_dabt_pending) {
ret = kvm_inject_sea_dabt(vcpu, kvm_vcpu_get_hfar(vcpu));
commit_pending_events(vcpu);
}
if (ext_dabt_pending)
kvm_inject_dabt(vcpu, kvm_vcpu_get_hfar(vcpu));
if (ret < 0)
return ret;
return 0;
if (!serror_pending)
return 0;
if (!cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && has_esr)
return -EINVAL;
if (has_esr && (esr & ~ESR_ELx_ISS_MASK))
return -EINVAL;
if (has_esr)
ret = kvm_inject_serror_esr(vcpu, esr);
else
ret = kvm_inject_serror(vcpu);
/*
* We could've decided that the SError is due for immediate software
* injection; commit the exception in case userspace decides it wants
* to inject more exceptions for some strange reason.
*/
commit_pending_events(vcpu);
return (ret < 0) ? ret : 0;
}
u32 __attribute_const__ kvm_target_cpu(void)
arch/arm64/kvm/handle_exit.c
+9 -15
View File
@@ -32,7 +32,7 @@ typedef int (*exit_handle_fn)(struct kvm_vcpu *);
static void kvm_handle_guest_serror(struct kvm_vcpu *vcpu, u64 esr)
{
if (!arm64_is_ras_serror(esr) || arm64_is_fatal_ras_serror(NULL, esr))
kvm_inject_vabt(vcpu);
kvm_inject_serror(vcpu);
}
static int handle_hvc(struct kvm_vcpu *vcpu)
@@ -252,7 +252,7 @@ static int kvm_handle_ptrauth(struct kvm_vcpu *vcpu)
return 1;
}
if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu)) {
if (is_nested_ctxt(vcpu)) {
kvm_inject_nested_sync(vcpu, kvm_vcpu_get_esr(vcpu));
return 1;
}
@@ -311,12 +311,11 @@ static int kvm_handle_gcs(struct kvm_vcpu *vcpu)
static int handle_other(struct kvm_vcpu *vcpu)
{
bool is_l2 = vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu);
bool allowed, fwd = is_nested_ctxt(vcpu);
u64 hcrx = __vcpu_sys_reg(vcpu, HCRX_EL2);
u64 esr = kvm_vcpu_get_esr(vcpu);
u64 iss = ESR_ELx_ISS(esr);
struct kvm *kvm = vcpu->kvm;
bool allowed, fwd = false;
/*
* We only trap for two reasons:
@@ -335,28 +334,23 @@ static int handle_other(struct kvm_vcpu *vcpu)
switch (iss) {
case ESR_ELx_ISS_OTHER_ST64BV:
allowed = kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_V);
if (is_l2)
fwd = !(hcrx & HCRX_EL2_EnASR);
fwd &= !(hcrx & HCRX_EL2_EnASR);
break;
case ESR_ELx_ISS_OTHER_ST64BV0:
allowed = kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_ACCDATA);
if (is_l2)
fwd = !(hcrx & HCRX_EL2_EnAS0);
fwd &= !(hcrx & HCRX_EL2_EnAS0);
break;
case ESR_ELx_ISS_OTHER_LDST64B:
allowed = kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64);
if (is_l2)
fwd = !(hcrx & HCRX_EL2_EnALS);
fwd &= !(hcrx & HCRX_EL2_EnALS);
break;
case ESR_ELx_ISS_OTHER_TSBCSYNC:
allowed = kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceBuffer, TRBE_V1P1);
if (is_l2)
fwd = (__vcpu_sys_reg(vcpu, HFGITR2_EL2) & HFGITR2_EL2_TSBCSYNC);
fwd &= (__vcpu_sys_reg(vcpu, HFGITR2_EL2) & HFGITR2_EL2_TSBCSYNC);
break;
case ESR_ELx_ISS_OTHER_PSBCSYNC:
allowed = kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, V1P5);
if (is_l2)
fwd = (__vcpu_sys_reg(vcpu, HFGITR_EL2) & HFGITR_EL2_PSBCSYNC);
fwd &= (__vcpu_sys_reg(vcpu, HFGITR_EL2) & HFGITR_EL2_PSBCSYNC);
break;
default:
/* Clearly, we're missing something. */
@@ -496,7 +490,7 @@ void handle_exit_early(struct kvm_vcpu *vcpu, int exception_index)
kvm_handle_guest_serror(vcpu, disr_to_esr(disr));
} else {
kvm_inject_vabt(vcpu);
kvm_inject_serror(vcpu);
}
return;
arch/arm64/kvm/hyp/exception.c
+13 -3
View File
@@ -26,7 +26,8 @@ static inline u64 __vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg)
if (unlikely(vcpu_has_nv(vcpu)))
return vcpu_read_sys_reg(vcpu, reg);
else if (__vcpu_read_sys_reg_from_cpu(reg, &val))
else if (vcpu_get_flag(vcpu, SYSREGS_ON_CPU) &&
__vcpu_read_sys_reg_from_cpu(reg, &val))
return val;
return __vcpu_sys_reg(vcpu, reg);
@@ -36,7 +37,8 @@ static inline void __vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg)
{
if (unlikely(vcpu_has_nv(vcpu)))
vcpu_write_sys_reg(vcpu, val, reg);
else if (!__vcpu_write_sys_reg_to_cpu(val, reg))
else if (!vcpu_get_flag(vcpu, SYSREGS_ON_CPU) ||
!__vcpu_write_sys_reg_to_cpu(val, reg))
__vcpu_assign_sys_reg(vcpu, reg, val);
}
@@ -339,6 +341,10 @@ static void kvm_inject_exception(struct kvm_vcpu *vcpu)
enter_exception64(vcpu, PSR_MODE_EL1h, except_type_sync);
break;
case unpack_vcpu_flag(EXCEPT_AA64_EL1_SERR):
enter_exception64(vcpu, PSR_MODE_EL1h, except_type_serror);
break;
case unpack_vcpu_flag(EXCEPT_AA64_EL2_SYNC):
enter_exception64(vcpu, PSR_MODE_EL2h, except_type_sync);
break;
@@ -347,9 +353,13 @@ static void kvm_inject_exception(struct kvm_vcpu *vcpu)
enter_exception64(vcpu, PSR_MODE_EL2h, except_type_irq);
break;
case unpack_vcpu_flag(EXCEPT_AA64_EL2_SERR):
enter_exception64(vcpu, PSR_MODE_EL2h, except_type_serror);
break;
default:
/*
* Only EL1_SYNC and EL2_{SYNC,IRQ} makes
* Only EL1_{SYNC,SERR} and EL2_{SYNC,IRQ,SERR} makes
* sense so far. Everything else gets silently
* ignored.
*/
arch/arm64/kvm/hyp/include/hyp/switch.h
+44 -9
View File
@@ -298,7 +298,7 @@ static inline void __deactivate_cptr_traps(struct kvm_vcpu *vcpu)
u64 val; \
\
ctxt_sys_reg(hctxt, reg) = read_sysreg_s(SYS_ ## reg); \
if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu)) \
if (is_nested_ctxt(vcpu)) \
compute_clr_set(vcpu, reg, c, s); \
\
compute_undef_clr_set(vcpu, kvm, reg, c, s); \
@@ -436,7 +436,7 @@ static inline void __activate_traps_common(struct kvm_vcpu *vcpu)
if (cpus_have_final_cap(ARM64_HAS_HCX)) {
u64 hcrx = vcpu->arch.hcrx_el2;
if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu)) {
if (is_nested_ctxt(vcpu)) {
u64 val = __vcpu_sys_reg(vcpu, HCRX_EL2);
hcrx |= val & __HCRX_EL2_MASK;
hcrx &= ~(~val & __HCRX_EL2_nMASK);
@@ -476,21 +476,56 @@ static inline void ___activate_traps(struct kvm_vcpu *vcpu, u64 hcr)
write_sysreg_hcr(hcr);
if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_VSE))
write_sysreg_s(vcpu->arch.vsesr_el2, SYS_VSESR_EL2);
if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_VSE)) {
u64 vsesr;
/*
* When HCR_EL2.AMO is set, physical SErrors are taken to EL2
* and vSError injection is enabled for EL1. Conveniently, for
* NV this means that it is never the case where a 'physical'
* SError (injected by KVM or userspace) and vSError are
* deliverable to the same context.
*
* As such, we can trivially select between the host or guest's
* VSESR_EL2. Except for the case that FEAT_RAS hasn't been
* exposed to the guest, where ESR propagation in hardware
* occurs unconditionally.
*
* Paper over the architectural wart and use an IMPLEMENTATION
* DEFINED ESR value in case FEAT_RAS is hidden from the guest.
*/
if (!vserror_state_is_nested(vcpu))
vsesr = vcpu->arch.vsesr_el2;
else if (kvm_has_ras(kern_hyp_va(vcpu->kvm)))
vsesr = __vcpu_sys_reg(vcpu, VSESR_EL2);
else
vsesr = ESR_ELx_ISV;
write_sysreg_s(vsesr, SYS_VSESR_EL2);
}
}
static inline void ___deactivate_traps(struct kvm_vcpu *vcpu)
{
u64 *hcr;
if (vserror_state_is_nested(vcpu))
hcr = __ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2);
else
hcr = &vcpu->arch.hcr_el2;
/*
* If we pended a virtual abort, preserve it until it gets
* cleared. See D1.14.3 (Virtual Interrupts) for details, but
* the crucial bit is "On taking a vSError interrupt,
* HCR_EL2.VSE is cleared to 0."
*
* Additionally, when in a nested context we need to propagate the
* updated state to the guest hypervisor's HCR_EL2.
*/
if (vcpu->arch.hcr_el2 & HCR_VSE) {
vcpu->arch.hcr_el2 &= ~HCR_VSE;
vcpu->arch.hcr_el2 |= read_sysreg(hcr_el2) & HCR_VSE;
if (*hcr & HCR_VSE) {
*hcr &= ~HCR_VSE;
*hcr |= read_sysreg(hcr_el2) & HCR_VSE;
}
}
@@ -531,7 +566,7 @@ static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu)
* nested guest, as the guest hypervisor could select a smaller VL. Slap
* that into hardware before wrapping up.
*/
if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu))
if (is_nested_ctxt(vcpu))
sve_cond_update_zcr_vq(__vcpu_sys_reg(vcpu, ZCR_EL2), SYS_ZCR_EL2);
write_sysreg_el1(__vcpu_sys_reg(vcpu, vcpu_sve_zcr_elx(vcpu)), SYS_ZCR);
@@ -557,7 +592,7 @@ static inline void fpsimd_lazy_switch_to_guest(struct kvm_vcpu *vcpu)
if (vcpu_has_sve(vcpu)) {
/* A guest hypervisor may restrict the effective max VL. */
if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu))
if (is_nested_ctxt(vcpu))
zcr_el2 = __vcpu_sys_reg(vcpu, ZCR_EL2);
else
zcr_el2 = vcpu_sve_max_vq(vcpu) - 1;
arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
+46 -3
View File
@@ -109,6 +109,28 @@ static inline bool ctxt_has_s1poe(struct kvm_cpu_context *ctxt)
return kvm_has_s1poe(kern_hyp_va(vcpu->kvm));
}
static inline bool ctxt_has_ras(struct kvm_cpu_context *ctxt)
{
struct kvm_vcpu *vcpu;
if (!cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
return false;
vcpu = ctxt_to_vcpu(ctxt);
return kvm_has_ras(kern_hyp_va(vcpu->kvm));
}
static inline bool ctxt_has_sctlr2(struct kvm_cpu_context *ctxt)
{
struct kvm_vcpu *vcpu;
if (!cpus_have_final_cap(ARM64_HAS_SCTLR2))
return false;
vcpu = ctxt_to_vcpu(ctxt);
return kvm_has_sctlr2(kern_hyp_va(vcpu->kvm));
}
static inline void __sysreg_save_el1_state(struct kvm_cpu_context *ctxt)
{
ctxt_sys_reg(ctxt, SCTLR_EL1) = read_sysreg_el1(SYS_SCTLR);
@@ -147,6 +169,9 @@ static inline void __sysreg_save_el1_state(struct kvm_cpu_context *ctxt)
ctxt_sys_reg(ctxt, SP_EL1) = read_sysreg(sp_el1);
ctxt_sys_reg(ctxt, ELR_EL1) = read_sysreg_el1(SYS_ELR);
ctxt_sys_reg(ctxt, SPSR_EL1) = read_sysreg_el1(SYS_SPSR);
if (ctxt_has_sctlr2(ctxt))
ctxt_sys_reg(ctxt, SCTLR2_EL1) = read_sysreg_el1(SYS_SCTLR2);
}
static inline void __sysreg_save_el2_return_state(struct kvm_cpu_context *ctxt)
@@ -159,8 +184,13 @@ static inline void __sysreg_save_el2_return_state(struct kvm_cpu_context *ctxt)
if (!has_vhe() && ctxt->__hyp_running_vcpu)
ctxt->regs.pstate = read_sysreg_el2(SYS_SPSR);
if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
if (!cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
return;
if (!vserror_state_is_nested(ctxt_to_vcpu(ctxt)))
ctxt_sys_reg(ctxt, DISR_EL1) = read_sysreg_s(SYS_VDISR_EL2);
else if (ctxt_has_ras(ctxt))
ctxt_sys_reg(ctxt, VDISR_EL2) = read_sysreg_s(SYS_VDISR_EL2);
}
static inline void __sysreg_restore_common_state(struct kvm_cpu_context *ctxt)
@@ -252,6 +282,9 @@ static inline void __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt,
write_sysreg(ctxt_sys_reg(ctxt, SP_EL1), sp_el1);
write_sysreg_el1(ctxt_sys_reg(ctxt, ELR_EL1), SYS_ELR);
write_sysreg_el1(ctxt_sys_reg(ctxt, SPSR_EL1), SYS_SPSR);
if (ctxt_has_sctlr2(ctxt))
write_sysreg_el1(ctxt_sys_reg(ctxt, SCTLR2_EL1), SYS_SCTLR2);
}
/* Read the VCPU state's PSTATE, but translate (v)EL2 to EL1. */
@@ -275,6 +308,7 @@ static inline void __sysreg_restore_el2_return_state(struct kvm_cpu_context *ctx
{
u64 pstate = to_hw_pstate(ctxt);
u64 mode = pstate & PSR_AA32_MODE_MASK;
u64 vdisr;
/*
* Safety check to ensure we're setting the CPU up to enter the guest
@@ -293,8 +327,17 @@ static inline void __sysreg_restore_el2_return_state(struct kvm_cpu_context *ctx
write_sysreg_el2(ctxt->regs.pc, SYS_ELR);
write_sysreg_el2(pstate, SYS_SPSR);
if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
write_sysreg_s(ctxt_sys_reg(ctxt, DISR_EL1), SYS_VDISR_EL2);
if (!cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
return;
if (!vserror_state_is_nested(ctxt_to_vcpu(ctxt)))
vdisr = ctxt_sys_reg(ctxt, DISR_EL1);
else if (ctxt_has_ras(ctxt))
vdisr = ctxt_sys_reg(ctxt, VDISR_EL2);
else
vdisr = 0;
write_sysreg_s(vdisr, SYS_VDISR_EL2);
}
static inline void __sysreg32_save_state(struct kvm_vcpu *vcpu)
arch/arm64/kvm/hyp/vgic-v3-sr.c
+44 -9
View File
@@ -296,12 +296,19 @@ void __vgic_v3_activate_traps(struct vgic_v3_cpu_if *cpu_if)
}
/*
* Prevent the guest from touching the ICC_SRE_EL1 system
* register. Note that this may not have any effect, as
* ICC_SRE_EL2.Enable being RAO/WI is a valid implementation.
* GICv5 BET0 FEAT_GCIE_LEGACY doesn't include ICC_SRE_EL2. This is due
* to be relaxed in a future spec release, at which point this in
* condition can be dropped.
*/
write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE,
ICC_SRE_EL2);
if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) {
/*
* Prevent the guest from touching the ICC_SRE_EL1 system
* register. Note that this may not have any effect, as
* ICC_SRE_EL2.Enable being RAO/WI is a valid implementation.
*/
write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE,
ICC_SRE_EL2);
}
/*
* If we need to trap system registers, we must write
@@ -322,8 +329,14 @@ void __vgic_v3_deactivate_traps(struct vgic_v3_cpu_if *cpu_if)
cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2);
}
val = read_gicreg(ICC_SRE_EL2);
write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2);
/*
* Can be dropped in the future when GICv5 spec is relaxed. See comment
* above.
*/
if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) {
val = read_gicreg(ICC_SRE_EL2);
write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2);
}
if (!cpu_if->vgic_sre) {
/* Make sure ENABLE is set at EL2 before setting SRE at EL1 */
@@ -423,9 +436,19 @@ void __vgic_v3_init_lrs(void)
*/
u64 __vgic_v3_get_gic_config(void)
{
u64 val, sre = read_gicreg(ICC_SRE_EL1);
u64 val, sre;
unsigned long flags = 0;
/*
* In compat mode, we cannot access ICC_SRE_EL1 at any EL
* other than EL1 itself; just return the
* ICH_VTR_EL2. ICC_IDR0_EL1 is only implemented on a GICv5
* system, so we first check if we have GICv5 support.
*/
if (cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF))
return read_gicreg(ICH_VTR_EL2);
sre = read_gicreg(ICC_SRE_EL1);
/*
* To check whether we have a MMIO-based (GICv2 compatible)
* CPU interface, we need to disable the system register
@@ -471,6 +494,16 @@ u64 __vgic_v3_get_gic_config(void)
return val;
}
static void __vgic_v3_compat_mode_enable(void)
{
if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF))
return;
sysreg_clear_set_s(SYS_ICH_VCTLR_EL2, 0, ICH_VCTLR_EL2_V3);
/* Wait for V3 to become enabled */
isb();
}
static u64 __vgic_v3_read_vmcr(void)
{
return read_gicreg(ICH_VMCR_EL2);
@@ -490,6 +523,8 @@ void __vgic_v3_save_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if)
void __vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if)
{
__vgic_v3_compat_mode_enable();
/*
* If dealing with a GICv2 emulation on GICv3, VMCR_EL2.VFIQen
* is dependent on ICC_SRE_EL1.SRE, and we have to perform the
@@ -1050,7 +1085,7 @@ static bool __vgic_v3_check_trap_forwarding(struct kvm_vcpu *vcpu,
{
u64 ich_hcr;
if (!vcpu_has_nv(vcpu) || is_hyp_ctxt(vcpu))
if (!is_nested_ctxt(vcpu))
return false;
ich_hcr = __vcpu_sys_reg(vcpu, ICH_HCR_EL2);
arch/arm64/kvm/hyp/vhe/switch.c
+12 -2
View File
@@ -48,8 +48,7 @@ DEFINE_PER_CPU(unsigned long, kvm_hyp_vector);
static u64 __compute_hcr(struct kvm_vcpu *vcpu)
{
u64 guest_hcr = __vcpu_sys_reg(vcpu, HCR_EL2);
u64 hcr = vcpu->arch.hcr_el2;
u64 guest_hcr, hcr = vcpu->arch.hcr_el2;
if (!vcpu_has_nv(vcpu))
return hcr;
@@ -68,10 +67,21 @@ static u64 __compute_hcr(struct kvm_vcpu *vcpu)
if (!vcpu_el2_e2h_is_set(vcpu))
hcr |= HCR_NV1;
/*
* Nothing in HCR_EL2 should impact running in hypervisor
* context, apart from bits we have defined as RESx (E2H,
* HCD and co), or that cannot be set directly (the EXCLUDE
* bits). Given that we OR the guest's view with the host's,
* we can use the 0 value as the starting point, and only
* use the config-driven RES1 bits.
*/
guest_hcr = kvm_vcpu_apply_reg_masks(vcpu, HCR_EL2, 0);
write_sysreg_s(vcpu->arch.ctxt.vncr_array, SYS_VNCR_EL2);
} else {
host_data_clear_flag(VCPU_IN_HYP_CONTEXT);
guest_hcr = __vcpu_sys_reg(vcpu, HCR_EL2);
if (guest_hcr & HCR_NV) {
u64 va = __fix_to_virt(vncr_fixmap(smp_processor_id()));
arch/arm64/kvm/hyp/vhe/sysreg-sr.c
+6
View File
@@ -77,6 +77,9 @@ static void __sysreg_save_vel2_state(struct kvm_vcpu *vcpu)
__vcpu_assign_sys_reg(vcpu, SP_EL2, read_sysreg(sp_el1));
__vcpu_assign_sys_reg(vcpu, ELR_EL2, read_sysreg_el1(SYS_ELR));
__vcpu_assign_sys_reg(vcpu, SPSR_EL2, read_sysreg_el1(SYS_SPSR));
if (ctxt_has_sctlr2(&vcpu->arch.ctxt))
__vcpu_assign_sys_reg(vcpu, SCTLR2_EL2, read_sysreg_el1(SYS_SCTLR2));
}
static void __sysreg_restore_vel2_state(struct kvm_vcpu *vcpu)
@@ -139,6 +142,9 @@ static void __sysreg_restore_vel2_state(struct kvm_vcpu *vcpu)
write_sysreg(__vcpu_sys_reg(vcpu, SP_EL2), sp_el1);
write_sysreg_el1(__vcpu_sys_reg(vcpu, ELR_EL2), SYS_ELR);
write_sysreg_el1(__vcpu_sys_reg(vcpu, SPSR_EL2), SYS_SPSR);
if (ctxt_has_sctlr2(&vcpu->arch.ctxt))
write_sysreg_el1(__vcpu_sys_reg(vcpu, SCTLR2_EL2), SYS_SCTLR2);
}
/*
arch/arm64/kvm/inject_fault.c
+160 -75
View File
@@ -15,13 +15,11 @@
#include <asm/kvm_nested.h>
#include <asm/esr.h>
static void pend_sync_exception(struct kvm_vcpu *vcpu)
static unsigned int exception_target_el(struct kvm_vcpu *vcpu)
{
/* If not nesting, EL1 is the only possible exception target */
if (likely(!vcpu_has_nv(vcpu))) {
kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC);
return;
}
if (likely(!vcpu_has_nv(vcpu)))
return PSR_MODE_EL1h;
/*
* With NV, we need to pick between EL1 and EL2. Note that we
@@ -32,26 +30,76 @@ static void pend_sync_exception(struct kvm_vcpu *vcpu)
switch(*vcpu_cpsr(vcpu) & PSR_MODE_MASK) {
case PSR_MODE_EL2h:
case PSR_MODE_EL2t:
kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SYNC);
break;
return PSR_MODE_EL2h;
case PSR_MODE_EL1h:
case PSR_MODE_EL1t:
kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC);
break;
return PSR_MODE_EL1h;
case PSR_MODE_EL0t:
if (vcpu_el2_tge_is_set(vcpu))
kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SYNC);
else
kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC);
break;
return vcpu_el2_tge_is_set(vcpu) ? PSR_MODE_EL2h : PSR_MODE_EL1h;
default:
BUG();
}
}
static bool match_target_el(struct kvm_vcpu *vcpu, unsigned long target)
static enum vcpu_sysreg exception_esr_elx(struct kvm_vcpu *vcpu)
{
return (vcpu_get_flag(vcpu, EXCEPT_MASK) == target);
if (exception_target_el(vcpu) == PSR_MODE_EL2h)
return ESR_EL2;
return ESR_EL1;
}
static enum vcpu_sysreg exception_far_elx(struct kvm_vcpu *vcpu)
{
if (exception_target_el(vcpu) == PSR_MODE_EL2h)
return FAR_EL2;
return FAR_EL1;
}
static void pend_sync_exception(struct kvm_vcpu *vcpu)
{
if (exception_target_el(vcpu) == PSR_MODE_EL1h)
kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC);
else
kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SYNC);
}
static void pend_serror_exception(struct kvm_vcpu *vcpu)
{
if (exception_target_el(vcpu) == PSR_MODE_EL1h)
kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SERR);
else
kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SERR);
}
static bool __effective_sctlr2_bit(struct kvm_vcpu *vcpu, unsigned int idx)
{
u64 sctlr2;
if (!kvm_has_sctlr2(vcpu->kvm))
return false;
if (is_nested_ctxt(vcpu) &&
!(__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_SCTLR2En))
return false;
if (exception_target_el(vcpu) == PSR_MODE_EL1h)
sctlr2 = vcpu_read_sys_reg(vcpu, SCTLR2_EL1);
else
sctlr2 = vcpu_read_sys_reg(vcpu, SCTLR2_EL2);
return sctlr2 & BIT(idx);
}
static bool effective_sctlr2_ease(struct kvm_vcpu *vcpu)
{
return __effective_sctlr2_bit(vcpu, SCTLR2_EL1_EASE_SHIFT);
}
static bool effective_sctlr2_nmea(struct kvm_vcpu *vcpu)
{
return __effective_sctlr2_bit(vcpu, SCTLR2_EL1_NMEA_SHIFT);
}
static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr)
@@ -60,7 +108,11 @@ static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr
bool is_aarch32 = vcpu_mode_is_32bit(vcpu);
u64 esr = 0;
pend_sync_exception(vcpu);
/* This delight is brought to you by FEAT_DoubleFault2. */
if (effective_sctlr2_ease(vcpu))
pend_serror_exception(vcpu);
else
pend_sync_exception(vcpu);
/*
* Build an {i,d}abort, depending on the level and the
@@ -83,13 +135,8 @@ static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr
esr |= ESR_ELx_FSC_EXTABT;
if (match_target_el(vcpu, unpack_vcpu_flag(EXCEPT_AA64_EL1_SYNC))) {
vcpu_write_sys_reg(vcpu, addr, FAR_EL1);
vcpu_write_sys_reg(vcpu, esr, ESR_EL1);
} else {
vcpu_write_sys_reg(vcpu, addr, FAR_EL2);
vcpu_write_sys_reg(vcpu, esr, ESR_EL2);
}
vcpu_write_sys_reg(vcpu, addr, exception_far_elx(vcpu));
vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));
}
static void inject_undef64(struct kvm_vcpu *vcpu)
@@ -105,10 +152,7 @@ static void inject_undef64(struct kvm_vcpu *vcpu)
if (kvm_vcpu_trap_il_is32bit(vcpu))
esr |= ESR_ELx_IL;
if (match_target_el(vcpu, unpack_vcpu_flag(EXCEPT_AA64_EL1_SYNC)))
vcpu_write_sys_reg(vcpu, esr, ESR_EL1);
else
vcpu_write_sys_reg(vcpu, esr, ESR_EL2);
vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));
}
#define DFSR_FSC_EXTABT_LPAE 0x10
@@ -155,36 +199,35 @@ static void inject_abt32(struct kvm_vcpu *vcpu, bool is_pabt, u32 addr)
vcpu_write_sys_reg(vcpu, far, FAR_EL1);
}
/**
* kvm_inject_dabt - inject a data abort into the guest
* @vcpu: The VCPU to receive the data abort
* @addr: The address to report in the DFAR
*
* It is assumed that this code is called from the VCPU thread and that the
* VCPU therefore is not currently executing guest code.
*/
void kvm_inject_dabt(struct kvm_vcpu *vcpu, unsigned long addr)
static void __kvm_inject_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr)
{
if (vcpu_el1_is_32bit(vcpu))
inject_abt32(vcpu, false, addr);
inject_abt32(vcpu, iabt, addr);
else
inject_abt64(vcpu, false, addr);
inject_abt64(vcpu, iabt, addr);
}
/**
* kvm_inject_pabt - inject a prefetch abort into the guest
* @vcpu: The VCPU to receive the prefetch abort
* @addr: The address to report in the DFAR
*
* It is assumed that this code is called from the VCPU thread and that the
* VCPU therefore is not currently executing guest code.
*/
void kvm_inject_pabt(struct kvm_vcpu *vcpu, unsigned long addr)
static bool kvm_sea_target_is_el2(struct kvm_vcpu *vcpu)
{
if (vcpu_el1_is_32bit(vcpu))
inject_abt32(vcpu, true, addr);
else
inject_abt64(vcpu, true, addr);
if (__vcpu_sys_reg(vcpu, HCR_EL2) & (HCR_TGE | HCR_TEA))
return true;
if (!vcpu_mode_priv(vcpu))
return false;
return (*vcpu_cpsr(vcpu) & PSR_A_BIT) &&
(__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TMEA);
}
int kvm_inject_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr)
{
lockdep_assert_held(&vcpu->mutex);
if (is_nested_ctxt(vcpu) && kvm_sea_target_is_el2(vcpu))
return kvm_inject_nested_sea(vcpu, iabt, addr);
__kvm_inject_sea(vcpu, iabt, addr);
return 1;
}
void kvm_inject_size_fault(struct kvm_vcpu *vcpu)
@@ -194,10 +237,7 @@ void kvm_inject_size_fault(struct kvm_vcpu *vcpu)
addr = kvm_vcpu_get_fault_ipa(vcpu);
addr |= kvm_vcpu_get_hfar(vcpu) & GENMASK(11, 0);
if (kvm_vcpu_trap_is_iabt(vcpu))
kvm_inject_pabt(vcpu, addr);
else
kvm_inject_dabt(vcpu, addr);
__kvm_inject_sea(vcpu, kvm_vcpu_trap_is_iabt(vcpu), addr);
/*
* If AArch64 or LPAE, set FSC to 0 to indicate an Address
@@ -210,9 +250,9 @@ void kvm_inject_size_fault(struct kvm_vcpu *vcpu)
!(vcpu_read_sys_reg(vcpu, TCR_EL1) & TTBCR_EAE))
return;
esr = vcpu_read_sys_reg(vcpu, ESR_EL1);
esr = vcpu_read_sys_reg(vcpu, exception_esr_elx(vcpu));
esr &= ~GENMASK_ULL(5, 0);
vcpu_write_sys_reg(vcpu, esr, ESR_EL1);
vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));
}
/**
@@ -230,25 +270,70 @@ void kvm_inject_undefined(struct kvm_vcpu *vcpu)
inject_undef64(vcpu);
}
void kvm_set_sei_esr(struct kvm_vcpu *vcpu, u64 esr)
static bool serror_is_masked(struct kvm_vcpu *vcpu)
{
vcpu_set_vsesr(vcpu, esr & ESR_ELx_ISS_MASK);
*vcpu_hcr(vcpu) |= HCR_VSE;
return (*vcpu_cpsr(vcpu) & PSR_A_BIT) && !effective_sctlr2_nmea(vcpu);
}
/**
* kvm_inject_vabt - inject an async abort / SError into the guest
* @vcpu: The VCPU to receive the exception
*
* It is assumed that this code is called from the VCPU thread and that the
* VCPU therefore is not currently executing guest code.
*
* Systems with the RAS Extensions specify an imp-def ESR (ISV/IDS = 1) with
* the remaining ISS all-zeros so that this error is not interpreted as an
* uncategorized RAS error. Without the RAS Extensions we can't specify an ESR
* value, so the CPU generates an imp-def value.
*/
void kvm_inject_vabt(struct kvm_vcpu *vcpu)
static bool kvm_serror_target_is_el2(struct kvm_vcpu *vcpu)
{
kvm_set_sei_esr(vcpu, ESR_ELx_ISV);
if (is_hyp_ctxt(vcpu) || vcpu_el2_amo_is_set(vcpu))
return true;
if (!(__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TMEA))
return false;
/*
* In another example where FEAT_DoubleFault2 is entirely backwards,
* "masked" as it relates to the routing effects of HCRX_EL2.TMEA
* doesn't consider SCTLR2_EL1.NMEA. That is to say, even if EL1 asked
* for non-maskable SErrors, the EL2 bit takes priority if A is set.
*/
if (vcpu_mode_priv(vcpu))
return *vcpu_cpsr(vcpu) & PSR_A_BIT;
/*
* Otherwise SErrors are considered unmasked when taken from EL0 and
* NMEA is set.
*/
return serror_is_masked(vcpu);
}
static bool kvm_serror_undeliverable_at_el2(struct kvm_vcpu *vcpu)
{
return !(vcpu_el2_tge_is_set(vcpu) || vcpu_el2_amo_is_set(vcpu));
}
int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr)
{
lockdep_assert_held(&vcpu->mutex);
if (is_nested_ctxt(vcpu) && kvm_serror_target_is_el2(vcpu))
return kvm_inject_nested_serror(vcpu, esr);
if (vcpu_is_el2(vcpu) && kvm_serror_undeliverable_at_el2(vcpu)) {
vcpu_set_vsesr(vcpu, esr);
vcpu_set_flag(vcpu, NESTED_SERROR_PENDING);
return 1;
}
/*
* Emulate the exception entry if SErrors are unmasked. This is useful if
* the vCPU is in a nested context w/ vSErrors enabled then we've already
* delegated he hardware vSError context (i.e. HCR_EL2.VSE, VSESR_EL2,
* VDISR_EL2) to the guest hypervisor.
*
* As we're emulating the SError injection we need to explicitly populate
* ESR_ELx.EC because hardware will not do it on our behalf.
*/
if (!serror_is_masked(vcpu)) {
pend_serror_exception(vcpu);
esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR);
vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));
return 1;
}
vcpu_set_vsesr(vcpu, esr & ESR_ELx_ISS_MASK);
*vcpu_hcr(vcpu) |= HCR_VSE;
return 1;
}
arch/arm64/kvm/mmio.c
+6 -6
View File
@@ -72,7 +72,7 @@ unsigned long kvm_mmio_read_buf(const void *buf, unsigned int len)
return data;
}
static bool kvm_pending_sync_exception(struct kvm_vcpu *vcpu)
static bool kvm_pending_external_abort(struct kvm_vcpu *vcpu)
{
if (!vcpu_get_flag(vcpu, PENDING_EXCEPTION))
return false;
@@ -90,6 +90,8 @@ static bool kvm_pending_sync_exception(struct kvm_vcpu *vcpu)
switch (vcpu_get_flag(vcpu, EXCEPT_MASK)) {
case unpack_vcpu_flag(EXCEPT_AA64_EL1_SYNC):
case unpack_vcpu_flag(EXCEPT_AA64_EL2_SYNC):
case unpack_vcpu_flag(EXCEPT_AA64_EL1_SERR):
case unpack_vcpu_flag(EXCEPT_AA64_EL2_SERR):
return true;
default:
return false;
@@ -113,7 +115,7 @@ int kvm_handle_mmio_return(struct kvm_vcpu *vcpu)
* Detect if the MMIO return was already handled or if userspace aborted
* the MMIO access.
*/
if (unlikely(!vcpu->mmio_needed || kvm_pending_sync_exception(vcpu)))
if (unlikely(!vcpu->mmio_needed || kvm_pending_external_abort(vcpu)))
return 1;
vcpu->mmio_needed = 0;
@@ -169,10 +171,8 @@ int io_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa)
trace_kvm_mmio_nisv(*vcpu_pc(vcpu), kvm_vcpu_get_esr(vcpu),
kvm_vcpu_get_hfar(vcpu), fault_ipa);
if (vcpu_is_protected(vcpu)) {
kvm_inject_dabt(vcpu, kvm_vcpu_get_hfar(vcpu));
return 1;
}
if (vcpu_is_protected(vcpu))
return kvm_inject_sea_dabt(vcpu, kvm_vcpu_get_hfar(vcpu));
if (test_bit(KVM_ARCH_FLAG_RETURN_NISV_IO_ABORT_TO_USER,
&vcpu->kvm->arch.flags)) {
arch/arm64/kvm/mmu.c
+70 -35
View File
@@ -193,11 +193,6 @@ int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm,
return 0;
}
static bool kvm_is_device_pfn(unsigned long pfn)
{
return !pfn_is_map_memory(pfn);
}
static void *stage2_memcache_zalloc_page(void *arg)
{
struct kvm_mmu_memory_cache *mc = arg;
@@ -1470,6 +1465,18 @@ static bool kvm_vma_mte_allowed(struct vm_area_struct *vma)
return vma->vm_flags & VM_MTE_ALLOWED;
}
static bool kvm_vma_is_cacheable(struct vm_area_struct *vma)
{
switch (FIELD_GET(PTE_ATTRINDX_MASK, pgprot_val(vma->vm_page_prot))) {
case MT_NORMAL_NC:
case MT_DEVICE_nGnRnE:
case MT_DEVICE_nGnRE:
return false;
default:
return true;
}
}
static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
struct kvm_s2_trans *nested,
struct kvm_memory_slot *memslot, unsigned long hva,
@@ -1477,8 +1484,8 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
{
int ret = 0;
bool write_fault, writable, force_pte = false;
bool exec_fault, mte_allowed;
bool device = false, vfio_allow_any_uc = false;
bool exec_fault, mte_allowed, is_vma_cacheable;
bool s2_force_noncacheable = false, vfio_allow_any_uc = false;
unsigned long mmu_seq;
phys_addr_t ipa = fault_ipa;
struct kvm *kvm = vcpu->kvm;
@@ -1492,6 +1499,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R;
struct kvm_pgtable *pgt;
struct page *page;
vm_flags_t vm_flags;
enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_HANDLE_FAULT | KVM_PGTABLE_WALK_SHARED;
if (fault_is_perm)
@@ -1619,6 +1627,10 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
vfio_allow_any_uc = vma->vm_flags & VM_ALLOW_ANY_UNCACHED;
vm_flags = vma->vm_flags;
is_vma_cacheable = kvm_vma_is_cacheable(vma);
/* Don't use the VMA after the unlock -- it may have vanished */
vma = NULL;
@@ -1642,18 +1654,39 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
if (is_error_noslot_pfn(pfn))
return -EFAULT;
if (kvm_is_device_pfn(pfn)) {
/*
* If the page was identified as device early by looking at
* the VMA flags, vma_pagesize is already representing the
* largest quantity we can map. If instead it was mapped
* via __kvm_faultin_pfn(), vma_pagesize is set to PAGE_SIZE
* and must not be upgraded.
*
* In both cases, we don't let transparent_hugepage_adjust()
* change things at the last minute.
*/
device = true;
/*
* Check if this is non-struct page memory PFN, and cannot support
* CMOs. It could potentially be unsafe to access as cachable.
*/
if (vm_flags & (VM_PFNMAP | VM_MIXEDMAP) && !pfn_is_map_memory(pfn)) {
if (is_vma_cacheable) {
/*
* Whilst the VMA owner expects cacheable mapping to this
* PFN, hardware also has to support the FWB and CACHE DIC
* features.
*
* ARM64 KVM relies on kernel VA mapping to the PFN to
* perform cache maintenance as the CMO instructions work on
* virtual addresses. VM_PFNMAP region are not necessarily
* mapped to a KVA and hence the presence of hardware features
* S2FWB and CACHE DIC are mandatory to avoid the need for
* cache maintenance.
*/
if (!kvm_supports_cacheable_pfnmap())
return -EFAULT;
} else {
/*
* If the page was identified as device early by looking at
* the VMA flags, vma_pagesize is already representing the
* largest quantity we can map. If instead it was mapped
* via __kvm_faultin_pfn(), vma_pagesize is set to PAGE_SIZE
* and must not be upgraded.
*
* In both cases, we don't let transparent_hugepage_adjust()
* change things at the last minute.
*/
s2_force_noncacheable = true;
}
} else if (logging_active && !write_fault) {
/*
* Only actually map the page as writable if this was a write
@@ -1662,7 +1695,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
writable = false;
}
if (exec_fault && device)
if (exec_fault && s2_force_noncacheable)
return -ENOEXEC;
/*
@@ -1695,7 +1728,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
* If we are not forced to use page mapping, check if we are
* backed by a THP and thus use block mapping if possible.
*/
if (vma_pagesize == PAGE_SIZE && !(force_pte || device)) {
if (vma_pagesize == PAGE_SIZE && !(force_pte || s2_force_noncacheable)) {
if (fault_is_perm && fault_granule > PAGE_SIZE)
vma_pagesize = fault_granule;
else
@@ -1709,7 +1742,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
}
}
if (!fault_is_perm && !device && kvm_has_mte(kvm)) {
if (!fault_is_perm && !s2_force_noncacheable && kvm_has_mte(kvm)) {
/* Check the VMM hasn't introduced a new disallowed VMA */
if (mte_allowed) {
sanitise_mte_tags(kvm, pfn, vma_pagesize);
@@ -1725,7 +1758,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
if (exec_fault)
prot |= KVM_PGTABLE_PROT_X;
if (device) {
if (s2_force_noncacheable) {
if (vfio_allow_any_uc)
prot |= KVM_PGTABLE_PROT_NORMAL_NC;
else
@@ -1808,7 +1841,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
* There is no need to pass the error into the guest.
*/
if (kvm_handle_guest_sea())
kvm_inject_vabt(vcpu);
return kvm_inject_serror(vcpu);
return 1;
}
@@ -1836,11 +1869,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
if (fault_ipa >= BIT_ULL(VTCR_EL2_IPA(vcpu->arch.hw_mmu->vtcr))) {
fault_ipa |= kvm_vcpu_get_hfar(vcpu) & GENMASK(11, 0);
if (is_iabt)
kvm_inject_pabt(vcpu, fault_ipa);
else
kvm_inject_dabt(vcpu, fault_ipa);
return 1;
return kvm_inject_sea(vcpu, is_iabt, fault_ipa);
}
}
@@ -1912,8 +1941,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
}
if (kvm_vcpu_abt_iss1tw(vcpu)) {
kvm_inject_dabt(vcpu, kvm_vcpu_get_hfar(vcpu));
ret = 1;
ret = kvm_inject_sea_dabt(vcpu, kvm_vcpu_get_hfar(vcpu));
goto out_unlock;
}
@@ -1958,10 +1986,8 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
if (ret == 0)
ret = 1;
out:
if (ret == -ENOEXEC) {
kvm_inject_pabt(vcpu, kvm_vcpu_get_hfar(vcpu));
ret = 1;
}
if (ret == -ENOEXEC)
ret = kvm_inject_sea_iabt(vcpu, kvm_vcpu_get_hfar(vcpu));
out_unlock:
srcu_read_unlock(&vcpu->kvm->srcu, idx);
return ret;
@@ -2221,6 +2247,15 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
ret = -EINVAL;
break;
}
/*
* Cacheable PFNMAP is allowed only if the hardware
* supports it.
*/
if (kvm_vma_is_cacheable(vma) && !kvm_supports_cacheable_pfnmap()) {
ret = -EINVAL;
break;
}
}
hva = min(reg_end, vma->vm_end);
} while (hva < reg_end);
arch/arm64/kvm/nested.c
+50 -59
View File
@@ -1441,12 +1441,11 @@ u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val)
break;
case SYS_ID_AA64PFR0_EL1:
/* No RME, AMU, MPAM, S-EL2, or RAS */
/* No RME, AMU, MPAM, or S-EL2 */
val &= ~(ID_AA64PFR0_EL1_RME |
ID_AA64PFR0_EL1_AMU |
ID_AA64PFR0_EL1_MPAM |
ID_AA64PFR0_EL1_SEL2 |
ID_AA64PFR0_EL1_RAS |
ID_AA64PFR0_EL1_EL3 |
ID_AA64PFR0_EL1_EL2 |
ID_AA64PFR0_EL1_EL1 |
@@ -1683,69 +1682,21 @@ int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu)
set_sysreg_masks(kvm, HFGITR2_EL2, res0, res1);
/* TCR2_EL2 */
res0 = TCR2_EL2_RES0;
res1 = TCR2_EL2_RES1;
if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, D128, IMP))
res0 |= (TCR2_EL2_DisCH0 | TCR2_EL2_DisCH1 | TCR2_EL2_D128);
if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, MEC, IMP))
res0 |= TCR2_EL2_AMEC1 | TCR2_EL2_AMEC0;
if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, HAFDBS, HAFT))
res0 |= TCR2_EL2_HAFT;
if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, THE, IMP))
res0 |= TCR2_EL2_PTTWI | TCR2_EL2_PnCH;
if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, AIE, IMP))
res0 |= TCR2_EL2_AIE;
if (!kvm_has_s1poe(kvm))
res0 |= TCR2_EL2_POE | TCR2_EL2_E0POE;
if (!kvm_has_s1pie(kvm))
res0 |= TCR2_EL2_PIE;
if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, VH, IMP))
res0 |= (TCR2_EL2_E0POE | TCR2_EL2_D128 |
TCR2_EL2_AMEC1 | TCR2_EL2_DisCH0 | TCR2_EL2_DisCH1);
get_reg_fixed_bits(kvm, TCR2_EL2, &res0, &res1);
set_sysreg_masks(kvm, TCR2_EL2, res0, res1);
/* SCTLR_EL1 */
res0 = SCTLR_EL1_RES0;
res1 = SCTLR_EL1_RES1;
if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, PAN, PAN3))
res0 |= SCTLR_EL1_EPAN;
get_reg_fixed_bits(kvm, SCTLR_EL1, &res0, &res1);
set_sysreg_masks(kvm, SCTLR_EL1, res0, res1);
/* SCTLR2_ELx */
get_reg_fixed_bits(kvm, SCTLR2_EL1, &res0, &res1);
set_sysreg_masks(kvm, SCTLR2_EL1, res0, res1);
get_reg_fixed_bits(kvm, SCTLR2_EL2, &res0, &res1);
set_sysreg_masks(kvm, SCTLR2_EL2, res0, res1);
/* MDCR_EL2 */
res0 = MDCR_EL2_RES0;
res1 = MDCR_EL2_RES1;
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP))
res0 |= (MDCR_EL2_HPMN | MDCR_EL2_TPMCR |
MDCR_EL2_TPM | MDCR_EL2_HPME);
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, IMP))
res0 |= MDCR_EL2_E2PB | MDCR_EL2_TPMS;
if (!kvm_has_feat(kvm, ID_AA64DFR1_EL1, SPMU, IMP))
res0 |= MDCR_EL2_EnSPM;
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, V3P1))
res0 |= MDCR_EL2_HPMD;
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceFilt, IMP))
res0 |= MDCR_EL2_TTRF;
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, V3P5))
res0 |= MDCR_EL2_HCCD | MDCR_EL2_HLP;
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceBuffer, IMP))
res0 |= MDCR_EL2_E2TB;
if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, FGT, IMP))
res0 |= MDCR_EL2_TDCC;
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, MTPMU, IMP) ||
kvm_has_feat(kvm, ID_AA64PFR0_EL1, EL3, IMP))
res0 |= MDCR_EL2_MTPME;
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, V3P7))
res0 |= MDCR_EL2_HPMFZO;
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSS, IMP))
res0 |= MDCR_EL2_PMSSE;
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, V1P2))
res0 |= MDCR_EL2_HPMFZS;
if (!kvm_has_feat(kvm, ID_AA64DFR1_EL1, EBEP, IMP))
res0 |= MDCR_EL2_PMEE;
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, DebugVer, V8P9))
res0 |= MDCR_EL2_EBWE;
if (!kvm_has_feat(kvm, ID_AA64DFR2_EL1, STEP, IMP))
res0 |= MDCR_EL2_EnSTEPOP;
get_reg_fixed_bits(kvm, MDCR_EL2, &res0, &res1);
set_sysreg_masks(kvm, MDCR_EL2, res0, res1);
/* CNTHCTL_EL2 */
@@ -1802,3 +1753,43 @@ void check_nested_vcpu_requests(struct kvm_vcpu *vcpu)
if (kvm_check_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu))
kvm_inject_nested_irq(vcpu);
}
/*
* One of the many architectural bugs in FEAT_NV2 is that the guest hypervisor
* can write to HCR_EL2 behind our back, potentially changing the exception
* routing / masking for even the host context.
*
* What follows is some slop to (1) react to exception routing / masking and (2)
* preserve the pending SError state across translation regimes.
*/
void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu)
{
if (!vcpu_has_nv(vcpu))
return;
if (unlikely(vcpu_test_and_clear_flag(vcpu, NESTED_SERROR_PENDING)))
kvm_inject_serror_esr(vcpu, vcpu_get_vsesr(vcpu));
}
void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu)
{
unsigned long *hcr = vcpu_hcr(vcpu);
if (!vcpu_has_nv(vcpu))
return;
/*
* We previously decided that an SError was deliverable to the guest.
* Reap the pending state from HCR_EL2 and...
*/
if (unlikely(__test_and_clear_bit(__ffs(HCR_VSE), hcr)))
vcpu_set_flag(vcpu, NESTED_SERROR_PENDING);
/*
* Re-attempt SError injection in case the deliverability has changed,
* which is necessary to faithfully emulate WFI the case of a pending
* SError being a wakeup condition.
*/
if (unlikely(vcpu_test_and_clear_flag(vcpu, NESTED_SERROR_PENDING)))
kvm_inject_serror_esr(vcpu, vcpu_get_vsesr(vcpu));
}
arch/arm64/kvm/sys_regs.c
+143 -62
View File
@@ -108,7 +108,6 @@ static bool get_el2_to_el1_mapping(unsigned int reg,
PURE_EL2_SYSREG( HACR_EL2 );
PURE_EL2_SYSREG( VTTBR_EL2 );
PURE_EL2_SYSREG( VTCR_EL2 );
PURE_EL2_SYSREG( RVBAR_EL2 );
PURE_EL2_SYSREG( TPIDR_EL2 );
PURE_EL2_SYSREG( HPFAR_EL2 );
PURE_EL2_SYSREG( HCRX_EL2 );
@@ -144,6 +143,7 @@ static bool get_el2_to_el1_mapping(unsigned int reg,
MAPPED_EL2_SYSREG(SPSR_EL2, SPSR_EL1, NULL );
MAPPED_EL2_SYSREG(ZCR_EL2, ZCR_EL1, NULL );
MAPPED_EL2_SYSREG(CONTEXTIDR_EL2, CONTEXTIDR_EL1, NULL );
MAPPED_EL2_SYSREG(SCTLR2_EL2, SCTLR2_EL1, NULL );
default:
return false;
}
@@ -533,8 +533,7 @@ static bool access_gic_sre(struct kvm_vcpu *vcpu,
return ignore_write(vcpu, p);
if (p->Op1 == 4) { /* ICC_SRE_EL2 */
p->regval = (ICC_SRE_EL2_ENABLE | ICC_SRE_EL2_SRE |
ICC_SRE_EL1_DIB | ICC_SRE_EL1_DFB);
p->regval = KVM_ICC_SRE_EL2;
} else { /* ICC_SRE_EL1 */
p->regval = vcpu->arch.vgic_cpu.vgic_v3.vgic_sre;
}
@@ -773,6 +772,12 @@ static u64 reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
return mpidr;
}
static unsigned int hidden_visibility(const struct kvm_vcpu *vcpu,
const struct sys_reg_desc *r)
{
return REG_HIDDEN;
}
static unsigned int pmu_visibility(const struct kvm_vcpu *vcpu,
const struct sys_reg_desc *r)
{
@@ -1612,7 +1617,6 @@ static u64 __kvm_read_sanitised_id_reg(const struct kvm_vcpu *vcpu,
val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_GCS);
val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_THE);
val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_MTEX);
val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_DF2);
val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_PFAR);
val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_MPAM_frac);
break;
@@ -1645,8 +1649,10 @@ static u64 __kvm_read_sanitised_id_reg(const struct kvm_vcpu *vcpu,
val &= ~ID_AA64MMFR2_EL1_NV;
break;
case SYS_ID_AA64MMFR3_EL1:
val &= ID_AA64MMFR3_EL1_TCRX | ID_AA64MMFR3_EL1_S1POE |
ID_AA64MMFR3_EL1_S1PIE;
val &= ID_AA64MMFR3_EL1_TCRX |
ID_AA64MMFR3_EL1_SCTLRX |
ID_AA64MMFR3_EL1_S1POE |
ID_AA64MMFR3_EL1_S1PIE;
break;
case SYS_ID_MMFR4_EL1:
val &= ~ARM64_FEATURE_MASK(ID_MMFR4_EL1_CCIDX);
@@ -1813,7 +1819,7 @@ static u64 sanitise_id_aa64pfr0_el1(const struct kvm_vcpu *vcpu, u64 val)
val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, CSV3, IMP);
}
if (kvm_vgic_global_state.type == VGIC_V3) {
if (vgic_is_v3(vcpu->kvm)) {
val &= ~ID_AA64PFR0_EL1_GIC_MASK;
val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, GIC, IMP);
}
@@ -1955,6 +1961,14 @@ static int set_id_aa64pfr0_el1(struct kvm_vcpu *vcpu,
(vcpu_has_nv(vcpu) && !FIELD_GET(ID_AA64PFR0_EL1_EL2, user_val)))
return -EINVAL;
/*
* If we are running on a GICv5 host and support FEAT_GCIE_LEGACY, then
* we support GICv3. Fail attempts to do anything but set that to IMP.
*/
if (vgic_is_v3_compat(vcpu->kvm) &&
FIELD_GET(ID_AA64PFR0_EL1_GIC_MASK, user_val) != ID_AA64PFR0_EL1_GIC_IMP)
return -EINVAL;
return set_id_reg(vcpu, rd, user_val);
}
@@ -2327,6 +2341,10 @@ static bool bad_redir_trap(struct kvm_vcpu *vcpu,
EL2_REG_FILTERED(name, acc, rst, v, el2_visibility)
#define EL2_REG_VNCR(name, rst, v) EL2_REG(name, bad_vncr_trap, rst, v)
#define EL2_REG_VNCR_FILT(name, vis) \
EL2_REG_FILTERED(name, bad_vncr_trap, reset_val, 0, vis)
#define EL2_REG_VNCR_GICv3(name) \
EL2_REG_VNCR_FILT(name, hidden_visibility)
#define EL2_REG_REDIR(name, rst, v) EL2_REG(name, bad_redir_trap, rst, v)
/*
@@ -2485,6 +2503,21 @@ static unsigned int vncr_el2_visibility(const struct kvm_vcpu *vcpu,
return REG_HIDDEN;
}
static unsigned int sctlr2_visibility(const struct kvm_vcpu *vcpu,
const struct sys_reg_desc *rd)
{
if (kvm_has_sctlr2(vcpu->kvm))
return 0;
return REG_HIDDEN;
}
static unsigned int sctlr2_el2_visibility(const struct kvm_vcpu *vcpu,
const struct sys_reg_desc *rd)
{
return __el2_visibility(vcpu, rd, sctlr2_visibility);
}
static bool access_zcr_el2(struct kvm_vcpu *vcpu,
struct sys_reg_params *p,
const struct sys_reg_desc *r)
@@ -2515,11 +2548,7 @@ static bool access_gic_vtr(struct kvm_vcpu *vcpu,
if (p->is_write)
return write_to_read_only(vcpu, p, r);
p->regval = kvm_vgic_global_state.ich_vtr_el2;
p->regval &= ~(ICH_VTR_EL2_DVIM |
ICH_VTR_EL2_A3V |
ICH_VTR_EL2_IDbits);
p->regval |= ICH_VTR_EL2_nV4;
p->regval = kvm_get_guest_vtr_el2();
return true;
}
@@ -2590,6 +2619,26 @@ static unsigned int tcr2_el2_visibility(const struct kvm_vcpu *vcpu,
return __el2_visibility(vcpu, rd, tcr2_visibility);
}
static unsigned int fgt2_visibility(const struct kvm_vcpu *vcpu,
const struct sys_reg_desc *rd)
{
if (el2_visibility(vcpu, rd) == 0 &&
kvm_has_feat(vcpu->kvm, ID_AA64MMFR0_EL1, FGT, FGT2))
return 0;
return REG_HIDDEN;
}
static unsigned int fgt_visibility(const struct kvm_vcpu *vcpu,
const struct sys_reg_desc *rd)
{
if (el2_visibility(vcpu, rd) == 0 &&
kvm_has_feat(vcpu->kvm, ID_AA64MMFR0_EL1, FGT, IMP))
return 0;
return REG_HIDDEN;
}
static unsigned int s1pie_visibility(const struct kvm_vcpu *vcpu,
const struct sys_reg_desc *rd)
{
@@ -2641,6 +2690,23 @@ static bool access_mdcr(struct kvm_vcpu *vcpu,
return true;
}
static bool access_ras(struct kvm_vcpu *vcpu,
struct sys_reg_params *p,
const struct sys_reg_desc *r)
{
struct kvm *kvm = vcpu->kvm;
switch(reg_to_encoding(r)) {
default:
if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP)) {
kvm_inject_undefined(vcpu);
return false;
}
}
return trap_raz_wi(vcpu, p, r);
}
/*
* For historical (ahem ABI) reasons, KVM treated MIDR_EL1, REVIDR_EL1, and
* AIDR_EL1 as "invariant" registers, meaning userspace cannot change them.
@@ -2868,7 +2934,6 @@ static const struct sys_reg_desc sys_reg_descs[] = {
ID_AA64PFR0_EL1_FP)),
ID_FILTERED(ID_AA64PFR1_EL1, id_aa64pfr1_el1,
~(ID_AA64PFR1_EL1_PFAR |
ID_AA64PFR1_EL1_DF2 |
ID_AA64PFR1_EL1_MTEX |
ID_AA64PFR1_EL1_THE |
ID_AA64PFR1_EL1_GCS |
@@ -2950,6 +3015,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
ID_AA64MMFR2_EL1_NV |
ID_AA64MMFR2_EL1_CCIDX)),
ID_WRITABLE(ID_AA64MMFR3_EL1, (ID_AA64MMFR3_EL1_TCRX |
ID_AA64MMFR3_EL1_SCTLRX |
ID_AA64MMFR3_EL1_S1PIE |
ID_AA64MMFR3_EL1_S1POE)),
ID_WRITABLE(ID_AA64MMFR4_EL1, ID_AA64MMFR4_EL1_NV_frac),
@@ -2960,6 +3026,8 @@ static const struct sys_reg_desc sys_reg_descs[] = {
{ SYS_DESC(SYS_SCTLR_EL1), access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },
{ SYS_DESC(SYS_ACTLR_EL1), access_actlr, reset_actlr, ACTLR_EL1 },
{ SYS_DESC(SYS_CPACR_EL1), NULL, reset_val, CPACR_EL1, 0 },
{ SYS_DESC(SYS_SCTLR2_EL1), access_vm_reg, reset_val, SCTLR2_EL1, 0,
.visibility = sctlr2_visibility },
MTE_REG(RGSR_EL1),
MTE_REG(GCR_EL1),
@@ -2989,14 +3057,14 @@ static const struct sys_reg_desc sys_reg_descs[] = {
{ SYS_DESC(SYS_AFSR1_EL1), access_vm_reg, reset_unknown, AFSR1_EL1 },
{ SYS_DESC(SYS_ESR_EL1), access_vm_reg, reset_unknown, ESR_EL1 },
{ SYS_DESC(SYS_ERRIDR_EL1), trap_raz_wi },
{ SYS_DESC(SYS_ERRSELR_EL1), trap_raz_wi },
{ SYS_DESC(SYS_ERXFR_EL1), trap_raz_wi },
{ SYS_DESC(SYS_ERXCTLR_EL1), trap_raz_wi },
{ SYS_DESC(SYS_ERXSTATUS_EL1), trap_raz_wi },
{ SYS_DESC(SYS_ERXADDR_EL1), trap_raz_wi },
{ SYS_DESC(SYS_ERXMISC0_EL1), trap_raz_wi },
{ SYS_DESC(SYS_ERXMISC1_EL1), trap_raz_wi },
{ SYS_DESC(SYS_ERRIDR_EL1), access_ras },
{ SYS_DESC(SYS_ERRSELR_EL1), access_ras },
{ SYS_DESC(SYS_ERXFR_EL1), access_ras },
{ SYS_DESC(SYS_ERXCTLR_EL1), access_ras },
{ SYS_DESC(SYS_ERXSTATUS_EL1), access_ras },
{ SYS_DESC(SYS_ERXADDR_EL1), access_ras },
{ SYS_DESC(SYS_ERXMISC0_EL1), access_ras },
{ SYS_DESC(SYS_ERXMISC1_EL1), access_ras },
MTE_REG(TFSR_EL1),
MTE_REG(TFSRE0_EL1),
@@ -3307,12 +3375,14 @@ static const struct sys_reg_desc sys_reg_descs[] = {
EL2_REG_VNCR(VMPIDR_EL2, reset_unknown, 0),
EL2_REG(SCTLR_EL2, access_rw, reset_val, SCTLR_EL2_RES1),
EL2_REG(ACTLR_EL2, access_rw, reset_val, 0),
EL2_REG_FILTERED(SCTLR2_EL2, access_vm_reg, reset_val, 0,
sctlr2_el2_visibility),
EL2_REG_VNCR(HCR_EL2, reset_hcr, 0),
EL2_REG(MDCR_EL2, access_mdcr, reset_mdcr, 0),
EL2_REG(CPTR_EL2, access_rw, reset_val, CPTR_NVHE_EL2_RES1),
EL2_REG_VNCR(HSTR_EL2, reset_val, 0),
EL2_REG_VNCR(HFGRTR_EL2, reset_val, 0),
EL2_REG_VNCR(HFGWTR_EL2, reset_val, 0),
EL2_REG_VNCR_FILT(HFGRTR_EL2, fgt_visibility),
EL2_REG_VNCR_FILT(HFGWTR_EL2, fgt_visibility),
EL2_REG_VNCR(HFGITR_EL2, reset_val, 0),
EL2_REG_VNCR(HACR_EL2, reset_val, 0),
@@ -3332,9 +3402,14 @@ static const struct sys_reg_desc sys_reg_descs[] = {
vncr_el2_visibility),
{ SYS_DESC(SYS_DACR32_EL2), undef_access, reset_unknown, DACR32_EL2 },
EL2_REG_VNCR(HDFGRTR_EL2, reset_val, 0),
EL2_REG_VNCR(HDFGWTR_EL2, reset_val, 0),
EL2_REG_VNCR(HAFGRTR_EL2, reset_val, 0),
EL2_REG_VNCR_FILT(HDFGRTR2_EL2, fgt2_visibility),
EL2_REG_VNCR_FILT(HDFGWTR2_EL2, fgt2_visibility),
EL2_REG_VNCR_FILT(HFGRTR2_EL2, fgt2_visibility),
EL2_REG_VNCR_FILT(HFGWTR2_EL2, fgt2_visibility),
EL2_REG_VNCR_FILT(HDFGRTR_EL2, fgt_visibility),
EL2_REG_VNCR_FILT(HDFGWTR_EL2, fgt_visibility),
EL2_REG_VNCR_FILT(HAFGRTR_EL2, fgt_visibility),
EL2_REG_VNCR_FILT(HFGITR2_EL2, fgt2_visibility),
EL2_REG_REDIR(SPSR_EL2, reset_val, 0),
EL2_REG_REDIR(ELR_EL2, reset_val, 0),
{ SYS_DESC(SYS_SP_EL1), access_sp_el1},
@@ -3349,6 +3424,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
EL2_REG(AFSR0_EL2, access_rw, reset_val, 0),
EL2_REG(AFSR1_EL2, access_rw, reset_val, 0),
EL2_REG_REDIR(ESR_EL2, reset_val, 0),
EL2_REG_VNCR(VSESR_EL2, reset_unknown, 0),
{ SYS_DESC(SYS_FPEXC32_EL2), undef_access, reset_val, FPEXC32_EL2, 0x700 },
EL2_REG_REDIR(FAR_EL2, reset_val, 0),
@@ -3375,43 +3451,44 @@ static const struct sys_reg_desc sys_reg_descs[] = {
{ SYS_DESC(SYS_MPAMVPM7_EL2), undef_access },
EL2_REG(VBAR_EL2, access_rw, reset_val, 0),
EL2_REG(RVBAR_EL2, access_rw, reset_val, 0),
{ SYS_DESC(SYS_RVBAR_EL2), undef_access },
{ SYS_DESC(SYS_RMR_EL2), undef_access },
EL2_REG_VNCR(VDISR_EL2, reset_unknown, 0),
EL2_REG_VNCR(ICH_AP0R0_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_AP0R1_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_AP0R2_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_AP0R3_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_AP1R0_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_AP1R1_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_AP1R2_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_AP1R3_EL2, reset_val, 0),
EL2_REG_VNCR_GICv3(ICH_AP0R0_EL2),
EL2_REG_VNCR_GICv3(ICH_AP0R1_EL2),
EL2_REG_VNCR_GICv3(ICH_AP0R2_EL2),
EL2_REG_VNCR_GICv3(ICH_AP0R3_EL2),
EL2_REG_VNCR_GICv3(ICH_AP1R0_EL2),
EL2_REG_VNCR_GICv3(ICH_AP1R1_EL2),
EL2_REG_VNCR_GICv3(ICH_AP1R2_EL2),
EL2_REG_VNCR_GICv3(ICH_AP1R3_EL2),
{ SYS_DESC(SYS_ICC_SRE_EL2), access_gic_sre },
EL2_REG_VNCR(ICH_HCR_EL2, reset_val, 0),
EL2_REG_VNCR_GICv3(ICH_HCR_EL2),
{ SYS_DESC(SYS_ICH_VTR_EL2), access_gic_vtr },
{ SYS_DESC(SYS_ICH_MISR_EL2), access_gic_misr },
{ SYS_DESC(SYS_ICH_EISR_EL2), access_gic_eisr },
{ SYS_DESC(SYS_ICH_ELRSR_EL2), access_gic_elrsr },
EL2_REG_VNCR(ICH_VMCR_EL2, reset_val, 0),
EL2_REG_VNCR_GICv3(ICH_VMCR_EL2),
EL2_REG_VNCR(ICH_LR0_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR1_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR2_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR3_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR4_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR5_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR6_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR7_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR8_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR9_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR10_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR11_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR12_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR13_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR14_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR15_EL2, reset_val, 0),
EL2_REG_VNCR_GICv3(ICH_LR0_EL2),
EL2_REG_VNCR_GICv3(ICH_LR1_EL2),
EL2_REG_VNCR_GICv3(ICH_LR2_EL2),
EL2_REG_VNCR_GICv3(ICH_LR3_EL2),
EL2_REG_VNCR_GICv3(ICH_LR4_EL2),
EL2_REG_VNCR_GICv3(ICH_LR5_EL2),
EL2_REG_VNCR_GICv3(ICH_LR6_EL2),
EL2_REG_VNCR_GICv3(ICH_LR7_EL2),
EL2_REG_VNCR_GICv3(ICH_LR8_EL2),
EL2_REG_VNCR_GICv3(ICH_LR9_EL2),
EL2_REG_VNCR_GICv3(ICH_LR10_EL2),
EL2_REG_VNCR_GICv3(ICH_LR11_EL2),
EL2_REG_VNCR_GICv3(ICH_LR12_EL2),
EL2_REG_VNCR_GICv3(ICH_LR13_EL2),
EL2_REG_VNCR_GICv3(ICH_LR14_EL2),
EL2_REG_VNCR_GICv3(ICH_LR15_EL2),
EL2_REG(CONTEXTIDR_EL2, access_rw, reset_val, 0),
EL2_REG(TPIDR_EL2, access_rw, reset_val, 0),
@@ -4280,12 +4357,12 @@ static const struct sys_reg_desc cp15_64_regs[] = {
};
static bool check_sysreg_table(const struct sys_reg_desc *table, unsigned int n,
bool is_32)
bool reset_check)
{
unsigned int i;
for (i = 0; i < n; i++) {
if (!is_32 && table[i].reg && !table[i].reset) {
if (reset_check && table[i].reg && !table[i].reset) {
kvm_err("sys_reg table %pS entry %d (%s) lacks reset\n",
&table[i], i, table[i].name);
return false;
@@ -4480,7 +4557,7 @@ static bool kvm_esr_cp10_id_to_sys64(u64 esr, struct sys_reg_params *params)
return true;
kvm_pr_unimpl("Unhandled cp10 register %s: %u\n",
params->is_write ? "write" : "read", reg_id);
str_write_read(params->is_write), reg_id);
return false;
}
@@ -5274,18 +5351,22 @@ int kvm_finalize_sys_regs(struct kvm_vcpu *vcpu)
int __init kvm_sys_reg_table_init(void)
{
const struct sys_reg_desc *gicv3_regs;
bool valid = true;
unsigned int i;
unsigned int i, sz;
int ret = 0;
/* Make sure tables are unique and in order. */
valid &= check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs), false);
valid &= check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs), true);
valid &= check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs), true);
valid &= check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs), true);
valid &= check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs), true);
valid &= check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs), true);
valid &= check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs), false);
valid &= check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs), false);
valid &= check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs), false);
valid &= check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs), false);
valid &= check_sysreg_table(sys_insn_descs, ARRAY_SIZE(sys_insn_descs), false);
gicv3_regs = vgic_v3_get_sysreg_table(&sz);
valid &= check_sysreg_table(gicv3_regs, sz, false);
if (!valid)
return -EINVAL;
arch/arm64/kvm/sys_regs.h
+1 -1
View File
@@ -108,7 +108,7 @@ inline void print_sys_reg_msg(const struct sys_reg_params *p,
/* Look, we even formatted it for you to paste into the table! */
kvm_pr_unimpl("%pV { Op0(%2u), Op1(%2u), CRn(%2u), CRm(%2u), Op2(%2u), func_%s },\n",
&(struct va_format){ fmt, &va },
p->Op0, p->Op1, p->CRn, p->CRm, p->Op2, p->is_write ? "write" : "read");
p->Op0, p->Op1, p->CRn, p->CRm, p->Op2, str_write_read(p->is_write));
va_end(va);
}
arch/arm64/kvm/trace_handle_exit.h
+1 -1
View File
@@ -113,7 +113,7 @@ TRACE_EVENT(kvm_sys_access,
__entry->vcpu_pc, __entry->name ?: "UNKN",
__entry->Op0, __entry->Op1, __entry->CRn,
__entry->CRm, __entry->Op2,
__entry->is_write ? "write" : "read")
str_write_read(__entry->is_write))
);
TRACE_EVENT(kvm_set_guest_debug,
arch/arm64/kvm/vgic-sys-reg-v3.c
+125 -2
View File
@@ -297,6 +297,91 @@ static int get_gic_sre(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
return 0;
}
static int set_gic_ich_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
u64 val)
{
__vcpu_assign_sys_reg(vcpu, r->reg, val);
return 0;
}
static int get_gic_ich_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
u64 *val)
{
*val = __vcpu_sys_reg(vcpu, r->reg);
return 0;
}
static int set_gic_ich_apr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
u64 val)
{
u8 idx = r->Op2 & 3;
if (idx > vgic_v3_max_apr_idx(vcpu))
return -EINVAL;
return set_gic_ich_reg(vcpu, r, val);
}
static int get_gic_ich_apr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
u64 *val)
{
u8 idx = r->Op2 & 3;
if (idx > vgic_v3_max_apr_idx(vcpu))
return -EINVAL;
return get_gic_ich_reg(vcpu, r, val);
}
static int set_gic_icc_sre(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
u64 val)
{
if (val != KVM_ICC_SRE_EL2)
return -EINVAL;
return 0;
}
static int get_gic_icc_sre(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
u64 *val)
{
*val = KVM_ICC_SRE_EL2;
return 0;
}
static int set_gic_ich_vtr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
u64 val)
{
if (val != kvm_get_guest_vtr_el2())
return -EINVAL;
return 0;
}
static int get_gic_ich_vtr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
u64 *val)
{
*val = kvm_get_guest_vtr_el2();
return 0;
}
static unsigned int el2_visibility(const struct kvm_vcpu *vcpu,
const struct sys_reg_desc *rd)
{
return vcpu_has_nv(vcpu) ? 0 : REG_HIDDEN;
}
#define __EL2_REG(r, acc, i) \
{ \
SYS_DESC(SYS_ ## r), \
.get_user = get_gic_ ## acc, \
.set_user = set_gic_ ## acc, \
.reg = i, \
.visibility = el2_visibility, \
}
#define EL2_REG(r, acc) __EL2_REG(r, acc, r)
#define EL2_REG_RO(r, acc) __EL2_REG(r, acc, 0)
static const struct sys_reg_desc gic_v3_icc_reg_descs[] = {
{ SYS_DESC(SYS_ICC_PMR_EL1),
.set_user = set_gic_pmr, .get_user = get_gic_pmr, },
@@ -328,8 +413,42 @@ static const struct sys_reg_desc gic_v3_icc_reg_descs[] = {
.set_user = set_gic_grpen0, .get_user = get_gic_grpen0, },
{ SYS_DESC(SYS_ICC_IGRPEN1_EL1),
.set_user = set_gic_grpen1, .get_user = get_gic_grpen1, },
EL2_REG(ICH_AP0R0_EL2, ich_apr),
EL2_REG(ICH_AP0R1_EL2, ich_apr),
EL2_REG(ICH_AP0R2_EL2, ich_apr),
EL2_REG(ICH_AP0R3_EL2, ich_apr),
EL2_REG(ICH_AP1R0_EL2, ich_apr),
EL2_REG(ICH_AP1R1_EL2, ich_apr),
EL2_REG(ICH_AP1R2_EL2, ich_apr),
EL2_REG(ICH_AP1R3_EL2, ich_apr),
EL2_REG_RO(ICC_SRE_EL2, icc_sre),
EL2_REG(ICH_HCR_EL2, ich_reg),
EL2_REG_RO(ICH_VTR_EL2, ich_vtr),
EL2_REG(ICH_VMCR_EL2, ich_reg),
EL2_REG(ICH_LR0_EL2, ich_reg),
EL2_REG(ICH_LR1_EL2, ich_reg),
EL2_REG(ICH_LR2_EL2, ich_reg),
EL2_REG(ICH_LR3_EL2, ich_reg),
EL2_REG(ICH_LR4_EL2, ich_reg),
EL2_REG(ICH_LR5_EL2, ich_reg),
EL2_REG(ICH_LR6_EL2, ich_reg),
EL2_REG(ICH_LR7_EL2, ich_reg),
EL2_REG(ICH_LR8_EL2, ich_reg),
EL2_REG(ICH_LR9_EL2, ich_reg),
EL2_REG(ICH_LR10_EL2, ich_reg),
EL2_REG(ICH_LR11_EL2, ich_reg),
EL2_REG(ICH_LR12_EL2, ich_reg),
EL2_REG(ICH_LR13_EL2, ich_reg),
EL2_REG(ICH_LR14_EL2, ich_reg),
EL2_REG(ICH_LR15_EL2, ich_reg),
};
const struct sys_reg_desc *vgic_v3_get_sysreg_table(unsigned int *sz)
{
*sz = ARRAY_SIZE(gic_v3_icc_reg_descs);
return gic_v3_icc_reg_descs;
}
static u64 attr_to_id(u64 attr)
{
return ARM64_SYS_REG(FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_OP0_MASK, attr),
@@ -341,8 +460,12 @@ static u64 attr_to_id(u64 attr)
int vgic_v3_has_cpu_sysregs_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
{
if (get_reg_by_id(attr_to_id(attr->attr), gic_v3_icc_reg_descs,
ARRAY_SIZE(gic_v3_icc_reg_descs)))
const struct sys_reg_desc *r;
r = get_reg_by_id(attr_to_id(attr->attr), gic_v3_icc_reg_descs,
ARRAY_SIZE(gic_v3_icc_reg_descs));
if (r && !sysreg_hidden(vcpu, r))
return 0;
return -ENXIO;
arch/arm64/kvm/vgic/vgic-init.c
+15 -15
View File
@@ -157,6 +157,7 @@ int kvm_vgic_create(struct kvm *kvm, u32 type)
kvm->arch.vgic.in_kernel = true;
kvm->arch.vgic.vgic_model = type;
kvm->arch.vgic.implementation_rev = KVM_VGIC_IMP_REV_LATEST;
kvm->arch.vgic.vgic_dist_base = VGIC_ADDR_UNDEF;
@@ -165,6 +166,9 @@ int kvm_vgic_create(struct kvm *kvm, u32 type)
else
INIT_LIST_HEAD(&kvm->arch.vgic.rd_regions);
if (type == KVM_DEV_TYPE_ARM_VGIC_V3)
kvm->arch.vgic.nassgicap = system_supports_direct_sgis();
out_unlock:
mutex_unlock(&kvm->arch.config_lock);
kvm_unlock_all_vcpus(kvm);
@@ -391,11 +395,10 @@ int vgic_init(struct kvm *kvm)
goto out;
/*
* If we have GICv4.1 enabled, unconditionally request enable the
* v4 support so that we get HW-accelerated vSGIs. Otherwise, only
* enable it if we present a virtual ITS to the guest.
* Ensure vPEs are allocated if direct IRQ injection (e.g. vSGIs,
* vLPIs) is supported.
*/
if (vgic_supports_direct_msis(kvm)) {
if (vgic_supports_direct_irqs(kvm)) {
ret = vgic_v4_init(kvm);
if (ret)
goto out;
@@ -409,15 +412,7 @@ int vgic_init(struct kvm *kvm)
goto out;
vgic_debug_init(kvm);
/*
* If userspace didn't set the GIC implementation revision,
* default to the latest and greatest. You know want it.
*/
if (!dist->implementation_rev)
dist->implementation_rev = KVM_VGIC_IMP_REV_LATEST;
dist->initialized = true;
out:
return ret;
}
@@ -443,7 +438,7 @@ static void kvm_vgic_dist_destroy(struct kvm *kvm)
dist->vgic_cpu_base = VGIC_ADDR_UNDEF;
}
if (vgic_supports_direct_msis(kvm))
if (vgic_supports_direct_irqs(kvm))
vgic_v4_teardown(kvm);
xa_destroy(&dist->lpi_xa);
@@ -674,10 +669,12 @@ void kvm_vgic_init_cpu_hardware(void)
* We want to make sure the list registers start out clear so that we
* only have the program the used registers.
*/
if (kvm_vgic_global_state.type == VGIC_V2)
if (kvm_vgic_global_state.type == VGIC_V2) {
vgic_v2_init_lrs();
else
} else if (kvm_vgic_global_state.type == VGIC_V3 ||
kvm_vgic_global_state.has_gcie_v3_compat) {
kvm_call_hyp(__vgic_v3_init_lrs);
}
}
/**
@@ -722,6 +719,9 @@ int kvm_vgic_hyp_init(void)
kvm_info("GIC system register CPU interface enabled\n");
}
break;
case GIC_V5:
ret = vgic_v5_probe(gic_kvm_info);
break;
default:
ret = -ENODEV;
}
arch/arm64/kvm/vgic/vgic-its.c
+4 -1
View File
@@ -758,7 +758,7 @@ static void its_free_ite(struct kvm *kvm, struct its_ite *ite)
if (irq) {
scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) {
if (irq->hw)
WARN_ON(its_unmap_vlpi(ite->irq->host_irq));
its_unmap_vlpi(ite->irq->host_irq);
irq->hw = false;
}
@@ -2694,6 +2694,9 @@ static int vgic_its_ctrl(struct kvm *kvm, struct vgic_its *its, u64 attr)
case KVM_DEV_ARM_ITS_RESTORE_TABLES:
ret = abi->restore_tables(its);
break;
default:
ret = -ENXIO;
break;
}
mutex_unlock(&its->its_lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
+44 -26
View File
@@ -5,6 +5,7 @@
* Copyright (C) 2015 ARM Ltd.
* Author: Marc Zyngier <marc.zyngier@arm.com>
*/
#include <linux/irqchip/arm-gic-v3.h>
#include <linux/kvm_host.h>
#include <kvm/arm_vgic.h>
#include <linux/uaccess.h>
@@ -303,12 +304,6 @@ static int vgic_get_common_attr(struct kvm_device *dev,
VGIC_NR_PRIVATE_IRQS, uaddr);
break;
}
case KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ: {
u32 __user *uaddr = (u32 __user *)(long)attr->addr;
r = put_user(dev->kvm->arch.vgic.mi_intid, uaddr);
break;
}
}
return r;
@@ -509,6 +504,24 @@ int vgic_v3_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr,
return 0;
}
/*
* Allow access to certain ID-like registers prior to VGIC initialization,
* thereby allowing the VMM to provision the features / sizing of the VGIC.
*/
static bool reg_allowed_pre_init(struct kvm_device_attr *attr)
{
if (attr->group != KVM_DEV_ARM_VGIC_GRP_DIST_REGS)
return false;
switch (attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK) {
case GICD_IIDR:
case GICD_TYPER2:
return true;
default:
return false;
}
}
/*
* vgic_v3_attr_regs_access - allows user space to access VGIC v3 state
*
@@ -523,7 +536,7 @@ static int vgic_v3_attr_regs_access(struct kvm_device *dev,
struct vgic_reg_attr reg_attr;
gpa_t addr;
struct kvm_vcpu *vcpu;
bool uaccess, post_init = true;
bool uaccess;
u32 val;
int ret;
@@ -539,9 +552,6 @@ static int vgic_v3_attr_regs_access(struct kvm_device *dev,
/* Sysregs uaccess is performed by the sysreg handling code */
uaccess = false;
break;
case KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ:
post_init = false;
fallthrough;
default:
uaccess = true;
}
@@ -561,7 +571,7 @@ static int vgic_v3_attr_regs_access(struct kvm_device *dev,
mutex_lock(&dev->kvm->arch.config_lock);
if (post_init != vgic_initialized(dev->kvm)) {
if (!(vgic_initialized(dev->kvm) || reg_allowed_pre_init(attr))) {
ret = -EBUSY;
goto out;
}
@@ -591,19 +601,6 @@ static int vgic_v3_attr_regs_access(struct kvm_device *dev,
}
break;
}
case KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ:
if (!is_write) {
val = dev->kvm->arch.vgic.mi_intid;
ret = 0;
break;
}
ret = -EINVAL;
if ((val < VGIC_NR_PRIVATE_IRQS) && (val >= VGIC_NR_SGIS)) {
dev->kvm->arch.vgic.mi_intid = val;
ret = 0;
}
break;
default:
ret = -EINVAL;
break;
@@ -630,8 +627,24 @@ static int vgic_v3_set_attr(struct kvm_device *dev,
case KVM_DEV_ARM_VGIC_GRP_REDIST_REGS:
case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS:
case KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO:
case KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ:
return vgic_v3_attr_regs_access(dev, attr, true);
case KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ: {
u32 __user *uaddr = (u32 __user *)attr->addr;
u32 val;
if (get_user(val, uaddr))
return -EFAULT;
guard(mutex)(&dev->kvm->arch.config_lock);
if (vgic_initialized(dev->kvm))
return -EBUSY;
if (!irq_is_ppi(val))
return -EINVAL;
dev->kvm->arch.vgic.mi_intid = val;
return 0;
}
default:
return vgic_set_common_attr(dev, attr);
}
@@ -645,8 +658,13 @@ static int vgic_v3_get_attr(struct kvm_device *dev,
case KVM_DEV_ARM_VGIC_GRP_REDIST_REGS:
case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS:
case KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO:
case KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ:
return vgic_v3_attr_regs_access(dev, attr, false);
case KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ: {
u32 __user *uaddr = (u32 __user *)(long)attr->addr;
guard(mutex)(&dev->kvm->arch.config_lock);
return put_user(dev->kvm->arch.vgic.mi_intid, uaddr);
}
default:
return vgic_get_common_attr(dev, attr);
}
arch/arm64/kvm/vgic/vgic-mmio-v3.c
+26 -7
View File
@@ -50,8 +50,17 @@ bool vgic_has_its(struct kvm *kvm)
bool vgic_supports_direct_msis(struct kvm *kvm)
{
return (kvm_vgic_global_state.has_gicv4_1 ||
(kvm_vgic_global_state.has_gicv4 && vgic_has_its(kvm)));
return kvm_vgic_global_state.has_gicv4 && vgic_has_its(kvm);
}
bool system_supports_direct_sgis(void)
{
return kvm_vgic_global_state.has_gicv4_1 && gic_cpuif_has_vsgi();
}
bool vgic_supports_direct_sgis(struct kvm *kvm)
{
return kvm->arch.vgic.nassgicap;
}
/*
@@ -86,7 +95,7 @@ static unsigned long vgic_mmio_read_v3_misc(struct kvm_vcpu *vcpu,
}
break;
case GICD_TYPER2:
if (kvm_vgic_global_state.has_gicv4_1 && gic_cpuif_has_vsgi())
if (vgic_supports_direct_sgis(vcpu->kvm))
value = GICD_TYPER2_nASSGIcap;
break;
case GICD_IIDR:
@@ -119,7 +128,7 @@ static void vgic_mmio_write_v3_misc(struct kvm_vcpu *vcpu,
dist->enabled = val & GICD_CTLR_ENABLE_SS_G1;
/* Not a GICv4.1? No HW SGIs */
if (!kvm_vgic_global_state.has_gicv4_1 || !gic_cpuif_has_vsgi())
if (!vgic_supports_direct_sgis(vcpu->kvm))
val &= ~GICD_CTLR_nASSGIreq;
/* Dist stays enabled? nASSGIreq is RO */
@@ -133,7 +142,7 @@ static void vgic_mmio_write_v3_misc(struct kvm_vcpu *vcpu,
if (is_hwsgi != dist->nassgireq)
vgic_v4_configure_vsgis(vcpu->kvm);
if (kvm_vgic_global_state.has_gicv4_1 &&
if (vgic_supports_direct_sgis(vcpu->kvm) &&
was_enabled != dist->enabled)
kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_RELOAD_GICv4);
else if (!was_enabled && dist->enabled)
@@ -159,8 +168,18 @@ static int vgic_mmio_uaccess_write_v3_misc(struct kvm_vcpu *vcpu,
switch (addr & 0x0c) {
case GICD_TYPER2:
if (val != vgic_mmio_read_v3_misc(vcpu, addr, len))
reg = vgic_mmio_read_v3_misc(vcpu, addr, len);
if (reg == val)
return 0;
if (vgic_initialized(vcpu->kvm))
return -EBUSY;
if ((reg ^ val) & ~GICD_TYPER2_nASSGIcap)
return -EINVAL;
if (!system_supports_direct_sgis() && val)
return -EINVAL;
dist->nassgicap = val & GICD_TYPER2_nASSGIcap;
return 0;
case GICD_IIDR:
reg = vgic_mmio_read_v3_misc(vcpu, addr, len);
@@ -178,7 +197,7 @@ static int vgic_mmio_uaccess_write_v3_misc(struct kvm_vcpu *vcpu,
}
case GICD_CTLR:
/* Not a GICv4.1? No HW SGIs */
if (!kvm_vgic_global_state.has_gicv4_1)
if (!vgic_supports_direct_sgis(vcpu->kvm))
val &= ~GICD_CTLR_nASSGIreq;
dist->enabled = val & GICD_CTLR_ENABLE_SS_G1;
arch/arm64/kvm/vgic/vgic-v3-nested.c
+1 -1
View File
@@ -116,7 +116,7 @@ bool vgic_state_is_nested(struct kvm_vcpu *vcpu)
{
u64 xmo;
if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu)) {
if (is_nested_ctxt(vcpu)) {
xmo = __vcpu_sys_reg(vcpu, HCR_EL2) & (HCR_IMO | HCR_FMO);
WARN_ONCE(xmo && xmo != (HCR_IMO | HCR_FMO),
"Separate virtual IRQ/FIQ settings not supported\n");
arch/arm64/kvm/vgic/vgic-v4.c
+6 -8
View File
@@ -356,7 +356,7 @@ int vgic_v4_put(struct kvm_vcpu *vcpu)
{
struct its_vpe *vpe = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe;
if (!vgic_supports_direct_msis(vcpu->kvm) || !vpe->resident)
if (!vgic_supports_direct_irqs(vcpu->kvm) || !vpe->resident)
return 0;
return its_make_vpe_non_resident(vpe, vgic_v4_want_doorbell(vcpu));
@@ -367,7 +367,7 @@ int vgic_v4_load(struct kvm_vcpu *vcpu)
struct its_vpe *vpe = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe;
int err;
if (!vgic_supports_direct_msis(vcpu->kvm) || vpe->resident)
if (!vgic_supports_direct_irqs(vcpu->kvm) || vpe->resident)
return 0;
if (vcpu_get_flag(vcpu, IN_WFI))
@@ -527,28 +527,26 @@ static struct vgic_irq *__vgic_host_irq_get_vlpi(struct kvm *kvm, int host_irq)
return NULL;
}
int kvm_vgic_v4_unset_forwarding(struct kvm *kvm, int host_irq)
void kvm_vgic_v4_unset_forwarding(struct kvm *kvm, int host_irq)
{
struct vgic_irq *irq;
unsigned long flags;
int ret = 0;
if (!vgic_supports_direct_msis(kvm))
return 0;
return;
irq = __vgic_host_irq_get_vlpi(kvm, host_irq);
if (!irq)
return 0;
return;
raw_spin_lock_irqsave(&irq->irq_lock, flags);
WARN_ON(irq->hw && irq->host_irq != host_irq);
if (irq->hw) {
atomic_dec(&irq->target_vcpu->arch.vgic_cpu.vgic_v3.its_vpe.vlpi_count);
irq->hw = false;
ret = its_unmap_vlpi(host_irq);
its_unmap_vlpi(host_irq);
}
raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
vgic_put_irq(kvm, irq);
return ret;
}
arch/arm64/kvm/vgic/vgic-v5.c
+52
View File
@@ -0,0 +1,52 @@
// SPDX-License-Identifier: GPL-2.0-only
#include <kvm/arm_vgic.h>
#include <linux/irqchip/arm-vgic-info.h>
#include "vgic.h"
/*
* Probe for a vGICv5 compatible interrupt controller, returning 0 on success.
* Currently only supports GICv3-based VMs on a GICv5 host, and hence only
* registers a VGIC_V3 device.
*/
int vgic_v5_probe(const struct gic_kvm_info *info)
{
u64 ich_vtr_el2;
int ret;
if (!info->has_gcie_v3_compat)
return -ENODEV;
kvm_vgic_global_state.type = VGIC_V5;
kvm_vgic_global_state.has_gcie_v3_compat = true;
/* We only support v3 compat mode - use vGICv3 limits */
kvm_vgic_global_state.max_gic_vcpus = VGIC_V3_MAX_CPUS;
kvm_vgic_global_state.vcpu_base = 0;
kvm_vgic_global_state.vctrl_base = NULL;
kvm_vgic_global_state.can_emulate_gicv2 = false;
kvm_vgic_global_state.has_gicv4 = false;
kvm_vgic_global_state.has_gicv4_1 = false;
ich_vtr_el2 = kvm_call_hyp_ret(__vgic_v3_get_gic_config);
kvm_vgic_global_state.ich_vtr_el2 = (u32)ich_vtr_el2;
/*
* The ListRegs field is 5 bits, but there is an architectural
* maximum of 16 list registers. Just ignore bit 4...
*/
kvm_vgic_global_state.nr_lr = (ich_vtr_el2 & 0xf) + 1;
ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V3);
if (ret) {
kvm_err("Cannot register GICv3-legacy KVM device.\n");
return ret;
}
static_branch_enable(&kvm_vgic_global_state.gicv3_cpuif);
kvm_info("GCIE legacy system register CPU interface\n");
return 0;
}
arch/arm64/kvm/vgic/vgic.c
+2 -2
View File
@@ -951,7 +951,7 @@ void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
* can be directly injected (GICv4).
*/
if (list_empty(&vcpu->arch.vgic_cpu.ap_list_head) &&
!vgic_supports_direct_msis(vcpu->kvm))
!vgic_supports_direct_irqs(vcpu->kvm))
return;
DEBUG_SPINLOCK_BUG_ON(!irqs_disabled());
@@ -965,7 +965,7 @@ void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
if (can_access_vgic_from_kernel())
vgic_restore_state(vcpu);
if (vgic_supports_direct_msis(vcpu->kvm))
if (vgic_supports_direct_irqs(vcpu->kvm))
vgic_v4_commit(vcpu);
}
arch/arm64/kvm/vgic/vgic.h
+48
View File
@@ -64,6 +64,24 @@
KVM_REG_ARM_VGIC_SYSREG_CRM_MASK | \
KVM_REG_ARM_VGIC_SYSREG_OP2_MASK)
#define KVM_ICC_SRE_EL2 (ICC_SRE_EL2_ENABLE | ICC_SRE_EL2_SRE | \
ICC_SRE_EL1_DIB | ICC_SRE_EL1_DFB)
#define KVM_ICH_VTR_EL2_RES0 (ICH_VTR_EL2_DVIM | \
ICH_VTR_EL2_A3V | \
ICH_VTR_EL2_IDbits)
#define KVM_ICH_VTR_EL2_RES1 ICH_VTR_EL2_nV4
static inline u64 kvm_get_guest_vtr_el2(void)
{
u64 vtr;
vtr = kvm_vgic_global_state.ich_vtr_el2;
vtr &= ~KVM_ICH_VTR_EL2_RES0;
vtr |= KVM_ICH_VTR_EL2_RES1;
return vtr;
}
/*
* As per Documentation/virt/kvm/devices/arm-vgic-its.rst,
* below macros are defined for ITS table entry encoding.
@@ -297,6 +315,7 @@ int vgic_v3_redist_uaccess(struct kvm_vcpu *vcpu, bool is_write,
int vgic_v3_cpu_sysregs_uaccess(struct kvm_vcpu *vcpu,
struct kvm_device_attr *attr, bool is_write);
int vgic_v3_has_cpu_sysregs_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr);
const struct sys_reg_desc *vgic_v3_get_sysreg_table(unsigned int *sz);
int vgic_v3_line_level_info_uaccess(struct kvm_vcpu *vcpu, bool is_write,
u32 intid, u32 *val);
int kvm_register_vgic_device(unsigned long type);
@@ -308,6 +327,8 @@ int vgic_init(struct kvm *kvm);
void vgic_debug_init(struct kvm *kvm);
void vgic_debug_destroy(struct kvm *kvm);
int vgic_v5_probe(const struct gic_kvm_info *info);
static inline int vgic_v3_max_apr_idx(struct kvm_vcpu *vcpu)
{
struct vgic_cpu *cpu_if = &vcpu->arch.vgic_cpu;
@@ -369,7 +390,23 @@ void vgic_its_invalidate_all_caches(struct kvm *kvm);
int vgic_its_inv_lpi(struct kvm *kvm, struct vgic_irq *irq);
int vgic_its_invall(struct kvm_vcpu *vcpu);
bool system_supports_direct_sgis(void);
bool vgic_supports_direct_msis(struct kvm *kvm);
bool vgic_supports_direct_sgis(struct kvm *kvm);
static inline bool vgic_supports_direct_irqs(struct kvm *kvm)
{
/*
* Deliberately conflate vLPI and vSGI support on GICv4.1 hardware,
* indirectly allowing userspace to control whether or not vPEs are
* allocated for the VM.
*/
if (system_supports_direct_sgis())
return vgic_supports_direct_sgis(kvm);
return vgic_supports_direct_msis(kvm);
}
int vgic_v4_init(struct kvm *kvm);
void vgic_v4_teardown(struct kvm *kvm);
void vgic_v4_configure_vsgis(struct kvm *kvm);
@@ -389,6 +426,17 @@ void vgic_v3_put_nested(struct kvm_vcpu *vcpu);
void vgic_v3_handle_nested_maint_irq(struct kvm_vcpu *vcpu);
void vgic_v3_nested_update_mi(struct kvm_vcpu *vcpu);
static inline bool vgic_is_v3_compat(struct kvm *kvm)
{
return cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF) &&
kvm_vgic_global_state.has_gcie_v3_compat;
}
static inline bool vgic_is_v3(struct kvm *kvm)
{
return kvm_vgic_global_state.type == VGIC_V3 || vgic_is_v3_compat(kvm);
}
int vgic_its_debug_init(struct kvm_device *dev);
void vgic_its_debug_destroy(struct kvm_device *dev);
arch/arm64/tools/cpucaps
+3 -1
View File
@@ -35,7 +35,8 @@ HAS_GENERIC_AUTH
HAS_GENERIC_AUTH_ARCH_QARMA3
HAS_GENERIC_AUTH_ARCH_QARMA5
HAS_GENERIC_AUTH_IMP_DEF
HAS_GIC_CPUIF_SYSREGS
HAS_GICV3_CPUIF
HAS_GICV5_CPUIF
HAS_GIC_PRIO_MASKING
HAS_GIC_PRIO_RELAXED_SYNC
HAS_HCR_NV1
@@ -50,6 +51,7 @@ HAS_PAN
HAS_PMUV3
HAS_S1PIE
HAS_S1POE
HAS_SCTLR2
HAS_RAS_EXTN
HAS_RNG
HAS_SB
arch/arm64/tools/sysreg
+511 -3
View File
@@ -1314,7 +1314,10 @@ UnsignedEnum 19:16 UINJ
0b0000 NI
0b0001 IMP
EndEnum
Res0 15:12
UnsignedEnum 15:12 GCIE
0b0000 NI
0b0001 IMP
EndEnum
UnsignedEnum 11:8 MTEFAR
0b0000 NI
0b0001 IMP
@@ -3153,6 +3156,435 @@ Sysreg PMIAR_EL1 3 0 9 14 7
Field 63:0 ADDRESS
EndSysreg
SysregFields ICC_PPI_HMRx_EL1
Field 63 HM63
Field 62 HM62
Field 61 HM61
Field 60 HM60
Field 59 HM59
Field 58 HM58
Field 57 HM57
Field 56 HM56
Field 55 HM55
Field 54 HM54
Field 53 HM53
Field 52 HM52
Field 51 HM51
Field 50 HM50
Field 49 HM49
Field 48 HM48
Field 47 HM47
Field 46 HM46
Field 45 HM45
Field 44 HM44
Field 43 HM43
Field 42 HM42
Field 41 HM41
Field 40 HM40
Field 39 HM39
Field 38 HM38
Field 37 HM37
Field 36 HM36
Field 35 HM35
Field 34 HM34
Field 33 HM33
Field 32 HM32
Field 31 HM31
Field 30 HM30
Field 29 HM29
Field 28 HM28
Field 27 HM27
Field 26 HM26
Field 25 HM25
Field 24 HM24
Field 23 HM23
Field 22 HM22
Field 21 HM21
Field 20 HM20
Field 19 HM19
Field 18 HM18
Field 17 HM17
Field 16 HM16
Field 15 HM15
Field 14 HM14
Field 13 HM13
Field 12 HM12
Field 11 HM11
Field 10 HM10
Field 9 HM9
Field 8 HM8
Field 7 HM7
Field 6 HM6
Field 5 HM5
Field 4 HM4
Field 3 HM3
Field 2 HM2
Field 1 HM1
Field 0 HM0
EndSysregFields
Sysreg ICC_PPI_HMR0_EL1 3 0 12 10 0
Fields ICC_PPI_HMRx_EL1
EndSysreg
Sysreg ICC_PPI_HMR1_EL1 3 0 12 10 1
Fields ICC_PPI_HMRx_EL1
EndSysreg
Sysreg ICC_IDR0_EL1 3 0 12 10 2
Res0 63:12
UnsignedEnum 11:8 GCIE_LEGACY
0b0000 NI
0b0001 IMP
EndEnum
UnsignedEnum 7:4 PRI_BITS
0b0011 4BITS
0b0100 5BITS
EndEnum
UnsignedEnum 3:0 ID_BITS
0b0000 16BITS
0b0001 24BITS
EndEnum
EndSysreg
Sysreg ICC_ICSR_EL1 3 0 12 10 4
Res0 63:48
Field 47:32 IAFFID
Res0 31:16
Field 15:11 Priority
Res0 10:6
Field 5 HM
Field 4 Active
Field 3 IRM
Field 2 Pending
Field 1 Enabled
Field 0 F
EndSysreg
SysregFields ICC_PPI_ENABLERx_EL1
Field 63 EN63
Field 62 EN62
Field 61 EN61
Field 60 EN60
Field 59 EN59
Field 58 EN58
Field 57 EN57
Field 56 EN56
Field 55 EN55
Field 54 EN54
Field 53 EN53
Field 52 EN52
Field 51 EN51
Field 50 EN50
Field 49 EN49
Field 48 EN48
Field 47 EN47
Field 46 EN46
Field 45 EN45
Field 44 EN44
Field 43 EN43
Field 42 EN42
Field 41 EN41
Field 40 EN40
Field 39 EN39
Field 38 EN38
Field 37 EN37
Field 36 EN36
Field 35 EN35
Field 34 EN34
Field 33 EN33
Field 32 EN32
Field 31 EN31
Field 30 EN30
Field 29 EN29
Field 28 EN28
Field 27 EN27
Field 26 EN26
Field 25 EN25
Field 24 EN24
Field 23 EN23
Field 22 EN22
Field 21 EN21
Field 20 EN20
Field 19 EN19
Field 18 EN18
Field 17 EN17
Field 16 EN16
Field 15 EN15
Field 14 EN14
Field 13 EN13
Field 12 EN12
Field 11 EN11
Field 10 EN10
Field 9 EN9
Field 8 EN8
Field 7 EN7
Field 6 EN6
Field 5 EN5
Field 4 EN4
Field 3 EN3
Field 2 EN2
Field 1 EN1
Field 0 EN0
EndSysregFields
Sysreg ICC_PPI_ENABLER0_EL1 3 0 12 10 6
Fields ICC_PPI_ENABLERx_EL1
EndSysreg
Sysreg ICC_PPI_ENABLER1_EL1 3 0 12 10 7
Fields ICC_PPI_ENABLERx_EL1
EndSysreg
SysregFields ICC_PPI_ACTIVERx_EL1
Field 63 Active63
Field 62 Active62
Field 61 Active61
Field 60 Active60
Field 59 Active59
Field 58 Active58
Field 57 Active57
Field 56 Active56
Field 55 Active55
Field 54 Active54
Field 53 Active53
Field 52 Active52
Field 51 Active51
Field 50 Active50
Field 49 Active49
Field 48 Active48
Field 47 Active47
Field 46 Active46
Field 45 Active45
Field 44 Active44
Field 43 Active43
Field 42 Active42
Field 41 Active41
Field 40 Active40
Field 39 Active39
Field 38 Active38
Field 37 Active37
Field 36 Active36
Field 35 Active35
Field 34 Active34
Field 33 Active33
Field 32 Active32
Field 31 Active31
Field 30 Active30
Field 29 Active29
Field 28 Active28
Field 27 Active27
Field 26 Active26
Field 25 Active25
Field 24 Active24
Field 23 Active23
Field 22 Active22
Field 21 Active21
Field 20 Active20
Field 19 Active19
Field 18 Active18
Field 17 Active17
Field 16 Active16
Field 15 Active15
Field 14 Active14
Field 13 Active13
Field 12 Active12
Field 11 Active11
Field 10 Active10
Field 9 Active9
Field 8 Active8
Field 7 Active7
Field 6 Active6
Field 5 Active5
Field 4 Active4
Field 3 Active3
Field 2 Active2
Field 1 Active1
Field 0 Active0
EndSysregFields
Sysreg ICC_PPI_CACTIVER0_EL1 3 0 12 13 0
Fields ICC_PPI_ACTIVERx_EL1
EndSysreg
Sysreg ICC_PPI_CACTIVER1_EL1 3 0 12 13 1
Fields ICC_PPI_ACTIVERx_EL1
EndSysreg
Sysreg ICC_PPI_SACTIVER0_EL1 3 0 12 13 2
Fields ICC_PPI_ACTIVERx_EL1
EndSysreg
Sysreg ICC_PPI_SACTIVER1_EL1 3 0 12 13 3
Fields ICC_PPI_ACTIVERx_EL1
EndSysreg
SysregFields ICC_PPI_PENDRx_EL1
Field 63 Pend63
Field 62 Pend62
Field 61 Pend61
Field 60 Pend60
Field 59 Pend59
Field 58 Pend58
Field 57 Pend57
Field 56 Pend56
Field 55 Pend55
Field 54 Pend54
Field 53 Pend53
Field 52 Pend52
Field 51 Pend51
Field 50 Pend50
Field 49 Pend49
Field 48 Pend48
Field 47 Pend47
Field 46 Pend46
Field 45 Pend45
Field 44 Pend44
Field 43 Pend43
Field 42 Pend42
Field 41 Pend41
Field 40 Pend40
Field 39 Pend39
Field 38 Pend38
Field 37 Pend37
Field 36 Pend36
Field 35 Pend35
Field 34 Pend34
Field 33 Pend33
Field 32 Pend32
Field 31 Pend31
Field 30 Pend30
Field 29 Pend29
Field 28 Pend28
Field 27 Pend27
Field 26 Pend26
Field 25 Pend25
Field 24 Pend24
Field 23 Pend23
Field 22 Pend22
Field 21 Pend21
Field 20 Pend20
Field 19 Pend19
Field 18 Pend18
Field 17 Pend17
Field 16 Pend16
Field 15 Pend15
Field 14 Pend14
Field 13 Pend13
Field 12 Pend12
Field 11 Pend11
Field 10 Pend10
Field 9 Pend9
Field 8 Pend8
Field 7 Pend7
Field 6 Pend6
Field 5 Pend5
Field 4 Pend4
Field 3 Pend3
Field 2 Pend2
Field 1 Pend1
Field 0 Pend0
EndSysregFields
Sysreg ICC_PPI_CPENDR0_EL1 3 0 12 13 4
Fields ICC_PPI_PENDRx_EL1
EndSysreg
Sysreg ICC_PPI_CPENDR1_EL1 3 0 12 13 5
Fields ICC_PPI_PENDRx_EL1
EndSysreg
Sysreg ICC_PPI_SPENDR0_EL1 3 0 12 13 6
Fields ICC_PPI_PENDRx_EL1
EndSysreg
Sysreg ICC_PPI_SPENDR1_EL1 3 0 12 13 7
Fields ICC_PPI_PENDRx_EL1
EndSysreg
SysregFields ICC_PPI_PRIORITYRx_EL1
Res0 63:61
Field 60:56 Priority7
Res0 55:53
Field 52:48 Priority6
Res0 47:45
Field 44:40 Priority5
Res0 39:37
Field 36:32 Priority4
Res0 31:29
Field 28:24 Priority3
Res0 23:21
Field 20:16 Priority2
Res0 15:13
Field 12:8 Priority1
Res0 7:5
Field 4:0 Priority0
EndSysregFields
Sysreg ICC_PPI_PRIORITYR0_EL1 3 0 12 14 0
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR1_EL1 3 0 12 14 1
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR2_EL1 3 0 12 14 2
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR3_EL1 3 0 12 14 3
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR4_EL1 3 0 12 14 4
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR5_EL1 3 0 12 14 5
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR6_EL1 3 0 12 14 6
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR7_EL1 3 0 12 14 7
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR8_EL1 3 0 12 15 0
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR9_EL1 3 0 12 15 1
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR10_EL1 3 0 12 15 2
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR11_EL1 3 0 12 15 3
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR12_EL1 3 0 12 15 4
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR13_EL1 3 0 12 15 5
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR14_EL1 3 0 12 15 6
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR15_EL1 3 0 12 15 7
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg PMSELR_EL0 3 3 9 12 5
Res0 63:5
Field 4:0 SEL
@@ -3235,6 +3667,19 @@ Res0 14:12
Field 11:0 AFFINITY
EndSysreg
Sysreg ICC_CR0_EL1 3 1 12 0 1
Res0 63:39
Field 38 PID
Field 37:32 IPPT
Res0 31:1
Field 0 EN
EndSysreg
Sysreg ICC_PCR_EL1 3 1 12 0 2
Res0 63:5
Field 4:0 PRIORITY
EndSysreg
Sysreg CSSELR_EL1 3 2 0 0 0
Res0 63:5
Field 4 TnD
@@ -4121,6 +4566,54 @@ Field 31:16 PhyPARTID29
Field 15:0 PhyPARTID28
EndSysreg
Sysreg ICH_HFGRTR_EL2 3 4 12 9 4
Res0 63:21
Field 20 ICC_PPI_ACTIVERn_EL1
Field 19 ICC_PPI_PRIORITYRn_EL1
Field 18 ICC_PPI_PENDRn_EL1
Field 17 ICC_PPI_ENABLERn_EL1
Field 16 ICC_PPI_HMRn_EL1
Res0 15:8
Field 7 ICC_IAFFIDR_EL1
Field 6 ICC_ICSR_EL1
Field 5 ICC_PCR_EL1
Field 4 ICC_HPPIR_EL1
Field 3 ICC_HAPR_EL1
Field 2 ICC_CR0_EL1
Field 1 ICC_IDRn_EL1
Field 0 ICC_APR_EL1
EndSysreg
Sysreg ICH_HFGWTR_EL2 3 4 12 9 6
Res0 63:21
Field 20 ICC_PPI_ACTIVERn_EL1
Field 19 ICC_PPI_PRIORITYRn_EL1
Field 18 ICC_PPI_PENDRn_EL1
Field 17 ICC_PPI_ENABLERn_EL1
Res0 16:7
Field 6 ICC_ICSR_EL1
Field 5 ICC_PCR_EL1
Res0 4:3
Field 2 ICC_CR0_EL1
Res0 1
Field 0 ICC_APR_EL1
EndSysreg
Sysreg ICH_HFGITR_EL2 3 4 12 9 7
Res0 63:11
Field 10 GICRCDNMIA
Field 9 GICRCDIA
Field 8 GICCDDI
Field 7 GICCDEOI
Field 6 GICCDHM
Field 5 GICCDRCFG
Field 4 GICCDPEND
Field 3 GICCDAFF
Field 2 GICCDPRI
Field 1 GICCDDIS
Field 0 GICCDEN
EndSysreg
Sysreg ICH_HCR_EL2 3 4 12 11 0
Res0 63:32
Field 31:27 EOIcount
@@ -4169,6 +4662,12 @@ Field 1 U
Field 0 EOI
EndSysreg
Sysreg ICH_VCTLR_EL2 3 4 12 11 4
Res0 63:2
Field 1 V3
Field 0 En
EndSysreg
Sysreg CONTEXTIDR_EL2 3 4 13 0 1
Fields CONTEXTIDR_ELx
EndSysreg
@@ -4282,7 +4781,13 @@ Mapping TCR_EL1
EndSysreg
Sysreg TCR2_EL1 3 0 2 0 3
Res0 63:16
Res0 63:22
Field 21 FNGNA1
Field 20 FNGNA0
Res0 19
Field 18 FNG1
Field 17 FNG0
Field 16 A2
Field 15 DisCH1
Field 14 DisCH0
Res0 13:12
@@ -4306,7 +4811,10 @@ Mapping TCR2_EL1
EndSysreg
Sysreg TCR2_EL2 3 4 2 0 3
Res0 63:16
Res0 63:19
Field 18 FNG1
Field 17 FNG0
Field 16 A2
Field 15 DisCH1
Field 14 DisCH0
Field 13 AMEC1
arch/loongarch/include/asm/kvm_host.h
+6 -6
View File
@@ -50,12 +50,6 @@ struct kvm_vm_stat {
struct kvm_vm_stat_generic generic;
u64 pages;
u64 hugepages;
u64 ipi_read_exits;
u64 ipi_write_exits;
u64 eiointc_read_exits;
u64 eiointc_write_exits;
u64 pch_pic_read_exits;
u64 pch_pic_write_exits;
};
struct kvm_vcpu_stat {
@@ -65,6 +59,12 @@ struct kvm_vcpu_stat {
u64 cpucfg_exits;
u64 signal_exits;
u64 hypercall_exits;
u64 ipi_read_exits;
u64 ipi_write_exits;
u64 eiointc_read_exits;
u64 eiointc_write_exits;
u64 pch_pic_read_exits;
u64 pch_pic_write_exits;
};
#define KVM_MEM_HUGEPAGE_CAPABLE (1UL << 0)
arch/loongarch/kvm/exit.c
+14 -19
View File
@@ -289,9 +289,11 @@ static int kvm_trap_handle_gspr(struct kvm_vcpu *vcpu)
er = EMULATE_FAIL;
switch (((inst.word >> 24) & 0xff)) {
case 0x0: /* CPUCFG GSPR */
trace_kvm_exit_cpucfg(vcpu, KVM_TRACE_EXIT_CPUCFG);
er = kvm_emu_cpucfg(vcpu, inst);
break;
case 0x4: /* CSR{RD,WR,XCHG} GSPR */
trace_kvm_exit_csr(vcpu, KVM_TRACE_EXIT_CSR);
er = kvm_handle_csr(vcpu, inst);
break;
case 0x6: /* Cache, Idle and IOCSR GSPR */
@@ -821,32 +823,25 @@ static int kvm_handle_lbt_disabled(struct kvm_vcpu *vcpu, int ecode)
return RESUME_GUEST;
}
static int kvm_send_pv_ipi(struct kvm_vcpu *vcpu)
static void kvm_send_pv_ipi(struct kvm_vcpu *vcpu)
{
unsigned int min, cpu, i;
unsigned long ipi_bitmap;
unsigned int min, cpu;
struct kvm_vcpu *dest;
DECLARE_BITMAP(ipi_bitmap, BITS_PER_LONG * 2) = {
kvm_read_reg(vcpu, LOONGARCH_GPR_A1),
kvm_read_reg(vcpu, LOONGARCH_GPR_A2)
};
min = kvm_read_reg(vcpu, LOONGARCH_GPR_A3);
for (i = 0; i < 2; i++, min += BITS_PER_LONG) {
ipi_bitmap = kvm_read_reg(vcpu, LOONGARCH_GPR_A1 + i);
if (!ipi_bitmap)
for_each_set_bit(cpu, ipi_bitmap, BITS_PER_LONG * 2) {
dest = kvm_get_vcpu_by_cpuid(vcpu->kvm, cpu + min);
if (!dest)
continue;
cpu = find_first_bit((void *)&ipi_bitmap, BITS_PER_LONG);
while (cpu < BITS_PER_LONG) {
dest = kvm_get_vcpu_by_cpuid(vcpu->kvm, cpu + min);
cpu = find_next_bit((void *)&ipi_bitmap, BITS_PER_LONG, cpu + 1);
if (!dest)
continue;
/* Send SWI0 to dest vcpu to emulate IPI interrupt */
kvm_queue_irq(dest, INT_SWI0);
kvm_vcpu_kick(dest);
}
/* Send SWI0 to dest vcpu to emulate IPI interrupt */
kvm_queue_irq(dest, INT_SWI0);
kvm_vcpu_kick(dest);
}
return 0;
}
/*
arch/loongarch/kvm/intc/eiointc.c
+82 -471
View File
@@ -9,7 +9,7 @@
static void eiointc_set_sw_coreisr(struct loongarch_eiointc *s)
{
int ipnum, cpu, cpuid, irq_index, irq_mask, irq;
int ipnum, cpu, cpuid, irq;
struct kvm_vcpu *vcpu;
for (irq = 0; irq < EIOINTC_IRQS; irq++) {
@@ -18,8 +18,6 @@ static void eiointc_set_sw_coreisr(struct loongarch_eiointc *s)
ipnum = count_trailing_zeros(ipnum);
ipnum = (ipnum >= 0 && ipnum < 4) ? ipnum : 0;
}
irq_index = irq / 32;
irq_mask = BIT(irq & 0x1f);
cpuid = s->coremap.reg_u8[irq];
vcpu = kvm_get_vcpu_by_cpuid(s->kvm, cpuid);
@@ -27,16 +25,16 @@ static void eiointc_set_sw_coreisr(struct loongarch_eiointc *s)
continue;
cpu = vcpu->vcpu_id;
if (!!(s->coreisr.reg_u32[cpu][irq_index] & irq_mask))
set_bit(irq, s->sw_coreisr[cpu][ipnum]);
if (test_bit(irq, (unsigned long *)s->coreisr.reg_u32[cpu]))
__set_bit(irq, s->sw_coreisr[cpu][ipnum]);
else
clear_bit(irq, s->sw_coreisr[cpu][ipnum]);
__clear_bit(irq, s->sw_coreisr[cpu][ipnum]);
}
}
static void eiointc_update_irq(struct loongarch_eiointc *s, int irq, int level)
{
int ipnum, cpu, found, irq_index, irq_mask;
int ipnum, cpu, found;
struct kvm_vcpu *vcpu;
struct kvm_interrupt vcpu_irq;
@@ -48,19 +46,16 @@ static void eiointc_update_irq(struct loongarch_eiointc *s, int irq, int level)
cpu = s->sw_coremap[irq];
vcpu = kvm_get_vcpu(s->kvm, cpu);
irq_index = irq / 32;
irq_mask = BIT(irq & 0x1f);
if (level) {
/* if not enable return false */
if (((s->enable.reg_u32[irq_index]) & irq_mask) == 0)
if (!test_bit(irq, (unsigned long *)s->enable.reg_u32))
return;
s->coreisr.reg_u32[cpu][irq_index] |= irq_mask;
__set_bit(irq, (unsigned long *)s->coreisr.reg_u32[cpu]);
found = find_first_bit(s->sw_coreisr[cpu][ipnum], EIOINTC_IRQS);
set_bit(irq, s->sw_coreisr[cpu][ipnum]);
__set_bit(irq, s->sw_coreisr[cpu][ipnum]);
} else {
s->coreisr.reg_u32[cpu][irq_index] &= ~irq_mask;
clear_bit(irq, s->sw_coreisr[cpu][ipnum]);
__clear_bit(irq, (unsigned long *)s->coreisr.reg_u32[cpu]);
__clear_bit(irq, s->sw_coreisr[cpu][ipnum]);
found = find_first_bit(s->sw_coreisr[cpu][ipnum], EIOINTC_IRQS);
}
@@ -110,159 +105,14 @@ void eiointc_set_irq(struct loongarch_eiointc *s, int irq, int level)
unsigned long flags;
unsigned long *isr = (unsigned long *)s->isr.reg_u8;
level ? set_bit(irq, isr) : clear_bit(irq, isr);
spin_lock_irqsave(&s->lock, flags);
level ? __set_bit(irq, isr) : __clear_bit(irq, isr);
eiointc_update_irq(s, irq, level);
spin_unlock_irqrestore(&s->lock, flags);
}
static inline void eiointc_enable_irq(struct kvm_vcpu *vcpu,
struct loongarch_eiointc *s, int index, u8 mask, int level)
{
u8 val;
int irq;
val = mask & s->isr.reg_u8[index];
irq = ffs(val);
while (irq != 0) {
/*
* enable bit change from 0 to 1,
* need to update irq by pending bits
*/
eiointc_update_irq(s, irq - 1 + index * 8, level);
val &= ~BIT(irq - 1);
irq = ffs(val);
}
}
static int loongarch_eiointc_readb(struct kvm_vcpu *vcpu, struct loongarch_eiointc *s,
gpa_t addr, int len, void *val)
{
int index, ret = 0;
u8 data = 0;
gpa_t offset;
offset = addr - EIOINTC_BASE;
switch (offset) {
case EIOINTC_NODETYPE_START ... EIOINTC_NODETYPE_END:
index = offset - EIOINTC_NODETYPE_START;
data = s->nodetype.reg_u8[index];
break;
case EIOINTC_IPMAP_START ... EIOINTC_IPMAP_END:
index = offset - EIOINTC_IPMAP_START;
data = s->ipmap.reg_u8[index];
break;
case EIOINTC_ENABLE_START ... EIOINTC_ENABLE_END:
index = offset - EIOINTC_ENABLE_START;
data = s->enable.reg_u8[index];
break;
case EIOINTC_BOUNCE_START ... EIOINTC_BOUNCE_END:
index = offset - EIOINTC_BOUNCE_START;
data = s->bounce.reg_u8[index];
break;
case EIOINTC_COREISR_START ... EIOINTC_COREISR_END:
index = offset - EIOINTC_COREISR_START;
data = s->coreisr.reg_u8[vcpu->vcpu_id][index];
break;
case EIOINTC_COREMAP_START ... EIOINTC_COREMAP_END:
index = offset - EIOINTC_COREMAP_START;
data = s->coremap.reg_u8[index];
break;
default:
ret = -EINVAL;
break;
}
*(u8 *)val = data;
return ret;
}
static int loongarch_eiointc_readw(struct kvm_vcpu *vcpu, struct loongarch_eiointc *s,
gpa_t addr, int len, void *val)
{
int index, ret = 0;
u16 data = 0;
gpa_t offset;
offset = addr - EIOINTC_BASE;
switch (offset) {
case EIOINTC_NODETYPE_START ... EIOINTC_NODETYPE_END:
index = (offset - EIOINTC_NODETYPE_START) >> 1;
data = s->nodetype.reg_u16[index];
break;
case EIOINTC_IPMAP_START ... EIOINTC_IPMAP_END:
index = (offset - EIOINTC_IPMAP_START) >> 1;
data = s->ipmap.reg_u16[index];
break;
case EIOINTC_ENABLE_START ... EIOINTC_ENABLE_END:
index = (offset - EIOINTC_ENABLE_START) >> 1;
data = s->enable.reg_u16[index];
break;
case EIOINTC_BOUNCE_START ... EIOINTC_BOUNCE_END:
index = (offset - EIOINTC_BOUNCE_START) >> 1;
data = s->bounce.reg_u16[index];
break;
case EIOINTC_COREISR_START ... EIOINTC_COREISR_END:
index = (offset - EIOINTC_COREISR_START) >> 1;
data = s->coreisr.reg_u16[vcpu->vcpu_id][index];
break;
case EIOINTC_COREMAP_START ... EIOINTC_COREMAP_END:
index = (offset - EIOINTC_COREMAP_START) >> 1;
data = s->coremap.reg_u16[index];
break;
default:
ret = -EINVAL;
break;
}
*(u16 *)val = data;
return ret;
}
static int loongarch_eiointc_readl(struct kvm_vcpu *vcpu, struct loongarch_eiointc *s,
gpa_t addr, int len, void *val)
{
int index, ret = 0;
u32 data = 0;
gpa_t offset;
offset = addr - EIOINTC_BASE;
switch (offset) {
case EIOINTC_NODETYPE_START ... EIOINTC_NODETYPE_END:
index = (offset - EIOINTC_NODETYPE_START) >> 2;
data = s->nodetype.reg_u32[index];
break;
case EIOINTC_IPMAP_START ... EIOINTC_IPMAP_END:
index = (offset - EIOINTC_IPMAP_START) >> 2;
data = s->ipmap.reg_u32[index];
break;
case EIOINTC_ENABLE_START ... EIOINTC_ENABLE_END:
index = (offset - EIOINTC_ENABLE_START) >> 2;
data = s->enable.reg_u32[index];
break;
case EIOINTC_BOUNCE_START ... EIOINTC_BOUNCE_END:
index = (offset - EIOINTC_BOUNCE_START) >> 2;
data = s->bounce.reg_u32[index];
break;
case EIOINTC_COREISR_START ... EIOINTC_COREISR_END:
index = (offset - EIOINTC_COREISR_START) >> 2;
data = s->coreisr.reg_u32[vcpu->vcpu_id][index];
break;
case EIOINTC_COREMAP_START ... EIOINTC_COREMAP_END:
index = (offset - EIOINTC_COREMAP_START) >> 2;
data = s->coremap.reg_u32[index];
break;
default:
ret = -EINVAL;
break;
}
*(u32 *)val = data;
return ret;
}
static int loongarch_eiointc_readq(struct kvm_vcpu *vcpu, struct loongarch_eiointc *s,
gpa_t addr, int len, void *val)
static int loongarch_eiointc_read(struct kvm_vcpu *vcpu, struct loongarch_eiointc *s,
gpa_t addr, unsigned long *val)
{
int index, ret = 0;
u64 data = 0;
@@ -298,7 +148,7 @@ static int loongarch_eiointc_readq(struct kvm_vcpu *vcpu, struct loongarch_eioin
ret = -EINVAL;
break;
}
*(u64 *)val = data;
*val = data;
return ret;
}
@@ -308,7 +158,7 @@ static int kvm_eiointc_read(struct kvm_vcpu *vcpu,
gpa_t addr, int len, void *val)
{
int ret = -EINVAL;
unsigned long flags;
unsigned long flags, data, offset;
struct loongarch_eiointc *eiointc = vcpu->kvm->arch.eiointc;
if (!eiointc) {
@@ -321,355 +171,115 @@ static int kvm_eiointc_read(struct kvm_vcpu *vcpu,
return -EINVAL;
}
vcpu->kvm->stat.eiointc_read_exits++;
offset = addr & 0x7;
addr -= offset;
vcpu->stat.eiointc_read_exits++;
spin_lock_irqsave(&eiointc->lock, flags);
ret = loongarch_eiointc_read(vcpu, eiointc, addr, &data);
spin_unlock_irqrestore(&eiointc->lock, flags);
if (ret)
return ret;
data = data >> (offset * 8);
switch (len) {
case 1:
ret = loongarch_eiointc_readb(vcpu, eiointc, addr, len, val);
*(long *)val = (s8)data;
break;
case 2:
ret = loongarch_eiointc_readw(vcpu, eiointc, addr, len, val);
*(long *)val = (s16)data;
break;
case 4:
ret = loongarch_eiointc_readl(vcpu, eiointc, addr, len, val);
break;
case 8:
ret = loongarch_eiointc_readq(vcpu, eiointc, addr, len, val);
*(long *)val = (s32)data;
break;
default:
WARN_ONCE(1, "%s: Abnormal address access: addr 0x%llx, size %d\n",
__func__, addr, len);
}
spin_unlock_irqrestore(&eiointc->lock, flags);
return ret;
}
static int loongarch_eiointc_writeb(struct kvm_vcpu *vcpu,
struct loongarch_eiointc *s,
gpa_t addr, int len, const void *val)
{
int index, irq, bits, ret = 0;
u8 cpu;
u8 data, old_data;
u8 coreisr, old_coreisr;
gpa_t offset;
data = *(u8 *)val;
offset = addr - EIOINTC_BASE;
switch (offset) {
case EIOINTC_NODETYPE_START ... EIOINTC_NODETYPE_END:
index = (offset - EIOINTC_NODETYPE_START);
s->nodetype.reg_u8[index] = data;
break;
case EIOINTC_IPMAP_START ... EIOINTC_IPMAP_END:
/*
* ipmap cannot be set at runtime, can be set only at the beginning
* of irqchip driver, need not update upper irq level
*/
index = (offset - EIOINTC_IPMAP_START);
s->ipmap.reg_u8[index] = data;
break;
case EIOINTC_ENABLE_START ... EIOINTC_ENABLE_END:
index = (offset - EIOINTC_ENABLE_START);
old_data = s->enable.reg_u8[index];
s->enable.reg_u8[index] = data;
/*
* 1: enable irq.
* update irq when isr is set.
*/
data = s->enable.reg_u8[index] & ~old_data & s->isr.reg_u8[index];
eiointc_enable_irq(vcpu, s, index, data, 1);
/*
* 0: disable irq.
* update irq when isr is set.
*/
data = ~s->enable.reg_u8[index] & old_data & s->isr.reg_u8[index];
eiointc_enable_irq(vcpu, s, index, data, 0);
break;
case EIOINTC_BOUNCE_START ... EIOINTC_BOUNCE_END:
/* do not emulate hw bounced irq routing */
index = offset - EIOINTC_BOUNCE_START;
s->bounce.reg_u8[index] = data;
break;
case EIOINTC_COREISR_START ... EIOINTC_COREISR_END:
index = (offset - EIOINTC_COREISR_START);
/* use attrs to get current cpu index */
cpu = vcpu->vcpu_id;
coreisr = data;
old_coreisr = s->coreisr.reg_u8[cpu][index];
/* write 1 to clear interrupt */
s->coreisr.reg_u8[cpu][index] = old_coreisr & ~coreisr;
coreisr &= old_coreisr;
bits = sizeof(data) * 8;
irq = find_first_bit((void *)&coreisr, bits);
while (irq < bits) {
eiointc_update_irq(s, irq + index * bits, 0);
bitmap_clear((void *)&coreisr, irq, 1);
irq = find_first_bit((void *)&coreisr, bits);
}
break;
case EIOINTC_COREMAP_START ... EIOINTC_COREMAP_END:
irq = offset - EIOINTC_COREMAP_START;
index = irq;
s->coremap.reg_u8[index] = data;
eiointc_update_sw_coremap(s, irq, data, sizeof(data), true);
break;
default:
ret = -EINVAL;
*(long *)val = (long)data;
break;
}
return ret;
return 0;
}
static int loongarch_eiointc_writew(struct kvm_vcpu *vcpu,
static int loongarch_eiointc_write(struct kvm_vcpu *vcpu,
struct loongarch_eiointc *s,
gpa_t addr, int len, const void *val)
gpa_t addr, u64 value, u64 field_mask)
{
int i, index, irq, bits, ret = 0;
int index, irq, ret = 0;
u8 cpu;
u16 data, old_data;
u16 coreisr, old_coreisr;
u64 data, old, mask;
gpa_t offset;
data = *(u16 *)val;
offset = addr - EIOINTC_BASE;
offset = addr & 7;
mask = field_mask << (offset * 8);
data = (value & field_mask) << (offset * 8);
switch (offset) {
case EIOINTC_NODETYPE_START ... EIOINTC_NODETYPE_END:
index = (offset - EIOINTC_NODETYPE_START) >> 1;
s->nodetype.reg_u16[index] = data;
break;
case EIOINTC_IPMAP_START ... EIOINTC_IPMAP_END:
/*
* ipmap cannot be set at runtime, can be set only at the beginning
* of irqchip driver, need not update upper irq level
*/
index = (offset - EIOINTC_IPMAP_START) >> 1;
s->ipmap.reg_u16[index] = data;
break;
case EIOINTC_ENABLE_START ... EIOINTC_ENABLE_END:
index = (offset - EIOINTC_ENABLE_START) >> 1;
old_data = s->enable.reg_u16[index];
s->enable.reg_u16[index] = data;
/*
* 1: enable irq.
* update irq when isr is set.
*/
data = s->enable.reg_u16[index] & ~old_data & s->isr.reg_u16[index];
for (i = 0; i < sizeof(data); i++) {
u8 mask = (data >> (i * 8)) & 0xff;
eiointc_enable_irq(vcpu, s, index * 2 + i, mask, 1);
}
/*
* 0: disable irq.
* update irq when isr is set.
*/
data = ~s->enable.reg_u16[index] & old_data & s->isr.reg_u16[index];
for (i = 0; i < sizeof(data); i++) {
u8 mask = (data >> (i * 8)) & 0xff;
eiointc_enable_irq(vcpu, s, index * 2 + i, mask, 0);
}
break;
case EIOINTC_BOUNCE_START ... EIOINTC_BOUNCE_END:
/* do not emulate hw bounced irq routing */
index = (offset - EIOINTC_BOUNCE_START) >> 1;
s->bounce.reg_u16[index] = data;
break;
case EIOINTC_COREISR_START ... EIOINTC_COREISR_END:
index = (offset - EIOINTC_COREISR_START) >> 1;
/* use attrs to get current cpu index */
cpu = vcpu->vcpu_id;
coreisr = data;
old_coreisr = s->coreisr.reg_u16[cpu][index];
/* write 1 to clear interrupt */
s->coreisr.reg_u16[cpu][index] = old_coreisr & ~coreisr;
coreisr &= old_coreisr;
bits = sizeof(data) * 8;
irq = find_first_bit((void *)&coreisr, bits);
while (irq < bits) {
eiointc_update_irq(s, irq + index * bits, 0);
bitmap_clear((void *)&coreisr, irq, 1);
irq = find_first_bit((void *)&coreisr, bits);
}
break;
case EIOINTC_COREMAP_START ... EIOINTC_COREMAP_END:
irq = offset - EIOINTC_COREMAP_START;
index = irq >> 1;
s->coremap.reg_u16[index] = data;
eiointc_update_sw_coremap(s, irq, data, sizeof(data), true);
break;
default:
ret = -EINVAL;
break;
}
return ret;
}
static int loongarch_eiointc_writel(struct kvm_vcpu *vcpu,
struct loongarch_eiointc *s,
gpa_t addr, int len, const void *val)
{
int i, index, irq, bits, ret = 0;
u8 cpu;
u32 data, old_data;
u32 coreisr, old_coreisr;
gpa_t offset;
data = *(u32 *)val;
offset = addr - EIOINTC_BASE;
switch (offset) {
case EIOINTC_NODETYPE_START ... EIOINTC_NODETYPE_END:
index = (offset - EIOINTC_NODETYPE_START) >> 2;
s->nodetype.reg_u32[index] = data;
break;
case EIOINTC_IPMAP_START ... EIOINTC_IPMAP_END:
/*
* ipmap cannot be set at runtime, can be set only at the beginning
* of irqchip driver, need not update upper irq level
*/
index = (offset - EIOINTC_IPMAP_START) >> 2;
s->ipmap.reg_u32[index] = data;
break;
case EIOINTC_ENABLE_START ... EIOINTC_ENABLE_END:
index = (offset - EIOINTC_ENABLE_START) >> 2;
old_data = s->enable.reg_u32[index];
s->enable.reg_u32[index] = data;
/*
* 1: enable irq.
* update irq when isr is set.
*/
data = s->enable.reg_u32[index] & ~old_data & s->isr.reg_u32[index];
for (i = 0; i < sizeof(data); i++) {
u8 mask = (data >> (i * 8)) & 0xff;
eiointc_enable_irq(vcpu, s, index * 4 + i, mask, 1);
}
/*
* 0: disable irq.
* update irq when isr is set.
*/
data = ~s->enable.reg_u32[index] & old_data & s->isr.reg_u32[index];
for (i = 0; i < sizeof(data); i++) {
u8 mask = (data >> (i * 8)) & 0xff;
eiointc_enable_irq(vcpu, s, index * 4 + i, mask, 0);
}
break;
case EIOINTC_BOUNCE_START ... EIOINTC_BOUNCE_END:
/* do not emulate hw bounced irq routing */
index = (offset - EIOINTC_BOUNCE_START) >> 2;
s->bounce.reg_u32[index] = data;
break;
case EIOINTC_COREISR_START ... EIOINTC_COREISR_END:
index = (offset - EIOINTC_COREISR_START) >> 2;
/* use attrs to get current cpu index */
cpu = vcpu->vcpu_id;
coreisr = data;
old_coreisr = s->coreisr.reg_u32[cpu][index];
/* write 1 to clear interrupt */
s->coreisr.reg_u32[cpu][index] = old_coreisr & ~coreisr;
coreisr &= old_coreisr;
bits = sizeof(data) * 8;
irq = find_first_bit((void *)&coreisr, bits);
while (irq < bits) {
eiointc_update_irq(s, irq + index * bits, 0);
bitmap_clear((void *)&coreisr, irq, 1);
irq = find_first_bit((void *)&coreisr, bits);
}
break;
case EIOINTC_COREMAP_START ... EIOINTC_COREMAP_END:
irq = offset - EIOINTC_COREMAP_START;
index = irq >> 2;
s->coremap.reg_u32[index] = data;
eiointc_update_sw_coremap(s, irq, data, sizeof(data), true);
break;
default:
ret = -EINVAL;
break;
}
return ret;
}
static int loongarch_eiointc_writeq(struct kvm_vcpu *vcpu,
struct loongarch_eiointc *s,
gpa_t addr, int len, const void *val)
{
int i, index, irq, bits, ret = 0;
u8 cpu;
u64 data, old_data;
u64 coreisr, old_coreisr;
gpa_t offset;
data = *(u64 *)val;
addr -= offset;
offset = addr - EIOINTC_BASE;
switch (offset) {
case EIOINTC_NODETYPE_START ... EIOINTC_NODETYPE_END:
index = (offset - EIOINTC_NODETYPE_START) >> 3;
s->nodetype.reg_u64[index] = data;
old = s->nodetype.reg_u64[index];
s->nodetype.reg_u64[index] = (old & ~mask) | data;
break;
case EIOINTC_IPMAP_START ... EIOINTC_IPMAP_END:
/*
* ipmap cannot be set at runtime, can be set only at the beginning
* of irqchip driver, need not update upper irq level
*/
index = (offset - EIOINTC_IPMAP_START) >> 3;
s->ipmap.reg_u64 = data;
old = s->ipmap.reg_u64;
s->ipmap.reg_u64 = (old & ~mask) | data;
break;
case EIOINTC_ENABLE_START ... EIOINTC_ENABLE_END:
index = (offset - EIOINTC_ENABLE_START) >> 3;
old_data = s->enable.reg_u64[index];
s->enable.reg_u64[index] = data;
old = s->enable.reg_u64[index];
s->enable.reg_u64[index] = (old & ~mask) | data;
/*
* 1: enable irq.
* update irq when isr is set.
*/
data = s->enable.reg_u64[index] & ~old_data & s->isr.reg_u64[index];
for (i = 0; i < sizeof(data); i++) {
u8 mask = (data >> (i * 8)) & 0xff;
eiointc_enable_irq(vcpu, s, index * 8 + i, mask, 1);
data = s->enable.reg_u64[index] & ~old & s->isr.reg_u64[index];
while (data) {
irq = __ffs(data);
eiointc_update_irq(s, irq + index * 64, 1);
data &= ~BIT_ULL(irq);
}
/*
* 0: disable irq.
* update irq when isr is set.
*/
data = ~s->enable.reg_u64[index] & old_data & s->isr.reg_u64[index];
for (i = 0; i < sizeof(data); i++) {
u8 mask = (data >> (i * 8)) & 0xff;
eiointc_enable_irq(vcpu, s, index * 8 + i, mask, 0);
data = ~s->enable.reg_u64[index] & old & s->isr.reg_u64[index];
while (data) {
irq = __ffs(data);
eiointc_update_irq(s, irq + index * 64, 0);
data &= ~BIT_ULL(irq);
}
break;
case EIOINTC_BOUNCE_START ... EIOINTC_BOUNCE_END:
/* do not emulate hw bounced irq routing */
index = (offset - EIOINTC_BOUNCE_START) >> 3;
s->bounce.reg_u64[index] = data;
old = s->bounce.reg_u64[index];
s->bounce.reg_u64[index] = (old & ~mask) | data;
break;
case EIOINTC_COREISR_START ... EIOINTC_COREISR_END:
index = (offset - EIOINTC_COREISR_START) >> 3;
/* use attrs to get current cpu index */
cpu = vcpu->vcpu_id;
coreisr = data;
old_coreisr = s->coreisr.reg_u64[cpu][index];
old = s->coreisr.reg_u64[cpu][index];
/* write 1 to clear interrupt */
s->coreisr.reg_u64[cpu][index] = old_coreisr & ~coreisr;
coreisr &= old_coreisr;
bits = sizeof(data) * 8;
irq = find_first_bit((void *)&coreisr, bits);
while (irq < bits) {
eiointc_update_irq(s, irq + index * bits, 0);
bitmap_clear((void *)&coreisr, irq, 1);
irq = find_first_bit((void *)&coreisr, bits);
s->coreisr.reg_u64[cpu][index] = old & ~data;
data &= old;
while (data) {
irq = __ffs(data);
eiointc_update_irq(s, irq + index * 64, 0);
data &= ~BIT_ULL(irq);
}
break;
case EIOINTC_COREMAP_START ... EIOINTC_COREMAP_END:
irq = offset - EIOINTC_COREMAP_START;
index = irq >> 3;
s->coremap.reg_u64[index] = data;
eiointc_update_sw_coremap(s, irq, data, sizeof(data), true);
index = (offset - EIOINTC_COREMAP_START) >> 3;
old = s->coremap.reg_u64[index];
s->coremap.reg_u64[index] = (old & ~mask) | data;
data = s->coremap.reg_u64[index];
eiointc_update_sw_coremap(s, index * 8, data, sizeof(data), true);
break;
default:
ret = -EINVAL;
@@ -684,7 +294,7 @@ static int kvm_eiointc_write(struct kvm_vcpu *vcpu,
gpa_t addr, int len, const void *val)
{
int ret = -EINVAL;
unsigned long flags;
unsigned long flags, value;
struct loongarch_eiointc *eiointc = vcpu->kvm->arch.eiointc;
if (!eiointc) {
@@ -697,24 +307,25 @@ static int kvm_eiointc_write(struct kvm_vcpu *vcpu,
return -EINVAL;
}
vcpu->kvm->stat.eiointc_write_exits++;
vcpu->stat.eiointc_write_exits++;
spin_lock_irqsave(&eiointc->lock, flags);
switch (len) {
case 1:
ret = loongarch_eiointc_writeb(vcpu, eiointc, addr, len, val);
value = *(unsigned char *)val;
ret = loongarch_eiointc_write(vcpu, eiointc, addr, value, 0xFF);
break;
case 2:
ret = loongarch_eiointc_writew(vcpu, eiointc, addr, len, val);
value = *(unsigned short *)val;
ret = loongarch_eiointc_write(vcpu, eiointc, addr, value, USHRT_MAX);
break;
case 4:
ret = loongarch_eiointc_writel(vcpu, eiointc, addr, len, val);
break;
case 8:
ret = loongarch_eiointc_writeq(vcpu, eiointc, addr, len, val);
value = *(unsigned int *)val;
ret = loongarch_eiointc_write(vcpu, eiointc, addr, value, UINT_MAX);
break;
default:
WARN_ONCE(1, "%s: Abnormal address access: addr 0x%llx, size %d\n",
__func__, addr, len);
value = *(unsigned long *)val;
ret = loongarch_eiointc_write(vcpu, eiointc, addr, value, ULONG_MAX);
break;
}
spin_unlock_irqrestore(&eiointc->lock, flags);
@@ -989,7 +600,7 @@ static int kvm_eiointc_create(struct kvm_device *dev, u32 type)
{
int ret;
struct loongarch_eiointc *s;
struct kvm_io_device *device, *device1;
struct kvm_io_device *device;
struct kvm *kvm = dev->kvm;
/* eiointc has been created */
@@ -1017,10 +628,10 @@ static int kvm_eiointc_create(struct kvm_device *dev, u32 type)
return ret;
}
device1 = &s->device_vext;
kvm_iodevice_init(device1, &kvm_eiointc_virt_ops);
device = &s->device_vext;
kvm_iodevice_init(device, &kvm_eiointc_virt_ops);
ret = kvm_io_bus_register_dev(kvm, KVM_IOCSR_BUS,
EIOINTC_VIRT_BASE, EIOINTC_VIRT_SIZE, device1);
EIOINTC_VIRT_BASE, EIOINTC_VIRT_SIZE, device);
if (ret < 0) {
kvm_io_bus_unregister_dev(kvm, KVM_IOCSR_BUS, &s->device);
kfree(s);
arch/loongarch/kvm/intc/ipi.c
+4 -24
View File
@@ -268,36 +268,16 @@ static int kvm_ipi_read(struct kvm_vcpu *vcpu,
struct kvm_io_device *dev,
gpa_t addr, int len, void *val)
{
int ret;
struct loongarch_ipi *ipi;
ipi = vcpu->kvm->arch.ipi;
if (!ipi) {
kvm_err("%s: ipi irqchip not valid!\n", __func__);
return -EINVAL;
}
ipi->kvm->stat.ipi_read_exits++;
ret = loongarch_ipi_readl(vcpu, addr, len, val);
return ret;
vcpu->stat.ipi_read_exits++;
return loongarch_ipi_readl(vcpu, addr, len, val);
}
static int kvm_ipi_write(struct kvm_vcpu *vcpu,
struct kvm_io_device *dev,
gpa_t addr, int len, const void *val)
{
int ret;
struct loongarch_ipi *ipi;
ipi = vcpu->kvm->arch.ipi;
if (!ipi) {
kvm_err("%s: ipi irqchip not valid!\n", __func__);
return -EINVAL;
}
ipi->kvm->stat.ipi_write_exits++;
ret = loongarch_ipi_writel(vcpu, addr, len, val);
return ret;
vcpu->stat.ipi_write_exits++;
return loongarch_ipi_writel(vcpu, addr, len, val);
}
static const struct kvm_io_device_ops kvm_ipi_ops = {
arch/loongarch/kvm/intc/pch_pic.c
+2 -2
View File
@@ -196,7 +196,7 @@ static int kvm_pch_pic_read(struct kvm_vcpu *vcpu,
}
/* statistics of pch pic reading */
vcpu->kvm->stat.pch_pic_read_exits++;
vcpu->stat.pch_pic_read_exits++;
ret = loongarch_pch_pic_read(s, addr, len, val);
return ret;
@@ -303,7 +303,7 @@ static int kvm_pch_pic_write(struct kvm_vcpu *vcpu,
}
/* statistics of pch pic writing */
vcpu->kvm->stat.pch_pic_write_exits++;
vcpu->stat.pch_pic_write_exits++;
ret = loongarch_pch_pic_write(s, addr, len, val);
return ret;
arch/loongarch/kvm/interrupt.c
+4 -21
View File
@@ -83,28 +83,11 @@ void kvm_deliver_intr(struct kvm_vcpu *vcpu)
unsigned long *pending = &vcpu->arch.irq_pending;
unsigned long *pending_clr = &vcpu->arch.irq_clear;
if (!(*pending) && !(*pending_clr))
return;
for_each_set_bit(priority, pending_clr, INT_IPI + 1)
kvm_irq_clear(vcpu, priority);
if (*pending_clr) {
priority = __ffs(*pending_clr);
while (priority <= INT_IPI) {
kvm_irq_clear(vcpu, priority);
priority = find_next_bit(pending_clr,
BITS_PER_BYTE * sizeof(*pending_clr),
priority + 1);
}
}
if (*pending) {
priority = __ffs(*pending);
while (priority <= INT_IPI) {
kvm_irq_deliver(vcpu, priority);
priority = find_next_bit(pending,
BITS_PER_BYTE * sizeof(*pending),
priority + 1);
}
}
for_each_set_bit(priority, pending, INT_IPI + 1)
kvm_irq_deliver(vcpu, priority);
}
int kvm_pending_timer(struct kvm_vcpu *vcpu)
arch/loongarch/kvm/trace.h
+13 -1
View File
@@ -46,11 +46,15 @@ DEFINE_EVENT(kvm_transition, kvm_out,
/* Further exit reasons */
#define KVM_TRACE_EXIT_IDLE 64
#define KVM_TRACE_EXIT_CACHE 65
#define KVM_TRACE_EXIT_CPUCFG 66
#define KVM_TRACE_EXIT_CSR 67
/* Tracepoints for VM exits */
#define kvm_trace_symbol_exit_types \
{ KVM_TRACE_EXIT_IDLE, "IDLE" }, \
{ KVM_TRACE_EXIT_CACHE, "CACHE" }
{ KVM_TRACE_EXIT_CACHE, "CACHE" }, \
{ KVM_TRACE_EXIT_CPUCFG, "CPUCFG" }, \
{ KVM_TRACE_EXIT_CSR, "CSR" }
DECLARE_EVENT_CLASS(kvm_exit,
TP_PROTO(struct kvm_vcpu *vcpu, unsigned int reason),
@@ -82,6 +86,14 @@ DEFINE_EVENT(kvm_exit, kvm_exit_cache,
TP_PROTO(struct kvm_vcpu *vcpu, unsigned int reason),
TP_ARGS(vcpu, reason));
DEFINE_EVENT(kvm_exit, kvm_exit_cpucfg,
TP_PROTO(struct kvm_vcpu *vcpu, unsigned int reason),
TP_ARGS(vcpu, reason));
DEFINE_EVENT(kvm_exit, kvm_exit_csr,
TP_PROTO(struct kvm_vcpu *vcpu, unsigned int reason),
TP_ARGS(vcpu, reason));
DEFINE_EVENT(kvm_exit, kvm_exit,
TP_PROTO(struct kvm_vcpu *vcpu, unsigned int reason),
TP_ARGS(vcpu, reason));
arch/loongarch/kvm/vcpu.c
+7 -1
View File
@@ -20,7 +20,13 @@ const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
STATS_DESC_COUNTER(VCPU, idle_exits),
STATS_DESC_COUNTER(VCPU, cpucfg_exits),
STATS_DESC_COUNTER(VCPU, signal_exits),
STATS_DESC_COUNTER(VCPU, hypercall_exits)
STATS_DESC_COUNTER(VCPU, hypercall_exits),
STATS_DESC_COUNTER(VCPU, ipi_read_exits),
STATS_DESC_COUNTER(VCPU, ipi_write_exits),
STATS_DESC_COUNTER(VCPU, eiointc_read_exits),
STATS_DESC_COUNTER(VCPU, eiointc_write_exits),
STATS_DESC_COUNTER(VCPU, pch_pic_read_exits),
STATS_DESC_COUNTER(VCPU, pch_pic_write_exits)
};
const struct kvm_stats_header kvm_vcpu_stats_header = {
arch/riscv/include/asm/kvm_aia.h
+1 -1
View File
@@ -150,7 +150,7 @@ int kvm_riscv_vcpu_aia_rmw_ireg(struct kvm_vcpu *vcpu, unsigned int csr_num,
int kvm_riscv_vcpu_aia_update(struct kvm_vcpu *vcpu);
void kvm_riscv_vcpu_aia_reset(struct kvm_vcpu *vcpu);
int kvm_riscv_vcpu_aia_init(struct kvm_vcpu *vcpu);
void kvm_riscv_vcpu_aia_init(struct kvm_vcpu *vcpu);
void kvm_riscv_vcpu_aia_deinit(struct kvm_vcpu *vcpu);
int kvm_riscv_aia_inject_msi_by_id(struct kvm *kvm, u32 hart_index,
arch/riscv/include/asm/kvm_gstage.h
+72
View File
@@ -0,0 +1,72 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2019 Western Digital Corporation or its affiliates.
* Copyright (c) 2025 Ventana Micro Systems Inc.
*/
#ifndef __RISCV_KVM_GSTAGE_H_
#define __RISCV_KVM_GSTAGE_H_
#include <linux/kvm_types.h>
struct kvm_gstage {
struct kvm *kvm;
unsigned long flags;
#define KVM_GSTAGE_FLAGS_LOCAL BIT(0)
unsigned long vmid;
pgd_t *pgd;
};
struct kvm_gstage_mapping {
gpa_t addr;
pte_t pte;
u32 level;
};
#ifdef CONFIG_64BIT
#define kvm_riscv_gstage_index_bits 9
#else
#define kvm_riscv_gstage_index_bits 10
#endif
extern unsigned long kvm_riscv_gstage_mode;
extern unsigned long kvm_riscv_gstage_pgd_levels;
#define kvm_riscv_gstage_pgd_xbits 2
#define kvm_riscv_gstage_pgd_size (1UL << (HGATP_PAGE_SHIFT + kvm_riscv_gstage_pgd_xbits))
#define kvm_riscv_gstage_gpa_bits (HGATP_PAGE_SHIFT + \
(kvm_riscv_gstage_pgd_levels * \
kvm_riscv_gstage_index_bits) + \
kvm_riscv_gstage_pgd_xbits)
#define kvm_riscv_gstage_gpa_size ((gpa_t)(1ULL << kvm_riscv_gstage_gpa_bits))
bool kvm_riscv_gstage_get_leaf(struct kvm_gstage *gstage, gpa_t addr,
pte_t **ptepp, u32 *ptep_level);
int kvm_riscv_gstage_set_pte(struct kvm_gstage *gstage,
struct kvm_mmu_memory_cache *pcache,
const struct kvm_gstage_mapping *map);
int kvm_riscv_gstage_map_page(struct kvm_gstage *gstage,
struct kvm_mmu_memory_cache *pcache,
gpa_t gpa, phys_addr_t hpa, unsigned long page_size,
bool page_rdonly, bool page_exec,
struct kvm_gstage_mapping *out_map);
enum kvm_riscv_gstage_op {
GSTAGE_OP_NOP = 0, /* Nothing */
GSTAGE_OP_CLEAR, /* Clear/Unmap */
GSTAGE_OP_WP, /* Write-protect */
};
void kvm_riscv_gstage_op_pte(struct kvm_gstage *gstage, gpa_t addr,
pte_t *ptep, u32 ptep_level, enum kvm_riscv_gstage_op op);
void kvm_riscv_gstage_unmap_range(struct kvm_gstage *gstage,
gpa_t start, gpa_t size, bool may_block);
void kvm_riscv_gstage_wp_range(struct kvm_gstage *gstage, gpa_t start, gpa_t end);
void kvm_riscv_gstage_mode_detect(void);
#endif
arch/riscv/include/asm/kvm_host.h
+5 -100
View File
@@ -16,6 +16,8 @@
#include <asm/hwcap.h>
#include <asm/kvm_aia.h>
#include <asm/ptrace.h>
#include <asm/kvm_tlb.h>
#include <asm/kvm_vmid.h>
#include <asm/kvm_vcpu_fp.h>
#include <asm/kvm_vcpu_insn.h>
#include <asm/kvm_vcpu_sbi.h>
@@ -36,14 +38,16 @@
#define KVM_REQ_UPDATE_HGATP KVM_ARCH_REQ(2)
#define KVM_REQ_FENCE_I \
KVM_ARCH_REQ_FLAGS(3, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
#define KVM_REQ_HFENCE_GVMA_VMID_ALL KVM_REQ_TLB_FLUSH
#define KVM_REQ_HFENCE_VVMA_ALL \
KVM_ARCH_REQ_FLAGS(4, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
#define KVM_REQ_HFENCE \
KVM_ARCH_REQ_FLAGS(5, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
#define KVM_REQ_STEAL_UPDATE KVM_ARCH_REQ(6)
#define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE
#define KVM_HEDELEG_DEFAULT (BIT(EXC_INST_MISALIGNED) | \
BIT(EXC_INST_ILLEGAL) | \
BIT(EXC_BREAKPOINT) | \
BIT(EXC_SYSCALL) | \
BIT(EXC_INST_PAGE_FAULT) | \
@@ -54,24 +58,6 @@
BIT(IRQ_VS_TIMER) | \
BIT(IRQ_VS_EXT))
enum kvm_riscv_hfence_type {
KVM_RISCV_HFENCE_UNKNOWN = 0,
KVM_RISCV_HFENCE_GVMA_VMID_GPA,
KVM_RISCV_HFENCE_VVMA_ASID_GVA,
KVM_RISCV_HFENCE_VVMA_ASID_ALL,
KVM_RISCV_HFENCE_VVMA_GVA,
};
struct kvm_riscv_hfence {
enum kvm_riscv_hfence_type type;
unsigned long asid;
unsigned long order;
gpa_t addr;
gpa_t size;
};
#define KVM_RISCV_VCPU_MAX_HFENCE 64
struct kvm_vm_stat {
struct kvm_vm_stat_generic generic;
};
@@ -97,15 +83,6 @@ struct kvm_vcpu_stat {
struct kvm_arch_memory_slot {
};
struct kvm_vmid {
/*
* Writes to vmid_version and vmid happen with vmid_lock held
* whereas reads happen without any lock held.
*/
unsigned long vmid_version;
unsigned long vmid;
};
struct kvm_arch {
/* G-stage vmid */
struct kvm_vmid vmid;
@@ -309,77 +286,6 @@ static inline bool kvm_arch_pmi_in_guest(struct kvm_vcpu *vcpu)
static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {}
static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {}
#define KVM_RISCV_GSTAGE_TLB_MIN_ORDER 12
void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid,
gpa_t gpa, gpa_t gpsz,
unsigned long order);
void kvm_riscv_local_hfence_gvma_vmid_all(unsigned long vmid);
void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz,
unsigned long order);
void kvm_riscv_local_hfence_gvma_all(void);
void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid,
unsigned long asid,
unsigned long gva,
unsigned long gvsz,
unsigned long order);
void kvm_riscv_local_hfence_vvma_asid_all(unsigned long vmid,
unsigned long asid);
void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid,
unsigned long gva, unsigned long gvsz,
unsigned long order);
void kvm_riscv_local_hfence_vvma_all(unsigned long vmid);
void kvm_riscv_local_tlb_sanitize(struct kvm_vcpu *vcpu);
void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu);
void kvm_riscv_hfence_gvma_vmid_all_process(struct kvm_vcpu *vcpu);
void kvm_riscv_hfence_vvma_all_process(struct kvm_vcpu *vcpu);
void kvm_riscv_hfence_process(struct kvm_vcpu *vcpu);
void kvm_riscv_fence_i(struct kvm *kvm,
unsigned long hbase, unsigned long hmask);
void kvm_riscv_hfence_gvma_vmid_gpa(struct kvm *kvm,
unsigned long hbase, unsigned long hmask,
gpa_t gpa, gpa_t gpsz,
unsigned long order);
void kvm_riscv_hfence_gvma_vmid_all(struct kvm *kvm,
unsigned long hbase, unsigned long hmask);
void kvm_riscv_hfence_vvma_asid_gva(struct kvm *kvm,
unsigned long hbase, unsigned long hmask,
unsigned long gva, unsigned long gvsz,
unsigned long order, unsigned long asid);
void kvm_riscv_hfence_vvma_asid_all(struct kvm *kvm,
unsigned long hbase, unsigned long hmask,
unsigned long asid);
void kvm_riscv_hfence_vvma_gva(struct kvm *kvm,
unsigned long hbase, unsigned long hmask,
unsigned long gva, unsigned long gvsz,
unsigned long order);
void kvm_riscv_hfence_vvma_all(struct kvm *kvm,
unsigned long hbase, unsigned long hmask);
int kvm_riscv_gstage_ioremap(struct kvm *kvm, gpa_t gpa,
phys_addr_t hpa, unsigned long size,
bool writable, bool in_atomic);
void kvm_riscv_gstage_iounmap(struct kvm *kvm, gpa_t gpa,
unsigned long size);
int kvm_riscv_gstage_map(struct kvm_vcpu *vcpu,
struct kvm_memory_slot *memslot,
gpa_t gpa, unsigned long hva, bool is_write);
int kvm_riscv_gstage_alloc_pgd(struct kvm *kvm);
void kvm_riscv_gstage_free_pgd(struct kvm *kvm);
void kvm_riscv_gstage_update_hgatp(struct kvm_vcpu *vcpu);
void __init kvm_riscv_gstage_mode_detect(void);
unsigned long __init kvm_riscv_gstage_mode(void);
int kvm_riscv_gstage_gpa_bits(void);
void __init kvm_riscv_gstage_vmid_detect(void);
unsigned long kvm_riscv_gstage_vmid_bits(void);
int kvm_riscv_gstage_vmid_init(struct kvm *kvm);
bool kvm_riscv_gstage_vmid_ver_changed(struct kvm_vmid *vmid);
void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu);
int kvm_riscv_setup_default_irq_routing(struct kvm *kvm, u32 lines);
void __kvm_riscv_unpriv_trap(void);
@@ -415,7 +321,6 @@ void __kvm_riscv_vcpu_power_on(struct kvm_vcpu *vcpu);
void kvm_riscv_vcpu_power_on(struct kvm_vcpu *vcpu);
bool kvm_riscv_vcpu_stopped(struct kvm_vcpu *vcpu);
void kvm_riscv_vcpu_sbi_sta_reset(struct kvm_vcpu *vcpu);
void kvm_riscv_vcpu_record_steal_time(struct kvm_vcpu *vcpu);
#endif /* __RISCV_KVM_HOST_H__ */
arch/riscv/include/asm/kvm_mmu.h
+21
View File
@@ -0,0 +1,21 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2025 Ventana Micro Systems Inc.
*/
#ifndef __RISCV_KVM_MMU_H_
#define __RISCV_KVM_MMU_H_
#include <asm/kvm_gstage.h>
int kvm_riscv_mmu_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa,
unsigned long size, bool writable, bool in_atomic);
void kvm_riscv_mmu_iounmap(struct kvm *kvm, gpa_t gpa, unsigned long size);
int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot,
gpa_t gpa, unsigned long hva, bool is_write,
struct kvm_gstage_mapping *out_map);
int kvm_riscv_mmu_alloc_pgd(struct kvm *kvm);
void kvm_riscv_mmu_free_pgd(struct kvm *kvm);
void kvm_riscv_mmu_update_hgatp(struct kvm_vcpu *vcpu);
#endif
arch/riscv/include/asm/kvm_tlb.h
+84
View File
@@ -0,0 +1,84 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2025 Ventana Micro Systems Inc.
*/
#ifndef __RISCV_KVM_TLB_H_
#define __RISCV_KVM_TLB_H_
#include <linux/kvm_types.h>
enum kvm_riscv_hfence_type {
KVM_RISCV_HFENCE_UNKNOWN = 0,
KVM_RISCV_HFENCE_GVMA_VMID_GPA,
KVM_RISCV_HFENCE_GVMA_VMID_ALL,
KVM_RISCV_HFENCE_VVMA_ASID_GVA,
KVM_RISCV_HFENCE_VVMA_ASID_ALL,
KVM_RISCV_HFENCE_VVMA_GVA,
KVM_RISCV_HFENCE_VVMA_ALL
};
struct kvm_riscv_hfence {
enum kvm_riscv_hfence_type type;
unsigned long asid;
unsigned long vmid;
unsigned long order;
gpa_t addr;
gpa_t size;
};
#define KVM_RISCV_VCPU_MAX_HFENCE 64
#define KVM_RISCV_GSTAGE_TLB_MIN_ORDER 12
void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid,
gpa_t gpa, gpa_t gpsz,
unsigned long order);
void kvm_riscv_local_hfence_gvma_vmid_all(unsigned long vmid);
void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz,
unsigned long order);
void kvm_riscv_local_hfence_gvma_all(void);
void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid,
unsigned long asid,
unsigned long gva,
unsigned long gvsz,
unsigned long order);
void kvm_riscv_local_hfence_vvma_asid_all(unsigned long vmid,
unsigned long asid);
void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid,
unsigned long gva, unsigned long gvsz,
unsigned long order);
void kvm_riscv_local_hfence_vvma_all(unsigned long vmid);
void kvm_riscv_tlb_flush_process(struct kvm_vcpu *vcpu);
void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu);
void kvm_riscv_hfence_vvma_all_process(struct kvm_vcpu *vcpu);
void kvm_riscv_hfence_process(struct kvm_vcpu *vcpu);
void kvm_riscv_fence_i(struct kvm *kvm,
unsigned long hbase, unsigned long hmask);
void kvm_riscv_hfence_gvma_vmid_gpa(struct kvm *kvm,
unsigned long hbase, unsigned long hmask,
gpa_t gpa, gpa_t gpsz,
unsigned long order, unsigned long vmid);
void kvm_riscv_hfence_gvma_vmid_all(struct kvm *kvm,
unsigned long hbase, unsigned long hmask,
unsigned long vmid);
void kvm_riscv_hfence_vvma_asid_gva(struct kvm *kvm,
unsigned long hbase, unsigned long hmask,
unsigned long gva, unsigned long gvsz,
unsigned long order, unsigned long asid,
unsigned long vmid);
void kvm_riscv_hfence_vvma_asid_all(struct kvm *kvm,
unsigned long hbase, unsigned long hmask,
unsigned long asid, unsigned long vmid);
void kvm_riscv_hfence_vvma_gva(struct kvm *kvm,
unsigned long hbase, unsigned long hmask,
unsigned long gva, unsigned long gvsz,
unsigned long order, unsigned long vmid);
void kvm_riscv_hfence_vvma_all(struct kvm *kvm,
unsigned long hbase, unsigned long hmask,
unsigned long vmid);
#endif
arch/riscv/include/asm/kvm_vcpu_sbi.h
+12
View File
@@ -49,6 +49,16 @@ struct kvm_vcpu_sbi_extension {
/* Extension specific probe function */
unsigned long (*probe)(struct kvm_vcpu *vcpu);
/*
* Init/deinit function called once during VCPU init/destroy. These
* might be use if the SBI extensions need to allocate or do specific
* init time only configuration.
*/
int (*init)(struct kvm_vcpu *vcpu);
void (*deinit)(struct kvm_vcpu *vcpu);
void (*reset)(struct kvm_vcpu *vcpu);
};
void kvm_riscv_vcpu_sbi_forward(struct kvm_vcpu *vcpu, struct kvm_run *run);
@@ -72,6 +82,8 @@ const struct kvm_vcpu_sbi_extension *kvm_vcpu_sbi_find_ext(
bool riscv_vcpu_supports_sbi_ext(struct kvm_vcpu *vcpu, int idx);
int kvm_riscv_vcpu_sbi_ecall(struct kvm_vcpu *vcpu, struct kvm_run *run);
void kvm_riscv_vcpu_sbi_init(struct kvm_vcpu *vcpu);
void kvm_riscv_vcpu_sbi_deinit(struct kvm_vcpu *vcpu);
void kvm_riscv_vcpu_sbi_reset(struct kvm_vcpu *vcpu);
int kvm_riscv_vcpu_get_reg_sbi_sta(struct kvm_vcpu *vcpu, unsigned long reg_num,
unsigned long *reg_val);
arch/riscv/include/asm/kvm_vmid.h
+27
View File
@@ -0,0 +1,27 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2025 Ventana Micro Systems Inc.
*/
#ifndef __RISCV_KVM_VMID_H_
#define __RISCV_KVM_VMID_H_
#include <linux/kvm_types.h>
struct kvm_vmid {
/*
* Writes to vmid_version and vmid happen with vmid_lock held
* whereas reads happen without any lock held.
*/
unsigned long vmid_version;
unsigned long vmid;
};
void __init kvm_riscv_gstage_vmid_detect(void);
unsigned long kvm_riscv_gstage_vmid_bits(void);
int kvm_riscv_gstage_vmid_init(struct kvm *kvm);
bool kvm_riscv_gstage_vmid_ver_changed(struct kvm_vmid *vmid);
void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu);
void kvm_riscv_gstage_vmid_sanitize(struct kvm_vcpu *vcpu);
#endif
arch/riscv/include/uapi/asm/kvm.h
+1
View File
@@ -18,6 +18,7 @@
#define __KVM_HAVE_IRQ_LINE
#define KVM_COALESCED_MMIO_PAGE_OFFSET 1
#define KVM_DIRTY_LOG_PAGE_OFFSET 64
#define KVM_INTERRUPT_SET -1U
#define KVM_INTERRUPT_UNSET -2U
arch/riscv/kvm/Kconfig
+1
View File
@@ -25,6 +25,7 @@ config KVM
select HAVE_KVM_MSI
select HAVE_KVM_VCPU_ASYNC_IOCTL
select HAVE_KVM_READONLY_MEM
select HAVE_KVM_DIRTY_RING_ACQ_REL
select KVM_COMMON
select KVM_GENERIC_DIRTYLOG_READ_PROTECT
select KVM_GENERIC_HARDWARE_ENABLING
arch/riscv/kvm/Makefile
+1
View File
@@ -14,6 +14,7 @@ kvm-y += aia.o
kvm-y += aia_aplic.o
kvm-y += aia_device.o
kvm-y += aia_imsic.o
kvm-y += gstage.o
kvm-y += main.o
kvm-y += mmu.o
kvm-y += nacl.o
arch/riscv/kvm/aia_device.c
+2 -4
View File
@@ -509,12 +509,12 @@ void kvm_riscv_vcpu_aia_reset(struct kvm_vcpu *vcpu)
kvm_riscv_vcpu_aia_imsic_reset(vcpu);
}
int kvm_riscv_vcpu_aia_init(struct kvm_vcpu *vcpu)
void kvm_riscv_vcpu_aia_init(struct kvm_vcpu *vcpu)
{
struct kvm_vcpu_aia *vaia = &vcpu->arch.aia_context;
if (!kvm_riscv_aia_available())
return 0;
return;
/*
* We don't do any memory allocations over here because these
@@ -526,8 +526,6 @@ int kvm_riscv_vcpu_aia_init(struct kvm_vcpu *vcpu)
/* Initialize default values in AIA vcpu context */
vaia->imsic_addr = KVM_RISCV_AIA_UNDEF_ADDR;
vaia->hart_index = vcpu->vcpu_idx;
return 0;
}
void kvm_riscv_vcpu_aia_deinit(struct kvm_vcpu *vcpu)
arch/riscv/kvm/aia_imsic.c
+6 -6
View File
@@ -16,6 +16,7 @@
#include <linux/swab.h>
#include <kvm/iodev.h>
#include <asm/csr.h>
#include <asm/kvm_mmu.h>
#define IMSIC_MAX_EIX (IMSIC_MAX_ID / BITS_PER_TYPE(u64))
@@ -745,9 +746,8 @@ void kvm_riscv_vcpu_aia_imsic_release(struct kvm_vcpu *vcpu)
*/
/* Purge the G-stage mapping */
kvm_riscv_gstage_iounmap(vcpu->kvm,
vcpu->arch.aia_context.imsic_addr,
IMSIC_MMIO_PAGE_SZ);
kvm_riscv_mmu_iounmap(vcpu->kvm, vcpu->arch.aia_context.imsic_addr,
IMSIC_MMIO_PAGE_SZ);
/* TODO: Purge the IOMMU mapping ??? */
@@ -830,9 +830,9 @@ int kvm_riscv_vcpu_aia_imsic_update(struct kvm_vcpu *vcpu)
imsic_vsfile_local_clear(new_vsfile_hgei, imsic->nr_hw_eix);
/* Update G-stage mapping for the new IMSIC VS-file */
ret = kvm_riscv_gstage_ioremap(kvm, vcpu->arch.aia_context.imsic_addr,
new_vsfile_pa, IMSIC_MMIO_PAGE_SZ,
true, true);
ret = kvm_riscv_mmu_ioremap(kvm, vcpu->arch.aia_context.imsic_addr,
new_vsfile_pa, IMSIC_MMIO_PAGE_SZ,
true, true);
if (ret)
goto fail_free_vsfile_hgei;
arch/riscv/kvm/gstage.c
+338
View File
@@ -0,0 +1,338 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2019 Western Digital Corporation or its affiliates.
* Copyright (c) 2025 Ventana Micro Systems Inc.
*/
#include <linux/bitops.h>
#include <linux/errno.h>
#include <linux/kvm_host.h>
#include <linux/module.h>
#include <linux/pgtable.h>
#include <asm/kvm_gstage.h>
#ifdef CONFIG_64BIT
unsigned long kvm_riscv_gstage_mode __ro_after_init = HGATP_MODE_SV39X4;
unsigned long kvm_riscv_gstage_pgd_levels __ro_after_init = 3;
#else
unsigned long kvm_riscv_gstage_mode __ro_after_init = HGATP_MODE_SV32X4;
unsigned long kvm_riscv_gstage_pgd_levels __ro_after_init = 2;
#endif
#define gstage_pte_leaf(__ptep) \
(pte_val(*(__ptep)) & (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC))
static inline unsigned long gstage_pte_index(gpa_t addr, u32 level)
{
unsigned long mask;
unsigned long shift = HGATP_PAGE_SHIFT + (kvm_riscv_gstage_index_bits * level);
if (level == (kvm_riscv_gstage_pgd_levels - 1))
mask = (PTRS_PER_PTE * (1UL << kvm_riscv_gstage_pgd_xbits)) - 1;
else
mask = PTRS_PER_PTE - 1;
return (addr >> shift) & mask;
}
static inline unsigned long gstage_pte_page_vaddr(pte_t pte)
{
return (unsigned long)pfn_to_virt(__page_val_to_pfn(pte_val(pte)));
}
static int gstage_page_size_to_level(unsigned long page_size, u32 *out_level)
{
u32 i;
unsigned long psz = 1UL << 12;
for (i = 0; i < kvm_riscv_gstage_pgd_levels; i++) {
if (page_size == (psz << (i * kvm_riscv_gstage_index_bits))) {
*out_level = i;
return 0;
}
}
return -EINVAL;
}
static int gstage_level_to_page_order(u32 level, unsigned long *out_pgorder)
{
if (kvm_riscv_gstage_pgd_levels < level)
return -EINVAL;
*out_pgorder = 12 + (level * kvm_riscv_gstage_index_bits);
return 0;
}
static int gstage_level_to_page_size(u32 level, unsigned long *out_pgsize)
{
int rc;
unsigned long page_order = PAGE_SHIFT;
rc = gstage_level_to_page_order(level, &page_order);
if (rc)
return rc;
*out_pgsize = BIT(page_order);
return 0;
}
bool kvm_riscv_gstage_get_leaf(struct kvm_gstage *gstage, gpa_t addr,
pte_t **ptepp, u32 *ptep_level)
{
pte_t *ptep;
u32 current_level = kvm_riscv_gstage_pgd_levels - 1;
*ptep_level = current_level;
ptep = (pte_t *)gstage->pgd;
ptep = &ptep[gstage_pte_index(addr, current_level)];
while (ptep && pte_val(ptep_get(ptep))) {
if (gstage_pte_leaf(ptep)) {
*ptep_level = current_level;
*ptepp = ptep;
return true;
}
if (current_level) {
current_level--;
*ptep_level = current_level;
ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep));
ptep = &ptep[gstage_pte_index(addr, current_level)];
} else {
ptep = NULL;
}
}
return false;
}
static void gstage_tlb_flush(struct kvm_gstage *gstage, u32 level, gpa_t addr)
{
unsigned long order = PAGE_SHIFT;
if (gstage_level_to_page_order(level, &order))
return;
addr &= ~(BIT(order) - 1);
if (gstage->flags & KVM_GSTAGE_FLAGS_LOCAL)
kvm_riscv_local_hfence_gvma_vmid_gpa(gstage->vmid, addr, BIT(order), order);
else
kvm_riscv_hfence_gvma_vmid_gpa(gstage->kvm, -1UL, 0, addr, BIT(order), order,
gstage->vmid);
}
int kvm_riscv_gstage_set_pte(struct kvm_gstage *gstage,
struct kvm_mmu_memory_cache *pcache,
const struct kvm_gstage_mapping *map)
{
u32 current_level = kvm_riscv_gstage_pgd_levels - 1;
pte_t *next_ptep = (pte_t *)gstage->pgd;
pte_t *ptep = &next_ptep[gstage_pte_index(map->addr, current_level)];
if (current_level < map->level)
return -EINVAL;
while (current_level != map->level) {
if (gstage_pte_leaf(ptep))
return -EEXIST;
if (!pte_val(ptep_get(ptep))) {
if (!pcache)
return -ENOMEM;
next_ptep = kvm_mmu_memory_cache_alloc(pcache);
if (!next_ptep)
return -ENOMEM;
set_pte(ptep, pfn_pte(PFN_DOWN(__pa(next_ptep)),
__pgprot(_PAGE_TABLE)));
} else {
if (gstage_pte_leaf(ptep))
return -EEXIST;
next_ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep));
}
current_level--;
ptep = &next_ptep[gstage_pte_index(map->addr, current_level)];
}
if (pte_val(*ptep) != pte_val(map->pte)) {
set_pte(ptep, map->pte);
if (gstage_pte_leaf(ptep))
gstage_tlb_flush(gstage, current_level, map->addr);
}
return 0;
}
int kvm_riscv_gstage_map_page(struct kvm_gstage *gstage,
struct kvm_mmu_memory_cache *pcache,
gpa_t gpa, phys_addr_t hpa, unsigned long page_size,
bool page_rdonly, bool page_exec,
struct kvm_gstage_mapping *out_map)
{
pgprot_t prot;
int ret;
out_map->addr = gpa;
out_map->level = 0;
ret = gstage_page_size_to_level(page_size, &out_map->level);
if (ret)
return ret;
/*
* A RISC-V implementation can choose to either:
* 1) Update 'A' and 'D' PTE bits in hardware
* 2) Generate page fault when 'A' and/or 'D' bits are not set
* PTE so that software can update these bits.
*
* We support both options mentioned above. To achieve this, we
* always set 'A' and 'D' PTE bits at time of creating G-stage
* mapping. To support KVM dirty page logging with both options
* mentioned above, we will write-protect G-stage PTEs to track
* dirty pages.
*/
if (page_exec) {
if (page_rdonly)
prot = PAGE_READ_EXEC;
else
prot = PAGE_WRITE_EXEC;
} else {
if (page_rdonly)
prot = PAGE_READ;
else
prot = PAGE_WRITE;
}
out_map->pte = pfn_pte(PFN_DOWN(hpa), prot);
out_map->pte = pte_mkdirty(out_map->pte);
return kvm_riscv_gstage_set_pte(gstage, pcache, out_map);
}
void kvm_riscv_gstage_op_pte(struct kvm_gstage *gstage, gpa_t addr,
pte_t *ptep, u32 ptep_level, enum kvm_riscv_gstage_op op)
{
int i, ret;
pte_t old_pte, *next_ptep;
u32 next_ptep_level;
unsigned long next_page_size, page_size;
ret = gstage_level_to_page_size(ptep_level, &page_size);
if (ret)
return;
WARN_ON(addr & (page_size - 1));
if (!pte_val(ptep_get(ptep)))
return;
if (ptep_level && !gstage_pte_leaf(ptep)) {
next_ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep));
next_ptep_level = ptep_level - 1;
ret = gstage_level_to_page_size(next_ptep_level, &next_page_size);
if (ret)
return;
if (op == GSTAGE_OP_CLEAR)
set_pte(ptep, __pte(0));
for (i = 0; i < PTRS_PER_PTE; i++)
kvm_riscv_gstage_op_pte(gstage, addr + i * next_page_size,
&next_ptep[i], next_ptep_level, op);
if (op == GSTAGE_OP_CLEAR)
put_page(virt_to_page(next_ptep));
} else {
old_pte = *ptep;
if (op == GSTAGE_OP_CLEAR)
set_pte(ptep, __pte(0));
else if (op == GSTAGE_OP_WP)
set_pte(ptep, __pte(pte_val(ptep_get(ptep)) & ~_PAGE_WRITE));
if (pte_val(*ptep) != pte_val(old_pte))
gstage_tlb_flush(gstage, ptep_level, addr);
}
}
void kvm_riscv_gstage_unmap_range(struct kvm_gstage *gstage,
gpa_t start, gpa_t size, bool may_block)
{
int ret;
pte_t *ptep;
u32 ptep_level;
bool found_leaf;
unsigned long page_size;
gpa_t addr = start, end = start + size;
while (addr < end) {
found_leaf = kvm_riscv_gstage_get_leaf(gstage, addr, &ptep, &ptep_level);
ret = gstage_level_to_page_size(ptep_level, &page_size);
if (ret)
break;
if (!found_leaf)
goto next;
if (!(addr & (page_size - 1)) && ((end - addr) >= page_size))
kvm_riscv_gstage_op_pte(gstage, addr, ptep,
ptep_level, GSTAGE_OP_CLEAR);
next:
addr += page_size;
/*
* If the range is too large, release the kvm->mmu_lock
* to prevent starvation and lockup detector warnings.
*/
if (!(gstage->flags & KVM_GSTAGE_FLAGS_LOCAL) && may_block && addr < end)
cond_resched_lock(&gstage->kvm->mmu_lock);
}
}
void kvm_riscv_gstage_wp_range(struct kvm_gstage *gstage, gpa_t start, gpa_t end)
{
int ret;
pte_t *ptep;
u32 ptep_level;
bool found_leaf;
gpa_t addr = start;
unsigned long page_size;
while (addr < end) {
found_leaf = kvm_riscv_gstage_get_leaf(gstage, addr, &ptep, &ptep_level);
ret = gstage_level_to_page_size(ptep_level, &page_size);
if (ret)
break;
if (!found_leaf)
goto next;
if (!(addr & (page_size - 1)) && ((end - addr) >= page_size))
kvm_riscv_gstage_op_pte(gstage, addr, ptep,
ptep_level, GSTAGE_OP_WP);
next:
addr += page_size;
}
}
void __init kvm_riscv_gstage_mode_detect(void)
{
#ifdef CONFIG_64BIT
/* Try Sv57x4 G-stage mode */
csr_write(CSR_HGATP, HGATP_MODE_SV57X4 << HGATP_MODE_SHIFT);
if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV57X4) {
kvm_riscv_gstage_mode = HGATP_MODE_SV57X4;
kvm_riscv_gstage_pgd_levels = 5;
goto skip_sv48x4_test;
}
/* Try Sv48x4 G-stage mode */
csr_write(CSR_HGATP, HGATP_MODE_SV48X4 << HGATP_MODE_SHIFT);
if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV48X4) {
kvm_riscv_gstage_mode = HGATP_MODE_SV48X4;
kvm_riscv_gstage_pgd_levels = 4;
}
skip_sv48x4_test:
csr_write(CSR_HGATP, 0);
kvm_riscv_local_hfence_gvma_all();
#endif
}
arch/riscv/kvm/main.c
+2 -1
View File
@@ -11,6 +11,7 @@
#include <linux/module.h>
#include <linux/kvm_host.h>
#include <asm/cpufeature.h>
#include <asm/kvm_mmu.h>
#include <asm/kvm_nacl.h>
#include <asm/sbi.h>
@@ -134,7 +135,7 @@ static int __init riscv_kvm_init(void)
(rc) ? slist : "no features");
}
switch (kvm_riscv_gstage_mode()) {
switch (kvm_riscv_gstage_mode) {
case HGATP_MODE_SV32X4:
str = "Sv32x4";
break;
arch/riscv/kvm/mmu.c
+112 -397
View File
@@ -6,348 +6,38 @@
* Anup Patel <anup.patel@wdc.com>
*/
#include <linux/bitops.h>
#include <linux/errno.h>
#include <linux/err.h>
#include <linux/hugetlb.h>
#include <linux/module.h>
#include <linux/uaccess.h>
#include <linux/vmalloc.h>
#include <linux/kvm_host.h>
#include <linux/sched/signal.h>
#include <asm/kvm_mmu.h>
#include <asm/kvm_nacl.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#ifdef CONFIG_64BIT
static unsigned long gstage_mode __ro_after_init = (HGATP_MODE_SV39X4 << HGATP_MODE_SHIFT);
static unsigned long gstage_pgd_levels __ro_after_init = 3;
#define gstage_index_bits 9
#else
static unsigned long gstage_mode __ro_after_init = (HGATP_MODE_SV32X4 << HGATP_MODE_SHIFT);
static unsigned long gstage_pgd_levels __ro_after_init = 2;
#define gstage_index_bits 10
#endif
#define gstage_pgd_xbits 2
#define gstage_pgd_size (1UL << (HGATP_PAGE_SHIFT + gstage_pgd_xbits))
#define gstage_gpa_bits (HGATP_PAGE_SHIFT + \
(gstage_pgd_levels * gstage_index_bits) + \
gstage_pgd_xbits)
#define gstage_gpa_size ((gpa_t)(1ULL << gstage_gpa_bits))
#define gstage_pte_leaf(__ptep) \
(pte_val(*(__ptep)) & (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC))
static inline unsigned long gstage_pte_index(gpa_t addr, u32 level)
{
unsigned long mask;
unsigned long shift = HGATP_PAGE_SHIFT + (gstage_index_bits * level);
if (level == (gstage_pgd_levels - 1))
mask = (PTRS_PER_PTE * (1UL << gstage_pgd_xbits)) - 1;
else
mask = PTRS_PER_PTE - 1;
return (addr >> shift) & mask;
}
static inline unsigned long gstage_pte_page_vaddr(pte_t pte)
{
return (unsigned long)pfn_to_virt(__page_val_to_pfn(pte_val(pte)));
}
static int gstage_page_size_to_level(unsigned long page_size, u32 *out_level)
{
u32 i;
unsigned long psz = 1UL << 12;
for (i = 0; i < gstage_pgd_levels; i++) {
if (page_size == (psz << (i * gstage_index_bits))) {
*out_level = i;
return 0;
}
}
return -EINVAL;
}
static int gstage_level_to_page_order(u32 level, unsigned long *out_pgorder)
{
if (gstage_pgd_levels < level)
return -EINVAL;
*out_pgorder = 12 + (level * gstage_index_bits);
return 0;
}
static int gstage_level_to_page_size(u32 level, unsigned long *out_pgsize)
{
int rc;
unsigned long page_order = PAGE_SHIFT;
rc = gstage_level_to_page_order(level, &page_order);
if (rc)
return rc;
*out_pgsize = BIT(page_order);
return 0;
}
static bool gstage_get_leaf_entry(struct kvm *kvm, gpa_t addr,
pte_t **ptepp, u32 *ptep_level)
{
pte_t *ptep;
u32 current_level = gstage_pgd_levels - 1;
*ptep_level = current_level;
ptep = (pte_t *)kvm->arch.pgd;
ptep = &ptep[gstage_pte_index(addr, current_level)];
while (ptep && pte_val(ptep_get(ptep))) {
if (gstage_pte_leaf(ptep)) {
*ptep_level = current_level;
*ptepp = ptep;
return true;
}
if (current_level) {
current_level--;
*ptep_level = current_level;
ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep));
ptep = &ptep[gstage_pte_index(addr, current_level)];
} else {
ptep = NULL;
}
}
return false;
}
static void gstage_remote_tlb_flush(struct kvm *kvm, u32 level, gpa_t addr)
{
unsigned long order = PAGE_SHIFT;
if (gstage_level_to_page_order(level, &order))
return;
addr &= ~(BIT(order) - 1);
kvm_riscv_hfence_gvma_vmid_gpa(kvm, -1UL, 0, addr, BIT(order), order);
}
static int gstage_set_pte(struct kvm *kvm, u32 level,
struct kvm_mmu_memory_cache *pcache,
gpa_t addr, const pte_t *new_pte)
{
u32 current_level = gstage_pgd_levels - 1;
pte_t *next_ptep = (pte_t *)kvm->arch.pgd;
pte_t *ptep = &next_ptep[gstage_pte_index(addr, current_level)];
if (current_level < level)
return -EINVAL;
while (current_level != level) {
if (gstage_pte_leaf(ptep))
return -EEXIST;
if (!pte_val(ptep_get(ptep))) {
if (!pcache)
return -ENOMEM;
next_ptep = kvm_mmu_memory_cache_alloc(pcache);
if (!next_ptep)
return -ENOMEM;
set_pte(ptep, pfn_pte(PFN_DOWN(__pa(next_ptep)),
__pgprot(_PAGE_TABLE)));
} else {
if (gstage_pte_leaf(ptep))
return -EEXIST;
next_ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep));
}
current_level--;
ptep = &next_ptep[gstage_pte_index(addr, current_level)];
}
set_pte(ptep, *new_pte);
if (gstage_pte_leaf(ptep))
gstage_remote_tlb_flush(kvm, current_level, addr);
return 0;
}
static int gstage_map_page(struct kvm *kvm,
struct kvm_mmu_memory_cache *pcache,
gpa_t gpa, phys_addr_t hpa,
unsigned long page_size,
bool page_rdonly, bool page_exec)
{
int ret;
u32 level = 0;
pte_t new_pte;
pgprot_t prot;
ret = gstage_page_size_to_level(page_size, &level);
if (ret)
return ret;
/*
* A RISC-V implementation can choose to either:
* 1) Update 'A' and 'D' PTE bits in hardware
* 2) Generate page fault when 'A' and/or 'D' bits are not set
* PTE so that software can update these bits.
*
* We support both options mentioned above. To achieve this, we
* always set 'A' and 'D' PTE bits at time of creating G-stage
* mapping. To support KVM dirty page logging with both options
* mentioned above, we will write-protect G-stage PTEs to track
* dirty pages.
*/
if (page_exec) {
if (page_rdonly)
prot = PAGE_READ_EXEC;
else
prot = PAGE_WRITE_EXEC;
} else {
if (page_rdonly)
prot = PAGE_READ;
else
prot = PAGE_WRITE;
}
new_pte = pfn_pte(PFN_DOWN(hpa), prot);
new_pte = pte_mkdirty(new_pte);
return gstage_set_pte(kvm, level, pcache, gpa, &new_pte);
}
enum gstage_op {
GSTAGE_OP_NOP = 0, /* Nothing */
GSTAGE_OP_CLEAR, /* Clear/Unmap */
GSTAGE_OP_WP, /* Write-protect */
};
static void gstage_op_pte(struct kvm *kvm, gpa_t addr,
pte_t *ptep, u32 ptep_level, enum gstage_op op)
{
int i, ret;
pte_t *next_ptep;
u32 next_ptep_level;
unsigned long next_page_size, page_size;
ret = gstage_level_to_page_size(ptep_level, &page_size);
if (ret)
return;
BUG_ON(addr & (page_size - 1));
if (!pte_val(ptep_get(ptep)))
return;
if (ptep_level && !gstage_pte_leaf(ptep)) {
next_ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep));
next_ptep_level = ptep_level - 1;
ret = gstage_level_to_page_size(next_ptep_level,
&next_page_size);
if (ret)
return;
if (op == GSTAGE_OP_CLEAR)
set_pte(ptep, __pte(0));
for (i = 0; i < PTRS_PER_PTE; i++)
gstage_op_pte(kvm, addr + i * next_page_size,
&next_ptep[i], next_ptep_level, op);
if (op == GSTAGE_OP_CLEAR)
put_page(virt_to_page(next_ptep));
} else {
if (op == GSTAGE_OP_CLEAR)
set_pte(ptep, __pte(0));
else if (op == GSTAGE_OP_WP)
set_pte(ptep, __pte(pte_val(ptep_get(ptep)) & ~_PAGE_WRITE));
gstage_remote_tlb_flush(kvm, ptep_level, addr);
}
}
static void gstage_unmap_range(struct kvm *kvm, gpa_t start,
gpa_t size, bool may_block)
{
int ret;
pte_t *ptep;
u32 ptep_level;
bool found_leaf;
unsigned long page_size;
gpa_t addr = start, end = start + size;
while (addr < end) {
found_leaf = gstage_get_leaf_entry(kvm, addr,
&ptep, &ptep_level);
ret = gstage_level_to_page_size(ptep_level, &page_size);
if (ret)
break;
if (!found_leaf)
goto next;
if (!(addr & (page_size - 1)) && ((end - addr) >= page_size))
gstage_op_pte(kvm, addr, ptep,
ptep_level, GSTAGE_OP_CLEAR);
next:
addr += page_size;
/*
* If the range is too large, release the kvm->mmu_lock
* to prevent starvation and lockup detector warnings.
*/
if (may_block && addr < end)
cond_resched_lock(&kvm->mmu_lock);
}
}
static void gstage_wp_range(struct kvm *kvm, gpa_t start, gpa_t end)
{
int ret;
pte_t *ptep;
u32 ptep_level;
bool found_leaf;
gpa_t addr = start;
unsigned long page_size;
while (addr < end) {
found_leaf = gstage_get_leaf_entry(kvm, addr,
&ptep, &ptep_level);
ret = gstage_level_to_page_size(ptep_level, &page_size);
if (ret)
break;
if (!found_leaf)
goto next;
if (!(addr & (page_size - 1)) && ((end - addr) >= page_size))
gstage_op_pte(kvm, addr, ptep,
ptep_level, GSTAGE_OP_WP);
next:
addr += page_size;
}
}
static void gstage_wp_memory_region(struct kvm *kvm, int slot)
static void mmu_wp_memory_region(struct kvm *kvm, int slot)
{
struct kvm_memslots *slots = kvm_memslots(kvm);
struct kvm_memory_slot *memslot = id_to_memslot(slots, slot);
phys_addr_t start = memslot->base_gfn << PAGE_SHIFT;
phys_addr_t end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT;
struct kvm_gstage gstage;
gstage.kvm = kvm;
gstage.flags = 0;
gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
gstage.pgd = kvm->arch.pgd;
spin_lock(&kvm->mmu_lock);
gstage_wp_range(kvm, start, end);
kvm_riscv_gstage_wp_range(&gstage, start, end);
spin_unlock(&kvm->mmu_lock);
kvm_flush_remote_tlbs(kvm);
kvm_flush_remote_tlbs_memslot(kvm, memslot);
}
int kvm_riscv_gstage_ioremap(struct kvm *kvm, gpa_t gpa,
phys_addr_t hpa, unsigned long size,
bool writable, bool in_atomic)
int kvm_riscv_mmu_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa,
unsigned long size, bool writable, bool in_atomic)
{
pte_t pte;
int ret = 0;
unsigned long pfn;
phys_addr_t addr, end;
@@ -355,22 +45,31 @@ int kvm_riscv_gstage_ioremap(struct kvm *kvm, gpa_t gpa,
.gfp_custom = (in_atomic) ? GFP_ATOMIC | __GFP_ACCOUNT : 0,
.gfp_zero = __GFP_ZERO,
};
struct kvm_gstage_mapping map;
struct kvm_gstage gstage;
gstage.kvm = kvm;
gstage.flags = 0;
gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
gstage.pgd = kvm->arch.pgd;
end = (gpa + size + PAGE_SIZE - 1) & PAGE_MASK;
pfn = __phys_to_pfn(hpa);
for (addr = gpa; addr < end; addr += PAGE_SIZE) {
pte = pfn_pte(pfn, PAGE_KERNEL_IO);
map.addr = addr;
map.pte = pfn_pte(pfn, PAGE_KERNEL_IO);
map.level = 0;
if (!writable)
pte = pte_wrprotect(pte);
map.pte = pte_wrprotect(map.pte);
ret = kvm_mmu_topup_memory_cache(&pcache, gstage_pgd_levels);
ret = kvm_mmu_topup_memory_cache(&pcache, kvm_riscv_gstage_pgd_levels);
if (ret)
goto out;
spin_lock(&kvm->mmu_lock);
ret = gstage_set_pte(kvm, 0, &pcache, addr, &pte);
ret = kvm_riscv_gstage_set_pte(&gstage, &pcache, &map);
spin_unlock(&kvm->mmu_lock);
if (ret)
goto out;
@@ -383,10 +82,17 @@ out:
return ret;
}
void kvm_riscv_gstage_iounmap(struct kvm *kvm, gpa_t gpa, unsigned long size)
void kvm_riscv_mmu_iounmap(struct kvm *kvm, gpa_t gpa, unsigned long size)
{
struct kvm_gstage gstage;
gstage.kvm = kvm;
gstage.flags = 0;
gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
gstage.pgd = kvm->arch.pgd;
spin_lock(&kvm->mmu_lock);
gstage_unmap_range(kvm, gpa, size, false);
kvm_riscv_gstage_unmap_range(&gstage, gpa, size, false);
spin_unlock(&kvm->mmu_lock);
}
@@ -398,8 +104,14 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
phys_addr_t base_gfn = slot->base_gfn + gfn_offset;
phys_addr_t start = (base_gfn + __ffs(mask)) << PAGE_SHIFT;
phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT;
struct kvm_gstage gstage;
gstage_wp_range(kvm, start, end);
gstage.kvm = kvm;
gstage.flags = 0;
gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
gstage.pgd = kvm->arch.pgd;
kvm_riscv_gstage_wp_range(&gstage, start, end);
}
void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
@@ -416,7 +128,7 @@ void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen)
void kvm_arch_flush_shadow_all(struct kvm *kvm)
{
kvm_riscv_gstage_free_pgd(kvm);
kvm_riscv_mmu_free_pgd(kvm);
}
void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
@@ -424,9 +136,15 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
{
gpa_t gpa = slot->base_gfn << PAGE_SHIFT;
phys_addr_t size = slot->npages << PAGE_SHIFT;
struct kvm_gstage gstage;
gstage.kvm = kvm;
gstage.flags = 0;
gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
gstage.pgd = kvm->arch.pgd;
spin_lock(&kvm->mmu_lock);
gstage_unmap_range(kvm, gpa, size, false);
kvm_riscv_gstage_unmap_range(&gstage, gpa, size, false);
spin_unlock(&kvm->mmu_lock);
}
@@ -441,7 +159,7 @@ void kvm_arch_commit_memory_region(struct kvm *kvm,
* the memory slot is write protected.
*/
if (change != KVM_MR_DELETE && new->flags & KVM_MEM_LOG_DIRTY_PAGES)
gstage_wp_memory_region(kvm, new->id);
mmu_wp_memory_region(kvm, new->id);
}
int kvm_arch_prepare_memory_region(struct kvm *kvm,
@@ -463,7 +181,7 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
* space addressable by the KVM guest GPA space.
*/
if ((new->base_gfn + new->npages) >=
(gstage_gpa_size >> PAGE_SHIFT))
(kvm_riscv_gstage_gpa_size >> PAGE_SHIFT))
return -EFAULT;
hva = new->userspace_addr;
@@ -487,10 +205,11 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
* +--------------------------------------------+
*/
do {
struct vm_area_struct *vma = find_vma(current->mm, hva);
struct vm_area_struct *vma;
hva_t vm_start, vm_end;
if (!vma || vma->vm_start >= reg_end)
vma = find_vma_intersection(current->mm, hva, reg_end);
if (!vma)
break;
/*
@@ -519,9 +238,8 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
goto out;
}
ret = kvm_riscv_gstage_ioremap(kvm, gpa, pa,
vm_end - vm_start,
writable, false);
ret = kvm_riscv_mmu_ioremap(kvm, gpa, pa, vm_end - vm_start,
writable, false);
if (ret)
break;
}
@@ -532,7 +250,7 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
goto out;
if (ret)
kvm_riscv_gstage_iounmap(kvm, base_gpa, size);
kvm_riscv_mmu_iounmap(kvm, base_gpa, size);
out:
mmap_read_unlock(current->mm);
@@ -541,12 +259,18 @@ out:
bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
{
struct kvm_gstage gstage;
if (!kvm->arch.pgd)
return false;
gstage_unmap_range(kvm, range->start << PAGE_SHIFT,
(range->end - range->start) << PAGE_SHIFT,
range->may_block);
gstage.kvm = kvm;
gstage.flags = 0;
gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
gstage.pgd = kvm->arch.pgd;
kvm_riscv_gstage_unmap_range(&gstage, range->start << PAGE_SHIFT,
(range->end - range->start) << PAGE_SHIFT,
range->may_block);
return false;
}
@@ -555,14 +279,19 @@ bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
pte_t *ptep;
u32 ptep_level = 0;
u64 size = (range->end - range->start) << PAGE_SHIFT;
struct kvm_gstage gstage;
if (!kvm->arch.pgd)
return false;
WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE);
if (!gstage_get_leaf_entry(kvm, range->start << PAGE_SHIFT,
&ptep, &ptep_level))
gstage.kvm = kvm;
gstage.flags = 0;
gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
gstage.pgd = kvm->arch.pgd;
if (!kvm_riscv_gstage_get_leaf(&gstage, range->start << PAGE_SHIFT,
&ptep, &ptep_level))
return false;
return ptep_test_and_clear_young(NULL, 0, ptep);
@@ -573,22 +302,27 @@ bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
pte_t *ptep;
u32 ptep_level = 0;
u64 size = (range->end - range->start) << PAGE_SHIFT;
struct kvm_gstage gstage;
if (!kvm->arch.pgd)
return false;
WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE);
if (!gstage_get_leaf_entry(kvm, range->start << PAGE_SHIFT,
&ptep, &ptep_level))
gstage.kvm = kvm;
gstage.flags = 0;
gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
gstage.pgd = kvm->arch.pgd;
if (!kvm_riscv_gstage_get_leaf(&gstage, range->start << PAGE_SHIFT,
&ptep, &ptep_level))
return false;
return pte_young(ptep_get(ptep));
}
int kvm_riscv_gstage_map(struct kvm_vcpu *vcpu,
struct kvm_memory_slot *memslot,
gpa_t gpa, unsigned long hva, bool is_write)
int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot,
gpa_t gpa, unsigned long hva, bool is_write,
struct kvm_gstage_mapping *out_map)
{
int ret;
kvm_pfn_t hfn;
@@ -601,10 +335,19 @@ int kvm_riscv_gstage_map(struct kvm_vcpu *vcpu,
bool logging = (memslot->dirty_bitmap &&
!(memslot->flags & KVM_MEM_READONLY)) ? true : false;
unsigned long vma_pagesize, mmu_seq;
struct kvm_gstage gstage;
struct page *page;
gstage.kvm = kvm;
gstage.flags = 0;
gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
gstage.pgd = kvm->arch.pgd;
/* Setup initial state of output mapping */
memset(out_map, 0, sizeof(*out_map));
/* We need minimum second+third level pages */
ret = kvm_mmu_topup_memory_cache(pcache, gstage_pgd_levels);
ret = kvm_mmu_topup_memory_cache(pcache, kvm_riscv_gstage_pgd_levels);
if (ret) {
kvm_err("Failed to topup G-stage cache\n");
return ret;
@@ -648,7 +391,8 @@ int kvm_riscv_gstage_map(struct kvm_vcpu *vcpu,
return -EFAULT;
}
hfn = kvm_faultin_pfn(vcpu, gfn, is_write, &writable, &page);
hfn = __kvm_faultin_pfn(memslot, gfn, is_write ? FOLL_WRITE : 0,
&writable, &page);
if (hfn == KVM_PFN_ERR_HWPOISON) {
send_sig_mceerr(BUS_MCEERR_AR, (void __user *)hva,
vma_pageshift, current);
@@ -670,12 +414,12 @@ int kvm_riscv_gstage_map(struct kvm_vcpu *vcpu,
goto out_unlock;
if (writable) {
mark_page_dirty(kvm, gfn);
ret = gstage_map_page(kvm, pcache, gpa, hfn << PAGE_SHIFT,
vma_pagesize, false, true);
mark_page_dirty_in_slot(kvm, memslot, gfn);
ret = kvm_riscv_gstage_map_page(&gstage, pcache, gpa, hfn << PAGE_SHIFT,
vma_pagesize, false, true, out_map);
} else {
ret = gstage_map_page(kvm, pcache, gpa, hfn << PAGE_SHIFT,
vma_pagesize, true, true);
ret = kvm_riscv_gstage_map_page(&gstage, pcache, gpa, hfn << PAGE_SHIFT,
vma_pagesize, true, true, out_map);
}
if (ret)
@@ -687,7 +431,7 @@ out_unlock:
return ret;
}
int kvm_riscv_gstage_alloc_pgd(struct kvm *kvm)
int kvm_riscv_mmu_alloc_pgd(struct kvm *kvm)
{
struct page *pgd_page;
@@ -697,7 +441,7 @@ int kvm_riscv_gstage_alloc_pgd(struct kvm *kvm)
}
pgd_page = alloc_pages(GFP_KERNEL | __GFP_ZERO,
get_order(gstage_pgd_size));
get_order(kvm_riscv_gstage_pgd_size));
if (!pgd_page)
return -ENOMEM;
kvm->arch.pgd = page_to_virt(pgd_page);
@@ -706,13 +450,18 @@ int kvm_riscv_gstage_alloc_pgd(struct kvm *kvm)
return 0;
}
void kvm_riscv_gstage_free_pgd(struct kvm *kvm)
void kvm_riscv_mmu_free_pgd(struct kvm *kvm)
{
struct kvm_gstage gstage;
void *pgd = NULL;
spin_lock(&kvm->mmu_lock);
if (kvm->arch.pgd) {
gstage_unmap_range(kvm, 0UL, gstage_gpa_size, false);
gstage.kvm = kvm;
gstage.flags = 0;
gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
gstage.pgd = kvm->arch.pgd;
kvm_riscv_gstage_unmap_range(&gstage, 0UL, kvm_riscv_gstage_gpa_size, false);
pgd = READ_ONCE(kvm->arch.pgd);
kvm->arch.pgd = NULL;
kvm->arch.pgd_phys = 0;
@@ -720,12 +469,12 @@ void kvm_riscv_gstage_free_pgd(struct kvm *kvm)
spin_unlock(&kvm->mmu_lock);
if (pgd)
free_pages((unsigned long)pgd, get_order(gstage_pgd_size));
free_pages((unsigned long)pgd, get_order(kvm_riscv_gstage_pgd_size));
}
void kvm_riscv_gstage_update_hgatp(struct kvm_vcpu *vcpu)
void kvm_riscv_mmu_update_hgatp(struct kvm_vcpu *vcpu)
{
unsigned long hgatp = gstage_mode;
unsigned long hgatp = kvm_riscv_gstage_mode << HGATP_MODE_SHIFT;
struct kvm_arch *k = &vcpu->kvm->arch;
hgatp |= (READ_ONCE(k->vmid.vmid) << HGATP_VMID_SHIFT) & HGATP_VMID;
@@ -736,37 +485,3 @@ void kvm_riscv_gstage_update_hgatp(struct kvm_vcpu *vcpu)
if (!kvm_riscv_gstage_vmid_bits())
kvm_riscv_local_hfence_gvma_all();
}
void __init kvm_riscv_gstage_mode_detect(void)
{
#ifdef CONFIG_64BIT
/* Try Sv57x4 G-stage mode */
csr_write(CSR_HGATP, HGATP_MODE_SV57X4 << HGATP_MODE_SHIFT);
if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV57X4) {
gstage_mode = (HGATP_MODE_SV57X4 << HGATP_MODE_SHIFT);
gstage_pgd_levels = 5;
goto skip_sv48x4_test;
}
/* Try Sv48x4 G-stage mode */
csr_write(CSR_HGATP, HGATP_MODE_SV48X4 << HGATP_MODE_SHIFT);
if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV48X4) {
gstage_mode = (HGATP_MODE_SV48X4 << HGATP_MODE_SHIFT);
gstage_pgd_levels = 4;
}
skip_sv48x4_test:
csr_write(CSR_HGATP, 0);
kvm_riscv_local_hfence_gvma_all();
#endif
}
unsigned long __init kvm_riscv_gstage_mode(void)
{
return gstage_mode >> HGATP_MODE_SHIFT;
}
int kvm_riscv_gstage_gpa_bits(void)
{
return gstage_gpa_bits;
}
arch/riscv/kvm/tlb.c
+59 -51
View File
@@ -15,6 +15,8 @@
#include <asm/cpufeature.h>
#include <asm/insn-def.h>
#include <asm/kvm_nacl.h>
#include <asm/kvm_tlb.h>
#include <asm/kvm_vmid.h>
#define has_svinval() riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)
@@ -156,36 +158,13 @@ void kvm_riscv_local_hfence_vvma_all(unsigned long vmid)
csr_write(CSR_HGATP, hgatp);
}
void kvm_riscv_local_tlb_sanitize(struct kvm_vcpu *vcpu)
{
unsigned long vmid;
if (!kvm_riscv_gstage_vmid_bits() ||
vcpu->arch.last_exit_cpu == vcpu->cpu)
return;
/*
* On RISC-V platforms with hardware VMID support, we share same
* VMID for all VCPUs of a particular Guest/VM. This means we might
* have stale G-stage TLB entries on the current Host CPU due to
* some other VCPU of the same Guest which ran previously on the
* current Host CPU.
*
* To cleanup stale TLB entries, we simply flush all G-stage TLB
* entries by VMID whenever underlying Host CPU changes for a VCPU.
*/
vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
kvm_riscv_local_hfence_gvma_vmid_all(vmid);
}
void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu)
{
kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_FENCE_I_RCVD);
local_flush_icache_all();
}
void kvm_riscv_hfence_gvma_vmid_all_process(struct kvm_vcpu *vcpu)
void kvm_riscv_tlb_flush_process(struct kvm_vcpu *vcpu)
{
struct kvm_vmid *v = &vcpu->kvm->arch.vmid;
unsigned long vmid = READ_ONCE(v->vmid);
@@ -258,51 +237,58 @@ static bool vcpu_hfence_enqueue(struct kvm_vcpu *vcpu,
void kvm_riscv_hfence_process(struct kvm_vcpu *vcpu)
{
unsigned long vmid;
struct kvm_riscv_hfence d = { 0 };
struct kvm_vmid *v = &vcpu->kvm->arch.vmid;
while (vcpu_hfence_dequeue(vcpu, &d)) {
switch (d.type) {
case KVM_RISCV_HFENCE_UNKNOWN:
break;
case KVM_RISCV_HFENCE_GVMA_VMID_GPA:
vmid = READ_ONCE(v->vmid);
if (kvm_riscv_nacl_available())
nacl_hfence_gvma_vmid(nacl_shmem(), vmid,
nacl_hfence_gvma_vmid(nacl_shmem(), d.vmid,
d.addr, d.size, d.order);
else
kvm_riscv_local_hfence_gvma_vmid_gpa(vmid, d.addr,
kvm_riscv_local_hfence_gvma_vmid_gpa(d.vmid, d.addr,
d.size, d.order);
break;
case KVM_RISCV_HFENCE_GVMA_VMID_ALL:
if (kvm_riscv_nacl_available())
nacl_hfence_gvma_vmid_all(nacl_shmem(), d.vmid);
else
kvm_riscv_local_hfence_gvma_vmid_all(d.vmid);
break;
case KVM_RISCV_HFENCE_VVMA_ASID_GVA:
kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_ASID_RCVD);
vmid = READ_ONCE(v->vmid);
if (kvm_riscv_nacl_available())
nacl_hfence_vvma_asid(nacl_shmem(), vmid, d.asid,
nacl_hfence_vvma_asid(nacl_shmem(), d.vmid, d.asid,
d.addr, d.size, d.order);
else
kvm_riscv_local_hfence_vvma_asid_gva(vmid, d.asid, d.addr,
kvm_riscv_local_hfence_vvma_asid_gva(d.vmid, d.asid, d.addr,
d.size, d.order);
break;
case KVM_RISCV_HFENCE_VVMA_ASID_ALL:
kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_ASID_RCVD);
vmid = READ_ONCE(v->vmid);
if (kvm_riscv_nacl_available())
nacl_hfence_vvma_asid_all(nacl_shmem(), vmid, d.asid);
nacl_hfence_vvma_asid_all(nacl_shmem(), d.vmid, d.asid);
else
kvm_riscv_local_hfence_vvma_asid_all(vmid, d.asid);
kvm_riscv_local_hfence_vvma_asid_all(d.vmid, d.asid);
break;
case KVM_RISCV_HFENCE_VVMA_GVA:
kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_RCVD);
vmid = READ_ONCE(v->vmid);
if (kvm_riscv_nacl_available())
nacl_hfence_vvma(nacl_shmem(), vmid,
nacl_hfence_vvma(nacl_shmem(), d.vmid,
d.addr, d.size, d.order);
else
kvm_riscv_local_hfence_vvma_gva(vmid, d.addr,
kvm_riscv_local_hfence_vvma_gva(d.vmid, d.addr,
d.size, d.order);
break;
case KVM_RISCV_HFENCE_VVMA_ALL:
kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_RCVD);
if (kvm_riscv_nacl_available())
nacl_hfence_vvma_all(nacl_shmem(), d.vmid);
else
kvm_riscv_local_hfence_vvma_all(d.vmid);
break;
default:
break;
}
@@ -355,35 +341,43 @@ void kvm_riscv_fence_i(struct kvm *kvm,
void kvm_riscv_hfence_gvma_vmid_gpa(struct kvm *kvm,
unsigned long hbase, unsigned long hmask,
gpa_t gpa, gpa_t gpsz,
unsigned long order)
unsigned long order, unsigned long vmid)
{
struct kvm_riscv_hfence data;
data.type = KVM_RISCV_HFENCE_GVMA_VMID_GPA;
data.asid = 0;
data.vmid = vmid;
data.addr = gpa;
data.size = gpsz;
data.order = order;
make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
KVM_REQ_HFENCE_GVMA_VMID_ALL, &data);
KVM_REQ_TLB_FLUSH, &data);
}
void kvm_riscv_hfence_gvma_vmid_all(struct kvm *kvm,
unsigned long hbase, unsigned long hmask)
unsigned long hbase, unsigned long hmask,
unsigned long vmid)
{
make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE_GVMA_VMID_ALL,
KVM_REQ_HFENCE_GVMA_VMID_ALL, NULL);
struct kvm_riscv_hfence data = {0};
data.type = KVM_RISCV_HFENCE_GVMA_VMID_ALL;
data.vmid = vmid;
make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
KVM_REQ_TLB_FLUSH, &data);
}
void kvm_riscv_hfence_vvma_asid_gva(struct kvm *kvm,
unsigned long hbase, unsigned long hmask,
unsigned long gva, unsigned long gvsz,
unsigned long order, unsigned long asid)
unsigned long order, unsigned long asid,
unsigned long vmid)
{
struct kvm_riscv_hfence data;
data.type = KVM_RISCV_HFENCE_VVMA_ASID_GVA;
data.asid = asid;
data.vmid = vmid;
data.addr = gva;
data.size = gvsz;
data.order = order;
@@ -393,13 +387,13 @@ void kvm_riscv_hfence_vvma_asid_gva(struct kvm *kvm,
void kvm_riscv_hfence_vvma_asid_all(struct kvm *kvm,
unsigned long hbase, unsigned long hmask,
unsigned long asid)
unsigned long asid, unsigned long vmid)
{
struct kvm_riscv_hfence data;
struct kvm_riscv_hfence data = {0};
data.type = KVM_RISCV_HFENCE_VVMA_ASID_ALL;
data.asid = asid;
data.addr = data.size = data.order = 0;
data.vmid = vmid;
make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
KVM_REQ_HFENCE_VVMA_ALL, &data);
}
@@ -407,12 +401,13 @@ void kvm_riscv_hfence_vvma_asid_all(struct kvm *kvm,
void kvm_riscv_hfence_vvma_gva(struct kvm *kvm,
unsigned long hbase, unsigned long hmask,
unsigned long gva, unsigned long gvsz,
unsigned long order)
unsigned long order, unsigned long vmid)
{
struct kvm_riscv_hfence data;
data.type = KVM_RISCV_HFENCE_VVMA_GVA;
data.asid = 0;
data.vmid = vmid;
data.addr = gva;
data.size = gvsz;
data.order = order;
@@ -421,8 +416,21 @@ void kvm_riscv_hfence_vvma_gva(struct kvm *kvm,
}
void kvm_riscv_hfence_vvma_all(struct kvm *kvm,
unsigned long hbase, unsigned long hmask)
unsigned long hbase, unsigned long hmask,
unsigned long vmid)
{
make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE_VVMA_ALL,
KVM_REQ_HFENCE_VVMA_ALL, NULL);
struct kvm_riscv_hfence data = {0};
data.type = KVM_RISCV_HFENCE_VVMA_ALL;
data.vmid = vmid;
make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
KVM_REQ_HFENCE_VVMA_ALL, &data);
}
int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn, u64 nr_pages)
{
kvm_riscv_hfence_gvma_vmid_gpa(kvm, -1UL, 0,
gfn << PAGE_SHIFT, nr_pages << PAGE_SHIFT,
PAGE_SHIFT, READ_ONCE(kvm->arch.vmid.vmid));
return 0;
}
arch/riscv/kvm/vcpu.c
+30 -18
View File
@@ -18,6 +18,7 @@
#include <linux/fs.h>
#include <linux/kvm_host.h>
#include <asm/cacheflush.h>
#include <asm/kvm_mmu.h>
#include <asm/kvm_nacl.h>
#include <asm/kvm_vcpu_vector.h>
@@ -111,7 +112,7 @@ static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu, bool kvm_sbi_reset)
vcpu->arch.hfence_tail = 0;
memset(vcpu->arch.hfence_queue, 0, sizeof(vcpu->arch.hfence_queue));
kvm_riscv_vcpu_sbi_sta_reset(vcpu);
kvm_riscv_vcpu_sbi_reset(vcpu);
/* Reset the guest CSRs for hotplug usecase */
if (loaded)
@@ -148,8 +149,9 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
spin_lock_init(&vcpu->arch.reset_state.lock);
if (kvm_riscv_vcpu_alloc_vector_context(vcpu))
return -ENOMEM;
rc = kvm_riscv_vcpu_alloc_vector_context(vcpu);
if (rc)
return rc;
/* Setup VCPU timer */
kvm_riscv_vcpu_timer_init(vcpu);
@@ -158,9 +160,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
kvm_riscv_vcpu_pmu_init(vcpu);
/* Setup VCPU AIA */
rc = kvm_riscv_vcpu_aia_init(vcpu);
if (rc)
return rc;
kvm_riscv_vcpu_aia_init(vcpu);
/*
* Setup SBI extensions
@@ -187,6 +187,8 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
{
kvm_riscv_vcpu_sbi_deinit(vcpu);
/* Cleanup VCPU AIA context */
kvm_riscv_vcpu_aia_deinit(vcpu);
@@ -620,7 +622,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
}
}
kvm_riscv_gstage_update_hgatp(vcpu);
kvm_riscv_mmu_update_hgatp(vcpu);
kvm_riscv_vcpu_timer_restore(vcpu);
@@ -680,7 +682,14 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
}
}
static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu)
/**
* check_vcpu_requests - check and handle pending vCPU requests
* @vcpu: the VCPU pointer
*
* Return: 1 if we should enter the guest
* 0 if we should exit to userspace
*/
static int kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu)
{
struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
@@ -705,17 +714,13 @@ static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu)
kvm_riscv_reset_vcpu(vcpu, true);
if (kvm_check_request(KVM_REQ_UPDATE_HGATP, vcpu))
kvm_riscv_gstage_update_hgatp(vcpu);
kvm_riscv_mmu_update_hgatp(vcpu);
if (kvm_check_request(KVM_REQ_FENCE_I, vcpu))
kvm_riscv_fence_i_process(vcpu);
/*
* The generic KVM_REQ_TLB_FLUSH is same as
* KVM_REQ_HFENCE_GVMA_VMID_ALL
*/
if (kvm_check_request(KVM_REQ_HFENCE_GVMA_VMID_ALL, vcpu))
kvm_riscv_hfence_gvma_vmid_all_process(vcpu);
if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))
kvm_riscv_tlb_flush_process(vcpu);
if (kvm_check_request(KVM_REQ_HFENCE_VVMA_ALL, vcpu))
kvm_riscv_hfence_vvma_all_process(vcpu);
@@ -725,7 +730,12 @@ static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu)
if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
kvm_riscv_vcpu_record_steal_time(vcpu);
if (kvm_dirty_ring_check_request(vcpu))
return 0;
}
return 1;
}
static void kvm_riscv_update_hvip(struct kvm_vcpu *vcpu)
@@ -907,7 +917,9 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
kvm_riscv_gstage_vmid_update(vcpu);
kvm_riscv_check_vcpu_requests(vcpu);
ret = kvm_riscv_check_vcpu_requests(vcpu);
if (ret <= 0)
continue;
preempt_disable();
@@ -951,12 +963,12 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
}
/*
* Cleanup stale TLB enteries
* Sanitize VMID mappings cached (TLB) on current CPU
*
* Note: This should be done after G-stage VMID has been
* updated using kvm_riscv_gstage_vmid_ver_changed()
*/
kvm_riscv_local_tlb_sanitize(vcpu);
kvm_riscv_gstage_vmid_sanitize(vcpu);
trace_kvm_entry(vcpu);
arch/riscv/kvm/vcpu_exit.c
+12 -8
View File
@@ -9,10 +9,13 @@
#include <linux/kvm_host.h>
#include <asm/csr.h>
#include <asm/insn-def.h>
#include <asm/kvm_mmu.h>
#include <asm/kvm_nacl.h>
static int gstage_page_fault(struct kvm_vcpu *vcpu, struct kvm_run *run,
struct kvm_cpu_trap *trap)
{
struct kvm_gstage_mapping host_map;
struct kvm_memory_slot *memslot;
unsigned long hva, fault_addr;
bool writable;
@@ -40,8 +43,9 @@ static int gstage_page_fault(struct kvm_vcpu *vcpu, struct kvm_run *run,
};
}
ret = kvm_riscv_gstage_map(vcpu, memslot, fault_addr, hva,
(trap->scause == EXC_STORE_GUEST_PAGE_FAULT) ? true : false);
ret = kvm_riscv_mmu_map(vcpu, memslot, fault_addr, hva,
(trap->scause == EXC_STORE_GUEST_PAGE_FAULT) ? true : false,
&host_map);
if (ret < 0)
return ret;
@@ -135,7 +139,7 @@ unsigned long kvm_riscv_vcpu_unpriv_read(struct kvm_vcpu *vcpu,
void kvm_riscv_vcpu_trap_redirect(struct kvm_vcpu *vcpu,
struct kvm_cpu_trap *trap)
{
unsigned long vsstatus = csr_read(CSR_VSSTATUS);
unsigned long vsstatus = ncsr_read(CSR_VSSTATUS);
/* Change Guest SSTATUS.SPP bit */
vsstatus &= ~SR_SPP;
@@ -151,15 +155,15 @@ void kvm_riscv_vcpu_trap_redirect(struct kvm_vcpu *vcpu,
vsstatus &= ~SR_SIE;
/* Update Guest SSTATUS */
csr_write(CSR_VSSTATUS, vsstatus);
ncsr_write(CSR_VSSTATUS, vsstatus);
/* Update Guest SCAUSE, STVAL, and SEPC */
csr_write(CSR_VSCAUSE, trap->scause);
csr_write(CSR_VSTVAL, trap->stval);
csr_write(CSR_VSEPC, trap->sepc);
ncsr_write(CSR_VSCAUSE, trap->scause);
ncsr_write(CSR_VSTVAL, trap->stval);
ncsr_write(CSR_VSEPC, trap->sepc);
/* Set Guest PC to Guest exception vector */
vcpu->arch.guest_context.sepc = csr_read(CSR_VSTVEC);
vcpu->arch.guest_context.sepc = ncsr_read(CSR_VSTVEC);
/* Set Guest privilege mode to supervisor */
vcpu->arch.guest_context.sstatus |= SR_SPP;
arch/riscv/kvm/vcpu_onereg.c
+53 -30
View File
@@ -23,7 +23,7 @@
#define KVM_ISA_EXT_ARR(ext) \
[KVM_RISCV_ISA_EXT_##ext] = RISCV_ISA_EXT_##ext
/* Mapping between KVM ISA Extension ID & Host ISA extension ID */
/* Mapping between KVM ISA Extension ID & guest ISA extension ID */
static const unsigned long kvm_isa_ext_arr[] = {
/* Single letter extensions (alphabetically sorted) */
[KVM_RISCV_ISA_EXT_A] = RISCV_ISA_EXT_a,
@@ -35,7 +35,7 @@ static const unsigned long kvm_isa_ext_arr[] = {
[KVM_RISCV_ISA_EXT_M] = RISCV_ISA_EXT_m,
[KVM_RISCV_ISA_EXT_V] = RISCV_ISA_EXT_v,
/* Multi letter extensions (alphabetically sorted) */
[KVM_RISCV_ISA_EXT_SMNPM] = RISCV_ISA_EXT_SSNPM,
KVM_ISA_EXT_ARR(SMNPM),
KVM_ISA_EXT_ARR(SMSTATEEN),
KVM_ISA_EXT_ARR(SSAIA),
KVM_ISA_EXT_ARR(SSCOFPMF),
@@ -112,6 +112,36 @@ static unsigned long kvm_riscv_vcpu_base2isa_ext(unsigned long base_ext)
return KVM_RISCV_ISA_EXT_MAX;
}
static int kvm_riscv_vcpu_isa_check_host(unsigned long kvm_ext, unsigned long *guest_ext)
{
unsigned long host_ext;
if (kvm_ext >= KVM_RISCV_ISA_EXT_MAX ||
kvm_ext >= ARRAY_SIZE(kvm_isa_ext_arr))
return -ENOENT;
*guest_ext = kvm_isa_ext_arr[kvm_ext];
switch (*guest_ext) {
case RISCV_ISA_EXT_SMNPM:
/*
* Pointer masking effective in (H)S-mode is provided by the
* Smnpm extension, so that extension is reported to the guest,
* even though the CSR bits for configuring VS-mode pointer
* masking on the host side are part of the Ssnpm extension.
*/
host_ext = RISCV_ISA_EXT_SSNPM;
break;
default:
host_ext = *guest_ext;
break;
}
if (!__riscv_isa_extension_available(NULL, host_ext))
return -ENOENT;
return 0;
}
static bool kvm_riscv_vcpu_isa_enable_allowed(unsigned long ext)
{
switch (ext) {
@@ -219,13 +249,13 @@ static bool kvm_riscv_vcpu_isa_disable_allowed(unsigned long ext)
void kvm_riscv_vcpu_setup_isa(struct kvm_vcpu *vcpu)
{
unsigned long host_isa, i;
unsigned long guest_ext, i;
for (i = 0; i < ARRAY_SIZE(kvm_isa_ext_arr); i++) {
host_isa = kvm_isa_ext_arr[i];
if (__riscv_isa_extension_available(NULL, host_isa) &&
kvm_riscv_vcpu_isa_enable_allowed(i))
set_bit(host_isa, vcpu->arch.isa);
if (kvm_riscv_vcpu_isa_check_host(i, &guest_ext))
continue;
if (kvm_riscv_vcpu_isa_enable_allowed(i))
set_bit(guest_ext, vcpu->arch.isa);
}
}
@@ -607,18 +637,15 @@ static int riscv_vcpu_get_isa_ext_single(struct kvm_vcpu *vcpu,
unsigned long reg_num,
unsigned long *reg_val)
{
unsigned long host_isa_ext;
unsigned long guest_ext;
int ret;
if (reg_num >= KVM_RISCV_ISA_EXT_MAX ||
reg_num >= ARRAY_SIZE(kvm_isa_ext_arr))
return -ENOENT;
host_isa_ext = kvm_isa_ext_arr[reg_num];
if (!__riscv_isa_extension_available(NULL, host_isa_ext))
return -ENOENT;
ret = kvm_riscv_vcpu_isa_check_host(reg_num, &guest_ext);
if (ret)
return ret;
*reg_val = 0;
if (__riscv_isa_extension_available(vcpu->arch.isa, host_isa_ext))
if (__riscv_isa_extension_available(vcpu->arch.isa, guest_ext))
*reg_val = 1; /* Mark the given extension as available */
return 0;
@@ -628,17 +655,14 @@ static int riscv_vcpu_set_isa_ext_single(struct kvm_vcpu *vcpu,
unsigned long reg_num,
unsigned long reg_val)
{
unsigned long host_isa_ext;
unsigned long guest_ext;
int ret;
if (reg_num >= KVM_RISCV_ISA_EXT_MAX ||
reg_num >= ARRAY_SIZE(kvm_isa_ext_arr))
return -ENOENT;
ret = kvm_riscv_vcpu_isa_check_host(reg_num, &guest_ext);
if (ret)
return ret;
host_isa_ext = kvm_isa_ext_arr[reg_num];
if (!__riscv_isa_extension_available(NULL, host_isa_ext))
return -ENOENT;
if (reg_val == test_bit(host_isa_ext, vcpu->arch.isa))
if (reg_val == test_bit(guest_ext, vcpu->arch.isa))
return 0;
if (!vcpu->arch.ran_atleast_once) {
@@ -648,10 +672,10 @@ static int riscv_vcpu_set_isa_ext_single(struct kvm_vcpu *vcpu,
*/
if (reg_val == 1 &&
kvm_riscv_vcpu_isa_enable_allowed(reg_num))
set_bit(host_isa_ext, vcpu->arch.isa);
set_bit(guest_ext, vcpu->arch.isa);
else if (!reg_val &&
kvm_riscv_vcpu_isa_disable_allowed(reg_num))
clear_bit(host_isa_ext, vcpu->arch.isa);
clear_bit(guest_ext, vcpu->arch.isa);
else
return -EINVAL;
kvm_riscv_vcpu_fp_reset(vcpu);
@@ -1009,16 +1033,15 @@ static int copy_fp_d_reg_indices(const struct kvm_vcpu *vcpu,
static int copy_isa_ext_reg_indices(const struct kvm_vcpu *vcpu,
u64 __user *uindices)
{
unsigned long guest_ext;
unsigned int n = 0;
unsigned long isa_ext;
for (int i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++) {
u64 size = IS_ENABLED(CONFIG_32BIT) ?
KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64;
u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_ISA_EXT | i;
isa_ext = kvm_isa_ext_arr[i];
if (!__riscv_isa_extension_available(NULL, isa_ext))
if (kvm_riscv_vcpu_isa_check_host(i, &guest_ext))
continue;
if (uindices) {
arch/riscv/kvm/vcpu_sbi.c
+49
View File
@@ -536,5 +536,54 @@ void kvm_riscv_vcpu_sbi_init(struct kvm_vcpu *vcpu)
scontext->ext_status[idx] = ext->default_disabled ?
KVM_RISCV_SBI_EXT_STATUS_DISABLED :
KVM_RISCV_SBI_EXT_STATUS_ENABLED;
if (ext->init && ext->init(vcpu) != 0)
scontext->ext_status[idx] = KVM_RISCV_SBI_EXT_STATUS_UNAVAILABLE;
}
}
void kvm_riscv_vcpu_sbi_deinit(struct kvm_vcpu *vcpu)
{
struct kvm_vcpu_sbi_context *scontext = &vcpu->arch.sbi_context;
const struct kvm_riscv_sbi_extension_entry *entry;
const struct kvm_vcpu_sbi_extension *ext;
int idx, i;
for (i = 0; i < ARRAY_SIZE(sbi_ext); i++) {
entry = &sbi_ext[i];
ext = entry->ext_ptr;
idx = entry->ext_idx;
if (idx < 0 || idx >= ARRAY_SIZE(scontext->ext_status))
continue;
if (scontext->ext_status[idx] == KVM_RISCV_SBI_EXT_STATUS_UNAVAILABLE ||
!ext->deinit)
continue;
ext->deinit(vcpu);
}
}
void kvm_riscv_vcpu_sbi_reset(struct kvm_vcpu *vcpu)
{
struct kvm_vcpu_sbi_context *scontext = &vcpu->arch.sbi_context;
const struct kvm_riscv_sbi_extension_entry *entry;
const struct kvm_vcpu_sbi_extension *ext;
int idx, i;
for (i = 0; i < ARRAY_SIZE(sbi_ext); i++) {
entry = &sbi_ext[i];
ext = entry->ext_ptr;
idx = entry->ext_idx;
if (idx < 0 || idx >= ARRAY_SIZE(scontext->ext_status))
continue;
if (scontext->ext_status[idx] != KVM_RISCV_SBI_EXT_STATUS_ENABLED ||
!ext->reset)
continue;
ext->reset(vcpu);
}
}
arch/riscv/kvm/vcpu_sbi_replace.c
+9 -8
View File
@@ -96,6 +96,7 @@ static int kvm_sbi_ext_rfence_handler(struct kvm_vcpu *vcpu, struct kvm_run *run
unsigned long hmask = cp->a0;
unsigned long hbase = cp->a1;
unsigned long funcid = cp->a6;
unsigned long vmid;
switch (funcid) {
case SBI_EXT_RFENCE_REMOTE_FENCE_I:
@@ -103,22 +104,22 @@ static int kvm_sbi_ext_rfence_handler(struct kvm_vcpu *vcpu, struct kvm_run *run
kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_FENCE_I_SENT);
break;
case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA:
vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
if ((cp->a2 == 0 && cp->a3 == 0) || cp->a3 == -1UL)
kvm_riscv_hfence_vvma_all(vcpu->kvm, hbase, hmask);
kvm_riscv_hfence_vvma_all(vcpu->kvm, hbase, hmask, vmid);
else
kvm_riscv_hfence_vvma_gva(vcpu->kvm, hbase, hmask,
cp->a2, cp->a3, PAGE_SHIFT);
cp->a2, cp->a3, PAGE_SHIFT, vmid);
kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_SENT);
break;
case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID:
vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
if ((cp->a2 == 0 && cp->a3 == 0) || cp->a3 == -1UL)
kvm_riscv_hfence_vvma_asid_all(vcpu->kvm,
hbase, hmask, cp->a4);
kvm_riscv_hfence_vvma_asid_all(vcpu->kvm, hbase, hmask,
cp->a4, vmid);
else
kvm_riscv_hfence_vvma_asid_gva(vcpu->kvm,
hbase, hmask,
cp->a2, cp->a3,
PAGE_SHIFT, cp->a4);
kvm_riscv_hfence_vvma_asid_gva(vcpu->kvm, hbase, hmask, cp->a2,
cp->a3, PAGE_SHIFT, cp->a4, vmid);
kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_ASID_SENT);
break;
case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA:
arch/riscv/kvm/vcpu_sbi_sta.c
+2 -1
View File
@@ -16,7 +16,7 @@
#include <asm/sbi.h>
#include <asm/uaccess.h>
void kvm_riscv_vcpu_sbi_sta_reset(struct kvm_vcpu *vcpu)
static void kvm_riscv_vcpu_sbi_sta_reset(struct kvm_vcpu *vcpu)
{
vcpu->arch.sta.shmem = INVALID_GPA;
vcpu->arch.sta.last_steal = 0;
@@ -156,6 +156,7 @@ const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_sta = {
.extid_end = SBI_EXT_STA,
.handler = kvm_sbi_ext_sta_handler,
.probe = kvm_sbi_ext_sta_probe,
.reset = kvm_riscv_vcpu_sbi_sta_reset,
};
int kvm_riscv_vcpu_get_reg_sbi_sta(struct kvm_vcpu *vcpu,
arch/riscv/kvm/vcpu_sbi_v01.c
+11 -14
View File
@@ -23,6 +23,7 @@ static int kvm_sbi_ext_v01_handler(struct kvm_vcpu *vcpu, struct kvm_run *run,
struct kvm *kvm = vcpu->kvm;
struct kvm_cpu_context *cp = &vcpu->arch.guest_context;
struct kvm_cpu_trap *utrap = retdata->utrap;
unsigned long vmid;
switch (cp->a7) {
case SBI_EXT_0_1_CONSOLE_GETCHAR:
@@ -78,25 +79,21 @@ static int kvm_sbi_ext_v01_handler(struct kvm_vcpu *vcpu, struct kvm_run *run,
if (cp->a7 == SBI_EXT_0_1_REMOTE_FENCE_I)
kvm_riscv_fence_i(vcpu->kvm, 0, hmask);
else if (cp->a7 == SBI_EXT_0_1_REMOTE_SFENCE_VMA) {
vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
if (cp->a1 == 0 && cp->a2 == 0)
kvm_riscv_hfence_vvma_all(vcpu->kvm,
0, hmask);
kvm_riscv_hfence_vvma_all(vcpu->kvm, 0, hmask, vmid);
else
kvm_riscv_hfence_vvma_gva(vcpu->kvm,
0, hmask,
cp->a1, cp->a2,
PAGE_SHIFT);
kvm_riscv_hfence_vvma_gva(vcpu->kvm, 0, hmask, cp->a1,
cp->a2, PAGE_SHIFT, vmid);
} else {
vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
if (cp->a1 == 0 && cp->a2 == 0)
kvm_riscv_hfence_vvma_asid_all(vcpu->kvm,
0, hmask,
cp->a3);
kvm_riscv_hfence_vvma_asid_all(vcpu->kvm, 0, hmask,
cp->a3, vmid);
else
kvm_riscv_hfence_vvma_asid_gva(vcpu->kvm,
0, hmask,
cp->a1, cp->a2,
PAGE_SHIFT,
cp->a3);
kvm_riscv_hfence_vvma_asid_gva(vcpu->kvm, 0, hmask,
cp->a1, cp->a2, PAGE_SHIFT,
cp->a3, vmid);
}
break;
default:
arch/riscv/kvm/vm.c
+4 -3
View File
@@ -11,6 +11,7 @@
#include <linux/module.h>
#include <linux/uaccess.h>
#include <linux/kvm_host.h>
#include <asm/kvm_mmu.h>
const struct _kvm_stats_desc kvm_vm_stats_desc[] = {
KVM_GENERIC_VM_STATS()
@@ -31,13 +32,13 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
{
int r;
r = kvm_riscv_gstage_alloc_pgd(kvm);
r = kvm_riscv_mmu_alloc_pgd(kvm);
if (r)
return r;
r = kvm_riscv_gstage_vmid_init(kvm);
if (r) {
kvm_riscv_gstage_free_pgd(kvm);
kvm_riscv_mmu_free_pgd(kvm);
return r;
}
@@ -199,7 +200,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
r = KVM_USER_MEM_SLOTS;
break;
case KVM_CAP_VM_GPA_BITS:
r = kvm_riscv_gstage_gpa_bits();
r = kvm_riscv_gstage_gpa_bits;
break;
default:
r = 0;
arch/riscv/kvm/vmid.c
+25
View File
@@ -14,6 +14,8 @@
#include <linux/smp.h>
#include <linux/kvm_host.h>
#include <asm/csr.h>
#include <asm/kvm_tlb.h>
#include <asm/kvm_vmid.h>
static unsigned long vmid_version = 1;
static unsigned long vmid_next;
@@ -122,3 +124,26 @@ void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu)
kvm_for_each_vcpu(i, v, vcpu->kvm)
kvm_make_request(KVM_REQ_UPDATE_HGATP, v);
}
void kvm_riscv_gstage_vmid_sanitize(struct kvm_vcpu *vcpu)
{
unsigned long vmid;
if (!kvm_riscv_gstage_vmid_bits() ||
vcpu->arch.last_exit_cpu == vcpu->cpu)
return;
/*
* On RISC-V platforms with hardware VMID support, we share same
* VMID for all VCPUs of a particular Guest/VM. This means we might
* have stale G-stage TLB entries on the current Host CPU due to
* some other VCPU of the same Guest which ran previously on the
* current Host CPU.
*
* To cleanup stale TLB entries, we simply flush all G-stage TLB
* entries by VMID whenever underlying Host CPU changes for a VCPU.
*/
vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
kvm_riscv_local_hfence_gvma_vmid_all(vmid);
}
arch/s390/include/asm/entry-common.h
+10
View File
@@ -59,4 +59,14 @@ static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
#define arch_exit_to_user_mode_prepare arch_exit_to_user_mode_prepare
static __always_inline bool arch_in_rcu_eqs(void)
{
if (IS_ENABLED(CONFIG_KVM))
return current->flags & PF_VCPU;
return false;
}
#define arch_in_rcu_eqs arch_in_rcu_eqs
#endif
arch/s390/include/asm/kvm_host.h
+3
View File
@@ -716,6 +716,9 @@ extern char sie_exit;
bool kvm_s390_pv_is_protected(struct kvm *kvm);
bool kvm_s390_pv_cpu_is_protected(struct kvm_vcpu *vcpu);
extern int kvm_s390_enter_exit_sie(struct kvm_s390_sie_block *scb,
u64 *gprs, unsigned long gasce);
extern int kvm_s390_gisc_register(struct kvm *kvm, u32 gisc);
extern int kvm_s390_gisc_unregister(struct kvm *kvm, u32 gisc);
arch/s390/kvm/kvm-s390.c
+39 -12
View File
@@ -5063,6 +5063,30 @@ static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
return vcpu_post_run_handle_fault(vcpu);
}
int noinstr kvm_s390_enter_exit_sie(struct kvm_s390_sie_block *scb,
u64 *gprs, unsigned long gasce)
{
int ret;
guest_state_enter_irqoff();
/*
* The guest_state_{enter,exit}_irqoff() functions inform lockdep and
* tracing that entry to the guest will enable host IRQs, and exit from
* the guest will disable host IRQs.
*
* We must not use lockdep/tracing/RCU in this critical section, so we
* use the low-level arch_local_irq_*() helpers to enable/disable IRQs.
*/
arch_local_irq_enable();
ret = sie64a(scb, gprs, gasce);
arch_local_irq_disable();
guest_state_exit_irqoff();
return ret;
}
#define PSW_INT_MASK (PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_MCHECK)
static int __vcpu_run(struct kvm_vcpu *vcpu)
{
@@ -5083,20 +5107,27 @@ static int __vcpu_run(struct kvm_vcpu *vcpu)
kvm_vcpu_srcu_read_unlock(vcpu);
/*
* As PF_VCPU will be used in fault handler, between
* guest_enter and guest_exit should be no uaccess.
* guest_timing_enter_irqoff and guest_timing_exit_irqoff
* should be no uaccess.
*/
local_irq_disable();
guest_enter_irqoff();
__disable_cpu_timer_accounting(vcpu);
local_irq_enable();
if (kvm_s390_pv_cpu_is_protected(vcpu)) {
memcpy(sie_page->pv_grregs,
vcpu->run->s.regs.gprs,
sizeof(sie_page->pv_grregs));
}
exit_reason = sie64a(vcpu->arch.sie_block,
vcpu->run->s.regs.gprs,
vcpu->arch.gmap->asce);
local_irq_disable();
guest_timing_enter_irqoff();
__disable_cpu_timer_accounting(vcpu);
exit_reason = kvm_s390_enter_exit_sie(vcpu->arch.sie_block,
vcpu->run->s.regs.gprs,
vcpu->arch.gmap->asce);
__enable_cpu_timer_accounting(vcpu);
guest_timing_exit_irqoff();
local_irq_enable();
if (kvm_s390_pv_cpu_is_protected(vcpu)) {
memcpy(vcpu->run->s.regs.gprs,
sie_page->pv_grregs,
@@ -5112,10 +5143,6 @@ static int __vcpu_run(struct kvm_vcpu *vcpu)
vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK;
}
}
local_irq_disable();
__enable_cpu_timer_accounting(vcpu);
guest_exit_irqoff();
local_irq_enable();
kvm_vcpu_srcu_read_lock(vcpu);
rc = vcpu_post_run(vcpu, exit_reason);
arch/s390/kvm/vsie.c
+7 -10
View File
@@ -1170,10 +1170,6 @@ static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
vcpu->arch.sie_block->fpf & FPF_BPBC)
set_thread_flag(TIF_ISOLATE_BP_GUEST);
local_irq_disable();
guest_enter_irqoff();
local_irq_enable();
/*
* Simulate a SIE entry of the VCPU (see sie64a), so VCPU blocking
* and VCPU requests also hinder the vSIE from running and lead
@@ -1183,15 +1179,16 @@ static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
vcpu->arch.sie_block->prog0c |= PROG_IN_SIE;
current->thread.gmap_int_code = 0;
barrier();
if (!kvm_s390_vcpu_sie_inhibited(vcpu))
rc = sie64a(scb_s, vcpu->run->s.regs.gprs, vsie_page->gmap->asce);
if (!kvm_s390_vcpu_sie_inhibited(vcpu)) {
local_irq_disable();
guest_timing_enter_irqoff();
rc = kvm_s390_enter_exit_sie(scb_s, vcpu->run->s.regs.gprs, vsie_page->gmap->asce);
guest_timing_exit_irqoff();
local_irq_enable();
}
barrier();
vcpu->arch.sie_block->prog0c &= ~PROG_IN_SIE;
local_irq_disable();
guest_exit_irqoff();
local_irq_enable();
/* restore guest state for bp isolation override */
if (!guest_bp_isolation)
clear_thread_flag(TIF_ISOLATE_BP_GUEST);
arch/x86/include/asm/apic.h
+64 -2
View File
@@ -488,11 +488,14 @@ static inline void apic_setup_apic_calls(void) { }
extern void apic_ack_irq(struct irq_data *data);
#define APIC_VECTOR_TO_BIT_NUMBER(v) ((unsigned int)(v) % 32)
#define APIC_VECTOR_TO_REG_OFFSET(v) ((unsigned int)(v) / 32 * 0x10)
static inline bool lapic_vector_set_in_irr(unsigned int vector)
{
u32 irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
u32 irr = apic_read(APIC_IRR + APIC_VECTOR_TO_REG_OFFSET(vector));
return !!(irr & (1U << (vector % 32)));
return !!(irr & (1U << APIC_VECTOR_TO_BIT_NUMBER(vector)));
}
static inline bool is_vector_pending(unsigned int vector)
@@ -500,6 +503,65 @@ static inline bool is_vector_pending(unsigned int vector)
return lapic_vector_set_in_irr(vector) || pi_pending_this_cpu(vector);
}
#define MAX_APIC_VECTOR 256
#define APIC_VECTORS_PER_REG 32
/*
* Vector states are maintained by APIC in 32-bit registers that are
* 16 bytes aligned. The status of each vector is kept in a single
* bit.
*/
static inline int apic_find_highest_vector(void *bitmap)
{
int vec;
u32 *reg;
for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG; vec >= 0; vec -= APIC_VECTORS_PER_REG) {
reg = bitmap + APIC_VECTOR_TO_REG_OFFSET(vec);
if (*reg)
return __fls(*reg) + vec;
}
return -1;
}
static inline u32 apic_get_reg(void *regs, int reg)
{
return *((u32 *) (regs + reg));
}
static inline void apic_set_reg(void *regs, int reg, u32 val)
{
*((u32 *) (regs + reg)) = val;
}
static __always_inline u64 apic_get_reg64(void *regs, int reg)
{
BUILD_BUG_ON(reg != APIC_ICR);
return *((u64 *) (regs + reg));
}
static __always_inline void apic_set_reg64(void *regs, int reg, u64 val)
{
BUILD_BUG_ON(reg != APIC_ICR);
*((u64 *) (regs + reg)) = val;
}
static inline void apic_clear_vector(int vec, void *bitmap)
{
clear_bit(APIC_VECTOR_TO_BIT_NUMBER(vec), bitmap + APIC_VECTOR_TO_REG_OFFSET(vec));
}
static inline void apic_set_vector(int vec, void *bitmap)
{
set_bit(APIC_VECTOR_TO_BIT_NUMBER(vec), bitmap + APIC_VECTOR_TO_REG_OFFSET(vec));
}
static inline int apic_test_vector(int vec, void *bitmap)
{
return test_bit(APIC_VECTOR_TO_BIT_NUMBER(vec), bitmap + APIC_VECTOR_TO_REG_OFFSET(vec));
}
/*
* Warm reset vector position:
*/
arch/x86/include/asm/irq_remapping.h
+16 -1
View File
@@ -26,7 +26,22 @@ enum {
IRQ_REMAP_X2APIC_MODE,
};
struct vcpu_data {
/*
* This is mainly used to communicate information back-and-forth
* between SVM and IOMMU for setting up and tearing down posted
* interrupt
*/
struct amd_iommu_pi_data {
u64 vapic_addr; /* Physical address of the vCPU's vAPIC. */
u32 ga_tag;
u32 vector; /* Guest vector of the interrupt */
int cpu;
bool ga_log_intr;
bool is_guest_mode;
void *ir_data;
};
struct intel_iommu_pi_data {
u64 pi_desc_addr; /* Physical address of PI Descriptor */
u32 vector; /* Guest vector of the interrupt */
};
arch/x86/include/asm/kvm-x86-ops.h
+2 -3
View File
@@ -49,7 +49,6 @@ KVM_X86_OP(set_idt)
KVM_X86_OP(get_gdt)
KVM_X86_OP(set_gdt)
KVM_X86_OP(sync_dirty_debug_regs)
KVM_X86_OP(set_dr6)
KVM_X86_OP(set_dr7)
KVM_X86_OP(cache_reg)
KVM_X86_OP(get_rflags)
@@ -112,7 +111,7 @@ KVM_X86_OP_OPTIONAL(update_cpu_dirty_logging)
KVM_X86_OP_OPTIONAL(vcpu_blocking)
KVM_X86_OP_OPTIONAL(vcpu_unblocking)
KVM_X86_OP_OPTIONAL(pi_update_irte)
KVM_X86_OP_OPTIONAL(pi_start_assignment)
KVM_X86_OP_OPTIONAL(pi_start_bypass)
KVM_X86_OP_OPTIONAL(apicv_pre_state_restore)
KVM_X86_OP_OPTIONAL(apicv_post_state_restore)
KVM_X86_OP_OPTIONAL_RET0(dy_apicv_has_pending_interrupt)
@@ -139,7 +138,7 @@ KVM_X86_OP(check_emulate_instruction)
KVM_X86_OP(apic_init_signal_blocked)
KVM_X86_OP_OPTIONAL(enable_l2_tlb_flush)
KVM_X86_OP_OPTIONAL(migrate_timers)
KVM_X86_OP(msr_filter_changed)
KVM_X86_OP(recalc_msr_intercepts)
KVM_X86_OP(complete_emulated_msr)
KVM_X86_OP(vcpu_deliver_sipi_vector)
KVM_X86_OP_OPTIONAL_RET0(vcpu_get_apicv_inhibit_reasons);
arch/x86/include/asm/kvm_host.h
+37 -39
View File
@@ -297,6 +297,7 @@ enum x86_intercept_stage;
*/
#define KVM_APIC_PV_EOI_PENDING 1
struct kvm_kernel_irqfd;
struct kvm_kernel_irq_routing_entry;
/*
@@ -1320,6 +1321,12 @@ enum kvm_apicv_inhibit {
*/
APICV_INHIBIT_REASON_LOGICAL_ID_ALIASED,
/*
* AVIC is disabled because the vCPU's APIC ID is beyond the max
* supported by AVIC/x2AVIC, i.e. the vCPU is unaddressable.
*/
APICV_INHIBIT_REASON_PHYSICAL_ID_TOO_BIG,
NR_APICV_INHIBIT_REASONS,
};
@@ -1338,7 +1345,8 @@ enum kvm_apicv_inhibit {
__APICV_INHIBIT_REASON(IRQWIN), \
__APICV_INHIBIT_REASON(PIT_REINJ), \
__APICV_INHIBIT_REASON(SEV), \
__APICV_INHIBIT_REASON(LOGICAL_ID_ALIASED)
__APICV_INHIBIT_REASON(LOGICAL_ID_ALIASED), \
__APICV_INHIBIT_REASON(PHYSICAL_ID_TOO_BIG)
struct kvm_arch {
unsigned long n_used_mmu_pages;
@@ -1350,7 +1358,7 @@ struct kvm_arch {
bool has_private_mem;
bool has_protected_state;
bool pre_fault_allowed;
struct hlist_head mmu_page_hash[KVM_NUM_MMU_PAGES];
struct hlist_head *mmu_page_hash;
struct list_head active_mmu_pages;
/*
* A list of kvm_mmu_page structs that, if zapped, could possibly be
@@ -1379,11 +1387,13 @@ struct kvm_arch {
#define __KVM_HAVE_ARCH_NONCOHERENT_DMA
atomic_t noncoherent_dma_count;
#define __KVM_HAVE_ARCH_ASSIGNED_DEVICE
atomic_t assigned_device_count;
unsigned long nr_possible_bypass_irqs;
#ifdef CONFIG_KVM_IOAPIC
struct kvm_pic *vpic;
struct kvm_ioapic *vioapic;
struct kvm_pit *vpit;
#endif
atomic_t vapics_in_nmi_mode;
struct mutex apic_map_lock;
struct kvm_apic_map __rcu *apic_map;
@@ -1398,12 +1408,8 @@ struct kvm_arch {
gpa_t wall_clock;
bool mwait_in_guest;
bool hlt_in_guest;
bool pause_in_guest;
bool cstate_in_guest;
u64 disabled_exits;
unsigned long irq_sources_bitmap;
s64 kvmclock_offset;
/*
@@ -1432,9 +1438,6 @@ struct kvm_arch {
struct delayed_work kvmclock_update_work;
struct delayed_work kvmclock_sync_work;
/* reads protected by irq_srcu, writes by irq_lock */
struct hlist_head mask_notifier_list;
#ifdef CONFIG_KVM_HYPERV
struct kvm_hv hyperv;
#endif
@@ -1457,6 +1460,7 @@ struct kvm_arch {
bool x2apic_format;
bool x2apic_broadcast_quirk_disabled;
bool has_mapped_host_mmio;
bool guest_can_read_msr_platform_info;
bool exception_payload_enabled;
@@ -1680,6 +1684,12 @@ static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical)
return dest_mode_logical ? APIC_DEST_LOGICAL : APIC_DEST_PHYSICAL;
}
enum kvm_x86_run_flags {
KVM_RUN_FORCE_IMMEDIATE_EXIT = BIT(0),
KVM_RUN_LOAD_GUEST_DR6 = BIT(1),
KVM_RUN_LOAD_DEBUGCTL = BIT(2),
};
struct kvm_x86_ops {
const char *name;
@@ -1708,6 +1718,12 @@ struct kvm_x86_ops {
void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu);
void (*vcpu_put)(struct kvm_vcpu *vcpu);
/*
* Mask of DEBUGCTL bits that are owned by the host, i.e. that need to
* match the host's value even while the guest is active.
*/
const u64 HOST_OWNED_DEBUGCTL;
void (*update_exception_bitmap)(struct kvm_vcpu *vcpu);
int (*get_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr);
int (*set_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr);
@@ -1730,7 +1746,6 @@ struct kvm_x86_ops {
void (*get_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
void (*set_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
void (*sync_dirty_debug_regs)(struct kvm_vcpu *vcpu);
void (*set_dr6)(struct kvm_vcpu *vcpu, unsigned long value);
void (*set_dr7)(struct kvm_vcpu *vcpu, unsigned long value);
void (*cache_reg)(struct kvm_vcpu *vcpu, enum kvm_reg reg);
unsigned long (*get_rflags)(struct kvm_vcpu *vcpu);
@@ -1761,7 +1776,7 @@ struct kvm_x86_ops {
int (*vcpu_pre_run)(struct kvm_vcpu *vcpu);
enum exit_fastpath_completion (*vcpu_run)(struct kvm_vcpu *vcpu,
bool force_immediate_exit);
u64 run_flags);
int (*handle_exit)(struct kvm_vcpu *vcpu,
enum exit_fastpath_completion exit_fastpath);
int (*skip_emulated_instruction)(struct kvm_vcpu *vcpu);
@@ -1853,9 +1868,10 @@ struct kvm_x86_ops {
void (*vcpu_blocking)(struct kvm_vcpu *vcpu);
void (*vcpu_unblocking)(struct kvm_vcpu *vcpu);
int (*pi_update_irte)(struct kvm *kvm, unsigned int host_irq,
uint32_t guest_irq, bool set);
void (*pi_start_assignment)(struct kvm *kvm);
int (*pi_update_irte)(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm,
unsigned int host_irq, uint32_t guest_irq,
struct kvm_vcpu *vcpu, u32 vector);
void (*pi_start_bypass)(struct kvm *kvm);
void (*apicv_pre_state_restore)(struct kvm_vcpu *vcpu);
void (*apicv_post_state_restore)(struct kvm_vcpu *vcpu);
bool (*dy_apicv_has_pending_interrupt)(struct kvm_vcpu *vcpu);
@@ -1892,7 +1908,7 @@ struct kvm_x86_ops {
int (*enable_l2_tlb_flush)(struct kvm_vcpu *vcpu);
void (*migrate_timers)(struct kvm_vcpu *vcpu);
void (*msr_filter_changed)(struct kvm_vcpu *vcpu);
void (*recalc_msr_intercepts)(struct kvm_vcpu *vcpu);
int (*complete_emulated_msr)(struct kvm_vcpu *vcpu, int err);
void (*vcpu_deliver_sipi_vector)(struct kvm_vcpu *vcpu, u8 vector);
@@ -1950,6 +1966,7 @@ struct kvm_arch_async_pf {
extern u32 __read_mostly kvm_nr_uret_msrs;
extern bool __read_mostly allow_smaller_maxphyaddr;
extern bool __read_mostly enable_apicv;
extern bool __read_mostly enable_ipiv;
extern bool __read_mostly enable_device_posted_irqs;
extern struct kvm_x86_ops kvm_x86_ops;
@@ -1968,7 +1985,7 @@ void kvm_x86_vendor_exit(void);
#define __KVM_HAVE_ARCH_VM_ALLOC
static inline struct kvm *kvm_arch_alloc_vm(void)
{
return __vmalloc(kvm_x86_ops.vm_size, GFP_KERNEL_ACCOUNT | __GFP_ZERO);
return kvzalloc(kvm_x86_ops.vm_size, GFP_KERNEL_ACCOUNT);
}
#define __KVM_HAVE_ARCH_VM_FREE
@@ -2013,7 +2030,7 @@ void kvm_mmu_vendor_module_exit(void);
void kvm_mmu_destroy(struct kvm_vcpu *vcpu);
int kvm_mmu_create(struct kvm_vcpu *vcpu);
void kvm_mmu_init_vm(struct kvm *kvm);
int kvm_mmu_init_vm(struct kvm *kvm);
void kvm_mmu_uninit_vm(struct kvm *kvm);
void kvm_mmu_init_memslot_memory_attributes(struct kvm *kvm,
@@ -2044,19 +2061,6 @@ int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3);
int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
const void *val, int bytes);
struct kvm_irq_mask_notifier {
void (*func)(struct kvm_irq_mask_notifier *kimn, bool masked);
int irq;
struct hlist_node link;
};
void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq,
struct kvm_irq_mask_notifier *kimn);
void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq,
struct kvm_irq_mask_notifier *kimn);
void kvm_fire_mask_notifiers(struct kvm *kvm, unsigned irqchip, unsigned pin,
bool mask);
extern bool tdp_enabled;
u64 vcpu_tsc_khz(struct kvm_vcpu *vcpu);
@@ -2215,9 +2219,6 @@ static inline int __kvm_irq_line_state(unsigned long *irq_state,
return !!(*irq_state);
}
int kvm_pic_set_irq(struct kvm_pic *pic, int irq, int irq_source_id, int level);
void kvm_pic_clear_all(struct kvm_pic *pic, int irq_source_id);
void kvm_inject_nmi(struct kvm_vcpu *vcpu);
int kvm_get_nr_pending_nmis(struct kvm_vcpu *vcpu);
@@ -2394,9 +2395,6 @@ bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu);
bool kvm_intr_is_single_vcpu(struct kvm *kvm, struct kvm_lapic_irq *irq,
struct kvm_vcpu **dest_vcpu);
void kvm_set_msi_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e,
struct kvm_lapic_irq *irq);
static inline bool kvm_irq_is_postable(struct kvm_lapic_irq *irq)
{
/* We can only post Fixed and LowPrio IRQs */
arch/x86/include/asm/msr-index.h
+1
View File
@@ -419,6 +419,7 @@
#define DEBUGCTLMSR_FREEZE_PERFMON_ON_PMI (1UL << 12)
#define DEBUGCTLMSR_FREEZE_IN_SMM_BIT 14
#define DEBUGCTLMSR_FREEZE_IN_SMM (1UL << DEBUGCTLMSR_FREEZE_IN_SMM_BIT)
#define DEBUGCTLMSR_RTM_DEBUG BIT(15)
#define MSR_PEBS_FRONTEND 0x000003f7
arch/x86/include/asm/svm.h
+8 -5
View File
@@ -252,16 +252,21 @@ struct __attribute__ ((__packed__)) vmcb_control_area {
#define AVIC_LOGICAL_ID_ENTRY_VALID_BIT 31
#define AVIC_LOGICAL_ID_ENTRY_VALID_MASK (1 << 31)
/*
* GA_LOG_INTR is a synthetic flag that's never propagated to hardware-visible
* tables. GA_LOG_INTR is set if the vCPU needs device posted IRQs to generate
* GA log interrupts to wake the vCPU (because it's blocking or about to block).
*/
#define AVIC_PHYSICAL_ID_ENTRY_GA_LOG_INTR BIT_ULL(61)
#define AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK GENMASK_ULL(11, 0)
#define AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK (0xFFFFFFFFFFULL << 12)
#define AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK GENMASK_ULL(51, 12)
#define AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK (1ULL << 62)
#define AVIC_PHYSICAL_ID_ENTRY_VALID_MASK (1ULL << 63)
#define AVIC_PHYSICAL_ID_TABLE_SIZE_MASK (0xFFULL)
#define AVIC_DOORBELL_PHYSICAL_ID_MASK GENMASK_ULL(11, 0)
#define VMCB_AVIC_APIC_BAR_MASK 0xFFFFFFFFFF000ULL
#define AVIC_UNACCEL_ACCESS_WRITE_MASK 1
#define AVIC_UNACCEL_ACCESS_OFFSET_MASK 0xFF0
#define AVIC_UNACCEL_ACCESS_VECTOR_MASK 0xFFFFFFFF
@@ -290,8 +295,6 @@ enum avic_ipi_failure_cause {
static_assert((AVIC_MAX_PHYSICAL_ID & AVIC_PHYSICAL_MAX_INDEX_MASK) == AVIC_MAX_PHYSICAL_ID);
static_assert((X2AVIC_MAX_PHYSICAL_ID & AVIC_PHYSICAL_MAX_INDEX_MASK) == X2AVIC_MAX_PHYSICAL_ID);
#define AVIC_HPA_MASK ~((0xFFFULL << 52) | 0xFFF)
#define SVM_SEV_FEAT_SNP_ACTIVE BIT(0)
#define SVM_SEV_FEAT_RESTRICTED_INJECTION BIT(3)
#define SVM_SEV_FEAT_ALTERNATE_INJECTION BIT(4)
arch/x86/kvm/Kconfig
+10
View File
@@ -166,6 +166,16 @@ config KVM_AMD_SEV
Encrypted State (SEV-ES), and Secure Encrypted Virtualization with
Secure Nested Paging (SEV-SNP) technologies on AMD processors.
config KVM_IOAPIC
bool "I/O APIC, PIC, and PIT emulation"
default y
depends on KVM
help
Provides support for KVM to emulate an I/O APIC, PIC, and PIT, i.e.
for full in-kernel APIC emulation.
If unsure, say Y.
config KVM_SMM
bool "System Management Mode emulation"
default y
arch/x86/kvm/Makefile
+3 -4
View File
@@ -5,12 +5,11 @@ ccflags-$(CONFIG_KVM_WERROR) += -Werror
include $(srctree)/virt/kvm/Makefile.kvm
kvm-y += x86.o emulate.o i8259.o irq.o lapic.o \
i8254.o ioapic.o irq_comm.o cpuid.o pmu.o mtrr.o \
debugfs.o mmu/mmu.o mmu/page_track.o \
mmu/spte.o
kvm-y += x86.o emulate.o irq.o lapic.o cpuid.o pmu.o mtrr.o \
debugfs.o mmu/mmu.o mmu/page_track.o mmu/spte.o
kvm-$(CONFIG_X86_64) += mmu/tdp_iter.o mmu/tdp_mmu.o
kvm-$(CONFIG_KVM_IOAPIC) += i8259.o i8254.o ioapic.o
kvm-$(CONFIG_KVM_HYPERV) += hyperv.o
kvm-$(CONFIG_KVM_XEN) += xen.o
kvm-$(CONFIG_KVM_SMM) += smm.o
arch/x86/kvm/cpuid.c
+1
View File
@@ -979,6 +979,7 @@ void kvm_set_cpu_caps(void)
F(FSRS),
F(FSRC),
F(WRMSRNS),
X86_64_F(LKGS),
F(AMX_FP16),
F(AVX_IFMA),
F(LAM),
arch/x86/kvm/hyperv.c
+7 -3
View File
@@ -497,15 +497,19 @@ static int synic_set_irq(struct kvm_vcpu_hv_synic *synic, u32 sint)
return ret;
}
int kvm_hv_synic_set_irq(struct kvm *kvm, u32 vpidx, u32 sint)
int kvm_hv_synic_set_irq(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
int irq_source_id, int level, bool line_status)
{
struct kvm_vcpu_hv_synic *synic;
synic = synic_get(kvm, vpidx);
if (!level)
return -1;
synic = synic_get(kvm, e->hv_sint.vcpu);
if (!synic)
return -EINVAL;
return synic_set_irq(synic, sint);
return synic_set_irq(synic, e->hv_sint.sint);
}
void kvm_hv_synic_send_eoi(struct kvm_vcpu *vcpu, int vector)
arch/x86/kvm/hyperv.h
+2 -1
View File
@@ -103,7 +103,8 @@ static inline bool kvm_hv_hypercall_enabled(struct kvm_vcpu *vcpu)
int kvm_hv_hypercall(struct kvm_vcpu *vcpu);
void kvm_hv_irq_routing_update(struct kvm *kvm);
int kvm_hv_synic_set_irq(struct kvm *kvm, u32 vcpu_id, u32 sint);
int kvm_hv_synic_set_irq(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
int irq_source_id, int level, bool line_status);
void kvm_hv_synic_send_eoi(struct kvm_vcpu *vcpu, int vector);
int kvm_hv_activate_synic(struct kvm_vcpu *vcpu, bool dont_zero_synic_pages);
arch/x86/kvm/i8254.c
+78 -12
View File
@@ -248,8 +248,8 @@ static void pit_do_work(struct kthread_work *work)
if (atomic_read(&ps->reinject) && !atomic_xchg(&ps->irq_ack, 0))
return;
kvm_set_irq(kvm, pit->irq_source_id, 0, 1, false);
kvm_set_irq(kvm, pit->irq_source_id, 0, 0, false);
kvm_set_irq(kvm, KVM_PIT_IRQ_SOURCE_ID, 0, 1, false);
kvm_set_irq(kvm, KVM_PIT_IRQ_SOURCE_ID, 0, 0, false);
/*
* Provides NMI watchdog support via Virtual Wire mode.
@@ -288,7 +288,7 @@ static inline void kvm_pit_reset_reinject(struct kvm_pit *pit)
atomic_set(&pit->pit_state.irq_ack, 1);
}
void kvm_pit_set_reinject(struct kvm_pit *pit, bool reinject)
static void kvm_pit_set_reinject(struct kvm_pit *pit, bool reinject)
{
struct kvm_kpit_state *ps = &pit->pit_state;
struct kvm *kvm = pit->kvm;
@@ -400,8 +400,8 @@ static void pit_load_count(struct kvm_pit *pit, int channel, u32 val)
}
}
void kvm_pit_load_count(struct kvm_pit *pit, int channel, u32 val,
int hpet_legacy_start)
static void kvm_pit_load_count(struct kvm_pit *pit, int channel, u32 val,
int hpet_legacy_start)
{
u8 saved_mode;
@@ -649,6 +649,79 @@ static void pit_mask_notifer(struct kvm_irq_mask_notifier *kimn, bool mask)
kvm_pit_reset_reinject(pit);
}
int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps)
{
struct kvm_kpit_state *kps = &kvm->arch.vpit->pit_state;
BUILD_BUG_ON(sizeof(*ps) != sizeof(kps->channels));
mutex_lock(&kps->lock);
memcpy(ps, &kps->channels, sizeof(*ps));
mutex_unlock(&kps->lock);
return 0;
}
int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps)
{
int i;
struct kvm_pit *pit = kvm->arch.vpit;
mutex_lock(&pit->pit_state.lock);
memcpy(&pit->pit_state.channels, ps, sizeof(*ps));
for (i = 0; i < 3; i++)
kvm_pit_load_count(pit, i, ps->channels[i].count, 0);
mutex_unlock(&pit->pit_state.lock);
return 0;
}
int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
{
mutex_lock(&kvm->arch.vpit->pit_state.lock);
memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels,
sizeof(ps->channels));
ps->flags = kvm->arch.vpit->pit_state.flags;
mutex_unlock(&kvm->arch.vpit->pit_state.lock);
memset(&ps->reserved, 0, sizeof(ps->reserved));
return 0;
}
int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
{
int start = 0;
int i;
u32 prev_legacy, cur_legacy;
struct kvm_pit *pit = kvm->arch.vpit;
mutex_lock(&pit->pit_state.lock);
prev_legacy = pit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY;
cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY;
if (!prev_legacy && cur_legacy)
start = 1;
memcpy(&pit->pit_state.channels, &ps->channels,
sizeof(pit->pit_state.channels));
pit->pit_state.flags = ps->flags;
for (i = 0; i < 3; i++)
kvm_pit_load_count(pit, i, pit->pit_state.channels[i].count,
start && i == 0);
mutex_unlock(&pit->pit_state.lock);
return 0;
}
int kvm_vm_ioctl_reinject(struct kvm *kvm, struct kvm_reinject_control *control)
{
struct kvm_pit *pit = kvm->arch.vpit;
/* pit->pit_state.lock was overloaded to prevent userspace from getting
* an inconsistent state after running multiple KVM_REINJECT_CONTROL
* ioctls in parallel. Use a separate lock if that ioctl isn't rare.
*/
mutex_lock(&pit->pit_state.lock);
kvm_pit_set_reinject(pit, control->pit_reinject);
mutex_unlock(&pit->pit_state.lock);
return 0;
}
static const struct kvm_io_device_ops pit_dev_ops = {
.read = pit_ioport_read,
.write = pit_ioport_write,
@@ -671,10 +744,6 @@ struct kvm_pit *kvm_create_pit(struct kvm *kvm, u32 flags)
if (!pit)
return NULL;
pit->irq_source_id = kvm_request_irq_source_id(kvm);
if (pit->irq_source_id < 0)
goto fail_request;
mutex_init(&pit->pit_state.lock);
pid = get_pid(task_tgid(current));
@@ -726,8 +795,6 @@ fail_register_pit:
kvm_pit_set_reinject(pit, false);
kthread_destroy_worker(pit->worker);
fail_kthread:
kvm_free_irq_source_id(kvm, pit->irq_source_id);
fail_request:
kfree(pit);
return NULL;
}
@@ -744,7 +811,6 @@ void kvm_free_pit(struct kvm *kvm)
kvm_pit_set_reinject(pit, false);
hrtimer_cancel(&pit->pit_state.timer);
kthread_destroy_worker(pit->worker);
kvm_free_irq_source_id(kvm, pit->irq_source_id);
kfree(pit);
}
}
Loaded 100 of 209 files, more files were not shown because too many files have changed in this diff. Show more