diff --git a/CREDITS b/CREDITS index 17962bdd6dbd..42a79083b98e 100644 --- a/CREDITS +++ b/CREDITS @@ -2804,6 +2804,7 @@ D: Some of PAS 16 mixer & PCM support, inet6-apps N: William (Bill) Metzenthen E: billm@suburbia.net +E: billm@melbpc.org.au D: Author of the FPU emulator. D: Minor kernel hacker for other lost causes (Hercules mono, etc). S: 22 Parker Street diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 947697ac93c1..da6026fa695c 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -4424,10 +4424,6 @@ Kernel parameters These settings can be accessed at runtime via the nmi_watchdog and hardlockup_panic sysctls. - no387 [BUGS=X86-32] Tells the kernel to use the 387 maths - emulation library even if a 387 maths coprocessor - is present. - no4lvl [RISCV,EARLY] Disable 4-level and 5-level paging modes. Forces kernel to use 3-level paging instead. diff --git a/Documentation/arch/arm64/elf_hwcaps.rst b/Documentation/arch/arm64/elf_hwcaps.rst index 97315ae6c0da..f60ca5612daa 100644 --- a/Documentation/arch/arm64/elf_hwcaps.rst +++ b/Documentation/arch/arm64/elf_hwcaps.rst @@ -451,6 +451,33 @@ HWCAP3_LS64 of CPU. User should only use ld64b/st64b on supported target (device) memory location, otherwise fallback to the non-atomic alternatives. +HWCAP3_SVE_B16MM + Functionality implied by ID_AA64PFR0_EL1.SVE == 0b0001 and + ID_AA64ZFR0_EL1.B16B16 == 0b0011 + +HWCAP3_SVE2P3 + Functionality implied by ID_AA64PFR0_EL1.SVE == 0b0001 and + ID_AA64ZFR0_EL1.SVEver == 0b0100 + +HWCAP3_SME_LUT6 + Functionality implied by ID_AA64SMFR0_EL1.LUT6 == 0b1 + +HWCAP3_SME2P3 + Functionality implied by ID_AA64SMFR0_EL1.SMEver == 0b0100 + +HWCAP3_F16MM + Functionality implied by ID_AA64FPFR0_EL1.F16MM2 == 0b1 + +HWCAP3_F16F32DOT + Functionality implied by ID_AA64ISAR0_EL1.FHM == 0b0010 + +HWCAP3_F16F32MM + Functionality implied by ID_AA64ISAR0_EL1.FHM == 0b0011 + +HWCAP3_SVE_LUT6 + Functionality implied by ID_AA64PFR0_EL1.SVE == 0b0001 and + ID_AA64ISAR2_EL1.LUT == 0b0010. + 4. Unused AT_HWCAP bits ----------------------- diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index 211119ce7adc..014aa1c215a1 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -116,7 +116,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A73 | #858921 | ARM64_ERRATUM_858921 | +----------------+-----------------+-----------------+-----------------------------+ -| ARM | Cortex-A76 | #1188873,1418040| ARM64_ERRATUM_1418040 | +| ARM | Cortex-A76 | #1188873, | ARM64_ERRATUM_1418040 | +| | | #1418040 | | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A76 | #1165522 | ARM64_ERRATUM_1165522 | +----------------+-----------------+-----------------+-----------------------------+ @@ -128,15 +129,28 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A76 | #3324349 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A76 | #4193800 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A76AE | #4193801 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A77 | #1491015 | N/A | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A77 | #1508412 | ARM64_ERRATUM_1508412 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A77 | #3324348 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A77 | #4193798 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A78 | #3324344 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ -| ARM | Cortex-A78C | #3324346,3324347| ARM64_ERRATUM_3194386 | +| ARM | Cortex-A78 | #4193791 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A78AE | #4193793 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A78C | #3324346, | ARM64_ERRATUM_3194386 | +| | | #3324347 | | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A78C | #4193794 | ARM64_ERRATUM_4118414 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A710 | #2119858 | ARM64_ERRATUM_2119858 | +----------------+-----------------+-----------------+-----------------------------+ @@ -146,6 +160,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A710 | #3324338 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A710 | #4193788 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A715 | #2645198 | ARM64_ERRATUM_2645198 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A715 | #3456084 | ARM64_ERRATUM_3194386 | @@ -158,30 +174,44 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X1 | #3324344 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X1 | #4193791 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X1C | #3324346 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X1C | #4193792 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X2 | #2119858 | ARM64_ERRATUM_2119858 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X2 | #2224489 | ARM64_ERRATUM_2224489 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X2 | #3324338 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X2 | #4193788 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X3 | #3324335 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X3 | #4193786 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X4 | #3194386 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X4 | #4118414 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X925 | #3324334 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ -| ARM | Neoverse-N1 | #1188873,1418040| ARM64_ERRATUM_1418040 | +| ARM | Cortex-X925 | #4193781 | ARM64_ERRATUM_4118414 | +----------------+-----------------+-----------------+-----------------------------+ -| ARM | Neoverse-N1 | #1349291 | N/A | +| ARM | Neoverse-N1 | #1188873, | ARM64_ERRATUM_1418040 | +| | | #1418040 | | +----------------+-----------------+-----------------+-----------------------------+ -| ARM | Neoverse-N1 | #1490853 | N/A | +| ARM | Neoverse-N1 | #1349291, | N/A | +| | | #1490853 | | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N1 | #1542419 | ARM64_ERRATUM_1542419 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N1 | #3324349 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-N1 | #4193800 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N2 | #2139208 | ARM64_ERRATUM_2139208 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N2 | #2067961 | ARM64_ERRATUM_2067961 | @@ -190,24 +220,41 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N2 | #3324339 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-N2 | #4193789 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N3 | #3456111 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V1 | #1619801 | N/A | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V1 | #3324341 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-V1 | #4193790 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V2 | #3324336 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-V2 | #4193787 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V3 | #3312417 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-V3 | #4193784 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V3AE | #3312417 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-V3AE | #4193784 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | C1-Premium | #4193780 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | C1-Pro | #4193714 | ARM64_ERRATUM_4193714 | +----------------+-----------------+-----------------+-----------------------------+ -| ARM | MMU-500 | #841119,826419 | ARM_SMMU_MMU_500_CPRE_ERRATA| -| | | #562869,1047329 | | +| ARM | C1-Ultra | #4193780 | ARM64_ERRATUM_4118414 | +----------------+-----------------+-----------------+-----------------------------+ -| ARM | MMU-600 | #1076982,1209401| N/A | +| ARM | MMU-500 | #562869, | ARM_SMMU_MMU_500_CPRE_ERRATA| +| | | #841119, | | +| | | #826419, | | +| | | #1047329 | | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | MMU-600 | #1076982, | N/A | +| | | #1209401 | | +----------------+-----------------+-----------------+-----------------------------+ | ARM | MMU-700 | #2133013, | N/A | | | | #2268618, | | @@ -230,11 +277,13 @@ stable kernels. | Broadcom | Brahma-B53 | N/A | ARM64_ERRATUM_843419 | +----------------+-----------------+-----------------+-----------------------------+ +----------------+-----------------+-----------------+-----------------------------+ -| Cavium | ThunderX ITS | #22375,24313 | CAVIUM_ERRATUM_22375 | +| Cavium | ThunderX ITS | #22375, | CAVIUM_ERRATUM_22375 | +| | | #24313 | | +----------------+-----------------+-----------------+-----------------------------+ | Cavium | ThunderX ITS | #23144 | CAVIUM_ERRATUM_23144 | +----------------+-----------------+-----------------+-----------------------------+ -| Cavium | ThunderX GICv3 | #23154,38545 | CAVIUM_ERRATUM_23154 | +| Cavium | ThunderX GICv3 | #23154, | CAVIUM_ERRATUM_23154 | +| | | #38545 | | +----------------+-----------------+-----------------+-----------------------------+ | Cavium | ThunderX GICv3 | #38539 | N/A | +----------------+-----------------+-----------------+-----------------------------+ @@ -244,9 +293,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | Cavium | ThunderX SMMUv2 | #27704 | N/A | +----------------+-----------------+-----------------+-----------------------------+ -| Cavium | ThunderX2 SMMUv3| #74 | N/A | -+----------------+-----------------+-----------------+-----------------------------+ -| Cavium | ThunderX2 SMMUv3| #126 | N/A | +| Cavium | ThunderX2 SMMUv3| #74, | N/A | +| | | #126 | | +----------------+-----------------+-----------------+-----------------------------+ | Cavium | ThunderX2 Core | #219 | CAVIUM_TX2_ERRATUM_219 | +----------------+-----------------+-----------------+-----------------------------+ @@ -256,13 +304,13 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | NVIDIA | Carmel Core | N/A | NVIDIA_CARMEL_CNP_ERRATUM | +----------------+-----------------+-----------------+-----------------------------+ +| NVIDIA | Olympus core | T410-OLY-1029 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | NVIDIA | T241 GICv3/4.x | T241-FABRIC-4 | N/A | +----------------+-----------------+-----------------+-----------------------------+ -| NVIDIA | T241 MPAM | T241-MPAM-1 | N/A | -+----------------+-----------------+-----------------+-----------------------------+ -| NVIDIA | T241 MPAM | T241-MPAM-4 | N/A | -+----------------+-----------------+-----------------+-----------------------------+ -| NVIDIA | T241 MPAM | T241-MPAM-6 | N/A | +| NVIDIA | T241 MPAM | T241-MPAM-1, | N/A | +| | | T241-MPAM-4, | | +| | | T241-MPAM-6 | | +----------------+-----------------+-----------------+-----------------------------+ +----------------+-----------------+-----------------+-----------------------------+ | Freescale/NXP | LS2080A/LS1043A | A-008585 | FSL_ERRATUM_A008585 | @@ -270,9 +318,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | Hisilicon | Hip0{5,6,7} | #161010101 | HISILICON_ERRATUM_161010101 | +----------------+-----------------+-----------------+-----------------------------+ -| Hisilicon | Hip0{6,7} | #161010701 | N/A | -+----------------+-----------------+-----------------+-----------------------------+ -| Hisilicon | Hip0{6,7} | #161010803 | N/A | +| Hisilicon | Hip0{6,7} | #161010701, | N/A | +| | | #161010803 | | +----------------+-----------------+-----------------+-----------------------------+ | Hisilicon | Hip07 | #161600802 | HISILICON_ERRATUM_161600802 | +----------------+-----------------+-----------------+-----------------------------+ @@ -284,6 +331,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | Hisilicon | Hip09 | #162100801 | HISILICON_ERRATUM_162100801 | +----------------+-----------------+-----------------+-----------------------------+ +| Hisilicon | Hip09 | #162100125 | HISILICON_ERRATUM_162100125 | ++----------------+-----------------+-----------------+-----------------------------+ +----------------+-----------------+-----------------+-----------------------------+ | Qualcomm Tech. | Kryo/Falkor v1 | E1003 | QCOM_FALKOR_ERRATUM_1003 | +----------------+-----------------+-----------------+-----------------------------+ @@ -323,3 +372,5 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | Microsoft | Azure Cobalt 100| #3324339 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| Microsoft | Azure Cobalt 100| #4193789 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ diff --git a/Documentation/arch/s390/s390dbf.rst b/Documentation/arch/s390/s390dbf.rst index aad6d88974fe..034374c88dba 100644 --- a/Documentation/arch/s390/s390dbf.rst +++ b/Documentation/arch/s390/s390dbf.rst @@ -106,6 +106,36 @@ the ``debug_stoppable`` sysctl. If you set ``debug_stoppable`` to 0 the debug feature cannot be stopped. If the debug feature is already stopped, it will stay deactivated. +Kernel parameters +----------------- +The size and log levels of debug logs can be configured early during boot using +the ``s390dbf`` kernel parameter. The parameter accepts a debug log name, a log +level, and a log size, separated by colon characters (``:``). To configure only +a single attribute, either the log level or the log size may be omitted. + +To configure multiple debug logs, the parameter may be specified multiple times, +or multiple parameter sets may be provided in a single instance, separated by +commas. + +Parameter format:: + + s390dbf=:[|-]:[][,...] + +where: + +- ``name`` specifies either an exact debug log name or a shell-style wildcard + pattern +- ``level`` specifies the log level, or ``-`` to completely deactivate the log +- ``pages`` specifies the debug area size in pages + +Example:: + + s390dbf=cio*:6:128,sclp_err::2 + +This example sets the log level to 6 and the log size to 128 pages for all debug +logs whose names start with ``cio``. It also sets the log level of the +``sclp_err`` debug log to 2. + Kernel Interfaces: ------------------ diff --git a/Documentation/core-api/cpu_hotplug.rst b/Documentation/core-api/cpu_hotplug.rst index 9b4afca9fd09..6de26d1c6a9a 100644 --- a/Documentation/core-api/cpu_hotplug.rst +++ b/Documentation/core-api/cpu_hotplug.rst @@ -45,11 +45,6 @@ Command Line Switches This option is limited to the X86 and S390 architecture. -``cpu0_hotplug`` - Allow to shutdown CPU0. - - This option is limited to the X86 architecture. - CPU maps ======== diff --git a/Documentation/filesystems/resctrl.rst b/Documentation/filesystems/resctrl.rst index b003bed339fd..e4b66af55ffb 100644 --- a/Documentation/filesystems/resctrl.rst +++ b/Documentation/filesystems/resctrl.rst @@ -427,9 +427,9 @@ with the following files: Two MBM events are supported by default: mbm_local_bytes and mbm_total_bytes. Each MBM event's sub-directory contains a file named "event_filter" that is - used to view and modify which memory transactions the MBM event is configured - with. The file is accessible only when "mbm_event" counter assignment mode is - enabled. + used to view and (if writable) modify which memory transactions the MBM event + is configured with. The file is accessible only when "mbm_event" counter + assignment mode is enabled. List of memory transaction types supported: @@ -454,9 +454,8 @@ with the following files: # cat /sys/fs/resctrl/info/L3_MON/event_configs/mbm_local_bytes/event_filter local_reads,local_non_temporal_writes,local_reads_slow_memory - Modify the event configuration by writing to the "event_filter" file within - the "event_configs" directory. The read/write "event_filter" file contains the - configuration of the event that reflects which memory transactions are counted by it. + The memory transactions the MBM event is configured with can be changed + if "event_filter" is writable. For example:: @@ -480,6 +479,12 @@ with the following files: "1": Auto assignment is enabled. + Automatic counter assignment is done with best effort. If auto + assignment is enabled but there are not enough available counters then + monitor group creation could succeed while one or more events belonging + to the group may not have a counter assigned in all domains. Consult + mbm_L3_assignments for counter assignment states of the new groups. + Example:: # echo 0 > /sys/fs/resctrl/info/L3_MON/mbm_assign_on_mkdir @@ -570,6 +575,11 @@ All groups contain the following files: then the task must already belong to the CTRL_MON parent of this group. The task is removed from any previous MON group. + When writing to this file, a task id of 0 is interpreted as the + task id of the currently running task. On reading the file, a task + id of 0 will never be shown and there is no representation of the + idle tasks. Instead, a CPU's idle task is always considered as a + member of the group owning the CPU. "cpus": Reading this file shows a bitmask of the logical CPUs owned by diff --git a/Documentation/rust/arch-support.rst b/Documentation/rust/arch-support.rst index 6e6a515d0899..4f980815e92a 100644 --- a/Documentation/rust/arch-support.rst +++ b/Documentation/rust/arch-support.rst @@ -19,6 +19,7 @@ Architecture Level of support Constraints ``arm64`` Maintained Little Endian only. ``loongarch`` Maintained \- ``riscv`` Maintained ``riscv64`` and LLVM/Clang only. +``s390`` Maintained ``CONFIG_EXPOLINE`` must be disabled. ``um`` Maintained \- ``x86`` Maintained ``x86_64`` only. ============= ================ ============================================== diff --git a/Documentation/trace/ftrace.rst b/Documentation/trace/ftrace.rst index 2ed1b96e440b..84f06bf0da9b 100644 --- a/Documentation/trace/ftrace.rst +++ b/Documentation/trace/ftrace.rst @@ -570,6 +570,10 @@ of ftrace. Here is a list of some of the key files: to correlate events across hypervisor/guest if tb_offset is known. + s390-tod: + This uses the s390 TOD clock value. This clock is usually in + sync across virtual machines and STP-enabled machines. + mono: This uses the fast monotonic clock (CLOCK_MONOTONIC) which is monotonic and is subject to NTP rate adjustments. diff --git a/MAINTAINERS b/MAINTAINERS index 3c776cfe2eaf..1c2efe5b8f21 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1122,10 +1122,9 @@ S: Orphan F: drivers/usb/gadget/udc/amd5536udc.* AMD GEODE PROCESSOR/CHIPSET SUPPORT -M: Andres Salomon L: linux-geode@lists.infradead.org (moderated for non-subscribers) -S: Supported -W: http://www.amd.com/us-en/ConnectivitySolutions/TechnicalResources/0,,50_2334_2452_11363,00.html +S: Orphaned +F: arch/x86/platform/geode/ F: arch/x86/include/asm/geode.h F: drivers/char/hw_random/geode-rng.c F: drivers/crypto/geode* @@ -10117,12 +10116,6 @@ F: Documentation/fpga/ F: drivers/fpga/ F: include/linux/fpga/ -FPU EMULATOR -M: Bill Metzenthen -S: Maintained -W: https://floatingpoint.billm.au/ -F: arch/x86/math-emu/ - FRAMEBUFFER CONSOLE M: Helge Deller M: Thomas Zimmermann @@ -11643,7 +11636,7 @@ F: Documentation/devicetree/bindings/net/hisilicon*.txt F: drivers/net/ethernet/hisilicon/ HISILICON PMU DRIVER -M: Jonathan Cameron +M: Yushan Wang S: Supported W: http://www.hisilicon.com F: Documentation/admin-guide/perf/hisi-pcie-pmu.rst @@ -28907,6 +28900,7 @@ R: Ahmed S. Darwish L: x86-cpuid@lists.linux.dev S: Maintained W: https://x86-cpuid.org +F: arch/x86/include/asm/cpuid/leaf_types.h F: tools/arch/x86/kcpuid/ X86 ENTRY CODE @@ -28926,6 +28920,7 @@ S: Maintained F: Documentation/admin-guide/hw-vuln/ F: arch/x86/include/asm/nospec-branch.h F: arch/x86/kernel/cpu/bugs.c +F: scripts/update-intel-ucode-defs.py X86 MCE INFRASTRUCTURE M: Tony Luck diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b0f3098c0388..22625c033ce7 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1153,6 +1153,44 @@ config ARM64_ERRATUM_4193714 If unsure, say Y. +config ARM64_ERRATUM_4118414 + bool "Various: Completion of affected memory accesses might not be guaranteed by completion of a TLBI" + default y + select ARM64_WORKAROUND_REPEAT_TLBI + help + This option adds a workaround for the following errata: + + * ARM C1-Premium erratum 4193780 + * ARM C1-Ultra erratum 4193780 + * ARM Cortex-A76 erratum 4193800 + * ARM Cortex-A76AE erratum 4193801 + * ARM Cortex-A77 erratum 4193798 + * ARM Cortex-A78 erratum 4193791 + * ARM Cortex-A78AE erratum 4193793 + * ARM Cortex-A78C erratum 4193794 + * ARM Cortex-A710 erratum 4193788 + * ARM Cortex-X1 erratum 4193791 + * ARM Cortex-X1C erratum 4193792 + * ARM Cortex-X2 erratum 4193788 + * ARM Cortex-X3 erratum 4193786 + * ARM Cortex-X4 erratum 4118414 + * ARM Cortex-X925 erratum 4193781 + * ARM Neoverse-N1 erratum 4193800 + * ARM Neoverse-N2 erratum 4193789 + * ARM Neoverse-V1 erratum 4193790 + * ARM Neoverse-V2 erratum 4193787 + * ARM Neoverse-V3 erratum 4193784 + * ARM Neoverse-V3AE erratum 4193784 + * Microsoft Azure Cobalt 100 4193789 + * NVIDIA Olympus erratum T410-OLY-1029 + + On affected cores, some memory accesses might not be completed by + broadcast TLB invalidation. + + This issue is also known as CVE-2025-10263. + + If unsure, say Y. + config CAVIUM_ERRATUM_22375 bool "Cavium erratum 22375, 24313" default y @@ -1272,6 +1310,22 @@ config HISILICON_ERRATUM_162100801 If unsure, say Y. +config HISILICON_ERRATUM_162100125 + bool "Hisilicon erratum 162100125" + default y + select ARM64_WORKAROUND_DISABLE_CNP + help + On HiSilicon HIP09, TLB entry matching behavior when CNP + (TTBRx.CNP=1) is enabled differs from the ARM architecture + specification. + + TLB entries may be incorrectly shared between CPUs, potentially + causing TLB conflicts and stale mappings. + + Disable CNP support for affected HiSilicon HIP09 cores. + + If unsure, say Y. + config QCOM_FALKOR_ERRATUM_1003 bool "Falkor E1003: Incorrect translation due to ASID change" default y @@ -1314,9 +1368,13 @@ config QCOM_FALKOR_ERRATUM_E1041 If unsure, say Y. +config ARM64_WORKAROUND_DISABLE_CNP + bool + config NVIDIA_CARMEL_CNP_ERRATUM bool "NVIDIA Carmel CNP: CNP on Carmel semantically different than ARM cores" default y + select ARM64_WORKAROUND_DISABLE_CNP help If CNP is enabled on Carmel cores, non-sharable TLBIs on a core will not invalidate shared TLB entries installed by a different core, as it would @@ -2246,10 +2304,15 @@ config ARM64_SVE booting the kernel. If unsure and you are not observing these symptoms, you should assume that it is safe to say Y. +config AS_HAS_SME + # Supported by LLVM 13+ and binutils 2.38+ + def_bool $(as-instr,.arch_extension sme) + config ARM64_SME bool "ARM Scalable Matrix Extension support" default y depends on ARM64_SVE + depends on AS_HAS_SME help The Scalable Matrix Extension (SME) is an extension to the AArch64 execution state which utilises a substantial subset of the SVE diff --git a/arch/arm64/include/asm/arch_timer.h b/arch/arm64/include/asm/arch_timer.h index f5794d50f51d..6717209df05b 100644 --- a/arch/arm64/include/asm/arch_timer.h +++ b/arch/arm64/include/asm/arch_timer.h @@ -178,12 +178,8 @@ static __always_inline u64 __arch_counter_get_cntpct_stable(void) static __always_inline u64 __arch_counter_get_cntpct(void) { - u64 cnt; + u64 cnt = arch_timer_read_cntpct_el0(); - asm volatile(ALTERNATIVE("isb\n mrs %0, cntpct_el0", - "nop\n" __mrs_s("%0", SYS_CNTPCTSS_EL0), - ARM64_HAS_ECV) - : "=r" (cnt)); arch_counter_enforce_ordering(cnt); return cnt; } @@ -199,12 +195,8 @@ static __always_inline u64 __arch_counter_get_cntvct_stable(void) static __always_inline u64 __arch_counter_get_cntvct(void) { - u64 cnt; + u64 cnt = arch_timer_read_cntvct_el0(); - asm volatile(ALTERNATIVE("isb\n mrs %0, cntvct_el0", - "nop\n" __mrs_s("%0", SYS_CNTVCTSS_EL0), - ARM64_HAS_ECV) - : "=r" (cnt)); arch_counter_enforce_ordering(cnt); return cnt; } diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h index d0d3cdd5763c..25c61cda901c 100644 --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -58,8 +58,8 @@ cpucap_is_possible(const unsigned int cap) return IS_ENABLED(CONFIG_ARM64_ERRATUM_2658417); case ARM64_WORKAROUND_CAVIUM_23154: return IS_ENABLED(CONFIG_CAVIUM_ERRATUM_23154); - case ARM64_WORKAROUND_NVIDIA_CARMEL_CNP: - return IS_ENABLED(CONFIG_NVIDIA_CARMEL_CNP_ERRATUM); + case ARM64_WORKAROUND_DISABLE_CNP: + return IS_ENABLED(CONFIG_ARM64_WORKAROUND_DISABLE_CNP); case ARM64_WORKAROUND_REPEAT_TLBI: return IS_ENABLED(CONFIG_ARM64_WORKAROUND_REPEAT_TLBI); case ARM64_WORKAROUND_SPECULATIVE_SSBS: diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 4de51f8d92cb..a57870fa96db 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -620,6 +620,13 @@ static inline bool id_aa64pfr0_mpam(u64 pfr0) return val > 0; } +static inline bool id_aa64pfr1_mpamfrac(u64 pfr1) +{ + u32 val = cpuid_feature_extract_unsigned_field(pfr1, ID_AA64PFR1_EL1_MPAM_frac_SHIFT); + + return val > 0; +} + static inline bool id_aa64pfr1_mte(u64 pfr1) { u32 val = cpuid_feature_extract_unsigned_field(pfr1, ID_AA64PFR1_EL1_MTE_SHIFT); diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 7b518e81dd15..1b9f0cda1336 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -97,8 +97,10 @@ #define ARM_CPU_PART_CORTEX_X925 0xD85 #define ARM_CPU_PART_CORTEX_A725 0xD87 #define ARM_CPU_PART_CORTEX_A720AE 0xD89 +#define ARM_CPU_PART_C1_ULTRA 0xD8C #define ARM_CPU_PART_NEOVERSE_N3 0xD8E #define ARM_CPU_PART_C1_PRO 0xD8B +#define ARM_CPU_PART_C1_PREMIUM 0xD90 #define APM_CPU_PART_XGENE 0x000 #define APM_CPU_VAR_POTENZA 0x00 @@ -189,8 +191,10 @@ #define MIDR_CORTEX_X925 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X925) #define MIDR_CORTEX_A725 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A725) #define MIDR_CORTEX_A720AE MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A720AE) +#define MIDR_C1_ULTRA MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_ULTRA) #define MIDR_NEOVERSE_N3 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N3) #define MIDR_C1_PRO MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_PRO) +#define MIDR_C1_PREMIUM MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_PREMIUM) #define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) #define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX) #define MIDR_THUNDERX_83XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX) diff --git a/arch/arm64/include/asm/daifflags.h b/arch/arm64/include/asm/daifflags.h index 5fca48009043..795b35128467 100644 --- a/arch/arm64/include/asm/daifflags.h +++ b/arch/arm64/include/asm/daifflags.h @@ -19,7 +19,7 @@ /* mask/save/unmask/restore all exceptions, including interrupts. */ -static inline void local_daif_mask(void) +static __always_inline void local_daif_mask(void) { WARN_ON(system_has_prio_mask_debugging() && (read_sysreg_s(SYS_ICC_PMR_EL1) == (GIC_PRIO_IRQOFF | @@ -38,7 +38,7 @@ static inline void local_daif_mask(void) trace_hardirqs_off(); } -static inline unsigned long local_daif_save_flags(void) +static __always_inline unsigned long local_daif_save_flags(void) { unsigned long flags; @@ -53,7 +53,7 @@ static inline unsigned long local_daif_save_flags(void) return flags; } -static inline unsigned long local_daif_save(void) +static __always_inline unsigned long local_daif_save(void) { unsigned long flags; @@ -64,7 +64,7 @@ static inline unsigned long local_daif_save(void) return flags; } -static inline void local_daif_restore(unsigned long flags) +static __always_inline void local_daif_restore(unsigned long flags) { bool irq_disabled = flags & PSR_I_BIT; @@ -124,7 +124,7 @@ static inline void local_daif_restore(unsigned long flags) * Called by synchronous exception handlers to restore the DAIF bits that were * modified by taking an exception. */ -static inline void local_daif_inherit(struct pt_regs *regs) +static __always_inline void local_daif_inherit(struct pt_regs *regs) { unsigned long flags = regs->pstate & DAIF_MASK; diff --git a/arch/arm64/include/asm/device.h b/arch/arm64/include/asm/device.h deleted file mode 100644 index 996498751318..000000000000 --- a/arch/arm64/include/asm/device.h +++ /dev/null @@ -1,14 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2012 ARM Ltd. - */ -#ifndef __ASM_DEVICE_H -#define __ASM_DEVICE_H - -struct dev_archdata { -}; - -struct pdev_archdata { -}; - -#endif diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h index 587507a9980e..aa8ec9df8024 100644 --- a/arch/arm64/include/asm/el2_setup.h +++ b/arch/arm64/include/asm/el2_setup.h @@ -510,7 +510,9 @@ #endif .macro finalise_el2_state - check_override id_aa64pfr0, ID_AA64PFR0_EL1_MPAM_SHIFT, .Linit_mpam_\@, .Lskip_mpam_\@, x1, x2 + check_override id_aa64pfr0, ID_AA64PFR0_EL1_MPAM_SHIFT, .Linit_mpam_\@, .Lmpam_minor_\@, x1, x2 +.Lmpam_minor_\@: + check_override id_aa64pfr1, ID_AA64PFR1_EL1_MPAM_frac_SHIFT, .Linit_mpam_\@, .Lskip_mpam_\@, x1, x2 .Linit_mpam_\@: mov x0, #MPAM2_EL2_EnMPAMSM_MASK diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h index 7e86d400864e..81c17320a588 100644 --- a/arch/arm64/include/asm/esr.h +++ b/arch/arm64/include/asm/esr.h @@ -160,6 +160,8 @@ #define ESR_ELx_CM (UL(1) << ESR_ELx_CM_SHIFT) /* ISS2 field definitions for Data Aborts */ +#define ESR_ELx_HDBSSF_SHIFT (11) +#define ESR_ELx_HDBSSF (UL(1) << ESR_ELx_HDBSSF_SHIFT) #define ESR_ELx_TnD_SHIFT (10) #define ESR_ELx_TnD (UL(1) << ESR_ELx_TnD_SHIFT) #define ESR_ELx_TagAccess_SHIFT (9) diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index d9d00b45ab11..a67d5774e672 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -22,6 +22,11 @@ #include #include +#define __FPSIMD_PREAMBLE ".arch_extension fp\n" \ + ".arch_extension simd\n" +#define __SVE_PREAMBLE ".arch_extension sve\n" +#define __SME_PREAMBLE ".arch_extension sme\n" + /* Masks for extracting the FPSR and FPCR from the FPSCR */ #define VFP_FPSCR_STAT_MASK 0xf800009f #define VFP_FPSCR_CTRL_MASK 0x07f79f00 @@ -71,8 +76,82 @@ static inline void cpacr_restore(unsigned long cpacr) struct task_struct; -extern void fpsimd_save_state(struct user_fpsimd_state *state); -extern void fpsimd_load_state(struct user_fpsimd_state *state); +static inline void fpsimd_save_common(struct user_fpsimd_state *state) +{ + state->fpsr = read_sysreg_s(SYS_FPSR); + state->fpcr = read_sysreg_s(SYS_FPCR); +} + +static inline void fpsimd_load_common(const struct user_fpsimd_state *state) +{ + write_sysreg_s(state->fpsr, SYS_FPSR); + write_sysreg_s(state->fpcr, SYS_FPCR); +} + +static inline void fpsimd_save_vregs(struct user_fpsimd_state *state) +{ + instrument_write(state->vregs, sizeof(state->vregs)); + asm volatile( + __FPSIMD_PREAMBLE + " stp q0, q1, [%[vregs], #16 * 0]\n" + " stp q2, q3, [%[vregs], #16 * 2]\n" + " stp q4, q5, [%[vregs], #16 * 4]\n" + " stp q6, q7, [%[vregs], #16 * 6]\n" + " stp q8, q9, [%[vregs], #16 * 8]\n" + " stp q10, q11, [%[vregs], #16 * 10]\n" + " stp q12, q13, [%[vregs], #16 * 12]\n" + " stp q14, q15, [%[vregs], #16 * 14]\n" + " stp q16, q17, [%[vregs], #16 * 16]\n" + " stp q18, q19, [%[vregs], #16 * 18]\n" + " stp q20, q21, [%[vregs], #16 * 20]\n" + " stp q22, q23, [%[vregs], #16 * 22]\n" + " stp q24, q25, [%[vregs], #16 * 24]\n" + " stp q26, q27, [%[vregs], #16 * 26]\n" + " stp q28, q29, [%[vregs], #16 * 28]\n" + " stp q30, q31, [%[vregs], #16 * 30]\n" + : "=Q" (state->vregs) + : [vregs] "r" (state->vregs) + ); +} + +static inline void fpsimd_load_vregs(const struct user_fpsimd_state *state) +{ + instrument_read(state->vregs, sizeof(state->vregs)); + asm volatile( + __FPSIMD_PREAMBLE + " ldp q0, q1, [%[vregs], #16 * 0]\n" + " ldp q2, q3, [%[vregs], #16 * 2]\n" + " ldp q4, q5, [%[vregs], #16 * 4]\n" + " ldp q6, q7, [%[vregs], #16 * 6]\n" + " ldp q8, q9, [%[vregs], #16 * 8]\n" + " ldp q10, q11, [%[vregs], #16 * 10]\n" + " ldp q12, q13, [%[vregs], #16 * 12]\n" + " ldp q14, q15, [%[vregs], #16 * 14]\n" + " ldp q16, q17, [%[vregs], #16 * 16]\n" + " ldp q18, q19, [%[vregs], #16 * 18]\n" + " ldp q20, q21, [%[vregs], #16 * 20]\n" + " ldp q22, q23, [%[vregs], #16 * 22]\n" + " ldp q24, q25, [%[vregs], #16 * 24]\n" + " ldp q26, q27, [%[vregs], #16 * 26]\n" + " ldp q28, q29, [%[vregs], #16 * 28]\n" + " ldp q30, q31, [%[vregs], #16 * 30]\n" + : + : "Q" (state->vregs), + [vregs] "r" (state->vregs) + ); +} + +static inline void fpsimd_save_state(struct user_fpsimd_state *state) +{ + fpsimd_save_vregs(state); + fpsimd_save_common(state); +} + +static inline void fpsimd_load_state(const struct user_fpsimd_state *state) +{ + fpsimd_load_vregs(state); + fpsimd_load_common(state); +} extern void fpsimd_thread_switch(struct task_struct *next); extern void fpsimd_flush_thread(void); @@ -83,8 +162,8 @@ extern void fpsimd_update_current_state(struct user_fpsimd_state const *state); struct cpu_fp_state { struct user_fpsimd_state *st; - void *sve_state; - void *sme_state; + struct arm64_sve_state *sve_state; + struct arm64_sme_state *sme_state; u64 *svcr; u64 *fpmr; unsigned int sve_vl; @@ -116,40 +195,166 @@ extern void task_smstop_sm(struct task_struct *task); /* Maximum VL that SVE/SME VL-agnostic software can transparently support */ #define VL_ARCH_MAX 0x100 -/* Offset of FFR in the SVE register dump */ -static inline size_t sve_ffr_offset(int vl) -{ - return SVE_SIG_FFR_OFFSET(sve_vq_from_vl(vl)) - SVE_SIG_REGS_OFFSET; -} - -static inline void *sve_pffr(struct thread_struct *thread) -{ - unsigned int vl; - - if (system_supports_sme() && thread_sm_enabled(thread)) - vl = thread_get_sme_vl(thread); - else - vl = thread_get_sve_vl(thread); - - return (char *)thread->sve_state + sve_ffr_offset(vl); -} - static inline void *thread_zt_state(struct thread_struct *thread) { /* The ZT register state is stored immediately after the ZA state */ unsigned int sme_vq = sve_vq_from_vl(thread_get_sme_vl(thread)); - return thread->sme_state + ZA_SIG_REGS_SIZE(sme_vq); + return (void *)thread->sme_state + ZA_SIG_REGS_SIZE(sme_vq); } -extern void sve_save_state(void *state, u32 *pfpsr, int save_ffr); -extern void sve_load_state(void const *state, u32 const *pfpsr, - int restore_ffr); -extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1); -extern unsigned int sve_get_vl(void); -extern void sve_set_vq(unsigned long vq_minus_1); -extern void sme_set_vq(unsigned long vq_minus_1); -extern void sme_save_state(void *state, int zt); -extern void sme_load_state(void const *state, int zt); +static inline unsigned int sve_get_vl(void) +{ + unsigned int vl; + + asm volatile( + __SVE_PREAMBLE + " rdvl %x[vl], #1\n" + : [vl] "=r" (vl) + ); + + return vl; +} + +#define FOR_EACH_Z_REG(idx_str, asm_str) \ + " .irp " idx_str ",0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31\n" \ + asm_str "\n" \ + " .endr\n" + +#define FOR_EACH_P_REG(idx_str, asm_str) \ + " .irp " idx_str ",0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15\n" \ + asm_str "\n" \ + " .endr\n" + +static inline void __sve_save_z(struct arm64_sve_state *state, unsigned long vl) +{ + instrument_write(state, SVE_NUM_ZREGS * vl); + asm volatile( + __SVE_PREAMBLE + FOR_EACH_Z_REG("n", "str z\\n, [%[zregs], #\\n, MUL VL]") + : + : [zregs] "r" (state) + : "memory" + ); +} + +static inline void __sve_load_z(const struct arm64_sve_state *state, unsigned long vl) +{ + instrument_read(state, SVE_NUM_ZREGS * vl); + asm volatile( + __SVE_PREAMBLE + FOR_EACH_Z_REG("n", "ldr z\\n, [%[zregs], #\\n, MUL VL]") + : + : [zregs] "r" (state) + : "memory" + ); +} + +static inline void __sve_save_p(struct arm64_sve_state *state, unsigned long vl, bool ffr) +{ + void *pregs = (void *)state + SVE_NUM_ZREGS * vl; + unsigned long pl = vl / 8; + void *pffr = pregs + SVE_NUM_PREGS * pl; + + instrument_write(pregs, SVE_NUM_PREGS * pl); + asm volatile( + __SVE_PREAMBLE + FOR_EACH_P_REG("n", "str p\\n, [%[pregs], #\\n, MUL VL]\n") + : + : [pregs] "r" (pregs) + : "memory" + ); + + instrument_write(pffr, pl); + if (ffr) { + asm volatile( + __SVE_PREAMBLE + " rdffr p0.b\n" + " str p0, [%[pffr]]\n" + " ldr p0, [%[pregs]]\n" + : + : [pregs] "r" (pregs), + [pffr] "r" (pffr) + : "memory" + ); + } else { + asm volatile( + __SVE_PREAMBLE + " pfalse p0.b\n" + " str p0, [%[pffr]]\n" + " ldr p0, [%[pregs]]\n" + : + : [pregs] "r" (pregs), + [pffr] "r" (pffr) + : "memory" + ); + } +} + +static inline void __sve_load_p(const struct arm64_sve_state *state, unsigned long vl, bool ffr) +{ + const void *pregs = (const void *)state + SVE_NUM_ZREGS * vl; + unsigned long pl = vl / 8; + const void *pffr = pregs + SVE_NUM_PREGS * pl; + + if (ffr) { + instrument_read(pffr, pl); + asm volatile( + __SVE_PREAMBLE + " ldr p0, [%[pffr]]\n" + " wrffr p0.b\n" + : + : [pffr] "r" (pffr) + : "memory" + ); + } + + instrument_read(pregs, SVE_NUM_PREGS * pl); + asm volatile( + __SVE_PREAMBLE + FOR_EACH_P_REG("n", "ldr p\\n, [%[pregs], #\\n, MUL VL]\n") + : + : [pregs] "r" (pregs) + : "memory" + ); +} + +static inline void sve_save_state(struct arm64_sve_state *state, bool ffr) +{ + unsigned long vl = sve_get_vl(); + __sve_save_z(state, vl); + __sve_save_p(state, vl, ffr); +} + +static inline void sve_load_state(const struct arm64_sve_state *state, bool ffr) +{ + unsigned long vl = sve_get_vl(); + __sve_load_z(state, vl); + __sve_load_p(state, vl, ffr); +} + +/* + * Zero all SVE registers except for the first 128 bits of each vector. + * + * The caller must ensure that the VL has been configured and the CPU must be + * in non-streaming mode. + */ +static inline void sve_flush_live(void) +{ + unsigned long vl = sve_get_vl(); + + if (vl > sizeof(__uint128_t)) { + asm volatile( + __FPSIMD_PREAMBLE + FOR_EACH_Z_REG("n", "mov v\\n\\().16b, v\\n\\().16b") + ); + } + + asm volatile( + __SVE_PREAMBLE + FOR_EACH_P_REG("n", "pfalse p\\n\\().b") + " wrffr p0.b\n" + ); +} struct arm64_cpu_capabilities; extern void cpu_enable_fpsimd(const struct arm64_cpu_capabilities *__unused); @@ -402,8 +607,20 @@ static inline int sme_max_virtualisable_vl(void) return vec_max_virtualisable_vl(ARM64_VEC_SME); } +static inline unsigned int sme_get_vl(void) +{ + unsigned int vl; + + asm volatile( + __SME_PREAMBLE + " rdsvl %x[vl], #1\n" + : [vl] "=r" (vl) + ); + + return vl; +} + extern void sme_alloc(struct task_struct *task, bool flush); -extern unsigned int sme_get_vl(void); extern int sme_set_current_vl(unsigned long arg); extern int sme_get_current_vl(void); extern void sme_suspend_exit(void); @@ -418,6 +635,106 @@ static inline size_t __sme_state_size(unsigned int sme_vl) return size; } +static inline void __sme_save_za(struct arm64_sme_state *state, unsigned long svl) +{ + /* + * The argument to LDR/STR (array vector) can only encode W12-W15. + * The "Ucj" constraint exists for this, but is only supported by GCC + * 14.1.0+ and LLVM 18.1.0+. + */ + register unsigned int v asm ("w12"); + + instrument_write(state, svl * svl); + for (v = 0; v < svl; v++) { + void *pav = (void *)state + v * svl; + + asm volatile( + __SME_PREAMBLE + " str za[%w[v], #0], [%[pav]]\n" + : + : [v] "r" (v), + [pav] "r" (pav) + : "memory" + ); + } +} + +static inline void __sme_load_za(const struct arm64_sme_state *state, unsigned long svl) +{ + /* See comment in __sme_save_za */ + register unsigned int v asm ("w12"); + + instrument_read(state, svl * svl); + for (v = 0; v < svl; v++) { + void *pav = (void *)state + v * svl; + + asm volatile( + __SME_PREAMBLE + " ldr za[%w[v], #0], [%[pav]]\n" + : + : [v] "r" (v), + [pav] "r" (pav) + : "memory" + ); + } +} + +static inline void __sme_save_zt(struct arm64_sme_state *state, unsigned long svl) +{ + void *pzt = (void *)state + svl * svl; + + instrument_write(pzt, 64); + asm volatile( + __DEFINE_ASM_GPR_NUMS + /* + * STR ZT0, [] + * Supported by binutils 2.41+. + * Supported by LLVM 16+ + */ + " .inst 0xe13f8000 | ((.L__gpr_num_%[pzt]) << 5)\n" + : + : [pzt] "r" (pzt) + : "memory" + ); +} + +static inline void __sme_load_zt(const struct arm64_sme_state *state, unsigned long svl) +{ + void *pzt = (void *)state + svl * svl; + + instrument_read(pzt, 64); + asm volatile( + __DEFINE_ASM_GPR_NUMS + /* + * LDR ZT0, [] + * Supported by binutils 2.41+. + * Supported by LLVM 16+ + */ + " .inst 0xe11f8000 | ((.L__gpr_num_%[pzt]) << 5)\n" + : + : [pzt] "r" (pzt) + : "memory" + ); +} + +static inline void sme_save_state(struct arm64_sme_state *state, bool zt) +{ + unsigned long svl = sme_get_vl(); + + __sme_save_za(state, svl); + if (zt) + __sme_save_zt(state, svl); +} + +static inline void sme_load_state(const struct arm64_sme_state *state, bool zt) +{ + unsigned long svl = sme_get_vl(); + + __sme_load_za(state, svl); + if (zt) + __sme_load_zt(state, svl); +} + /* * Return how many bytes of memory are required to store the full SME * specific state for task, given task's currently configured vector @@ -474,6 +791,9 @@ static inline size_t sme_state_size(struct task_struct const *task) return 0; } +static inline void sme_save_state(struct arm64_sme_state *state, bool zt) { BUILD_BUG(); } +static inline void sme_load_state(const struct arm64_sme_state *state, bool zt) { BUILD_BUG(); } + static inline void sme_enter_from_user_mode(void) { } static inline void sme_exit_to_user_mode(void) { } diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h deleted file mode 100644 index cda81d009c9b..000000000000 --- a/arch/arm64/include/asm/fpsimdmacros.h +++ /dev/null @@ -1,357 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * FP/SIMD state saving and restoring macros - * - * Copyright (C) 2012 ARM Ltd. - * Author: Catalin Marinas - */ - -#include - -.macro fpsimd_save state, tmpnr - stp q0, q1, [\state, #16 * 0] - stp q2, q3, [\state, #16 * 2] - stp q4, q5, [\state, #16 * 4] - stp q6, q7, [\state, #16 * 6] - stp q8, q9, [\state, #16 * 8] - stp q10, q11, [\state, #16 * 10] - stp q12, q13, [\state, #16 * 12] - stp q14, q15, [\state, #16 * 14] - stp q16, q17, [\state, #16 * 16] - stp q18, q19, [\state, #16 * 18] - stp q20, q21, [\state, #16 * 20] - stp q22, q23, [\state, #16 * 22] - stp q24, q25, [\state, #16 * 24] - stp q26, q27, [\state, #16 * 26] - stp q28, q29, [\state, #16 * 28] - stp q30, q31, [\state, #16 * 30]! - mrs x\tmpnr, fpsr - str w\tmpnr, [\state, #16 * 2] - mrs x\tmpnr, fpcr - str w\tmpnr, [\state, #16 * 2 + 4] -.endm - -.macro fpsimd_restore_fpcr state, tmp - /* - * Writes to fpcr may be self-synchronising, so avoid restoring - * the register if it hasn't changed. - */ - mrs \tmp, fpcr - cmp \tmp, \state - b.eq 9999f - msr fpcr, \state -9999: -.endm - -/* Clobbers \state */ -.macro fpsimd_restore state, tmpnr - ldp q0, q1, [\state, #16 * 0] - ldp q2, q3, [\state, #16 * 2] - ldp q4, q5, [\state, #16 * 4] - ldp q6, q7, [\state, #16 * 6] - ldp q8, q9, [\state, #16 * 8] - ldp q10, q11, [\state, #16 * 10] - ldp q12, q13, [\state, #16 * 12] - ldp q14, q15, [\state, #16 * 14] - ldp q16, q17, [\state, #16 * 16] - ldp q18, q19, [\state, #16 * 18] - ldp q20, q21, [\state, #16 * 20] - ldp q22, q23, [\state, #16 * 22] - ldp q24, q25, [\state, #16 * 24] - ldp q26, q27, [\state, #16 * 26] - ldp q28, q29, [\state, #16 * 28] - ldp q30, q31, [\state, #16 * 30]! - ldr w\tmpnr, [\state, #16 * 2] - msr fpsr, x\tmpnr - ldr w\tmpnr, [\state, #16 * 2 + 4] - fpsimd_restore_fpcr x\tmpnr, \state -.endm - -/* Sanity-check macros to help avoid encoding garbage instructions */ - -.macro _check_general_reg nr - .if (\nr) < 0 || (\nr) > 30 - .error "Bad register number \nr." - .endif -.endm - -.macro _sve_check_zreg znr - .if (\znr) < 0 || (\znr) > 31 - .error "Bad Scalable Vector Extension vector register number \znr." - .endif -.endm - -.macro _sve_check_preg pnr - .if (\pnr) < 0 || (\pnr) > 15 - .error "Bad Scalable Vector Extension predicate register number \pnr." - .endif -.endm - -.macro _check_num n, min, max - .if (\n) < (\min) || (\n) > (\max) - .error "Number \n out of range [\min,\max]" - .endif -.endm - -.macro _sme_check_wv v - .if (\v) < 12 || (\v) > 15 - .error "Bad vector select register \v." - .endif -.endm - -/* SVE instruction encodings for non-SVE-capable assemblers */ -/* (pre binutils 2.28, all kernel capable clang versions support SVE) */ - -/* STR (vector): STR Z\nz, [X\nxbase, #\offset, MUL VL] */ -.macro _sve_str_v nz, nxbase, offset=0 - _sve_check_zreg \nz - _check_general_reg \nxbase - _check_num (\offset), -0x100, 0xff - .inst 0xe5804000 \ - | (\nz) \ - | ((\nxbase) << 5) \ - | (((\offset) & 7) << 10) \ - | (((\offset) & 0x1f8) << 13) -.endm - -/* LDR (vector): LDR Z\nz, [X\nxbase, #\offset, MUL VL] */ -.macro _sve_ldr_v nz, nxbase, offset=0 - _sve_check_zreg \nz - _check_general_reg \nxbase - _check_num (\offset), -0x100, 0xff - .inst 0x85804000 \ - | (\nz) \ - | ((\nxbase) << 5) \ - | (((\offset) & 7) << 10) \ - | (((\offset) & 0x1f8) << 13) -.endm - -/* STR (predicate): STR P\np, [X\nxbase, #\offset, MUL VL] */ -.macro _sve_str_p np, nxbase, offset=0 - _sve_check_preg \np - _check_general_reg \nxbase - _check_num (\offset), -0x100, 0xff - .inst 0xe5800000 \ - | (\np) \ - | ((\nxbase) << 5) \ - | (((\offset) & 7) << 10) \ - | (((\offset) & 0x1f8) << 13) -.endm - -/* LDR (predicate): LDR P\np, [X\nxbase, #\offset, MUL VL] */ -.macro _sve_ldr_p np, nxbase, offset=0 - _sve_check_preg \np - _check_general_reg \nxbase - _check_num (\offset), -0x100, 0xff - .inst 0x85800000 \ - | (\np) \ - | ((\nxbase) << 5) \ - | (((\offset) & 7) << 10) \ - | (((\offset) & 0x1f8) << 13) -.endm - -/* RDVL X\nx, #\imm */ -.macro _sve_rdvl nx, imm - _check_general_reg \nx - _check_num (\imm), -0x20, 0x1f - .inst 0x04bf5000 \ - | (\nx) \ - | (((\imm) & 0x3f) << 5) -.endm - -/* RDFFR (unpredicated): RDFFR P\np.B */ -.macro _sve_rdffr np - _sve_check_preg \np - .inst 0x2519f000 \ - | (\np) -.endm - -/* WRFFR P\np.B */ -.macro _sve_wrffr np - _sve_check_preg \np - .inst 0x25289000 \ - | ((\np) << 5) -.endm - -/* PFALSE P\np.B */ -.macro _sve_pfalse np - _sve_check_preg \np - .inst 0x2518e400 \ - | (\np) -.endm - -/* SME instruction encodings for non-SME-capable assemblers */ -/* (pre binutils 2.38/LLVM 13) */ - -/* RDSVL X\nx, #\imm */ -.macro _sme_rdsvl nx, imm - _check_general_reg \nx - _check_num (\imm), -0x20, 0x1f - .inst 0x04bf5800 \ - | (\nx) \ - | (((\imm) & 0x3f) << 5) -.endm - -/* - * STR (vector from ZA array): - * STR ZA[\nw, #\offset], [X\nxbase, #\offset, MUL VL] - */ -.macro _sme_str_zav nw, nxbase, offset=0 - _sme_check_wv \nw - _check_general_reg \nxbase - _check_num (\offset), -0x100, 0xff - .inst 0xe1200000 \ - | (((\nw) & 3) << 13) \ - | ((\nxbase) << 5) \ - | ((\offset) & 7) -.endm - -/* - * LDR (vector to ZA array): - * LDR ZA[\nw, #\offset], [X\nxbase, #\offset, MUL VL] - */ -.macro _sme_ldr_zav nw, nxbase, offset=0 - _sme_check_wv \nw - _check_general_reg \nxbase - _check_num (\offset), -0x100, 0xff - .inst 0xe1000000 \ - | (((\nw) & 3) << 13) \ - | ((\nxbase) << 5) \ - | ((\offset) & 7) -.endm - -/* - * LDR (ZT0) - * - * LDR ZT0, nx - */ -.macro _ldr_zt nx - _check_general_reg \nx - .inst 0xe11f8000 \ - | (\nx << 5) -.endm - -/* - * STR (ZT0) - * - * STR ZT0, nx - */ -.macro _str_zt nx - _check_general_reg \nx - .inst 0xe13f8000 \ - | (\nx << 5) -.endm - -.macro __for from:req, to:req - .if (\from) == (\to) - _for__body %\from - .else - __for %\from, %((\from) + ((\to) - (\from)) / 2) - __for %((\from) + ((\to) - (\from)) / 2 + 1), %\to - .endif -.endm - -.macro _for var:req, from:req, to:req, insn:vararg - .macro _for__body \var:req - .noaltmacro - \insn - .altmacro - .endm - - .altmacro - __for \from, \to - .noaltmacro - - .purgem _for__body -.endm - -/* Update ZCR_EL1.LEN with the new VQ */ -.macro sve_load_vq xvqminus1, xtmp, xtmp2 - mrs_s \xtmp, SYS_ZCR_EL1 - bic \xtmp2, \xtmp, ZCR_ELx_LEN_MASK - orr \xtmp2, \xtmp2, \xvqminus1 - cmp \xtmp2, \xtmp - b.eq 921f - msr_s SYS_ZCR_EL1, \xtmp2 //self-synchronising -921: -.endm - -/* Update SMCR_EL1.LEN with the new VQ */ -.macro sme_load_vq xvqminus1, xtmp, xtmp2 - mrs_s \xtmp, SYS_SMCR_EL1 - bic \xtmp2, \xtmp, SMCR_ELx_LEN_MASK - orr \xtmp2, \xtmp2, \xvqminus1 - cmp \xtmp2, \xtmp - b.eq 921f - msr_s SYS_SMCR_EL1, \xtmp2 //self-synchronising -921: -.endm - -/* Preserve the first 128-bits of Znz and zero the rest. */ -.macro _sve_flush_z nz - _sve_check_zreg \nz - mov v\nz\().16b, v\nz\().16b -.endm - -.macro sve_flush_z - _for n, 0, 31, _sve_flush_z \n -.endm -.macro sve_flush_p - _for n, 0, 15, _sve_pfalse \n -.endm -.macro sve_flush_ffr - _sve_wrffr 0 -.endm - -.macro sve_save nxbase, xpfpsr, save_ffr, nxtmp - _for n, 0, 31, _sve_str_v \n, \nxbase, \n - 34 - _for n, 0, 15, _sve_str_p \n, \nxbase, \n - 16 - cbz \save_ffr, 921f - _sve_rdffr 0 - b 922f -921: - _sve_pfalse 0 // Zero out FFR -922: - _sve_str_p 0, \nxbase - _sve_ldr_p 0, \nxbase, -16 - mrs x\nxtmp, fpsr - str w\nxtmp, [\xpfpsr] - mrs x\nxtmp, fpcr - str w\nxtmp, [\xpfpsr, #4] -.endm - -.macro sve_load nxbase, xpfpsr, restore_ffr, nxtmp - _for n, 0, 31, _sve_ldr_v \n, \nxbase, \n - 34 - cbz \restore_ffr, 921f - _sve_ldr_p 0, \nxbase - _sve_wrffr 0 -921: - _for n, 0, 15, _sve_ldr_p \n, \nxbase, \n - 16 - - ldr w\nxtmp, [\xpfpsr] - msr fpsr, x\nxtmp - ldr w\nxtmp, [\xpfpsr, #4] - msr fpcr, x\nxtmp -.endm - -.macro sme_save_za nxbase, xvl, nw - mov w\nw, #0 - -423: - _sme_str_zav \nw, \nxbase - add x\nxbase, x\nxbase, \xvl - add x\nw, x\nw, #1 - cmp \xvl, x\nw - bne 423b -.endm - -.macro sme_load_za nxbase, xvl, nw - mov w\nw, #0 - -423: - _sme_ldr_zav \nw, \nxbase - add x\nxbase, x\nxbase, \xvl - add x\nw, x\nw, #1 - cmp \xvl, x\nw - bne 423b -.endm diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h index 8cbd1e96fd50..21c8e400107c 100644 --- a/arch/arm64/include/asm/io.h +++ b/arch/arm64/include/asm/io.h @@ -270,7 +270,7 @@ static inline void __iomem *ioremap_prot(phys_addr_t phys, size_t size, pgprot_t user_prot) { pgprot_t prot; - ptdesc_t user_prot_val = pgprot_val(user_prot); + ptval_t user_prot_val = pgprot_val(user_prot); if (WARN_ON_ONCE(!(user_prot_val & PTE_USER))) return NULL; diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index a49042bfa801..9209c54350c7 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -732,20 +732,6 @@ struct kvm_cpu_context { u64 *vncr_array; }; -struct cpu_sve_state { - __u64 zcr_el1; - - /* - * Ordering is important since __sve_save_state/__sve_restore_state - * relies on it. - */ - __u32 fpsr; - __u32 fpcr; - - /* Must be SVE_VQ_BYTES (128 bit) aligned. */ - __u8 sve_regs[]; -}; - /* * This structure is instantiated on a per-CPU basis, and contains * data that is: @@ -771,12 +757,9 @@ struct kvm_host_data { /* * Hyp VA. - * sve_state is only used in pKVM and if system_supports_sve(). + * sve_regs is only used in pKVM and if system_supports_sve(). */ - struct cpu_sve_state *sve_state; - - /* Used by pKVM only. */ - u64 fpmr; + struct arm64_sve_state *sve_regs; /* Ownership of the FP regs */ enum { @@ -870,7 +853,7 @@ struct kvm_vcpu_arch { * floating point code saves the register state of a task it * records which view it saved in fp_type. */ - void *sve_state; + struct arm64_sve_state *sve_state; enum fp_type fp_type; unsigned int sve_max_vl; @@ -1114,10 +1097,6 @@ struct kvm_vcpu_arch { #define NESTED_SERROR_PENDING __vcpu_single_flag(sflags, BIT(8)) -/* Pointer to the vcpu's SVE FFR for sve_{save,load}_state() */ -#define vcpu_sve_pffr(vcpu) (kern_hyp_va((vcpu)->arch.sve_state) + \ - sve_ffr_offset((vcpu)->arch.sve_max_vl)) - #define vcpu_sve_max_vq(vcpu) sve_vq_from_vl((vcpu)->arch.sve_max_vl) #define vcpu_sve_zcr_elx(vcpu) \ diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index 8d06b62e7188..ad19de1d0654 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -121,11 +121,6 @@ void __debug_save_host_buffers_nvhe(struct kvm_vcpu *vcpu); void __debug_restore_host_buffers_nvhe(struct kvm_vcpu *vcpu); #endif -void __fpsimd_save_state(struct user_fpsimd_state *fp_regs); -void __fpsimd_restore_state(struct user_fpsimd_state *fp_regs); -void __sve_save_state(void *sve_pffr, u32 *fpsr, int save_ffr); -void __sve_restore_state(void *sve_pffr, u32 *fpsr, int restore_ffr); - u64 __guest_enter(struct kvm_vcpu *vcpu); bool kvm_host_psci_handler(struct kvm_cpu_context *host_ctxt, u32 func_id); diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index 2954b311128c..74fedd9c5ff0 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -188,8 +188,7 @@ static inline size_t pkvm_host_sve_state_size(void) if (!system_supports_sve()) return 0; - return size_add(sizeof(struct cpu_sve_state), - SVE_SIG_REGS_SIZE(sve_vq_from_vl(kvm_host_sve_max_vl))); + return SVE_SIG_REGS_SIZE(sve_vq_from_vl(kvm_host_sve_max_vl)); } struct pkvm_mapping { diff --git a/arch/arm64/include/asm/linkage.h b/arch/arm64/include/asm/linkage.h index 40bd17add539..d1f7a16729d2 100644 --- a/arch/arm64/include/asm/linkage.h +++ b/arch/arm64/include/asm/linkage.h @@ -43,4 +43,8 @@ SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \ bti c ; +#define _THIS_IP_ ({ unsigned long __ip; asm volatile("adr %0, ." : "=r" (__ip)); __ip; }) + +#define __bss_pgtbl __section(".bss..pgtbl") __aligned(PAGE_SIZE) + #endif diff --git a/arch/arm64/include/asm/pgtable-types.h b/arch/arm64/include/asm/pgtable-types.h index 265e8301d7ba..2f2f5527930f 100644 --- a/arch/arm64/include/asm/pgtable-types.h +++ b/arch/arm64/include/asm/pgtable-types.h @@ -17,13 +17,13 @@ * Generic page table descriptor format from which * all level specific descriptors can be derived. */ -typedef u64 ptdesc_t; +typedef u64 ptval_t; -typedef ptdesc_t pteval_t; -typedef ptdesc_t pmdval_t; -typedef ptdesc_t pudval_t; -typedef ptdesc_t p4dval_t; -typedef ptdesc_t pgdval_t; +typedef ptval_t pteval_t; +typedef ptval_t pmdval_t; +typedef ptval_t pudval_t; +typedef ptval_t p4dval_t; +typedef ptval_t pgdval_t; /* * These are used to make use of C type-checking.. @@ -54,7 +54,7 @@ typedef struct { pgdval_t pgd; } pgd_t; #define pgd_val(x) ((x).pgd) #define __pgd(x) ((pgd_t) { (x) } ) -typedef struct { ptdesc_t pgprot; } pgprot_t; +typedef struct { ptval_t pgprot; } pgprot_t; #define pgprot_val(x) ((x).pgprot) #define __pgprot(x) ((pgprot_t) { (x) } ) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 4dfa42b7d053..c9e4e00a9af2 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -1007,7 +1007,7 @@ static inline pud_t *p4d_pgtable(p4d_t p4d) static inline phys_addr_t pud_offset_phys(p4d_t *p4dp, unsigned long addr) { - BUG_ON(!pgtable_l4_enabled()); + VM_WARN_ON_ONCE(!pgtable_l4_enabled()); return p4d_page_paddr(READ_ONCE(*p4dp)) + pud_index(addr) * sizeof(pud_t); } @@ -1130,7 +1130,7 @@ static inline p4d_t *pgd_to_folded_p4d(pgd_t *pgdp, unsigned long addr) static inline phys_addr_t p4d_offset_phys(pgd_t *pgdp, unsigned long addr) { - BUG_ON(!pgtable_l5_enabled()); + VM_WARN_ON_ONCE(!pgtable_l5_enabled()); return pgd_page_paddr(READ_ONCE(*pgdp)) + p4d_index(addr) * sizeof(p4d_t); } diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index e30c4c8e3a7a..c2a627f39314 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -130,6 +130,9 @@ enum fp_type { FP_STATE_SVE, }; +struct arm64_sve_state; /* Opaque type */ +struct arm64_sme_state; /* Opaque type */ + struct cpu_context { unsigned long x19; unsigned long x20; @@ -164,8 +167,8 @@ struct thread_struct { enum fp_type fp_type; /* registers FPSIMD or SVE? */ unsigned int fpsimd_cpu; - void *sve_state; /* SVE registers, if any */ - void *sme_state; /* ZA and ZT state, if any */ + struct arm64_sve_state *sve_state; /* SVE registers, if any */ + struct arm64_sme_state *sme_state; /* ZA and ZT state, if any */ unsigned int vl[ARM64_VEC_MAX]; /* vector length */ unsigned int vl_onexec[ARM64_VEC_MAX]; /* vl after next exec */ unsigned long fault_address; /* fault info */ diff --git a/arch/arm64/include/asm/ptdump.h b/arch/arm64/include/asm/ptdump.h index baff24004459..5b374a6ab34a 100644 --- a/arch/arm64/include/asm/ptdump.h +++ b/arch/arm64/include/asm/ptdump.h @@ -26,8 +26,8 @@ struct ptdump_info { }; struct ptdump_prot_bits { - ptdesc_t mask; - ptdesc_t val; + ptval_t mask; + ptval_t val; const char *set; const char *clear; }; @@ -36,7 +36,7 @@ struct ptdump_pg_level { const struct ptdump_prot_bits *bits; char name[4]; int num; - ptdesc_t mask; + ptval_t mask; }; /* @@ -53,7 +53,7 @@ struct ptdump_pg_state { const struct mm_struct *mm; unsigned long start_address; int level; - ptdesc_t current_prot; + ptval_t current_prot; bool check_wx; unsigned long wx_pages; unsigned long uxn_pages; diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index c0bf5b398041..d52ac8c17190 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -725,9 +725,9 @@ static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *b sme_dvmsync_add_pending(batch, mm); } -static inline bool __pte_flags_need_flush(ptdesc_t oldval, ptdesc_t newval) +static inline bool __pte_flags_need_flush(ptval_t oldval, ptval_t newval) { - ptdesc_t diff = oldval ^ newval; + ptval_t diff = oldval ^ newval; /* invalid to valid transition requires no flush */ if (!(oldval & PTE_VALID)) diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h index 06f83ca8de56..10272ddb4d6f 100644 --- a/arch/arm64/include/uapi/asm/hwcap.h +++ b/arch/arm64/include/uapi/asm/hwcap.h @@ -147,5 +147,13 @@ #define HWCAP3_MTE_STORE_ONLY (1UL << 1) #define HWCAP3_LSFE (1UL << 2) #define HWCAP3_LS64 (1UL << 3) +#define HWCAP3_SVE_B16MM (1UL << 4) +#define HWCAP3_SVE2P3 (1UL << 5) +#define HWCAP3_SME_LUT6 (1UL << 6) +#define HWCAP3_SME2P3 (1UL << 7) +#define HWCAP3_F16MM (1UL << 8) +#define HWCAP3_F16F32DOT (1UL << 9) +#define HWCAP3_F16F32MM (1UL << 10) +#define HWCAP3_SVE_LUT6 (1UL << 11) #endif /* _UAPI__ASM_HWCAP_H */ diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 74b76bb70452..d2690c3ec528 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -27,7 +27,7 @@ KCOV_INSTRUMENT_idle.o := n # Object file lists. obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ - entry-common.o entry-fpsimd.o process.o ptrace.o \ + entry-common.o process.o ptrace.o \ setup.o signal.o sys.o stacktrace.o time.o traps.o \ io.o vdso.o hyp-stub.o psci.o cpu_ops.o \ return_address.o cpuinfo.o cpu_errata.o \ diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 5377e4c2eba2..1995e1198648 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -340,7 +340,37 @@ static const struct arm64_cpu_capabilities arm64_repeat_tlbi_list[] = { ERRATA_MIDR_RANGE(MIDR_CORTEX_A510, 0, 0, 1, 1), }, #endif - {}, +#ifdef CONFIG_ARM64_ERRATUM_4118414 + { + ERRATA_MIDR_RANGE_LIST(((const struct midr_range[]) { + MIDR_ALL_VERSIONS(MIDR_C1_PREMIUM), + MIDR_ALL_VERSIONS(MIDR_C1_ULTRA), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A76), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A76AE), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A77), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78AE), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A710), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X1), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X1C), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X2), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X3), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X4), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X925), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V2), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3AE), + MIDR_ALL_VERSIONS(MIDR_NVIDIA_OLYMPUS), + MIDR_ALL_VERSIONS(MIDR_MICROSOFT_AZURE_COBALT_100), + {} + })), + }, +#endif + {} }; #endif @@ -608,6 +638,18 @@ static const struct midr_range erratum_ac04_cpu_23_list[] = { }; #endif +#ifdef CONFIG_ARM64_WORKAROUND_DISABLE_CNP +static const struct midr_range cnp_erratum_cpus[] = { +#ifdef CONFIG_NVIDIA_CARMEL_CNP_ERRATUM + MIDR_ALL_VERSIONS(MIDR_NVIDIA_CARMEL), +#endif +#ifdef CONFIG_HISILICON_ERRATUM_162100125 + MIDR_ALL_VERSIONS(MIDR_HISI_HIP09), +#endif + {}, +}; +#endif + const struct arm64_cpu_capabilities arm64_errata[] = { #ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE { @@ -693,7 +735,7 @@ const struct arm64_cpu_capabilities arm64_errata[] = { #endif #ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI { - .desc = "Qualcomm erratum 1009, or ARM erratum 1286807, 2441009", + .desc = "Broken broadcast TLBI completion", .capability = ARM64_WORKAROUND_REPEAT_TLBI, .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, .matches = cpucap_multi_entry_cap_matches, @@ -801,12 +843,11 @@ const struct arm64_cpu_capabilities arm64_errata[] = { 1, 0), }, #endif -#ifdef CONFIG_NVIDIA_CARMEL_CNP_ERRATUM +#ifdef CONFIG_ARM64_WORKAROUND_DISABLE_CNP { - /* NVIDIA Carmel */ - .desc = "NVIDIA Carmel CNP erratum", - .capability = ARM64_WORKAROUND_NVIDIA_CARMEL_CNP, - ERRATA_MIDR_ALL_VERSIONS(MIDR_NVIDIA_CARMEL), + .desc = "NVIDIA Carmel CNP erratum, or Hisilicon erratum 162100125", + .capability = ARM64_WORKAROUND_DISABLE_CNP, + ERRATA_MIDR_RANGE_LIST(cnp_erratum_cpus), }, #endif #ifdef CONFIG_ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 4f8670760b9a..9583ba295d9e 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -365,6 +365,8 @@ static const struct arm64_ftr_bits ftr_id_aa64zfr0[] = { static const struct arm64_ftr_bits ftr_id_aa64smfr0[] = { ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_FA64_SHIFT, 1, 0), + ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), + FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_LUT6_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_LUTv2_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), @@ -419,6 +421,7 @@ static const struct arm64_ftr_bits ftr_id_aa64fpfr0[] = { ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8DP2_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8MM8_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8MM4_SHIFT, 1, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F16MM2_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8E4M3_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8E5M2_SHIFT, 1, 0), ARM64_FTR_END, @@ -1164,6 +1167,14 @@ static __init void detect_system_supports_pseudo_nmi(void) static inline void detect_system_supports_pseudo_nmi(void) { } #endif +static bool detect_ftr_has_mpam(void) +{ + u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); + u64 pfr1 = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1); + + return id_aa64pfr0_mpam(pfr0) || id_aa64pfr1_mpamfrac(pfr1); +} + void __init init_cpu_features(struct cpuinfo_arm64 *info) { /* Before we start using the tables, make sure it is sorted */ @@ -1211,7 +1222,7 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info) cpacr_restore(cpacr); } - if (id_aa64pfr0_mpam(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1))) { + if (detect_ftr_has_mpam()) { info->reg_mpamidr = read_cpuid(MPAMIDR_EL1); init_cpu_ftr_reg(SYS_MPAMIDR_EL1, info->reg_mpamidr); } @@ -1467,7 +1478,7 @@ void update_cpu_features(int cpu, cpacr_restore(cpacr); } - if (id_aa64pfr0_mpam(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1))) { + if (detect_ftr_has_mpam()) { info->reg_mpamidr = read_cpuid(MPAMIDR_EL1); taint |= check_update_ftr_reg(SYS_MPAMIDR_EL1, cpu, info->reg_mpamidr, boot->reg_mpamidr); @@ -1786,7 +1797,7 @@ has_useable_cnp(const struct arm64_cpu_capabilities *entry, int scope) if (is_kdump_kernel()) return false; - if (cpus_have_cap(ARM64_WORKAROUND_NVIDIA_CARMEL_CNP)) + if (cpus_have_cap(ARM64_WORKAROUND_DISABLE_CNP)) return false; return has_cpuid_feature(entry, scope); @@ -2487,7 +2498,7 @@ cpucap_panic_on_conflict(const struct arm64_cpu_capabilities *cap) static bool test_has_mpam(const struct arm64_cpu_capabilities *entry, int scope) { - if (!has_cpuid_feature(entry, scope)) + if (!detect_ftr_has_mpam()) return false; /* Check firmware actually enabled MPAM on this cpu. */ @@ -3094,7 +3105,6 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .capability = ARM64_MPAM, .matches = test_has_mpam, .cpu_enable = cpu_enable_mpam, - ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, MPAM, 1) }, { .desc = "Memory Partitioning And Monitoring Virtualisation", @@ -3285,6 +3295,8 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(ID_AA64ISAR0_EL1, SM4, IMP, CAP_HWCAP, KERNEL_HWCAP_SM4), HWCAP_CAP(ID_AA64ISAR0_EL1, DP, IMP, CAP_HWCAP, KERNEL_HWCAP_ASIMDDP), HWCAP_CAP(ID_AA64ISAR0_EL1, FHM, IMP, CAP_HWCAP, KERNEL_HWCAP_ASIMDFHM), + HWCAP_CAP(ID_AA64ISAR0_EL1, FHM, F16F32DOT, CAP_HWCAP, KERNEL_HWCAP_F16F32DOT), + HWCAP_CAP(ID_AA64ISAR0_EL1, FHM, F16F32MM, CAP_HWCAP, KERNEL_HWCAP_F16F32MM), HWCAP_CAP(ID_AA64ISAR0_EL1, TS, FLAGM, CAP_HWCAP, KERNEL_HWCAP_FLAGM), HWCAP_CAP(ID_AA64ISAR0_EL1, TS, FLAGM2, CAP_HWCAP, KERNEL_HWCAP_FLAGM2), HWCAP_CAP(ID_AA64ISAR0_EL1, RNDR, IMP, CAP_HWCAP, KERNEL_HWCAP_RNG), @@ -3314,7 +3326,9 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(ID_AA64ISAR3_EL1, LSFE, IMP, CAP_HWCAP, KERNEL_HWCAP_LSFE), HWCAP_CAP(ID_AA64MMFR2_EL1, AT, IMP, CAP_HWCAP, KERNEL_HWCAP_USCAT), #ifdef CONFIG_ARM64_SVE + HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ISAR2_EL1, LUT, LUT6, CAP_HWCAP, KERNEL_HWCAP_SVE_LUT6), HWCAP_CAP(ID_AA64PFR0_EL1, SVE, IMP, CAP_HWCAP, KERNEL_HWCAP_SVE), + HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2p3, CAP_HWCAP, KERNEL_HWCAP_SVE2P3), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2p2, CAP_HWCAP, KERNEL_HWCAP_SVE2P2), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2p1, CAP_HWCAP, KERNEL_HWCAP_SVE2P1), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2, CAP_HWCAP, KERNEL_HWCAP_SVE2), @@ -3324,6 +3338,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BitPerm, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEBITPERM), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, B16B16, IMP, CAP_HWCAP, KERNEL_HWCAP_SVE_B16B16), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, B16B16, BFSCALE, CAP_HWCAP, KERNEL_HWCAP_SVE_BFSCALE), + HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, B16B16, B16MM, CAP_HWCAP, KERNEL_HWCAP_SVE_B16MM), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BF16, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEBF16), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BF16, EBF16, CAP_HWCAP, KERNEL_HWCAP_SVE_EBF16), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SHA3, IMP, CAP_HWCAP, KERNEL_HWCAP_SVESHA3), @@ -3363,7 +3378,9 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { #ifdef CONFIG_ARM64_SME HWCAP_CAP(ID_AA64PFR1_EL1, SME, IMP, CAP_HWCAP, KERNEL_HWCAP_SME), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, FA64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_FA64), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, LUT6, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_LUT6), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, LUTv2, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_LUTV2), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2p3, CAP_HWCAP, KERNEL_HWCAP_SME2P3), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2p2, CAP_HWCAP, KERNEL_HWCAP_SME2P2), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2p1, CAP_HWCAP, KERNEL_HWCAP_SME2P1), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2, CAP_HWCAP, KERNEL_HWCAP_SME2), @@ -3394,6 +3411,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(ID_AA64FPFR0_EL1, F8DP2, IMP, CAP_HWCAP, KERNEL_HWCAP_F8DP2), HWCAP_CAP(ID_AA64FPFR0_EL1, F8MM8, IMP, CAP_HWCAP, KERNEL_HWCAP_F8MM8), HWCAP_CAP(ID_AA64FPFR0_EL1, F8MM4, IMP, CAP_HWCAP, KERNEL_HWCAP_F8MM4), + HWCAP_CAP(ID_AA64FPFR0_EL1, F16MM2, IMP, CAP_HWCAP, KERNEL_HWCAP_F16MM), HWCAP_CAP(ID_AA64FPFR0_EL1, F8E4M3, IMP, CAP_HWCAP, KERNEL_HWCAP_F8E4M3), HWCAP_CAP(ID_AA64FPFR0_EL1, F8E5M2, IMP, CAP_HWCAP, KERNEL_HWCAP_F8E5M2), #ifdef CONFIG_ARM64_POE diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 6149bc91251d..d50e2a9b066b 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -164,6 +164,14 @@ static const char *const hwcap_str[] = { [KERNEL_HWCAP_MTE_FAR] = "mtefar", [KERNEL_HWCAP_MTE_STORE_ONLY] = "mtestoreonly", [KERNEL_HWCAP_LSFE] = "lsfe", + [KERNEL_HWCAP_SVE_B16MM] = "sveb16mm", + [KERNEL_HWCAP_SVE2P3] = "sve2p3", + [KERNEL_HWCAP_SME_LUT6] = "smelut6", + [KERNEL_HWCAP_SME2P3] = "sme2p3", + [KERNEL_HWCAP_F16MM] = "f16mm", + [KERNEL_HWCAP_F16F32DOT] = "f16f32dot", + [KERNEL_HWCAP_F16F32MM] = "f16f32mm", + [KERNEL_HWCAP_SVE_LUT6] = "svelut6", }; #ifdef CONFIG_COMPAT diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c index a81cb4aa4738..30cd7f804398 100644 --- a/arch/arm64/kernel/efi.c +++ b/arch/arm64/kernel/efi.c @@ -31,7 +31,7 @@ static bool region_is_misaligned(const efi_memory_desc_t *md) * executable, everything else can be mapped with the XN bits * set. Also take the new (optional) RO/XP bits into account. */ -static __init ptdesc_t create_mapping_protection(efi_memory_desc_t *md) +static __init ptval_t create_mapping_protection(efi_memory_desc_t *md) { u64 attr = md->attribute; u32 type = md->type; @@ -85,7 +85,7 @@ static __init ptdesc_t create_mapping_protection(efi_memory_desc_t *md) int __init efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md) { - ptdesc_t prot_val = create_mapping_protection(md); + ptval_t prot_val = create_mapping_protection(md); bool page_mappings_only = (md->type == EFI_RUNTIME_SERVICES_CODE || md->type == EFI_RUNTIME_SERVICES_DATA); diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index c7a23f7c2212..ceb4eb11232a 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -254,12 +254,8 @@ static inline void fpsimd_syscall_enter(void) if (!system_supports_sve()) return; - if (test_thread_flag(TIF_SVE)) { - unsigned int sve_vq_minus_one; - - sve_vq_minus_one = sve_vq_from_vl(task_get_sve_vl(current)) - 1; - sve_flush_live(true, sve_vq_minus_one); - } + if (test_thread_flag(TIF_SVE)) + sve_flush_live(); /* * Any live non-FPSIMD SVE state has been zeroed. Allow diff --git a/arch/arm64/kernel/entry-fpsimd.S b/arch/arm64/kernel/entry-fpsimd.S deleted file mode 100644 index 6325db1a2179..000000000000 --- a/arch/arm64/kernel/entry-fpsimd.S +++ /dev/null @@ -1,134 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * FP/SIMD state saving and restoring - * - * Copyright (C) 2012 ARM Ltd. - * Author: Catalin Marinas - */ - -#include - -#include -#include - -/* - * Save the FP registers. - * - * x0 - pointer to struct fpsimd_state - */ -SYM_FUNC_START(fpsimd_save_state) - fpsimd_save x0, 8 - ret -SYM_FUNC_END(fpsimd_save_state) - -/* - * Load the FP registers. - * - * x0 - pointer to struct fpsimd_state - */ -SYM_FUNC_START(fpsimd_load_state) - fpsimd_restore x0, 8 - ret -SYM_FUNC_END(fpsimd_load_state) - -#ifdef CONFIG_ARM64_SVE - -/* - * Save the SVE state - * - * x0 - pointer to buffer for state - * x1 - pointer to storage for FPSR - * x2 - Save FFR if non-zero - */ -SYM_FUNC_START(sve_save_state) - sve_save 0, x1, x2, 3 - ret -SYM_FUNC_END(sve_save_state) - -/* - * Load the SVE state - * - * x0 - pointer to buffer for state - * x1 - pointer to storage for FPSR - * x2 - Restore FFR if non-zero - */ -SYM_FUNC_START(sve_load_state) - sve_load 0, x1, x2, 4 - ret -SYM_FUNC_END(sve_load_state) - -SYM_FUNC_START(sve_get_vl) - _sve_rdvl 0, 1 - ret -SYM_FUNC_END(sve_get_vl) - -SYM_FUNC_START(sve_set_vq) - sve_load_vq x0, x1, x2 - ret -SYM_FUNC_END(sve_set_vq) - -/* - * Zero all SVE registers but the first 128-bits of each vector - * - * VQ must already be configured by caller, any further updates of VQ - * will need to ensure that the register state remains valid. - * - * x0 = include FFR? - * x1 = VQ - 1 - */ -SYM_FUNC_START(sve_flush_live) - cbz x1, 1f // A VQ-1 of 0 is 128 bits so no extra Z state - sve_flush_z -1: sve_flush_p - tbz x0, #0, 2f - sve_flush_ffr -2: ret -SYM_FUNC_END(sve_flush_live) - -#endif /* CONFIG_ARM64_SVE */ - -#ifdef CONFIG_ARM64_SME - -SYM_FUNC_START(sme_get_vl) - _sme_rdsvl 0, 1 - ret -SYM_FUNC_END(sme_get_vl) - -SYM_FUNC_START(sme_set_vq) - sme_load_vq x0, x1, x2 - ret -SYM_FUNC_END(sme_set_vq) - -/* - * Save the ZA and ZT state - * - * x0 - pointer to buffer for state - * x1 - number of ZT registers to save - */ -SYM_FUNC_START(sme_save_state) - _sme_rdsvl 2, 1 // x2 = VL/8 - sme_save_za 0, x2, 12 // Leaves x0 pointing to the end of ZA - - cbz x1, 1f - _str_zt 0 -1: - ret -SYM_FUNC_END(sme_save_state) - -/* - * Load the ZA and ZT state - * - * x0 - pointer to buffer for state - * x1 - number of ZT registers to save - */ -SYM_FUNC_START(sme_load_state) - _sme_rdsvl 2, 1 // x2 = VL/8 - sme_load_za 0, x2, 12 // Leaves x0 pointing to the end of ZA - - cbz x1, 1f - _ldr_zt 0 -1: - ret -SYM_FUNC_END(sme_load_state) - -#endif /* CONFIG_ARM64_SME */ diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 60a45d600b46..25dc5afe9ba0 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -377,8 +377,10 @@ static void task_fpsimd_load(void) if (!thread_sm_enabled(¤t->thread)) WARN_ON_ONCE(!test_and_set_thread_flag(TIF_SVE)); - if (test_thread_flag(TIF_SVE)) - sve_set_vq(sve_vq_from_vl(task_get_sve_vl(current)) - 1); + if (test_thread_flag(TIF_SVE)) { + unsigned long vq = sve_vq_from_vl(task_get_sve_vl(current)); + sysreg_clear_set_s(SYS_ZCR_EL1, ZCR_ELx_LEN, vq - 1); + } restore_sve_regs = true; restore_ffr = true; @@ -403,8 +405,10 @@ static void task_fpsimd_load(void) unsigned long sme_vl = task_get_sme_vl(current); /* Ensure VL is set up for restoring data */ - if (test_thread_flag(TIF_SME)) - sme_set_vq(sve_vq_from_vl(sme_vl) - 1); + if (test_thread_flag(TIF_SME)) { + unsigned long vq = sve_vq_from_vl(sme_vl); + sysreg_clear_set_s(SYS_SMCR_EL1, SMCR_ELx_LEN, vq - 1); + } write_sysreg_s(current->thread.svcr, SYS_SVCR); @@ -421,9 +425,8 @@ static void task_fpsimd_load(void) if (restore_sve_regs) { WARN_ON_ONCE(current->thread.fp_type != FP_STATE_SVE); - sve_load_state(sve_pffr(¤t->thread), - ¤t->thread.uw.fpsimd_state.fpsr, - restore_ffr); + sve_load_state(current->thread.sve_state, restore_ffr); + fpsimd_load_common(¤t->thread.uw.fpsimd_state); } else { WARN_ON_ONCE(current->thread.fp_type != FP_STATE_FPSIMD); fpsimd_load_state(¤t->thread.uw.fpsimd_state); @@ -503,9 +506,8 @@ static void fpsimd_save_user_state(void) return; } - sve_save_state((char *)last->sve_state + - sve_ffr_offset(vl), - &last->st->fpsr, save_ffr); + sve_save_state(last->sve_state, save_ffr); + fpsimd_save_common(last->st); *last->fp_type = FP_STATE_SVE; } else { fpsimd_save_state(last->st); @@ -636,7 +638,8 @@ static __uint128_t arm64_cpu_to_le128(__uint128_t x) #define arm64_le128_to_cpu(x) arm64_cpu_to_le128(x) -static void __fpsimd_to_sve(void *sst, struct user_fpsimd_state const *fst, +static void __fpsimd_to_sve(struct arm64_sve_state *sst, + struct user_fpsimd_state const *fst, unsigned int vq) { unsigned int i; @@ -663,7 +666,7 @@ static void __fpsimd_to_sve(void *sst, struct user_fpsimd_state const *fst, static inline void fpsimd_to_sve(struct task_struct *task) { unsigned int vq; - void *sst = task->thread.sve_state; + struct arm64_sve_state *sst = task->thread.sve_state; struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state; if (!system_supports_sve() && !system_supports_sme()) @@ -687,7 +690,7 @@ static inline void fpsimd_to_sve(struct task_struct *task) static inline void sve_to_fpsimd(struct task_struct *task) { unsigned int vq, vl; - void const *sst = task->thread.sve_state; + const struct arm64_sve_state *sst = task->thread.sve_state; struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state; unsigned int i; __uint128_t const *p; @@ -786,7 +789,7 @@ void fpsimd_sync_from_effective_state(struct task_struct *task) void fpsimd_sync_to_effective_state_zeropad(struct task_struct *task) { unsigned int vq; - void *sst = task->thread.sve_state; + struct arm64_sve_state *sst = task->thread.sve_state; struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state; if (task->thread.fp_type != FP_STATE_SVE) @@ -804,7 +807,8 @@ static int change_live_vector_length(struct task_struct *task, { unsigned int sve_vl = task_get_sve_vl(task); unsigned int sme_vl = task_get_sme_vl(task); - void *sve_state = NULL, *sme_state = NULL; + struct arm64_sve_state *sve_state = NULL; + struct arm64_sme_state *sme_state = NULL; if (type == ARM64_VEC_SME) sme_vl = vl; @@ -1293,31 +1297,6 @@ void sme_suspend_exit(void) #endif /* CONFIG_ARM64_SME */ -static void sve_init_regs(void) -{ - /* - * Convert the FPSIMD state to SVE, zeroing all the state that - * is not shared with FPSIMD. If (as is likely) the current - * state is live in the registers then do this there and - * update our metadata for the current task including - * disabling the trap, otherwise update our in-memory copy. - * We are guaranteed to not be in streaming mode, we can only - * take a SVE trap when not in streaming mode and we can't be - * in streaming mode when taking a SME trap. - */ - if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) { - unsigned long vq_minus_one = - sve_vq_from_vl(task_get_sve_vl(current)) - 1; - sve_set_vq(vq_minus_one); - sve_flush_live(true, vq_minus_one); - fpsimd_bind_task_to_cpu(); - } else { - fpsimd_to_sve(current); - current->thread.fp_type = FP_STATE_SVE; - fpsimd_flush_task_state(current); - } -} - /* * Trapped SVE access * @@ -1349,13 +1328,23 @@ void do_sve_acc(unsigned long esr, struct pt_regs *regs) WARN_ON(1); /* SVE access shouldn't have trapped */ /* - * Even if the task can have used streaming mode we can only - * generate SVE access traps in normal SVE mode and - * transitioning out of streaming mode may discard any - * streaming mode state. Always clear the high bits to avoid - * any potential errors tracking what is properly initialised. + * Convert the FPSIMD state to SVE. Stale SVE state can be present in + * registers or memory, so we must zero all state that is not shared + * with FPSIMD. + * + * SVE traps cannot be taken from streaming mode, so there cannot be + * any effective streaming mode SVE state. */ - sve_init_regs(); + if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) { + unsigned long vq = sve_vq_from_vl(task_get_sve_vl(current)); + sysreg_clear_set_s(SYS_ZCR_EL1, ZCR_ELx_LEN, vq - 1); + sve_flush_live(); + fpsimd_bind_task_to_cpu(); + } else { + fpsimd_to_sve(current); + current->thread.fp_type = FP_STATE_SVE; + fpsimd_flush_task_state(current); + } put_cpu_fpsimd_context(); } @@ -1479,9 +1468,8 @@ void do_sme_acc(unsigned long esr, struct pt_regs *regs) WARN_ON(1); if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) { - unsigned long vq_minus_one = - sve_vq_from_vl(task_get_sme_vl(current)) - 1; - sme_set_vq(vq_minus_one); + unsigned long vq = sve_vq_from_vl(task_get_sme_vl(current)); + sysreg_clear_set_s(SYS_SMCR_EL1, SMCR_ELx_LEN, vq - 1); fpsimd_bind_task_to_cpu(); } else { @@ -1656,8 +1644,8 @@ static void fpsimd_flush_thread_vl(enum vec_type type) void fpsimd_flush_thread(void) { - void *sve_state = NULL; - void *sme_state = NULL; + struct arm64_sve_state *sve_state = NULL; + struct arm64_sme_state *sme_state = NULL; if (!system_supports_fpsimd()) return; diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c index 15dedb385b9e..9fafd826002b 100644 --- a/arch/arm64/kernel/irq.c +++ b/arch/arm64/kernel/irq.c @@ -10,6 +10,7 @@ * Copyright (C) 2012 ARM Ltd. */ +#include #include #include #include @@ -32,34 +33,43 @@ DEFINE_PER_CPU(struct nmi_ctx, nmi_contexts); DEFINE_PER_CPU(unsigned long *, irq_stack_ptr); - DECLARE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr); #ifdef CONFIG_SHADOW_CALL_STACK DEFINE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr); #endif -static void init_irq_scs(void) +static int __init init_irq_scs(void) { int cpu; + void *s; if (!scs_is_enabled()) - return; + return 0; - for_each_possible_cpu(cpu) - per_cpu(irq_shadow_call_stack_ptr, cpu) = - scs_alloc(early_cpu_to_node(cpu)); + for_each_possible_cpu(cpu) { + s = scs_alloc(early_cpu_to_node(cpu)); + if (!s) + return -ENOMEM; + per_cpu(irq_shadow_call_stack_ptr, cpu) = s; + } + + return 0; } -static void __init init_irq_stacks(void) +static int __init init_irq_stacks(void) { int cpu; unsigned long *p; for_each_possible_cpu(cpu) { p = arch_alloc_vmap_stack(IRQ_STACK_SIZE, early_cpu_to_node(cpu)); + if (!p) + return -ENOMEM; per_cpu(irq_stack_ptr, cpu) = p; } + + return 0; } #ifdef CONFIG_SOFTIRQ_ON_OWN_STACK @@ -109,8 +119,9 @@ int __init set_handle_fiq(void (*handle_fiq)(struct pt_regs *)) void __init init_IRQ(void) { - init_irq_stacks(); - init_irq_scs(); + if (init_irq_stacks() || init_irq_scs()) + panic("Failed to allocate IRQ stack resources\n"); + irqchip_init(); if (system_uses_irq_prio_masking()) { diff --git a/arch/arm64/kernel/patching.c b/arch/arm64/kernel/patching.c index 1041bc67a3ee..09f019c6547a 100644 --- a/arch/arm64/kernel/patching.c +++ b/arch/arm64/kernel/patching.c @@ -116,8 +116,7 @@ static void *__text_poke(text_poke_f func, void *addr, void *src, size_t len) while (patched < len) { ptr = addr + patched; - size = min_t(size_t, PAGE_SIZE - offset_in_page(ptr), - len - patched); + size = min(PAGE_SIZE - offset_in_page(ptr), len - patched); waddr = patch_map(ptr, FIX_TEXT_POKE0); func(waddr, src, patched, size); diff --git a/arch/arm64/kernel/pi/map_kernel.c b/arch/arm64/kernel/pi/map_kernel.c index a852264958c3..fb44cbdd2f29 100644 --- a/arch/arm64/kernel/pi/map_kernel.c +++ b/arch/arm64/kernel/pi/map_kernel.c @@ -165,7 +165,7 @@ static void noinline __section(".idmap.text") set_ttbr0_for_lpa2(phys_addr_t ttb static void __init remap_idmap_for_lpa2(void) { /* clear the bits that change meaning once LPA2 is turned on */ - ptdesc_t mask = PTE_SHARED; + ptval_t mask = PTE_SHARED; /* * We have to clear bits [9:8] in all block or page descriptors in the diff --git a/arch/arm64/kernel/pi/map_range.c b/arch/arm64/kernel/pi/map_range.c index de52cd85c691..761b14893f74 100644 --- a/arch/arm64/kernel/pi/map_range.c +++ b/arch/arm64/kernel/pi/map_range.c @@ -31,7 +31,7 @@ void __init map_range(phys_addr_t *pte, u64 start, u64 end, phys_addr_t pa, u64 va_offset) { u64 cmask = (level == 3) ? CONT_PTE_SIZE - 1 : U64_MAX; - ptdesc_t protval = pgprot_val(prot) & ~PTE_TYPE_MASK; + ptval_t protval = pgprot_val(prot) & ~PTE_TYPE_MASK; int lshift = (3 - level) * PTDESC_TABLE_SHIFT; u64 lmask = (PAGE_SIZE << lshift) - 1; @@ -88,7 +88,7 @@ void __init map_range(phys_addr_t *pte, u64 start, u64 end, phys_addr_t pa, } } -asmlinkage phys_addr_t __init create_init_idmap(pgd_t *pg_dir, ptdesc_t clrmask) +asmlinkage phys_addr_t __init create_init_idmap(pgd_t *pg_dir, ptval_t clrmask) { phys_addr_t ptep = (phys_addr_t)pg_dir + PAGE_SIZE; /* MMU is off */ pgprot_t text_prot = PAGE_KERNEL_ROX; diff --git a/arch/arm64/kernel/pi/pi.h b/arch/arm64/kernel/pi/pi.h index aec3172d4003..5dfd8484d200 100644 --- a/arch/arm64/kernel/pi/pi.h +++ b/arch/arm64/kernel/pi/pi.h @@ -35,4 +35,4 @@ void map_range(phys_addr_t *pte, u64 start, u64 end, phys_addr_t pa, asmlinkage void early_map_kernel(u64 boot_status, phys_addr_t fdt); -asmlinkage phys_addr_t create_init_idmap(pgd_t *pgd, ptdesc_t clrmask); +asmlinkage phys_addr_t create_init_idmap(pgd_t *pgd, ptval_t clrmask); diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c index b3801f532b10..7bb6553fec08 100644 --- a/arch/arm64/kernel/proton-pack.c +++ b/arch/arm64/kernel/proton-pack.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -61,7 +62,7 @@ static void update_mitigation_state(enum mitigation_state *oldp, ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf) { - return sprintf(buf, "Mitigation: __user pointer sanitization\n"); + return sysfs_emit(buf, "Mitigation: __user pointer sanitization\n"); } /* @@ -126,7 +127,7 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, switch (spectre_v2_state) { case SPECTRE_UNAFFECTED: if (bhb_state == SPECTRE_UNAFFECTED) - return sprintf(buf, "Not affected\n"); + return sysfs_emit(buf, "Not affected\n"); /* * Platforms affected by Spectre-BHB can't report @@ -136,13 +137,13 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, fallthrough; case SPECTRE_MITIGATED: if (bhb_state == SPECTRE_MITIGATED && _unprivileged_ebpf_enabled()) - return sprintf(buf, "Vulnerable: Unprivileged eBPF enabled\n"); + return sysfs_emit(buf, "Vulnerable: Unprivileged eBPF enabled\n"); - return sprintf(buf, "Mitigation: %s%s\n", v2_str, bhb_str); + return sysfs_emit(buf, "Mitigation: %s%s\n", v2_str, bhb_str); case SPECTRE_VULNERABLE: fallthrough; default: - return sprintf(buf, "Vulnerable\n"); + return sysfs_emit(buf, "Vulnerable\n"); } } @@ -438,13 +439,13 @@ ssize_t cpu_show_spec_store_bypass(struct device *dev, { switch (spectre_v4_state) { case SPECTRE_UNAFFECTED: - return sprintf(buf, "Not affected\n"); + return sysfs_emit(buf, "Not affected\n"); case SPECTRE_MITIGATED: - return sprintf(buf, "Mitigation: Speculative Store Bypass disabled via prctl\n"); + return sysfs_emit(buf, "Mitigation: Speculative Store Bypass disabled via prctl\n"); case SPECTRE_VULNERABLE: fallthrough; default: - return sprintf(buf, "Vulnerable\n"); + return sysfs_emit(buf, "Vulnerable\n"); } } diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 5f0228e0119f..2b6951877868 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -751,15 +751,21 @@ void __init smp_init_cpus(void) else acpi_parse_and_init_cpus(); - if (cpu_count > nr_cpu_ids) - pr_warn("Number of cores (%d) exceeds configured maximum of %u - clipping\n", - cpu_count, nr_cpu_ids); - if (!bootcpu_valid) { pr_err("missing boot CPU MPIDR, not enabling secondaries\n"); return; } + /* + * For the nosmp/maxcpus=0 case, do not mark the secondary CPUs + * possible. + */ + if (!setup_max_cpus) + return; + + if (cpu_count > nr_cpu_ids) + pr_warn("Number of cores (%d) exceeds configured maximum of %u - clipping\n", + cpu_count, nr_cpu_ids); /* * We need to set the cpu_logical_map entries before enabling * the cpus so that cpu processor description entries (DT cpu nodes diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index 8aaf404980a7..af1d72020976 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -349,9 +349,15 @@ SECTIONS _edata = .; /* start of zero-init region */ - BSS_SECTION(SBSS_ALIGN, 0, 0) + BSS_SECTION(SBSS_ALIGN, 0, PAGE_SIZE) __pi___bss_start = __bss_start; + /* page table BSS starts here - preceding data/BSS is omitted from the linear map */ + .pgtbl : ALIGN(PAGE_SIZE) { + *(.bss..pgtbl) + } + ASSERT(ADDR(.pgtbl) == __bss_stop, ".pgtbl must follow BSS") + . = ALIGN(PAGE_SIZE); __pi_init_pg_dir = .; . += INIT_DIR_SIZE; diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 9453321ef8c6..7def2fad21e8 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -2502,10 +2502,10 @@ static void __init teardown_hyp_mode(void) continue; if (free_sve) { - struct cpu_sve_state *sve_state; + struct arm64_sve_state *sve_regs; - sve_state = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state; - free_pages((unsigned long) sve_state, pkvm_host_sve_state_order()); + sve_regs = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_regs; + free_pages((unsigned long) sve_regs, pkvm_host_sve_state_order()); } free_pages(kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu], nvhe_percpu_order()); @@ -2630,7 +2630,7 @@ static int init_pkvm_host_sve_state(void) if (!page) return -ENOMEM; - per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state = page_address(page); + per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_regs = page_address(page); } /* @@ -2667,11 +2667,11 @@ static void finalize_init_hyp_mode(void) if (system_supports_sve() && is_protected_kvm_enabled()) { for_each_possible_cpu(cpu) { - struct cpu_sve_state *sve_state; + struct arm64_sve_state *sve_regs; - sve_state = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state; - per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state = - kern_hyp_va(sve_state); + sve_regs = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_regs; + per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_regs = + kern_hyp_va(sve_regs); } } } diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index 332c453b87cf..b01d6622b872 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -500,7 +500,7 @@ static int get_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) if (!kvm_arm_vcpu_sve_finalized(vcpu)) return -EPERM; - if (copy_to_user(uptr, vcpu->arch.sve_state + region.koffset, + if (copy_to_user(uptr, (void *)vcpu->arch.sve_state + region.koffset, region.klen) || clear_user(uptr + region.klen, region.upad)) return -EFAULT; @@ -526,7 +526,7 @@ static int set_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) if (!kvm_arm_vcpu_sve_finalized(vcpu)) return -EPERM; - if (copy_from_user(vcpu->arch.sve_state + region.koffset, uptr, + if (copy_from_user((void *)vcpu->arch.sve_state + region.koffset, uptr, region.klen)) return -EFAULT; diff --git a/arch/arm64/kvm/hyp/entry.S b/arch/arm64/kvm/hyp/entry.S index 11a10d8f5beb..308100ed25de 100644 --- a/arch/arm64/kvm/hyp/entry.S +++ b/arch/arm64/kvm/hyp/entry.S @@ -8,7 +8,6 @@ #include #include -#include #include #include #include diff --git a/arch/arm64/kvm/hyp/fpsimd.S b/arch/arm64/kvm/hyp/fpsimd.S deleted file mode 100644 index e950875e31ce..000000000000 --- a/arch/arm64/kvm/hyp/fpsimd.S +++ /dev/null @@ -1,33 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2015 - ARM Ltd - * Author: Marc Zyngier - */ - -#include - -#include - - .text - -SYM_FUNC_START(__fpsimd_save_state) - fpsimd_save x0, 1 - ret -SYM_FUNC_END(__fpsimd_save_state) - -SYM_FUNC_START(__fpsimd_restore_state) - fpsimd_restore x0, 1 - ret -SYM_FUNC_END(__fpsimd_restore_state) - -SYM_FUNC_START(__sve_restore_state) - mov x2, #1 - sve_load 0, x1, x2, 3 - ret -SYM_FUNC_END(__sve_restore_state) - -SYM_FUNC_START(__sve_save_state) - mov x2, #1 - sve_save 0, x1, x2, 3 - ret -SYM_FUNC_END(__sve_save_state) diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index e9b36a3b27bb..d549d550b6e1 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -471,9 +471,8 @@ static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu) * vCPU. Start off with the max VL so we can load the SVE state. */ sve_cond_update_zcr_vq(zcr_el2, SYS_ZCR_EL2); - __sve_restore_state(vcpu_sve_pffr(vcpu), - &vcpu->arch.ctxt.fp_regs.fpsr, - true); + sve_load_state(kern_hyp_va(vcpu->arch.sve_state), true); + fpsimd_load_common(&vcpu->arch.ctxt.fp_regs); /* * The effective VL for a VM could differ from the max VL when running a @@ -490,13 +489,13 @@ static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu) static inline void __hyp_sve_save_host(void) { - struct cpu_sve_state *sve_state = *host_data_ptr(sve_state); + struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); + struct arm64_sve_state *sve_regs = *host_data_ptr(sve_regs); - sve_state->zcr_el1 = read_sysreg_el1(SYS_ZCR); + ctxt_sys_reg(hctxt, ZCR_EL1) = read_sysreg_el1(SYS_ZCR); write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2); - __sve_save_state(sve_state->sve_regs + sve_ffr_offset(kvm_host_sve_max_vl), - &sve_state->fpsr, - true); + sve_save_state(sve_regs, true); + fpsimd_save_common(&hctxt->fp_regs); } static inline void fpsimd_lazy_switch_to_guest(struct kvm_vcpu *vcpu) @@ -560,6 +559,8 @@ static inline void fpsimd_lazy_switch_to_host(struct kvm_vcpu *vcpu) static void kvm_hyp_save_fpsimd_host(struct kvm_vcpu *vcpu) { + struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); + /* * Non-protected kvm relies on the host restoring its sve state. * Protected kvm restores the host's sve state as not to reveal that @@ -568,11 +569,11 @@ static void kvm_hyp_save_fpsimd_host(struct kvm_vcpu *vcpu) if (system_supports_sve()) { __hyp_sve_save_host(); } else { - __fpsimd_save_state(host_data_ptr(host_ctxt.fp_regs)); + fpsimd_save_state(&hctxt->fp_regs); } if (kvm_has_fpmr(kern_hyp_va(vcpu->kvm))) - *host_data_ptr(fpmr) = read_sysreg_s(SYS_FPMR); + ctxt_sys_reg(hctxt, FPMR) = read_sysreg_s(SYS_FPMR); } @@ -628,7 +629,7 @@ static inline bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) if (sve_guest) __hyp_sve_restore_guest(vcpu); else - __fpsimd_restore_state(&vcpu->arch.ctxt.fp_regs); + fpsimd_load_state(&vcpu->arch.ctxt.fp_regs); if (kvm_has_fpmr(kern_hyp_va(vcpu->kvm))) write_sysreg_s(__vcpu_sys_reg(vcpu, FPMR), SYS_FPMR); diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile index 62cdfbff7562..f57450ebcb49 100644 --- a/arch/arm64/kvm/hyp/nvhe/Makefile +++ b/arch/arm64/kvm/hyp/nvhe/Makefile @@ -26,7 +26,7 @@ hyp-obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o hyp-main.o hyp-smp.o psci-relay.o early_alloc.o page_alloc.o \ cache.o setup.o mm.o mem_protect.o sys_regs.o pkvm.o stacktrace.o ffa.o hyp-obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \ - ../fpsimd.o ../hyp-entry.o ../exception.o ../pgtable.o ../vgic-v5-sr.o + ../hyp-entry.o ../exception.o ../pgtable.o ../vgic-v5-sr.o hyp-obj-y += ../../../kernel/smccc-call.o hyp-obj-$(CONFIG_LIST_HARDENED) += list_debug.o hyp-obj-$(CONFIG_NVHE_EL2_TRACING) += clock.o trace.o events.o diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 06db299c37a8..51dad35000f5 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -35,13 +35,15 @@ static void __hyp_sve_save_guest(struct kvm_vcpu *vcpu) * on the VL, so use a consistent (i.e., the maximum) guest VL. */ sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2); - __sve_save_state(vcpu_sve_pffr(vcpu), &vcpu->arch.ctxt.fp_regs.fpsr, true); + sve_save_state(kern_hyp_va(vcpu->arch.sve_state), true); + fpsimd_save_common(&vcpu->arch.ctxt.fp_regs); write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2); } static void __hyp_sve_restore_host(void) { - struct cpu_sve_state *sve_state = *host_data_ptr(sve_state); + struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); + struct arm64_sve_state *sve_regs = *host_data_ptr(sve_regs); /* * On saving/restoring host sve state, always use the maximum VL for @@ -53,10 +55,9 @@ static void __hyp_sve_restore_host(void) * need to be revisited. */ write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2); - __sve_restore_state(sve_state->sve_regs + sve_ffr_offset(kvm_host_sve_max_vl), - &sve_state->fpsr, - true); - write_sysreg_el1(sve_state->zcr_el1, SYS_ZCR); + sve_load_state(sve_regs, true); + fpsimd_load_common(&hctxt->fp_regs); + write_sysreg_el1(ctxt_sys_reg(hctxt, ZCR_EL1), SYS_ZCR); } static void fpsimd_sve_flush(void) @@ -66,6 +67,7 @@ static void fpsimd_sve_flush(void) static void fpsimd_sve_sync(struct kvm_vcpu *vcpu) { + struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); bool has_fpmr; if (!guest_owns_fp_regs()) @@ -80,7 +82,7 @@ static void fpsimd_sve_sync(struct kvm_vcpu *vcpu) if (vcpu_has_sve(vcpu)) __hyp_sve_save_guest(vcpu); else - __fpsimd_save_state(&vcpu->arch.ctxt.fp_regs); + fpsimd_save_state(&vcpu->arch.ctxt.fp_regs); has_fpmr = kvm_has_fpmr(kern_hyp_va(vcpu->kvm)); if (has_fpmr) @@ -89,10 +91,10 @@ static void fpsimd_sve_sync(struct kvm_vcpu *vcpu) if (system_supports_sve()) __hyp_sve_restore_host(); else - __fpsimd_restore_state(host_data_ptr(host_ctxt.fp_regs)); + fpsimd_load_state(&hctxt->fp_regs); if (has_fpmr) - write_sysreg_s(*host_data_ptr(fpmr), SYS_FPMR); + write_sysreg_s(ctxt_sys_reg(hctxt, FPMR), SYS_FPMR); *host_data_ptr(fp_owner) = FP_STATE_HOST_OWNED; } diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c index d461981616d9..75b00c323310 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -82,9 +82,9 @@ static int pkvm_create_host_sve_mappings(void) for (i = 0; i < hyp_nr_cpus; i++) { struct kvm_host_data *host_data = per_cpu_ptr(&kvm_host_data, i); - struct cpu_sve_state *sve_state = host_data->sve_state; + struct arm64_sve_state *sve_regs = host_data->sve_regs; - start = kern_hyp_va(sve_state); + start = kern_hyp_va(sve_regs); end = start + PAGE_ALIGN(pkvm_host_sve_state_size()); ret = pkvm_create_mappings(start, end, PAGE_HYP); if (ret) diff --git a/arch/arm64/kvm/hyp/vhe/Makefile b/arch/arm64/kvm/hyp/vhe/Makefile index 9695328bbd96..d6b3475145c0 100644 --- a/arch/arm64/kvm/hyp/vhe/Makefile +++ b/arch/arm64/kvm/hyp/vhe/Makefile @@ -10,4 +10,4 @@ CFLAGS_switch.o += -Wno-override-init obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \ - ../fpsimd.o ../hyp-entry.o ../exception.o ../vgic-v5-sr.o + ../hyp-entry.o ../exception.o ../vgic-v5-sr.o diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 4da9281312eb..6e95514d5a76 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1479,6 +1479,11 @@ static void sanitise_mte_tags(struct kvm *kvm, kvm_pfn_t pfn, if (!kvm_has_mte(kvm)) return; + if (is_zero_pfn(pfn)) { + WARN_ON_ONCE(nr_pages != 1); + return; + } + if (folio_test_hugetlb(folio)) { /* Hugetlb has MTE flags set on head page only */ if (folio_try_hugetlb_mte_tagging(folio)) { diff --git a/arch/arm64/mm/fixmap.c b/arch/arm64/mm/fixmap.c index c5c5425791da..f66a0016dd02 100644 --- a/arch/arm64/mm/fixmap.c +++ b/arch/arm64/mm/fixmap.c @@ -31,9 +31,9 @@ static_assert(NR_BM_PMD_TABLES == 1); #define BM_PTE_TABLE_IDX(addr) __BM_TABLE_IDX(addr, PMD_SHIFT) -static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __page_aligned_bss; -static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused; -static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused; +static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __bss_pgtbl; +static pmd_t bm_pmd[PTRS_PER_PMD] __bss_pgtbl __maybe_unused; +static pud_t bm_pud[PTRS_PER_PUD] __bss_pgtbl __maybe_unused; static inline pte_t *fixmap_pte(unsigned long addr) { diff --git a/arch/arm64/mm/kasan_init.c b/arch/arm64/mm/kasan_init.c index abeb81bf6ebd..3fcad956fdf7 100644 --- a/arch/arm64/mm/kasan_init.c +++ b/arch/arm64/mm/kasan_init.c @@ -214,7 +214,7 @@ asmlinkage void __init kasan_early_init(void) * shadow pud_t[]/p4d_t[], which could end up getting corrupted * when the linear region is mapped. */ - static pte_t tbl[PTRS_PER_PTE] __page_aligned_bss; + static pte_t tbl[PTRS_PER_PTE] __bss_pgtbl; pgd_t *pgdp = pgd_offset_k(KASAN_SHADOW_START); set_pgd(pgdp, __pgd(__pa_symbol(tbl) | PGD_TYPE_TABLE)); diff --git a/arch/arm64/mm/mmap.c b/arch/arm64/mm/mmap.c index 92b2f5097a96..32e0771d6477 100644 --- a/arch/arm64/mm/mmap.c +++ b/arch/arm64/mm/mmap.c @@ -34,7 +34,7 @@ static pgprot_t protection_map[16] __ro_after_init = { [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = PAGE_SHARED_EXEC }; -static ptdesc_t gcs_page_prot __ro_after_init = _PAGE_GCS_RO; +static ptval_t gcs_page_prot __ro_after_init = _PAGE_GCS_RO; /* * You really shouldn't be using read() or write() on /dev/mem. This might go @@ -87,7 +87,7 @@ arch_initcall(adjust_protection_map); pgprot_t vm_get_page_prot(vm_flags_t vm_flags) { - ptdesc_t prot; + ptval_t prot; /* Short circuit GCS to avoid bloating the table. */ if (system_supports_gcs() && (vm_flags & VM_SHADOW_STACK)) { diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 8242f93f05e4..9f354971b7e4 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -134,10 +134,6 @@ bool pgattr_change_is_safe(pteval_t old, pteval_t new) if (pte_pfn(__pte(old)) != pte_pfn(__pte(new))) return false; - /* live contiguous mappings may not be manipulated at all */ - if ((old | new) & PTE_CONT) - return false; - /* Transitioning from Non-Global to Global is unsafe */ if (old & ~new & PTE_NG) return false; @@ -187,6 +183,17 @@ static void init_pte(pte_t *ptep, unsigned long addr, unsigned long end, } while (ptep++, addr += PAGE_SIZE, addr != end); } +static bool pte_range_has_valid_noncont(pte_t *ptep) +{ + for (int i = 0; i < CONT_PTES; i++) { + pte_t pte = __ptep_get(&ptep[i]); + + if (pte_valid(pte) && !pte_cont(pte)) + return true; + } + return false; +} + static int alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr, unsigned long end, phys_addr_t phys, pgprot_t prot, @@ -224,7 +231,8 @@ static int alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr, /* use a contiguous mapping if the range is suitably aligned */ if ((((addr | next | phys) & ~CONT_PTE_MASK) == 0) && - (flags & NO_CONT_MAPPINGS) == 0) + (flags & NO_CONT_MAPPINGS) == 0 && + !pte_range_has_valid_noncont(ptep)) __prot = __pgprot(pgprot_val(prot) | PTE_CONT); init_pte(ptep, addr, next, phys, __prot); @@ -256,8 +264,9 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end, /* try section mapping first */ if (((addr | next | phys) & ~PMD_MASK) == 0 && - (flags & NO_BLOCK_MAPPINGS) == 0) { - pmd_set_huge(pmdp, phys, prot); + (flags & NO_BLOCK_MAPPINGS) == 0 && + !pmd_table(old_pmd)) { + WARN_ON(!pmd_set_huge(pmdp, phys, prot)); /* * After the PMD entry has been populated once, we @@ -273,8 +282,8 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end, if (ret) return ret; - BUG_ON(pmd_val(old_pmd) != 0 && - pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp))); + VM_WARN_ON_ONCE(pmd_val(old_pmd) != 0 && + pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp))); } phys += next - addr; } while (pmdp++, addr = next, addr != end); @@ -282,6 +291,17 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end, return 0; } +static bool pmd_range_has_valid_noncont(pmd_t *pmdp) +{ + for (int i = 0; i < CONT_PMDS; i++) { + pte_t pte = pmd_pte(READ_ONCE(pmdp[i])); + + if (pte_valid(pte) && !pte_cont(pte)) + return true; + } + return false; +} + static int alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, unsigned long end, phys_addr_t phys, pgprot_t prot, @@ -323,7 +343,8 @@ static int alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, /* use a contiguous mapping if the range is suitably aligned */ if ((((addr | next | phys) & ~CONT_PMD_MASK) == 0) && - (flags & NO_CONT_MAPPINGS) == 0) + (flags & NO_CONT_MAPPINGS) == 0 && + !pmd_range_has_valid_noncont(pmdp)) __prot = __pgprot(pgprot_val(prot) | PTE_CONT); ret = init_pmd(pmdp, addr, next, phys, __prot, pgtable_alloc, flags); @@ -379,8 +400,9 @@ static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end, */ if (pud_sect_supported() && ((addr | next | phys) & ~PUD_MASK) == 0 && - (flags & NO_BLOCK_MAPPINGS) == 0) { - pud_set_huge(pudp, phys, prot); + (flags & NO_BLOCK_MAPPINGS) == 0 && + !pud_table(old_pud)) { + WARN_ON(!pud_set_huge(pudp, phys, prot)); /* * After the PUD entry has been populated once, we @@ -394,8 +416,8 @@ static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end, if (ret) goto out; - BUG_ON(pud_val(old_pud) != 0 && - pud_val(old_pud) != READ_ONCE(pud_val(*pudp))); + VM_WARN_ON_ONCE(pud_val(old_pud) != 0 && + pud_val(old_pud) != READ_ONCE(pud_val(*pudp))); } phys += next - addr; } while (pudp++, addr = next, addr != end); @@ -445,8 +467,8 @@ static int alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end, if (ret) goto out; - BUG_ON(p4d_val(old_p4d) != 0 && - p4d_val(old_p4d) != READ_ONCE(p4d_val(*p4dp))); + VM_WARN_ON_ONCE(p4d_val(old_p4d) != 0 && + p4d_val(old_p4d) != READ_ONCE(p4d_val(*p4dp))); phys += next - addr; } while (p4dp++, addr = next, addr != end); @@ -1000,8 +1022,7 @@ void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt, &phys, virt); return; } - early_create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, - NO_CONT_MAPPINGS); + early_create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, 0); } void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, @@ -1028,18 +1049,17 @@ static void update_mapping_prot(phys_addr_t phys, unsigned long virt, return; } - early_create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, - NO_CONT_MAPPINGS); + early_create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, 0); /* flush the TLBs after updating live kernel mappings */ flush_tlb_kernel_range(virt, virt + size); } -static void __init __map_memblock(pgd_t *pgdp, phys_addr_t start, - phys_addr_t end, pgprot_t prot, int flags) +static void __init __map_memblock(phys_addr_t start, phys_addr_t end, + pgprot_t prot, int flags) { - early_create_pgd_mapping(pgdp, start, __phys_to_virt(start), end - start, - prot, early_pgtable_alloc, flags); + early_create_pgd_mapping(swapper_pg_dir, start, __phys_to_virt(start), + end - start, prot, early_pgtable_alloc, flags); } void __init mark_linear_text_alias_ro(void) @@ -1067,36 +1087,24 @@ static int __init parse_kfence_early_init(char *arg) } early_param("kfence.sample_interval", parse_kfence_early_init); -static phys_addr_t __init arm64_kfence_alloc_pool(void) +static void __init arm64_kfence_map_pool(void) { phys_addr_t kfence_pool; if (!kfence_early_init) - return 0; + return; kfence_pool = memblock_phys_alloc(KFENCE_POOL_SIZE, PAGE_SIZE); if (!kfence_pool) { pr_err("failed to allocate kfence pool\n"); kfence_early_init = false; - return 0; + return; } - /* Temporarily mark as NOMAP. */ - memblock_mark_nomap(kfence_pool, KFENCE_POOL_SIZE); - - return kfence_pool; -} - -static void __init arm64_kfence_map_pool(phys_addr_t kfence_pool, pgd_t *pgdp) -{ - if (!kfence_pool) - return; - /* KFENCE pool needs page-level mapping. */ - __map_memblock(pgdp, kfence_pool, kfence_pool + KFENCE_POOL_SIZE, + __map_memblock(kfence_pool, kfence_pool + KFENCE_POOL_SIZE, pgprot_tagged(PAGE_KERNEL), - NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS); - memblock_clear_nomap(kfence_pool, KFENCE_POOL_SIZE); + NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS | NO_EXEC_MAPPINGS); __kfence_pool = phys_to_virt(kfence_pool); } @@ -1128,18 +1136,18 @@ bool arch_kfence_init_pool(void) } #else /* CONFIG_KFENCE */ -static inline phys_addr_t arm64_kfence_alloc_pool(void) { return 0; } -static inline void arm64_kfence_map_pool(phys_addr_t kfence_pool, pgd_t *pgdp) { } +static inline void arm64_kfence_map_pool(void) { } #endif /* CONFIG_KFENCE */ -static void __init map_mem(pgd_t *pgdp) +static void __init map_mem(void) { static const u64 direct_map_end = _PAGE_END(VA_BITS_MIN); phys_addr_t kernel_start = __pa_symbol(_text); - phys_addr_t kernel_end = __pa_symbol(__init_begin); + phys_addr_t init_begin = __pa_symbol(__init_begin); + phys_addr_t init_end = __pa_symbol(__init_end); + phys_addr_t kernel_end = __pa_symbol(__bss_stop); phys_addr_t start, end; - phys_addr_t early_kfence_pool; int flags = NO_EXEC_MAPPINGS; u64 i; @@ -1156,7 +1164,7 @@ static void __init map_mem(pgd_t *pgdp) BUILD_BUG_ON(pgd_index(direct_map_end - 1) == pgd_index(direct_map_end) && pgd_index(_PAGE_OFFSET(VA_BITS_MIN)) != PTRS_PER_PGD - 1); - early_kfence_pool = arm64_kfence_alloc_pool(); + arm64_kfence_map_pool(); linear_map_requires_bbml2 = !force_pte_mapping() && can_set_direct_map(); @@ -1164,40 +1172,37 @@ static void __init map_mem(pgd_t *pgdp) flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; /* - * Take care not to create a writable alias for the - * read-only text and rodata sections of the kernel image. - * So temporarily mark them as NOMAP to skip mappings in - * the following for-loop + * Map the linear alias of the [_text, __init_begin) interval first + * so that its write permissions can be removed later without the need + * to split any block mappings created by the loop below. + * + * Write permissions are needed for alternatives patching, and will be + * removed later by mark_linear_text_alias_ro() above. This makes the + * contents of the region accessible to subsystems such as hibernate, + * but protects it from inadvertent modification or execution. */ - memblock_mark_nomap(kernel_start, kernel_end - kernel_start); + __map_memblock(kernel_start, init_begin, pgprot_tagged(PAGE_KERNEL), + flags); + + /* Map the kernel data/bss so it can be remapped later */ + __map_memblock(init_end, kernel_end, pgprot_tagged(PAGE_KERNEL), + flags); /* map all the memory banks */ for_each_mem_range(i, &start, &end) { - if (start >= end) - break; /* * The linear map must allow allocation tags reading/writing * if MTE is present. Otherwise, it has the same attributes as * PAGE_KERNEL. */ - __map_memblock(pgdp, start, end, pgprot_tagged(PAGE_KERNEL), + __map_memblock(start, end, pgprot_tagged(PAGE_KERNEL), flags); } - /* - * Map the linear alias of the [_text, __init_begin) interval - * as non-executable now, and remove the write permission in - * mark_linear_text_alias_ro() below (which will be called after - * alternative patching has completed). This makes the contents - * of the region accessible to subsystems such as hibernate, - * but protects it from inadvertent modification or execution. - * Note that contiguous mappings cannot be remapped in this way, - * so we should avoid them here. - */ - __map_memblock(pgdp, kernel_start, kernel_end, - PAGE_KERNEL, NO_CONT_MAPPINGS); - memblock_clear_nomap(kernel_start, kernel_end - kernel_start); - arm64_kfence_map_pool(early_kfence_pool, pgdp); + /* Map the kernel data/bss read-only in the linear map */ + __map_memblock(init_end, kernel_end, PAGE_KERNEL_RO, flags); + flush_tlb_kernel_range((unsigned long)lm_alias(__init_end), + (unsigned long)lm_alias(__bss_stop)); } void mark_rodata_ro(void) @@ -1419,7 +1424,7 @@ static void __init create_idmap(void) void __init paging_init(void) { - map_mem(swapper_pg_dir); + map_mem(); memblock_allow_resize(); diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c index ce035e1b4eaf..bbe98ac9ad8c 100644 --- a/arch/arm64/mm/pageattr.c +++ b/arch/arm64/mm/pageattr.c @@ -21,7 +21,7 @@ struct page_change_data { pgprot_t clear_mask; }; -static ptdesc_t set_pageattr_masks(ptdesc_t val, struct mm_walk *walk) +static ptval_t set_pageattr_masks(ptval_t val, struct mm_walk *walk) { struct page_change_data *masks = walk->private; diff --git a/arch/arm64/mm/ptdump.c b/arch/arm64/mm/ptdump.c index ab9899ca1e5f..1c20144700d7 100644 --- a/arch/arm64/mm/ptdump.c +++ b/arch/arm64/mm/ptdump.c @@ -194,7 +194,7 @@ void note_page(struct ptdump_state *pt_st, unsigned long addr, int level, struct ptdump_pg_state *st = container_of(pt_st, struct ptdump_pg_state, ptdump); struct ptdump_pg_level *pg_level = st->pg_level; static const char units[] = "KMGTPE"; - ptdesc_t prot = 0; + ptval_t prot = 0; /* check if the current level has been folded dynamically */ if (st->mm && ((level == 1 && mm_p4d_folded(st->mm)) || diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 811c2479e82d..9b85a84f6fd4 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -120,7 +120,7 @@ WORKAROUND_CAVIUM_TX2_219_PRFM WORKAROUND_CAVIUM_TX2_219_TVM WORKAROUND_CLEAN_CACHE WORKAROUND_DEVICE_LOAD_ACQUIRE -WORKAROUND_NVIDIA_CARMEL_CNP +WORKAROUND_DISABLE_CNP WORKAROUND_PMUV3_IMPDEF_TRAPS WORKAROUND_QCOM_FALKOR_E1003 WORKAROUND_QCOM_ORYON_CNTVOFF diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 6c3ff14e561e..bc1788b1662b 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -3790,6 +3790,51 @@ Field 1 ZA Field 0 SM EndSysreg +Sysreg FPCR 3 3 4 4 0 +Res0 63:27 +Field 26 AHP +Field 25 DN +Field 24 FZ +Enum 23:22 RMode + 0b00 RN + 0b01 RP + 0b10 RM + 0b11 RZ +EndEnum +Field 21:20 Stride +Field 19 FZ16 +Field 18:16 Len +Field 15 IDE +Res0 14 +Field 13 EBF +Field 12 IXE +Field 11 UFE +Field 10 OFE +Field 9 DZE +Field 8 IOE +Res0 7:3 +Field 2 NEP +Field 1 AH +Field 0 FIZ +EndSysreg + +Sysreg FPSR 3 3 4 4 1 +Res0 63:32 +Field 31 N +Field 30 Z +Field 29 C +Field 28 V +Field 27 QC +Res0 26:8 +Field 7 IDC +Res0 6:5 +Field 4 IXC +Field 3 UFC +Field 2 OFC +Field 1 DZC +Field 0 IOC +EndSysreg + Sysreg FPMR 3 3 4 4 2 Res0 63:38 Field 37:32 LSCALE2 @@ -4597,6 +4642,35 @@ Sysreg GCSPR_EL2 3 4 2 5 1 Fields GCSPR_ELx EndSysreg +Sysreg HDBSSBR_EL2 3 4 2 3 2 +Res0 63:56 +Field 55:12 BADDR +Res0 11:4 +Enum 3:0 SZ + 0b0000 4KB + 0b0001 8KB + 0b0010 16KB + 0b0011 32KB + 0b0100 64KB + 0b0101 128KB + 0b0110 256KB + 0b0111 512KB + 0b1000 1MB + 0b1001 2MB +EndEnum +EndSysreg + +Sysreg HDBSSPROD_EL2 3 4 2 3 3 +Res0 63:32 +Enum 31:26 FSC + 0b000000 OK + 0b010000 ExternalAbort + 0b101000 GPF +EndEnum +Res0 25:19 +Field 18:0 INDEX +EndSysreg + Sysreg DACR32_EL2 3 4 3 0 0 Res0 63:32 Field 31:30 D15 diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig index 47e48c18e55c..1a4f0739c6bb 100644 --- a/arch/m68k/configs/amiga_defconfig +++ b/arch/m68k/configs/amiga_defconfig @@ -244,7 +244,6 @@ CONFIG_NETLINK_DIAG=m CONFIG_MPLS=y CONFIG_NET_MPLS_GSO=m CONFIG_MPLS_ROUTING=m -CONFIG_MPLS_IPTUNNEL=m CONFIG_NET_NSH=m CONFIG_AF_KCM=m # CONFIG_WIRELESS is not set @@ -394,8 +393,6 @@ CONFIG_HID=m CONFIG_HIDRAW=y CONFIG_UHID=m # CONFIG_HID_GENERIC is not set -# CONFIG_HID_ITE is not set -# CONFIG_HID_REDRAGON is not set # CONFIG_USB_SUPPORT is not set CONFIG_RTC_CLASS=y # CONFIG_RTC_NVMEM is not set @@ -539,9 +536,13 @@ CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_CMAC=m CONFIG_CRYPTO_MD4=m +CONFIG_CRYPTO_MD5=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m +CONFIG_CRYPTO_SHA512=m +CONFIG_CRYPTO_SHA3=m CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m @@ -550,14 +551,13 @@ CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m CONFIG_CRYPTO_ZSTD=m -CONFIG_CRYPTO_DRBG_HASH=y -CONFIG_CRYPTO_DRBG_CTR=y CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m # CONFIG_CRYPTO_HW is not set CONFIG_PRIME_NUMBERS=m +CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=m CONFIG_CRC_BENCHMARK=y CONFIG_XZ_DEC_TEST=m # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set @@ -574,6 +574,7 @@ CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m CONFIG_TEST_HEXDUMP=m +CONFIG_STRING_KUNIT_BENCH=y CONFIG_TEST_KSTRTOX=m CONFIG_TEST_BITMAP=m CONFIG_TEST_XARRAY=m @@ -582,11 +583,13 @@ CONFIG_TEST_RHASHTABLE=m CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m +CONFIG_TEST_WORKQUEUE=m CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m CONFIG_LINEAR_RANGES_TEST=m +CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig index 161586d611ab..dacce863e37f 100644 --- a/arch/m68k/configs/apollo_defconfig +++ b/arch/m68k/configs/apollo_defconfig @@ -240,7 +240,6 @@ CONFIG_NETLINK_DIAG=m CONFIG_MPLS=y CONFIG_NET_MPLS_GSO=m CONFIG_MPLS_ROUTING=m -CONFIG_MPLS_IPTUNNEL=m CONFIG_NET_NSH=m CONFIG_AF_KCM=m # CONFIG_WIRELESS is not set @@ -350,8 +349,6 @@ CONFIG_HID=m CONFIG_HIDRAW=y CONFIG_UHID=m # CONFIG_HID_GENERIC is not set -# CONFIG_HID_ITE is not set -# CONFIG_HID_REDRAGON is not set # CONFIG_USB_SUPPORT is not set CONFIG_RTC_CLASS=y # CONFIG_RTC_NVMEM is not set @@ -494,9 +491,13 @@ CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_CMAC=m CONFIG_CRYPTO_MD4=m +CONFIG_CRYPTO_MD5=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m +CONFIG_CRYPTO_SHA512=m +CONFIG_CRYPTO_SHA3=m CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m @@ -505,14 +506,13 @@ CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m CONFIG_CRYPTO_ZSTD=m -CONFIG_CRYPTO_DRBG_HASH=y -CONFIG_CRYPTO_DRBG_CTR=y CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m # CONFIG_CRYPTO_HW is not set CONFIG_PRIME_NUMBERS=m +CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=m CONFIG_CRC_BENCHMARK=y CONFIG_XZ_DEC_TEST=m # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set @@ -529,6 +529,7 @@ CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m CONFIG_TEST_HEXDUMP=m +CONFIG_STRING_KUNIT_BENCH=y CONFIG_TEST_KSTRTOX=m CONFIG_TEST_BITMAP=m CONFIG_TEST_XARRAY=m @@ -537,11 +538,13 @@ CONFIG_TEST_RHASHTABLE=m CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m +CONFIG_TEST_WORKQUEUE=m CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m CONFIG_LINEAR_RANGES_TEST=m +CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig index c13c6deeac22..ee0349fc3d9e 100644 --- a/arch/m68k/configs/atari_defconfig +++ b/arch/m68k/configs/atari_defconfig @@ -247,7 +247,6 @@ CONFIG_NETLINK_DIAG=m CONFIG_MPLS=y CONFIG_NET_MPLS_GSO=m CONFIG_MPLS_ROUTING=m -CONFIG_MPLS_IPTUNNEL=m CONFIG_NET_NSH=m CONFIG_AF_KCM=m # CONFIG_WIRELESS is not set @@ -374,7 +373,6 @@ CONFIG_HID=m CONFIG_HIDRAW=y CONFIG_UHID=m # CONFIG_HID_GENERIC is not set -# CONFIG_HID_ITE is not set CONFIG_RTC_CLASS=y # CONFIG_RTC_NVMEM is not set CONFIG_RTC_DRV_GENERIC=m @@ -516,9 +514,13 @@ CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_CMAC=m CONFIG_CRYPTO_MD4=m +CONFIG_CRYPTO_MD5=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m +CONFIG_CRYPTO_SHA512=m +CONFIG_CRYPTO_SHA3=m CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m @@ -527,14 +529,13 @@ CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m CONFIG_CRYPTO_ZSTD=m -CONFIG_CRYPTO_DRBG_HASH=y -CONFIG_CRYPTO_DRBG_CTR=y CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m # CONFIG_CRYPTO_HW is not set CONFIG_PRIME_NUMBERS=m +CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=m CONFIG_CRC_BENCHMARK=y CONFIG_XZ_DEC_TEST=m # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set @@ -551,6 +552,7 @@ CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m CONFIG_TEST_HEXDUMP=m +CONFIG_STRING_KUNIT_BENCH=y CONFIG_TEST_KSTRTOX=m CONFIG_TEST_BITMAP=m CONFIG_TEST_XARRAY=m @@ -559,11 +561,13 @@ CONFIG_TEST_RHASHTABLE=m CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m +CONFIG_TEST_WORKQUEUE=m CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m CONFIG_LINEAR_RANGES_TEST=m +CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig index d4f3f94b61ff..f1d0acf2afba 100644 --- a/arch/m68k/configs/bvme6000_defconfig +++ b/arch/m68k/configs/bvme6000_defconfig @@ -237,7 +237,6 @@ CONFIG_NETLINK_DIAG=m CONFIG_MPLS=y CONFIG_NET_MPLS_GSO=m CONFIG_MPLS_ROUTING=m -CONFIG_MPLS_IPTUNNEL=m CONFIG_NET_NSH=m CONFIG_AF_KCM=m # CONFIG_WIRELESS is not set @@ -342,8 +341,6 @@ CONFIG_HID=m CONFIG_HIDRAW=y CONFIG_UHID=m # CONFIG_HID_GENERIC is not set -# CONFIG_HID_ITE is not set -# CONFIG_HID_REDRAGON is not set # CONFIG_USB_SUPPORT is not set CONFIG_RTC_CLASS=y # CONFIG_RTC_NVMEM is not set @@ -486,9 +483,13 @@ CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_CMAC=m CONFIG_CRYPTO_MD4=m +CONFIG_CRYPTO_MD5=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m +CONFIG_CRYPTO_SHA512=m +CONFIG_CRYPTO_SHA3=m CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m @@ -497,14 +498,13 @@ CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m CONFIG_CRYPTO_ZSTD=m -CONFIG_CRYPTO_DRBG_HASH=y -CONFIG_CRYPTO_DRBG_CTR=y CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m # CONFIG_CRYPTO_HW is not set CONFIG_PRIME_NUMBERS=m +CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=m CONFIG_CRC_BENCHMARK=y CONFIG_XZ_DEC_TEST=m # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set @@ -521,6 +521,7 @@ CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m CONFIG_TEST_HEXDUMP=m +CONFIG_STRING_KUNIT_BENCH=y CONFIG_TEST_KSTRTOX=m CONFIG_TEST_BITMAP=m CONFIG_TEST_XARRAY=m @@ -529,11 +530,13 @@ CONFIG_TEST_RHASHTABLE=m CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m +CONFIG_TEST_WORKQUEUE=m CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m CONFIG_LINEAR_RANGES_TEST=m +CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig index 58288f83349d..d808468f803b 100644 --- a/arch/m68k/configs/hp300_defconfig +++ b/arch/m68k/configs/hp300_defconfig @@ -239,7 +239,6 @@ CONFIG_NETLINK_DIAG=m CONFIG_MPLS=y CONFIG_NET_MPLS_GSO=m CONFIG_MPLS_ROUTING=m -CONFIG_MPLS_IPTUNNEL=m CONFIG_NET_NSH=m CONFIG_AF_KCM=m # CONFIG_WIRELESS is not set @@ -352,8 +351,6 @@ CONFIG_HID=m CONFIG_HIDRAW=y CONFIG_UHID=m # CONFIG_HID_GENERIC is not set -# CONFIG_HID_ITE is not set -# CONFIG_HID_REDRAGON is not set # CONFIG_USB_SUPPORT is not set CONFIG_RTC_CLASS=y # CONFIG_RTC_NVMEM is not set @@ -496,9 +493,13 @@ CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_CMAC=m CONFIG_CRYPTO_MD4=m +CONFIG_CRYPTO_MD5=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m +CONFIG_CRYPTO_SHA512=m +CONFIG_CRYPTO_SHA3=m CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m @@ -507,14 +508,13 @@ CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m CONFIG_CRYPTO_ZSTD=m -CONFIG_CRYPTO_DRBG_HASH=y -CONFIG_CRYPTO_DRBG_CTR=y CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m # CONFIG_CRYPTO_HW is not set CONFIG_PRIME_NUMBERS=m +CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=m CONFIG_CRC_BENCHMARK=y CONFIG_XZ_DEC_TEST=m # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set @@ -531,6 +531,7 @@ CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m CONFIG_TEST_HEXDUMP=m +CONFIG_STRING_KUNIT_BENCH=y CONFIG_TEST_KSTRTOX=m CONFIG_TEST_BITMAP=m CONFIG_TEST_XARRAY=m @@ -539,11 +540,13 @@ CONFIG_TEST_RHASHTABLE=m CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m +CONFIG_TEST_WORKQUEUE=m CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m CONFIG_LINEAR_RANGES_TEST=m +CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig index abb369fd1f55..e594b644c0a6 100644 --- a/arch/m68k/configs/mac_defconfig +++ b/arch/m68k/configs/mac_defconfig @@ -238,7 +238,6 @@ CONFIG_NETLINK_DIAG=m CONFIG_MPLS=y CONFIG_NET_MPLS_GSO=m CONFIG_MPLS_ROUTING=m -CONFIG_MPLS_IPTUNNEL=m CONFIG_NET_NSH=m CONFIG_AF_KCM=m # CONFIG_WIRELESS is not set @@ -371,8 +370,6 @@ CONFIG_HID=m CONFIG_HIDRAW=y CONFIG_UHID=m # CONFIG_HID_GENERIC is not set -# CONFIG_HID_ITE is not set -# CONFIG_HID_REDRAGON is not set # CONFIG_USB_SUPPORT is not set CONFIG_RTC_CLASS=y # CONFIG_RTC_NVMEM is not set @@ -515,9 +512,13 @@ CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_CMAC=m CONFIG_CRYPTO_MD4=m +CONFIG_CRYPTO_MD5=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m +CONFIG_CRYPTO_SHA512=m +CONFIG_CRYPTO_SHA3=m CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m @@ -526,14 +527,13 @@ CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m CONFIG_CRYPTO_ZSTD=m -CONFIG_CRYPTO_DRBG_HASH=y -CONFIG_CRYPTO_DRBG_CTR=y CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m # CONFIG_CRYPTO_HW is not set CONFIG_PRIME_NUMBERS=m +CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=m CONFIG_CRC_BENCHMARK=y CONFIG_XZ_DEC_TEST=m # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set @@ -550,6 +550,7 @@ CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m CONFIG_TEST_HEXDUMP=m +CONFIG_STRING_KUNIT_BENCH=y CONFIG_TEST_KSTRTOX=m CONFIG_TEST_BITMAP=m CONFIG_TEST_XARRAY=m @@ -558,11 +559,13 @@ CONFIG_TEST_RHASHTABLE=m CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m +CONFIG_TEST_WORKQUEUE=m CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m CONFIG_LINEAR_RANGES_TEST=m +CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig index cb8de979700f..0c1af1894605 100644 --- a/arch/m68k/configs/multi_defconfig +++ b/arch/m68k/configs/multi_defconfig @@ -258,7 +258,6 @@ CONFIG_NETLINK_DIAG=m CONFIG_MPLS=y CONFIG_NET_MPLS_GSO=m CONFIG_MPLS_ROUTING=m -CONFIG_MPLS_IPTUNNEL=m CONFIG_NET_NSH=m CONFIG_AF_KCM=m # CONFIG_WIRELESS is not set @@ -455,8 +454,6 @@ CONFIG_HID=m CONFIG_HIDRAW=y CONFIG_UHID=m # CONFIG_HID_GENERIC is not set -# CONFIG_HID_ITE is not set -# CONFIG_HID_REDRAGON is not set # CONFIG_USB_SUPPORT is not set CONFIG_RTC_CLASS=y # CONFIG_RTC_NVMEM is not set @@ -602,9 +599,13 @@ CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_CMAC=m CONFIG_CRYPTO_MD4=m +CONFIG_CRYPTO_MD5=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m +CONFIG_CRYPTO_SHA512=m +CONFIG_CRYPTO_SHA3=m CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m @@ -613,14 +614,13 @@ CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m CONFIG_CRYPTO_ZSTD=m -CONFIG_CRYPTO_DRBG_HASH=y -CONFIG_CRYPTO_DRBG_CTR=y CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m # CONFIG_CRYPTO_HW is not set CONFIG_PRIME_NUMBERS=m +CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=m CONFIG_CRC_BENCHMARK=y CONFIG_XZ_DEC_TEST=m # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set @@ -637,6 +637,7 @@ CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m CONFIG_TEST_HEXDUMP=m +CONFIG_STRING_KUNIT_BENCH=y CONFIG_TEST_KSTRTOX=m CONFIG_TEST_BITMAP=m CONFIG_TEST_XARRAY=m @@ -645,11 +646,13 @@ CONFIG_TEST_RHASHTABLE=m CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m +CONFIG_TEST_WORKQUEUE=m CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m CONFIG_LINEAR_RANGES_TEST=m +CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig index 176540bd5074..a17e771b4760 100644 --- a/arch/m68k/configs/mvme147_defconfig +++ b/arch/m68k/configs/mvme147_defconfig @@ -236,7 +236,6 @@ CONFIG_NETLINK_DIAG=m CONFIG_MPLS=y CONFIG_NET_MPLS_GSO=m CONFIG_MPLS_ROUTING=m -CONFIG_MPLS_IPTUNNEL=m CONFIG_NET_NSH=m CONFIG_AF_KCM=m # CONFIG_WIRELESS is not set @@ -341,8 +340,6 @@ CONFIG_HID=m CONFIG_HIDRAW=y CONFIG_UHID=m # CONFIG_HID_GENERIC is not set -# CONFIG_HID_ITE is not set -# CONFIG_HID_REDRAGON is not set # CONFIG_USB_SUPPORT is not set CONFIG_RTC_CLASS=y # CONFIG_RTC_NVMEM is not set @@ -486,9 +483,13 @@ CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_CMAC=m CONFIG_CRYPTO_MD4=m +CONFIG_CRYPTO_MD5=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m +CONFIG_CRYPTO_SHA512=m +CONFIG_CRYPTO_SHA3=m CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m @@ -497,14 +498,13 @@ CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m CONFIG_CRYPTO_ZSTD=m -CONFIG_CRYPTO_DRBG_HASH=y -CONFIG_CRYPTO_DRBG_CTR=y CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m # CONFIG_CRYPTO_HW is not set CONFIG_PRIME_NUMBERS=m +CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=m CONFIG_CRC_BENCHMARK=y CONFIG_XZ_DEC_TEST=m # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set @@ -521,6 +521,7 @@ CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m CONFIG_TEST_HEXDUMP=m +CONFIG_STRING_KUNIT_BENCH=y CONFIG_TEST_KSTRTOX=m CONFIG_TEST_BITMAP=m CONFIG_TEST_XARRAY=m @@ -529,11 +530,13 @@ CONFIG_TEST_RHASHTABLE=m CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m +CONFIG_TEST_WORKQUEUE=m CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m CONFIG_LINEAR_RANGES_TEST=m +CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig index 8b2e5cf4d2f2..34e48a518ccc 100644 --- a/arch/m68k/configs/mvme16x_defconfig +++ b/arch/m68k/configs/mvme16x_defconfig @@ -237,7 +237,6 @@ CONFIG_NETLINK_DIAG=m CONFIG_MPLS=y CONFIG_NET_MPLS_GSO=m CONFIG_MPLS_ROUTING=m -CONFIG_MPLS_IPTUNNEL=m CONFIG_NET_NSH=m CONFIG_AF_KCM=m # CONFIG_WIRELESS is not set @@ -342,8 +341,6 @@ CONFIG_HID=m CONFIG_HIDRAW=y CONFIG_UHID=m # CONFIG_HID_GENERIC is not set -# CONFIG_HID_ITE is not set -# CONFIG_HID_REDRAGON is not set # CONFIG_USB_SUPPORT is not set CONFIG_RTC_CLASS=y # CONFIG_RTC_NVMEM is not set @@ -487,9 +484,13 @@ CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_CMAC=m CONFIG_CRYPTO_MD4=m +CONFIG_CRYPTO_MD5=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m +CONFIG_CRYPTO_SHA512=m +CONFIG_CRYPTO_SHA3=m CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m @@ -498,14 +499,13 @@ CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m CONFIG_CRYPTO_ZSTD=m -CONFIG_CRYPTO_DRBG_HASH=y -CONFIG_CRYPTO_DRBG_CTR=y CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m # CONFIG_CRYPTO_HW is not set CONFIG_PRIME_NUMBERS=m +CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=m CONFIG_CRC_BENCHMARK=y CONFIG_XZ_DEC_TEST=m # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set @@ -522,6 +522,7 @@ CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m CONFIG_TEST_HEXDUMP=m +CONFIG_STRING_KUNIT_BENCH=y CONFIG_TEST_KSTRTOX=m CONFIG_TEST_BITMAP=m CONFIG_TEST_XARRAY=m @@ -530,11 +531,13 @@ CONFIG_TEST_RHASHTABLE=m CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m +CONFIG_TEST_WORKQUEUE=m CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m CONFIG_LINEAR_RANGES_TEST=m +CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig index d48f3cf5285b..8294c1533bda 100644 --- a/arch/m68k/configs/q40_defconfig +++ b/arch/m68k/configs/q40_defconfig @@ -238,7 +238,6 @@ CONFIG_NETLINK_DIAG=m CONFIG_MPLS=y CONFIG_NET_MPLS_GSO=m CONFIG_MPLS_ROUTING=m -CONFIG_MPLS_IPTUNNEL=m CONFIG_NET_NSH=m CONFIG_AF_KCM=m # CONFIG_WIRELESS is not set @@ -361,8 +360,6 @@ CONFIG_HID=m CONFIG_HIDRAW=y CONFIG_UHID=m # CONFIG_HID_GENERIC is not set -# CONFIG_HID_ITE is not set -# CONFIG_HID_REDRAGON is not set # CONFIG_USB_SUPPORT is not set CONFIG_RTC_CLASS=y # CONFIG_RTC_NVMEM is not set @@ -505,9 +502,13 @@ CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_CMAC=m CONFIG_CRYPTO_MD4=m +CONFIG_CRYPTO_MD5=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m +CONFIG_CRYPTO_SHA512=m +CONFIG_CRYPTO_SHA3=m CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m @@ -516,14 +517,13 @@ CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m CONFIG_CRYPTO_ZSTD=m -CONFIG_CRYPTO_DRBG_HASH=y -CONFIG_CRYPTO_DRBG_CTR=y CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m # CONFIG_CRYPTO_HW is not set CONFIG_PRIME_NUMBERS=m +CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=m CONFIG_CRC_BENCHMARK=y CONFIG_XZ_DEC_TEST=m # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set @@ -540,6 +540,7 @@ CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m CONFIG_TEST_HEXDUMP=m +CONFIG_STRING_KUNIT_BENCH=y CONFIG_TEST_KSTRTOX=m CONFIG_TEST_BITMAP=m CONFIG_TEST_XARRAY=m @@ -548,11 +549,13 @@ CONFIG_TEST_RHASHTABLE=m CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m +CONFIG_TEST_WORKQUEUE=m CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m CONFIG_LINEAR_RANGES_TEST=m +CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig index 0b96428f25d4..53fa21deb7bd 100644 --- a/arch/m68k/configs/sun3_defconfig +++ b/arch/m68k/configs/sun3_defconfig @@ -233,7 +233,6 @@ CONFIG_NETLINK_DIAG=m CONFIG_MPLS=y CONFIG_NET_MPLS_GSO=m CONFIG_MPLS_ROUTING=m -CONFIG_MPLS_IPTUNNEL=m CONFIG_NET_NSH=m CONFIG_AF_KCM=m # CONFIG_WIRELESS is not set @@ -340,8 +339,6 @@ CONFIG_HID=m CONFIG_HIDRAW=y CONFIG_UHID=m # CONFIG_HID_GENERIC is not set -# CONFIG_HID_ITE is not set -# CONFIG_HID_REDRAGON is not set # CONFIG_USB_SUPPORT is not set CONFIG_RTC_CLASS=y # CONFIG_RTC_NVMEM is not set @@ -484,9 +481,13 @@ CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_CMAC=m CONFIG_CRYPTO_MD4=m +CONFIG_CRYPTO_MD5=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m +CONFIG_CRYPTO_SHA512=m +CONFIG_CRYPTO_SHA3=m CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m @@ -495,14 +496,13 @@ CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m CONFIG_CRYPTO_ZSTD=m -CONFIG_CRYPTO_DRBG_HASH=y -CONFIG_CRYPTO_DRBG_CTR=y CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m # CONFIG_CRYPTO_HW is not set CONFIG_PRIME_NUMBERS=m +CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=m CONFIG_CRC_BENCHMARK=y CONFIG_XZ_DEC_TEST=m # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set @@ -518,6 +518,7 @@ CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m CONFIG_TEST_HEXDUMP=m +CONFIG_STRING_KUNIT_BENCH=y CONFIG_TEST_KSTRTOX=m CONFIG_TEST_BITMAP=m CONFIG_TEST_XARRAY=m @@ -526,11 +527,13 @@ CONFIG_TEST_RHASHTABLE=m CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m +CONFIG_TEST_WORKQUEUE=m CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m CONFIG_LINEAR_RANGES_TEST=m +CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig index 6140e18244a1..6c58d82276e8 100644 --- a/arch/m68k/configs/sun3x_defconfig +++ b/arch/m68k/configs/sun3x_defconfig @@ -234,7 +234,6 @@ CONFIG_NETLINK_DIAG=m CONFIG_MPLS=y CONFIG_NET_MPLS_GSO=m CONFIG_MPLS_ROUTING=m -CONFIG_MPLS_IPTUNNEL=m CONFIG_NET_NSH=m CONFIG_AF_KCM=m # CONFIG_WIRELESS is not set @@ -340,8 +339,6 @@ CONFIG_HID=m CONFIG_HIDRAW=y CONFIG_UHID=m # CONFIG_HID_GENERIC is not set -# CONFIG_HID_ITE is not set -# CONFIG_HID_REDRAGON is not set # CONFIG_USB_SUPPORT is not set CONFIG_RTC_CLASS=y # CONFIG_RTC_NVMEM is not set @@ -484,9 +481,13 @@ CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_CMAC=m CONFIG_CRYPTO_MD4=m +CONFIG_CRYPTO_MD5=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m +CONFIG_CRYPTO_SHA512=m +CONFIG_CRYPTO_SHA3=m CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m @@ -495,14 +496,13 @@ CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m CONFIG_CRYPTO_ZSTD=m -CONFIG_CRYPTO_DRBG_HASH=y -CONFIG_CRYPTO_DRBG_CTR=y CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m # CONFIG_CRYPTO_HW is not set CONFIG_PRIME_NUMBERS=m +CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=m CONFIG_CRC_BENCHMARK=y CONFIG_XZ_DEC_TEST=m # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set @@ -519,6 +519,7 @@ CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m CONFIG_TEST_HEXDUMP=m +CONFIG_STRING_KUNIT_BENCH=y CONFIG_TEST_KSTRTOX=m CONFIG_TEST_BITMAP=m CONFIG_TEST_XARRAY=m @@ -527,11 +528,13 @@ CONFIG_TEST_RHASHTABLE=m CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m +CONFIG_TEST_WORKQUEUE=m CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m CONFIG_LINEAR_RANGES_TEST=m +CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m diff --git a/arch/m68k/include/asm/hash.h b/arch/m68k/include/asm/hash.h index 6d0d0c893f16..6dadf4937066 100644 --- a/arch/m68k/include/asm/hash.h +++ b/arch/m68k/include/asm/hash.h @@ -2,6 +2,8 @@ #ifndef _ASM_HASH_H #define _ASM_HASH_H +#include + /* * If CONFIG_M68000=y (original mc68000/010), this file is #included * to work around the lack of a MULU.L instruction. @@ -54,7 +56,7 @@ static inline u32 __attribute_const__ __hash_32(u32 x) : "=&d,d" (a), "=&r,r" (b) : "r,roi?" (x)); /* a+b = x*0x8647 */ - return ((u16)(x*0x61c8) << 16) + a + b; + return (lower_16_bits(x * 0x61c8) << 16) + a + b; } #endif /* _ASM_HASH_H */ diff --git a/arch/m68k/kernel/head.S b/arch/m68k/kernel/head.S index 2e4ef0358887..502eeae49eaf 100644 --- a/arch/m68k/kernel/head.S +++ b/arch/m68k/kernel/head.S @@ -1110,7 +1110,7 @@ L(not147): jbra L(mmu_init_done) L(not16x): -#endif /* CONFIG_MVME162 | CONFIG_MVME167 */ +#endif /* CONFIG_MVME16x */ #ifdef CONFIG_BVME6000 diff --git a/arch/powerpc/lib/code-patching.c b/arch/powerpc/lib/code-patching.c index f84e0337cc02..44ff9f684bef 100644 --- a/arch/powerpc/lib/code-patching.c +++ b/arch/powerpc/lib/code-patching.c @@ -60,9 +60,6 @@ struct patch_context { static DEFINE_PER_CPU(struct patch_context, cpu_patching_context); -static int map_patch_area(void *addr, unsigned long text_poke_addr); -static void unmap_patch_area(unsigned long addr); - static bool mm_patch_enabled(void) { return IS_ENABLED(CONFIG_SMP) && radix_enabled(); @@ -117,11 +114,11 @@ static int text_area_cpu_up(unsigned int cpu) // Map/unmap the area to ensure all page tables are pre-allocated addr = (unsigned long)area->addr; - err = map_patch_area(empty_zero_page, addr); + err = map_kernel_page(addr, __pa_symbol(empty_zero_page), PAGE_KERNEL_RO); if (err) return err; - unmap_patch_area(addr); + unmap_kernel_page(addr); this_cpu_write(cpu_patching_context.area, area); this_cpu_write(cpu_patching_context.addr, addr); @@ -233,51 +230,6 @@ static unsigned long get_patch_pfn(void *addr) return __pa_symbol(addr) >> PAGE_SHIFT; } -/* - * This can be called for kernel text or a module. - */ -static int map_patch_area(void *addr, unsigned long text_poke_addr) -{ - unsigned long pfn = get_patch_pfn(addr); - - return map_kernel_page(text_poke_addr, (pfn << PAGE_SHIFT), PAGE_KERNEL); -} - -static void unmap_patch_area(unsigned long addr) -{ - pte_t *ptep; - pmd_t *pmdp; - pud_t *pudp; - p4d_t *p4dp; - pgd_t *pgdp; - - pgdp = pgd_offset_k(addr); - if (WARN_ON(pgd_none(*pgdp))) - return; - - p4dp = p4d_offset(pgdp, addr); - if (WARN_ON(p4d_none(*p4dp))) - return; - - pudp = pud_offset(p4dp, addr); - if (WARN_ON(pud_none(*pudp))) - return; - - pmdp = pmd_offset(pudp, addr); - if (WARN_ON(pmd_none(*pmdp))) - return; - - ptep = pte_offset_kernel(pmdp, addr); - if (WARN_ON(pte_none(*ptep))) - return; - - /* - * In hash, pte_clear flushes the tlb, in radix, we have to - */ - pte_clear(&init_mm, addr, ptep); - flush_tlb_kernel_range(addr, addr + PAGE_SIZE); -} - static int __do_patch_mem_mm(void *addr, unsigned long val, bool is_dword) { int err; diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 81185b19e106..84404e6778d5 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -163,7 +163,6 @@ config S390 select ARCH_WANTS_THP_SWAP select BUILDTIME_TABLE_SORT select CLONE_BACKWARDS2 - select DCACHE_WORD_ACCESS if !KMSAN select DYNAMIC_FTRACE if FUNCTION_TRACER select FUNCTION_ALIGNMENT_8B if CC_IS_GCC select FUNCTION_ALIGNMENT_16B if !CC_IS_GCC @@ -175,6 +174,7 @@ config S390 select GENERIC_GETTIMEOFDAY select GENERIC_SMP_IDLE_THREAD select GENERIC_IOREMAP if PCI + select GLOB select HAVE_ALIGNED_STRUCT_PAGE select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_GET_SECUREBOOT @@ -244,6 +244,7 @@ config S390 select HAVE_RELIABLE_STACKTRACE select HAVE_RETHOOK select HAVE_RSEQ + select HAVE_RUST if !EXPOLINE select HAVE_SAMPLE_FTRACE_DIRECT select HAVE_SAMPLE_FTRACE_DIRECT_MULTI select HAVE_SETUP_PER_CPU_AREA @@ -995,9 +996,8 @@ config S390_MODULES_SANITY_TEST_HELPERS menu "Selftests" config S390_UNWIND_SELFTEST - def_tristate n + def_tristate KUNIT_ALL_TESTS depends on KUNIT - default KUNIT_ALL_TESTS prompt "Test unwind functions" help This option enables s390 specific stack unwinder testing kernel @@ -1007,7 +1007,7 @@ config S390_UNWIND_SELFTEST Say N if you are unsure. config S390_KPROBES_SANITY_TEST - def_tristate n + def_tristate KUNIT_ALL_TESTS prompt "Enable s390 specific kprobes tests" depends on KPROBES depends on KUNIT @@ -1019,9 +1019,8 @@ config S390_KPROBES_SANITY_TEST Say N if you are unsure. config S390_MODULES_SANITY_TEST - def_tristate n + def_tristate KUNIT_ALL_TESTS depends on KUNIT && m - default KUNIT_ALL_TESTS prompt "Enable s390 specific modules tests" select S390_MODULES_SANITY_TEST_HELPERS help diff --git a/arch/s390/Makefile b/arch/s390/Makefile index 297976b41088..8b712cd85fcd 100644 --- a/arch/s390/Makefile +++ b/arch/s390/Makefile @@ -35,25 +35,31 @@ KBUILD_CFLAGS_DECOMPRESSOR += $(if $(CONFIG_DEBUG_INFO),-g) KBUILD_CFLAGS_DECOMPRESSOR += $(if $(CONFIG_DEBUG_INFO_DWARF4), $(call cc-option, -gdwarf-4,)) KBUILD_CFLAGS_DECOMPRESSOR += $(if $(CONFIG_CC_NO_ARRAY_BOUNDS),-Wno-array-bounds) +KBUILD_RUSTFLAGS += --target=s390x-unknown-none-softfloat -Zpacked-stack -Ctarget-feature=+backchain + UTS_MACHINE := s390x STACK_SIZE := $(if $(CONFIG_KASAN),65536,$(if $(CONFIG_KMSAN),65536,16384)) CHECKFLAGS += -D__s390__ -D__s390x__ export LD_BFD -mflags-$(CONFIG_MARCH_Z10) := -march=z10 -mflags-$(CONFIG_MARCH_Z196) := -march=z196 -mflags-$(CONFIG_MARCH_ZEC12) := -march=zEC12 -mflags-$(CONFIG_MARCH_Z13) := -march=z13 -mflags-$(CONFIG_MARCH_Z14) := -march=z14 -mflags-$(CONFIG_MARCH_Z15) := -march=z15 -mflags-$(CONFIG_MARCH_Z16) := -march=z16 -mflags-$(CONFIG_MARCH_Z17) := -march=z17 +march-name-$(CONFIG_MARCH_Z10) := z10 +march-name-$(CONFIG_MARCH_Z196) := z196 +march-name-$(CONFIG_MARCH_ZEC12) := zEC12 +march-name-$(CONFIG_MARCH_Z13) := z13 +march-name-$(CONFIG_MARCH_Z14) := z14 +march-name-$(CONFIG_MARCH_Z15) := z15 +march-name-$(CONFIG_MARCH_Z16) := z16 +march-name-$(CONFIG_MARCH_Z17) := z17 -export CC_FLAGS_MARCH := $(mflags-y) +mflags := -march=$(march-name-y) -aflags-y += $(mflags-y) -cflags-y += $(mflags-y) +export CC_FLAGS_MARCH := $(mflags) + +aflags-y += $(mflags) +cflags-y += $(mflags) + +KBUILD_RUSTFLAGS += -Ctarget-cpu=$(march-name-y) cflags-$(CONFIG_MARCH_Z10_TUNE) += -mtune=z10 cflags-$(CONFIG_MARCH_Z196_TUNE) += -mtune=z196 diff --git a/arch/s390/appldata/appldata_base.c b/arch/s390/appldata/appldata_base.c index edbedbb2a773..9cba4633c3f3 100644 --- a/arch/s390/appldata/appldata_base.c +++ b/arch/s390/appldata/appldata_base.c @@ -51,7 +51,6 @@ static int appldata_timer_handler(const struct ctl_table *ctl, int write, static int appldata_interval_handler(const struct ctl_table *ctl, int write, void *buffer, size_t *lenp, loff_t *ppos); -static struct ctl_table_header *appldata_sysctl_header; static const struct ctl_table appldata_table[] = { { .procname = "timer", @@ -406,7 +405,7 @@ static int __init appldata_init(void) appldata_wq = alloc_ordered_workqueue("appldata", 0); if (!appldata_wq) return -ENOMEM; - appldata_sysctl_header = register_sysctl(appldata_proc_name, appldata_table); + register_sysctl(appldata_proc_name, appldata_table); return 0; } diff --git a/arch/s390/boot/Makefile b/arch/s390/boot/Makefile index a1e719a79d38..10b75e053a6f 100644 --- a/arch/s390/boot/Makefile +++ b/arch/s390/boot/Makefile @@ -25,8 +25,13 @@ KBUILD_CFLAGS += $(call cc-option, -Wno-default-const-init-unsafe) CFLAGS_sclp_early_core.o += -I$(srctree)/drivers/s390/char +# string.o implements standard library functions like memset/memcpy etc. +# Use -ffreestanding to ensure that the compiler does not try to "optimize" +# them into calls to themselves. +CFLAGS_string.o = -ffreestanding + obj-y := head.o als.o startup.o physmem_info.o ipl_parm.o ipl_report.o vmem.o -obj-y += string.o ebcdic.o sclp_early_core.o mem.o ipl_vmparm.o cmdline.o +obj-y += string.o ebcdic.o sclp_early_core.o ipl_vmparm.o cmdline.o obj-y += version.o pgm_check.o ctype.o ipl_data.o relocs.o alternative.o obj-y += uv.o printk.o trampoline.o obj-$(CONFIG_RANDOMIZE_BASE) += kaslr.o diff --git a/arch/s390/boot/mem.S b/arch/s390/boot/mem.S deleted file mode 100644 index b33463633f03..000000000000 --- a/arch/s390/boot/mem.S +++ /dev/null @@ -1,2 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#include "../lib/mem.S" diff --git a/arch/s390/boot/string.c b/arch/s390/boot/string.c index bd68161434a6..e4ad196cb720 100644 --- a/arch/s390/boot/string.c +++ b/arch/s390/boot/string.c @@ -43,11 +43,7 @@ ssize_t sized_strscpy(char *dst, const char *src, size_t count) void *memset64(uint64_t *s, uint64_t v, size_t count) { - uint64_t *xs = s; - - while (count--) - *xs++ = v; - return s; + return __memset64(s, v, count * sizeof(v)); } char *skip_spaces(const char *str) diff --git a/arch/s390/boot/vmlinux.lds.S b/arch/s390/boot/vmlinux.lds.S index 070bc18babd0..d44964592541 100644 --- a/arch/s390/boot/vmlinux.lds.S +++ b/arch/s390/boot/vmlinux.lds.S @@ -31,6 +31,7 @@ SECTIONS _text = .; /* Text */ *(.text) *(.text.*) + *(.noinstr.text) INIT_TEXT _etext = . ; } diff --git a/arch/s390/include/asm/asm-extable.h b/arch/s390/include/asm/asm-extable.h index d23ea0c94e4e..99748c20e767 100644 --- a/arch/s390/include/asm/asm-extable.h +++ b/arch/s390/include/asm/asm-extable.h @@ -12,7 +12,6 @@ #define EX_TYPE_UA_FAULT 3 #define EX_TYPE_UA_LOAD_REG 5 #define EX_TYPE_UA_LOAD_REGPAIR 6 -#define EX_TYPE_ZEROPAD 7 #define EX_TYPE_FPC 8 #define EX_TYPE_UA_MVCOS_TO 9 #define EX_TYPE_UA_MVCOS_FROM 10 @@ -80,9 +79,6 @@ #define EX_TABLE_UA_LOAD_REGPAIR(_fault, _target, _regerr, _regzero) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REGPAIR, _regerr, _regzero, 0) -#define EX_TABLE_ZEROPAD(_fault, _target, _regdata, _regaddr) \ - __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_ZEROPAD, _regdata, _regaddr, 0) - #define EX_TABLE_FPC(_fault, _target) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_FPC, __stringify(%%r0), __stringify(%%r0), 0) diff --git a/arch/s390/include/asm/asm-prototypes.h b/arch/s390/include/asm/asm-prototypes.h index 7bd1801cf241..d4da4436d02b 100644 --- a/arch/s390/include/asm/asm-prototypes.h +++ b/arch/s390/include/asm/asm-prototypes.h @@ -8,8 +8,4 @@ #include #include -__int128_t __ashlti3(__int128_t a, int b); -__int128_t __ashrti3(__int128_t a, int b); -__int128_t __lshrti3(__int128_t a, int b); - #endif /* _ASM_S390_PROTOTYPES_H */ diff --git a/arch/s390/include/asm/barrier.h b/arch/s390/include/asm/barrier.h index dad02f5b3c8d..d1e4201f3850 100644 --- a/arch/s390/include/asm/barrier.h +++ b/arch/s390/include/asm/barrier.h @@ -8,6 +8,7 @@ #ifndef __ASM_BARRIER_H #define __ASM_BARRIER_H +#include #include /* @@ -16,16 +17,11 @@ * to devices. */ -#ifdef MARCH_HAS_Z196_FEATURES -/* Fast-BCR without checkpoint synchronization */ -#define __ASM_BCR_SERIALIZE "bcr 14,0" -#else -#define __ASM_BCR_SERIALIZE "bcr 15,0" -#endif - static __always_inline void bcr_serialize(void) { - asm volatile(__ASM_BCR_SERIALIZE : : : "memory"); + asm_inline volatile( + ALTERNATIVE("bcr 15,0", "bcr 14,0", ALT_FACILITY(45)) + : : : "memory"); } #define __mb() bcr_serialize() diff --git a/arch/s390/include/asm/bug.h b/arch/s390/include/asm/bug.h index 50a270edb020..2010342c97b1 100644 --- a/arch/s390/include/asm/bug.h +++ b/arch/s390/include/asm/bug.h @@ -4,6 +4,7 @@ #include #include +#include #define MONCODE_BUG _AC(0, U) #define MONCODE_BUG_ARG _AC(1, U) @@ -121,6 +122,17 @@ do { \ #define HAVE_ARCH_BUG_FORMAT #define HAVE_ARCH_BUG_FORMAT_ARGS +#define ARCH_WARN_ASM(file, line, flags, size) \ + ".section .rodata.str,\"aMS\",@progbits,1\n" \ + "9:\n" \ + ".asciz \"\"\n" /* Empty string for compatibility */ \ + ".previous\n" \ + "0:\n" \ + __stringify(mc 0(%r0),0) "\n" \ + __BUG_ENTRY("9b", file, line, flags, size) + +#define ARCH_WARN_REACHABLE + #endif /* CONFIG_BUG && CONFIG_CC_HAS_ASM_IMMEDIATE_STRINGS */ #endif /* __ASSEMBLER__ */ diff --git a/arch/s390/include/asm/cmpxchg.h b/arch/s390/include/asm/cmpxchg.h index 008357996262..e6ac55cf3c17 100644 --- a/arch/s390/include/asm/cmpxchg.h +++ b/arch/s390/include/asm/cmpxchg.h @@ -35,7 +35,7 @@ static __always_inline u64 __csg_asm(u64 ptr, u64 old, u64 new) return old; } -static inline u8 __arch_cmpxchg1(u64 ptr, u8 old, u8 new) +static __no_sanitize_or_inline u8 __arch_cmpxchg1(u64 ptr, u8 old, u8 new) { union { u8 b[4]; @@ -58,7 +58,7 @@ static inline u8 __arch_cmpxchg1(u64 ptr, u8 old, u8 new) return old; } -static inline u16 __arch_cmpxchg2(u64 ptr, u16 old, u16 new) +static __no_sanitize_or_inline u16 __arch_cmpxchg2(u64 ptr, u16 old, u16 new) { union { u16 b[2]; @@ -173,7 +173,7 @@ static __always_inline u64 __arch_cmpxchg(u64 ptr, u64 old, u64 new, int size) void __xchg_called_with_bad_pointer(void); -static inline u8 __arch_xchg1(u64 ptr, u8 x) +static __no_sanitize_or_inline u8 __arch_xchg1(u64 ptr, u8 x) { int shift = (3 ^ (ptr & 3)) << 3; u32 mask, old, new; @@ -188,7 +188,7 @@ static inline u8 __arch_xchg1(u64 ptr, u8 x) return old >> shift; } -static inline u16 __arch_xchg2(u64 ptr, u16 x) +static __no_sanitize_or_inline u16 __arch_xchg2(u64 ptr, u16 x) { int shift = (2 ^ (ptr & 2)) << 3; u32 mask, old, new; diff --git a/arch/s390/include/asm/debug.h b/arch/s390/include/asm/debug.h index c5440b3ee53d..39d484c59774 100644 --- a/arch/s390/include/asm/debug.h +++ b/arch/s390/include/asm/debug.h @@ -490,6 +490,7 @@ arch_initcall(VNAME(var, reg)) __DEFINE_STATIC_AREA(var); \ static debug_info_t __refdata var = \ __DEBUG_INFO_INIT(var, (name), (buf_size)); \ +static debug_info_t __used __section(".s390dbf_info") *VNAME(var, info) = &var; \ __REGISTER_STATIC_DEBUG_INFO(var, name, pages, nr_areas, view) void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas); diff --git a/arch/s390/include/asm/entry-percpu.h b/arch/s390/include/asm/entry-percpu.h new file mode 100644 index 000000000000..4ef47265cfce --- /dev/null +++ b/arch/s390/include/asm/entry-percpu.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ARCH_S390_ENTRY_PERCPU_H +#define ARCH_S390_ENTRY_PERCPU_H + +#include +#include +#include +#include +#include + +static __always_inline void percpu_entry(struct pt_regs *regs) +{ + struct lowcore *lc = get_lowcore(); + + if (user_mode(regs)) + return; + regs->cpu = lc->cpu_nr; + regs->percpu_register = lc->percpu_register; + lc->percpu_register = 0; +} + +static __always_inline bool percpu_code_check(struct pt_regs *regs) +{ + unsigned int insn, disp; + struct kprobe *p; + + if (likely(user_mode(regs) || !regs->percpu_register)) + return false; + /* + * Within a percpu code section - check if the percpu base register + * needs to be updated. This is the case if the PSW does not point to + * the ADD instruction within the section. + * - AG %rx,percpu_offset_in_lowcore(%r0,%r0) + * which adds the percpu offset to the percpu base register. + */ + lockdep_assert_preemption_disabled(); +again: + insn = READ_ONCE(*(u16 *)psw_bits(regs->psw).ia); + if (unlikely(insn == BREAKPOINT_INSTRUCTION)) { + p = get_kprobe((void *)psw_bits(regs->psw).ia); + /* + * If the kprobe is concurrently removed on a different CPU + * it might not be found anymore. However text must have + * been restored - try again. + */ + if (!p) + goto again; + insn = p->opcode; + } + if ((insn & 0xff0f) != 0xe300) + return true; + disp = offsetof(struct lowcore, percpu_offset); + if (machine_has_relocated_lowcore()) + disp += LOWCORE_ALT_ADDRESS; + insn = (disp & 0xff000) >> 4 | (disp & 0x00fff) << 16 | 0x8; + if (*(u32 *)(psw_bits(regs->psw).ia + 2) != insn) + return true; + return false; +} + +static __always_inline void percpu_exit(struct pt_regs *regs, bool needs_fixup) +{ + struct lowcore *lc = get_lowcore(); + unsigned char reg; + + if (user_mode(regs)) + return; + reg = regs->percpu_register; + lc->percpu_register = reg; + if (likely(!needs_fixup)) + return; + /* Check if process has been migrated to a different CPU. */ + if (regs->cpu == lc->cpu_nr) + return; + /* Fixup percpu base register */ + regs->gprs[reg] -= __per_cpu_offset[regs->cpu]; + regs->gprs[reg] += lc->percpu_offset; +} + +#endif diff --git a/arch/s390/include/asm/fpu-insn-asm.h b/arch/s390/include/asm/fpu-insn-asm.h index cc0468fdf2d0..4a1bfc0f578e 100644 --- a/arch/s390/include/asm/fpu-insn-asm.h +++ b/arch/s390/include/asm/fpu-insn-asm.h @@ -16,182 +16,10 @@ #error only can be included directly #endif +#include + #ifdef __ASSEMBLER__ -/* Macros to generate vector instruction byte code */ - -/* GR_NUM - Retrieve general-purpose register number - * - * @opd: Operand to store register number - * @r64: String designation register in the format "%rN" - */ -.macro GR_NUM opd gr - \opd = 255 - .ifc \gr,%r0 - \opd = 0 - .endif - .ifc \gr,%r1 - \opd = 1 - .endif - .ifc \gr,%r2 - \opd = 2 - .endif - .ifc \gr,%r3 - \opd = 3 - .endif - .ifc \gr,%r4 - \opd = 4 - .endif - .ifc \gr,%r5 - \opd = 5 - .endif - .ifc \gr,%r6 - \opd = 6 - .endif - .ifc \gr,%r7 - \opd = 7 - .endif - .ifc \gr,%r8 - \opd = 8 - .endif - .ifc \gr,%r9 - \opd = 9 - .endif - .ifc \gr,%r10 - \opd = 10 - .endif - .ifc \gr,%r11 - \opd = 11 - .endif - .ifc \gr,%r12 - \opd = 12 - .endif - .ifc \gr,%r13 - \opd = 13 - .endif - .ifc \gr,%r14 - \opd = 14 - .endif - .ifc \gr,%r15 - \opd = 15 - .endif - .if \opd == 255 - \opd = \gr - .endif -.endm - -/* VX_NUM - Retrieve vector register number - * - * @opd: Operand to store register number - * @vxr: String designation register in the format "%vN" - * - * The vector register number is used for as input number to the - * instruction and, as well as, to compute the RXB field of the - * instruction. - */ -.macro VX_NUM opd vxr - \opd = 255 - .ifc \vxr,%v0 - \opd = 0 - .endif - .ifc \vxr,%v1 - \opd = 1 - .endif - .ifc \vxr,%v2 - \opd = 2 - .endif - .ifc \vxr,%v3 - \opd = 3 - .endif - .ifc \vxr,%v4 - \opd = 4 - .endif - .ifc \vxr,%v5 - \opd = 5 - .endif - .ifc \vxr,%v6 - \opd = 6 - .endif - .ifc \vxr,%v7 - \opd = 7 - .endif - .ifc \vxr,%v8 - \opd = 8 - .endif - .ifc \vxr,%v9 - \opd = 9 - .endif - .ifc \vxr,%v10 - \opd = 10 - .endif - .ifc \vxr,%v11 - \opd = 11 - .endif - .ifc \vxr,%v12 - \opd = 12 - .endif - .ifc \vxr,%v13 - \opd = 13 - .endif - .ifc \vxr,%v14 - \opd = 14 - .endif - .ifc \vxr,%v15 - \opd = 15 - .endif - .ifc \vxr,%v16 - \opd = 16 - .endif - .ifc \vxr,%v17 - \opd = 17 - .endif - .ifc \vxr,%v18 - \opd = 18 - .endif - .ifc \vxr,%v19 - \opd = 19 - .endif - .ifc \vxr,%v20 - \opd = 20 - .endif - .ifc \vxr,%v21 - \opd = 21 - .endif - .ifc \vxr,%v22 - \opd = 22 - .endif - .ifc \vxr,%v23 - \opd = 23 - .endif - .ifc \vxr,%v24 - \opd = 24 - .endif - .ifc \vxr,%v25 - \opd = 25 - .endif - .ifc \vxr,%v26 - \opd = 26 - .endif - .ifc \vxr,%v27 - \opd = 27 - .endif - .ifc \vxr,%v28 - \opd = 28 - .endif - .ifc \vxr,%v29 - \opd = 29 - .endif - .ifc \vxr,%v30 - \opd = 30 - .endif - .ifc \vxr,%v31 - \opd = 31 - .endif - .if \opd == 255 - \opd = \vxr - .endif -.endm - /* RXB - Compute most significant bit used vector registers * * @rxb: Operand to store computed RXB value diff --git a/arch/s390/include/asm/fpu-insn.h b/arch/s390/include/asm/fpu-insn.h index 96727f3bd0dc..ae8b7033cfd2 100644 --- a/arch/s390/include/asm/fpu-insn.h +++ b/arch/s390/include/asm/fpu-insn.h @@ -15,6 +15,7 @@ #include #include +asm(".include \"asm/insn-common-asm.h\"\n"); asm(".include \"asm/fpu-insn-asm.h\"\n"); /* diff --git a/arch/s390/include/asm/insn-common-asm.h b/arch/s390/include/asm/insn-common-asm.h new file mode 100644 index 000000000000..fd9b3cacb7c5 --- /dev/null +++ b/arch/s390/include/asm/insn-common-asm.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Assembler helper macros to generate .byte/.word code for instructions + * that are unknown to older binutils versions. + */ + +#ifndef __ASM_S390_INSN_COMMON_ASM_H +#define __ASM_S390_INSN_COMMON_ASM_H + +#ifdef __ASSEMBLER__ + +/* + * GR_NUM - Retrieve general-purpose register number + * + * @opd: Operand to store register number + * @gr: String designation register in the format "%rN" + */ +.macro GR_NUM opd gr + \opd = 255 + .irp rs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 + .ifc \gr,%r\rs + \opd = \rs + .endif + .endr + .if \opd == 255 + \opd = \gr + .endif +.endm + +/* + * VX_NUM - Retrieve vector register number + * + * @opd: Operand to store register number + * @vxr: String designation register in the format "%vN" + * + * The vector register number is used for as input number to the + * instruction and, as well as, to compute the RXB field of the + * instruction. + */ +.macro VX_NUM opd vxr + \opd = 255 + .irp vs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 + .ifc \vxr,%v\vs + \opd = \vs + .endif + .endr + .if \opd == 255 + \opd = \vxr + .endif +.endm + +#endif /* __ASSEMBLER__ */ +#endif /* __ASM_S390_INSN_COMMON_ASM_H */ diff --git a/arch/s390/include/asm/jump_label.h b/arch/s390/include/asm/jump_label.h index d9cbc18f6b2e..0e28c90dc242 100644 --- a/arch/s390/include/asm/jump_label.h +++ b/arch/s390/include/asm/jump_label.h @@ -19,19 +19,29 @@ #define JUMP_LABEL_STATIC_KEY_CONSTRAINT "jdd" #endif +#define ARCH_JUMP_TABLE_ENTRY(key, label, local_label) \ + ".pushsection __jump_table,\"aw\"\n" \ + ".balign 8\n" \ + ".long " local_label "-.," label "-.\n" \ + ".quad " key "-.\n" \ + ".popsection\n" + /* * We use a brcl 0, instruction for jump labels so it * can be easily distinguished from a hotpatch generated instruction. */ +#define ARCH_STATIC_BRANCH_ASM(key, label) \ + "0: brcl 0," label "\n" \ + ARCH_JUMP_TABLE_ENTRY(key, label, "0b") + +#define ARCH_STATIC_BRANCH_JUMP_ASM(key, label) \ + "0: brcl 15," label "\n" \ + ARCH_JUMP_TABLE_ENTRY(key, label, "0b") + static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { - asm goto("0: brcl 0,%l[label]\n" - ".pushsection __jump_table,\"aw\"\n" - ".balign 8\n" - ".long 0b-.,%l[label]-.\n" - ".quad %0+%1-.\n" - ".popsection\n" - : : JUMP_LABEL_STATIC_KEY_CONSTRAINT (key), "i" (branch) : : label); + asm goto(ARCH_STATIC_BRANCH_ASM("%0+%1", "%l[label]") + : : JUMP_LABEL_STATIC_KEY_CONSTRAINT (key), "i" (branch) : : label); return false; label: return true; @@ -39,13 +49,8 @@ label: static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) { - asm goto("0: brcl 15,%l[label]\n" - ".pushsection __jump_table,\"aw\"\n" - ".balign 8\n" - ".long 0b-.,%l[label]-.\n" - ".quad %0+%1-.\n" - ".popsection\n" - : : JUMP_LABEL_STATIC_KEY_CONSTRAINT (key), "i" (branch) : : label); + asm goto(ARCH_STATIC_BRANCH_JUMP_ASM("%0+%1", "%l[label]") + : : JUMP_LABEL_STATIC_KEY_CONSTRAINT (key), "i" (branch) : : label); return false; label: return true; diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 50ffe75adeb4..cd1ddfdb5d35 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -165,7 +165,8 @@ struct lowcore { __u32 spinlock_index; /* 0x03b0 */ __u8 pad_0x03b4[0x03b8-0x03b4]; /* 0x03b4 */ __u64 percpu_offset; /* 0x03b8 */ - __u8 pad_0x03c0[0x0400-0x03c0]; /* 0x03c0 */ + __u8 percpu_register; /* 0x03c0 */ + __u8 pad_0x03c1[0x0400-0x03c1]; /* 0x03c1 */ __u32 return_lpswe; /* 0x0400 */ __u32 return_mcck_lpswe; /* 0x0404 */ diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h index b18a96f3a334..1d955dd0defa 100644 --- a/arch/s390/include/asm/percpu.h +++ b/arch/s390/include/asm/percpu.h @@ -60,6 +60,68 @@ #define this_cpu_or_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) #define this_cpu_or_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) +/* + * Macros to be used for percpu code section based on atomic instructions. + * + * Avoid the need to use preempt_disable() / preempt_disable() pairs and the + * conditional preempt_schedule_notrace() function calls which come with + * this. The idea is that this_cpu operations based on atomic instructions are + * guarded with mviy instructions: + * + * - The first mviy instruction writes the register number, which contains the + * percpu address variable to lowcore. This also indicates that a percpu + * code section is executed. + * + * - The first mviy instruction following the mviy instruction must be the ag + * instruction which adds the percpu offset to the percpu address register. + * + * - Afterwards the atomic percpu operation follows. + * + * - Then a second mviy instruction writes a zero to lowcore, which indicates + * the end of the percpu code section. + * + * - In case of an interrupt/exception/nmi the register number which was + * written to lowcore is copied to the exception frame (pt_regs), and a zero + * is written to lowcore. + * + * - On return to the previous context it is checked if a percpu code section + * was executed (saved register number not zero), and if the process was + * migrated to a different cpu. If the percpu offset was already added to + * the percpu address register (instruction address does _not_ point to the + * ag instruction) the content of the percpu address register is adjusted so + * it points to percpu variable of the new cpu. + * + * Inline assemblies making use of this typically have a code sequence like: + * + * MVIY_PERCPU(...) <- start of percpu code section + * AG_ALT(...) <- add percpu offset; must be the second instruction + * atomic_op <- atomic op + * MVIY_ALT(...) <- end of percpu code section + */ + +#define MVIY_PERCPU(disp, dispalt, reg) \ + ".macro GEN_MVIY disp reg\n" \ + ".irp rs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15\n" \ + " .ifc \\reg,%%r\\rs\n" \ + " mviy \\disp(%%r0),\\rs\n" \ + " .endif\n" \ + ".endr\n" \ + ".endm\n" \ + ALTERNATIVE("GEN_MVIY " __stringify(disp) " " __stringify(reg) "\n", \ + "GEN_MVIY " __stringify(dispalt) " " __stringify(reg) "\n", \ + ALT_FEATURE(MFEATURE_LOWCORE)) \ + ".purgem GEN_MVIY\n" + +#define MVIY_ALT(disp, dispalt) \ + ALTERNATIVE(" mviy " disp "(%%r0),0\n", \ + " mviy " dispalt "(%%r0),0\n", \ + ALT_FEATURE(MFEATURE_LOWCORE)) + +#define AG_ALT(disp, dispalt, reg) \ + ALTERNATIVE(" ag " reg ", " disp "(%%r0)\n", \ + " ag " reg ", " dispalt "(%%r0)\n", \ + ALT_FEATURE(MFEATURE_LOWCORE)) + #ifndef MARCH_HAS_Z196_FEATURES #define this_cpu_add_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) @@ -73,46 +135,79 @@ #else /* MARCH_HAS_Z196_FEATURES */ -#define arch_this_cpu_add(pcp, val, op1, op2, szcast) \ -{ \ - typedef typeof(pcp) pcp_op_T__; \ - pcp_op_T__ val__ = (val); \ - pcp_op_T__ old__, *ptr__; \ - preempt_disable_notrace(); \ - ptr__ = raw_cpu_ptr(&(pcp)); \ - if (__builtin_constant_p(val__) && \ - ((szcast)val__ > -129) && ((szcast)val__ < 128)) { \ - asm volatile( \ - op2 " %[ptr__],%[val__]" \ - : [ptr__] "+Q" (*ptr__) \ - : [val__] "i" ((szcast)val__) \ - : "cc"); \ - } else { \ - asm volatile( \ - op1 " %[old__],%[val__],%[ptr__]" \ - : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ - : [val__] "d" (val__) \ - : "cc"); \ - } \ - preempt_enable_notrace(); \ -} +#define arch_this_cpu_add(pcp, val, op1, op2, szcast) \ +do { \ + unsigned long lc_pcpr, lc_pcpo; \ + typedef typeof(pcp) pcp_op_T__; \ + pcp_op_T__ val__ = (val); \ + pcp_op_T__ old__, *ptr__; \ + \ + lc_pcpr = offsetof(struct lowcore, percpu_register); \ + lc_pcpo = offsetof(struct lowcore, percpu_offset); \ + ptr__ = PERCPU_PTR(&(pcp)); \ + if (__builtin_constant_p(val__) && \ + ((szcast)val__ > -129) && ((szcast)val__ < 128)) { \ + asm volatile( \ + MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ + AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + op2 " 0(%[ptr__]),%[val__]\n" \ + MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + : [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [val__] "i" ((szcast)val__), \ + [disppcpr] "i" (lc_pcpr), \ + [disppcpo] "i" (lc_pcpo), \ + [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ + [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + "m" (((struct lowcore *)0)->percpu_offset) \ + : "cc"); \ + } else { \ + asm volatile( \ + MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ + AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + op1 " %[old__],%[val__],0(%[ptr__])\n" \ + MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + : [old__] "=&d" (old__), \ + [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [val__] "d" (val__), \ + [disppcpr] "i" (lc_pcpr), \ + [disppcpo] "i" (lc_pcpo), \ + [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ + [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + "m" (((struct lowcore *)0)->percpu_offset) \ + : "cc"); \ + } \ +} while (0) #define this_cpu_add_4(pcp, val) arch_this_cpu_add(pcp, val, "laa", "asi", int) #define this_cpu_add_8(pcp, val) arch_this_cpu_add(pcp, val, "laag", "agsi", long) #define arch_this_cpu_add_return(pcp, val, op) \ ({ \ + unsigned long lc_pcpr, lc_pcpo; \ typedef typeof(pcp) pcp_op_T__; \ pcp_op_T__ val__ = (val); \ pcp_op_T__ old__, *ptr__; \ - preempt_disable_notrace(); \ - ptr__ = raw_cpu_ptr(&(pcp)); \ - asm volatile( \ - op " %[old__],%[val__],%[ptr__]" \ - : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ - : [val__] "d" (val__) \ + \ + lc_pcpr = offsetof(struct lowcore, percpu_register); \ + lc_pcpo = offsetof(struct lowcore, percpu_offset); \ + ptr__ = PERCPU_PTR(&(pcp)); \ + asm_inline volatile( \ + MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ + AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + op " %[old__],%[val__],0(%[ptr__])\n" \ + MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + : [old__] "=&d" (old__), \ + [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [val__] "d" (val__), \ + [disppcpr] "i" (lc_pcpr), \ + [disppcpo] "i" (lc_pcpo), \ + [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ + [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ - preempt_enable_notrace(); \ old__ + val__; \ }) @@ -120,19 +215,31 @@ #define this_cpu_add_return_8(pcp, val) arch_this_cpu_add_return(pcp, val, "laag") #define arch_this_cpu_to_op(pcp, val, op) \ -{ \ +do { \ + unsigned long lc_pcpr, lc_pcpo; \ typedef typeof(pcp) pcp_op_T__; \ pcp_op_T__ val__ = (val); \ pcp_op_T__ old__, *ptr__; \ - preempt_disable_notrace(); \ - ptr__ = raw_cpu_ptr(&(pcp)); \ - asm volatile( \ - op " %[old__],%[val__],%[ptr__]" \ - : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ - : [val__] "d" (val__) \ + \ + lc_pcpr = offsetof(struct lowcore, percpu_register); \ + lc_pcpo = offsetof(struct lowcore, percpu_offset); \ + ptr__ = PERCPU_PTR(&(pcp)); \ + asm_inline volatile( \ + MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ + AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + op " %[old__],%[val__],0(%[ptr__])\n" \ + MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + : [old__] "=&d" (old__), \ + [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [val__] "d" (val__), \ + [disppcpr] "i" (lc_pcpr), \ + [disppcpo] "i" (lc_pcpo), \ + [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ + [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ - preempt_enable_notrace(); \ -} +} while (0) #define this_cpu_and_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lan") #define this_cpu_and_8(pcp, val) arch_this_cpu_to_op(pcp, val, "lang") @@ -141,6 +248,67 @@ #endif /* MARCH_HAS_Z196_FEATURES */ +#define arch_this_cpu_read(pcp, op) \ +({ \ + unsigned long lc_pcpr, lc_pcpo, res__; \ + typedef typeof(pcp) pcp_op_T__; \ + pcp_op_T__ *ptr__; \ + \ + lc_pcpr = offsetof(struct lowcore, percpu_register); \ + lc_pcpo = offsetof(struct lowcore, percpu_offset); \ + ptr__ = PERCPU_PTR(&(pcp)); \ + asm_inline volatile( \ + MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ + AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + op " %[res__],0(%[ptr__])\n" \ + MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + : [res__] "=&d" (res__), [ptr__] "+&a" (ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [disppcpr] "i" (lc_pcpr), \ + [disppcpo] "i" (lc_pcpo), \ + [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ + [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + "m" (*ptr__), \ + "m" (((struct lowcore *)0)->percpu_offset) \ + : "cc"); \ + (pcp_op_T__)res__; \ +}) + +#define this_cpu_read_1(pcp) arch_this_cpu_read(pcp, "llgc") +#define this_cpu_read_2(pcp) arch_this_cpu_read(pcp, "llgh") +#define this_cpu_read_4(pcp) arch_this_cpu_read(pcp, "llgf") +#define this_cpu_read_8(pcp) arch_this_cpu_read(pcp, "lg") + +#define arch_this_cpu_write(pcp, val, op) \ +do { \ + unsigned long lc_pcpr, lc_pcpo; \ + typedef typeof(pcp) pcp_op_T__; \ + pcp_op_T__ *ptr__, val__ = (val); \ + \ + lc_pcpr = offsetof(struct lowcore, percpu_register); \ + lc_pcpo = offsetof(struct lowcore, percpu_offset); \ + ptr__ = PERCPU_PTR(&(pcp)); \ + asm_inline volatile( \ + MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ + AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + op " %[val__],0(%[ptr__])\n" \ + MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + : [ptr__] "+&a" (ptr__), "=m" (*ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [val__] "d" (val__), \ + [disppcpr] "i" (lc_pcpr), \ + [disppcpo] "i" (lc_pcpo), \ + [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ + [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + "m" (((struct lowcore *)0)->percpu_offset) \ + : "cc"); \ +} while (0) + +#define this_cpu_write_1(pcp, val) arch_this_cpu_write(pcp, val, "stc") +#define this_cpu_write_2(pcp, val) arch_this_cpu_write(pcp, val, "sth") +#define this_cpu_write_4(pcp, val) arch_this_cpu_write(pcp, val, "st") +#define this_cpu_write_8(pcp, val) arch_this_cpu_write(pcp, val, "stg") + #define arch_this_cpu_cmpxchg(pcp, oval, nval) \ ({ \ typedef typeof(pcp) pcp_op_T__; \ diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 2c6cee8241e0..3197b8b372a2 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -1189,10 +1189,10 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, pte_t res; res = ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID)); + page_table_check_pte_clear(mm, addr, res); /* At this point the reference through the mapping is still present */ if (mm_is_protected(mm) && pte_present(res)) WARN_ON_ONCE(uv_convert_from_secure_pte(res)); - page_table_check_pte_clear(mm, addr, res); return res; } @@ -1208,10 +1208,10 @@ static inline pte_t ptep_clear_flush(struct vm_area_struct *vma, pte_t res; res = ptep_xchg_direct(vma->vm_mm, addr, ptep, __pte(_PAGE_INVALID)); + page_table_check_pte_clear(vma->vm_mm, addr, res); /* At this point the reference through the mapping is still present */ if (mm_is_protected(vma->vm_mm) && pte_present(res)) WARN_ON_ONCE(uv_convert_from_secure_pte(res)); - page_table_check_pte_clear(vma->vm_mm, addr, res); return res; } @@ -1235,26 +1235,23 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, } else { res = ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID)); } - page_table_check_pte_clear(mm, addr, res); - - /* Nothing to do */ - if (!mm_is_protected(mm) || !pte_present(res)) - return res; - /* - * At this point the reference through the mapping is still present. - * The notifier should have destroyed all protected vCPUs at this - * point, so the destroy should be successful. - */ - if (full && !uv_destroy_pte(res)) - return res; - /* - * If something went wrong and the page could not be destroyed, or - * if this is not a mm teardown, the slower export is used as - * fallback instead. If even that fails, print a warning and leak - * the page, to avoid crashing the whole system. - */ - WARN_ON_ONCE(uv_convert_from_secure_pte(res)); + /* At this point the reference through the mapping is still present */ + if (mm_is_protected(mm) && pte_present(res)) { + /* + * The notifier should have destroyed all protected vCPUs at + * this point, so the destroy should be successful. + */ + if (full && !uv_destroy_pte(res)) + return res; + /* + * If something went wrong and the page could not be destroyed, + * or if this is not a mm teardown, the slower export is used + * as fallback instead. If even that fails, print a warning and + * leak the page, to avoid crashing the whole system. + */ + WARN_ON_ONCE(uv_convert_from_secure_pte(res)); + } return res; } diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index 78195ee5e99f..ecd3341686eb 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -282,8 +283,6 @@ static __always_inline unsigned short stap(void) return cpu_address; } -#define cpu_relax() barrier() - #define ECAG_CACHE_ATTRIBUTE 0 #define ECAG_CPU_ATTRIBUTE 1 diff --git a/arch/s390/include/asm/ptrace.h b/arch/s390/include/asm/ptrace.h index aaceb1d9110a..495e310c3d6d 100644 --- a/arch/s390/include/asm/ptrace.h +++ b/arch/s390/include/asm/ptrace.h @@ -134,6 +134,8 @@ struct pt_regs { }; unsigned long flags; unsigned long last_break; + unsigned int cpu; + unsigned char percpu_register; }; /* diff --git a/arch/s390/include/asm/sclp.h b/arch/s390/include/asm/sclp.h index 0f184dbdbe5e..d928a9ddfe40 100644 --- a/arch/s390/include/asm/sclp.h +++ b/arch/s390/include/asm/sclp.h @@ -20,6 +20,7 @@ #define SCLP_ERRNOTIFY_AQ_REPAIR 1 #define SCLP_ERRNOTIFY_AQ_INFO_LOG 2 #define SCLP_ERRNOTIFY_AQ_OPTICS_DATA 3 +#define SCLP_ERRNOTIFY_AQ_NVME_SMART_LOG 4 #ifndef __ASSEMBLER__ #include diff --git a/arch/s390/include/asm/string.h b/arch/s390/include/asm/string.h index 238e721e5a22..378f85304ef5 100644 --- a/arch/s390/include/asm/string.h +++ b/arch/s390/include/asm/string.h @@ -26,7 +26,6 @@ void *memmove(void *dest, const void *src, size_t n); #define __HAVE_ARCH_MEMSCAN /* inline & arch function */ #define __HAVE_ARCH_STRCAT /* inline & arch function */ #define __HAVE_ARCH_STRCMP /* arch function */ -#define __HAVE_ARCH_STRLCAT /* arch function */ #define __HAVE_ARCH_STRLEN /* inline & arch function */ #define __HAVE_ARCH_STRNCAT /* arch function */ #define __HAVE_ARCH_STRNLEN /* inline & arch function */ @@ -38,7 +37,6 @@ void *memmove(void *dest, const void *src, size_t n); /* Prototypes for non-inlined arch strings functions. */ int memcmp(const void *s1, const void *s2, size_t n); int strcmp(const char *s1, const char *s2); -size_t strlcat(char *dest, const char *src, size_t n); char *strncat(char *dest, const char *src, size_t n); char *strstr(const char *s1, const char *s2); #endif /* !defined(CONFIG_KASAN) && !defined(CONFIG_KMSAN) */ diff --git a/arch/s390/include/asm/trace_clock.h b/arch/s390/include/asm/trace_clock.h new file mode 100644 index 000000000000..273e05cbdae0 --- /dev/null +++ b/arch/s390/include/asm/trace_clock.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_S390_TRACE_CLOCK_H +#define _ASM_S390_TRACE_CLOCK_H + +#include +#include + +u64 notrace trace_clock_s390_tod(void); + +#define ARCH_TRACE_CLOCKS \ + { trace_clock_s390_tod, "s390-tod", .in_ns = 0 }, + +#endif /* _ASM_S390_TRACE_CLOCK_H */ diff --git a/arch/s390/include/asm/vdso/processor.h b/arch/s390/include/asm/vdso/processor.h index cfcc3e117c4c..6775621e5a5a 100644 --- a/arch/s390/include/asm/vdso/processor.h +++ b/arch/s390/include/asm/vdso/processor.h @@ -2,6 +2,8 @@ #ifndef __ASM_VDSO_PROCESSOR_H #define __ASM_VDSO_PROCESSOR_H -#define cpu_relax() barrier() +#include + +#define cpu_relax() bcr_serialize() #endif /* __ASM_VDSO_PROCESSOR_H */ diff --git a/arch/s390/include/asm/word-at-a-time.h b/arch/s390/include/asm/word-at-a-time.h index eaa19dee7699..e9287036392d 100644 --- a/arch/s390/include/asm/word-at-a-time.h +++ b/arch/s390/include/asm/word-at-a-time.h @@ -4,7 +4,6 @@ #include #include -#include #include struct word_at_a_time { @@ -41,25 +40,4 @@ static inline unsigned long zero_bytemask(unsigned long data) return ~1UL << data; } -/* - * Load an unaligned word from kernel space. - * - * In the (very unlikely) case of the word being a page-crosser - * and the next page not being mapped, take the exception and - * return zeroes in the non-existing part. - */ -static inline unsigned long load_unaligned_zeropad(const void *addr) -{ - unsigned long data; - - asm_inline volatile( - "0: lg %[data],0(%[addr])\n" - "1: nopr %%r7\n" - EX_TABLE_ZEROPAD(0b, 1b, %[data], %[addr]) - EX_TABLE_ZEROPAD(1b, 1b, %[data], %[addr]) - : [data] "=d" (data) - : [addr] "a" (addr), "m" (*(unsigned long *)addr)); - return data; -} - #endif /* _ASM_WORD_AT_A_TIME_H */ diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 137e4793ec11..6c88476d79a3 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -62,6 +62,7 @@ obj-$(CONFIG_KPROBES) += mcount.o obj-$(CONFIG_RETHOOK) += rethook.o obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o obj-$(CONFIG_FUNCTION_TRACER) += mcount.o +obj-$(CONFIG_TRACING) += trace_clock.o obj-$(CONFIG_CRASH_DUMP) += crash_dump.o obj-$(CONFIG_KEXEC_CORE) += machine_kexec.o relocate_kernel.o obj-$(CONFIG_VMCORE_INFO) += vmcore_info.o diff --git a/arch/s390/kernel/debug.c b/arch/s390/kernel/debug.c index 7650f2adb5cf..dbf430f479bd 100644 --- a/arch/s390/kernel/debug.c +++ b/arch/s390/kernel/debug.c @@ -26,6 +26,8 @@ #include #include #include +#include +#include #include @@ -154,6 +156,7 @@ static unsigned int __used debug_feature_version = __DEBUG_FEATURE_VERSION; static debug_info_t *debug_area_first; static debug_info_t *debug_area_last; static DEFINE_MUTEX(debug_mutex); +extern debug_info_t *__s390dbf_info[], *__s390dbf_info_end[]; static int initialized; static int debug_critical; @@ -168,7 +171,91 @@ static const struct file_operations debug_file_ops = { static struct dentry *debug_debugfs_root_entry; +/* List of debug area parameters to override */ +#define PARAM_UNSET -2 +#define PARAM_NUM 16 +static struct debug_param_t { + char name[DEBUG_MAX_NAME_LEN + 1]; + int level; + int pages; +} debug_param[PARAM_NUM]; +static int debug_param_num; + /* functions */ +static void debug_get_param(const char *name, int *level, int *pages) +{ + struct debug_param_t *p; + int i; + + for (i = 0; i < debug_param_num; i++) { + p = &debug_param[i]; + if (!glob_match(p->name, name)) + continue; + if (level && p->level != PARAM_UNSET) { + pr_info("%s: override level to %d\n", name, p->level); + *level = p->level; + } + if (pages && p->pages != PARAM_UNSET) { + pr_info("%s: override pages to %d\n", name, p->pages); + *pages = p->pages; + } + } +} + +#define LVL_LEN 10 +#define LVL_FMT "%" __stringify(LVL_LEN) "[^:]" +#define NAME_FMT "%" __stringify(DEBUG_MAX_NAME_LEN) "[^:]" + +static bool __init s390dbf_parse_one(const char *arg, struct debug_param_t *param) +{ + struct debug_param_t p = { { 0 }, PARAM_UNSET, PARAM_UNSET }; + char level[LVL_LEN + 1] = { 0 }; + + /* arg: :[|-]:[] */ + if (sscanf(arg, NAME_FMT ":" LVL_FMT ":%d", p.name, level, &p.pages) > 1) { + if (strcmp(level, "-") == 0) + p.level = DEBUG_OFF_LEVEL; + else if (kstrtoint(level, 0, &p.level) != 0) + return false; + } else if (sscanf(arg, NAME_FMT "::%d", p.name, &p.pages) != 2) { + return false; + } + + if (p.level != PARAM_UNSET && p.level != DEBUG_OFF_LEVEL && + (p.level < 0 || p.level > DEBUG_MAX_LEVEL)) + return false; + if (p.pages != PARAM_UNSET && p.pages < 0) + return false; + *param = p; + + return true; +} + +static int __init s390dbf_parse(char *arg) +{ + debug_info_t **id; + int i, rc = 0; + + while (arg && debug_param_num < PARAM_NUM) { + if (s390dbf_parse_one(arg, &debug_param[debug_param_num])) + debug_param_num++; + else + rc = -EINVAL; + arg = strchr(arg, ','); + if (arg) + arg++; + } + + /* + * Apply level to static debug areas, delay buffer size changes until + * regular memory allocations are possible. + */ + for (i = 0, id = __s390dbf_info; &id[i] < __s390dbf_info_end; i++) + debug_get_param(id[i]->name, &id[i]->level, NULL); + + return rc; +} +early_param("s390dbf", s390dbf_parse); /* * debug_areas_alloc @@ -305,10 +392,11 @@ static void debug_info_free(debug_info_t *db_info) static debug_info_t *debug_info_create(const char *name, int pages_per_area, int nr_areas, int buf_size, umode_t mode) { + int level = DEBUG_DEFAULT_LEVEL; debug_info_t *rc; - rc = debug_info_alloc(name, pages_per_area, nr_areas, buf_size, - DEBUG_DEFAULT_LEVEL, ALL_AREAS); + debug_get_param(name, &level, &pages_per_area); + rc = debug_info_alloc(name, pages_per_area, nr_areas, buf_size, level, ALL_AREAS); if (!rc) goto out; @@ -872,6 +960,7 @@ void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas) return; } + debug_get_param(id->name, &id->level, &pages_per_area); copy = debug_info_alloc("", pages_per_area, nr_areas, id->buf_size, id->level, ALL_AREAS); if (!copy) { @@ -975,16 +1064,7 @@ static int debug_set_size(debug_info_t *id, int nr_areas, int pages_per_area) return 0; } -/** - * debug_set_level() - Sets new actual debug level if new_level is valid. - * - * @id: handle for debug log - * @new_level: new debug level - * - * Return: - * none - */ -void debug_set_level(debug_info_t *id, int new_level) +static void _debug_set_level(debug_info_t *id, int new_level) { unsigned long flags; @@ -1003,6 +1083,23 @@ void debug_set_level(debug_info_t *id, int new_level) id->level = new_level; raw_spin_unlock_irqrestore(&id->lock, flags); } + +/** + * debug_set_level() - Sets new actual debug level if new_level is valid. + * + * @id: handle for debug log + * @new_level: new debug level + * + * Return: + * none + */ +void debug_set_level(debug_info_t *id, int new_level) +{ + /* Level specified via kernel parameter takes precedence */ + debug_get_param(id->name, &new_level, NULL); + + _debug_set_level(id, new_level); +} EXPORT_SYMBOL(debug_set_level); /* @@ -1137,8 +1234,6 @@ static const struct ctl_table s390dbf_table[] = { }, }; -static struct ctl_table_header *s390dbf_sysctl_header; - /** * debug_stop_all() - stops the debug feature if stopping is allowed. * @@ -1529,7 +1624,7 @@ static int debug_input_level_fn(debug_info_t *id, struct debug_view *view, goto out; } if (str[0] == '-') { - debug_set_level(id, DEBUG_OFF_LEVEL); + _debug_set_level(id, DEBUG_OFF_LEVEL); rc = user_len; goto free_str; } else { @@ -1539,7 +1634,7 @@ static int debug_input_level_fn(debug_info_t *id, struct debug_view *view, pr_warn("%s is not a valid level for a debug feature\n", str); rc = -EINVAL; } else { - debug_set_level(id, new_level); + _debug_set_level(id, new_level); rc = user_len; } free_str: @@ -1728,7 +1823,7 @@ EXPORT_SYMBOL(debug_sprintf_format_fn); */ static int __init debug_init(void) { - s390dbf_sysctl_header = register_sysctl("s390dbf", s390dbf_table); + register_sysctl("s390dbf", s390dbf_table); mutex_lock(&debug_mutex); debug_debugfs_root_entry = debugfs_create_dir(DEBUG_DIR_ROOT, NULL); initialized = 1; diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index d10a17e6531d..efb988833c88 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -33,6 +33,7 @@ #include #include #include +#include #include "entry.h" DEFINE_PER_CPU_SHARED_ALIGNED(struct irq_stat, irq_stat); @@ -142,10 +143,13 @@ static int irq_pending(struct pt_regs *regs) void noinstr do_io_irq(struct pt_regs *regs) { - irqentry_state_t state = irqentry_enter(regs); - struct pt_regs *old_regs = set_irq_regs(regs); - bool from_idle; + bool from_idle, percpu_needs_fixup; + struct pt_regs *old_regs; + irqentry_state_t state; + percpu_entry(regs); + state = irqentry_enter(regs); + old_regs = set_irq_regs(regs); from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT); if (from_idle) update_timer_idle(); @@ -170,21 +174,25 @@ void noinstr do_io_irq(struct pt_regs *regs) do_irq_async(regs, IO_INTERRUPT); } while (machine_is_lpar() && irq_pending(regs)); + percpu_needs_fixup = percpu_code_check(regs); irq_exit_rcu(); - set_irq_regs(old_regs); irqentry_exit(regs, state); if (from_idle) regs->psw.mask &= ~(PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_WAIT); + percpu_exit(regs, percpu_needs_fixup); } void noinstr do_ext_irq(struct pt_regs *regs) { - irqentry_state_t state = irqentry_enter(regs); - struct pt_regs *old_regs = set_irq_regs(regs); - bool from_idle; + bool from_idle, percpu_needs_fixup; + struct pt_regs *old_regs; + irqentry_state_t state; + percpu_entry(regs); + state = irqentry_enter(regs); + old_regs = set_irq_regs(regs); from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT); if (from_idle) update_timer_idle(); @@ -206,12 +214,14 @@ void noinstr do_ext_irq(struct pt_regs *regs) do_irq_async(regs, EXT_INTERRUPT); + percpu_needs_fixup = percpu_code_check(regs); irq_exit_rcu(); set_irq_regs(old_regs); irqentry_exit(regs, state); if (from_idle) regs->psw.mask &= ~(PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_WAIT); + percpu_exit(regs, percpu_needs_fixup); } static void show_msi_interrupt(struct seq_file *p, int irq) diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 94fbfad49f62..e17a59d4d5a4 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -363,6 +364,7 @@ NOKPROBE_SYMBOL(s390_backup_mcck_info); */ void notrace s390_do_machine_check(struct pt_regs *regs) { + bool percpu_needs_fixup; static int ipd_count; static DEFINE_SPINLOCK(ipd_lock); static unsigned long long last_ipd; @@ -374,6 +376,7 @@ void notrace s390_do_machine_check(struct pt_regs *regs) unsigned long mcck_dam_code; int mcck_pending = 0; + percpu_entry(regs); irq_state = irqentry_nmi_enter(regs); if (user_mode(regs)) @@ -495,7 +498,9 @@ void notrace s390_do_machine_check(struct pt_regs *regs) if (mcck_pending) schedule_mcck_handler(); + percpu_needs_fixup = percpu_code_check(regs); irqentry_nmi_exit(regs, irq_state); + percpu_exit(regs, percpu_needs_fixup); } NOKPROBE_SYMBOL(s390_do_machine_check); diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index 0df95dcb2101..416650ae4871 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -50,7 +50,7 @@ void ret_from_fork(void) asm("ret_from_fork"); void __ret_from_fork(struct task_struct *prev, struct pt_regs *regs) { - void (*func)(void *arg); + int (*func)(void *arg); schedule_tail(prev); @@ -203,9 +203,6 @@ unsigned long __get_wchan(struct task_struct *p) struct unwind_state state; unsigned long ip = 0; - if (!task_stack_page(p)) - return 0; - if (!try_get_task_stack(p)) return 0; diff --git a/arch/s390/kernel/trace_clock.c b/arch/s390/kernel/trace_clock.c new file mode 100644 index 000000000000..9ca568058cbc --- /dev/null +++ b/arch/s390/kernel/trace_clock.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +/* + * trace_clock_s390_tod(): trace clock based on the s390 TOD clock + * + * Unlike the other clocks, this is not in nanoseconds. + */ +u64 notrace trace_clock_s390_tod(void) +{ + return get_tod_clock(); +} diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c index 1b5c6fc431cc..564403496a7c 100644 --- a/arch/s390/kernel/traps.c +++ b/arch/s390/kernel/traps.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -329,6 +330,7 @@ static void (*pgm_check_table[128])(struct pt_regs *regs); void noinstr __do_pgm_check(struct pt_regs *regs) { struct lowcore *lc = get_lowcore(); + bool percpu_needs_fixup; irqentry_state_t state; unsigned int trapnr; union teid teid; @@ -349,6 +351,7 @@ void noinstr __do_pgm_check(struct pt_regs *regs) current->thread.gmap_int_code = regs->int_code & 0xffff; return; } + percpu_entry(regs); state = irqentry_enter(regs); if (user_mode(regs)) { update_timer_sys(); @@ -385,7 +388,9 @@ void noinstr __do_pgm_check(struct pt_regs *regs) pgm_check_table[trapnr](regs); out: local_irq_disable(); + percpu_needs_fixup = percpu_code_check(regs); irqentry_exit(regs, state); + percpu_exit(regs, percpu_needs_fixup); } /* diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S index 2b62395e35bf..1f0e71c58eb9 100644 --- a/arch/s390/kernel/vmlinux.lds.S +++ b/arch/s390/kernel/vmlinux.lds.S @@ -159,6 +159,13 @@ SECTIONS } #endif + . = ALIGN(8); + .s390dbf_info : { + __s390dbf_info = .; + *(.s390dbf_info) + __s390dbf_info_end = .; + } + /* * Table with the patch locations to undo expolines */ diff --git a/arch/s390/lib/Makefile b/arch/s390/lib/Makefile index 2bf47204f6ab..aa6cc6a1fe88 100644 --- a/arch/s390/lib/Makefile +++ b/arch/s390/lib/Makefile @@ -3,9 +3,13 @@ # Makefile for s390-specific library files.. # +# string.o implements standard library functions like memset/memcpy etc. +# Use -ffreestanding to ensure that the compiler does not try to "optimize" +# them into calls to themselves. +CFLAGS_string.o = -ffreestanding + lib-y += delay.o string.o uaccess.o find.o spinlock.o tishift.o lib-y += csum-partial.o -obj-y += mem.o lib-$(CONFIG_KPROBES) += probes.o lib-$(CONFIG_UPROBES) += probes.o obj-$(CONFIG_S390_KPROBES_SANITY_TEST) += test_kprobes_s390.o diff --git a/arch/s390/lib/mem.S b/arch/s390/lib/mem.S deleted file mode 100644 index d026debf250c..000000000000 --- a/arch/s390/lib/mem.S +++ /dev/null @@ -1,192 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * String handling functions. - * - * Copyright IBM Corp. 2012 - */ - -#include -#include -#include - - GEN_BR_THUNK %r14 - -/* - * void *memmove(void *dest, const void *src, size_t n) - */ -SYM_FUNC_START(__memmove) - ltgr %r4,%r4 - lgr %r1,%r2 - jz .Lmemmove_exit - aghi %r4,-1 - clgr %r2,%r3 - jnh .Lmemmove_forward - la %r5,1(%r4,%r3) - clgr %r2,%r5 - jl .Lmemmove_reverse -.Lmemmove_forward: - srlg %r0,%r4,8 - ltgr %r0,%r0 - jz .Lmemmove_forward_remainder -.Lmemmove_forward_loop: - mvc 0(256,%r1),0(%r3) - la %r1,256(%r1) - la %r3,256(%r3) - brctg %r0,.Lmemmove_forward_loop -.Lmemmove_forward_remainder: - exrl %r4,.Lmemmove_mvc -.Lmemmove_exit: - BR_EX %r14 -.Lmemmove_reverse: - ic %r0,0(%r4,%r3) - stc %r0,0(%r4,%r1) - brctg %r4,.Lmemmove_reverse - ic %r0,0(%r4,%r3) - stc %r0,0(%r4,%r1) - BR_EX %r14 -.Lmemmove_mvc: - mvc 0(1,%r1),0(%r3) -SYM_FUNC_END(__memmove) -EXPORT_SYMBOL(__memmove) - -SYM_FUNC_ALIAS(memmove, __memmove) -EXPORT_SYMBOL(memmove) - -/* - * memset implementation - * - * This code corresponds to the C construct below. We do distinguish - * between clearing (c == 0) and setting a memory array (c != 0) simply - * because nearly all memset invocations in the kernel clear memory and - * the xc instruction is preferred in such cases. - * - * void *memset(void *s, int c, size_t n) - * { - * if (likely(c == 0)) - * return __builtin_memset(s, 0, n); - * return __builtin_memset(s, c, n); - * } - */ -SYM_FUNC_START(__memset) - ltgr %r4,%r4 - jz .Lmemset_exit - ltgr %r3,%r3 - jnz .Lmemset_fill - aghi %r4,-1 - srlg %r3,%r4,8 - ltgr %r3,%r3 - lgr %r1,%r2 - jz .Lmemset_clear_remainder -.Lmemset_clear_loop: - xc 0(256,%r1),0(%r1) - la %r1,256(%r1) - brctg %r3,.Lmemset_clear_loop -.Lmemset_clear_remainder: - exrl %r4,.Lmemset_xc -.Lmemset_exit: - BR_EX %r14 -.Lmemset_fill: - cghi %r4,1 - lgr %r1,%r2 - je .Lmemset_fill_exit - aghi %r4,-2 - srlg %r5,%r4,8 - ltgr %r5,%r5 - jz .Lmemset_fill_remainder -.Lmemset_fill_loop: - stc %r3,0(%r1) - mvc 1(255,%r1),0(%r1) - la %r1,256(%r1) - brctg %r5,.Lmemset_fill_loop -.Lmemset_fill_remainder: - stc %r3,0(%r1) - exrl %r4,.Lmemset_mvc - BR_EX %r14 -.Lmemset_fill_exit: - stc %r3,0(%r1) - BR_EX %r14 -.Lmemset_xc: - xc 0(1,%r1),0(%r1) -.Lmemset_mvc: - mvc 1(1,%r1),0(%r1) -SYM_FUNC_END(__memset) -EXPORT_SYMBOL(__memset) - -SYM_FUNC_ALIAS(memset, __memset) -EXPORT_SYMBOL(memset) - -/* - * memcpy implementation - * - * void *memcpy(void *dest, const void *src, size_t n) - */ -SYM_FUNC_START(__memcpy) - ltgr %r4,%r4 - jz .Lmemcpy_exit - aghi %r4,-1 - srlg %r5,%r4,8 - ltgr %r5,%r5 - lgr %r1,%r2 - jnz .Lmemcpy_loop -.Lmemcpy_remainder: - exrl %r4,.Lmemcpy_mvc -.Lmemcpy_exit: - BR_EX %r14 -.Lmemcpy_loop: - mvc 0(256,%r1),0(%r3) - la %r1,256(%r1) - la %r3,256(%r3) - brctg %r5,.Lmemcpy_loop - j .Lmemcpy_remainder -.Lmemcpy_mvc: - mvc 0(1,%r1),0(%r3) -SYM_FUNC_END(__memcpy) -EXPORT_SYMBOL(__memcpy) - -SYM_FUNC_ALIAS(memcpy, __memcpy) -EXPORT_SYMBOL(memcpy) - -/* - * __memset16/32/64 - * - * void *__memset16(uint16_t *s, uint16_t v, size_t count) - * void *__memset32(uint32_t *s, uint32_t v, size_t count) - * void *__memset64(uint64_t *s, uint64_t v, size_t count) - */ -.macro __MEMSET bits,bytes,insn -SYM_FUNC_START(__memset\bits) - ltgr %r4,%r4 - jz .L__memset_exit\bits - cghi %r4,\bytes - je .L__memset_store\bits - aghi %r4,-(\bytes+1) - srlg %r5,%r4,8 - ltgr %r5,%r5 - lgr %r1,%r2 - jz .L__memset_remainder\bits -.L__memset_loop\bits: - \insn %r3,0(%r1) - mvc \bytes(256-\bytes,%r1),0(%r1) - la %r1,256(%r1) - brctg %r5,.L__memset_loop\bits -.L__memset_remainder\bits: - \insn %r3,0(%r1) - exrl %r4,.L__memset_mvc\bits - BR_EX %r14 -.L__memset_store\bits: - \insn %r3,0(%r2) -.L__memset_exit\bits: - BR_EX %r14 -.L__memset_mvc\bits: - mvc \bytes(1,%r1),0(%r1) -SYM_FUNC_END(__memset\bits) -.endm - -__MEMSET 16,2,sth -EXPORT_SYMBOL(__memset16) - -__MEMSET 32,4,st -EXPORT_SYMBOL(__memset32) - -__MEMSET 64,8,stg -EXPORT_SYMBOL(__memset64) diff --git a/arch/s390/lib/string.c b/arch/s390/lib/string.c index 757f58960198..32e0e6b1e623 100644 --- a/arch/s390/lib/string.c +++ b/arch/s390/lib/string.c @@ -15,8 +15,218 @@ #include #include #include +#include #include +#define SYMBOL_FUNCTION_ALIAS(alias, name) \ +asm(".globl " __stringify(alias) "\n\t" \ + ".set " __stringify(alias) "," __stringify(name)) + +#ifdef __HAVE_ARCH_MEMMOVE +noinstr void *__memmove(void *dest, const void *src, size_t n) +{ + const char *s = src; + char *d = dest; + + if (!n) + return dest; + if ((d <= s || d >= s + n)) { + /* Forward copy */ + while (n >= 256) { + asm volatile( + " mvc 0(256,%[d]),0(%[s])\n" + : + : [d] "a" (d), [s] "a" (s) + : "memory"); + d += 256; + s += 256; + n -= 256; + } + if (n) { + asm volatile( + " exrl %[n],0f\n" + " j 1f\n" + "0: mvc 0(1,%[d]),0(%[s])\n" + "1:" + : + : [d] "a" (d), [s] "a" (s), [n] "a" (n - 1) + : "memory"); + } + return dest; + } + /* Backward copy */ + if (test_facility(61)) { + /* Use mvcrl instruction if available */ + while (n >= 256) { + asm volatile( + " lghi %%r0,255\n" + " .insn sse,0xe50a00000000,%[d],%[s]\n" + : [d] "=Q" (*(d + n - 256)) + : [s] "Q" (*(s + n - 256)) + : "0", "memory"); + n -= 256; + } + if (n) { + asm volatile( + " lgr %%r0,%[n]\n" + " .insn sse,0xe50a00000000,%[d],%[s]\n" + : [d] "=Q" (*d) + : [s] "Q" (*s), [n] "d" (n - 1) + : "0", "memory"); + } + } else { + while (n--) + d[n] = s[n]; + } + return dest; +} +SYMBOL_FUNCTION_ALIAS(memmove, __memmove); +EXPORT_SYMBOL(__memmove); +EXPORT_SYMBOL(memmove); +#endif + +#ifdef __HAVE_ARCH_MEMSET +noinstr void *__memset(void *s, int c, size_t n) +{ + char *xs = s; + + if (!n) + return s; + if (!c) { + /* Clear memory */ + while (n >= 256) { + asm volatile( + " xc 0(256,%[xs]),0(%[xs])" + : + : [xs] "a" (xs) + : "cc", "memory"); + xs += 256; + n -= 256; + } + if (!n) + return s; + asm volatile( + " exrl %[n],0f\n" + " j 1f\n" + "0: xc 0(1,%[xs]),0(%[xs])\n" + "1:" + : + : [xs] "a" (xs), [n] "a" (n - 1) + : "cc", "memory"); + } else { + /* Fill memory */ + while (n >= 256) { + *xs = c; + asm volatile( + " mvc 1(255,%[xs]),0(%[xs])" + : + : [xs] "a" (xs) + : "memory"); + xs += 256; + n -= 256; + } + if (!n) + return s; + *xs = c; + if (n == 1) + return s; + asm volatile( + " exrl %[n],0f\n" + " j 1f\n" + "0: mvc 1(1,%[xs]),0(%[xs])\n" + "1:" + : + : [xs] "a" (xs), [n] "a" (n - 2) + : "memory"); + } + return s; +} +SYMBOL_FUNCTION_ALIAS(memset, __memset); +EXPORT_SYMBOL(__memset); +EXPORT_SYMBOL(memset); +#endif + +#ifdef __HAVE_ARCH_MEMCPY +noinstr void *__memcpy(void *dest, const void *src, size_t n) +{ + void *d = dest; + + if (!n) + return d; + while (n >= 256) { + asm volatile( + " mvc 0(256,%[dest]),0(%[src])" + : + : [dest] "a" (dest), [src] "a" (src) + : "memory"); + dest += 256; + src += 256; + n -= 256; + } + if (!n) + return d; + asm volatile( + " exrl %[n],1f\n" + " j 2f\n" + "1: mvc 0(1,%[dest]),0(%[src])\n" + "2:" + : + : [dest] "a" (dest), [src] "a" (src), [n] "a" (n - 1) + : "memory"); + return d; +} +SYMBOL_FUNCTION_ALIAS(memcpy, __memcpy); +EXPORT_SYMBOL(__memcpy); +EXPORT_SYMBOL(memcpy); +#endif + +#define DEFINE_MEMSET(_bits, _bytes, _type) \ +void *__memset##_bits(_type *s, _type v, size_t n) \ +{ \ + _type *xs = s; \ + \ + if (!n) \ + return s; \ + while (n >= 256) { \ + *xs = v; \ + asm volatile( \ + " mvc %[_b](256-%[_b],%[xs]),0(%[xs])\n" \ + : \ + : [xs] "a" (xs), [_b] "i" (_bytes) \ + : "memory"); \ + xs = (_type *)((char *)xs + 256); \ + n -= 256; \ + } \ + if (!n) \ + return s; \ + *xs = v; \ + if (n == _bytes) \ + return s; \ + n -= _bytes + 1; \ + asm volatile( \ + " exrl %[n],1f\n" \ + " j 2f\n" \ + "1: mvc %[_b](1,%[xs]),0(%[xs])\n" \ + "2:" \ + : \ + : [n] "a" (n), [xs] "a" (xs), [_b] "i" (_bytes) \ + : "memory"); \ + return s; \ +} \ +EXPORT_SYMBOL(__memset##_bits) + +#ifdef __HAVE_ARCH_MEMSET16 +DEFINE_MEMSET(16, 2, uint16_t); +#endif + +#ifdef __HAVE_ARCH_MEMSET32 +DEFINE_MEMSET(32, 4, uint32_t); +#endif + +#ifdef __HAVE_ARCH_MEMSET64 +DEFINE_MEMSET(64, 8, uint64_t); +#endif + /* * Helper functions to find the end of a string */ @@ -104,32 +314,6 @@ char *strcat(char *dest, const char *src) EXPORT_SYMBOL(strcat); #endif -/** - * strlcat - Append a length-limited, %NUL-terminated string to another - * @dest: The string to be appended to - * @src: The string to append to it - * @n: The size of the destination buffer. - */ -#ifdef __HAVE_ARCH_STRLCAT -size_t strlcat(char *dest, const char *src, size_t n) -{ - size_t dsize = __strend(dest) - dest; - size_t len = __strend(src) - src; - size_t res = dsize + len; - - if (dsize < n) { - dest += dsize; - n -= dsize; - if (len >= n) - len = n - 1; - dest[len] = '\0'; - memcpy(dest, src, len); - } - return res; -} -EXPORT_SYMBOL(strlcat); -#endif - /** * strncat - Append a length-limited, %NUL-terminated string to another * @dest: The string to be appended to diff --git a/arch/s390/lib/tishift.S b/arch/s390/lib/tishift.S deleted file mode 100644 index 96214f51f49b..000000000000 --- a/arch/s390/lib/tishift.S +++ /dev/null @@ -1,63 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - -#include -#include -#include - - .section .noinstr.text, "ax" - - GEN_BR_THUNK %r14 - -SYM_FUNC_START(__ashlti3) - lmg %r0,%r1,0(%r3) - cije %r4,0,1f - lhi %r3,64 - sr %r3,%r4 - jnh 0f - srlg %r3,%r1,0(%r3) - sllg %r0,%r0,0(%r4) - sllg %r1,%r1,0(%r4) - ogr %r0,%r3 - j 1f -0: sllg %r0,%r1,-64(%r4) - lghi %r1,0 -1: stmg %r0,%r1,0(%r2) - BR_EX %r14 -SYM_FUNC_END(__ashlti3) -EXPORT_SYMBOL(__ashlti3) - -SYM_FUNC_START(__ashrti3) - lmg %r0,%r1,0(%r3) - cije %r4,0,1f - lhi %r3,64 - sr %r3,%r4 - jnh 0f - sllg %r3,%r0,0(%r3) - srlg %r1,%r1,0(%r4) - srag %r0,%r0,0(%r4) - ogr %r1,%r3 - j 1f -0: srag %r1,%r0,-64(%r4) - srag %r0,%r0,63 -1: stmg %r0,%r1,0(%r2) - BR_EX %r14 -SYM_FUNC_END(__ashrti3) -EXPORT_SYMBOL(__ashrti3) - -SYM_FUNC_START(__lshrti3) - lmg %r0,%r1,0(%r3) - cije %r4,0,1f - lhi %r3,64 - sr %r3,%r4 - jnh 0f - sllg %r3,%r0,0(%r3) - srlg %r1,%r1,0(%r4) - srlg %r0,%r0,0(%r4) - ogr %r1,%r3 - j 1f -0: srlg %r1,%r0,-64(%r4) - lghi %r0,0 -1: stmg %r0,%r1,0(%r2) - BR_EX %r14 -SYM_FUNC_END(__lshrti3) -EXPORT_SYMBOL(__lshrti3) diff --git a/arch/s390/lib/tishift.c b/arch/s390/lib/tishift.c new file mode 100644 index 000000000000..bb16cf639af3 --- /dev/null +++ b/arch/s390/lib/tishift.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include "tishift.h" + +union ti { + __int128_t val; + struct { + u64 high; + u64 low; + }; +}; + +noinstr __int128_t __ashlti3(__int128_t a, int shift) +{ + union ti ti = { .val = a }; + + if (!shift) + return ti.val; + if (shift < 64) { + ti.high = (ti.high << shift) | (ti.low >> (64 - shift)); + ti.low = ti.low << shift; + } else { + ti.high = ti.low << (shift - 64); + ti.low = 0; + } + return ti.val; +} +EXPORT_SYMBOL(__ashlti3); + +noinstr __int128_t __ashrti3(__int128_t a, int shift) +{ + union ti ti = { .val = a }; + + if (!shift) + return ti.val; + if (shift < 64) { + ti.low = (ti.low >> shift) | (ti.high << (64 - shift)); + ti.high = (int64_t)ti.high >> shift; + } else { + ti.low = (int64_t)ti.high >> (shift - 64); + ti.high = (int64_t)ti.high >> 63; + } + return ti.val; +} +EXPORT_SYMBOL(__ashrti3); + +noinstr __int128_t __lshrti3(__int128_t a, int shift) +{ + union ti ti = { .val = a }; + + if (!shift) + return ti.val; + if (shift < 64) { + ti.low = (ti.low >> shift) | (ti.high << (64 - shift)); + ti.high = ti.high >> shift; + } else { + ti.low = ti.high >> (shift - 64); + ti.high = 0; + } + return ti.val; +} +EXPORT_SYMBOL(__lshrti3); diff --git a/arch/s390/lib/tishift.h b/arch/s390/lib/tishift.h new file mode 100644 index 000000000000..43a9b8c8e545 --- /dev/null +++ b/arch/s390/lib/tishift.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _S390_LIB_TISHIFT_H +#define _S390_LIB_TISHIFT_H + +__int128_t __ashlti3(__int128_t a, int b); +__int128_t __ashrti3(__int128_t a, int b); +__int128_t __lshrti3(__int128_t a, int b); + +#endif /* _S390_LIB_TISHIFT_H */ diff --git a/arch/s390/mm/extable.c b/arch/s390/mm/extable.c index 7498e858c401..063b4346742d 100644 --- a/arch/s390/mm/extable.c +++ b/arch/s390/mm/extable.c @@ -50,22 +50,6 @@ static bool ex_handler_ua_load_reg(const struct exception_table_entry *ex, return true; } -static bool ex_handler_zeropad(const struct exception_table_entry *ex, struct pt_regs *regs) -{ - unsigned int reg_addr = FIELD_GET(EX_DATA_REG_ADDR, ex->data); - unsigned int reg_data = FIELD_GET(EX_DATA_REG_ERR, ex->data); - unsigned long data, addr, offset; - - addr = regs->gprs[reg_addr]; - offset = addr & (sizeof(unsigned long) - 1); - addr &= ~(sizeof(unsigned long) - 1); - data = *(unsigned long *)addr; - data <<= BITS_PER_BYTE * offset; - regs->gprs[reg_data] = data; - regs->psw.addr = extable_fixup(ex); - return true; -} - static bool ex_handler_fpc(const struct exception_table_entry *ex, struct pt_regs *regs) { fpu_sfpc(0); @@ -134,8 +118,6 @@ bool fixup_exception(struct pt_regs *regs) return ex_handler_ua_load_reg(ex, false, regs); case EX_TYPE_UA_LOAD_REGPAIR: return ex_handler_ua_load_reg(ex, true, regs); - case EX_TYPE_ZEROPAD: - return ex_handler_zeropad(ex, regs); case EX_TYPE_FPC: return ex_handler_fpc(ex, regs); case EX_TYPE_UA_MVCOS_TO: diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index 1f72efc2a579..36bd9530db52 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -84,6 +84,8 @@ void __init arch_setup_zero_pages(void) empty_zero_page = (unsigned long)memblock_alloc_or_panic(PAGE_SIZE << order, PAGE_SIZE); zero_page_mask = ((PAGE_SIZE << order) - 1) & PAGE_MASK; + + set_memory_ro(empty_zero_page, 1UL << order); } void __init arch_zone_limits_init(unsigned long *max_zone_pfns) diff --git a/arch/s390/purgatory/Makefile b/arch/s390/purgatory/Makefile index 95a8ac45b67e..e74410bb1b88 100644 --- a/arch/s390/purgatory/Makefile +++ b/arch/s390/purgatory/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 -purgatory-y := head.o purgatory.o string.o sha256.o mem.o +purgatory-y := head.o purgatory.o string.o sha256.o targets += $(purgatory-y) purgatory.lds purgatory purgatory.chk purgatory.ro PURGATORY_OBJS = $(addprefix $(obj)/,$(purgatory-y)) @@ -10,8 +10,7 @@ $(obj)/sha256.o: $(srctree)/lib/crypto/sha256.c FORCE CFLAGS_sha256.o := -D__NO_FORTIFY -$(obj)/mem.o: $(srctree)/arch/s390/lib/mem.S FORCE - $(call if_changed_rule,as_o_S) +CC_FLAGS_MARCH_MINIMUM := -march=z10 KBUILD_CFLAGS := $(CC_FLAGS_DIALECT) -fno-strict-aliasing -Wall -Wstrict-prototypes KBUILD_CFLAGS += -Wno-pointer-sign -Wno-sign-compare @@ -19,12 +18,12 @@ KBUILD_CFLAGS += -fno-zero-initialized-in-bss -fno-builtin -ffreestanding KBUILD_CFLAGS += -Os -m64 -msoft-float -fno-common KBUILD_CFLAGS += -fno-stack-protector KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING -KBUILD_CFLAGS += -D__DISABLE_EXPORTS +KBUILD_CFLAGS += $(CC_FLAGS_MARCH_MINIMUM) -D__DISABLE_EXPORTS KBUILD_CFLAGS += $(CLANG_FLAGS) KBUILD_CFLAGS += $(call cc-option,-fno-PIE) KBUILD_CFLAGS += $(call cc-option, -Wno-default-const-init-unsafe) KBUILD_AFLAGS := $(filter-out -DCC_USING_EXPOLINE,$(KBUILD_AFLAGS)) -KBUILD_AFLAGS += -D__DISABLE_EXPORTS +KBUILD_AFLAGS += $(CC_FLAGS_MARCH_MINIMUM) -D__DISABLE_EXPORTS # Since we link purgatory with -r unresolved symbols are not checked, so we # also link a purgatory.chk binary without -r to check for unresolved symbols. diff --git a/arch/s390/purgatory/purgatory.lds.S b/arch/s390/purgatory/purgatory.lds.S index 482eb4fbcef1..387d0db4085f 100644 --- a/arch/s390/purgatory/purgatory.lds.S +++ b/arch/s390/purgatory/purgatory.lds.S @@ -19,6 +19,7 @@ SECTIONS _text = .; /* Text */ *(.text) *(.text.*) + *(.noinstr.text) _etext = . ; } .rodata : { diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c index 4e40d5e96be9..110308bdef01 100644 --- a/arch/sh/mm/init.c +++ b/arch/sh/mm/init.c @@ -331,9 +331,6 @@ void __init mem_init(void) /* Set this up early, so we can take care of the zero page */ cpu_cache_init(); - /* clear the zero-page */ - __flush_wback_region(empty_zero_page, PAGE_SIZE); - vsyscall_init(); pr_info("virtual kernel memory layout:\n" diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 8af8e9070ce4..fa4f0079614c 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -132,7 +132,7 @@ config X86 select ARCH_SUPPORTS_LTO_CLANG_THIN select ARCH_SUPPORTS_RT select ARCH_USE_BUILTIN_BSWAP - select ARCH_USE_CMPXCHG_LOCKREF if X86_CX8 + select ARCH_USE_CMPXCHG_LOCKREF select ARCH_USE_MEMTEST select ARCH_USE_QUEUED_RWLOCKS select ARCH_USE_QUEUED_SPINLOCKS @@ -682,7 +682,6 @@ config X86_INTEL_QUARK depends on X86_32 depends on X86_EXTENDED_PLATFORM depends on X86_PLATFORM_DEVICES - depends on X86_TSC depends on PCI depends on PCI_GOANY depends on X86_IO_APIC @@ -694,17 +693,6 @@ config X86_INTEL_QUARK Say Y here if you have a Quark based system such as the Arduino compatible Intel Galileo. -config X86_RDC321X - bool "RDC R-321x SoC" - depends on X86_32 - depends on X86_EXTENDED_PLATFORM - select M486 - select X86_REBOOTFIXUPS - help - This option is needed for RDC R-321x system-on-chip, also known - as R-8610-(G). - If you don't have one of these chips, you should say N here. - config X86_INTEL_LPSS bool "Intel Low Power Subsystem Support" depends on X86 && ACPI && PCI @@ -1643,33 +1631,6 @@ config X86_BOOTPARAM_MEMORY_CORRUPTION_CHECK Set whether the default state of memory_corruption_check is on or off. -config MATH_EMULATION - bool - depends on MODIFY_LDT_SYSCALL - prompt "Math emulation" if X86_32 && (M486SX || MELAN) - help - Linux can emulate a math coprocessor (used for floating point - operations) if you don't have one. 486DX and Pentium processors have - a math coprocessor built in, 486SX and 386 do not, unless you added - a 487DX or 387, respectively. (The messages during boot time can - give you some hints here ["man dmesg"].) Everyone needs either a - coprocessor or this emulation. - - If you don't have a math coprocessor, you need to say Y here; if you - say Y here even though you have a coprocessor, the coprocessor will - be used nevertheless. (This behavior can be changed with the kernel - command line option "no387", which comes handy if your coprocessor - is broken. Try "man bootparam" or see the documentation of your boot - loader (lilo or loadlin) about how to pass options to the kernel at - boot time.) This means that it is a good idea to say Y here if you - intend to use this kernel on different machines. - - More information about the internals of the Linux math coprocessor - emulation can be found in . - - If you are not sure, say Y; apart from resulting in a 66 KB bigger - kernel, it won't hurt. - config MTRR def_bool y prompt "MTRR (Memory Type Range Register) support" if EXPERT @@ -2775,18 +2736,16 @@ menuconfig APM 1) make sure that you have enough swap space and that it is enabled. 2) pass the "idle=poll" option to the kernel - 3) switch on floating point emulation in the kernel and pass - the "no387" option to the kernel - 4) pass the "floppy=nodma" option to the kernel - 5) pass the "mem=4M" option to the kernel (thereby disabling + 3) pass the "floppy=nodma" option to the kernel + 4) pass the "mem=4M" option to the kernel (thereby disabling all but the first 4 MB of RAM) - 6) make sure that the CPU is not over clocked. - 7) read the sig11 FAQ at - 8) disable the cache from your BIOS settings - 9) install a fan for the video card or exchange video RAM - 10) install a better fan for the CPU - 11) exchange RAM chips - 12) exchange the motherboard. + 5) make sure that the CPU is not over clocked. + 6) read the sig11 FAQ at + 7) disable the cache from your BIOS settings + 8) install a fan for the video card or exchange video RAM + 9) install a better fan for the CPU + 10) exchange RAM chips + 11) exchange the motherboard. To compile this driver as a module, choose M here: the module will be called apm. diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu index d7ba9219cb47..e4654388d794 100644 --- a/arch/x86/Kconfig.cpu +++ b/arch/x86/Kconfig.cpu @@ -32,8 +32,6 @@ choice - "Athlon" for the AMD K7 family (Athlon/Duron/Thunderbird). - "Crusoe" for the Transmeta Crusoe series. - "Efficeon" for the Transmeta Efficeon series. - - "Winchip-C6" for original IDT Winchip. - - "Winchip-2" for IDT Winchips with 3dNow! capabilities. - "AMD Elan" for the 32-bit AMD Elan embedded CPU. - "GeodeGX1" for Geode GX1 (Cyrix MediaGX). - "Geode GX/LX" For AMD Geode GX and LX processors. @@ -45,14 +43,6 @@ choice See each option's help text for additional details. If you don't know what to do, choose "Pentium-Pro". -config M586 - bool "586/K5/5x86/6x86/6x86MX" - depends on X86_32 - help - Select this for an 586 or 686 series processor such as the AMD K5, - the Cyrix 5x86, 6x86 and 6x86MX. This choice does not - assume the RDTSC (Read Time Stamp Counter) instruction. - config M586TSC bool "Pentium-Classic" depends on X86_32 @@ -155,24 +145,6 @@ config MEFFICEON help Select this for a Transmeta Efficeon processor. -config MWINCHIPC6 - bool "Winchip-C6" - depends on X86_32 - help - Select this for an IDT Winchip C6 chip. Linux and GCC - treat this chip as a 586TSC with some extended instructions - and alignment requirements. - -config MWINCHIP3D - bool "Winchip-2/Winchip-2A/Winchip-3" - depends on X86_32 - help - Select this for an IDT Winchip-2, 2A or 3. Linux and GCC - treat this chip as a 586TSC with some extended instructions - and alignment requirements. Also enable out of order memory - stores for this CPU, which can increase performance of some - operations. - config MGEODEGX1 bool "GeodeGX1" depends on X86_32 @@ -270,19 +242,15 @@ config X86_L1_CACHE_SHIFT default "7" if MPENTIUM4 default "6" if MK7 || MPENTIUMM || MATOM || MVIAC7 || X86_GENERIC || X86_64 default "4" if MGEODEGX1 - default "5" if MWINCHIP3D || MWINCHIPC6 || MCRUSOE || MEFFICEON || MCYRIXIII || MK6 || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || M586 || MVIAC3_2 || MGEODE_LX + default "5" if MCRUSOE || MEFFICEON || MCYRIXIII || MK6 || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MVIAC3_2 || MGEODE_LX config X86_F00F_BUG def_bool y depends on M586MMX || M586TSC || M586 -config X86_INVD_BUG - def_bool y - depends on M486SX || M486 - config X86_ALIGNMENT_16 def_bool y - depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK6 || M586MMX || M586TSC || M586 || MVIAC3_2 || MGEODEGX1 + depends on MCYRIXIII || MK6 || M586MMX || M586TSC || M586 || MVIAC3_2 || MGEODEGX1 config X86_INTEL_USERCOPY def_bool y @@ -290,11 +258,10 @@ config X86_INTEL_USERCOPY config X86_USE_PPRO_CHECKSUM def_bool y - depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MATOM + depends on MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MATOM config X86_TSC def_bool y - depends on (MWINCHIP3D || MCRUSOE || MEFFICEON || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MVIAC3_2 || MVIAC7 || MGEODEGX1 || MGEODE_LX || MATOM) || X86_64 config X86_HAVE_PAE def_bool y @@ -302,7 +269,6 @@ config X86_HAVE_PAE config X86_CX8 def_bool y - depends on X86_HAVE_PAE || M586TSC || M586MMX || MK6 || MK7 || MGEODEGX1 || MGEODE_LX # this should be set for all -march=.. options where the compiler # generates cmov. @@ -318,7 +284,7 @@ config X86_MINIMUM_CPU_FAMILY config X86_DEBUGCTLMSR def_bool y - depends on !(MK6 || MWINCHIPC6 || MWINCHIP3D || MCYRIXIII || M586MMX || M586TSC || M586) && !UML + depends on !(MK6 || MCYRIXIII || M586MMX || M586TSC || M586) && !UML config IA32_FEAT_CTL def_bool y @@ -334,10 +300,6 @@ menuconfig PROCESSOR_SELECT This lets you choose what x86 vendor support code your kernel will include. -config BROADCAST_TLB_FLUSH - def_bool y - depends on CPU_SUP_AMD && 64BIT - config CPU_SUP_INTEL default y bool "Support Intel processors" if PROCESSOR_SELECT @@ -419,20 +381,6 @@ config CPU_SUP_TRANSMETA_32 If unsure, say N. -config CPU_SUP_UMC_32 - default y - bool "Support UMC processors" if PROCESSOR_SELECT - depends on M486SX || M486 || (EXPERT && !64BIT) - help - This enables detection, tunings and quirks for UMC processors - - You need this enabled if you want your kernel to run on a - UMC CPU. Disabling this option on other types of CPUs - makes the kernel a tiny bit smaller. Disabling it on a UMC - CPU might render the kernel unbootable. - - If unsure, say N. - config CPU_SUP_ZHAOXIN default y bool "Support Zhaoxin processors" if PROCESSOR_SELECT @@ -458,3 +406,7 @@ config CPU_SUP_VORTEX_32 makes the kernel a tiny bit smaller. If unsure, say N. + +config BROADCAST_TLB_FLUSH + def_bool y + depends on CPU_SUP_AMD && 64BIT diff --git a/arch/x86/Kconfig.cpufeatures b/arch/x86/Kconfig.cpufeatures index 532cbc276b1e..09833c88f9ff 100644 --- a/arch/x86/Kconfig.cpufeatures +++ b/arch/x86/Kconfig.cpufeatures @@ -42,7 +42,6 @@ config X86_REQUIRED_FEATURE_NOPL config X86_REQUIRED_FEATURE_CX8 def_bool y - depends on X86_CX8 # this should be set for all -march=.. options where the compiler # generates cmov. @@ -70,7 +69,6 @@ config X86_REQUIRED_FEATURE_UP config X86_REQUIRED_FEATURE_FPU def_bool y - depends on !MATH_EMULATION config X86_REQUIRED_FEATURE_PAE def_bool y diff --git a/arch/x86/Makefile b/arch/x86/Makefile index 1e5457a9a4cd..598f178102ee 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -275,7 +275,6 @@ archprepare: $(cpufeaturemasks.hdr) libs-y += arch/x86/lib/ # drivers-y are linked after core-y -drivers-$(CONFIG_MATH_EMULATION) += arch/x86/math-emu/ drivers-$(CONFIG_PCI) += arch/x86/pci/ # suspend and hibernation support diff --git a/arch/x86/Makefile_32.cpu b/arch/x86/Makefile_32.cpu index 7c9898c15376..c5aa169b596d 100644 --- a/arch/x86/Makefile_32.cpu +++ b/arch/x86/Makefile_32.cpu @@ -10,7 +10,6 @@ else align := -falign-functions=0 -falign-jumps=0 -falign-loops=0 endif -cflags-$(CONFIG_M586) += -march=i586 cflags-$(CONFIG_M586TSC) += -march=i586 cflags-$(CONFIG_M586MMX) += -march=pentium-mmx cflags-$(CONFIG_M686) += -march=i686 @@ -24,16 +23,11 @@ cflags-$(CONFIG_MK6) += -march=k6 cflags-$(CONFIG_MK7) += -march=athlon cflags-$(CONFIG_MCRUSOE) += -march=i686 $(align) cflags-$(CONFIG_MEFFICEON) += -march=i686 $(call tune,pentium3) $(align) -cflags-$(CONFIG_MWINCHIPC6) += $(call cc-option,-march=winchip-c6,-march=i586) -cflags-$(CONFIG_MWINCHIP3D) += $(call cc-option,-march=winchip2,-march=i586) cflags-$(CONFIG_MCYRIXIII) += $(call cc-option,-march=c3,-march=i486) $(align) cflags-$(CONFIG_MVIAC3_2) += $(call cc-option,-march=c3-2,-march=i686) cflags-$(CONFIG_MVIAC7) += -march=i686 cflags-$(CONFIG_MATOM) += -march=atom -# AMD Elan support -cflags-$(CONFIG_MELAN) += -march=i486 - # Geode GX1 support cflags-$(CONFIG_MGEODEGX1) += -march=pentium-mmx cflags-$(CONFIG_MGEODE_LX) += $(call cc-option,-march=geode,-march=pentium-mmx) diff --git a/arch/x86/boot/compressed/kaslr.c b/arch/x86/boot/compressed/kaslr.c index 3b0948ad449f..8e4bf5365ac6 100644 --- a/arch/x86/boot/compressed/kaslr.c +++ b/arch/x86/boot/compressed/kaslr.c @@ -219,7 +219,8 @@ static void parse_gb_huge_pages(char *param, char *val) if (!strcmp(param, "hugepages") && gbpage_sz) { p = val; - max_gb_huge_pages = simple_strtoull(p, &p, 0); + if (boot_kstrtoul(p, 0, &max_gb_huge_pages)) + warn("Failed to parse hugepages= boot parameter\n"); return; } } diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c index 0e89e197e112..1b2fb35704f9 100644 --- a/arch/x86/boot/compressed/pgtable_64.c +++ b/arch/x86/boot/compressed/pgtable_64.c @@ -2,6 +2,7 @@ #include "misc.h" #include #include +#include #include #include #include diff --git a/arch/x86/boot/startup/sme.c b/arch/x86/boot/startup/sme.c index b76a7c95dfe1..c07a2c381ed1 100644 --- a/arch/x86/boot/startup/sme.c +++ b/arch/x86/boot/startup/sme.c @@ -43,6 +43,7 @@ #include #include #include +#include #include #include diff --git a/arch/x86/boot/string.c b/arch/x86/boot/string.c index b25c6a9303b7..ac0f900ebc47 100644 --- a/arch/x86/boot/string.c +++ b/arch/x86/boot/string.c @@ -289,6 +289,9 @@ static int _kstrtoull(const char *s, unsigned int base, unsigned long long *res) unsigned long long _res; unsigned int rv; + if (s[0] == '+') + s++; + s = _parse_integer_fixup_radix(s, &base); rv = _parse_integer(s, base, &_res); if (rv & KSTRTOX_OVERFLOW) @@ -304,35 +307,12 @@ static int _kstrtoull(const char *s, unsigned int base, unsigned long long *res) return 0; } -/** - * kstrtoull - convert a string to an unsigned long long - * @s: The start of the string. The string must be null-terminated, and may also - * include a single newline before its terminating null. The first character - * may also be a plus sign, but not a minus sign. - * @base: The number base to use. The maximum supported base is 16. If base is - * given as 0, then the base of the string is automatically detected with the - * conventional semantics - If it begins with 0x the number will be parsed as a - * hexadecimal (case insensitive), if it otherwise begins with 0, it will be - * parsed as an octal number. Otherwise it will be parsed as a decimal. - * @res: Where to write the result of the conversion on success. - * - * Returns 0 on success, -ERANGE on overflow and -EINVAL on parsing error. - * Used as a replacement for the obsolete simple_strtoull. Return code must - * be checked. - */ -int kstrtoull(const char *s, unsigned int base, unsigned long long *res) -{ - if (s[0] == '+') - s++; - return _kstrtoull(s, base, res); -} - static int _kstrtoul(const char *s, unsigned int base, unsigned long *res) { unsigned long long tmp; int rv; - rv = kstrtoull(s, base, &tmp); + rv = _kstrtoull(s, base, &tmp); if (rv < 0) return rv; if (tmp != (unsigned long)tmp) @@ -364,7 +344,7 @@ int boot_kstrtoul(const char *s, unsigned int base, unsigned long *res) */ if (sizeof(unsigned long) == sizeof(unsigned long long) && __alignof__(unsigned long) == __alignof__(unsigned long long)) - return kstrtoull(s, base, (unsigned long long *)res); + return _kstrtoull(s, base, (unsigned long long *)res); else return _kstrtoul(s, base, res); } diff --git a/arch/x86/boot/string.h b/arch/x86/boot/string.h index a5b05ebc037d..4092bf2ed1ee 100644 --- a/arch/x86/boot/string.h +++ b/arch/x86/boot/string.h @@ -28,6 +28,5 @@ extern unsigned long long simple_strtoull(const char *cp, char **endp, unsigned int base); long simple_strtol(const char *cp, char **endp, unsigned int base); -int kstrtoull(const char *s, unsigned int base, unsigned long long *res); int boot_kstrtoul(const char *s, unsigned int base, unsigned long *res); #endif /* BOOT_STRING_H */ diff --git a/arch/x86/coco/tdx/tdx.c b/arch/x86/coco/tdx/tdx.c index 186915a17c50..29b6f1ed59ec 100644 --- a/arch/x86/coco/tdx/tdx.c +++ b/arch/x86/coco/tdx/tdx.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/x86/events/amd/core.c b/arch/x86/events/amd/core.c index 8ad771f1a0de..6569048a8c1c 100644 --- a/arch/x86/events/amd/core.c +++ b/arch/x86/events/amd/core.c @@ -8,8 +8,10 @@ #include #include #include + #include #include +#include #include #include diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index ed18a6d7e1a8..3531f9c23b8c 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -15,6 +15,7 @@ #include #include +#include #include #include "../perf_event.h" diff --git a/arch/x86/events/amd/lbr.c b/arch/x86/events/amd/lbr.c index d24da377df77..5b437dc8e4ce 100644 --- a/arch/x86/events/amd/lbr.c +++ b/arch/x86/events/amd/lbr.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include + +#include #include #include diff --git a/arch/x86/events/amd/power.c b/arch/x86/events/amd/power.c index dad42790cf7d..744dffa42dee 100644 --- a/arch/x86/events/amd/power.c +++ b/arch/x86/events/amd/power.c @@ -10,8 +10,11 @@ #include #include #include + #include +#include #include + #include "../perf_event.h" /* Event code: LSB 8 bits, passed in attr->config any other bit is reserved. */ diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index a0364ca2f917..dbc00b6dd69e 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -16,6 +16,7 @@ #include #include +#include #include #define NUM_COUNTERS_NB 4 @@ -966,7 +967,7 @@ static void amd_uncore_umc_read(struct perf_event *event) * UMC counters do not have RDPMC assignments. Read counts directly * from the corresponding PERF_CTR. */ - rdmsrl(hwc->event_base, new); + rdmsrq(hwc->event_base, new); /* * Unlike the other uncore counters, UMC counters saturate and set the @@ -975,7 +976,7 @@ static void amd_uncore_umc_read(struct perf_event *event) * that the counter never gets a chance to saturate. */ if (new & BIT_ULL(63 - COUNTER_SHIFT)) { - wrmsrl(hwc->event_base, 0); + wrmsrq(hwc->event_base, 0); local64_set(&hwc->prev_count, 0); } else { local64_set(&hwc->prev_count, new); diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 6f4e0e5a2416..2b35483e2b70 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -3417,12 +3418,12 @@ static void intel_pmu_config_acr(int idx, u64 mask, u32 reload) } if (cpuc->acr_cfg_b[idx] != mask) { - wrmsrl(msr_b + msr_offset, mask); + wrmsrq(msr_b + msr_offset, mask); cpuc->acr_cfg_b[idx] = mask; } /* Only update CFG_C reload when ACR is actively enabled (mask != 0) */ if (mask && ((cpuc->cfg_c_val[idx] & ARCH_PEBS_RELOAD) != reload)) { - wrmsrl(msr_c + msr_offset, reload); + wrmsrq(msr_c + msr_offset, reload); cpuc->cfg_c_val[idx] = reload; } } diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index cb72af9b61ce..91a093d8cf2e 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -780,9 +780,7 @@ void init_debug_store_on_cpu(int cpu) if (!ds) return; - wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA, - (u32)((u64)(unsigned long)ds), - (u32)((u64)(unsigned long)ds >> 32)); + wrmsrq_on_cpu(cpu, MSR_IA32_DS_AREA, (u64)(unsigned long)ds); } void fini_debug_store_on_cpu(int cpu) @@ -790,7 +788,7 @@ void fini_debug_store_on_cpu(int cpu) if (!per_cpu(cpu_hw_events, cpu).ds) return; - wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA, 0, 0); + wrmsrq_on_cpu(cpu, MSR_IA32_DS_AREA, 0); } static DEFINE_PER_CPU(void *, insn_buffer); @@ -1095,8 +1093,7 @@ void init_arch_pebs_on_cpu(int cpu) * contiguous physical buffer (__alloc_pages_node() with order) */ arch_pebs_base = virt_to_phys(cpuc->pebs_vaddr) | PEBS_BUFFER_SHIFT; - wrmsr_on_cpu(cpu, MSR_IA32_PEBS_BASE, (u32)arch_pebs_base, - (u32)(arch_pebs_base >> 32)); + wrmsrq_on_cpu(cpu, MSR_IA32_PEBS_BASE, arch_pebs_base); x86_pmu.pebs_active = 1; } @@ -1105,7 +1102,7 @@ inline void fini_arch_pebs_on_cpu(int cpu) if (!x86_pmu.arch_pebs) return; - wrmsr_on_cpu(cpu, MSR_IA32_PEBS_BASE, 0, 0); + wrmsrq_on_cpu(cpu, MSR_IA32_PEBS_BASE, 0); } /* diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c index 72f2adcda7c6..cae2e02fe6cc 100644 --- a/arch/x86/events/intel/lbr.c +++ b/arch/x86/events/intel/lbr.c @@ -4,6 +4,7 @@ #include #include +#include #include #include diff --git a/arch/x86/events/zhaoxin/core.c b/arch/x86/events/zhaoxin/core.c index 4bc177badac2..e506f677db57 100644 --- a/arch/x86/events/zhaoxin/core.c +++ b/arch/x86/events/zhaoxin/core.c @@ -13,6 +13,7 @@ #include #include +#include #include #include #include diff --git a/arch/x86/include/asm/acrn.h b/arch/x86/include/asm/acrn.h index fab11192c60a..db42b477c41d 100644 --- a/arch/x86/include/asm/acrn.h +++ b/arch/x86/include/asm/acrn.h @@ -2,6 +2,8 @@ #ifndef _ASM_X86_ACRN_H #define _ASM_X86_ACRN_H +#include + /* * This CPUID returns feature bitmaps in EAX. * Guest VM uses this to detect the appropriate feature bit. diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h index 3ddc1d33399b..90680f978d43 100644 --- a/arch/x86/include/asm/cpufeature.h +++ b/arch/x86/include/asm/cpufeature.h @@ -125,7 +125,6 @@ t_no: #define cpu_has_bug(c, bit) cpu_has(c, (bit)) #define set_cpu_bug(c, bit) set_cpu_cap(c, (bit)) -#define clear_cpu_bug(c, bit) clear_cpu_cap(c, (bit)) #define static_cpu_has_bug(bit) static_cpu_has((bit)) #define boot_cpu_has_bug(bit) cpu_has_bug(&boot_cpu_data, (bit)) diff --git a/arch/x86/include/asm/cpuid/api.h b/arch/x86/include/asm/cpuid/api.h index 44fa82e1267c..82eddfa2347b 100644 --- a/arch/x86/include/asm/cpuid/api.h +++ b/arch/x86/include/asm/cpuid/api.h @@ -7,6 +7,7 @@ #include #include +#include #include /* @@ -131,12 +132,12 @@ static inline void __cpuid_read(u32 leaf, u32 subleaf, u32 *regs) __cpuid(regs + CPUID_EAX, regs + CPUID_EBX, regs + CPUID_ECX, regs + CPUID_EDX); } -#define cpuid_subleaf(leaf, subleaf, regs) { \ +#define cpuid_read_subleaf(leaf, subleaf, regs) { \ static_assert(sizeof(*(regs)) == 16); \ __cpuid_read(leaf, subleaf, (u32 *)(regs)); \ } -#define cpuid_leaf(leaf, regs) { \ +#define cpuid_read(leaf, regs) { \ static_assert(sizeof(*(regs)) == 16); \ __cpuid_read(leaf, 0, (u32 *)(regs)); \ } @@ -228,7 +229,7 @@ static inline u32 cpuid_base_hypervisor(const char *sig, u32 leaves) */ static inline void cpuid_leaf_0x2(union leaf_0x2_regs *regs) { - cpuid_leaf(0x2, regs); + cpuid_read(0x2, regs); /* * All Intel CPUs must report an iteration count of 1. In case @@ -289,4 +290,250 @@ static inline bool cpuid_amd_hygon_has_l3_cache(void) return cpuid_edx(0x80000006); } +/* + * 'struct cpuid_leaves' accessors (without sanity checks): + * + * For internal use by the CPUID parser. + */ + +/* Return constified pointers for all call-site APIs */ +#define __const_ptr(_ptr) \ + ((const __typeof__(*(_ptr)) *)(_ptr)) + +#define __cpuid_leaves_subleaf(_leaves, _leaf, _subleaf) \ + __const_ptr(&((_leaves)->leaf_ ## _leaf ## _ ## _subleaf)[0]) + +#define __cpuid_leaves_subleaf_n(_leaves, _leaf, _index) \ + __const_ptr(&((_leaves)->leaf_ ## _leaf ## _ ## n)[_index]) + +#define __cpuid_leaves_subleaf_info(_leaves, _leaf, _subleaf) \ + __const_ptr(&((_leaves)->leaf_ ## _leaf ## _ ## _subleaf ## _ ## info)) + +/* + * 'struct cpuid_table' accessors (with sanity checks): + * + * For internal use by the CPUID parser. + */ + +#define __cpuid_table_nr_filled_subleaves(_table, _leaf, _subleaf) \ + __cpuid_leaves_subleaf_info(&((_table)->leaves), _leaf, _subleaf)->nr_entries + +#define __cpuid_table_subleaf_range_size(_table, _leaf) \ + ARRAY_SIZE((_table)->leaves.leaf_ ## _leaf ## _n) + +#define __cpuid_table_invalid_subleaf(_table, _leaf, _subleaf) \ + (((_subleaf) < (__cpuid_leaf_first_subleaf(_leaf))) || \ + ((_subleaf) > (__cpuid_leaf_first_subleaf(_leaf) + \ + __cpuid_table_subleaf_range_size(_table, _leaf) - 1))) + +/* Return NULL if the parser did not fill that leaf. Check cpuid_subleaf(). */ +#define __cpuid_table_subleaf(_table, _leaf, _subleaf) \ +({ \ + unsigned int ____f = __cpuid_table_nr_filled_subleaves(_table, _leaf, _subleaf); \ + \ + (____f != 1) ? NULL : __cpuid_leaves_subleaf(&((_table)->leaves), _leaf, _subleaf); \ +}) + +/* + * Return NULL if the CPUID parser did not fill this leaf, or if the given + * dynamic subleaf value is out of range. Check cpuid_subleaf_n(). + */ +#define __cpuid_table_subleaf_n(_table, _leaf, _subleaf) \ +({ \ + unsigned int ____i = (_subleaf) - __cpuid_leaf_first_subleaf(_leaf); \ + unsigned int ____f = __cpuid_table_nr_filled_subleaves(_table, _leaf, n); \ + \ + /* CPUID parser might not have filled the entire subleaf range */ \ + ((____i >= ____f) || __cpuid_table_invalid_subleaf(_table, _leaf, _subleaf)) ? \ + NULL : __cpuid_leaves_subleaf_n(&((_table)->leaves), _leaf, ____i); \ +}) + +/* + * Compile-time checks for leaves with a subleaf range: + */ + +#define __cpuid_assert_subleaf_range(_cpuinfo, _leaf) \ + static_assert(__cpuid_table_subleaf_range_size(&(_cpuinfo)->cpuid, _leaf) > 1) + +#define __cpuid_assert_subleaf_within_range(_cpuinfo, _leaf, _subleaf) \ + BUILD_BUG_ON(__builtin_constant_p(_subleaf) && \ + __cpuid_table_invalid_subleaf(&(_cpuinfo)->cpuid, _leaf, _subleaf)) + +/* + * CPUID Parser Call-site APIs + * + * Call sites should use below APIs instead of invoking direct CPUID queries. + * + * Benefits include: + * + * - Return CPUID output as typed C structures that are auto-generated from a + * centralized database (see data type: 'struct leaf_0xM_N', where + * 0xM is the token provided at @_leaf, and N is the token provided at + * @_subleaf; e.g. struct leaf_0x7_0. + * + * Returns NULL if the requested CPUID @_leaf/@_subleaf query output is not + * present at the parsed CPUID table inside @_cpuinfo. This can happen if: + * + * - The CPUID table inside @_cpuinfo has not yet been populated. + * - The CPUID table inside @_cpuinfo was populated, but the CPU does not + * implement the requested CPUID @_leaf/@_subleaf combination. + * - The CPUID table inside @_cpuinfo was populated, but the kernel's CPUID + * parser has predetermined that the requested CPUID @_leaf/@_subleaf + * hardware output is invalid or unsupported. + * + * Example usage:: + * + * const struct leaf_0x7_0 *l7_0 = cpuid_subleaf(c, 0x7, 0); + * if (!l7_0) { + * // Handle error + * } + * + * const struct leaf_0x7_1 *l7_1 = cpuid_subleaf(c, 0x7, 1); + * if (!l7_1) { + * // Handle error + * } + */ +#define cpuid_subleaf(_cpuinfo, _leaf, _subleaf) \ + __cpuid_table_subleaf(&(_cpuinfo)->cpuid, _leaf, _subleaf) \ + +/** + * cpuid_leaf() - Access parsed CPUID data + * @_cpuinfo: CPU capability structure reference ('struct cpuinfo_x86') + * @_leaf: CPUID leaf, in compile-time 0xN format; e.g. 0x0, 0x2, 0x80000000 + * + * Similar to cpuid_subleaf(), but with a CPUID subleaf = 0. + * + * Example usage:: + * + * const struct leaf_0x0_0 *l0 = cpuid_leaf(c, 0x0); + * if (!l0) { + * // Handle error + * } + * + * const struct leaf_0x80000000_0 *el0 = cpuid_leaf(c, 0x80000000); + * if (!el0) { + * // Handle error + * } + */ +#define cpuid_leaf(_cpuinfo, _leaf) \ + cpuid_subleaf(_cpuinfo, _leaf, 0) + +/** + * cpuid_leaf_raw() - Access parsed CPUID data in raw format + * @_cpuinfo: CPU capability structure reference ('struct cpuinfo_x86') + * @_leaf: CPUID leaf, in compile-time 0xN format + * + * Similar to cpuid_leaf(), but returns a raw 'struct cpuid_regs' pointer to + * the parsed CPUID data instead of a "typed" pointer. + */ +#define cpuid_leaf_raw(_cpuinfo, _leaf) \ + ((const struct cpuid_regs *)(cpuid_leaf(_cpuinfo, _leaf))) + +/* + * Call-site APIs for CPUID leaves with a subleaf range: + */ + +/** + * cpuid_subleaf_n() - Access parsed CPUID data for leaf with a subleaf range + * @_cpuinfo: CPU capability structure reference ('struct cpuinfo_x86') + * @_leaf: CPUID leaf, in compile-time 0xN format; e.g. 0x4, 0x8000001d + * @_subleaf: Subleaf number, which can be passed dynamically. It must be smaller + * than cpuid_subleaf_count(@_cpuinfo, @_leaf). + * + * Build-time errors will be emitted in the following cases: + * + * - @_leaf has no subleaf range. Leaves with a subleaf range have an '_n' type + * suffix and are listed at using the CPUID_LEAF_N() macro. + * + * - @_subleaf is known at compile-time but is out of range. + * + * Example usage:: + * + * const struct leaf_0x4_n *l4; + * + * for (int i = 0; i < cpuid_subleaf_count(c, 0x4); i++) { + * l4 = cpuid_subleaf_n(c, 0x4, i); + * if (!l4) { + * // Handle error + * } + * ... + * } + * + * Beside the standard error situations detailed at cpuid_subleaf(), this + * macro will also return NULL if @_subleaf is out of the leaf's subleaf range. + */ +#define cpuid_subleaf_n(_cpuinfo, _leaf, _subleaf) \ +({ \ + __cpuid_assert_subleaf_range(_cpuinfo, _leaf); \ + __cpuid_assert_subleaf_within_range(_cpuinfo, _leaf, _subleaf); \ + __cpuid_table_subleaf_n(&(_cpuinfo)->cpuid, _leaf, _subleaf); \ +}) + +/** + * cpuid_subleaf_n_raw() - Access parsed CPUID data for leaf with subleaf range + * @_cpuinfo: CPU capability structure reference ('struct cpuinfo_x86') + * @_leaf: CPUID leaf, in compile-time 0xN format; e.g. 0x4, 0x8000001d + * @_subleaf: Subleaf number, which can be passed dynamically. It must be smaller + * than cpuid_subleaf_count(@_cpuinfo, @_leaf). + * + * Similar to cpuid_subleaf_n(), but returns a raw 'struct cpuid_regs' pointer to + * the parsed CPUID data instead of a "typed" pointer. + */ +#define cpuid_subleaf_n_raw(_cpuinfo, _leaf, _subleaf) \ + ((const struct cpuid_regs *)cpuid_subleaf_n(_cpuinfo, _leaf, _subleaf)) + +/** + * cpuid_subleaf_count() - Number of filled subleaves for @_leaf + * @_cpuinfo: CPU capability structure reference ('struct cpuinfo_x86') + * @_leaf: CPUID leaf, in compile-time 0xN format; e.g. 0x4, 0x8000001d + * + * Return the number of subleaves filled by the CPUID parser for @_leaf. + * + * @_leaf must have subleaf range. Leaves with a subleaf range have an '_n' type + * suffix and are listed at using the CPUID_LEAF_N() macro. + */ +#define cpuid_subleaf_count(_cpuinfo, _leaf) \ +({ \ + __cpuid_assert_subleaf_range(_cpuinfo, _leaf); \ + __cpuid_table_nr_filled_subleaves(&(_cpuinfo)->cpuid, _leaf, n); \ +}) + +/* + * CPUID parser exported APIs: + */ + +void cpuid_scan_cpu(struct cpuinfo_x86 *c); +void cpuid_refresh_leaf(struct cpuinfo_x86 *c, u32 leaf); +void cpuid_refresh_range(struct cpuinfo_x86 *c, u32 start, u32 end); + #endif /* _ASM_X86_CPUID_API_H */ diff --git a/arch/x86/include/asm/cpuid/leaf_types.h b/arch/x86/include/asm/cpuid/leaf_types.h new file mode 100644 index 000000000000..222d2d2682c8 --- /dev/null +++ b/arch/x86/include/asm/cpuid/leaf_types.h @@ -0,0 +1,2350 @@ +/* SPDX-License-Identifier: MIT */ +/* Generator: x86-cpuid-db v3.1 */ + +/* + * Auto-generated file. + * Please submit all updates and bugfixes to https://x86-cpuid.org + */ + +#ifndef _ASM_X86_CPUID_LEAF_TYPES +#define _ASM_X86_CPUID_LEAF_TYPES + +#include + +/* + * Leaf 0x0 + * Maximum standard leaf + CPU vendor string + */ + +struct leaf_0x0_0 { + // eax + u32 max_std_leaf : 32; // Highest standard CPUID leaf + // ebx + u32 cpu_vendorid_0 : 32; // CPU vendor ID string bytes 0 - 3 + // ecx + u32 cpu_vendorid_2 : 32; // CPU vendor ID string bytes 8 - 11 + // edx + u32 cpu_vendorid_1 : 32; // CPU vendor ID string bytes 4 - 7 +}; + +/* + * Leaf 0x1 + * CPU FMS (Family/Model/Stepping) + standard feature flags + */ + +struct leaf_0x1_0 { + // eax + u32 stepping : 4, // Stepping ID + base_model : 4, // Base CPU model ID + base_family_id : 4, // Base CPU family ID + cpu_type : 2, // CPU type + : 2, // Reserved + ext_model : 4, // Extended CPU model ID + ext_family : 8, // Extended CPU family ID + : 4; // Reserved + // ebx + u32 brand_id : 8, // Brand index + clflush_size : 8, // CLFLUSH instruction cache line size + n_logical_cpu : 8, // Logical CPU count + local_apic_id : 8; // Initial local APIC physical ID + // ecx + u32 sse3 : 1, // Streaming SIMD Extensions 3 (SSE3) + pclmulqdq : 1, // PCLMULQDQ instruction support + dtes64 : 1, // 64-bit DS save area + monitor : 1, // MONITOR/MWAIT support + dscpl : 1, // CPL Qualified Debug Store + vmx : 1, // Virtual Machine Extensions + smx : 1, // Safer Mode Extensions + est : 1, // Enhanced Intel SpeedStep + tm2 : 1, // Thermal Monitor 2 + ssse3 : 1, // Supplemental SSE3 + cntxt_id : 1, // L1 Context ID + sdbg : 1, // Silicon Debug + fma : 1, // FMA extensions using YMM state + cx16 : 1, // CMPXCHG16B instruction support + xtpr_update : 1, // xTPR Update Control + pdcm : 1, // Perfmon and Debug Capability + : 1, // Reserved + pcid : 1, // Process-context identifiers + dca : 1, // Direct Cache Access + sse4_1 : 1, // SSE4.1 + sse4_2 : 1, // SSE4.2 + x2apic : 1, // X2APIC support + movbe : 1, // MOVBE instruction support + popcnt : 1, // POPCNT instruction support + tsc_deadline_timer : 1, // APIC timer one-shot operation + aes : 1, // AES instructions + xsave : 1, // XSAVE (and related instructions) support + osxsave : 1, // XSAVE (and related instructions) are enabled by OS + avx : 1, // AVX instructions support + f16c : 1, // Half-precision floating-point conversion support + rdrand : 1, // RDRAND instruction support + guest_status : 1; // System is running as guest; (para-)virtualized system + // edx + u32 fpu : 1, // Floating-Point Unit on-chip (x87) + vme : 1, // Virtual-8086 Mode Extensions + de : 1, // Debugging Extensions + pse : 1, // Page Size Extension + tsc : 1, // Time Stamp Counter + msr : 1, // Model-Specific Registers (RDMSR and WRMSR support) + pae : 1, // Physical Address Extensions + mce : 1, // Machine Check Exception + cx8 : 1, // CMPXCHG8B instruction + apic : 1, // APIC on-chip + : 1, // Reserved + sep : 1, // SYSENTER, SYSEXIT, and associated MSRs + mtrr : 1, // Memory Type Range Registers + pge : 1, // Page Global Extensions + mca : 1, // Machine Check Architecture + cmov : 1, // Conditional Move Instruction + pat : 1, // Page Attribute Table + pse36 : 1, // Page Size Extension (36-bit) + psn : 1, // Processor Serial Number + clflush : 1, // CLFLUSH instruction + : 1, // Reserved + ds : 1, // Debug Store + acpi : 1, // Thermal monitor and clock control + mmx : 1, // MMX instructions + fxsr : 1, // FXSAVE and FXRSTOR instructions + sse : 1, // SSE instructions + sse2 : 1, // SSE2 instructions + selfsnoop : 1, // Self Snoop + htt : 1, // Hyper-threading + tm : 1, // Thermal Monitor + ia64 : 1, // Legacy IA-64 (Itanium) support bit, now reserved + pbe : 1; // Pending Break Enable +}; + +/* + * Leaf 0x2 + * Intel cache and TLB information one-byte descriptors + */ + +struct leaf_0x2_0 { + // eax + u32 iteration_count : 8, // Number of times this leaf must be queried + desc1 : 8, // Descriptor #1 + desc2 : 8, // Descriptor #2 + desc3 : 7, // Descriptor #3 + eax_invalid : 1; // Descriptors 1-3 are invalid if set + // ebx + u32 desc4 : 8, // Descriptor #4 + desc5 : 8, // Descriptor #5 + desc6 : 8, // Descriptor #6 + desc7 : 7, // Descriptor #7 + ebx_invalid : 1; // Descriptors 4-7 are invalid if set + // ecx + u32 desc8 : 8, // Descriptor #8 + desc9 : 8, // Descriptor #9 + desc10 : 8, // Descriptor #10 + desc11 : 7, // Descriptor #11 + ecx_invalid : 1; // Descriptors 8-11 are invalid if set + // edx + u32 desc12 : 8, // Descriptor #12 + desc13 : 8, // Descriptor #13 + desc14 : 8, // Descriptor #14 + desc15 : 7, // Descriptor #15 + edx_invalid : 1; // Descriptors 12-15 are invalid if set +}; + +/* + * Leaf 0x4 + * Intel deterministic cache parameters + */ + +struct leaf_0x4_n { + // eax + u32 cache_type : 5, // Cache type field + cache_level : 3, // Cache level (1-based) + cache_self_init : 1, // Self-initializing cache level + fully_associative : 1, // Fully-associative cache + : 4, // Reserved + num_threads_sharing : 12, // Number logical CPUs sharing this cache + num_cores_on_die : 6; // Number of cores in the physical package + // ebx + u32 cache_linesize : 12, // System coherency line size (0-based) + cache_npartitions : 10, // Physical line partitions (0-based) + cache_nways : 10; // Ways of associativity (0-based) + // ecx + u32 cache_nsets : 31, // Cache number of sets (0-based) + : 1; // Reserved + // edx + u32 wbinvd_rll_no_guarantee : 1, // WBINVD/INVD not guaranteed for Remote Lower-Level caches + ll_inclusive : 1, // Cache is inclusive of Lower-Level caches + complex_indexing : 1, // Not a direct-mapped cache (complex function) + : 29; // Reserved +}; + +#define LEAF_0x4_SUBLEAF_N_FIRST 0 +#define LEAF_0x4_SUBLEAF_N_LAST 31 + +/* + * Leaf 0x5 + * MONITOR/MWAIT instructions + */ + +struct leaf_0x5_0 { + // eax + u32 min_mon_size : 16, // Smallest monitor-line size, in bytes + : 16; // Reserved + // ebx + u32 max_mon_size : 16, // Largest monitor-line size, in bytes + : 16; // Reserved + // ecx + u32 mwait_ext : 1, // MONITOR/MWAIT extensions + mwait_irq_break : 1, // Interrupts as a break event for MWAIT + : 30; // Reserved + // edx + u32 n_c0_substates : 4, // Number of C0 sub C-states + n_c1_substates : 4, // Number of C1 sub C-states + n_c2_substates : 4, // Number of C2 sub C-states + n_c3_substates : 4, // Number of C3 sub C-states + n_c4_substates : 4, // Number of C4 sub C-states + n_c5_substates : 4, // Number of C5 sub C-states + n_c6_substates : 4, // Number of C6 sub C-states + n_c7_substates : 4; // Number of C7 sub C-states +}; + +/* + * Leaf 0x6 + * Thermal and power management + */ + +struct leaf_0x6_0 { + // eax + u32 digital_temp : 1, // Digital temperature sensor + turbo_boost : 1, // Intel Turbo Boost + lapic_timer_always_on : 1, // Always-Running APIC Timer (not affected by p-state) + : 1, // Reserved + power_limit_event : 1, // Power Limit Notification (PLN) event + ecmd : 1, // Clock modulation duty cycle extension + package_thermal : 1, // Package thermal management + hwp_base_regs : 1, // HWP (Hardware P-states) base registers + hwp_notify : 1, // HWP notification (IA32_HWP_INTERRUPT MSR) + hwp_activity_window : 1, // HWP activity window (IA32_HWP_REQUEST[bits 41:32]) + hwp_energy_perf_pr : 1, // HWP Energy Performance Preference + hwp_package_req : 1, // HWP Package Level Request + : 1, // Reserved + hdc_base_regs : 1, // HDC base registers + turbo_boost_3_0 : 1, // Intel Turbo Boost Max 3.0 + hwp_capabilities : 1, // HWP Highest Performance change + hwp_peci_override : 1, // HWP PECI override + hwp_flexible : 1, // Flexible HWP + hwp_fast : 1, // IA32_HWP_REQUEST MSR fast access mode + hw_feedback : 1, // HW_FEEDBACK MSRs + hwp_ignore_idle : 1, // Ignoring idle logical CPU HWP request is supported + : 1, // Reserved + hwp_ctl : 1, // IA32_HWP_CTL MSR + thread_director : 1, // Intel thread director + therm_interrupt_bit25 : 1, // IA32_THERM_INTERRUPT MSR bit 25 + : 7; // Reserved + // ebx + u32 n_therm_thresholds : 4, // Digital thermometer thresholds + : 28; // Reserved + // ecx + u32 aperf_mperf : 1, // MPERF/APERF MSRs (effective frequency interface) + : 2, // Reserved + energy_perf_bias : 1, // IA32_ENERGY_PERF_BIAS MSR + : 4, // Reserved + hw_feedback_nclasses : 8, // Number of Intel Thread Director classes + : 16; // Reserved + // edx + u32 perfcap_reporting : 1, // Performance capability reporting + encap_reporting : 1, // Energy efficiency capability reporting + : 6, // Reserved + feedback_sz : 4, // Feedback interface structure size, in 4K pages + : 4, // Reserved + this_lcpu_hwfdbk_idx : 16; // This logical CPU hardware feedback interface index +}; + +/* + * Leaf 0x7 + * Extended CPU features + */ + +struct leaf_0x7_0 { + // eax + u32 leaf7_n_subleaves : 32; // Number of leaf 0x7 subleaves + // ebx + u32 fsgsbase : 1, // FSBASE/GSBASE read/write + tsc_adjust : 1, // IA32_TSC_ADJUST MSR + sgx : 1, // Intel SGX (Software Guard Extensions) + bmi1 : 1, // Bit manipulation extensions group 1 + hle : 1, // Hardware Lock Elision + avx2 : 1, // AVX2 instruction set + fdp_excptn_only : 1, // FPU Data Pointer updated only on x87 exceptions + smep : 1, // Supervisor Mode Execution Protection + bmi2 : 1, // Bit manipulation extensions group 2 + erms : 1, // Enhanced REP MOVSB/STOSB + invpcid : 1, // INVPCID instruction (Invalidate Processor Context ID) + rtm : 1, // Intel restricted transactional memory + pqm : 1, // Intel RDT-CMT / AMD Platform-QoS cache monitoring + zero_fcs_fds : 1, // Deprecated FPU CS/DS (stored as zero) + mpx : 1, // Intel memory protection extensions + rdt_a : 1, // Intel RDT / AMD Platform-QoS Enforcement + avx512f : 1, // AVX-512 foundation instructions + avx512dq : 1, // AVX-512 double/quadword instructions + rdseed : 1, // RDSEED instruction + adx : 1, // ADCX/ADOX instructions + smap : 1, // Supervisor mode access prevention + avx512ifma : 1, // AVX-512 integer fused multiply add + : 1, // Reserved + clflushopt : 1, // CLFLUSHOPT instruction + clwb : 1, // CLWB instruction + intel_pt : 1, // Intel processor trace + avx512pf : 1, // AVX-512 prefetch instructions + avx512er : 1, // AVX-512 exponent/reciprocal instructions + avx512cd : 1, // AVX-512 conflict detection instructions + sha : 1, // SHA/SHA256 instructions + avx512bw : 1, // AVX-512 byte/word instructions + avx512vl : 1; // AVX-512 VL (128/256 vector length) extensions + // ecx + u32 prefetchwt1 : 1, // PREFETCHWT1 (Intel Xeon Phi only) + avx512vbmi : 1, // AVX-512 Vector byte manipulation instructions + umip : 1, // User mode instruction protection + pku : 1, // Protection keys for user-space + ospke : 1, // OS protection keys enable + waitpkg : 1, // WAITPKG instructions + avx512_vbmi2 : 1, // AVX-512 vector byte manipulation instructions group 2 + cet_ss : 1, // CET shadow stack features + gfni : 1, // Galois field new instructions + vaes : 1, // Vector AES instructions + vpclmulqdq : 1, // VPCLMULQDQ 256-bit instruction + avx512_vnni : 1, // Vector neural network instructions + avx512_bitalg : 1, // AVX-512 bitwise algorithms + tme : 1, // Intel total memory encryption + avx512_vpopcntdq : 1, // AVX-512: POPCNT for vectors of DWORD/QWORD + : 1, // Reserved + la57 : 1, // 57-bit linear addresses (five-level paging) + mawau_val_lm : 5, // BNDLDX/BNDSTX MAWAU value in 64-bit mode + rdpid : 1, // RDPID instruction + key_locker : 1, // Intel key locker + bus_lock_detect : 1, // OS bus-lock detection + cldemote : 1, // CLDEMOTE instruction + : 1, // Reserved + movdiri : 1, // MOVDIRI instruction + movdir64b : 1, // MOVDIR64B instruction + enqcmd : 1, // Enqueue stores (ENQCMD{,S}) + sgx_lc : 1, // Intel SGX launch configuration + pks : 1; // Protection keys for supervisor-mode pages + // edx + u32 : 1, // Reserved + sgx_keys : 1, // Intel SGX attestation services + avx512_4vnniw : 1, // AVX-512 neural network instructions + avx512_4fmaps : 1, // AVX-512 multiply accumulation single precision + fsrm : 1, // Fast short REP MOVSB + uintr : 1, // User interrupts + : 2, // Reserved + avx512_vp2intersect : 1, // VP2INTERSECT{D,Q} instructions + srbds_ctrl : 1, // SRBDS mitigation MSR + md_clear : 1, // VERW MD_CLEAR microcode + rtm_always_abort : 1, // XBEGIN (RTM transaction) always aborts + : 1, // Reserved + tsx_force_abort : 1, // MSR TSX_FORCE_ABORT, RTM_ABORT bit + serialize : 1, // SERIALIZE instruction + hybrid_cpu : 1, // The CPU is identified as a 'hybrid part' + tsxldtrk : 1, // TSX suspend/resume load address tracking + : 1, // Reserved + pconfig : 1, // PCONFIG instruction + arch_lbr : 1, // Intel architectural LBRs + cet_ibt : 1, // CET indirect branch tracking + : 1, // Reserved + amx_bf16 : 1, // AMX-BF16: tile bfloat16 + avx512_fp16 : 1, // AVX-512 FP16 instructions + amx_tile : 1, // AMX-TILE: tile architecture + amx_int8 : 1, // AMX-INT8: tile 8-bit integer + spec_ctrl : 1, // Speculation Control (IBRS/IBPB: indirect branch restrictions) + intel_stibp : 1, // Single thread indirect branch predictors + flush_l1d : 1, // FLUSH L1D cache: IA32_FLUSH_CMD MSR + arch_capabilities : 1, // Intel IA32_ARCH_CAPABILITIES MSR + core_capabilities : 1, // IA32_CORE_CAPABILITIES MSR + spec_ctrl_ssbd : 1; // Speculative store bypass disable +}; + +struct leaf_0x7_1 { + // eax + u32 : 4, // Reserved + avx_vnni : 1, // AVX-VNNI instructions + avx512_bf16 : 1, // AVX-512 bfloat16 instructions + lass : 1, // Linear address space separation + cmpccxadd : 1, // CMPccXADD instructions + arch_perfmon_ext : 1, // ArchPerfmonExt: leaf 0x23 + : 1, // Reserved + fzrm : 1, // Fast zero-length REP MOVSB + fsrs : 1, // Fast short REP STOSB + fsrc : 1, // Fast Short REP CMPSB/SCASB + : 4, // Reserved + fred : 1, // FRED: Flexible return and event delivery transitions + lkgs : 1, // LKGS: Load 'kernel' (userspace) GS + wrmsrns : 1, // WRMSRNS instruction (WRMSR-non-serializing) + nmi_src : 1, // NMI-source reporting with FRED event data + amx_fp16 : 1, // AMX-FP16: FP16 tile operations + hreset : 1, // HRESET (Thread director history reset) + avx_ifma : 1, // Integer fused multiply add + : 2, // Reserved + lam : 1, // Linear address masking + rd_wr_msrlist : 1, // RDMSRLIST/WRMSRLIST instructions + : 4; // Reserved + // ebx + u32 intel_ppin : 1, // Protected processor inventory number (PPIN{,_CTL} MSRs) + : 31; // Reserved + // ecx + u32 : 32; // Reserved + // edx + u32 : 4, // Reserved + avx_vnni_int8 : 1, // AVX-VNNI-INT8 instructions + avx_ne_convert : 1, // AVX-NE-CONVERT instructions + : 2, // Reserved + amx_complex : 1, // AMX-COMPLEX instructions (starting from Granite Rapids) + : 5, // Reserved + prefetchit_0_1 : 1, // PREFETCHIT0/1 instructions + : 3, // Reserved + cet_sss : 1, // CET supervisor shadow stacks safe to use + : 13; // Reserved +}; + +struct leaf_0x7_2 { + // eax + u32 : 32; // Reserved + // ebx + u32 : 32; // Reserved + // ecx + u32 : 32; // Reserved + // edx + u32 intel_psfd : 1, // Intel predictive store forward disable + ipred_ctrl : 1, // MSR bits IA32_SPEC_CTRL.IPRED_DIS_{U,S} + rrsba_ctrl : 1, // MSR bits IA32_SPEC_CTRL.RRSBA_DIS_{U,S} + ddp_ctrl : 1, // MSR bit IA32_SPEC_CTRL.DDPD_U + bhi_ctrl : 1, // MSR bit IA32_SPEC_CTRL.BHI_DIS_S + mcdt_no : 1, // MCDT mitigation not needed + uclock_disable : 1, // UC-lock disable + : 25; // Reserved +}; + +/* + * Leaf 0x9 + * Intel DCA (Direct Cache Access) + */ + +struct leaf_0x9_0 { + // eax + u32 dca_enabled_in_bios : 1, // DCA is enabled in BIOS + : 31; // Reserved + // ebx + u32 : 32; // Reserved + // ecx + u32 : 32; // Reserved + // edx + u32 : 32; // Reserved +}; + +/* + * Leaf 0xa + * Intel PMU (Performance Monitoring Unit) + */ + +struct leaf_0xa_0 { + // eax + u32 pmu_version : 8, // Performance monitoring unit version ID + num_counters_gp : 8, // Number of general-purpose PMU counters per logical CPU + bit_width_gp : 8, // Bitwidth of PMU general-purpose counters + events_mask_len : 8; // Length of CPUID(0xa).EBX bit vector + // ebx + u32 no_core_cycle : 1, // Core cycle event not available + no_instruction_retired : 1, // Instruction retired event not available + no_reference_cycles : 1, // Reference cycles event not available + no_llc_reference : 1, // LLC-reference event not available + no_llc_misses : 1, // LLC-misses event not available + no_br_insn_retired : 1, // Branch instruction retired event not available + no_br_misses_retired : 1, // Branch mispredict retired event not available + no_topdown_slots : 1, // Topdown slots event not available + no_backend_bound : 1, // Topdown backend bound not available + no_bad_speculation : 1, // Topdown bad speculation not available + no_frontend_bound : 1, // Topdown frontend bound not available + no_retiring : 1, // Topdown retiring not available + no_lbr_inserts : 1, // LBR inserts not available + : 19; // Reserved + // ecx + u32 pmu_fcounters_bitmap : 32; // Fixed-function PMU counters support bitmap + // edx + u32 num_counters_fixed : 5, // Number of fixed PMU counters + bitwidth_fixed : 8, // Bitwidth of PMU fixed counters + : 2, // Reserved + anythread_deprecation : 1, // AnyThread mode deprecation + : 16; // Reserved +}; + +/* + * Leaf 0xb + * CPU extended topology v1 + */ + +struct leaf_0xb_n { + // eax + u32 x2apic_id_shift : 5, // Bit width of this level (previous levels inclusive) + : 27; // Reserved + // ebx + u32 domain_lcpus_count : 16, // Logical CPUs count across all instances of this domain + : 16; // Reserved + // ecx + u32 domain_nr : 8, // This domain level (subleaf ID) + domain_type : 8, // This domain type + : 16; // Reserved + // edx + u32 x2apic_id : 32; // x2APIC ID of current logical CPU +}; + +#define LEAF_0xb_SUBLEAF_N_FIRST 0 +#define LEAF_0xb_SUBLEAF_N_LAST 1 + +/* + * Leaf 0xd + * CPU extended state + */ + +struct leaf_0xd_0 { + // eax + u32 xcr0_x87 : 1, // XCR0.X87 + xcr0_sse : 1, // XCR0.SSE + xcr0_avx : 1, // XCR0.AVX + xcr0_mpx_bndregs : 1, // XCR0.BNDREGS: MPX BND0-BND3 registers + xcr0_mpx_bndcsr : 1, // XCR0.BNDCSR: MPX BNDCFGU/BNDSTATUS registers + xcr0_avx512_opmask : 1, // XCR0.OPMASK: AVX-512 k0-k7 registers + xcr0_avx512_zmm_hi256 : 1, // XCR0.ZMM_Hi256: AVX-512 ZMM0->ZMM7/15 registers + xcr0_avx512_hi16_zmm : 1, // XCR0.HI16_ZMM: AVX-512 ZMM16->ZMM31 registers + : 1, // Reserved + xcr0_pkru : 1, // XCR0.PKRU: XSAVE PKRU registers + : 1, // Reserved + xcr0_cet_u : 1, // XCR0.CET_U: CET user state + xcr0_cet_s : 1, // XCR0.CET_S: CET supervisor state + : 4, // Reserved + xcr0_tileconfig : 1, // XCR0.TILECONFIG: AMX can manage TILECONFIG + xcr0_tiledata : 1, // XCR0.TILEDATA: AMX can manage TILEDATA + : 13; // Reserved + // ebx + u32 xsave_sz_xcr0 : 32; // XSAVE/XRSTOR area byte size, for XCR0 enabled features + // ecx + u32 xsave_sz_max : 32; // XSAVE/XRSTOR area max byte size, all CPU features + // edx + u32 : 30, // Reserved + xcr0_lwp : 1, // AMD XCR0.LWP: Light-weight Profiling + : 1; // Reserved +}; + +struct leaf_0xd_1 { + // eax + u32 xsaveopt : 1, // XSAVEOPT instruction + xsavec : 1, // XSAVEC instruction + xgetbv1 : 1, // XGETBV instruction with ECX = 1 + xsaves : 1, // XSAVES/XRSTORS instructions (and XSS MSR) + xfd : 1, // Extended feature disable + : 27; // Reserved + // ebx + u32 xsave_sz_xcr0_xss : 32; // XSAVES/XSAVEC area byte size, for XCR0|XSS enabled features + // ecx + u32 : 8, // Reserved + xss_pt : 1, // PT state + : 1, // Reserved + xss_pasid : 1, // PASID state + xss_cet_u : 1, // CET user state + xss_cet_s : 1, // CET supervisor state + xss_hdc : 1, // HDC state + xss_uintr : 1, // UINTR state + xss_lbr : 1, // LBR state + xss_hwp : 1, // HWP state + : 15; // Reserved + // edx + u32 : 32; // Reserved +}; + +struct leaf_0xd_n { + // eax + u32 xsave_sz : 32; // Subleaf-N feature save area size, in bytes + // ebx + u32 xsave_offset : 32; // Subleaf-N feature save area offset, in bytes + // ecx + u32 is_xss_bit : 1, // Subleaf N describes an XSS bit (otherwise XCR0) + compacted_xsave_64byte_aligned : 1, // When compacted, subleaf-N XSAVE area is 64-byte aligned + : 30; // Reserved + // edx + u32 : 32; // Reserved +}; + +#define LEAF_0xd_SUBLEAF_N_FIRST 2 +#define LEAF_0xd_SUBLEAF_N_LAST 63 + +/* + * Leaf 0xf + * Intel RDT / AMD PQoS resource monitoring + */ + +struct leaf_0xf_0 { + // eax + u32 : 32; // Reserved + // ebx + u32 core_rmid_max : 32; // RMID max within this core (0-based) + // ecx + u32 : 32; // Reserved + // edx + u32 : 1, // Reserved + llc_qos_mon : 1, // LLC QoS-monitoring + : 30; // Reserved +}; + +struct leaf_0xf_1 { + // eax + u32 l3c_qm_bitwidth : 8, // L3 QoS-monitoring counter bitwidth (24-based) + l3c_qm_overflow_bit : 1, // QM_CTR MSR bit 61 is an overflow bit + io_rdt_cmt : 1, // non-CPU agent supporting Intel RDT CMT present + io_rdt_mbm : 1, // non-CPU agent supporting Intel RDT MBM present + : 21; // Reserved + // ebx + u32 l3c_qm_conver_factor : 32; // QM_CTR MSR conversion factor to bytes + // ecx + u32 l3c_qm_rmid_max : 32; // L3 QoS-monitoring max RMID + // edx + u32 l3c_qm_occupancy : 1, // L3 QoS occupancy monitoring + l3c_qm_mbm_total : 1, // L3 QoS total bandwidth monitoring + l3c_qm_mbm_local : 1, // L3 QoS local bandwidth monitoring + : 29; // Reserved +}; + +/* + * Leaf 0x10 + * Intel RDT / AMD PQoS allocation + */ + +struct leaf_0x10_0 { + // eax + u32 : 32; // Reserved + // ebx + u32 : 1, // Reserved + cat_l3 : 1, // L3 Cache Allocation Technology + cat_l2 : 1, // L2 Cache Allocation Technology + mba : 1, // Memory Bandwidth Allocation + : 28; // Reserved + // ecx + u32 : 32; // Reserved + // edx + u32 : 32; // Reserved +}; + +struct leaf_0x10_n { + // eax + u32 cat_cbm_len : 5, // L3/L2_CAT capacity bitmask length, minus-one notation + : 27; // Reserved + // ebx + u32 cat_units_bitmap : 32; // L3/L2_CAT allocation units bitmap + // ecx + u32 : 1, // Reserved + l3_cat_cos_infreq_updates : 1, // L3_CAT COS updates should be infrequent + cat_cdp_supported : 1, // L3/L2_CAT Code and Data Prioritization + cat_sparse_1s : 1, // L3/L2_CAT non-contiguous 1s value + : 28; // Reserved + // edx + u32 cat_cos_max : 16, // L3/L2_CAT max Class of Service + : 16; // Reserved +}; + +#define LEAF_0x10_SUBLEAF_N_FIRST 1 +#define LEAF_0x10_SUBLEAF_N_LAST 2 + +struct leaf_0x10_3 { + // eax + u32 mba_max_delay : 12, // Max MBA throttling value; minus-one notation + : 20; // Reserved + // ebx + u32 : 32; // Reserved + // ecx + u32 mba_per_thread : 1, // Per-thread MBA controls + : 1, // Reserved + mba_delay_linear : 1, // Delay values are linear + : 29; // Reserved + // edx + u32 mba_cos_max : 16, // MBA max Class of Service + : 16; // Reserved +}; + +/* + * Leaf 0x12 + * Intel SGX (Software Guard Extensions) + */ + +struct leaf_0x12_0 { + // eax + u32 sgx1 : 1, // SGX1 leaf functions + sgx2 : 1, // SGX2 leaf functions + : 3, // Reserved + enclv_leaves : 1, // ENCLV leaves + encls_leaves : 1, // ENCLS leaves + enclu_everifyreport2 : 1, // ENCLU leaf EVERIFYREPORT2 + : 2, // Reserved + encls_eupdatesvn : 1, // ENCLS leaf EUPDATESVN + enclu_edeccssa : 1, // ENCLU leaf EDECCSSA + : 20; // Reserved + // ebx + u32 miscselect_exinfo : 1, // SSA.MISC frame: Enclave #PF and #GP reporting + miscselect_cpinfo : 1, // SSA.MISC frame: Enclave #CP reporting + : 30; // Reserved + // ecx + u32 : 32; // Reserved + // edx + u32 max_enclave_sz_not64 : 8, // Maximum enclave size in non-64-bit mode (log2) + max_enclave_sz_64 : 8, // Maximum enclave size in 64-bit mode (log2) + : 16; // Reserved +}; + +struct leaf_0x12_1 { + // eax + u32 secs_attr_init : 1, // Enclave initialized by EINIT + secs_attr_debug : 1, // Enclave permits debugger read/write + secs_attr_mode64bit : 1, // Enclave runs in 64-bit mode + : 1, // Reserved + secs_attr_provisionkey : 1, // Provisioning key + secs_attr_einittoken_key : 1, // EINIT token key + secs_attr_cet : 1, // CET attributes + secs_attr_kss : 1, // Key Separation and Sharing + : 2, // Reserved + secs_attr_aexnotify : 1, // Enclave threads: AEX notifications + : 21; // Reserved + // ebx + u32 : 32; // Reserved + // ecx + u32 xfrm_x87 : 1, // Enclave XFRM.X87 + xfrm_sse : 1, // Enclave XFRM.SSE + xfrm_avx : 1, // Enclave XFRM.AVX + xfrm_mpx_bndregs : 1, // Enclave XFRM.BNDREGS (MPX BND0-BND3 registers) + xfrm_mpx_bndcsr : 1, // Enclave XFRM.BNDCSR (MPX BNDCFGU/BNDSTATUS registers) + xfrm_avx512_opmask : 1, // Enclave XFRM.OPMASK (AVX-512 k0-k7 registers) + xfrm_avx512_zmm_hi256 : 1, // Enclave XFRM.ZMM_Hi256 (AVX-512 ZMM0->ZMM7/15 registers) + xfrm_avx512_hi16_zmm : 1, // Enclave XFRM.HI16_ZMM (AVX-512 ZMM16->ZMM31 registers) + : 1, // Reserved + xfrm_pkru : 1, // Enclave XFRM.PKRU (XSAVE PKRU registers) + : 7, // Reserved + xfrm_tileconfig : 1, // Enclave XFRM.TILECONFIG (AMX can manage TILECONFIG) + xfrm_tiledata : 1, // Enclave XFRM.TILEDATA (AMX can manage TILEDATA) + : 13; // Reserved + // edx + u32 : 32; // Reserved +}; + +struct leaf_0x12_n { + // eax + u32 subleaf_type : 4, // Subleaf type + : 8, // Reserved + epc_sec_base_addr_0 : 20; // EPC section base address, bits[12:31] + // ebx + u32 epc_sec_base_addr_1 : 20, // EPC section base address, bits[32:51] + : 12; // Reserved + // ecx + u32 epc_sec_type : 4, // EPC section type / property encoding + : 8, // Reserved + epc_sec_size_0 : 20; // EPC section size, bits[12:31] + // edx + u32 epc_sec_size_1 : 20, // EPC section size, bits[32:51] + : 12; // Reserved +}; + +#define LEAF_0x12_SUBLEAF_N_FIRST 2 +#define LEAF_0x12_SUBLEAF_N_LAST 31 + +/* + * Leaf 0x14 + * Intel Processor Trace + */ + +struct leaf_0x14_0 { + // eax + u32 pt_max_subleaf : 32; // Maximum leaf 0x14 subleaf + // ebx + u32 cr3_filtering : 1, // IA32_RTIT_CR3_MATCH is accessible + psb_cyc : 1, // Configurable PSB and cycle-accurate mode + ip_filtering : 1, // IP/TraceStop filtering; Warm-reset PT MSRs preservation + mtc_timing : 1, // MTC timing packet; COFI-based packets suppression + ptwrite : 1, // PTWRITE instruction + power_event_trace : 1, // Power Event Trace + psb_pmi_preserve : 1, // PSB and PMI preservation + event_trace : 1, // Event Trace packet generation + tnt_disable : 1, // TNT packet generation disable + : 23; // Reserved + // ecx + u32 topa_output : 1, // ToPA output scheme + topa_multiple_entries : 1, // ToPA tables can hold multiple entries + single_range_output : 1, // Single-range output + trace_transport_output : 1, // Trace Transport subsystem output + : 27, // Reserved + ip_payloads_lip : 1; // IP payloads have LIP values (CS base included) + // edx + u32 : 32; // Reserved +}; + +struct leaf_0x14_1 { + // eax + u32 num_address_ranges : 3, // Number of configurable address ranges + : 13, // Reserved + mtc_periods_bmp : 16; // MTC period encodings bitmap + // ebx + u32 cycle_thresholds_bmp : 16, // Cycle Threshold encodings bitmap + psb_periods_bmp : 16; // Configurable PSB frequency encodings bitmap + // ecx + u32 : 32; // Reserved + // edx + u32 : 32; // Reserved +}; + +/* + * Leaf 0x15 + * Intel TSC (Time Stamp Counter) + */ + +struct leaf_0x15_0 { + // eax + u32 tsc_denominator : 32; // Denominator of the TSC/'core crystal clock' ratio + // ebx + u32 tsc_numerator : 32; // Numerator of the TSC/'core crystal clock' ratio + // ecx + u32 cpu_crystal_hz : 32; // Core crystal clock nominal frequency, in Hz + // edx + u32 : 32; // Reserved +}; + +/* + * Leaf 0x16 + * Intel processor frequency + */ + +struct leaf_0x16_0 { + // eax + u32 cpu_base_mhz : 16, // Processor base frequency, in MHz + : 16; // Reserved + // ebx + u32 cpu_max_mhz : 16, // Processor max frequency, in MHz + : 16; // Reserved + // ecx + u32 bus_mhz : 16, // Bus reference frequency, in MHz + : 16; // Reserved + // edx + u32 : 32; // Reserved +}; + +/* + * Leaf 0x17 + * Intel SoC vendor attributes + */ + +struct leaf_0x17_0 { + // eax + u32 soc_max_subleaf : 32; // Maximum leaf 0x17 subleaf + // ebx + u32 soc_vendor_id : 16, // SoC vendor ID + is_vendor_scheme : 1, // Assigned by industry enumeration scheme (not Intel) + : 15; // Reserved + // ecx + u32 soc_proj_id : 32; // SoC project ID, assigned by vendor + // edx + u32 soc_stepping_id : 32; // SoC project stepping ID, assigned by vendor +}; + +struct leaf_0x17_n { + // eax + u32 vendor_brand_a : 32; // Vendor Brand ID string, bytes subleaf_nr * (0 -> 3) + // ebx + u32 vendor_brand_b : 32; // Vendor Brand ID string, bytes subleaf_nr * (4 -> 7) + // ecx + u32 vendor_brand_c : 32; // Vendor Brand ID string, bytes subleaf_nr * (8 -> 11) + // edx + u32 vendor_brand_d : 32; // Vendor Brand ID string, bytes subleaf_nr * (12 -> 15) +}; + +#define LEAF_0x17_SUBLEAF_N_FIRST 1 +#define LEAF_0x17_SUBLEAF_N_LAST 3 + +/* + * Leaf 0x18 + * Intel deterministic address translation (TLB) parameters + */ + +struct leaf_0x18_n { + // eax + u32 tlb_max_subleaf : 32; // Maximum leaf 0x18 subleaf + // ebx + u32 tlb_4k_page : 1, // TLB supports 4KB-page entries + tlb_2m_page : 1, // TLB supports 2MB-page entries + tlb_4m_page : 1, // TLB supports 4MB-page entries + tlb_1g_page : 1, // TLB supports 1GB-page entries + : 4, // Reserved + hard_partitioning : 3, // Partitioning between logical CPUs + : 5, // Reserved + n_way_associative : 16; // Ways of associativity + // ecx + u32 n_sets : 32; // Number of sets + // edx + u32 tlb_type : 5, // Translation cache type (TLB type) + tlb_cache_level : 3, // Translation cache level (1-based) + is_fully_associative : 1, // Fully-associative + : 5, // Reserved + tlb_max_addressable_ids : 12, // Max number of addressable IDs - 1 + : 6; // Reserved +}; + +#define LEAF_0x18_SUBLEAF_N_FIRST 0 +#define LEAF_0x18_SUBLEAF_N_LAST 31 + +/* + * Leaf 0x19 + * Intel key locker + */ + +struct leaf_0x19_0 { + // eax + u32 kl_cpl0_only : 1, // CPL0-only key locker restriction + kl_no_encrypt : 1, // No-encrypt key locker restriction + kl_no_decrypt : 1, // No-decrypt key locker restriction + : 29; // Reserved + // ebx + u32 aes_keylocker : 1, // AES key locker instructions + : 1, // Reserved + aes_keylocker_wide : 1, // AES wide key locker instructions + : 1, // Reserved + kl_msr_iwkey : 1, // Key locker MSRs and IWKEY backups + : 27; // Reserved + // ecx + u32 loadiwkey_no_backup : 1, // LOADIWKEY NoBackup parameter + iwkey_rand : 1, // IWKEY randomization + : 30; // Reserved + // edx + u32 : 32; // Reserved +}; + +/* + * Leaf 0x1a + * Intel hybrid CPUs identification (e.g. Atom, Core) + */ + +struct leaf_0x1a_0 { + // eax + u32 core_native_model : 24, // This core's native model ID + core_type : 8; // This core's type + // ebx + u32 : 32; // Reserved + // ecx + u32 : 32; // Reserved + // edx + u32 : 32; // Reserved +}; + +/* + * Leaf 0x1b + * Intel PCONFIG (Platform configuration) + */ + +struct leaf_0x1b_n { + // eax + u32 pconfig_subleaf_type : 12, // CPUID 0x1b subleaf type + : 20; // Reserved + // ebx + u32 pconfig_target_id_x : 32; // A supported PCONFIG target ID + // ecx + u32 pconfig_target_id_y : 32; // A supported PCONFIG target ID + // edx + u32 pconfig_target_id_z : 32; // A supported PCONFIG target ID +}; + +#define LEAF_0x1b_SUBLEAF_N_FIRST 0 +#define LEAF_0x1b_SUBLEAF_N_LAST 31 + +/* + * Leaf 0x1c + * Intel LBR (Last Branch Record) + */ + +struct leaf_0x1c_0 { + // eax + u32 lbr_depth_mask : 8, // Max LBR stack depth bitmask + : 22, // Reserved + lbr_deep_c_reset : 1, // LBRs may be cleared on MWAIT C-state > C1 + lbr_ip_is_lip : 1; // LBR IP contain Last IP (otherwise effective IP) + // ebx + u32 lbr_cpl : 1, // CPL filtering + lbr_branch_filter : 1, // Branch filtering + lbr_call_stack : 1, // Call-stack mode + : 29; // Reserved + // ecx + u32 lbr_mispredict : 1, // Branch misprediction bit + lbr_timed_lbr : 1, // Timed LBRs (CPU cycles since last LBR entry) + lbr_branch_type : 1, // Branch type field + : 13, // Reserved + lbr_events_gpc_bmp : 4, // PMU-events logging support + : 12; // Reserved + // edx + u32 : 32; // Reserved +}; + +/* + * Leaf 0x1d + * Intel AMX (Advanced Matrix Extensions) tile information + */ + +struct leaf_0x1d_0 { + // eax + u32 amx_max_palette : 32; // Highest palette ID / subleaf ID + // ebx + u32 : 32; // Reserved + // ecx + u32 : 32; // Reserved + // edx + u32 : 32; // Reserved +}; + +struct leaf_0x1d_1 { + // eax + u32 amx_palette_size : 16, // AMX palette total tiles size, in bytes + amx_tile_size : 16; // AMX single tile's size, in bytes + // ebx + u32 amx_tile_row_size : 16, // AMX tile single row's size, in bytes + amx_palette_nr_tiles : 16; // AMX palette number of tiles + // ecx + u32 amx_tile_nr_rows : 16, // AMX tile max number of rows + : 16; // Reserved + // edx + u32 : 32; // Reserved +}; + +/* + * Leaf 0x1e + * Intel TMUL (Tile-matrix Multiply) + */ + +struct leaf_0x1e_0 { + // eax + u32 : 32; // Reserved + // ebx + u32 tmul_maxk : 8, // TMUL unit maximum height, K (rows or columns) + tmul_maxn : 16, // TMUL unit maximum SIMD dimension, N (column bytes) + : 8; // Reserved + // ecx + u32 : 32; // Reserved + // edx + u32 : 32; // Reserved +}; + +/* + * Leaf 0x1f + * Intel extended topology v2 + */ + +struct leaf_0x1f_n { + // eax + u32 x2apic_id_shift : 5, // Bit width of this level (previous levels inclusive) + : 27; // Reserved + // ebx + u32 domain_lcpus_count : 16, // Logical CPUs count across all instances of this domain + : 16; // Reserved + // ecx + u32 domain_level : 8, // This domain level (subleaf ID) + domain_type : 8, // This domain type + : 16; // Reserved + // edx + u32 x2apic_id : 32; // x2APIC ID of current logical CPU +}; + +#define LEAF_0x1f_SUBLEAF_N_FIRST 0 +#define LEAF_0x1f_SUBLEAF_N_LAST 5 + +/* + * Leaf 0x20 + * Intel HRESET (History Reset) + */ + +struct leaf_0x20_0 { + // eax + u32 hreset_nr_subleaves : 32; // CPUID 0x20 max subleaf + 1 + // ebx + u32 hreset_thread_director : 1, // Intel thread director HRESET + : 31; // Reserved + // ecx + u32 : 32; // Reserved + // edx + u32 : 32; // Reserved +}; + +/* + * Leaf 0x21 + * Intel TD (Trust Domain) + */ + +struct leaf_0x21_0 { + // eax + u32 : 32; // Reserved + // ebx + u32 tdx_vendorid_0 : 32; // TDX vendor ID string bytes 0 - 3 + // ecx + u32 tdx_vendorid_2 : 32; // TDX vendor ID string bytes 8 - 11 + // edx + u32 tdx_vendorid_1 : 32; // TDX vendor ID string bytes 4 - 7 +}; + +/* + * Leaf 0x23 + * Intel Architectural Performance Monitoring Extended (ArchPerfmonExt) + */ + +struct leaf_0x23_0 { + // eax + u32 subleaf_0 : 1, // Subleaf 0, this subleaf + counters_subleaf : 1, // Subleaf 1, PMU counter bitmaps + acr_subleaf : 1, // Subleaf 2, Auto Counter Reload bitmaps + events_subleaf : 1, // Subleaf 3, PMU event bitmaps + pebs_caps_subleaf : 1, // Subleaf 4, PEBS capabilities + pebs_subleaf : 1, // Subleaf 5, Arch PEBS bitmaps + : 26; // Reserved + // ebx + u32 unitmask2 : 1, // IA32_PERFEVTSELx MSRs UnitMask2 bit + eq : 1, // IA32_PERFEVTSELx MSRs EQ bit + rdpmc_user_disable : 1, // RDPMC userspace disable + : 29; // Reserved + // ecx + u32 : 32; // Reserved + // edx + u32 : 32; // Reserved +}; + +struct leaf_0x23_1 { + // eax + u32 gp_counters : 32; // Bitmap of general-purpose PMU counters + // ebx + u32 fixed_counters : 32; // Bitmap of fixed PMU counters + // ecx + u32 : 32; // Reserved + // edx + u32 : 32; // Reserved +}; + +struct leaf_0x23_2 { + // eax + u32 acr_gp_reload : 32; // Bitmap of general-purpose counters that can be reloaded + // ebx + u32 acr_fixed_reload : 32; // Bitmap of fixed counters that can be reloaded + // ecx + u32 acr_gp_trigger : 32; // Bitmap of general-purpose counters that can trigger reloads + // edx + u32 acr_fixed_trigger : 32; // Bitmap of fixed counters that can trigger reloads +}; + +struct leaf_0x23_3 { + // eax + u32 core_cycles_evt : 1, // Core cycles event + insn_retired_evt : 1, // Instructions retired event + ref_cycles_evt : 1, // Reference cycles event + llc_refs_evt : 1, // Last-level cache references event + llc_misses_evt : 1, // Last-level cache misses event + br_insn_ret_evt : 1, // Branch instruction retired event + br_mispr_evt : 1, // Branch mispredict retired event + td_slots_evt : 1, // Topdown slots event + td_backend_bound_evt : 1, // Topdown backend bound event + td_bad_spec_evt : 1, // Topdown bad speculation event + td_frontend_bound_evt : 1, // Topdown frontend bound event + td_retiring_evt : 1, // Topdown retiring event + : 20; // Reserved + // ebx + u32 : 32; // Reserved + // ecx + u32 : 32; // Reserved + // edx + u32 : 32; // Reserved +}; + +struct leaf_0x23_4 { + // eax + u32 : 32; // Reserved + // ebx + u32 : 3, // Reserved + allow_in_record : 1, // ALLOW_IN_RECORD bit in MSRs + counters_gp : 1, // Counters group sub-group general-purpose counters + counters_fixed : 1, // Counters group sub-group fixed-function counters + counters_metrics : 1, // Counters group sub-group performance metrics + : 1, // Reserved + lbr : 2, // LBR group + : 6, // Reserved + xer : 8, // XER group + : 5, // Reserved + gpr : 1, // GPR group + aux : 1, // AUX group + : 1; // Reserved + // ecx + u32 : 32; // Reserved + // edx + u32 : 32; // Reserved +}; + +struct leaf_0x23_5 { + // eax + u32 pebs_gp : 32; // Architectural PEBS general-purpose counters + // ebx + u32 pebs_pdist_gp : 32; // Architectural PEBS PDIST general-purpose counters + // ecx + u32 pebs_fixed : 32; // Architectural PEBS fixed counters + // edx + u32 pebs_pdist_fixed : 32; // Architectural PEBS PDIST fixed counters +}; + +/* + * Leaf 0x40000000 + * Maximum hypervisor leaf + hypervisor vendor string + */ + +struct leaf_0x40000000_0 { + // eax + u32 max_hyp_leaf : 32; // Maximum hypervisor leaf + // ebx + u32 hypervisor_id_0 : 32; // Hypervisor ID string bytes 0 - 3 + // ecx + u32 hypervisor_id_1 : 32; // Hypervisor ID string bytes 4 - 7 + // edx + u32 hypervisor_id_2 : 32; // Hypervisor ID string bytes 8 - 11 +}; + +/* + * Leaf 0x4c780001 + * Linux-defined synthetic feature flags + */ + +struct leaf_0x4c780001_0 { + // eax + u32 cxmmx : 1, // Cyrix MMX extensions + k6_mtrr : 1, // AMD K6 nonstandard MTRRs + cyrix_arr : 1, // Cyrix ARRs (= MTRRs) + centaur_mcr : 1, // Centaur MCRs (= MTRRs) + k8 : 1, // Opteron, Athlon64 + zen5 : 1, // CPU based on Zen5 micro-architecture + zen6 : 1, // CPU based on Zen6 micro-architecture + : 1, // Reserved + constant_tsc : 1, // TSC ticks at a constant rate + up : 1, // SMP kernel running on UP + art : 1, // Always running timer (ART) + arch_perfmon : 1, // Intel Architectural PerfMon + pebs : 1, // Precise-Event Based Sampling + bts : 1, // Branch Trace Store + syscall32 : 1, // SYSCALL in IA32 userspace + sysenter32 : 1, // SYSENTER in IA32 userspace + rep_good : 1, // REP microcode works well + amd_lbr_v2 : 1, // AMD Last Branch Record Extension version 2 + clear_cpu_buf : 1, // Clear CPU buffers using VERW + acc_power : 1, // AMD Accumulated Power Mechanism + nopl : 1, // The NOPL instructions + always : 1, // Always-present feature + xtopology : 1, // CPU topology enumeration extensions + tsc_reliable : 1, // TSC is known to be reliable + nonstop_tsc : 1, // TSC does not stop in C states + cpuid : 1, // CPU has the CPUID instruction + extd_apicid : 1, // Extended APIC ID (8 bits) + amd_dcm : 1, // AMD multi-node processor + aperfmperf : 1, // APERF/MPERF MSRs: P-State hardware coordination feedback + rapl : 1, // AMD/Hygon RAPL interface + nonstop_tsc_s3 : 1, // TSC does not stop in S3 state + tsc_known_freq : 1; // TSC has known frequency + // ebx + u32 ring3mwait : 1, // Ring 3 MONITOR/MWAIT instructions + cpuid_fault : 1, // Intel CPUID faulting + cpb : 1, // AMD Core Performance Boost + epb : 1, // IA32_ENERGY_PERF_BIAS support + cat_l3 : 1, // Cache Allocation Technology L3 + cat_l2 : 1, // Cache Allocation Technology L2 + cdp_l3 : 1, // Code and Data Prioritization L3 + tdx_host_platform : 1, // Platform supports being a TDX host + hw_pstate : 1, // AMD Hardware P-state control + proc_feedback : 1, // AMD Processor Feedback Interface + xcompacted : 1, // Use compacted XSTATE (XSAVES or XSAVEC) + pti : 1, // Kernel Page Table Isolation enabled + kernel_ibrs : 1, // Set/clear IBRS on kernel entry/exit + rsb_vmexit : 1, // Fill RSB on VM-Exit + intel_ppin : 1, // Intel Processor Inventory Number + cdp_l2 : 1, // Code and Data Prioritization L2 + msr_spec_ctrl : 1, // MSR SPEC_CTRL is implemented + ssbd : 1, // Speculative Store Bypass Disable + mba : 1, // Memory Bandwidth Allocation + rsb_ctxsw : 1, // Fill RSB on context switches + perfmon_v2 : 1, // AMD Performance Monitoring Version 2 + : 1, // Reserved + use_ibrs_fw : 1, // Use IBRS during runtime firmware calls + ss_bypass_disable : 1, // Disable Speculative Store Bypass + ls_cfg_ssbd : 1, // AMD SSBD implementation via LS_CFG MSR + ibrs : 1, // Indirect Branch Restricted Speculation + ibpb : 1, // Indirect Branch Prediction Barrier (without RSB flush guarantee) + stibp : 1, // Single Thread Indirect Branch Predictors + zen : 1, // Generic flag for all Zen and newer + l1tf_pteinv : 1, // L1TF workaround PTE inversion + ibrs_enhanced : 1, // Enhanced IBRS + msr_ia32_feat_ctl : 1; // MSR IA32_FEAT_CTL configured + // ecx + u32 tpr_shadow : 1, // Intel TPR Shadow + flexpriority : 1, // Intel FlexPriority + ept : 1, // Intel Extended Page Table + vpid : 1, // Intel Virtual Processor ID + coherency_sfw_no : 1, // SNP cache coherency software workaround not needed + : 10, // Reserved + vmmcall : 1, // Prefer VMMCALL to VMCALL + xenpv : 1, // Xen paravirtual guest + ept_ad : 1, // Intel Extended Page Table access-dirty bit + vmcall : 1, // Hypervisor supports the VMCALL instruction + vmw_vmmcall : 1, // VMware prefers the VMMCALL instruction + pvunlock : 1, // PV unlock function + vcpupreempt : 1, // PV vcpu_is_preempted function + tdx_guest : 1, // Intel Trust Domain Extensions Guest + : 9; // Reserved + // edx + u32 cqm_llc : 1, // LLC QoS + cqm_occup_llc : 1, // LLC occupancy monitoring + cqm_mbm_total : 1, // LLC Total MBM monitoring + cqm_mbm_local : 1, // LLC Local MBM monitoring + fence_swapgs_user : 1, // LFENCE in user entry SWAPGS path + fence_swapgs_kernel : 1, // LFENCE in kernel entry SWAPGS path + split_lock_detect : 1, // #AC for split lock + per_thread_mba : 1, // Per-thread Memory Bandwidth Allocation + sgx1 : 1, // SGX Basic + sgx2 : 1, // SGX Enclave Dynamic Memory Management (EDMM) + entry_ibpb : 1, // Issue an IBPB on kernel entry + rrsba_ctrl : 1, // RET prediction control + retpoline : 1, // Generic Retpoline mitigation for Spectre variant 2 + retpoline_lfence : 1, // Use LFENCE for Spectre variant 2 + rethunk : 1, // Use Return THUNK + unret : 1, // AMD BTB untrain return + use_ibpb_fw : 1, // Use IBPB during runtime firmware calls + rsb_vmexit_lite : 1, // Fill RSB on VM exit when EIBRS is enabled + sgx_edeccssa : 1, // SGX EDECCSSA user leaf function + call_depth : 1, // Call depth tracking for RSB stuffing + msr_tsx_ctrl : 1, // MSR IA32_TSX_CTRL (Intel) implemented + smba : 1, // Slow Memory Bandwidth Allocation + bmec : 1, // Bandwidth Monitoring Event Configuration + user_shstk : 1, // Shadow stack support for user mode applications + srso : 1, // AMD BTB untrain RETs + srso_alias : 1, // AMD BTB untrain RETs through aliasing + ibpb_on_vmexit : 1, // Issue an IBPB only on VMEXIT + apic_msrs_fence : 1, // IA32_TSC_DEADLINE and X2APIC MSRs need fencing + zen2 : 1, // CPU based on Zen2 microarchitecture + zen3 : 1, // CPU based on Zen3 microarchitecture + zen4 : 1, // CPU based on Zen4 microarchitecture + zen1 : 1; // CPU based on Zen1 microarchitecture +}; + +struct leaf_0x4c780001_1 { + // eax + u32 overflow_recov : 1, // MCA overflow recovery support + succor : 1, // Uncorrectable error containment and recovery + : 1, // Reserved + smca : 1, // Scalable MCA + : 28; // Reserved + // ebx + u32 amd_lbr_pmc_freeze : 1, // AMD LBR and PMC Freeze + clear_bhb_loop : 1, // Clear branch history at SYSCALL entry using SW loop + bhi_ctrl : 1, // BHI_DIS_S HW control available + clear_bhb_hw : 1, // BHI_DIS_S HW control enabled + clear_bhb_vmexit : 1, // Clear branch history at VMEXIT using SW loop + amd_fast_cppc : 1, // AMD fast Collaborative Processor Performance Control + amd_htr_cores : 1, // Heterogeneous Core Topology + amd_workload_class : 1, // Workload Classification + prefer_ymm : 1, // Avoid ZMM registers due to downclocking + apx : 1, // Advanced Performance Extensions + indirect_thunk_its : 1, // Use thunk for indirect branches in lower half of cache line + tsa_sq_no : 1, // AMD CPU not vulnerable to TSA-SQ + tsa_l1_no : 1, // AMD CPU not vulnerable to TSA-L1 + clear_cpu_buf_vm : 1, // Clear CPU buffers using VERW before VMRUN + ibpb_exit_to_user : 1, // Use IBPB on exit-to-userspace, see VMSCAPE bug + : 17; // Reserved + // ecx + u32 : 32; // Reserved + // edx + u32 : 32; // Reserved +}; + +/* + * Leaf 0x4c780002 + * Linux-defined synthetic CPU bug flags + */ + +struct leaf_0x4c780002_0 { + // eax + u32 f00f : 1, // Intel F00F + fdiv : 1, // FPU FDIV + coma : 1, // Cyrix 6x86 coma + amd_tlb_mmatch : 1, // AMD Erratum 383 + amd_apic_c1e : 1, // AMD Erratum 400 + bug_11ap : 1, // Bad local APIC aka 11AP + fxsave_leak : 1, // FXSAVE leaks FOP/FIP/FOP + clflush_monitor : 1, // AAI65, CLFLUSH required before MONITOR + sysret_ss_attrs : 1, // SYSRET does not fix up SS attributes + espfix : 1, // IRET to 16-bit SS corrupts ESP/RSP high bits (x86-32) + null_seg : 1, // Setting a selector to NULL preserves the base + swapgs_fence : 1, // SWAPGS without input dep on GS + monitor : 1, // IPI required to wake up remote CPU + amd_e400 : 1, // CPU is among the affected by Erratum 400 + cpu_meltdown : 1, // CPU affected by meltdown; needs kernel page table isolation + spectre_v1 : 1, // CPU affected by Spectre variant 1 with conditional branches + spectre_v2 : 1, // CPU affected by Spectre variant 2 with indirect branches + spec_store_bypass : 1, // CPU affected by speculative store bypass attack + l1tf : 1, // CPU affected by L1 Terminal Fault + mds : 1, // CPU affected by Microarchitectural data sampling + msbds_only : 1, // Microarchitectural data sampling: CPU only affected by the MSBDS variant + swapgs : 1, // CPU affected by speculation through SWAPGS + taa : 1, // CPU is affected by TSX Async Abort (TAA) + itlb_multihit : 1, // CPU may incur MCE during certain page attribute changes + srbds : 1, // CPU may leak RNG bits if not mitigated + mmio_stale_data : 1, // CPU affected by Processor MMIO Stale Data vulnerabilities + : 1, // Reserved + retbleed : 1, // CPU affected by Retbleed + eibrs_pbrsb : 1, // EIBRS is vulnerable to Post Barrier RSB Predictions + smt_rsb : 1, // CPU vulnerable to Cross-Thread Return Address Predictions + gds : 1, // CPU affected by Gather Data Sampling + tdx_pw_mce : 1; // CPU may incur #MC if non-TD software does partial write to TDX private memory + // ebx + u32 srso : 1, // AMD SRSO bug + div0 : 1, // AMD DIV0 speculation bug + rfds : 1, // CPU vulnerable to Register File Data Sampling + bhi : 1, // CPU affected by Branch History Injection + ibpb_no_ret : 1, // IBPB omits return target predictions + spectre_v2_user : 1, // CPU affected by Spectre variant 2 between user processes + old_microcode : 1, // CPU has old microcode; it must be vulnerable to something + its : 1, // CPU affected by Indirect Target Selection + its_native_only : 1, // CPU affected by ITS; VMX is not affected + tsa : 1, // CPU affected by Transient Scheduler Attacks + vmscape : 1, // CPU affected by VMSCAPE attacks from guests + : 21; // Reserved + // ecx + u32 : 32; // Reserved + // edx + u32 : 32; // Reserved +}; + +/* + * Leaf 0x80000000 + * Maximum extended leaf + CPU vendor string + */ + +struct leaf_0x80000000_0 { + // eax + u32 max_ext_leaf : 32; // Maximum extended CPUID leaf + // ebx + u32 cpu_vendorid_0 : 32; // Vendor ID string bytes 0 - 3 + // ecx + u32 cpu_vendorid_2 : 32; // Vendor ID string bytes 8 - 11 + // edx + u32 cpu_vendorid_1 : 32; // Vendor ID string bytes 4 - 7 +}; + +/* + * Leaf 0x80000001 + * Extended CPU features + */ + +struct leaf_0x80000001_0 { + // eax + u32 e_stepping_id : 4, // Stepping ID + e_base_model : 4, // Base processor model + e_base_family : 4, // Base processor family + e_base_type : 2, // Base processor type (Transmeta) + : 2, // Reserved + e_ext_model : 4, // Extended processor model + e_ext_family : 8, // Extended processor family + : 4; // Reserved + // ebx + u32 brand_id : 16, // Brand ID + : 12, // Reserved + pkg_type : 4; // Package type + // ecx + u32 lahf_lm : 1, // LAHF and SAHF in 64-bit mode + cmp_legacy : 1, // Multi-processing legacy mode (No HT) + svm : 1, // Secure Virtual Machine + extapic : 1, // Extended APIC space + cr8_legacy : 1, // LOCK MOV CR0 means MOV CR8 + lzcnt_abm : 1, // LZCNT advanced bit manipulation + sse4a : 1, // SSE4A support + misaligned_sse : 1, // Misaligned SSE mode + _3dnow_prefetch : 1, // 3DNow PREFETCH/PREFETCHW support + osvw : 1, // OS visible workaround + ibs : 1, // Instruction based sampling + xop : 1, // XOP: extended operation (AVX instructions) + skinit : 1, // SKINIT/STGI support + wdt : 1, // Watchdog timer support + : 1, // Reserved + lwp : 1, // Lightweight profiling + fma4 : 1, // 4-operand FMA instruction + tce : 1, // Translation cache extension + : 1, // Reserved + nodeid_msr : 1, // NodeId MSR (0xc001100c) + : 1, // Reserved + tbm : 1, // Trailing bit manipulations + topoext : 1, // Topology Extensions (leaf 0x8000001d) + perfctr_core : 1, // Core performance counter extensions + perfctr_nb : 1, // NB/DF performance counter extensions + : 1, // Reserved + data_bp_ext : 1, // Data access breakpoint extension + perf_tsc : 1, // Performance time-stamp counter + perfctr_llc : 1, // LLC (L3) performance counter extensions + mwaitx : 1, // MWAITX/MONITORX support + addr_mask_ext : 1, // Breakpoint address mask extension (to bit 31) + : 1; // Reserved + // edx + u32 e_fpu : 1, // Floating-Point Unit on-chip (x87) + e_vme : 1, // Virtual-8086 Mode Extensions + e_de : 1, // Debugging Extensions + e_pse : 1, // Page Size Extension + e_tsc : 1, // Time Stamp Counter + e_msr : 1, // Model-Specific Registers (RDMSR and WRMSR support) + pae : 1, // Physical Address Extensions + mce : 1, // Machine Check Exception + cx8 : 1, // CMPXCHG8B instruction + apic : 1, // APIC on-chip + : 1, // Reserved + syscall : 1, // SYSCALL and SYSRET instructions + mtrr : 1, // Memory Type Range Registers + pge : 1, // Page Global Extensions + mca : 1, // Machine Check Architecture + cmov : 1, // Conditional Move Instruction + pat : 1, // Page Attribute Table + pse36 : 1, // Page Size Extension (36-bit) + : 1, // Reserved + obsolete_mp_bit : 1, // Out-of-spec AMD Multiprocessing bit + nx : 1, // No-execute page protection + : 1, // Reserved + mmxext : 1, // AMD MMX extensions + e_mmx : 1, // MMX instructions + e_fxsr : 1, // FXSAVE and FXRSTOR instructions + fxsr_opt : 1, // FXSAVE and FXRSTOR optimizations + page1gb : 1, // 1-GB large page support + rdtscp : 1, // RDTSCP instruction + : 1, // Reserved + lm : 1, // Long mode (x86-64, 64-bit support) + _3dnowext : 1, // AMD 3DNow extensions + _3dnow : 1; // 3DNow instructions +}; + +/* + * Leaf 0x80000002 + * CPU brand ID string, bytes 0 - 15 + */ + +struct leaf_0x80000002_0 { + // eax + u32 cpu_brandid_0 : 32; // CPU brand ID string, bytes 0 - 3 + // ebx + u32 cpu_brandid_1 : 32; // CPU brand ID string, bytes 4 - 7 + // ecx + u32 cpu_brandid_2 : 32; // CPU brand ID string, bytes 8 - 11 + // edx + u32 cpu_brandid_3 : 32; // CPU brand ID string, bytes 12 - 15 +}; + +/* + * Leaf 0x80000003 + * CPU brand ID string, bytes 16 - 31 + */ + +struct leaf_0x80000003_0 { + // eax + u32 cpu_brandid_4 : 32; // CPU brand ID string bytes, 16 - 19 + // ebx + u32 cpu_brandid_5 : 32; // CPU brand ID string bytes, 20 - 23 + // ecx + u32 cpu_brandid_6 : 32; // CPU brand ID string bytes, 24 - 27 + // edx + u32 cpu_brandid_7 : 32; // CPU brand ID string bytes, 28 - 31 +}; + +/* + * Leaf 0x80000004 + * CPU brand ID string, bytes 32 - 47 + */ + +struct leaf_0x80000004_0 { + // eax + u32 cpu_brandid_8 : 32; // CPU brand ID string, bytes 32 - 35 + // ebx + u32 cpu_brandid_9 : 32; // CPU brand ID string, bytes 36 - 39 + // ecx + u32 cpu_brandid_10 : 32; // CPU brand ID string, bytes 40 - 43 + // edx + u32 cpu_brandid_11 : 32; // CPU brand ID string, bytes 44 - 47 +}; + +/* + * Leaf 0x80000005 + * AMD/Transmeta L1 cache and TLB + */ + +struct leaf_0x80000005_0 { + // eax + u32 l1_itlb_2m_4m_nentries : 8, // L1 ITLB #entries, 2M and 4M pages + l1_itlb_2m_4m_assoc : 8, // L1 ITLB associativity, 2M and 4M pages + l1_dtlb_2m_4m_nentries : 8, // L1 DTLB #entries, 2M and 4M pages + l1_dtlb_2m_4m_assoc : 8; // L1 DTLB associativity, 2M and 4M pages + // ebx + u32 l1_itlb_4k_nentries : 8, // L1 ITLB #entries, 4K pages + l1_itlb_4k_assoc : 8, // L1 ITLB associativity, 4K pages + l1_dtlb_4k_nentries : 8, // L1 DTLB #entries, 4K pages + l1_dtlb_4k_assoc : 8; // L1 DTLB associativity, 4K pages + // ecx + u32 l1_dcache_line_size : 8, // L1 dcache line size, in bytes + l1_dcache_nlines : 8, // L1 dcache lines per tag + l1_dcache_assoc : 8, // L1 dcache associativity + l1_dcache_size_kb : 8; // L1 dcache size, in KB + // edx + u32 l1_icache_line_size : 8, // L1 icache line size, in bytes + l1_icache_nlines : 8, // L1 icache lines per tag + l1_icache_assoc : 8, // L1 icache associativity + l1_icache_size_kb : 8; // L1 icache size, in KB +}; + +/* + * Leaf 0x80000006 + * (Mostly AMD) L2/L3 cache and TLB + */ + +struct leaf_0x80000006_0 { + // eax + u32 l2_itlb_2m_4m_nentries : 12, // L2 iTLB #entries, 2M and 4M pages + l2_itlb_2m_4m_assoc : 4, // L2 iTLB associativity, 2M and 4M pages + l2_dtlb_2m_4m_nentries : 12, // L2 dTLB #entries, 2M and 4M pages + l2_dtlb_2m_4m_assoc : 4; // L2 dTLB associativity, 2M and 4M pages + // ebx + u32 l2_itlb_4k_nentries : 12, // L2 iTLB #entries, 4K pages + l2_itlb_4k_assoc : 4, // L2 iTLB associativity, 4K pages + l2_dtlb_4k_nentries : 12, // L2 dTLB #entries, 4K pages + l2_dtlb_4k_assoc : 4; // L2 dTLB associativity, 4K pages + // ecx + u32 l2_line_size : 8, // L2 cache line size, in bytes + l2_nlines : 4, // L2 cache number of lines per tag + l2_assoc : 4, // L2 cache associativity + l2_size_kb : 16; // L2 cache size, in KB + // edx + u32 l3_line_size : 8, // L3 cache line size, in bytes + l3_nlines : 4, // L3 cache number of lines per tag + l3_assoc : 4, // L3 cache associativity + : 2, // Reserved + l3_size_range : 14; // L3 cache size range +}; + +/* + * Leaf 0x80000007 + * CPU power management (mostly AMD) and AMD RAS + */ + +struct leaf_0x80000007_0 { + // eax + u32 : 32; // Reserved + // ebx + u32 mca_overflow_recovery : 1, // MCA overflow conditions not fatal + succor : 1, // Software containment of uncorrectable errors + hw_assert : 1, // Hardware assert MSRs + scalable_mca : 1, // Scalable MCA (MCAX MSRs) + : 28; // Reserved + // ecx + u32 cpu_pwr_sample_ratio : 32; // CPU power sample time ratio + // edx + u32 digital_temp : 1, // Digital temperature sensor + powernow_freq_id : 1, // PowerNOW! frequency scaling + powernow_volt_id : 1, // PowerNOW! voltage scaling + thermal_trip : 1, // THERMTRIP (Thermal Trip) + hw_thermal_control : 1, // Hardware thermal control + sw_thermal_control : 1, // Software thermal control + _100mhz_steps : 1, // 100 MHz multiplier control + hw_pstate : 1, // Hardware P-state control + constant_tsc : 1, // TSC ticks at constant rate across all P and C states + core_perf_boost : 1, // Core performance boost + eff_freq_ro : 1, // Read-only effective frequency interface + proc_feedback : 1, // Processor feedback interface (deprecated) + proc_power_reporting : 1, // Processor power reporting interface + connected_standby : 1, // CPU Connected Standby support + rapl_interface : 1, // Runtime Average Power Limit interface + : 17; // Reserved +}; + +/* + * Leaf 0x80000008 + * CPU capacity parameters and extended feature flags (mostly AMD) + */ + +struct leaf_0x80000008_0 { + // eax + u32 phys_addr_bits : 8, // Max physical address bits + virt_addr_bits : 8, // Max virtual address bits + guest_phys_addr_bits : 8, // Max nested-paging guest physical address bits + : 8; // Reserved + // ebx + u32 clzero : 1, // CLZERO instruction + insn_retired_perf : 1, // Instruction retired counter MSR + xsave_err_ptr : 1, // XSAVE/XRSTOR always saves/restores FPU error pointers + invlpgb : 1, // INVLPGB broadcasts a TLB invalidate + rdpru : 1, // RDPRU (Read Processor Register at User level) + : 1, // Reserved + mba : 1, // Memory Bandwidth Allocation (AMD bit) + : 1, // Reserved + mcommit : 1, // MCOMMIT instruction + wbnoinvd : 1, // WBNOINVD instruction + : 2, // Reserved + ibpb : 1, // Indirect Branch Prediction Barrier + wbinvd_int : 1, // Interruptible WBINVD/WBNOINVD + ibrs : 1, // Indirect Branch Restricted Speculation + stibp : 1, // Single Thread Indirect Branch Prediction mode + ibrs_always_on : 1, // IBRS always-on preferred + stibp_always_on : 1, // STIBP always-on preferred + ibrs_fast : 1, // IBRS is preferred over software solution + ibrs_same_mode : 1, // IBRS provides same mode protection + no_efer_lmsle : 1, // Long-Mode Segment Limit Enable unsupported + tlb_flush_nested : 1, // INVLPGB RAX[5] bit can be set + : 1, // Reserved + amd_ppin : 1, // Protected Processor Inventory Number + amd_ssbd : 1, // Speculative Store Bypass Disable + virt_ssbd : 1, // virtualized SSBD (Speculative Store Bypass Disable) + amd_ssb_no : 1, // SSBD is not needed (fixed in hardware) + cppc : 1, // Collaborative Processor Performance Control + amd_psfd : 1, // Predictive Store Forward Disable + btc_no : 1, // CPU not affected by Branch Type Confusion + ibpb_ret : 1, // IBPB clears RSB/RAS too + branch_sampling : 1; // Branch Sampling + // ecx + u32 cpu_nthreads : 8, // Number of physical threads - 1 + : 4, // Reserved + apicid_coreid_len : 4, // Number of thread core ID bits (shift) in APIC ID + perf_tsc_len : 2, // Performance time-stamp counter size + : 14; // Reserved + // edx + u32 invlpgb_max_pages : 16, // INVLPGB maximum page count + rdpru_max_reg_id : 16; // RDPRU max register ID (ECX input) +}; + +/* + * Leaf 0x8000000a + * AMD SVM (Secure Virtual Machine) + */ + +struct leaf_0x8000000a_0 { + // eax + u32 svm_version : 8, // SVM revision number + : 24; // Reserved + // ebx + u32 svm_nasid : 32; // Number of address space identifiers (ASID) + // ecx + u32 : 4, // Reserved + pml : 1, // Page Modification Logging (PML) + : 27; // Reserved + // edx + u32 nested_pt : 1, // Nested paging + lbr_virt : 1, // LBR virtualization + svm_lock : 1, // SVM lock + nrip_save : 1, // NRIP save support on #VMEXIT + tsc_rate_msr : 1, // MSR-based TSC rate control + vmcb_clean : 1, // VMCB clean bits support + flush_by_asid : 1, // Flush by ASID + Extended VMCB TLB_Control + decode_assists : 1, // Decode Assists support + : 2, // Reserved + pause_filter : 1, // Pause intercept filter + : 1, // Reserved + pf_threshold : 1, // Pause filter threshold + avic : 1, // Advanced virtual interrupt controller + : 1, // Reserved + v_vmsave_vmload : 1, // Virtual VMSAVE/VMLOAD (nested virtualization) + v_gif : 1, // Virtualize the Global Interrupt Flag + gmet : 1, // Guest mode execution trap + x2avic : 1, // Virtual x2APIC + sss_check : 1, // Supervisor Shadow Stack restrictions + v_spec_ctrl : 1, // Virtual SPEC_CTRL + ro_gpt : 1, // Read-Only guest page table support + : 1, // Reserved + h_mce_override : 1, // Host MCE override + tlbsync_int : 1, // TLBSYNC intercept + INVLPGB/TLBSYNC in VMCB + nmi_virt : 1, // NMI virtualization + ibs_virt : 1, // IBS Virtualization + ext_lvt_off_chg : 1, // Extended LVT offset fault change + svme_addr_chk : 1, // Guest SVME address check + : 3; // Reserved +}; + +/* + * Leaf 0x80000019 + * AMD TLB characteristics for 1GB pages + */ + +struct leaf_0x80000019_0 { + // eax + u32 l1_itlb_1g_nentries : 12, // L1 iTLB #entries, 1G pages + l1_itlb_1g_assoc : 4, // L1 iTLB associativity, 1G pages + l1_dtlb_1g_nentries : 12, // L1 dTLB #entries, 1G pages + l1_dtlb_1g_assoc : 4; // L1 dTLB associativity, 1G pages + // ebx + u32 l2_itlb_1g_nentries : 12, // L2 iTLB #entries, 1G pages + l2_itlb_1g_assoc : 4, // L2 iTLB associativity, 1G pages + l2_dtlb_1g_nentries : 12, // L2 dTLB #entries, 1G pages + l2_dtlb_1g_assoc : 4; // L2 dTLB associativity, 1G pages + // ecx + u32 : 32; // Reserved + // edx + u32 : 32; // Reserved +}; + +/* + * Leaf 0x8000001a + * AMD instruction optimizations + */ + +struct leaf_0x8000001a_0 { + // eax + u32 fp_128 : 1, // Internal FP/SIMD exec data path is 128-bits wide + movu_preferred : 1, // SSE: MOVU* better than MOVL*/MOVH* + fp_256 : 1, // Internal FP/SSE exec data path is 256-bits wide + : 29; // Reserved + // ebx + u32 : 32; // Reserved + // ecx + u32 : 32; // Reserved + // edx + u32 : 32; // Reserved +}; + +/* + * Leaf 0x8000001b + * AMD IBS (Instruction-Based Sampling) + */ + +struct leaf_0x8000001b_0 { + // eax + u32 ibs_flags : 1, // IBS feature flags + ibs_fetch_sampling : 1, // IBS fetch sampling + ibs_op_sampling : 1, // IBS execution sampling + ibs_rdwr_op_counter : 1, // IBS read/write of op counter + ibs_op_count : 1, // IBS OP counting mode + ibs_branch_target : 1, // IBS branch target address reporting + ibs_op_counters_ext : 1, // IBS IbsOpCurCnt/IbsOpMaxCnt extend by 7 bits + ibs_rip_invalid_chk : 1, // IBS invalid RIP indication + ibs_op_branch_fuse : 1, // IBS fused branch micro-op indication + ibs_fetch_ctl_ext : 1, // IBS Fetch Control Extended MSR + ibs_op_data_4 : 1, // IBS op data 4 MSR + ibs_l3_miss_filter : 1, // IBS L3-miss filtering (Zen4+) + : 20; // Reserved + // ebx + u32 : 32; // Reserved + // ecx + u32 : 32; // Reserved + // edx + u32 : 32; // Reserved +}; + +/* + * Leaf 0x8000001c + * AMD LWP (Lightweight Profiling) + */ + +struct leaf_0x8000001c_0 { + // eax + u32 os_lwp_avail : 1, // OS: LWP is available to application programs + os_lwpval : 1, // OS: LWPVAL instruction + os_lwp_ire : 1, // OS: Instructions Retired Event + os_lwp_bre : 1, // OS: Branch Retired Event + os_lwp_dme : 1, // OS: Dcache Miss Event + os_lwp_cnh : 1, // OS: CPU Clocks Not Halted event + os_lwp_rnh : 1, // OS: CPU Reference clocks Not Halted event + : 22, // Reserved + os_lwp_cont : 1, // OS: LWP sampling in continuous mode + os_lwp_ptsc : 1, // OS: Performance Time Stamp Counter in event records + os_lwp_int : 1; // OS: Interrupt on threshold overflow + // ebx + u32 lwp_lwpcb_sz : 8, // Control Block size, in quadwords + lwp_event_sz : 8, // Event record size, in bytes + lwp_max_events : 8, // Max EventID supported + lwp_event_offset : 8; // Control Block events area offset + // ecx + u32 lwp_latency_max : 5, // Cache latency counters number of bits + lwp_data_addr : 1, // Cache miss events report data cache address + lwp_latency_rnd : 3, // Cache latency rounding amount + lwp_version : 7, // LWP version + lwp_buf_min_sz : 8, // LWP event ring buffer min size, 32 event record units + : 4, // Reserved + lwp_branch_predict : 1, // Branches Retired events can be filtered + lwp_ip_filtering : 1, // IP filtering (IPI, IPF, BaseIP, and LimitIP @ LWPCP) + lwp_cache_levels : 1, // Cache-related events: filter by cache level + lwp_cache_latency : 1; // Cache-related events: filter by latency + // edx + u32 hw_lwp_avail : 1, // HW: LWP available + hw_lwpval : 1, // HW: LWPVAL available + hw_lwp_ire : 1, // HW: Instructions Retired Event + hw_lwp_bre : 1, // HW: Branch Retired Event + hw_lwp_dme : 1, // HW: Dcache Miss Event + hw_lwp_cnh : 1, // HW: Clocks Not Halted event + hw_lwp_rnh : 1, // HW: Reference clocks Not Halted event + : 22, // Reserved + hw_lwp_cont : 1, // HW: LWP sampling in continuous mode + hw_lwp_ptsc : 1, // HW: Performance Time Stamp Counter in event records + hw_lwp_int : 1; // HW: Interrupt on threshold overflow +}; + +/* + * Leaf 0x8000001d + * AMD deterministic cache parameters + */ + +struct leaf_0x8000001d_n { + // eax + u32 cache_type : 5, // Cache type field + cache_level : 3, // Cache level (1-based) + cache_self_init : 1, // Self-initializing cache level + fully_associative : 1, // Fully-associative cache + : 4, // Reserved + num_threads_sharing : 12, // Number of logical CPUs sharing cache + : 6; // Reserved + // ebx + u32 cache_linesize : 12, // System coherency line size (0-based) + cache_npartitions : 10, // Physical line partitions (0-based) + cache_nways : 10; // Ways of associativity (0-based) + // ecx + u32 cache_nsets : 31, // Cache number of sets (0-based) + : 1; // Reserved + // edx + u32 wbinvd_rll_no_guarantee : 1, // WBINVD/INVD not guaranteed for Remote Lower-Level caches + ll_inclusive : 1, // Cache is inclusive of Lower-Level caches + : 30; // Reserved +}; + +#define LEAF_0x8000001d_SUBLEAF_N_FIRST 0 +#define LEAF_0x8000001d_SUBLEAF_N_LAST 31 + +/* + * Leaf 0x8000001e + * AMD CPU topology + */ + +struct leaf_0x8000001e_0 { + // eax + u32 ext_apic_id : 32; // Extended APIC ID + // ebx + u32 core_id : 8, // Unique per-socket logical core unit ID + core_nthreads : 8, // #Threads per core (zero-based) + : 16; // Reserved + // ecx + u32 node_id : 8, // Node (die) ID of invoking logical CPU + nnodes_per_socket : 3, // #nodes in invoking logical CPU's package/socket + : 21; // Reserved + // edx + u32 : 32; // Reserved +}; + +/* + * Leaf 0x8000001f + * AMD encrypted memory capabilities (SME/SEV) + */ + +struct leaf_0x8000001f_0 { + // eax + u32 sme : 1, // Secure Memory Encryption + sev : 1, // Secure Encrypted Virtualization + vm_page_flush : 1, // VM Page Flush MSR + sev_encrypted_state : 1, // SEV Encrypted State + sev_nested_paging : 1, // SEV secure nested paging + vm_permission_levels : 1, // VMPL + rpmquery : 1, // RPMQUERY instruction + vmpl_sss : 1, // VMPL supervisor shadow stack + secure_tsc : 1, // Secure TSC + virt_tsc_aux : 1, // Hardware virtualizes TSC_AUX + sme_coherent : 1, // Cache coherency enforcement across encryption domains + req_64bit_hypervisor : 1, // SEV guest mandates 64-bit hypervisor + restricted_injection : 1, // Restricted Injection supported + alternate_injection : 1, // Alternate Injection supported + debug_swap : 1, // SEV-ES: Full debug state swap + disallow_host_ibs : 1, // SEV-ES: Disallowing IBS use by the host + virt_transparent_enc : 1, // Virtual Transparent Encryption + vmgexit_parameter : 1, // SEV_FEATURES: VmgexitParameter + virt_tom_msr : 1, // Virtual TOM MSR + virt_ibs : 1, // SEV-ES guests: IBS state virtualization + : 4, // Reserved + vmsa_reg_protection : 1, // VMSA register protection + smt_protection : 1, // SMT protection + : 2, // Reserved + svsm_page_msr : 1, // SVSM communication page MSR + nested_virt_snp_msr : 1, // VIRT_RMPUPDATE/VIRT_PSMASH MSRs + : 2; // Reserved + // ebx + u32 pte_cbit_pos : 6, // PTE bit number to enable memory encryption + phys_addr_reduction_nbits : 6, // Reduction of phys address space in bits + vmpl_count : 4, // Number of VM permission levels (VMPL) + : 16; // Reserved + // ecx + u32 enc_guests_max : 32; // Max number of simultaneous encrypted guests + // edx + u32 min_sev_asid_no_sev_es : 32; // Minimum ASID for SEV-enabled SEV-ES-disabled guest +}; + +/* + * Leaf 0x80000020 + * AMD PQoS (Platform QoS) extended features + */ + +struct leaf_0x80000020_0 { + // eax + u32 : 32; // Reserved + // ebx + u32 : 1, // Reserved + mba : 1, // Memory Bandwidth Allocation support + smba : 1, // Slow Memory Bandwidth Allocation support + bmec : 1, // Bandwidth Monitoring Event Configuration support + l3rr : 1, // L3 Range Reservation support + abmc : 1, // Assignable Bandwidth Monitoring Counters + sdciae : 1, // Smart Data Cache Injection (SDCI) Allocation Enforcement + : 25; // Reserved + // ecx + u32 : 32; // Reserved + // edx + u32 : 32; // Reserved +}; + +struct leaf_0x80000020_1 { + // eax + u32 mba_limit_len : 32; // MBA enforcement limit size + // ebx + u32 : 32; // Reserved + // ecx + u32 : 32; // Reserved + // edx + u32 mba_cos_max : 32; // MBA max Class of Service number (zero-based) +}; + +struct leaf_0x80000020_2 { + // eax + u32 smba_limit_len : 32; // SMBA enforcement limit size + // ebx + u32 : 32; // Reserved + // ecx + u32 : 32; // Reserved + // edx + u32 smba_cos_max : 32; // SMBA max Class of Service number (zero-based) +}; + +struct leaf_0x80000020_3 { + // eax + u32 : 32; // Reserved + // ebx + u32 bmec_num_events : 8, // BMEC number of bandwidth events available + : 24; // Reserved + // ecx + u32 bmec_local_reads : 1, // Local NUMA reads can be tracked + bmec_remote_reads : 1, // Remote NUMA reads can be tracked + bmec_local_nontemp_wr : 1, // Local NUMA non-temporal writes can be tracked + bmec_remote_nontemp_wr : 1, // Remote NUMA non-temporal writes can be tracked + bmec_local_slow_mem_rd : 1, // Local NUMA slow-memory reads can be tracked + bmec_remote_slow_mem_rd : 1, // Remote NUMA slow-memory reads can be tracked + bmec_all_dirty_victims : 1, // Dirty QoS victims to all types of memory can be tracked + : 25; // Reserved + // edx + u32 : 32; // Reserved +}; + +/* + * Leaf 0x80000021 + * AMD extended CPU features 2 + */ + +struct leaf_0x80000021_0 { + // eax + u32 no_nested_data_bp : 1, // No nested data breakpoints + fsgs_non_serializing : 1, // WRMSR to {FS,GS,KERNEL_GS}_BASE is non-serializing + lfence_serializing : 1, // LFENCE always serializing / synchronizes RDTSC + smm_page_cfg_lock : 1, // SMM paging configuration lock + : 2, // Reserved + null_sel_clr_base : 1, // Null selector clears base + upper_addr_ignore : 1, // EFER MSR Upper Address Ignore + auto_ibrs : 1, // EFER MSR Automatic IBRS + no_smm_ctl_msr : 1, // SMM_CTL MSR not available + fsrs : 1, // Fast Short REP STOSB + fsrc : 1, // Fast Short REP CMPSB + : 1, // Reserved + prefetch_ctl_msr : 1, // Prefetch control MSR + : 2, // Reserved + opcode_reclaim : 1, // Reserves opcode space + user_cpuid_disable : 1, // #GP when executing CPUID at CPL > 0 + epsf : 1, // Enhanced Predictive Store Forwarding + : 3, // Reserved + wl_feedback : 1, // Workload-based heuristic feedback to OS + : 1, // Reserved + eraps : 1, // Enhanced Return Address Predictor Security + : 2, // Reserved + sbpb : 1, // Selective Branch Predictor Barrier + ibpb_brtype : 1, // Branch predictions flushed from CPU branch predictor + srso_no : 1, // No SRSO vulnerability + srso_uk_no : 1, // No SRSO at user-kernel boundary + srso_msr_fix : 1; // MSR BP_CFG[BpSpecReduce] SRSO mitigation + // ebx + u32 microcode_patch_size : 16, // Microcode patch size, in 16-byte units + rap_size : 8, // Return Address Predictor size + : 8; // Reserved + // ecx + u32 : 32; // Reserved + // edx + u32 : 32; // Reserved +}; + +/* + * Leaf 0x80000022 + * AMD extended performance monitoring + */ + +struct leaf_0x80000022_0 { + // eax + u32 perfmon_v2 : 1, // Performance monitoring v2 + lbr_v2 : 1, // Last Branch Record v2 extensions (LBR Stack) + lbr_pmc_freeze : 1, // Freezing core performance counters / LBR Stack + : 29; // Reserved + // ebx + u32 n_pmc_core : 4, // Number of core performance counters + lbr_v2_stack_size : 6, // Number of LBR stack entries + n_pmc_northbridge : 6, // Number of northbridge performance counters + n_pmc_umc : 6, // Number of UMC performance counters + : 10; // Reserved + // ecx + u32 active_umc_bitmask : 32; // Active UMCs bitmask + // edx + u32 : 32; // Reserved +}; + +/* + * Leaf 0x80000023 + * AMD multi-key encrypted memory + */ + +struct leaf_0x80000023_0 { + // eax + u32 mem_hmk_mode : 1, // MEM-HMK encryption mode + : 31; // Reserved + // ebx + u32 mem_hmk_avail_keys : 16, // Total number of available encryption keys + : 16; // Reserved + // ecx + u32 : 32; // Reserved + // edx + u32 : 32; // Reserved +}; + +/* + * Leaf 0x80000026 + * AMD extended CPU topology + */ + +struct leaf_0x80000026_n { + // eax + u32 x2apic_id_shift : 5, // Bit width of this level (previous levels inclusive) + : 24, // Reserved + core_has_pwreff_ranking : 1, // This core has a power efficiency ranking + domain_has_hybrid_cores : 1, // This domain level has hybrid (E, P) cores + domain_core_count_asymm : 1; // The 'Core' domain has asymmetric cores count + // ebx + u32 domain_lcpus_count : 16, // Number of logical CPUs at this domain instance + core_pwreff_ranking : 8, // This core's static power efficiency ranking + core_native_model_id : 4, // This core's native model ID + core_type : 4; // This core's type + // ecx + u32 domain_level : 8, // This domain level (subleaf ID) + domain_type : 8, // This domain type + : 16; // Reserved + // edx + u32 x2apic_id : 32; // x2APIC ID of current logical CPU +}; + +#define LEAF_0x80000026_SUBLEAF_N_FIRST 0 +#define LEAF_0x80000026_SUBLEAF_N_LAST 3 + +/* + * Leaf 0x80860000 + * Maximum Transmeta leaf + CPU vendor string + */ + +struct leaf_0x80860000_0 { + // eax + u32 max_tra_leaf : 32; // Maximum Transmeta leaf + // ebx + u32 cpu_vendorid_0 : 32; // Transmeta vendor ID string bytes 0 - 3 + // ecx + u32 cpu_vendorid_2 : 32; // Transmeta vendor ID string bytes 8 - 11 + // edx + u32 cpu_vendorid_1 : 32; // Transmeta vendor ID string bytes 4 - 7 +}; + +/* + * Leaf 0x80860001 + * Transmeta extended CPU features + */ + +struct leaf_0x80860001_0 { + // eax + u32 stepping : 4, // Stepping ID + base_model : 4, // Base CPU model ID + base_family_id : 4, // Base CPU family ID + cpu_type : 2, // CPU type + : 18; // Reserved + // ebx + u32 cpu_rev_mask_minor : 8, // CPU revision ID, mask minor + cpu_rev_mask_major : 8, // CPU revision ID, mask major + cpu_rev_minor : 8, // CPU revision ID, minor + cpu_rev_major : 8; // CPU revision ID, major + // ecx + u32 cpu_base_mhz : 32; // CPU nominal frequency, in MHz + // edx + u32 recovery : 1, // Recovery CMS is active (after bad flush) + longrun : 1, // LongRun power management capabilities + : 1, // Reserved + lrti : 1, // LongRun Table Interface + : 28; // Reserved +}; + +/* + * Leaf 0x80860002 + * Transmeta CMS (Code Morphing Software) + */ + +struct leaf_0x80860002_0 { + // eax + u32 cpu_rev_id : 32; // CPU revision ID + // ebx + u32 cms_rev_mask_2 : 8, // CMS revision ID, mask component 2 + cms_rev_mask_1 : 8, // CMS revision ID, mask component 1 + cms_rev_minor : 8, // CMS revision ID, minor + cms_rev_major : 8; // CMS revision ID, major + // ecx + u32 cms_rev_mask_3 : 32; // CMS revision ID, mask component 3 + // edx + u32 : 32; // Reserved +}; + +/* + * Leaf 0x80860003 + * Transmeta CPU information string, bytes 0 - 15 + */ + +struct leaf_0x80860003_0 { + // eax + u32 cpu_info_0 : 32; // CPU info string bytes 0 - 3 + // ebx + u32 cpu_info_1 : 32; // CPU info string bytes 4 - 7 + // ecx + u32 cpu_info_2 : 32; // CPU info string bytes 8 - 11 + // edx + u32 cpu_info_3 : 32; // CPU info string bytes 12 - 15 +}; + +/* + * Leaf 0x80860004 + * Transmeta CPU information string, bytes 16 - 31 + */ + +struct leaf_0x80860004_0 { + // eax + u32 cpu_info_4 : 32; // CPU info string bytes 16 - 19 + // ebx + u32 cpu_info_5 : 32; // CPU info string bytes 20 - 23 + // ecx + u32 cpu_info_6 : 32; // CPU info string bytes 24 - 27 + // edx + u32 cpu_info_7 : 32; // CPU info string bytes 28 - 31 +}; + +/* + * Leaf 0x80860005 + * Transmeta CPU information string, bytes 32 - 47 + */ + +struct leaf_0x80860005_0 { + // eax + u32 cpu_info_8 : 32; // CPU info string bytes 32 - 35 + // ebx + u32 cpu_info_9 : 32; // CPU info string bytes 36 - 39 + // ecx + u32 cpu_info_10 : 32; // CPU info string bytes 40 - 43 + // edx + u32 cpu_info_11 : 32; // CPU info string bytes 44 - 47 +}; + +/* + * Leaf 0x80860006 + * Transmeta CPU information string, bytes 48 - 63 + */ + +struct leaf_0x80860006_0 { + // eax + u32 cpu_info_12 : 32; // CPU info string bytes 48 - 51 + // ebx + u32 cpu_info_13 : 32; // CPU info string bytes 52 - 55 + // ecx + u32 cpu_info_14 : 32; // CPU info string bytes 56 - 59 + // edx + u32 cpu_info_15 : 32; // CPU info string bytes 60 - 63 +}; + +/* + * Leaf 0x80860007 + * Transmeta live CPU information + */ + +struct leaf_0x80860007_0 { + // eax + u32 cpu_cur_mhz : 32; // Current CPU frequency, in MHz + // ebx + u32 cpu_cur_voltage : 32; // Current CPU voltage, in millivolts + // ecx + u32 cpu_cur_perf_pctg : 32; // Current CPU performance percentage, 0 - 100 + // edx + u32 cpu_cur_gate_delay : 32; // Current CPU gate delay, in femtoseconds +}; + +/* + * Leaf 0xc0000000 + * Maximum Centaur/Zhaoxin leaf + */ + +struct leaf_0xc0000000_0 { + // eax + u32 max_cntr_leaf : 32; // Maximum Centaur/Zhaoxin leaf + // ebx + u32 : 32; // Reserved + // ecx + u32 : 32; // Reserved + // edx + u32 : 32; // Reserved +}; + +/* + * Leaf 0xc0000001 + * Centaur/Zhaoxin extended CPU features + */ + +struct leaf_0xc0000001_0 { + // eax + u32 : 32; // Reserved + // ebx + u32 : 32; // Reserved + // ecx + u32 : 32; // Reserved + // edx + u32 ccs_sm2 : 1, // CCS SM2 instructions + ccs_sm2_en : 1, // CCS SM2 enabled + rng : 1, // Random Number Generator + rng_en : 1, // RNG enabled + ccs_sm3_sm4 : 1, // CCS SM3 and SM4 instructions + ccs_sm3_sm4_en : 1, // CCS SM3/SM4 enabled + ace : 1, // Advanced Cryptography Engine + ace_en : 1, // ACE enabled + ace2 : 1, // Advanced Cryptography Engine v2 + ace2_en : 1, // ACE v2 enabled + phe : 1, // PadLock Hash Engine + phe_en : 1, // PHE enabled + pmm : 1, // PadLock Montgomery Multiplier + pmm_en : 1, // PMM enabled + : 2, // Reserved + parallax : 1, // Parallax auto adjust processor voltage + parallax_en : 1, // Parallax enabled + : 2, // Reserved + tm3 : 1, // Thermal Monitor v3 + tm3_en : 1, // TM v3 enabled + : 3, // Reserved + phe2 : 1, // PadLock Hash Engine v2 (SHA384/SHA512) + phe2_en : 1, // PHE v2 enabled + rsa : 1, // RSA instructions (XMODEXP/MONTMUL2) + rsa_en : 1, // RSA instructions enabled + : 3; // Reserved +}; + +#endif /* _ASM_X86_CPUID_LEAF_TYPES */ diff --git a/arch/x86/include/asm/cpuid/types.h b/arch/x86/include/asm/cpuid/types.h index 8a00364b79de..3d0e611c97ba 100644 --- a/arch/x86/include/asm/cpuid/types.h +++ b/arch/x86/include/asm/cpuid/types.h @@ -5,6 +5,8 @@ #include #include +#include + /* * Types for raw CPUID access: */ @@ -30,6 +32,12 @@ enum cpuid_regs_idx { #define CPUID_LEAF_FREQ 0x16 #define CPUID_LEAF_TILE 0x1d +#define CPUID_RANGE(idx) ((idx) & 0xffff0000) +#define CPUID_RANGE_MAX(idx) (CPUID_RANGE(idx) + 0xffff) + +#define CPUID_BASE_START 0x00000000 +#define CPUID_BASE_END CPUID_RANGE_MAX(CPUID_BASE_START) + /* * Types for CPUID(0x2) parsing: */ @@ -124,4 +132,94 @@ extern const struct leaf_0x2_table cpuid_0x2_table[256]; */ #define TLB_0x63_2M_4M_ENTRIES 32 +/* + * Types for centralized CPUID tables: + * + * For internal use by the CPUID parser. + */ + +/** + * struct leaf_parse_info - CPUID query parse info + * @nr_entries: Number of valid entries filled by the CPUID parser + */ +struct leaf_parse_info { + unsigned int nr_entries; +}; + +/** + * __CPUID_LEAF() - Define a CPUID output and parse info entry + * @_name: Struct type name of the CPUID leaf/subleaf (e.g. 'leaf_0x7_0'). Such + * types are defined at and follow the leaf_0xM_N + * format, where 0xM is the leaf and N is the subleaf. + * @_count: Number of storage entries to allocate for this leaf/subleaf. + * + * For a given leaf/subleaf, define an array of CPUID storage entries and an associated + * query info structure. + * + * Use an array of storage entries to accommodate CPUID leaves with multiple subleaves + * having the same output format. This is common for hierarchical enumeration; e.g., + * CPUID(0x4), CPUID(0x12), and CPUID(0x8000001d). + */ +#define __CPUID_LEAF(_name, _count) \ + struct _name _name[_count]; \ + struct leaf_parse_info _name##_info + +/** + * CPUID_LEAF() - Define a 'struct cpuid_leaves' storage entry + * @_leaf: Leaf number, in compile-time 0xN format + * @_subleaf: Subleaf number, in compile-time decimal format + * + * Convenience wrapper around __CPUID_LEAF(). + */ +#define CPUID_LEAF(_leaf, _subleaf) \ + __CPUID_LEAF(leaf_ ## _leaf ## _ ## _subleaf, 1) + +#define __cpuid_leaf_first_subleaf(_l) \ + LEAF_ ## _l ## _ ## SUBLEAF_N_FIRST +#define __cpuid_leaf_last_subleaf(_l) \ + LEAF_ ## _l ## _ ## SUBLEAF_N_LAST + +#define __cpuid_leaf_subleaf_count_min(_l) 2 +#define __cpuid_leaf_subleaf_count_max(_l) \ + (__cpuid_leaf_last_subleaf(_l) - __cpuid_leaf_first_subleaf(_l) + 1) + +/** + * CPUID_LEAF_N() - Define a 'struct cpuid_leaves' storage entry + * @_leaf: Leaf number, in compile-time 0xN format + * @_count: Number of storage entries to allocate for that leaf. It must not exceed + * the limits defined at . + * + * Convenience wrapper around __CPUID_LEAF(). + */ +#define CPUID_LEAF_N(_leaf, _count) \ + static_assert(_count >= __cpuid_leaf_subleaf_count_min(_leaf)); \ + static_assert(_count <= __cpuid_leaf_subleaf_count_max(_leaf)); \ + __CPUID_LEAF(leaf_ ## _leaf ## _ ## n, _count) + +/* + * struct cpuid_leaves - Parsed CPUID data + */ +struct cpuid_leaves { + /* Leaf Subleaf number (or max number of subleaves) */ + CPUID_LEAF ( 0x0, 0 ); + CPUID_LEAF ( 0x1, 0 ); +}; + +/* + * Types for centralized CPUID tables: + * + * For external use. + */ + +/** + * struct cpuid_table - Per-CPU CPUID data repository + * @leaves: Parsed CPUID queries output and their metadata + * + * This is to be embedded inside 'struct cpuinfo_x86' to provide parsed and + * sanitized CPUID data per CPU. + */ +struct cpuid_table { + struct cpuid_leaves leaves; +}; + #endif /* _ASM_X86_CPUID_TYPES_H */ diff --git a/arch/x86/include/asm/fpu/api.h b/arch/x86/include/asm/fpu/api.h index cd6f194a912b..90c63fe19c0f 100644 --- a/arch/x86/include/asm/fpu/api.h +++ b/arch/x86/include/asm/fpu/api.h @@ -119,12 +119,6 @@ extern void fpu__init_system(void); extern void fpu__init_check_bugs(void); extern void fpu__resume_cpu(void); -#ifdef CONFIG_MATH_EMULATION -extern void fpstate_init_soft(struct swregs_state *soft); -#else -static inline void fpstate_init_soft(struct swregs_state *soft) {} -#endif - /* State tracking */ DECLARE_PER_CPU(bool, kernel_fpu_allowed); DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx); diff --git a/arch/x86/include/asm/intel-family.h b/arch/x86/include/asm/intel-family.h index 950bfd006905..66d964afe18a 100644 --- a/arch/x86/include/asm/intel-family.h +++ b/arch/x86/include/asm/intel-family.h @@ -29,6 +29,7 @@ * _N,_P - other mobile parts * _H - premium mobile parts * _S - other client parts + * _R - ruggedized for harsh environment * * Historical OPTDIFFs: * @@ -151,6 +152,7 @@ #define INTEL_LUNARLAKE_M IFM(6, 0xBD) /* Lion Cove / Skymont */ #define INTEL_PANTHERLAKE_L IFM(6, 0xCC) /* Cougar Cove / Darkmont */ +#define INTEL_PANTHERLAKE_R IFM(6, 0xE5) /* Cougar Cove / Darkmont */ #define INTEL_WILDCATLAKE_L IFM(6, 0xD5) diff --git a/arch/x86/include/asm/microcode.h b/arch/x86/include/asm/microcode.h index 3c317d155771..9cd136d4515c 100644 --- a/arch/x86/include/asm/microcode.h +++ b/arch/x86/include/asm/microcode.h @@ -3,6 +3,7 @@ #define _ASM_X86_MICROCODE_H #include +#include struct cpu_signature { unsigned int sig; diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index 9c2ea29e12a9..2f41a9289d79 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -256,29 +256,15 @@ int msr_set_bit(u32 msr, u8 bit); int msr_clear_bit(u32 msr, u8 bit); #ifdef CONFIG_SMP -int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h); -int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h); int rdmsrq_on_cpu(unsigned int cpu, u32 msr_no, u64 *q); int wrmsrq_on_cpu(unsigned int cpu, u32 msr_no, u64 q); void rdmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr __percpu *msrs); void wrmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr __percpu *msrs); -int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h); -int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h); int rdmsrq_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q); int wrmsrq_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q); int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]); int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]); #else /* CONFIG_SMP */ -static inline int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h) -{ - rdmsr(msr_no, *l, *h); - return 0; -} -static inline int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) -{ - wrmsr(msr_no, l, h); - return 0; -} static inline int rdmsrq_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) { rdmsrq(msr_no, *q); @@ -292,21 +278,12 @@ static inline int wrmsrq_on_cpu(unsigned int cpu, u32 msr_no, u64 q) static inline void rdmsr_on_cpus(const struct cpumask *m, u32 msr_no, struct msr __percpu *msrs) { - rdmsr_on_cpu(0, msr_no, raw_cpu_ptr(&msrs->l), raw_cpu_ptr(&msrs->h)); + rdmsrq_on_cpu(0, msr_no, raw_cpu_ptr(&msrs->q)); } static inline void wrmsr_on_cpus(const struct cpumask *m, u32 msr_no, struct msr __percpu *msrs) { - wrmsr_on_cpu(0, msr_no, raw_cpu_read(msrs->l), raw_cpu_read(msrs->h)); -} -static inline int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, - u32 *l, u32 *h) -{ - return rdmsr_safe(msr_no, l, h); -} -static inline int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) -{ - return wrmsr_safe(msr_no, l, h); + wrmsrq_on_cpu(0, msr_no, raw_cpu_read(msrs->q)); } static inline int rdmsrq_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) { @@ -325,11 +302,5 @@ static inline int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]) return wrmsr_safe_regs(regs); } #endif /* CONFIG_SMP */ - -/* Compatibility wrappers: */ -#define rdmsrl(msr, val) rdmsrq(msr, val) -#define wrmsrl(msr, val) wrmsrq(msr, val) -#define rdmsrl_on_cpu(cpu, msr, q) rdmsrq_on_cpu(cpu, msr, q) - #endif /* __ASSEMBLER__ */ #endif /* _ASM_X86_MSR_H */ diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 81f79a7fe94f..87b1d4c0727e 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -16,7 +16,7 @@ struct vm86; #include #include #include -#include +#include #include #include #include @@ -170,6 +170,7 @@ struct cpuinfo_x86 { char x86_vendor_id[16]; char x86_model_id[64]; struct cpuinfo_topology topo; + struct cpuid_table cpuid; /* in KB - valid for CPUS which support this call: */ unsigned int x86_cache_size; int x86_cache_alignment; /* In bytes */ diff --git a/arch/x86/include/asm/purgatory.h b/arch/x86/include/asm/purgatory.h index 2fee5e9f1ccc..56a9e81edb15 100644 --- a/arch/x86/include/asm/purgatory.h +++ b/arch/x86/include/asm/purgatory.h @@ -8,4 +8,4 @@ extern void purgatory(void); #endif /* __ASSEMBLER__ */ -#endif /* _ASM_PURGATORY_H */ +#endif /* _ASM_X86_PURGATORY_H */ diff --git a/arch/x86/include/asm/timex.h b/arch/x86/include/asm/timex.h index 956e4145311b..6e57e3c0fdd2 100644 --- a/arch/x86/include/asm/timex.h +++ b/arch/x86/include/asm/timex.h @@ -7,8 +7,7 @@ static inline unsigned long random_get_entropy(void) { - if (!IS_ENABLED(CONFIG_X86_TSC) && - !cpu_feature_enabled(X86_FEATURE_TSC)) + if (!cpu_feature_enabled(X86_FEATURE_TSC)) return random_get_entropy_fallback(); return rdtsc(); } diff --git a/arch/x86/include/asm/trace_clock.h b/arch/x86/include/asm/trace_clock.h index 7061a5650969..1efab284c32a 100644 --- a/arch/x86/include/asm/trace_clock.h +++ b/arch/x86/include/asm/trace_clock.h @@ -5,17 +5,9 @@ #include #include -#ifdef CONFIG_X86_TSC - extern u64 notrace trace_clock_x86_tsc(void); # define ARCH_TRACE_CLOCKS \ { trace_clock_x86_tsc, "x86-tsc", .in_ns = 0 }, -#else /* !CONFIG_X86_TSC */ - -#define ARCH_TRACE_CLOCKS - -#endif - #endif /* _ASM_X86_TRACE_CLOCK_H */ diff --git a/arch/x86/include/asm/tsc.h b/arch/x86/include/asm/tsc.h index 4f7f09f50552..4d2d2f21ff06 100644 --- a/arch/x86/include/asm/tsc.h +++ b/arch/x86/include/asm/tsc.h @@ -76,8 +76,7 @@ extern void disable_TSC(void); static inline cycles_t get_cycles(void) { - if (!IS_ENABLED(CONFIG_X86_TSC) && - !cpu_feature_enabled(X86_FEATURE_TSC)) + if (!cpu_feature_enabled(X86_FEATURE_TSC)) return 0; return rdtsc(); } @@ -94,25 +93,15 @@ extern unsigned long native_calibrate_tsc(void); extern unsigned long long native_sched_clock_from_tsc(u64 tsc); extern int tsc_clocksource_reliable; -#ifdef CONFIG_X86_TSC extern bool tsc_async_resets; -#else -# define tsc_async_resets false -#endif /* * Boot-time check whether the TSCs are synchronized across * all CPUs/cores: */ -#ifdef CONFIG_X86_TSC extern bool tsc_store_and_check_tsc_adjust(bool bootcpu); extern void tsc_verify_tsc_adjust(bool resume); extern void check_tsc_sync_target(void); -#else -static inline bool tsc_store_and_check_tsc_adjust(bool bootcpu) { return false; } -static inline void tsc_verify_tsc_adjust(bool resume) { } -static inline void check_tsc_sync_target(void) { } -#endif extern int notsc_setup(char *); extern void tsc_save_sched_clock_state(void); diff --git a/arch/x86/include/asm/vermagic.h b/arch/x86/include/asm/vermagic.h index eda233a90ea8..e26061df0c9b 100644 --- a/arch/x86/include/asm/vermagic.h +++ b/arch/x86/include/asm/vermagic.h @@ -5,8 +5,6 @@ #ifdef CONFIG_X86_64 /* X86_64 does not define MODULE_PROC_FAMILY */ -#elif defined CONFIG_M586 -#define MODULE_PROC_FAMILY "586 " #elif defined CONFIG_M586TSC #define MODULE_PROC_FAMILY "586TSC " #elif defined CONFIG_M586MMX @@ -31,10 +29,6 @@ #define MODULE_PROC_FAMILY "CRUSOE " #elif defined CONFIG_MEFFICEON #define MODULE_PROC_FAMILY "EFFICEON " -#elif defined CONFIG_MWINCHIPC6 -#define MODULE_PROC_FAMILY "WINCHIPC6 " -#elif defined CONFIG_MWINCHIP3D -#define MODULE_PROC_FAMILY "WINCHIP3D " #elif defined CONFIG_MCYRIXIII #define MODULE_PROC_FAMILY "CYRIXIII " #elif defined CONFIG_MVIAC3_2 diff --git a/arch/x86/include/asm/xen/hypervisor.h b/arch/x86/include/asm/xen/hypervisor.h index f666637d773e..9cd2fec1782f 100644 --- a/arch/x86/include/asm/xen/hypervisor.h +++ b/arch/x86/include/asm/xen/hypervisor.h @@ -37,6 +37,7 @@ extern struct shared_info *HYPERVISOR_shared_info; extern struct start_info *xen_start_info; #include +#include #include #define XEN_SIGNATURE "XenVMMXenVMM" diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 47a32f583930..31f46fd00527 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -108,7 +108,7 @@ apm-y := apm_32.o obj-$(CONFIG_APM) += apm.o obj-$(CONFIG_SMP) += smp.o obj-$(CONFIG_SMP) += smpboot.o -obj-$(CONFIG_X86_TSC) += tsc_sync.o +obj-y += tsc_sync.o obj-$(CONFIG_SMP) += setup_percpu.o obj-$(CONFIG_X86_MPPARSE) += mpparse.o obj-y += apic/ @@ -117,7 +117,7 @@ obj-$(CONFIG_DYNAMIC_FTRACE) += ftrace.o obj-$(CONFIG_FUNCTION_TRACER) += ftrace_$(BITS).o obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += ftrace.o obj-$(CONFIG_FTRACE_SYSCALLS) += ftrace.o -obj-$(CONFIG_X86_TSC) += trace_clock.o +obj-y += trace_clock.o obj-$(CONFIG_TRACING) += trace.o obj-$(CONFIG_RETHOOK) += rethook.o obj-$(CONFIG_VMCORE_INFO) += vmcore_info_$(BITS).o diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 3eeebc2c5a1d..aa1e19979aa8 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -64,6 +64,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile index 2f8a58ef690e..d2e8a849f180 100644 --- a/arch/x86/kernel/cpu/Makefile +++ b/arch/x86/kernel/cpu/Makefile @@ -19,6 +19,7 @@ KCSAN_SANITIZE_common.o := n obj-y := cacheinfo.o scattered.o obj-y += topology_common.o topology_ext.o topology_amd.o +obj-y += cpuid_parser.o obj-y += common.o obj-y += rdrand.o obj-y += match.o diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 31f01e9c7114..487ac147e11f 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/x86/kernel/cpu/centaur.c b/arch/x86/kernel/cpu/centaur.c index 81695da9c524..681d2da49341 100644 --- a/arch/x86/kernel/cpu/centaur.c +++ b/arch/x86/kernel/cpu/centaur.c @@ -5,6 +5,7 @@ #include #include +#include #include #include #include diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index a4268c47f2bc..a3df21d26460 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1737,13 +1737,6 @@ static void __init cpu_parse_early_param(void) int arglen; #ifdef CONFIG_X86_32 - if (cmdline_find_option_bool(boot_command_line, "no387")) -#ifdef CONFIG_MATH_EMULATION - setup_clear_cpu_cap(X86_FEATURE_FPU); -#else - pr_err("Option 'no387' required CONFIG_MATH_EMULATION enabled.\n"); -#endif - if (cmdline_find_option_bool(boot_command_line, "nofxsr")) setup_clear_cpu_cap(X86_FEATURE_FXSR); #endif @@ -1791,6 +1784,7 @@ static void __init cpu_parse_early_param(void) static void __init early_identify_cpu(struct cpuinfo_x86 *c) { memset(&c->x86_capability, 0, sizeof(c->x86_capability)); + memset(&c->cpuid, 0, sizeof(c->cpuid)); c->extended_cpuid_level = 0; if (!cpuid_feature()) @@ -1798,6 +1792,7 @@ static void __init early_identify_cpu(struct cpuinfo_x86 *c) /* cyrix could have cpuid enabled via c_identify()*/ if (cpuid_feature()) { + cpuid_scan_cpu(c); cpu_detect(c); get_cpu_vendor(c); intel_unlock_cpuid_leafs(c); @@ -1970,8 +1965,8 @@ static void generic_identify(struct cpuinfo_x86 *c) if (!cpuid_feature()) return; + cpuid_scan_cpu(c); cpu_detect(c); - get_cpu_vendor(c); intel_unlock_cpuid_leafs(c); get_cpu_cap(c); @@ -2023,6 +2018,7 @@ static void identify_cpu(struct cpuinfo_x86 *c) #endif c->x86_cache_alignment = c->x86_clflush_size; memset(&c->x86_capability, 0, sizeof(c->x86_capability)); + memset(&c->cpuid, 0, sizeof(c->cpuid)); #ifdef CONFIG_X86_VMX_FEATURE_NAMES memset(&c->vmx_capability, 0, sizeof(c->vmx_capability)); #endif diff --git a/arch/x86/kernel/cpu/cpu.h b/arch/x86/kernel/cpu/cpu.h index 5c7a3a71191a..dca2d5845e42 100644 --- a/arch/x86/kernel/cpu/cpu.h +++ b/arch/x86/kernel/cpu/cpu.h @@ -75,7 +75,6 @@ static inline struct amd_northbridge *amd_init_l3_cache(int index) } #endif -unsigned int aperfmperf_get_khz(int cpu); void cpu_select_mitigations(void); extern void x86_spec_ctrl_setup_ap(void); diff --git a/arch/x86/kernel/cpu/cpuid_parser.c b/arch/x86/kernel/cpu/cpuid_parser.c new file mode 100644 index 000000000000..898b0c441431 --- /dev/null +++ b/arch/x86/kernel/cpu/cpuid_parser.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * CPUID parser; for populating the system's CPUID tables. + */ + +#include + +#include +#include + +#include "cpuid_parser.h" + +/* Clear a single CPUID table entry */ +static void cpuid_clear(const struct cpuid_parse_entry *e, const struct cpuid_read_output *output) +{ + struct cpuid_regs *regs = output->regs; + + for (int i = 0; i < e->maxcnt; i++, regs++) + memset(regs, 0, sizeof(*regs)); + + memset(output->info, 0, sizeof(*output->info)); +} + +/* + * Leaf read functions: + */ + +/* + * Default CPUID read function + * Satisfies the requirements stated at 'struct cpuid_parse_entry'->read(). + */ +static void +cpuid_read_generic(const struct cpuid_parse_entry *e, const struct cpuid_read_output *output) +{ + struct cpuid_regs *regs = output->regs; + + for (int i = 0; i < e->maxcnt; i++, regs++, output->info->nr_entries++) + cpuid_read_subleaf(e->leaf, e->subleaf + i, regs); +} + +/* + * CPUID parser table: + */ + +static const struct cpuid_parse_entry cpuid_parse_entries[] = { + CPUID_PARSE_ENTRIES +}; + +/* + * Leaf-independent parser code: + */ + +static unsigned int cpuid_range_max_leaf(const struct cpuid_table *t, unsigned int range) +{ + const struct leaf_0x0_0 *l0 = __cpuid_table_subleaf(t, 0x0, 0); + + switch (range) { + case CPUID_BASE_START: return l0 ? l0->max_std_leaf : 0; + default: return 0; + } +} + +static void +__cpuid_reset_table(struct cpuid_table *t, const struct cpuid_parse_entry entries[], + unsigned int nr_entries, unsigned int start, unsigned int end, bool fill) +{ + const struct cpuid_parse_entry *entry = entries; + unsigned int range = CPUID_RANGE(start); + + for (unsigned int i = 0; i < nr_entries; i++, entry++) { + struct cpuid_read_output output = { + .regs = cpuid_table_regs_p(t, entry->regs_offs), + .info = cpuid_table_info_p(t, entry->info_offs), + }; + + if (entry->leaf < start || entry->leaf > end) + continue; + + cpuid_clear(entry, &output); + + /* + * Read the range's anchor leaf unconditionally so that the cached + * maximum valid leaf value is available for the remaining entries. + */ + if (fill && (entry->leaf == range || entry->leaf <= cpuid_range_max_leaf(t, range))) + entry->read(entry, &output); + } +} + +/* + * Zero all cached CPUID entries within [@start-@end] range. This is needed when + * certain operations like MSR writes induce changes to the CPU's CPUID layout. + */ +static void +__cpuid_zero_table(struct cpuid_table *t, const struct cpuid_parse_entry entries[], + unsigned int nr_entries, unsigned int start, unsigned int end) +{ + __cpuid_reset_table(t, entries, nr_entries, start, end, false); +} + +static void +__cpuid_fill_table(struct cpuid_table *t, const struct cpuid_parse_entry entries[], + unsigned int nr_entries, unsigned int start, unsigned int end) +{ + __cpuid_reset_table(t, entries, nr_entries, start, end, true); +} + +static void +cpuid_fill_table(struct cpuid_table *t, const struct cpuid_parse_entry entries[], unsigned int nr_entries) +{ + static const struct { + unsigned int start; + unsigned int end; + } ranges[] = { + { CPUID_BASE_START, CPUID_BASE_END }, + }; + + for (unsigned int i = 0; i < ARRAY_SIZE(ranges); i++) + __cpuid_fill_table(t, entries, nr_entries, ranges[i].start, ranges[i].end); +} + +static void __cpuid_scan_cpu_full(struct cpuinfo_x86 *c) +{ + unsigned int nr_entries = ARRAY_SIZE(cpuid_parse_entries); + struct cpuid_table *table = &c->cpuid; + + cpuid_fill_table(table, cpuid_parse_entries, nr_entries); +} + +static void +__cpuid_scan_cpu_partial(struct cpuinfo_x86 *c, unsigned int start_leaf, unsigned int end_leaf) +{ + unsigned int nr_entries = ARRAY_SIZE(cpuid_parse_entries); + struct cpuid_table *table = &c->cpuid; + + __cpuid_zero_table(table, cpuid_parse_entries, nr_entries, start_leaf, end_leaf); + __cpuid_fill_table(table, cpuid_parse_entries, nr_entries, start_leaf, end_leaf); +} + +/* + * Call-site APIs: + */ + +/** + * cpuid_scan_cpu() - Populate current CPU's CPUID table + * @c: CPU capability structure associated with the current CPU + * + * Populate the CPUID table embedded within @c with parsed CPUID data. All CPUID + * instructions are invoked locally, so this must be called on the CPU associated + * with @c. + */ +void cpuid_scan_cpu(struct cpuinfo_x86 *c) +{ + __cpuid_scan_cpu_full(c); +} + +/** + * cpuid_refresh_range() - Rescan a CPUID table's leaf range + * @c: CPU capability structure associated with the current CPU + * @start: Start of leaf range to be re-scanned + * @end: End of leaf range + */ +void cpuid_refresh_range(struct cpuinfo_x86 *c, u32 start, u32 end) +{ + if (WARN_ON_ONCE(start > end)) + return; + + if (WARN_ON_ONCE(CPUID_RANGE(start) != CPUID_RANGE(end))) + return; + + __cpuid_scan_cpu_partial(c, start, end); +} + +/** + * cpuid_refresh_leaf() - Rescan a CPUID table's leaf + * @c: CPU capability structure associated with the current CPU + * @leaf: Leaf to be re-scanned + */ +void cpuid_refresh_leaf(struct cpuinfo_x86 *c, u32 leaf) +{ + cpuid_refresh_range(c, leaf, leaf); +} diff --git a/arch/x86/kernel/cpu/cpuid_parser.h b/arch/x86/kernel/cpu/cpuid_parser.h new file mode 100644 index 000000000000..df627306cc8c --- /dev/null +++ b/arch/x86/kernel/cpu/cpuid_parser.h @@ -0,0 +1,120 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ARCH_X86_CPUID_PARSER_H +#define _ARCH_X86_CPUID_PARSER_H + +#include + +/* + * Since accessing the CPUID leaves at 'struct cpuid_leaves' require compile time + * tokenization, split the CPUID parser into two stages: compile time macros for + * tokenizing the leaf/subleaf output offsets within the table, and generic runtime + * code to write to the relevant CPUID leaves using such offsets. + * + * The output of the compile time macros is cached by a compile time "parse entry" + * table (see 'struct cpuid_parse_entry'). The runtime parser code will utilize + * such offsets by passing them to the cpuid_table_*_p() functions. + */ + +/* + * Compile time CPUID table offset calculations: + * + * @_leaf: CPUID leaf, in 0xN format + * @_subleaf: CPUID subleaf, in decimal format + */ + +#define __cpuid_leaves_regs_offset(_leaf, _subleaf) \ + offsetof(struct cpuid_leaves, leaf_ ## _leaf ## _ ## _subleaf) + +#define __cpuid_leaves_info_offset(_leaf, _subleaf) \ + offsetof(struct cpuid_leaves, leaf_ ## _leaf ## _ ## _subleaf ## _ ## info) + +#define __cpuid_leaves_regs_maxcnt(_leaf, _subleaf) \ + ARRAY_SIZE(((struct cpuid_leaves *)NULL)->leaf_ ## _leaf ## _ ## _subleaf) + +/* + * Translation of compile time offsets to generic runtime pointers: + */ + +static inline struct cpuid_regs * +cpuid_table_regs_p(const struct cpuid_table *t, unsigned long regs_offset) +{ + return (struct cpuid_regs *)((unsigned long)(&t->leaves) + regs_offset); +} + +static inline struct leaf_parse_info * +cpuid_table_info_p(const struct cpuid_table *t, unsigned long info_offset) +{ + return (struct leaf_parse_info *)((unsigned long)(&t->leaves) + info_offset); +} + +/** + * struct cpuid_read_output - Output of a CPUID read operation + * @regs: Pointer to an array of CPUID outputs, where each array element covers the + * full EAX->EDX output range. + * @info: Pointer to query info; for saving the number of filled elements at @regs. + * + * A CPUID parser read function like cpuid_read_generic() or cpuid_read_0xN() uses this + * structure to save the CPUID query outputs. Actual storage for @regs and @info is + * provided by the read function caller, and is typically within the CPU's CPUID table. + * + * See struct cpuid_parse_entry.read(). + */ +struct cpuid_read_output { + struct cpuid_regs *regs; + struct leaf_parse_info *info; +}; + +/** + * struct cpuid_parse_entry - CPUID parse table entry + * @leaf: Leaf number to be parsed + * @subleaf: Subleaf number to be parsed + * @regs_offs: Offset within 'struct cpuid_leaves' for saving the CPUID query output; to be + * passed to cpuid_table_regs_p(). + * @info_offs: Offset within 'struct cpuid_leaves' for saving the CPUID query parse info; to be + * passed to cpuid_table_info_p(). + * @maxcnt: Maximum number of output storage entries available for the CPUID query. + * @read: Read function for this entry. It must save the parsed CPUID output to the passed + * 'struct cpuid_read_output'->regs array of size >= @maxcnt. It must set + * 'struct cpuid_read_output'->info.nr_entries to the number of CPUID output entries + * parsed and filled. A generic implementation is provided at cpuid_read_generic(). + */ +struct cpuid_parse_entry { + unsigned int leaf; + unsigned int subleaf; + unsigned int regs_offs; + unsigned int info_offs; + unsigned int maxcnt; + void (*read)(const struct cpuid_parse_entry *e, const struct cpuid_read_output *o); +}; + +#define __CPUID_PARSE_ENTRY(_leaf, _subleaf, _suffix, _reader_fn) \ + { \ + .leaf = _leaf, \ + .subleaf = _subleaf, \ + .regs_offs = __cpuid_leaves_regs_offset(_leaf, _suffix), \ + .info_offs = __cpuid_leaves_info_offset(_leaf, _suffix), \ + .maxcnt = __cpuid_leaves_regs_maxcnt(_leaf, _suffix), \ + .read = cpuid_read_ ## _reader_fn, \ + } + +/* + * CPUID_PARSE_ENTRY_N() is for parsing CPUID leaves with a subleaf range. + * Check __CPUID_LEAF() vs. CPUID_LEAF_N(). + */ + +#define CPUID_PARSE_ENTRY(_leaf, _subleaf, _reader_fn) \ + __CPUID_PARSE_ENTRY(_leaf, _subleaf, _subleaf, _reader_fn) + +#define CPUID_PARSE_ENTRY_N(_leaf, _reader_fn) \ + __CPUID_PARSE_ENTRY(_leaf, __cpuid_leaf_first_subleaf(_leaf), n, _reader_fn) + +/* + * CPUID parser table: + */ + +#define CPUID_PARSE_ENTRIES \ + /* Leaf Subleaf Reader function */ \ + CPUID_PARSE_ENTRY ( 0x0, 0, generic ), \ + CPUID_PARSE_ENTRY ( 0x1, 0, generic ), \ + +#endif /* _ARCH_X86_CPUID_PARSER_H */ diff --git a/arch/x86/kernel/cpu/hygon.c b/arch/x86/kernel/cpu/hygon.c index 7f95a74e4c65..3e8891a9caf2 100644 --- a/arch/x86/kernel/cpu/hygon.c +++ b/arch/x86/kernel/cpu/hygon.c @@ -10,6 +10,7 @@ #include #include +#include #include #include #include diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index f28c0efb7c8f..abb3984336eb 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -388,24 +388,15 @@ __setup("forcepae", forcepae_setup); static void intel_workarounds(struct cpuinfo_x86 *c) { -#ifdef CONFIG_X86_F00F_BUG /* * All models of Pentium and Pentium with MMX technology CPUs * have the F0 0F bug, which lets nonprivileged users lock up the - * system. Announce that the fault handler will be checking for it. + * system. The fault handler always checks for it. * The Quark is also family 5, but does not have the same bug. */ - clear_cpu_bug(c, X86_BUG_F00F); - if (c->x86_vfm >= INTEL_FAM5_START && c->x86_vfm < INTEL_QUARK_X1000) { - static int f00f_workaround_enabled; - + if (IS_ENABLED(CONFIG_X86_F00F_BUG) && + (c->x86_vfm >= INTEL_FAM5_START && c->x86_vfm < INTEL_QUARK_X1000)) set_cpu_bug(c, X86_BUG_F00F); - if (!f00f_workaround_enabled) { - pr_notice("Intel Pentium with F0 0F bug - workaround enabled.\n"); - f00f_workaround_enabled = 1; - } - } -#endif /* * SEP CPUID bug: Pentium Pro reports SEP but doesn't have it until diff --git a/arch/x86/kernel/cpu/mce/amd.c b/arch/x86/kernel/cpu/mce/amd.c index 222fa9cb181b..36e0df4d1342 100644 --- a/arch/x86/kernel/cpu/mce/amd.c +++ b/arch/x86/kernel/cpu/mce/amd.c @@ -969,13 +969,13 @@ store_threshold_limit(struct threshold_block *b, const char *buf, size_t size) static ssize_t show_error_count(struct threshold_block *b, char *buf) { - u32 lo, hi; + struct msr val; /* CPU might be offline by now */ - if (rdmsr_on_cpu(b->cpu, b->address, &lo, &hi)) + if (rdmsrq_on_cpu(b->cpu, b->address, &val.q)) return -ENODEV; - return sprintf(buf, "%u\n", ((hi & THRESHOLD_MAX) - + return sprintf(buf, "%u\n", ((val.h & THRESHOLD_MAX) - (THRESHOLD_MAX - b->threshold_limit))); } diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index 3c1faface683..9bba1e2f03af 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -49,6 +49,7 @@ #include #include +#include #include #include #include diff --git a/arch/x86/kernel/cpu/mce/inject.c b/arch/x86/kernel/cpu/mce/inject.c index d02c4f556cd0..6f8a49d8baeb 100644 --- a/arch/x86/kernel/cpu/mce/inject.c +++ b/arch/x86/kernel/cpu/mce/inject.c @@ -26,6 +26,7 @@ #include #include +#include #include #include #include @@ -316,18 +317,18 @@ static struct notifier_block inject_nb = { */ static int toggle_hw_mce_inject(unsigned int cpu, bool enable) { - u32 l, h; + struct msr val; int err; - err = rdmsr_on_cpu(cpu, MSR_K7_HWCR, &l, &h); + err = rdmsrq_on_cpu(cpu, MSR_K7_HWCR, &val.q); if (err) { pr_err("%s: error reading HWCR\n", __func__); return err; } - enable ? (l |= BIT(18)) : (l &= ~BIT(18)); + enable ? (val.l |= BIT(18)) : (val.l &= ~BIT(18)); - err = wrmsr_on_cpu(cpu, MSR_K7_HWCR, l, h); + err = wrmsrq_on_cpu(cpu, MSR_K7_HWCR, val.q); if (err) pr_err("%s: error writing HWCR\n", __func__); diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c index 5c0afae75e9f..531dfb771c8b 100644 --- a/arch/x86/kernel/cpu/microcode/amd.c +++ b/arch/x86/kernel/cpu/microcode/amd.c @@ -34,6 +34,7 @@ #include #include +#include #include #include #include @@ -233,11 +234,6 @@ static bool need_sha_check(u32 cur_rev) { u32 cutoff; - if (!cur_rev) { - cur_rev = cpuid_to_ucode_rev(bsp_cpuid_1_eax); - pr_info_once("No current revision, generating the lowest one: 0x%x\n", cur_rev); - } - cutoff = get_cutoff_revision(cur_rev); if (cutoff) return cur_rev <= cutoff; @@ -338,6 +334,13 @@ static u32 get_patch_level(void) } native_rdmsr(MSR_AMD64_PATCH_LEVEL, rev, dummy); + if (!rev) { + if (x86_family(bsp_cpuid_1_eax) < 0x17) + return rev; + + rev = cpuid_to_ucode_rev(bsp_cpuid_1_eax); + pr_info_once("No current revision, generating the lowest one: 0x%x\n", rev); + } return rev; } diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c index 45ca406a8112..0dd0c7241c57 100644 --- a/arch/x86/kernel/cpu/microcode/core.c +++ b/arch/x86/kernel/cpu/microcode/core.c @@ -34,6 +34,7 @@ #include #include +#include #include #include #include @@ -121,7 +122,7 @@ bool __init microcode_loader_disabled(void) * 1) The CPU does not support CPUID, detected below in * load_ucode_bsp(). * - * 2) Bit 31 in CPUID[1]:ECX is clear + * 2) Bit 31 in CPUID[1]:ECX is set * The bit is reserved for hypervisor use. This is still not * completely accurate as XEN PV guests don't see that CPUID bit * set, but that's good enough as they don't land on the BSP diff --git a/arch/x86/kernel/cpu/microcode/intel-ucode-defs.h b/arch/x86/kernel/cpu/microcode/intel-ucode-defs.h index 72c8809f88e5..af8b1d8b26f6 100644 --- a/arch/x86/kernel/cpu/microcode/intel-ucode-defs.h +++ b/arch/x86/kernel/cpu/microcode/intel-ucode-defs.h @@ -1,238 +1,241 @@ -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x03, .steppings = 0x0004, .platform_mask = 0x00, .driver_data = 0x2 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0001, .platform_mask = 0x01, .driver_data = 0x40 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0001, .platform_mask = 0x02, .driver_data = 0x41 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0001, .platform_mask = 0x08, .driver_data = 0x45 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x40 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x2a }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0004, .platform_mask = 0x02, .driver_data = 0x2c }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x2b }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0008, .platform_mask = 0x01, .driver_data = 0x10 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0008, .platform_mask = 0x02, .driver_data = 0xc }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0008, .platform_mask = 0x04, .driver_data = 0xb }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0008, .platform_mask = 0x08, .driver_data = 0xd }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0001, .platform_mask = 0x01, .driver_data = 0xa }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x3 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0400, .platform_mask = 0x02, .driver_data = 0xc }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0400, .platform_mask = 0x08, .driver_data = 0xd }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0400, .platform_mask = 0x20, .driver_data = 0xb }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x2000, .platform_mask = 0x02, .driver_data = 0x5 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x2000, .platform_mask = 0x08, .driver_data = 0x6 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x2000, .platform_mask = 0x20, .driver_data = 0x7 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x07, .steppings = 0x0002, .platform_mask = 0x04, .driver_data = 0x14 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x07, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x38 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x07, .steppings = 0x0008, .platform_mask = 0x04, .driver_data = 0x2e }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0xd }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0002, .platform_mask = 0x04, .driver_data = 0x10 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0002, .platform_mask = 0x08, .driver_data = 0xf }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x11 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0002, .platform_mask = 0x20, .driver_data = 0xe }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0008, .platform_mask = 0x08, .driver_data = 0x8 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0008, .platform_mask = 0x20, .driver_data = 0x7 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0040, .platform_mask = 0x01, .driver_data = 0x7 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0040, .platform_mask = 0x02, .driver_data = 0xa }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0x2 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0040, .platform_mask = 0x10, .driver_data = 0x8 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0040, .platform_mask = 0x80, .driver_data = 0xc }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0400, .platform_mask = 0x10, .driver_data = 0x1 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0400, .platform_mask = 0x20, .driver_data = 0x4 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0400, .platform_mask = 0x80, .driver_data = 0x5 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x09, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x7 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x09, .steppings = 0x0020, .platform_mask = 0x20, .driver_data = 0x7 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x09, .steppings = 0x0020, .platform_mask = 0x80, .driver_data = 0x47 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0a, .steppings = 0x0001, .platform_mask = 0x04, .driver_data = 0x3 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0a, .steppings = 0x0002, .platform_mask = 0x04, .driver_data = 0x1 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0b, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x1c }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0b, .steppings = 0x0002, .platform_mask = 0x20, .driver_data = 0x1d }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0b, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0x1 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0b, .steppings = 0x0010, .platform_mask = 0x20, .driver_data = 0x2 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0d, .steppings = 0x0040, .platform_mask = 0x20, .driver_data = 0x18 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0e, .steppings = 0x0100, .platform_mask = 0x20, .driver_data = 0x39 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0e, .steppings = 0x1000, .platform_mask = 0x20, .driver_data = 0x54 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0e, .steppings = 0x1000, .platform_mask = 0x80, .driver_data = 0x59 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x5d }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0004, .platform_mask = 0x20, .driver_data = 0x5c }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0040, .platform_mask = 0x01, .driver_data = 0xd0 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0xd2 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0040, .platform_mask = 0x20, .driver_data = 0xd1 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0080, .platform_mask = 0x10, .driver_data = 0x6a }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0080, .platform_mask = 0x40, .driver_data = 0x6b }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0400, .platform_mask = 0x80, .driver_data = 0x95 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x01, .driver_data = 0xba }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x04, .driver_data = 0xbc }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x08, .driver_data = 0xbb }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x10, .driver_data = 0xba }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x20, .driver_data = 0xba }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x40, .driver_data = 0xbc }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x80, .driver_data = 0xba }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x2000, .platform_mask = 0x01, .driver_data = 0xa4 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x2000, .platform_mask = 0x20, .driver_data = 0xa4 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x2000, .platform_mask = 0x80, .driver_data = 0xa4 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x16, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x43 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x16, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x42 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x16, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0x44 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0040, .platform_mask = 0x01, .driver_data = 0x60f }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0x60f }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0040, .platform_mask = 0x10, .driver_data = 0x60f }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0040, .platform_mask = 0x40, .driver_data = 0x60f }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0040, .platform_mask = 0x80, .driver_data = 0x60f }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0080, .platform_mask = 0x10, .driver_data = 0x70a }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0400, .platform_mask = 0x11, .driver_data = 0xa0b }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0400, .platform_mask = 0x44, .driver_data = 0xa0b }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0400, .platform_mask = 0xa0, .driver_data = 0xa0b }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1a, .steppings = 0x0010, .platform_mask = 0x03, .driver_data = 0x12 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1a, .steppings = 0x0020, .platform_mask = 0x03, .driver_data = 0x1d }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x217 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x218 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0004, .platform_mask = 0x08, .driver_data = 0x219 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x01, .driver_data = 0x107 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x04, .driver_data = 0x107 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x08, .driver_data = 0x107 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x10, .driver_data = 0x107 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1d, .steppings = 0x0002, .platform_mask = 0x08, .driver_data = 0x29 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1e, .steppings = 0x0020, .platform_mask = 0x13, .driver_data = 0xa }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x25, .steppings = 0x0004, .platform_mask = 0x12, .driver_data = 0x11 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x25, .steppings = 0x0020, .platform_mask = 0x92, .driver_data = 0x7 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x26, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x104 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x26, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x105 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2a, .steppings = 0x0080, .platform_mask = 0x12, .driver_data = 0x2f }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2c, .steppings = 0x0004, .platform_mask = 0x03, .driver_data = 0x1f }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2d, .steppings = 0x0040, .platform_mask = 0x6d, .driver_data = 0x621 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2d, .steppings = 0x0080, .platform_mask = 0x6d, .driver_data = 0x71a }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2e, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0xd }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2f, .steppings = 0x0004, .platform_mask = 0x05, .driver_data = 0x3b }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x37, .steppings = 0x0100, .platform_mask = 0x02, .driver_data = 0x838 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x37, .steppings = 0x0100, .platform_mask = 0x0c, .driver_data = 0x838 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x37, .steppings = 0x0200, .platform_mask = 0x0f, .driver_data = 0x90d }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3a, .steppings = 0x0200, .platform_mask = 0x12, .driver_data = 0x21 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3c, .steppings = 0x0008, .platform_mask = 0x32, .driver_data = 0x28 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3d, .steppings = 0x0010, .platform_mask = 0xc0, .driver_data = 0x2f }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3e, .steppings = 0x0010, .platform_mask = 0xed, .driver_data = 0x42e }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3e, .steppings = 0x0040, .platform_mask = 0xed, .driver_data = 0x600 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3e, .steppings = 0x0080, .platform_mask = 0xed, .driver_data = 0x715 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3f, .steppings = 0x0004, .platform_mask = 0x6f, .driver_data = 0x49 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3f, .steppings = 0x0010, .platform_mask = 0x80, .driver_data = 0x1a }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x45, .steppings = 0x0002, .platform_mask = 0x72, .driver_data = 0x26 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x46, .steppings = 0x0002, .platform_mask = 0x32, .driver_data = 0x1c }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x47, .steppings = 0x0002, .platform_mask = 0x22, .driver_data = 0x22 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x4c, .steppings = 0x0008, .platform_mask = 0x01, .driver_data = 0x368 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x4c, .steppings = 0x0010, .platform_mask = 0x01, .driver_data = 0x411 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x4d, .steppings = 0x0100, .platform_mask = 0x01, .driver_data = 0x12d }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x4e, .steppings = 0x0008, .platform_mask = 0xc0, .driver_data = 0xf0 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0008, .platform_mask = 0x97, .driver_data = 0x1000191 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0010, .platform_mask = 0xb7, .driver_data = 0x2007006 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0020, .platform_mask = 0xb7, .driver_data = 0x3000010 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0080, .platform_mask = 0xbf, .driver_data = 0x5003901 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0800, .platform_mask = 0xbf, .driver_data = 0x7002b01 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x56, .steppings = 0x0004, .platform_mask = 0x10, .driver_data = 0x1c }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x56, .steppings = 0x0008, .platform_mask = 0x10, .driver_data = 0x700001c }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x56, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0xf00001a }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x56, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0xe000015 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5c, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x14 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5c, .steppings = 0x0200, .platform_mask = 0x03, .driver_data = 0x48 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5c, .steppings = 0x0400, .platform_mask = 0x03, .driver_data = 0x28 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5e, .steppings = 0x0008, .platform_mask = 0x36, .driver_data = 0xf0 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5f, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x3e }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x66, .steppings = 0x0008, .platform_mask = 0x80, .driver_data = 0x2a }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x6a, .steppings = 0x0020, .platform_mask = 0x87, .driver_data = 0xc0002f0 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x6a, .steppings = 0x0040, .platform_mask = 0x87, .driver_data = 0xd000404 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x6c, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x10002d0 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x7a, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x42 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x7a, .steppings = 0x0100, .platform_mask = 0x01, .driver_data = 0x26 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x7e, .steppings = 0x0020, .platform_mask = 0x80, .driver_data = 0xca }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8a, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x33 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8c, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0xbc }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8c, .steppings = 0x0004, .platform_mask = 0xc2, .driver_data = 0x3c }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8d, .steppings = 0x0002, .platform_mask = 0xc2, .driver_data = 0x56 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0200, .platform_mask = 0x10, .driver_data = 0xf6 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0200, .platform_mask = 0xc0, .driver_data = 0xf6 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0400, .platform_mask = 0xc0, .driver_data = 0xf6 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0800, .platform_mask = 0xd0, .driver_data = 0xf6 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x1000, .platform_mask = 0x94, .driver_data = 0x100 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0x2c0003f7 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0010, .platform_mask = 0x87, .driver_data = 0x2b000639 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x2c0003f7 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0020, .platform_mask = 0x87, .driver_data = 0x2b000639 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0040, .platform_mask = 0x10, .driver_data = 0x2c0003f7 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0040, .platform_mask = 0x87, .driver_data = 0x2b000639 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0080, .platform_mask = 0x87, .driver_data = 0x2b000639 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0100, .platform_mask = 0x10, .driver_data = 0x2c0003f7 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0100, .platform_mask = 0x87, .driver_data = 0x2b000639 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x96, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x1a }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x97, .steppings = 0x0004, .platform_mask = 0x07, .driver_data = 0x3a }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x97, .steppings = 0x0020, .platform_mask = 0x07, .driver_data = 0x3a }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9a, .steppings = 0x0008, .platform_mask = 0x80, .driver_data = 0x437 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9a, .steppings = 0x0010, .platform_mask = 0x40, .driver_data = 0xa }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9a, .steppings = 0x0010, .platform_mask = 0x80, .driver_data = 0x437 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9c, .steppings = 0x0001, .platform_mask = 0x01, .driver_data = 0x24000026 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x0200, .platform_mask = 0x2a, .driver_data = 0xf8 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x0400, .platform_mask = 0x22, .driver_data = 0xfa }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x0800, .platform_mask = 0x02, .driver_data = 0xf6 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x1000, .platform_mask = 0x22, .driver_data = 0xf8 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x2000, .platform_mask = 0x22, .driver_data = 0x104 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa5, .steppings = 0x0004, .platform_mask = 0x20, .driver_data = 0x100 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa5, .steppings = 0x0008, .platform_mask = 0x22, .driver_data = 0x100 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa5, .steppings = 0x0020, .platform_mask = 0x22, .driver_data = 0x100 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa6, .steppings = 0x0001, .platform_mask = 0x80, .driver_data = 0x102 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa6, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0x100 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa7, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x64 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xaa, .steppings = 0x0010, .platform_mask = 0xe6, .driver_data = 0x24 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xad, .steppings = 0x0002, .platform_mask = 0x20, .driver_data = 0xa0000d1 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xad, .steppings = 0x0002, .platform_mask = 0x95, .driver_data = 0x10003a2 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xaf, .steppings = 0x0008, .platform_mask = 0x01, .driver_data = 0x3000341 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xb5, .steppings = 0x0001, .platform_mask = 0x80, .driver_data = 0xa }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xb7, .steppings = 0x0002, .platform_mask = 0x32, .driver_data = 0x12f }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xb7, .steppings = 0x0010, .platform_mask = 0x32, .driver_data = 0x12f }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xba, .steppings = 0x0004, .platform_mask = 0xe0, .driver_data = 0x4128 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xba, .steppings = 0x0008, .platform_mask = 0xe0, .driver_data = 0x4128 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xba, .steppings = 0x0100, .platform_mask = 0xe0, .driver_data = 0x4128 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbd, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0x11f }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbe, .steppings = 0x0001, .platform_mask = 0x19, .driver_data = 0x1d }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0004, .platform_mask = 0x07, .driver_data = 0x3a }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0020, .platform_mask = 0x07, .driver_data = 0x3a }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0040, .platform_mask = 0x07, .driver_data = 0x3a }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0080, .platform_mask = 0x07, .driver_data = 0x3a }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xc5, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x118 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xc6, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x118 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xc6, .steppings = 0x0010, .platform_mask = 0x82, .driver_data = 0x118 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xca, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x118 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xcf, .steppings = 0x0002, .platform_mask = 0x87, .driver_data = 0x210002a9 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xcf, .steppings = 0x0004, .platform_mask = 0x87, .driver_data = 0x210002a9 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0080, .platform_mask = 0x01, .driver_data = 0x12 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0080, .platform_mask = 0x02, .driver_data = 0x8 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0400, .platform_mask = 0x01, .driver_data = 0x13 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0400, .platform_mask = 0x02, .driver_data = 0x15 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0400, .platform_mask = 0x04, .driver_data = 0x14 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x01, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x2e }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0010, .platform_mask = 0x02, .driver_data = 0x1f }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0010, .platform_mask = 0x04, .driver_data = 0x1e }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0x21 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0020, .platform_mask = 0x01, .driver_data = 0x29 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0020, .platform_mask = 0x02, .driver_data = 0x2a }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0020, .platform_mask = 0x04, .driver_data = 0x2b }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x2c }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0040, .platform_mask = 0x02, .driver_data = 0x10 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0080, .platform_mask = 0x02, .driver_data = 0x38 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0080, .platform_mask = 0x04, .driver_data = 0x37 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0080, .platform_mask = 0x08, .driver_data = 0x39 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0200, .platform_mask = 0x02, .driver_data = 0x2d }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0200, .platform_mask = 0x04, .driver_data = 0x2e }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0200, .platform_mask = 0x08, .driver_data = 0x2f }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x03, .steppings = 0x0004, .platform_mask = 0x0d, .driver_data = 0xa }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x03, .steppings = 0x0008, .platform_mask = 0x0d, .driver_data = 0xc }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x03, .steppings = 0x0010, .platform_mask = 0x1d, .driver_data = 0x17 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x16 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0002, .platform_mask = 0xbd, .driver_data = 0x17 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0008, .platform_mask = 0x9d, .driver_data = 0x5 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0010, .platform_mask = 0x9d, .driver_data = 0x6 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0080, .platform_mask = 0x9d, .driver_data = 0x3 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0100, .platform_mask = 0x01, .driver_data = 0xc }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0100, .platform_mask = 0x02, .driver_data = 0xe }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0100, .platform_mask = 0x5f, .driver_data = 0x7 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0200, .platform_mask = 0xbd, .driver_data = 0x3 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0400, .platform_mask = 0x5c, .driver_data = 0x4 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0400, .platform_mask = 0x5d, .driver_data = 0x2 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0xf }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0010, .platform_mask = 0x01, .driver_data = 0x2 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0010, .platform_mask = 0x34, .driver_data = 0x4 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0020, .platform_mask = 0x01, .driver_data = 0x8 }, -{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0100, .platform_mask = 0x22, .driver_data = 0x9 }, +/* SPDX-License-Identifier: GPL-2.0 */ +/* Auto-generated by scripts/update-intel-ucode-defs.py */ +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x03, .steppings = 0x0004, .platform_mask = 0x00, .driver_data = 0x2 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0001, .platform_mask = 0x01, .driver_data = 0x40 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0001, .platform_mask = 0x02, .driver_data = 0x41 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0001, .platform_mask = 0x08, .driver_data = 0x45 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x40 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x2a }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0004, .platform_mask = 0x02, .driver_data = 0x2c }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x2b }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0008, .platform_mask = 0x01, .driver_data = 0x10 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0008, .platform_mask = 0x02, .driver_data = 0xc }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0008, .platform_mask = 0x04, .driver_data = 0xb }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0008, .platform_mask = 0x08, .driver_data = 0xd }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0001, .platform_mask = 0x01, .driver_data = 0xa }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x3 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0400, .platform_mask = 0x02, .driver_data = 0xc }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0400, .platform_mask = 0x08, .driver_data = 0xd }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0400, .platform_mask = 0x20, .driver_data = 0xb }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x2000, .platform_mask = 0x02, .driver_data = 0x5 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x2000, .platform_mask = 0x08, .driver_data = 0x6 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x2000, .platform_mask = 0x20, .driver_data = 0x7 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x07, .steppings = 0x0002, .platform_mask = 0x04, .driver_data = 0x14 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x07, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x38 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x07, .steppings = 0x0008, .platform_mask = 0x04, .driver_data = 0x2e }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0xd }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0002, .platform_mask = 0x04, .driver_data = 0x10 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0002, .platform_mask = 0x08, .driver_data = 0xf }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x11 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0002, .platform_mask = 0x20, .driver_data = 0xe }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0008, .platform_mask = 0x08, .driver_data = 0x8 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0008, .platform_mask = 0x20, .driver_data = 0x7 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0040, .platform_mask = 0x01, .driver_data = 0x7 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0040, .platform_mask = 0x02, .driver_data = 0xa }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0x2 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0040, .platform_mask = 0x10, .driver_data = 0x8 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0040, .platform_mask = 0x80, .driver_data = 0xc }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0400, .platform_mask = 0x10, .driver_data = 0x1 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0400, .platform_mask = 0x20, .driver_data = 0x4 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0400, .platform_mask = 0x80, .driver_data = 0x5 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x09, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x7 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x09, .steppings = 0x0020, .platform_mask = 0x20, .driver_data = 0x7 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x09, .steppings = 0x0020, .platform_mask = 0x80, .driver_data = 0x47 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0a, .steppings = 0x0001, .platform_mask = 0x04, .driver_data = 0x3 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0a, .steppings = 0x0002, .platform_mask = 0x04, .driver_data = 0x1 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0b, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x1c }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0b, .steppings = 0x0002, .platform_mask = 0x20, .driver_data = 0x1d }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0b, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0x1 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0b, .steppings = 0x0010, .platform_mask = 0x20, .driver_data = 0x2 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0d, .steppings = 0x0040, .platform_mask = 0x20, .driver_data = 0x18 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0e, .steppings = 0x0100, .platform_mask = 0x20, .driver_data = 0x39 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0e, .steppings = 0x1000, .platform_mask = 0x20, .driver_data = 0x54 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0e, .steppings = 0x1000, .platform_mask = 0x80, .driver_data = 0x59 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x5d }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0004, .platform_mask = 0x20, .driver_data = 0x5c }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0040, .platform_mask = 0x01, .driver_data = 0xd0 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0xd2 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0040, .platform_mask = 0x20, .driver_data = 0xd1 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0080, .platform_mask = 0x10, .driver_data = 0x6a }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0080, .platform_mask = 0x40, .driver_data = 0x6b }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0400, .platform_mask = 0x80, .driver_data = 0x95 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x01, .driver_data = 0xba }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x04, .driver_data = 0xbc }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x08, .driver_data = 0xbb }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x10, .driver_data = 0xba }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x20, .driver_data = 0xba }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x40, .driver_data = 0xbc }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x80, .driver_data = 0xba }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x2000, .platform_mask = 0x01, .driver_data = 0xa4 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x2000, .platform_mask = 0x20, .driver_data = 0xa4 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x2000, .platform_mask = 0x80, .driver_data = 0xa4 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x16, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x43 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x16, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x42 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x16, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0x44 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0040, .platform_mask = 0x01, .driver_data = 0x60f }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0x60f }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0040, .platform_mask = 0x10, .driver_data = 0x60f }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0040, .platform_mask = 0x40, .driver_data = 0x60f }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0040, .platform_mask = 0x80, .driver_data = 0x60f }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0080, .platform_mask = 0x10, .driver_data = 0x70a }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0400, .platform_mask = 0x11, .driver_data = 0xa0b }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0400, .platform_mask = 0x44, .driver_data = 0xa0b }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0400, .platform_mask = 0xa0, .driver_data = 0xa0b }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1a, .steppings = 0x0010, .platform_mask = 0x03, .driver_data = 0x12 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1a, .steppings = 0x0020, .platform_mask = 0x03, .driver_data = 0x1d }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x217 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x218 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0004, .platform_mask = 0x08, .driver_data = 0x219 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x01, .driver_data = 0x107 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x04, .driver_data = 0x107 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x08, .driver_data = 0x107 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x10, .driver_data = 0x107 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1d, .steppings = 0x0002, .platform_mask = 0x08, .driver_data = 0x29 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1e, .steppings = 0x0020, .platform_mask = 0x13, .driver_data = 0xa }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x25, .steppings = 0x0004, .platform_mask = 0x12, .driver_data = 0x11 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x25, .steppings = 0x0020, .platform_mask = 0x92, .driver_data = 0x7 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x26, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x104 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x26, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x105 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2a, .steppings = 0x0080, .platform_mask = 0x12, .driver_data = 0x2f }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2c, .steppings = 0x0004, .platform_mask = 0x03, .driver_data = 0x1f }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2d, .steppings = 0x0040, .platform_mask = 0x6d, .driver_data = 0x621 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2d, .steppings = 0x0080, .platform_mask = 0x6d, .driver_data = 0x71a }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2e, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0xd }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2f, .steppings = 0x0004, .platform_mask = 0x05, .driver_data = 0x3b }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x37, .steppings = 0x0100, .platform_mask = 0x02, .driver_data = 0x838 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x37, .steppings = 0x0100, .platform_mask = 0x0c, .driver_data = 0x838 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x37, .steppings = 0x0200, .platform_mask = 0x0f, .driver_data = 0x90d }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3a, .steppings = 0x0200, .platform_mask = 0x12, .driver_data = 0x21 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3c, .steppings = 0x0008, .platform_mask = 0x32, .driver_data = 0x28 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3d, .steppings = 0x0010, .platform_mask = 0xc0, .driver_data = 0x2f }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3e, .steppings = 0x0010, .platform_mask = 0xed, .driver_data = 0x42e }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3e, .steppings = 0x0040, .platform_mask = 0xed, .driver_data = 0x600 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3e, .steppings = 0x0080, .platform_mask = 0xed, .driver_data = 0x715 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3f, .steppings = 0x0004, .platform_mask = 0x6f, .driver_data = 0x49 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3f, .steppings = 0x0010, .platform_mask = 0x80, .driver_data = 0x1a }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x45, .steppings = 0x0002, .platform_mask = 0x72, .driver_data = 0x26 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x46, .steppings = 0x0002, .platform_mask = 0x32, .driver_data = 0x1c }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x47, .steppings = 0x0002, .platform_mask = 0x22, .driver_data = 0x22 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x4c, .steppings = 0x0008, .platform_mask = 0x01, .driver_data = 0x368 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x4c, .steppings = 0x0010, .platform_mask = 0x01, .driver_data = 0x411 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x4d, .steppings = 0x0100, .platform_mask = 0x01, .driver_data = 0x12d }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x4e, .steppings = 0x0008, .platform_mask = 0xc0, .driver_data = 0xf0 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0008, .platform_mask = 0x97, .driver_data = 0x1000191 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0010, .platform_mask = 0xb7, .driver_data = 0x2007006 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0020, .platform_mask = 0xb7, .driver_data = 0x3000010 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0080, .platform_mask = 0xbf, .driver_data = 0x5003901 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0800, .platform_mask = 0xbf, .driver_data = 0x7002b01 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x56, .steppings = 0x0004, .platform_mask = 0x10, .driver_data = 0x1c }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x56, .steppings = 0x0008, .platform_mask = 0x10, .driver_data = 0x700001c }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x56, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0xf00001a }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x56, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0xe000015 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5c, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x14 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5c, .steppings = 0x0200, .platform_mask = 0x03, .driver_data = 0x48 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5c, .steppings = 0x0400, .platform_mask = 0x03, .driver_data = 0x28 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5e, .steppings = 0x0008, .platform_mask = 0x36, .driver_data = 0xf0 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5f, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x3e }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x66, .steppings = 0x0008, .platform_mask = 0x80, .driver_data = 0x2a }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x6a, .steppings = 0x0020, .platform_mask = 0x87, .driver_data = 0xc0002f0 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x6a, .steppings = 0x0040, .platform_mask = 0x87, .driver_data = 0xd000410 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x6c, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x10002e0 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x7a, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x42 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x7a, .steppings = 0x0100, .platform_mask = 0x01, .driver_data = 0x26 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x7e, .steppings = 0x0020, .platform_mask = 0x80, .driver_data = 0xca }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8a, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x33 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8c, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0xbc }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8c, .steppings = 0x0004, .platform_mask = 0xc2, .driver_data = 0x3c }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8d, .steppings = 0x0002, .platform_mask = 0xc2, .driver_data = 0x56 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0200, .platform_mask = 0x10, .driver_data = 0xf6 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0200, .platform_mask = 0xc0, .driver_data = 0xf6 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0400, .platform_mask = 0xc0, .driver_data = 0xf6 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0800, .platform_mask = 0xd0, .driver_data = 0xf6 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x1000, .platform_mask = 0x94, .driver_data = 0x100 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0x2c000410 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0010, .platform_mask = 0x87, .driver_data = 0x2b000650 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x2c000410 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0020, .platform_mask = 0x87, .driver_data = 0x2b000650 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0040, .platform_mask = 0x10, .driver_data = 0x2c000410 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0040, .platform_mask = 0x87, .driver_data = 0x2b000650 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0080, .platform_mask = 0x87, .driver_data = 0x2b000650 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0100, .platform_mask = 0x10, .driver_data = 0x2c000410 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0100, .platform_mask = 0x87, .driver_data = 0x2b000650 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x96, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x1a }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x97, .steppings = 0x0004, .platform_mask = 0x07, .driver_data = 0x3d }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x97, .steppings = 0x0020, .platform_mask = 0x07, .driver_data = 0x3d }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9a, .steppings = 0x0008, .platform_mask = 0x80, .driver_data = 0x43a }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9a, .steppings = 0x0010, .platform_mask = 0x40, .driver_data = 0xb }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9a, .steppings = 0x0010, .platform_mask = 0x80, .driver_data = 0x43a }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9c, .steppings = 0x0001, .platform_mask = 0x01, .driver_data = 0x24000026 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x0200, .platform_mask = 0x2a, .driver_data = 0xf8 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x0400, .platform_mask = 0x22, .driver_data = 0xfa }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x0800, .platform_mask = 0x02, .driver_data = 0xf6 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x1000, .platform_mask = 0x22, .driver_data = 0xf8 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x2000, .platform_mask = 0x22, .driver_data = 0x104 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa5, .steppings = 0x0004, .platform_mask = 0x20, .driver_data = 0x100 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa5, .steppings = 0x0008, .platform_mask = 0x22, .driver_data = 0x100 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa5, .steppings = 0x0020, .platform_mask = 0x22, .driver_data = 0x100 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa6, .steppings = 0x0001, .platform_mask = 0x80, .driver_data = 0x102 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa6, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0x100 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa7, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x64 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xaa, .steppings = 0x0010, .platform_mask = 0xe6, .driver_data = 0x25 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xad, .steppings = 0x0002, .platform_mask = 0x20, .driver_data = 0xa000124 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xad, .steppings = 0x0002, .platform_mask = 0x95, .driver_data = 0x10003f0 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xae, .steppings = 0x0002, .platform_mask = 0x97, .driver_data = 0x1000273 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xaf, .steppings = 0x0008, .platform_mask = 0x01, .driver_data = 0x3000382 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xb5, .steppings = 0x0001, .platform_mask = 0x80, .driver_data = 0xa }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xb7, .steppings = 0x0002, .platform_mask = 0x32, .driver_data = 0x132 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xb7, .steppings = 0x0010, .platform_mask = 0x32, .driver_data = 0x132 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xba, .steppings = 0x0004, .platform_mask = 0xe0, .driver_data = 0x6133 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xba, .steppings = 0x0008, .platform_mask = 0xe0, .driver_data = 0x6133 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xba, .steppings = 0x0100, .platform_mask = 0xe0, .driver_data = 0x6133 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbd, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0x125 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbe, .steppings = 0x0001, .platform_mask = 0x19, .driver_data = 0x1e }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0004, .platform_mask = 0x07, .driver_data = 0x3d }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0020, .platform_mask = 0x07, .driver_data = 0x3d }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0040, .platform_mask = 0x07, .driver_data = 0x3d }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0080, .platform_mask = 0x07, .driver_data = 0x3d }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xc5, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x11a }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xc6, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x11a }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xc6, .steppings = 0x0010, .platform_mask = 0x82, .driver_data = 0x11a }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xca, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x11a }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xcf, .steppings = 0x0002, .platform_mask = 0x87, .driver_data = 0x210002c0 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xcf, .steppings = 0x0004, .platform_mask = 0x87, .driver_data = 0x210002c0 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0080, .platform_mask = 0x01, .driver_data = 0x12 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0080, .platform_mask = 0x02, .driver_data = 0x8 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0400, .platform_mask = 0x01, .driver_data = 0x13 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0400, .platform_mask = 0x02, .driver_data = 0x15 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0400, .platform_mask = 0x04, .driver_data = 0x14 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x01, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x2e }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0010, .platform_mask = 0x02, .driver_data = 0x1f }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0010, .platform_mask = 0x04, .driver_data = 0x1e }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0x21 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0020, .platform_mask = 0x01, .driver_data = 0x29 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0020, .platform_mask = 0x02, .driver_data = 0x2a }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0020, .platform_mask = 0x04, .driver_data = 0x2b }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x2c }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0040, .platform_mask = 0x02, .driver_data = 0x10 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0080, .platform_mask = 0x02, .driver_data = 0x38 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0080, .platform_mask = 0x04, .driver_data = 0x37 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0080, .platform_mask = 0x08, .driver_data = 0x39 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0200, .platform_mask = 0x02, .driver_data = 0x2d }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0200, .platform_mask = 0x04, .driver_data = 0x2e }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0200, .platform_mask = 0x08, .driver_data = 0x2f }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x03, .steppings = 0x0004, .platform_mask = 0x0d, .driver_data = 0xa }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x03, .steppings = 0x0008, .platform_mask = 0x0d, .driver_data = 0xc }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x03, .steppings = 0x0010, .platform_mask = 0x1d, .driver_data = 0x17 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x16 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0002, .platform_mask = 0xbd, .driver_data = 0x17 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0008, .platform_mask = 0x9d, .driver_data = 0x5 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0010, .platform_mask = 0x9d, .driver_data = 0x6 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0080, .platform_mask = 0x9d, .driver_data = 0x3 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0100, .platform_mask = 0x01, .driver_data = 0xc }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0100, .platform_mask = 0x02, .driver_data = 0xe }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0100, .platform_mask = 0x5f, .driver_data = 0x7 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0200, .platform_mask = 0xbd, .driver_data = 0x3 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0400, .platform_mask = 0x5c, .driver_data = 0x4 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0400, .platform_mask = 0x5d, .driver_data = 0x2 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0xf }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0010, .platform_mask = 0x01, .driver_data = 0x2 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0010, .platform_mask = 0x34, .driver_data = 0x4 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0020, .platform_mask = 0x01, .driver_data = 0x8 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0100, .platform_mask = 0x22, .driver_data = 0x9 }, diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c index a4c0a0cf928b..f4a444e6114d 100644 --- a/arch/x86/kernel/cpu/microcode/intel.c +++ b/arch/x86/kernel/cpu/microcode/intel.c @@ -25,6 +25,7 @@ #include #include +#include #include #include #include diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c index 9381102e884a..185d4f677ec0 100644 --- a/arch/x86/kernel/cpu/mshyperv.c +++ b/arch/x86/kernel/cpu/mshyperv.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c index 7667cf7c4e94..9c01d2562b7a 100644 --- a/arch/x86/kernel/cpu/resctrl/core.c +++ b/arch/x86/kernel/cpu/resctrl/core.c @@ -22,6 +22,7 @@ #include #include +#include #include #include #include "internal.h" diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c index 59215fef3924..03ee6102ab07 100644 --- a/arch/x86/kernel/cpu/resctrl/monitor.c +++ b/arch/x86/kernel/cpu/resctrl/monitor.c @@ -21,6 +21,7 @@ #include #include +#include #include #include "internal.h" @@ -301,7 +302,7 @@ static int __cntr_id_read(u32 cntr_id, u64 *val) * is set if the counter data is unavailable. */ wrmsr(MSR_IA32_QM_EVTSEL, ABMC_EXTENDED_EVT_ID | ABMC_EVT_ID, cntr_id); - rdmsrl(MSR_IA32_QM_CTR, msr_val); + rdmsrq(MSR_IA32_QM_CTR, msr_val); if (msr_val & RMID_VAL_ERROR) return -EIO; @@ -459,6 +460,7 @@ int __init rdt_get_l3_mon_config(struct rdt_resource *r) (rdt_cpu_has(X86_FEATURE_CQM_MBM_TOTAL) || rdt_cpu_has(X86_FEATURE_CQM_MBM_LOCAL))) { r->mon.mbm_cntr_assignable = true; + r->mon.mbm_cntr_configurable = true; cpuid_count(0x80000020, 5, &eax, &ebx, &ecx, &edx); r->mon.num_mbm_cntrs = (ebx & GENMASK(15, 0)) + 1; hw_res->mbm_cntr_assign_enabled = true; @@ -532,7 +534,7 @@ static void resctrl_abmc_config_one_amd(void *info) { union l3_qos_abmc_cfg *abmc_cfg = info; - wrmsrl(MSR_IA32_L3_QOS_ABMC_CFG, abmc_cfg->full); + wrmsrq(MSR_IA32_L3_QOS_ABMC_CFG, abmc_cfg->full); } /* diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c index 837d6a4b0c28..937129ce6a96 100644 --- a/arch/x86/kernel/cpu/scattered.c +++ b/arch/x86/kernel/cpu/scattered.c @@ -6,6 +6,7 @@ #include #include +#include #include #include "cpu.h" diff --git a/arch/x86/kernel/cpu/sgx/driver.c b/arch/x86/kernel/cpu/sgx/driver.c index 473619741bc4..9268289cd9f9 100644 --- a/arch/x86/kernel/cpu/sgx/driver.c +++ b/arch/x86/kernel/cpu/sgx/driver.c @@ -6,7 +6,10 @@ #include #include #include + +#include #include + #include "driver.h" #include "encl.h" diff --git a/arch/x86/kernel/cpu/sgx/main.c b/arch/x86/kernel/cpu/sgx/main.c index 38b7fd2f63be..4505f808af5e 100644 --- a/arch/x86/kernel/cpu/sgx/main.c +++ b/arch/x86/kernel/cpu/sgx/main.c @@ -15,9 +15,12 @@ #include #include #include + +#include #include #include #include + #include "driver.h" #include "encl.h" #include "encls.h" diff --git a/arch/x86/kernel/cpu/topology_amd.c b/arch/x86/kernel/cpu/topology_amd.c index 6ac097e13106..da080d732e10 100644 --- a/arch/x86/kernel/cpu/topology_amd.c +++ b/arch/x86/kernel/cpu/topology_amd.c @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -79,7 +80,7 @@ static bool parse_8000_001e(struct topo_scan *tscan) if (!boot_cpu_has(X86_FEATURE_TOPOEXT)) return false; - cpuid_leaf(0x8000001e, &leaf); + cpuid_read(0x8000001e, &leaf); /* * If leaf 0xb/0x26 is available, then the APIC ID and the domain diff --git a/arch/x86/kernel/cpu/topology_common.c b/arch/x86/kernel/cpu/topology_common.c index d0d79d5b8eb9..cf7513416b70 100644 --- a/arch/x86/kernel/cpu/topology_common.c +++ b/arch/x86/kernel/cpu/topology_common.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include "cpu.h" diff --git a/arch/x86/kernel/cpu/topology_ext.c b/arch/x86/kernel/cpu/topology_ext.c index 467b0326bf1a..60dfaa02ffd0 100644 --- a/arch/x86/kernel/cpu/topology_ext.c +++ b/arch/x86/kernel/cpu/topology_ext.c @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -70,7 +71,7 @@ static inline bool topo_subleaf(struct topo_scan *tscan, u32 leaf, u32 subleaf, default: return false; } - cpuid_subleaf(leaf, subleaf, &sl); + cpuid_read_subleaf(leaf, subleaf, &sl); if (!sl.num_processors || sl.type == INVALID_TYPE) return false; diff --git a/arch/x86/kernel/cpu/transmeta.c b/arch/x86/kernel/cpu/transmeta.c index 42c939827621..1fdcd69c625c 100644 --- a/arch/x86/kernel/cpu/transmeta.c +++ b/arch/x86/kernel/cpu/transmeta.c @@ -3,8 +3,11 @@ #include #include #include + #include +#include #include + #include "cpu.h" static void early_init_transmeta(struct cpuinfo_x86 *c) diff --git a/arch/x86/kernel/cpu/umc.c b/arch/x86/kernel/cpu/umc.c deleted file mode 100644 index 65a58a390fc3..000000000000 --- a/arch/x86/kernel/cpu/umc.c +++ /dev/null @@ -1,26 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include -#include -#include "cpu.h" - -/* - * UMC chips appear to be only either 386 or 486, - * so no special init takes place. - */ - -static const struct cpu_dev umc_cpu_dev = { - .c_vendor = "UMC", - .c_ident = { "UMC UMC UMC" }, - .legacy_models = { - { .family = 4, .model_names = - { - [1] = "U5D", - [2] = "U5S", - } - }, - }, - .c_x86_vendor = X86_VENDOR_UMC, -}; - -cpu_dev_register(umc_cpu_dev); - diff --git a/arch/x86/kernel/cpu/vmware.c b/arch/x86/kernel/cpu/vmware.c index eee0d1a48802..34b73573b108 100644 --- a/arch/x86/kernel/cpu/vmware.c +++ b/arch/x86/kernel/cpu/vmware.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/x86/kernel/cpu/zhaoxin.c b/arch/x86/kernel/cpu/zhaoxin.c index 031379b7d4fa..761aef5590ac 100644 --- a/arch/x86/kernel/cpu/zhaoxin.c +++ b/arch/x86/kernel/cpu/zhaoxin.c @@ -4,6 +4,7 @@ #include #include +#include #include #include "cpu.h" diff --git a/arch/x86/kernel/cpuid.c b/arch/x86/kernel/cpuid.c index dae436253de4..b55fe9c7359a 100644 --- a/arch/x86/kernel/cpuid.c +++ b/arch/x86/kernel/cpuid.c @@ -37,6 +37,7 @@ #include #include +#include #include #include @@ -58,8 +59,7 @@ static void cpuid_smp_cpuid(void *cmd_block) complete(&cmd->done); } -static ssize_t cpuid_read(struct file *file, char __user *buf, - size_t count, loff_t *ppos) +static ssize_t cpuid_read_f(struct file *file, char __user *buf, size_t count, loff_t *ppos) { char __user *tmp = buf; struct cpuid_regs_done cmd; @@ -119,7 +119,7 @@ static int cpuid_open(struct inode *inode, struct file *file) static const struct file_operations cpuid_fops = { .owner = THIS_MODULE, .llseek = no_seek_end_llseek, - .read = cpuid_read, + .read = cpuid_read_f, .open = cpuid_open, }; diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 608983806fd7..584fb9913be4 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -558,11 +558,6 @@ static inline void fpstate_init_fstate(struct fpstate *fpstate) */ void fpstate_init_user(struct fpstate *fpstate) { - if (!cpu_feature_enabled(X86_FEATURE_FPU)) { - fpstate_init_soft(&fpstate->regs.soft); - return; - } - xstate_init_xcomp_bv(&fpstate->regs.xsave, fpstate->xfeatures); if (cpu_feature_enabled(X86_FEATURE_FXSR)) diff --git a/arch/x86/kernel/fpu/init.c b/arch/x86/kernel/fpu/init.c index ff988b9ea39f..0d33c217b71c 100644 --- a/arch/x86/kernel/fpu/init.c +++ b/arch/x86/kernel/fpu/init.c @@ -36,12 +36,7 @@ static void fpu__init_cpu_generic(void) write_cr0(cr0); /* Flush out any pending x87 state: */ -#ifdef CONFIG_MATH_EMULATION - if (!boot_cpu_has(X86_FEATURE_FPU)) - ; - else -#endif - asm volatile ("fninit"); + asm volatile ("fninit"); } /* @@ -86,13 +81,11 @@ static void __init fpu__init_system_early_generic(void) setup_clear_cpu_cap(X86_FEATURE_FPU); } -#ifndef CONFIG_MATH_EMULATION if (!test_cpu_cap(&boot_cpu_data, X86_FEATURE_FPU)) { pr_emerg("x86/fpu: Giving up, no FPU found and no math emulation present\n"); for (;;) asm volatile("hlt"); } -#endif } /* diff --git a/arch/x86/kernel/i8253.c b/arch/x86/kernel/i8253.c index cb9852ad6098..0b991035f127 100644 --- a/arch/x86/kernel/i8253.c +++ b/arch/x86/kernel/i8253.c @@ -31,7 +31,7 @@ struct clock_event_device *global_clock_event; */ static bool __init use_pit(void) { - if (!IS_ENABLED(CONFIG_X86_TSC) || !boot_cpu_has(X86_FEATURE_TSC)) + if (!boot_cpu_has(X86_FEATURE_TSC)) return true; /* This also returns true when APIC is disabled */ diff --git a/arch/x86/kernel/jailhouse.c b/arch/x86/kernel/jailhouse.c index 9e9a591a5fec..f58ce9220e0f 100644 --- a/arch/x86/kernel/jailhouse.c +++ b/arch/x86/kernel/jailhouse.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index d1f3f320168c..dcef84da304b 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c index 4469c784eaa0..34bdb752f892 100644 --- a/arch/x86/kernel/msr.c +++ b/arch/x86/kernel/msr.c @@ -53,7 +53,7 @@ static ssize_t msr_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) { u32 __user *tmp = (u32 __user *) buf; - u32 data[2]; + u64 data; u32 reg = *ppos; int cpu = iminor(file_inode(file)); int err = 0; @@ -63,7 +63,7 @@ static ssize_t msr_read(struct file *file, char __user *buf, return -EINVAL; /* Invalid chunk size */ for (; count; count -= 8) { - err = rdmsr_safe_on_cpu(cpu, reg, &data[0], &data[1]); + err = rdmsrq_safe_on_cpu(cpu, reg, &data); if (err) break; if (copy_to_user(tmp, &data, 8)) { @@ -109,7 +109,7 @@ static ssize_t msr_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { const u32 __user *tmp = (const u32 __user *)buf; - u32 data[2]; + u64 data; u32 reg = *ppos; int cpu = iminor(file_inode(file)); int err = 0; @@ -134,7 +134,7 @@ static ssize_t msr_write(struct file *file, const char __user *buf, add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK); - err = wrmsr_safe_on_cpu(cpu, reg, data[0], data[1]); + err = wrmsrq_safe_on_cpu(cpu, reg, data); if (err) break; diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c index 22f72034470f..00b59d774389 100644 --- a/arch/x86/kernel/paravirt.c +++ b/arch/x86/kernel/paravirt.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/x86/kernel/pmem.c b/arch/x86/kernel/pmem.c index 23154d24b117..04fb221716ff 100644 --- a/arch/x86/kernel/pmem.c +++ b/arch/x86/kernel/pmem.c @@ -27,6 +27,8 @@ static __init int register_e820_pmem(void) * simply here to trigger the module to load on demand. */ pdev = platform_device_alloc("e820_pmem", -1); + if (!pdev) + return -ENOMEM; rc = platform_device_add(pdev); if (rc) diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 4c718f8adc59..a554f19c9973 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -969,7 +969,7 @@ void amd_e400_c1e_apic_setup(void) void __init arch_post_acpi_subsys_init(void) { - u32 lo, hi; + u64 val; if (!boot_cpu_has_bug(X86_BUG_AMD_E400)) return; @@ -979,8 +979,8 @@ void __init arch_post_acpi_subsys_init(void) * the machine is affected K8_INTP_C1E_ACTIVE_MASK bits are set in * MSR_K8_INT_PENDING_MSG. */ - rdmsr(MSR_K8_INT_PENDING_MSG, lo, hi); - if (!(lo & K8_INTP_C1E_ACTIVE_MASK)) + rdmsrq(MSR_K8_INT_PENDING_MSG, val); + if (!(val & K8_INTP_C1E_ACTIVE_MASK)) return; boot_cpu_set_bug(X86_BUG_AMD_APIC_C1E); diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index b85e715ebb30..d44afbe005bb 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -708,7 +708,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p) /* Reset hw history on AMD CPUs */ if (cpu_feature_enabled(X86_FEATURE_AMD_WORKLOAD_CLASS)) - wrmsrl(MSR_AMD_WORKLOAD_HRST, 0x1); + wrmsrq(MSR_AMD_WORKLOAD_HRST, 0x1); return prev_p; } diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index 0ca3912ecb7f..30aa8369957e 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -1475,13 +1475,6 @@ DEFINE_IDTENTRY(exc_coprocessor_error) DEFINE_IDTENTRY(exc_simd_coprocessor_error) { - if (IS_ENABLED(CONFIG_X86_INVD_BUG)) { - /* AMD 486 bug: INVD in CPL 0 raises #XF instead of #GP */ - if (!static_cpu_has(X86_FEATURE_XMM)) { - __exc_general_protection(regs, 0); - return; - } - } math_error(regs, X86_TRAP_XF); } @@ -1550,20 +1543,6 @@ DEFINE_IDTENTRY(exc_device_not_available) if (handle_xfd_event(regs)) return; -#ifdef CONFIG_MATH_EMULATION - if (!boot_cpu_has(X86_FEATURE_FPU) && (cr0 & X86_CR0_EM)) { - struct math_emu_info info = { }; - - cond_local_irq_enable(regs); - - info.regs = regs; - math_emulate(&info); - - cond_local_irq_disable(regs); - return; - } -#endif - /* This should not happen. */ if (WARN(cr0 & X86_CR0_TS, "CR0.TS was set")) { /* Try to fix it up and carry on. */ diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index c5110eb554bc..ce10ae4b298b 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -298,30 +298,17 @@ notrace u64 sched_clock(void) preempt_enable_notrace(); return now; } - int check_tsc_unstable(void) { return tsc_unstable; } EXPORT_SYMBOL_GPL(check_tsc_unstable); -#ifdef CONFIG_X86_TSC int __init notsc_setup(char *str) { mark_tsc_unstable("boot parameter notsc"); return 1; } -#else -/* - * disable flag for tsc. Takes effect by clearing the TSC cpu flag - * in cpu/common.c - */ -int __init notsc_setup(char *str) -{ - setup_clear_cpu_cap(X86_FEATURE_TSC); - return 1; -} -#endif __setup("notsc", notsc_setup); enum { diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index f0144ae8d891..91843e9224d0 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -52,6 +52,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/x86/kvm/mmu/spte.c b/arch/x86/kvm/mmu/spte.c index 85a0473809b0..4e753386c8d4 100644 --- a/arch/x86/kvm/mmu/spte.c +++ b/arch/x86/kvm/mmu/spte.c @@ -15,6 +15,7 @@ #include "x86.h" #include "spte.h" +#include #include #include #include diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index e218352e3423..aee70e5dc15d 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -1313,14 +1313,14 @@ static void kvm_pmu_load_guest_pmcs(struct kvm_vcpu *vcpu) pmc = &pmu->gp_counters[i]; if (pmc->counter != rdpmc(i)) - wrmsrl(gp_counter_msr(i), pmc->counter); - wrmsrl(gp_eventsel_msr(i), pmc->eventsel_hw); + wrmsrq(gp_counter_msr(i), pmc->counter); + wrmsrq(gp_eventsel_msr(i), pmc->eventsel_hw); } for (i = 0; i < pmu->nr_arch_fixed_counters; i++) { pmc = &pmu->fixed_counters[i]; if (pmc->counter != rdpmc(INTEL_PMC_FIXED_RDPMC_BASE | i)) - wrmsrl(fixed_counter_msr(i), pmc->counter); + wrmsrq(fixed_counter_msr(i), pmc->counter); } } diff --git a/arch/x86/kvm/reverse_cpuid.h b/arch/x86/kvm/reverse_cpuid.h index 657f5f743ed9..2ad25781cefb 100644 --- a/arch/x86/kvm/reverse_cpuid.h +++ b/arch/x86/kvm/reverse_cpuid.h @@ -3,8 +3,10 @@ #define ARCH_X86_KVM_REVERSE_CPUID_H #include + #include #include +#include /* * Define a KVM-only feature flag. diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 6c6a6d663e29..75e4517e8119 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -23,6 +23,7 @@ #include #include +#include #include #include #include diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index e02a38da5296..d38a21be099d 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index 27eb76e6b6a0..74e0b01185b8 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "x86.h" #include "cpuid.h" #include "lapic.h" diff --git a/arch/x86/kvm/vmx/sgx.c b/arch/x86/kvm/vmx/sgx.c index df1d0cf76947..29a1f8e3be60 100644 --- a/arch/x86/kvm/vmx/sgx.c +++ b/arch/x86/kvm/vmx/sgx.c @@ -2,6 +2,7 @@ /* Copyright(c) 2021 Intel Corporation. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include #include diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 04ce321ebdf3..cb50e23c39ca 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -823,7 +823,7 @@ static void tdx_prepare_switch_to_host(struct kvm_vcpu *vcpu) return; ++vcpu->stat.host_state_reload; - wrmsrl(MSR_KERNEL_GS_BASE, vt->msr_host_kernel_gs_base); + wrmsrq(MSR_KERNEL_GS_BASE, vt->msr_host_kernel_gs_base); vt->guest_state_loaded = false; } @@ -1048,10 +1048,10 @@ static void tdx_load_host_xsave_state(struct kvm_vcpu *vcpu) /* * Likewise, even if a TDX hosts didn't support XSS both arms of - * the comparison would be 0 and the wrmsrl would be skipped. + * the comparison would be 0 and the wrmsrq would be skipped. */ if (kvm_host.xss != (kvm_tdx->xfam & kvm_caps.supported_xss)) - wrmsrl(MSR_IA32_XSS, kvm_host.xss); + wrmsrq(MSR_IA32_XSS, kvm_host.xss); } #define TDX_DEBUGCTL_PRESERVED (DEBUGCTLMSR_BTF | \ diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index b9103de01428..ede773ce065a 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/x86/lib/msr-smp.c b/arch/x86/lib/msr-smp.c index b8f63419e6ae..0c6627773c80 100644 --- a/arch/x86/lib/msr-smp.c +++ b/arch/x86/lib/msr-smp.c @@ -31,22 +31,6 @@ static void __wrmsr_on_cpu(void *info) wrmsr(rv->msr_no, reg->l, reg->h); } -int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h) -{ - int err; - struct msr_info rv; - - memset(&rv, 0, sizeof(rv)); - - rv.msr_no = msr_no; - err = smp_call_function_single(cpu, __rdmsr_on_cpu, &rv, 1); - *l = rv.reg.l; - *h = rv.reg.h; - - return err; -} -EXPORT_SYMBOL(rdmsr_on_cpu); - int rdmsrq_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) { int err; @@ -62,22 +46,6 @@ int rdmsrq_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) } EXPORT_SYMBOL(rdmsrq_on_cpu); -int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) -{ - int err; - struct msr_info rv; - - memset(&rv, 0, sizeof(rv)); - - rv.msr_no = msr_no; - rv.reg.l = l; - rv.reg.h = h; - err = smp_call_function_single(cpu, __wrmsr_on_cpu, &rv, 1); - - return err; -} -EXPORT_SYMBOL(wrmsr_on_cpu); - int wrmsrq_on_cpu(unsigned int cpu, u32 msr_no, u64 q) { int err; @@ -164,46 +132,6 @@ static void __wrmsr_safe_on_cpu(void *info) rv->err = wrmsr_safe(rv->msr_no, rv->reg.l, rv->reg.h); } -int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h) -{ - struct msr_info_completion rv; - call_single_data_t csd; - int err; - - INIT_CSD(&csd, __rdmsr_safe_on_cpu, &rv); - - memset(&rv, 0, sizeof(rv)); - init_completion(&rv.done); - rv.msr.msr_no = msr_no; - - err = smp_call_function_single_async(cpu, &csd); - if (!err) { - wait_for_completion(&rv.done); - err = rv.msr.err; - } - *l = rv.msr.reg.l; - *h = rv.msr.reg.h; - - return err; -} -EXPORT_SYMBOL(rdmsr_safe_on_cpu); - -int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) -{ - int err; - struct msr_info rv; - - memset(&rv, 0, sizeof(rv)); - - rv.msr_no = msr_no; - rv.reg.l = l; - rv.reg.h = h; - err = smp_call_function_single(cpu, __wrmsr_safe_on_cpu, &rv, 1); - - return err ? err : rv.err; -} -EXPORT_SYMBOL(wrmsr_safe_on_cpu); - int wrmsrq_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q) { int err; @@ -222,11 +150,22 @@ EXPORT_SYMBOL(wrmsrq_safe_on_cpu); int rdmsrq_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) { - u32 low, high; + struct msr_info_completion rv; + call_single_data_t csd; int err; - err = rdmsr_safe_on_cpu(cpu, msr_no, &low, &high); - *q = (u64)high << 32 | low; + INIT_CSD(&csd, __rdmsr_safe_on_cpu, &rv); + + memset(&rv, 0, sizeof(rv)); + init_completion(&rv.done); + rv.msr.msr_no = msr_no; + + err = smp_call_function_single_async(cpu, &csd); + if (!err) { + wait_for_completion(&rv.done); + err = rv.msr.err; + } + *q = rv.msr.reg.q; return err; } diff --git a/arch/x86/math-emu/Makefile b/arch/x86/math-emu/Makefile deleted file mode 100644 index 02211fc6f2ec..000000000000 --- a/arch/x86/math-emu/Makefile +++ /dev/null @@ -1,30 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# Makefile for wm-FPU-emu -# - -#DEBUG = -DDEBUGGING -DEBUG = -PARANOID = -DPARANOID -ccflags-y += $(PARANOID) $(DEBUG) -fno-builtin $(MATH_EMULATION) -asflags-y += $(PARANOID) - -# From 'C' language sources: -C_OBJS =fpu_entry.o errors.o \ - fpu_arith.o fpu_aux.o fpu_etc.o fpu_tags.o fpu_trig.o \ - load_store.o get_address.o \ - poly_atan.o poly_l2.o poly_2xm1.o poly_sin.o poly_tan.o \ - reg_add_sub.o reg_compare.o reg_constant.o reg_convert.o \ - reg_ld_str.o reg_divide.o reg_mul.o - -# From 80x86 assembler sources: -A_OBJS =reg_u_add.o reg_u_div.o reg_u_mul.o reg_u_sub.o \ - div_small.o reg_norm.o reg_round.o \ - wm_shrx.o wm_sqrt.o \ - div_Xsig.o polynom_Xsig.o round_Xsig.o \ - shr_Xsig.o mul_Xsig.o - -obj-y =$(C_OBJS) $(A_OBJS) - -proto: - cproto -e -DMAKING_PROTO *.c >fpu_proto.h diff --git a/arch/x86/math-emu/README b/arch/x86/math-emu/README deleted file mode 100644 index e6235491d6eb..000000000000 --- a/arch/x86/math-emu/README +++ /dev/null @@ -1,427 +0,0 @@ - +---------------------------------------------------------------------------+ - | wm-FPU-emu an FPU emulator for 80386 and 80486SX microprocessors. | - | | - | Copyright (C) 1992,1993,1994,1995,1996,1997,1999 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, | - | Australia. E-mail billm@melbpc.org.au | - | | - | This program is free software; you can redistribute it and/or modify | - | it under the terms of the GNU General Public License version 2 as | - | published by the Free Software Foundation. | - | | - | This program is distributed in the hope that it will be useful, | - | but WITHOUT ANY WARRANTY; without even the implied warranty of | - | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | - | GNU General Public License for more details. | - | | - | You should have received a copy of the GNU General Public License | - | along with this program; if not, write to the Free Software | - | Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. | - | | - +---------------------------------------------------------------------------+ - - - -wm-FPU-emu is an FPU emulator for Linux. It is derived from wm-emu387 -which was my 80387 emulator for early versions of djgpp (gcc under -msdos); wm-emu387 was in turn based upon emu387 which was written by -DJ Delorie for djgpp. The interface to the Linux kernel is based upon -the original Linux math emulator by Linus Torvalds. - -My target FPU for wm-FPU-emu is that described in the Intel486 -Programmer's Reference Manual (1992 edition). Unfortunately, numerous -facets of the functioning of the FPU are not well covered in the -Reference Manual. The information in the manual has been supplemented -with measurements on real 80486's. Unfortunately, it is simply not -possible to be sure that all of the peculiarities of the 80486 have -been discovered, so there is always likely to be obscure differences -in the detailed behaviour of the emulator and a real 80486. - -wm-FPU-emu does not implement all of the behaviour of the 80486 FPU, -but is very close. See "Limitations" later in this file for a list of -some differences. - -Please report bugs, etc to me at: - billm@melbpc.org.au -or b.metzenthen@medoto.unimelb.edu.au - -For more information on the emulator and on floating point topics, see -my web pages, currently at http://www.suburbia.net/~billm/ - - ---Bill Metzenthen - December 1999 - - ------------------------ Internals of wm-FPU-emu ----------------------- - -Numeric algorithms: -(1) Add, subtract, and multiply. Nothing remarkable in these. -(2) Divide has been tuned to get reasonable performance. The algorithm - is not the obvious one which most people seem to use, but is designed - to take advantage of the characteristics of the 80386. I expect that - it has been invented many times before I discovered it, but I have not - seen it. It is based upon one of those ideas which one carries around - for years without ever bothering to check it out. -(3) The sqrt function has been tuned to get good performance. It is based - upon Newton's classic method. Performance was improved by capitalizing - upon the properties of Newton's method, and the code is once again - structured taking account of the 80386 characteristics. -(4) The trig, log, and exp functions are based in each case upon quasi- - "optimal" polynomial approximations. My definition of "optimal" was - based upon getting good accuracy with reasonable speed. -(5) The argument reducing code for the trig function effectively uses - a value of pi which is accurate to more than 128 bits. As a consequence, - the reduced argument is accurate to more than 64 bits for arguments up - to a few pi, and accurate to more than 64 bits for most arguments, - even for arguments approaching 2^63. This is far superior to an - 80486, which uses a value of pi which is accurate to 66 bits. - -The code of the emulator is complicated slightly by the need to -account for a limited form of re-entrancy. Normally, the emulator will -emulate each FPU instruction to completion without interruption. -However, it may happen that when the emulator is accessing the user -memory space, swapping may be needed. In this case the emulator may be -temporarily suspended while disk i/o takes place. During this time -another process may use the emulator, thereby perhaps changing static -variables. The code which accesses user memory is confined to five -files: - fpu_entry.c - reg_ld_str.c - load_store.c - get_address.c - errors.c -As from version 1.12 of the emulator, no static variables are used -(apart from those in the kernel's per-process tables). The emulator is -therefore now fully re-entrant, rather than having just the restricted -form of re-entrancy which is required by the Linux kernel. - ------------------------ Limitations of wm-FPU-emu ----------------------- - -There are a number of differences between the current wm-FPU-emu -(version 2.01) and the 80486 FPU (apart from bugs). The differences -are fewer than those which applied to the 1.xx series of the emulator. -Some of the more important differences are listed below: - -The Roundup flag does not have much meaning for the transcendental -functions and its 80486 value with these functions is likely to differ -from its emulator value. - -In a few rare cases the Underflow flag obtained with the emulator will -be different from that obtained with an 80486. This occurs when the -following conditions apply simultaneously: -(a) the operands have a higher precision than the current setting of the - precision control (PC) flags. -(b) the underflow exception is masked. -(c) the magnitude of the exact result (before rounding) is less than 2^-16382. -(d) the magnitude of the final result (after rounding) is exactly 2^-16382. -(e) the magnitude of the exact result would be exactly 2^-16382 if the - operands were rounded to the current precision before the arithmetic - operation was performed. -If all of these apply, the emulator will set the Underflow flag but a real -80486 will not. - -NOTE: Certain formats of Extended Real are UNSUPPORTED. They are -unsupported by the 80486. They are the Pseudo-NaNs, Pseudoinfinities, -and Unnormals. None of these will be generated by an 80486 or by the -emulator. Do not use them. The emulator treats them differently in -detail from the way an 80486 does. - -Self modifying code can cause the emulator to fail. An example of such -code is: - movl %esp,[%ebx] - fld1 -The FPU instruction may be (usually will be) loaded into the pre-fetch -queue of the CPU before the mov instruction is executed. If the -destination of the 'movl' overlaps the FPU instruction then the bytes -in the prefetch queue and memory will be inconsistent when the FPU -instruction is executed. The emulator will be invoked but will not be -able to find the instruction which caused the device-not-present -exception. For this case, the emulator cannot emulate the behaviour of -an 80486DX. - -Handling of the address size override prefix byte (0x67) has not been -extensively tested yet. A major problem exists because using it in -vm86 mode can cause a general protection fault. Address offsets -greater than 0xffff appear to be illegal in vm86 mode but are quite -acceptable (and work) in real mode. A small test program developed to -check the addressing, and which runs successfully in real mode, -crashes dosemu under Linux and also brings Windows down with a general -protection fault message when run under the MS-DOS prompt of Windows -3.1. (The program simply reads data from a valid address). - -The emulator supports 16-bit protected mode, with one difference from -an 80486DX. A 80486DX will allow some floating point instructions to -write a few bytes below the lowest address of the stack. The emulator -will not allow this in 16-bit protected mode: no instructions are -allowed to write outside the bounds set by the protection. - ------------------------ Performance of wm-FPU-emu ----------------------- - -Speed. ------ - -The speed of floating point computation with the emulator will depend -upon instruction mix. Relative performance is best for the instructions -which require most computation. The simple instructions are adversely -affected by the FPU instruction trap overhead. - - -Timing: Some simple timing tests have been made on the emulator functions. -The times include load/store instructions. All times are in microseconds -measured on a 33MHz 386 with 64k cache. The Turbo C tests were under -ms-dos, the next two columns are for emulators running with the djgpp -ms-dos extender. The final column is for wm-FPU-emu in Linux 0.97, -using libm4.0 (hard). - -function Turbo C djgpp 1.06 WM-emu387 wm-FPU-emu - - + 60.5 154.8 76.5 139.4 - - 61.1-65.5 157.3-160.8 76.2-79.5 142.9-144.7 - * 71.0 190.8 79.6 146.6 - / 61.2-75.0 261.4-266.9 75.3-91.6 142.2-158.1 - - sin() 310.8 4692.0 319.0 398.5 - cos() 284.4 4855.2 308.0 388.7 - tan() 495.0 8807.1 394.9 504.7 - atan() 328.9 4866.4 601.1 419.5-491.9 - - sqrt() 128.7 crashed 145.2 227.0 - log() 413.1-419.1 5103.4-5354.21 254.7-282.2 409.4-437.1 - exp() 479.1 6619.2 469.1 850.8 - - -The performance under Linux is improved by the use of look-ahead code. -The following results show the improvement which is obtained under -Linux due to the look-ahead code. Also given are the times for the -original Linux emulator with the 4.1 'soft' lib. - - [ Linus' note: I changed look-ahead to be the default under linux, as - there was no reason not to use it after I had edited it to be - disabled during tracing ] - - wm-FPU-emu w original w - look-ahead 'soft' lib - + 106.4 190.2 - - 108.6-111.6 192.4-216.2 - * 113.4 193.1 - / 108.8-124.4 700.1-706.2 - - sin() 390.5 2642.0 - cos() 381.5 2767.4 - tan() 496.5 3153.3 - atan() 367.2-435.5 2439.4-3396.8 - - sqrt() 195.1 4732.5 - log() 358.0-387.5 3359.2-3390.3 - exp() 619.3 4046.4 - - -These figures are now somewhat out-of-date. The emulator has become -progressively slower for most functions as more of the 80486 features -have been implemented. - - ------------------------ Accuracy of wm-FPU-emu ----------------------- - - -The accuracy of the emulator is in almost all cases equal to or better -than that of an Intel 80486 FPU. - -The results of the basic arithmetic functions (+,-,*,/), and fsqrt -match those of an 80486 FPU. They are the best possible; the error for -these never exceeds 1/2 an lsb. The fprem and fprem1 instructions -return exact results; they have no error. - - -The following table compares the emulator accuracy for the sqrt(), -trig and log functions against the Turbo C "emulator". For this table, -each function was tested at about 400 points. Ideal worst-case results -would be 64 bits. The reduced Turbo C accuracy of cos() and tan() for -arguments greater than pi/4 can be thought of as being related to the -precision of the argument x; e.g. an argument of pi/2-(1e-10) which is -accurate to 64 bits can result in a relative accuracy in cos() of -about 64 + log2(cos(x)) = 31 bits. - - -Function Tested x range Worst result Turbo C - (relative bits) - -sqrt(x) 1 .. 2 64.1 63.2 -atan(x) 1e-10 .. 200 64.2 62.8 -cos(x) 0 .. pi/2-(1e-10) 64.4 (x <= pi/4) 62.4 - 64.1 (x = pi/2-(1e-10)) 31.9 -sin(x) 1e-10 .. pi/2 64.0 62.8 -tan(x) 1e-10 .. pi/2-(1e-10) 64.0 (x <= pi/4) 62.1 - 64.1 (x = pi/2-(1e-10)) 31.9 -exp(x) 0 .. 1 63.1 ** 62.9 -log(x) 1+1e-6 .. 2 63.8 ** 62.1 - -** The accuracy for exp() and log() is low because the FPU (emulator) -does not compute them directly; two operations are required. - - -The emulator passes the "paranoia" tests (compiled with gcc 2.3.3 or -later) for 'float' variables (24 bit precision numbers) when precision -control is set to 24, 53 or 64 bits, and for 'double' variables (53 -bit precision numbers) when precision control is set to 53 bits (a -properly performing FPU cannot pass the 'paranoia' tests for 'double' -variables when precision control is set to 64 bits). - -The code for reducing the argument for the trig functions (fsin, fcos, -fptan and fsincos) has been improved and now effectively uses a value -for pi which is accurate to more than 128 bits precision. As a -consequence, the accuracy of these functions for large arguments has -been dramatically improved (and is now very much better than an 80486 -FPU). There is also now no degradation of accuracy for fcos and fptan -for operands close to pi/2. Measured results are (note that the -definition of accuracy has changed slightly from that used for the -above table): - -Function Tested x range Worst result - (absolute bits) - -cos(x) 0 .. 9.22e+18 62.0 -sin(x) 1e-16 .. 9.22e+18 62.1 -tan(x) 1e-16 .. 9.22e+18 61.8 - -It is possible with some effort to find very large arguments which -give much degraded precision. For example, the integer number - 8227740058411162616.0 -is within about 10e-7 of a multiple of pi. To find the tan (for -example) of this number to 64 bits precision it would be necessary to -have a value of pi which had about 150 bits precision. The FPU -emulator computes the result to about 42.6 bits precision (the correct -result is about -9.739715e-8). On the other hand, an 80486 FPU returns -0.01059, which in relative terms is hopelessly inaccurate. - -For arguments close to critical angles (which occur at multiples of -pi/2) the emulator is more accurate than an 80486 FPU. For very large -arguments, the emulator is far more accurate. - - -Prior to version 1.20 of the emulator, the accuracy of the results for -the transcendental functions (in their principal range) was not as -good as the results from an 80486 FPU. From version 1.20, the accuracy -has been considerably improved and these functions now give measured -worst-case results which are better than the worst-case results given -by an 80486 FPU. - -The following table gives the measured results for the emulator. The -number of randomly selected arguments in each case is about half a -million. The group of three columns gives the frequency of the given -accuracy in number of times per million, thus the second of these -columns shows that an accuracy of between 63.80 and 63.89 bits was -found at a rate of 133 times per one million measurements for fsin. -The results show that the fsin, fcos and fptan instructions return -results which are in error (i.e. less accurate than the best possible -result (which is 64 bits)) for about one per cent of all arguments -between -pi/2 and +pi/2. The other instructions have a lower -frequency of results which are in error. The last two columns give -the worst accuracy which was found (in bits) and the approximate value -of the argument which produced it. - - frequency (per M) - ------------------- --------------- -instr arg range # tests 63.7 63.8 63.9 worst at arg - bits bits bits bits ------ ------------ ------- ---- ---- ----- ----- -------- -fsin (0,pi/2) 547756 0 133 10673 63.89 0.451317 -fcos (0,pi/2) 547563 0 126 10532 63.85 0.700801 -fptan (0,pi/2) 536274 11 267 10059 63.74 0.784876 -fpatan 4 quadrants 517087 0 8 1855 63.88 0.435121 (4q) -fyl2x (0,20) 541861 0 0 1323 63.94 1.40923 (x) -fyl2xp1 (-.293,.414) 520256 0 0 5678 63.93 0.408542 (x) -f2xm1 (-1,1) 538847 4 481 6488 63.79 0.167709 - - -Tests performed on an 80486 FPU showed results of lower accuracy. The -following table gives the results which were obtained with an AMD -486DX2/66 (other tests indicate that an Intel 486DX produces -identical results). The tests were basically the same as those used -to measure the emulator (the values, being random, were in general not -the same). The total number of tests for each instruction are given -at the end of the table, in case each about 100k tests were performed. -Another line of figures at the end of the table shows that most of the -instructions return results which are in error for more than 10 -percent of the arguments tested. - -The numbers in the body of the table give the approx number of times a -result of the given accuracy in bits (given in the left-most column) -was obtained per one million arguments. For three of the instructions, -two columns of results are given: * The second column for f2xm1 gives -the number cases where the results of the first column were for a -positive argument, this shows that this instruction gives better -results for positive arguments than it does for negative. * In the -cases of fcos and fptan, the first column gives the results when all -cases where arguments greater than 1.5 were removed from the results -given in the second column. Unlike the emulator, an 80486 FPU returns -results of relatively poor accuracy for these instructions when the -argument approaches pi/2. The table does not show those cases when the -accuracy of the results were less than 62 bits, which occurs quite -often for fsin and fptan when the argument approaches pi/2. This poor -accuracy is discussed above in relation to the Turbo C "emulator", and -the accuracy of the value of pi. - - -bits f2xm1 f2xm1 fpatan fcos fcos fyl2x fyl2xp1 fsin fptan fptan -62.0 0 0 0 0 437 0 0 0 0 925 -62.1 0 0 10 0 894 0 0 0 0 1023 -62.2 14 0 0 0 1033 0 0 0 0 945 -62.3 57 0 0 0 1202 0 0 0 0 1023 -62.4 385 0 0 10 1292 0 23 0 0 1178 -62.5 1140 0 0 119 1649 0 39 0 0 1149 -62.6 2037 0 0 189 1620 0 16 0 0 1169 -62.7 5086 14 0 646 2315 10 101 35 39 1402 -62.8 8818 86 0 984 3050 59 287 131 224 2036 -62.9 11340 1355 0 2126 4153 79 605 357 321 1948 -63.0 15557 4750 0 3319 5376 246 1281 862 808 2688 -63.1 20016 8288 0 4620 6628 511 2569 1723 1510 3302 -63.2 24945 11127 10 6588 8098 1120 4470 2968 2990 4724 -63.3 25686 12382 69 8774 10682 1906 6775 4482 5474 7236 -63.4 29219 14722 79 11109 12311 3094 9414 7259 8912 10587 -63.5 30458 14936 393 13802 15014 5874 12666 9609 13762 15262 -63.6 32439 16448 1277 17945 19028 10226 15537 14657 19158 20346 -63.7 35031 16805 4067 23003 23947 18910 20116 21333 25001 26209 -63.8 33251 15820 7673 24781 25675 24617 25354 24440 29433 30329 -63.9 33293 16833 18529 28318 29233 31267 31470 27748 29676 30601 - -Per cent with error: - 30.9 3.2 18.5 9.8 13.1 11.6 17.4 -Total arguments tested: - 70194 70099 101784 100641 100641 101799 128853 114893 102675 102675 - - -------------------------- Contributors ------------------------------- - -A number of people have contributed to the development of the -emulator, often by just reporting bugs, sometimes with suggested -fixes, and a few kind people have provided me with access in one way -or another to an 80486 machine. Contributors include (to those people -who I may have forgotten, please forgive me): - -Linus Torvalds -Tommy.Thorn@daimi.aau.dk -Andrew.Tridgell@anu.edu.au -Nick Holloway, alfie@dcs.warwick.ac.uk -Hermano Moura, moura@dcs.gla.ac.uk -Jon Jagger, J.Jagger@scp.ac.uk -Lennart Benschop -Brian Gallew, geek+@CMU.EDU -Thomas Staniszewski, ts3v+@andrew.cmu.edu -Martin Howell, mph@plasma.apana.org.au -M Saggaf, alsaggaf@athena.mit.edu -Peter Barker, PETER@socpsy.sci.fau.edu -tom@vlsivie.tuwien.ac.at -Dan Russel, russed@rpi.edu -Daniel Carosone, danielce@ee.mu.oz.au -cae@jpmorgan.com -Hamish Coleman, t933093@minyos.xx.rmit.oz.au -Bruce Evans, bde@kralizec.zeta.org.au -Timo Korvola, Timo.Korvola@hut.fi -Rick Lyons, rick@razorback.brisnet.org.au -Rick, jrs@world.std.com - -...and numerous others who responded to my request for help with -a real 80486. - diff --git a/arch/x86/math-emu/control_w.h b/arch/x86/math-emu/control_w.h deleted file mode 100644 index 93cbc89b34e2..000000000000 --- a/arch/x86/math-emu/control_w.h +++ /dev/null @@ -1,46 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/*---------------------------------------------------------------------------+ - | control_w.h | - | | - | Copyright (C) 1992,1993 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, | - | Australia. E-mail billm@vaxc.cc.monash.edu.au | - | | - +---------------------------------------------------------------------------*/ - -#ifndef _CONTROLW_H_ -#define _CONTROLW_H_ - -#ifdef __ASSEMBLER__ -#define _Const_(x) $##x -#else -#define _Const_(x) x -#endif - -#define CW_RC _Const_(0x0C00) /* rounding control */ -#define CW_PC _Const_(0x0300) /* precision control */ - -#define CW_Precision Const_(0x0020) /* loss of precision mask */ -#define CW_Underflow Const_(0x0010) /* underflow mask */ -#define CW_Overflow Const_(0x0008) /* overflow mask */ -#define CW_ZeroDiv Const_(0x0004) /* divide by zero mask */ -#define CW_Denormal Const_(0x0002) /* denormalized operand mask */ -#define CW_Invalid Const_(0x0001) /* invalid operation mask */ - -#define CW_Exceptions _Const_(0x003f) /* all masks */ - -#define RC_RND _Const_(0x0000) -#define RC_DOWN _Const_(0x0400) -#define RC_UP _Const_(0x0800) -#define RC_CHOP _Const_(0x0C00) - -/* p 15-5: Precision control bits affect only the following: - ADD, SUB(R), MUL, DIV(R), and SQRT */ -#define PR_24_BITS _Const_(0x000) -#define PR_53_BITS _Const_(0x200) -#define PR_64_BITS _Const_(0x300) -#define PR_RESERVED_BITS _Const_(0x100) -/* FULL_PRECISION simulates all exceptions masked */ -#define FULL_PRECISION (PR_64_BITS | RC_RND | 0x3f) - -#endif /* _CONTROLW_H_ */ diff --git a/arch/x86/math-emu/div_Xsig.S b/arch/x86/math-emu/div_Xsig.S deleted file mode 100644 index 8c270ab415be..000000000000 --- a/arch/x86/math-emu/div_Xsig.S +++ /dev/null @@ -1,367 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - .file "div_Xsig.S" -/*---------------------------------------------------------------------------+ - | div_Xsig.S | - | | - | Division subroutine for 96 bit quantities | - | | - | Copyright (C) 1994,1995 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, | - | Australia. E-mail billm@jacobi.maths.monash.edu.au | - | | - | | - +---------------------------------------------------------------------------*/ - -/*---------------------------------------------------------------------------+ - | Divide the 96 bit quantity pointed to by a, by that pointed to by b, and | - | put the 96 bit result at the location d. | - | | - | The result may not be accurate to 96 bits. It is intended for use where | - | a result better than 64 bits is required. The result should usually be | - | good to at least 94 bits. | - | The returned result is actually divided by one half. This is done to | - | prevent overflow. | - | | - | .aaaaaaaaaaaaaa / .bbbbbbbbbbbbb -> .dddddddddddd | - | | - | void div_Xsig(Xsig *a, Xsig *b, Xsig *dest) | - | | - +---------------------------------------------------------------------------*/ - -#include "exception.h" -#include "fpu_emu.h" - - -#define XsigLL(x) (x) -#define XsigL(x) 4(x) -#define XsigH(x) 8(x) - - -#ifndef NON_REENTRANT_FPU -/* - Local storage on the stack: - Accumulator: FPU_accum_3:FPU_accum_2:FPU_accum_1:FPU_accum_0 - */ -#define FPU_accum_3 -4(%ebp) -#define FPU_accum_2 -8(%ebp) -#define FPU_accum_1 -12(%ebp) -#define FPU_accum_0 -16(%ebp) -#define FPU_result_3 -20(%ebp) -#define FPU_result_2 -24(%ebp) -#define FPU_result_1 -28(%ebp) - -#else -.data -/* - Local storage in a static area: - Accumulator: FPU_accum_3:FPU_accum_2:FPU_accum_1:FPU_accum_0 - */ - .align 4,0 -FPU_accum_3: - .long 0 -FPU_accum_2: - .long 0 -FPU_accum_1: - .long 0 -FPU_accum_0: - .long 0 -FPU_result_3: - .long 0 -FPU_result_2: - .long 0 -FPU_result_1: - .long 0 -#endif /* NON_REENTRANT_FPU */ - - -.text -SYM_FUNC_START(div_Xsig) - pushl %ebp - movl %esp,%ebp -#ifndef NON_REENTRANT_FPU - subl $28,%esp -#endif /* NON_REENTRANT_FPU */ - - pushl %esi - pushl %edi - pushl %ebx - - movl PARAM1,%esi /* pointer to num */ - movl PARAM2,%ebx /* pointer to denom */ - -#ifdef PARANOID - testl $0x80000000, XsigH(%ebx) /* Divisor */ - je L_bugged -#endif /* PARANOID */ - - -/*---------------------------------------------------------------------------+ - | Divide: Return arg1/arg2 to arg3. | - | | - | The maximum returned value is (ignoring exponents) | - | .ffffffff ffffffff | - | ------------------ = 1.ffffffff fffffffe | - | .80000000 00000000 | - | and the minimum is | - | .80000000 00000000 | - | ------------------ = .80000000 00000001 (rounded) | - | .ffffffff ffffffff | - | | - +---------------------------------------------------------------------------*/ - - /* Save extended dividend in local register */ - - /* Divide by 2 to prevent overflow */ - clc - movl XsigH(%esi),%eax - rcrl %eax - movl %eax,FPU_accum_3 - movl XsigL(%esi),%eax - rcrl %eax - movl %eax,FPU_accum_2 - movl XsigLL(%esi),%eax - rcrl %eax - movl %eax,FPU_accum_1 - movl $0,%eax - rcrl %eax - movl %eax,FPU_accum_0 - - movl FPU_accum_2,%eax /* Get the current num */ - movl FPU_accum_3,%edx - -/*----------------------------------------------------------------------*/ -/* Initialization done. - Do the first 32 bits. */ - - /* We will divide by a number which is too large */ - movl XsigH(%ebx),%ecx - addl $1,%ecx - jnc LFirst_div_not_1 - - /* here we need to divide by 100000000h, - i.e., no division at all.. */ - mov %edx,%eax - jmp LFirst_div_done - -LFirst_div_not_1: - divl %ecx /* Divide the numerator by the augmented - denom ms dw */ - -LFirst_div_done: - movl %eax,FPU_result_3 /* Put the result in the answer */ - - mull XsigH(%ebx) /* mul by the ms dw of the denom */ - - subl %eax,FPU_accum_2 /* Subtract from the num local reg */ - sbbl %edx,FPU_accum_3 - - movl FPU_result_3,%eax /* Get the result back */ - mull XsigL(%ebx) /* now mul the ls dw of the denom */ - - subl %eax,FPU_accum_1 /* Subtract from the num local reg */ - sbbl %edx,FPU_accum_2 - sbbl $0,FPU_accum_3 - je LDo_2nd_32_bits /* Must check for non-zero result here */ - -#ifdef PARANOID - jb L_bugged_1 -#endif /* PARANOID */ - - /* need to subtract another once of the denom */ - incl FPU_result_3 /* Correct the answer */ - - movl XsigL(%ebx),%eax - movl XsigH(%ebx),%edx - subl %eax,FPU_accum_1 /* Subtract from the num local reg */ - sbbl %edx,FPU_accum_2 - -#ifdef PARANOID - sbbl $0,FPU_accum_3 - jne L_bugged_1 /* Must check for non-zero result here */ -#endif /* PARANOID */ - -/*----------------------------------------------------------------------*/ -/* Half of the main problem is done, there is just a reduced numerator - to handle now. - Work with the second 32 bits, FPU_accum_0 not used from now on */ -LDo_2nd_32_bits: - movl FPU_accum_2,%edx /* get the reduced num */ - movl FPU_accum_1,%eax - - /* need to check for possible subsequent overflow */ - cmpl XsigH(%ebx),%edx - jb LDo_2nd_div - ja LPrevent_2nd_overflow - - cmpl XsigL(%ebx),%eax - jb LDo_2nd_div - -LPrevent_2nd_overflow: -/* The numerator is greater or equal, would cause overflow */ - /* prevent overflow */ - subl XsigL(%ebx),%eax - sbbl XsigH(%ebx),%edx - movl %edx,FPU_accum_2 - movl %eax,FPU_accum_1 - - incl FPU_result_3 /* Reflect the subtraction in the answer */ - -#ifdef PARANOID - je L_bugged_2 /* Can't bump the result to 1.0 */ -#endif /* PARANOID */ - -LDo_2nd_div: - cmpl $0,%ecx /* augmented denom msw */ - jnz LSecond_div_not_1 - - /* %ecx == 0, we are dividing by 1.0 */ - mov %edx,%eax - jmp LSecond_div_done - -LSecond_div_not_1: - divl %ecx /* Divide the numerator by the denom ms dw */ - -LSecond_div_done: - movl %eax,FPU_result_2 /* Put the result in the answer */ - - mull XsigH(%ebx) /* mul by the ms dw of the denom */ - - subl %eax,FPU_accum_1 /* Subtract from the num local reg */ - sbbl %edx,FPU_accum_2 - -#ifdef PARANOID - jc L_bugged_2 -#endif /* PARANOID */ - - movl FPU_result_2,%eax /* Get the result back */ - mull XsigL(%ebx) /* now mul the ls dw of the denom */ - - subl %eax,FPU_accum_0 /* Subtract from the num local reg */ - sbbl %edx,FPU_accum_1 /* Subtract from the num local reg */ - sbbl $0,FPU_accum_2 - -#ifdef PARANOID - jc L_bugged_2 -#endif /* PARANOID */ - - jz LDo_3rd_32_bits - -#ifdef PARANOID - cmpl $1,FPU_accum_2 - jne L_bugged_2 -#endif /* PARANOID */ - - /* need to subtract another once of the denom */ - movl XsigL(%ebx),%eax - movl XsigH(%ebx),%edx - subl %eax,FPU_accum_0 /* Subtract from the num local reg */ - sbbl %edx,FPU_accum_1 - sbbl $0,FPU_accum_2 - -#ifdef PARANOID - jc L_bugged_2 - jne L_bugged_2 -#endif /* PARANOID */ - - addl $1,FPU_result_2 /* Correct the answer */ - adcl $0,FPU_result_3 - -#ifdef PARANOID - jc L_bugged_2 /* Must check for non-zero result here */ -#endif /* PARANOID */ - -/*----------------------------------------------------------------------*/ -/* The division is essentially finished here, we just need to perform - tidying operations. - Deal with the 3rd 32 bits */ -LDo_3rd_32_bits: - /* We use an approximation for the third 32 bits. - To take account of the 3rd 32 bits of the divisor - (call them del), we subtract del * (a/b) */ - - movl FPU_result_3,%eax /* a/b */ - mull XsigLL(%ebx) /* del */ - - subl %edx,FPU_accum_1 - - /* A borrow indicates that the result is negative */ - jnb LTest_over - - movl XsigH(%ebx),%edx - addl %edx,FPU_accum_1 - - subl $1,FPU_result_2 /* Adjust the answer */ - sbbl $0,FPU_result_3 - - /* The above addition might not have been enough, check again. */ - movl FPU_accum_1,%edx /* get the reduced num */ - cmpl XsigH(%ebx),%edx /* denom */ - jb LDo_3rd_div - - movl XsigH(%ebx),%edx - addl %edx,FPU_accum_1 - - subl $1,FPU_result_2 /* Adjust the answer */ - sbbl $0,FPU_result_3 - jmp LDo_3rd_div - -LTest_over: - movl FPU_accum_1,%edx /* get the reduced num */ - - /* need to check for possible subsequent overflow */ - cmpl XsigH(%ebx),%edx /* denom */ - jb LDo_3rd_div - - /* prevent overflow */ - subl XsigH(%ebx),%edx - movl %edx,FPU_accum_1 - - addl $1,FPU_result_2 /* Reflect the subtraction in the answer */ - adcl $0,FPU_result_3 - -LDo_3rd_div: - movl FPU_accum_0,%eax - movl FPU_accum_1,%edx - divl XsigH(%ebx) - - movl %eax,FPU_result_1 /* Rough estimate of third word */ - - movl PARAM3,%esi /* pointer to answer */ - - movl FPU_result_1,%eax - movl %eax,XsigLL(%esi) - movl FPU_result_2,%eax - movl %eax,XsigL(%esi) - movl FPU_result_3,%eax - movl %eax,XsigH(%esi) - -L_exit: - popl %ebx - popl %edi - popl %esi - - leave - RET - - -#ifdef PARANOID -/* The logic is wrong if we got here */ -L_bugged: - pushl EX_INTERNAL|0x240 - call EXCEPTION - pop %ebx - jmp L_exit - -L_bugged_1: - pushl EX_INTERNAL|0x241 - call EXCEPTION - pop %ebx - jmp L_exit - -L_bugged_2: - pushl EX_INTERNAL|0x242 - call EXCEPTION - pop %ebx - jmp L_exit -#endif /* PARANOID */ -SYM_FUNC_END(div_Xsig) diff --git a/arch/x86/math-emu/div_small.S b/arch/x86/math-emu/div_small.S deleted file mode 100644 index 637439bfefa4..000000000000 --- a/arch/x86/math-emu/div_small.S +++ /dev/null @@ -1,48 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - .file "div_small.S" -/*---------------------------------------------------------------------------+ - | div_small.S | - | | - | Divide a 64 bit integer by a 32 bit integer & return remainder. | - | | - | Copyright (C) 1992,1995 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, | - | Australia. E-mail billm@jacobi.maths.monash.edu.au | - | | - | | - +---------------------------------------------------------------------------*/ - -/*---------------------------------------------------------------------------+ - | unsigned long FPU_div_small(unsigned long long *x, unsigned long y) | - +---------------------------------------------------------------------------*/ - -#include "fpu_emu.h" - -.text -SYM_FUNC_START(FPU_div_small) - pushl %ebp - movl %esp,%ebp - - pushl %esi - - movl PARAM1,%esi /* pointer to num */ - movl PARAM2,%ecx /* The denominator */ - - movl 4(%esi),%eax /* Get the current num msw */ - xorl %edx,%edx - divl %ecx - - movl %eax,4(%esi) - - movl (%esi),%eax /* Get the num lsw */ - divl %ecx - - movl %eax,(%esi) - - movl %edx,%eax /* Return the remainder in eax */ - - popl %esi - - leave - RET -SYM_FUNC_END(FPU_div_small) diff --git a/arch/x86/math-emu/errors.c b/arch/x86/math-emu/errors.c deleted file mode 100644 index ec071cbb0804..000000000000 --- a/arch/x86/math-emu/errors.c +++ /dev/null @@ -1,686 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/*---------------------------------------------------------------------------+ - | errors.c | - | | - | The error handling functions for wm-FPU-emu | - | | - | Copyright (C) 1992,1993,1994,1996 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia | - | E-mail billm@jacobi.maths.monash.edu.au | - | | - | | - +---------------------------------------------------------------------------*/ - -/*---------------------------------------------------------------------------+ - | Note: | - | The file contains code which accesses user memory. | - | Emulator static data may change when user memory is accessed, due to | - | other processes using the emulator while swapping is in progress. | - +---------------------------------------------------------------------------*/ - -#include - -#include - -#include "fpu_emu.h" -#include "fpu_system.h" -#include "exception.h" -#include "status_w.h" -#include "control_w.h" -#include "reg_constant.h" -#include "version.h" - -/* */ -#undef PRINT_MESSAGES -/* */ - -#if 0 -void Un_impl(void) -{ - u_char byte1, FPU_modrm; - unsigned long address = FPU_ORIG_EIP; - - RE_ENTRANT_CHECK_OFF; - /* No need to check access_ok(), we have previously fetched these bytes. */ - printk("Unimplemented FPU Opcode at eip=%p : ", (void __user *)address); - if (FPU_CS == __USER_CS) { - while (1) { - FPU_get_user(byte1, (u_char __user *) address); - if ((byte1 & 0xf8) == 0xd8) - break; - printk("[%02x]", byte1); - address++; - } - printk("%02x ", byte1); - FPU_get_user(FPU_modrm, 1 + (u_char __user *) address); - - if (FPU_modrm >= 0300) - printk("%02x (%02x+%d)\n", FPU_modrm, FPU_modrm & 0xf8, - FPU_modrm & 7); - else - printk("/%d\n", (FPU_modrm >> 3) & 7); - } else { - printk("cs selector = %04x\n", FPU_CS); - } - - RE_ENTRANT_CHECK_ON; - - EXCEPTION(EX_Invalid); - -} -#endif /* 0 */ - -/* - Called for opcodes which are illegal and which are known to result in a - SIGILL with a real 80486. - */ -void FPU_illegal(void) -{ - math_abort(FPU_info, SIGILL); -} - -void FPU_printall(void) -{ - int i; - static const char *tag_desc[] = { "Valid", "Zero", "ERROR", "Empty", - "DeNorm", "Inf", "NaN" - }; - u_char byte1, FPU_modrm; - unsigned long address = FPU_ORIG_EIP; - - RE_ENTRANT_CHECK_OFF; - /* No need to check access_ok(), we have previously fetched these bytes. */ - printk("At %p:", (void *)address); - if (FPU_CS == __USER_CS) { -#define MAX_PRINTED_BYTES 20 - for (i = 0; i < MAX_PRINTED_BYTES; i++) { - FPU_get_user(byte1, (u_char __user *) address); - if ((byte1 & 0xf8) == 0xd8) { - printk(" %02x", byte1); - break; - } - printk(" [%02x]", byte1); - address++; - } - if (i == MAX_PRINTED_BYTES) - printk(" [more..]\n"); - else { - FPU_get_user(FPU_modrm, 1 + (u_char __user *) address); - - if (FPU_modrm >= 0300) - printk(" %02x (%02x+%d)\n", FPU_modrm, - FPU_modrm & 0xf8, FPU_modrm & 7); - else - printk(" /%d, mod=%d rm=%d\n", - (FPU_modrm >> 3) & 7, - (FPU_modrm >> 6) & 3, FPU_modrm & 7); - } - } else { - printk("%04x\n", FPU_CS); - } - - partial_status = status_word(); - -#ifdef DEBUGGING - if (partial_status & SW_Backward) - printk("SW: backward compatibility\n"); - if (partial_status & SW_C3) - printk("SW: condition bit 3\n"); - if (partial_status & SW_C2) - printk("SW: condition bit 2\n"); - if (partial_status & SW_C1) - printk("SW: condition bit 1\n"); - if (partial_status & SW_C0) - printk("SW: condition bit 0\n"); - if (partial_status & SW_Summary) - printk("SW: exception summary\n"); - if (partial_status & SW_Stack_Fault) - printk("SW: stack fault\n"); - if (partial_status & SW_Precision) - printk("SW: loss of precision\n"); - if (partial_status & SW_Underflow) - printk("SW: underflow\n"); - if (partial_status & SW_Overflow) - printk("SW: overflow\n"); - if (partial_status & SW_Zero_Div) - printk("SW: divide by zero\n"); - if (partial_status & SW_Denorm_Op) - printk("SW: denormalized operand\n"); - if (partial_status & SW_Invalid) - printk("SW: invalid operation\n"); -#endif /* DEBUGGING */ - - printk(" SW: b=%d st=%d es=%d sf=%d cc=%d%d%d%d ef=%d%d%d%d%d%d\n", partial_status & 0x8000 ? 1 : 0, /* busy */ - (partial_status & 0x3800) >> 11, /* stack top pointer */ - partial_status & 0x80 ? 1 : 0, /* Error summary status */ - partial_status & 0x40 ? 1 : 0, /* Stack flag */ - partial_status & SW_C3 ? 1 : 0, partial_status & SW_C2 ? 1 : 0, /* cc */ - partial_status & SW_C1 ? 1 : 0, partial_status & SW_C0 ? 1 : 0, /* cc */ - partial_status & SW_Precision ? 1 : 0, - partial_status & SW_Underflow ? 1 : 0, - partial_status & SW_Overflow ? 1 : 0, - partial_status & SW_Zero_Div ? 1 : 0, - partial_status & SW_Denorm_Op ? 1 : 0, - partial_status & SW_Invalid ? 1 : 0); - - printk(" CW: ic=%d rc=%d%d pc=%d%d iem=%d ef=%d%d%d%d%d%d\n", - control_word & 0x1000 ? 1 : 0, - (control_word & 0x800) >> 11, (control_word & 0x400) >> 10, - (control_word & 0x200) >> 9, (control_word & 0x100) >> 8, - control_word & 0x80 ? 1 : 0, - control_word & SW_Precision ? 1 : 0, - control_word & SW_Underflow ? 1 : 0, - control_word & SW_Overflow ? 1 : 0, - control_word & SW_Zero_Div ? 1 : 0, - control_word & SW_Denorm_Op ? 1 : 0, - control_word & SW_Invalid ? 1 : 0); - - for (i = 0; i < 8; i++) { - FPU_REG *r = &st(i); - u_char tagi = FPU_gettagi(i); - - switch (tagi) { - case TAG_Empty: - continue; - case TAG_Zero: - case TAG_Special: - /* Update tagi for the printk below */ - tagi = FPU_Special(r); - fallthrough; - case TAG_Valid: - printk("st(%d) %c .%04lx %04lx %04lx %04lx e%+-6d ", i, - getsign(r) ? '-' : '+', - (long)(r->sigh >> 16), - (long)(r->sigh & 0xFFFF), - (long)(r->sigl >> 16), - (long)(r->sigl & 0xFFFF), - exponent(r) - EXP_BIAS + 1); - break; - default: - printk("Whoops! Error in errors.c: tag%d is %d ", i, - tagi); - continue; - } - printk("%s\n", tag_desc[(int)(unsigned)tagi]); - } - - RE_ENTRANT_CHECK_ON; - -} - -static struct { - int type; - const char *name; -} exception_names[] = { - { - EX_StackOver, "stack overflow"}, { - EX_StackUnder, "stack underflow"}, { - EX_Precision, "loss of precision"}, { - EX_Underflow, "underflow"}, { - EX_Overflow, "overflow"}, { - EX_ZeroDiv, "divide by zero"}, { - EX_Denormal, "denormalized operand"}, { - EX_Invalid, "invalid operation"}, { - EX_INTERNAL, "INTERNAL BUG in " FPU_VERSION}, { - 0, NULL} -}; - -/* - EX_INTERNAL is always given with a code which indicates where the - error was detected. - - Internal error types: - 0x14 in fpu_etc.c - 0x1nn in a *.c file: - 0x101 in reg_add_sub.c - 0x102 in reg_mul.c - 0x104 in poly_atan.c - 0x105 in reg_mul.c - 0x107 in fpu_trig.c - 0x108 in reg_compare.c - 0x109 in reg_compare.c - 0x110 in reg_add_sub.c - 0x111 in fpe_entry.c - 0x112 in fpu_trig.c - 0x113 in errors.c - 0x115 in fpu_trig.c - 0x116 in fpu_trig.c - 0x117 in fpu_trig.c - 0x118 in fpu_trig.c - 0x119 in fpu_trig.c - 0x120 in poly_atan.c - 0x121 in reg_compare.c - 0x122 in reg_compare.c - 0x123 in reg_compare.c - 0x125 in fpu_trig.c - 0x126 in fpu_entry.c - 0x127 in poly_2xm1.c - 0x128 in fpu_entry.c - 0x129 in fpu_entry.c - 0x130 in get_address.c - 0x131 in get_address.c - 0x132 in get_address.c - 0x133 in get_address.c - 0x140 in load_store.c - 0x141 in load_store.c - 0x150 in poly_sin.c - 0x151 in poly_sin.c - 0x160 in reg_ld_str.c - 0x161 in reg_ld_str.c - 0x162 in reg_ld_str.c - 0x163 in reg_ld_str.c - 0x164 in reg_ld_str.c - 0x170 in fpu_tags.c - 0x171 in fpu_tags.c - 0x172 in fpu_tags.c - 0x180 in reg_convert.c - 0x2nn in an *.S file: - 0x201 in reg_u_add.S - 0x202 in reg_u_div.S - 0x203 in reg_u_div.S - 0x204 in reg_u_div.S - 0x205 in reg_u_mul.S - 0x206 in reg_u_sub.S - 0x207 in wm_sqrt.S - 0x208 in reg_div.S - 0x209 in reg_u_sub.S - 0x210 in reg_u_sub.S - 0x211 in reg_u_sub.S - 0x212 in reg_u_sub.S - 0x213 in wm_sqrt.S - 0x214 in wm_sqrt.S - 0x215 in wm_sqrt.S - 0x220 in reg_norm.S - 0x221 in reg_norm.S - 0x230 in reg_round.S - 0x231 in reg_round.S - 0x232 in reg_round.S - 0x233 in reg_round.S - 0x234 in reg_round.S - 0x235 in reg_round.S - 0x236 in reg_round.S - 0x240 in div_Xsig.S - 0x241 in div_Xsig.S - 0x242 in div_Xsig.S - */ - -asmlinkage __visible void FPU_exception(int n) -{ - int i, int_type; - - int_type = 0; /* Needed only to stop compiler warnings */ - if (n & EX_INTERNAL) { - int_type = n - EX_INTERNAL; - n = EX_INTERNAL; - /* Set lots of exception bits! */ - partial_status |= (SW_Exc_Mask | SW_Summary | SW_Backward); - } else { - /* Extract only the bits which we use to set the status word */ - n &= (SW_Exc_Mask); - /* Set the corresponding exception bit */ - partial_status |= n; - /* Set summary bits iff exception isn't masked */ - if (partial_status & ~control_word & CW_Exceptions) - partial_status |= (SW_Summary | SW_Backward); - if (n & (SW_Stack_Fault | EX_Precision)) { - if (!(n & SW_C1)) - /* This bit distinguishes over- from underflow for a stack fault, - and roundup from round-down for precision loss. */ - partial_status &= ~SW_C1; - } - } - - RE_ENTRANT_CHECK_OFF; - if ((~control_word & n & CW_Exceptions) || (n == EX_INTERNAL)) { - /* Get a name string for error reporting */ - for (i = 0; exception_names[i].type; i++) - if ((exception_names[i].type & n) == - exception_names[i].type) - break; - - if (exception_names[i].type) { -#ifdef PRINT_MESSAGES - printk("FP Exception: %s!\n", exception_names[i].name); -#endif /* PRINT_MESSAGES */ - } else - printk("FPU emulator: Unknown Exception: 0x%04x!\n", n); - - if (n == EX_INTERNAL) { - printk("FPU emulator: Internal error type 0x%04x\n", - int_type); - FPU_printall(); - } -#ifdef PRINT_MESSAGES - else - FPU_printall(); -#endif /* PRINT_MESSAGES */ - - /* - * The 80486 generates an interrupt on the next non-control FPU - * instruction. So we need some means of flagging it. - * We use the ES (Error Summary) bit for this. - */ - } - RE_ENTRANT_CHECK_ON; - -#ifdef __DEBUG__ - math_abort(FPU_info, SIGFPE); -#endif /* __DEBUG__ */ - -} - -/* Real operation attempted on a NaN. */ -/* Returns < 0 if the exception is unmasked */ -int real_1op_NaN(FPU_REG *a) -{ - int signalling, isNaN; - - isNaN = (exponent(a) == EXP_OVER) && (a->sigh & 0x80000000); - - /* The default result for the case of two "equal" NaNs (signs may - differ) is chosen to reproduce 80486 behaviour */ - signalling = isNaN && !(a->sigh & 0x40000000); - - if (!signalling) { - if (!isNaN) { /* pseudo-NaN, or other unsupported? */ - if (control_word & CW_Invalid) { - /* Masked response */ - reg_copy(&CONST_QNaN, a); - } - EXCEPTION(EX_Invalid); - return (!(control_word & CW_Invalid) ? FPU_Exception : - 0) | TAG_Special; - } - return TAG_Special; - } - - if (control_word & CW_Invalid) { - /* The masked response */ - if (!(a->sigh & 0x80000000)) { /* pseudo-NaN ? */ - reg_copy(&CONST_QNaN, a); - } - /* ensure a Quiet NaN */ - a->sigh |= 0x40000000; - } - - EXCEPTION(EX_Invalid); - - return (!(control_word & CW_Invalid) ? FPU_Exception : 0) | TAG_Special; -} - -/* Real operation attempted on two operands, one a NaN. */ -/* Returns < 0 if the exception is unmasked */ -int real_2op_NaN(FPU_REG const *b, u_char tagb, - int deststnr, FPU_REG const *defaultNaN) -{ - FPU_REG *dest = &st(deststnr); - FPU_REG const *a = dest; - u_char taga = FPU_gettagi(deststnr); - FPU_REG const *x; - int signalling, unsupported; - - if (taga == TAG_Special) - taga = FPU_Special(a); - if (tagb == TAG_Special) - tagb = FPU_Special(b); - - /* TW_NaN is also used for unsupported data types. */ - unsupported = ((taga == TW_NaN) - && !((exponent(a) == EXP_OVER) - && (a->sigh & 0x80000000))) - || ((tagb == TW_NaN) - && !((exponent(b) == EXP_OVER) && (b->sigh & 0x80000000))); - if (unsupported) { - if (control_word & CW_Invalid) { - /* Masked response */ - FPU_copy_to_regi(&CONST_QNaN, TAG_Special, deststnr); - } - EXCEPTION(EX_Invalid); - return (!(control_word & CW_Invalid) ? FPU_Exception : 0) | - TAG_Special; - } - - if (taga == TW_NaN) { - x = a; - if (tagb == TW_NaN) { - signalling = !(a->sigh & b->sigh & 0x40000000); - if (significand(b) > significand(a)) - x = b; - else if (significand(b) == significand(a)) { - /* The default result for the case of two "equal" NaNs (signs may - differ) is chosen to reproduce 80486 behaviour */ - x = defaultNaN; - } - } else { - /* return the quiet version of the NaN in a */ - signalling = !(a->sigh & 0x40000000); - } - } else -#ifdef PARANOID - if (tagb == TW_NaN) -#endif /* PARANOID */ - { - signalling = !(b->sigh & 0x40000000); - x = b; - } -#ifdef PARANOID - else { - signalling = 0; - EXCEPTION(EX_INTERNAL | 0x113); - x = &CONST_QNaN; - } -#endif /* PARANOID */ - - if ((!signalling) || (control_word & CW_Invalid)) { - if (!x) - x = b; - - if (!(x->sigh & 0x80000000)) /* pseudo-NaN ? */ - x = &CONST_QNaN; - - FPU_copy_to_regi(x, TAG_Special, deststnr); - - if (!signalling) - return TAG_Special; - - /* ensure a Quiet NaN */ - dest->sigh |= 0x40000000; - } - - EXCEPTION(EX_Invalid); - - return (!(control_word & CW_Invalid) ? FPU_Exception : 0) | TAG_Special; -} - -/* Invalid arith operation on Valid registers */ -/* Returns < 0 if the exception is unmasked */ -asmlinkage __visible int arith_invalid(int deststnr) -{ - - EXCEPTION(EX_Invalid); - - if (control_word & CW_Invalid) { - /* The masked response */ - FPU_copy_to_regi(&CONST_QNaN, TAG_Special, deststnr); - } - - return (!(control_word & CW_Invalid) ? FPU_Exception : 0) | TAG_Valid; - -} - -/* Divide a finite number by zero */ -asmlinkage __visible int FPU_divide_by_zero(int deststnr, u_char sign) -{ - FPU_REG *dest = &st(deststnr); - int tag = TAG_Valid; - - if (control_word & CW_ZeroDiv) { - /* The masked response */ - FPU_copy_to_regi(&CONST_INF, TAG_Special, deststnr); - setsign(dest, sign); - tag = TAG_Special; - } - - EXCEPTION(EX_ZeroDiv); - - return (!(control_word & CW_ZeroDiv) ? FPU_Exception : 0) | tag; - -} - -/* This may be called often, so keep it lean */ -int set_precision_flag(int flags) -{ - if (control_word & CW_Precision) { - partial_status &= ~(SW_C1 & flags); - partial_status |= flags; /* The masked response */ - return 0; - } else { - EXCEPTION(flags); - return 1; - } -} - -/* This may be called often, so keep it lean */ -asmlinkage __visible void set_precision_flag_up(void) -{ - if (control_word & CW_Precision) - partial_status |= (SW_Precision | SW_C1); /* The masked response */ - else - EXCEPTION(EX_Precision | SW_C1); -} - -/* This may be called often, so keep it lean */ -asmlinkage __visible void set_precision_flag_down(void) -{ - if (control_word & CW_Precision) { /* The masked response */ - partial_status &= ~SW_C1; - partial_status |= SW_Precision; - } else - EXCEPTION(EX_Precision); -} - -asmlinkage __visible int denormal_operand(void) -{ - if (control_word & CW_Denormal) { /* The masked response */ - partial_status |= SW_Denorm_Op; - return TAG_Special; - } else { - EXCEPTION(EX_Denormal); - return TAG_Special | FPU_Exception; - } -} - -asmlinkage __visible int arith_overflow(FPU_REG *dest) -{ - int tag = TAG_Valid; - - if (control_word & CW_Overflow) { - /* The masked response */ -/* ###### The response here depends upon the rounding mode */ - reg_copy(&CONST_INF, dest); - tag = TAG_Special; - } else { - /* Subtract the magic number from the exponent */ - addexponent(dest, (-3 * (1 << 13))); - } - - EXCEPTION(EX_Overflow); - if (control_word & CW_Overflow) { - /* The overflow exception is masked. */ - /* By definition, precision is lost. - The roundup bit (C1) is also set because we have - "rounded" upwards to Infinity. */ - EXCEPTION(EX_Precision | SW_C1); - return tag; - } - - return tag; - -} - -asmlinkage __visible int arith_underflow(FPU_REG *dest) -{ - int tag = TAG_Valid; - - if (control_word & CW_Underflow) { - /* The masked response */ - if (exponent16(dest) <= EXP_UNDER - 63) { - reg_copy(&CONST_Z, dest); - partial_status &= ~SW_C1; /* Round down. */ - tag = TAG_Zero; - } else { - stdexp(dest); - } - } else { - /* Add the magic number to the exponent. */ - addexponent(dest, (3 * (1 << 13)) + EXTENDED_Ebias); - } - - EXCEPTION(EX_Underflow); - if (control_word & CW_Underflow) { - /* The underflow exception is masked. */ - EXCEPTION(EX_Precision); - return tag; - } - - return tag; - -} - -void FPU_stack_overflow(void) -{ - - if (control_word & CW_Invalid) { - /* The masked response */ - top--; - FPU_copy_to_reg0(&CONST_QNaN, TAG_Special); - } - - EXCEPTION(EX_StackOver); - - return; - -} - -void FPU_stack_underflow(void) -{ - - if (control_word & CW_Invalid) { - /* The masked response */ - FPU_copy_to_reg0(&CONST_QNaN, TAG_Special); - } - - EXCEPTION(EX_StackUnder); - - return; - -} - -void FPU_stack_underflow_i(int i) -{ - - if (control_word & CW_Invalid) { - /* The masked response */ - FPU_copy_to_regi(&CONST_QNaN, TAG_Special, i); - } - - EXCEPTION(EX_StackUnder); - - return; - -} - -void FPU_stack_underflow_pop(int i) -{ - - if (control_word & CW_Invalid) { - /* The masked response */ - FPU_copy_to_regi(&CONST_QNaN, TAG_Special, i); - FPU_pop(); - } - - EXCEPTION(EX_StackUnder); - - return; - -} diff --git a/arch/x86/math-emu/exception.h b/arch/x86/math-emu/exception.h deleted file mode 100644 index 59961d350bc4..000000000000 --- a/arch/x86/math-emu/exception.h +++ /dev/null @@ -1,51 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/*---------------------------------------------------------------------------+ - | exception.h | - | | - | Copyright (C) 1992 W. Metzenthen, 22 Parker St, Ormond, Vic 3163, | - | Australia. E-mail billm@vaxc.cc.monash.edu.au | - | | - +---------------------------------------------------------------------------*/ - -#ifndef _EXCEPTION_H_ -#define _EXCEPTION_H_ - -#ifdef __ASSEMBLER__ -#define Const_(x) $##x -#else -#define Const_(x) x -#endif - -#ifndef SW_C1 -#include "fpu_emu.h" -#endif /* SW_C1 */ - -#define FPU_BUSY Const_(0x8000) /* FPU busy bit (8087 compatibility) */ -#define EX_ErrorSummary Const_(0x0080) /* Error summary status */ -/* Special exceptions: */ -#define EX_INTERNAL Const_(0x8000) /* Internal error in wm-FPU-emu */ -#define EX_StackOver Const_(0x0041|SW_C1) /* stack overflow */ -#define EX_StackUnder Const_(0x0041) /* stack underflow */ -/* Exception flags: */ -#define EX_Precision Const_(0x0020) /* loss of precision */ -#define EX_Underflow Const_(0x0010) /* underflow */ -#define EX_Overflow Const_(0x0008) /* overflow */ -#define EX_ZeroDiv Const_(0x0004) /* divide by zero */ -#define EX_Denormal Const_(0x0002) /* denormalized operand */ -#define EX_Invalid Const_(0x0001) /* invalid operation */ - -#define PRECISION_LOST_UP Const_((EX_Precision | SW_C1)) -#define PRECISION_LOST_DOWN Const_(EX_Precision) - -#ifndef __ASSEMBLER__ - -#ifdef DEBUG -#define EXCEPTION(x) { printk("exception in %s at line %d\n", \ - __FILE__, __LINE__); FPU_exception(x); } -#else -#define EXCEPTION(x) FPU_exception(x) -#endif - -#endif /* __ASSEMBLER__ */ - -#endif /* _EXCEPTION_H_ */ diff --git a/arch/x86/math-emu/fpu_arith.c b/arch/x86/math-emu/fpu_arith.c deleted file mode 100644 index 09006dc474a0..000000000000 --- a/arch/x86/math-emu/fpu_arith.c +++ /dev/null @@ -1,153 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/*---------------------------------------------------------------------------+ - | fpu_arith.c | - | | - | Code to implement the FPU register/register arithmetic instructions | - | | - | Copyright (C) 1992,1993,1997 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia | - | E-mail billm@suburbia.net | - | | - | | - +---------------------------------------------------------------------------*/ - -#include "fpu_system.h" -#include "fpu_emu.h" -#include "control_w.h" -#include "status_w.h" - -void fadd__(void) -{ - /* fadd st,st(i) */ - int i = FPU_rm; - clear_C1(); - FPU_add(&st(i), FPU_gettagi(i), 0, control_word); -} - -void fmul__(void) -{ - /* fmul st,st(i) */ - int i = FPU_rm; - clear_C1(); - FPU_mul(&st(i), FPU_gettagi(i), 0, control_word); -} - -void fsub__(void) -{ - /* fsub st,st(i) */ - clear_C1(); - FPU_sub(0, FPU_rm, control_word); -} - -void fsubr_(void) -{ - /* fsubr st,st(i) */ - clear_C1(); - FPU_sub(REV, FPU_rm, control_word); -} - -void fdiv__(void) -{ - /* fdiv st,st(i) */ - clear_C1(); - FPU_div(0, FPU_rm, control_word); -} - -void fdivr_(void) -{ - /* fdivr st,st(i) */ - clear_C1(); - FPU_div(REV, FPU_rm, control_word); -} - -void fadd_i(void) -{ - /* fadd st(i),st */ - int i = FPU_rm; - clear_C1(); - FPU_add(&st(i), FPU_gettagi(i), i, control_word); -} - -void fmul_i(void) -{ - /* fmul st(i),st */ - clear_C1(); - FPU_mul(&st(0), FPU_gettag0(), FPU_rm, control_word); -} - -void fsubri(void) -{ - /* fsubr st(i),st */ - clear_C1(); - FPU_sub(DEST_RM, FPU_rm, control_word); -} - -void fsub_i(void) -{ - /* fsub st(i),st */ - clear_C1(); - FPU_sub(REV | DEST_RM, FPU_rm, control_word); -} - -void fdivri(void) -{ - /* fdivr st(i),st */ - clear_C1(); - FPU_div(DEST_RM, FPU_rm, control_word); -} - -void fdiv_i(void) -{ - /* fdiv st(i),st */ - clear_C1(); - FPU_div(REV | DEST_RM, FPU_rm, control_word); -} - -void faddp_(void) -{ - /* faddp st(i),st */ - int i = FPU_rm; - clear_C1(); - if (FPU_add(&st(i), FPU_gettagi(i), i, control_word) >= 0) - FPU_pop(); -} - -void fmulp_(void) -{ - /* fmulp st(i),st */ - clear_C1(); - if (FPU_mul(&st(0), FPU_gettag0(), FPU_rm, control_word) >= 0) - FPU_pop(); -} - -void fsubrp(void) -{ - /* fsubrp st(i),st */ - clear_C1(); - if (FPU_sub(DEST_RM, FPU_rm, control_word) >= 0) - FPU_pop(); -} - -void fsubp_(void) -{ - /* fsubp st(i),st */ - clear_C1(); - if (FPU_sub(REV | DEST_RM, FPU_rm, control_word) >= 0) - FPU_pop(); -} - -void fdivrp(void) -{ - /* fdivrp st(i),st */ - clear_C1(); - if (FPU_div(DEST_RM, FPU_rm, control_word) >= 0) - FPU_pop(); -} - -void fdivp_(void) -{ - /* fdivp st(i),st */ - clear_C1(); - if (FPU_div(REV | DEST_RM, FPU_rm, control_word) >= 0) - FPU_pop(); -} diff --git a/arch/x86/math-emu/fpu_asm.h b/arch/x86/math-emu/fpu_asm.h deleted file mode 100644 index a83353d5271c..000000000000 --- a/arch/x86/math-emu/fpu_asm.h +++ /dev/null @@ -1,32 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/*---------------------------------------------------------------------------+ - | fpu_asm.h | - | | - | Copyright (C) 1992,1995,1997 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, | - | Australia. E-mail billm@suburbia.net | - | | - +---------------------------------------------------------------------------*/ - -#ifndef _FPU_ASM_H_ -#define _FPU_ASM_H_ - -#include - -#define EXCEPTION FPU_exception - -#define PARAM1 8(%ebp) -#define PARAM2 12(%ebp) -#define PARAM3 16(%ebp) -#define PARAM4 20(%ebp) -#define PARAM5 24(%ebp) -#define PARAM6 28(%ebp) -#define PARAM7 32(%ebp) - -#define SIGL_OFFSET 0 -#define EXP(x) 8(x) -#define SIG(x) SIGL_OFFSET##(x) -#define SIGL(x) SIGL_OFFSET##(x) -#define SIGH(x) 4(x) - -#endif /* _FPU_ASM_H_ */ diff --git a/arch/x86/math-emu/fpu_aux.c b/arch/x86/math-emu/fpu_aux.c deleted file mode 100644 index 5f253ae406b6..000000000000 --- a/arch/x86/math-emu/fpu_aux.c +++ /dev/null @@ -1,267 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/*---------------------------------------------------------------------------+ - | fpu_aux.c | - | | - | Code to implement some of the FPU auxiliary instructions. | - | | - | Copyright (C) 1992,1993,1994,1997 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia | - | E-mail billm@suburbia.net | - | | - | | - +---------------------------------------------------------------------------*/ - -#include "fpu_system.h" -#include "exception.h" -#include "fpu_emu.h" -#include "status_w.h" -#include "control_w.h" - -static void fnop(void) -{ -} - -static void fclex(void) -{ - partial_status &= - ~(SW_Backward | SW_Summary | SW_Stack_Fault | SW_Precision | - SW_Underflow | SW_Overflow | SW_Zero_Div | SW_Denorm_Op | - SW_Invalid); - no_ip_update = 1; -} - -/* Needs to be externally visible */ -void fpstate_init_soft(struct swregs_state *soft) -{ - struct address *oaddr, *iaddr; - memset(soft, 0, sizeof(*soft)); - soft->cwd = 0x037f; - soft->swd = 0; - soft->ftop = 0; /* We don't keep top in the status word internally. */ - soft->twd = 0xffff; - /* The behaviour is different from that detailed in - Section 15.1.6 of the Intel manual */ - oaddr = (struct address *)&soft->foo; - oaddr->offset = 0; - oaddr->selector = 0; - iaddr = (struct address *)&soft->fip; - iaddr->offset = 0; - iaddr->selector = 0; - iaddr->opcode = 0; - soft->no_update = 1; -} - -void finit(void) -{ - fpstate_init_soft(&x86_task_fpu(current)->fpstate->regs.soft); -} - -/* - * These are nops on the i387.. - */ -#define feni fnop -#define fdisi fnop -#define fsetpm fnop - -static FUNC const finit_table[] = { - feni, fdisi, fclex, finit, - fsetpm, FPU_illegal, FPU_illegal, FPU_illegal -}; - -void finit_(void) -{ - (finit_table[FPU_rm]) (); -} - -static void fstsw_ax(void) -{ - *(short *)&FPU_EAX = status_word(); - no_ip_update = 1; -} - -static FUNC const fstsw_table[] = { - fstsw_ax, FPU_illegal, FPU_illegal, FPU_illegal, - FPU_illegal, FPU_illegal, FPU_illegal, FPU_illegal -}; - -void fstsw_(void) -{ - (fstsw_table[FPU_rm]) (); -} - -static FUNC const fp_nop_table[] = { - fnop, FPU_illegal, FPU_illegal, FPU_illegal, - FPU_illegal, FPU_illegal, FPU_illegal, FPU_illegal -}; - -void fp_nop(void) -{ - (fp_nop_table[FPU_rm]) (); -} - -void fld_i_(void) -{ - FPU_REG *st_new_ptr; - int i; - u_char tag; - - if (STACK_OVERFLOW) { - FPU_stack_overflow(); - return; - } - - /* fld st(i) */ - i = FPU_rm; - if (NOT_EMPTY(i)) { - reg_copy(&st(i), st_new_ptr); - tag = FPU_gettagi(i); - push(); - FPU_settag0(tag); - } else { - if (control_word & CW_Invalid) { - /* The masked response */ - FPU_stack_underflow(); - } else - EXCEPTION(EX_StackUnder); - } - -} - -void fxch_i(void) -{ - /* fxch st(i) */ - FPU_REG t; - int i = FPU_rm; - FPU_REG *st0_ptr = &st(0), *sti_ptr = &st(i); - long tag_word = fpu_tag_word; - int regnr = top & 7, regnri = ((regnr + i) & 7); - u_char st0_tag = (tag_word >> (regnr * 2)) & 3; - u_char sti_tag = (tag_word >> (regnri * 2)) & 3; - - if (st0_tag == TAG_Empty) { - if (sti_tag == TAG_Empty) { - FPU_stack_underflow(); - FPU_stack_underflow_i(i); - return; - } - if (control_word & CW_Invalid) { - /* Masked response */ - FPU_copy_to_reg0(sti_ptr, sti_tag); - } - FPU_stack_underflow_i(i); - return; - } - if (sti_tag == TAG_Empty) { - if (control_word & CW_Invalid) { - /* Masked response */ - FPU_copy_to_regi(st0_ptr, st0_tag, i); - } - FPU_stack_underflow(); - return; - } - clear_C1(); - - reg_copy(st0_ptr, &t); - reg_copy(sti_ptr, st0_ptr); - reg_copy(&t, sti_ptr); - - tag_word &= ~(3 << (regnr * 2)) & ~(3 << (regnri * 2)); - tag_word |= (sti_tag << (regnr * 2)) | (st0_tag << (regnri * 2)); - fpu_tag_word = tag_word; -} - -static void fcmovCC(void) -{ - /* fcmovCC st(i) */ - int i = FPU_rm; - FPU_REG *st0_ptr = &st(0); - FPU_REG *sti_ptr = &st(i); - long tag_word = fpu_tag_word; - int regnr = top & 7; - int regnri = (top + i) & 7; - u_char sti_tag = (tag_word >> (regnri * 2)) & 3; - - if (sti_tag == TAG_Empty) { - FPU_stack_underflow(); - clear_C1(); - return; - } - reg_copy(sti_ptr, st0_ptr); - tag_word &= ~(3 << (regnr * 2)); - tag_word |= (sti_tag << (regnr * 2)); - fpu_tag_word = tag_word; -} - -void fcmovb(void) -{ - if (FPU_EFLAGS & X86_EFLAGS_CF) - fcmovCC(); -} - -void fcmove(void) -{ - if (FPU_EFLAGS & X86_EFLAGS_ZF) - fcmovCC(); -} - -void fcmovbe(void) -{ - if (FPU_EFLAGS & (X86_EFLAGS_CF|X86_EFLAGS_ZF)) - fcmovCC(); -} - -void fcmovu(void) -{ - if (FPU_EFLAGS & X86_EFLAGS_PF) - fcmovCC(); -} - -void fcmovnb(void) -{ - if (!(FPU_EFLAGS & X86_EFLAGS_CF)) - fcmovCC(); -} - -void fcmovne(void) -{ - if (!(FPU_EFLAGS & X86_EFLAGS_ZF)) - fcmovCC(); -} - -void fcmovnbe(void) -{ - if (!(FPU_EFLAGS & (X86_EFLAGS_CF|X86_EFLAGS_ZF))) - fcmovCC(); -} - -void fcmovnu(void) -{ - if (!(FPU_EFLAGS & X86_EFLAGS_PF)) - fcmovCC(); -} - -void ffree_(void) -{ - /* ffree st(i) */ - FPU_settagi(FPU_rm, TAG_Empty); -} - -void ffreep(void) -{ - /* ffree st(i) + pop - unofficial code */ - FPU_settagi(FPU_rm, TAG_Empty); - FPU_pop(); -} - -void fst_i_(void) -{ - /* fst st(i) */ - FPU_copy_to_regi(&st(0), FPU_gettag0(), FPU_rm); -} - -void fstp_i(void) -{ - /* fstp st(i) */ - FPU_copy_to_regi(&st(0), FPU_gettag0(), FPU_rm); - FPU_pop(); -} diff --git a/arch/x86/math-emu/fpu_emu.h b/arch/x86/math-emu/fpu_emu.h deleted file mode 100644 index def569c50b76..000000000000 --- a/arch/x86/math-emu/fpu_emu.h +++ /dev/null @@ -1,218 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/*---------------------------------------------------------------------------+ - | fpu_emu.h | - | | - | Copyright (C) 1992,1993,1994,1997 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, | - | Australia. E-mail billm@suburbia.net | - | | - +---------------------------------------------------------------------------*/ - -#ifndef _FPU_EMU_H_ -#define _FPU_EMU_H_ - -/* - * Define PECULIAR_486 to get a closer approximation to 80486 behaviour, - * rather than behaviour which appears to be cleaner. - * This is a matter of opinion: for all I know, the 80486 may simply - * be complying with the IEEE spec. Maybe one day I'll get to see the - * spec... - */ -#define PECULIAR_486 - -#ifdef __ASSEMBLER__ -#include "fpu_asm.h" -#define Const(x) $##x -#else -#define Const(x) x -#endif - -#define EXP_BIAS Const(0) -#define EXP_OVER Const(0x4000) /* smallest invalid large exponent */ -#define EXP_UNDER Const(-0x3fff) /* largest invalid small exponent */ -#define EXP_WAY_UNDER Const(-0x6000) /* Below the smallest denormal, but - still a 16 bit nr. */ -#define EXP_Infinity EXP_OVER -#define EXP_NaN EXP_OVER - -#define EXTENDED_Ebias Const(0x3fff) -#define EXTENDED_Emin (-0x3ffe) /* smallest valid exponent */ - -#define SIGN_POS Const(0) -#define SIGN_NEG Const(0x80) - -#define SIGN_Positive Const(0) -#define SIGN_Negative Const(0x8000) - -/* Keep the order TAG_Valid, TAG_Zero, TW_Denormal */ -/* The following fold to 2 (Special) in the Tag Word */ -#define TW_Denormal Const(4) /* De-normal */ -#define TW_Infinity Const(5) /* + or - infinity */ -#define TW_NaN Const(6) /* Not a Number */ -#define TW_Unsupported Const(7) /* Not supported by an 80486 */ - -#define TAG_Valid Const(0) /* valid */ -#define TAG_Zero Const(1) /* zero */ -#define TAG_Special Const(2) /* De-normal, + or - infinity, - or Not a Number */ -#define TAG_Empty Const(3) /* empty */ -#define TAG_Error Const(0x80) /* probably need to abort */ - -#define LOADED_DATA Const(10101) /* Special st() number to identify - loaded data (not on stack). */ - -/* A few flags (must be >= 0x10). */ -#define REV 0x10 -#define DEST_RM 0x20 -#define LOADED 0x40 - -#define FPU_Exception Const(0x80000000) /* Added to tag returns. */ - -#ifndef __ASSEMBLER__ - -#include "fpu_system.h" - -#include /* for struct _fpstate */ -#include -#include - -/* -#define RE_ENTRANT_CHECKING - */ - -#ifdef RE_ENTRANT_CHECKING -extern u_char emulating; -# define RE_ENTRANT_CHECK_OFF emulating = 0 -# define RE_ENTRANT_CHECK_ON emulating = 1 -#else -# define RE_ENTRANT_CHECK_OFF -# define RE_ENTRANT_CHECK_ON -#endif /* RE_ENTRANT_CHECKING */ - -#define FWAIT_OPCODE 0x9b -#define OP_SIZE_PREFIX 0x66 -#define ADDR_SIZE_PREFIX 0x67 -#define PREFIX_CS 0x2e -#define PREFIX_DS 0x3e -#define PREFIX_ES 0x26 -#define PREFIX_SS 0x36 -#define PREFIX_FS 0x64 -#define PREFIX_GS 0x65 -#define PREFIX_REPE 0xf3 -#define PREFIX_REPNE 0xf2 -#define PREFIX_LOCK 0xf0 -#define PREFIX_CS_ 1 -#define PREFIX_DS_ 2 -#define PREFIX_ES_ 3 -#define PREFIX_FS_ 4 -#define PREFIX_GS_ 5 -#define PREFIX_SS_ 6 -#define PREFIX_DEFAULT 7 - -struct address { - unsigned int offset; - unsigned int selector:16; - unsigned int opcode:11; - unsigned int empty:5; -}; -struct fpu__reg { - unsigned sigl; - unsigned sigh; - short exp; -}; - -typedef void (*FUNC) (void); -typedef struct fpu__reg FPU_REG; -typedef void (*FUNC_ST0) (FPU_REG *st0_ptr, u_char st0_tag); -typedef struct { - u_char address_size, operand_size, segment; -} overrides; -/* This structure is 32 bits: */ -typedef struct { - overrides override; - u_char default_mode; -} fpu_addr_modes; -/* PROTECTED has a restricted meaning in the emulator; it is used - to signal that the emulator needs to do special things to ensure - that protection is respected in a segmented model. */ -#define PROTECTED 4 -#define SIXTEEN 1 /* We rely upon this being 1 (true) */ -#define VM86 SIXTEEN -#define PM16 (SIXTEEN | PROTECTED) -#define SEG32 PROTECTED -extern u_char const data_sizes_16[32]; - -#define register_base ((u_char *) registers ) -#define fpu_register(x) ( * ((FPU_REG *)( register_base + 10 * (x & 7) )) ) -#define st(x) ( * ((FPU_REG *)( register_base + 10 * ((top+x) & 7) )) ) - -#define STACK_OVERFLOW (FPU_stackoverflow(&st_new_ptr)) -#define NOT_EMPTY(i) (!FPU_empty_i(i)) - -#define NOT_EMPTY_ST0 (st0_tag ^ TAG_Empty) - -#define poppop() { FPU_pop(); FPU_pop(); } - -/* push() does not affect the tags */ -#define push() { top--; } - -#define signbyte(a) (((u_char *)(a))[9]) -#define getsign(a) (signbyte(a) & 0x80) -#define setsign(a,b) { if ((b) != 0) signbyte(a) |= 0x80; else signbyte(a) &= 0x7f; } -#define copysign(a,b) { if (getsign(a)) signbyte(b) |= 0x80; \ - else signbyte(b) &= 0x7f; } -#define changesign(a) { signbyte(a) ^= 0x80; } -#define setpositive(a) { signbyte(a) &= 0x7f; } -#define setnegative(a) { signbyte(a) |= 0x80; } -#define signpositive(a) ( (signbyte(a) & 0x80) == 0 ) -#define signnegative(a) (signbyte(a) & 0x80) - -static inline void reg_copy(FPU_REG const *x, FPU_REG *y) -{ - *(short *)&(y->exp) = *(const short *)&(x->exp); - *(long long *)&(y->sigl) = *(const long long *)&(x->sigl); -} - -#define exponent(x) (((*(short *)&((x)->exp)) & 0x7fff) - EXTENDED_Ebias) -#define setexponentpos(x,y) { (*(short *)&((x)->exp)) = \ - ((y) + EXTENDED_Ebias) & 0x7fff; } -#define exponent16(x) (*(short *)&((x)->exp)) -#define setexponent16(x,y) { (*(short *)&((x)->exp)) = (u16)(y); } -#define addexponent(x,y) { (*(short *)&((x)->exp)) += (y); } -#define stdexp(x) { (*(short *)&((x)->exp)) += EXTENDED_Ebias; } - -#define isdenormal(ptr) (exponent(ptr) == EXP_BIAS+EXP_UNDER) - -#define significand(x) ( ((unsigned long long *)&((x)->sigl))[0] ) - -/*----- Prototypes for functions written in assembler -----*/ -/* extern void reg_move(FPU_REG *a, FPU_REG *b); */ - -asmlinkage int FPU_normalize(FPU_REG *x); -asmlinkage int FPU_normalize_nuo(FPU_REG *x); -asmlinkage int FPU_u_sub(FPU_REG const *arg1, FPU_REG const *arg2, - FPU_REG * answ, unsigned int control_w, u_char sign, - int expa, int expb); -asmlinkage int FPU_u_mul(FPU_REG const *arg1, FPU_REG const *arg2, - FPU_REG * answ, unsigned int control_w, u_char sign, - int expon); -asmlinkage int FPU_u_div(FPU_REG const *arg1, FPU_REG const *arg2, - FPU_REG * answ, unsigned int control_w, u_char sign); -asmlinkage int FPU_u_add(FPU_REG const *arg1, FPU_REG const *arg2, - FPU_REG * answ, unsigned int control_w, u_char sign, - int expa, int expb); -asmlinkage int wm_sqrt(FPU_REG *n, int dummy1, int dummy2, - unsigned int control_w, u_char sign); -asmlinkage unsigned FPU_shrx(void *l, unsigned x); -asmlinkage unsigned FPU_shrxs(void *v, unsigned x); -asmlinkage unsigned long FPU_div_small(unsigned long long *x, unsigned long y); -asmlinkage int FPU_round(FPU_REG *arg, unsigned int extent, int dummy, - unsigned int control_w, u_char sign); - -#ifndef MAKING_PROTO -#include "fpu_proto.h" -#endif - -#endif /* __ASSEMBLER__ */ - -#endif /* _FPU_EMU_H_ */ diff --git a/arch/x86/math-emu/fpu_entry.c b/arch/x86/math-emu/fpu_entry.c deleted file mode 100644 index 5034df617740..000000000000 --- a/arch/x86/math-emu/fpu_entry.c +++ /dev/null @@ -1,718 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/*---------------------------------------------------------------------------+ - | fpu_entry.c | - | | - | The entry functions for wm-FPU-emu | - | | - | Copyright (C) 1992,1993,1994,1996,1997 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia | - | E-mail billm@suburbia.net | - | | - | See the files "README" and "COPYING" for further copyright and warranty | - | information. | - | | - +---------------------------------------------------------------------------*/ - -/*---------------------------------------------------------------------------+ - | Note: | - | The file contains code which accesses user memory. | - | Emulator static data may change when user memory is accessed, due to | - | other processes using the emulator while swapping is in progress. | - +---------------------------------------------------------------------------*/ - -/*---------------------------------------------------------------------------+ - | math_emulate(), restore_i387_soft() and save_i387_soft() are the only | - | entry points for wm-FPU-emu. | - +---------------------------------------------------------------------------*/ - -#include -#include - -#include -#include -#include -#include -#include - -#include "fpu_system.h" -#include "fpu_emu.h" -#include "exception.h" -#include "control_w.h" -#include "status_w.h" - -#define __BAD__ FPU_illegal /* Illegal on an 80486, causes SIGILL */ - -/* fcmovCC and f(u)comi(p) are enabled if CPUID(1).EDX(15) "cmov" is set */ - -/* WARNING: "u" entries are not documented by Intel in their 80486 manual - and may not work on FPU clones or later Intel FPUs. - Changes to support them provided by Linus Torvalds. */ - -static FUNC const st_instr_table[64] = { -/* Opcode: d8 d9 da db */ -/* dc dd de df */ -/* c0..7 */ fadd__, fld_i_, fcmovb, fcmovnb, -/* c0..7 */ fadd_i, ffree_, faddp_, ffreep,/*u*/ -/* c8..f */ fmul__, fxch_i, fcmove, fcmovne, -/* c8..f */ fmul_i, fxch_i,/*u*/ fmulp_, fxch_i,/*u*/ -/* d0..7 */ fcom_st, fp_nop, fcmovbe, fcmovnbe, -/* d0..7 */ fcom_st,/*u*/ fst_i_, fcompst,/*u*/ fstp_i,/*u*/ -/* d8..f */ fcompst, fstp_i,/*u*/ fcmovu, fcmovnu, -/* d8..f */ fcompst,/*u*/ fstp_i, fcompp, fstp_i,/*u*/ -/* e0..7 */ fsub__, FPU_etc, __BAD__, finit_, -/* e0..7 */ fsubri, fucom_, fsubrp, fstsw_, -/* e8..f */ fsubr_, fconst, fucompp, fucomi_, -/* e8..f */ fsub_i, fucomp, fsubp_, fucomip, -/* f0..7 */ fdiv__, FPU_triga, __BAD__, fcomi_, -/* f0..7 */ fdivri, __BAD__, fdivrp, fcomip, -/* f8..f */ fdivr_, FPU_trigb, __BAD__, __BAD__, -/* f8..f */ fdiv_i, __BAD__, fdivp_, __BAD__, -}; - -#define _NONE_ 0 /* Take no special action */ -#define _REG0_ 1 /* Need to check for not empty st(0) */ -#define _REGI_ 2 /* Need to check for not empty st(0) and st(rm) */ -#define _REGi_ 0 /* Uses st(rm) */ -#define _PUSH_ 3 /* Need to check for space to push onto stack */ -#define _null_ 4 /* Function illegal or not implemented */ -#define _REGIi 5 /* Uses st(0) and st(rm), result to st(rm) */ -#define _REGIp 6 /* Uses st(0) and st(rm), result to st(rm) then pop */ -#define _REGIc 0 /* Compare st(0) and st(rm) */ -#define _REGIn 0 /* Uses st(0) and st(rm), but handle checks later */ - -static u_char const type_table[64] = { -/* Opcode: d8 d9 da db dc dd de df */ -/* c0..7 */ _REGI_, _NONE_, _REGIn, _REGIn, _REGIi, _REGi_, _REGIp, _REGi_, -/* c8..f */ _REGI_, _REGIn, _REGIn, _REGIn, _REGIi, _REGI_, _REGIp, _REGI_, -/* d0..7 */ _REGIc, _NONE_, _REGIn, _REGIn, _REGIc, _REG0_, _REGIc, _REG0_, -/* d8..f */ _REGIc, _REG0_, _REGIn, _REGIn, _REGIc, _REG0_, _REGIc, _REG0_, -/* e0..7 */ _REGI_, _NONE_, _null_, _NONE_, _REGIi, _REGIc, _REGIp, _NONE_, -/* e8..f */ _REGI_, _NONE_, _REGIc, _REGIc, _REGIi, _REGIc, _REGIp, _REGIc, -/* f0..7 */ _REGI_, _NONE_, _null_, _REGIc, _REGIi, _null_, _REGIp, _REGIc, -/* f8..f */ _REGI_, _NONE_, _null_, _null_, _REGIi, _null_, _REGIp, _null_, -}; - -#ifdef RE_ENTRANT_CHECKING -u_char emulating = 0; -#endif /* RE_ENTRANT_CHECKING */ - -static int valid_prefix(u_char *Byte, u_char __user ** fpu_eip, - overrides * override); - -void math_emulate(struct math_emu_info *info) -{ - u_char FPU_modrm, byte1; - unsigned short code; - fpu_addr_modes addr_modes; - int unmasked; - FPU_REG loaded_data; - FPU_REG *st0_ptr; - u_char loaded_tag, st0_tag; - void __user *data_address; - struct address data_sel_off; - struct address entry_sel_off; - unsigned long code_base = 0; - unsigned long code_limit = 0; /* Initialized to stop compiler warnings */ - struct desc_struct code_descriptor; - -#ifdef RE_ENTRANT_CHECKING - if (emulating) { - printk("ERROR: wm-FPU-emu is not RE-ENTRANT!\n"); - } - RE_ENTRANT_CHECK_ON; -#endif /* RE_ENTRANT_CHECKING */ - - FPU_info = info; - - FPU_ORIG_EIP = FPU_EIP; - - if ((FPU_EFLAGS & 0x00020000) != 0) { - /* Virtual 8086 mode */ - addr_modes.default_mode = VM86; - FPU_EIP += code_base = FPU_CS << 4; - code_limit = code_base + 0xffff; /* Assumes code_base <= 0xffff0000 */ - } else if (FPU_CS == __USER_CS && FPU_DS == __USER_DS) { - addr_modes.default_mode = 0; - } else if (FPU_CS == __KERNEL_CS) { - printk("math_emulate: %04x:%08lx\n", FPU_CS, FPU_EIP); - panic("Math emulation needed in kernel"); - } else { - - if ((FPU_CS & 4) != 4) { /* Must be in the LDT */ - /* Can only handle segmented addressing via the LDT - for now, and it must be 16 bit */ - printk("FPU emulator: Unsupported addressing mode\n"); - math_abort(FPU_info, SIGILL); - } - - code_descriptor = FPU_get_ldt_descriptor(FPU_CS); - if (code_descriptor.d) { - /* The above test may be wrong, the book is not clear */ - /* Segmented 32 bit protected mode */ - addr_modes.default_mode = SEG32; - } else { - /* 16 bit protected mode */ - addr_modes.default_mode = PM16; - } - FPU_EIP += code_base = seg_get_base(&code_descriptor); - code_limit = seg_get_limit(&code_descriptor) + 1; - code_limit *= seg_get_granularity(&code_descriptor); - code_limit += code_base - 1; - if (code_limit < code_base) - code_limit = 0xffffffff; - } - - FPU_lookahead = !(FPU_EFLAGS & X86_EFLAGS_TF); - - if (!valid_prefix(&byte1, (u_char __user **) & FPU_EIP, - &addr_modes.override)) { - RE_ENTRANT_CHECK_OFF; - printk - ("FPU emulator: Unknown prefix byte 0x%02x, probably due to\n" - "FPU emulator: self-modifying code! (emulation impossible)\n", - byte1); - RE_ENTRANT_CHECK_ON; - EXCEPTION(EX_INTERNAL | 0x126); - math_abort(FPU_info, SIGILL); - } - - do_another_FPU_instruction: - - no_ip_update = 0; - - FPU_EIP++; /* We have fetched the prefix and first code bytes. */ - - if (addr_modes.default_mode) { - /* This checks for the minimum instruction bytes. - We also need to check any extra (address mode) code access. */ - if (FPU_EIP > code_limit) - math_abort(FPU_info, SIGSEGV); - } - - if ((byte1 & 0xf8) != 0xd8) { - if (byte1 == FWAIT_OPCODE) { - if (partial_status & SW_Summary) - goto do_the_FPU_interrupt; - else - goto FPU_fwait_done; - } -#ifdef PARANOID - EXCEPTION(EX_INTERNAL | 0x128); - math_abort(FPU_info, SIGILL); -#endif /* PARANOID */ - } - - RE_ENTRANT_CHECK_OFF; - FPU_code_access_ok(1); - FPU_get_user(FPU_modrm, (u_char __user *) FPU_EIP); - RE_ENTRANT_CHECK_ON; - FPU_EIP++; - - if (partial_status & SW_Summary) { - /* Ignore the error for now if the current instruction is a no-wait - control instruction */ - /* The 80486 manual contradicts itself on this topic, - but a real 80486 uses the following instructions: - fninit, fnstenv, fnsave, fnstsw, fnstenv, fnclex. - */ - code = (FPU_modrm << 8) | byte1; - if (!((((code & 0xf803) == 0xe003) || /* fnclex, fninit, fnstsw */ - (((code & 0x3003) == 0x3001) && /* fnsave, fnstcw, fnstenv, - fnstsw */ - ((code & 0xc000) != 0xc000))))) { - /* - * We need to simulate the action of the kernel to FPU - * interrupts here. - */ - do_the_FPU_interrupt: - - FPU_EIP = FPU_ORIG_EIP; /* Point to current FPU instruction. */ - - RE_ENTRANT_CHECK_OFF; - current->thread.trap_nr = X86_TRAP_MF; - current->thread.error_code = 0; - send_sig(SIGFPE, current, 1); - return; - } - } - - entry_sel_off.offset = FPU_ORIG_EIP; - entry_sel_off.selector = FPU_CS; - entry_sel_off.opcode = (byte1 << 8) | FPU_modrm; - entry_sel_off.empty = 0; - - FPU_rm = FPU_modrm & 7; - - if (FPU_modrm < 0300) { - /* All of these instructions use the mod/rm byte to get a data address */ - - if ((addr_modes.default_mode & SIXTEEN) - ^ (addr_modes.override.address_size == ADDR_SIZE_PREFIX)) - data_address = - FPU_get_address_16(FPU_modrm, &FPU_EIP, - &data_sel_off, addr_modes); - else - data_address = - FPU_get_address(FPU_modrm, &FPU_EIP, &data_sel_off, - addr_modes); - - if (addr_modes.default_mode) { - if (FPU_EIP - 1 > code_limit) - math_abort(FPU_info, SIGSEGV); - } - - if (!(byte1 & 1)) { - unsigned short status1 = partial_status; - - st0_ptr = &st(0); - st0_tag = FPU_gettag0(); - - /* Stack underflow has priority */ - if (NOT_EMPTY_ST0) { - if (addr_modes.default_mode & PROTECTED) { - /* This table works for 16 and 32 bit protected mode */ - if (access_limit < - data_sizes_16[(byte1 >> 1) & 3]) - math_abort(FPU_info, SIGSEGV); - } - - unmasked = 0; /* Do this here to stop compiler warnings. */ - switch ((byte1 >> 1) & 3) { - case 0: - unmasked = - FPU_load_single((float __user *) - data_address, - &loaded_data); - loaded_tag = unmasked & 0xff; - unmasked &= ~0xff; - break; - case 1: - loaded_tag = - FPU_load_int32((long __user *) - data_address, - &loaded_data); - break; - case 2: - unmasked = - FPU_load_double((double __user *) - data_address, - &loaded_data); - loaded_tag = unmasked & 0xff; - unmasked &= ~0xff; - break; - case 3: - default: /* Used here to suppress gcc warnings. */ - loaded_tag = - FPU_load_int16((short __user *) - data_address, - &loaded_data); - break; - } - - /* No more access to user memory, it is safe - to use static data now */ - - /* NaN operands have the next priority. */ - /* We have to delay looking at st(0) until after - loading the data, because that data might contain an SNaN */ - if (((st0_tag == TAG_Special) && isNaN(st0_ptr)) - || ((loaded_tag == TAG_Special) - && isNaN(&loaded_data))) { - /* Restore the status word; we might have loaded a - denormal. */ - partial_status = status1; - if ((FPU_modrm & 0x30) == 0x10) { - /* fcom or fcomp */ - EXCEPTION(EX_Invalid); - setcc(SW_C3 | SW_C2 | SW_C0); - if ((FPU_modrm & 0x08) - && (control_word & - CW_Invalid)) - FPU_pop(); /* fcomp, masked, so we pop. */ - } else { - if (loaded_tag == TAG_Special) - loaded_tag = - FPU_Special - (&loaded_data); -#ifdef PECULIAR_486 - /* This is not really needed, but gives behaviour - identical to an 80486 */ - if ((FPU_modrm & 0x28) == 0x20) - /* fdiv or fsub */ - real_2op_NaN - (&loaded_data, - loaded_tag, 0, - &loaded_data); - else -#endif /* PECULIAR_486 */ - /* fadd, fdivr, fmul, or fsubr */ - real_2op_NaN - (&loaded_data, - loaded_tag, 0, - st0_ptr); - } - goto reg_mem_instr_done; - } - - if (unmasked && !((FPU_modrm & 0x30) == 0x10)) { - /* Is not a comparison instruction. */ - if ((FPU_modrm & 0x38) == 0x38) { - /* fdivr */ - if ((st0_tag == TAG_Zero) && - ((loaded_tag == TAG_Valid) - || (loaded_tag == - TAG_Special - && - isdenormal - (&loaded_data)))) { - if (FPU_divide_by_zero - (0, - getsign - (&loaded_data)) - < 0) { - /* We use the fact here that the unmasked - exception in the loaded data was for a - denormal operand */ - /* Restore the state of the denormal op bit */ - partial_status - &= - ~SW_Denorm_Op; - partial_status - |= - status1 & - SW_Denorm_Op; - } else - setsign(st0_ptr, - getsign - (&loaded_data)); - } - } - goto reg_mem_instr_done; - } - - switch ((FPU_modrm >> 3) & 7) { - case 0: /* fadd */ - clear_C1(); - FPU_add(&loaded_data, loaded_tag, 0, - control_word); - break; - case 1: /* fmul */ - clear_C1(); - FPU_mul(&loaded_data, loaded_tag, 0, - control_word); - break; - case 2: /* fcom */ - FPU_compare_st_data(&loaded_data, - loaded_tag); - break; - case 3: /* fcomp */ - if (!FPU_compare_st_data - (&loaded_data, loaded_tag) - && !unmasked) - FPU_pop(); - break; - case 4: /* fsub */ - clear_C1(); - FPU_sub(LOADED | loaded_tag, - (int)&loaded_data, - control_word); - break; - case 5: /* fsubr */ - clear_C1(); - FPU_sub(REV | LOADED | loaded_tag, - (int)&loaded_data, - control_word); - break; - case 6: /* fdiv */ - clear_C1(); - FPU_div(LOADED | loaded_tag, - (int)&loaded_data, - control_word); - break; - case 7: /* fdivr */ - clear_C1(); - if (st0_tag == TAG_Zero) - partial_status = status1; /* Undo any denorm tag, - zero-divide has priority. */ - FPU_div(REV | LOADED | loaded_tag, - (int)&loaded_data, - control_word); - break; - } - } else { - if ((FPU_modrm & 0x30) == 0x10) { - /* The instruction is fcom or fcomp */ - EXCEPTION(EX_StackUnder); - setcc(SW_C3 | SW_C2 | SW_C0); - if ((FPU_modrm & 0x08) - && (control_word & CW_Invalid)) - FPU_pop(); /* fcomp */ - } else - FPU_stack_underflow(); - } - reg_mem_instr_done: - operand_address = data_sel_off; - } else { - if (!(no_ip_update = - FPU_load_store(((FPU_modrm & 0x38) | (byte1 & 6)) - >> 1, addr_modes, data_address))) { - operand_address = data_sel_off; - } - } - - } else { - /* None of these instructions access user memory */ - u_char instr_index = (FPU_modrm & 0x38) | (byte1 & 7); - -#ifdef PECULIAR_486 - /* This is supposed to be undefined, but a real 80486 seems - to do this: */ - operand_address.offset = 0; - operand_address.selector = FPU_DS; -#endif /* PECULIAR_486 */ - - st0_ptr = &st(0); - st0_tag = FPU_gettag0(); - switch (type_table[(int)instr_index]) { - case _NONE_: /* also _REGIc: _REGIn */ - break; - case _REG0_: - if (!NOT_EMPTY_ST0) { - FPU_stack_underflow(); - goto FPU_instruction_done; - } - break; - case _REGIi: - if (!NOT_EMPTY_ST0 || !NOT_EMPTY(FPU_rm)) { - FPU_stack_underflow_i(FPU_rm); - goto FPU_instruction_done; - } - break; - case _REGIp: - if (!NOT_EMPTY_ST0 || !NOT_EMPTY(FPU_rm)) { - FPU_stack_underflow_pop(FPU_rm); - goto FPU_instruction_done; - } - break; - case _REGI_: - if (!NOT_EMPTY_ST0 || !NOT_EMPTY(FPU_rm)) { - FPU_stack_underflow(); - goto FPU_instruction_done; - } - break; - case _PUSH_: /* Only used by the fld st(i) instruction */ - break; - case _null_: - FPU_illegal(); - goto FPU_instruction_done; - default: - EXCEPTION(EX_INTERNAL | 0x111); - goto FPU_instruction_done; - } - (*st_instr_table[(int)instr_index]) (); - - FPU_instruction_done: - ; - } - - if (!no_ip_update) - instruction_address = entry_sel_off; - - FPU_fwait_done: - -#ifdef DEBUG - RE_ENTRANT_CHECK_OFF; - FPU_printall(); - RE_ENTRANT_CHECK_ON; -#endif /* DEBUG */ - - if (FPU_lookahead && !need_resched()) { - FPU_ORIG_EIP = FPU_EIP - code_base; - if (valid_prefix(&byte1, (u_char __user **) & FPU_EIP, - &addr_modes.override)) - goto do_another_FPU_instruction; - } - - if (addr_modes.default_mode) - FPU_EIP -= code_base; - - RE_ENTRANT_CHECK_OFF; -} - -/* Support for prefix bytes is not yet complete. To properly handle - all prefix bytes, further changes are needed in the emulator code - which accesses user address space. Access to separate segments is - important for msdos emulation. */ -static int valid_prefix(u_char *Byte, u_char __user **fpu_eip, - overrides * override) -{ - u_char byte; - u_char __user *ip = *fpu_eip; - - *override = (overrides) { - 0, 0, PREFIX_DEFAULT}; /* defaults */ - - RE_ENTRANT_CHECK_OFF; - FPU_code_access_ok(1); - FPU_get_user(byte, ip); - RE_ENTRANT_CHECK_ON; - - while (1) { - switch (byte) { - case ADDR_SIZE_PREFIX: - override->address_size = ADDR_SIZE_PREFIX; - goto do_next_byte; - - case OP_SIZE_PREFIX: - override->operand_size = OP_SIZE_PREFIX; - goto do_next_byte; - - case PREFIX_CS: - override->segment = PREFIX_CS_; - goto do_next_byte; - case PREFIX_ES: - override->segment = PREFIX_ES_; - goto do_next_byte; - case PREFIX_SS: - override->segment = PREFIX_SS_; - goto do_next_byte; - case PREFIX_FS: - override->segment = PREFIX_FS_; - goto do_next_byte; - case PREFIX_GS: - override->segment = PREFIX_GS_; - goto do_next_byte; - case PREFIX_DS: - override->segment = PREFIX_DS_; - goto do_next_byte; - -/* lock is not a valid prefix for FPU instructions, - let the cpu handle it to generate a SIGILL. */ -/* case PREFIX_LOCK: */ - - /* rep.. prefixes have no meaning for FPU instructions */ - case PREFIX_REPE: - case PREFIX_REPNE: - - do_next_byte: - ip++; - RE_ENTRANT_CHECK_OFF; - FPU_code_access_ok(1); - FPU_get_user(byte, ip); - RE_ENTRANT_CHECK_ON; - break; - case FWAIT_OPCODE: - *Byte = byte; - return 1; - default: - if ((byte & 0xf8) == 0xd8) { - *Byte = byte; - *fpu_eip = ip; - return 1; - } else { - /* Not a valid sequence of prefix bytes followed by - an FPU instruction. */ - *Byte = byte; /* Needed for error message. */ - return 0; - } - } - } -} - -void math_abort(struct math_emu_info *info, unsigned int signal) -{ - FPU_EIP = FPU_ORIG_EIP; - current->thread.trap_nr = X86_TRAP_MF; - current->thread.error_code = 0; - send_sig(signal, current, 1); - RE_ENTRANT_CHECK_OFF; - __asm__("movl %0,%%esp ; ret": :"g"(((long)info) - 4)); -#ifdef PARANOID - printk("ERROR: wm-FPU-emu math_abort failed!\n"); -#endif /* PARANOID */ -} - -#define S387 ((struct swregs_state *)s387) -#define sstatus_word() \ - ((S387->swd & ~SW_Top & 0xffff) | ((S387->ftop << SW_Top_Shift) & SW_Top)) - -int fpregs_soft_set(struct task_struct *target, - const struct user_regset *regset, - unsigned int pos, unsigned int count, - const void *kbuf, const void __user *ubuf) -{ - struct swregs_state *s387 = &x86_task_fpu(target)->fpstate->regs.soft; - void *space = s387->st_space; - int ret; - int offset, other, i, tags, regnr, tag, newtop; - - RE_ENTRANT_CHECK_OFF; - ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, s387, 0, - offsetof(struct swregs_state, st_space)); - RE_ENTRANT_CHECK_ON; - - if (ret) - return ret; - - S387->ftop = (S387->swd >> SW_Top_Shift) & 7; - offset = (S387->ftop & 7) * 10; - other = 80 - offset; - - RE_ENTRANT_CHECK_OFF; - - /* Copy all registers in stack order. */ - ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, - space + offset, 0, other); - if (!ret && offset) - ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, - space, 0, offset); - - RE_ENTRANT_CHECK_ON; - - /* The tags may need to be corrected now. */ - tags = S387->twd; - newtop = S387->ftop; - for (i = 0; i < 8; i++) { - regnr = (i + newtop) & 7; - if (((tags >> ((regnr & 7) * 2)) & 3) != TAG_Empty) { - /* The loaded data over-rides all other cases. */ - tag = - FPU_tagof((FPU_REG *) ((u_char *) S387->st_space + - 10 * regnr)); - tags &= ~(3 << (regnr * 2)); - tags |= (tag & 3) << (regnr * 2); - } - } - S387->twd = tags; - - return ret; -} - -int fpregs_soft_get(struct task_struct *target, - const struct user_regset *regset, - struct membuf to) -{ - struct swregs_state *s387 = &x86_task_fpu(target)->fpstate->regs.soft; - const void *space = s387->st_space; - int offset = (S387->ftop & 7) * 10, other = 80 - offset; - - RE_ENTRANT_CHECK_OFF; - -#ifdef PECULIAR_486 - S387->cwd &= ~0xe080; - /* An 80486 sets nearly all of the reserved bits to 1. */ - S387->cwd |= 0xffff0040; - S387->swd = sstatus_word() | 0xffff0000; - S387->twd |= 0xffff0000; - S387->fcs &= ~0xf8000000; - S387->fos |= 0xffff0000; -#endif /* PECULIAR_486 */ - - membuf_write(&to, s387, offsetof(struct swregs_state, st_space)); - membuf_write(&to, space + offset, other); - membuf_write(&to, space, offset); - - RE_ENTRANT_CHECK_ON; - - return 0; -} diff --git a/arch/x86/math-emu/fpu_etc.c b/arch/x86/math-emu/fpu_etc.c deleted file mode 100644 index 39423ec409e1..000000000000 --- a/arch/x86/math-emu/fpu_etc.c +++ /dev/null @@ -1,136 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/*---------------------------------------------------------------------------+ - | fpu_etc.c | - | | - | Implement a few FPU instructions. | - | | - | Copyright (C) 1992,1993,1994,1997 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, | - | Australia. E-mail billm@suburbia.net | - | | - | | - +---------------------------------------------------------------------------*/ - -#include "fpu_system.h" -#include "exception.h" -#include "fpu_emu.h" -#include "status_w.h" -#include "reg_constant.h" - -static void fchs(FPU_REG *st0_ptr, u_char st0tag) -{ - if (st0tag ^ TAG_Empty) { - signbyte(st0_ptr) ^= SIGN_NEG; - clear_C1(); - } else - FPU_stack_underflow(); -} - -static void fabs(FPU_REG *st0_ptr, u_char st0tag) -{ - if (st0tag ^ TAG_Empty) { - setpositive(st0_ptr); - clear_C1(); - } else - FPU_stack_underflow(); -} - -static void ftst_(FPU_REG *st0_ptr, u_char st0tag) -{ - switch (st0tag) { - case TAG_Zero: - setcc(SW_C3); - break; - case TAG_Valid: - if (getsign(st0_ptr) == SIGN_POS) - setcc(0); - else - setcc(SW_C0); - break; - case TAG_Special: - switch (FPU_Special(st0_ptr)) { - case TW_Denormal: - if (getsign(st0_ptr) == SIGN_POS) - setcc(0); - else - setcc(SW_C0); - if (denormal_operand() < 0) { -#ifdef PECULIAR_486 - /* This is weird! */ - if (getsign(st0_ptr) == SIGN_POS) - setcc(SW_C3); -#endif /* PECULIAR_486 */ - return; - } - break; - case TW_NaN: - setcc(SW_C0 | SW_C2 | SW_C3); /* Operand is not comparable */ - EXCEPTION(EX_Invalid); - break; - case TW_Infinity: - if (getsign(st0_ptr) == SIGN_POS) - setcc(0); - else - setcc(SW_C0); - break; - default: - setcc(SW_C0 | SW_C2 | SW_C3); /* Operand is not comparable */ - EXCEPTION(EX_INTERNAL | 0x14); - break; - } - break; - case TAG_Empty: - setcc(SW_C0 | SW_C2 | SW_C3); - EXCEPTION(EX_StackUnder); - break; - } -} - -static void fxam(FPU_REG *st0_ptr, u_char st0tag) -{ - int c = 0; - switch (st0tag) { - case TAG_Empty: - c = SW_C3 | SW_C0; - break; - case TAG_Zero: - c = SW_C3; - break; - case TAG_Valid: - c = SW_C2; - break; - case TAG_Special: - switch (FPU_Special(st0_ptr)) { - case TW_Denormal: - c = SW_C2 | SW_C3; /* Denormal */ - break; - case TW_NaN: - /* We also use NaN for unsupported types. */ - if ((st0_ptr->sigh & 0x80000000) - && (exponent(st0_ptr) == EXP_OVER)) - c = SW_C0; - break; - case TW_Infinity: - c = SW_C2 | SW_C0; - break; - } - } - if (getsign(st0_ptr) == SIGN_NEG) - c |= SW_C1; - setcc(c); -} - -static void FPU_ST0_illegal(FPU_REG *st0_ptr, u_char st0_tag) -{ - FPU_illegal(); -} - -static FUNC_ST0 const fp_etc_table[] = { - fchs, fabs, FPU_ST0_illegal, FPU_ST0_illegal, - ftst_, fxam, FPU_ST0_illegal, FPU_ST0_illegal, -}; - -void FPU_etc(void) -{ - (fp_etc_table[FPU_rm]) (&st(0), FPU_gettag0()); -} diff --git a/arch/x86/math-emu/fpu_proto.h b/arch/x86/math-emu/fpu_proto.h deleted file mode 100644 index 94c4023092f3..000000000000 --- a/arch/x86/math-emu/fpu_proto.h +++ /dev/null @@ -1,157 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _FPU_PROTO_H -#define _FPU_PROTO_H - -/* errors.c */ -extern void FPU_illegal(void); -extern void FPU_printall(void); -asmlinkage void FPU_exception(int n); -extern int real_1op_NaN(FPU_REG *a); -extern int real_2op_NaN(FPU_REG const *b, u_char tagb, int deststnr, - FPU_REG const *defaultNaN); -asmlinkage int arith_invalid(int deststnr); -asmlinkage int FPU_divide_by_zero(int deststnr, u_char sign); -extern int set_precision_flag(int flags); -asmlinkage void set_precision_flag_up(void); -asmlinkage void set_precision_flag_down(void); -asmlinkage int denormal_operand(void); -asmlinkage int arith_overflow(FPU_REG *dest); -asmlinkage int arith_underflow(FPU_REG *dest); -extern void FPU_stack_overflow(void); -extern void FPU_stack_underflow(void); -extern void FPU_stack_underflow_i(int i); -extern void FPU_stack_underflow_pop(int i); -/* fpu_arith.c */ -extern void fadd__(void); -extern void fmul__(void); -extern void fsub__(void); -extern void fsubr_(void); -extern void fdiv__(void); -extern void fdivr_(void); -extern void fadd_i(void); -extern void fmul_i(void); -extern void fsubri(void); -extern void fsub_i(void); -extern void fdivri(void); -extern void fdiv_i(void); -extern void faddp_(void); -extern void fmulp_(void); -extern void fsubrp(void); -extern void fsubp_(void); -extern void fdivrp(void); -extern void fdivp_(void); -/* fpu_aux.c */ -extern void finit(void); -extern void finit_(void); -extern void fstsw_(void); -extern void fp_nop(void); -extern void fld_i_(void); -extern void fxch_i(void); -extern void fcmovb(void); -extern void fcmove(void); -extern void fcmovbe(void); -extern void fcmovu(void); -extern void fcmovnb(void); -extern void fcmovne(void); -extern void fcmovnbe(void); -extern void fcmovnu(void); -extern void ffree_(void); -extern void ffreep(void); -extern void fst_i_(void); -extern void fstp_i(void); -/* fpu_entry.c */ -extern void math_emulate(struct math_emu_info *info); -extern void math_abort(struct math_emu_info *info, unsigned int signal); -/* fpu_etc.c */ -extern void FPU_etc(void); -/* fpu_tags.c */ -extern int FPU_gettag0(void); -extern int FPU_gettagi(int stnr); -extern int FPU_gettag(int regnr); -extern void FPU_settag0(int tag); -extern void FPU_settagi(int stnr, int tag); -extern void FPU_settag(int regnr, int tag); -extern int FPU_Special(FPU_REG const *ptr); -extern int isNaN(FPU_REG const *ptr); -extern void FPU_pop(void); -extern int FPU_empty_i(int stnr); -extern int FPU_stackoverflow(FPU_REG ** st_new_ptr); -extern void FPU_copy_to_regi(FPU_REG const *r, u_char tag, int stnr); -extern void FPU_copy_to_reg1(FPU_REG const *r, u_char tag); -extern void FPU_copy_to_reg0(FPU_REG const *r, u_char tag); -/* fpu_trig.c */ -extern void FPU_triga(void); -extern void FPU_trigb(void); -/* get_address.c */ -extern void __user *FPU_get_address(u_char FPU_modrm, unsigned long *fpu_eip, - struct address *addr, - fpu_addr_modes addr_modes); -extern void __user *FPU_get_address_16(u_char FPU_modrm, unsigned long *fpu_eip, - struct address *addr, - fpu_addr_modes addr_modes); -/* load_store.c */ -extern int FPU_load_store(u_char type, fpu_addr_modes addr_modes, - void __user * data_address); -/* poly_2xm1.c */ -extern int poly_2xm1(u_char sign, FPU_REG * arg, FPU_REG *result); -/* poly_atan.c */ -extern void poly_atan(FPU_REG * st0_ptr, u_char st0_tag, FPU_REG *st1_ptr, - u_char st1_tag); -/* poly_l2.c */ -extern void poly_l2(FPU_REG *st0_ptr, FPU_REG *st1_ptr, u_char st1_sign); -extern int poly_l2p1(u_char s0, u_char s1, FPU_REG *r0, FPU_REG *r1, - FPU_REG * d); -/* poly_sin.c */ -extern void poly_sine(FPU_REG *st0_ptr); -extern void poly_cos(FPU_REG *st0_ptr); -/* poly_tan.c */ -extern void poly_tan(FPU_REG *st0_ptr); -/* reg_add_sub.c */ -extern int FPU_add(FPU_REG const *b, u_char tagb, int destrnr, int control_w); -extern int FPU_sub(int flags, int rm, int control_w); -/* reg_compare.c */ -extern int FPU_compare_st_data(FPU_REG const *loaded_data, u_char loaded_tag); -extern void fcom_st(void); -extern void fcompst(void); -extern void fcompp(void); -extern void fucom_(void); -extern void fucomp(void); -extern void fucompp(void); -extern void fcomi_(void); -extern void fcomip(void); -extern void fucomi_(void); -extern void fucomip(void); -/* reg_constant.c */ -extern void fconst(void); -/* reg_ld_str.c */ -extern int FPU_load_extended(long double __user *s, int stnr); -extern int FPU_load_double(double __user *dfloat, FPU_REG *loaded_data); -extern int FPU_load_single(float __user *single, FPU_REG *loaded_data); -extern int FPU_load_int64(long long __user *_s); -extern int FPU_load_int32(long __user *_s, FPU_REG *loaded_data); -extern int FPU_load_int16(short __user *_s, FPU_REG *loaded_data); -extern int FPU_load_bcd(u_char __user *s); -extern int FPU_store_extended(FPU_REG *st0_ptr, u_char st0_tag, - long double __user * d); -extern int FPU_store_double(FPU_REG *st0_ptr, u_char st0_tag, - double __user * dfloat); -extern int FPU_store_single(FPU_REG *st0_ptr, u_char st0_tag, - float __user * single); -extern int FPU_store_int64(FPU_REG *st0_ptr, u_char st0_tag, - long long __user * d); -extern int FPU_store_int32(FPU_REG *st0_ptr, u_char st0_tag, long __user *d); -extern int FPU_store_int16(FPU_REG *st0_ptr, u_char st0_tag, short __user *d); -extern int FPU_store_bcd(FPU_REG *st0_ptr, u_char st0_tag, u_char __user *d); -extern int FPU_round_to_int(FPU_REG *r, u_char tag); -extern u_char __user *fldenv(fpu_addr_modes addr_modes, u_char __user *s); -extern void FPU_frstor(fpu_addr_modes addr_modes, u_char __user *data_address); -extern u_char __user *fstenv(fpu_addr_modes addr_modes, u_char __user *d); -extern void fsave(fpu_addr_modes addr_modes, u_char __user *data_address); -extern int FPU_tagof(FPU_REG *ptr); -/* reg_mul.c */ -extern int FPU_mul(FPU_REG const *b, u_char tagb, int deststnr, int control_w); - -extern int FPU_div(int flags, int regrm, int control_w); -/* reg_convert.c */ -extern int FPU_to_exp16(FPU_REG const *a, FPU_REG *x); -#endif /* _FPU_PROTO_H */ diff --git a/arch/x86/math-emu/fpu_system.h b/arch/x86/math-emu/fpu_system.h deleted file mode 100644 index 5e238e930fe3..000000000000 --- a/arch/x86/math-emu/fpu_system.h +++ /dev/null @@ -1,130 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/*---------------------------------------------------------------------------+ - | fpu_system.h | - | | - | Copyright (C) 1992,1994,1997 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, | - | Australia. E-mail billm@suburbia.net | - | | - +---------------------------------------------------------------------------*/ - -#ifndef _FPU_SYSTEM_H -#define _FPU_SYSTEM_H - -/* system dependent definitions */ - -#include -#include -#include - -#include -#include - -static inline struct desc_struct FPU_get_ldt_descriptor(unsigned seg) -{ - static struct desc_struct zero_desc; - struct desc_struct ret = zero_desc; - -#ifdef CONFIG_MODIFY_LDT_SYSCALL - seg >>= 3; - mutex_lock(¤t->mm->context.lock); - if (current->mm->context.ldt && seg < current->mm->context.ldt->nr_entries) - ret = current->mm->context.ldt->entries[seg]; - mutex_unlock(¤t->mm->context.lock); -#endif - return ret; -} - -#define SEG_TYPE_WRITABLE (1U << 1) -#define SEG_TYPE_EXPANDS_DOWN (1U << 2) -#define SEG_TYPE_EXECUTE (1U << 3) -#define SEG_TYPE_EXPAND_MASK (SEG_TYPE_EXPANDS_DOWN | SEG_TYPE_EXECUTE) -#define SEG_TYPE_EXECUTE_MASK (SEG_TYPE_WRITABLE | SEG_TYPE_EXECUTE) - -static inline unsigned long seg_get_base(struct desc_struct *d) -{ - unsigned long base = (unsigned long)d->base2 << 24; - - return base | ((unsigned long)d->base1 << 16) | d->base0; -} - -static inline unsigned long seg_get_limit(struct desc_struct *d) -{ - return ((unsigned long)d->limit1 << 16) | d->limit0; -} - -static inline unsigned long seg_get_granularity(struct desc_struct *d) -{ - return d->g ? 4096 : 1; -} - -static inline bool seg_expands_down(struct desc_struct *d) -{ - return (d->type & SEG_TYPE_EXPAND_MASK) == SEG_TYPE_EXPANDS_DOWN; -} - -static inline bool seg_execute_only(struct desc_struct *d) -{ - return (d->type & SEG_TYPE_EXECUTE_MASK) == SEG_TYPE_EXECUTE; -} - -static inline bool seg_writable(struct desc_struct *d) -{ - return (d->type & SEG_TYPE_EXECUTE_MASK) == SEG_TYPE_WRITABLE; -} - -#define I387 (&x86_task_fpu(current)->fpstate->regs) -#define FPU_info (I387->soft.info) - -#define FPU_CS (*(unsigned short *) &(FPU_info->regs->cs)) -#define FPU_SS (*(unsigned short *) &(FPU_info->regs->ss)) -#define FPU_DS (*(unsigned short *) &(FPU_info->regs->ds)) -#define FPU_EAX (FPU_info->regs->ax) -#define FPU_EFLAGS (FPU_info->regs->flags) -#define FPU_EIP (FPU_info->regs->ip) -#define FPU_ORIG_EIP (FPU_info->___orig_eip) - -#define FPU_lookahead (I387->soft.lookahead) - -/* nz if ip_offset and cs_selector are not to be set for the current - instruction. */ -#define no_ip_update (*(u_char *)&(I387->soft.no_update)) -#define FPU_rm (*(u_char *)&(I387->soft.rm)) - -/* Number of bytes of data which can be legally accessed by the current - instruction. This only needs to hold a number <= 108, so a byte will do. */ -#define access_limit (*(u_char *)&(I387->soft.alimit)) - -#define partial_status (I387->soft.swd) -#define control_word (I387->soft.cwd) -#define fpu_tag_word (I387->soft.twd) -#define registers (I387->soft.st_space) -#define top (I387->soft.ftop) - -#define instruction_address (*(struct address *)&I387->soft.fip) -#define operand_address (*(struct address *)&I387->soft.foo) - -#define FPU_access_ok(y,z) if ( !access_ok(y,z) ) \ - math_abort(FPU_info,SIGSEGV) -#define FPU_abort math_abort(FPU_info, SIGSEGV) -#define FPU_copy_from_user(to, from, n) \ - do { if (copy_from_user(to, from, n)) FPU_abort; } while (0) - -#undef FPU_IGNORE_CODE_SEGV -#ifdef FPU_IGNORE_CODE_SEGV -/* access_ok() is very expensive, and causes the emulator to run - about 20% slower if applied to the code. Anyway, errors due to bad - code addresses should be much rarer than errors due to bad data - addresses. */ -#define FPU_code_access_ok(z) -#else -/* A simpler test than access_ok() can probably be done for - FPU_code_access_ok() because the only possible error is to step - past the upper boundary of a legal code area. */ -#define FPU_code_access_ok(z) FPU_access_ok((void __user *)FPU_EIP,z) -#endif - -#define FPU_get_user(x,y) do { if (get_user((x),(y))) FPU_abort; } while (0) -#define FPU_put_user(x,y) do { if (put_user((x),(y))) FPU_abort; } while (0) - -#endif diff --git a/arch/x86/math-emu/fpu_tags.c b/arch/x86/math-emu/fpu_tags.c deleted file mode 100644 index bff95d4e7dca..000000000000 --- a/arch/x86/math-emu/fpu_tags.c +++ /dev/null @@ -1,116 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/*---------------------------------------------------------------------------+ - | fpu_tags.c | - | | - | Set FPU register tags. | - | | - | Copyright (C) 1997 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia | - | E-mail billm@jacobi.maths.monash.edu.au | - | | - | | - +---------------------------------------------------------------------------*/ - -#include "fpu_emu.h" -#include "fpu_system.h" -#include "exception.h" - -void FPU_pop(void) -{ - fpu_tag_word |= 3 << ((top & 7) * 2); - top++; -} - -int FPU_gettag0(void) -{ - return (fpu_tag_word >> ((top & 7) * 2)) & 3; -} - -int FPU_gettagi(int stnr) -{ - return (fpu_tag_word >> (((top + stnr) & 7) * 2)) & 3; -} - -int FPU_gettag(int regnr) -{ - return (fpu_tag_word >> ((regnr & 7) * 2)) & 3; -} - -void FPU_settag0(int tag) -{ - int regnr = top; - regnr &= 7; - fpu_tag_word &= ~(3 << (regnr * 2)); - fpu_tag_word |= (tag & 3) << (regnr * 2); -} - -void FPU_settagi(int stnr, int tag) -{ - int regnr = stnr + top; - regnr &= 7; - fpu_tag_word &= ~(3 << (regnr * 2)); - fpu_tag_word |= (tag & 3) << (regnr * 2); -} - -void FPU_settag(int regnr, int tag) -{ - regnr &= 7; - fpu_tag_word &= ~(3 << (regnr * 2)); - fpu_tag_word |= (tag & 3) << (regnr * 2); -} - -int FPU_Special(FPU_REG const *ptr) -{ - int exp = exponent(ptr); - - if (exp == EXP_BIAS + EXP_UNDER) - return TW_Denormal; - else if (exp != EXP_BIAS + EXP_OVER) - return TW_NaN; - else if ((ptr->sigh == 0x80000000) && (ptr->sigl == 0)) - return TW_Infinity; - return TW_NaN; -} - -int isNaN(FPU_REG const *ptr) -{ - return ((exponent(ptr) == EXP_BIAS + EXP_OVER) - && !((ptr->sigh == 0x80000000) && (ptr->sigl == 0))); -} - -int FPU_empty_i(int stnr) -{ - int regnr = (top + stnr) & 7; - - return ((fpu_tag_word >> (regnr * 2)) & 3) == TAG_Empty; -} - -int FPU_stackoverflow(FPU_REG ** st_new_ptr) -{ - *st_new_ptr = &st(-1); - - return ((fpu_tag_word >> (((top - 1) & 7) * 2)) & 3) != TAG_Empty; -} - -void FPU_copy_to_regi(FPU_REG const *r, u_char tag, int stnr) -{ - reg_copy(r, &st(stnr)); - FPU_settagi(stnr, tag); -} - -void FPU_copy_to_reg1(FPU_REG const *r, u_char tag) -{ - reg_copy(r, &st(1)); - FPU_settagi(1, tag); -} - -void FPU_copy_to_reg0(FPU_REG const *r, u_char tag) -{ - int regnr = top; - regnr &= 7; - - reg_copy(r, &st(0)); - - fpu_tag_word &= ~(3 << (regnr * 2)); - fpu_tag_word |= (tag & 3) << (regnr * 2); -} diff --git a/arch/x86/math-emu/fpu_trig.c b/arch/x86/math-emu/fpu_trig.c deleted file mode 100644 index 85daf98c81c3..000000000000 --- a/arch/x86/math-emu/fpu_trig.c +++ /dev/null @@ -1,1649 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/*---------------------------------------------------------------------------+ - | fpu_trig.c | - | | - | Implementation of the FPU "transcendental" functions. | - | | - | Copyright (C) 1992,1993,1994,1997,1999 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, | - | Australia. E-mail billm@melbpc.org.au | - | | - | | - +---------------------------------------------------------------------------*/ - -#include "fpu_system.h" -#include "exception.h" -#include "fpu_emu.h" -#include "status_w.h" -#include "control_w.h" -#include "reg_constant.h" - -static void rem_kernel(unsigned long long st0, unsigned long long *y, - unsigned long long st1, unsigned long long q, int n); - -#define BETTER_THAN_486 - -#define FCOS 4 - -/* Used only by fptan, fsin, fcos, and fsincos. */ -/* This routine produces very accurate results, similar to - using a value of pi with more than 128 bits precision. */ -/* Limited measurements show no results worse than 64 bit precision - except for the results for arguments close to 2^63, where the - precision of the result sometimes degrades to about 63.9 bits */ -static int trig_arg(FPU_REG *st0_ptr, int even) -{ - FPU_REG tmp; - u_char tmptag; - unsigned long long q; - int old_cw = control_word, saved_status = partial_status; - int tag, st0_tag = TAG_Valid; - - if (exponent(st0_ptr) >= 63) { - partial_status |= SW_C2; /* Reduction incomplete. */ - return -1; - } - - control_word &= ~CW_RC; - control_word |= RC_CHOP; - - setpositive(st0_ptr); - tag = FPU_u_div(st0_ptr, &CONST_PI2, &tmp, PR_64_BITS | RC_CHOP | 0x3f, - SIGN_POS); - - FPU_round_to_int(&tmp, tag); /* Fortunately, this can't overflow - to 2^64 */ - q = significand(&tmp); - if (q) { - rem_kernel(significand(st0_ptr), - &significand(&tmp), - significand(&CONST_PI2), - q, exponent(st0_ptr) - exponent(&CONST_PI2)); - setexponent16(&tmp, exponent(&CONST_PI2)); - st0_tag = FPU_normalize(&tmp); - FPU_copy_to_reg0(&tmp, st0_tag); - } - - if ((even && !(q & 1)) || (!even && (q & 1))) { - st0_tag = - FPU_sub(REV | LOADED | TAG_Valid, (int)&CONST_PI2, - FULL_PRECISION); - -#ifdef BETTER_THAN_486 - /* So far, the results are exact but based upon a 64 bit - precision approximation to pi/2. The technique used - now is equivalent to using an approximation to pi/2 which - is accurate to about 128 bits. */ - if ((exponent(st0_ptr) <= exponent(&CONST_PI2extra) + 64) - || (q > 1)) { - /* This code gives the effect of having pi/2 to better than - 128 bits precision. */ - - significand(&tmp) = q + 1; - setexponent16(&tmp, 63); - FPU_normalize(&tmp); - tmptag = - FPU_u_mul(&CONST_PI2extra, &tmp, &tmp, - FULL_PRECISION, SIGN_POS, - exponent(&CONST_PI2extra) + - exponent(&tmp)); - setsign(&tmp, getsign(&CONST_PI2extra)); - st0_tag = FPU_add(&tmp, tmptag, 0, FULL_PRECISION); - if (signnegative(st0_ptr)) { - /* CONST_PI2extra is negative, so the result of the addition - can be negative. This means that the argument is actually - in a different quadrant. The correction is always < pi/2, - so it can't overflow into yet another quadrant. */ - setpositive(st0_ptr); - q++; - } - } -#endif /* BETTER_THAN_486 */ - } -#ifdef BETTER_THAN_486 - else { - /* So far, the results are exact but based upon a 64 bit - precision approximation to pi/2. The technique used - now is equivalent to using an approximation to pi/2 which - is accurate to about 128 bits. */ - if (((q > 0) - && (exponent(st0_ptr) <= exponent(&CONST_PI2extra) + 64)) - || (q > 1)) { - /* This code gives the effect of having p/2 to better than - 128 bits precision. */ - - significand(&tmp) = q; - setexponent16(&tmp, 63); - FPU_normalize(&tmp); /* This must return TAG_Valid */ - tmptag = - FPU_u_mul(&CONST_PI2extra, &tmp, &tmp, - FULL_PRECISION, SIGN_POS, - exponent(&CONST_PI2extra) + - exponent(&tmp)); - setsign(&tmp, getsign(&CONST_PI2extra)); - st0_tag = FPU_sub(LOADED | (tmptag & 0x0f), (int)&tmp, - FULL_PRECISION); - if ((exponent(st0_ptr) == exponent(&CONST_PI2)) && - ((st0_ptr->sigh > CONST_PI2.sigh) - || ((st0_ptr->sigh == CONST_PI2.sigh) - && (st0_ptr->sigl > CONST_PI2.sigl)))) { - /* CONST_PI2extra is negative, so the result of the - subtraction can be larger than pi/2. This means - that the argument is actually in a different quadrant. - The correction is always < pi/2, so it can't overflow - into yet another quadrant. */ - st0_tag = - FPU_sub(REV | LOADED | TAG_Valid, - (int)&CONST_PI2, FULL_PRECISION); - q++; - } - } - } -#endif /* BETTER_THAN_486 */ - - FPU_settag0(st0_tag); - control_word = old_cw; - partial_status = saved_status & ~SW_C2; /* Reduction complete. */ - - return (q & 3) | even; -} - -/* Convert a long to register */ -static void convert_l2reg(long const *arg, int deststnr) -{ - int tag; - long num = *arg; - u_char sign; - FPU_REG *dest = &st(deststnr); - - if (num == 0) { - FPU_copy_to_regi(&CONST_Z, TAG_Zero, deststnr); - return; - } - - if (num > 0) { - sign = SIGN_POS; - } else { - num = -num; - sign = SIGN_NEG; - } - - dest->sigh = num; - dest->sigl = 0; - setexponent16(dest, 31); - tag = FPU_normalize(dest); - FPU_settagi(deststnr, tag); - setsign(dest, sign); - return; -} - -static void single_arg_error(FPU_REG *st0_ptr, u_char st0_tag) -{ - if (st0_tag == TAG_Empty) - FPU_stack_underflow(); /* Puts a QNaN in st(0) */ - else if (st0_tag == TW_NaN) - real_1op_NaN(st0_ptr); /* return with a NaN in st(0) */ -#ifdef PARANOID - else - EXCEPTION(EX_INTERNAL | 0x0112); -#endif /* PARANOID */ -} - -static void single_arg_2_error(FPU_REG *st0_ptr, u_char st0_tag) -{ - int isNaN; - - switch (st0_tag) { - case TW_NaN: - isNaN = (exponent(st0_ptr) == EXP_OVER) - && (st0_ptr->sigh & 0x80000000); - if (isNaN && !(st0_ptr->sigh & 0x40000000)) { /* Signaling ? */ - EXCEPTION(EX_Invalid); - if (control_word & CW_Invalid) { - /* The masked response */ - /* Convert to a QNaN */ - st0_ptr->sigh |= 0x40000000; - push(); - FPU_copy_to_reg0(st0_ptr, TAG_Special); - } - } else if (isNaN) { - /* A QNaN */ - push(); - FPU_copy_to_reg0(st0_ptr, TAG_Special); - } else { - /* pseudoNaN or other unsupported */ - EXCEPTION(EX_Invalid); - if (control_word & CW_Invalid) { - /* The masked response */ - FPU_copy_to_reg0(&CONST_QNaN, TAG_Special); - push(); - FPU_copy_to_reg0(&CONST_QNaN, TAG_Special); - } - } - break; /* return with a NaN in st(0) */ -#ifdef PARANOID - default: - EXCEPTION(EX_INTERNAL | 0x0112); -#endif /* PARANOID */ - } -} - -/*---------------------------------------------------------------------------*/ - -static void f2xm1(FPU_REG *st0_ptr, u_char tag) -{ - FPU_REG a; - - clear_C1(); - - if (tag == TAG_Valid) { - /* For an 80486 FPU, the result is undefined if the arg is >= 1.0 */ - if (exponent(st0_ptr) < 0) { - denormal_arg: - - FPU_to_exp16(st0_ptr, &a); - - /* poly_2xm1(x) requires 0 < st(0) < 1. */ - poly_2xm1(getsign(st0_ptr), &a, st0_ptr); - } - set_precision_flag_up(); /* 80486 appears to always do this */ - return; - } - - if (tag == TAG_Zero) - return; - - if (tag == TAG_Special) - tag = FPU_Special(st0_ptr); - - switch (tag) { - case TW_Denormal: - if (denormal_operand() < 0) - return; - goto denormal_arg; - case TW_Infinity: - if (signnegative(st0_ptr)) { - /* -infinity gives -1 (p16-10) */ - FPU_copy_to_reg0(&CONST_1, TAG_Valid); - setnegative(st0_ptr); - } - return; - default: - single_arg_error(st0_ptr, tag); - } -} - -static void fptan(FPU_REG *st0_ptr, u_char st0_tag) -{ - FPU_REG *st_new_ptr; - int q; - u_char arg_sign = getsign(st0_ptr); - - /* Stack underflow has higher priority */ - if (st0_tag == TAG_Empty) { - FPU_stack_underflow(); /* Puts a QNaN in st(0) */ - if (control_word & CW_Invalid) { - st_new_ptr = &st(-1); - push(); - FPU_stack_underflow(); /* Puts a QNaN in the new st(0) */ - } - return; - } - - if (STACK_OVERFLOW) { - FPU_stack_overflow(); - return; - } - - if (st0_tag == TAG_Valid) { - if (exponent(st0_ptr) > -40) { - if ((q = trig_arg(st0_ptr, 0)) == -1) { - /* Operand is out of range */ - return; - } - - poly_tan(st0_ptr); - setsign(st0_ptr, (q & 1) ^ (arg_sign != 0)); - set_precision_flag_up(); /* We do not really know if up or down */ - } else { - /* For a small arg, the result == the argument */ - /* Underflow may happen */ - - denormal_arg: - - FPU_to_exp16(st0_ptr, st0_ptr); - - st0_tag = - FPU_round(st0_ptr, 1, 0, FULL_PRECISION, arg_sign); - FPU_settag0(st0_tag); - } - push(); - FPU_copy_to_reg0(&CONST_1, TAG_Valid); - return; - } - - if (st0_tag == TAG_Zero) { - push(); - FPU_copy_to_reg0(&CONST_1, TAG_Valid); - setcc(0); - return; - } - - if (st0_tag == TAG_Special) - st0_tag = FPU_Special(st0_ptr); - - if (st0_tag == TW_Denormal) { - if (denormal_operand() < 0) - return; - - goto denormal_arg; - } - - if (st0_tag == TW_Infinity) { - /* The 80486 treats infinity as an invalid operand */ - if (arith_invalid(0) >= 0) { - st_new_ptr = &st(-1); - push(); - arith_invalid(0); - } - return; - } - - single_arg_2_error(st0_ptr, st0_tag); -} - -static void fxtract(FPU_REG *st0_ptr, u_char st0_tag) -{ - FPU_REG *st_new_ptr; - u_char sign; - register FPU_REG *st1_ptr = st0_ptr; /* anticipate */ - - if (STACK_OVERFLOW) { - FPU_stack_overflow(); - return; - } - - clear_C1(); - - if (st0_tag == TAG_Valid) { - long e; - - push(); - sign = getsign(st1_ptr); - reg_copy(st1_ptr, st_new_ptr); - setexponent16(st_new_ptr, exponent(st_new_ptr)); - - denormal_arg: - - e = exponent16(st_new_ptr); - convert_l2reg(&e, 1); - setexponentpos(st_new_ptr, 0); - setsign(st_new_ptr, sign); - FPU_settag0(TAG_Valid); /* Needed if arg was a denormal */ - return; - } else if (st0_tag == TAG_Zero) { - sign = getsign(st0_ptr); - - if (FPU_divide_by_zero(0, SIGN_NEG) < 0) - return; - - push(); - FPU_copy_to_reg0(&CONST_Z, TAG_Zero); - setsign(st_new_ptr, sign); - return; - } - - if (st0_tag == TAG_Special) - st0_tag = FPU_Special(st0_ptr); - - if (st0_tag == TW_Denormal) { - if (denormal_operand() < 0) - return; - - push(); - sign = getsign(st1_ptr); - FPU_to_exp16(st1_ptr, st_new_ptr); - goto denormal_arg; - } else if (st0_tag == TW_Infinity) { - sign = getsign(st0_ptr); - setpositive(st0_ptr); - push(); - FPU_copy_to_reg0(&CONST_INF, TAG_Special); - setsign(st_new_ptr, sign); - return; - } else if (st0_tag == TW_NaN) { - if (real_1op_NaN(st0_ptr) < 0) - return; - - push(); - FPU_copy_to_reg0(st0_ptr, TAG_Special); - return; - } else if (st0_tag == TAG_Empty) { - /* Is this the correct behaviour? */ - if (control_word & EX_Invalid) { - FPU_stack_underflow(); - push(); - FPU_stack_underflow(); - } else - EXCEPTION(EX_StackUnder); - } -#ifdef PARANOID - else - EXCEPTION(EX_INTERNAL | 0x119); -#endif /* PARANOID */ -} - -static void fdecstp(FPU_REG *st0_ptr, u_char st0_tag) -{ - clear_C1(); - top--; -} - -static void fincstp(FPU_REG *st0_ptr, u_char st0_tag) -{ - clear_C1(); - top++; -} - -static void fsqrt_(FPU_REG *st0_ptr, u_char st0_tag) -{ - int expon; - - clear_C1(); - - if (st0_tag == TAG_Valid) { - u_char tag; - - if (signnegative(st0_ptr)) { - arith_invalid(0); /* sqrt(negative) is invalid */ - return; - } - - /* make st(0) in [1.0 .. 4.0) */ - expon = exponent(st0_ptr); - - denormal_arg: - - setexponent16(st0_ptr, (expon & 1)); - - /* Do the computation, the sign of the result will be positive. */ - tag = wm_sqrt(st0_ptr, 0, 0, control_word, SIGN_POS); - addexponent(st0_ptr, expon >> 1); - FPU_settag0(tag); - return; - } - - if (st0_tag == TAG_Zero) - return; - - if (st0_tag == TAG_Special) - st0_tag = FPU_Special(st0_ptr); - - if (st0_tag == TW_Infinity) { - if (signnegative(st0_ptr)) - arith_invalid(0); /* sqrt(-Infinity) is invalid */ - return; - } else if (st0_tag == TW_Denormal) { - if (signnegative(st0_ptr)) { - arith_invalid(0); /* sqrt(negative) is invalid */ - return; - } - - if (denormal_operand() < 0) - return; - - FPU_to_exp16(st0_ptr, st0_ptr); - - expon = exponent16(st0_ptr); - - goto denormal_arg; - } - - single_arg_error(st0_ptr, st0_tag); - -} - -static void frndint_(FPU_REG *st0_ptr, u_char st0_tag) -{ - int flags, tag; - - if (st0_tag == TAG_Valid) { - u_char sign; - - denormal_arg: - - sign = getsign(st0_ptr); - - if (exponent(st0_ptr) > 63) - return; - - if (st0_tag == TW_Denormal) { - if (denormal_operand() < 0) - return; - } - - /* Fortunately, this can't overflow to 2^64 */ - if ((flags = FPU_round_to_int(st0_ptr, st0_tag))) - set_precision_flag(flags); - - setexponent16(st0_ptr, 63); - tag = FPU_normalize(st0_ptr); - setsign(st0_ptr, sign); - FPU_settag0(tag); - return; - } - - if (st0_tag == TAG_Zero) - return; - - if (st0_tag == TAG_Special) - st0_tag = FPU_Special(st0_ptr); - - if (st0_tag == TW_Denormal) - goto denormal_arg; - else if (st0_tag == TW_Infinity) - return; - else - single_arg_error(st0_ptr, st0_tag); -} - -static int f_sin(FPU_REG *st0_ptr, u_char tag) -{ - u_char arg_sign = getsign(st0_ptr); - - if (tag == TAG_Valid) { - int q; - - if (exponent(st0_ptr) > -40) { - if ((q = trig_arg(st0_ptr, 0)) == -1) { - /* Operand is out of range */ - return 1; - } - - poly_sine(st0_ptr); - - if (q & 2) - changesign(st0_ptr); - - setsign(st0_ptr, getsign(st0_ptr) ^ arg_sign); - - /* We do not really know if up or down */ - set_precision_flag_up(); - return 0; - } else { - /* For a small arg, the result == the argument */ - set_precision_flag_up(); /* Must be up. */ - return 0; - } - } - - if (tag == TAG_Zero) { - setcc(0); - return 0; - } - - if (tag == TAG_Special) - tag = FPU_Special(st0_ptr); - - if (tag == TW_Denormal) { - if (denormal_operand() < 0) - return 1; - - /* For a small arg, the result == the argument */ - /* Underflow may happen */ - FPU_to_exp16(st0_ptr, st0_ptr); - - tag = FPU_round(st0_ptr, 1, 0, FULL_PRECISION, arg_sign); - - FPU_settag0(tag); - - return 0; - } else if (tag == TW_Infinity) { - /* The 80486 treats infinity as an invalid operand */ - arith_invalid(0); - return 1; - } else { - single_arg_error(st0_ptr, tag); - return 1; - } -} - -static void fsin(FPU_REG *st0_ptr, u_char tag) -{ - f_sin(st0_ptr, tag); -} - -static int f_cos(FPU_REG *st0_ptr, u_char tag) -{ - u_char st0_sign; - - st0_sign = getsign(st0_ptr); - - if (tag == TAG_Valid) { - int q; - - if (exponent(st0_ptr) > -40) { - if ((exponent(st0_ptr) < 0) - || ((exponent(st0_ptr) == 0) - && (significand(st0_ptr) <= - 0xc90fdaa22168c234LL))) { - poly_cos(st0_ptr); - - /* We do not really know if up or down */ - set_precision_flag_down(); - - return 0; - } else if ((q = trig_arg(st0_ptr, FCOS)) != -1) { - poly_sine(st0_ptr); - - if ((q + 1) & 2) - changesign(st0_ptr); - - /* We do not really know if up or down */ - set_precision_flag_down(); - - return 0; - } else { - /* Operand is out of range */ - return 1; - } - } else { - denormal_arg: - - setcc(0); - FPU_copy_to_reg0(&CONST_1, TAG_Valid); -#ifdef PECULIAR_486 - set_precision_flag_down(); /* 80486 appears to do this. */ -#else - set_precision_flag_up(); /* Must be up. */ -#endif /* PECULIAR_486 */ - return 0; - } - } else if (tag == TAG_Zero) { - FPU_copy_to_reg0(&CONST_1, TAG_Valid); - setcc(0); - return 0; - } - - if (tag == TAG_Special) - tag = FPU_Special(st0_ptr); - - if (tag == TW_Denormal) { - if (denormal_operand() < 0) - return 1; - - goto denormal_arg; - } else if (tag == TW_Infinity) { - /* The 80486 treats infinity as an invalid operand */ - arith_invalid(0); - return 1; - } else { - single_arg_error(st0_ptr, tag); /* requires st0_ptr == &st(0) */ - return 1; - } -} - -static void fcos(FPU_REG *st0_ptr, u_char st0_tag) -{ - f_cos(st0_ptr, st0_tag); -} - -static void fsincos(FPU_REG *st0_ptr, u_char st0_tag) -{ - FPU_REG *st_new_ptr; - FPU_REG arg; - u_char tag; - - /* Stack underflow has higher priority */ - if (st0_tag == TAG_Empty) { - FPU_stack_underflow(); /* Puts a QNaN in st(0) */ - if (control_word & CW_Invalid) { - st_new_ptr = &st(-1); - push(); - FPU_stack_underflow(); /* Puts a QNaN in the new st(0) */ - } - return; - } - - if (STACK_OVERFLOW) { - FPU_stack_overflow(); - return; - } - - if (st0_tag == TAG_Special) - tag = FPU_Special(st0_ptr); - else - tag = st0_tag; - - if (tag == TW_NaN) { - single_arg_2_error(st0_ptr, TW_NaN); - return; - } else if (tag == TW_Infinity) { - /* The 80486 treats infinity as an invalid operand */ - if (arith_invalid(0) >= 0) { - /* Masked response */ - push(); - arith_invalid(0); - } - return; - } - - reg_copy(st0_ptr, &arg); - if (!f_sin(st0_ptr, st0_tag)) { - push(); - FPU_copy_to_reg0(&arg, st0_tag); - f_cos(&st(0), st0_tag); - } else { - /* An error, so restore st(0) */ - FPU_copy_to_reg0(&arg, st0_tag); - } -} - -/*---------------------------------------------------------------------------*/ -/* The following all require two arguments: st(0) and st(1) */ - -/* A lean, mean kernel for the fprem instructions. This relies upon - the division and rounding to an integer in do_fprem giving an - exact result. Because of this, rem_kernel() needs to deal only with - the least significant 64 bits, the more significant bits of the - result must be zero. - */ -static void rem_kernel(unsigned long long st0, unsigned long long *y, - unsigned long long st1, unsigned long long q, int n) -{ - int dummy; - unsigned long long x; - - x = st0 << n; - - /* Do the required multiplication and subtraction in the one operation */ - - /* lsw x -= lsw st1 * lsw q */ - asm volatile ("mull %4; subl %%eax,%0; sbbl %%edx,%1":"=m" - (((unsigned *)&x)[0]), "=m"(((unsigned *)&x)[1]), - "=a"(dummy) - :"2"(((unsigned *)&st1)[0]), "m"(((unsigned *)&q)[0]) - :"%dx"); - /* msw x -= msw st1 * lsw q */ - asm volatile ("mull %3; subl %%eax,%0":"=m" (((unsigned *)&x)[1]), - "=a"(dummy) - :"1"(((unsigned *)&st1)[1]), "m"(((unsigned *)&q)[0]) - :"%dx"); - /* msw x -= lsw st1 * msw q */ - asm volatile ("mull %3; subl %%eax,%0":"=m" (((unsigned *)&x)[1]), - "=a"(dummy) - :"1"(((unsigned *)&st1)[0]), "m"(((unsigned *)&q)[1]) - :"%dx"); - - *y = x; -} - -/* Remainder of st(0) / st(1) */ -/* This routine produces exact results, i.e. there is never any - rounding or truncation, etc of the result. */ -static void do_fprem(FPU_REG *st0_ptr, u_char st0_tag, int round) -{ - FPU_REG *st1_ptr = &st(1); - u_char st1_tag = FPU_gettagi(1); - - if (!((st0_tag ^ TAG_Valid) | (st1_tag ^ TAG_Valid))) { - FPU_REG tmp, st0, st1; - u_char st0_sign, st1_sign; - u_char tmptag; - int tag; - int old_cw; - int expdif; - long long q; - unsigned short saved_status; - int cc; - - fprem_valid: - /* Convert registers for internal use. */ - st0_sign = FPU_to_exp16(st0_ptr, &st0); - st1_sign = FPU_to_exp16(st1_ptr, &st1); - expdif = exponent16(&st0) - exponent16(&st1); - - old_cw = control_word; - cc = 0; - - /* We want the status following the denorm tests, but don't want - the status changed by the arithmetic operations. */ - saved_status = partial_status; - control_word &= ~CW_RC; - control_word |= RC_CHOP; - - if (expdif < 64) { - /* This should be the most common case */ - - if (expdif > -2) { - u_char sign = st0_sign ^ st1_sign; - tag = FPU_u_div(&st0, &st1, &tmp, - PR_64_BITS | RC_CHOP | 0x3f, - sign); - setsign(&tmp, sign); - - if (exponent(&tmp) >= 0) { - FPU_round_to_int(&tmp, tag); /* Fortunately, this can't - overflow to 2^64 */ - q = significand(&tmp); - - rem_kernel(significand(&st0), - &significand(&tmp), - significand(&st1), - q, expdif); - - setexponent16(&tmp, exponent16(&st1)); - } else { - reg_copy(&st0, &tmp); - q = 0; - } - - if ((round == RC_RND) - && (tmp.sigh & 0xc0000000)) { - /* We may need to subtract st(1) once more, - to get a result <= 1/2 of st(1). */ - unsigned long long x; - expdif = - exponent16(&st1) - exponent16(&tmp); - if (expdif <= 1) { - if (expdif == 0) - x = significand(&st1) - - significand(&tmp); - else /* expdif is 1 */ - x = (significand(&st1) - << 1) - - significand(&tmp); - if ((x < significand(&tmp)) || - /* or equi-distant (from 0 & st(1)) and q is odd */ - ((x == significand(&tmp)) - && (q & 1))) { - st0_sign = !st0_sign; - significand(&tmp) = x; - q++; - } - } - } - - if (q & 4) - cc |= SW_C0; - if (q & 2) - cc |= SW_C3; - if (q & 1) - cc |= SW_C1; - } else { - control_word = old_cw; - setcc(0); - return; - } - } else { - /* There is a large exponent difference ( >= 64 ) */ - /* To make much sense, the code in this section should - be done at high precision. */ - int exp_1, N; - u_char sign; - - /* prevent overflow here */ - /* N is 'a number between 32 and 63' (p26-113) */ - reg_copy(&st0, &tmp); - tmptag = st0_tag; - N = (expdif & 0x0000001f) + 32; /* This choice gives results - identical to an AMD 486 */ - setexponent16(&tmp, N); - exp_1 = exponent16(&st1); - setexponent16(&st1, 0); - expdif -= N; - - sign = getsign(&tmp) ^ st1_sign; - tag = - FPU_u_div(&tmp, &st1, &tmp, - PR_64_BITS | RC_CHOP | 0x3f, sign); - setsign(&tmp, sign); - - FPU_round_to_int(&tmp, tag); /* Fortunately, this can't - overflow to 2^64 */ - - rem_kernel(significand(&st0), - &significand(&tmp), - significand(&st1), - significand(&tmp), exponent(&tmp) - ); - setexponent16(&tmp, exp_1 + expdif); - - /* It is possible for the operation to be complete here. - What does the IEEE standard say? The Intel 80486 manual - implies that the operation will never be completed at this - point, and the behaviour of a real 80486 confirms this. - */ - if (!(tmp.sigh | tmp.sigl)) { - /* The result is zero */ - control_word = old_cw; - partial_status = saved_status; - FPU_copy_to_reg0(&CONST_Z, TAG_Zero); - setsign(&st0, st0_sign); -#ifdef PECULIAR_486 - setcc(SW_C2); -#else - setcc(0); -#endif /* PECULIAR_486 */ - return; - } - cc = SW_C2; - } - - control_word = old_cw; - partial_status = saved_status; - tag = FPU_normalize_nuo(&tmp); - reg_copy(&tmp, st0_ptr); - - /* The only condition to be looked for is underflow, - and it can occur here only if underflow is unmasked. */ - if ((exponent16(&tmp) <= EXP_UNDER) && (tag != TAG_Zero) - && !(control_word & CW_Underflow)) { - setcc(cc); - tag = arith_underflow(st0_ptr); - setsign(st0_ptr, st0_sign); - FPU_settag0(tag); - return; - } else if ((exponent16(&tmp) > EXP_UNDER) || (tag == TAG_Zero)) { - stdexp(st0_ptr); - setsign(st0_ptr, st0_sign); - } else { - tag = - FPU_round(st0_ptr, 0, 0, FULL_PRECISION, st0_sign); - } - FPU_settag0(tag); - setcc(cc); - - return; - } - - if (st0_tag == TAG_Special) - st0_tag = FPU_Special(st0_ptr); - if (st1_tag == TAG_Special) - st1_tag = FPU_Special(st1_ptr); - - if (((st0_tag == TAG_Valid) && (st1_tag == TW_Denormal)) - || ((st0_tag == TW_Denormal) && (st1_tag == TAG_Valid)) - || ((st0_tag == TW_Denormal) && (st1_tag == TW_Denormal))) { - if (denormal_operand() < 0) - return; - goto fprem_valid; - } else if ((st0_tag == TAG_Empty) || (st1_tag == TAG_Empty)) { - FPU_stack_underflow(); - return; - } else if (st0_tag == TAG_Zero) { - if (st1_tag == TAG_Valid) { - setcc(0); - return; - } else if (st1_tag == TW_Denormal) { - if (denormal_operand() < 0) - return; - setcc(0); - return; - } else if (st1_tag == TAG_Zero) { - arith_invalid(0); - return; - } /* fprem(?,0) always invalid */ - else if (st1_tag == TW_Infinity) { - setcc(0); - return; - } - } else if ((st0_tag == TAG_Valid) || (st0_tag == TW_Denormal)) { - if (st1_tag == TAG_Zero) { - arith_invalid(0); /* fprem(Valid,Zero) is invalid */ - return; - } else if (st1_tag != TW_NaN) { - if (((st0_tag == TW_Denormal) - || (st1_tag == TW_Denormal)) - && (denormal_operand() < 0)) - return; - - if (st1_tag == TW_Infinity) { - /* fprem(Valid,Infinity) is o.k. */ - setcc(0); - return; - } - } - } else if (st0_tag == TW_Infinity) { - if (st1_tag != TW_NaN) { - arith_invalid(0); /* fprem(Infinity,?) is invalid */ - return; - } - } - - /* One of the registers must contain a NaN if we got here. */ - -#ifdef PARANOID - if ((st0_tag != TW_NaN) && (st1_tag != TW_NaN)) - EXCEPTION(EX_INTERNAL | 0x118); -#endif /* PARANOID */ - - real_2op_NaN(st1_ptr, st1_tag, 0, st1_ptr); - -} - -/* ST(1) <- ST(1) * log ST; pop ST */ -static void fyl2x(FPU_REG *st0_ptr, u_char st0_tag) -{ - FPU_REG *st1_ptr = &st(1), exponent; - u_char st1_tag = FPU_gettagi(1); - u_char sign; - int e, tag; - - clear_C1(); - - if ((st0_tag == TAG_Valid) && (st1_tag == TAG_Valid)) { - both_valid: - /* Both regs are Valid or Denormal */ - if (signpositive(st0_ptr)) { - if (st0_tag == TW_Denormal) - FPU_to_exp16(st0_ptr, st0_ptr); - else - /* Convert st(0) for internal use. */ - setexponent16(st0_ptr, exponent(st0_ptr)); - - if ((st0_ptr->sigh == 0x80000000) - && (st0_ptr->sigl == 0)) { - /* Special case. The result can be precise. */ - u_char esign; - e = exponent16(st0_ptr); - if (e >= 0) { - exponent.sigh = e; - esign = SIGN_POS; - } else { - exponent.sigh = -e; - esign = SIGN_NEG; - } - exponent.sigl = 0; - setexponent16(&exponent, 31); - tag = FPU_normalize_nuo(&exponent); - stdexp(&exponent); - setsign(&exponent, esign); - tag = - FPU_mul(&exponent, tag, 1, FULL_PRECISION); - if (tag >= 0) - FPU_settagi(1, tag); - } else { - /* The usual case */ - sign = getsign(st1_ptr); - if (st1_tag == TW_Denormal) - FPU_to_exp16(st1_ptr, st1_ptr); - else - /* Convert st(1) for internal use. */ - setexponent16(st1_ptr, - exponent(st1_ptr)); - poly_l2(st0_ptr, st1_ptr, sign); - } - } else { - /* negative */ - if (arith_invalid(1) < 0) - return; - } - - FPU_pop(); - - return; - } - - if (st0_tag == TAG_Special) - st0_tag = FPU_Special(st0_ptr); - if (st1_tag == TAG_Special) - st1_tag = FPU_Special(st1_ptr); - - if ((st0_tag == TAG_Empty) || (st1_tag == TAG_Empty)) { - FPU_stack_underflow_pop(1); - return; - } else if ((st0_tag <= TW_Denormal) && (st1_tag <= TW_Denormal)) { - if (st0_tag == TAG_Zero) { - if (st1_tag == TAG_Zero) { - /* Both args zero is invalid */ - if (arith_invalid(1) < 0) - return; - } else { - u_char sign; - sign = getsign(st1_ptr) ^ SIGN_NEG; - if (FPU_divide_by_zero(1, sign) < 0) - return; - - setsign(st1_ptr, sign); - } - } else if (st1_tag == TAG_Zero) { - /* st(1) contains zero, st(0) valid <> 0 */ - /* Zero is the valid answer */ - sign = getsign(st1_ptr); - - if (signnegative(st0_ptr)) { - /* log(negative) */ - if (arith_invalid(1) < 0) - return; - } else if ((st0_tag == TW_Denormal) - && (denormal_operand() < 0)) - return; - else { - if (exponent(st0_ptr) < 0) - sign ^= SIGN_NEG; - - FPU_copy_to_reg1(&CONST_Z, TAG_Zero); - setsign(st1_ptr, sign); - } - } else { - /* One or both operands are denormals. */ - if (denormal_operand() < 0) - return; - goto both_valid; - } - } else if ((st0_tag == TW_NaN) || (st1_tag == TW_NaN)) { - if (real_2op_NaN(st0_ptr, st0_tag, 1, st0_ptr) < 0) - return; - } - /* One or both arg must be an infinity */ - else if (st0_tag == TW_Infinity) { - if ((signnegative(st0_ptr)) || (st1_tag == TAG_Zero)) { - /* log(-infinity) or 0*log(infinity) */ - if (arith_invalid(1) < 0) - return; - } else { - u_char sign = getsign(st1_ptr); - - if ((st1_tag == TW_Denormal) - && (denormal_operand() < 0)) - return; - - FPU_copy_to_reg1(&CONST_INF, TAG_Special); - setsign(st1_ptr, sign); - } - } - /* st(1) must be infinity here */ - else if (((st0_tag == TAG_Valid) || (st0_tag == TW_Denormal)) - && (signpositive(st0_ptr))) { - if (exponent(st0_ptr) >= 0) { - if ((exponent(st0_ptr) == 0) && - (st0_ptr->sigh == 0x80000000) && - (st0_ptr->sigl == 0)) { - /* st(0) holds 1.0 */ - /* infinity*log(1) */ - if (arith_invalid(1) < 0) - return; - } - /* else st(0) is positive and > 1.0 */ - } else { - /* st(0) is positive and < 1.0 */ - - if ((st0_tag == TW_Denormal) - && (denormal_operand() < 0)) - return; - - changesign(st1_ptr); - } - } else { - /* st(0) must be zero or negative */ - if (st0_tag == TAG_Zero) { - /* This should be invalid, but a real 80486 is happy with it. */ - -#ifndef PECULIAR_486 - sign = getsign(st1_ptr); - if (FPU_divide_by_zero(1, sign) < 0) - return; -#endif /* PECULIAR_486 */ - - changesign(st1_ptr); - } else if (arith_invalid(1) < 0) /* log(negative) */ - return; - } - - FPU_pop(); -} - -static void fpatan(FPU_REG *st0_ptr, u_char st0_tag) -{ - FPU_REG *st1_ptr = &st(1); - u_char st1_tag = FPU_gettagi(1); - int tag; - - clear_C1(); - if (!((st0_tag ^ TAG_Valid) | (st1_tag ^ TAG_Valid))) { - valid_atan: - - poly_atan(st0_ptr, st0_tag, st1_ptr, st1_tag); - - FPU_pop(); - - return; - } - - if (st0_tag == TAG_Special) - st0_tag = FPU_Special(st0_ptr); - if (st1_tag == TAG_Special) - st1_tag = FPU_Special(st1_ptr); - - if (((st0_tag == TAG_Valid) && (st1_tag == TW_Denormal)) - || ((st0_tag == TW_Denormal) && (st1_tag == TAG_Valid)) - || ((st0_tag == TW_Denormal) && (st1_tag == TW_Denormal))) { - if (denormal_operand() < 0) - return; - - goto valid_atan; - } else if ((st0_tag == TAG_Empty) || (st1_tag == TAG_Empty)) { - FPU_stack_underflow_pop(1); - return; - } else if ((st0_tag == TW_NaN) || (st1_tag == TW_NaN)) { - if (real_2op_NaN(st0_ptr, st0_tag, 1, st0_ptr) >= 0) - FPU_pop(); - return; - } else if ((st0_tag == TW_Infinity) || (st1_tag == TW_Infinity)) { - u_char sign = getsign(st1_ptr); - if (st0_tag == TW_Infinity) { - if (st1_tag == TW_Infinity) { - if (signpositive(st0_ptr)) { - FPU_copy_to_reg1(&CONST_PI4, TAG_Valid); - } else { - setpositive(st1_ptr); - tag = - FPU_u_add(&CONST_PI4, &CONST_PI2, - st1_ptr, FULL_PRECISION, - SIGN_POS, - exponent(&CONST_PI4), - exponent(&CONST_PI2)); - if (tag >= 0) - FPU_settagi(1, tag); - } - } else { - if ((st1_tag == TW_Denormal) - && (denormal_operand() < 0)) - return; - - if (signpositive(st0_ptr)) { - FPU_copy_to_reg1(&CONST_Z, TAG_Zero); - setsign(st1_ptr, sign); /* An 80486 preserves the sign */ - FPU_pop(); - return; - } else { - FPU_copy_to_reg1(&CONST_PI, TAG_Valid); - } - } - } else { - /* st(1) is infinity, st(0) not infinity */ - if ((st0_tag == TW_Denormal) - && (denormal_operand() < 0)) - return; - - FPU_copy_to_reg1(&CONST_PI2, TAG_Valid); - } - setsign(st1_ptr, sign); - } else if (st1_tag == TAG_Zero) { - /* st(0) must be valid or zero */ - u_char sign = getsign(st1_ptr); - - if ((st0_tag == TW_Denormal) && (denormal_operand() < 0)) - return; - - if (signpositive(st0_ptr)) { - /* An 80486 preserves the sign */ - FPU_pop(); - return; - } - - FPU_copy_to_reg1(&CONST_PI, TAG_Valid); - setsign(st1_ptr, sign); - } else if (st0_tag == TAG_Zero) { - /* st(1) must be TAG_Valid here */ - u_char sign = getsign(st1_ptr); - - if ((st1_tag == TW_Denormal) && (denormal_operand() < 0)) - return; - - FPU_copy_to_reg1(&CONST_PI2, TAG_Valid); - setsign(st1_ptr, sign); - } -#ifdef PARANOID - else - EXCEPTION(EX_INTERNAL | 0x125); -#endif /* PARANOID */ - - FPU_pop(); - set_precision_flag_up(); /* We do not really know if up or down */ -} - -static void fprem(FPU_REG *st0_ptr, u_char st0_tag) -{ - do_fprem(st0_ptr, st0_tag, RC_CHOP); -} - -static void fprem1(FPU_REG *st0_ptr, u_char st0_tag) -{ - do_fprem(st0_ptr, st0_tag, RC_RND); -} - -static void fyl2xp1(FPU_REG *st0_ptr, u_char st0_tag) -{ - u_char sign, sign1; - FPU_REG *st1_ptr = &st(1), a, b; - u_char st1_tag = FPU_gettagi(1); - - clear_C1(); - if (!((st0_tag ^ TAG_Valid) | (st1_tag ^ TAG_Valid))) { - valid_yl2xp1: - - sign = getsign(st0_ptr); - sign1 = getsign(st1_ptr); - - FPU_to_exp16(st0_ptr, &a); - FPU_to_exp16(st1_ptr, &b); - - if (poly_l2p1(sign, sign1, &a, &b, st1_ptr)) - return; - - FPU_pop(); - return; - } - - if (st0_tag == TAG_Special) - st0_tag = FPU_Special(st0_ptr); - if (st1_tag == TAG_Special) - st1_tag = FPU_Special(st1_ptr); - - if (((st0_tag == TAG_Valid) && (st1_tag == TW_Denormal)) - || ((st0_tag == TW_Denormal) && (st1_tag == TAG_Valid)) - || ((st0_tag == TW_Denormal) && (st1_tag == TW_Denormal))) { - if (denormal_operand() < 0) - return; - - goto valid_yl2xp1; - } else if ((st0_tag == TAG_Empty) | (st1_tag == TAG_Empty)) { - FPU_stack_underflow_pop(1); - return; - } else if (st0_tag == TAG_Zero) { - switch (st1_tag) { - case TW_Denormal: - if (denormal_operand() < 0) - return; - fallthrough; - case TAG_Zero: - case TAG_Valid: - setsign(st0_ptr, getsign(st0_ptr) ^ getsign(st1_ptr)); - FPU_copy_to_reg1(st0_ptr, st0_tag); - break; - - case TW_Infinity: - /* Infinity*log(1) */ - if (arith_invalid(1) < 0) - return; - break; - - case TW_NaN: - if (real_2op_NaN(st0_ptr, st0_tag, 1, st0_ptr) < 0) - return; - break; - - default: -#ifdef PARANOID - EXCEPTION(EX_INTERNAL | 0x116); - return; -#endif /* PARANOID */ - break; - } - } else if ((st0_tag == TAG_Valid) || (st0_tag == TW_Denormal)) { - switch (st1_tag) { - case TAG_Zero: - if (signnegative(st0_ptr)) { - if (exponent(st0_ptr) >= 0) { - /* st(0) holds <= -1.0 */ -#ifdef PECULIAR_486 /* Stupid 80486 doesn't worry about log(negative). */ - changesign(st1_ptr); -#else - if (arith_invalid(1) < 0) - return; -#endif /* PECULIAR_486 */ - } else if ((st0_tag == TW_Denormal) - && (denormal_operand() < 0)) - return; - else - changesign(st1_ptr); - } else if ((st0_tag == TW_Denormal) - && (denormal_operand() < 0)) - return; - break; - - case TW_Infinity: - if (signnegative(st0_ptr)) { - if ((exponent(st0_ptr) >= 0) && - !((st0_ptr->sigh == 0x80000000) && - (st0_ptr->sigl == 0))) { - /* st(0) holds < -1.0 */ -#ifdef PECULIAR_486 /* Stupid 80486 doesn't worry about log(negative). */ - changesign(st1_ptr); -#else - if (arith_invalid(1) < 0) - return; -#endif /* PECULIAR_486 */ - } else if ((st0_tag == TW_Denormal) - && (denormal_operand() < 0)) - return; - else - changesign(st1_ptr); - } else if ((st0_tag == TW_Denormal) - && (denormal_operand() < 0)) - return; - break; - - case TW_NaN: - if (real_2op_NaN(st0_ptr, st0_tag, 1, st0_ptr) < 0) - return; - } - - } else if (st0_tag == TW_NaN) { - if (real_2op_NaN(st0_ptr, st0_tag, 1, st0_ptr) < 0) - return; - } else if (st0_tag == TW_Infinity) { - if (st1_tag == TW_NaN) { - if (real_2op_NaN(st0_ptr, st0_tag, 1, st0_ptr) < 0) - return; - } else if (signnegative(st0_ptr)) { -#ifndef PECULIAR_486 - /* This should have higher priority than denormals, but... */ - if (arith_invalid(1) < 0) /* log(-infinity) */ - return; -#endif /* PECULIAR_486 */ - if ((st1_tag == TW_Denormal) - && (denormal_operand() < 0)) - return; -#ifdef PECULIAR_486 - /* Denormal operands actually get higher priority */ - if (arith_invalid(1) < 0) /* log(-infinity) */ - return; -#endif /* PECULIAR_486 */ - } else if (st1_tag == TAG_Zero) { - /* log(infinity) */ - if (arith_invalid(1) < 0) - return; - } - - /* st(1) must be valid here. */ - - else if ((st1_tag == TW_Denormal) && (denormal_operand() < 0)) - return; - - /* The Manual says that log(Infinity) is invalid, but a real - 80486 sensibly says that it is o.k. */ - else { - u_char sign = getsign(st1_ptr); - FPU_copy_to_reg1(&CONST_INF, TAG_Special); - setsign(st1_ptr, sign); - } - } -#ifdef PARANOID - else { - EXCEPTION(EX_INTERNAL | 0x117); - return; - } -#endif /* PARANOID */ - - FPU_pop(); - return; - -} - -static void fscale(FPU_REG *st0_ptr, u_char st0_tag) -{ - FPU_REG *st1_ptr = &st(1); - u_char st1_tag = FPU_gettagi(1); - int old_cw = control_word; - u_char sign = getsign(st0_ptr); - - clear_C1(); - if (!((st0_tag ^ TAG_Valid) | (st1_tag ^ TAG_Valid))) { - long scale; - FPU_REG tmp; - - /* Convert register for internal use. */ - setexponent16(st0_ptr, exponent(st0_ptr)); - - valid_scale: - - if (exponent(st1_ptr) > 30) { - /* 2^31 is far too large, would require 2^(2^30) or 2^(-2^30) */ - - if (signpositive(st1_ptr)) { - EXCEPTION(EX_Overflow); - FPU_copy_to_reg0(&CONST_INF, TAG_Special); - } else { - EXCEPTION(EX_Underflow); - FPU_copy_to_reg0(&CONST_Z, TAG_Zero); - } - setsign(st0_ptr, sign); - return; - } - - control_word &= ~CW_RC; - control_word |= RC_CHOP; - reg_copy(st1_ptr, &tmp); - FPU_round_to_int(&tmp, st1_tag); /* This can never overflow here */ - control_word = old_cw; - scale = signnegative(st1_ptr) ? -tmp.sigl : tmp.sigl; - scale += exponent16(st0_ptr); - - setexponent16(st0_ptr, scale); - - /* Use FPU_round() to properly detect under/overflow etc */ - FPU_round(st0_ptr, 0, 0, control_word, sign); - - return; - } - - if (st0_tag == TAG_Special) - st0_tag = FPU_Special(st0_ptr); - if (st1_tag == TAG_Special) - st1_tag = FPU_Special(st1_ptr); - - if ((st0_tag == TAG_Valid) || (st0_tag == TW_Denormal)) { - switch (st1_tag) { - case TAG_Valid: - /* st(0) must be a denormal */ - if ((st0_tag == TW_Denormal) - && (denormal_operand() < 0)) - return; - - FPU_to_exp16(st0_ptr, st0_ptr); /* Will not be left on stack */ - goto valid_scale; - - case TAG_Zero: - if (st0_tag == TW_Denormal) - denormal_operand(); - return; - - case TW_Denormal: - denormal_operand(); - return; - - case TW_Infinity: - if ((st0_tag == TW_Denormal) - && (denormal_operand() < 0)) - return; - - if (signpositive(st1_ptr)) - FPU_copy_to_reg0(&CONST_INF, TAG_Special); - else - FPU_copy_to_reg0(&CONST_Z, TAG_Zero); - setsign(st0_ptr, sign); - return; - - case TW_NaN: - real_2op_NaN(st1_ptr, st1_tag, 0, st0_ptr); - return; - } - } else if (st0_tag == TAG_Zero) { - switch (st1_tag) { - case TAG_Valid: - case TAG_Zero: - return; - - case TW_Denormal: - denormal_operand(); - return; - - case TW_Infinity: - if (signpositive(st1_ptr)) - arith_invalid(0); /* Zero scaled by +Infinity */ - return; - - case TW_NaN: - real_2op_NaN(st1_ptr, st1_tag, 0, st0_ptr); - return; - } - } else if (st0_tag == TW_Infinity) { - switch (st1_tag) { - case TAG_Valid: - case TAG_Zero: - return; - - case TW_Denormal: - denormal_operand(); - return; - - case TW_Infinity: - if (signnegative(st1_ptr)) - arith_invalid(0); /* Infinity scaled by -Infinity */ - return; - - case TW_NaN: - real_2op_NaN(st1_ptr, st1_tag, 0, st0_ptr); - return; - } - } else if (st0_tag == TW_NaN) { - if (st1_tag != TAG_Empty) { - real_2op_NaN(st1_ptr, st1_tag, 0, st0_ptr); - return; - } - } -#ifdef PARANOID - if (!((st0_tag == TAG_Empty) || (st1_tag == TAG_Empty))) { - EXCEPTION(EX_INTERNAL | 0x115); - return; - } -#endif - - /* At least one of st(0), st(1) must be empty */ - FPU_stack_underflow(); - -} - -/*---------------------------------------------------------------------------*/ - -static FUNC_ST0 const trig_table_a[] = { - f2xm1, fyl2x, fptan, fpatan, - fxtract, fprem1, fdecstp, fincstp, -}; - -void FPU_triga(void) -{ - (trig_table_a[FPU_rm]) (&st(0), FPU_gettag0()); -} - -static FUNC_ST0 const trig_table_b[] = { - fprem, fyl2xp1, fsqrt_, fsincos, frndint_, fscale, fsin, fcos -}; - -void FPU_trigb(void) -{ - (trig_table_b[FPU_rm]) (&st(0), FPU_gettag0()); -} diff --git a/arch/x86/math-emu/get_address.c b/arch/x86/math-emu/get_address.c deleted file mode 100644 index 4a9fd9029a53..000000000000 --- a/arch/x86/math-emu/get_address.c +++ /dev/null @@ -1,401 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/*---------------------------------------------------------------------------+ - | get_address.c | - | | - | Get the effective address from an FPU instruction. | - | | - | Copyright (C) 1992,1993,1994,1997 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, | - | Australia. E-mail billm@suburbia.net | - | | - | | - +---------------------------------------------------------------------------*/ - -/*---------------------------------------------------------------------------+ - | Note: | - | The file contains code which accesses user memory. | - | Emulator static data may change when user memory is accessed, due to | - | other processes using the emulator while swapping is in progress. | - +---------------------------------------------------------------------------*/ - -#include - -#include -#include - -#include "fpu_system.h" -#include "exception.h" -#include "fpu_emu.h" - -#define FPU_WRITE_BIT 0x10 - -static int reg_offset[] = { - offsetof(struct pt_regs, ax), - offsetof(struct pt_regs, cx), - offsetof(struct pt_regs, dx), - offsetof(struct pt_regs, bx), - offsetof(struct pt_regs, sp), - offsetof(struct pt_regs, bp), - offsetof(struct pt_regs, si), - offsetof(struct pt_regs, di) -}; - -#define REG_(x) (*(long *)(reg_offset[(x)] + (u_char *)FPU_info->regs)) - -static int reg_offset_vm86[] = { - offsetof(struct pt_regs, cs), - offsetof(struct kernel_vm86_regs, ds), - offsetof(struct kernel_vm86_regs, es), - offsetof(struct kernel_vm86_regs, fs), - offsetof(struct kernel_vm86_regs, gs), - offsetof(struct pt_regs, ss), - offsetof(struct kernel_vm86_regs, ds) -}; - -#define VM86_REG_(x) (*(unsigned short *) \ - (reg_offset_vm86[((unsigned)x)] + (u_char *)FPU_info->regs)) - -static int reg_offset_pm[] = { - offsetof(struct pt_regs, cs), - offsetof(struct pt_regs, ds), - offsetof(struct pt_regs, es), - offsetof(struct pt_regs, fs), - offsetof(struct pt_regs, ds), /* dummy, not saved on stack */ - offsetof(struct pt_regs, ss), - offsetof(struct pt_regs, ds) -}; - -#define PM_REG_(x) (*(unsigned short *) \ - (reg_offset_pm[((unsigned)x)] + (u_char *)FPU_info->regs)) - -/* Decode the SIB byte. This function assumes mod != 0 */ -static int sib(int mod, unsigned long *fpu_eip) -{ - u_char ss, index, base; - long offset; - - RE_ENTRANT_CHECK_OFF; - FPU_code_access_ok(1); - FPU_get_user(base, (u_char __user *) (*fpu_eip)); /* The SIB byte */ - RE_ENTRANT_CHECK_ON; - (*fpu_eip)++; - ss = base >> 6; - index = (base >> 3) & 7; - base &= 7; - - if ((mod == 0) && (base == 5)) - offset = 0; /* No base register */ - else - offset = REG_(base); - - if (index == 4) { - /* No index register */ - /* A non-zero ss is illegal */ - if (ss) - EXCEPTION(EX_Invalid); - } else { - offset += (REG_(index)) << ss; - } - - if (mod == 1) { - /* 8 bit signed displacement */ - long displacement; - RE_ENTRANT_CHECK_OFF; - FPU_code_access_ok(1); - FPU_get_user(displacement, (signed char __user *)(*fpu_eip)); - offset += displacement; - RE_ENTRANT_CHECK_ON; - (*fpu_eip)++; - } else if (mod == 2 || base == 5) { /* The second condition also has mod==0 */ - /* 32 bit displacement */ - long displacement; - RE_ENTRANT_CHECK_OFF; - FPU_code_access_ok(4); - FPU_get_user(displacement, (long __user *)(*fpu_eip)); - offset += displacement; - RE_ENTRANT_CHECK_ON; - (*fpu_eip) += 4; - } - - return offset; -} - -static unsigned long vm86_segment(u_char segment, struct address *addr) -{ - segment--; -#ifdef PARANOID - if (segment > PREFIX_SS_) { - EXCEPTION(EX_INTERNAL | 0x130); - math_abort(FPU_info, SIGSEGV); - } -#endif /* PARANOID */ - addr->selector = VM86_REG_(segment); - return (unsigned long)VM86_REG_(segment) << 4; -} - -/* This should work for 16 and 32 bit protected mode. */ -static long pm_address(u_char FPU_modrm, u_char segment, - struct address *addr, long offset) -{ - struct desc_struct descriptor; - unsigned long base_address, limit, address, seg_top; - - segment--; - -#ifdef PARANOID - /* segment is unsigned, so this also detects if segment was 0: */ - if (segment > PREFIX_SS_) { - EXCEPTION(EX_INTERNAL | 0x132); - math_abort(FPU_info, SIGSEGV); - } -#endif /* PARANOID */ - - switch (segment) { - case PREFIX_GS_ - 1: - /* user gs handling can be lazy, use special accessors */ - savesegment(gs, addr->selector); - break; - default: - addr->selector = PM_REG_(segment); - } - - descriptor = FPU_get_ldt_descriptor(addr->selector); - base_address = seg_get_base(&descriptor); - address = base_address + offset; - limit = seg_get_limit(&descriptor) + 1; - limit *= seg_get_granularity(&descriptor); - limit += base_address - 1; - if (limit < base_address) - limit = 0xffffffff; - - if (seg_expands_down(&descriptor)) { - if (descriptor.g) { - seg_top = 0xffffffff; - } else { - seg_top = base_address + (1 << 20); - if (seg_top < base_address) - seg_top = 0xffffffff; - } - access_limit = - (address <= limit) || (address >= seg_top) ? 0 : - ((seg_top - address) >= 255 ? 255 : seg_top - address); - } else { - access_limit = - (address > limit) || (address < base_address) ? 0 : - ((limit - address) >= 254 ? 255 : limit - address + 1); - } - if (seg_execute_only(&descriptor) || - (!seg_writable(&descriptor) && (FPU_modrm & FPU_WRITE_BIT))) { - access_limit = 0; - } - return address; -} - -/* - MOD R/M byte: MOD == 3 has a special use for the FPU - SIB byte used iff R/M = 100b - - 7 6 5 4 3 2 1 0 - ..... ......... ......... - MOD OPCODE(2) R/M - - SIB byte - - 7 6 5 4 3 2 1 0 - ..... ......... ......... - SS INDEX BASE - -*/ - -void __user *FPU_get_address(u_char FPU_modrm, unsigned long *fpu_eip, - struct address *addr, fpu_addr_modes addr_modes) -{ - u_char mod; - unsigned rm = FPU_modrm & 7; - long *cpu_reg_ptr; - int address = 0; /* Initialized just to stop compiler warnings. */ - - /* Memory accessed via the cs selector is write protected - in `non-segmented' 32 bit protected mode. */ - if (!addr_modes.default_mode && (FPU_modrm & FPU_WRITE_BIT) - && (addr_modes.override.segment == PREFIX_CS_)) { - math_abort(FPU_info, SIGSEGV); - } - - addr->selector = FPU_DS; /* Default, for 32 bit non-segmented mode. */ - - mod = (FPU_modrm >> 6) & 3; - - if (rm == 4 && mod != 3) { - address = sib(mod, fpu_eip); - } else { - cpu_reg_ptr = ®_(rm); - switch (mod) { - case 0: - if (rm == 5) { - /* Special case: disp32 */ - RE_ENTRANT_CHECK_OFF; - FPU_code_access_ok(4); - FPU_get_user(address, - (unsigned long __user - *)(*fpu_eip)); - (*fpu_eip) += 4; - RE_ENTRANT_CHECK_ON; - addr->offset = address; - return (void __user *)address; - } else { - address = *cpu_reg_ptr; /* Just return the contents - of the cpu register */ - addr->offset = address; - return (void __user *)address; - } - case 1: - /* 8 bit signed displacement */ - RE_ENTRANT_CHECK_OFF; - FPU_code_access_ok(1); - FPU_get_user(address, (signed char __user *)(*fpu_eip)); - RE_ENTRANT_CHECK_ON; - (*fpu_eip)++; - break; - case 2: - /* 32 bit displacement */ - RE_ENTRANT_CHECK_OFF; - FPU_code_access_ok(4); - FPU_get_user(address, (long __user *)(*fpu_eip)); - (*fpu_eip) += 4; - RE_ENTRANT_CHECK_ON; - break; - case 3: - /* Not legal for the FPU */ - EXCEPTION(EX_Invalid); - } - address += *cpu_reg_ptr; - } - - addr->offset = address; - - switch (addr_modes.default_mode) { - case 0: - break; - case VM86: - address += vm86_segment(addr_modes.override.segment, addr); - break; - case PM16: - case SEG32: - address = pm_address(FPU_modrm, addr_modes.override.segment, - addr, address); - break; - default: - EXCEPTION(EX_INTERNAL | 0x133); - } - - return (void __user *)address; -} - -void __user *FPU_get_address_16(u_char FPU_modrm, unsigned long *fpu_eip, - struct address *addr, fpu_addr_modes addr_modes) -{ - u_char mod; - unsigned rm = FPU_modrm & 7; - int address = 0; /* Default used for mod == 0 */ - - /* Memory accessed via the cs selector is write protected - in `non-segmented' 32 bit protected mode. */ - if (!addr_modes.default_mode && (FPU_modrm & FPU_WRITE_BIT) - && (addr_modes.override.segment == PREFIX_CS_)) { - math_abort(FPU_info, SIGSEGV); - } - - addr->selector = FPU_DS; /* Default, for 32 bit non-segmented mode. */ - - mod = (FPU_modrm >> 6) & 3; - - switch (mod) { - case 0: - if (rm == 6) { - /* Special case: disp16 */ - RE_ENTRANT_CHECK_OFF; - FPU_code_access_ok(2); - FPU_get_user(address, - (unsigned short __user *)(*fpu_eip)); - (*fpu_eip) += 2; - RE_ENTRANT_CHECK_ON; - goto add_segment; - } - break; - case 1: - /* 8 bit signed displacement */ - RE_ENTRANT_CHECK_OFF; - FPU_code_access_ok(1); - FPU_get_user(address, (signed char __user *)(*fpu_eip)); - RE_ENTRANT_CHECK_ON; - (*fpu_eip)++; - break; - case 2: - /* 16 bit displacement */ - RE_ENTRANT_CHECK_OFF; - FPU_code_access_ok(2); - FPU_get_user(address, (unsigned short __user *)(*fpu_eip)); - (*fpu_eip) += 2; - RE_ENTRANT_CHECK_ON; - break; - case 3: - /* Not legal for the FPU */ - EXCEPTION(EX_Invalid); - break; - } - switch (rm) { - case 0: - address += FPU_info->regs->bx + FPU_info->regs->si; - break; - case 1: - address += FPU_info->regs->bx + FPU_info->regs->di; - break; - case 2: - address += FPU_info->regs->bp + FPU_info->regs->si; - if (addr_modes.override.segment == PREFIX_DEFAULT) - addr_modes.override.segment = PREFIX_SS_; - break; - case 3: - address += FPU_info->regs->bp + FPU_info->regs->di; - if (addr_modes.override.segment == PREFIX_DEFAULT) - addr_modes.override.segment = PREFIX_SS_; - break; - case 4: - address += FPU_info->regs->si; - break; - case 5: - address += FPU_info->regs->di; - break; - case 6: - address += FPU_info->regs->bp; - if (addr_modes.override.segment == PREFIX_DEFAULT) - addr_modes.override.segment = PREFIX_SS_; - break; - case 7: - address += FPU_info->regs->bx; - break; - } - - add_segment: - address &= 0xffff; - - addr->offset = address; - - switch (addr_modes.default_mode) { - case 0: - break; - case VM86: - address += vm86_segment(addr_modes.override.segment, addr); - break; - case PM16: - case SEG32: - address = pm_address(FPU_modrm, addr_modes.override.segment, - addr, address); - break; - default: - EXCEPTION(EX_INTERNAL | 0x131); - } - - return (void __user *)address; -} diff --git a/arch/x86/math-emu/load_store.c b/arch/x86/math-emu/load_store.c deleted file mode 100644 index 4092df79de4f..000000000000 --- a/arch/x86/math-emu/load_store.c +++ /dev/null @@ -1,322 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/*---------------------------------------------------------------------------+ - | load_store.c | - | | - | This file contains most of the code to interpret the FPU instructions | - | which load and store from user memory. | - | | - | Copyright (C) 1992,1993,1994,1997 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, | - | Australia. E-mail billm@suburbia.net | - | | - | | - +---------------------------------------------------------------------------*/ - -/*---------------------------------------------------------------------------+ - | Note: | - | The file contains code which accesses user memory. | - | Emulator static data may change when user memory is accessed, due to | - | other processes using the emulator while swapping is in progress. | - +---------------------------------------------------------------------------*/ - -#include - -#include "fpu_system.h" -#include "exception.h" -#include "fpu_emu.h" -#include "status_w.h" -#include "control_w.h" - -#define _NONE_ 0 /* st0_ptr etc not needed */ -#define _REG0_ 1 /* Will be storing st(0) */ -#define _PUSH_ 3 /* Need to check for space to push onto stack */ -#define _null_ 4 /* Function illegal or not implemented */ - -#define pop_0() { FPU_settag0(TAG_Empty); top++; } - -/* index is a 5-bit value: (3-bit FPU_modrm.reg field | opcode[2,1]) */ -static u_char const type_table[32] = { - _PUSH_, _PUSH_, _PUSH_, _PUSH_, /* /0: d9:fld f32, db:fild m32, dd:fld f64, df:fild m16 */ - _null_, _REG0_, _REG0_, _REG0_, /* /1: d9:undef, db,dd,df:fisttp m32/64/16 */ - _REG0_, _REG0_, _REG0_, _REG0_, /* /2: d9:fst f32, db:fist m32, dd:fst f64, df:fist m16 */ - _REG0_, _REG0_, _REG0_, _REG0_, /* /3: d9:fstp f32, db:fistp m32, dd:fstp f64, df:fistp m16 */ - _NONE_, _null_, _NONE_, _PUSH_, - _NONE_, _PUSH_, _null_, _PUSH_, - _NONE_, _null_, _NONE_, _REG0_, - _NONE_, _REG0_, _NONE_, _REG0_ -}; - -u_char const data_sizes_16[32] = { - 4, 4, 8, 2, - 0, 4, 8, 2, /* /1: d9:undef, db,dd,df:fisttp */ - 4, 4, 8, 2, - 4, 4, 8, 2, - 14, 0, 94, 10, 2, 10, 0, 8, - 14, 0, 94, 10, 2, 10, 2, 8 -}; - -static u_char const data_sizes_32[32] = { - 4, 4, 8, 2, - 0, 4, 8, 2, /* /1: d9:undef, db,dd,df:fisttp */ - 4, 4, 8, 2, - 4, 4, 8, 2, - 28, 0, 108, 10, 2, 10, 0, 8, - 28, 0, 108, 10, 2, 10, 2, 8 -}; - -int FPU_load_store(u_char type, fpu_addr_modes addr_modes, - void __user * data_address) -{ - FPU_REG loaded_data; - FPU_REG *st0_ptr; - u_char st0_tag = TAG_Empty; /* This is just to stop a gcc warning. */ - u_char loaded_tag; - int sv_cw; - - st0_ptr = NULL; /* Initialized just to stop compiler warnings. */ - - if (addr_modes.default_mode & PROTECTED) { - if (addr_modes.default_mode == SEG32) { - if (access_limit < data_sizes_32[type]) - math_abort(FPU_info, SIGSEGV); - } else if (addr_modes.default_mode == PM16) { - if (access_limit < data_sizes_16[type]) - math_abort(FPU_info, SIGSEGV); - } -#ifdef PARANOID - else - EXCEPTION(EX_INTERNAL | 0x140); -#endif /* PARANOID */ - } - - switch (type_table[type]) { - case _NONE_: - break; - case _REG0_: - st0_ptr = &st(0); /* Some of these instructions pop after - storing */ - st0_tag = FPU_gettag0(); - break; - case _PUSH_: - { - if (FPU_gettagi(-1) != TAG_Empty) { - FPU_stack_overflow(); - return 0; - } - top--; - st0_ptr = &st(0); - } - break; - case _null_: - FPU_illegal(); - return 0; -#ifdef PARANOID - default: - EXCEPTION(EX_INTERNAL | 0x141); - return 0; -#endif /* PARANOID */ - } - - switch (type) { - /* type is a 5-bit value: (3-bit FPU_modrm.reg field | opcode[2,1]) */ - case 000: /* fld m32real (d9 /0) */ - clear_C1(); - loaded_tag = - FPU_load_single((float __user *)data_address, &loaded_data); - if ((loaded_tag == TAG_Special) - && isNaN(&loaded_data) - && (real_1op_NaN(&loaded_data) < 0)) { - top++; - break; - } - FPU_copy_to_reg0(&loaded_data, loaded_tag); - break; - case 001: /* fild m32int (db /0) */ - clear_C1(); - loaded_tag = - FPU_load_int32((long __user *)data_address, &loaded_data); - FPU_copy_to_reg0(&loaded_data, loaded_tag); - break; - case 002: /* fld m64real (dd /0) */ - clear_C1(); - loaded_tag = - FPU_load_double((double __user *)data_address, - &loaded_data); - if ((loaded_tag == TAG_Special) - && isNaN(&loaded_data) - && (real_1op_NaN(&loaded_data) < 0)) { - top++; - break; - } - FPU_copy_to_reg0(&loaded_data, loaded_tag); - break; - case 003: /* fild m16int (df /0) */ - clear_C1(); - loaded_tag = - FPU_load_int16((short __user *)data_address, &loaded_data); - FPU_copy_to_reg0(&loaded_data, loaded_tag); - break; - /* case 004: undefined (d9 /1) */ - /* fisttp are enabled if CPUID(1).ECX(0) "sse3" is set */ - case 005: /* fisttp m32int (db /1) */ - clear_C1(); - sv_cw = control_word; - control_word |= RC_CHOP; - if (FPU_store_int32 - (st0_ptr, st0_tag, (long __user *)data_address)) - pop_0(); /* pop only if the number was actually stored - (see the 80486 manual p16-28) */ - control_word = sv_cw; - break; - case 006: /* fisttp m64int (dd /1) */ - clear_C1(); - sv_cw = control_word; - control_word |= RC_CHOP; - if (FPU_store_int64 - (st0_ptr, st0_tag, (long long __user *)data_address)) - pop_0(); /* pop only if the number was actually stored - (see the 80486 manual p16-28) */ - control_word = sv_cw; - break; - case 007: /* fisttp m16int (df /1) */ - clear_C1(); - sv_cw = control_word; - control_word |= RC_CHOP; - if (FPU_store_int16 - (st0_ptr, st0_tag, (short __user *)data_address)) - pop_0(); /* pop only if the number was actually stored - (see the 80486 manual p16-28) */ - control_word = sv_cw; - break; - case 010: /* fst m32real */ - clear_C1(); - FPU_store_single(st0_ptr, st0_tag, - (float __user *)data_address); - break; - case 011: /* fist m32int */ - clear_C1(); - FPU_store_int32(st0_ptr, st0_tag, (long __user *)data_address); - break; - case 012: /* fst m64real */ - clear_C1(); - FPU_store_double(st0_ptr, st0_tag, - (double __user *)data_address); - break; - case 013: /* fist m16int */ - clear_C1(); - FPU_store_int16(st0_ptr, st0_tag, (short __user *)data_address); - break; - case 014: /* fstp m32real */ - clear_C1(); - if (FPU_store_single - (st0_ptr, st0_tag, (float __user *)data_address)) - pop_0(); /* pop only if the number was actually stored - (see the 80486 manual p16-28) */ - break; - case 015: /* fistp m32int */ - clear_C1(); - if (FPU_store_int32 - (st0_ptr, st0_tag, (long __user *)data_address)) - pop_0(); /* pop only if the number was actually stored - (see the 80486 manual p16-28) */ - break; - case 016: /* fstp m64real */ - clear_C1(); - if (FPU_store_double - (st0_ptr, st0_tag, (double __user *)data_address)) - pop_0(); /* pop only if the number was actually stored - (see the 80486 manual p16-28) */ - break; - case 017: /* fistp m16int */ - clear_C1(); - if (FPU_store_int16 - (st0_ptr, st0_tag, (short __user *)data_address)) - pop_0(); /* pop only if the number was actually stored - (see the 80486 manual p16-28) */ - break; - case 020: /* fldenv m14/28byte */ - fldenv(addr_modes, (u_char __user *) data_address); - /* Ensure that the values just loaded are not changed by - fix-up operations. */ - return 1; - case 022: /* frstor m94/108byte */ - FPU_frstor(addr_modes, (u_char __user *) data_address); - /* Ensure that the values just loaded are not changed by - fix-up operations. */ - return 1; - case 023: /* fbld m80dec */ - clear_C1(); - loaded_tag = FPU_load_bcd((u_char __user *) data_address); - FPU_settag0(loaded_tag); - break; - case 024: /* fldcw */ - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(data_address, 2); - FPU_get_user(control_word, - (unsigned short __user *)data_address); - RE_ENTRANT_CHECK_ON; - if (partial_status & ~control_word & CW_Exceptions) - partial_status |= (SW_Summary | SW_Backward); - else - partial_status &= ~(SW_Summary | SW_Backward); -#ifdef PECULIAR_486 - control_word |= 0x40; /* An 80486 appears to always set this bit */ -#endif /* PECULIAR_486 */ - return 1; - case 025: /* fld m80real */ - clear_C1(); - loaded_tag = - FPU_load_extended((long double __user *)data_address, 0); - FPU_settag0(loaded_tag); - break; - case 027: /* fild m64int */ - clear_C1(); - loaded_tag = FPU_load_int64((long long __user *)data_address); - if (loaded_tag == TAG_Error) - return 0; - FPU_settag0(loaded_tag); - break; - case 030: /* fstenv m14/28byte */ - fstenv(addr_modes, (u_char __user *) data_address); - return 1; - case 032: /* fsave */ - fsave(addr_modes, (u_char __user *) data_address); - return 1; - case 033: /* fbstp m80dec */ - clear_C1(); - if (FPU_store_bcd - (st0_ptr, st0_tag, (u_char __user *) data_address)) - pop_0(); /* pop only if the number was actually stored - (see the 80486 manual p16-28) */ - break; - case 034: /* fstcw m16int */ - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(data_address, 2); - FPU_put_user(control_word, - (unsigned short __user *)data_address); - RE_ENTRANT_CHECK_ON; - return 1; - case 035: /* fstp m80real */ - clear_C1(); - if (FPU_store_extended - (st0_ptr, st0_tag, (long double __user *)data_address)) - pop_0(); /* pop only if the number was actually stored - (see the 80486 manual p16-28) */ - break; - case 036: /* fstsw m2byte */ - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(data_address, 2); - FPU_put_user(status_word(), - (unsigned short __user *)data_address); - RE_ENTRANT_CHECK_ON; - return 1; - case 037: /* fistp m64int */ - clear_C1(); - if (FPU_store_int64 - (st0_ptr, st0_tag, (long long __user *)data_address)) - pop_0(); /* pop only if the number was actually stored - (see the 80486 manual p16-28) */ - break; - } - return 0; -} diff --git a/arch/x86/math-emu/mul_Xsig.S b/arch/x86/math-emu/mul_Xsig.S deleted file mode 100644 index 54a031b66142..000000000000 --- a/arch/x86/math-emu/mul_Xsig.S +++ /dev/null @@ -1,179 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/*---------------------------------------------------------------------------+ - | mul_Xsig.S | - | | - | Multiply a 12 byte fixed point number by another fixed point number. | - | | - | Copyright (C) 1992,1994,1995 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, | - | Australia. E-mail billm@jacobi.maths.monash.edu.au | - | | - | Call from C as: | - | void mul32_Xsig(Xsig *x, unsigned b) | - | | - | void mul64_Xsig(Xsig *x, unsigned long long *b) | - | | - | void mul_Xsig_Xsig(Xsig *x, unsigned *b) | - | | - | The result is neither rounded nor normalized, and the ls bit or so may | - | be wrong. | - | | - +---------------------------------------------------------------------------*/ - .file "mul_Xsig.S" - - -#include "fpu_emu.h" - -.text -SYM_FUNC_START(mul32_Xsig) - pushl %ebp - movl %esp,%ebp - subl $16,%esp - pushl %esi - - movl PARAM1,%esi - movl PARAM2,%ecx - - xor %eax,%eax - movl %eax,-4(%ebp) - movl %eax,-8(%ebp) - - movl (%esi),%eax /* lsl of Xsig */ - mull %ecx /* msl of b */ - movl %edx,-12(%ebp) - - movl 4(%esi),%eax /* midl of Xsig */ - mull %ecx /* msl of b */ - addl %eax,-12(%ebp) - adcl %edx,-8(%ebp) - adcl $0,-4(%ebp) - - movl 8(%esi),%eax /* msl of Xsig */ - mull %ecx /* msl of b */ - addl %eax,-8(%ebp) - adcl %edx,-4(%ebp) - - movl -12(%ebp),%eax - movl %eax,(%esi) - movl -8(%ebp),%eax - movl %eax,4(%esi) - movl -4(%ebp),%eax - movl %eax,8(%esi) - - popl %esi - leave - RET -SYM_FUNC_END(mul32_Xsig) - - -SYM_FUNC_START(mul64_Xsig) - pushl %ebp - movl %esp,%ebp - subl $16,%esp - pushl %esi - - movl PARAM1,%esi - movl PARAM2,%ecx - - xor %eax,%eax - movl %eax,-4(%ebp) - movl %eax,-8(%ebp) - - movl (%esi),%eax /* lsl of Xsig */ - mull 4(%ecx) /* msl of b */ - movl %edx,-12(%ebp) - - movl 4(%esi),%eax /* midl of Xsig */ - mull (%ecx) /* lsl of b */ - addl %edx,-12(%ebp) - adcl $0,-8(%ebp) - adcl $0,-4(%ebp) - - movl 4(%esi),%eax /* midl of Xsig */ - mull 4(%ecx) /* msl of b */ - addl %eax,-12(%ebp) - adcl %edx,-8(%ebp) - adcl $0,-4(%ebp) - - movl 8(%esi),%eax /* msl of Xsig */ - mull (%ecx) /* lsl of b */ - addl %eax,-12(%ebp) - adcl %edx,-8(%ebp) - adcl $0,-4(%ebp) - - movl 8(%esi),%eax /* msl of Xsig */ - mull 4(%ecx) /* msl of b */ - addl %eax,-8(%ebp) - adcl %edx,-4(%ebp) - - movl -12(%ebp),%eax - movl %eax,(%esi) - movl -8(%ebp),%eax - movl %eax,4(%esi) - movl -4(%ebp),%eax - movl %eax,8(%esi) - - popl %esi - leave - RET -SYM_FUNC_END(mul64_Xsig) - - - -SYM_FUNC_START(mul_Xsig_Xsig) - pushl %ebp - movl %esp,%ebp - subl $16,%esp - pushl %esi - - movl PARAM1,%esi - movl PARAM2,%ecx - - xor %eax,%eax - movl %eax,-4(%ebp) - movl %eax,-8(%ebp) - - movl (%esi),%eax /* lsl of Xsig */ - mull 8(%ecx) /* msl of b */ - movl %edx,-12(%ebp) - - movl 4(%esi),%eax /* midl of Xsig */ - mull 4(%ecx) /* midl of b */ - addl %edx,-12(%ebp) - adcl $0,-8(%ebp) - adcl $0,-4(%ebp) - - movl 8(%esi),%eax /* msl of Xsig */ - mull (%ecx) /* lsl of b */ - addl %edx,-12(%ebp) - adcl $0,-8(%ebp) - adcl $0,-4(%ebp) - - movl 4(%esi),%eax /* midl of Xsig */ - mull 8(%ecx) /* msl of b */ - addl %eax,-12(%ebp) - adcl %edx,-8(%ebp) - adcl $0,-4(%ebp) - - movl 8(%esi),%eax /* msl of Xsig */ - mull 4(%ecx) /* midl of b */ - addl %eax,-12(%ebp) - adcl %edx,-8(%ebp) - adcl $0,-4(%ebp) - - movl 8(%esi),%eax /* msl of Xsig */ - mull 8(%ecx) /* msl of b */ - addl %eax,-8(%ebp) - adcl %edx,-4(%ebp) - - movl -12(%ebp),%edx - movl %edx,(%esi) - movl -8(%ebp),%edx - movl %edx,4(%esi) - movl -4(%ebp),%edx - movl %edx,8(%esi) - - popl %esi - leave - RET -SYM_FUNC_END(mul_Xsig_Xsig) diff --git a/arch/x86/math-emu/poly.h b/arch/x86/math-emu/poly.h deleted file mode 100644 index 654bfe4e29a0..000000000000 --- a/arch/x86/math-emu/poly.h +++ /dev/null @@ -1,115 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/*---------------------------------------------------------------------------+ - | poly.h | - | | - | Header file for the FPU-emu poly*.c source files. | - | | - | Copyright (C) 1994,1999 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, | - | Australia. E-mail billm@melbpc.org.au | - | | - | Declarations and definitions for functions operating on Xsig (12-byte | - | extended-significand) quantities. | - | | - +---------------------------------------------------------------------------*/ - -#ifndef _POLY_H -#define _POLY_H - -/* This 12-byte structure is used to improve the accuracy of computation - of transcendental functions. - Intended to be used to get results better than 8-byte computation - allows. 9-byte would probably be sufficient. - */ -typedef struct { - unsigned long lsw; - unsigned long midw; - unsigned long msw; -} Xsig; - -asmlinkage void mul64(unsigned long long const *a, unsigned long long const *b, - unsigned long long *result); -asmlinkage void polynomial_Xsig(Xsig *, const unsigned long long *x, - const unsigned long long terms[], const int n); - -asmlinkage void mul32_Xsig(Xsig *, const unsigned long mult); -asmlinkage void mul64_Xsig(Xsig *, const unsigned long long *mult); -asmlinkage void mul_Xsig_Xsig(Xsig *dest, const Xsig *mult); - -asmlinkage void shr_Xsig(Xsig *, const int n); -asmlinkage int round_Xsig(Xsig *); -asmlinkage int norm_Xsig(Xsig *); -asmlinkage void div_Xsig(Xsig *x1, const Xsig *x2, Xsig *dest); - -/* Macro to extract the most significant 32 bits from a long long */ -#define LL_MSW(x) (((unsigned long *)&x)[1]) - -/* Macro to initialize an Xsig struct */ -#define MK_XSIG(a,b,c) { c, b, a } - -/* Macro to access the 8 ms bytes of an Xsig as a long long */ -#define XSIG_LL(x) (*(unsigned long long *)&x.midw) - -/* - Need to run gcc with optimizations on to get these to - actually be in-line. - */ - -/* Multiply two fixed-point 32 bit numbers, producing a 32 bit result. - The answer is the ms word of the product. */ -/* Some versions of gcc make it difficult to stop eax from being clobbered. - Merely specifying that it is used doesn't work... - */ -static inline unsigned long mul_32_32(const unsigned long arg1, - const unsigned long arg2) -{ - int retval; - asm volatile ("mull %2; movl %%edx,%%eax":"=a" (retval) - :"0"(arg1), "g"(arg2) - :"dx"); - return retval; -} - -/* Add the 12 byte Xsig x2 to Xsig dest, with no checks for overflow. */ -static inline void add_Xsig_Xsig(Xsig *dest, const Xsig *x2) -{ - asm volatile ("movl %1,%%edi; movl %2,%%esi;\n" - "movl (%%esi),%%eax; addl %%eax,(%%edi);\n" - "movl 4(%%esi),%%eax; adcl %%eax,4(%%edi);\n" - "movl 8(%%esi),%%eax; adcl %%eax,8(%%edi);\n":"=g" - (*dest):"g"(dest), "g"(x2) - :"ax", "si", "di"); -} - -/* Add the 12 byte Xsig x2 to Xsig dest, adjust exp if overflow occurs. */ -/* Note: the constraints in the asm statement didn't always work properly - with gcc 2.5.8. Changing from using edi to using ecx got around the - problem, but keep fingers crossed! */ -static inline void add_two_Xsig(Xsig *dest, const Xsig *x2, long int *exp) -{ - asm volatile ("movl %2,%%ecx; movl %3,%%esi;\n" - "movl (%%esi),%%eax; addl %%eax,(%%ecx);\n" - "movl 4(%%esi),%%eax; adcl %%eax,4(%%ecx);\n" - "movl 8(%%esi),%%eax; adcl %%eax,8(%%ecx);\n" - "jnc 0f;\n" - "rcrl 8(%%ecx); rcrl 4(%%ecx); rcrl (%%ecx)\n" - "movl %4,%%ecx; incl (%%ecx)\n" - "movl $1,%%eax; jmp 1f;\n" - "0: xorl %%eax,%%eax;\n" "1:\n":"=g" (*exp), "=g"(*dest) - :"g"(dest), "g"(x2), "g"(exp) - :"cx", "si", "ax"); -} - -/* Negate (subtract from 1.0) the 12 byte Xsig */ -/* This is faster in a loop on my 386 than using the "neg" instruction. */ -static inline void negate_Xsig(Xsig *x) -{ - asm volatile ("movl %1,%%esi;\n" - "xorl %%ecx,%%ecx;\n" - "movl %%ecx,%%eax; subl (%%esi),%%eax; movl %%eax,(%%esi);\n" - "movl %%ecx,%%eax; sbbl 4(%%esi),%%eax; movl %%eax,4(%%esi);\n" - "movl %%ecx,%%eax; sbbl 8(%%esi),%%eax; movl %%eax,8(%%esi);\n":"=g" - (*x):"g"(x):"si", "ax", "cx"); -} - -#endif /* _POLY_H */ diff --git a/arch/x86/math-emu/poly_2xm1.c b/arch/x86/math-emu/poly_2xm1.c deleted file mode 100644 index aa33006bafd5..000000000000 --- a/arch/x86/math-emu/poly_2xm1.c +++ /dev/null @@ -1,146 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/*---------------------------------------------------------------------------+ - | poly_2xm1.c | - | | - | Function to compute 2^x-1 by a polynomial approximation. | - | | - | Copyright (C) 1992,1993,1994,1997 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia | - | E-mail billm@suburbia.net | - | | - | | - +---------------------------------------------------------------------------*/ - -#include "exception.h" -#include "reg_constant.h" -#include "fpu_emu.h" -#include "fpu_system.h" -#include "control_w.h" -#include "poly.h" - -#define HIPOWER 11 -static const unsigned long long lterms[HIPOWER] = { - 0x0000000000000000LL, /* This term done separately as 12 bytes */ - 0xf5fdeffc162c7543LL, - 0x1c6b08d704a0bfa6LL, - 0x0276556df749cc21LL, - 0x002bb0ffcf14f6b8LL, - 0x0002861225ef751cLL, - 0x00001ffcbfcd5422LL, - 0x00000162c005d5f1LL, - 0x0000000da96ccb1bLL, - 0x0000000078d1b897LL, - 0x000000000422b029LL -}; - -static const Xsig hiterm = MK_XSIG(0xb17217f7, 0xd1cf79ab, 0xc8a39194); - -/* Four slices: 0.0 : 0.25 : 0.50 : 0.75 : 1.0, - These numbers are 2^(1/4), 2^(1/2), and 2^(3/4) - */ -static const Xsig shiftterm0 = MK_XSIG(0, 0, 0); -static const Xsig shiftterm1 = MK_XSIG(0x9837f051, 0x8db8a96f, 0x46ad2318); -static const Xsig shiftterm2 = MK_XSIG(0xb504f333, 0xf9de6484, 0x597d89b3); -static const Xsig shiftterm3 = MK_XSIG(0xd744fcca, 0xd69d6af4, 0x39a68bb9); - -static const Xsig *shiftterm[] = { &shiftterm0, &shiftterm1, - &shiftterm2, &shiftterm3 -}; - -/*--- poly_2xm1() -----------------------------------------------------------+ - | Requires st(0) which is TAG_Valid and < 1. | - +---------------------------------------------------------------------------*/ -int poly_2xm1(u_char sign, FPU_REG *arg, FPU_REG *result) -{ - long int exponent, shift; - unsigned long long Xll; - Xsig accumulator, Denom, argSignif; - u_char tag; - - exponent = exponent16(arg); - -#ifdef PARANOID - if (exponent >= 0) { /* Don't want a |number| >= 1.0 */ - /* Number negative, too large, or not Valid. */ - EXCEPTION(EX_INTERNAL | 0x127); - return 1; - } -#endif /* PARANOID */ - - argSignif.lsw = 0; - XSIG_LL(argSignif) = Xll = significand(arg); - - if (exponent == -1) { - shift = (argSignif.msw & 0x40000000) ? 3 : 2; - /* subtract 0.5 or 0.75 */ - exponent -= 2; - XSIG_LL(argSignif) <<= 2; - Xll <<= 2; - } else if (exponent == -2) { - shift = 1; - /* subtract 0.25 */ - exponent--; - XSIG_LL(argSignif) <<= 1; - Xll <<= 1; - } else - shift = 0; - - if (exponent < -2) { - /* Shift the argument right by the required places. */ - if (FPU_shrx(&Xll, -2 - exponent) >= 0x80000000U) - Xll++; /* round up */ - } - - accumulator.lsw = accumulator.midw = accumulator.msw = 0; - polynomial_Xsig(&accumulator, &Xll, lterms, HIPOWER - 1); - mul_Xsig_Xsig(&accumulator, &argSignif); - shr_Xsig(&accumulator, 3); - - mul_Xsig_Xsig(&argSignif, &hiterm); /* The leading term */ - add_two_Xsig(&accumulator, &argSignif, &exponent); - - if (shift) { - /* The argument is large, use the identity: - f(x+a) = f(a) * (f(x) + 1) - 1; - */ - shr_Xsig(&accumulator, -exponent); - accumulator.msw |= 0x80000000; /* add 1.0 */ - mul_Xsig_Xsig(&accumulator, shiftterm[shift]); - accumulator.msw &= 0x3fffffff; /* subtract 1.0 */ - exponent = 1; - } - - if (sign != SIGN_POS) { - /* The argument is negative, use the identity: - f(-x) = -f(x) / (1 + f(x)) - */ - Denom.lsw = accumulator.lsw; - XSIG_LL(Denom) = XSIG_LL(accumulator); - if (exponent < 0) - shr_Xsig(&Denom, -exponent); - else if (exponent > 0) { - /* exponent must be 1 here */ - XSIG_LL(Denom) <<= 1; - if (Denom.lsw & 0x80000000) - XSIG_LL(Denom) |= 1; - (Denom.lsw) <<= 1; - } - Denom.msw |= 0x80000000; /* add 1.0 */ - div_Xsig(&accumulator, &Denom, &accumulator); - } - - /* Convert to 64 bit signed-compatible */ - exponent += round_Xsig(&accumulator); - - result = &st(0); - significand(result) = XSIG_LL(accumulator); - setexponent16(result, exponent); - - tag = FPU_round(result, 1, 0, FULL_PRECISION, sign); - - setsign(result, sign); - FPU_settag0(tag); - - return 0; - -} diff --git a/arch/x86/math-emu/poly_atan.c b/arch/x86/math-emu/poly_atan.c deleted file mode 100644 index 7e7412c5a1fe..000000000000 --- a/arch/x86/math-emu/poly_atan.c +++ /dev/null @@ -1,209 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/*---------------------------------------------------------------------------+ - | poly_atan.c | - | | - | Compute the arctan of a FPU_REG, using a polynomial approximation. | - | | - | Copyright (C) 1992,1993,1994,1997 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia | - | E-mail billm@suburbia.net | - | | - | | - +---------------------------------------------------------------------------*/ - -#include "exception.h" -#include "reg_constant.h" -#include "fpu_emu.h" -#include "fpu_system.h" -#include "status_w.h" -#include "control_w.h" -#include "poly.h" - -#define HIPOWERon 6 /* odd poly, negative terms */ -static const unsigned long long oddnegterms[HIPOWERon] = { - 0x0000000000000000LL, /* Dummy (not for - 1.0) */ - 0x015328437f756467LL, - 0x0005dda27b73dec6LL, - 0x0000226bf2bfb91aLL, - 0x000000ccc439c5f7LL, - 0x0000000355438407LL -}; - -#define HIPOWERop 6 /* odd poly, positive terms */ -static const unsigned long long oddplterms[HIPOWERop] = { -/* 0xaaaaaaaaaaaaaaabLL, transferred to fixedpterm[] */ - 0x0db55a71875c9ac2LL, - 0x0029fce2d67880b0LL, - 0x0000dfd3908b4596LL, - 0x00000550fd61dab4LL, - 0x0000001c9422b3f9LL, - 0x000000003e3301e1LL -}; - -static const unsigned long long denomterm = 0xebd9b842c5c53a0eLL; - -static const Xsig fixedpterm = MK_XSIG(0xaaaaaaaa, 0xaaaaaaaa, 0xaaaaaaaa); - -static const Xsig pi_signif = MK_XSIG(0xc90fdaa2, 0x2168c234, 0xc4c6628b); - -/*--- poly_atan() -----------------------------------------------------------+ - | | - +---------------------------------------------------------------------------*/ -void poly_atan(FPU_REG *st0_ptr, u_char st0_tag, - FPU_REG *st1_ptr, u_char st1_tag) -{ - u_char transformed, inverted, sign1, sign2; - int exponent; - long int dummy_exp; - Xsig accumulator, Numer, Denom, accumulatore, argSignif, argSq, argSqSq; - u_char tag; - - sign1 = getsign(st0_ptr); - sign2 = getsign(st1_ptr); - if (st0_tag == TAG_Valid) { - exponent = exponent(st0_ptr); - } else { - /* This gives non-compatible stack contents... */ - FPU_to_exp16(st0_ptr, st0_ptr); - exponent = exponent16(st0_ptr); - } - if (st1_tag == TAG_Valid) { - exponent -= exponent(st1_ptr); - } else { - /* This gives non-compatible stack contents... */ - FPU_to_exp16(st1_ptr, st1_ptr); - exponent -= exponent16(st1_ptr); - } - - if ((exponent < 0) || ((exponent == 0) && - ((st0_ptr->sigh < st1_ptr->sigh) || - ((st0_ptr->sigh == st1_ptr->sigh) && - (st0_ptr->sigl < st1_ptr->sigl))))) { - inverted = 1; - Numer.lsw = Denom.lsw = 0; - XSIG_LL(Numer) = significand(st0_ptr); - XSIG_LL(Denom) = significand(st1_ptr); - } else { - inverted = 0; - exponent = -exponent; - Numer.lsw = Denom.lsw = 0; - XSIG_LL(Numer) = significand(st1_ptr); - XSIG_LL(Denom) = significand(st0_ptr); - } - div_Xsig(&Numer, &Denom, &argSignif); - exponent += norm_Xsig(&argSignif); - - if ((exponent >= -1) - || ((exponent == -2) && (argSignif.msw > 0xd413ccd0))) { - /* The argument is greater than sqrt(2)-1 (=0.414213562...) */ - /* Convert the argument by an identity for atan */ - transformed = 1; - - if (exponent >= 0) { -#ifdef PARANOID - if (!((exponent == 0) && - (argSignif.lsw == 0) && (argSignif.midw == 0) && - (argSignif.msw == 0x80000000))) { - EXCEPTION(EX_INTERNAL | 0x104); /* There must be a logic error */ - return; - } -#endif /* PARANOID */ - argSignif.msw = 0; /* Make the transformed arg -> 0.0 */ - } else { - Numer.lsw = Denom.lsw = argSignif.lsw; - XSIG_LL(Numer) = XSIG_LL(Denom) = XSIG_LL(argSignif); - - if (exponent < -1) - shr_Xsig(&Numer, -1 - exponent); - negate_Xsig(&Numer); - - shr_Xsig(&Denom, -exponent); - Denom.msw |= 0x80000000; - - div_Xsig(&Numer, &Denom, &argSignif); - - exponent = -1 + norm_Xsig(&argSignif); - } - } else { - transformed = 0; - } - - argSq.lsw = argSignif.lsw; - argSq.midw = argSignif.midw; - argSq.msw = argSignif.msw; - mul_Xsig_Xsig(&argSq, &argSq); - - argSqSq.lsw = argSq.lsw; - argSqSq.midw = argSq.midw; - argSqSq.msw = argSq.msw; - mul_Xsig_Xsig(&argSqSq, &argSqSq); - - accumulatore.lsw = argSq.lsw; - XSIG_LL(accumulatore) = XSIG_LL(argSq); - - shr_Xsig(&argSq, 2 * (-1 - exponent - 1)); - shr_Xsig(&argSqSq, 4 * (-1 - exponent - 1)); - - /* Now have argSq etc with binary point at the left - .1xxxxxxxx */ - - /* Do the basic fixed point polynomial evaluation */ - accumulator.msw = accumulator.midw = accumulator.lsw = 0; - polynomial_Xsig(&accumulator, &XSIG_LL(argSqSq), - oddplterms, HIPOWERop - 1); - mul64_Xsig(&accumulator, &XSIG_LL(argSq)); - negate_Xsig(&accumulator); - polynomial_Xsig(&accumulator, &XSIG_LL(argSqSq), oddnegterms, - HIPOWERon - 1); - negate_Xsig(&accumulator); - add_two_Xsig(&accumulator, &fixedpterm, &dummy_exp); - - mul64_Xsig(&accumulatore, &denomterm); - shr_Xsig(&accumulatore, 1 + 2 * (-1 - exponent)); - accumulatore.msw |= 0x80000000; - - div_Xsig(&accumulator, &accumulatore, &accumulator); - - mul_Xsig_Xsig(&accumulator, &argSignif); - mul_Xsig_Xsig(&accumulator, &argSq); - - shr_Xsig(&accumulator, 3); - negate_Xsig(&accumulator); - add_Xsig_Xsig(&accumulator, &argSignif); - - if (transformed) { - /* compute pi/4 - accumulator */ - shr_Xsig(&accumulator, -1 - exponent); - negate_Xsig(&accumulator); - add_Xsig_Xsig(&accumulator, &pi_signif); - exponent = -1; - } - - if (inverted) { - /* compute pi/2 - accumulator */ - shr_Xsig(&accumulator, -exponent); - negate_Xsig(&accumulator); - add_Xsig_Xsig(&accumulator, &pi_signif); - exponent = 0; - } - - if (sign1) { - /* compute pi - accumulator */ - shr_Xsig(&accumulator, 1 - exponent); - negate_Xsig(&accumulator); - add_Xsig_Xsig(&accumulator, &pi_signif); - exponent = 1; - } - - exponent += round_Xsig(&accumulator); - - significand(st1_ptr) = XSIG_LL(accumulator); - setexponent16(st1_ptr, exponent); - - tag = FPU_round(st1_ptr, 1, 0, FULL_PRECISION, sign2); - FPU_settagi(1, tag); - - set_precision_flag_up(); /* We do not really know if up or down, - use this as the default. */ - -} diff --git a/arch/x86/math-emu/poly_l2.c b/arch/x86/math-emu/poly_l2.c deleted file mode 100644 index 98b6949bb854..000000000000 --- a/arch/x86/math-emu/poly_l2.c +++ /dev/null @@ -1,245 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/*---------------------------------------------------------------------------+ - | poly_l2.c | - | | - | Compute the base 2 log of a FPU_REG, using a polynomial approximation. | - | | - | Copyright (C) 1992,1993,1994,1997 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia | - | E-mail billm@suburbia.net | - | | - | | - +---------------------------------------------------------------------------*/ - -#include "exception.h" -#include "reg_constant.h" -#include "fpu_emu.h" -#include "fpu_system.h" -#include "control_w.h" -#include "poly.h" - -static void log2_kernel(FPU_REG const *arg, u_char argsign, - Xsig * accum_result, long int *expon); - -/*--- poly_l2() -------------------------------------------------------------+ - | Base 2 logarithm by a polynomial approximation. | - +---------------------------------------------------------------------------*/ -void poly_l2(FPU_REG *st0_ptr, FPU_REG *st1_ptr, u_char st1_sign) -{ - long int exponent, expon, expon_expon; - Xsig accumulator, expon_accum, yaccum; - u_char sign, argsign; - FPU_REG x; - int tag; - - exponent = exponent16(st0_ptr); - - /* From st0_ptr, make a number > sqrt(2)/2 and < sqrt(2) */ - if (st0_ptr->sigh > (unsigned)0xb504f334) { - /* Treat as sqrt(2)/2 < st0_ptr < 1 */ - significand(&x) = -significand(st0_ptr); - setexponent16(&x, -1); - exponent++; - argsign = SIGN_NEG; - } else { - /* Treat as 1 <= st0_ptr < sqrt(2) */ - x.sigh = st0_ptr->sigh - 0x80000000; - x.sigl = st0_ptr->sigl; - setexponent16(&x, 0); - argsign = SIGN_POS; - } - tag = FPU_normalize_nuo(&x); - - if (tag == TAG_Zero) { - expon = 0; - accumulator.msw = accumulator.midw = accumulator.lsw = 0; - } else { - log2_kernel(&x, argsign, &accumulator, &expon); - } - - if (exponent < 0) { - sign = SIGN_NEG; - exponent = -exponent; - } else - sign = SIGN_POS; - expon_accum.msw = exponent; - expon_accum.midw = expon_accum.lsw = 0; - if (exponent) { - expon_expon = 31 + norm_Xsig(&expon_accum); - shr_Xsig(&accumulator, expon_expon - expon); - - if (sign ^ argsign) - negate_Xsig(&accumulator); - add_Xsig_Xsig(&accumulator, &expon_accum); - } else { - expon_expon = expon; - sign = argsign; - } - - yaccum.lsw = 0; - XSIG_LL(yaccum) = significand(st1_ptr); - mul_Xsig_Xsig(&accumulator, &yaccum); - - expon_expon += round_Xsig(&accumulator); - - if (accumulator.msw == 0) { - FPU_copy_to_reg1(&CONST_Z, TAG_Zero); - return; - } - - significand(st1_ptr) = XSIG_LL(accumulator); - setexponent16(st1_ptr, expon_expon + exponent16(st1_ptr) + 1); - - tag = FPU_round(st1_ptr, 1, 0, FULL_PRECISION, sign ^ st1_sign); - FPU_settagi(1, tag); - - set_precision_flag_up(); /* 80486 appears to always do this */ - - return; - -} - -/*--- poly_l2p1() -----------------------------------------------------------+ - | Base 2 logarithm by a polynomial approximation. | - | log2(x+1) | - +---------------------------------------------------------------------------*/ -int poly_l2p1(u_char sign0, u_char sign1, - FPU_REG * st0_ptr, FPU_REG * st1_ptr, FPU_REG * dest) -{ - u_char tag; - long int exponent; - Xsig accumulator, yaccum; - - if (exponent16(st0_ptr) < 0) { - log2_kernel(st0_ptr, sign0, &accumulator, &exponent); - - yaccum.lsw = 0; - XSIG_LL(yaccum) = significand(st1_ptr); - mul_Xsig_Xsig(&accumulator, &yaccum); - - exponent += round_Xsig(&accumulator); - - exponent += exponent16(st1_ptr) + 1; - if (exponent < EXP_WAY_UNDER) - exponent = EXP_WAY_UNDER; - - significand(dest) = XSIG_LL(accumulator); - setexponent16(dest, exponent); - - tag = FPU_round(dest, 1, 0, FULL_PRECISION, sign0 ^ sign1); - FPU_settagi(1, tag); - - if (tag == TAG_Valid) - set_precision_flag_up(); /* 80486 appears to always do this */ - } else { - /* The magnitude of st0_ptr is far too large. */ - - if (sign0 != SIGN_POS) { - /* Trying to get the log of a negative number. */ -#ifdef PECULIAR_486 /* Stupid 80486 doesn't worry about log(negative). */ - changesign(st1_ptr); -#else - if (arith_invalid(1) < 0) - return 1; -#endif /* PECULIAR_486 */ - } - - /* 80486 appears to do this */ - if (sign0 == SIGN_NEG) - set_precision_flag_down(); - else - set_precision_flag_up(); - } - - if (exponent(dest) <= EXP_UNDER) - EXCEPTION(EX_Underflow); - - return 0; - -} - -#undef HIPOWER -#define HIPOWER 10 -static const unsigned long long logterms[HIPOWER] = { - 0x2a8eca5705fc2ef0LL, - 0xf6384ee1d01febceLL, - 0x093bb62877cdf642LL, - 0x006985d8a9ec439bLL, - 0x0005212c4f55a9c8LL, - 0x00004326a16927f0LL, - 0x0000038d1d80a0e7LL, - 0x0000003141cc80c6LL, - 0x00000002b1668c9fLL, - 0x000000002c7a46aaLL -}; - -static const unsigned long leadterm = 0xb8000000; - -/*--- log2_kernel() ---------------------------------------------------------+ - | Base 2 logarithm by a polynomial approximation. | - | log2(x+1) | - +---------------------------------------------------------------------------*/ -static void log2_kernel(FPU_REG const *arg, u_char argsign, Xsig *accum_result, - long int *expon) -{ - long int exponent, adj; - unsigned long long Xsq; - Xsig accumulator, Numer, Denom, argSignif, arg_signif; - - exponent = exponent16(arg); - Numer.lsw = Denom.lsw = 0; - XSIG_LL(Numer) = XSIG_LL(Denom) = significand(arg); - if (argsign == SIGN_POS) { - shr_Xsig(&Denom, 2 - (1 + exponent)); - Denom.msw |= 0x80000000; - div_Xsig(&Numer, &Denom, &argSignif); - } else { - shr_Xsig(&Denom, 1 - (1 + exponent)); - negate_Xsig(&Denom); - if (Denom.msw & 0x80000000) { - div_Xsig(&Numer, &Denom, &argSignif); - exponent++; - } else { - /* Denom must be 1.0 */ - argSignif.lsw = Numer.lsw; - argSignif.midw = Numer.midw; - argSignif.msw = Numer.msw; - } - } - -#ifndef PECULIAR_486 - /* Should check here that |local_arg| is within the valid range */ - if (exponent >= -2) { - if ((exponent > -2) || (argSignif.msw > (unsigned)0xafb0ccc0)) { - /* The argument is too large */ - } - } -#endif /* PECULIAR_486 */ - - arg_signif.lsw = argSignif.lsw; - XSIG_LL(arg_signif) = XSIG_LL(argSignif); - adj = norm_Xsig(&argSignif); - accumulator.lsw = argSignif.lsw; - XSIG_LL(accumulator) = XSIG_LL(argSignif); - mul_Xsig_Xsig(&accumulator, &accumulator); - shr_Xsig(&accumulator, 2 * (-1 - (1 + exponent + adj))); - Xsq = XSIG_LL(accumulator); - if (accumulator.lsw & 0x80000000) - Xsq++; - - accumulator.msw = accumulator.midw = accumulator.lsw = 0; - /* Do the basic fixed point polynomial evaluation */ - polynomial_Xsig(&accumulator, &Xsq, logterms, HIPOWER - 1); - - mul_Xsig_Xsig(&accumulator, &argSignif); - shr_Xsig(&accumulator, 6 - adj); - - mul32_Xsig(&arg_signif, leadterm); - add_two_Xsig(&accumulator, &arg_signif, &exponent); - - *expon = exponent + 1; - accum_result->lsw = accumulator.lsw; - accum_result->midw = accumulator.midw; - accum_result->msw = accumulator.msw; - -} diff --git a/arch/x86/math-emu/poly_sin.c b/arch/x86/math-emu/poly_sin.c deleted file mode 100644 index c192fba51526..000000000000 --- a/arch/x86/math-emu/poly_sin.c +++ /dev/null @@ -1,379 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/*---------------------------------------------------------------------------+ - | poly_sin.c | - | | - | Computation of an approximation of the sin function and the cosine | - | function by a polynomial. | - | | - | Copyright (C) 1992,1993,1994,1997,1999 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia | - | E-mail billm@melbpc.org.au | - | | - | | - +---------------------------------------------------------------------------*/ - -#include "exception.h" -#include "reg_constant.h" -#include "fpu_emu.h" -#include "fpu_system.h" -#include "control_w.h" -#include "poly.h" - -#define N_COEFF_P 4 -#define N_COEFF_N 4 - -static const unsigned long long pos_terms_l[N_COEFF_P] = { - 0xaaaaaaaaaaaaaaabLL, - 0x00d00d00d00cf906LL, - 0x000006b99159a8bbLL, - 0x000000000d7392e6LL -}; - -static const unsigned long long neg_terms_l[N_COEFF_N] = { - 0x2222222222222167LL, - 0x0002e3bc74aab624LL, - 0x0000000b09229062LL, - 0x00000000000c7973LL -}; - -#define N_COEFF_PH 4 -#define N_COEFF_NH 4 -static const unsigned long long pos_terms_h[N_COEFF_PH] = { - 0x0000000000000000LL, - 0x05b05b05b05b0406LL, - 0x000049f93edd91a9LL, - 0x00000000c9c9ed62LL -}; - -static const unsigned long long neg_terms_h[N_COEFF_NH] = { - 0xaaaaaaaaaaaaaa98LL, - 0x001a01a01a019064LL, - 0x0000008f76c68a77LL, - 0x0000000000d58f5eLL -}; - -/*--- poly_sine() -----------------------------------------------------------+ - | | - +---------------------------------------------------------------------------*/ -void poly_sine(FPU_REG *st0_ptr) -{ - int exponent, echange; - Xsig accumulator, argSqrd, argTo4; - unsigned long fix_up, adj; - unsigned long long fixed_arg; - FPU_REG result; - - exponent = exponent(st0_ptr); - - accumulator.lsw = accumulator.midw = accumulator.msw = 0; - - /* Split into two ranges, for arguments below and above 1.0 */ - /* The boundary between upper and lower is approx 0.88309101259 */ - if ((exponent < -1) - || ((exponent == -1) && (st0_ptr->sigh <= 0xe21240aa))) { - /* The argument is <= 0.88309101259 */ - - argSqrd.msw = st0_ptr->sigh; - argSqrd.midw = st0_ptr->sigl; - argSqrd.lsw = 0; - mul64_Xsig(&argSqrd, &significand(st0_ptr)); - shr_Xsig(&argSqrd, 2 * (-1 - exponent)); - argTo4.msw = argSqrd.msw; - argTo4.midw = argSqrd.midw; - argTo4.lsw = argSqrd.lsw; - mul_Xsig_Xsig(&argTo4, &argTo4); - - polynomial_Xsig(&accumulator, &XSIG_LL(argTo4), neg_terms_l, - N_COEFF_N - 1); - mul_Xsig_Xsig(&accumulator, &argSqrd); - negate_Xsig(&accumulator); - - polynomial_Xsig(&accumulator, &XSIG_LL(argTo4), pos_terms_l, - N_COEFF_P - 1); - - shr_Xsig(&accumulator, 2); /* Divide by four */ - accumulator.msw |= 0x80000000; /* Add 1.0 */ - - mul64_Xsig(&accumulator, &significand(st0_ptr)); - mul64_Xsig(&accumulator, &significand(st0_ptr)); - mul64_Xsig(&accumulator, &significand(st0_ptr)); - - /* Divide by four, FPU_REG compatible, etc */ - exponent = 3 * exponent; - - /* The minimum exponent difference is 3 */ - shr_Xsig(&accumulator, exponent(st0_ptr) - exponent); - - negate_Xsig(&accumulator); - XSIG_LL(accumulator) += significand(st0_ptr); - - echange = round_Xsig(&accumulator); - - setexponentpos(&result, exponent(st0_ptr) + echange); - } else { - /* The argument is > 0.88309101259 */ - /* We use sin(st(0)) = cos(pi/2-st(0)) */ - - fixed_arg = significand(st0_ptr); - - if (exponent == 0) { - /* The argument is >= 1.0 */ - - /* Put the binary point at the left. */ - fixed_arg <<= 1; - } - /* pi/2 in hex is: 1.921fb54442d18469 898CC51701B839A2 52049C1 */ - fixed_arg = 0x921fb54442d18469LL - fixed_arg; - /* There is a special case which arises due to rounding, to fix here. */ - if (fixed_arg == 0xffffffffffffffffLL) - fixed_arg = 0; - - XSIG_LL(argSqrd) = fixed_arg; - argSqrd.lsw = 0; - mul64_Xsig(&argSqrd, &fixed_arg); - - XSIG_LL(argTo4) = XSIG_LL(argSqrd); - argTo4.lsw = argSqrd.lsw; - mul_Xsig_Xsig(&argTo4, &argTo4); - - polynomial_Xsig(&accumulator, &XSIG_LL(argTo4), neg_terms_h, - N_COEFF_NH - 1); - mul_Xsig_Xsig(&accumulator, &argSqrd); - negate_Xsig(&accumulator); - - polynomial_Xsig(&accumulator, &XSIG_LL(argTo4), pos_terms_h, - N_COEFF_PH - 1); - negate_Xsig(&accumulator); - - mul64_Xsig(&accumulator, &fixed_arg); - mul64_Xsig(&accumulator, &fixed_arg); - - shr_Xsig(&accumulator, 3); - negate_Xsig(&accumulator); - - add_Xsig_Xsig(&accumulator, &argSqrd); - - shr_Xsig(&accumulator, 1); - - accumulator.lsw |= 1; /* A zero accumulator here would cause problems */ - negate_Xsig(&accumulator); - - /* The basic computation is complete. Now fix the answer to - compensate for the error due to the approximation used for - pi/2 - */ - - /* This has an exponent of -65 */ - fix_up = 0x898cc517; - /* The fix-up needs to be improved for larger args */ - if (argSqrd.msw & 0xffc00000) { - /* Get about 32 bit precision in these: */ - fix_up -= mul_32_32(0x898cc517, argSqrd.msw) / 6; - } - fix_up = mul_32_32(fix_up, LL_MSW(fixed_arg)); - - adj = accumulator.lsw; /* temp save */ - accumulator.lsw -= fix_up; - if (accumulator.lsw > adj) - XSIG_LL(accumulator)--; - - echange = round_Xsig(&accumulator); - - setexponentpos(&result, echange - 1); - } - - significand(&result) = XSIG_LL(accumulator); - setsign(&result, getsign(st0_ptr)); - FPU_copy_to_reg0(&result, TAG_Valid); - -#ifdef PARANOID - if ((exponent(&result) >= 0) - && (significand(&result) > 0x8000000000000000LL)) { - EXCEPTION(EX_INTERNAL | 0x150); - } -#endif /* PARANOID */ - -} - -/*--- poly_cos() ------------------------------------------------------------+ - | | - +---------------------------------------------------------------------------*/ -void poly_cos(FPU_REG *st0_ptr) -{ - FPU_REG result; - long int exponent, exp2, echange; - Xsig accumulator, argSqrd, fix_up, argTo4; - unsigned long long fixed_arg; - -#ifdef PARANOID - if ((exponent(st0_ptr) > 0) - || ((exponent(st0_ptr) == 0) - && (significand(st0_ptr) > 0xc90fdaa22168c234LL))) { - EXCEPTION(EX_Invalid); - FPU_copy_to_reg0(&CONST_QNaN, TAG_Special); - return; - } -#endif /* PARANOID */ - - exponent = exponent(st0_ptr); - - accumulator.lsw = accumulator.midw = accumulator.msw = 0; - - if ((exponent < -1) - || ((exponent == -1) && (st0_ptr->sigh <= 0xb00d6f54))) { - /* arg is < 0.687705 */ - - argSqrd.msw = st0_ptr->sigh; - argSqrd.midw = st0_ptr->sigl; - argSqrd.lsw = 0; - mul64_Xsig(&argSqrd, &significand(st0_ptr)); - - if (exponent < -1) { - /* shift the argument right by the required places */ - shr_Xsig(&argSqrd, 2 * (-1 - exponent)); - } - - argTo4.msw = argSqrd.msw; - argTo4.midw = argSqrd.midw; - argTo4.lsw = argSqrd.lsw; - mul_Xsig_Xsig(&argTo4, &argTo4); - - polynomial_Xsig(&accumulator, &XSIG_LL(argTo4), neg_terms_h, - N_COEFF_NH - 1); - mul_Xsig_Xsig(&accumulator, &argSqrd); - negate_Xsig(&accumulator); - - polynomial_Xsig(&accumulator, &XSIG_LL(argTo4), pos_terms_h, - N_COEFF_PH - 1); - negate_Xsig(&accumulator); - - mul64_Xsig(&accumulator, &significand(st0_ptr)); - mul64_Xsig(&accumulator, &significand(st0_ptr)); - shr_Xsig(&accumulator, -2 * (1 + exponent)); - - shr_Xsig(&accumulator, 3); - negate_Xsig(&accumulator); - - add_Xsig_Xsig(&accumulator, &argSqrd); - - shr_Xsig(&accumulator, 1); - - /* It doesn't matter if accumulator is all zero here, the - following code will work ok */ - negate_Xsig(&accumulator); - - if (accumulator.lsw & 0x80000000) - XSIG_LL(accumulator)++; - if (accumulator.msw == 0) { - /* The result is 1.0 */ - FPU_copy_to_reg0(&CONST_1, TAG_Valid); - return; - } else { - significand(&result) = XSIG_LL(accumulator); - - /* will be a valid positive nr with expon = -1 */ - setexponentpos(&result, -1); - } - } else { - fixed_arg = significand(st0_ptr); - - if (exponent == 0) { - /* The argument is >= 1.0 */ - - /* Put the binary point at the left. */ - fixed_arg <<= 1; - } - /* pi/2 in hex is: 1.921fb54442d18469 898CC51701B839A2 52049C1 */ - fixed_arg = 0x921fb54442d18469LL - fixed_arg; - /* There is a special case which arises due to rounding, to fix here. */ - if (fixed_arg == 0xffffffffffffffffLL) - fixed_arg = 0; - - exponent = -1; - exp2 = -1; - - /* A shift is needed here only for a narrow range of arguments, - i.e. for fixed_arg approx 2^-32, but we pick up more... */ - if (!(LL_MSW(fixed_arg) & 0xffff0000)) { - fixed_arg <<= 16; - exponent -= 16; - exp2 -= 16; - } - - XSIG_LL(argSqrd) = fixed_arg; - argSqrd.lsw = 0; - mul64_Xsig(&argSqrd, &fixed_arg); - - if (exponent < -1) { - /* shift the argument right by the required places */ - shr_Xsig(&argSqrd, 2 * (-1 - exponent)); - } - - argTo4.msw = argSqrd.msw; - argTo4.midw = argSqrd.midw; - argTo4.lsw = argSqrd.lsw; - mul_Xsig_Xsig(&argTo4, &argTo4); - - polynomial_Xsig(&accumulator, &XSIG_LL(argTo4), neg_terms_l, - N_COEFF_N - 1); - mul_Xsig_Xsig(&accumulator, &argSqrd); - negate_Xsig(&accumulator); - - polynomial_Xsig(&accumulator, &XSIG_LL(argTo4), pos_terms_l, - N_COEFF_P - 1); - - shr_Xsig(&accumulator, 2); /* Divide by four */ - accumulator.msw |= 0x80000000; /* Add 1.0 */ - - mul64_Xsig(&accumulator, &fixed_arg); - mul64_Xsig(&accumulator, &fixed_arg); - mul64_Xsig(&accumulator, &fixed_arg); - - /* Divide by four, FPU_REG compatible, etc */ - exponent = 3 * exponent; - - /* The minimum exponent difference is 3 */ - shr_Xsig(&accumulator, exp2 - exponent); - - negate_Xsig(&accumulator); - XSIG_LL(accumulator) += fixed_arg; - - /* The basic computation is complete. Now fix the answer to - compensate for the error due to the approximation used for - pi/2 - */ - - /* This has an exponent of -65 */ - XSIG_LL(fix_up) = 0x898cc51701b839a2ll; - fix_up.lsw = 0; - - /* The fix-up needs to be improved for larger args */ - if (argSqrd.msw & 0xffc00000) { - /* Get about 32 bit precision in these: */ - fix_up.msw -= mul_32_32(0x898cc517, argSqrd.msw) / 2; - fix_up.msw += mul_32_32(0x898cc517, argTo4.msw) / 24; - } - - exp2 += norm_Xsig(&accumulator); - shr_Xsig(&accumulator, 1); /* Prevent overflow */ - exp2++; - shr_Xsig(&fix_up, 65 + exp2); - - add_Xsig_Xsig(&accumulator, &fix_up); - - echange = round_Xsig(&accumulator); - - setexponentpos(&result, exp2 + echange); - significand(&result) = XSIG_LL(accumulator); - } - - FPU_copy_to_reg0(&result, TAG_Valid); - -#ifdef PARANOID - if ((exponent(&result) >= 0) - && (significand(&result) > 0x8000000000000000LL)) { - EXCEPTION(EX_INTERNAL | 0x151); - } -#endif /* PARANOID */ - -} diff --git a/arch/x86/math-emu/poly_tan.c b/arch/x86/math-emu/poly_tan.c deleted file mode 100644 index 1f5b1d712323..000000000000 --- a/arch/x86/math-emu/poly_tan.c +++ /dev/null @@ -1,213 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/*---------------------------------------------------------------------------+ - | poly_tan.c | - | | - | Compute the tan of a FPU_REG, using a polynomial approximation. | - | | - | Copyright (C) 1992,1993,1994,1997,1999 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, | - | Australia. E-mail billm@melbpc.org.au | - | | - | | - +---------------------------------------------------------------------------*/ - -#include "exception.h" -#include "reg_constant.h" -#include "fpu_emu.h" -#include "fpu_system.h" -#include "control_w.h" -#include "poly.h" - -#define HiPOWERop 3 /* odd poly, positive terms */ -static const unsigned long long oddplterm[HiPOWERop] = { - 0x0000000000000000LL, - 0x0051a1cf08fca228LL, - 0x0000000071284ff7LL -}; - -#define HiPOWERon 2 /* odd poly, negative terms */ -static const unsigned long long oddnegterm[HiPOWERon] = { - 0x1291a9a184244e80LL, - 0x0000583245819c21LL -}; - -#define HiPOWERep 2 /* even poly, positive terms */ -static const unsigned long long evenplterm[HiPOWERep] = { - 0x0e848884b539e888LL, - 0x00003c7f18b887daLL -}; - -#define HiPOWERen 2 /* even poly, negative terms */ -static const unsigned long long evennegterm[HiPOWERen] = { - 0xf1f0200fd51569ccLL, - 0x003afb46105c4432LL -}; - -static const unsigned long long twothirds = 0xaaaaaaaaaaaaaaabLL; - -/*--- poly_tan() ------------------------------------------------------------+ - | | - +---------------------------------------------------------------------------*/ -void poly_tan(FPU_REG *st0_ptr) -{ - long int exponent; - int invert; - Xsig argSq, argSqSq, accumulatoro, accumulatore, accum, - argSignif, fix_up; - unsigned long adj; - - exponent = exponent(st0_ptr); - -#ifdef PARANOID - if (signnegative(st0_ptr)) { /* Can't hack a number < 0.0 */ - arith_invalid(0); - return; - } /* Need a positive number */ -#endif /* PARANOID */ - - /* Split the problem into two domains, smaller and larger than pi/4 */ - if ((exponent == 0) - || ((exponent == -1) && (st0_ptr->sigh > 0xc90fdaa2))) { - /* The argument is greater than (approx) pi/4 */ - invert = 1; - accum.lsw = 0; - XSIG_LL(accum) = significand(st0_ptr); - - if (exponent == 0) { - /* The argument is >= 1.0 */ - /* Put the binary point at the left. */ - XSIG_LL(accum) <<= 1; - } - /* pi/2 in hex is: 1.921fb54442d18469 898CC51701B839A2 52049C1 */ - XSIG_LL(accum) = 0x921fb54442d18469LL - XSIG_LL(accum); - /* This is a special case which arises due to rounding. */ - if (XSIG_LL(accum) == 0xffffffffffffffffLL) { - FPU_settag0(TAG_Valid); - significand(st0_ptr) = 0x8a51e04daabda360LL; - setexponent16(st0_ptr, - (0x41 + EXTENDED_Ebias) | SIGN_Negative); - return; - } - - argSignif.lsw = accum.lsw; - XSIG_LL(argSignif) = XSIG_LL(accum); - exponent = -1 + norm_Xsig(&argSignif); - } else { - invert = 0; - argSignif.lsw = 0; - XSIG_LL(accum) = XSIG_LL(argSignif) = significand(st0_ptr); - - if (exponent < -1) { - /* shift the argument right by the required places */ - if (FPU_shrx(&XSIG_LL(accum), -1 - exponent) >= - 0x80000000U) - XSIG_LL(accum)++; /* round up */ - } - } - - XSIG_LL(argSq) = XSIG_LL(accum); - argSq.lsw = accum.lsw; - mul_Xsig_Xsig(&argSq, &argSq); - XSIG_LL(argSqSq) = XSIG_LL(argSq); - argSqSq.lsw = argSq.lsw; - mul_Xsig_Xsig(&argSqSq, &argSqSq); - - /* Compute the negative terms for the numerator polynomial */ - accumulatoro.msw = accumulatoro.midw = accumulatoro.lsw = 0; - polynomial_Xsig(&accumulatoro, &XSIG_LL(argSqSq), oddnegterm, - HiPOWERon - 1); - mul_Xsig_Xsig(&accumulatoro, &argSq); - negate_Xsig(&accumulatoro); - /* Add the positive terms */ - polynomial_Xsig(&accumulatoro, &XSIG_LL(argSqSq), oddplterm, - HiPOWERop - 1); - - /* Compute the positive terms for the denominator polynomial */ - accumulatore.msw = accumulatore.midw = accumulatore.lsw = 0; - polynomial_Xsig(&accumulatore, &XSIG_LL(argSqSq), evenplterm, - HiPOWERep - 1); - mul_Xsig_Xsig(&accumulatore, &argSq); - negate_Xsig(&accumulatore); - /* Add the negative terms */ - polynomial_Xsig(&accumulatore, &XSIG_LL(argSqSq), evennegterm, - HiPOWERen - 1); - /* Multiply by arg^2 */ - mul64_Xsig(&accumulatore, &XSIG_LL(argSignif)); - mul64_Xsig(&accumulatore, &XSIG_LL(argSignif)); - /* de-normalize and divide by 2 */ - shr_Xsig(&accumulatore, -2 * (1 + exponent) + 1); - negate_Xsig(&accumulatore); /* This does 1 - accumulator */ - - /* Now find the ratio. */ - if (accumulatore.msw == 0) { - /* accumulatoro must contain 1.0 here, (actually, 0) but it - really doesn't matter what value we use because it will - have negligible effect in later calculations - */ - XSIG_LL(accum) = 0x8000000000000000LL; - accum.lsw = 0; - } else { - div_Xsig(&accumulatoro, &accumulatore, &accum); - } - - /* Multiply by 1/3 * arg^3 */ - mul64_Xsig(&accum, &XSIG_LL(argSignif)); - mul64_Xsig(&accum, &XSIG_LL(argSignif)); - mul64_Xsig(&accum, &XSIG_LL(argSignif)); - mul64_Xsig(&accum, &twothirds); - shr_Xsig(&accum, -2 * (exponent + 1)); - - /* tan(arg) = arg + accum */ - add_two_Xsig(&accum, &argSignif, &exponent); - - if (invert) { - /* We now have the value of tan(pi_2 - arg) where pi_2 is an - approximation for pi/2 - */ - /* The next step is to fix the answer to compensate for the - error due to the approximation used for pi/2 - */ - - /* This is (approx) delta, the error in our approx for pi/2 - (see above). It has an exponent of -65 - */ - XSIG_LL(fix_up) = 0x898cc51701b839a2LL; - fix_up.lsw = 0; - - if (exponent == 0) - adj = 0xffffffff; /* We want approx 1.0 here, but - this is close enough. */ - else if (exponent > -30) { - adj = accum.msw >> -(exponent + 1); /* tan */ - adj = mul_32_32(adj, adj); /* tan^2 */ - } else - adj = 0; - adj = mul_32_32(0x898cc517, adj); /* delta * tan^2 */ - - fix_up.msw += adj; - if (!(fix_up.msw & 0x80000000)) { /* did fix_up overflow ? */ - /* Yes, we need to add an msb */ - shr_Xsig(&fix_up, 1); - fix_up.msw |= 0x80000000; - shr_Xsig(&fix_up, 64 + exponent); - } else - shr_Xsig(&fix_up, 65 + exponent); - - add_two_Xsig(&accum, &fix_up, &exponent); - - /* accum now contains tan(pi/2 - arg). - Use tan(arg) = 1.0 / tan(pi/2 - arg) - */ - accumulatoro.lsw = accumulatoro.midw = 0; - accumulatoro.msw = 0x80000000; - div_Xsig(&accumulatoro, &accum, &accum); - exponent = -exponent - 1; - } - - /* Transfer the result */ - round_Xsig(&accum); - FPU_settag0(TAG_Valid); - significand(st0_ptr) = XSIG_LL(accum); - setexponent16(st0_ptr, exponent + EXTENDED_Ebias); /* Result is positive. */ - -} diff --git a/arch/x86/math-emu/polynom_Xsig.S b/arch/x86/math-emu/polynom_Xsig.S deleted file mode 100644 index 35fd723fc0df..000000000000 --- a/arch/x86/math-emu/polynom_Xsig.S +++ /dev/null @@ -1,137 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/*---------------------------------------------------------------------------+ - | polynomial_Xsig.S | - | | - | Fixed point arithmetic polynomial evaluation. | - | | - | Copyright (C) 1992,1993,1994,1995 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, | - | Australia. E-mail billm@jacobi.maths.monash.edu.au | - | | - | Call from C as: | - | void polynomial_Xsig(Xsig *accum, unsigned long long x, | - | unsigned long long terms[], int n) | - | | - | Computes: | - | terms[0] + (terms[1] + (terms[2] + ... + (terms[n-1]*x)*x)*x)*x) ... )*x | - | and adds the result to the 12 byte Xsig. | - | The terms[] are each 8 bytes, but all computation is performed to 12 byte | - | precision. | - | | - | This function must be used carefully: most overflow of intermediate | - | results is controlled, but overflow of the result is not. | - | | - +---------------------------------------------------------------------------*/ - .file "polynomial_Xsig.S" - -#include "fpu_emu.h" - - -#define TERM_SIZE $8 -#define SUM_MS -20(%ebp) /* sum ms long */ -#define SUM_MIDDLE -24(%ebp) /* sum middle long */ -#define SUM_LS -28(%ebp) /* sum ls long */ -#define ACCUM_MS -4(%ebp) /* accum ms long */ -#define ACCUM_MIDDLE -8(%ebp) /* accum middle long */ -#define ACCUM_LS -12(%ebp) /* accum ls long */ -#define OVERFLOWED -16(%ebp) /* addition overflow flag */ - -.text -SYM_FUNC_START(polynomial_Xsig) - pushl %ebp - movl %esp,%ebp - subl $32,%esp - pushl %esi - pushl %edi - pushl %ebx - - movl PARAM2,%esi /* x */ - movl PARAM3,%edi /* terms */ - - movl TERM_SIZE,%eax - mull PARAM4 /* n */ - addl %eax,%edi - - movl 4(%edi),%edx /* terms[n] */ - movl %edx,SUM_MS - movl (%edi),%edx /* terms[n] */ - movl %edx,SUM_MIDDLE - xor %eax,%eax - movl %eax,SUM_LS - movb %al,OVERFLOWED - - subl TERM_SIZE,%edi - decl PARAM4 - js L_accum_done - -L_accum_loop: - xor %eax,%eax - movl %eax,ACCUM_MS - movl %eax,ACCUM_MIDDLE - - movl SUM_MIDDLE,%eax - mull (%esi) /* x ls long */ - movl %edx,ACCUM_LS - - movl SUM_MIDDLE,%eax - mull 4(%esi) /* x ms long */ - addl %eax,ACCUM_LS - adcl %edx,ACCUM_MIDDLE - adcl $0,ACCUM_MS - - movl SUM_MS,%eax - mull (%esi) /* x ls long */ - addl %eax,ACCUM_LS - adcl %edx,ACCUM_MIDDLE - adcl $0,ACCUM_MS - - movl SUM_MS,%eax - mull 4(%esi) /* x ms long */ - addl %eax,ACCUM_MIDDLE - adcl %edx,ACCUM_MS - - testb $0xff,OVERFLOWED - jz L_no_overflow - - movl (%esi),%eax - addl %eax,ACCUM_MIDDLE - movl 4(%esi),%eax - adcl %eax,ACCUM_MS /* This could overflow too */ - -L_no_overflow: - -/* - * Now put the sum of next term and the accumulator - * into the sum register - */ - movl ACCUM_LS,%eax - addl (%edi),%eax /* term ls long */ - movl %eax,SUM_LS - movl ACCUM_MIDDLE,%eax - adcl (%edi),%eax /* term ls long */ - movl %eax,SUM_MIDDLE - movl ACCUM_MS,%eax - adcl 4(%edi),%eax /* term ms long */ - movl %eax,SUM_MS - sbbb %al,%al - movb %al,OVERFLOWED /* Used in the next iteration */ - - subl TERM_SIZE,%edi - decl PARAM4 - jns L_accum_loop - -L_accum_done: - movl PARAM1,%edi /* accum */ - movl SUM_LS,%eax - addl %eax,(%edi) - movl SUM_MIDDLE,%eax - adcl %eax,4(%edi) - movl SUM_MS,%eax - adcl %eax,8(%edi) - - popl %ebx - popl %edi - popl %esi - leave - RET -SYM_FUNC_END(polynomial_Xsig) diff --git a/arch/x86/math-emu/reg_add_sub.c b/arch/x86/math-emu/reg_add_sub.c deleted file mode 100644 index 29451dd07556..000000000000 --- a/arch/x86/math-emu/reg_add_sub.c +++ /dev/null @@ -1,334 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/*---------------------------------------------------------------------------+ - | reg_add_sub.c | - | | - | Functions to add or subtract two registers and put the result in a third. | - | | - | Copyright (C) 1992,1993,1997 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia | - | E-mail billm@suburbia.net | - | | - | | - +---------------------------------------------------------------------------*/ - -/*---------------------------------------------------------------------------+ - | For each function, the destination may be any FPU_REG, including one of | - | the source FPU_REGs. | - | Each function returns 0 if the answer is o.k., otherwise a non-zero | - | value is returned, indicating either an exception condition or an | - | internal error. | - +---------------------------------------------------------------------------*/ - -#include "exception.h" -#include "reg_constant.h" -#include "fpu_emu.h" -#include "control_w.h" -#include "fpu_system.h" - -static -int add_sub_specials(FPU_REG const *a, u_char taga, u_char signa, - FPU_REG const *b, u_char tagb, u_char signb, - FPU_REG * dest, int deststnr, int control_w); - -/* - Operates on st(0) and st(n), or on st(0) and temporary data. - The destination must be one of the source st(x). - */ -int FPU_add(FPU_REG const *b, u_char tagb, int deststnr, int control_w) -{ - FPU_REG *a = &st(0); - FPU_REG *dest = &st(deststnr); - u_char signb = getsign(b); - u_char taga = FPU_gettag0(); - u_char signa = getsign(a); - u_char saved_sign = getsign(dest); - int diff, tag, expa, expb; - - if (!(taga | tagb)) { - expa = exponent(a); - expb = exponent(b); - - valid_add: - /* Both registers are valid */ - if (!(signa ^ signb)) { - /* signs are the same */ - tag = - FPU_u_add(a, b, dest, control_w, signa, expa, expb); - } else { - /* The signs are different, so do a subtraction */ - diff = expa - expb; - if (!diff) { - diff = a->sigh - b->sigh; /* This works only if the ms bits - are identical. */ - if (!diff) { - diff = a->sigl > b->sigl; - if (!diff) - diff = -(a->sigl < b->sigl); - } - } - - if (diff > 0) { - tag = - FPU_u_sub(a, b, dest, control_w, signa, - expa, expb); - } else if (diff < 0) { - tag = - FPU_u_sub(b, a, dest, control_w, signb, - expb, expa); - } else { - FPU_copy_to_regi(&CONST_Z, TAG_Zero, deststnr); - /* sign depends upon rounding mode */ - setsign(dest, ((control_w & CW_RC) != RC_DOWN) - ? SIGN_POS : SIGN_NEG); - return TAG_Zero; - } - } - - if (tag < 0) { - setsign(dest, saved_sign); - return tag; - } - FPU_settagi(deststnr, tag); - return tag; - } - - if (taga == TAG_Special) - taga = FPU_Special(a); - if (tagb == TAG_Special) - tagb = FPU_Special(b); - - if (((taga == TAG_Valid) && (tagb == TW_Denormal)) - || ((taga == TW_Denormal) && (tagb == TAG_Valid)) - || ((taga == TW_Denormal) && (tagb == TW_Denormal))) { - FPU_REG x, y; - - if (denormal_operand() < 0) - return FPU_Exception; - - FPU_to_exp16(a, &x); - FPU_to_exp16(b, &y); - a = &x; - b = &y; - expa = exponent16(a); - expb = exponent16(b); - goto valid_add; - } - - if ((taga == TW_NaN) || (tagb == TW_NaN)) { - if (deststnr == 0) - return real_2op_NaN(b, tagb, deststnr, a); - else - return real_2op_NaN(a, taga, deststnr, a); - } - - return add_sub_specials(a, taga, signa, b, tagb, signb, - dest, deststnr, control_w); -} - -/* Subtract b from a. (a-b) -> dest */ -int FPU_sub(int flags, int rm, int control_w) -{ - FPU_REG const *a, *b; - FPU_REG *dest; - u_char taga, tagb, signa, signb, saved_sign, sign; - int diff, tag = 0, expa, expb, deststnr; - - a = &st(0); - taga = FPU_gettag0(); - - deststnr = 0; - if (flags & LOADED) { - b = (FPU_REG *) rm; - tagb = flags & 0x0f; - } else { - b = &st(rm); - tagb = FPU_gettagi(rm); - - if (flags & DEST_RM) - deststnr = rm; - } - - signa = getsign(a); - signb = getsign(b); - - if (flags & REV) { - signa ^= SIGN_NEG; - signb ^= SIGN_NEG; - } - - dest = &st(deststnr); - saved_sign = getsign(dest); - - if (!(taga | tagb)) { - expa = exponent(a); - expb = exponent(b); - - valid_subtract: - /* Both registers are valid */ - - diff = expa - expb; - - if (!diff) { - diff = a->sigh - b->sigh; /* Works only if ms bits are identical */ - if (!diff) { - diff = a->sigl > b->sigl; - if (!diff) - diff = -(a->sigl < b->sigl); - } - } - - switch ((((int)signa) * 2 + signb) / SIGN_NEG) { - case 0: /* P - P */ - case 3: /* N - N */ - if (diff > 0) { - /* |a| > |b| */ - tag = - FPU_u_sub(a, b, dest, control_w, signa, - expa, expb); - } else if (diff == 0) { - FPU_copy_to_regi(&CONST_Z, TAG_Zero, deststnr); - - /* sign depends upon rounding mode */ - setsign(dest, ((control_w & CW_RC) != RC_DOWN) - ? SIGN_POS : SIGN_NEG); - return TAG_Zero; - } else { - sign = signa ^ SIGN_NEG; - tag = - FPU_u_sub(b, a, dest, control_w, sign, expb, - expa); - } - break; - case 1: /* P - N */ - tag = - FPU_u_add(a, b, dest, control_w, SIGN_POS, expa, - expb); - break; - case 2: /* N - P */ - tag = - FPU_u_add(a, b, dest, control_w, SIGN_NEG, expa, - expb); - break; -#ifdef PARANOID - default: - EXCEPTION(EX_INTERNAL | 0x111); - return -1; -#endif - } - if (tag < 0) { - setsign(dest, saved_sign); - return tag; - } - FPU_settagi(deststnr, tag); - return tag; - } - - if (taga == TAG_Special) - taga = FPU_Special(a); - if (tagb == TAG_Special) - tagb = FPU_Special(b); - - if (((taga == TAG_Valid) && (tagb == TW_Denormal)) - || ((taga == TW_Denormal) && (tagb == TAG_Valid)) - || ((taga == TW_Denormal) && (tagb == TW_Denormal))) { - FPU_REG x, y; - - if (denormal_operand() < 0) - return FPU_Exception; - - FPU_to_exp16(a, &x); - FPU_to_exp16(b, &y); - a = &x; - b = &y; - expa = exponent16(a); - expb = exponent16(b); - - goto valid_subtract; - } - - if ((taga == TW_NaN) || (tagb == TW_NaN)) { - FPU_REG const *d1, *d2; - if (flags & REV) { - d1 = b; - d2 = a; - } else { - d1 = a; - d2 = b; - } - if (flags & LOADED) - return real_2op_NaN(b, tagb, deststnr, d1); - if (flags & DEST_RM) - return real_2op_NaN(a, taga, deststnr, d2); - else - return real_2op_NaN(b, tagb, deststnr, d2); - } - - return add_sub_specials(a, taga, signa, b, tagb, signb ^ SIGN_NEG, - dest, deststnr, control_w); -} - -static -int add_sub_specials(FPU_REG const *a, u_char taga, u_char signa, - FPU_REG const *b, u_char tagb, u_char signb, - FPU_REG * dest, int deststnr, int control_w) -{ - if (((taga == TW_Denormal) || (tagb == TW_Denormal)) - && (denormal_operand() < 0)) - return FPU_Exception; - - if (taga == TAG_Zero) { - if (tagb == TAG_Zero) { - /* Both are zero, result will be zero. */ - u_char different_signs = signa ^ signb; - - FPU_copy_to_regi(a, TAG_Zero, deststnr); - if (different_signs) { - /* Signs are different. */ - /* Sign of answer depends upon rounding mode. */ - setsign(dest, ((control_w & CW_RC) != RC_DOWN) - ? SIGN_POS : SIGN_NEG); - } else - setsign(dest, signa); /* signa may differ from the sign of a. */ - return TAG_Zero; - } else { - reg_copy(b, dest); - if ((tagb == TW_Denormal) && (b->sigh & 0x80000000)) { - /* A pseudoDenormal, convert it. */ - addexponent(dest, 1); - tagb = TAG_Valid; - } else if (tagb > TAG_Empty) - tagb = TAG_Special; - setsign(dest, signb); /* signb may differ from the sign of b. */ - FPU_settagi(deststnr, tagb); - return tagb; - } - } else if (tagb == TAG_Zero) { - reg_copy(a, dest); - if ((taga == TW_Denormal) && (a->sigh & 0x80000000)) { - /* A pseudoDenormal */ - addexponent(dest, 1); - taga = TAG_Valid; - } else if (taga > TAG_Empty) - taga = TAG_Special; - setsign(dest, signa); /* signa may differ from the sign of a. */ - FPU_settagi(deststnr, taga); - return taga; - } else if (taga == TW_Infinity) { - if ((tagb != TW_Infinity) || (signa == signb)) { - FPU_copy_to_regi(a, TAG_Special, deststnr); - setsign(dest, signa); /* signa may differ from the sign of a. */ - return taga; - } - /* Infinity-Infinity is undefined. */ - return arith_invalid(deststnr); - } else if (tagb == TW_Infinity) { - FPU_copy_to_regi(b, TAG_Special, deststnr); - setsign(dest, signb); /* signb may differ from the sign of b. */ - return tagb; - } -#ifdef PARANOID - EXCEPTION(EX_INTERNAL | 0x101); -#endif - - return FPU_Exception; -} diff --git a/arch/x86/math-emu/reg_compare.c b/arch/x86/math-emu/reg_compare.c deleted file mode 100644 index eacb5128fc09..000000000000 --- a/arch/x86/math-emu/reg_compare.c +++ /dev/null @@ -1,479 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/*---------------------------------------------------------------------------+ - | reg_compare.c | - | | - | Compare two floating point registers | - | | - | Copyright (C) 1992,1993,1994,1997 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia | - | E-mail billm@suburbia.net | - | | - | | - +---------------------------------------------------------------------------*/ - -/*---------------------------------------------------------------------------+ - | compare() is the core FPU_REG comparison function | - +---------------------------------------------------------------------------*/ - -#include "fpu_system.h" -#include "exception.h" -#include "fpu_emu.h" -#include "control_w.h" -#include "status_w.h" - -static int compare(FPU_REG const *b, int tagb) -{ - int diff, exp0, expb; - u_char st0_tag; - FPU_REG *st0_ptr; - FPU_REG x, y; - u_char st0_sign, signb = getsign(b); - - st0_ptr = &st(0); - st0_tag = FPU_gettag0(); - st0_sign = getsign(st0_ptr); - - if (tagb == TAG_Special) - tagb = FPU_Special(b); - if (st0_tag == TAG_Special) - st0_tag = FPU_Special(st0_ptr); - - if (((st0_tag != TAG_Valid) && (st0_tag != TW_Denormal)) - || ((tagb != TAG_Valid) && (tagb != TW_Denormal))) { - if (st0_tag == TAG_Zero) { - if (tagb == TAG_Zero) - return COMP_A_eq_B; - if (tagb == TAG_Valid) - return ((signb == - SIGN_POS) ? COMP_A_lt_B : COMP_A_gt_B); - if (tagb == TW_Denormal) - return ((signb == - SIGN_POS) ? COMP_A_lt_B : COMP_A_gt_B) - | COMP_Denormal; - } else if (tagb == TAG_Zero) { - if (st0_tag == TAG_Valid) - return ((st0_sign == - SIGN_POS) ? COMP_A_gt_B : COMP_A_lt_B); - if (st0_tag == TW_Denormal) - return ((st0_sign == - SIGN_POS) ? COMP_A_gt_B : COMP_A_lt_B) - | COMP_Denormal; - } - - if (st0_tag == TW_Infinity) { - if ((tagb == TAG_Valid) || (tagb == TAG_Zero)) - return ((st0_sign == - SIGN_POS) ? COMP_A_gt_B : COMP_A_lt_B); - else if (tagb == TW_Denormal) - return ((st0_sign == - SIGN_POS) ? COMP_A_gt_B : COMP_A_lt_B) - | COMP_Denormal; - else if (tagb == TW_Infinity) { - /* The 80486 book says that infinities can be equal! */ - return (st0_sign == signb) ? COMP_A_eq_B : - ((st0_sign == - SIGN_POS) ? COMP_A_gt_B : COMP_A_lt_B); - } - /* Fall through to the NaN code */ - } else if (tagb == TW_Infinity) { - if ((st0_tag == TAG_Valid) || (st0_tag == TAG_Zero)) - return ((signb == - SIGN_POS) ? COMP_A_lt_B : COMP_A_gt_B); - if (st0_tag == TW_Denormal) - return ((signb == - SIGN_POS) ? COMP_A_lt_B : COMP_A_gt_B) - | COMP_Denormal; - /* Fall through to the NaN code */ - } - - /* The only possibility now should be that one of the arguments - is a NaN */ - if ((st0_tag == TW_NaN) || (tagb == TW_NaN)) { - int signalling = 0, unsupported = 0; - if (st0_tag == TW_NaN) { - signalling = - (st0_ptr->sigh & 0xc0000000) == 0x80000000; - unsupported = !((exponent(st0_ptr) == EXP_OVER) - && (st0_ptr-> - sigh & 0x80000000)); - } - if (tagb == TW_NaN) { - signalling |= - (b->sigh & 0xc0000000) == 0x80000000; - unsupported |= !((exponent(b) == EXP_OVER) - && (b->sigh & 0x80000000)); - } - if (signalling || unsupported) - return COMP_No_Comp | COMP_SNaN | COMP_NaN; - else - /* Neither is a signaling NaN */ - return COMP_No_Comp | COMP_NaN; - } - - EXCEPTION(EX_Invalid); - } - - if (st0_sign != signb) { - return ((st0_sign == SIGN_POS) ? COMP_A_gt_B : COMP_A_lt_B) - | (((st0_tag == TW_Denormal) || (tagb == TW_Denormal)) ? - COMP_Denormal : 0); - } - - if ((st0_tag == TW_Denormal) || (tagb == TW_Denormal)) { - FPU_to_exp16(st0_ptr, &x); - FPU_to_exp16(b, &y); - st0_ptr = &x; - b = &y; - exp0 = exponent16(st0_ptr); - expb = exponent16(b); - } else { - exp0 = exponent(st0_ptr); - expb = exponent(b); - } - -#ifdef PARANOID - if (!(st0_ptr->sigh & 0x80000000)) - EXCEPTION(EX_Invalid); - if (!(b->sigh & 0x80000000)) - EXCEPTION(EX_Invalid); -#endif /* PARANOID */ - - diff = exp0 - expb; - if (diff == 0) { - diff = st0_ptr->sigh - b->sigh; /* Works only if ms bits are - identical */ - if (diff == 0) { - diff = st0_ptr->sigl > b->sigl; - if (diff == 0) - diff = -(st0_ptr->sigl < b->sigl); - } - } - - if (diff > 0) { - return ((st0_sign == SIGN_POS) ? COMP_A_gt_B : COMP_A_lt_B) - | (((st0_tag == TW_Denormal) || (tagb == TW_Denormal)) ? - COMP_Denormal : 0); - } - if (diff < 0) { - return ((st0_sign == SIGN_POS) ? COMP_A_lt_B : COMP_A_gt_B) - | (((st0_tag == TW_Denormal) || (tagb == TW_Denormal)) ? - COMP_Denormal : 0); - } - - return COMP_A_eq_B - | (((st0_tag == TW_Denormal) || (tagb == TW_Denormal)) ? - COMP_Denormal : 0); - -} - -/* This function requires that st(0) is not empty */ -int FPU_compare_st_data(FPU_REG const *loaded_data, u_char loaded_tag) -{ - int f, c; - - c = compare(loaded_data, loaded_tag); - - if (c & COMP_NaN) { - EXCEPTION(EX_Invalid); - f = SW_C3 | SW_C2 | SW_C0; - } else - switch (c & 7) { - case COMP_A_lt_B: - f = SW_C0; - break; - case COMP_A_eq_B: - f = SW_C3; - break; - case COMP_A_gt_B: - f = 0; - break; - case COMP_No_Comp: - f = SW_C3 | SW_C2 | SW_C0; - break; - default: -#ifdef PARANOID - EXCEPTION(EX_INTERNAL | 0x121); -#endif /* PARANOID */ - f = SW_C3 | SW_C2 | SW_C0; - break; - } - setcc(f); - if (c & COMP_Denormal) { - return denormal_operand() < 0; - } - return 0; -} - -static int compare_st_st(int nr) -{ - int f, c; - FPU_REG *st_ptr; - - if (!NOT_EMPTY(0) || !NOT_EMPTY(nr)) { - setcc(SW_C3 | SW_C2 | SW_C0); - /* Stack fault */ - EXCEPTION(EX_StackUnder); - return !(control_word & CW_Invalid); - } - - st_ptr = &st(nr); - c = compare(st_ptr, FPU_gettagi(nr)); - if (c & COMP_NaN) { - setcc(SW_C3 | SW_C2 | SW_C0); - EXCEPTION(EX_Invalid); - return !(control_word & CW_Invalid); - } else - switch (c & 7) { - case COMP_A_lt_B: - f = SW_C0; - break; - case COMP_A_eq_B: - f = SW_C3; - break; - case COMP_A_gt_B: - f = 0; - break; - case COMP_No_Comp: - f = SW_C3 | SW_C2 | SW_C0; - break; - default: -#ifdef PARANOID - EXCEPTION(EX_INTERNAL | 0x122); -#endif /* PARANOID */ - f = SW_C3 | SW_C2 | SW_C0; - break; - } - setcc(f); - if (c & COMP_Denormal) { - return denormal_operand() < 0; - } - return 0; -} - -static int compare_i_st_st(int nr) -{ - int f, c; - FPU_REG *st_ptr; - - if (!NOT_EMPTY(0) || !NOT_EMPTY(nr)) { - FPU_EFLAGS |= (X86_EFLAGS_ZF | X86_EFLAGS_PF | X86_EFLAGS_CF); - /* Stack fault */ - EXCEPTION(EX_StackUnder); - return !(control_word & CW_Invalid); - } - - partial_status &= ~SW_C0; - st_ptr = &st(nr); - c = compare(st_ptr, FPU_gettagi(nr)); - if (c & COMP_NaN) { - FPU_EFLAGS |= (X86_EFLAGS_ZF | X86_EFLAGS_PF | X86_EFLAGS_CF); - EXCEPTION(EX_Invalid); - return !(control_word & CW_Invalid); - } - - switch (c & 7) { - case COMP_A_lt_B: - f = X86_EFLAGS_CF; - break; - case COMP_A_eq_B: - f = X86_EFLAGS_ZF; - break; - case COMP_A_gt_B: - f = 0; - break; - case COMP_No_Comp: - f = X86_EFLAGS_ZF | X86_EFLAGS_PF | X86_EFLAGS_CF; - break; - default: -#ifdef PARANOID - EXCEPTION(EX_INTERNAL | 0x122); -#endif /* PARANOID */ - f = 0; - break; - } - FPU_EFLAGS = (FPU_EFLAGS & ~(X86_EFLAGS_ZF | X86_EFLAGS_PF | X86_EFLAGS_CF)) | f; - if (c & COMP_Denormal) { - return denormal_operand() < 0; - } - return 0; -} - -static int compare_u_st_st(int nr) -{ - int f = 0, c; - FPU_REG *st_ptr; - - if (!NOT_EMPTY(0) || !NOT_EMPTY(nr)) { - setcc(SW_C3 | SW_C2 | SW_C0); - /* Stack fault */ - EXCEPTION(EX_StackUnder); - return !(control_word & CW_Invalid); - } - - st_ptr = &st(nr); - c = compare(st_ptr, FPU_gettagi(nr)); - if (c & COMP_NaN) { - setcc(SW_C3 | SW_C2 | SW_C0); - if (c & COMP_SNaN) { /* This is the only difference between - un-ordered and ordinary comparisons */ - EXCEPTION(EX_Invalid); - return !(control_word & CW_Invalid); - } - return 0; - } else - switch (c & 7) { - case COMP_A_lt_B: - f = SW_C0; - break; - case COMP_A_eq_B: - f = SW_C3; - break; - case COMP_A_gt_B: - f = 0; - break; - case COMP_No_Comp: - f = SW_C3 | SW_C2 | SW_C0; - break; -#ifdef PARANOID - default: - EXCEPTION(EX_INTERNAL | 0x123); - f = SW_C3 | SW_C2 | SW_C0; - break; -#endif /* PARANOID */ - } - setcc(f); - if (c & COMP_Denormal) { - return denormal_operand() < 0; - } - return 0; -} - -static int compare_ui_st_st(int nr) -{ - int f = 0, c; - FPU_REG *st_ptr; - - if (!NOT_EMPTY(0) || !NOT_EMPTY(nr)) { - FPU_EFLAGS |= (X86_EFLAGS_ZF | X86_EFLAGS_PF | X86_EFLAGS_CF); - /* Stack fault */ - EXCEPTION(EX_StackUnder); - return !(control_word & CW_Invalid); - } - - partial_status &= ~SW_C0; - st_ptr = &st(nr); - c = compare(st_ptr, FPU_gettagi(nr)); - if (c & COMP_NaN) { - FPU_EFLAGS |= (X86_EFLAGS_ZF | X86_EFLAGS_PF | X86_EFLAGS_CF); - if (c & COMP_SNaN) { /* This is the only difference between - un-ordered and ordinary comparisons */ - EXCEPTION(EX_Invalid); - return !(control_word & CW_Invalid); - } - return 0; - } - - switch (c & 7) { - case COMP_A_lt_B: - f = X86_EFLAGS_CF; - break; - case COMP_A_eq_B: - f = X86_EFLAGS_ZF; - break; - case COMP_A_gt_B: - f = 0; - break; - case COMP_No_Comp: - f = X86_EFLAGS_ZF | X86_EFLAGS_PF | X86_EFLAGS_CF; - break; -#ifdef PARANOID - default: - EXCEPTION(EX_INTERNAL | 0x123); - f = 0; - break; -#endif /* PARANOID */ - } - FPU_EFLAGS = (FPU_EFLAGS & ~(X86_EFLAGS_ZF | X86_EFLAGS_PF | X86_EFLAGS_CF)) | f; - if (c & COMP_Denormal) { - return denormal_operand() < 0; - } - return 0; -} - -/*---------------------------------------------------------------------------*/ - -void fcom_st(void) -{ - /* fcom st(i) */ - compare_st_st(FPU_rm); -} - -void fcompst(void) -{ - /* fcomp st(i) */ - if (!compare_st_st(FPU_rm)) - FPU_pop(); -} - -void fcompp(void) -{ - /* fcompp */ - if (FPU_rm != 1) { - FPU_illegal(); - return; - } - if (!compare_st_st(1)) - poppop(); -} - -void fucom_(void) -{ - /* fucom st(i) */ - compare_u_st_st(FPU_rm); - -} - -void fucomp(void) -{ - /* fucomp st(i) */ - if (!compare_u_st_st(FPU_rm)) - FPU_pop(); -} - -void fucompp(void) -{ - /* fucompp */ - if (FPU_rm == 1) { - if (!compare_u_st_st(1)) - poppop(); - } else - FPU_illegal(); -} - -/* P6+ compare-to-EFLAGS ops */ - -void fcomi_(void) -{ - /* fcomi st(i) */ - compare_i_st_st(FPU_rm); -} - -void fcomip(void) -{ - /* fcomip st(i) */ - if (!compare_i_st_st(FPU_rm)) - FPU_pop(); -} - -void fucomi_(void) -{ - /* fucomi st(i) */ - compare_ui_st_st(FPU_rm); -} - -void fucomip(void) -{ - /* fucomip st(i) */ - if (!compare_ui_st_st(FPU_rm)) - FPU_pop(); -} diff --git a/arch/x86/math-emu/reg_constant.c b/arch/x86/math-emu/reg_constant.c deleted file mode 100644 index 003a0b2753e6..000000000000 --- a/arch/x86/math-emu/reg_constant.c +++ /dev/null @@ -1,123 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/*---------------------------------------------------------------------------+ - | reg_constant.c | - | | - | All of the constant FPU_REGs | - | | - | Copyright (C) 1992,1993,1994,1997 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, | - | Australia. E-mail billm@suburbia.net | - | | - | | - +---------------------------------------------------------------------------*/ - -#include "fpu_system.h" -#include "fpu_emu.h" -#include "status_w.h" -#include "reg_constant.h" -#include "control_w.h" - -#define MAKE_REG(s, e, l, h) { l, h, \ - (u16)((EXTENDED_Ebias+(e)) | ((SIGN_##s != 0)*0x8000)) } - -FPU_REG const CONST_1 = MAKE_REG(POS, 0, 0x00000000, 0x80000000); -#if 0 -FPU_REG const CONST_2 = MAKE_REG(POS, 1, 0x00000000, 0x80000000); -FPU_REG const CONST_HALF = MAKE_REG(POS, -1, 0x00000000, 0x80000000); -#endif /* 0 */ -static FPU_REG const CONST_L2T = MAKE_REG(POS, 1, 0xcd1b8afe, 0xd49a784b); -static FPU_REG const CONST_L2E = MAKE_REG(POS, 0, 0x5c17f0bc, 0xb8aa3b29); -FPU_REG const CONST_PI = MAKE_REG(POS, 1, 0x2168c235, 0xc90fdaa2); -FPU_REG const CONST_PI2 = MAKE_REG(POS, 0, 0x2168c235, 0xc90fdaa2); -FPU_REG const CONST_PI4 = MAKE_REG(POS, -1, 0x2168c235, 0xc90fdaa2); -static FPU_REG const CONST_LG2 = MAKE_REG(POS, -2, 0xfbcff799, 0x9a209a84); -static FPU_REG const CONST_LN2 = MAKE_REG(POS, -1, 0xd1cf79ac, 0xb17217f7); - -/* Extra bits to take pi/2 to more than 128 bits precision. */ -FPU_REG const CONST_PI2extra = MAKE_REG(NEG, -66, - 0xfc8f8cbb, 0xece675d1); - -/* Only the sign (and tag) is used in internal zeroes */ -FPU_REG const CONST_Z = MAKE_REG(POS, EXP_UNDER, 0x0, 0x0); - -/* Only the sign and significand (and tag) are used in internal NaNs */ -/* The 80486 never generates one of these -FPU_REG const CONST_SNAN = MAKE_REG(POS, EXP_OVER, 0x00000001, 0x80000000); - */ -/* This is the real indefinite QNaN */ -FPU_REG const CONST_QNaN = MAKE_REG(NEG, EXP_OVER, 0x00000000, 0xC0000000); - -/* Only the sign (and tag) is used in internal infinities */ -FPU_REG const CONST_INF = MAKE_REG(POS, EXP_OVER, 0x00000000, 0x80000000); - -static void fld_const(FPU_REG const * c, int adj, u_char tag) -{ - FPU_REG *st_new_ptr; - - if (STACK_OVERFLOW) { - FPU_stack_overflow(); - return; - } - push(); - reg_copy(c, st_new_ptr); - st_new_ptr->sigl += adj; /* For all our fldxxx constants, we don't need to - borrow or carry. */ - FPU_settag0(tag); - clear_C1(); -} - -/* A fast way to find out whether x is one of RC_DOWN or RC_CHOP - (and not one of RC_RND or RC_UP). - */ -#define DOWN_OR_CHOP(x) (x & RC_DOWN) - -static void fld1(int rc) -{ - fld_const(&CONST_1, 0, TAG_Valid); -} - -static void fldl2t(int rc) -{ - fld_const(&CONST_L2T, (rc == RC_UP) ? 1 : 0, TAG_Valid); -} - -static void fldl2e(int rc) -{ - fld_const(&CONST_L2E, DOWN_OR_CHOP(rc) ? -1 : 0, TAG_Valid); -} - -static void fldpi(int rc) -{ - fld_const(&CONST_PI, DOWN_OR_CHOP(rc) ? -1 : 0, TAG_Valid); -} - -static void fldlg2(int rc) -{ - fld_const(&CONST_LG2, DOWN_OR_CHOP(rc) ? -1 : 0, TAG_Valid); -} - -static void fldln2(int rc) -{ - fld_const(&CONST_LN2, DOWN_OR_CHOP(rc) ? -1 : 0, TAG_Valid); -} - -static void fldz(int rc) -{ - fld_const(&CONST_Z, 0, TAG_Zero); -} - -typedef void (*FUNC_RC) (int); - -static void FPU_RC_illegal(int unused) -{ - FPU_illegal(); -} - -static FUNC_RC constants_table[] = { - fld1, fldl2t, fldl2e, fldpi, fldlg2, fldln2, fldz, FPU_RC_illegal -}; - -void fconst(void) -{ - (constants_table[FPU_rm]) (control_word & CW_RC); -} diff --git a/arch/x86/math-emu/reg_constant.h b/arch/x86/math-emu/reg_constant.h deleted file mode 100644 index f2fdd344dcaa..000000000000 --- a/arch/x86/math-emu/reg_constant.h +++ /dev/null @@ -1,26 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/*---------------------------------------------------------------------------+ - | reg_constant.h | - | | - | Copyright (C) 1992 W. Metzenthen, 22 Parker St, Ormond, Vic 3163, | - | Australia. E-mail billm@vaxc.cc.monash.edu.au | - | | - +---------------------------------------------------------------------------*/ - -#ifndef _REG_CONSTANT_H_ -#define _REG_CONSTANT_H_ - -#include "fpu_emu.h" - -extern FPU_REG const CONST_1; -extern FPU_REG const CONST_PI; -extern FPU_REG const CONST_PI2; -extern FPU_REG const CONST_PI2extra; -extern FPU_REG const CONST_PI4; -extern FPU_REG const CONST_Z; -extern FPU_REG const CONST_PINF; -extern FPU_REG const CONST_INF; -extern FPU_REG const CONST_MINF; -extern FPU_REG const CONST_QNaN; - -#endif /* _REG_CONSTANT_H_ */ diff --git a/arch/x86/math-emu/reg_convert.c b/arch/x86/math-emu/reg_convert.c deleted file mode 100644 index 251180623ce0..000000000000 --- a/arch/x86/math-emu/reg_convert.c +++ /dev/null @@ -1,47 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/*---------------------------------------------------------------------------+ - | reg_convert.c | - | | - | Convert register representation. | - | | - | Copyright (C) 1992,1993,1994,1996,1997 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia | - | E-mail billm@suburbia.net | - | | - | | - +---------------------------------------------------------------------------*/ - -#include "exception.h" -#include "fpu_emu.h" - -int FPU_to_exp16(FPU_REG const *a, FPU_REG *x) -{ - int sign = getsign(a); - - *(long long *)&(x->sigl) = *(const long long *)&(a->sigl); - - /* Set up the exponent as a 16 bit quantity. */ - setexponent16(x, exponent(a)); - - if (exponent16(x) == EXP_UNDER) { - /* The number is a de-normal or pseudodenormal. */ - /* We only deal with the significand and exponent. */ - - if (x->sigh & 0x80000000) { - /* Is a pseudodenormal. */ - /* This is non-80486 behaviour because the number - loses its 'denormal' identity. */ - addexponent(x, 1); - } else { - /* Is a denormal. */ - addexponent(x, 1); - FPU_normalize_nuo(x); - } - } - - if (!(x->sigh & 0x80000000)) { - EXCEPTION(EX_INTERNAL | 0x180); - } - - return sign; -} diff --git a/arch/x86/math-emu/reg_divide.c b/arch/x86/math-emu/reg_divide.c deleted file mode 100644 index 08c2f6de0288..000000000000 --- a/arch/x86/math-emu/reg_divide.c +++ /dev/null @@ -1,183 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/*---------------------------------------------------------------------------+ - | reg_divide.c | - | | - | Divide one FPU_REG by another and put the result in a destination FPU_REG.| - | | - | Copyright (C) 1996 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia | - | E-mail billm@jacobi.maths.monash.edu.au | - | | - | Return value is the tag of the answer, or-ed with FPU_Exception if | - | one was raised, or -1 on internal error. | - | | - +---------------------------------------------------------------------------*/ - -/*---------------------------------------------------------------------------+ - | The destination may be any FPU_REG, including one of the source FPU_REGs. | - +---------------------------------------------------------------------------*/ - -#include "exception.h" -#include "reg_constant.h" -#include "fpu_emu.h" -#include "fpu_system.h" - -/* - Divide one register by another and put the result into a third register. - */ -int FPU_div(int flags, int rm, int control_w) -{ - FPU_REG x, y; - FPU_REG const *a, *b, *st0_ptr, *st_ptr; - FPU_REG *dest; - u_char taga, tagb, signa, signb, sign, saved_sign; - int tag, deststnr; - - if (flags & DEST_RM) - deststnr = rm; - else - deststnr = 0; - - if (flags & REV) { - b = &st(0); - st0_ptr = b; - tagb = FPU_gettag0(); - if (flags & LOADED) { - a = (FPU_REG *) rm; - taga = flags & 0x0f; - } else { - a = &st(rm); - st_ptr = a; - taga = FPU_gettagi(rm); - } - } else { - a = &st(0); - st0_ptr = a; - taga = FPU_gettag0(); - if (flags & LOADED) { - b = (FPU_REG *) rm; - tagb = flags & 0x0f; - } else { - b = &st(rm); - st_ptr = b; - tagb = FPU_gettagi(rm); - } - } - - signa = getsign(a); - signb = getsign(b); - - sign = signa ^ signb; - - dest = &st(deststnr); - saved_sign = getsign(dest); - - if (!(taga | tagb)) { - /* Both regs Valid, this should be the most common case. */ - reg_copy(a, &x); - reg_copy(b, &y); - setpositive(&x); - setpositive(&y); - tag = FPU_u_div(&x, &y, dest, control_w, sign); - - if (tag < 0) - return tag; - - FPU_settagi(deststnr, tag); - return tag; - } - - if (taga == TAG_Special) - taga = FPU_Special(a); - if (tagb == TAG_Special) - tagb = FPU_Special(b); - - if (((taga == TAG_Valid) && (tagb == TW_Denormal)) - || ((taga == TW_Denormal) && (tagb == TAG_Valid)) - || ((taga == TW_Denormal) && (tagb == TW_Denormal))) { - if (denormal_operand() < 0) - return FPU_Exception; - - FPU_to_exp16(a, &x); - FPU_to_exp16(b, &y); - tag = FPU_u_div(&x, &y, dest, control_w, sign); - if (tag < 0) - return tag; - - FPU_settagi(deststnr, tag); - return tag; - } else if ((taga <= TW_Denormal) && (tagb <= TW_Denormal)) { - if (tagb != TAG_Zero) { - /* Want to find Zero/Valid */ - if (tagb == TW_Denormal) { - if (denormal_operand() < 0) - return FPU_Exception; - } - - /* The result is zero. */ - FPU_copy_to_regi(&CONST_Z, TAG_Zero, deststnr); - setsign(dest, sign); - return TAG_Zero; - } - /* We have an exception condition, either 0/0 or Valid/Zero. */ - if (taga == TAG_Zero) { - /* 0/0 */ - return arith_invalid(deststnr); - } - /* Valid/Zero */ - return FPU_divide_by_zero(deststnr, sign); - } - /* Must have infinities, NaNs, etc */ - else if ((taga == TW_NaN) || (tagb == TW_NaN)) { - if (flags & LOADED) - return real_2op_NaN((FPU_REG *) rm, flags & 0x0f, 0, - st0_ptr); - - if (flags & DEST_RM) { - int tag; - tag = FPU_gettag0(); - if (tag == TAG_Special) - tag = FPU_Special(st0_ptr); - return real_2op_NaN(st0_ptr, tag, rm, - (flags & REV) ? st0_ptr : &st(rm)); - } else { - int tag; - tag = FPU_gettagi(rm); - if (tag == TAG_Special) - tag = FPU_Special(&st(rm)); - return real_2op_NaN(&st(rm), tag, 0, - (flags & REV) ? st0_ptr : &st(rm)); - } - } else if (taga == TW_Infinity) { - if (tagb == TW_Infinity) { - /* infinity/infinity */ - return arith_invalid(deststnr); - } else { - /* tagb must be Valid or Zero */ - if ((tagb == TW_Denormal) && (denormal_operand() < 0)) - return FPU_Exception; - - /* Infinity divided by Zero or Valid does - not raise and exception, but returns Infinity */ - FPU_copy_to_regi(a, TAG_Special, deststnr); - setsign(dest, sign); - return taga; - } - } else if (tagb == TW_Infinity) { - if ((taga == TW_Denormal) && (denormal_operand() < 0)) - return FPU_Exception; - - /* The result is zero. */ - FPU_copy_to_regi(&CONST_Z, TAG_Zero, deststnr); - setsign(dest, sign); - return TAG_Zero; - } -#ifdef PARANOID - else { - EXCEPTION(EX_INTERNAL | 0x102); - return FPU_Exception; - } -#endif /* PARANOID */ - - return 0; -} diff --git a/arch/x86/math-emu/reg_ld_str.c b/arch/x86/math-emu/reg_ld_str.c deleted file mode 100644 index 7e4521fbe7da..000000000000 --- a/arch/x86/math-emu/reg_ld_str.c +++ /dev/null @@ -1,1220 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/*---------------------------------------------------------------------------+ - | reg_ld_str.c | - | | - | All of the functions which transfer data between user memory and FPU_REGs.| - | | - | Copyright (C) 1992,1993,1994,1996,1997 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia | - | E-mail billm@suburbia.net | - | | - | | - +---------------------------------------------------------------------------*/ - -/*---------------------------------------------------------------------------+ - | Note: | - | The file contains code which accesses user memory. | - | Emulator static data may change when user memory is accessed, due to | - | other processes using the emulator while swapping is in progress. | - +---------------------------------------------------------------------------*/ - -#include "fpu_emu.h" - -#include - -#include "fpu_system.h" -#include "exception.h" -#include "reg_constant.h" -#include "control_w.h" -#include "status_w.h" - -#define DOUBLE_Emax 1023 /* largest valid exponent */ -#define DOUBLE_Ebias 1023 -#define DOUBLE_Emin (-1022) /* smallest valid exponent */ - -#define SINGLE_Emax 127 /* largest valid exponent */ -#define SINGLE_Ebias 127 -#define SINGLE_Emin (-126) /* smallest valid exponent */ - -static u_char normalize_no_excep(FPU_REG *r, int exp, int sign) -{ - u_char tag; - - setexponent16(r, exp); - - tag = FPU_normalize_nuo(r); - stdexp(r); - if (sign) - setnegative(r); - - return tag; -} - -int FPU_tagof(FPU_REG *ptr) -{ - int exp; - - exp = exponent16(ptr) & 0x7fff; - if (exp == 0) { - if (!(ptr->sigh | ptr->sigl)) { - return TAG_Zero; - } - /* The number is a de-normal or pseudodenormal. */ - return TAG_Special; - } - - if (exp == 0x7fff) { - /* Is an Infinity, a NaN, or an unsupported data type. */ - return TAG_Special; - } - - if (!(ptr->sigh & 0x80000000)) { - /* Unsupported data type. */ - /* Valid numbers have the ms bit set to 1. */ - /* Unnormal. */ - return TAG_Special; - } - - return TAG_Valid; -} - -/* Get a long double from user memory */ -int FPU_load_extended(long double __user *s, int stnr) -{ - FPU_REG *sti_ptr = &st(stnr); - - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(s, 10); - FPU_copy_from_user(sti_ptr, s, 10); - RE_ENTRANT_CHECK_ON; - - return FPU_tagof(sti_ptr); -} - -/* Get a double from user memory */ -int FPU_load_double(double __user *dfloat, FPU_REG *loaded_data) -{ - int exp, tag, negative; - unsigned m64, l64; - - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(dfloat, 8); - FPU_get_user(m64, 1 + (unsigned long __user *)dfloat); - FPU_get_user(l64, (unsigned long __user *)dfloat); - RE_ENTRANT_CHECK_ON; - - negative = (m64 & 0x80000000) ? SIGN_Negative : SIGN_Positive; - exp = ((m64 & 0x7ff00000) >> 20) - DOUBLE_Ebias + EXTENDED_Ebias; - m64 &= 0xfffff; - if (exp > DOUBLE_Emax + EXTENDED_Ebias) { - /* Infinity or NaN */ - if ((m64 == 0) && (l64 == 0)) { - /* +- infinity */ - loaded_data->sigh = 0x80000000; - loaded_data->sigl = 0x00000000; - exp = EXP_Infinity + EXTENDED_Ebias; - tag = TAG_Special; - } else { - /* Must be a signaling or quiet NaN */ - exp = EXP_NaN + EXTENDED_Ebias; - loaded_data->sigh = (m64 << 11) | 0x80000000; - loaded_data->sigh |= l64 >> 21; - loaded_data->sigl = l64 << 11; - tag = TAG_Special; /* The calling function must look for NaNs */ - } - } else if (exp < DOUBLE_Emin + EXTENDED_Ebias) { - /* Zero or de-normal */ - if ((m64 == 0) && (l64 == 0)) { - /* Zero */ - reg_copy(&CONST_Z, loaded_data); - exp = 0; - tag = TAG_Zero; - } else { - /* De-normal */ - loaded_data->sigh = m64 << 11; - loaded_data->sigh |= l64 >> 21; - loaded_data->sigl = l64 << 11; - - return normalize_no_excep(loaded_data, DOUBLE_Emin, - negative) - | (denormal_operand() < 0 ? FPU_Exception : 0); - } - } else { - loaded_data->sigh = (m64 << 11) | 0x80000000; - loaded_data->sigh |= l64 >> 21; - loaded_data->sigl = l64 << 11; - - tag = TAG_Valid; - } - - setexponent16(loaded_data, exp | negative); - - return tag; -} - -/* Get a float from user memory */ -int FPU_load_single(float __user *single, FPU_REG *loaded_data) -{ - unsigned m32; - int exp, tag, negative; - - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(single, 4); - FPU_get_user(m32, (unsigned long __user *)single); - RE_ENTRANT_CHECK_ON; - - negative = (m32 & 0x80000000) ? SIGN_Negative : SIGN_Positive; - - if (!(m32 & 0x7fffffff)) { - /* Zero */ - reg_copy(&CONST_Z, loaded_data); - addexponent(loaded_data, negative); - return TAG_Zero; - } - exp = ((m32 & 0x7f800000) >> 23) - SINGLE_Ebias + EXTENDED_Ebias; - m32 = (m32 & 0x7fffff) << 8; - if (exp < SINGLE_Emin + EXTENDED_Ebias) { - /* De-normals */ - loaded_data->sigh = m32; - loaded_data->sigl = 0; - - return normalize_no_excep(loaded_data, SINGLE_Emin, negative) - | (denormal_operand() < 0 ? FPU_Exception : 0); - } else if (exp > SINGLE_Emax + EXTENDED_Ebias) { - /* Infinity or NaN */ - if (m32 == 0) { - /* +- infinity */ - loaded_data->sigh = 0x80000000; - loaded_data->sigl = 0x00000000; - exp = EXP_Infinity + EXTENDED_Ebias; - tag = TAG_Special; - } else { - /* Must be a signaling or quiet NaN */ - exp = EXP_NaN + EXTENDED_Ebias; - loaded_data->sigh = m32 | 0x80000000; - loaded_data->sigl = 0; - tag = TAG_Special; /* The calling function must look for NaNs */ - } - } else { - loaded_data->sigh = m32 | 0x80000000; - loaded_data->sigl = 0; - tag = TAG_Valid; - } - - setexponent16(loaded_data, exp | negative); /* Set the sign. */ - - return tag; -} - -/* Get a long long from user memory */ -int FPU_load_int64(long long __user *_s) -{ - long long s; - int sign; - FPU_REG *st0_ptr = &st(0); - - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(_s, 8); - if (copy_from_user(&s, _s, 8)) - FPU_abort; - RE_ENTRANT_CHECK_ON; - - if (s == 0) { - reg_copy(&CONST_Z, st0_ptr); - return TAG_Zero; - } - - if (s > 0) - sign = SIGN_Positive; - else { - s = -s; - sign = SIGN_Negative; - } - - significand(st0_ptr) = s; - - return normalize_no_excep(st0_ptr, 63, sign); -} - -/* Get a long from user memory */ -int FPU_load_int32(long __user *_s, FPU_REG *loaded_data) -{ - long s; - int negative; - - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(_s, 4); - FPU_get_user(s, _s); - RE_ENTRANT_CHECK_ON; - - if (s == 0) { - reg_copy(&CONST_Z, loaded_data); - return TAG_Zero; - } - - if (s > 0) - negative = SIGN_Positive; - else { - s = -s; - negative = SIGN_Negative; - } - - loaded_data->sigh = s; - loaded_data->sigl = 0; - - return normalize_no_excep(loaded_data, 31, negative); -} - -/* Get a short from user memory */ -int FPU_load_int16(short __user *_s, FPU_REG *loaded_data) -{ - int s, negative; - - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(_s, 2); - /* Cast as short to get the sign extended. */ - FPU_get_user(s, _s); - RE_ENTRANT_CHECK_ON; - - if (s == 0) { - reg_copy(&CONST_Z, loaded_data); - return TAG_Zero; - } - - if (s > 0) - negative = SIGN_Positive; - else { - s = -s; - negative = SIGN_Negative; - } - - loaded_data->sigh = s << 16; - loaded_data->sigl = 0; - - return normalize_no_excep(loaded_data, 15, negative); -} - -/* Get a packed bcd array from user memory */ -int FPU_load_bcd(u_char __user *s) -{ - FPU_REG *st0_ptr = &st(0); - int pos; - u_char bcd; - long long l = 0; - int sign; - - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(s, 10); - RE_ENTRANT_CHECK_ON; - for (pos = 8; pos >= 0; pos--) { - l *= 10; - RE_ENTRANT_CHECK_OFF; - FPU_get_user(bcd, s + pos); - RE_ENTRANT_CHECK_ON; - l += bcd >> 4; - l *= 10; - l += bcd & 0x0f; - } - - RE_ENTRANT_CHECK_OFF; - FPU_get_user(sign, s + 9); - sign = sign & 0x80 ? SIGN_Negative : SIGN_Positive; - RE_ENTRANT_CHECK_ON; - - if (l == 0) { - reg_copy(&CONST_Z, st0_ptr); - addexponent(st0_ptr, sign); /* Set the sign. */ - return TAG_Zero; - } else { - significand(st0_ptr) = l; - return normalize_no_excep(st0_ptr, 63, sign); - } -} - -/*===========================================================================*/ - -/* Put a long double into user memory */ -int FPU_store_extended(FPU_REG *st0_ptr, u_char st0_tag, - long double __user * d) -{ - /* - The only exception raised by an attempt to store to an - extended format is the Invalid Stack exception, i.e. - attempting to store from an empty register. - */ - - if (st0_tag != TAG_Empty) { - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(d, 10); - - FPU_put_user(st0_ptr->sigl, (unsigned long __user *)d); - FPU_put_user(st0_ptr->sigh, - (unsigned long __user *)((u_char __user *) d + 4)); - FPU_put_user(exponent16(st0_ptr), - (unsigned short __user *)((u_char __user *) d + - 8)); - RE_ENTRANT_CHECK_ON; - - return 1; - } - - /* Empty register (stack underflow) */ - EXCEPTION(EX_StackUnder); - if (control_word & CW_Invalid) { - /* The masked response */ - /* Put out the QNaN indefinite */ - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(d, 10); - FPU_put_user(0, (unsigned long __user *)d); - FPU_put_user(0xc0000000, 1 + (unsigned long __user *)d); - FPU_put_user(0xffff, 4 + (short __user *)d); - RE_ENTRANT_CHECK_ON; - return 1; - } else - return 0; - -} - -/* Put a double into user memory */ -int FPU_store_double(FPU_REG *st0_ptr, u_char st0_tag, double __user *dfloat) -{ - unsigned long l[2]; - unsigned long increment = 0; /* avoid gcc warnings */ - int precision_loss; - int exp; - FPU_REG tmp; - - l[0] = 0; - l[1] = 0; - if (st0_tag == TAG_Valid) { - reg_copy(st0_ptr, &tmp); - exp = exponent(&tmp); - - if (exp < DOUBLE_Emin) { /* It may be a denormal */ - addexponent(&tmp, -DOUBLE_Emin + 52); /* largest exp to be 51 */ -denormal_arg: - if ((precision_loss = FPU_round_to_int(&tmp, st0_tag))) { -#ifdef PECULIAR_486 - /* Did it round to a non-denormal ? */ - /* This behaviour might be regarded as peculiar, it appears - that the 80486 rounds to the dest precision, then - converts to decide underflow. */ - if (! - ((tmp.sigh == 0x00100000) && (tmp.sigl == 0) - && (st0_ptr->sigl & 0x000007ff))) -#endif /* PECULIAR_486 */ - { - EXCEPTION(EX_Underflow); - /* This is a special case: see sec 16.2.5.1 of - the 80486 book */ - if (!(control_word & CW_Underflow)) - return 0; - } - EXCEPTION(precision_loss); - if (!(control_word & CW_Precision)) - return 0; - } - l[0] = tmp.sigl; - l[1] = tmp.sigh; - } else { - if (tmp.sigl & 0x000007ff) { - precision_loss = 1; - switch (control_word & CW_RC) { - case RC_RND: - /* Rounding can get a little messy.. */ - increment = ((tmp.sigl & 0x7ff) > 0x400) | /* nearest */ - ((tmp.sigl & 0xc00) == 0xc00); /* odd -> even */ - break; - case RC_DOWN: /* towards -infinity */ - increment = - signpositive(&tmp) ? 0 : tmp. - sigl & 0x7ff; - break; - case RC_UP: /* towards +infinity */ - increment = - signpositive(&tmp) ? tmp. - sigl & 0x7ff : 0; - break; - case RC_CHOP: - increment = 0; - break; - } - - /* Truncate the mantissa */ - tmp.sigl &= 0xfffff800; - - if (increment) { - if (tmp.sigl >= 0xfffff800) { - /* the sigl part overflows */ - if (tmp.sigh == 0xffffffff) { - /* The sigh part overflows */ - tmp.sigh = 0x80000000; - exp++; - if (exp >= EXP_OVER) - goto overflow; - } else { - tmp.sigh++; - } - tmp.sigl = 0x00000000; - } else { - /* We only need to increment sigl */ - tmp.sigl += 0x00000800; - } - } - } else - precision_loss = 0; - - l[0] = (tmp.sigl >> 11) | (tmp.sigh << 21); - l[1] = ((tmp.sigh >> 11) & 0xfffff); - - if (exp > DOUBLE_Emax) { - overflow: - EXCEPTION(EX_Overflow); - if (!(control_word & CW_Overflow)) - return 0; - set_precision_flag_up(); - if (!(control_word & CW_Precision)) - return 0; - - /* This is a special case: see sec 16.2.5.1 of the 80486 book */ - /* Overflow to infinity */ - l[1] = 0x7ff00000; /* Set to + INF */ - } else { - if (precision_loss) { - if (increment) - set_precision_flag_up(); - else - set_precision_flag_down(); - } - /* Add the exponent */ - l[1] |= (((exp + DOUBLE_Ebias) & 0x7ff) << 20); - } - } - } else if (st0_tag == TAG_Zero) { - /* Number is zero */ - } else if (st0_tag == TAG_Special) { - st0_tag = FPU_Special(st0_ptr); - if (st0_tag == TW_Denormal) { - /* A denormal will always underflow. */ -#ifndef PECULIAR_486 - /* An 80486 is supposed to be able to generate - a denormal exception here, but... */ - /* Underflow has priority. */ - if (control_word & CW_Underflow) - denormal_operand(); -#endif /* PECULIAR_486 */ - reg_copy(st0_ptr, &tmp); - goto denormal_arg; - } else if (st0_tag == TW_Infinity) { - l[1] = 0x7ff00000; - } else if (st0_tag == TW_NaN) { - /* Is it really a NaN ? */ - if ((exponent(st0_ptr) == EXP_OVER) - && (st0_ptr->sigh & 0x80000000)) { - /* See if we can get a valid NaN from the FPU_REG */ - l[0] = - (st0_ptr->sigl >> 11) | (st0_ptr-> - sigh << 21); - l[1] = ((st0_ptr->sigh >> 11) & 0xfffff); - if (!(st0_ptr->sigh & 0x40000000)) { - /* It is a signalling NaN */ - EXCEPTION(EX_Invalid); - if (!(control_word & CW_Invalid)) - return 0; - l[1] |= (0x40000000 >> 11); - } - l[1] |= 0x7ff00000; - } else { - /* It is an unsupported data type */ - EXCEPTION(EX_Invalid); - if (!(control_word & CW_Invalid)) - return 0; - l[1] = 0xfff80000; - } - } - } else if (st0_tag == TAG_Empty) { - /* Empty register (stack underflow) */ - EXCEPTION(EX_StackUnder); - if (control_word & CW_Invalid) { - /* The masked response */ - /* Put out the QNaN indefinite */ - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(dfloat, 8); - FPU_put_user(0, (unsigned long __user *)dfloat); - FPU_put_user(0xfff80000, - 1 + (unsigned long __user *)dfloat); - RE_ENTRANT_CHECK_ON; - return 1; - } else - return 0; - } - if (getsign(st0_ptr)) - l[1] |= 0x80000000; - - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(dfloat, 8); - FPU_put_user(l[0], (unsigned long __user *)dfloat); - FPU_put_user(l[1], 1 + (unsigned long __user *)dfloat); - RE_ENTRANT_CHECK_ON; - - return 1; -} - -/* Put a float into user memory */ -int FPU_store_single(FPU_REG *st0_ptr, u_char st0_tag, float __user *single) -{ - long templ = 0; - unsigned long increment = 0; /* avoid gcc warnings */ - int precision_loss; - int exp; - FPU_REG tmp; - - if (st0_tag == TAG_Valid) { - - reg_copy(st0_ptr, &tmp); - exp = exponent(&tmp); - - if (exp < SINGLE_Emin) { - addexponent(&tmp, -SINGLE_Emin + 23); /* largest exp to be 22 */ - - denormal_arg: - - if ((precision_loss = FPU_round_to_int(&tmp, st0_tag))) { -#ifdef PECULIAR_486 - /* Did it round to a non-denormal ? */ - /* This behaviour might be regarded as peculiar, it appears - that the 80486 rounds to the dest precision, then - converts to decide underflow. */ - if (!((tmp.sigl == 0x00800000) && - ((st0_ptr->sigh & 0x000000ff) - || st0_ptr->sigl))) -#endif /* PECULIAR_486 */ - { - EXCEPTION(EX_Underflow); - /* This is a special case: see sec 16.2.5.1 of - the 80486 book */ - if (!(control_word & CW_Underflow)) - return 0; - } - EXCEPTION(precision_loss); - if (!(control_word & CW_Precision)) - return 0; - } - templ = tmp.sigl; - } else { - if (tmp.sigl | (tmp.sigh & 0x000000ff)) { - unsigned long sigh = tmp.sigh; - unsigned long sigl = tmp.sigl; - - precision_loss = 1; - switch (control_word & CW_RC) { - case RC_RND: - increment = ((sigh & 0xff) > 0x80) /* more than half */ - ||(((sigh & 0xff) == 0x80) && sigl) /* more than half */ - ||((sigh & 0x180) == 0x180); /* round to even */ - break; - case RC_DOWN: /* towards -infinity */ - increment = signpositive(&tmp) - ? 0 : (sigl | (sigh & 0xff)); - break; - case RC_UP: /* towards +infinity */ - increment = signpositive(&tmp) - ? (sigl | (sigh & 0xff)) : 0; - break; - case RC_CHOP: - increment = 0; - break; - } - - /* Truncate part of the mantissa */ - tmp.sigl = 0; - - if (increment) { - if (sigh >= 0xffffff00) { - /* The sigh part overflows */ - tmp.sigh = 0x80000000; - exp++; - if (exp >= EXP_OVER) - goto overflow; - } else { - tmp.sigh &= 0xffffff00; - tmp.sigh += 0x100; - } - } else { - tmp.sigh &= 0xffffff00; /* Finish the truncation */ - } - } else - precision_loss = 0; - - templ = (tmp.sigh >> 8) & 0x007fffff; - - if (exp > SINGLE_Emax) { - overflow: - EXCEPTION(EX_Overflow); - if (!(control_word & CW_Overflow)) - return 0; - set_precision_flag_up(); - if (!(control_word & CW_Precision)) - return 0; - - /* This is a special case: see sec 16.2.5.1 of the 80486 book. */ - /* Masked response is overflow to infinity. */ - templ = 0x7f800000; - } else { - if (precision_loss) { - if (increment) - set_precision_flag_up(); - else - set_precision_flag_down(); - } - /* Add the exponent */ - templ |= ((exp + SINGLE_Ebias) & 0xff) << 23; - } - } - } else if (st0_tag == TAG_Zero) { - templ = 0; - } else if (st0_tag == TAG_Special) { - st0_tag = FPU_Special(st0_ptr); - if (st0_tag == TW_Denormal) { - reg_copy(st0_ptr, &tmp); - - /* A denormal will always underflow. */ -#ifndef PECULIAR_486 - /* An 80486 is supposed to be able to generate - a denormal exception here, but... */ - /* Underflow has priority. */ - if (control_word & CW_Underflow) - denormal_operand(); -#endif /* PECULIAR_486 */ - goto denormal_arg; - } else if (st0_tag == TW_Infinity) { - templ = 0x7f800000; - } else if (st0_tag == TW_NaN) { - /* Is it really a NaN ? */ - if ((exponent(st0_ptr) == EXP_OVER) - && (st0_ptr->sigh & 0x80000000)) { - /* See if we can get a valid NaN from the FPU_REG */ - templ = st0_ptr->sigh >> 8; - if (!(st0_ptr->sigh & 0x40000000)) { - /* It is a signalling NaN */ - EXCEPTION(EX_Invalid); - if (!(control_word & CW_Invalid)) - return 0; - templ |= (0x40000000 >> 8); - } - templ |= 0x7f800000; - } else { - /* It is an unsupported data type */ - EXCEPTION(EX_Invalid); - if (!(control_word & CW_Invalid)) - return 0; - templ = 0xffc00000; - } - } -#ifdef PARANOID - else { - EXCEPTION(EX_INTERNAL | 0x164); - return 0; - } -#endif - } else if (st0_tag == TAG_Empty) { - /* Empty register (stack underflow) */ - EXCEPTION(EX_StackUnder); - if (control_word & EX_Invalid) { - /* The masked response */ - /* Put out the QNaN indefinite */ - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(single, 4); - FPU_put_user(0xffc00000, - (unsigned long __user *)single); - RE_ENTRANT_CHECK_ON; - return 1; - } else - return 0; - } -#ifdef PARANOID - else { - EXCEPTION(EX_INTERNAL | 0x163); - return 0; - } -#endif - if (getsign(st0_ptr)) - templ |= 0x80000000; - - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(single, 4); - FPU_put_user(templ, (unsigned long __user *)single); - RE_ENTRANT_CHECK_ON; - - return 1; -} - -/* Put a long long into user memory */ -int FPU_store_int64(FPU_REG *st0_ptr, u_char st0_tag, long long __user *d) -{ - FPU_REG t; - long long tll; - int precision_loss; - - if (st0_tag == TAG_Empty) { - /* Empty register (stack underflow) */ - EXCEPTION(EX_StackUnder); - goto invalid_operand; - } else if (st0_tag == TAG_Special) { - st0_tag = FPU_Special(st0_ptr); - if ((st0_tag == TW_Infinity) || (st0_tag == TW_NaN)) { - EXCEPTION(EX_Invalid); - goto invalid_operand; - } - } - - reg_copy(st0_ptr, &t); - precision_loss = FPU_round_to_int(&t, st0_tag); - ((long *)&tll)[0] = t.sigl; - ((long *)&tll)[1] = t.sigh; - if ((precision_loss == 1) || - ((t.sigh & 0x80000000) && - !((t.sigh == 0x80000000) && (t.sigl == 0) && signnegative(&t)))) { - EXCEPTION(EX_Invalid); - /* This is a special case: see sec 16.2.5.1 of the 80486 book */ - invalid_operand: - if (control_word & EX_Invalid) { - /* Produce something like QNaN "indefinite" */ - tll = 0x8000000000000000LL; - } else - return 0; - } else { - if (precision_loss) - set_precision_flag(precision_loss); - if (signnegative(&t)) - tll = -tll; - } - - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(d, 8); - if (copy_to_user(d, &tll, 8)) - FPU_abort; - RE_ENTRANT_CHECK_ON; - - return 1; -} - -/* Put a long into user memory */ -int FPU_store_int32(FPU_REG *st0_ptr, u_char st0_tag, long __user *d) -{ - FPU_REG t; - int precision_loss; - - if (st0_tag == TAG_Empty) { - /* Empty register (stack underflow) */ - EXCEPTION(EX_StackUnder); - goto invalid_operand; - } else if (st0_tag == TAG_Special) { - st0_tag = FPU_Special(st0_ptr); - if ((st0_tag == TW_Infinity) || (st0_tag == TW_NaN)) { - EXCEPTION(EX_Invalid); - goto invalid_operand; - } - } - - reg_copy(st0_ptr, &t); - precision_loss = FPU_round_to_int(&t, st0_tag); - if (t.sigh || - ((t.sigl & 0x80000000) && - !((t.sigl == 0x80000000) && signnegative(&t)))) { - EXCEPTION(EX_Invalid); - /* This is a special case: see sec 16.2.5.1 of the 80486 book */ - invalid_operand: - if (control_word & EX_Invalid) { - /* Produce something like QNaN "indefinite" */ - t.sigl = 0x80000000; - } else - return 0; - } else { - if (precision_loss) - set_precision_flag(precision_loss); - if (signnegative(&t)) - t.sigl = -(long)t.sigl; - } - - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(d, 4); - FPU_put_user(t.sigl, (unsigned long __user *)d); - RE_ENTRANT_CHECK_ON; - - return 1; -} - -/* Put a short into user memory */ -int FPU_store_int16(FPU_REG *st0_ptr, u_char st0_tag, short __user *d) -{ - FPU_REG t; - int precision_loss; - - if (st0_tag == TAG_Empty) { - /* Empty register (stack underflow) */ - EXCEPTION(EX_StackUnder); - goto invalid_operand; - } else if (st0_tag == TAG_Special) { - st0_tag = FPU_Special(st0_ptr); - if ((st0_tag == TW_Infinity) || (st0_tag == TW_NaN)) { - EXCEPTION(EX_Invalid); - goto invalid_operand; - } - } - - reg_copy(st0_ptr, &t); - precision_loss = FPU_round_to_int(&t, st0_tag); - if (t.sigh || - ((t.sigl & 0xffff8000) && - !((t.sigl == 0x8000) && signnegative(&t)))) { - EXCEPTION(EX_Invalid); - /* This is a special case: see sec 16.2.5.1 of the 80486 book */ - invalid_operand: - if (control_word & EX_Invalid) { - /* Produce something like QNaN "indefinite" */ - t.sigl = 0x8000; - } else - return 0; - } else { - if (precision_loss) - set_precision_flag(precision_loss); - if (signnegative(&t)) - t.sigl = -t.sigl; - } - - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(d, 2); - FPU_put_user((short)t.sigl, d); - RE_ENTRANT_CHECK_ON; - - return 1; -} - -/* Put a packed bcd array into user memory */ -int FPU_store_bcd(FPU_REG *st0_ptr, u_char st0_tag, u_char __user *d) -{ - FPU_REG t; - unsigned long long ll; - u_char b; - int i, precision_loss; - u_char sign = (getsign(st0_ptr) == SIGN_NEG) ? 0x80 : 0; - - if (st0_tag == TAG_Empty) { - /* Empty register (stack underflow) */ - EXCEPTION(EX_StackUnder); - goto invalid_operand; - } else if (st0_tag == TAG_Special) { - st0_tag = FPU_Special(st0_ptr); - if ((st0_tag == TW_Infinity) || (st0_tag == TW_NaN)) { - EXCEPTION(EX_Invalid); - goto invalid_operand; - } - } - - reg_copy(st0_ptr, &t); - precision_loss = FPU_round_to_int(&t, st0_tag); - ll = significand(&t); - - /* Check for overflow, by comparing with 999999999999999999 decimal. */ - if ((t.sigh > 0x0de0b6b3) || - ((t.sigh == 0x0de0b6b3) && (t.sigl > 0xa763ffff))) { - EXCEPTION(EX_Invalid); - /* This is a special case: see sec 16.2.5.1 of the 80486 book */ - invalid_operand: - if (control_word & CW_Invalid) { - /* Produce the QNaN "indefinite" */ - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(d, 10); - for (i = 0; i < 7; i++) - FPU_put_user(0, d + i); /* These bytes "undefined" */ - FPU_put_user(0xc0, d + 7); /* This byte "undefined" */ - FPU_put_user(0xff, d + 8); - FPU_put_user(0xff, d + 9); - RE_ENTRANT_CHECK_ON; - return 1; - } else - return 0; - } else if (precision_loss) { - /* Precision loss doesn't stop the data transfer */ - set_precision_flag(precision_loss); - } - - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(d, 10); - RE_ENTRANT_CHECK_ON; - for (i = 0; i < 9; i++) { - b = FPU_div_small(&ll, 10); - b |= (FPU_div_small(&ll, 10)) << 4; - RE_ENTRANT_CHECK_OFF; - FPU_put_user(b, d + i); - RE_ENTRANT_CHECK_ON; - } - RE_ENTRANT_CHECK_OFF; - FPU_put_user(sign, d + 9); - RE_ENTRANT_CHECK_ON; - - return 1; -} - -/*===========================================================================*/ - -/* r gets mangled such that sig is int, sign: - it is NOT normalized */ -/* The return value (in eax) is zero if the result is exact, - if bits are changed due to rounding, truncation, etc, then - a non-zero value is returned */ -/* Overflow is signaled by a non-zero return value (in eax). - In the case of overflow, the returned significand always has the - largest possible value */ -int FPU_round_to_int(FPU_REG *r, u_char tag) -{ - u_char very_big; - unsigned eax; - - if (tag == TAG_Zero) { - /* Make sure that zero is returned */ - significand(r) = 0; - return 0; /* o.k. */ - } - - if (exponent(r) > 63) { - r->sigl = r->sigh = ~0; /* The largest representable number */ - return 1; /* overflow */ - } - - eax = FPU_shrxs(&r->sigl, 63 - exponent(r)); - very_big = !(~(r->sigh) | ~(r->sigl)); /* test for 0xfff...fff */ -#define half_or_more (eax & 0x80000000) -#define frac_part (eax) -#define more_than_half ((eax & 0x80000001) == 0x80000001) - switch (control_word & CW_RC) { - case RC_RND: - if (more_than_half /* nearest */ - || (half_or_more && (r->sigl & 1))) { /* odd -> even */ - if (very_big) - return 1; /* overflow */ - significand(r)++; - return PRECISION_LOST_UP; - } - break; - case RC_DOWN: - if (frac_part && getsign(r)) { - if (very_big) - return 1; /* overflow */ - significand(r)++; - return PRECISION_LOST_UP; - } - break; - case RC_UP: - if (frac_part && !getsign(r)) { - if (very_big) - return 1; /* overflow */ - significand(r)++; - return PRECISION_LOST_UP; - } - break; - case RC_CHOP: - break; - } - - return eax ? PRECISION_LOST_DOWN : 0; - -} - -/*===========================================================================*/ - -u_char __user *fldenv(fpu_addr_modes addr_modes, u_char __user *s) -{ - unsigned short tag_word = 0; - u_char tag; - int i; - - if ((addr_modes.default_mode == VM86) || - ((addr_modes.default_mode == PM16) - ^ (addr_modes.override.operand_size == OP_SIZE_PREFIX))) { - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(s, 0x0e); - FPU_get_user(control_word, (unsigned short __user *)s); - FPU_get_user(partial_status, (unsigned short __user *)(s + 2)); - FPU_get_user(tag_word, (unsigned short __user *)(s + 4)); - FPU_get_user(instruction_address.offset, - (unsigned short __user *)(s + 6)); - FPU_get_user(instruction_address.selector, - (unsigned short __user *)(s + 8)); - FPU_get_user(operand_address.offset, - (unsigned short __user *)(s + 0x0a)); - FPU_get_user(operand_address.selector, - (unsigned short __user *)(s + 0x0c)); - RE_ENTRANT_CHECK_ON; - s += 0x0e; - if (addr_modes.default_mode == VM86) { - instruction_address.offset - += (instruction_address.selector & 0xf000) << 4; - operand_address.offset += - (operand_address.selector & 0xf000) << 4; - } - } else { - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(s, 0x1c); - FPU_get_user(control_word, (unsigned short __user *)s); - FPU_get_user(partial_status, (unsigned short __user *)(s + 4)); - FPU_get_user(tag_word, (unsigned short __user *)(s + 8)); - FPU_get_user(instruction_address.offset, - (unsigned long __user *)(s + 0x0c)); - FPU_get_user(instruction_address.selector, - (unsigned short __user *)(s + 0x10)); - FPU_get_user(instruction_address.opcode, - (unsigned short __user *)(s + 0x12)); - FPU_get_user(operand_address.offset, - (unsigned long __user *)(s + 0x14)); - FPU_get_user(operand_address.selector, - (unsigned long __user *)(s + 0x18)); - RE_ENTRANT_CHECK_ON; - s += 0x1c; - } - -#ifdef PECULIAR_486 - control_word &= ~0xe080; -#endif /* PECULIAR_486 */ - - top = (partial_status >> SW_Top_Shift) & 7; - - if (partial_status & ~control_word & CW_Exceptions) - partial_status |= (SW_Summary | SW_Backward); - else - partial_status &= ~(SW_Summary | SW_Backward); - - for (i = 0; i < 8; i++) { - tag = tag_word & 3; - tag_word >>= 2; - - if (tag == TAG_Empty) - /* New tag is empty. Accept it */ - FPU_settag(i, TAG_Empty); - else if (FPU_gettag(i) == TAG_Empty) { - /* Old tag is empty and new tag is not empty. New tag is determined - by old reg contents */ - if (exponent(&fpu_register(i)) == -EXTENDED_Ebias) { - if (! - (fpu_register(i).sigl | fpu_register(i). - sigh)) - FPU_settag(i, TAG_Zero); - else - FPU_settag(i, TAG_Special); - } else if (exponent(&fpu_register(i)) == - 0x7fff - EXTENDED_Ebias) { - FPU_settag(i, TAG_Special); - } else if (fpu_register(i).sigh & 0x80000000) - FPU_settag(i, TAG_Valid); - else - FPU_settag(i, TAG_Special); /* An Un-normal */ - } - /* Else old tag is not empty and new tag is not empty. Old tag - remains correct */ - } - - return s; -} - -void FPU_frstor(fpu_addr_modes addr_modes, u_char __user *data_address) -{ - int i, regnr; - u_char __user *s = fldenv(addr_modes, data_address); - int offset = (top & 7) * 10, other = 80 - offset; - - /* Copy all registers in stack order. */ - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(s, 80); - FPU_copy_from_user(register_base + offset, s, other); - if (offset) - FPU_copy_from_user(register_base, s + other, offset); - RE_ENTRANT_CHECK_ON; - - for (i = 0; i < 8; i++) { - regnr = (i + top) & 7; - if (FPU_gettag(regnr) != TAG_Empty) - /* The loaded data over-rides all other cases. */ - FPU_settag(regnr, FPU_tagof(&st(i))); - } - -} - -u_char __user *fstenv(fpu_addr_modes addr_modes, u_char __user *d) -{ - if ((addr_modes.default_mode == VM86) || - ((addr_modes.default_mode == PM16) - ^ (addr_modes.override.operand_size == OP_SIZE_PREFIX))) { - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(d, 14); -#ifdef PECULIAR_486 - FPU_put_user(control_word & ~0xe080, (unsigned long __user *)d); -#else - FPU_put_user(control_word, (unsigned short __user *)d); -#endif /* PECULIAR_486 */ - FPU_put_user(status_word(), (unsigned short __user *)(d + 2)); - FPU_put_user(fpu_tag_word, (unsigned short __user *)(d + 4)); - FPU_put_user(instruction_address.offset, - (unsigned short __user *)(d + 6)); - FPU_put_user(operand_address.offset, - (unsigned short __user *)(d + 0x0a)); - if (addr_modes.default_mode == VM86) { - FPU_put_user((instruction_address. - offset & 0xf0000) >> 4, - (unsigned short __user *)(d + 8)); - FPU_put_user((operand_address.offset & 0xf0000) >> 4, - (unsigned short __user *)(d + 0x0c)); - } else { - FPU_put_user(instruction_address.selector, - (unsigned short __user *)(d + 8)); - FPU_put_user(operand_address.selector, - (unsigned short __user *)(d + 0x0c)); - } - RE_ENTRANT_CHECK_ON; - d += 0x0e; - } else { - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(d, 7 * 4); -#ifdef PECULIAR_486 - control_word &= ~0xe080; - /* An 80486 sets nearly all of the reserved bits to 1. */ - control_word |= 0xffff0040; - partial_status = status_word() | 0xffff0000; - fpu_tag_word |= 0xffff0000; - I387->soft.fcs &= ~0xf8000000; - I387->soft.fos |= 0xffff0000; -#endif /* PECULIAR_486 */ - if (__copy_to_user(d, &control_word, 7 * 4)) - FPU_abort; - RE_ENTRANT_CHECK_ON; - d += 0x1c; - } - - control_word |= CW_Exceptions; - partial_status &= ~(SW_Summary | SW_Backward); - - return d; -} - -void fsave(fpu_addr_modes addr_modes, u_char __user *data_address) -{ - u_char __user *d; - int offset = (top & 7) * 10, other = 80 - offset; - - d = fstenv(addr_modes, data_address); - - RE_ENTRANT_CHECK_OFF; - FPU_access_ok(d, 80); - - /* Copy all registers in stack order. */ - if (__copy_to_user(d, register_base + offset, other)) - FPU_abort; - if (offset) - if (__copy_to_user(d + other, register_base, offset)) - FPU_abort; - RE_ENTRANT_CHECK_ON; - - finit(); -} - -/*===========================================================================*/ diff --git a/arch/x86/math-emu/reg_mul.c b/arch/x86/math-emu/reg_mul.c deleted file mode 100644 index d69618572f14..000000000000 --- a/arch/x86/math-emu/reg_mul.c +++ /dev/null @@ -1,116 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/*---------------------------------------------------------------------------+ - | reg_mul.c | - | | - | Multiply one FPU_REG by another, put the result in a destination FPU_REG. | - | | - | Copyright (C) 1992,1993,1997 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia | - | E-mail billm@suburbia.net | - | | - | Returns the tag of the result if no exceptions or errors occurred. | - | | - +---------------------------------------------------------------------------*/ - -/*---------------------------------------------------------------------------+ - | The destination may be any FPU_REG, including one of the source FPU_REGs. | - +---------------------------------------------------------------------------*/ - -#include "fpu_emu.h" -#include "exception.h" -#include "reg_constant.h" -#include "fpu_system.h" - -/* - Multiply two registers to give a register result. - The sources are st(deststnr) and (b,tagb,signb). - The destination is st(deststnr). - */ -/* This routine must be called with non-empty source registers */ -int FPU_mul(FPU_REG const *b, u_char tagb, int deststnr, int control_w) -{ - FPU_REG *a = &st(deststnr); - FPU_REG *dest = a; - u_char taga = FPU_gettagi(deststnr); - u_char saved_sign = getsign(dest); - u_char sign = (getsign(a) ^ getsign(b)); - int tag; - - if (!(taga | tagb)) { - /* Both regs Valid, this should be the most common case. */ - - tag = - FPU_u_mul(a, b, dest, control_w, sign, - exponent(a) + exponent(b)); - if (tag < 0) { - setsign(dest, saved_sign); - return tag; - } - FPU_settagi(deststnr, tag); - return tag; - } - - if (taga == TAG_Special) - taga = FPU_Special(a); - if (tagb == TAG_Special) - tagb = FPU_Special(b); - - if (((taga == TAG_Valid) && (tagb == TW_Denormal)) - || ((taga == TW_Denormal) && (tagb == TAG_Valid)) - || ((taga == TW_Denormal) && (tagb == TW_Denormal))) { - FPU_REG x, y; - if (denormal_operand() < 0) - return FPU_Exception; - - FPU_to_exp16(a, &x); - FPU_to_exp16(b, &y); - tag = FPU_u_mul(&x, &y, dest, control_w, sign, - exponent16(&x) + exponent16(&y)); - if (tag < 0) { - setsign(dest, saved_sign); - return tag; - } - FPU_settagi(deststnr, tag); - return tag; - } else if ((taga <= TW_Denormal) && (tagb <= TW_Denormal)) { - if (((tagb == TW_Denormal) || (taga == TW_Denormal)) - && (denormal_operand() < 0)) - return FPU_Exception; - - /* Must have either both arguments == zero, or - one valid and the other zero. - The result is therefore zero. */ - FPU_copy_to_regi(&CONST_Z, TAG_Zero, deststnr); - /* The 80486 book says that the answer is +0, but a real - 80486 behaves this way. - IEEE-754 apparently says it should be this way. */ - setsign(dest, sign); - return TAG_Zero; - } - /* Must have infinities, NaNs, etc */ - else if ((taga == TW_NaN) || (tagb == TW_NaN)) { - return real_2op_NaN(b, tagb, deststnr, &st(0)); - } else if (((taga == TW_Infinity) && (tagb == TAG_Zero)) - || ((tagb == TW_Infinity) && (taga == TAG_Zero))) { - return arith_invalid(deststnr); /* Zero*Infinity is invalid */ - } else if (((taga == TW_Denormal) || (tagb == TW_Denormal)) - && (denormal_operand() < 0)) { - return FPU_Exception; - } else if (taga == TW_Infinity) { - FPU_copy_to_regi(a, TAG_Special, deststnr); - setsign(dest, sign); - return TAG_Special; - } else if (tagb == TW_Infinity) { - FPU_copy_to_regi(b, TAG_Special, deststnr); - setsign(dest, sign); - return TAG_Special; - } -#ifdef PARANOID - else { - EXCEPTION(EX_INTERNAL | 0x102); - return FPU_Exception; - } -#endif /* PARANOID */ - - return 0; -} diff --git a/arch/x86/math-emu/reg_norm.S b/arch/x86/math-emu/reg_norm.S deleted file mode 100644 index 594936eeed67..000000000000 --- a/arch/x86/math-emu/reg_norm.S +++ /dev/null @@ -1,150 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/*---------------------------------------------------------------------------+ - | reg_norm.S | - | | - | Copyright (C) 1992,1993,1994,1995,1997 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, | - | Australia. E-mail billm@suburbia.net | - | | - | Normalize the value in a FPU_REG. | - | | - | Call from C as: | - | int FPU_normalize(FPU_REG *n) | - | | - | int FPU_normalize_nuo(FPU_REG *n) | - | | - | Return value is the tag of the answer, or-ed with FPU_Exception if | - | one was raised, or -1 on internal error. | - | | - +---------------------------------------------------------------------------*/ - -#include "fpu_emu.h" - - -.text -SYM_FUNC_START(FPU_normalize) - pushl %ebp - movl %esp,%ebp - pushl %ebx - - movl PARAM1,%ebx - - movl SIGH(%ebx),%edx - movl SIGL(%ebx),%eax - - orl %edx,%edx /* ms bits */ - js L_done /* Already normalized */ - jnz L_shift_1 /* Shift left 1 - 31 bits */ - - orl %eax,%eax - jz L_zero /* The contents are zero */ - - movl %eax,%edx - xorl %eax,%eax - subw $32,EXP(%ebx) /* This can cause an underflow */ - -/* We need to shift left by 1 - 31 bits */ -L_shift_1: - bsrl %edx,%ecx /* get the required shift in %ecx */ - subl $31,%ecx - negl %ecx - shld %cl,%eax,%edx - shl %cl,%eax - subw %cx,EXP(%ebx) /* This can cause an underflow */ - - movl %edx,SIGH(%ebx) - movl %eax,SIGL(%ebx) - -L_done: - cmpw EXP_OVER,EXP(%ebx) - jge L_overflow - - cmpw EXP_UNDER,EXP(%ebx) - jle L_underflow - -L_exit_valid: - movl TAG_Valid,%eax - - /* Convert the exponent to 80x87 form. */ - addw EXTENDED_Ebias,EXP(%ebx) - andw $0x7fff,EXP(%ebx) - -L_exit: - popl %ebx - leave - RET - - -L_zero: - movw $0,EXP(%ebx) - movl TAG_Zero,%eax - jmp L_exit - -L_underflow: - /* Convert the exponent to 80x87 form. */ - addw EXTENDED_Ebias,EXP(%ebx) - push %ebx - call arith_underflow - pop %ebx - jmp L_exit - -L_overflow: - /* Convert the exponent to 80x87 form. */ - addw EXTENDED_Ebias,EXP(%ebx) - push %ebx - call arith_overflow - pop %ebx - jmp L_exit -SYM_FUNC_END(FPU_normalize) - - - -/* Normalise without reporting underflow or overflow */ -SYM_FUNC_START(FPU_normalize_nuo) - pushl %ebp - movl %esp,%ebp - pushl %ebx - - movl PARAM1,%ebx - - movl SIGH(%ebx),%edx - movl SIGL(%ebx),%eax - - orl %edx,%edx /* ms bits */ - js L_exit_nuo_valid /* Already normalized */ - jnz L_nuo_shift_1 /* Shift left 1 - 31 bits */ - - orl %eax,%eax - jz L_exit_nuo_zero /* The contents are zero */ - - movl %eax,%edx - xorl %eax,%eax - subw $32,EXP(%ebx) /* This can cause an underflow */ - -/* We need to shift left by 1 - 31 bits */ -L_nuo_shift_1: - bsrl %edx,%ecx /* get the required shift in %ecx */ - subl $31,%ecx - negl %ecx - shld %cl,%eax,%edx - shl %cl,%eax - subw %cx,EXP(%ebx) /* This can cause an underflow */ - - movl %edx,SIGH(%ebx) - movl %eax,SIGL(%ebx) - -L_exit_nuo_valid: - movl TAG_Valid,%eax - - popl %ebx - leave - RET - -L_exit_nuo_zero: - movl TAG_Zero,%eax - movw EXP_UNDER,EXP(%ebx) - - popl %ebx - leave - RET -SYM_FUNC_END(FPU_normalize_nuo) diff --git a/arch/x86/math-emu/reg_round.S b/arch/x86/math-emu/reg_round.S deleted file mode 100644 index 0bb2a092161a..000000000000 --- a/arch/x86/math-emu/reg_round.S +++ /dev/null @@ -1,711 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - .file "reg_round.S" -/*---------------------------------------------------------------------------+ - | reg_round.S | - | | - | Rounding/truncation/etc for FPU basic arithmetic functions. | - | | - | Copyright (C) 1993,1995,1997 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, | - | Australia. E-mail billm@suburbia.net | - | | - | This code has four possible entry points. | - | The following must be entered by a jmp instruction: | - | fpu_reg_round, fpu_reg_round_sqrt, and fpu_Arith_exit. | - | | - | The FPU_round entry point is intended to be used by C code. | - | From C, call as: | - | int FPU_round(FPU_REG *arg, unsigned int extent, unsigned int control_w) | - | | - | Return value is the tag of the answer, or-ed with FPU_Exception if | - | one was raised, or -1 on internal error. | - | | - | For correct "up" and "down" rounding, the argument must have the correct | - | sign. | - | | - +---------------------------------------------------------------------------*/ - -/*---------------------------------------------------------------------------+ - | Four entry points. | - | | - | Needed by both the fpu_reg_round and fpu_reg_round_sqrt entry points: | - | %eax:%ebx 64 bit significand | - | %edx 32 bit extension of the significand | - | %edi pointer to an FPU_REG for the result to be stored | - | stack calling function must have set up a C stack frame and | - | pushed %esi, %edi, and %ebx | - | | - | Needed just for the fpu_reg_round_sqrt entry point: | - | %cx A control word in the same format as the FPU control word. | - | Otherwise, PARAM4 must give such a value. | - | | - | | - | The significand and its extension are assumed to be exact in the | - | following sense: | - | If the significand by itself is the exact result then the significand | - | extension (%edx) must contain 0, otherwise the significand extension | - | must be non-zero. | - | If the significand extension is non-zero then the significand is | - | smaller than the magnitude of the correct exact result by an amount | - | greater than zero and less than one ls bit of the significand. | - | The significand extension is only required to have three possible | - | non-zero values: | - | less than 0x80000000 <=> the significand is less than 1/2 an ls | - | bit smaller than the magnitude of the | - | true exact result. | - | exactly 0x80000000 <=> the significand is exactly 1/2 an ls bit | - | smaller than the magnitude of the true | - | exact result. | - | greater than 0x80000000 <=> the significand is more than 1/2 an ls | - | bit smaller than the magnitude of the | - | true exact result. | - | | - +---------------------------------------------------------------------------*/ - -/*---------------------------------------------------------------------------+ - | The code in this module has become quite complex, but it should handle | - | all of the FPU flags which are set at this stage of the basic arithmetic | - | computations. | - | There are a few rare cases where the results are not set identically to | - | a real FPU. These require a bit more thought because at this stage the | - | results of the code here appear to be more consistent... | - | This may be changed in a future version. | - +---------------------------------------------------------------------------*/ - - -#include "fpu_emu.h" -#include "exception.h" -#include "control_w.h" - -/* Flags for FPU_bits_lost */ -#define LOST_DOWN $1 -#define LOST_UP $2 - -/* Flags for FPU_denormal */ -#define DENORMAL $1 -#define UNMASKED_UNDERFLOW $2 - - -#ifndef NON_REENTRANT_FPU -/* Make the code re-entrant by putting - local storage on the stack: */ -#define FPU_bits_lost (%esp) -#define FPU_denormal 1(%esp) - -#else -/* Not re-entrant, so we can gain speed by putting - local storage in a static area: */ -.data - .align 4,0 -FPU_bits_lost: - .byte 0 -FPU_denormal: - .byte 0 -#endif /* NON_REENTRANT_FPU */ - - -.text -.globl fpu_reg_round -.globl fpu_Arith_exit - -/* Entry point when called from C */ -SYM_FUNC_START(FPU_round) - pushl %ebp - movl %esp,%ebp - pushl %esi - pushl %edi - pushl %ebx - - movl PARAM1,%edi - movl SIGH(%edi),%eax - movl SIGL(%edi),%ebx - movl PARAM2,%edx - -fpu_reg_round: /* Normal entry point */ - movl PARAM4,%ecx - -#ifndef NON_REENTRANT_FPU - pushl %ebx /* adjust the stack pointer */ -#endif /* NON_REENTRANT_FPU */ - -#ifdef PARANOID -/* Cannot use this here yet */ -/* orl %eax,%eax */ -/* jns L_entry_bugged */ -#endif /* PARANOID */ - - cmpw EXP_UNDER,EXP(%edi) - jle L_Make_denorm /* The number is a de-normal */ - - movb $0,FPU_denormal /* 0 -> not a de-normal */ - -Denorm_done: - movb $0,FPU_bits_lost /* No bits yet lost in rounding */ - - movl %ecx,%esi - andl CW_PC,%ecx - cmpl PR_64_BITS,%ecx - je LRound_To_64 - - cmpl PR_53_BITS,%ecx - je LRound_To_53 - - cmpl PR_24_BITS,%ecx - je LRound_To_24 - -#ifdef PECULIAR_486 -/* With the precision control bits set to 01 "(reserved)", a real 80486 - behaves as if the precision control bits were set to 11 "64 bits" */ - cmpl PR_RESERVED_BITS,%ecx - je LRound_To_64 -#ifdef PARANOID - jmp L_bugged_denorm_486 -#endif /* PARANOID */ -#else -#ifdef PARANOID - jmp L_bugged_denorm /* There is no bug, just a bad control word */ -#endif /* PARANOID */ -#endif /* PECULIAR_486 */ - - -/* Round etc to 24 bit precision */ -LRound_To_24: - movl %esi,%ecx - andl CW_RC,%ecx - cmpl RC_RND,%ecx - je LRound_nearest_24 - - cmpl RC_CHOP,%ecx - je LCheck_truncate_24 - - cmpl RC_UP,%ecx /* Towards +infinity */ - je LUp_24 - - cmpl RC_DOWN,%ecx /* Towards -infinity */ - je LDown_24 - -#ifdef PARANOID - jmp L_bugged_round24 -#endif /* PARANOID */ - -LUp_24: - cmpb SIGN_POS,PARAM5 - jne LCheck_truncate_24 /* If negative then up==truncate */ - - jmp LCheck_24_round_up - -LDown_24: - cmpb SIGN_POS,PARAM5 - je LCheck_truncate_24 /* If positive then down==truncate */ - -LCheck_24_round_up: - movl %eax,%ecx - andl $0x000000ff,%ecx - orl %ebx,%ecx - orl %edx,%ecx - jnz LDo_24_round_up - jmp L_Re_normalise - -LRound_nearest_24: - /* Do rounding of the 24th bit if needed (nearest or even) */ - movl %eax,%ecx - andl $0x000000ff,%ecx - cmpl $0x00000080,%ecx - jc LCheck_truncate_24 /* less than half, no increment needed */ - - jne LGreater_Half_24 /* greater than half, increment needed */ - - /* Possibly half, we need to check the ls bits */ - orl %ebx,%ebx - jnz LGreater_Half_24 /* greater than half, increment needed */ - - orl %edx,%edx - jnz LGreater_Half_24 /* greater than half, increment needed */ - - /* Exactly half, increment only if 24th bit is 1 (round to even) */ - testl $0x00000100,%eax - jz LDo_truncate_24 - -LGreater_Half_24: /* Rounding: increment at the 24th bit */ -LDo_24_round_up: - andl $0xffffff00,%eax /* Truncate to 24 bits */ - xorl %ebx,%ebx - movb LOST_UP,FPU_bits_lost - addl $0x00000100,%eax - jmp LCheck_Round_Overflow - -LCheck_truncate_24: - movl %eax,%ecx - andl $0x000000ff,%ecx - orl %ebx,%ecx - orl %edx,%ecx - jz L_Re_normalise /* No truncation needed */ - -LDo_truncate_24: - andl $0xffffff00,%eax /* Truncate to 24 bits */ - xorl %ebx,%ebx - movb LOST_DOWN,FPU_bits_lost - jmp L_Re_normalise - - -/* Round etc to 53 bit precision */ -LRound_To_53: - movl %esi,%ecx - andl CW_RC,%ecx - cmpl RC_RND,%ecx - je LRound_nearest_53 - - cmpl RC_CHOP,%ecx - je LCheck_truncate_53 - - cmpl RC_UP,%ecx /* Towards +infinity */ - je LUp_53 - - cmpl RC_DOWN,%ecx /* Towards -infinity */ - je LDown_53 - -#ifdef PARANOID - jmp L_bugged_round53 -#endif /* PARANOID */ - -LUp_53: - cmpb SIGN_POS,PARAM5 - jne LCheck_truncate_53 /* If negative then up==truncate */ - - jmp LCheck_53_round_up - -LDown_53: - cmpb SIGN_POS,PARAM5 - je LCheck_truncate_53 /* If positive then down==truncate */ - -LCheck_53_round_up: - movl %ebx,%ecx - andl $0x000007ff,%ecx - orl %edx,%ecx - jnz LDo_53_round_up - jmp L_Re_normalise - -LRound_nearest_53: - /* Do rounding of the 53rd bit if needed (nearest or even) */ - movl %ebx,%ecx - andl $0x000007ff,%ecx - cmpl $0x00000400,%ecx - jc LCheck_truncate_53 /* less than half, no increment needed */ - - jnz LGreater_Half_53 /* greater than half, increment needed */ - - /* Possibly half, we need to check the ls bits */ - orl %edx,%edx - jnz LGreater_Half_53 /* greater than half, increment needed */ - - /* Exactly half, increment only if 53rd bit is 1 (round to even) */ - testl $0x00000800,%ebx - jz LTruncate_53 - -LGreater_Half_53: /* Rounding: increment at the 53rd bit */ -LDo_53_round_up: - movb LOST_UP,FPU_bits_lost - andl $0xfffff800,%ebx /* Truncate to 53 bits */ - addl $0x00000800,%ebx - adcl $0,%eax - jmp LCheck_Round_Overflow - -LCheck_truncate_53: - movl %ebx,%ecx - andl $0x000007ff,%ecx - orl %edx,%ecx - jz L_Re_normalise - -LTruncate_53: - movb LOST_DOWN,FPU_bits_lost - andl $0xfffff800,%ebx /* Truncate to 53 bits */ - jmp L_Re_normalise - - -/* Round etc to 64 bit precision */ -LRound_To_64: - movl %esi,%ecx - andl CW_RC,%ecx - cmpl RC_RND,%ecx - je LRound_nearest_64 - - cmpl RC_CHOP,%ecx - je LCheck_truncate_64 - - cmpl RC_UP,%ecx /* Towards +infinity */ - je LUp_64 - - cmpl RC_DOWN,%ecx /* Towards -infinity */ - je LDown_64 - -#ifdef PARANOID - jmp L_bugged_round64 -#endif /* PARANOID */ - -LUp_64: - cmpb SIGN_POS,PARAM5 - jne LCheck_truncate_64 /* If negative then up==truncate */ - - orl %edx,%edx - jnz LDo_64_round_up - jmp L_Re_normalise - -LDown_64: - cmpb SIGN_POS,PARAM5 - je LCheck_truncate_64 /* If positive then down==truncate */ - - orl %edx,%edx - jnz LDo_64_round_up - jmp L_Re_normalise - -LRound_nearest_64: - cmpl $0x80000000,%edx - jc LCheck_truncate_64 - - jne LDo_64_round_up - - /* Now test for round-to-even */ - testb $1,%bl - jz LCheck_truncate_64 - -LDo_64_round_up: - movb LOST_UP,FPU_bits_lost - addl $1,%ebx - adcl $0,%eax - -LCheck_Round_Overflow: - jnc L_Re_normalise - - /* Overflow, adjust the result (significand to 1.0) */ - rcrl $1,%eax - rcrl $1,%ebx - incw EXP(%edi) - jmp L_Re_normalise - -LCheck_truncate_64: - orl %edx,%edx - jz L_Re_normalise - -LTruncate_64: - movb LOST_DOWN,FPU_bits_lost - -L_Re_normalise: - testb $0xff,FPU_denormal - jnz Normalise_result - -L_Normalised: - movl TAG_Valid,%edx - -L_deNormalised: - cmpb LOST_UP,FPU_bits_lost - je L_precision_lost_up - - cmpb LOST_DOWN,FPU_bits_lost - je L_precision_lost_down - -L_no_precision_loss: - /* store the result */ - -L_Store_significand: - movl %eax,SIGH(%edi) - movl %ebx,SIGL(%edi) - - cmpw EXP_OVER,EXP(%edi) - jge L_overflow - - movl %edx,%eax - - /* Convert the exponent to 80x87 form. */ - addw EXTENDED_Ebias,EXP(%edi) - andw $0x7fff,EXP(%edi) - -fpu_reg_round_signed_special_exit: - - cmpb SIGN_POS,PARAM5 - je fpu_reg_round_special_exit - - orw $0x8000,EXP(%edi) /* Negative sign for the result. */ - -fpu_reg_round_special_exit: - -#ifndef NON_REENTRANT_FPU - popl %ebx /* adjust the stack pointer */ -#endif /* NON_REENTRANT_FPU */ - -fpu_Arith_exit: - popl %ebx - popl %edi - popl %esi - leave - RET - - -/* - * Set the FPU status flags to represent precision loss due to - * round-up. - */ -L_precision_lost_up: - push %edx - push %eax - call set_precision_flag_up - popl %eax - popl %edx - jmp L_no_precision_loss - -/* - * Set the FPU status flags to represent precision loss due to - * truncation. - */ -L_precision_lost_down: - push %edx - push %eax - call set_precision_flag_down - popl %eax - popl %edx - jmp L_no_precision_loss - - -/* - * The number is a denormal (which might get rounded up to a normal) - * Shift the number right the required number of bits, which will - * have to be undone later... - */ -L_Make_denorm: - /* The action to be taken depends upon whether the underflow - exception is masked */ - testb CW_Underflow,%cl /* Underflow mask. */ - jz Unmasked_underflow /* Do not make a denormal. */ - - movb DENORMAL,FPU_denormal - - pushl %ecx /* Save */ - movw EXP_UNDER+1,%cx - subw EXP(%edi),%cx - - cmpw $64,%cx /* shrd only works for 0..31 bits */ - jnc Denorm_shift_more_than_63 - - cmpw $32,%cx /* shrd only works for 0..31 bits */ - jnc Denorm_shift_more_than_32 - -/* - * We got here without jumps by assuming that the most common requirement - * is for a small de-normalising shift. - * Shift by [1..31] bits - */ - addw %cx,EXP(%edi) - orl %edx,%edx /* extension */ - setne %ch /* Save whether %edx is non-zero */ - xorl %edx,%edx - shrd %cl,%ebx,%edx - shrd %cl,%eax,%ebx - shr %cl,%eax - orb %ch,%dl - popl %ecx - jmp Denorm_done - -/* Shift by [32..63] bits */ -Denorm_shift_more_than_32: - addw %cx,EXP(%edi) - subb $32,%cl - orl %edx,%edx - setne %ch - orb %ch,%bl - xorl %edx,%edx - shrd %cl,%ebx,%edx - shrd %cl,%eax,%ebx - shr %cl,%eax - orl %edx,%edx /* test these 32 bits */ - setne %cl - orb %ch,%bl - orb %cl,%bl - movl %ebx,%edx - movl %eax,%ebx - xorl %eax,%eax - popl %ecx - jmp Denorm_done - -/* Shift by [64..) bits */ -Denorm_shift_more_than_63: - cmpw $64,%cx - jne Denorm_shift_more_than_64 - -/* Exactly 64 bit shift */ - addw %cx,EXP(%edi) - xorl %ecx,%ecx - orl %edx,%edx - setne %cl - orl %ebx,%ebx - setne %ch - orb %ch,%cl - orb %cl,%al - movl %eax,%edx - xorl %eax,%eax - xorl %ebx,%ebx - popl %ecx - jmp Denorm_done - -Denorm_shift_more_than_64: - movw EXP_UNDER+1,EXP(%edi) -/* This is easy, %eax must be non-zero, so.. */ - movl $1,%edx - xorl %eax,%eax - xorl %ebx,%ebx - popl %ecx - jmp Denorm_done - - -Unmasked_underflow: - movb UNMASKED_UNDERFLOW,FPU_denormal - jmp Denorm_done - - -/* Undo the de-normalisation. */ -Normalise_result: - cmpb UNMASKED_UNDERFLOW,FPU_denormal - je Signal_underflow - -/* The number must be a denormal if we got here. */ -#ifdef PARANOID - /* But check it... just in case. */ - cmpw EXP_UNDER+1,EXP(%edi) - jne L_norm_bugged -#endif /* PARANOID */ - -#ifdef PECULIAR_486 - /* - * This implements a special feature of 80486 behaviour. - * Underflow will be signaled even if the number is - * not a denormal after rounding. - * This difference occurs only for masked underflow, and not - * in the unmasked case. - * Actual 80486 behaviour differs from this in some circumstances. - */ - orl %eax,%eax /* ms bits */ - js LPseudoDenormal /* Will be masked underflow */ -#else - orl %eax,%eax /* ms bits */ - js L_Normalised /* No longer a denormal */ -#endif /* PECULIAR_486 */ - - jnz LDenormal_adj_exponent - - orl %ebx,%ebx - jz L_underflow_to_zero /* The contents are zero */ - -LDenormal_adj_exponent: - decw EXP(%edi) - -LPseudoDenormal: - testb $0xff,FPU_bits_lost /* bits lost == underflow */ - movl TAG_Special,%edx - jz L_deNormalised - - /* There must be a masked underflow */ - push %eax - pushl EX_Underflow - call EXCEPTION - popl %eax - popl %eax - movl TAG_Special,%edx - jmp L_deNormalised - - -/* - * The operations resulted in a number too small to represent. - * Masked response. - */ -L_underflow_to_zero: - push %eax - call set_precision_flag_down - popl %eax - - push %eax - pushl EX_Underflow - call EXCEPTION - popl %eax - popl %eax - -/* Reduce the exponent to EXP_UNDER */ - movw EXP_UNDER,EXP(%edi) - movl TAG_Zero,%edx - jmp L_Store_significand - - -/* The operations resulted in a number too large to represent. */ -L_overflow: - addw EXTENDED_Ebias,EXP(%edi) /* Set for unmasked response. */ - push %edi - call arith_overflow - pop %edi - jmp fpu_reg_round_signed_special_exit - - -Signal_underflow: - /* The number may have been changed to a non-denormal */ - /* by the rounding operations. */ - cmpw EXP_UNDER,EXP(%edi) - jle Do_unmasked_underflow - - jmp L_Normalised - -Do_unmasked_underflow: - /* Increase the exponent by the magic number */ - addw $(3*(1<<13)),EXP(%edi) - push %eax - pushl EX_Underflow - call EXCEPTION - popl %eax - popl %eax - jmp L_Normalised - - -#ifdef PARANOID -#ifdef PECULIAR_486 -L_bugged_denorm_486: - pushl EX_INTERNAL|0x236 - call EXCEPTION - popl %ebx - jmp L_exception_exit -#else -L_bugged_denorm: - pushl EX_INTERNAL|0x230 - call EXCEPTION - popl %ebx - jmp L_exception_exit -#endif /* PECULIAR_486 */ - -L_bugged_round24: - pushl EX_INTERNAL|0x231 - call EXCEPTION - popl %ebx - jmp L_exception_exit - -L_bugged_round53: - pushl EX_INTERNAL|0x232 - call EXCEPTION - popl %ebx - jmp L_exception_exit - -L_bugged_round64: - pushl EX_INTERNAL|0x233 - call EXCEPTION - popl %ebx - jmp L_exception_exit - -L_norm_bugged: - pushl EX_INTERNAL|0x234 - call EXCEPTION - popl %ebx - jmp L_exception_exit - -L_entry_bugged: - pushl EX_INTERNAL|0x235 - call EXCEPTION - popl %ebx -L_exception_exit: - mov $-1,%eax - jmp fpu_reg_round_special_exit -#endif /* PARANOID */ - -SYM_FUNC_END(FPU_round) diff --git a/arch/x86/math-emu/reg_u_add.S b/arch/x86/math-emu/reg_u_add.S deleted file mode 100644 index 07247287a3af..000000000000 --- a/arch/x86/math-emu/reg_u_add.S +++ /dev/null @@ -1,169 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - .file "reg_u_add.S" -/*---------------------------------------------------------------------------+ - | reg_u_add.S | - | | - | Add two valid (TAG_Valid) FPU_REG numbers, of the same sign, and put the | - | result in a destination FPU_REG. | - | | - | Copyright (C) 1992,1993,1995,1997 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia | - | E-mail billm@suburbia.net | - | | - | Call from C as: | - | int FPU_u_add(FPU_REG *arg1, FPU_REG *arg2, FPU_REG *answ, | - | int control_w) | - | Return value is the tag of the answer, or-ed with FPU_Exception if | - | one was raised, or -1 on internal error. | - | | - +---------------------------------------------------------------------------*/ - -/* - | Kernel addition routine FPU_u_add(reg *arg1, reg *arg2, reg *answ). - | Takes two valid reg f.p. numbers (TAG_Valid), which are - | treated as unsigned numbers, - | and returns their sum as a TAG_Valid or TAG_Special f.p. number. - | The returned number is normalized. - | Basic checks are performed if PARANOID is defined. - */ - -#include "exception.h" -#include "fpu_emu.h" -#include "control_w.h" - -.text -SYM_FUNC_START(FPU_u_add) - pushl %ebp - movl %esp,%ebp - pushl %esi - pushl %edi - pushl %ebx - - movl PARAM1,%esi /* source 1 */ - movl PARAM2,%edi /* source 2 */ - - movl PARAM6,%ecx - movl %ecx,%edx - subl PARAM7,%ecx /* exp1 - exp2 */ - jge L_arg1_larger - - /* num1 is smaller */ - movl SIGL(%esi),%ebx - movl SIGH(%esi),%eax - - movl %edi,%esi - movl PARAM7,%edx - negw %cx - jmp L_accum_loaded - -L_arg1_larger: - /* num1 has larger or equal exponent */ - movl SIGL(%edi),%ebx - movl SIGH(%edi),%eax - -L_accum_loaded: - movl PARAM3,%edi /* destination */ - movw %dx,EXP(%edi) /* Copy exponent to destination */ - - xorl %edx,%edx /* clear the extension */ - -#ifdef PARANOID - testl $0x80000000,%eax - je L_bugged - - testl $0x80000000,SIGH(%esi) - je L_bugged -#endif /* PARANOID */ - -/* The number to be shifted is in %eax:%ebx:%edx */ - cmpw $32,%cx /* shrd only works for 0..31 bits */ - jnc L_more_than_31 - -/* less than 32 bits */ - shrd %cl,%ebx,%edx - shrd %cl,%eax,%ebx - shr %cl,%eax - jmp L_shift_done - -L_more_than_31: - cmpw $64,%cx - jnc L_more_than_63 - - subb $32,%cl - jz L_exactly_32 - - shrd %cl,%eax,%edx - shr %cl,%eax - orl %ebx,%ebx - jz L_more_31_no_low /* none of the lowest bits is set */ - - orl $1,%edx /* record the fact in the extension */ - -L_more_31_no_low: - movl %eax,%ebx - xorl %eax,%eax - jmp L_shift_done - -L_exactly_32: - movl %ebx,%edx - movl %eax,%ebx - xorl %eax,%eax - jmp L_shift_done - -L_more_than_63: - cmpw $65,%cx - jnc L_more_than_64 - - movl %eax,%edx - orl %ebx,%ebx - jz L_more_63_no_low - - orl $1,%edx - jmp L_more_63_no_low - -L_more_than_64: - movl $1,%edx /* The shifted nr always at least one '1' */ - -L_more_63_no_low: - xorl %ebx,%ebx - xorl %eax,%eax - -L_shift_done: - /* Now do the addition */ - addl SIGL(%esi),%ebx - adcl SIGH(%esi),%eax - jnc L_round_the_result - - /* Overflow, adjust the result */ - rcrl $1,%eax - rcrl $1,%ebx - rcrl $1,%edx - jnc L_no_bit_lost - - orl $1,%edx - -L_no_bit_lost: - incw EXP(%edi) - -L_round_the_result: - jmp fpu_reg_round /* Round the result */ - - - -#ifdef PARANOID -/* If we ever get here then we have problems! */ -L_bugged: - pushl EX_INTERNAL|0x201 - call EXCEPTION - pop %ebx - movl $-1,%eax - jmp L_exit - -L_exit: - popl %ebx - popl %edi - popl %esi - leave - RET -#endif /* PARANOID */ -SYM_FUNC_END(FPU_u_add) diff --git a/arch/x86/math-emu/reg_u_div.S b/arch/x86/math-emu/reg_u_div.S deleted file mode 100644 index b5a41e2fc484..000000000000 --- a/arch/x86/math-emu/reg_u_div.S +++ /dev/null @@ -1,474 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - .file "reg_u_div.S" -/*---------------------------------------------------------------------------+ - | reg_u_div.S | - | | - | Divide one FPU_REG by another and put the result in a destination FPU_REG.| - | | - | Copyright (C) 1992,1993,1995,1997 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia | - | E-mail billm@suburbia.net | - | | - | | - +---------------------------------------------------------------------------*/ - -/*---------------------------------------------------------------------------+ - | Call from C as: | - | int FPU_u_div(FPU_REG *a, FPU_REG *b, FPU_REG *dest, | - | unsigned int control_word, char *sign) | - | | - | Does not compute the destination exponent, but does adjust it. | - | | - | Return value is the tag of the answer, or-ed with FPU_Exception if | - | one was raised, or -1 on internal error. | - +---------------------------------------------------------------------------*/ - -#include "exception.h" -#include "fpu_emu.h" -#include "control_w.h" - - -/* #define dSIGL(x) (x) */ -/* #define dSIGH(x) 4(x) */ - - -#ifndef NON_REENTRANT_FPU -/* - Local storage on the stack: - Result: FPU_accum_3:FPU_accum_2:FPU_accum_1:FPU_accum_0 - Overflow flag: ovfl_flag - */ -#define FPU_accum_3 -4(%ebp) -#define FPU_accum_2 -8(%ebp) -#define FPU_accum_1 -12(%ebp) -#define FPU_accum_0 -16(%ebp) -#define FPU_result_1 -20(%ebp) -#define FPU_result_2 -24(%ebp) -#define FPU_ovfl_flag -28(%ebp) - -#else -.data -/* - Local storage in a static area: - Result: FPU_accum_3:FPU_accum_2:FPU_accum_1:FPU_accum_0 - Overflow flag: ovfl_flag - */ - .align 4,0 -FPU_accum_3: - .long 0 -FPU_accum_2: - .long 0 -FPU_accum_1: - .long 0 -FPU_accum_0: - .long 0 -FPU_result_1: - .long 0 -FPU_result_2: - .long 0 -FPU_ovfl_flag: - .byte 0 -#endif /* NON_REENTRANT_FPU */ - -#define REGA PARAM1 -#define REGB PARAM2 -#define DEST PARAM3 - -.text -SYM_FUNC_START(FPU_u_div) - pushl %ebp - movl %esp,%ebp -#ifndef NON_REENTRANT_FPU - subl $28,%esp -#endif /* NON_REENTRANT_FPU */ - - pushl %esi - pushl %edi - pushl %ebx - - movl REGA,%esi - movl REGB,%ebx - movl DEST,%edi - - movswl EXP(%esi),%edx - movswl EXP(%ebx),%eax - subl %eax,%edx - addl EXP_BIAS,%edx - - /* A denormal and a large number can cause an exponent underflow */ - cmpl EXP_WAY_UNDER,%edx - jg xExp_not_underflow - - /* Set to a really low value allow correct handling */ - movl EXP_WAY_UNDER,%edx - -xExp_not_underflow: - - movw %dx,EXP(%edi) - -#ifdef PARANOID -/* testl $0x80000000, SIGH(%esi) // Dividend */ -/* je L_bugged */ - testl $0x80000000, SIGH(%ebx) /* Divisor */ - je L_bugged -#endif /* PARANOID */ - -/* Check if the divisor can be treated as having just 32 bits */ - cmpl $0,SIGL(%ebx) - jnz L_Full_Division /* Can't do a quick divide */ - -/* We should be able to zip through the division here */ - movl SIGH(%ebx),%ecx /* The divisor */ - movl SIGH(%esi),%edx /* Dividend */ - movl SIGL(%esi),%eax /* Dividend */ - - cmpl %ecx,%edx - setaeb FPU_ovfl_flag /* Keep a record */ - jb L_no_adjust - - subl %ecx,%edx /* Prevent the overflow */ - -L_no_adjust: - /* Divide the 64 bit number by the 32 bit denominator */ - divl %ecx - movl %eax,FPU_result_2 - - /* Work on the remainder of the first division */ - xorl %eax,%eax - divl %ecx - movl %eax,FPU_result_1 - - /* Work on the remainder of the 64 bit division */ - xorl %eax,%eax - divl %ecx - - testb $255,FPU_ovfl_flag /* was the num > denom ? */ - je L_no_overflow - - /* Do the shifting here */ - /* increase the exponent */ - incw EXP(%edi) - - /* shift the mantissa right one bit */ - stc /* To set the ms bit */ - rcrl FPU_result_2 - rcrl FPU_result_1 - rcrl %eax - -L_no_overflow: - jmp LRound_precision /* Do the rounding as required */ - - -/*---------------------------------------------------------------------------+ - | Divide: Return arg1/arg2 to arg3. | - | | - | This routine does not use the exponents of arg1 and arg2, but does | - | adjust the exponent of arg3. | - | | - | The maximum returned value is (ignoring exponents) | - | .ffffffff ffffffff | - | ------------------ = 1.ffffffff fffffffe | - | .80000000 00000000 | - | and the minimum is | - | .80000000 00000000 | - | ------------------ = .80000000 00000001 (rounded) | - | .ffffffff ffffffff | - | | - +---------------------------------------------------------------------------*/ - - -L_Full_Division: - /* Save extended dividend in local register */ - movl SIGL(%esi),%eax - movl %eax,FPU_accum_2 - movl SIGH(%esi),%eax - movl %eax,FPU_accum_3 - xorl %eax,%eax - movl %eax,FPU_accum_1 /* zero the extension */ - movl %eax,FPU_accum_0 /* zero the extension */ - - movl SIGL(%esi),%eax /* Get the current num */ - movl SIGH(%esi),%edx - -/*----------------------------------------------------------------------*/ -/* Initialization done. - Do the first 32 bits. */ - - movb $0,FPU_ovfl_flag - cmpl SIGH(%ebx),%edx /* Test for imminent overflow */ - jb LLess_than_1 - ja LGreater_than_1 - - cmpl SIGL(%ebx),%eax - jb LLess_than_1 - -LGreater_than_1: -/* The dividend is greater or equal, would cause overflow */ - setaeb FPU_ovfl_flag /* Keep a record */ - - subl SIGL(%ebx),%eax - sbbl SIGH(%ebx),%edx /* Prevent the overflow */ - movl %eax,FPU_accum_2 - movl %edx,FPU_accum_3 - -LLess_than_1: -/* At this point, we have a dividend < divisor, with a record of - adjustment in FPU_ovfl_flag */ - - /* We will divide by a number which is too large */ - movl SIGH(%ebx),%ecx - addl $1,%ecx - jnc LFirst_div_not_1 - - /* here we need to divide by 100000000h, - i.e., no division at all.. */ - mov %edx,%eax - jmp LFirst_div_done - -LFirst_div_not_1: - divl %ecx /* Divide the numerator by the augmented - denom ms dw */ - -LFirst_div_done: - movl %eax,FPU_result_2 /* Put the result in the answer */ - - mull SIGH(%ebx) /* mul by the ms dw of the denom */ - - subl %eax,FPU_accum_2 /* Subtract from the num local reg */ - sbbl %edx,FPU_accum_3 - - movl FPU_result_2,%eax /* Get the result back */ - mull SIGL(%ebx) /* now mul the ls dw of the denom */ - - subl %eax,FPU_accum_1 /* Subtract from the num local reg */ - sbbl %edx,FPU_accum_2 - sbbl $0,FPU_accum_3 - je LDo_2nd_32_bits /* Must check for non-zero result here */ - -#ifdef PARANOID - jb L_bugged_1 -#endif /* PARANOID */ - - /* need to subtract another once of the denom */ - incl FPU_result_2 /* Correct the answer */ - - movl SIGL(%ebx),%eax - movl SIGH(%ebx),%edx - subl %eax,FPU_accum_1 /* Subtract from the num local reg */ - sbbl %edx,FPU_accum_2 - -#ifdef PARANOID - sbbl $0,FPU_accum_3 - jne L_bugged_1 /* Must check for non-zero result here */ -#endif /* PARANOID */ - -/*----------------------------------------------------------------------*/ -/* Half of the main problem is done, there is just a reduced numerator - to handle now. - Work with the second 32 bits, FPU_accum_0 not used from now on */ -LDo_2nd_32_bits: - movl FPU_accum_2,%edx /* get the reduced num */ - movl FPU_accum_1,%eax - - /* need to check for possible subsequent overflow */ - cmpl SIGH(%ebx),%edx - jb LDo_2nd_div - ja LPrevent_2nd_overflow - - cmpl SIGL(%ebx),%eax - jb LDo_2nd_div - -LPrevent_2nd_overflow: -/* The numerator is greater or equal, would cause overflow */ - /* prevent overflow */ - subl SIGL(%ebx),%eax - sbbl SIGH(%ebx),%edx - movl %edx,FPU_accum_2 - movl %eax,FPU_accum_1 - - incl FPU_result_2 /* Reflect the subtraction in the answer */ - -#ifdef PARANOID - je L_bugged_2 /* Can't bump the result to 1.0 */ -#endif /* PARANOID */ - -LDo_2nd_div: - cmpl $0,%ecx /* augmented denom msw */ - jnz LSecond_div_not_1 - - /* %ecx == 0, we are dividing by 1.0 */ - mov %edx,%eax - jmp LSecond_div_done - -LSecond_div_not_1: - divl %ecx /* Divide the numerator by the denom ms dw */ - -LSecond_div_done: - movl %eax,FPU_result_1 /* Put the result in the answer */ - - mull SIGH(%ebx) /* mul by the ms dw of the denom */ - - subl %eax,FPU_accum_1 /* Subtract from the num local reg */ - sbbl %edx,FPU_accum_2 - -#ifdef PARANOID - jc L_bugged_2 -#endif /* PARANOID */ - - movl FPU_result_1,%eax /* Get the result back */ - mull SIGL(%ebx) /* now mul the ls dw of the denom */ - - subl %eax,FPU_accum_0 /* Subtract from the num local reg */ - sbbl %edx,FPU_accum_1 /* Subtract from the num local reg */ - sbbl $0,FPU_accum_2 - -#ifdef PARANOID - jc L_bugged_2 -#endif /* PARANOID */ - - jz LDo_3rd_32_bits - -#ifdef PARANOID - cmpl $1,FPU_accum_2 - jne L_bugged_2 -#endif /* PARANOID */ - - /* need to subtract another once of the denom */ - movl SIGL(%ebx),%eax - movl SIGH(%ebx),%edx - subl %eax,FPU_accum_0 /* Subtract from the num local reg */ - sbbl %edx,FPU_accum_1 - sbbl $0,FPU_accum_2 - -#ifdef PARANOID - jc L_bugged_2 - jne L_bugged_2 -#endif /* PARANOID */ - - addl $1,FPU_result_1 /* Correct the answer */ - adcl $0,FPU_result_2 - -#ifdef PARANOID - jc L_bugged_2 /* Must check for non-zero result here */ -#endif /* PARANOID */ - -/*----------------------------------------------------------------------*/ -/* The division is essentially finished here, we just need to perform - tidying operations. - Deal with the 3rd 32 bits */ -LDo_3rd_32_bits: - movl FPU_accum_1,%edx /* get the reduced num */ - movl FPU_accum_0,%eax - - /* need to check for possible subsequent overflow */ - cmpl SIGH(%ebx),%edx /* denom */ - jb LRound_prep - ja LPrevent_3rd_overflow - - cmpl SIGL(%ebx),%eax /* denom */ - jb LRound_prep - -LPrevent_3rd_overflow: - /* prevent overflow */ - subl SIGL(%ebx),%eax - sbbl SIGH(%ebx),%edx - movl %edx,FPU_accum_1 - movl %eax,FPU_accum_0 - - addl $1,FPU_result_1 /* Reflect the subtraction in the answer */ - adcl $0,FPU_result_2 - jne LRound_prep - jnc LRound_prep - - /* This is a tricky spot, there is an overflow of the answer */ - movb $255,FPU_ovfl_flag /* Overflow -> 1.000 */ - -LRound_prep: -/* - * Prepare for rounding. - * To test for rounding, we just need to compare 2*accum with the - * denom. - */ - movl FPU_accum_0,%ecx - movl FPU_accum_1,%edx - movl %ecx,%eax - orl %edx,%eax - jz LRound_ovfl /* The accumulator contains zero. */ - - /* Multiply by 2 */ - clc - rcll $1,%ecx - rcll $1,%edx - jc LRound_large /* No need to compare, denom smaller */ - - subl SIGL(%ebx),%ecx - sbbl SIGH(%ebx),%edx - jnc LRound_not_small - - movl $0x70000000,%eax /* Denom was larger */ - jmp LRound_ovfl - -LRound_not_small: - jnz LRound_large - - movl $0x80000000,%eax /* Remainder was exactly 1/2 denom */ - jmp LRound_ovfl - -LRound_large: - movl $0xff000000,%eax /* Denom was smaller */ - -LRound_ovfl: -/* We are now ready to deal with rounding, but first we must get - the bits properly aligned */ - testb $255,FPU_ovfl_flag /* was the num > denom ? */ - je LRound_precision - - incw EXP(%edi) - - /* shift the mantissa right one bit */ - stc /* Will set the ms bit */ - rcrl FPU_result_2 - rcrl FPU_result_1 - rcrl %eax - -/* Round the result as required */ -LRound_precision: - decw EXP(%edi) /* binary point between 1st & 2nd bits */ - - movl %eax,%edx - movl FPU_result_1,%ebx - movl FPU_result_2,%eax - jmp fpu_reg_round - - -#ifdef PARANOID -/* The logic is wrong if we got here */ -L_bugged: - pushl EX_INTERNAL|0x202 - call EXCEPTION - pop %ebx - jmp L_exit - -L_bugged_1: - pushl EX_INTERNAL|0x203 - call EXCEPTION - pop %ebx - jmp L_exit - -L_bugged_2: - pushl EX_INTERNAL|0x204 - call EXCEPTION - pop %ebx - jmp L_exit - -L_exit: - movl $-1,%eax - popl %ebx - popl %edi - popl %esi - - leave - RET -#endif /* PARANOID */ - -SYM_FUNC_END(FPU_u_div) diff --git a/arch/x86/math-emu/reg_u_mul.S b/arch/x86/math-emu/reg_u_mul.S deleted file mode 100644 index e2588b24b8c2..000000000000 --- a/arch/x86/math-emu/reg_u_mul.S +++ /dev/null @@ -1,150 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - .file "reg_u_mul.S" -/*---------------------------------------------------------------------------+ - | reg_u_mul.S | - | | - | Core multiplication routine | - | | - | Copyright (C) 1992,1993,1995,1997 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia | - | E-mail billm@suburbia.net | - | | - | | - +---------------------------------------------------------------------------*/ - -/*---------------------------------------------------------------------------+ - | Basic multiplication routine. | - | Does not check the resulting exponent for overflow/underflow | - | | - | FPU_u_mul(FPU_REG *a, FPU_REG *b, FPU_REG *c, unsigned int cw); | - | | - | Internal working is at approx 128 bits. | - | Result is rounded to nearest 53 or 64 bits, using "nearest or even". | - +---------------------------------------------------------------------------*/ - -#include "exception.h" -#include "fpu_emu.h" -#include "control_w.h" - - - -#ifndef NON_REENTRANT_FPU -/* Local storage on the stack: */ -#define FPU_accum_0 -4(%ebp) /* ms word */ -#define FPU_accum_1 -8(%ebp) - -#else -/* Local storage in a static area: */ -.data - .align 4,0 -FPU_accum_0: - .long 0 -FPU_accum_1: - .long 0 -#endif /* NON_REENTRANT_FPU */ - - -.text -SYM_FUNC_START(FPU_u_mul) - pushl %ebp - movl %esp,%ebp -#ifndef NON_REENTRANT_FPU - subl $8,%esp -#endif /* NON_REENTRANT_FPU */ - - pushl %esi - pushl %edi - pushl %ebx - - movl PARAM1,%esi - movl PARAM2,%edi - -#ifdef PARANOID - testl $0x80000000,SIGH(%esi) - jz L_bugged - testl $0x80000000,SIGH(%edi) - jz L_bugged -#endif /* PARANOID */ - - xorl %ecx,%ecx - xorl %ebx,%ebx - - movl SIGL(%esi),%eax - mull SIGL(%edi) - movl %eax,FPU_accum_0 - movl %edx,FPU_accum_1 - - movl SIGL(%esi),%eax - mull SIGH(%edi) - addl %eax,FPU_accum_1 - adcl %edx,%ebx -/* adcl $0,%ecx // overflow here is not possible */ - - movl SIGH(%esi),%eax - mull SIGL(%edi) - addl %eax,FPU_accum_1 - adcl %edx,%ebx - adcl $0,%ecx - - movl SIGH(%esi),%eax - mull SIGH(%edi) - addl %eax,%ebx - adcl %edx,%ecx - - /* Get the sum of the exponents. */ - movl PARAM6,%eax - subl EXP_BIAS-1,%eax - - /* Two denormals can cause an exponent underflow */ - cmpl EXP_WAY_UNDER,%eax - jg Exp_not_underflow - - /* Set to a really low value allow correct handling */ - movl EXP_WAY_UNDER,%eax - -Exp_not_underflow: - -/* Have now finished with the sources */ - movl PARAM3,%edi /* Point to the destination */ - movw %ax,EXP(%edi) - -/* Now make sure that the result is normalized */ - testl $0x80000000,%ecx - jnz LResult_Normalised - - /* Normalize by shifting left one bit */ - shll $1,FPU_accum_0 - rcll $1,FPU_accum_1 - rcll $1,%ebx - rcll $1,%ecx - decw EXP(%edi) - -LResult_Normalised: - movl FPU_accum_0,%eax - movl FPU_accum_1,%edx - orl %eax,%eax - jz L_extent_zero - - orl $1,%edx - -L_extent_zero: - movl %ecx,%eax - jmp fpu_reg_round - - -#ifdef PARANOID -L_bugged: - pushl EX_INTERNAL|0x205 - call EXCEPTION - pop %ebx - jmp L_exit - -L_exit: - popl %ebx - popl %edi - popl %esi - leave - RET -#endif /* PARANOID */ - -SYM_FUNC_END(FPU_u_mul) diff --git a/arch/x86/math-emu/reg_u_sub.S b/arch/x86/math-emu/reg_u_sub.S deleted file mode 100644 index 4c900c29e4ff..000000000000 --- a/arch/x86/math-emu/reg_u_sub.S +++ /dev/null @@ -1,274 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - .file "reg_u_sub.S" -/*---------------------------------------------------------------------------+ - | reg_u_sub.S | - | | - | Core floating point subtraction routine. | - | | - | Copyright (C) 1992,1993,1995,1997 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia | - | E-mail billm@suburbia.net | - | | - | Call from C as: | - | int FPU_u_sub(FPU_REG *arg1, FPU_REG *arg2, FPU_REG *answ, | - | int control_w) | - | Return value is the tag of the answer, or-ed with FPU_Exception if | - | one was raised, or -1 on internal error. | - | | - +---------------------------------------------------------------------------*/ - -/* - | Kernel subtraction routine FPU_u_sub(reg *arg1, reg *arg2, reg *answ). - | Takes two valid reg f.p. numbers (TAG_Valid), which are - | treated as unsigned numbers, - | and returns their difference as a TAG_Valid or TAG_Zero f.p. - | number. - | The first number (arg1) must be the larger. - | The returned number is normalized. - | Basic checks are performed if PARANOID is defined. - */ - -#include "exception.h" -#include "fpu_emu.h" -#include "control_w.h" - -.text -SYM_FUNC_START(FPU_u_sub) - pushl %ebp - movl %esp,%ebp - pushl %esi - pushl %edi - pushl %ebx - - movl PARAM1,%esi /* source 1 */ - movl PARAM2,%edi /* source 2 */ - - movl PARAM6,%ecx - subl PARAM7,%ecx /* exp1 - exp2 */ - -#ifdef PARANOID - /* source 2 is always smaller than source 1 */ - js L_bugged_1 - - testl $0x80000000,SIGH(%edi) /* The args are assumed to be be normalized */ - je L_bugged_2 - - testl $0x80000000,SIGH(%esi) - je L_bugged_2 -#endif /* PARANOID */ - -/*--------------------------------------+ - | Form a register holding the | - | smaller number | - +--------------------------------------*/ - movl SIGH(%edi),%eax /* register ms word */ - movl SIGL(%edi),%ebx /* register ls word */ - - movl PARAM3,%edi /* destination */ - movl PARAM6,%edx - movw %dx,EXP(%edi) /* Copy exponent to destination */ - - xorl %edx,%edx /* register extension */ - -/*--------------------------------------+ - | Shift the temporary register | - | right the required number of | - | places. | - +--------------------------------------*/ - - cmpw $32,%cx /* shrd only works for 0..31 bits */ - jnc L_more_than_31 - -/* less than 32 bits */ - shrd %cl,%ebx,%edx - shrd %cl,%eax,%ebx - shr %cl,%eax - jmp L_shift_done - -L_more_than_31: - cmpw $64,%cx - jnc L_more_than_63 - - subb $32,%cl - jz L_exactly_32 - - shrd %cl,%eax,%edx - shr %cl,%eax - orl %ebx,%ebx - jz L_more_31_no_low /* none of the lowest bits is set */ - - orl $1,%edx /* record the fact in the extension */ - -L_more_31_no_low: - movl %eax,%ebx - xorl %eax,%eax - jmp L_shift_done - -L_exactly_32: - movl %ebx,%edx - movl %eax,%ebx - xorl %eax,%eax - jmp L_shift_done - -L_more_than_63: - cmpw $65,%cx - jnc L_more_than_64 - - /* Shift right by 64 bits */ - movl %eax,%edx - orl %ebx,%ebx - jz L_more_63_no_low - - orl $1,%edx - jmp L_more_63_no_low - -L_more_than_64: - jne L_more_than_65 - - /* Shift right by 65 bits */ - /* Carry is clear if we get here */ - movl %eax,%edx - rcrl %edx - jnc L_shift_65_nc - - orl $1,%edx - jmp L_more_63_no_low - -L_shift_65_nc: - orl %ebx,%ebx - jz L_more_63_no_low - - orl $1,%edx - jmp L_more_63_no_low - -L_more_than_65: - movl $1,%edx /* The shifted nr always at least one '1' */ - -L_more_63_no_low: - xorl %ebx,%ebx - xorl %eax,%eax - -L_shift_done: -L_subtr: -/*------------------------------+ - | Do the subtraction | - +------------------------------*/ - xorl %ecx,%ecx - subl %edx,%ecx - movl %ecx,%edx - movl SIGL(%esi),%ecx - sbbl %ebx,%ecx - movl %ecx,%ebx - movl SIGH(%esi),%ecx - sbbl %eax,%ecx - movl %ecx,%eax - -#ifdef PARANOID - /* We can never get a borrow */ - jc L_bugged -#endif /* PARANOID */ - -/*--------------------------------------+ - | Normalize the result | - +--------------------------------------*/ - testl $0x80000000,%eax - jnz L_round /* no shifting needed */ - - orl %eax,%eax - jnz L_shift_1 /* shift left 1 - 31 bits */ - - orl %ebx,%ebx - jnz L_shift_32 /* shift left 32 - 63 bits */ - -/* - * A rare case, the only one which is non-zero if we got here - * is: 1000000 .... 0000 - * -0111111 .... 1111 1 - * -------------------- - * 0000000 .... 0000 1 - */ - - cmpl $0x80000000,%edx - jnz L_must_be_zero - - /* Shift left 64 bits */ - subw $64,EXP(%edi) - xchg %edx,%eax - jmp fpu_reg_round - -L_must_be_zero: -#ifdef PARANOID - orl %edx,%edx - jnz L_bugged_3 -#endif /* PARANOID */ - - /* The result is zero */ - movw $0,EXP(%edi) /* exponent */ - movl $0,SIGL(%edi) - movl $0,SIGH(%edi) - movl TAG_Zero,%eax - jmp L_exit - -L_shift_32: - movl %ebx,%eax - movl %edx,%ebx - movl $0,%edx - subw $32,EXP(%edi) /* Can get underflow here */ - -/* We need to shift left by 1 - 31 bits */ -L_shift_1: - bsrl %eax,%ecx /* get the required shift in %ecx */ - subl $31,%ecx - negl %ecx - shld %cl,%ebx,%eax - shld %cl,%edx,%ebx - shl %cl,%edx - subw %cx,EXP(%edi) /* Can get underflow here */ - -L_round: - jmp fpu_reg_round /* Round the result */ - - -#ifdef PARANOID -L_bugged_1: - pushl EX_INTERNAL|0x206 - call EXCEPTION - pop %ebx - jmp L_error_exit - -L_bugged_2: - pushl EX_INTERNAL|0x209 - call EXCEPTION - pop %ebx - jmp L_error_exit - -L_bugged_3: - pushl EX_INTERNAL|0x210 - call EXCEPTION - pop %ebx - jmp L_error_exit - -L_bugged_4: - pushl EX_INTERNAL|0x211 - call EXCEPTION - pop %ebx - jmp L_error_exit - -L_bugged: - pushl EX_INTERNAL|0x212 - call EXCEPTION - pop %ebx - jmp L_error_exit - -L_error_exit: - movl $-1,%eax - -#endif /* PARANOID */ - -L_exit: - popl %ebx - popl %edi - popl %esi - leave - RET -SYM_FUNC_END(FPU_u_sub) diff --git a/arch/x86/math-emu/round_Xsig.S b/arch/x86/math-emu/round_Xsig.S deleted file mode 100644 index 126c40473bad..000000000000 --- a/arch/x86/math-emu/round_Xsig.S +++ /dev/null @@ -1,142 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/*---------------------------------------------------------------------------+ - | round_Xsig.S | - | | - | Copyright (C) 1992,1993,1994,1995 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, | - | Australia. E-mail billm@jacobi.maths.monash.edu.au | - | | - | Normalize and round a 12 byte quantity. | - | Call from C as: | - | int round_Xsig(Xsig *n) | - | | - | Normalize a 12 byte quantity. | - | Call from C as: | - | int norm_Xsig(Xsig *n) | - | | - | Each function returns the size of the shift (nr of bits). | - | | - +---------------------------------------------------------------------------*/ - .file "round_Xsig.S" - -#include "fpu_emu.h" - - -.text -SYM_FUNC_START(round_Xsig) - pushl %ebp - movl %esp,%ebp - pushl %ebx /* Reserve some space */ - pushl %ebx - pushl %esi - - movl PARAM1,%esi - - movl 8(%esi),%edx - movl 4(%esi),%ebx - movl (%esi),%eax - - movl $0,-4(%ebp) - - orl %edx,%edx /* ms bits */ - js L_round /* Already normalized */ - jnz L_shift_1 /* Shift left 1 - 31 bits */ - - movl %ebx,%edx - movl %eax,%ebx - xorl %eax,%eax - movl $-32,-4(%ebp) - -/* We need to shift left by 1 - 31 bits */ -L_shift_1: - bsrl %edx,%ecx /* get the required shift in %ecx */ - subl $31,%ecx - negl %ecx - subl %ecx,-4(%ebp) - shld %cl,%ebx,%edx - shld %cl,%eax,%ebx - shl %cl,%eax - -L_round: - testl $0x80000000,%eax - jz L_exit - - addl $1,%ebx - adcl $0,%edx - jnz L_exit - - movl $0x80000000,%edx - incl -4(%ebp) - -L_exit: - movl %edx,8(%esi) - movl %ebx,4(%esi) - movl %eax,(%esi) - - movl -4(%ebp),%eax - - popl %esi - popl %ebx - leave - RET -SYM_FUNC_END(round_Xsig) - - - -SYM_FUNC_START(norm_Xsig) - pushl %ebp - movl %esp,%ebp - pushl %ebx /* Reserve some space */ - pushl %ebx - pushl %esi - - movl PARAM1,%esi - - movl 8(%esi),%edx - movl 4(%esi),%ebx - movl (%esi),%eax - - movl $0,-4(%ebp) - - orl %edx,%edx /* ms bits */ - js L_n_exit /* Already normalized */ - jnz L_n_shift_1 /* Shift left 1 - 31 bits */ - - movl %ebx,%edx - movl %eax,%ebx - xorl %eax,%eax - movl $-32,-4(%ebp) - - orl %edx,%edx /* ms bits */ - js L_n_exit /* Normalized now */ - jnz L_n_shift_1 /* Shift left 1 - 31 bits */ - - movl %ebx,%edx - movl %eax,%ebx - xorl %eax,%eax - addl $-32,-4(%ebp) - jmp L_n_exit /* Might not be normalized, - but shift no more. */ - -/* We need to shift left by 1 - 31 bits */ -L_n_shift_1: - bsrl %edx,%ecx /* get the required shift in %ecx */ - subl $31,%ecx - negl %ecx - subl %ecx,-4(%ebp) - shld %cl,%ebx,%edx - shld %cl,%eax,%ebx - shl %cl,%eax - -L_n_exit: - movl %edx,8(%esi) - movl %ebx,4(%esi) - movl %eax,(%esi) - - movl -4(%ebp),%eax - - popl %esi - popl %ebx - leave - RET -SYM_FUNC_END(norm_Xsig) diff --git a/arch/x86/math-emu/shr_Xsig.S b/arch/x86/math-emu/shr_Xsig.S deleted file mode 100644 index f726bf6f6396..000000000000 --- a/arch/x86/math-emu/shr_Xsig.S +++ /dev/null @@ -1,89 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - .file "shr_Xsig.S" -/*---------------------------------------------------------------------------+ - | shr_Xsig.S | - | | - | 12 byte right shift function | - | | - | Copyright (C) 1992,1994,1995 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, | - | Australia. E-mail billm@jacobi.maths.monash.edu.au | - | | - | Call from C as: | - | void shr_Xsig(Xsig *arg, unsigned nr) | - | | - | Extended shift right function. | - | Fastest for small shifts. | - | Shifts the 12 byte quantity pointed to by the first arg (arg) | - | right by the number of bits specified by the second arg (nr). | - | | - +---------------------------------------------------------------------------*/ - -#include "fpu_emu.h" - -.text -SYM_FUNC_START(shr_Xsig) - push %ebp - movl %esp,%ebp - pushl %esi - movl PARAM2,%ecx - movl PARAM1,%esi - cmpl $32,%ecx /* shrd only works for 0..31 bits */ - jnc L_more_than_31 - -/* less than 32 bits */ - pushl %ebx - movl (%esi),%eax /* lsl */ - movl 4(%esi),%ebx /* midl */ - movl 8(%esi),%edx /* msl */ - shrd %cl,%ebx,%eax - shrd %cl,%edx,%ebx - shr %cl,%edx - movl %eax,(%esi) - movl %ebx,4(%esi) - movl %edx,8(%esi) - popl %ebx - popl %esi - leave - RET - -L_more_than_31: - cmpl $64,%ecx - jnc L_more_than_63 - - subb $32,%cl - movl 4(%esi),%eax /* midl */ - movl 8(%esi),%edx /* msl */ - shrd %cl,%edx,%eax - shr %cl,%edx - movl %eax,(%esi) - movl %edx,4(%esi) - movl $0,8(%esi) - popl %esi - leave - RET - -L_more_than_63: - cmpl $96,%ecx - jnc L_more_than_95 - - subb $64,%cl - movl 8(%esi),%eax /* msl */ - shr %cl,%eax - xorl %edx,%edx - movl %eax,(%esi) - movl %edx,4(%esi) - movl %edx,8(%esi) - popl %esi - leave - RET - -L_more_than_95: - xorl %eax,%eax - movl %eax,(%esi) - movl %eax,4(%esi) - movl %eax,8(%esi) - popl %esi - leave - RET -SYM_FUNC_END(shr_Xsig) diff --git a/arch/x86/math-emu/status_w.h b/arch/x86/math-emu/status_w.h deleted file mode 100644 index f642957330ef..000000000000 --- a/arch/x86/math-emu/status_w.h +++ /dev/null @@ -1,68 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/*---------------------------------------------------------------------------+ - | status_w.h | - | | - | Copyright (C) 1992,1993 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, | - | Australia. E-mail billm@vaxc.cc.monash.edu.au | - | | - +---------------------------------------------------------------------------*/ - -#ifndef _STATUS_H_ -#define _STATUS_H_ - -#include "fpu_emu.h" /* for definition of PECULIAR_486 */ - -#ifdef __ASSEMBLER__ -#define Const__(x) $##x -#else -#define Const__(x) x -#endif - -#define SW_Backward Const__(0x8000) /* backward compatibility */ -#define SW_C3 Const__(0x4000) /* condition bit 3 */ -#define SW_Top Const__(0x3800) /* top of stack */ -#define SW_Top_Shift Const__(11) /* shift for top of stack bits */ -#define SW_C2 Const__(0x0400) /* condition bit 2 */ -#define SW_C1 Const__(0x0200) /* condition bit 1 */ -#define SW_C0 Const__(0x0100) /* condition bit 0 */ -#define SW_Summary Const__(0x0080) /* exception summary */ -#define SW_Stack_Fault Const__(0x0040) /* stack fault */ -#define SW_Precision Const__(0x0020) /* loss of precision */ -#define SW_Underflow Const__(0x0010) /* underflow */ -#define SW_Overflow Const__(0x0008) /* overflow */ -#define SW_Zero_Div Const__(0x0004) /* divide by zero */ -#define SW_Denorm_Op Const__(0x0002) /* denormalized operand */ -#define SW_Invalid Const__(0x0001) /* invalid operation */ - -#define SW_Exc_Mask Const__(0x27f) /* Status word exception bit mask */ - -#ifndef __ASSEMBLER__ - -#define COMP_A_gt_B 1 -#define COMP_A_eq_B 2 -#define COMP_A_lt_B 3 -#define COMP_No_Comp 4 -#define COMP_Denormal 0x20 -#define COMP_NaN 0x40 -#define COMP_SNaN 0x80 - -#define status_word() \ - ((partial_status & ~SW_Top & 0xffff) | ((top << SW_Top_Shift) & SW_Top)) -static inline void setcc(int cc) -{ - partial_status &= ~(SW_C0 | SW_C1 | SW_C2 | SW_C3); - partial_status |= (cc) & (SW_C0 | SW_C1 | SW_C2 | SW_C3); -} - -#ifdef PECULIAR_486 - /* Default, this conveys no information, but an 80486 does it. */ - /* Clear the SW_C1 bit, "other bits undefined". */ -# define clear_C1() { partial_status &= ~SW_C1; } -# else -# define clear_C1() -#endif /* PECULIAR_486 */ - -#endif /* __ASSEMBLER__ */ - -#endif /* _STATUS_H_ */ diff --git a/arch/x86/math-emu/version.h b/arch/x86/math-emu/version.h deleted file mode 100644 index a0d73a1d2b67..000000000000 --- a/arch/x86/math-emu/version.h +++ /dev/null @@ -1,12 +0,0 @@ -/*---------------------------------------------------------------------------+ - | version.h | - | | - | | - | Copyright (C) 1992,1993,1994,1996,1997,1999 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia | - | E-mail billm@melbpc.org.au | - | | - | | - +---------------------------------------------------------------------------*/ - -#define FPU_VERSION "wm-FPU-emu version 2.01" diff --git a/arch/x86/math-emu/wm_shrx.S b/arch/x86/math-emu/wm_shrx.S deleted file mode 100644 index f608a28a4c43..000000000000 --- a/arch/x86/math-emu/wm_shrx.S +++ /dev/null @@ -1,207 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - .file "wm_shrx.S" -/*---------------------------------------------------------------------------+ - | wm_shrx.S | - | | - | 64 bit right shift functions | - | | - | Copyright (C) 1992,1995 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, | - | Australia. E-mail billm@jacobi.maths.monash.edu.au | - | | - | Call from C as: | - | unsigned FPU_shrx(void *arg1, unsigned arg2) | - | and | - | unsigned FPU_shrxs(void *arg1, unsigned arg2) | - | | - +---------------------------------------------------------------------------*/ - -#include "fpu_emu.h" - -.text -/*---------------------------------------------------------------------------+ - | unsigned FPU_shrx(void *arg1, unsigned arg2) | - | | - | Extended shift right function. | - | Fastest for small shifts. | - | Shifts the 64 bit quantity pointed to by the first arg (arg1) | - | right by the number of bits specified by the second arg (arg2). | - | Forms a 96 bit quantity from the 64 bit arg and eax: | - | [ 64 bit arg ][ eax ] | - | shift right ---------> | - | The eax register is initialized to 0 before the shifting. | - | Results returned in the 64 bit arg and eax. | - +---------------------------------------------------------------------------*/ - -SYM_FUNC_START(FPU_shrx) - push %ebp - movl %esp,%ebp - pushl %esi - movl PARAM2,%ecx - movl PARAM1,%esi - cmpl $32,%ecx /* shrd only works for 0..31 bits */ - jnc L_more_than_31 - -/* less than 32 bits */ - pushl %ebx - movl (%esi),%ebx /* lsl */ - movl 4(%esi),%edx /* msl */ - xorl %eax,%eax /* extension */ - shrd %cl,%ebx,%eax - shrd %cl,%edx,%ebx - shr %cl,%edx - movl %ebx,(%esi) - movl %edx,4(%esi) - popl %ebx - popl %esi - leave - RET - -L_more_than_31: - cmpl $64,%ecx - jnc L_more_than_63 - - subb $32,%cl - movl (%esi),%eax /* lsl */ - movl 4(%esi),%edx /* msl */ - shrd %cl,%edx,%eax - shr %cl,%edx - movl %edx,(%esi) - movl $0,4(%esi) - popl %esi - leave - RET - -L_more_than_63: - cmpl $96,%ecx - jnc L_more_than_95 - - subb $64,%cl - movl 4(%esi),%eax /* msl */ - shr %cl,%eax - xorl %edx,%edx - movl %edx,(%esi) - movl %edx,4(%esi) - popl %esi - leave - RET - -L_more_than_95: - xorl %eax,%eax - movl %eax,(%esi) - movl %eax,4(%esi) - popl %esi - leave - RET -SYM_FUNC_END(FPU_shrx) - - -/*---------------------------------------------------------------------------+ - | unsigned FPU_shrxs(void *arg1, unsigned arg2) | - | | - | Extended shift right function (optimized for small floating point | - | integers). | - | Shifts the 64 bit quantity pointed to by the first arg (arg1) | - | right by the number of bits specified by the second arg (arg2). | - | Forms a 96 bit quantity from the 64 bit arg and eax: | - | [ 64 bit arg ][ eax ] | - | shift right ---------> | - | The eax register is initialized to 0 before the shifting. | - | The lower 8 bits of eax are lost and replaced by a flag which is | - | set (to 0x01) if any bit, apart from the first one, is set in the | - | part which has been shifted out of the arg. | - | Results returned in the 64 bit arg and eax. | - +---------------------------------------------------------------------------*/ -SYM_FUNC_START(FPU_shrxs) - push %ebp - movl %esp,%ebp - pushl %esi - pushl %ebx - movl PARAM2,%ecx - movl PARAM1,%esi - cmpl $64,%ecx /* shrd only works for 0..31 bits */ - jnc Ls_more_than_63 - - cmpl $32,%ecx /* shrd only works for 0..31 bits */ - jc Ls_less_than_32 - -/* We got here without jumps by assuming that the most common requirement - is for small integers */ -/* Shift by [32..63] bits */ - subb $32,%cl - movl (%esi),%eax /* lsl */ - movl 4(%esi),%edx /* msl */ - xorl %ebx,%ebx - shrd %cl,%eax,%ebx - shrd %cl,%edx,%eax - shr %cl,%edx - orl %ebx,%ebx /* test these 32 bits */ - setne %bl - test $0x7fffffff,%eax /* and 31 bits here */ - setne %bh - orw %bx,%bx /* Any of the 63 bit set ? */ - setne %al - movl %edx,(%esi) - movl $0,4(%esi) - popl %ebx - popl %esi - leave - RET - -/* Shift by [0..31] bits */ -Ls_less_than_32: - movl (%esi),%ebx /* lsl */ - movl 4(%esi),%edx /* msl */ - xorl %eax,%eax /* extension */ - shrd %cl,%ebx,%eax - shrd %cl,%edx,%ebx - shr %cl,%edx - test $0x7fffffff,%eax /* only need to look at eax here */ - setne %al - movl %ebx,(%esi) - movl %edx,4(%esi) - popl %ebx - popl %esi - leave - RET - -/* Shift by [64..95] bits */ -Ls_more_than_63: - cmpl $96,%ecx - jnc Ls_more_than_95 - - subb $64,%cl - movl (%esi),%ebx /* lsl */ - movl 4(%esi),%eax /* msl */ - xorl %edx,%edx /* extension */ - shrd %cl,%ebx,%edx - shrd %cl,%eax,%ebx - shr %cl,%eax - orl %ebx,%edx - setne %bl - test $0x7fffffff,%eax /* only need to look at eax here */ - setne %bh - orw %bx,%bx - setne %al - xorl %edx,%edx - movl %edx,(%esi) /* set to zero */ - movl %edx,4(%esi) /* set to zero */ - popl %ebx - popl %esi - leave - RET - -Ls_more_than_95: -/* Shift by [96..inf) bits */ - xorl %eax,%eax - movl (%esi),%ebx - orl 4(%esi),%ebx - setne %al - xorl %ebx,%ebx - movl %ebx,(%esi) - movl %ebx,4(%esi) - popl %ebx - popl %esi - leave - RET -SYM_FUNC_END(FPU_shrxs) diff --git a/arch/x86/math-emu/wm_sqrt.S b/arch/x86/math-emu/wm_sqrt.S deleted file mode 100644 index 40526dd85137..000000000000 --- a/arch/x86/math-emu/wm_sqrt.S +++ /dev/null @@ -1,472 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - .file "wm_sqrt.S" -/*---------------------------------------------------------------------------+ - | wm_sqrt.S | - | | - | Fixed point arithmetic square root evaluation. | - | | - | Copyright (C) 1992,1993,1995,1997 | - | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, | - | Australia. E-mail billm@suburbia.net | - | | - | Call from C as: | - | int wm_sqrt(FPU_REG *n, unsigned int control_word) | - | | - +---------------------------------------------------------------------------*/ - -/*---------------------------------------------------------------------------+ - | wm_sqrt(FPU_REG *n, unsigned int control_word) | - | returns the square root of n in n. | - | | - | Use Newton's method to compute the square root of a number, which must | - | be in the range [1.0 .. 4.0), to 64 bits accuracy. | - | Does not check the sign or tag of the argument. | - | Sets the exponent, but not the sign or tag of the result. | - | | - | The guess is kept in %esi:%edi | - +---------------------------------------------------------------------------*/ - -#include "exception.h" -#include "fpu_emu.h" - - -#ifndef NON_REENTRANT_FPU -/* Local storage on the stack: */ -#define FPU_accum_3 -4(%ebp) /* ms word */ -#define FPU_accum_2 -8(%ebp) -#define FPU_accum_1 -12(%ebp) -#define FPU_accum_0 -16(%ebp) - -/* - * The de-normalised argument: - * sq_2 sq_1 sq_0 - * b b b b b b b ... b b b b b b .... b b b b 0 0 0 ... 0 - * ^ binary point here - */ -#define FPU_fsqrt_arg_2 -20(%ebp) /* ms word */ -#define FPU_fsqrt_arg_1 -24(%ebp) -#define FPU_fsqrt_arg_0 -28(%ebp) /* ls word, at most the ms bit is set */ - -#else -/* Local storage in a static area: */ -.data - .align 4,0 -FPU_accum_3: - .long 0 /* ms word */ -FPU_accum_2: - .long 0 -FPU_accum_1: - .long 0 -FPU_accum_0: - .long 0 - -/* The de-normalised argument: - sq_2 sq_1 sq_0 - b b b b b b b ... b b b b b b .... b b b b 0 0 0 ... 0 - ^ binary point here - */ -FPU_fsqrt_arg_2: - .long 0 /* ms word */ -FPU_fsqrt_arg_1: - .long 0 -FPU_fsqrt_arg_0: - .long 0 /* ls word, at most the ms bit is set */ -#endif /* NON_REENTRANT_FPU */ - - -.text -SYM_FUNC_START(wm_sqrt) - pushl %ebp - movl %esp,%ebp -#ifndef NON_REENTRANT_FPU - subl $28,%esp -#endif /* NON_REENTRANT_FPU */ - pushl %esi - pushl %edi - pushl %ebx - - movl PARAM1,%esi - - movl SIGH(%esi),%eax - movl SIGL(%esi),%ecx - xorl %edx,%edx - -/* We use a rough linear estimate for the first guess.. */ - - cmpw EXP_BIAS,EXP(%esi) - jnz sqrt_arg_ge_2 - - shrl $1,%eax /* arg is in the range [1.0 .. 2.0) */ - rcrl $1,%ecx - rcrl $1,%edx - -sqrt_arg_ge_2: -/* From here on, n is never accessed directly again until it is - replaced by the answer. */ - - movl %eax,FPU_fsqrt_arg_2 /* ms word of n */ - movl %ecx,FPU_fsqrt_arg_1 - movl %edx,FPU_fsqrt_arg_0 - -/* Make a linear first estimate */ - shrl $1,%eax - addl $0x40000000,%eax - movl $0xaaaaaaaa,%ecx - mull %ecx - shll %edx /* max result was 7fff... */ - testl $0x80000000,%edx /* but min was 3fff... */ - jnz sqrt_prelim_no_adjust - - movl $0x80000000,%edx /* round up */ - -sqrt_prelim_no_adjust: - movl %edx,%esi /* Our first guess */ - -/* We have now computed (approx) (2 + x) / 3, which forms the basis - for a few iterations of Newton's method */ - - movl FPU_fsqrt_arg_2,%ecx /* ms word */ - -/* - * From our initial estimate, three iterations are enough to get us - * to 30 bits or so. This will then allow two iterations at better - * precision to complete the process. - */ - -/* Compute (g + n/g)/2 at each iteration (g is the guess). */ - shrl %ecx /* Doing this first will prevent a divide */ - /* overflow later. */ - - movl %ecx,%edx /* msw of the arg / 2 */ - divl %esi /* current estimate */ - shrl %esi /* divide by 2 */ - addl %eax,%esi /* the new estimate */ - - movl %ecx,%edx - divl %esi - shrl %esi - addl %eax,%esi - - movl %ecx,%edx - divl %esi - shrl %esi - addl %eax,%esi - -/* - * Now that an estimate accurate to about 30 bits has been obtained (in %esi), - * we improve it to 60 bits or so. - * - * The strategy from now on is to compute new estimates from - * guess := guess + (n - guess^2) / (2 * guess) - */ - -/* First, find the square of the guess */ - movl %esi,%eax - mull %esi -/* guess^2 now in %edx:%eax */ - - movl FPU_fsqrt_arg_1,%ecx - subl %ecx,%eax - movl FPU_fsqrt_arg_2,%ecx /* ms word of normalized n */ - sbbl %ecx,%edx - jnc sqrt_stage_2_positive - -/* Subtraction gives a negative result, - negate the result before division. */ - notl %edx - notl %eax - addl $1,%eax - adcl $0,%edx - - divl %esi - movl %eax,%ecx - - movl %edx,%eax - divl %esi - jmp sqrt_stage_2_finish - -sqrt_stage_2_positive: - divl %esi - movl %eax,%ecx - - movl %edx,%eax - divl %esi - - notl %ecx - notl %eax - addl $1,%eax - adcl $0,%ecx - -sqrt_stage_2_finish: - sarl $1,%ecx /* divide by 2 */ - rcrl $1,%eax - - /* Form the new estimate in %esi:%edi */ - movl %eax,%edi - addl %ecx,%esi - - jnz sqrt_stage_2_done /* result should be [1..2) */ - -#ifdef PARANOID -/* It should be possible to get here only if the arg is ffff....ffff */ - cmpl $0xffffffff,FPU_fsqrt_arg_1 - jnz sqrt_stage_2_error -#endif /* PARANOID */ - -/* The best rounded result. */ - xorl %eax,%eax - decl %eax - movl %eax,%edi - movl %eax,%esi - movl $0x7fffffff,%eax - jmp sqrt_round_result - -#ifdef PARANOID -sqrt_stage_2_error: - pushl EX_INTERNAL|0x213 - call EXCEPTION -#endif /* PARANOID */ - -sqrt_stage_2_done: - -/* Now the square root has been computed to better than 60 bits. */ - -/* Find the square of the guess. */ - movl %edi,%eax /* ls word of guess */ - mull %edi - movl %edx,FPU_accum_1 - - movl %esi,%eax - mull %esi - movl %edx,FPU_accum_3 - movl %eax,FPU_accum_2 - - movl %edi,%eax - mull %esi - addl %eax,FPU_accum_1 - adcl %edx,FPU_accum_2 - adcl $0,FPU_accum_3 - -/* movl %esi,%eax */ -/* mull %edi */ - addl %eax,FPU_accum_1 - adcl %edx,FPU_accum_2 - adcl $0,FPU_accum_3 - -/* guess^2 now in FPU_accum_3:FPU_accum_2:FPU_accum_1 */ - - movl FPU_fsqrt_arg_0,%eax /* get normalized n */ - subl %eax,FPU_accum_1 - movl FPU_fsqrt_arg_1,%eax - sbbl %eax,FPU_accum_2 - movl FPU_fsqrt_arg_2,%eax /* ms word of normalized n */ - sbbl %eax,FPU_accum_3 - jnc sqrt_stage_3_positive - -/* Subtraction gives a negative result, - negate the result before division */ - notl FPU_accum_1 - notl FPU_accum_2 - notl FPU_accum_3 - addl $1,FPU_accum_1 - adcl $0,FPU_accum_2 - -#ifdef PARANOID - adcl $0,FPU_accum_3 /* This must be zero */ - jz sqrt_stage_3_no_error - -sqrt_stage_3_error: - pushl EX_INTERNAL|0x207 - call EXCEPTION - -sqrt_stage_3_no_error: -#endif /* PARANOID */ - - movl FPU_accum_2,%edx - movl FPU_accum_1,%eax - divl %esi - movl %eax,%ecx - - movl %edx,%eax - divl %esi - - sarl $1,%ecx /* divide by 2 */ - rcrl $1,%eax - - /* prepare to round the result */ - - addl %ecx,%edi - adcl $0,%esi - - jmp sqrt_stage_3_finished - -sqrt_stage_3_positive: - movl FPU_accum_2,%edx - movl FPU_accum_1,%eax - divl %esi - movl %eax,%ecx - - movl %edx,%eax - divl %esi - - sarl $1,%ecx /* divide by 2 */ - rcrl $1,%eax - - /* prepare to round the result */ - - notl %eax /* Negate the correction term */ - notl %ecx - addl $1,%eax - adcl $0,%ecx /* carry here ==> correction == 0 */ - adcl $0xffffffff,%esi - - addl %ecx,%edi - adcl $0,%esi - -sqrt_stage_3_finished: - -/* - * The result in %esi:%edi:%esi should be good to about 90 bits here, - * and the rounding information here does not have sufficient accuracy - * in a few rare cases. - */ - cmpl $0xffffffe0,%eax - ja sqrt_near_exact_x - - cmpl $0x00000020,%eax - jb sqrt_near_exact - - cmpl $0x7fffffe0,%eax - jb sqrt_round_result - - cmpl $0x80000020,%eax - jb sqrt_get_more_precision - -sqrt_round_result: -/* Set up for rounding operations */ - movl %eax,%edx - movl %esi,%eax - movl %edi,%ebx - movl PARAM1,%edi - movw EXP_BIAS,EXP(%edi) /* Result is in [1.0 .. 2.0) */ - jmp fpu_reg_round - - -sqrt_near_exact_x: -/* First, the estimate must be rounded up. */ - addl $1,%edi - adcl $0,%esi - -sqrt_near_exact: -/* - * This is an easy case because x^1/2 is monotonic. - * We need just find the square of our estimate, compare it - * with the argument, and deduce whether our estimate is - * above, below, or exact. We use the fact that the estimate - * is known to be accurate to about 90 bits. - */ - movl %edi,%eax /* ls word of guess */ - mull %edi - movl %edx,%ebx /* 2nd ls word of square */ - movl %eax,%ecx /* ls word of square */ - - movl %edi,%eax - mull %esi - addl %eax,%ebx - addl %eax,%ebx - -#ifdef PARANOID - cmp $0xffffffb0,%ebx - jb sqrt_near_exact_ok - - cmp $0x00000050,%ebx - ja sqrt_near_exact_ok - - pushl EX_INTERNAL|0x214 - call EXCEPTION - -sqrt_near_exact_ok: -#endif /* PARANOID */ - - or %ebx,%ebx - js sqrt_near_exact_small - - jnz sqrt_near_exact_large - - or %ebx,%edx - jnz sqrt_near_exact_large - -/* Our estimate is exactly the right answer */ - xorl %eax,%eax - jmp sqrt_round_result - -sqrt_near_exact_small: -/* Our estimate is too small */ - movl $0x000000ff,%eax - jmp sqrt_round_result - -sqrt_near_exact_large: -/* Our estimate is too large, we need to decrement it */ - subl $1,%edi - sbbl $0,%esi - movl $0xffffff00,%eax - jmp sqrt_round_result - - -sqrt_get_more_precision: -/* This case is almost the same as the above, except we start - with an extra bit of precision in the estimate. */ - stc /* The extra bit. */ - rcll $1,%edi /* Shift the estimate left one bit */ - rcll $1,%esi - - movl %edi,%eax /* ls word of guess */ - mull %edi - movl %edx,%ebx /* 2nd ls word of square */ - movl %eax,%ecx /* ls word of square */ - - movl %edi,%eax - mull %esi - addl %eax,%ebx - addl %eax,%ebx - -/* Put our estimate back to its original value */ - stc /* The ms bit. */ - rcrl $1,%esi /* Shift the estimate left one bit */ - rcrl $1,%edi - -#ifdef PARANOID - cmp $0xffffff60,%ebx - jb sqrt_more_prec_ok - - cmp $0x000000a0,%ebx - ja sqrt_more_prec_ok - - pushl EX_INTERNAL|0x215 - call EXCEPTION - -sqrt_more_prec_ok: -#endif /* PARANOID */ - - or %ebx,%ebx - js sqrt_more_prec_small - - jnz sqrt_more_prec_large - - or %ebx,%ecx - jnz sqrt_more_prec_large - -/* Our estimate is exactly the right answer */ - movl $0x80000000,%eax - jmp sqrt_round_result - -sqrt_more_prec_small: -/* Our estimate is too small */ - movl $0x800000ff,%eax - jmp sqrt_round_result - -sqrt_more_prec_large: -/* Our estimate is too large */ - movl $0x7fffff00,%eax - jmp sqrt_round_result -SYM_FUNC_END(wm_sqrt) diff --git a/arch/x86/mm/pti.c b/arch/x86/mm/pti.c index 631f0375bd42..598f553cc871 100644 --- a/arch/x86/mm/pti.c +++ b/arch/x86/mm/pti.c @@ -31,6 +31,7 @@ #include #include +#include #include #include #include diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index 4c045f844492..1023acadd8f8 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -1769,7 +1769,7 @@ bool nmi_uaccess_okay(void) } static ssize_t tlbflush_read_file(struct file *file, char __user *user_buf, - size_t count, loff_t *ppos) + size_t count, loff_t *ppos) { char buf[32]; unsigned int len; @@ -1778,20 +1778,15 @@ static ssize_t tlbflush_read_file(struct file *file, char __user *user_buf, return simple_read_from_buffer(user_buf, count, ppos, buf, len); } -static ssize_t tlbflush_write_file(struct file *file, - const char __user *user_buf, size_t count, loff_t *ppos) +static ssize_t tlbflush_write_file(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) { - char buf[32]; - ssize_t len; int ceiling; + int err; - len = min(count, sizeof(buf) - 1); - if (copy_from_user(buf, user_buf, len)) - return -EFAULT; - - buf[len] = '\0'; - if (kstrtoint(buf, 0, &ceiling)) - return -EINVAL; + err = kstrtoint_from_user(user_buf, count, 0, &ceiling); + if (err) + return err; if (ceiling < 0) return -EINVAL; diff --git a/arch/x86/pci/xen.c b/arch/x86/pci/xen.c index 6818515a501b..550c631bc77f 100644 --- a/arch/x86/pci/xen.c +++ b/arch/x86/pci/xen.c @@ -18,6 +18,7 @@ #include #include #include +#include #include diff --git a/arch/x86/platform/uv/uv_nmi.c b/arch/x86/platform/uv/uv_nmi.c index 5c50e550ab63..565ab43fa6b4 100644 --- a/arch/x86/platform/uv/uv_nmi.c +++ b/arch/x86/platform/uv/uv_nmi.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -340,7 +341,7 @@ static void uv_nmi_setup_hubless_intr(void) uv_pch_intr_now_enabled ? GPIROUTNMI : 0); nmi_debug("UV:NMI: GPP_D_0 interrupt %s\n", - uv_pch_intr_now_enabled ? "enabled" : "disabled"); + str_enabled_disabled(uv_pch_intr_now_enabled)); } static struct init_nmi { diff --git a/arch/x86/xen/Kconfig b/arch/x86/xen/Kconfig index 51b53ce66efb..99b06f5c47cd 100644 --- a/arch/x86/xen/Kconfig +++ b/arch/x86/xen/Kconfig @@ -11,7 +11,7 @@ config XEN select HIBERNATE_CALLBACKS depends on X86_64 || (X86_32 && X86_PAE) depends on X86_64 || (X86_GENERIC || MPENTIUM4 || MATOM) - depends on X86_LOCAL_APIC && X86_TSC + depends on X86_LOCAL_APIC help This is the Linux Xen port. Enabling this will allow the kernel to boot in a paravirtualized environment under the diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c index 23b91bf9b663..cf061ed45ce8 100644 --- a/arch/x86/xen/enlighten.c +++ b/arch/x86/xen/enlighten.c @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include "xen-ops.h" @@ -76,6 +76,7 @@ unsigned long xen_released_pages; static __ref void xen_get_vendor(void) { init_cpu_devs(); + cpuid_scan_cpu(&boot_cpu_data); cpu_detect(&boot_cpu_data); get_cpu_vendor(&boot_cpu_data); } diff --git a/arch/x86/xen/enlighten_hvm.c b/arch/x86/xen/enlighten_hvm.c index 6c4eac4ff13a..89ae28415a3b 100644 --- a/arch/x86/xen/enlighten_hvm.c +++ b/arch/x86/xen/enlighten_hvm.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c index 54a714077eee..2c64b388f616 100644 --- a/arch/x86/xen/enlighten_pv.c +++ b/arch/x86/xen/enlighten_pv.c @@ -1423,6 +1423,7 @@ asmlinkage __visible void __init xen_start_kernel(struct start_info *si) xen_build_dynamic_phys_to_machine(); /* Work out if we support NX */ + cpuid_scan_cpu(&boot_cpu_data); get_cpu_cap(&boot_cpu_data); x86_configure_nx(); diff --git a/arch/x86/xen/pmu.c b/arch/x86/xen/pmu.c index 8f89ce0b67e3..5f50a3ee08f5 100644 --- a/arch/x86/xen/pmu.c +++ b/arch/x86/xen/pmu.c @@ -2,6 +2,7 @@ #include #include +#include #include #include #include diff --git a/arch/x86/xen/time.c b/arch/x86/xen/time.c index 6f9f665bb7ae..d62c14334b35 100644 --- a/arch/x86/xen/time.c +++ b/arch/x86/xen/time.c @@ -18,6 +18,7 @@ #include #include +#include #include #include #include diff --git a/drivers/char/agp/efficeon-agp.c b/drivers/char/agp/efficeon-agp.c index 0d25bbdc7e6a..4d0b7d7c0aad 100644 --- a/drivers/char/agp/efficeon-agp.c +++ b/drivers/char/agp/efficeon-agp.c @@ -27,6 +27,7 @@ #include #include #include +#include #include "agp.h" #include "intel-agp.h" diff --git a/drivers/char/hw_random/s390-trng.c b/drivers/char/hw_random/s390-trng.c index 3024d5e9fd61..5520f66274b3 100644 --- a/drivers/char/hw_random/s390-trng.c +++ b/drivers/char/hw_random/s390-trng.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -67,7 +68,7 @@ static ssize_t trng_read(struct file *file, char __user *ubuf, */ if (nbytes > sizeof(buf)) { - p = (u8 *) __get_free_page(GFP_KERNEL); + p = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!p) return -ENOMEM; } @@ -94,7 +95,7 @@ static ssize_t trng_read(struct file *file, char __user *ubuf, } if (p != buf) - free_page((unsigned long) p); + kfree(p); DEBUG_DBG("trng_read()=%zd\n", ret); return ret; diff --git a/drivers/cpufreq/Kconfig.x86 b/drivers/cpufreq/Kconfig.x86 index 46b9742a1688..d6190b8dde4b 100644 --- a/drivers/cpufreq/Kconfig.x86 +++ b/drivers/cpufreq/Kconfig.x86 @@ -113,17 +113,6 @@ config X86_ACPI_CPUFREQ_CPB By enabling this option the acpi_cpufreq driver provides the old entry in addition to the new boost ones, for compatibility reasons. -config SC520_CPUFREQ - tristate "AMD Elan SC520" - depends on MELAN - help - This adds the CPUFreq driver for AMD Elan SC520 processor. - - For details, take a look at . - - If in doubt, say N. - - config X86_POWERNOW_K6 tristate "AMD Mobile K6-2/K6-3 PowerNow!" depends on X86_32 diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile index 2583c02b061c..50dc220fc66f 100644 --- a/drivers/cpufreq/Makefile +++ b/drivers/cpufreq/Makefile @@ -45,7 +45,6 @@ obj-$(CONFIG_X86_POWERNOW_K6) += powernow-k6.o obj-$(CONFIG_X86_POWERNOW_K7) += powernow-k7.o obj-$(CONFIG_X86_LONGHAUL) += longhaul.o obj-$(CONFIG_X86_E_POWERSAVER) += e_powersaver.o -obj-$(CONFIG_SC520_CPUFREQ) += sc520_freq.o obj-$(CONFIG_X86_LONGRUN) += longrun.o obj-$(CONFIG_X86_GX_SUSPMOD) += gx-suspmod.o obj-$(CONFIG_X86_SPEEDSTEP_ICH) += speedstep-ich.o diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index f5154c77c31b..3a80acee9a7c 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -472,7 +472,7 @@ static int msr_init_perf(struct amd_cpudata *cpudata) if (ret) return ret; - ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &cppc_req); + ret = rdmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &cppc_req); if (ret) return ret; diff --git a/drivers/cpufreq/amd_freq_sensitivity.c b/drivers/cpufreq/amd_freq_sensitivity.c index 13fed4b9e02b..739d54dc9f2b 100644 --- a/drivers/cpufreq/amd_freq_sensitivity.c +++ b/drivers/cpufreq/amd_freq_sensitivity.c @@ -51,10 +51,8 @@ static unsigned int amd_powersave_bias_target(struct cpufreq_policy *policy, if (!policy->freq_table) return freq_next; - rdmsr_on_cpu(policy->cpu, MSR_AMD64_FREQ_SENSITIVITY_ACTUAL, - &actual.l, &actual.h); - rdmsr_on_cpu(policy->cpu, MSR_AMD64_FREQ_SENSITIVITY_REFERENCE, - &reference.l, &reference.h); + rdmsrq_on_cpu(policy->cpu, MSR_AMD64_FREQ_SENSITIVITY_ACTUAL, &actual.q); + rdmsrq_on_cpu(policy->cpu, MSR_AMD64_FREQ_SENSITIVITY_REFERENCE, &reference.q); actual.h &= 0x00ffffff; reference.h &= 0x00ffffff; diff --git a/drivers/cpufreq/longrun.c b/drivers/cpufreq/longrun.c index 1caaec7c280b..f3aaca0496a4 100644 --- a/drivers/cpufreq/longrun.c +++ b/drivers/cpufreq/longrun.c @@ -14,6 +14,7 @@ #include #include #include +#include static struct cpufreq_driver longrun_driver; diff --git a/drivers/cpufreq/p4-clockmod.c b/drivers/cpufreq/p4-clockmod.c index 69c19233fcd4..c1690aa48193 100644 --- a/drivers/cpufreq/p4-clockmod.c +++ b/drivers/cpufreq/p4-clockmod.c @@ -51,24 +51,24 @@ static unsigned int cpufreq_p4_get(unsigned int cpu); static int cpufreq_p4_setdc(unsigned int cpu, unsigned int newstate) { - u32 l, h; + struct msr val; if ((newstate > DC_DISABLE) || (newstate == DC_RESV)) return -EINVAL; - rdmsr_on_cpu(cpu, MSR_IA32_THERM_STATUS, &l, &h); + rdmsrq_on_cpu(cpu, MSR_IA32_THERM_STATUS, &val.q); - if (l & 0x01) + if (val.l & 0x01) pr_debug("CPU#%d currently thermal throttled\n", cpu); if (has_N44_O17_errata[cpu] && (newstate == DC_25PT || newstate == DC_DFLT)) newstate = DC_38PT; - rdmsr_on_cpu(cpu, MSR_IA32_THERM_CONTROL, &l, &h); + rdmsrq_on_cpu(cpu, MSR_IA32_THERM_CONTROL, &val.q); if (newstate == DC_DISABLE) { pr_debug("CPU#%d disabling modulation\n", cpu); - wrmsr_on_cpu(cpu, MSR_IA32_THERM_CONTROL, l & ~(1<<4), h); + wrmsrq_on_cpu(cpu, MSR_IA32_THERM_CONTROL, val.q & ~(1ULL << 4)); } else { pr_debug("CPU#%d setting duty cycle to %d%%\n", cpu, ((125 * newstate) / 10)); @@ -77,9 +77,9 @@ static int cpufreq_p4_setdc(unsigned int cpu, unsigned int newstate) * bits 3-1 : duty cycle * bit 0 : reserved */ - l = (l & ~14); - l = l | (1<<4) | ((newstate & 0x7)<<1); - wrmsr_on_cpu(cpu, MSR_IA32_THERM_CONTROL, l, h); + val.l = (val.l & ~14); + val.l = val.l | (1<<4) | ((newstate & 0x7)<<1); + wrmsrq_on_cpu(cpu, MSR_IA32_THERM_CONTROL, val.q); } return 0; @@ -205,18 +205,18 @@ static int cpufreq_p4_cpu_init(struct cpufreq_policy *policy) static unsigned int cpufreq_p4_get(unsigned int cpu) { - u32 l, h; + struct msr val; - rdmsr_on_cpu(cpu, MSR_IA32_THERM_CONTROL, &l, &h); + rdmsrq_on_cpu(cpu, MSR_IA32_THERM_CONTROL, &val.q); - if (l & 0x10) { - l = l >> 1; - l &= 0x7; + if (val.l & 0x10) { + val.l = val.l >> 1; + val.l &= 0x7; } else - l = DC_DISABLE; + val.l = DC_DISABLE; - if (l != DC_DISABLE) - return stock_freq * l / 8; + if (val.l != DC_DISABLE) + return stock_freq * val.l / 8; return stock_freq; } diff --git a/drivers/cpufreq/powernow-k7.c b/drivers/cpufreq/powernow-k7.c index 6b7caf4ae20d..6a930d7e6a5c 100644 --- a/drivers/cpufreq/powernow-k7.c +++ b/drivers/cpufreq/powernow-k7.c @@ -29,6 +29,7 @@ #include /* Needed for recalibrate_cpu_khz() */ #include #include +#include #ifdef CONFIG_X86_POWERNOW_K7_ACPI #include diff --git a/drivers/cpufreq/powernow-k8.c b/drivers/cpufreq/powernow-k8.c index 4d77eef53fe0..2b791f1ec51b 100644 --- a/drivers/cpufreq/powernow-k8.c +++ b/drivers/cpufreq/powernow-k8.c @@ -39,6 +39,7 @@ #include #include +#include #include #include diff --git a/drivers/cpufreq/sc520_freq.c b/drivers/cpufreq/sc520_freq.c deleted file mode 100644 index b360f03a116f..000000000000 --- a/drivers/cpufreq/sc520_freq.c +++ /dev/null @@ -1,136 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * sc520_freq.c: cpufreq driver for the AMD Elan sc520 - * - * Copyright (C) 2005 Sean Young - * - * Based on elanfreq.c - * - * 2005-03-30: - initial revision - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include -#include -#include - -#include -#include -#include -#include - -#include - -#define MMCR_BASE 0xfffef000 /* The default base address */ -#define OFFS_CPUCTL 0x2 /* CPU Control Register */ - -static __u8 __iomem *cpuctl; - -static struct cpufreq_frequency_table sc520_freq_table[] = { - {0, 0x01, 100000}, - {0, 0x02, 133000}, - {0, 0, CPUFREQ_TABLE_END}, -}; - -static unsigned int sc520_freq_get_cpu_frequency(unsigned int cpu) -{ - u8 clockspeed_reg = *cpuctl; - - switch (clockspeed_reg & 0x03) { - default: - pr_err("error: cpuctl register has unexpected value %02x\n", - clockspeed_reg); - fallthrough; - case 0x01: - return 100000; - case 0x02: - return 133000; - } -} - -static int sc520_freq_target(struct cpufreq_policy *policy, unsigned int state) -{ - - u8 clockspeed_reg; - - local_irq_disable(); - - clockspeed_reg = *cpuctl & ~0x03; - *cpuctl = clockspeed_reg | sc520_freq_table[state].driver_data; - - local_irq_enable(); - - return 0; -} - -/* - * Module init and exit code - */ - -static int sc520_freq_cpu_init(struct cpufreq_policy *policy) -{ - struct cpuinfo_x86 *c = &cpu_data(0); - - /* capability check */ - if (c->x86_vendor != X86_VENDOR_AMD || - c->x86 != 4 || c->x86_model != 9) - return -ENODEV; - - /* cpuinfo and default policy values */ - policy->cpuinfo.transition_latency = 1000000; /* 1ms */ - policy->freq_table = sc520_freq_table; - - return 0; -} - - -static struct cpufreq_driver sc520_freq_driver = { - .get = sc520_freq_get_cpu_frequency, - .verify = cpufreq_generic_frequency_table_verify, - .target_index = sc520_freq_target, - .init = sc520_freq_cpu_init, - .name = "sc520_freq", -}; - -static const struct x86_cpu_id sc520_ids[] = { - X86_MATCH_VENDOR_FAM_MODEL(AMD, 4, 9, NULL), - {} -}; -MODULE_DEVICE_TABLE(x86cpu, sc520_ids); - -static int __init sc520_freq_init(void) -{ - int err; - - if (!x86_match_cpu(sc520_ids)) - return -ENODEV; - - cpuctl = ioremap((unsigned long)(MMCR_BASE + OFFS_CPUCTL), 1); - if (!cpuctl) { - pr_err("sc520_freq: error: failed to remap memory\n"); - return -ENOMEM; - } - - err = cpufreq_register_driver(&sc520_freq_driver); - if (err) - iounmap(cpuctl); - - return err; -} - - -static void __exit sc520_freq_exit(void) -{ - cpufreq_unregister_driver(&sc520_freq_driver); - iounmap(cpuctl); -} - - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Sean Young "); -MODULE_DESCRIPTION("cpufreq driver for AMD's Elan sc520 CPU"); - -module_init(sc520_freq_init); -module_exit(sc520_freq_exit); - diff --git a/drivers/cpufreq/speedstep-centrino.c b/drivers/cpufreq/speedstep-centrino.c index 3e6e85a92212..9237ed8f2b1f 100644 --- a/drivers/cpufreq/speedstep-centrino.c +++ b/drivers/cpufreq/speedstep-centrino.c @@ -322,11 +322,11 @@ static unsigned extract_clock(unsigned msr, unsigned int cpu, int failsafe) /* Return the current CPU frequency in kHz */ static unsigned int get_cur_freq(unsigned int cpu) { - unsigned l, h; + struct msr val; unsigned clock_freq; - rdmsr_on_cpu(cpu, MSR_IA32_PERF_STATUS, &l, &h); - clock_freq = extract_clock(l, cpu, 0); + rdmsrq_on_cpu(cpu, MSR_IA32_PERF_STATUS, &val.q); + clock_freq = extract_clock(val.l, cpu, 0); if (unlikely(clock_freq == 0)) { /* @@ -335,8 +335,8 @@ static unsigned int get_cur_freq(unsigned int cpu) * P-state transition (like TM2). Get the last freq set * in PERF_CTL. */ - rdmsr_on_cpu(cpu, MSR_IA32_PERF_CTL, &l, &h); - clock_freq = extract_clock(l, cpu, 1); + rdmsrq_on_cpu(cpu, MSR_IA32_PERF_CTL, &val.q); + clock_freq = extract_clock(val.l, cpu, 1); } return clock_freq; } @@ -417,7 +417,8 @@ static void centrino_cpu_exit(struct cpufreq_policy *policy) */ static int centrino_target(struct cpufreq_policy *policy, unsigned int index) { - unsigned int msr, oldmsr = 0, h = 0, cpu = policy->cpu; + unsigned int msr, cpu = policy->cpu; + struct msr oldmsr = { .q = 0 }; int retval = 0; unsigned int j, first_cpu; struct cpufreq_frequency_table *op_points; @@ -459,22 +460,22 @@ static int centrino_target(struct cpufreq_policy *policy, unsigned int index) msr = op_points->driver_data; if (first_cpu) { - rdmsr_on_cpu(good_cpu, MSR_IA32_PERF_CTL, &oldmsr, &h); - if (msr == (oldmsr & 0xffff)) { + rdmsrq_on_cpu(good_cpu, MSR_IA32_PERF_CTL, &oldmsr.q); + if (msr == (oldmsr.l & 0xffff)) { pr_debug("no change needed - msr was and needs " - "to be %x\n", oldmsr); + "to be %x\n", oldmsr.l); retval = 0; goto out; } first_cpu = 0; /* all but 16 LSB are reserved, treat them with care */ - oldmsr &= ~0xffff; + oldmsr.l &= ~0xffff; msr &= 0xffff; - oldmsr |= msr; + oldmsr.l |= msr; } - wrmsr_on_cpu(good_cpu, MSR_IA32_PERF_CTL, oldmsr, h); + wrmsrq_on_cpu(good_cpu, MSR_IA32_PERF_CTL, oldmsr.q); if (policy->shared_type == CPUFREQ_SHARED_TYPE_ANY) break; @@ -490,7 +491,7 @@ static int centrino_target(struct cpufreq_policy *policy, unsigned int index) */ for_each_cpu(j, covered_cpus) - wrmsr_on_cpu(j, MSR_IA32_PERF_CTL, oldmsr, h); + wrmsrq_on_cpu(j, MSR_IA32_PERF_CTL, oldmsr.q); } retval = 0; diff --git a/drivers/cpufreq/speedstep-lib.c b/drivers/cpufreq/speedstep-lib.c index f8b42e981635..973716c1c29c 100644 --- a/drivers/cpufreq/speedstep-lib.c +++ b/drivers/cpufreq/speedstep-lib.c @@ -15,6 +15,7 @@ #include #include +#include #include #include #include "speedstep-lib.h" diff --git a/drivers/dio/dio.c b/drivers/dio/dio.c index 419b3c13d491..7722d9eae6d0 100644 --- a/drivers/dio/dio.c +++ b/drivers/dio/dio.c @@ -178,7 +178,7 @@ static int __init dio_init(void) if (!MACH_IS_HP300) return 0; - printk(KERN_INFO "Scanning for DIO devices...\n"); + pr_info("Scanning for DIO devices...\n"); /* Initialize the DIO bus */ INIT_LIST_HEAD(&dio_bus.devices); @@ -247,18 +247,19 @@ static int __init dio_init(void) dev->id = prid; dev->ipl = DIO_IPL(va); - strcpy(dev->name, dio_getname(dev->id)); - printk(KERN_INFO "select code %3d: ipl %d: ID %02X", dev->scode, dev->ipl, prid); + strscpy(dev->name, dio_getname(dev->id)); if (DIO_NEEDSSECID(prid)) - printk(":%02X", secid); - printk(": %s\n", dev->name); + pr_info("select code %3d: ipl %u: ID %02X:%02X: %s\n", + dev->scode, dev->ipl, prid, secid, dev->name); + else + pr_info("select code %3d: ipl %u: ID %02X: %s\n", + dev->scode, dev->ipl, prid, dev->name); if (scode >= DIOII_SCBASE) iounmap(va); error = device_register(&dev->dev); if (error) { - pr_err("DIO: Error registering device %s\n", - dev->name); + pr_err("DIO: Error registering device %s\n", dev->name); put_device(&dev->dev); continue; } diff --git a/drivers/edac/amd76x_edac.c b/drivers/edac/amd76x_edac.c index 2a49f68a7cf9..7bb11ffdb0c9 100644 --- a/drivers/edac/amd76x_edac.c +++ b/drivers/edac/amd76x_edac.c @@ -332,14 +332,14 @@ static void amd76x_remove_one(struct pci_dev *pdev) static const struct pci_device_id amd76x_pci_tbl[] = { { - PCI_VEND_DEV(AMD, FE_GATE_700C), PCI_ANY_ID, PCI_ANY_ID, 0, 0, - AMD762}, - { - PCI_VEND_DEV(AMD, FE_GATE_700E), PCI_ANY_ID, PCI_ANY_ID, 0, 0, - AMD761}, - { - 0, - } /* 0 terminated list. */ + PCI_VEND_DEV(AMD, FE_GATE_700C), + .driver_data = AMD762 + }, { + PCI_VEND_DEV(AMD, FE_GATE_700E), + .driver_data = AMD761, + }, { + /* 0 terminated list. */ + } }; MODULE_DEVICE_TABLE(pci, amd76x_pci_tbl); diff --git a/drivers/edac/e752x_edac.c b/drivers/edac/e752x_edac.c index 7221b4bb6df2..77c1fe75451e 100644 --- a/drivers/edac/e752x_edac.c +++ b/drivers/edac/e752x_edac.c @@ -1414,20 +1414,20 @@ static void e752x_remove_one(struct pci_dev *pdev) static const struct pci_device_id e752x_pci_tbl[] = { { - PCI_VEND_DEV(INTEL, 7520_0), PCI_ANY_ID, PCI_ANY_ID, 0, 0, - E7520}, - { - PCI_VEND_DEV(INTEL, 7525_0), PCI_ANY_ID, PCI_ANY_ID, 0, 0, - E7525}, - { - PCI_VEND_DEV(INTEL, 7320_0), PCI_ANY_ID, PCI_ANY_ID, 0, 0, - E7320}, - { - PCI_VEND_DEV(INTEL, 3100_0), PCI_ANY_ID, PCI_ANY_ID, 0, 0, - I3100}, - { - 0, - } /* 0 terminated list. */ + PCI_VEND_DEV(INTEL, 7520_0), + .driver_data = E7520, + }, { + PCI_VEND_DEV(INTEL, 7525_0), + .driver_data = E7525, + }, { + PCI_VEND_DEV(INTEL, 7320_0), + .driver_data = E7320, + }, { + PCI_VEND_DEV(INTEL, 3100_0), + .driver_data = I3100, + }, { + /* 0 terminated list. */ + } }; MODULE_DEVICE_TABLE(pci, e752x_pci_tbl); diff --git a/drivers/edac/e7xxx_edac.c b/drivers/edac/e7xxx_edac.c index 5852b95fa470..02071180b638 100644 --- a/drivers/edac/e7xxx_edac.c +++ b/drivers/edac/e7xxx_edac.c @@ -554,20 +554,20 @@ static void e7xxx_remove_one(struct pci_dev *pdev) static const struct pci_device_id e7xxx_pci_tbl[] = { { - PCI_VEND_DEV(INTEL, 7205_0), PCI_ANY_ID, PCI_ANY_ID, 0, 0, - E7205}, - { - PCI_VEND_DEV(INTEL, 7500_0), PCI_ANY_ID, PCI_ANY_ID, 0, 0, - E7500}, - { - PCI_VEND_DEV(INTEL, 7501_0), PCI_ANY_ID, PCI_ANY_ID, 0, 0, - E7501}, - { - PCI_VEND_DEV(INTEL, 7505_0), PCI_ANY_ID, PCI_ANY_ID, 0, 0, - E7505}, - { - 0, - } /* 0 terminated list. */ + PCI_VEND_DEV(INTEL, 7205_0), + .driver_data = E7205, + }, { + PCI_VEND_DEV(INTEL, 7500_0), + .driver_data = E7500, + }, { + PCI_VEND_DEV(INTEL, 7501_0), + .driver_data = E7501, + }, { + PCI_VEND_DEV(INTEL, 7505_0), + .driver_data = E7505 + }, { + /* 0 terminated list. */ + } }; MODULE_DEVICE_TABLE(pci, e7xxx_pci_tbl); diff --git a/drivers/edac/edac_mc.h b/drivers/edac/edac_mc.h index 881b00eadf7a..9505bbd41784 100644 --- a/drivers/edac/edac_mc.h +++ b/drivers/edac/edac_mc.h @@ -88,8 +88,8 @@ do { \ #endif /* !CONFIG_EDAC_DEBUG */ -#define PCI_VEND_DEV(vend, dev) PCI_VENDOR_ID_ ## vend, \ - PCI_DEVICE_ID_ ## vend ## _ ## dev +#define PCI_VEND_DEV(vend, dev) \ + PCI_DEVICE(PCI_VENDOR_ID_ ## vend, PCI_DEVICE_ID_ ## vend ## _ ## dev) #define edac_dev_name(dev) (dev)->dev_name diff --git a/drivers/edac/i10nm_base.c b/drivers/edac/i10nm_base.c index 63df35444214..fa1853f252b0 100644 --- a/drivers/edac/i10nm_base.c +++ b/drivers/edac/i10nm_base.c @@ -47,12 +47,6 @@ readl((m)->mbase + ((m)->hbm_mc ? 0xef8 : \ (res_cfg->type == GNR ? 0xaf8 : 0x20ef8)) + \ (i) * (m)->chan_mmio_sz) -#define I10NM_GET_REG32(m, i, offset) \ - readl((m)->mbase + (i) * (m)->chan_mmio_sz + (offset)) -#define I10NM_GET_REG64(m, i, offset) \ - readq((m)->mbase + (i) * (m)->chan_mmio_sz + (offset)) -#define I10NM_SET_REG32(m, i, offset, v) \ - writel(v, (m)->mbase + (i) * (m)->chan_mmio_sz + (offset)) #define I10NM_GET_SCK_MMIO_BASE(reg) (GET_BITFIELD(reg, 0, 28) << 23) #define I10NM_GET_IMC_MMIO_OFFSET(reg) (GET_BITFIELD(reg, 0, 10) << 12) @@ -79,11 +73,12 @@ static struct res_config *res_cfg; static int retry_rd_err_log; static int decoding_via_mca; static bool mem_cfg_2lm; +static bool no_adxl; static struct reg_rrl icx_reg_rrl_ddr = { .set_num = 2, .reg_num = 6, - .modes = {LRE_SCRUB, LRE_DEMAND}, + .sources = {RRL_SRC_LRE_SCRUB, RRL_SRC_LRE_DEMAND}, .offsets = { {0x22c60, 0x22c54, 0x22c5c, 0x22c58, 0x22c28, 0x20ed8}, {0x22e54, 0x22e60, 0x22e64, 0x22e58, 0x22e5c, 0x20ee0}, @@ -104,7 +99,7 @@ static struct reg_rrl icx_reg_rrl_ddr = { static struct reg_rrl spr_reg_rrl_ddr = { .set_num = 3, .reg_num = 6, - .modes = {LRE_SCRUB, LRE_DEMAND, FRE_DEMAND}, + .sources = {RRL_SRC_LRE_SCRUB, RRL_SRC_LRE_DEMAND, RRL_SRC_FRE_DEMAND}, .offsets = { {0x22c60, 0x22c54, 0x22f08, 0x22c58, 0x22c28, 0x20ed8}, {0x22e54, 0x22e60, 0x22f10, 0x22e58, 0x22e5c, 0x20ee0}, @@ -126,7 +121,7 @@ static struct reg_rrl spr_reg_rrl_ddr = { static struct reg_rrl spr_reg_rrl_hbm_pch0 = { .set_num = 2, .reg_num = 6, - .modes = {LRE_SCRUB, LRE_DEMAND}, + .sources = {RRL_SRC_LRE_SCRUB, RRL_SRC_LRE_DEMAND}, .offsets = { {0x2860, 0x2854, 0x2b08, 0x2858, 0x2828, 0x0ed8}, {0x2a54, 0x2a60, 0x2b10, 0x2a58, 0x2a5c, 0x0ee0}, @@ -147,7 +142,7 @@ static struct reg_rrl spr_reg_rrl_hbm_pch0 = { static struct reg_rrl spr_reg_rrl_hbm_pch1 = { .set_num = 2, .reg_num = 6, - .modes = {LRE_SCRUB, LRE_DEMAND}, + .sources = {RRL_SRC_LRE_SCRUB, RRL_SRC_LRE_DEMAND}, .offsets = { {0x2c60, 0x2c54, 0x2f08, 0x2c58, 0x2c28, 0x0fa8}, {0x2e54, 0x2e60, 0x2f10, 0x2e58, 0x2e5c, 0x0fb0}, @@ -168,7 +163,7 @@ static struct reg_rrl spr_reg_rrl_hbm_pch1 = { static struct reg_rrl gnr_reg_rrl_ddr = { .set_num = 4, .reg_num = 6, - .modes = {FRE_SCRUB, FRE_DEMAND, LRE_SCRUB, LRE_DEMAND}, + .sources = {RRL_SRC_FRE_SCRUB, RRL_SRC_FRE_DEMAND, RRL_SRC_LRE_SCRUB, RRL_SRC_LRE_DEMAND}, .offsets = { {0x2f10, 0x2f20, 0x2f30, 0x2f50, 0x2f60, 0xba0}, {0x2f14, 0x2f24, 0x2f38, 0x2f54, 0x2f64, 0xba8}, @@ -188,214 +183,6 @@ static struct reg_rrl gnr_reg_rrl_ddr = { .cecnt_widths = {4, 4, 4, 4, 4, 4, 4, 4}, }; -static u64 read_imc_reg(struct skx_imc *imc, int chan, u32 offset, u8 width) -{ - switch (width) { - case 4: - return I10NM_GET_REG32(imc, chan, offset); - case 8: - return I10NM_GET_REG64(imc, chan, offset); - default: - i10nm_printk(KERN_ERR, "Invalid read RRL 0x%x width %d\n", offset, width); - return 0; - } -} - -static void write_imc_reg(struct skx_imc *imc, int chan, u32 offset, u8 width, u64 val) -{ - switch (width) { - case 4: - return I10NM_SET_REG32(imc, chan, offset, (u32)val); - default: - i10nm_printk(KERN_ERR, "Invalid write RRL 0x%x width %d\n", offset, width); - } -} - -static void enable_rrl(struct skx_imc *imc, int chan, struct reg_rrl *rrl, - int rrl_set, bool enable, u32 *rrl_ctl) -{ - enum rrl_mode mode = rrl->modes[rrl_set]; - u32 offset = rrl->offsets[rrl_set][0], v; - u8 width = rrl->widths[0]; - bool first, scrub; - - /* First or last read error. */ - first = (mode == FRE_SCRUB || mode == FRE_DEMAND); - /* Patrol scrub or on-demand read error. */ - scrub = (mode == FRE_SCRUB || mode == LRE_SCRUB); - - v = read_imc_reg(imc, chan, offset, width); - - if (enable) { - /* Save default configurations. */ - *rrl_ctl = v; - v &= ~rrl->uc_mask; - - if (first) - v |= rrl->noover_mask; - else - v &= ~rrl->noover_mask; - - if (scrub) - v |= rrl->en_patspr_mask; - else - v &= ~rrl->en_patspr_mask; - - v |= rrl->en_mask; - } else { - /* Restore default configurations. */ - if (*rrl_ctl & rrl->uc_mask) - v |= rrl->uc_mask; - - if (first) { - if (!(*rrl_ctl & rrl->noover_mask)) - v &= ~rrl->noover_mask; - } else { - if (*rrl_ctl & rrl->noover_mask) - v |= rrl->noover_mask; - } - - if (scrub) { - if (!(*rrl_ctl & rrl->en_patspr_mask)) - v &= ~rrl->en_patspr_mask; - } else { - if (*rrl_ctl & rrl->en_patspr_mask) - v |= rrl->en_patspr_mask; - } - - if (!(*rrl_ctl & rrl->en_mask)) - v &= ~rrl->en_mask; - } - - write_imc_reg(imc, chan, offset, width, v); -} - -static void enable_rrls(struct skx_imc *imc, int chan, struct reg_rrl *rrl, - bool enable, u32 *rrl_ctl) -{ - for (int i = 0; i < rrl->set_num; i++) - enable_rrl(imc, chan, rrl, i, enable, rrl_ctl + i); -} - -static void enable_rrls_ddr(struct skx_imc *imc, bool enable) -{ - struct reg_rrl *rrl_ddr = res_cfg->reg_rrl_ddr; - int i, chan_num = res_cfg->ddr_chan_num; - struct skx_channel *chan = imc->chan; - - if (!imc->mbase) - return; - - for (i = 0; i < chan_num; i++) - enable_rrls(imc, i, rrl_ddr, enable, chan[i].rrl_ctl[0]); -} - -static void enable_rrls_hbm(struct skx_imc *imc, bool enable) -{ - struct reg_rrl **rrl_hbm = res_cfg->reg_rrl_hbm; - int i, chan_num = res_cfg->hbm_chan_num; - struct skx_channel *chan = imc->chan; - - if (!imc->mbase || !imc->hbm_mc || !rrl_hbm[0] || !rrl_hbm[1]) - return; - - for (i = 0; i < chan_num; i++) { - enable_rrls(imc, i, rrl_hbm[0], enable, chan[i].rrl_ctl[0]); - enable_rrls(imc, i, rrl_hbm[1], enable, chan[i].rrl_ctl[1]); - } -} - -static void enable_retry_rd_err_log(bool enable) -{ - struct skx_dev *d; - int i, imc_num; - - edac_dbg(2, "\n"); - - list_for_each_entry(d, i10nm_edac_list, list) { - imc_num = res_cfg->ddr_imc_num; - for (i = 0; i < imc_num; i++) - enable_rrls_ddr(&d->imc[i], enable); - - imc_num += res_cfg->hbm_imc_num; - for (; i < imc_num; i++) - enable_rrls_hbm(&d->imc[i], enable); - } -} - -static void show_retry_rd_err_log(struct decoded_addr *res, char *msg, - int len, bool scrub_err) -{ - int i, j, n, ch = res->channel, pch = res->cs & 1; - struct skx_imc *imc = &res->dev->imc[res->imc]; - u64 log, corr, status_mask; - struct reg_rrl *rrl; - bool scrub; - u32 offset; - u8 width; - - if (!imc->mbase) - return; - - rrl = imc->hbm_mc ? res_cfg->reg_rrl_hbm[pch] : res_cfg->reg_rrl_ddr; - - if (!rrl) - return; - - status_mask = rrl->over_mask | rrl->uc_mask | rrl->v_mask; - - n = scnprintf(msg, len, " retry_rd_err_log["); - for (i = 0; i < rrl->set_num; i++) { - scrub = (rrl->modes[i] == FRE_SCRUB || rrl->modes[i] == LRE_SCRUB); - if (scrub_err != scrub) - continue; - - for (j = 0; j < rrl->reg_num && len - n > 0; j++) { - offset = rrl->offsets[i][j]; - width = rrl->widths[j]; - log = read_imc_reg(imc, ch, offset, width); - - if (width == 4) - n += scnprintf(msg + n, len - n, "%.8llx ", log); - else - n += scnprintf(msg + n, len - n, "%.16llx ", log); - - /* Clear RRL status if RRL in Linux control mode. */ - if (retry_rd_err_log == 2 && !j && (log & status_mask)) - write_imc_reg(imc, ch, offset, width, log & ~status_mask); - } - } - - /* Move back one space. */ - n--; - n += scnprintf(msg + n, len - n, "]"); - - if (len - n > 0) { - n += scnprintf(msg + n, len - n, " correrrcnt["); - for (i = 0; i < rrl->cecnt_num && len - n > 0; i++) { - offset = rrl->cecnt_offsets[i]; - width = rrl->cecnt_widths[i]; - corr = read_imc_reg(imc, ch, offset, width); - - /* CPUs {ICX,SPR} encode two counters per 4-byte CORRERRCNT register. */ - if (res_cfg->type <= SPR) { - n += scnprintf(msg + n, len - n, "%.4llx %.4llx ", - corr & 0xffff, corr >> 16); - } else { - /* CPUs {GNR} encode one counter per CORRERRCNT register. */ - if (width == 4) - n += scnprintf(msg + n, len - n, "%.8llx ", corr); - else - n += scnprintf(msg + n, len - n, "%.16llx ", corr); - } - } - - /* Move back one space. */ - n--; - n += scnprintf(msg + n, len - n, "]"); - } -} - static struct pci_dev *pci_get_dev_wrapper(int dom, unsigned int bus, unsigned int dev, unsigned int fun) { @@ -980,7 +767,7 @@ static struct res_config i10nm_cfg0 = { .ddr_mdev_bdf = {0, 12, 0}, .hbm_mdev_bdf = {0, 12, 1}, .sad_all_offset = 0x108, - .reg_rrl_ddr = &icx_reg_rrl_ddr, + .reg_rrl_ddr[0] = &icx_reg_rrl_ddr, }; static struct res_config i10nm_cfg1 = { @@ -998,7 +785,7 @@ static struct res_config i10nm_cfg1 = { .ddr_mdev_bdf = {0, 12, 0}, .hbm_mdev_bdf = {0, 12, 1}, .sad_all_offset = 0x108, - .reg_rrl_ddr = &icx_reg_rrl_ddr, + .reg_rrl_ddr[0] = &icx_reg_rrl_ddr, }; static struct res_config spr_cfg = { @@ -1021,7 +808,7 @@ static struct res_config spr_cfg = { .ddr_mdev_bdf = {0, 12, 0}, .hbm_mdev_bdf = {0, 12, 1}, .sad_all_offset = 0x300, - .reg_rrl_ddr = &spr_reg_rrl_ddr, + .reg_rrl_ddr[0] = &spr_reg_rrl_ddr, .reg_rrl_hbm[0] = &spr_reg_rrl_hbm_pch0, .reg_rrl_hbm[1] = &spr_reg_rrl_hbm_pch1, }; @@ -1041,7 +828,7 @@ static struct res_config gnr_cfg = { .uracu_bdf = {0, 0, 1}, .ddr_mdev_bdf = {0, 5, 1}, .sad_all_offset = 0x300, - .reg_rrl_ddr = &gnr_reg_rrl_ddr, + .reg_rrl_ddr[0] = &gnr_reg_rrl_ddr, }; static const struct x86_cpu_id i10nm_cpuids[] = { @@ -1222,21 +1009,28 @@ static int __init i10nm_init(void) } rc = skx_adxl_get(); - if (rc) - goto fail; + if (rc) { + /* Decoding errors via MCA banks for 2LM isn't supported yet */ + if (rc != -ENODEV || mem_cfg_2lm) + goto fail; + i10nm_printk(KERN_INFO, "ADXL not found, falling back to MCA-based decoding.\n"); + no_adxl = true; + decoding_via_mca = true; + } opstate_init(); mce_register_decode_chain(&i10nm_mce_dec); skx_setup_debug("i10nm_test"); - if (retry_rd_err_log && res_cfg->reg_rrl_ddr) { - skx_set_decode(i10nm_mc_decode, show_retry_rd_err_log); - if (retry_rd_err_log == 2) - enable_retry_rd_err_log(true); - } else { - skx_set_decode(i10nm_mc_decode, NULL); + res_cfg->rrl_ctrl_mode = retry_rd_err_log; + if (retry_rd_err_log && res_cfg->reg_rrl_ddr[0]) { + skx_set_show_rrl(skx_show_rrl); + if (retry_rd_err_log == RRL_CTRL_LINUX) + skx_enable_rrl(true); } + skx_set_decode(i10nm_mc_decode); + i10nm_printk(KERN_INFO, "%s\n", I10NM_REVISION); return 0; @@ -1249,15 +1043,18 @@ static void __exit i10nm_exit(void) { edac_dbg(2, "\n"); - if (retry_rd_err_log && res_cfg->reg_rrl_ddr) { - skx_set_decode(NULL, NULL); - if (retry_rd_err_log == 2) - enable_retry_rd_err_log(false); + skx_set_decode(NULL); + + if (retry_rd_err_log && res_cfg->reg_rrl_ddr[0]) { + if (retry_rd_err_log == RRL_CTRL_LINUX) + skx_enable_rrl(false); + skx_set_show_rrl(NULL); } skx_teardown_debug(); mce_unregister_decode_chain(&i10nm_mce_dec); - skx_adxl_put(); + if (!no_adxl) + skx_adxl_put(); skx_remove(); } diff --git a/drivers/edac/i3000_edac.c b/drivers/edac/i3000_edac.c index 9065bc4386ff..04a231660b88 100644 --- a/drivers/edac/i3000_edac.c +++ b/drivers/edac/i3000_edac.c @@ -485,11 +485,11 @@ static void i3000_remove_one(struct pci_dev *pdev) static const struct pci_device_id i3000_pci_tbl[] = { { - PCI_VEND_DEV(INTEL, 3000_HB), PCI_ANY_ID, PCI_ANY_ID, 0, 0, - I3000}, - { - 0, - } /* 0 terminated list. */ + PCI_VEND_DEV(INTEL, 3000_HB), + .driver_data = I3000, + }, { + /* 0 terminated list. */ + } }; MODULE_DEVICE_TABLE(pci, i3000_pci_tbl); diff --git a/drivers/edac/i3200_edac.c b/drivers/edac/i3200_edac.c index 6cade6d7ceff..d600b6c05217 100644 --- a/drivers/edac/i3200_edac.c +++ b/drivers/edac/i3200_edac.c @@ -466,11 +466,11 @@ static void i3200_remove_one(struct pci_dev *pdev) static const struct pci_device_id i3200_pci_tbl[] = { { - PCI_VEND_DEV(INTEL, 3200_HB), PCI_ANY_ID, PCI_ANY_ID, 0, 0, - I3200}, - { - 0, - } /* 0 terminated list. */ + PCI_VEND_DEV(INTEL, 3200_HB), + .driver_data = I3200, + }, { + /* 0 terminated list. */ + } }; MODULE_DEVICE_TABLE(pci, i3200_pci_tbl); diff --git a/drivers/edac/i5000_edac.c b/drivers/edac/i5000_edac.c index 471b8540d18b..c0faf55f7812 100644 --- a/drivers/edac/i5000_edac.c +++ b/drivers/edac/i5000_edac.c @@ -352,6 +352,9 @@ struct i5000_pvt { /* Actual values for this controller */ int maxch; /* Max channels */ int maxdimmperch; /* Max DIMMs per channel */ + + /* Hardware error reporting status */ + bool enabled_error_reporting; }; /* I5000 MCH error information retrieved from Hardware */ @@ -1302,10 +1305,10 @@ static int i5000_init_csrows(struct mem_ctl_info *mci) } /* - * i5000_enable_error_reporting - * Turn on the memory reporting features of the hardware + * i5000_set_error_reporting + * Turn on/off the memory reporting features of the hardware */ -static void i5000_enable_error_reporting(struct mem_ctl_info *mci) +static void i5000_set_error_reporting(struct mem_ctl_info *mci, bool enable) { struct i5000_pvt *pvt; u32 fbd_error_mask; @@ -1316,8 +1319,11 @@ static void i5000_enable_error_reporting(struct mem_ctl_info *mci) pci_read_config_dword(pvt->branchmap_werrors, EMASK_FBD, &fbd_error_mask); - /* Enable with a '0' */ - fbd_error_mask &= ~(ENABLE_EMASK_ALL); + /* Enable with 0, disable with 1 */ + if (enable) + fbd_error_mask &= ~(ENABLE_EMASK_ALL); + else + fbd_error_mask |= ENABLE_EMASK_ALL; pci_write_config_dword(pvt->branchmap_werrors, EMASK_FBD, fbd_error_mask); @@ -1435,17 +1441,19 @@ static int i5000_probe1(struct pci_dev *pdev, int dev_idx) if (i5000_init_csrows(mci)) { edac_dbg(0, "MC: Setting mci->edac_cap to EDAC_FLAG_NONE because i5000_init_csrows() returned nonzero value\n"); mci->edac_cap = EDAC_FLAG_NONE; /* no csrows found */ + pvt->enabled_error_reporting = false; } else { edac_dbg(1, "MC: Enable error reporting now\n"); - i5000_enable_error_reporting(mci); + i5000_set_error_reporting(mci, true); + pvt->enabled_error_reporting = true; } /* add this new MC control structure to EDAC's list of MCs */ if (edac_mc_add_mc(mci)) { edac_dbg(0, "MC: failed edac_mc_add_mc()\n"); - /* FIXME: perhaps some code should go here that disables error - * reporting if we just enabled it - */ + /* Disable error reporting if we previously enabled it */ + if (pvt->enabled_error_reporting) + i5000_set_error_reporting(mci, false); goto fail1; } @@ -1503,6 +1511,7 @@ static int i5000_init_one(struct pci_dev *pdev, const struct pci_device_id *id) static void i5000_remove_one(struct pci_dev *pdev) { struct mem_ctl_info *mci; + struct i5000_pvt *pvt; edac_dbg(0, "\n"); @@ -1512,6 +1521,12 @@ static void i5000_remove_one(struct pci_dev *pdev) if ((mci = edac_mc_del_mc(&pdev->dev)) == NULL) return; + pvt = mci->pvt_info; + + /* Disable error reporting on teardown */ + if (pvt->enabled_error_reporting) + i5000_set_error_reporting(mci, false); + /* retrieve references to resources, and free those resources */ i5000_put_devices(mci); edac_mc_free(mci); diff --git a/drivers/edac/i5100_edac.c b/drivers/edac/i5100_edac.c index d470afe65001..d30919ceb22b 100644 --- a/drivers/edac/i5100_edac.c +++ b/drivers/edac/i5100_edac.c @@ -859,6 +859,21 @@ static void i5100_init_csrows(struct mem_ctl_info *mci) } } +static void i5100_set_error_reporting(struct pci_dev *pdev, bool enable) +{ + u32 dw; + + pci_read_config_dword(pdev, I5100_EMASK_MEM, &dw); + + /* Enable with 0, disable with 1 */ + if (enable) + dw &= ~I5100_FERR_NF_MEM_ANY_MASK; + else + dw |= I5100_FERR_NF_MEM_ANY_MASK; + + pci_write_config_dword(pdev, I5100_EMASK_MEM, dw); +} + /**************************************************************************** * Error injection routines ****************************************************************************/ @@ -1004,11 +1019,6 @@ static int i5100_init_one(struct pci_dev *pdev, const struct pci_device_id *id) pci_read_config_dword(pdev, I5100_MS, &dw); ranksperch = !!(dw & (1 << 8)) * 2 + 4; - /* enable error reporting... */ - pci_read_config_dword(pdev, I5100_EMASK_MEM, &dw); - dw &= ~I5100_FERR_NF_MEM_ANY_MASK; - pci_write_config_dword(pdev, I5100_EMASK_MEM, dw); - /* device 21, func 0, Channel 0 Memory Map, Error Flag/Mask, etc... */ ch0mm = pci_get_device_func(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_5100_21, 0); @@ -1125,6 +1135,9 @@ static int i5100_init_one(struct pci_dev *pdev, const struct pci_device_id *id) i5100_setup_debugfs(mci); + /* Enable error reporting on success */ + i5100_set_error_reporting(pdev, true); + return ret; bail_scrub: @@ -1169,6 +1182,9 @@ static void i5100_remove_one(struct pci_dev *pdev) priv = mci->pvt_info; + /* Disable error reporting at teardown */ + i5100_set_error_reporting(pdev, false); + edac_debugfs_remove_recursive(priv->debugfs); priv->scrub_enable = 0; diff --git a/drivers/edac/i5400_edac.c b/drivers/edac/i5400_edac.c index fb49a1d1df11..ae4f9298952d 100644 --- a/drivers/edac/i5400_edac.c +++ b/drivers/edac/i5400_edac.c @@ -353,6 +353,9 @@ struct i5400_pvt { /* Actual values for this controller */ int maxch; /* Max channels */ int maxdimmperch; /* Max DIMMs per channel */ + + /* Hardware error reporting status */ + bool enabled_error_reporting; }; /* I5400 MCH error information retrieved from Hardware */ @@ -1223,10 +1226,10 @@ static int i5400_init_dimms(struct mem_ctl_info *mci) } /* - * i5400_enable_error_reporting - * Turn on the memory reporting features of the hardware + * i5400_set_error_reporting + * Turn on/off the memory reporting features of the hardware */ -static void i5400_enable_error_reporting(struct mem_ctl_info *mci) +static void i5400_set_error_reporting(struct mem_ctl_info *mci, bool enable) { struct i5400_pvt *pvt; u32 fbd_error_mask; @@ -1237,8 +1240,11 @@ static void i5400_enable_error_reporting(struct mem_ctl_info *mci) pci_read_config_dword(pvt->branchmap_werrors, EMASK_FBD, &fbd_error_mask); - /* Enable with a '0' */ - fbd_error_mask &= ~(ENABLE_EMASK_ALL); + /* Enable with 0, disable with 1 */ + if (enable) + fbd_error_mask &= ~(ENABLE_EMASK_ALL); + else + fbd_error_mask |= ENABLE_EMASK_ALL; pci_write_config_dword(pvt->branchmap_werrors, EMASK_FBD, fbd_error_mask); @@ -1319,17 +1325,19 @@ static int i5400_probe1(struct pci_dev *pdev, int dev_idx) if (i5400_init_dimms(mci)) { edac_dbg(0, "MC: Setting mci->edac_cap to EDAC_FLAG_NONE because i5400_init_dimms() returned nonzero value\n"); mci->edac_cap = EDAC_FLAG_NONE; /* no dimms found */ + pvt->enabled_error_reporting = false; } else { edac_dbg(1, "MC: Enable error reporting now\n"); - i5400_enable_error_reporting(mci); + i5400_set_error_reporting(mci, true); + pvt->enabled_error_reporting = true; } /* add this new MC control structure to EDAC's list of MCs */ if (edac_mc_add_mc(mci)) { edac_dbg(0, "MC: failed edac_mc_add_mc()\n"); - /* FIXME: perhaps some code should go here that disables error - * reporting if we just enabled it - */ + /* Disable error reporting if we just enabled it */ + if (pvt->enabled_error_reporting) + i5400_set_error_reporting(mci, false); goto fail1; } @@ -1387,6 +1395,7 @@ static int i5400_init_one(struct pci_dev *pdev, const struct pci_device_id *id) static void i5400_remove_one(struct pci_dev *pdev) { struct mem_ctl_info *mci; + struct i5400_pvt *pvt; edac_dbg(0, "\n"); @@ -1397,6 +1406,12 @@ static void i5400_remove_one(struct pci_dev *pdev) if (!mci) return; + pvt = mci->pvt_info; + + /* Disable error reporting on teardown */ + if (pvt->enabled_error_reporting) + i5400_set_error_reporting(mci, false); + /* retrieve references to resources, and free those resources */ i5400_put_devices(mci); diff --git a/drivers/edac/i7300_edac.c b/drivers/edac/i7300_edac.c index 69068f8d0cad..64bc2d805a62 100644 --- a/drivers/edac/i7300_edac.c +++ b/drivers/edac/i7300_edac.c @@ -111,6 +111,9 @@ struct i7300_pvt { /* Temporary buffer for use when preparing error messages */ char *tmp_prt_buffer; + + /* Hardware error reporting status */ + bool enabled_error_reporting; }; /* FIXME: Why do we need to have this static? */ @@ -550,11 +553,12 @@ static void i7300_clear_error(struct mem_ctl_info *mci) } /** - * i7300_enable_error_reporting() - Enable the memory reporting logic at the + * i7300_set_error_reporting() - Enable or disable the memory reporting logic at the * hardware * @mci: struct mem_ctl_info pointer + * @enable: enables if 'true', disables if 'false' */ -static void i7300_enable_error_reporting(struct mem_ctl_info *mci) +static void i7300_set_error_reporting(struct mem_ctl_info *mci, bool enable) { struct i7300_pvt *pvt = mci->pvt_info; u32 fbd_error_mask; @@ -563,8 +567,11 @@ static void i7300_enable_error_reporting(struct mem_ctl_info *mci) pci_read_config_dword(pvt->pci_dev_16_1_fsb_addr_map, EMASK_FBD, &fbd_error_mask); - /* Enable with a '0' */ - fbd_error_mask &= ~(EMASK_FBD_ERR_MASK); + /* Enable with 0, disable with 1 */ + if (enable) + fbd_error_mask &= ~(EMASK_FBD_ERR_MASK); + else + fbd_error_mask |= EMASK_FBD_ERR_MASK; pci_write_config_dword(pvt->pci_dev_16_1_fsb_addr_map, EMASK_FBD, fbd_error_mask); @@ -1087,17 +1094,19 @@ static int i7300_init_one(struct pci_dev *pdev, const struct pci_device_id *id) if (i7300_get_mc_regs(mci)) { edac_dbg(0, "MC: Setting mci->edac_cap to EDAC_FLAG_NONE because i7300_init_csrows() returned nonzero value\n"); mci->edac_cap = EDAC_FLAG_NONE; /* no csrows found */ + pvt->enabled_error_reporting = false; } else { edac_dbg(1, "MC: Enable error reporting now\n"); - i7300_enable_error_reporting(mci); + i7300_set_error_reporting(mci, true); + pvt->enabled_error_reporting = true; } /* add this new MC control structure to EDAC's list of MCs */ if (edac_mc_add_mc(mci)) { edac_dbg(0, "MC: failed edac_mc_add_mc()\n"); - /* FIXME: perhaps some code should go here that disables error - * reporting if we just enabled it - */ + /* Disable error reporting if we just enabled it */ + if (pvt->enabled_error_reporting) + i7300_set_error_reporting(mci, false); goto fail1; } @@ -1134,6 +1143,7 @@ fail0: static void i7300_remove_one(struct pci_dev *pdev) { struct mem_ctl_info *mci; + struct i7300_pvt *pvt; char *tmp; edac_dbg(0, "\n"); @@ -1145,7 +1155,12 @@ static void i7300_remove_one(struct pci_dev *pdev) if (!mci) return; - tmp = ((struct i7300_pvt *)mci->pvt_info)->tmp_prt_buffer; + pvt = (struct i7300_pvt *)mci->pvt_info; + tmp = pvt->tmp_prt_buffer; + + /* Disable error reporting before unregistering device */ + if (pvt->enabled_error_reporting) + i7300_set_error_reporting(mci, false); /* retrieve references to resources, and free those resources */ i7300_put_devices(mci); diff --git a/drivers/edac/i82860_edac.c b/drivers/edac/i82860_edac.c index b8a497f0de28..e8c1ee80bba8 100644 --- a/drivers/edac/i82860_edac.c +++ b/drivers/edac/i82860_edac.c @@ -287,11 +287,11 @@ static void i82860_remove_one(struct pci_dev *pdev) static const struct pci_device_id i82860_pci_tbl[] = { { - PCI_VEND_DEV(INTEL, 82860_0), PCI_ANY_ID, PCI_ANY_ID, 0, 0, - I82860}, - { - 0, - } /* 0 terminated list. */ + PCI_VEND_DEV(INTEL, 82860_0), + .driver_data = I82860, + }, { + /* 0 terminated list. */ + } }; MODULE_DEVICE_TABLE(pci, i82860_pci_tbl); diff --git a/drivers/edac/i82875p_edac.c b/drivers/edac/i82875p_edac.c index 553880b9fc12..869de8e372b3 100644 --- a/drivers/edac/i82875p_edac.c +++ b/drivers/edac/i82875p_edac.c @@ -276,7 +276,7 @@ static int i82875p_setup_overfl_dev(struct pci_dev *pdev, *ovrfl_pdev = NULL; *ovrfl_window = NULL; - dev = pci_get_device(PCI_VEND_DEV(INTEL, 82875_6), NULL); + dev = pci_get_device(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82875_6, NULL); if (dev == NULL) { /* Intel tells BIOS developers to hide device 6 which @@ -518,11 +518,11 @@ static void i82875p_remove_one(struct pci_dev *pdev) static const struct pci_device_id i82875p_pci_tbl[] = { { - PCI_VEND_DEV(INTEL, 82875_0), PCI_ANY_ID, PCI_ANY_ID, 0, 0, - I82875P}, - { - 0, - } /* 0 terminated list. */ + PCI_VEND_DEV(INTEL, 82875_0), + .driver_data = I82875P, + }, { + /* 0 terminated list. */ + } }; MODULE_DEVICE_TABLE(pci, i82875p_pci_tbl); diff --git a/drivers/edac/i82975x_edac.c b/drivers/edac/i82975x_edac.c index d99f005832cf..09a79eaaa486 100644 --- a/drivers/edac/i82975x_edac.c +++ b/drivers/edac/i82975x_edac.c @@ -624,12 +624,11 @@ static void i82975x_remove_one(struct pci_dev *pdev) static const struct pci_device_id i82975x_pci_tbl[] = { { - PCI_VEND_DEV(INTEL, 82975_0), PCI_ANY_ID, PCI_ANY_ID, 0, 0, - I82975X - }, - { - 0, - } /* 0 terminated list. */ + PCI_VEND_DEV(INTEL, 82975_0), + .driver_data = I82975X + }, { + /* 0 terminated list. */ + } }; MODULE_DEVICE_TABLE(pci, i82975x_pci_tbl); diff --git a/drivers/edac/ie31200_edac.c b/drivers/edac/ie31200_edac.c index a5dc4b88097f..e3bd6436669b 100644 --- a/drivers/edac/ie31200_edac.c +++ b/drivers/edac/ie31200_edac.c @@ -733,49 +733,49 @@ static struct res_config rpl_s_cfg = { }; static const struct pci_device_id ie31200_pci_tbl[] = { - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_1), (kernel_ulong_t)&snb_cfg }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_2), (kernel_ulong_t)&snb_cfg }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_3), (kernel_ulong_t)&snb_cfg }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_4), (kernel_ulong_t)&snb_cfg }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_5), (kernel_ulong_t)&snb_cfg }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_6), (kernel_ulong_t)&snb_cfg }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_7), (kernel_ulong_t)&snb_cfg }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_8), (kernel_ulong_t)&skl_cfg }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_9), (kernel_ulong_t)&skl_cfg }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_10), (kernel_ulong_t)&skl_cfg }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_11), (kernel_ulong_t)&skl_cfg }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_12), (kernel_ulong_t)&skl_cfg }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_1), (kernel_ulong_t)&skl_cfg }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_2), (kernel_ulong_t)&skl_cfg }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_3), (kernel_ulong_t)&skl_cfg }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_4), (kernel_ulong_t)&skl_cfg }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_5), (kernel_ulong_t)&skl_cfg }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_6), (kernel_ulong_t)&skl_cfg }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_7), (kernel_ulong_t)&skl_cfg }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_8), (kernel_ulong_t)&skl_cfg }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_9), (kernel_ulong_t)&skl_cfg }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_10), (kernel_ulong_t)&skl_cfg }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_RPL_S_1), (kernel_ulong_t)&rpl_s_cfg}, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_RPL_S_2), (kernel_ulong_t)&rpl_s_cfg}, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_RPL_S_3), (kernel_ulong_t)&rpl_s_cfg}, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_RPL_S_4), (kernel_ulong_t)&rpl_s_cfg}, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_RPL_S_5), (kernel_ulong_t)&rpl_s_cfg}, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_RPL_S_6), (kernel_ulong_t)&rpl_s_cfg}, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_RPL_HX_1), (kernel_ulong_t)&rpl_s_cfg}, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_ADL_S_1), (kernel_ulong_t)&rpl_s_cfg}, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_ADL_S_2), (kernel_ulong_t)&rpl_s_cfg}, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_ADL_S_3), (kernel_ulong_t)&rpl_s_cfg}, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_BTL_S_1), (kernel_ulong_t)&rpl_s_cfg}, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_BTL_S_2), (kernel_ulong_t)&rpl_s_cfg}, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_BTL_S_3), (kernel_ulong_t)&rpl_s_cfg}, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_BTL_S_4), (kernel_ulong_t)&rpl_s_cfg}, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_BTL_S_5), (kernel_ulong_t)&rpl_s_cfg}, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_BTL_S_6), (kernel_ulong_t)&rpl_s_cfg}, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_BTL_S_7), (kernel_ulong_t)&rpl_s_cfg}, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_BTL_S_8), (kernel_ulong_t)&rpl_s_cfg}, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_BTL_S_9), (kernel_ulong_t)&rpl_s_cfg}, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_BTL_S_10), (kernel_ulong_t)&rpl_s_cfg}, - { 0, } /* 0 terminated list. */ + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_1), .driver_data = (kernel_ulong_t)&snb_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_2), .driver_data = (kernel_ulong_t)&snb_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_3), .driver_data = (kernel_ulong_t)&snb_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_4), .driver_data = (kernel_ulong_t)&snb_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_5), .driver_data = (kernel_ulong_t)&snb_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_6), .driver_data = (kernel_ulong_t)&snb_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_7), .driver_data = (kernel_ulong_t)&snb_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_8), .driver_data = (kernel_ulong_t)&skl_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_9), .driver_data = (kernel_ulong_t)&skl_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_10), .driver_data = (kernel_ulong_t)&skl_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_11), .driver_data = (kernel_ulong_t)&skl_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_12), .driver_data = (kernel_ulong_t)&skl_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_1), .driver_data = (kernel_ulong_t)&skl_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_2), .driver_data = (kernel_ulong_t)&skl_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_3), .driver_data = (kernel_ulong_t)&skl_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_4), .driver_data = (kernel_ulong_t)&skl_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_5), .driver_data = (kernel_ulong_t)&skl_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_6), .driver_data = (kernel_ulong_t)&skl_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_7), .driver_data = (kernel_ulong_t)&skl_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_8), .driver_data = (kernel_ulong_t)&skl_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_9), .driver_data = (kernel_ulong_t)&skl_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_10), .driver_data = (kernel_ulong_t)&skl_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_RPL_S_1), .driver_data = (kernel_ulong_t)&rpl_s_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_RPL_S_2), .driver_data = (kernel_ulong_t)&rpl_s_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_RPL_S_3), .driver_data = (kernel_ulong_t)&rpl_s_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_RPL_S_4), .driver_data = (kernel_ulong_t)&rpl_s_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_RPL_S_5), .driver_data = (kernel_ulong_t)&rpl_s_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_RPL_S_6), .driver_data = (kernel_ulong_t)&rpl_s_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_RPL_HX_1), .driver_data = (kernel_ulong_t)&rpl_s_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_ADL_S_1), .driver_data = (kernel_ulong_t)&rpl_s_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_ADL_S_2), .driver_data = (kernel_ulong_t)&rpl_s_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_ADL_S_3), .driver_data = (kernel_ulong_t)&rpl_s_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_BTL_S_1), .driver_data = (kernel_ulong_t)&rpl_s_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_BTL_S_2), .driver_data = (kernel_ulong_t)&rpl_s_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_BTL_S_3), .driver_data = (kernel_ulong_t)&rpl_s_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_BTL_S_4), .driver_data = (kernel_ulong_t)&rpl_s_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_BTL_S_5), .driver_data = (kernel_ulong_t)&rpl_s_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_BTL_S_6), .driver_data = (kernel_ulong_t)&rpl_s_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_BTL_S_7), .driver_data = (kernel_ulong_t)&rpl_s_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_BTL_S_8), .driver_data = (kernel_ulong_t)&rpl_s_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_BTL_S_9), .driver_data = (kernel_ulong_t)&rpl_s_cfg }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IE31200_BTL_S_10), .driver_data = (kernel_ulong_t)&rpl_s_cfg }, + { } /* 0 terminated list. */ }; MODULE_DEVICE_TABLE(pci, ie31200_pci_tbl); diff --git a/drivers/edac/igen6_edac.c b/drivers/edac/igen6_edac.c index fcb8ab44cba5..f1fc20d4ebf6 100644 --- a/drivers/edac/igen6_edac.c +++ b/drivers/edac/igen6_edac.c @@ -122,33 +122,6 @@ #define MEM_SLICE_HASH_MASK(v) (GET_BITFIELD(v, 6, 19) << 6) #define MEM_SLICE_HASH_LSB_MASK_BIT(v) GET_BITFIELD(v, 24, 26) -static struct res_config { - bool machine_check; - /* The number of present memory controllers. */ - int num_imc; - /* Host MMIO configuration */ - u64 reg_mchbar_mask; - /* Top of memory */ - u64 reg_tom_mask; - /* Top of upper usable DRAM */ - u64 reg_touud_mask; - /* IBECC error log */ - u64 reg_eccerrlog_addr_mask; - u32 imc_base; - u32 cmf_base; - u32 cmf_size; - u32 ms_hash_offset; - u32 ibecc_base; - u32 ibecc_error_log_offset; - bool (*ibecc_available)(struct pci_dev *pdev); - /* Extract error address logged in IBECC */ - u64 (*err_addr)(u64 ecclog); - /* Convert error address logged in IBECC to system physical address */ - u64 (*err_addr_to_sys_addr)(u64 eaddr, int mc); - /* Convert error address logged in IBECC to integrated memory controller address */ - u64 (*err_addr_to_imc_addr)(u64 eaddr, int mc); -} *res_cfg; - struct igen6_imc { int mc; struct mem_ctl_info *mci; @@ -163,8 +136,56 @@ struct igen6_imc { int dimm_l_map[NUM_CHANNELS]; }; +static struct res_config { + bool machine_check; + /* The number of present memory controllers. */ + int num_imc; + /* Host MMIO configuration */ + u64 reg_mchbar_mask; + /* Top of memory */ + u64 reg_tom_mask; + /* Top of upper usable DRAM */ + u64 reg_touud_mask; + /* IBECC error log */ + u64 reg_eccerrlog_addr_mask; + /* MEMSS_PMA_CR registers. */ + u32 reg_mem_config_offset; + u32 reg_mem_config_ddr_type_mask; + u32 reg_mem_config_ibecc_en_mask; + u32 reg_capabilities_misc_offset; + u32 reg_capabilities_misc_ibecc_dis; + /* Memory controller registers. */ + u32 reg_mad_inter_size_mask[NUM_CHANNELS]; + u64 reg_mad_inter_size_granularity; + u32 reg_mad_intra_rank_mask[NUM_DIMMS]; + u32 reg_mad_intra_width_mask[NUM_DIMMS]; + u32 reg_mad_intra_density_mask[NUM_DIMMS]; + u32 imc_base; + u32 cmf_base; + u32 cmf_size; + u32 ms_hash_offset; + u32 ibecc_base; + u32 ibecc_error_log_offset; + /* Get memory type. */ + enum mem_type (*get_mem_type)(struct igen6_imc *imc); + /* Get DRAM chip type. */ + enum dev_type (*get_dev_type)(struct igen6_imc *imc, int chan, int dimm_l); + /* Set imc->ch_{s_size,l_map}. */ + void (*set_chan_params)(struct igen6_imc *imc); + /* Set imc->dimm_{l_size,s_size,l_map}[chan]. */ + void (*set_dimm_params)(struct igen6_imc *imc, int chan); + bool (*ibecc_available)(struct pci_dev *pdev); + /* Extract error address logged in IBECC */ + u64 (*err_addr)(u64 ecclog); + /* Convert error address logged in IBECC to system physical address */ + u64 (*err_addr_to_sys_addr)(u64 eaddr, int mc); + /* Convert error address logged in IBECC to integrated memory controller address */ + u64 (*err_addr_to_imc_addr)(u64 eaddr, int mc); +} *res_cfg; + static struct igen6_pvt { struct igen6_imc imc[NUM_IMC]; + void __iomem *memss_pma_cr; u64 ms_hash; u64 ms_s_size; int ms_l_map; @@ -291,10 +312,17 @@ static struct work_struct ecclog_work; #define DID_PTL_H_SKU11 0xb028 #define DID_PTL_H_SKU12 0xb029 #define DID_PTL_H_SKU13 0xb02a +#define DID_PTL_H_SKU14 0xb00a /* Compute die IDs for Wildcat Lake with IBECC */ #define DID_WCL_SKU1 0xfd00 +/* Compute die IDs for Nova Lake-H/HX with IBECC */ +#define DID_NVL_H_SKU1 0xd701 +#define DID_NVL_H_SKU2 0xd702 +#define DID_NVL_H_SKU3 0xd704 +#define DID_NVL_H_SKU4 0xd705 + static int get_mchbar(struct pci_dev *pdev, u64 *mchbar) { union { @@ -386,27 +414,26 @@ static bool mtl_p_ibecc_available(struct pci_dev *pdev) return !(CAPID_E_IBECC_BIT18 & v); } -static bool mtl_ps_ibecc_available(struct pci_dev *pdev) +static bool generic_ibecc_available(struct pci_dev *pdev) { -#define MCHBAR_MEMSS_IBECCDIS 0x13c00 - void __iomem *window; - u64 mchbar; + void __iomem *base = igen6_pvt->memss_pma_cr; + bool present; u32 val; - if (get_mchbar(pdev, &mchbar)) - return false; - - window = ioremap(mchbar, MCHBAR_SIZE * 2); - if (!window) { - igen6_printk(KERN_ERR, "Failed to ioremap 0x%llx\n", mchbar); - return false; + if (res_cfg->reg_capabilities_misc_offset) { + val = readl(base + res_cfg->reg_capabilities_misc_offset); + present = !(val & res_cfg->reg_capabilities_misc_ibecc_dis); + edac_dbg(2, "capabilities misc reg 0x%x\n", val); + } else if (res_cfg->reg_mem_config_offset) { + val = readl(base + res_cfg->reg_mem_config_offset); + present = !!(val & res_cfg->reg_mem_config_ibecc_en_mask); + edac_dbg(2, "mem config reg 0x%x\n", val); + } else { + igen6_printk(KERN_ERR, "No register for detecting IBECC presence.\n"); + present = false; } - val = readl(window + MCHBAR_MEMSS_IBECCDIS); - iounmap(window); - - /* Bit6: 1 - IBECC is disabled, 0 - IBECC isn't disabled */ - return !GET_BITFIELD(val, 6, 6); + return present; } static u64 mem_addr_to_sys_addr(u64 maddr) @@ -500,6 +527,119 @@ static u64 rpl_p_err_addr(u64 ecclog) return field_get(res_cfg->reg_eccerrlog_addr_mask, ecclog); } +static enum mem_type ptl_h_get_mem_type(struct igen6_imc *imc) +{ + u32 mtype, val; + + val = readl(igen6_pvt->memss_pma_cr + res_cfg->reg_mem_config_offset); + mtype = field_get(res_cfg->reg_mem_config_ddr_type_mask, val); + + edac_dbg(2, "mtype %u (reg 0x%x)\n", mtype, val); + + switch (mtype) { + case 1: + return MEM_DDR5; + case 2: + return MEM_LPDDR5; + case 3: + return MEM_LPDDR4; + default: + return MEM_UNKNOWN; + } +} + +static enum dev_type ptl_h_get_dev_type(struct igen6_imc *imc, int chan, int dimm) +{ + u32 width, val; + + val = readl(imc->window + MAD_INTRA_CH0_OFFSET + chan * 4); + width = field_get(res_cfg->reg_mad_intra_width_mask[dimm], val); + + switch (width) { + case 1: + return DEV_X8; + default: + return DEV_X16; + } +} + +static u64 ptl_h_get_chan_size(struct igen6_imc *imc, int chan) +{ + u32 val = readl(imc->window + MAD_INTER_CHANNEL_OFFSET); + + return field_get(res_cfg->reg_mad_inter_size_mask[chan], val) * + res_cfg->reg_mad_inter_size_granularity; +} + +static u64 ptl_h_get_dimm_size(struct igen6_imc *imc, int chan, int dimm) +{ + u32 val = readl(imc->window + MAD_INTRA_CH0_OFFSET + chan * 4); + u32 ranks = 1 << field_get(res_cfg->reg_mad_intra_rank_mask[dimm], val); + /* DRAM device density in Gb */ + u64 density = field_get(res_cfg->reg_mad_intra_density_mask[dimm], val) * 4; + + enum mem_type mtype = ptl_h_get_mem_type(imc); + enum dev_type dtype = ptl_h_get_dev_type(imc, chan, dimm); + u64 sub_ch_width, dev_num; + + switch (mtype) { + case MEM_DDR5: + sub_ch_width = 32; + break; + case MEM_LPDDR5: + case MEM_LPDDR4: + sub_ch_width = 16; + break; + default: + sub_ch_width = 0; + } + + switch (dtype) { + case DEV_X8: + dev_num = sub_ch_width / 8; + break; + case DEV_X16: + dev_num = sub_ch_width / 16; + break; + default: + dev_num = 0; + } + + edac_dbg(2, "ranks %d, density %lluGb, sub_ch_width %llu, dev_num %llu (reg 0x%x)\n", ranks, density, sub_ch_width, dev_num, val); + + return ((dev_num * density / 8) * ranks) << 30; +} + +static void ptl_h_set_chan_params(struct igen6_imc *imc) +{ + u64 ch0_size = ptl_h_get_chan_size(imc, 0); + u64 ch1_size = ptl_h_get_chan_size(imc, 1); + + if (ch0_size <= ch1_size) { + imc->ch_s_size = ch0_size; + imc->ch_l_map = 1; + } else { + imc->ch_s_size = ch1_size; + imc->ch_l_map = 0; + } +} + +static void ptl_h_set_dimm_params(struct igen6_imc *imc, int chan) +{ + u64 dimm0_size = ptl_h_get_dimm_size(imc, chan, 0); + u64 dimm1_size = ptl_h_get_dimm_size(imc, chan, 1); + + if (dimm0_size <= dimm1_size) { + imc->dimm_s_size[chan] = dimm0_size; + imc->dimm_l_size[chan] = dimm1_size; + imc->dimm_l_map[chan] = 1; + } else { + imc->dimm_s_size[chan] = dimm1_size; + imc->dimm_l_size[chan] = dimm0_size; + imc->dimm_l_map[chan] = 0; + } +} + static struct res_config ehl_cfg = { .num_imc = 1, .reg_mchbar_mask = GENMASK_ULL(38, 16), @@ -593,18 +733,20 @@ static struct res_config rpl_p_cfg = { }; static struct res_config mtl_ps_cfg = { - .machine_check = true, - .num_imc = 2, - .reg_mchbar_mask = GENMASK_ULL(41, 17), - .reg_tom_mask = GENMASK_ULL(41, 20), - .reg_touud_mask = GENMASK_ULL(41, 20), - .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5), - .imc_base = 0xd800, - .ibecc_base = 0xd400, - .ibecc_error_log_offset = 0x170, - .ibecc_available = mtl_ps_ibecc_available, - .err_addr_to_sys_addr = adl_err_addr_to_sys_addr, - .err_addr_to_imc_addr = adl_err_addr_to_imc_addr, + .machine_check = true, + .num_imc = 2, + .reg_mchbar_mask = GENMASK_ULL(41, 17), + .reg_tom_mask = GENMASK_ULL(41, 20), + .reg_touud_mask = GENMASK_ULL(41, 20), + .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5), + .reg_capabilities_misc_offset = 0x13c00, + .reg_capabilities_misc_ibecc_dis = BIT(6), + .imc_base = 0xd800, + .ibecc_base = 0xd400, + .ibecc_error_log_offset = 0x170, + .ibecc_available = generic_ibecc_available, + .err_addr_to_sys_addr = adl_err_addr_to_sys_addr, + .err_addr_to_imc_addr = adl_err_addr_to_imc_addr, }; static struct res_config mtl_p_cfg = { @@ -622,6 +764,36 @@ static struct res_config mtl_p_cfg = { .err_addr_to_imc_addr = adl_err_addr_to_imc_addr, }; +static struct res_config ptl_h_cfg = { + .machine_check = true, + .num_imc = 2, + .reg_mchbar_mask = GENMASK_ULL(41, 17), + .reg_tom_mask = GENMASK_ULL(41, 20), + .reg_touud_mask = GENMASK_ULL(41, 20), + .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5), + .reg_mem_config_offset = 0x13d04, + .reg_mem_config_ddr_type_mask = GENMASK(8, 6), + .reg_mad_inter_size_mask[0] = GENMASK(15, 8), + .reg_mad_inter_size_mask[1] = GENMASK(23, 16), + .reg_mad_inter_size_granularity = BIT_ULL(29), + .reg_mad_intra_rank_mask[0] = BIT(7), + .reg_mad_intra_rank_mask[1] = BIT(15), + .reg_mad_intra_width_mask[0] = BIT(6), + .reg_mad_intra_width_mask[1] = BIT(14), + .reg_mad_intra_density_mask[0] = GENMASK(3, 0), + .reg_mad_intra_density_mask[1] = GENMASK(11, 8), + .imc_base = 0xd800, + .ibecc_base = 0xd400, + .ibecc_error_log_offset = 0x170, + .get_mem_type = ptl_h_get_mem_type, + .get_dev_type = ptl_h_get_dev_type, + .set_chan_params = ptl_h_set_chan_params, + .set_dimm_params = ptl_h_set_dimm_params, + .ibecc_available = mtl_p_ibecc_available, + .err_addr_to_sys_addr = adl_err_addr_to_sys_addr, + .err_addr_to_imc_addr = adl_err_addr_to_imc_addr, +}; + static struct res_config wcl_cfg = { .machine_check = true, .num_imc = 1, @@ -637,98 +809,122 @@ static struct res_config wcl_cfg = { .err_addr_to_imc_addr = adl_err_addr_to_imc_addr, }; +static struct res_config nvl_h_cfg = { + .machine_check = true, + .num_imc = 2, + .reg_mchbar_mask = GENMASK_ULL(41, 17), + .reg_tom_mask = GENMASK_ULL(41, 20), + .reg_touud_mask = GENMASK_ULL(41, 20), + .reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5), + .reg_mem_config_offset = 0x12904, + .reg_mem_config_ddr_type_mask = GENMASK(8, 6), + .reg_mem_config_ibecc_en_mask = GENMASK(3, 2), + .reg_mad_inter_size_mask[0] = GENMASK(15, 8), + .reg_mad_inter_size_mask[1] = GENMASK(23, 16), + .reg_mad_inter_size_granularity = BIT_ULL(29), + .reg_mad_intra_rank_mask[0] = BIT(7), + .reg_mad_intra_rank_mask[1] = BIT(15), + .reg_mad_intra_width_mask[0] = BIT(6), + .reg_mad_intra_width_mask[1] = BIT(14), + .reg_mad_intra_density_mask[0] = GENMASK(3, 0), + .reg_mad_intra_density_mask[1] = GENMASK(11, 8), + .imc_base = 0xd800, + .ibecc_base = 0xd400, + .ibecc_error_log_offset = 0x170, + .get_mem_type = ptl_h_get_mem_type, + .get_dev_type = ptl_h_get_dev_type, + .set_chan_params = ptl_h_set_chan_params, + .set_dimm_params = ptl_h_set_dimm_params, + .ibecc_available = generic_ibecc_available, + .err_addr_to_sys_addr = adl_err_addr_to_sys_addr, + .err_addr_to_imc_addr = adl_err_addr_to_imc_addr, +}; + static struct pci_device_id igen6_pci_tbl[] = { - { PCI_VDEVICE(INTEL, DID_EHL_SKU5), (kernel_ulong_t)&ehl_cfg }, - { PCI_VDEVICE(INTEL, DID_EHL_SKU6), (kernel_ulong_t)&ehl_cfg }, - { PCI_VDEVICE(INTEL, DID_EHL_SKU7), (kernel_ulong_t)&ehl_cfg }, - { PCI_VDEVICE(INTEL, DID_EHL_SKU8), (kernel_ulong_t)&ehl_cfg }, - { PCI_VDEVICE(INTEL, DID_EHL_SKU9), (kernel_ulong_t)&ehl_cfg }, - { PCI_VDEVICE(INTEL, DID_EHL_SKU10), (kernel_ulong_t)&ehl_cfg }, - { PCI_VDEVICE(INTEL, DID_EHL_SKU11), (kernel_ulong_t)&ehl_cfg }, - { PCI_VDEVICE(INTEL, DID_EHL_SKU12), (kernel_ulong_t)&ehl_cfg }, - { PCI_VDEVICE(INTEL, DID_EHL_SKU13), (kernel_ulong_t)&ehl_cfg }, - { PCI_VDEVICE(INTEL, DID_EHL_SKU14), (kernel_ulong_t)&ehl_cfg }, - { PCI_VDEVICE(INTEL, DID_EHL_SKU15), (kernel_ulong_t)&ehl_cfg }, - { PCI_VDEVICE(INTEL, DID_ICL_SKU8), (kernel_ulong_t)&icl_cfg }, - { PCI_VDEVICE(INTEL, DID_ICL_SKU10), (kernel_ulong_t)&icl_cfg }, - { PCI_VDEVICE(INTEL, DID_ICL_SKU11), (kernel_ulong_t)&icl_cfg }, - { PCI_VDEVICE(INTEL, DID_ICL_SKU12), (kernel_ulong_t)&icl_cfg }, - { PCI_VDEVICE(INTEL, DID_TGL_SKU), (kernel_ulong_t)&tgl_cfg }, - { PCI_VDEVICE(INTEL, DID_ADL_SKU1), (kernel_ulong_t)&adl_cfg }, - { PCI_VDEVICE(INTEL, DID_ADL_SKU2), (kernel_ulong_t)&adl_cfg }, - { PCI_VDEVICE(INTEL, DID_ADL_SKU3), (kernel_ulong_t)&adl_cfg }, - { PCI_VDEVICE(INTEL, DID_ADL_SKU4), (kernel_ulong_t)&adl_cfg }, - { PCI_VDEVICE(INTEL, DID_ADL_N_SKU1), (kernel_ulong_t)&adl_n_cfg }, - { PCI_VDEVICE(INTEL, DID_ADL_N_SKU2), (kernel_ulong_t)&adl_n_cfg }, - { PCI_VDEVICE(INTEL, DID_ADL_N_SKU3), (kernel_ulong_t)&adl_n_cfg }, - { PCI_VDEVICE(INTEL, DID_ADL_N_SKU4), (kernel_ulong_t)&adl_n_cfg }, - { PCI_VDEVICE(INTEL, DID_ADL_N_SKU5), (kernel_ulong_t)&adl_n_cfg }, - { PCI_VDEVICE(INTEL, DID_ADL_N_SKU6), (kernel_ulong_t)&adl_n_cfg }, - { PCI_VDEVICE(INTEL, DID_ADL_N_SKU7), (kernel_ulong_t)&adl_n_cfg }, - { PCI_VDEVICE(INTEL, DID_ADL_N_SKU8), (kernel_ulong_t)&adl_n_cfg }, - { PCI_VDEVICE(INTEL, DID_ADL_N_SKU9), (kernel_ulong_t)&adl_n_cfg }, - { PCI_VDEVICE(INTEL, DID_ADL_N_SKU10), (kernel_ulong_t)&adl_n_cfg }, - { PCI_VDEVICE(INTEL, DID_ADL_N_SKU11), (kernel_ulong_t)&adl_n_cfg }, - { PCI_VDEVICE(INTEL, DID_ADL_N_SKU12), (kernel_ulong_t)&adl_n_cfg }, - { PCI_VDEVICE(INTEL, DID_AZB_SKU1), (kernel_ulong_t)&adl_n_cfg }, - { PCI_VDEVICE(INTEL, DID_ASL_SKU1), (kernel_ulong_t)&adl_n_cfg }, - { PCI_VDEVICE(INTEL, DID_ASL_SKU2), (kernel_ulong_t)&adl_n_cfg }, - { PCI_VDEVICE(INTEL, DID_ASL_SKU3), (kernel_ulong_t)&adl_n_cfg }, - { PCI_VDEVICE(INTEL, DID_RPL_P_SKU1), (kernel_ulong_t)&rpl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_RPL_P_SKU2), (kernel_ulong_t)&rpl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_RPL_P_SKU3), (kernel_ulong_t)&rpl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_RPL_P_SKU4), (kernel_ulong_t)&rpl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_RPL_P_SKU5), (kernel_ulong_t)&rpl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU1), (kernel_ulong_t)&mtl_ps_cfg }, - { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU2), (kernel_ulong_t)&mtl_ps_cfg }, - { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU3), (kernel_ulong_t)&mtl_ps_cfg }, - { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU4), (kernel_ulong_t)&mtl_ps_cfg }, - { PCI_VDEVICE(INTEL, DID_MTL_P_SKU1), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_MTL_P_SKU2), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_MTL_P_SKU3), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_ARL_UH_SKU1), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_ARL_UH_SKU2), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_ARL_UH_SKU3), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_PTL_H_SKU1), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_PTL_H_SKU2), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_PTL_H_SKU3), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_PTL_H_SKU4), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_PTL_H_SKU5), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_PTL_H_SKU6), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_PTL_H_SKU7), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_PTL_H_SKU8), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_PTL_H_SKU9), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_PTL_H_SKU10), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_PTL_H_SKU11), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_PTL_H_SKU12), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_PTL_H_SKU13), (kernel_ulong_t)&mtl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_WCL_SKU1), (kernel_ulong_t)&wcl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU5), .driver_data = (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU6), .driver_data = (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU7), .driver_data = (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU8), .driver_data = (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU9), .driver_data = (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU10), .driver_data = (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU11), .driver_data = (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU12), .driver_data = (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU13), .driver_data = (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU14), .driver_data = (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU15), .driver_data = (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_ICL_SKU8), .driver_data = (kernel_ulong_t)&icl_cfg }, + { PCI_VDEVICE(INTEL, DID_ICL_SKU10), .driver_data = (kernel_ulong_t)&icl_cfg }, + { PCI_VDEVICE(INTEL, DID_ICL_SKU11), .driver_data = (kernel_ulong_t)&icl_cfg }, + { PCI_VDEVICE(INTEL, DID_ICL_SKU12), .driver_data = (kernel_ulong_t)&icl_cfg }, + { PCI_VDEVICE(INTEL, DID_TGL_SKU), .driver_data = (kernel_ulong_t)&tgl_cfg }, + { PCI_VDEVICE(INTEL, DID_ADL_SKU1), .driver_data = (kernel_ulong_t)&adl_cfg }, + { PCI_VDEVICE(INTEL, DID_ADL_SKU2), .driver_data = (kernel_ulong_t)&adl_cfg }, + { PCI_VDEVICE(INTEL, DID_ADL_SKU3), .driver_data = (kernel_ulong_t)&adl_cfg }, + { PCI_VDEVICE(INTEL, DID_ADL_SKU4), .driver_data = (kernel_ulong_t)&adl_cfg }, + { PCI_VDEVICE(INTEL, DID_ADL_N_SKU1), .driver_data = (kernel_ulong_t)&adl_n_cfg }, + { PCI_VDEVICE(INTEL, DID_ADL_N_SKU2), .driver_data = (kernel_ulong_t)&adl_n_cfg }, + { PCI_VDEVICE(INTEL, DID_ADL_N_SKU3), .driver_data = (kernel_ulong_t)&adl_n_cfg }, + { PCI_VDEVICE(INTEL, DID_ADL_N_SKU4), .driver_data = (kernel_ulong_t)&adl_n_cfg }, + { PCI_VDEVICE(INTEL, DID_ADL_N_SKU5), .driver_data = (kernel_ulong_t)&adl_n_cfg }, + { PCI_VDEVICE(INTEL, DID_ADL_N_SKU6), .driver_data = (kernel_ulong_t)&adl_n_cfg }, + { PCI_VDEVICE(INTEL, DID_ADL_N_SKU7), .driver_data = (kernel_ulong_t)&adl_n_cfg }, + { PCI_VDEVICE(INTEL, DID_ADL_N_SKU8), .driver_data = (kernel_ulong_t)&adl_n_cfg }, + { PCI_VDEVICE(INTEL, DID_ADL_N_SKU9), .driver_data = (kernel_ulong_t)&adl_n_cfg }, + { PCI_VDEVICE(INTEL, DID_ADL_N_SKU10), .driver_data = (kernel_ulong_t)&adl_n_cfg }, + { PCI_VDEVICE(INTEL, DID_ADL_N_SKU11), .driver_data = (kernel_ulong_t)&adl_n_cfg }, + { PCI_VDEVICE(INTEL, DID_ADL_N_SKU12), .driver_data = (kernel_ulong_t)&adl_n_cfg }, + { PCI_VDEVICE(INTEL, DID_AZB_SKU1), .driver_data = (kernel_ulong_t)&adl_n_cfg }, + { PCI_VDEVICE(INTEL, DID_ASL_SKU1), .driver_data = (kernel_ulong_t)&adl_n_cfg }, + { PCI_VDEVICE(INTEL, DID_ASL_SKU2), .driver_data = (kernel_ulong_t)&adl_n_cfg }, + { PCI_VDEVICE(INTEL, DID_ASL_SKU3), .driver_data = (kernel_ulong_t)&adl_n_cfg }, + { PCI_VDEVICE(INTEL, DID_RPL_P_SKU1), .driver_data = (kernel_ulong_t)&rpl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_RPL_P_SKU2), .driver_data = (kernel_ulong_t)&rpl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_RPL_P_SKU3), .driver_data = (kernel_ulong_t)&rpl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_RPL_P_SKU4), .driver_data = (kernel_ulong_t)&rpl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_RPL_P_SKU5), .driver_data = (kernel_ulong_t)&rpl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU1), .driver_data = (kernel_ulong_t)&mtl_ps_cfg }, + { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU2), .driver_data = (kernel_ulong_t)&mtl_ps_cfg }, + { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU3), .driver_data = (kernel_ulong_t)&mtl_ps_cfg }, + { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU4), .driver_data = (kernel_ulong_t)&mtl_ps_cfg }, + { PCI_VDEVICE(INTEL, DID_MTL_P_SKU1), .driver_data = (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_MTL_P_SKU2), .driver_data = (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_MTL_P_SKU3), .driver_data = (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_ARL_UH_SKU1), .driver_data = (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_ARL_UH_SKU2), .driver_data = (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_ARL_UH_SKU3), .driver_data = (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU1), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU2), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU3), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU4), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU5), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU6), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU7), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU8), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU9), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU10), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU11), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU12), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU13), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU14), .driver_data = (kernel_ulong_t)&ptl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_WCL_SKU1), .driver_data = (kernel_ulong_t)&wcl_cfg }, + { PCI_VDEVICE(INTEL, DID_NVL_H_SKU1), .driver_data = (kernel_ulong_t)&nvl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_NVL_H_SKU2), .driver_data = (kernel_ulong_t)&nvl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_NVL_H_SKU3), .driver_data = (kernel_ulong_t)&nvl_h_cfg }, + { PCI_VDEVICE(INTEL, DID_NVL_H_SKU4), .driver_data = (kernel_ulong_t)&nvl_h_cfg }, { }, }; MODULE_DEVICE_TABLE(pci, igen6_pci_tbl); -static enum dev_type get_width(int dimm_l, u32 mad_dimm) +static enum mem_type get_mem_type(struct igen6_imc *imc) { - u32 w = dimm_l ? MAD_DIMM_CH_DLW(mad_dimm) : - MAD_DIMM_CH_DSW(mad_dimm); + u32 val; - switch (w) { - case 0: - return DEV_X8; - case 1: - return DEV_X16; - case 2: - return DEV_X32; - default: - return DEV_UNKNOWN; - } -} + if (res_cfg->get_mem_type) + return res_cfg->get_mem_type(imc); -static enum mem_type get_memory_type(u32 mad_inter) -{ - u32 t = MAD_INTER_CHANNEL_DDR_TYPE(mad_inter); + val = readl(imc->window + MAD_INTER_CHANNEL_OFFSET); - switch (t) { + switch (MAD_INTER_CHANNEL_DDR_TYPE(val)) { case 0: return MEM_DDR4; case 1: @@ -744,6 +940,73 @@ static enum mem_type get_memory_type(u32 mad_inter) } } +static bool large_dimm(struct igen6_imc *imc, int chan, int dimm) +{ + return dimm == imc->dimm_l_map[chan]; +} + +static enum dev_type get_dev_type(struct igen6_imc *imc, int chan, int dimm) +{ + u32 width, val; + + if (res_cfg->get_dev_type) + return res_cfg->get_dev_type(imc, chan, dimm); + + val = readl(imc->window + MAD_DIMM_CH0_OFFSET + chan * 4); + width = large_dimm(imc, chan, dimm) ? MAD_DIMM_CH_DLW(val) : + MAD_DIMM_CH_DSW(val); + + switch (width) { + case 0: + return DEV_X8; + case 1: + return DEV_X16; + case 2: + return DEV_X32; + default: + return DEV_UNKNOWN; + } +} + +static u64 get_dimm_size(struct igen6_imc *imc, int chan, int dimm) +{ + if (large_dimm(imc, chan, dimm)) + return imc->dimm_l_size[chan]; + + return imc->dimm_s_size[chan]; +} + +static void set_chan_params(struct igen6_imc *imc) +{ + u32 val; + + if (res_cfg->set_chan_params) { + res_cfg->set_chan_params(imc); + return; + } + + val = readl(imc->window + MAD_INTER_CHANNEL_OFFSET); + imc->ch_s_size = MAD_INTER_CHANNEL_CH_S_SIZE(val); + imc->ch_l_map = MAD_INTER_CHANNEL_CH_L_MAP(val); +} + +static void set_dimm_params(struct igen6_imc *imc, int chan) +{ + u32 val; + + if (res_cfg->set_dimm_params) { + res_cfg->set_dimm_params(imc, chan); + return; + } + + val = readl(imc->window + MAD_INTRA_CH0_OFFSET + chan * 4); + imc->dimm_l_map[chan] = MAD_INTRA_CH_DIMM_L_MAP(val); + + val = readl(imc->window + MAD_DIMM_CH0_OFFSET + chan * 4); + imc->dimm_l_size[chan] = MAD_DIMM_CH_DIMM_L_SIZE(val); + imc->dimm_s_size[chan] = MAD_DIMM_CH_DIMM_S_SIZE(val); +} + static int decode_chan_idx(u64 addr, u64 mask, int intlv_bit) { u64 hash_addr = addr & mask, hash = 0; @@ -1084,7 +1347,6 @@ static bool igen6_check_ecc(struct igen6_imc *imc) static int igen6_get_dimm_config(struct mem_ctl_info *mci) { struct igen6_imc *imc = mci->pvt_info; - u32 mad_inter, mad_intra, mad_dimm; int i, j, ndimms, mc = imc->mc; struct dimm_info *dimm; enum mem_type mtype; @@ -1094,33 +1356,20 @@ static int igen6_get_dimm_config(struct mem_ctl_info *mci) edac_dbg(2, "\n"); - mad_inter = readl(imc->window + MAD_INTER_CHANNEL_OFFSET); - mtype = get_memory_type(mad_inter); + mtype = get_mem_type(imc); ecc = igen6_check_ecc(imc); - imc->ch_s_size = MAD_INTER_CHANNEL_CH_S_SIZE(mad_inter); - imc->ch_l_map = MAD_INTER_CHANNEL_CH_L_MAP(mad_inter); + set_chan_params(imc); for (i = 0; i < NUM_CHANNELS; i++) { - mad_intra = readl(imc->window + MAD_INTRA_CH0_OFFSET + i * 4); - mad_dimm = readl(imc->window + MAD_DIMM_CH0_OFFSET + i * 4); - - imc->dimm_l_size[i] = MAD_DIMM_CH_DIMM_L_SIZE(mad_dimm); - imc->dimm_s_size[i] = MAD_DIMM_CH_DIMM_S_SIZE(mad_dimm); - imc->dimm_l_map[i] = MAD_INTRA_CH_DIMM_L_MAP(mad_intra); + set_dimm_params(imc, i); imc->size += imc->dimm_s_size[i]; imc->size += imc->dimm_l_size[i]; ndimms = 0; for (j = 0; j < NUM_DIMMS; j++) { dimm = edac_get_dimm(mci, i, j, 0); - - if (j ^ imc->dimm_l_map[i]) { - dtype = get_width(0, mad_dimm); - dsize = imc->dimm_s_size[i]; - } else { - dtype = get_width(1, mad_dimm); - dsize = imc->dimm_l_size[i]; - } + dtype = get_dev_type(imc, i, j); + dsize = get_dimm_size(imc, i, j); if (!dsize) continue; @@ -1223,6 +1472,39 @@ static void igen6_debug_setup(void) {} static void igen6_debug_teardown(void) {} #endif +static struct igen6_pvt *igen6_pvt_setup(struct pci_dev *pdev) +{ + void __iomem *memss_pma_cr; + struct igen6_pvt *pvt; + u64 mchbar; + int rc; + + pvt = kzalloc_obj(*igen6_pvt); + if (!pvt) + return NULL; + + rc = get_mchbar(pdev, &mchbar); + if (rc) { + kfree(pvt); + return NULL; + } + + memss_pma_cr = ioremap(mchbar, MCHBAR_SIZE * 2); + if (!memss_pma_cr) { + kfree(pvt); + return NULL; + } + pvt->memss_pma_cr = memss_pma_cr; + + return pvt; +} + +static void igen6_pvt_release(struct igen6_pvt *pvt) +{ + iounmap(pvt->memss_pma_cr); + kfree(pvt); +} + static int igen6_pci_setup(struct pci_dev *pdev, u64 *mchbar) { union { @@ -1296,6 +1578,11 @@ static bool igen6_imc_absent(void __iomem *window) return readl(window + MAD_INTER_CHANNEL_OFFSET) == ~0; } +static void imc_release(struct device *dev) +{ + /* Nothing to do, the 'imc' owns the 'dev' and will also release it. */ +} + static int igen6_register_mci(int mc, void __iomem *window, struct pci_dev *pdev) { struct edac_mc_layer layers[2]; @@ -1334,6 +1621,7 @@ static int igen6_register_mci(int mc, void __iomem *window, struct pci_dev *pdev mci->pvt_info = &igen6_pvt->imc[mc]; imc = mci->pvt_info; + imc->dev.release = imc_release; device_initialize(&imc->dev); /* * EDAC core uses mci->pdev(pointer of structure device) as @@ -1549,12 +1837,12 @@ static int igen6_probe(struct pci_dev *pdev, const struct pci_device_id *ent) edac_dbg(2, "\n"); - igen6_pvt = kzalloc_obj(*igen6_pvt); + res_cfg = (struct res_config *)ent->driver_data; + + igen6_pvt = igen6_pvt_setup(pdev); if (!igen6_pvt) return -ENOMEM; - res_cfg = (struct res_config *)ent->driver_data; - rc = igen6_pci_setup(pdev, &mchbar); if (rc) goto fail; @@ -1603,7 +1891,7 @@ fail3: fail2: igen6_unregister_mcis(); fail: - kfree(igen6_pvt); + igen6_pvt_release(igen6_pvt); return rc; } @@ -1618,7 +1906,7 @@ static void igen6_remove(struct pci_dev *pdev) flush_work(&ecclog_work); gen_pool_destroy(ecclog_pool); igen6_unregister_mcis(); - kfree(igen6_pvt); + igen6_pvt_release(igen6_pvt); } static struct pci_driver igen6_driver = { diff --git a/drivers/edac/imh_base.c b/drivers/edac/imh_base.c index 40082ba45e62..6ca0df031bf5 100644 --- a/drivers/edac/imh_base.c +++ b/drivers/edac/imh_base.c @@ -71,28 +71,44 @@ struct local_reg { .width = (cfg)->ip_name##_reg_##reg_name##_width, \ } -static u64 readx(void __iomem *addr, u8 width) -{ - switch (width) { - case 1: - return readb(addr); - case 2: - return readw(addr); - case 4: - return readl(addr); - case 8: - return readq(addr); - default: - imh_printk(KERN_ERR, "Invalid reg 0x%p width %d\n", addr, width); - return 0; - } +static struct res_config *res_cfg; +static int retry_rd_err_log; + +#define REG_RRL_DEFINE(a0, a1, a2, a3, a4, a5, a6, b0, b1, b2, b3) \ + { \ + .set_num = 4, \ + .reg_num = 7, \ + .sources = {RRL_SRC_FRE_SCRUB, RRL_SRC_FRE_DEMAND, RRL_SRC_LRE_SCRUB, RRL_SRC_LRE_DEMAND}, \ + .offsets = { \ + {a0, a1, a2, a3, a4, a5, a6}, \ + {a0 + 4, a1 + 4, a2 + 8, a3 + 4, a4 + 4, a5 + 8, a6 + 8}, \ + {a0 + 8, a1 + 8, a2 + 16, a3 + 8, a4 + 8, a5 + 16, a6 + 16}, \ + {a0 + 12, a1 + 12, a2 + 24, a3 + 12, a4 + 12, a5 + 24, a6 + 24}, \ + }, \ + .widths = {4, 4, 8, 4, 4, 8, 8}, \ + .v_mask = BIT(0), \ + .uc_mask = BIT(1), \ + .over_mask = BIT(2), \ + .en_mask = BIT(12), \ + .en_patspr_mask = BIT(14), \ + .noover_mask = BIT(15), \ + .cecnt_num = 4, \ + .cecnt_offsets = {b0, b1, b2, b3}, \ + .cecnt_widths = {8, 8, 8, 8}, \ } +static struct reg_rrl dmr_reg_rrl_ddr_subch0 = REG_RRL_DEFINE( + 0x2dc0, 0x2dd0, 0x2de0, 0x2e00, 0x2e10, 0x2f70, 0x0200, + 0x2c10, 0x2c18, 0x2c20, 0x2c28); +static struct reg_rrl dmr_reg_rrl_ddr_subch1 = REG_RRL_DEFINE( + 0x6dc0, 0x6dd0, 0x6de0, 0x6e00, 0x6e10, 0x6f70, 0x4200, + 0x6c10, 0x6c18, 0x6c20, 0x6c28); + static void __read_local_reg(void *reg) { struct local_reg *r = (struct local_reg *)reg; - r->val = readx(r->vbase + r->offset, r->width); + r->val = skx_readx(r->vbase + r->offset, r->width); } /* Read a local-view register. */ @@ -378,22 +394,16 @@ static bool imh_2lm_enabled(struct res_config *cfg, struct list_head *head) return false; } -/* Helpers to read memory controller registers */ -static u64 read_imc_reg(struct skx_imc *imc, int chan, u32 offset, u8 width) -{ - return readx(imc->mbase + imc->chan_mmio_sz * chan + offset, width); -} - static u32 read_imc_mcmtr(struct res_config *cfg, struct skx_imc *imc, int chan) { - return (u32)read_imc_reg(imc, chan, cfg->ddr_reg_mcmtr_offset, cfg->ddr_reg_mcmtr_width); + return (u32)skx_read_imc_reg(imc, chan, cfg->ddr_reg_mcmtr_offset, cfg->ddr_reg_mcmtr_width); } static u32 read_imc_dimmmtr(struct res_config *cfg, struct skx_imc *imc, int chan, int dimm) { - return (u32)read_imc_reg(imc, chan, cfg->ddr_reg_dimmmtr_offset + - cfg->ddr_reg_dimmmtr_width * dimm, - cfg->ddr_reg_dimmmtr_width); + return (u32)skx_read_imc_reg(imc, chan, cfg->ddr_reg_dimmmtr_offset + + cfg->ddr_reg_dimmmtr_width * dimm, + cfg->ddr_reg_dimmmtr_width); } static bool ecc_enabled(u32 mcmtr) @@ -503,6 +513,8 @@ static struct res_config dmr_cfg = { .ha_size = 0x1000, .ha_reg_mode_offset = 0x4a0, .ha_reg_mode_width = 4, + .reg_rrl_ddr[0] = &dmr_reg_rrl_ddr_subch0, + .reg_rrl_ddr[1] = &dmr_reg_rrl_ddr_subch1, }; static const struct x86_cpu_id imh_cpuids[] = { @@ -542,6 +554,7 @@ static int __init imh_init(void) return -ENODEV; cfg = (struct res_config *)id->driver_data; skx_set_res_cfg(cfg); + res_cfg = cfg; if (!imh_get_tolm_tohm(cfg, &tolm, &tohm)) return -ENODEV; @@ -576,6 +589,13 @@ static int __init imh_init(void) mce_register_decode_chain(&imh_mce_dec); skx_setup_debug("imh_test"); + cfg->rrl_ctrl_mode = retry_rd_err_log; + if (retry_rd_err_log && cfg->reg_rrl_ddr[0]) { + skx_set_show_rrl(skx_show_rrl); + if (retry_rd_err_log == RRL_CTRL_LINUX) + skx_enable_rrl(true); + } + imh_printk(KERN_INFO, "%s\n", IMH_REVISION); return 0; @@ -588,6 +608,12 @@ static void __exit imh_exit(void) { edac_dbg(2, "\n"); + if (retry_rd_err_log && res_cfg->reg_rrl_ddr[0]) { + if (retry_rd_err_log == RRL_CTRL_LINUX) + skx_enable_rrl(false); + skx_set_show_rrl(NULL); + } + skx_teardown_debug(); mce_unregister_decode_chain(&imh_mce_dec); skx_adxl_put(); @@ -597,6 +623,9 @@ static void __exit imh_exit(void) module_init(imh_init); module_exit(imh_exit); +module_param(retry_rd_err_log, int, 0444); +MODULE_PARM_DESC(retry_rd_err_log, "retry_rd_err_log: 0=off(default), 1=bios(Linux doesn't reset any control bits, but just reports values.), 2=linux(Linux tries to take control and resets mode bits, clear valid/UC bits after reading.)"); + MODULE_LICENSE("GPL"); MODULE_AUTHOR("Qiuxu Zhuo"); MODULE_DESCRIPTION("MC Driver for Intel servers using IMH-based memory controller"); diff --git a/drivers/edac/sb_edac.c b/drivers/edac/sb_edac.c index 7b282dfd093f..35eb7a2038ab 100644 --- a/drivers/edac/sb_edac.c +++ b/drivers/edac/sb_edac.c @@ -2016,7 +2016,7 @@ static bool sb_decode_ddr4(struct mem_ctl_info *mci, int ch, u8 rank, static bool sb_decode_ddr3(struct mem_ctl_info *mci, int ch, u8 rank, u64 rank_addr, char *msg) { - pr_warn_once("DDR3 row/column decode not support yet!\n"); + pr_warn_once("DDR3 row/column decode is not supported yet!\n"); msg[0] = '\0'; return false; } diff --git a/drivers/edac/skx_base.c b/drivers/edac/skx_base.c index aa6593ccda2d..de749413ff9a 100644 --- a/drivers/edac/skx_base.c +++ b/drivers/edac/skx_base.c @@ -671,14 +671,14 @@ static int __init skx_init(void) } } - skx_set_decode(skx_decode, skx_show_retry_rd_err_log); + skx_set_show_rrl(skx_show_retry_rd_err_log); if (nvdimm_count && skx_adxl_get() != -ENODEV) { - skx_set_decode(NULL, skx_show_retry_rd_err_log); + skx_set_decode(NULL); } else { if (nvdimm_count) skx_printk(KERN_NOTICE, "Only decoding DDR4 address!\n"); - skx_set_decode(skx_decode, skx_show_retry_rd_err_log); + skx_set_decode(skx_decode); } /* Ensure that the OPSTATE is set correctly for POLL or NMI */ diff --git a/drivers/edac/skx_common.c b/drivers/edac/skx_common.c index a9557c8344bc..bfd0cd1689ed 100644 --- a/drivers/edac/skx_common.c +++ b/drivers/edac/skx_common.c @@ -31,10 +31,12 @@ static const char * const component_names[] = { [INDEX_CHANNEL] = "ChannelId", [INDEX_DIMM] = "DimmSlotId", [INDEX_CS] = "ChipSelect", + [INDEX_SUBCH] = "SubChId", [INDEX_NM_MEMCTRL] = "NmMemoryControllerId", [INDEX_NM_CHANNEL] = "NmChannelId", [INDEX_NM_DIMM] = "NmDimmSlotId", [INDEX_NM_CS] = "NmChipSelect", + [INDEX_NM_SUBCH] = "NmSubChId", }; static int component_indices[ARRAY_SIZE(component_names)]; @@ -43,15 +45,284 @@ static const char * const *adxl_component_names; static u64 *adxl_values; static char *adxl_msg; static unsigned long adxl_nm_bitmap; +static unsigned long adxl_bitmap; static char skx_msg[MSG_SIZE]; static skx_decode_f driver_decode; -static skx_show_retry_log_f skx_show_retry_rd_err_log; +static skx_show_rrl_f show_rrl; static u64 skx_tolm, skx_tohm; static LIST_HEAD(dev_edac_list); static bool skx_mem_cfg_2lm; static struct res_config *skx_res_cfg; +u64 skx_readx(void __iomem *addr, u8 width) +{ + switch (width) { + case 1: + return readb(addr); + case 2: + return readw(addr); + case 4: + return readl(addr); + case 8: + return readq(addr); + default: + skx_printk(KERN_ERR, "Invalid reg 0x%p width %u to read.\n", addr, width); + return 0; + } +} +EXPORT_SYMBOL_GPL(skx_readx); + +static void skx_writex(void __iomem *addr, u8 width, u64 val) +{ + switch (width) { + case 1: + writeb((u8)val, addr); + return; + case 2: + writew((u16)val, addr); + return; + case 4: + writel((u32)val, addr); + return; + case 8: + writeq(val, addr); + return; + default: + skx_printk(KERN_ERR, "Invalid reg 0x%p width %u to write 0x%llx.\n", addr, width, val); + } +} + +u64 skx_read_imc_reg(struct skx_imc *imc, int chan, u32 offset, u8 width) +{ + return skx_readx(imc->mbase + imc->chan_mmio_sz * chan + offset, width); +} +EXPORT_SYMBOL_GPL(skx_read_imc_reg); + +void skx_write_imc_reg(struct skx_imc *imc, int chan, u32 offset, u8 width, u64 val) +{ + skx_writex(imc->mbase + imc->chan_mmio_sz * chan + offset, width, val); +} +EXPORT_SYMBOL_GPL(skx_write_imc_reg); + +static void enable_rrl(struct skx_imc *imc, int chan, struct reg_rrl *rrl, + int rrl_set, bool enable, u32 *rrl_ctl) +{ + enum rrl_source_type source = rrl->sources[rrl_set]; + u32 offset = rrl->offsets[rrl_set][0], v; + u8 width = rrl->widths[0]; + bool first, scrub; + + /* First or last read error. */ + first = (source == RRL_SRC_FRE_SCRUB || source == RRL_SRC_FRE_DEMAND); + /* Patrol scrub or on-demand read error. */ + scrub = (source == RRL_SRC_FRE_SCRUB || source == RRL_SRC_LRE_SCRUB); + + v = skx_read_imc_reg(imc, chan, offset, width); + + if (enable) { + /* Save default configurations. */ + *rrl_ctl = v; + v &= ~rrl->uc_mask; + + if (first) + v |= rrl->noover_mask; + else + v &= ~rrl->noover_mask; + + if (scrub) + v |= rrl->en_patspr_mask; + else + v &= ~rrl->en_patspr_mask; + + v |= rrl->en_mask; + } else { + /* Restore default configurations. */ + if (*rrl_ctl & rrl->uc_mask) + v |= rrl->uc_mask; + + if (first) { + if (!(*rrl_ctl & rrl->noover_mask)) + v &= ~rrl->noover_mask; + } else { + if (*rrl_ctl & rrl->noover_mask) + v |= rrl->noover_mask; + } + + if (scrub) { + if (!(*rrl_ctl & rrl->en_patspr_mask)) + v &= ~rrl->en_patspr_mask; + } else { + if (*rrl_ctl & rrl->en_patspr_mask) + v |= rrl->en_patspr_mask; + } + + if (!(*rrl_ctl & rrl->en_mask)) + v &= ~rrl->en_mask; + } + + skx_write_imc_reg(imc, chan, offset, width, v); +} + +static void enable_rrls(struct skx_imc *imc, int chan, struct reg_rrl *rrl, + bool enable, u32 *rrl_ctl) +{ + for (int i = 0; i < rrl->set_num; i++) + enable_rrl(imc, chan, rrl, i, enable, rrl_ctl + i); +} + +static void enable_rrls_ddr(struct skx_imc *imc, bool enable) +{ + struct reg_rrl **rrl_ddr = skx_res_cfg->reg_rrl_ddr; + int i, chan_num = skx_res_cfg->ddr_chan_num; + struct skx_channel *chan = imc->chan; + + if (!imc->mbase) + return; + + for (i = 0; i < chan_num; i++) { + enable_rrls(imc, i, rrl_ddr[0], enable, chan[i].rrl_ctl[0]); + if (rrl_ddr[1]) + enable_rrls(imc, i, rrl_ddr[1], enable, chan[i].rrl_ctl[1]); + } +} + +static void enable_rrls_hbm(struct skx_imc *imc, bool enable) +{ + struct reg_rrl **rrl_hbm = skx_res_cfg->reg_rrl_hbm; + int i, chan_num = skx_res_cfg->hbm_chan_num; + struct skx_channel *chan = imc->chan; + + if (!imc->mbase || !imc->hbm_mc || !rrl_hbm[0] || !rrl_hbm[1]) + return; + + for (i = 0; i < chan_num; i++) { + enable_rrls(imc, i, rrl_hbm[0], enable, chan[i].rrl_ctl[0]); + enable_rrls(imc, i, rrl_hbm[1], enable, chan[i].rrl_ctl[1]); + } +} + +void skx_enable_rrl(bool enable) +{ + struct skx_dev *d; + int i, imc_num; + + edac_dbg(2, "\n"); + + list_for_each_entry(d, &dev_edac_list, list) { + imc_num = skx_res_cfg->ddr_imc_num; + for (i = 0; i < imc_num; i++) + enable_rrls_ddr(&d->imc[i], enable); + + imc_num += skx_res_cfg->hbm_imc_num; + for (; i < imc_num; i++) + enable_rrls_hbm(&d->imc[i], enable); + } +} +EXPORT_SYMBOL_GPL(skx_enable_rrl); + +static struct reg_rrl *get_rrl_reg(struct decoded_addr *res, struct res_config *cfg) +{ + struct skx_imc *imc = &res->dev->imc[res->imc]; + + /* HBM has two groups of RRL sets, one per pseudo-channel. */ + if (imc->hbm_mc) + return cfg->reg_rrl_hbm[res->cs & 1]; + + /* One group of RRL sets per DDR channel. */ + if (!cfg->reg_rrl_ddr[1]) + return cfg->reg_rrl_ddr[0]; + + if (res->subch == -1) { + skx_printk(KERN_ERR, "Invalid sub-channel id (-1), possibly missing %s ADXL component.\n", component_names[INDEX_SUBCH]); + return NULL; + } + + /* Two groups of RRL sets per DDR channel (e.g., DMR: one group per sub-channel). */ + return cfg->reg_rrl_ddr[res->subch & 1]; +} + +void skx_show_rrl(struct decoded_addr *res, char *msg, int len, bool scrub_err) +{ + struct skx_imc *imc = &res->dev->imc[res->imc]; + int i, j, n, ch = res->channel; + u64 log, corr, status_mask; + struct reg_rrl *rrl; + bool scrub; + u32 offset; + u8 width; + + if (!imc->mbase) + return; + + rrl = get_rrl_reg(res, skx_res_cfg); + if (!rrl) + return; + + status_mask = rrl->over_mask | rrl->uc_mask | rrl->v_mask; + + n = scnprintf(msg, len, " retry_rd_err_log["); + for (i = 0; i < rrl->set_num; i++) { + scrub = (rrl->sources[i] == RRL_SRC_FRE_SCRUB || rrl->sources[i] == RRL_SRC_LRE_SCRUB); + if (scrub_err != scrub) + continue; + + for (j = 0; j < rrl->reg_num && len - n > 0; j++) { + offset = rrl->offsets[i][j]; + width = rrl->widths[j]; + log = skx_read_imc_reg(imc, ch, offset, width); + + if (width == 4) + n += scnprintf(msg + n, len - n, "%.8llx ", log); + else + n += scnprintf(msg + n, len - n, "%.16llx ", log); + + /* Clear RRL status if RRL in Linux control mode. */ + if (skx_res_cfg->rrl_ctrl_mode == RRL_CTRL_LINUX && !j && (log & status_mask)) + skx_write_imc_reg(imc, ch, offset, width, log & ~status_mask); + } + } + + /* Move back one space. */ + n--; + n += scnprintf(msg + n, len - n, "]"); + + if (len - n > 0) { + n += scnprintf(msg + n, len - n, " correrrcnt["); + for (i = 0; i < rrl->cecnt_num && len - n > 0; i++) { + offset = rrl->cecnt_offsets[i]; + width = rrl->cecnt_widths[i]; + corr = skx_read_imc_reg(imc, ch, offset, width); + + /* CPUs {ICX,SPR} encode two counters per 4-byte CORRERRCNT register. */ + if (skx_res_cfg->type <= SPR) { + n += scnprintf(msg + n, len - n, "%.4llx %.4llx ", + corr & 0xffff, corr >> 16); + } else { + /* CPUs {GNR} encode one counter per CORRERRCNT register. */ + if (width == 4) + n += scnprintf(msg + n, len - n, "%.8llx ", corr); + else + n += scnprintf(msg + n, len - n, "%.16llx ", corr); + } + } + + /* Move back one space. */ + n--; + n += scnprintf(msg + n, len - n, "]"); + } +} +EXPORT_SYMBOL_GPL(skx_show_rrl); + +static bool adxl_component_required(int idx) +{ + return idx == INDEX_SOCKET || + idx == INDEX_MEMCTRL || + idx == INDEX_CHANNEL || + idx == INDEX_DIMM || + idx == INDEX_CS; +} + int skx_adxl_get(void) { const char * const *names; @@ -70,12 +341,14 @@ int skx_adxl_get(void) if (i >= INDEX_NM_FIRST) adxl_nm_bitmap |= 1 << i; + else + adxl_bitmap |= 1 << i; break; } } - if (!names[j] && i < INDEX_NM_FIRST) + if (!names[j] && adxl_component_required(i)) goto err; } @@ -202,11 +475,15 @@ static bool skx_adxl_decode(struct decoded_addr *res, enum error_source err_src) (int)adxl_values[component_indices[INDEX_NM_DIMM]] : -1; res->cs = (adxl_nm_bitmap & BIT_NM_CS) ? (int)adxl_values[component_indices[INDEX_NM_CS]] : -1; + res->subch = (adxl_nm_bitmap & BIT_NM_SUBCH) ? + (int)adxl_values[component_indices[INDEX_NM_SUBCH]] : -1; } else { res->imc = (int)adxl_values[component_indices[INDEX_MEMCTRL]]; res->channel = (int)adxl_values[component_indices[INDEX_CHANNEL]]; res->dimm = (int)adxl_values[component_indices[INDEX_DIMM]]; res->cs = (int)adxl_values[component_indices[INDEX_CS]]; + res->subch = (adxl_bitmap & BIT_SUBCH) ? + (int)adxl_values[component_indices[INDEX_SUBCH]] : -1; } if (res->imc < 0) { @@ -262,13 +539,18 @@ void skx_set_res_cfg(struct res_config *cfg) } EXPORT_SYMBOL_GPL(skx_set_res_cfg); -void skx_set_decode(skx_decode_f decode, skx_show_retry_log_f show_retry_log) +void skx_set_decode(skx_decode_f decode) { driver_decode = decode; - skx_show_retry_rd_err_log = show_retry_log; } EXPORT_SYMBOL_GPL(skx_set_decode); +void skx_set_show_rrl(skx_show_rrl_f rrl) +{ + show_rrl = rrl; +} +EXPORT_SYMBOL_GPL(skx_set_show_rrl); + static int skx_get_pkg_id(struct skx_dev *d, u8 *id) { int node; @@ -466,6 +748,9 @@ int skx_get_dimm_info(u32 mtr, u32 mcmtr, u32 amap, struct dimm_info *dimm, rows = numrow(mtr); cols = imc->hbm_mc ? 6 : numcol(mtr); + if (ranks < 0 || rows < 0 || cols < 0) + return 0; + if (imc->hbm_mc) { banks = 32; mtype = MEM_HBM2; @@ -714,8 +999,8 @@ static void skx_mce_output_error(struct mem_ctl_info *mci, res->row, res->column, res->bank_address, res->bank_group); } - if (skx_show_retry_rd_err_log) - skx_show_retry_rd_err_log(res, skx_msg + len, MSG_SIZE - len, scrub_err); + if (show_rrl) + show_rrl(res, skx_msg + len, MSG_SIZE - len, scrub_err); edac_dbg(0, "%s\n", skx_msg); diff --git a/drivers/edac/skx_common.h b/drivers/edac/skx_common.h index f88038e5b18c..777252cca809 100644 --- a/drivers/edac/skx_common.h +++ b/drivers/edac/skx_common.h @@ -77,27 +77,36 @@ /* Max RRL register sets per {,sub-,pseudo-}channel. */ #define NUM_RRL_SET 4 /* Max RRL registers per set. */ -#define NUM_RRL_REG 6 +#define NUM_RRL_REG 7 /* Max correctable error count registers. */ #define NUM_CECNT_REG 8 -/* Modes of RRL register set. */ -enum rrl_mode { +/* Error source from which the RRL registers log errors. */ +enum rrl_source_type { /* Last read error from patrol scrub. */ - LRE_SCRUB, + RRL_SRC_LRE_SCRUB, /* Last read error from demand. */ - LRE_DEMAND, + RRL_SRC_LRE_DEMAND, /* First read error from patrol scrub. */ - FRE_SCRUB, + RRL_SRC_FRE_SCRUB, /* First read error from demand. */ - FRE_DEMAND, + RRL_SRC_FRE_DEMAND, +}; + +enum rrl_ctrl_mode { + /* Linux does not control RRL or reports values. */ + RRL_CTRL_NONE, + /* Firmware retains control. Linux only reports values. */ + RRL_CTRL_BIOS, + /* Linux takes control, resets mode bits, and clears valid/UC bits; reports values. */ + RRL_CTRL_LINUX, }; /* RRL registers per {,sub-,pseudo-}channel. */ struct reg_rrl { /* RRL register parts. */ int set_num, reg_num; - enum rrl_mode modes[NUM_RRL_SET]; + enum rrl_source_type sources[NUM_RRL_SET]; u32 offsets[NUM_RRL_SET][NUM_RRL_REG]; /* RRL register widths in byte per set. */ u8 widths[NUM_RRL_REG]; @@ -201,11 +210,13 @@ enum { INDEX_CHANNEL, INDEX_DIMM, INDEX_CS, + INDEX_SUBCH, INDEX_NM_FIRST, INDEX_NM_MEMCTRL = INDEX_NM_FIRST, INDEX_NM_CHANNEL, INDEX_NM_DIMM, INDEX_NM_CS, + INDEX_NM_SUBCH, INDEX_MAX }; @@ -216,10 +227,12 @@ enum error_source { ERR_SRC_NOT_MEMORY, }; +#define BIT_SUBCH BIT_ULL(INDEX_SUBCH) #define BIT_NM_MEMCTRL BIT_ULL(INDEX_NM_MEMCTRL) #define BIT_NM_CHANNEL BIT_ULL(INDEX_NM_CHANNEL) #define BIT_NM_DIMM BIT_ULL(INDEX_NM_DIMM) #define BIT_NM_CS BIT_ULL(INDEX_NM_CS) +#define BIT_NM_SUBCH BIT_ULL(INDEX_NM_SUBCH) struct decoded_addr { struct mce *mce; @@ -233,6 +246,7 @@ struct decoded_addr { int chanways; int dimm; int cs; + int subch; int rank; int channel_rank; u64 rank_address; @@ -269,9 +283,11 @@ struct res_config { int hbm_chan_mmio_sz; bool support_ddr5; /* RRL register sets per DDR channel */ - struct reg_rrl *reg_rrl_ddr; + struct reg_rrl *reg_rrl_ddr[2]; /* RRL register sets per HBM channel */ struct reg_rrl *reg_rrl_hbm[2]; + /* RRL control mode */ + enum rrl_ctrl_mode rrl_ctrl_mode; union { /* {skx,i10nm}_edac */ struct { @@ -324,11 +340,17 @@ struct res_config { typedef int (*get_dimm_config_f)(struct mem_ctl_info *mci, struct res_config *cfg); typedef bool (*skx_decode_f)(struct decoded_addr *res); -typedef void (*skx_show_retry_log_f)(struct decoded_addr *res, char *msg, int len, bool scrub_err); +typedef void (*skx_show_rrl_f)(struct decoded_addr *res, char *msg, int len, bool scrub_err); +u64 skx_readx(void __iomem *addr, u8 width); +u64 skx_read_imc_reg(struct skx_imc *imc, int chan, u32 offset, u8 width); +void skx_write_imc_reg(struct skx_imc *imc, int chan, u32 offset, u8 width, u64 val); int skx_adxl_get(void); void skx_adxl_put(void); -void skx_set_decode(skx_decode_f decode, skx_show_retry_log_f show_retry_log); +void skx_set_decode(skx_decode_f decode); +void skx_set_show_rrl(skx_show_rrl_f rrl); +void skx_show_rrl(struct decoded_addr *res, char *msg, int len, bool scrub_err); +void skx_enable_rrl(bool enable); void skx_set_mem_cfg(bool mem_cfg_2lm); void skx_set_res_cfg(struct res_config *cfg); void skx_init_mc_mapping(struct skx_dev *d); diff --git a/drivers/edac/x38_edac.c b/drivers/edac/x38_edac.c index 292dda754c23..2b55daca33b0 100644 --- a/drivers/edac/x38_edac.c +++ b/drivers/edac/x38_edac.c @@ -446,11 +446,11 @@ static void x38_remove_one(struct pci_dev *pdev) static const struct pci_device_id x38_pci_tbl[] = { { - PCI_VEND_DEV(INTEL, X38_HB), PCI_ANY_ID, PCI_ANY_ID, 0, 0, - X38}, - { - 0, - } /* 0 terminated list. */ + PCI_VEND_DEV(INTEL, X38_HB), + .driver_data = X38, + }, { + /* 0 terminated list. */ + } }; MODULE_DEVICE_TABLE(pci, x38_pci_tbl); diff --git a/drivers/firmware/efi/libstub/x86-5lvl.c b/drivers/firmware/efi/libstub/x86-5lvl.c index c00d0ae7ed5d..c3da05c0df8b 100644 --- a/drivers/firmware/efi/libstub/x86-5lvl.c +++ b/drivers/firmware/efi/libstub/x86-5lvl.c @@ -2,6 +2,7 @@ #include #include +#include #include #include diff --git a/drivers/gpu/drm/gma500/mmu.c b/drivers/gpu/drm/gma500/mmu.c index 6b6b44e426cf..4fbc22a59ac7 100644 --- a/drivers/gpu/drm/gma500/mmu.c +++ b/drivers/gpu/drm/gma500/mmu.c @@ -7,6 +7,8 @@ #include #include +#include + #include "mmu.h" #include "psb_drv.h" #include "psb_reg.h" diff --git a/drivers/hv/mshv_vtl_main.c b/drivers/hv/mshv_vtl_main.c index c19400701467..0d3d4161974f 100644 --- a/drivers/hv/mshv_vtl_main.c +++ b/drivers/hv/mshv_vtl_main.c @@ -596,9 +596,9 @@ static int mshv_vtl_get_set_reg(struct hv_register_assoc *regs, bool set) } else { /* Handle MSRs */ if (set) - wrmsrl(reg_table[i].msr_addr, *reg64); + wrmsrq(reg_table[i].msr_addr, *reg64); else - rdmsrl(reg_table[i].msr_addr, *reg64); + rdmsrq(reg_table[i].msr_addr, *reg64); } return 0; } diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c index 6a0d94711ead..70711a7cca12 100644 --- a/drivers/hwmon/coretemp.c +++ b/drivers/hwmon/coretemp.c @@ -169,7 +169,7 @@ static int adjust_tjmax(struct cpuinfo_x86 *c, u32 id, struct device *dev) int tjmax_ee = 85000; int usemsr_ee = 1; int err; - u32 eax, edx; + u64 val; int i; u16 devfn = PCI_DEVFN(0, 0); struct pci_dev *host_bridge = pci_get_domain_bus_and_slot(0, 0, devfn); @@ -220,14 +220,14 @@ static int adjust_tjmax(struct cpuinfo_x86 *c, u32 id, struct device *dev) * http://softwarecommunity.intel.com/Wiki/Mobility/720.htm * For Core2 cores, check MSR 0x17, bit 28 1 = Mobile CPU */ - err = rdmsr_safe_on_cpu(id, 0x17, &eax, &edx); + err = rdmsrq_safe_on_cpu(id, 0x17, &val); if (err) { dev_warn(dev, "Unable to access MSR 0x17, assuming desktop" " CPU\n"); usemsr_ee = 0; } else if (c->x86_vfm < INTEL_CORE2_PENRYN && - !(eax & 0x10000000)) { + !(val & 0x10000000)) { /* * Trust bit 28 up to Penryn, I could not find any * documentation on that; if you happen to know @@ -235,8 +235,8 @@ static int adjust_tjmax(struct cpuinfo_x86 *c, u32 id, struct device *dev) */ usemsr_ee = 0; } else { - /* Platform ID bits 52:50 (EDX starts at bit 32) */ - platform_id = (edx >> 18) & 0x7; + /* Platform ID bits 52:50 */ + platform_id = (val >> 50) & 0x7; /* * Mobile Penryn CPU seems to be platform ID 7 or 5 @@ -255,12 +255,12 @@ static int adjust_tjmax(struct cpuinfo_x86 *c, u32 id, struct device *dev) } if (usemsr_ee) { - err = rdmsr_safe_on_cpu(id, 0xee, &eax, &edx); + err = rdmsrq_safe_on_cpu(id, 0xee, &val); if (err) { dev_warn(dev, "Unable to access MSR 0xEE, for Tjmax, left" " at default\n"); - } else if (eax & 0x40000000) { + } else if (val & 0x40000000) { tjmax = tjmax_ee; } } else if (tjmax == 100000) { @@ -278,7 +278,7 @@ static int get_tjmax(struct temp_data *tdata, struct device *dev) { struct cpuinfo_x86 *c = &cpu_data(tdata->cpu); int err; - u32 eax, edx; + u64 msrval; u32 val; /* use static tjmax once it is set */ @@ -289,11 +289,11 @@ static int get_tjmax(struct temp_data *tdata, struct device *dev) * A new feature of current Intel(R) processors, the * IA32_TEMPERATURE_TARGET contains the TjMax value */ - err = rdmsr_safe_on_cpu(tdata->cpu, MSR_IA32_TEMPERATURE_TARGET, &eax, &edx); + err = rdmsrq_safe_on_cpu(tdata->cpu, MSR_IA32_TEMPERATURE_TARGET, &msrval); if (err) { dev_warn_once(dev, "Unable to read TjMax from CPU %u\n", tdata->cpu); } else { - val = (eax >> 16) & 0xff; + val = (msrval >> 16) & 0xff; if (val) return val * 1000; } @@ -314,7 +314,7 @@ static int get_tjmax(struct temp_data *tdata, struct device *dev) static int get_ttarget(struct temp_data *tdata, struct device *dev) { - u32 eax, edx; + u64 val; int tjmax, ttarget_offset, ret; /* @@ -324,14 +324,14 @@ static int get_ttarget(struct temp_data *tdata, struct device *dev) if (tdata->tjmax) return -ENODEV; - ret = rdmsr_safe_on_cpu(tdata->cpu, MSR_IA32_TEMPERATURE_TARGET, &eax, &edx); + ret = rdmsrq_safe_on_cpu(tdata->cpu, MSR_IA32_TEMPERATURE_TARGET, &val); if (ret) return ret; - tjmax = (eax >> 16) & 0xff; + tjmax = (val >> 16) & 0xff; /* Read the still undocumented bits 8:15 of IA32_TEMPERATURE_TARGET. */ - ttarget_offset = (eax >> 8) & 0xff; + ttarget_offset = (val >> 8) & 0xff; return (tjmax - ttarget_offset) * 1000; } @@ -356,15 +356,15 @@ static ssize_t show_label(struct device *dev, static ssize_t show_crit_alarm(struct device *dev, struct device_attribute *devattr, char *buf) { - u32 eax, edx; + struct msr val; struct temp_data *tdata = container_of(devattr, struct temp_data, sd_attrs[ATTR_CRIT_ALARM]); mutex_lock(&tdata->update_lock); - rdmsr_on_cpu(tdata->cpu, tdata->status_reg, &eax, &edx); + rdmsrq_on_cpu(tdata->cpu, tdata->status_reg, &val.q); mutex_unlock(&tdata->update_lock); - return sprintf(buf, "%d\n", (eax >> 5) & 1); + return sprintf(buf, "%d\n", (val.l >> 5) & 1); } static ssize_t show_tjmax(struct device *dev, @@ -398,7 +398,7 @@ static ssize_t show_ttarget(struct device *dev, static ssize_t show_temp(struct device *dev, struct device_attribute *devattr, char *buf) { - u32 eax, edx; + struct msr val; struct temp_data *tdata = container_of(devattr, struct temp_data, sd_attrs[ATTR_TEMP]); int tjmax; @@ -407,14 +407,14 @@ static ssize_t show_temp(struct device *dev, tjmax = get_tjmax(tdata, dev); /* Check whether the time interval has elapsed */ if (time_after(jiffies, tdata->last_updated + HZ)) { - rdmsr_on_cpu(tdata->cpu, tdata->status_reg, &eax, &edx); + rdmsrq_on_cpu(tdata->cpu, tdata->status_reg, &val.q); /* * Ignore the valid bit. In all observed cases the register * value is either low or zero if the valid bit is 0. * Return it instead of reporting an error which doesn't * really help at all. */ - tdata->temp = tjmax - ((eax >> 16) & 0xff) * 1000; + tdata->temp = tjmax - ((val.l >> 16) & 0xff) * 1000; tdata->last_updated = jiffies; } @@ -560,7 +560,7 @@ static int create_core_data(struct platform_device *pdev, unsigned int cpu, struct temp_data *tdata; struct platform_data *pdata = platform_get_drvdata(pdev); struct cpuinfo_x86 *c = &cpu_data(cpu); - u32 eax, edx; + u64 val; int err; if (!housekeeping_cpu(cpu, HK_TYPE_MISC)) @@ -571,7 +571,7 @@ static int create_core_data(struct platform_device *pdev, unsigned int cpu, return -ENOMEM; /* Test if we can access the status register */ - err = rdmsr_safe_on_cpu(cpu, tdata->status_reg, &eax, &edx); + err = rdmsrq_safe_on_cpu(cpu, tdata->status_reg, &val); if (err) goto err; diff --git a/drivers/hwmon/fam15h_power.c b/drivers/hwmon/fam15h_power.c index efcbea2d070e..ad4ed4162b57 100644 --- a/drivers/hwmon/fam15h_power.c +++ b/drivers/hwmon/fam15h_power.c @@ -19,6 +19,7 @@ #include #include #include +#include #include MODULE_DESCRIPTION("AMD Family 15h CPU processor power monitor"); diff --git a/drivers/hwmon/k10temp.c b/drivers/hwmon/k10temp.c index a5d8f45b7881..de0760dc597d 100644 --- a/drivers/hwmon/k10temp.c +++ b/drivers/hwmon/k10temp.c @@ -20,7 +20,9 @@ #include #include #include + #include +#include #include MODULE_DESCRIPTION("AMD Family 10h+ CPU core temperature monitor"); diff --git a/drivers/hwmon/k8temp.c b/drivers/hwmon/k8temp.c index 2b80ac410cd1..53241164570e 100644 --- a/drivers/hwmon/k8temp.c +++ b/drivers/hwmon/k8temp.c @@ -15,6 +15,7 @@ #include #include #include +#include #define TEMP_FROM_REG(val) (((((val) >> 16) & 0xff) - 49) * 1000) #define REG_TEMP 0xe4 diff --git a/drivers/hwmon/via-cputemp.c b/drivers/hwmon/via-cputemp.c index a5c03ed59c1f..ec421201049d 100644 --- a/drivers/hwmon/via-cputemp.c +++ b/drivers/hwmon/via-cputemp.c @@ -65,28 +65,28 @@ static ssize_t temp_show(struct device *dev, struct device_attribute *devattr, char *buf) { struct via_cputemp_data *data = dev_get_drvdata(dev); - u32 eax, edx; + u64 val; int err; - err = rdmsr_safe_on_cpu(data->id, data->msr_temp, &eax, &edx); + err = rdmsrq_safe_on_cpu(data->id, data->msr_temp, &val); if (err) return -EAGAIN; - return sprintf(buf, "%lu\n", ((unsigned long)eax & 0xffffff) * 1000); + return sprintf(buf, "%lu\n", ((unsigned long)val & 0xffffff) * 1000); } static ssize_t cpu0_vid_show(struct device *dev, struct device_attribute *devattr, char *buf) { struct via_cputemp_data *data = dev_get_drvdata(dev); - u32 eax, edx; + u64 val; int err; - err = rdmsr_safe_on_cpu(data->id, data->msr_vid, &eax, &edx); + err = rdmsrq_safe_on_cpu(data->id, data->msr_vid, &val); if (err) return -EAGAIN; - return sprintf(buf, "%d\n", vid_from_reg(~edx & 0x7f, data->vrm)); + return sprintf(buf, "%d\n", vid_from_reg(~(val >> 32) & 0x7f, data->vrm)); } static SENSOR_DEVICE_ATTR_RO(temp1_input, temp, SHOW_TEMP); @@ -112,7 +112,7 @@ static int via_cputemp_probe(struct platform_device *pdev) struct via_cputemp_data *data; struct cpuinfo_x86 *c = &cpu_data(pdev->id); int err; - u32 eax, edx; + u64 val; data = devm_kzalloc(&pdev->dev, sizeof(struct via_cputemp_data), GFP_KERNEL); @@ -143,7 +143,7 @@ static int via_cputemp_probe(struct platform_device *pdev) } /* test if we can access the TEMPERATURE MSR */ - err = rdmsr_safe_on_cpu(data->id, data->msr_temp, &eax, &edx); + err = rdmsrq_safe_on_cpu(data->id, data->msr_temp, &val); if (err) { dev_err(&pdev->dev, "Unable to access TEMPERATURE MSR, giving up\n"); diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c index b2705d79a4ee..d74b478db280 100644 --- a/drivers/idle/intel_idle.c +++ b/drivers/idle/intel_idle.c @@ -2428,7 +2428,7 @@ static void intel_c1_demotion_toggle(void *enable) { unsigned long long msr_val; - rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_val); + rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr_val); /* * Enable/disable C1 undemotion along with C1 demotion, as this is the * most sensible configuration in general. @@ -2437,7 +2437,7 @@ static void intel_c1_demotion_toggle(void *enable) msr_val |= NHM_C1_AUTO_DEMOTE | SNB_C1_AUTO_UNDEMOTE; else msr_val &= ~(NHM_C1_AUTO_DEMOTE | SNB_C1_AUTO_UNDEMOTE); - wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_val); + wrmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr_val); } static ssize_t intel_c1_demotion_store(struct device *dev, @@ -2468,7 +2468,7 @@ static ssize_t intel_c1_demotion_show(struct device *dev, * Read the MSR value for a CPU and assume it is the same for all CPUs. Any other * configuration would be a BIOS bug. */ - rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_val); + rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr_val); return sysfs_emit(buf, "%d\n", !!(msr_val & NHM_C1_AUTO_DEMOTE)); } static DEVICE_ATTR_RW(intel_c1_demotion); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c index 7898b5075a8b..b8d467ba6d72 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c @@ -6,6 +6,7 @@ #include #include #include +#include #include "dwmac-intel.h" #include "dwmac4.h" #include "stmmac.h" diff --git a/drivers/nubus/nubus.c b/drivers/nubus/nubus.c index 559dce302d06..40ce4991c356 100644 --- a/drivers/nubus/nubus.c +++ b/drivers/nubus/nubus.c @@ -41,9 +41,7 @@ module_param_named(populate_procfs, nubus_populate_procfs, bool, 0); LIST_HEAD(nubus_func_rsrcs); -static struct device nubus_parent = { - .init_name = "nubus", -}; +static struct device *nubus_parent; /* Meaning of "bytelanes": @@ -833,7 +831,7 @@ static void __init nubus_add_board(int slot, int bytelanes) list_add_tail(&fres->list, &nubus_func_rsrcs); } - if (nubus_device_register(&nubus_parent, board)) + if (nubus_device_register(nubus_parent, board)) put_device(&board->dev); } @@ -880,18 +878,17 @@ static void __init nubus_scan_bus(void) static int __init nubus_init(void) { - int err; - if (!MACH_IS_MAC) return 0; nubus_proc_init(); - err = device_register(&nubus_parent); - if (err) { - put_device(&nubus_parent); - return err; - } + + nubus_parent = root_device_register("nubus"); + if (IS_ERR(nubus_parent)) + return PTR_ERR(nubus_parent); + nubus_scan_bus(); + return 0; } diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig index ab90932fc2d0..245e7bb763b9 100644 --- a/drivers/perf/Kconfig +++ b/drivers/perf/Kconfig @@ -188,7 +188,7 @@ config FUJITSU_UNCORE_PMU monitoring Uncore events. config QCOM_L2_PMU - bool "Qualcomm Technologies L2-cache PMU" + bool "Qualcomm L2-cache PMU" depends on ARCH_QCOM && ARM64 && ACPI select QCOM_KRYO_L2_ACCESSORS help @@ -198,7 +198,7 @@ config QCOM_L2_PMU monitoring L2 cache events. config QCOM_L3_PMU - bool "Qualcomm Technologies L3-cache PMU" + bool "Qualcomm L3-cache PMU" depends on ARCH_QCOM && ARM64 && ACPI select QCOM_IRQ_COMBINER help diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c index f5305c8fdca4..6e5cc4086a9e 100644 --- a/drivers/perf/arm-cmn.c +++ b/drivers/perf/arm-cmn.c @@ -197,13 +197,14 @@ enum cmn_model { CMN600 = 1, CMN650 = 2, - CMN700 = 4, - CI700 = 8, + CI700 = 4, + CMN700 = 8, CMNS3 = 16, /* ...and then we can use bitmap tricks for commonality */ CMN_ANY = -1, NOT_CMN600 = -2, - CMN_650ON = CMN650 | CMN700 | CMNS3, + CMN_700ON = ~(CMN700 - 1), + CMN_650ON = CMN_700ON | CMN650, }; /* Actual part numbers and revision IDs defined by the hardware */ @@ -919,14 +920,14 @@ static struct attribute *arm_cmn_event_attrs[] = { CMN_EVENT_DVM(NOT_CMN600, txsnp_stall, 0x0a), CMN_EVENT_DVM(NOT_CMN600, trkfull, 0x0b), CMN_EVENT_DVM_OCC(NOT_CMN600, trk_occupancy, 0x0c), - CMN_EVENT_DVM_OCC(CMN700, trk_occupancy_cxha, 0x0d), - CMN_EVENT_DVM_OCC(CMN700, trk_occupancy_pdn, 0x0e), - CMN_EVENT_DVM(CMN700, trk_alloc, 0x0f), - CMN_EVENT_DVM(CMN700, trk_cxha_alloc, 0x10), - CMN_EVENT_DVM(CMN700, trk_pdn_alloc, 0x11), - CMN_EVENT_DVM(CMN700, txsnp_stall_limit, 0x12), - CMN_EVENT_DVM(CMN700, rxsnp_stall_starv, 0x13), - CMN_EVENT_DVM(CMN700, txsnp_sync_stall_op, 0x14), + CMN_EVENT_DVM_OCC(CMN_700ON, trk_occupancy_cxha, 0x0d), + CMN_EVENT_DVM_OCC(CMN_700ON, trk_occupancy_pdn, 0x0e), + CMN_EVENT_DVM(CMN_700ON, trk_alloc, 0x0f), + CMN_EVENT_DVM(CMN_700ON, trk_cxha_alloc, 0x10), + CMN_EVENT_DVM(CMN_700ON, trk_pdn_alloc, 0x11), + CMN_EVENT_DVM(CMN_700ON, txsnp_stall_limit, 0x12), + CMN_EVENT_DVM(CMN_700ON, rxsnp_stall_starv, 0x13), + CMN_EVENT_DVM(CMN_700ON, txsnp_sync_stall_op, 0x14), CMN_EVENT_HNF(CMN_ANY, cache_miss, 0x01), CMN_EVENT_HNF(CMN_ANY, slc_sf_cache_access, 0x02), diff --git a/drivers/ras/amd/atl/Kconfig b/drivers/ras/amd/atl/Kconfig index 6e03942cd7da..44c2fd7febc5 100644 --- a/drivers/ras/amd/atl/Kconfig +++ b/drivers/ras/amd/atl/Kconfig @@ -12,7 +12,6 @@ config AMD_ATL depends on AMD_NB && X86_64 && RAS depends on AMD_NODE depends on MEMORY_FAILURE - default N help This library includes support for implementation-specific address translation procedures needed for various error diff --git a/drivers/ras/amd/fmpm.c b/drivers/ras/amd/fmpm.c index 34ef75af31cb..4ccaaf7b70bf 100644 --- a/drivers/ras/amd/fmpm.c +++ b/drivers/ras/amd/fmpm.c @@ -52,6 +52,7 @@ #include #include +#include #include #include "../debugfs.h" diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index 988fc291241d..b69f99488111 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -224,6 +224,24 @@ static inline void _mpam_write_monsel_reg(struct mpam_msc *msc, u16 reg, u32 val #define mpam_write_monsel_reg(msc, reg, val) _mpam_write_monsel_reg(msc, MSMON_##reg, val) +static bool mpam_msc_check_aidr(struct mpam_msc *msc) +{ + u32 aidr = __mpam_read_reg(msc, MPAMF_AIDR); + u32 major = FIELD_GET(MPAMF_AIDR_ARCH_MAJOR_REV, aidr); + u32 minor = FIELD_GET(MPAMF_AIDR_ARCH_MINOR_REV, aidr); + + /* + * v0.0 and >v2.x aren't supported, but anything else should be backward + * compatible to v0.1 or v1.0. + */ + if (!major && !minor) + return false; + if (major > 1) + return false; + + return true; +} + static u64 mpam_msc_read_idr(struct mpam_msc *msc) { u64 idr_high = 0, idr_low; @@ -945,9 +963,8 @@ static int mpam_msc_hw_probe(struct mpam_msc *msc) lockdep_assert_held(&msc->probe_lock); - idr = __mpam_read_reg(msc, MPAMF_AIDR); - if ((idr & MPAMF_AIDR_ARCH_MAJOR_REV) != MPAM_ARCHITECTURE_V1) { - dev_err_once(dev, "MSC does not match MPAM architecture v1.x\n"); + if (!mpam_msc_check_aidr(msc)) { + dev_err_once(dev, "MSC does not match architecture v1.x\n"); return -EIO; } diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index 14e58c336baa..74fe73b5738a 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -5569,7 +5569,7 @@ static void dasd_eckd_dump_sense_ccw(struct dasd_device *device, dev = &device->cdev->dev; - page = (char *) get_zeroed_page(GFP_ATOMIC); + page = kzalloc(PAGE_SIZE, GFP_ATOMIC); if (page == NULL) { DBF_DEV_EVENT(DBF_WARNING, device, "%s", "No memory to dump sense data\n"); @@ -5644,7 +5644,7 @@ static void dasd_eckd_dump_sense_ccw(struct dasd_device *device, } dasd_eckd_dump_ccw_range(device, from, last, page + len); } - free_page((unsigned long) page); + kfree(page); } @@ -5659,7 +5659,7 @@ static void dasd_eckd_dump_sense_tcw(struct dasd_device *device, struct tsb *tsb; u8 *sense, *rcq; - page = (char *) get_zeroed_page(GFP_ATOMIC); + page = kzalloc(PAGE_SIZE, GFP_ATOMIC); if (page == NULL) { DBF_DEV_EVENT(DBF_WARNING, device, " %s", "No memory to dump sense data"); @@ -5759,7 +5759,7 @@ static void dasd_eckd_dump_sense_tcw(struct dasd_device *device, sprintf(page + len, "SORRY - NO TSB DATA AVAILABLE\n"); } dev_err(&device->cdev->dev, "%s", page); - free_page((unsigned long) page); + kfree(page); } static void dasd_eckd_dump_sense(struct dasd_device *device, diff --git a/drivers/s390/block/dasd_eer.c b/drivers/s390/block/dasd_eer.c index 78d66e2711cd..e96d1805b7bb 100644 --- a/drivers/s390/block/dasd_eer.c +++ b/drivers/s390/block/dasd_eer.c @@ -211,7 +211,7 @@ static void dasd_eer_free_buffer_pages(char **buf, int no_pages) int i; for (i = 0; i < no_pages; i++) - free_page((unsigned long) buf[i]); + kfree(buf[i]); } /* @@ -222,7 +222,7 @@ static int dasd_eer_allocate_buffer_pages(char **buf, int no_pages) int i; for (i = 0; i < no_pages; i++) { - buf[i] = (char *) get_zeroed_page(GFP_KERNEL); + buf[i] = kzalloc(PAGE_SIZE, GFP_KERNEL); if (!buf[i]) { dasd_eer_free_buffer_pages(buf, i); return -ENOMEM; diff --git a/drivers/s390/char/con3270.c b/drivers/s390/char/con3270.c index 644d3679748d..c39da2ec22b4 100644 --- a/drivers/s390/char/con3270.c +++ b/drivers/s390/char/con3270.c @@ -880,7 +880,7 @@ static void tty3270_free_view(struct tty3270 *tp) raw3270_request_free(tp->kreset); raw3270_request_free(tp->read); raw3270_request_free(tp->write); - free_page((unsigned long)tp->converted_line); + kfree(tp->converted_line); tty_port_destroy(&tp->port); kfree(tp); } @@ -1063,7 +1063,7 @@ static void tty3270_free(struct raw3270_view *view) timer_delete_sync(&tp->timer); tty3270_free_screen(tp->screen, tp->allocated_lines); - free_page((unsigned long)tp->converted_line); + kfree(tp->converted_line); kfree(tp->input); kfree(tp->prompt); tty3270_free_view(tp); @@ -1121,7 +1121,7 @@ tty3270_create_view(int index, struct tty3270 **newtp) goto out_put_view; } - tp->converted_line = (void *)__get_free_page(GFP_KERNEL); + tp->converted_line = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!tp->converted_line) { rc = -ENOMEM; goto out_free_screen; @@ -1167,7 +1167,7 @@ out_free_prompt: out_free_input: kfree(tp->input); out_free_converted_line: - free_page((unsigned long)tp->converted_line); + kfree(tp->converted_line); out_free_screen: tty3270_free_screen(tp->screen, tp->view.rows); out_put_view: diff --git a/drivers/s390/char/sclp_pci.c b/drivers/s390/char/sclp_pci.c index 899063e64aef..d61a7fc0dd61 100644 --- a/drivers/s390/char/sclp_pci.c +++ b/drivers/s390/char/sclp_pci.c @@ -98,6 +98,7 @@ static int sclp_pci_check_report(struct zpci_report_error_header *report) case SCLP_ERRNOTIFY_AQ_REPAIR: case SCLP_ERRNOTIFY_AQ_INFO_LOG: case SCLP_ERRNOTIFY_AQ_OPTICS_DATA: + case SCLP_ERRNOTIFY_AQ_NVME_SMART_LOG: break; default: return -EINVAL; diff --git a/drivers/s390/char/sclp_vt220.c b/drivers/s390/char/sclp_vt220.c index 62979adcb381..4e2bf8186a5d 100644 --- a/drivers/s390/char/sclp_vt220.c +++ b/drivers/s390/char/sclp_vt220.c @@ -81,9 +81,6 @@ static struct timer_list sclp_vt220_timer; * yet sent */ static struct sclp_vt220_request *sclp_vt220_current_request; -/* Number of characters in current request buffer */ -static int sclp_vt220_buffered_chars; - /* Counter controlling core driver initialization. */ static int __initdata sclp_vt220_init_count; @@ -689,7 +686,6 @@ static int __init __sclp_vt220_init(int num_pages) timer_setup(&sclp_vt220_timer, sclp_vt220_timeout, 0); tty_port_init(&sclp_vt220_port); sclp_vt220_current_request = NULL; - sclp_vt220_buffered_chars = 0; sclp_vt220_flush_later = 0; /* Allocate pages for output buffering */ diff --git a/drivers/s390/char/zcore.c b/drivers/s390/char/zcore.c index b26b5fca6ce8..1ab7400a3c10 100644 --- a/drivers/s390/char/zcore.c +++ b/drivers/s390/char/zcore.c @@ -46,8 +46,6 @@ struct ipib_info { static struct debug_info *zcore_dbf; static int hsa_available; static struct dentry *zcore_dir; -static struct dentry *zcore_reipl_file; -static struct dentry *zcore_hsa_file; static struct ipl_parameter_block *zcore_ipl_block; static unsigned long os_info_flags; @@ -353,10 +351,8 @@ static int __init zcore_init(void) goto fail; zcore_dir = debugfs_create_dir("zcore" , NULL); - zcore_reipl_file = debugfs_create_file("reipl", S_IRUSR, zcore_dir, - NULL, &zcore_reipl_fops); - zcore_hsa_file = debugfs_create_file("hsa", S_IRUSR|S_IWUSR, zcore_dir, - NULL, &zcore_hsa_fops); + debugfs_create_file("reipl", 0400, zcore_dir, NULL, &zcore_reipl_fops); + debugfs_create_file("hsa", 0600, zcore_dir, NULL, &zcore_hsa_fops); register_reboot_notifier(&zcore_reboot_notifier); atomic_notifier_chain_register(&panic_notifier_list, &zcore_on_panic_notifier); diff --git a/drivers/s390/cio/device.c b/drivers/s390/cio/device.c index 5bbb112b1619..fb591118ecb2 100644 --- a/drivers/s390/cio/device.c +++ b/drivers/s390/cio/device.c @@ -1328,7 +1328,7 @@ static int purge_fn(struct subchannel *sch, void *data) cdev = sch_get_cdev(sch); if (cdev) { - if (cdev->private->state != DEV_STATE_OFFLINE) + if (cdev->online) goto unlock; if (atomic_cmpxchg(&cdev->private->onoff, 0, 1) != 0) diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index f24e27add721..6a7497db5fb9 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -744,6 +744,23 @@ void ap_send_online_uevent(struct ap_device *ap_dev, int online) } EXPORT_SYMBOL(ap_send_online_uevent); +void ap_send_se_bind_uevent(struct ap_device *ap_dev) +{ + char *envp[] = { "SE_BIND=1", NULL }; + + kobject_uevent_env(&ap_dev->device.kobj, KOBJ_CHANGE, envp); +} + +void ap_send_se_assoc_uevent(struct ap_device *ap_dev, unsigned int assoc_idx) +{ + char buf[32]; + char *envp[] = { buf, NULL }; + + snprintf(buf, sizeof(buf), "SE_ASSOC=%u", assoc_idx); + + kobject_uevent_env(&ap_dev->device.kobj, KOBJ_CHANGE, envp); +} + static void ap_send_mask_changed_uevent(unsigned long *newapm, unsigned long *newaqm) { diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h index 04ea256ecf91..b2e57e5d6c3f 100644 --- a/drivers/s390/crypto/ap_bus.h +++ b/drivers/s390/crypto/ap_bus.h @@ -134,8 +134,6 @@ struct ap_message; struct ap_driver { struct device_driver driver; - struct ap_device_id *ids; - unsigned int flags; int (*probe)(struct ap_device *); void (*remove)(struct ap_device *); @@ -156,6 +154,9 @@ struct ap_driver { */ void (*on_scan_complete)(struct ap_config_info *new_config_info, struct ap_config_info *old_config_info); + + struct ap_device_id *ids; + unsigned int flags; }; #define to_ap_drv(x) container_of_const((x), struct ap_driver, driver) @@ -173,11 +174,11 @@ struct ap_device { struct ap_card { struct ap_device ap_dev; struct ap_tapq_hwinfo hwinfo; /* TAPQ GR2 content */ - int id; /* AP card number. */ + atomic64_t total_request_count; /* # requests ever for this AP device.*/ unsigned int maxmsgsize; /* AP msg limit for this card */ + int id; /* AP card number. */ bool config; /* configured state */ bool chkstop; /* checkstop state */ - atomic64_t total_request_count; /* # requests ever for this AP device.*/ }; #define TAPQ_CARD_HWINFO_MASK 0xFFFF0000FFFF0F0FUL @@ -190,16 +191,14 @@ struct ap_queue { struct hlist_node hnode; /* Node for the ap_queues hashtable */ struct ap_card *card; /* Ptr to assoc. AP card. */ spinlock_t lock; /* Per device lock. */ + u64 total_request_count; /* # requests ever for this AP device.*/ enum ap_dev_state dev_state; /* queue device state */ - bool config; /* configured state */ - bool chkstop; /* checkstop state */ ap_qid_t qid; /* AP queue id. */ unsigned int se_bstate; /* SE bind state (BS) */ unsigned int assoc_idx; /* SE association index */ int queue_count; /* # messages currently on AP queue. */ int pendingq_count; /* # requests on pendingq list. */ int requestq_count; /* # requests on requestq list. */ - u64 total_request_count; /* # requests ever for this AP device.*/ int request_timeout; /* Request timeout in jiffies. */ struct timer_list timeout; /* Timer for request timeouts. */ struct list_head pendingq; /* List of message sent to AP queue. */ @@ -208,6 +207,8 @@ struct ap_queue { enum ap_sm_state sm_state; /* ap queue state machine state */ int rapq_fbit; /* fbit arg for next rapq invocation */ int last_err_rc; /* last error state response code */ + bool config; /* configured state */ + bool chkstop; /* checkstop state */ }; #define to_ap_queue(x) container_of((x), struct ap_queue, ap_dev.device) @@ -225,12 +226,12 @@ struct ap_message { void *msg; /* Pointer to message buffer. */ size_t len; /* actual msg len in msg buffer */ size_t bufsize; /* allocated msg buffer size */ - u16 flags; /* Flags, see AP_MSG_FLAG_xxx */ - int rc; /* Return code for this message */ - struct ap_response_type response; /* receive is called from tasklet context */ void (*receive)(struct ap_queue *, struct ap_message *, struct ap_message *); + struct ap_response_type response; + int rc; /* Return code for this message */ + u16 flags; /* Flags, see AP_MSG_FLAG_xxx */ }; #define AP_MSG_FLAG_SPECIAL 0x0001 /* flag msg as 'special' with NQAP */ @@ -373,5 +374,7 @@ int ap_wait_apqn_bindings_complete(unsigned long timeout); void ap_send_config_uevent(struct ap_device *ap_dev, bool cfg); void ap_send_online_uevent(struct ap_device *ap_dev, int online); +void ap_send_se_bind_uevent(struct ap_device *ap_dev); +void ap_send_se_assoc_uevent(struct ap_device *ap_dev, unsigned int assoc_idx); #endif /* _AP_BUS_H_ */ diff --git a/drivers/s390/crypto/ap_queue.c b/drivers/s390/crypto/ap_queue.c index ca9819e6f7e7..bc03b2101e39 100644 --- a/drivers/s390/crypto/ap_queue.c +++ b/drivers/s390/crypto/ap_queue.c @@ -478,6 +478,7 @@ static enum ap_sm_wait ap_sm_assoc_wait(struct ap_queue *aq) pr_debug("queue 0x%02x.%04x associated with %u\n", AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid), aq->assoc_idx); + ap_send_se_assoc_uevent(&aq->ap_dev, aq->assoc_idx); return AP_SM_WAIT_NONE; case AP_BS_Q_USABLE_NO_SECURE_KEY: /* association still pending */ @@ -960,34 +961,37 @@ static ssize_t se_bind_store(struct device *dev, __ap_flush_queue(aq); aq->rapq_fbit = 1; _ap_queue_init_state(aq); - rc = count; - goto out; + spin_unlock_bh(&aq->lock); + return count; } + /* lock this ap to have fetch and update in an atomic way */ + spin_lock_bh(&aq->lock); + /* Bind. Check current SE bind state */ status = ap_test_queue(aq->qid, 1, &hwinfo); if (status.response_code) { AP_DBF_WARN("%s RC 0x%02x on tapq(0x%02x.%04x)\n", __func__, status.response_code, AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); - return -EIO; + rc = -EIO; + goto error; } /* Update BS state */ - spin_lock_bh(&aq->lock); aq->se_bstate = hwinfo.bs; if (hwinfo.bs != AP_BS_Q_AVAIL_FOR_BINDING) { AP_DBF_WARN("%s bind attempt with bs %d on queue 0x%02x.%04x\n", __func__, hwinfo.bs, AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); rc = -EINVAL; - goto out; + goto error; } /* Check SM state */ if (aq->sm_state < AP_SM_STATE_IDLE) { rc = -EBUSY; - goto out; + goto error; } /* invoke BAPQ */ @@ -997,7 +1001,7 @@ static ssize_t se_bind_store(struct device *dev, __func__, status.response_code, AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); rc = -EIO; - goto out; + goto error; } aq->assoc_idx = ASSOC_IDX_INVALID; @@ -1008,7 +1012,7 @@ static ssize_t se_bind_store(struct device *dev, __func__, status.response_code, AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); rc = -EIO; - goto out; + goto error; } aq->se_bstate = hwinfo.bs; if (!(hwinfo.bs == AP_BS_Q_USABLE || @@ -1017,15 +1021,19 @@ static ssize_t se_bind_store(struct device *dev, __func__, hwinfo.bs, AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); rc = -EIO; - goto out; + goto error; } /* SE bind was successful */ + spin_unlock_bh(&aq->lock); + AP_DBF_INFO("%s bapq(0x%02x.%04x) success\n", __func__, AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); - rc = count; + ap_send_se_bind_uevent(&aq->ap_dev); -out: + return count; + +error: spin_unlock_bh(&aq->lock); return rc; } @@ -1091,15 +1099,18 @@ static ssize_t se_associate_store(struct device *dev, if (value >= ASSOC_IDX_INVALID) return -EINVAL; + /* lock this ap to have fetch and update in an atomic way */ + spin_lock_bh(&aq->lock); + /* check current SE bind state */ status = ap_test_queue(aq->qid, 1, &hwinfo); if (status.response_code) { AP_DBF_WARN("%s RC 0x%02x on tapq(0x%02x.%04x)\n", __func__, status.response_code, AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); - return -EIO; + rc = -EIO; + goto out; } - spin_lock_bh(&aq->lock); aq->se_bstate = hwinfo.bs; if (hwinfo.bs != AP_BS_Q_USABLE_NO_SECURE_KEY) { AP_DBF_WARN("%s association attempt with bs %d on queue 0x%02x.%04x\n", @@ -1123,17 +1134,16 @@ static ssize_t se_associate_store(struct device *dev, aq->sm_state = AP_SM_STATE_ASSOC_WAIT; aq->assoc_idx = value; ap_wait(ap_sm_event(aq, AP_SM_EVENT_POLL)); + rc = count; break; default: AP_DBF_WARN("%s RC 0x%02x on aapq(0x%02x.%04x)\n", __func__, status.response_code, AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); rc = -EIO; - goto out; + break; } - rc = count; - out: spin_unlock_bh(&aq->lock); return rc; diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index d6a455df228d..f57189c2b839 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -1782,7 +1782,7 @@ int zcrypt_rng_device_add(void) mutex_lock(&zcrypt_rng_mutex); if (zcrypt_rng_device_count == 0) { - zcrypt_rng_buffer = (u32 *)get_zeroed_page(GFP_KERNEL); + zcrypt_rng_buffer = kzalloc(PAGE_SIZE, GFP_KERNEL); if (!zcrypt_rng_buffer) { rc = -ENOMEM; goto out; @@ -1799,7 +1799,7 @@ int zcrypt_rng_device_add(void) return 0; out_free: - free_page((unsigned long)zcrypt_rng_buffer); + kfree(zcrypt_rng_buffer); out: mutex_unlock(&zcrypt_rng_mutex); return rc; @@ -1811,7 +1811,7 @@ void zcrypt_rng_device_remove(void) zcrypt_rng_device_count--; if (zcrypt_rng_device_count == 0) { hwrng_unregister(&zcrypt_rng_dev); - free_page((unsigned long)zcrypt_rng_buffer); + kfree(zcrypt_rng_buffer); } mutex_unlock(&zcrypt_rng_mutex); } diff --git a/drivers/s390/crypto/zcrypt_api.h b/drivers/s390/crypto/zcrypt_api.h index 6ef8850a42df..9f8df809bb85 100644 --- a/drivers/s390/crypto/zcrypt_api.h +++ b/drivers/s390/crypto/zcrypt_api.h @@ -104,33 +104,28 @@ struct zcrypt_card { struct list_head list; /* Device list. */ struct list_head zqueues; /* List of zcrypt queues */ struct kref refcount; /* device refcounting */ - struct ap_card *card; /* The "real" ap card device. */ int online; /* User online/offline */ - - int user_space_type; /* User space device id. */ + struct ap_card *card; /* The "real" ap card device. */ char *type_string; /* User space device name. */ + int user_space_type; /* User space device id. */ int min_mod_size; /* Min number of bits. */ int max_mod_size; /* Max number of bits. */ int max_exp_bit_length; const int *speed_rating; /* Speed idx of crypto ops. */ atomic_t load; /* Utilization of the crypto device */ - int request_count; /* # current requests. */ }; struct zcrypt_queue { struct list_head list; /* Device list. */ struct kref refcount; /* device refcounting */ + int online; /* User online/offline */ struct zcrypt_card *zcard; struct zcrypt_ops *ops; /* Crypto operations. */ struct ap_queue *queue; /* The "real" ap queue device. */ - int online; /* User online/offline */ - - atomic_t load; /* Utilization of the crypto device */ - - int request_count; /* # current requests. */ - struct ap_message reply; /* Per-device reply structure. */ + atomic_t load; /* Utilization of the crypto device */ + int request_count; /* # current requests. */ }; /* transport layer rescanning */ diff --git a/drivers/s390/crypto/zcrypt_ccamisc.h b/drivers/s390/crypto/zcrypt_ccamisc.h index 06507363947b..07bbb1c20022 100644 --- a/drivers/s390/crypto/zcrypt_ccamisc.h +++ b/drivers/s390/crypto/zcrypt_ccamisc.h @@ -235,7 +235,16 @@ int cca_findcard2(u32 *apqns, u32 *nr_apqns, u16 cardnr, u16 domain, /* struct to hold info for each CCA queue */ struct cca_info { - int hwtype; /* one of the defined AP_DEVICE_TYPE_* */ + u8 new_asym_mkvp[16]; /* verify pattern of new asym master key */ + u8 cur_asym_mkvp[16]; /* verify pattern of current asym master key */ + u8 old_asym_mkvp[16]; /* verify pattern of old asym master key */ + u8 new_aes_mkvp[8]; /* truncated sha256 of new aes master key */ + u8 cur_aes_mkvp[8]; /* truncated sha256 of current aes master key */ + u8 old_aes_mkvp[8]; /* truncated sha256 of old aes master key */ + u8 new_apka_mkvp[8]; /* truncated sha256 of new apka master key */ + u8 cur_apka_mkvp[8]; /* truncated sha256 of current apka mk */ + u8 old_apka_mkvp[8]; /* truncated sha256 of old apka mk */ + char serial[9]; /* serial number (8 ascii numbers + 0x00) */ char new_aes_mk_state; /* '1' empty, '2' partially full, '3' full */ char cur_aes_mk_state; /* '1' invalid, '2' valid */ char old_aes_mk_state; /* '1' invalid, '2' valid */ @@ -245,16 +254,7 @@ struct cca_info { char new_asym_mk_state; /* '1' empty, '2' partially full, '3' full */ char cur_asym_mk_state; /* '1' invalid, '2' valid */ char old_asym_mk_state; /* '1' invalid, '2' valid */ - u8 new_aes_mkvp[8]; /* truncated sha256 of new aes master key */ - u8 cur_aes_mkvp[8]; /* truncated sha256 of current aes master key */ - u8 old_aes_mkvp[8]; /* truncated sha256 of old aes master key */ - u8 new_apka_mkvp[8]; /* truncated sha256 of new apka master key */ - u8 cur_apka_mkvp[8]; /* truncated sha256 of current apka mk */ - u8 old_apka_mkvp[8]; /* truncated sha256 of old apka mk */ - u8 new_asym_mkvp[16]; /* verify pattern of new asym master key */ - u8 cur_asym_mkvp[16]; /* verify pattern of current asym master key */ - u8 old_asym_mkvp[16]; /* verify pattern of old asym master key */ - char serial[9]; /* serial number (8 ascii numbers + 0x00) */ + int hwtype; /* one of the defined AP_DEVICE_TYPE_* */ }; /* diff --git a/drivers/s390/crypto/zcrypt_ep11misc.h b/drivers/s390/crypto/zcrypt_ep11misc.h index b5e6fd861815..05006817b6c0 100644 --- a/drivers/s390/crypto/zcrypt_ep11misc.h +++ b/drivers/s390/crypto/zcrypt_ep11misc.h @@ -86,19 +86,19 @@ int ep11_check_aes_key(debug_info_t *dbg, int dbflvl, /* EP11 card info struct */ struct ep11_card_info { + u64 op_mode; /* card operational mode(s) */ + char serial[16]; /* serial number string (16 ascii, no 0x00 !) */ u32 API_ord_nr; /* API ordinal number */ u16 FW_version; /* Firmware major and minor version */ - char serial[16]; /* serial number string (16 ascii, no 0x00 !) */ - u64 op_mode; /* card operational mode(s) */ }; /* EP11 domain info struct */ struct ep11_domain_info { - char cur_wk_state; /* '0' invalid, '1' valid */ - char new_wk_state; /* '0' empty, '1' uncommitted, '2' committed */ + u64 op_mode; /* domain operational mode(s) */ u8 cur_wkvp[32]; /* current wrapping key verification pattern */ u8 new_wkvp[32]; /* new wrapping key verification pattern */ - u64 op_mode; /* domain operational mode(s) */ + char cur_wk_state; /* '0' invalid, '1' valid */ + char new_wk_state; /* '0' empty, '1' uncommitted, '2' committed */ }; /* diff --git a/drivers/s390/net/qeth_core_main.c b/drivers/s390/net/qeth_core_main.c index cf5f760d0e02..20fb0d2e02a9 100644 --- a/drivers/s390/net/qeth_core_main.c +++ b/drivers/s390/net/qeth_core_main.c @@ -3362,9 +3362,9 @@ static int qeth_query_setdiagass(struct qeth_card *card) static void qeth_get_trap_id(struct qeth_card *card, struct qeth_trap_id *tid) { - unsigned long info = get_zeroed_page(GFP_KERNEL); - struct sysinfo_2_2_2 *info222 = (struct sysinfo_2_2_2 *)info; - struct sysinfo_3_2_2 *info322 = (struct sysinfo_3_2_2 *)info; + void *info = kzalloc(PAGE_SIZE, GFP_KERNEL); + struct sysinfo_2_2_2 *info222 = info; + struct sysinfo_3_2_2 *info322 = info; struct ccw_dev_id ccwid; int level; @@ -3381,7 +3381,7 @@ static void qeth_get_trap_id(struct qeth_card *card, struct qeth_trap_id *tid) EBCASC(info322->vm[0].name, sizeof(info322->vm[0].name)); memcpy(tid->vmname, info322->vm[0].name, sizeof(tid->vmname)); } - free_page(info); + kfree(info); } static int qeth_hw_trap_cb(struct qeth_card *card, diff --git a/drivers/thermal/intel/intel_hfi.c b/drivers/thermal/intel/intel_hfi.c index 8c4ae75231f8..3273b8fe3d4d 100644 --- a/drivers/thermal/intel/intel_hfi.c +++ b/drivers/thermal/intel/intel_hfi.c @@ -41,6 +41,7 @@ #include #include +#include #include #include "intel_hfi.h" diff --git a/drivers/thermal/intel/intel_tcc.c b/drivers/thermal/intel/intel_tcc.c index ab61fb122937..59f70bb5ffa5 100644 --- a/drivers/thermal/intel/intel_tcc.c +++ b/drivers/thermal/intel/intel_tcc.c @@ -181,17 +181,17 @@ static u32 get_temp_mask(bool pkg) */ int intel_tcc_get_tjmax(int cpu) { - u32 low, high; + struct msr msrval; int val, err; if (cpu < 0) - err = rdmsr_safe(MSR_IA32_TEMPERATURE_TARGET, &low, &high); + err = rdmsr_safe(MSR_IA32_TEMPERATURE_TARGET, &msrval.l, &msrval.h); else - err = rdmsr_safe_on_cpu(cpu, MSR_IA32_TEMPERATURE_TARGET, &low, &high); + err = rdmsrq_safe_on_cpu(cpu, MSR_IA32_TEMPERATURE_TARGET, &msrval.q); if (err) return err; - val = (low >> 16) & 0xff; + val = (msrval.l >> 16) & 0xff; return val ? val : -ENODATA; } @@ -208,17 +208,17 @@ EXPORT_SYMBOL_NS_GPL(intel_tcc_get_tjmax, "INTEL_TCC"); */ int intel_tcc_get_offset(int cpu) { - u32 low, high; + struct msr val; int err; if (cpu < 0) - err = rdmsr_safe(MSR_IA32_TEMPERATURE_TARGET, &low, &high); + err = rdmsr_safe(MSR_IA32_TEMPERATURE_TARGET, &val.l, &val.h); else - err = rdmsr_safe_on_cpu(cpu, MSR_IA32_TEMPERATURE_TARGET, &low, &high); + err = rdmsrq_safe_on_cpu(cpu, MSR_IA32_TEMPERATURE_TARGET, &val.q); if (err) return err; - return (low >> 24) & intel_tcc_temp_masks.tcc_offset; + return (val.l >> 24) & intel_tcc_temp_masks.tcc_offset; } EXPORT_SYMBOL_NS_GPL(intel_tcc_get_offset, "INTEL_TCC"); @@ -235,7 +235,7 @@ EXPORT_SYMBOL_NS_GPL(intel_tcc_get_offset, "INTEL_TCC"); int intel_tcc_set_offset(int cpu, int offset) { - u32 low, high; + struct msr val; int err; if (!intel_tcc_temp_masks.tcc_offset) @@ -245,23 +245,23 @@ int intel_tcc_set_offset(int cpu, int offset) return -EINVAL; if (cpu < 0) - err = rdmsr_safe(MSR_IA32_TEMPERATURE_TARGET, &low, &high); + err = rdmsr_safe(MSR_IA32_TEMPERATURE_TARGET, &val.l, &val.h); else - err = rdmsr_safe_on_cpu(cpu, MSR_IA32_TEMPERATURE_TARGET, &low, &high); + err = rdmsrq_safe_on_cpu(cpu, MSR_IA32_TEMPERATURE_TARGET, &val.q); if (err) return err; /* MSR Locked */ - if (low & BIT(31)) + if (val.l & BIT(31)) return -EPERM; - low &= ~(intel_tcc_temp_masks.tcc_offset << 24); - low |= offset << 24; + val.l &= ~(intel_tcc_temp_masks.tcc_offset << 24); + val.l |= offset << 24; if (cpu < 0) - return wrmsr_safe(MSR_IA32_TEMPERATURE_TARGET, low, high); + return wrmsr_safe(MSR_IA32_TEMPERATURE_TARGET, val.l, val.h); else - return wrmsr_safe_on_cpu(cpu, MSR_IA32_TEMPERATURE_TARGET, low, high); + return wrmsrq_safe_on_cpu(cpu, MSR_IA32_TEMPERATURE_TARGET, val.q); } EXPORT_SYMBOL_NS_GPL(intel_tcc_set_offset, "INTEL_TCC"); @@ -279,7 +279,8 @@ EXPORT_SYMBOL_NS_GPL(intel_tcc_set_offset, "INTEL_TCC"); int intel_tcc_get_temp(int cpu, int *temp, bool pkg) { u32 msr = pkg ? MSR_IA32_PACKAGE_THERM_STATUS : MSR_IA32_THERM_STATUS; - u32 low, high, mask; + u32 mask; + struct msr val; int tjmax, err; tjmax = intel_tcc_get_tjmax(cpu); @@ -287,19 +288,19 @@ int intel_tcc_get_temp(int cpu, int *temp, bool pkg) return tjmax; if (cpu < 0) - err = rdmsr_safe(msr, &low, &high); + err = rdmsr_safe(msr, &val.l, &val.h); else - err = rdmsr_safe_on_cpu(cpu, msr, &low, &high); + err = rdmsrq_safe_on_cpu(cpu, msr, &val.q); if (err) return err; /* Temperature is beyond the valid thermal sensor range */ - if (!(low & BIT(31))) + if (!(val.l & BIT(31))) return -ENODATA; mask = get_temp_mask(pkg); - *temp = tjmax - ((low >> 16) & mask); + *temp = tjmax - ((val.l >> 16) & mask); return 0; } diff --git a/drivers/thermal/intel/x86_pkg_temp_thermal.c b/drivers/thermal/intel/x86_pkg_temp_thermal.c index 540109761f0a..688e04c63761 100644 --- a/drivers/thermal/intel/x86_pkg_temp_thermal.c +++ b/drivers/thermal/intel/x86_pkg_temp_thermal.c @@ -20,6 +20,7 @@ #include #include +#include #include #include "thermal_interrupt.h" @@ -125,8 +126,9 @@ sys_set_trip_temp(struct thermal_zone_device *tzd, { struct zone_device *zonedev = thermal_zone_device_priv(tzd); unsigned int trip_index = THERMAL_TRIP_PRIV_TO_INT(trip->priv); - u32 l, h, mask, shift, intr; + u32 mask, shift, intr; int tj_max, val, ret; + struct msr v; if (temp == THERMAL_TEMP_INVALID) temp = 0; @@ -141,8 +143,7 @@ sys_set_trip_temp(struct thermal_zone_device *tzd, if (trip_index >= MAX_NUMBER_OF_TRIPS || val < 0 || val > 0x7f) return -EINVAL; - ret = rdmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, - &l, &h); + ret = rdmsrq_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, &v.q); if (ret < 0) return ret; @@ -155,20 +156,19 @@ sys_set_trip_temp(struct thermal_zone_device *tzd, shift = THERM_SHIFT_THRESHOLD0; intr = THERM_INT_THRESHOLD0_ENABLE; } - l &= ~mask; + v.l &= ~mask; /* * When users space sets a trip temperature == 0, which is indication * that, it is no longer interested in receiving notifications. */ if (!temp) { - l &= ~intr; + v.l &= ~intr; } else { - l |= val << shift; - l |= intr; + v.l |= val << shift; + v.l |= intr; } - return wrmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, - l, h); + return wrmsrq_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, v.q); } /* Thermal zone callback registry */ @@ -277,7 +277,8 @@ static int pkg_temp_thermal_trips_init(int cpu, int tj_max, struct thermal_trip *trips, int num_trips) { unsigned long thres_reg_value; - u32 mask, shift, eax, edx; + u32 mask, shift; + struct msr val; int ret, i; for (i = 0; i < num_trips; i++) { @@ -290,12 +291,11 @@ static int pkg_temp_thermal_trips_init(int cpu, int tj_max, shift = THERM_SHIFT_THRESHOLD0; } - ret = rdmsr_on_cpu(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, - &eax, &edx); + ret = rdmsrq_on_cpu(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, &val.q); if (ret < 0) return ret; - thres_reg_value = (eax & mask) >> shift; + thres_reg_value = (val.l & mask) >> shift; trips[i].temperature = thres_reg_value ? tj_max - thres_reg_value * 1000 : THERMAL_TEMP_INVALID; diff --git a/drivers/tty/hvc/hvc_iucv.c b/drivers/tty/hvc/hvc_iucv.c index 37db8a3e5158..d29a86d161f6 100644 --- a/drivers/tty/hvc/hvc_iucv.c +++ b/drivers/tty/hvc/hvc_iucv.c @@ -1060,7 +1060,7 @@ static int __init hvc_iucv_alloc(int id, unsigned int is_console) INIT_DELAYED_WORK(&priv->sndbuf_work, hvc_iucv_sndbuf_work); init_waitqueue_head(&priv->sndbuf_waitq); - priv->sndbuf = (void *) get_zeroed_page(GFP_KERNEL); + priv->sndbuf = kzalloc(PAGE_SIZE, GFP_KERNEL); if (!priv->sndbuf) { kfree(priv); return -ENOMEM; @@ -1103,7 +1103,7 @@ static int __init hvc_iucv_alloc(int id, unsigned int is_console) out_error_dev: hvc_remove(priv->hvc); out_error_hvc: - free_page((unsigned long) priv->sndbuf); + kfree(priv->sndbuf); kfree(priv); return rc; @@ -1116,7 +1116,7 @@ static void __init hvc_iucv_destroy(struct hvc_iucv_private *priv) { hvc_remove(priv->hvc); device_unregister(priv->dev); - free_page((unsigned long) priv->sndbuf); + kfree(priv->sndbuf); kfree(priv); } diff --git a/drivers/virt/acrn/hsm.c b/drivers/virt/acrn/hsm.c index 74f2086fa59f..f170ff4617fd 100644 --- a/drivers/virt/acrn/hsm.c +++ b/drivers/virt/acrn/hsm.c @@ -16,6 +16,7 @@ #include #include +#include #include #include "acrn_drv.h" diff --git a/drivers/watchdog/Kconfig b/drivers/watchdog/Kconfig index dc78729ba2a5..6a9695a16d5e 100644 --- a/drivers/watchdog/Kconfig +++ b/drivers/watchdog/Kconfig @@ -1547,7 +1547,7 @@ config NV_TCO config RDC321X_WDT tristate "RDC R-321x SoC watchdog" - depends on X86_RDC321X || COMPILE_TEST + depends on X86_32 || COMPILE_TEST depends on PCI help This is the driver for the built in hardware watchdog diff --git a/drivers/xen/events/events_base.c b/drivers/xen/events/events_base.c index bc9a41662efc..6ea945508a89 100644 --- a/drivers/xen/events/events_base.c +++ b/drivers/xen/events/events_base.c @@ -40,6 +40,7 @@ #include #ifdef CONFIG_X86 +#include #include #include #include diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c index a6abf1ccd54c..35f879dc5dfb 100644 --- a/drivers/xen/grant-table.c +++ b/drivers/xen/grant-table.c @@ -59,6 +59,7 @@ #include #include #ifdef CONFIG_X86 +#include #include #endif #include diff --git a/drivers/xen/xenbus/xenbus_xs.c b/drivers/xen/xenbus/xenbus_xs.c index 82b0a34ded70..c202e7c553a6 100644 --- a/drivers/xen/xenbus/xenbus_xs.c +++ b/drivers/xen/xenbus/xenbus_xs.c @@ -47,6 +47,9 @@ #include #include #include +#ifdef CONFIG_X86 +#include +#endif #include #include #include "xenbus.h" diff --git a/drivers/zorro/zorro-sysfs.c b/drivers/zorro/zorro-sysfs.c index 4e967754d8ad..9989240fb76b 100644 --- a/drivers/zorro/zorro-sysfs.c +++ b/drivers/zorro/zorro-sysfs.c @@ -15,6 +15,7 @@ #include #include #include +#include #include @@ -29,7 +30,7 @@ static ssize_t name##_show(struct device *dev, \ struct zorro_dev *z; \ \ z = to_zorro_dev(dev); \ - return sprintf(buf, format_string, z->field); \ + return sysfs_emit(buf, format_string, z->field); \ } \ static DEVICE_ATTR_RO(name); @@ -44,7 +45,7 @@ static ssize_t serial_show(struct device *dev, struct device_attribute *attr, struct zorro_dev *z; z = to_zorro_dev(dev); - return sprintf(buf, "0x%08x\n", be32_to_cpu(z->rom.er_SerialNumber)); + return sysfs_emit(buf, "0x%08x\n", be32_to_cpu(z->rom.er_SerialNumber)); } static DEVICE_ATTR_RO(serial); @@ -53,10 +54,10 @@ static ssize_t resource_show(struct device *dev, struct device_attribute *attr, { struct zorro_dev *z = to_zorro_dev(dev); - return sprintf(buf, "0x%08lx 0x%08lx 0x%08lx\n", - (unsigned long)zorro_resource_start(z), - (unsigned long)zorro_resource_end(z), - zorro_resource_flags(z)); + return sysfs_emit(buf, "0x%08lx 0x%08lx 0x%08lx\n", + (unsigned long)zorro_resource_start(z), + (unsigned long)zorro_resource_end(z), + zorro_resource_flags(z)); } static DEVICE_ATTR_RO(resource); @@ -65,7 +66,7 @@ static ssize_t modalias_show(struct device *dev, struct device_attribute *attr, { struct zorro_dev *z = to_zorro_dev(dev); - return sprintf(buf, ZORRO_DEVICE_MODALIAS_FMT "\n", z->id); + return sysfs_emit(buf, ZORRO_DEVICE_MODALIAS_FMT "\n", z->id); } static DEVICE_ATTR_RO(modalias); diff --git a/fs/resctrl/internal.h b/fs/resctrl/internal.h index 1a9b29119f88..48af75b9dc85 100644 --- a/fs/resctrl/internal.h +++ b/fs/resctrl/internal.h @@ -408,6 +408,8 @@ void __check_limbo(struct rdt_l3_mon_domain *d, bool force_free); void resctrl_file_fflags_init(const char *config, unsigned long fflags); +void resctrl_file_mode_init(const char *config, umode_t mode); + void rdt_staged_configs_clear(void); bool closid_allocated(unsigned int closid); diff --git a/fs/resctrl/monitor.c b/fs/resctrl/monitor.c index 9fd901c78dc6..0e6a389a16bf 100644 --- a/fs/resctrl/monitor.c +++ b/fs/resctrl/monitor.c @@ -1211,9 +1211,10 @@ static int rdtgroup_alloc_assign_cntr(struct rdt_resource *r, struct rdt_l3_mon_ * NULL; otherwise, assign the counter to the specified domain @d. * * If all counters in a domain are already in use, rdtgroup_alloc_assign_cntr() - * will fail. The assignment process will abort at the first failure encountered - * during domain traversal, which may result in the event being only partially - * assigned. + * will fail. When attempting to assign counters to all domains, carry on trying + * to assign counters after a failure since only some domains may have counters + * and the goal is to assign counters where possible. If any counter assignment + * fails, return the error from the last failing assignment. * * Return: * 0 on success, < 0 on failure. @@ -1226,9 +1227,11 @@ static int rdtgroup_assign_cntr_event(struct rdt_l3_mon_domain *d, struct rdtgro if (!d) { list_for_each_entry(d, &r->mon_domains, hdr.list) { - ret = rdtgroup_alloc_assign_cntr(r, d, rdtgrp, mevt); - if (ret) - return ret; + int err; + + err = rdtgroup_alloc_assign_cntr(r, d, rdtgrp, mevt); + if (err) + ret = err; } } else { ret = rdtgroup_alloc_assign_cntr(r, d, rdtgrp, mevt); @@ -1422,6 +1425,11 @@ ssize_t event_filter_write(struct kernfs_open_file *of, char *buf, size_t nbytes ret = -EINVAL; goto out_unlock; } + if (!r->mon.mbm_cntr_configurable) { + rdt_last_cmd_puts("event_filter is not configurable\n"); + ret = -EPERM; + goto out_unlock; + } ret = resctrl_parse_mem_transactions(buf, &evt_cfg); if (!ret && mevt->evt_cfg != evt_cfg) { @@ -1451,7 +1459,7 @@ int resctrl_mbm_assign_mode_show(struct kernfs_open_file *of, else seq_puts(s, "[default]\n"); - if (!IS_ENABLED(CONFIG_RESCTRL_ASSIGN_FIXED)) { + if (!r->mon.mbm_cntr_assign_fixed) { if (enabled) seq_puts(s, "default\n"); else @@ -1502,6 +1510,12 @@ ssize_t resctrl_mbm_assign_mode_write(struct kernfs_open_file *of, char *buf, } if (enable != resctrl_arch_mbm_cntr_assign_enabled(r)) { + if (r->mon.mbm_cntr_assign_fixed) { + ret = -EINVAL; + rdt_last_cmd_puts("Counter assignment mode is not configurable\n"); + goto out_unlock; + } + ret = resctrl_arch_mbm_cntr_assign_set(r, enable); if (ret) goto out_unlock; @@ -1886,6 +1900,8 @@ int resctrl_l3_mon_resource_init(void) resctrl_file_fflags_init("available_mbm_cntrs", RFTYPE_MON_INFO | RFTYPE_RES_CACHE); resctrl_file_fflags_init("event_filter", RFTYPE_ASSIGN_CONFIG); + if (r->mon.mbm_cntr_configurable) + resctrl_file_mode_init("event_filter", 0644); resctrl_file_fflags_init("mbm_assign_on_mkdir", RFTYPE_MON_INFO | RFTYPE_RES_CACHE); resctrl_file_fflags_init("mbm_L3_assignments", RFTYPE_MON_BASE); diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c index 5dfdaa6f9d8f..af2cbab14497 100644 --- a/fs/resctrl/rdtgroup.c +++ b/fs/resctrl/rdtgroup.c @@ -2022,7 +2022,7 @@ static struct rftype res_common_files[] = { }, { .name = "event_filter", - .mode = 0644, + .mode = 0444, .kf_ops = &rdtgroup_kf_single_ops, .seq_show = event_filter_show, .write = event_filter_write, @@ -2215,6 +2215,15 @@ void resctrl_file_fflags_init(const char *config, unsigned long fflags) rft->fflags = fflags; } +void resctrl_file_mode_init(const char *config, umode_t mode) +{ + struct rftype *rft; + + rft = rdtgroup_get_rftype_by_name(config); + if (rft) + rft->mode = mode; +} + /** * rdtgroup_kn_mode_restrict - Restrict user access to named resctrl file * @r: The resource group with which the file is associated. @@ -2331,22 +2340,19 @@ static int resctrl_mkdir_event_configs(struct rdt_resource *r, struct kernfs_nod continue; kn_subdir2 = kernfs_create_dir(kn_subdir, mevt->name, kn_subdir->mode, mevt); - if (IS_ERR(kn_subdir2)) { - ret = PTR_ERR(kn_subdir2); - goto out; - } + if (IS_ERR(kn_subdir2)) + return PTR_ERR(kn_subdir2); ret = rdtgroup_kn_set_ugid(kn_subdir2); if (ret) - goto out; + return ret; ret = rdtgroup_add_files(kn_subdir2, RFTYPE_ASSIGN_CONFIG); if (ret) - break; + return ret; } -out: - return ret; + return 0; } static int rdtgroup_mkdir_info_resdir(void *priv, char *name, @@ -2510,10 +2516,13 @@ static void mba_sc_domain_destroy(struct rdt_resource *r, } /* - * MBA software controller is supported only if - * MBM is supported and MBA is in linear scale, - * and the MBM monitor scope is the same as MBA - * control scope. + * The MBA software controller is supported only if MBM is supported and MBA is + * in linear scale, and the MBM monitor scope is the same as MBA control scope. + * + * The software controller cannot be supported when the MBM counters are + * assignable. There is no guarantee that MBM counters are assigned to the + * event backing the software controller in all monitoring domains of all + * monitoring groups. */ static bool supports_mba_mbps(void) { @@ -2522,7 +2531,8 @@ static bool supports_mba_mbps(void) return (resctrl_is_mbm_enabled() && r->alloc_capable && is_mba_linear() && - r->ctrl_scope == rmbm->mon_scope); + r->ctrl_scope == rmbm->mon_scope && + !rmbm->mon.mbm_cntr_assignable); } /* @@ -2937,7 +2947,7 @@ static int rdt_parse_param(struct fs_context *fc, struct fs_parameter *param) ctx->enable_cdpl2 = true; return 0; case Opt_mba_mbps: - msg = "mba_MBps requires MBM and linear scale MBA at L3 scope"; + msg = "mba_MBps requires MBM (mbm_event mode not supported) and linear scale MBA at L3 scope"; if (!supports_mba_mbps()) return invalfc(fc, msg); ctx->enable_mba_mbps = true; diff --git a/fs/udf/super.c b/fs/udf/super.c index cad2e15d633b..be4085da2104 100644 --- a/fs/udf/super.c +++ b/fs/udf/super.c @@ -2321,7 +2321,7 @@ static int udf_fill_super(struct super_block *sb, struct fs_context *fc) error_out: iput(sbi->s_vat_inode); - unload_nls(uopt->nls_map); + unload_nls(sbi->s_nls_map); if (lvid_open) udf_close_lvid(sb); brelse(sbi->s_lvid_bh); diff --git a/include/linux/dio.h b/include/linux/dio.h index 464331c4c4a7..894e50b91792 100644 --- a/include/linux/dio.h +++ b/include/linux/dio.h @@ -112,9 +112,7 @@ struct dio_driver { #define DIOII_END 0x20000000 /* end of DIO-II space */ #define DIOII_DEVSIZE 0x00400000 /* size of a DIO-II device */ -/* Highest valid select code. If we add DIO-II support this should become - * 256 for everything except HP320, which only has DIO. - */ +/* highest valid select code */ #define DIO_SCMAX (hp300_model == HP_320 ? 32 : 256) #define DIOII_SCBASE 132 /* lowest DIO-II select code */ #define DIO_SCINHOLE(scode) (((scode) >= 32) && ((scode) < DIOII_SCBASE)) diff --git a/include/linux/edac.h b/include/linux/edac.h index deba46b3ee25..e6b4e51130e5 100644 --- a/include/linux/edac.h +++ b/include/linux/edac.h @@ -184,6 +184,7 @@ static inline char *mc_event_error_type(const unsigned int err_type) * @MEM_DDR5: Unbuffered DDR5 RAM * @MEM_RDDR5: Registered DDR5 RAM * @MEM_LRDDR5: Load-Reduced DDR5 memory. + * @MEM_LPDDR5: Low-Power DDR5 memory. * @MEM_NVDIMM: Non-volatile RAM * @MEM_WIO2: Wide I/O 2. * @MEM_HBM2: High bandwidth Memory Gen 2. @@ -216,6 +217,7 @@ enum mem_type { MEM_DDR5, MEM_RDDR5, MEM_LRDDR5, + MEM_LPDDR5, MEM_NVDIMM, MEM_WIO2, MEM_HBM2, @@ -247,6 +249,7 @@ enum mem_type { #define MEM_FLAG_DDR5 BIT(MEM_DDR5) #define MEM_FLAG_RDDR5 BIT(MEM_RDDR5) #define MEM_FLAG_LRDDR5 BIT(MEM_LRDDR5) +#define MEM_FLAG_LPDDR5 BIT(MEM_LPDDR5) #define MEM_FLAG_NVDIMM BIT(MEM_NVDIMM) #define MEM_FLAG_WIO2 BIT(MEM_WIO2) #define MEM_FLAG_HBM2 BIT(MEM_HBM2) diff --git a/include/linux/linkage.h b/include/linux/linkage.h index b11660b706c5..53fe1f48fd28 100644 --- a/include/linux/linkage.h +++ b/include/linux/linkage.h @@ -39,6 +39,10 @@ #define __page_aligned_data __section(".data..page_aligned") __aligned(PAGE_SIZE) #define __page_aligned_bss __section(".bss..page_aligned") __aligned(PAGE_SIZE) +#ifndef __bss_pgtbl +#define __bss_pgtbl __page_aligned_bss +#endif + /* * For assembly routines. * diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 07483ed9b3ce..eca14547f9c1 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -2027,7 +2027,7 @@ static inline unsigned long zero_pfn(unsigned long addr) return zero_page_pfn; } -extern uint8_t empty_zero_page[PAGE_SIZE]; +extern const uint8_t empty_zero_page[PAGE_SIZE]; extern struct page *__zero_page; static inline struct page *_zero_page(unsigned long addr) diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h index 006e57fd7ca5..73ff522448a0 100644 --- a/include/linux/resctrl.h +++ b/include/linux/resctrl.h @@ -286,13 +286,15 @@ enum resctrl_schema_fmt { /** * struct resctrl_mon - Monitoring related data of a resctrl resource. - * @num_rmid: Number of RMIDs available. - * @mbm_cfg_mask: Memory transactions that can be tracked when bandwidth - * monitoring events can be configured. - * @num_mbm_cntrs: Number of assignable counters. - * @mbm_cntr_assignable:Is system capable of supporting counter assignment? - * @mbm_assign_on_mkdir:True if counters should automatically be assigned to MBM - * events of monitor groups created via mkdir. + * @num_rmid: Number of RMIDs available. + * @mbm_cfg_mask: Memory transactions that can be tracked when + * bandwidth monitoring events can be configured. + * @num_mbm_cntrs: Number of assignable counters. + * @mbm_cntr_assignable: Is system capable of supporting counter assignment? + * @mbm_assign_on_mkdir: True if counters should automatically be assigned to MBM + * events of monitor groups created via mkdir. + * @mbm_cntr_configurable: True if assignable counters are configurable. + * @mbm_cntr_assign_fixed: True if the counter assignment mode is fixed. */ struct resctrl_mon { u32 num_rmid; @@ -300,6 +302,8 @@ struct resctrl_mon { int num_mbm_cntrs; bool mbm_cntr_assignable; bool mbm_assign_on_mkdir; + bool mbm_cntr_configurable; + bool mbm_cntr_assign_fixed; }; /** diff --git a/mm/kasan/init.c b/mm/kasan/init.c index 9c880f607c6a..66a883887987 100644 --- a/mm/kasan/init.c +++ b/mm/kasan/init.c @@ -26,10 +26,10 @@ * - Latter it reused it as zero shadow to cover large ranges of memory * that allowed to access, but not handled by kasan (vmalloc/vmemmap ...). */ -unsigned char kasan_early_shadow_page[PAGE_SIZE] __page_aligned_bss; +unsigned char kasan_early_shadow_page[PAGE_SIZE] __bss_pgtbl; #if CONFIG_PGTABLE_LEVELS > 4 -p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D] __page_aligned_bss; +p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D] __bss_pgtbl; static inline bool kasan_p4d_table(pgd_t pgd) { return pgd_page(pgd) == virt_to_page(lm_alias(kasan_early_shadow_p4d)); @@ -41,7 +41,7 @@ static inline bool kasan_p4d_table(pgd_t pgd) } #endif #if CONFIG_PGTABLE_LEVELS > 3 -pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD] __page_aligned_bss; +pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD] __bss_pgtbl; static inline bool kasan_pud_table(p4d_t p4d) { return p4d_page(p4d) == virt_to_page(lm_alias(kasan_early_shadow_pud)); @@ -53,7 +53,7 @@ static inline bool kasan_pud_table(p4d_t p4d) } #endif #if CONFIG_PGTABLE_LEVELS > 2 -pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD] __page_aligned_bss; +pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD] __bss_pgtbl; static inline bool kasan_pmd_table(pud_t pud) { return pud_page(pud) == virt_to_page(lm_alias(kasan_early_shadow_pmd)); @@ -65,7 +65,7 @@ static inline bool kasan_pmd_table(pud_t pud) } #endif pte_t kasan_early_shadow_pte[MAX_PTRS_PER_PTE + PTE_HWTABLE_PTRS] - __page_aligned_bss; + __bss_pgtbl; static inline bool kasan_pte_table(pmd_t pmd) { diff --git a/mm/mm_init.c b/mm/mm_init.c index f9f8e1af921c..46cf001238c5 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -57,7 +57,7 @@ unsigned long zero_page_pfn __ro_after_init; EXPORT_SYMBOL(zero_page_pfn); #ifndef __HAVE_COLOR_ZERO_PAGE -uint8_t empty_zero_page[PAGE_SIZE] __page_aligned_bss; +const uint8_t empty_zero_page[PAGE_SIZE] __aligned(PAGE_SIZE); EXPORT_SYMBOL(empty_zero_page); struct page *__zero_page __ro_after_init; diff --git a/rust/Makefile b/rust/Makefile index 63b1e355321d..a870d1616c71 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -451,6 +451,7 @@ BINDGEN_TARGET_x86 := x86_64-linux-gnu BINDGEN_TARGET_arm64 := aarch64-linux-gnu BINDGEN_TARGET_arm := arm-linux-gnueabi BINDGEN_TARGET_loongarch := loongarch64-linux-gnusf +BINDGEN_TARGET_s390 := s390x-linux-gnu # This is only for i386 UM builds, which need the 32-bit target not -m32 BINDGEN_TARGET_i386 := i386-linux-gnu BINDGEN_TARGET_um := $(BINDGEN_TARGET_$(SUBARCH)) diff --git a/rust/bindgen_parameters b/rust/bindgen_parameters index 6f02d9720ad2..8402b0c93545 100644 --- a/rust/bindgen_parameters +++ b/rust/bindgen_parameters @@ -14,6 +14,13 @@ --opaque-type alt_instr --opaque-type x86_msi_data --opaque-type x86_msi_addr_lo +# s390-only: same packed/align issue as above (E0588). +--opaque-type lowcore +--opaque-type tod_clock +--opaque-type tpi_info +--opaque-type uv_cb.* +--opaque-type uv_secret.* +--opaque-type zpci_fib # If SMP is disabled, `arch_spinlock_t` is defined as a ZST which triggers a Rust # warning. We don't need to peek into it anyway. diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 29f345b5745d..ac413625af16 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -91,6 +91,7 @@ #include "signal.c" #include "slab.c" #include "spinlock.c" +#include "string.c" #include "sync.c" #include "task.c" #include "time.c" diff --git a/rust/helpers/string.c b/rust/helpers/string.c new file mode 100644 index 000000000000..8ef30eb07a15 --- /dev/null +++ b/rust/helpers/string.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void *rust_helper_memchr(const void *s, int c, size_t n) +{ + return memchr(s, c, n); +} diff --git a/scripts/generate_rust_target.rs b/scripts/generate_rust_target.rs index 16f7e855e012..3bf296581a88 100644 --- a/scripts/generate_rust_target.rs +++ b/scripts/generate_rust_target.rs @@ -260,6 +260,8 @@ fn main() { } } else if cfg.has("LOONGARCH") { panic!("loongarch uses the builtin rustc loongarch64-unknown-none-softfloat target"); + } else if cfg.has("S390") { + panic!("s390 uses the builtin rustc s390x-unknown-none-softfloat target"); } else { panic!("Unsupported architecture"); } diff --git a/scripts/min-tool-version.sh b/scripts/min-tool-version.sh index ea2689bc9641..031f2192b390 100755 --- a/scripts/min-tool-version.sh +++ b/scripts/min-tool-version.sh @@ -31,7 +31,11 @@ llvm) fi ;; rustc) - echo 1.85.0 + if [ "$SRCARCH" = "s390" ]; then + echo 1.96.0 + else + echo 1.85.0 + fi ;; bindgen) echo 0.71.1 diff --git a/scripts/update-intel-ucode-defs.py b/scripts/update-intel-ucode-defs.py new file mode 100755 index 000000000000..9d6cc2c6075f --- /dev/null +++ b/scripts/update-intel-ucode-defs.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +import argparse +import re +import shutil +import subprocess +import sys +import os + +script = os.path.relpath(__file__) + +DESCRIPTION = f""" +For Intel CPUs, update the microcode revisions that determine +X86_BUG_OLD_MICROCODE. + +This script is intended to be run in response to releases of the +official Intel microcode GitHub repository: +https://github.com/intel/Intel-Linux-Processor-Microcode-Data-Files.git + +It takes the Intel microcode files as input and uses iucode-tool to +extract the revision information. It prints the output in the format +expected by intel-ucode-defs.h. + +Usage: + ./{script} /path/to/microcode/files > /path/to/intel-ucode-defs.h + +Typically, someone at Intel would see a new public release, wait for at +least three months to ensure the update is stable, run this script to +refresh the intel-ucode-defs.h file, and send a patch upstream to update +the mainline and stable versions. + +Any exception to this process should be supported with an appropriate +justification. +""" + +SIG_RE = re.compile(r'sig (0x[0-9a-fA-F]+)') +PFM_RE = re.compile(r'pf_mask (0x[0-9a-fA-F]+)') +REV_RE = re.compile(r'rev (0x[0-9a-fA-F]+)') + +# Functions to extract family, model, and stepping +def bits(val, bottom, top): + mask = (1 << (top + 1 - bottom)) - 1 + return (val >> bottom) & mask + +def family(sig): + if bits(sig, 8, 11) == 0xf: + return bits(sig, 8, 11) + bits(sig, 20, 27) + return bits(sig, 8, 11) + +def model(sig): + return bits(sig, 4, 7) | (bits(sig, 16, 19) << 4) + +def step(sig): + return bits(sig, 0, 3) + +class Ucode: + def __init__(self, sig, pfm, rev): + self.family = family(sig) + self.model = model(sig) + self.steppings = 1 << step(sig) + self.platforms = pfm + self.rev = rev + + self.key = (self.family, self.model, self.steppings, self.platforms) + + def __eq__(self, other): + return self.key == other.key + + def __hash__(self): + return hash(self.key) + + def __str__(self): + return "{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x%x, .model = 0x%02x, .steppings = 0x%04x, .platform_mask = 0x%02x, .driver_data = 0x%x }," % \ + (self.family, self.model, self.steppings, self.platforms, self.rev) + +def main(): + parser = argparse.ArgumentParser(description=DESCRIPTION, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('ucode_files', nargs='+', help='Path(s) to the microcode files') + + args = parser.parse_args() + + # Process the microcode files using iucode-tool + iucode_tool = shutil.which("iucode-tool") or shutil.which("iucode_tool") + if iucode_tool is None: + print("Error: iucode-tool not found, please install it", file=sys.stderr) + sys.exit(1) + + cmd = [iucode_tool, '--list-all'] + args.ucode_files + + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print("Error: iucode-tool ran into an error, exiting", file=sys.stderr) + if result.stderr: + print(result.stderr, file=sys.stderr, end='') + sys.exit(1) + + ucodes = set() + + # Parse the output of iucode-tool + for line in result.stdout.splitlines(): + sig_match = SIG_RE.search(line) + pfm_match = PFM_RE.search(line) + rev_match = REV_RE.search(line) + + if not (sig_match and pfm_match and rev_match): + continue + + sig = int(sig_match.group(1), 16) + pfm = int(pfm_match.group(1), 16) + rev = int(rev_match.group(1), 16) + debug_rev = bits(rev, 31, 31) + if debug_rev != 0: + print("Error: Debug ucode file found, exiting", file=sys.stderr) + sys.exit(1) + + ucodes.add(Ucode(sig, pfm, rev)) + + if not ucodes: + print("Error: No valid microcode files found, exiting", file=sys.stderr) + sys.exit(1) + + # Sort and print the microcode entries + print("/* SPDX-License-Identifier: GPL-2.0 */") + print("/* Auto-generated by scripts/update-intel-ucode-defs.py */") + for u in sorted(ucodes, key=lambda x: x.key): + print(u) + +if __name__ == "__main__": + main() diff --git a/tools/arch/x86/kcpuid/cpuid.csv b/tools/arch/x86/kcpuid/cpuid.csv index 9f5155c825ca..45a876d3519f 100644 --- a/tools/arch/x86/kcpuid/cpuid.csv +++ b/tools/arch/x86/kcpuid/cpuid.csv @@ -1,5 +1,5 @@ # SPDX-License-Identifier: CC0-1.0 -# Generator: x86-cpuid-db v3.0 +# Generator: x86-cpuid-db v3.1 # # Auto-generated file. @@ -177,7 +177,7 @@ 0x6, 0, ebx, 3:0, n_therm_thresholds , Digital thermometer thresholds 0x6, 0, ecx, 0, aperfmperf , MPERF/APERF MSRs (effective frequency interface) 0x6, 0, ecx, 3, epb , IA32_ENERGY_PERF_BIAS MSR - 0x6, 0, ecx, 15:8, thrd_director_nclasses , Number of classes, Intel thread director + 0x6, 0, ecx, 15:8, hw_feedback_nclasses , Number of Intel Thread Director classes 0x6, 0, edx, 0, perfcap_reporting , Performance capability reporting 0x6, 0, edx, 1, encap_reporting , Energy efficiency capability reporting 0x6, 0, edx, 11:8, feedback_sz , Feedback interface structure size, in 4K pages @@ -247,10 +247,10 @@ 0x7, 0, edx, 1, sgx_keys , Intel SGX attestation services 0x7, 0, edx, 2, avx512_4vnniw , AVX-512 neural network instructions 0x7, 0, edx, 3, avx512_4fmaps , AVX-512 multiply accumulation single precision - 0x7, 0, edx, 4, fsrm , Fast short REP MOV + 0x7, 0, edx, 4, fsrm , Fast short REP MOVSB 0x7, 0, edx, 5, uintr , User interrupts 0x7, 0, edx, 8, avx512_vp2intersect , VP2INTERSECT{D,Q} instructions - 0x7, 0, edx, 9, srdbs_ctrl , SRBDS mitigation MSR + 0x7, 0, edx, 9, srbds_ctrl , SRBDS mitigation MSR 0x7, 0, edx, 10, md_clear , VERW MD_CLEAR microcode 0x7, 0, edx, 11, rtm_always_abort , XBEGIN (RTM transaction) always aborts 0x7, 0, edx, 13, tsx_force_abort , MSR TSX_FORCE_ABORT, RTM_ABORT bit @@ -296,8 +296,8 @@ 0x7, 2, edx, 0, intel_psfd , Intel predictive store forward disable 0x7, 2, edx, 1, ipred_ctrl , MSR bits IA32_SPEC_CTRL.IPRED_DIS_{U,S} 0x7, 2, edx, 2, rrsba_ctrl , MSR bits IA32_SPEC_CTRL.RRSBA_DIS_{U,S} - 0x7, 2, edx, 3, ddp_ctrl , MSR bit IA32_SPEC_CTRL.DDPD_U - 0x7, 2, edx, 4, bhi_ctrl , MSR bit IA32_SPEC_CTRL.BHI_DIS_S + 0x7, 2, edx, 3, ddp_ctrl , MSR bit IA32_SPEC_CTRL.DDPD_U + 0x7, 2, edx, 4, bhi_ctrl , MSR bit IA32_SPEC_CTRL.BHI_DIS_S 0x7, 2, edx, 5, mcdt_no , MCDT mitigation not needed 0x7, 2, edx, 6, uclock_disable , UC-lock disable @@ -368,7 +368,7 @@ 0xd, 1, ecx, 8, xss_pt , PT state 0xd, 1, ecx, 10, xss_pasid , PASID state 0xd, 1, ecx, 11, xss_cet_u , CET user state - 0xd, 1, ecx, 12, xss_cet_p , CET supervisor state + 0xd, 1, ecx, 12, xss_cet_s , CET supervisor state 0xd, 1, ecx, 13, xss_hdc , HDC state 0xd, 1, ecx, 14, xss_uintr , UINTR state 0xd, 1, ecx, 15, xss_lbr , LBR state @@ -433,7 +433,7 @@ 0x12, 1, eax, 7, secs_attr_kss , Key Separation and Sharing 0x12, 1, eax, 10, secs_attr_aexnotify , Enclave threads: AEX notifications 0x12, 1, ecx, 0, xfrm_x87 , Enclave XFRM.X87 - 0x12, 1, ecx, 1, xfrm_sse , Enclave XFRM.SEE + 0x12, 1, ecx, 1, xfrm_sse , Enclave XFRM.SSE 0x12, 1, ecx, 2, xfrm_avx , Enclave XFRM.AVX 0x12, 1, ecx, 3, xfrm_mpx_bndregs , Enclave XFRM.BNDREGS (MPX BND0-BND3 registers) 0x12, 1, ecx, 4, xfrm_mpx_bndcsr , Enclave XFRM.BNDCSR (MPX BNDCFGU/BNDSTATUS registers) @@ -466,9 +466,9 @@ 0x14, 0, ecx, 0, topa_output , ToPA output scheme 0x14, 0, ecx, 1, topa_multiple_entries , ToPA tables can hold multiple entries 0x14, 0, ecx, 2, single_range_output , Single-range output - 0x14, 0, ecx, 3, trance_transport_output, Trace Transport subsystem output + 0x14, 0, ecx, 3, trace_transport_output , Trace Transport subsystem output 0x14, 0, ecx, 31, ip_payloads_lip , IP payloads have LIP values (CS base included) - 0x14, 1, eax, 2:0, num_address_ranges , Number of configurable Address Ranges + 0x14, 1, eax, 2:0, num_address_ranges , Number of configurable address ranges 0x14, 1, eax, 31:16, mtc_periods_bmp , MTC period encodings bitmap 0x14, 1, ebx, 15:0, cycle_thresholds_bmp , Cycle Threshold encodings bitmap 0x14, 1, ebx, 31:16, psb_periods_bmp , Configurable PSB frequency encodings bitmap @@ -494,7 +494,7 @@ 0x17, 0, ebx, 15:0, soc_vendor_id , SoC vendor ID 0x17, 0, ebx, 16, is_vendor_scheme , Assigned by industry enumeration scheme (not Intel) 0x17, 0, ecx, 31:0, soc_proj_id , SoC project ID, assigned by vendor - 0x17, 0, edx, 31:0, soc_stepping_id , Soc project stepping ID, assigned by vendor + 0x17, 0, edx, 31:0, soc_stepping_id , SoC project stepping ID, assigned by vendor 0x17, 3:1, eax, 31:0, vendor_brand_a , Vendor Brand ID string, bytes subleaf_nr * (0 -> 3) 0x17, 3:1, ebx, 31:0, vendor_brand_b , Vendor Brand ID string, bytes subleaf_nr * (4 -> 7) 0x17, 3:1, ecx, 31:0, vendor_brand_c , Vendor Brand ID string, bytes subleaf_nr * (8 -> 11) @@ -514,12 +514,12 @@ 0x18, 31:0, edx, 4:0, tlb_type , Translation cache type (TLB type) 0x18, 31:0, edx, 7:5, tlb_cache_level , Translation cache level (1-based) 0x18, 31:0, edx, 8, is_fully_associative , Fully-associative - 0x18, 31:0, edx, 25:14, tlb_max_addressible_ids, Max number of addressable IDs - 1 + 0x18, 31:0, edx, 25:14, tlb_max_addressable_ids, Max number of addressable IDs - 1 # Leaf 19H # Intel key locker - 0x19, 0, eax, 0, kl_cpl0_only , CPL0-only key Locker restriction + 0x19, 0, eax, 0, kl_cpl0_only , CPL0-only key locker restriction 0x19, 0, eax, 1, kl_no_encrypt , No-encrypt key locker restriction 0x19, 0, eax, 2, kl_no_decrypt , No-decrypt key locker restriction 0x19, 0, ebx, 0, aes_keylocker , AES key locker instructions @@ -546,7 +546,7 @@ # Intel LBR (Last Branch Record) 0x1c, 0, eax, 7:0, lbr_depth_mask , Max LBR stack depth bitmask - 0x1c, 0, eax, 30, lbr_deep_c_reset , LBRs maybe cleared on MWAIT C-state > C1 + 0x1c, 0, eax, 30, lbr_deep_c_reset , LBRs may be cleared on MWAIT C-state > C1 0x1c, 0, eax, 31, lbr_ip_is_lip , LBR IP contain Last IP (otherwise effective IP) 0x1c, 0, ebx, 0, lbr_cpl , CPL filtering 0x1c, 0, ebx, 1, lbr_branch_filter , Branch filtering @@ -591,8 +591,8 @@ # Intel TD (Trust Domain) 0x21, 0, ebx, 31:0, tdx_vendorid_0 , TDX vendor ID string bytes 0 - 3 - 0x21, 0, ecx, 31:0, tdx_vendorid_2 , CPU vendor ID string bytes 8 - 11 - 0x21, 0, edx, 31:0, tdx_vendorid_1 , CPU vendor ID string bytes 4 - 7 + 0x21, 0, ecx, 31:0, tdx_vendorid_2 , TDX vendor ID string bytes 8 - 11 + 0x21, 0, edx, 31:0, tdx_vendorid_1 , TDX vendor ID string bytes 4 - 7 # Leaf 23H # Intel Architectural Performance Monitoring Extended (ArchPerfmonExt) @@ -857,7 +857,7 @@ 0x8000000a, 0, edx, 1, lbrv , LBR virtualization 0x8000000a, 0, edx, 2, svm_lock , SVM lock 0x8000000a, 0, edx, 3, nrip_save , NRIP save support on #VMEXIT -0x8000000a, 0, edx, 4, tsc_scale , MSR based TSC rate control +0x8000000a, 0, edx, 4, tsc_scale , MSR-based TSC rate control 0x8000000a, 0, edx, 5, vmcb_clean , VMCB clean bits support 0x8000000a, 0, edx, 6, flushbyasid , Flush by ASID + Extended VMCB TLB_Control 0x8000000a, 0, edx, 7, decodeassists , Decode Assists support @@ -895,7 +895,7 @@ 0x8000001a, 0, eax, 0, fp_128 , Internal FP/SIMD exec data path is 128-bits wide 0x8000001a, 0, eax, 1, movu_preferred , SSE: MOVU* better than MOVL*/MOVH* -0x8000001a, 0, eax, 2, fp_256 , internal FP/SSE exec data path is 256-bits wide +0x8000001a, 0, eax, 2, fp_256 , Internal FP/SSE exec data path is 256-bits wide # Leaf 8000001BH # AMD IBS (Instruction-Based Sampling) @@ -917,7 +917,7 @@ # AMD LWP (Lightweight Profiling) 0x8000001c, 0, eax, 0, os_lwp_avail , OS: LWP is available to application programs -0x8000001c, 0, eax, 1, os_lpwval , OS: LWPVAL instruction +0x8000001c, 0, eax, 1, os_lwpval , OS: LWPVAL instruction 0x8000001c, 0, eax, 2, os_lwp_ire , OS: Instructions Retired Event 0x8000001c, 0, eax, 3, os_lwp_bre , OS: Branch Retired Event 0x8000001c, 0, eax, 4, os_lwp_dme , OS: Dcache Miss Event @@ -934,13 +934,13 @@ 0x8000001c, 0, ecx, 5, lwp_data_addr , Cache miss events report data cache address 0x8000001c, 0, ecx, 8:6, lwp_latency_rnd , Cache latency rounding amount 0x8000001c, 0, ecx, 15:9, lwp_version , LWP version -0x8000001c, 0, ecx, 23:16, lwp_buf_min_sz , LWP event ring buffer min size, 32 event records units +0x8000001c, 0, ecx, 23:16, lwp_buf_min_sz , LWP event ring buffer min size, 32 event record units 0x8000001c, 0, ecx, 28, lwp_branch_predict , Branches Retired events can be filtered 0x8000001c, 0, ecx, 29, lwp_ip_filtering , IP filtering (IPI, IPF, BaseIP, and LimitIP @ LWPCP) 0x8000001c, 0, ecx, 30, lwp_cache_levels , Cache-related events: filter by cache level 0x8000001c, 0, ecx, 31, lwp_cache_latency , Cache-related events: filter by latency 0x8000001c, 0, edx, 0, hw_lwp_avail , HW: LWP available -0x8000001c, 0, edx, 1, hw_lpwval , HW: LWPVAL available +0x8000001c, 0, edx, 1, hw_lwpval , HW: LWPVAL available 0x8000001c, 0, edx, 2, hw_lwp_ire , HW: Instructions Retired Event 0x8000001c, 0, edx, 3, hw_lwp_bre , HW: Branch Retired Event 0x8000001c, 0, edx, 4, hw_lwp_dme , HW: Dcache Miss Event @@ -1040,8 +1040,8 @@ 0x80000021, 0, eax, 7, upper_addr_ignore , EFER MSR Upper Address Ignore 0x80000021, 0, eax, 8, autoibrs , EFER MSR Automatic IBRS 0x80000021, 0, eax, 9, no_smm_ctl_msr , SMM_CTL MSR not available -0x80000021, 0, eax, 10, fsrs , Fast Short Rep STOSB -0x80000021, 0, eax, 11, fsrc , Fast Short Rep CMPSB +0x80000021, 0, eax, 10, fsrs , Fast Short REP STOSB +0x80000021, 0, eax, 11, fsrc , Fast Short REP CMPSB 0x80000021, 0, eax, 13, prefetch_ctl_msr , Prefetch control MSR 0x80000021, 0, eax, 16, opcode_reclaim , Reserves opcode space 0x80000021, 0, eax, 17, user_cpuid_disable , #GP when executing CPUID at CPL > 0 @@ -1093,9 +1093,9 @@ # Maximum Transmeta leaf + CPU vendor string 0x80860000, 0, eax, 31:0, max_tra_leaf , Maximum Transmeta leaf -0x80860000, 0, ebx, 31:0, cpu_vendorid_0 , Transmeta Vendor ID string bytes 0 - 3 -0x80860000, 0, ecx, 31:0, cpu_vendorid_2 , Transmeta Vendor ID string bytes 8 - 11 -0x80860000, 0, edx, 31:0, cpu_vendorid_1 , Transmeta Vendor ID string bytes 4 - 7 +0x80860000, 0, ebx, 31:0, cpu_vendorid_0 , Transmeta vendor ID string bytes 0 - 3 +0x80860000, 0, ecx, 31:0, cpu_vendorid_2 , Transmeta vendor ID string bytes 8 - 11 +0x80860000, 0, edx, 31:0, cpu_vendorid_1 , Transmeta vendor ID string bytes 4 - 7 # Leaf 80860001H # Transmeta extended CPU features diff --git a/tools/testing/selftests/arm64/abi/hwcap.c b/tools/testing/selftests/arm64/abi/hwcap.c index e22703d6b97c..19fca95f7c22 100644 --- a/tools/testing/selftests/arm64/abi/hwcap.c +++ b/tools/testing/selftests/arm64/abi/hwcap.c @@ -108,6 +108,24 @@ static void f8mm8_sigill(void) asm volatile(".inst 0x6e80ec00"); } +static void f16f32dot_sigill(void) +{ + /* FDOT V0.2S, V0.4H, V0.2H[0] */ + asm volatile(".inst 0xf409000"); +} + +static void f16f32mm_sigill(void) +{ + /* FMMLA V0.4S, V0.8H, V0.8H */ + asm volatile(".inst 0x4e40ec00"); +} + +static void f16mm_sigill(void) +{ + /* FMMLA V0.8H, V0.8H, V0.8H */ + asm volatile(".inst 0x4ec0ec00"); +} + static void faminmax_sigill(void) { /* FAMIN V0.4H, V0.4H, V0.4H */ @@ -191,6 +209,12 @@ static void lut_sigill(void) asm volatile(".inst 0x4e801000"); } +static void sve_lut6_sigill(void) +{ + /* LUTI6 Z0.H, { Z0.H, Z1.H }, Z0[0] */ + asm volatile(".inst 0x4560ac00"); +} + static void mops_sigill(void) { char dst[1], src[1]; @@ -282,6 +306,18 @@ static void sme2p2_sigill(void) asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); } +static void sme2p3_sigill(void) +{ + /* SMSTART SM */ + asm volatile("msr S0_3_C4_C3_3, xzr" : : : ); + + /* ADDQP Z0.B, Z0.B, Z0.B */ + asm volatile(".inst 0x4207800" : : : "z0"); + + /* SMSTOP */ + asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); +} + static void sme_aes_sigill(void) { /* SMSTART SM */ @@ -378,6 +414,18 @@ static void smef8f32_sigill(void) asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); } +static void smelut6_sigill(void) +{ + /* SMSTART */ + asm volatile("msr S0_3_C4_C7_3, xzr" : : : ); + + /* LUTI6 { Z0.B-Z3.B }, ZT0, { Z0-Z2 } */ + asm volatile(".inst 0xc08a0000" : : : ); + + /* SMSTOP */ + asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); +} + static void smelutv2_sigill(void) { /* SMSTART */ @@ -486,6 +534,12 @@ static void sve2p2_sigill(void) asm volatile(".inst 0x4cea000" : : : "z0"); } +static void sve2p3_sigill(void) +{ + /* ADDQP Z0.B, Z0.B, Z0.B */ + asm volatile(".inst 0x4207800" : : : "z0"); +} + static void sveaes_sigill(void) { /* AESD z0.b, z0.b, z0.b */ @@ -504,6 +558,12 @@ static void sveb16b16_sigill(void) asm volatile(".inst 0x65000000" : : : ); } +static void sveb16mm_sigill(void) +{ + /* BFMMLA Z0.H, Z0.H, Z0.H */ + asm volatile(".inst 0x64e0e000" : : : ); +} + static void svebfscale_sigill(void) { /* BFSCALE Z0.H, P0/M, Z0.H, Z0.H */ @@ -729,6 +789,27 @@ static const struct hwcap_data { .cpuinfo = "f8mm4", .sigill_fn = f8mm4_sigill, }, + { + .name = "F16MM", + .at_hwcap = AT_HWCAP3, + .hwcap_bit = HWCAP3_F16MM, + .cpuinfo = "f16mm", + .sigill_fn = f16mm_sigill, + }, + { + .name = "F16F32DOT", + .at_hwcap = AT_HWCAP3, + .hwcap_bit = HWCAP3_F16F32DOT, + .cpuinfo = "f16f32dot", + .sigill_fn = f16f32dot_sigill, + }, + { + .name = "F16F32MM", + .at_hwcap = AT_HWCAP3, + .hwcap_bit = HWCAP3_F16F32MM, + .cpuinfo = "f16f32mm", + .sigill_fn = f16f32mm_sigill, + }, { .name = "FAMINMAX", .at_hwcap = AT_HWCAP2, @@ -918,6 +999,13 @@ static const struct hwcap_data { .cpuinfo = "sme2p2", .sigill_fn = sme2p2_sigill, }, + { + .name = "SME 2.3", + .at_hwcap = AT_HWCAP3, + .hwcap_bit = HWCAP3_SME2P3, + .cpuinfo = "sme2p3", + .sigill_fn = sme2p3_sigill, + }, { .name = "SME AES", .at_hwcap = AT_HWCAP, @@ -967,6 +1055,13 @@ static const struct hwcap_data { .cpuinfo = "smef8f32", .sigill_fn = smef8f32_sigill, }, + { + .name = "SME LUT6", + .at_hwcap = AT_HWCAP3, + .hwcap_bit = HWCAP3_SME_LUT6, + .cpuinfo = "smelut6", + .sigill_fn = smelut6_sigill, + }, { .name = "SME LUTV2", .at_hwcap = AT_HWCAP2, @@ -1052,6 +1147,13 @@ static const struct hwcap_data { .cpuinfo = "sve2p2", .sigill_fn = sve2p2_sigill, }, + { + .name = "SVE 2.3", + .at_hwcap = AT_HWCAP3, + .hwcap_bit = HWCAP3_SVE2P3, + .cpuinfo = "sve2p3", + .sigill_fn = sve2p3_sigill, + }, { .name = "SVE AES", .at_hwcap = AT_HWCAP2, @@ -1066,6 +1168,13 @@ static const struct hwcap_data { .cpuinfo = "sveaes2", .sigill_fn = sveaes2_sigill, }, + { + .name = "SVE B16MM", + .at_hwcap = AT_HWCAP3, + .hwcap_bit = HWCAP3_SVE_B16MM, + .cpuinfo = "sveb16mm", + .sigill_fn = sveb16mm_sigill, + }, { .name = "SVE BFSCALE", .at_hwcap = AT_HWCAP, @@ -1087,6 +1196,13 @@ static const struct hwcap_data { .cpuinfo = "svef16mm", .sigill_fn = svef16mm_sigill, }, + { + .name = "SVE_LUT6", + .at_hwcap = AT_HWCAP3, + .hwcap_bit = HWCAP3_SVE_LUT6, + .cpuinfo = "svelut6", + .sigill_fn = sve_lut6_sigill, + }, { .name = "SVE2 B16B16", .at_hwcap = AT_HWCAP2, diff --git a/tools/testing/selftests/arm64/signal/test_signals.h b/tools/testing/selftests/arm64/signal/test_signals.h index ee75a2c25ce7..c7c343494cb8 100644 --- a/tools/testing/selftests/arm64/signal/test_signals.h +++ b/tools/testing/selftests/arm64/signal/test_signals.h @@ -36,6 +36,7 @@ enum { FSME_FA64_BIT, FSME2_BIT, FGCS_BIT, + FPOE_BIT, FMAX_END }; @@ -45,6 +46,7 @@ enum { #define FEAT_SME_FA64 (1UL << FSME_FA64_BIT) #define FEAT_SME2 (1UL << FSME2_BIT) #define FEAT_GCS (1UL << FGCS_BIT) +#define FEAT_POE (1UL << FPOE_BIT) /* * A descriptor used to describe and configure a test case. diff --git a/tools/testing/selftests/arm64/signal/test_signals_utils.c b/tools/testing/selftests/arm64/signal/test_signals_utils.c index 5d3621921cfe..4b12dbd7669d 100644 --- a/tools/testing/selftests/arm64/signal/test_signals_utils.c +++ b/tools/testing/selftests/arm64/signal/test_signals_utils.c @@ -31,6 +31,7 @@ static char const *const feats_names[FMAX_END] = { " FA64 ", " SME2 ", " GCS ", + " POE ", }; #define MAX_FEATS_SZ 128 @@ -341,6 +342,8 @@ int test_init(struct tdescr *td) td->feats_supported |= FEAT_SME2; if (getauxval(AT_HWCAP) & HWCAP_GCS) td->feats_supported |= FEAT_GCS; + if (getauxval(AT_HWCAP2) & HWCAP2_POE) + td->feats_supported |= FEAT_POE; if (feats_ok(td)) { if (td->feats_required & td->feats_supported) fprintf(stderr, diff --git a/tools/testing/selftests/arm64/signal/test_signals_utils.h b/tools/testing/selftests/arm64/signal/test_signals_utils.h index 36fc12b3cd60..2c7b8c64a35a 100644 --- a/tools/testing/selftests/arm64/signal/test_signals_utils.h +++ b/tools/testing/selftests/arm64/signal/test_signals_utils.h @@ -57,6 +57,22 @@ static inline __attribute__((always_inline)) uint64_t get_gcspr_el0(void) return val; } +#define SYS_POR_EL0 "S3_3_C10_C2_4" + +static inline uint64_t get_por_el0(void) +{ + uint64_t val; + + asm volatile("mrs %0, " SYS_POR_EL0 "\n" : "=r"(val)); + + return val; +} + +static inline void set_por_el0(uint64_t val) +{ + asm volatile("msr " SYS_POR_EL0 ", %0\n" :: "r"(val)); +} + static inline bool feats_ok(struct tdescr *td) { if (td->feats_incompatible & td->feats_supported) diff --git a/tools/testing/selftests/arm64/signal/testcases/poe_missing_poe_context.c b/tools/testing/selftests/arm64/signal/testcases/poe_missing_poe_context.c new file mode 100644 index 000000000000..3f22d8cf6106 --- /dev/null +++ b/tools/testing/selftests/arm64/signal/testcases/poe_missing_poe_context.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2026 Arm Ltd + * + * Verify that the POR_EL0 register is left untouched on sigreturn if the + * POE frame record is missing. + */ + +#include + +#include "test_signals_utils.h" +#include "testcases.h" + +#define POR_EL0_INIT 0x07ul +#define POR_EL0_CUSTOM 0x77ul + +static bool failed_check; + +static bool modify_por_el0(struct tdescr *td) +{ + set_por_el0(POR_EL0_CUSTOM); + + return true; +} + +static int signal_remove_poe_context(struct tdescr *td, siginfo_t *si, + ucontext_t *uc) +{ + struct _aarch64_ctx *ctx = GET_UC_RESV_HEAD(uc); + size_t resv_size = GET_UCP_RESV_SIZE(uc); + struct _aarch64_ctx *poe_ctx_head; + + poe_ctx_head = get_header(ctx, POE_MAGIC, resv_size, NULL); + if (!poe_ctx_head) { + fprintf(stderr, "Missing poe_context record\n"); + failed_check = true; + return 0; + } + + /* + * Actually removing the record would require moving down the next + * records. An easier option is to turn it into an ESR record, which is + * ignored by sigreturn(). + */ + poe_ctx_head->magic = ESR_MAGIC; + + return 0; +} + +static void check_por_el0_preserved(struct tdescr *td) +{ + uint64_t por_el0 = get_por_el0(); + + if (por_el0 == POR_EL0_INIT) { + fprintf(stderr, "POR_EL0 preserved\n"); + } else { + fprintf(stderr, "POR_EL0 unexpectedly set to %lx\n", por_el0); + failed_check = true; + } + + td->pass = !failed_check; +} + +struct tdescr tde = { + .name = "POR_EL0 missing poe_context", + .descr = "Remove poe_context record and check POR_EL0 is preserved", + .feats_required = FEAT_POE, + .timeout = 3, + .sig_trig = SIGUSR1, + .init = modify_por_el0, + .run = signal_remove_poe_context, + .check_result = check_por_el0_preserved, +}; diff --git a/tools/testing/selftests/arm64/signal/testcases/poe_restore.c b/tools/testing/selftests/arm64/signal/testcases/poe_restore.c new file mode 100644 index 000000000000..9f9a61a4214d --- /dev/null +++ b/tools/testing/selftests/arm64/signal/testcases/poe_restore.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2026 Arm Ltd + * + * Verify that the POR_EL0 register is saved and restored as expected on signal + * entry/return. + */ + +#include + +#include "test_signals_utils.h" +#include "testcases.h" + +#define POR_EL0_INIT 0x07ul +#define POR_EL0_CUSTOM 0x77ul + +static bool failed_check; + +static bool modify_por_el0(struct tdescr *td) +{ + set_por_el0(POR_EL0_CUSTOM); + + return true; +} + +static int signal_check_por_el0_reset(struct tdescr *td, siginfo_t *si, + ucontext_t *uc) +{ + uint64_t signal_por_el0 = get_por_el0(); + + if (signal_por_el0 != POR_EL0_INIT) { + fprintf(stderr, "POR_EL0 is %lx in signal handler (expected %lx)\n", + signal_por_el0, POR_EL0_INIT); + failed_check = true; + } + + return 0; +} + +static void check_por_el0_restored(struct tdescr *td) +{ + uint64_t por_el0 = get_por_el0(); + + if (por_el0 == POR_EL0_CUSTOM) { + fprintf(stderr, "POR_EL0 restored\n"); + } else { + fprintf(stderr, "POR_EL0 was %lx but is now %lx\n", + POR_EL0_CUSTOM, por_el0); + failed_check = true; + } + + td->pass = !failed_check; +} + +struct tdescr tde = { + .name = "POR_EL0 restore", + .descr = "Validate that POR_EL0 is saved/restored on signal entry/return", + .feats_required = FEAT_POE, + .timeout = 3, + .sig_trig = SIGUSR1, + .init = modify_por_el0, + .run = signal_check_por_el0_reset, + .check_result = check_por_el0_restored, +}; diff --git a/tools/testing/selftests/arm64/signal/testcases/poe_siginfo.c b/tools/testing/selftests/arm64/signal/testcases/poe_siginfo.c index 36bd9940ee05..e15fedf4da6e 100644 --- a/tools/testing/selftests/arm64/signal/testcases/poe_siginfo.c +++ b/tools/testing/selftests/arm64/signal/testcases/poe_siginfo.c @@ -21,21 +21,6 @@ static union { char buf[1024 * 128]; } context; -#define SYS_POR_EL0 "S3_3_C10_C2_4" - -static uint64_t get_por_el0(void) -{ - uint64_t val; - - asm volatile( - "mrs %0, " SYS_POR_EL0 "\n" - : "=r"(val) - : - : ); - - return val; -} - int poe_present(struct tdescr *td, siginfo_t *si, ucontext_t *uc) { struct _aarch64_ctx *head = GET_BUF_RESV_HEAD(context); diff --git a/tools/testing/selftests/mm/pkey-arm64.h b/tools/testing/selftests/mm/pkey-arm64.h index 8e9685e03c44..c5a78a2f211d 100644 --- a/tools/testing/selftests/mm/pkey-arm64.h +++ b/tools/testing/selftests/mm/pkey-arm64.h @@ -130,9 +130,10 @@ static inline u64 get_pkey_bits(u64 reg, int pkey) static inline void aarch64_write_signal_pkey(ucontext_t *uctxt, u64 pkey) { struct _aarch64_ctx *ctx = GET_UC_RESV_HEAD(uctxt); + size_t resv_size = GET_UCP_RESV_SIZE(uctxt); struct poe_context *poe_ctx = (struct poe_context *) get_header(ctx, POE_MAGIC, - sizeof(uctxt->uc_mcontext), NULL); + resv_size, NULL); if (poe_ctx) poe_ctx->por_el0 = pkey; }