diff --git a/Documentation/ABI/testing/sysfs-driver-qat_kpt b/Documentation/ABI/testing/sysfs-driver-qat_kpt new file mode 100644 index 000000000000..c6480ea1fa4f --- /dev/null +++ b/Documentation/ABI/testing/sysfs-driver-qat_kpt @@ -0,0 +1,97 @@ +What: /sys/bus/pci/devices//qat_kpt/ +Date: August 2026 +KernelVersion: 7.2 +Contact: qat-linux@intel.com +Description: + Directory containing attributes related to the QAT Key Protection + Technology (KPT) feature. KPT allows cryptographic keys to be used + by the accelerator without being exposed in plaintext to the host. + +What: /sys/bus/pci/devices//qat_kpt/enable +Date: August 2026 +KernelVersion: 7.2 +Contact: qat-linux@intel.com +Description: + (RW) Enables or disables Key Protection Technology (KPT). + + Write 1 to enable KPT, or 0 to disable it. + + Example usage:: + + # cat /sys/bus/pci/devices//qat_kpt/enable + 0 + # echo 1 > /sys/bus/pci/devices//qat_kpt/enable + + This attribute is only available on devices that support KPT. + +What: /sys/bus/pci/devices//qat_kpt/swk_cnt_per_fn +Date: August 2026 +KernelVersion: 7.2 +Contact: qat-linux@intel.com +Description: + (RW) Configures the maximum number of KPT symmetric wrapping keys + (SWKs) that a Virtual Function (VF) may be associated with. + + Valid values range from 0 to 128. A value of 0 indicates no limit. + + Example usage:: + + # cat /sys/bus/pci/devices//qat_kpt/swk_cnt_per_fn + 128 + # echo 128 > /sys/bus/pci/devices//qat_kpt/swk_cnt_per_fn + + This attribute is only available on devices that support KPT. + +What: /sys/bus/pci/devices//qat_kpt/swk_cnt_per_pasid +Date: August 2026 +KernelVersion: 7.2 +Contact: qat-linux@intel.com +Description: + (RW) Configures the maximum number of KPT symmetric wrapping keys + (SWKs) per Process Address Space ID (PASID). + + Valid values range from 0 to 128. A value of 0 indicates no limit. + + Example usage:: + + # cat /sys/bus/pci/devices//qat_kpt/swk_cnt_per_pasid + 128 + # echo 128 > /sys/bus/pci/devices//qat_kpt/swk_cnt_per_pasid + + This attribute is only available on devices that support KPT. + +What: /sys/bus/pci/devices//qat_kpt/swk_max_ttl +Date: August 2026 +KernelVersion: 7.2 +Contact: qat-linux@intel.com +Description: + (RW) Configures the maximum Time To Live (TTL) for KPT symmetric + wrapping keys (SWK). + + Valid values range from 0 to 31536000 seconds. A value of 0 + indicates that the SWK TTL is unlimited. + + Example usage:: + + # cat /sys/bus/pci/devices//qat_kpt/swk_max_ttl + 1000 + # echo 1000 > /sys/bus/pci/devices//qat_kpt/swk_max_ttl + + This attribute is only available on devices that support KPT. + +What: /sys/bus/pci/devices//qat_kpt/swk_shared +Date: August 2026 +KernelVersion: 7.2 +Contact: qat-linux@intel.com +Description: + (RW) Controls shared mode for KPT symmetric wrapping keys (SWK). + + Write 1 to enable shared mode, or 0 to disable it (non-shared mode). + + Example usage:: + + # cat /sys/bus/pci/devices//qat_kpt/swk_shared + 0 + # echo 1 > /sys/bus/pci/devices//qat_kpt/swk_shared + + This attribute is only available on devices that support KPT. diff --git a/Documentation/ABI/testing/sysfs-driver-qat_rl b/Documentation/ABI/testing/sysfs-driver-qat_rl index d534f89b4971..422333a0eb69 100644 --- a/Documentation/ABI/testing/sysfs-driver-qat_rl +++ b/Documentation/ABI/testing/sysfs-driver-qat_rl @@ -209,7 +209,7 @@ Date: January 2024 KernelVersion: 6.7 Contact: qat-linux@intel.com Description: - (RW) This file will return the remaining capability for a + (RW) This file will return the remaining capacity for a particular service/sla. This is the remaining value that a new SLA can be set to or a current SLA can be increased with. diff --git a/Documentation/crypto/api-samples.rst b/Documentation/crypto/api-samples.rst index e923f17bc2bd..388bb7d7a460 100644 --- a/Documentation/crypto/api-samples.rst +++ b/Documentation/crypto/api-samples.rst @@ -159,7 +159,7 @@ Code Example For Random Number Generator Usage static int get_random_numbers(u8 *buf, unsigned int len) { struct crypto_rng *rng = NULL; - char *drbg = "drbg_nopr_sha256"; /* Hash DRBG with SHA-256, no PR */ + char *drbg = "stdrng"; int ret; if (!buf || !len) { diff --git a/Documentation/crypto/userspace-if.rst b/Documentation/crypto/userspace-if.rst index 021759198fe7..ab93300c8e04 100644 --- a/Documentation/crypto/userspace-if.rst +++ b/Documentation/crypto/userspace-if.rst @@ -4,26 +4,88 @@ User Space Interface Introduction ------------ -The concepts of the kernel crypto API visible to kernel space is fully -applicable to the user space interface as well. Therefore, the kernel -crypto API high level discussion for the in-kernel use cases applies -here as well. +AF_ALG provides unprivileged userspace programs access to arbitrary hash, +symmetric cipher, AEAD, and RNG algorithms that are implemented in kernel-mode +code. -The major difference, however, is that user space can only act as a -consumer and never as a provider of a transformation or cipher -algorithm. +AF_ALG is insecure and is deprecated. Originally added to the kernel in 2010, +most kernel developers now consider it to be a mistake. Support for hardware +accelerators, which was the original purpose of AF_ALG, has been removed. -The following covers the user space interface exported by the kernel -crypto API. A working example of this description is libkcapi that can -be obtained from [1]. That library can be used by user space -applications that require cryptographic services from the kernel. +AF_ALG continues to be supported only for backwards compatibility. On systems +where no programs using AF_ALG remain, the support for it should be disabled by +disabling ``CONFIG_CRYPTO_USER_API_*``. -Some details of the in-kernel kernel crypto API aspects do not apply to -user space, however. This includes the difference between synchronous -and asynchronous invocations. The user space API call is fully -synchronous. +Deprecation +----------- -[1] https://www.chronox.de/libkcapi/index.html +AF_ALG was originally intended to provide userspace programs access to crypto +accelerators that they wouldn't otherwise have access to. + +However, that capability turned out to not be useful on very many systems. More +significantly, the actual implementation exposes a vastly greater amount of +functionality than that. It actually provides access to all software algorithms. + +This includes arbitrary compositions of different algorithms created via a +complex template system, as well as algorithms that only make sense as internal +implementation details of other algorithms. In the past, it also included full +zero-copy support, which was difficult for the kernel to implement securely. + +Ultimately, these algorithms are just math computations. They use the same +instructions that userspace programs already have access to, just accessed in a +much more convoluted and less efficient way. + +Indeed, userspace code is nearly always what is being used anyway. These same +algorithms are widely implemented in userspace crypto libraries. + +Even when zero-copy and off-CPU accelerators were supported, AF_ALG was usually +much slower than optimized software cryptography in userspace. This was +especially true for the small message sizes usually seen in performance-critical +workloads. While it was possible to demonstrate performance wins for hashing +large files on embedded devices, it is hard to imagine a situation where this +would be performance-critical. + +Nowadays, AF_ALG no longer supports zero-copy or off-CPU accelerators. +Therefore, it is *always* slower than an optimized userspace implementation, +even for large messages. The only possible advantage left is that it avoids +duplicating code between kernel and userspace. However, userspace +implementations, especially hardware-accelerated ones, do not need to be large. +Just because OpenSSL is huge does not mean that all userspace cryptography +libraries are. + +Meanwhile, AF_ALG hasn't been withstanding modern vulnerability discovery tools +such as syzbot and large language models. It receives a steady stream of CVEs. +Some of the examples include: + +- CVE-2026-31677 +- CVE-2026-31431 (https://copy.fail) +- CVE-2025-38079 +- CVE-2025-37808 +- CVE-2024-26824 +- CVE-2022-48781 +- CVE-2019-8912 +- CVE-2018-14619 +- CVE-2017-18075 +- CVE-2017-17806 +- CVE-2017-17805 +- CVE-2016-10147 +- CVE-2015-8970 +- CVE-2015-3331 +- CVE-2014-9644 +- CVE-2013-7421 +- CVE-2011-4081 + +Hardware accelerator drivers are frequently buggy. To reduce attack surface, +AF_ALG now only provides access to algorithms implemented in software. This +means that AF_ALG no longer fulfills its original purpose. + +It is recommended that, whenever possible, userspace programs be migrated to +userspace crypto code (which again, is what is normally used anyway) and +``CONFIG_CRYPTO_USER_API_*`` be disabled. On systems that use SELinux, SELinux +can also be used to restrict the use of AF_ALG to trusted programs. + +The remainder of this documentation provides the historical documentation for +the deprecated AF_ALG interface. User Space API General Remarks ------------------------------ @@ -297,7 +359,7 @@ follows: struct sockaddr_alg sa = { .salg_family = AF_ALG, .salg_type = "rng", /* this selects the random number generator */ - .salg_name = "drbg_nopr_sha256" /* this is the RNG name */ + .salg_name = "stdrng" /* this is the RNG name */ }; @@ -327,33 +389,10 @@ CRYPTO_USER_API_RNG_CAVP option: Zero-Copy Interface ------------------- -In addition to the send/write/read/recv system call family, the AF_ALG -interface can be accessed with the zero-copy interface of -splice/vmsplice. As the name indicates, the kernel tries to avoid a copy -operation into kernel space. - -The zero-copy operation requires data to be aligned at the page -boundary. Non-aligned data can be used as well, but may require more -operations of the kernel which would defeat the speed gains obtained -from the zero-copy interface. - -The system-inherent limit for the size of one zero-copy operation is 16 -pages. If more data is to be sent to AF_ALG, user space must slice the -input into segments with a maximum size of 16 pages. - -Zero-copy can be used with the following code example (a complete -working example is provided with libkcapi): - -:: - - int pipes[2]; - - pipe(pipes); - /* input data in iov */ - vmsplice(pipes[1], iov, iovlen, SPLICE_F_GIFT); - /* opfd is the file descriptor returned from accept() system call */ - splice(pipes[0], NULL, opfd, NULL, ret, 0); - read(opfd, out, outlen); +AF_ALG used to have zero-copy support, but it was removed due to it being a +frequent source of vulnerabilities. For backwards compatibility the splice() +and sendfile() system calls are still supported, but the kernel will make an +internal copy of the data before passing it to the crypto code. Setsockopt Interface diff --git a/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml b/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml index ccb6b8dd8e11..db895c50e2d2 100644 --- a/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml +++ b/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml @@ -14,6 +14,7 @@ properties: items: - enum: - qcom,eliza-inline-crypto-engine + - qcom,hawi-inline-crypto-engine - qcom,kaanapali-inline-crypto-engine - qcom,milos-inline-crypto-engine - qcom,qcs8300-inline-crypto-engine diff --git a/Documentation/devicetree/bindings/crypto/qcom,prng.yaml b/Documentation/devicetree/bindings/crypto/qcom,prng.yaml index 41402599e9ab..dc270c8aedf3 100644 --- a/Documentation/devicetree/bindings/crypto/qcom,prng.yaml +++ b/Documentation/devicetree/bindings/crypto/qcom,prng.yaml @@ -17,11 +17,14 @@ properties: - qcom,prng-ee # 8996 and later using EE - items: - enum: + - qcom,glymur-trng + - qcom,hawi-trng - qcom,ipq5332-trng - qcom,ipq5424-trng - qcom,ipq9574-trng - qcom,kaanapali-trng - qcom,milos-trng + - qcom,nord-trng - qcom,qcs615-trng - qcom,qcs8300-trng - qcom,sa8255p-trng diff --git a/Documentation/devicetree/bindings/crypto/qcom-qce.yaml b/Documentation/devicetree/bindings/crypto/qcom-qce.yaml index 79d5be2548bc..08febd66c22b 100644 --- a/Documentation/devicetree/bindings/crypto/qcom-qce.yaml +++ b/Documentation/devicetree/bindings/crypto/qcom-qce.yaml @@ -45,7 +45,11 @@ properties: - items: - enum: + - qcom,eliza-qce + - qcom,glymur-qce - qcom,kaanapali-qce + - qcom,milos-qce + - qcom,nord-qce - qcom,qcs615-qce - qcom,qcs8300-qce - qcom,sa8775p-qce diff --git a/Documentation/devicetree/bindings/rng/mtk-rng.yaml b/Documentation/devicetree/bindings/rng/mtk-rng.yaml index 7e8dc62e5d3a..38e67861b88f 100644 --- a/Documentation/devicetree/bindings/rng/mtk-rng.yaml +++ b/Documentation/devicetree/bindings/rng/mtk-rng.yaml @@ -11,12 +11,13 @@ maintainers: properties: $nodename: - pattern: "^rng@[0-9a-f]+$" + pattern: "^rng(@[0-9a-f]+)?$" compatible: oneOf: - enum: - mediatek,mt7623-rng + - mediatek,mt7981-rng - items: - enum: - mediatek,mt7622-rng @@ -25,6 +26,11 @@ properties: - mediatek,mt8365-rng - mediatek,mt8516-rng - const: mediatek,mt7623-rng + - items: + - enum: + - mediatek,mt7987-rng + - mediatek,mt7988-rng + - const: mediatek,mt7981-rng reg: maxItems: 1 @@ -38,18 +44,37 @@ properties: required: - compatible - - reg - - clocks - - clock-names + +allOf: + - if: + properties: + compatible: + contains: + const: mediatek,mt7981-rng + then: + properties: + reg: false + clocks: false + clock-names: false + else: + required: + - reg + - clocks + - clock-names additionalProperties: false examples: - | #include - rng: rng@1020f000 { - compatible = "mediatek,mt7623-rng"; - reg = <0x1020f000 0x1000>; - clocks = <&infracfg CLK_INFRA_TRNG>; - clock-names = "rng"; + + rng@1020f000 { + compatible = "mediatek,mt7623-rng"; + reg = <0x1020f000 0x1000>; + clocks = <&infracfg CLK_INFRA_TRNG>; + clock-names = "rng"; + }; + - | + rng { + compatible = "mediatek,mt7981-rng"; }; diff --git a/Documentation/devicetree/bindings/trivial-devices.yaml b/Documentation/devicetree/bindings/trivial-devices.yaml index 1e870bb1bb6e..1b35b411e73b 100644 --- a/Documentation/devicetree/bindings/trivial-devices.yaml +++ b/Documentation/devicetree/bindings/trivial-devices.yaml @@ -67,8 +67,10 @@ properties: - arduino,unoq-mcu # Temperature monitoring of Astera Labs PT5161L PCIe retimer - asteralabs,pt5161l - # i2c h/w elliptic curve crypto module + # ATECC508A - i2c h/w elliptic curve crypto module - atmel,atecc508a + # ATECC608B - i2c h/w elliptic curve crypto module + - atmel,atecc608b # ATSHA204 - i2c h/w symmetric crypto module - atmel,atsha204 # ATSHA204A - i2c h/w symmetric crypto module diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst index 331223761fff..29a08bc059dd 100644 --- a/Documentation/userspace-api/ioctl/ioctl-number.rst +++ b/Documentation/userspace-api/ioctl/ioctl-number.rst @@ -229,7 +229,6 @@ Code Seq# Include File Comments 'a' all linux/atm*.h, linux/sonet.h ATM on linux -'a' 00-0F drivers/crypto/qat/qat_common/adf_cfg_common.h conflict! qat driver 'b' 00-FF conflict! bit3 vme host bridge 'b' 00-0F linux/dma-buf.h conflict! diff --git a/MAINTAINERS b/MAINTAINERS index ab92f3668e92..69e679e8c96e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -11721,7 +11721,7 @@ F: drivers/mfd/hi6421-spmi-pmic.c HISILICON TRUE RANDOM NUMBER GENERATOR V2 SUPPORT M: Weili Qian S: Maintained -F: drivers/crypto/hisilicon/trng/trng.c +F: drivers/char/hw_random/hisi-trng-v2.c HISILICON V3XX SPI NOR FLASH Controller Driver M: Yang Shen @@ -15090,7 +15090,6 @@ M: Qunqin Zhao L: linux-crypto@vger.kernel.org S: Maintained F: drivers/char/tpm/tpm_loongson.c -F: drivers/crypto/loongson/ F: drivers/mfd/loongson-se.c F: include/linux/mfd/loongson-se.h @@ -21968,10 +21967,10 @@ F: Documentation/devicetree/bindings/opp/opp-v2-kryo-cpu.yaml F: drivers/cpufreq/qcom-cpufreq-nvmem.c QUALCOMM CRYPTO DRIVERS -M: Thara Gopinath +M: Bartosz Golaszewski L: linux-crypto@vger.kernel.org L: linux-arm-msm@vger.kernel.org -S: Maintained +S: Supported F: Documentation/devicetree/bindings/crypto/qcom-qce.yaml F: drivers/crypto/qce/ @@ -23797,14 +23796,6 @@ F: Documentation/devicetree/bindings/mailbox/google,gs101-mbox.yaml F: drivers/mailbox/exynos-mailbox.c F: include/linux/mailbox/exynos-message.h -SAMSUNG EXYNOS PSEUDO RANDOM NUMBER GENERATOR (RNG) DRIVER -M: Krzysztof Kozlowski -L: linux-crypto@vger.kernel.org -L: linux-samsung-soc@vger.kernel.org -S: Maintained -F: Documentation/devicetree/bindings/rng/samsung,exynos4-rng.yaml -F: drivers/crypto/exynos-rng.c - SAMSUNG EXYNOS TRUE RANDOM NUMBER GENERATOR (TRNG) DRIVER M: Łukasz Stelmach L: linux-samsung-soc@vger.kernel.org @@ -29315,7 +29306,7 @@ XILINX TRNG DRIVER M: Mounika Botcha M: Harsh Jain S: Maintained -F: drivers/crypto/xilinx/xilinx-trng.c +F: drivers/char/hw_random/xilinx-trng.c XILINX UARTLITE SERIAL DRIVER M: Peter Korsgaard diff --git a/arch/arm/configs/exynos_defconfig b/arch/arm/configs/exynos_defconfig index 84070e9698e8..8b072a5c0a5e 100644 --- a/arch/arm/configs/exynos_defconfig +++ b/arch/arm/configs/exynos_defconfig @@ -364,7 +364,6 @@ CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m CONFIG_CRYPTO_AES_ARM_BS=m -CONFIG_CRYPTO_DEV_EXYNOS_RNG=y CONFIG_CRYPTO_DEV_S5P=y CONFIG_DMA_CMA=y CONFIG_CMA_SIZE_MBYTES=96 diff --git a/arch/arm/configs/multi_v7_defconfig b/arch/arm/configs/multi_v7_defconfig index bcc9aabc1202..3672dd12df60 100644 --- a/arch/arm/configs/multi_v7_defconfig +++ b/arch/arm/configs/multi_v7_defconfig @@ -1329,7 +1329,6 @@ CONFIG_CRYPTO_AES_ARM_BS=m CONFIG_CRYPTO_AES_ARM_CE=m CONFIG_CRYPTO_DEV_SUN4I_SS=m CONFIG_CRYPTO_DEV_FSL_CAAM=m -CONFIG_CRYPTO_DEV_EXYNOS_RNG=m CONFIG_CRYPTO_DEV_S5P=m CONFIG_CRYPTO_DEV_ATMEL_AES=m CONFIG_CRYPTO_DEV_ATMEL_TDES=m diff --git a/arch/arm/configs/sunxi_defconfig b/arch/arm/configs/sunxi_defconfig index a83d29fed175..f4b8d8f7dbef 100644 --- a/arch/arm/configs/sunxi_defconfig +++ b/arch/arm/configs/sunxi_defconfig @@ -170,7 +170,6 @@ CONFIG_ROOT_NFS=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y CONFIG_CRYPTO_DEV_SUN4I_SS=y -CONFIG_CRYPTO_DEV_SUN4I_SS_PRNG=y CONFIG_CRYPTO_DEV_SUN8I_CE=y CONFIG_CRYPTO_DEV_SUN8I_SS=y CONFIG_DMA_CMA=y diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index 96ce783f24e7..1771c2215a0d 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -551,6 +551,8 @@ CONFIG_IPMI_DEVICE_INTERFACE=m CONFIG_IPMI_SI=m CONFIG_HW_RANDOM=y CONFIG_HW_RANDOM_VIRTIO=y +CONFIG_HW_RANDOM_HISI_TRNG=m +CONFIG_HW_RANDOM_XILINX=m CONFIG_TCG_TPM=y CONFIG_TCG_TIS=m CONFIG_TCG_TIS_SPI=m @@ -1956,14 +1958,12 @@ CONFIG_CRYPTO_DEV_FSL_DPAA2_CAAM=m CONFIG_CRYPTO_DEV_QCE=m CONFIG_CRYPTO_DEV_QCOM_RNG=m CONFIG_CRYPTO_DEV_TEGRA=m -CONFIG_CRYPTO_DEV_XILINX_TRNG=m CONFIG_CRYPTO_DEV_ZYNQMP_AES=m CONFIG_CRYPTO_DEV_ZYNQMP_SHA3=m CONFIG_CRYPTO_DEV_CCREE=m CONFIG_CRYPTO_DEV_HISI_SEC2=m CONFIG_CRYPTO_DEV_HISI_ZIP=m CONFIG_CRYPTO_DEV_HISI_HPRE=m -CONFIG_CRYPTO_DEV_HISI_TRNG=m CONFIG_CRYPTO_DEV_SA2UL=m CONFIG_DMA_RESTRICTED_POOL=y CONFIG_CMA_SIZE_MBYTES=32 diff --git a/arch/loongarch/Makefile b/arch/loongarch/Makefile index 54fcfa1eac1f..4b22f95aaafb 100644 --- a/arch/loongarch/Makefile +++ b/arch/loongarch/Makefile @@ -197,8 +197,6 @@ endif libs-y += arch/loongarch/lib/ libs-$(CONFIG_EFI_STUB) += $(objtree)/drivers/firmware/efi/libstub/lib.a -drivers-y += arch/loongarch/crypto/ - # suspend and hibernation support drivers-$(CONFIG_PM) += arch/loongarch/power/ diff --git a/arch/loongarch/configs/loongson32_defconfig b/arch/loongarch/configs/loongson32_defconfig index d5ef396dffe3..82897236863f 100644 --- a/arch/loongarch/configs/loongson32_defconfig +++ b/arch/loongarch/configs/loongson32_defconfig @@ -1091,7 +1091,6 @@ CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m CONFIG_CRYPTO_DEV_VIRTIO=m -CONFIG_CRYPTO_DEV_LOONGSON_RNG=m CONFIG_DMA_CMA=y CONFIG_CMA_SIZE_MBYTES=0 CONFIG_PRINTK_TIME=y diff --git a/arch/loongarch/configs/loongson64_defconfig b/arch/loongarch/configs/loongson64_defconfig index cba4cdff5acd..a94e88bd7ec5 100644 --- a/arch/loongarch/configs/loongson64_defconfig +++ b/arch/loongarch/configs/loongson64_defconfig @@ -1124,7 +1124,6 @@ CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m CONFIG_CRYPTO_DEV_VIRTIO=m -CONFIG_CRYPTO_DEV_LOONGSON_RNG=m CONFIG_DMA_CMA=y CONFIG_DMA_NUMA_CMA=y CONFIG_CMA_SIZE_MBYTES=0 diff --git a/arch/loongarch/crypto/Kconfig b/arch/loongarch/crypto/Kconfig deleted file mode 100644 index a0270b3e5b30..000000000000 --- a/arch/loongarch/crypto/Kconfig +++ /dev/null @@ -1,5 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 - -menu "Accelerated Cryptographic Algorithms for CPU (loongarch)" - -endmenu diff --git a/arch/loongarch/crypto/Makefile b/arch/loongarch/crypto/Makefile deleted file mode 100644 index ba83755dde2b..000000000000 --- a/arch/loongarch/crypto/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# Makefile for LoongArch crypto files.. -# diff --git a/arch/mips/Makefile b/arch/mips/Makefile index 6705fa5d9211..cff1a9a43b89 100644 --- a/arch/mips/Makefile +++ b/arch/mips/Makefile @@ -350,8 +350,6 @@ OBJCOPYFLAGS += --remove-section=.reginfo libs-y += arch/mips/lib/ libs-$(CONFIG_MIPS_FP_SUPPORT) += arch/mips/math-emu/ -drivers-y += arch/mips/crypto/ - # suspend and hibernation support drivers-$(CONFIG_PM) += arch/mips/power/ diff --git a/arch/mips/configs/decstation_64_defconfig b/arch/mips/configs/decstation_64_defconfig index 7c43352fac6b..509515cf4751 100644 --- a/arch/mips/configs/decstation_64_defconfig +++ b/arch/mips/configs/decstation_64_defconfig @@ -199,8 +199,6 @@ CONFIG_CRYPTO_LZO=m CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m -CONFIG_CRYPTO_DRBG_HASH=y -CONFIG_CRYPTO_DRBG_CTR=y # CONFIG_CRYPTO_HW is not set CONFIG_MAGIC_SYSRQ=y # CONFIG_FTRACE is not set diff --git a/arch/mips/configs/decstation_defconfig b/arch/mips/configs/decstation_defconfig index aee10274f048..3a17c080292f 100644 --- a/arch/mips/configs/decstation_defconfig +++ b/arch/mips/configs/decstation_defconfig @@ -194,8 +194,6 @@ CONFIG_CRYPTO_LZO=m CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m -CONFIG_CRYPTO_DRBG_HASH=y -CONFIG_CRYPTO_DRBG_CTR=y # CONFIG_CRYPTO_HW is not set CONFIG_FRAME_WARN=2048 CONFIG_MAGIC_SYSRQ=y diff --git a/arch/mips/configs/decstation_r4k_defconfig b/arch/mips/configs/decstation_r4k_defconfig index a1698049aa7a..7f5823663702 100644 --- a/arch/mips/configs/decstation_r4k_defconfig +++ b/arch/mips/configs/decstation_r4k_defconfig @@ -194,8 +194,6 @@ CONFIG_CRYPTO_LZO=m CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m -CONFIG_CRYPTO_DRBG_HASH=y -CONFIG_CRYPTO_DRBG_CTR=y # CONFIG_CRYPTO_HW is not set CONFIG_FRAME_WARN=2048 CONFIG_MAGIC_SYSRQ=y diff --git a/arch/mips/crypto/.gitignore b/arch/mips/crypto/.gitignore deleted file mode 100644 index 0d47d4f21c6d..000000000000 --- a/arch/mips/crypto/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -poly1305-core.S diff --git a/arch/mips/crypto/Kconfig b/arch/mips/crypto/Kconfig deleted file mode 100644 index 6a5bd5074867..000000000000 --- a/arch/mips/crypto/Kconfig +++ /dev/null @@ -1,5 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 - -menu "Accelerated Cryptographic Algorithms for CPU (mips)" - -endmenu diff --git a/arch/mips/crypto/Makefile b/arch/mips/crypto/Makefile deleted file mode 100644 index 5adb631a69c1..000000000000 --- a/arch/mips/crypto/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# Makefile for MIPS crypto files.. -# - diff --git a/arch/powerpc/crypto/aes-spe-glue.c b/arch/powerpc/crypto/aes-spe-glue.c index 7d2827e65240..e038488087e6 100644 --- a/arch/powerpc/crypto/aes-spe-glue.c +++ b/arch/powerpc/crypto/aes-spe-glue.c @@ -9,6 +9,7 @@ */ #include +#include #include #include #include @@ -140,7 +141,7 @@ static int ppc_ecb_crypt(struct skcipher_request *req, bool enc) err = skcipher_walk_virt(&walk, req, false); while ((nbytes = walk.nbytes) != 0) { - nbytes = min_t(unsigned int, nbytes, MAX_BYTES); + nbytes = min(nbytes, MAX_BYTES); nbytes = round_down(nbytes, AES_BLOCK_SIZE); spe_begin(); @@ -179,7 +180,7 @@ static int ppc_cbc_crypt(struct skcipher_request *req, bool enc) err = skcipher_walk_virt(&walk, req, false); while ((nbytes = walk.nbytes) != 0) { - nbytes = min_t(unsigned int, nbytes, MAX_BYTES); + nbytes = min(nbytes, MAX_BYTES); nbytes = round_down(nbytes, AES_BLOCK_SIZE); spe_begin(); @@ -220,7 +221,7 @@ static int ppc_ctr_crypt(struct skcipher_request *req) err = skcipher_walk_virt(&walk, req, false); while ((nbytes = walk.nbytes) != 0) { - nbytes = min_t(unsigned int, nbytes, MAX_BYTES); + nbytes = min(nbytes, MAX_BYTES); if (nbytes < walk.total) nbytes = round_down(nbytes, AES_BLOCK_SIZE); @@ -248,7 +249,7 @@ static int ppc_xts_crypt(struct skcipher_request *req, bool enc) twk = ctx->key_twk; while ((nbytes = walk.nbytes) != 0) { - nbytes = min_t(unsigned int, nbytes, MAX_BYTES); + nbytes = min(nbytes, MAX_BYTES); nbytes = round_down(nbytes, AES_BLOCK_SIZE); spe_begin(); diff --git a/arch/riscv/crypto/aes-riscv64-glue.c b/arch/riscv/crypto/aes-riscv64-glue.c index 8bbf7f348c23..bbd920c9e29d 100644 --- a/arch/riscv/crypto/aes-riscv64-glue.c +++ b/arch/riscv/crypto/aes-riscv64-glue.c @@ -19,6 +19,7 @@ #include #include #include +#include #include asmlinkage void aes_ecb_encrypt_zvkned(const struct crypto_aes_ctx *key, @@ -266,8 +267,7 @@ static int riscv64_aes_ctr_crypt(struct skcipher_request *req) * operation into two at the point where the overflow * will occur. After the first part, add the carry bit. */ - p1_nbytes = min_t(unsigned int, nbytes, - (nblocks - ctr32) * AES_BLOCK_SIZE); + p1_nbytes = min(nbytes, (nblocks - ctr32) * AES_BLOCK_SIZE); aes_ctr32_crypt_zvkned_zvkb(ctx, walk.src.virt.addr, walk.dst.virt.addr, p1_nbytes, req->iv); diff --git a/crypto/Kconfig b/crypto/Kconfig index 103d1f58cb7c..c3d7a20d5cb1 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1113,44 +1113,16 @@ endmenu menu "Random number generation" -menuconfig CRYPTO_DRBG_MENU +config CRYPTO_DRBG tristate "NIST SP800-90A DRBG (Deterministic Random Bit Generator)" + select CRYPTO_JITTERENTROPY + select CRYPTO_LIB_SHA512 + select CRYPTO_RNG help DRBG (Deterministic Random Bit Generator) (NIST SP800-90A) - In the following submenu, one or more of the DRBG types must be selected. - -if CRYPTO_DRBG_MENU - -config CRYPTO_DRBG_HMAC - bool - default y - select CRYPTO_HMAC - select CRYPTO_SHA512 - -config CRYPTO_DRBG_HASH - bool "Hash_DRBG" - select CRYPTO_SHA256 - help - Hash_DRBG variant as defined in NIST SP800-90A. - - This uses the SHA-1, SHA-256, SHA-384, or SHA-512 hash algorithms. - -config CRYPTO_DRBG_CTR - bool "CTR_DRBG" - select CRYPTO_DF80090A - help - CTR_DRBG variant as defined in NIST SP800-90A. - - This uses the AES cipher algorithm with the counter block mode. - -config CRYPTO_DRBG - tristate - default CRYPTO_DRBG_MENU - select CRYPTO_RNG - select CRYPTO_JITTERENTROPY - -endif # if CRYPTO_DRBG_MENU + Enable this only if you need it for a FIPS 140 certification. + It's otherwise redundant with the kernel's regular RNG. config CRYPTO_JITTERENTROPY tristate "CPU Jitter Non-Deterministic RNG (Random Number Generator)" @@ -1274,50 +1246,69 @@ config CRYPTO_KDF800108_CTR select CRYPTO_HMAC select CRYPTO_SHA256 -config CRYPTO_DF80090A - tristate - select CRYPTO_AES - select CRYPTO_CTR - endmenu -menu "Userspace interface" +menu "Userspace interface (deprecated)" config CRYPTO_USER_API tristate config CRYPTO_USER_API_HASH - tristate "Hash algorithms" + tristate "Hash algorithms (deprecated)" depends on NET select CRYPTO_HASH select CRYPTO_USER_API help - Enable the userspace interface for hash algorithms. + Enable the AF_ALG userspace interface for hash algorithms. This + provides unprivileged userspace programs access to arbitrary hash + algorithms implemented in the kernel's privileged execution context. - See Documentation/crypto/userspace-if.rst and - https://www.chronox.de/libkcapi/html/index.html + This interface is deprecated and is supported only for backwards + compatibility. It regularly has vulnerabilities, and the capabilities + it provides are redundant with userspace crypto libraries. + + Enable this only if needed for support for a program that hasn't yet + been converted to userspace crypto, for example iwd. + + See also Documentation/crypto/userspace-if.rst config CRYPTO_USER_API_SKCIPHER - tristate "Symmetric key cipher algorithms" + tristate "Symmetric key cipher algorithms (deprecated)" depends on NET select CRYPTO_SKCIPHER select CRYPTO_USER_API help - Enable the userspace interface for symmetric key cipher algorithms. + Enable the AF_ALG userspace interface for symmetric key algorithms. + This provides unprivileged userspace programs access to arbitrary + symmetric key algorithms implemented in the kernel's privileged + execution context. - See Documentation/crypto/userspace-if.rst and - https://www.chronox.de/libkcapi/html/index.html + This interface is deprecated and is supported only for backwards + compatibility. It regularly has vulnerabilities, and the capabilities + it provides are redundant with userspace crypto libraries. + + Enable this only if needed for support for a program that hasn't yet + been converted to userspace crypto, for example iwd, or cryptsetup + with certain algorithms. + + See also Documentation/crypto/userspace-if.rst config CRYPTO_USER_API_RNG - tristate "RNG (random number generator) algorithms" + tristate "Random number generation algorithms (deprecated)" depends on NET select CRYPTO_RNG select CRYPTO_USER_API help - Enable the userspace interface for RNG (random number generator) - algorithms. + Enable the AF_ALG userspace interface for random number generation + (RNG) algorithms. This provides unprivileged userspace programs + access to arbitrary RNG algorithms implemented in the kernel's + privileged execution context. - See Documentation/crypto/userspace-if.rst and - https://www.chronox.de/libkcapi/html/index.html + This interface is deprecated and is supported only for backwards + compatibility. It regularly has vulnerabilities, and the capabilities + it provides are redundant with userspace crypto libraries as well as + the normal kernel RNG (e.g., /dev/urandom and getrandom(2)). + + See also Documentation/crypto/userspace-if.rst config CRYPTO_USER_API_RNG_CAVP bool "Enable CAVP testing of DRBG" @@ -1332,16 +1323,25 @@ config CRYPTO_USER_API_RNG_CAVP no unless you know what this is. config CRYPTO_USER_API_AEAD - tristate "AEAD cipher algorithms" + tristate "AEAD cipher algorithms (deprecated)" depends on NET select CRYPTO_AEAD select CRYPTO_SKCIPHER select CRYPTO_USER_API help - Enable the userspace interface for AEAD cipher algorithms. + Enable the AF_ALG userspace interface for authenticated encryption + with associated data (AEAD) algorithms. This provides unprivileged + userspace programs access to arbitrary AEAD algorithms implemented in + the kernel's privileged execution context. - See Documentation/crypto/userspace-if.rst and - https://www.chronox.de/libkcapi/html/index.html + This interface is deprecated and is supported only for backwards + compatibility. It regularly has vulnerabilities, and the capabilities + it provides are redundant with userspace crypto libraries. + + Enable this only if needed for support for a program that hasn't yet + been converted to userspace crypto, for example iwd. + + See also Documentation/crypto/userspace-if.rst config CRYPTO_USER_API_ENABLE_OBSOLETE bool "Obsolete cryptographic algorithms" @@ -1361,12 +1361,6 @@ endif if ARM64 source "arch/arm64/crypto/Kconfig" endif -if LOONGARCH -source "arch/loongarch/crypto/Kconfig" -endif -if MIPS -source "arch/mips/crypto/Kconfig" -endif if PPC source "arch/powerpc/crypto/Kconfig" endif diff --git a/crypto/Makefile b/crypto/Makefile index 162242593c7c..f98f57c7a49f 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -178,6 +178,11 @@ obj-$(CONFIG_CRYPTO_ZSTD) += zstd.o obj-$(CONFIG_CRYPTO_ECC) += ecc.o obj-$(CONFIG_CRYPTO_ESSIV) += essiv.o +# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=124949 +ifeq ($(CONFIG_ARM)$(CONFIG_KASAN_STACK)$(CONFIG_CC_IS_GCC),yyy) +CFLAGS_ecc.o += $(call cc-option,-Wframe-larger-than=1536) +endif + ecdh_generic-y += ecdh.o ecdh_generic-y += ecdh_helper.o obj-$(CONFIG_CRYPTO_ECDH) += ecdh_generic.o @@ -203,6 +208,4 @@ obj-$(CONFIG_CRYPTO_SIMD) += crypto_simd.o # obj-$(CONFIG_CRYPTO_KDF800108_CTR) += kdf_sp800108.o -obj-$(CONFIG_CRYPTO_DF80090A) += df_sp80090a.o - obj-$(CONFIG_CRYPTO_KRB5) += krb5/ diff --git a/crypto/acompress.c b/crypto/acompress.c index 6025c1acce49..032de704eb2c 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -51,11 +51,9 @@ static inline struct acomp_alg *crypto_acomp_alg(struct crypto_acomp *tfm) static int __maybe_unused crypto_acomp_report( struct sk_buff *skb, struct crypto_alg *alg) { - struct crypto_report_acomp racomp; - - memset(&racomp, 0, sizeof(racomp)); - - strscpy(racomp.type, "acomp", sizeof(racomp.type)); + struct crypto_report_acomp racomp = { + .type = "acomp", + }; return nla_put(skb, CRYPTOCFGA_REPORT_ACOMP, sizeof(racomp), &racomp); } diff --git a/crypto/aead.c b/crypto/aead.c index e009937bf3a5..045b74c3779f 100644 --- a/crypto/aead.c +++ b/crypto/aead.c @@ -136,13 +136,11 @@ static int crypto_aead_init_tfm(struct crypto_tfm *tfm) static int __maybe_unused crypto_aead_report( struct sk_buff *skb, struct crypto_alg *alg) { - struct crypto_report_aead raead; struct aead_alg *aead = container_of(alg, struct aead_alg, base); - - memset(&raead, 0, sizeof(raead)); - - strscpy(raead.type, "aead", sizeof(raead.type)); - strscpy(raead.geniv, "", sizeof(raead.geniv)); + struct crypto_report_aead raead = { + .type = "aead", + .geniv = "", + }; raead.blocksize = alg->cra_blocksize; raead.maxauthsize = aead->maxauthsize; diff --git a/crypto/af_alg.c b/crypto/af_alg.c index 5a00c18eb145..cce000e8590e 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -181,7 +181,7 @@ static int alg_bind(struct socket *sock, struct sockaddr_unsized *uaddr, int add if (IS_ERR(type)) return PTR_ERR(type); - private = type->bind(sa->salg_name, sa->salg_feat, sa->salg_mask); + private = type->bind(sa->salg_name); if (IS_ERR(private)) { module_put(type->owner); return PTR_ERR(private); @@ -584,6 +584,8 @@ static int af_alg_cmsg_send(struct msghdr *msg, struct af_alg_control *con) if (cmsg->cmsg_len < CMSG_LEN(sizeof(u32))) return -EINVAL; con->aead_assoclen = *(u32 *)CMSG_DATA(cmsg); + if (con->aead_assoclen >= 0x80000000u) + return -EINVAL; break; default: @@ -973,7 +975,7 @@ int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size, ssize_t plen; /* use the existing memory in an allocated page */ - if (ctx->merge && !(msg->msg_flags & MSG_SPLICE_PAGES)) { + if (ctx->merge) { sgl = list_entry(ctx->tsgl_list.prev, struct af_alg_tsgl, list); sg = sgl->sg + sgl->cur - 1; @@ -1017,60 +1019,37 @@ int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size, if (sgl->cur) sg_unmark_end(sg + sgl->cur - 1); - if (msg->msg_flags & MSG_SPLICE_PAGES) { - struct sg_table sgtable = { - .sgl = sg, - .nents = sgl->cur, - .orig_nents = sgl->cur, - }; + do { + struct page *pg; + unsigned int i = sgl->cur; - plen = extract_iter_to_sg(&msg->msg_iter, len, &sgtable, - MAX_SGL_ENTS - sgl->cur, 0); - if (plen < 0) { - err = plen; + plen = min_t(size_t, len, PAGE_SIZE); + + pg = alloc_page(GFP_KERNEL); + if (!pg) { + err = -ENOMEM; goto unlock; } - for (; sgl->cur < sgtable.nents; sgl->cur++) - get_page(sg_page(&sg[sgl->cur])); + sg_assign_page(sg + i, pg); + + err = memcpy_from_msg(page_address(sg_page(sg + i)), + msg, plen); + if (err) { + __free_page(sg_page(sg + i)); + sg_assign_page(sg + i, NULL); + goto unlock; + } + + sg[i].length = plen; len -= plen; ctx->used += plen; copied += plen; size -= plen; - } else { - do { - struct page *pg; - unsigned int i = sgl->cur; + sgl->cur++; + } while (len && sgl->cur < MAX_SGL_ENTS); - plen = min_t(size_t, len, PAGE_SIZE); - - pg = alloc_page(GFP_KERNEL); - if (!pg) { - err = -ENOMEM; - goto unlock; - } - - sg_assign_page(sg + i, pg); - - err = memcpy_from_msg( - page_address(sg_page(sg + i)), - msg, plen); - if (err) { - __free_page(sg_page(sg + i)); - sg_assign_page(sg + i, NULL); - goto unlock; - } - - sg[i].length = plen; - len -= plen; - ctx->used += plen; - copied += plen; - size -= plen; - sgl->cur++; - } while (len && sgl->cur < MAX_SGL_ENTS); - - ctx->merge = plen & (PAGE_SIZE - 1); - } + ctx->merge = plen & (PAGE_SIZE - 1); if (!size) sg_mark_end(sg + sgl->cur - 1); @@ -1106,35 +1085,6 @@ void af_alg_free_resources(struct af_alg_async_req *areq) } EXPORT_SYMBOL_GPL(af_alg_free_resources); -/** - * af_alg_async_cb - AIO callback handler - * @data: async request completion data - * @err: if non-zero, error result to be returned via ki_complete(); - * otherwise return the AIO output length via ki_complete(). - * - * This handler cleans up the struct af_alg_async_req upon completion of the - * AIO operation. - * - * The number of bytes to be generated with the AIO operation must be set - * in areq->outlen before the AIO callback handler is invoked. - */ -void af_alg_async_cb(void *data, int err) -{ - struct af_alg_async_req *areq = data; - struct sock *sk = areq->sk; - struct kiocb *iocb = areq->iocb; - unsigned int resultlen; - - /* Buffer size written by crypto operation. */ - resultlen = areq->outlen; - - af_alg_free_resources(areq); - sock_put(sk); - - iocb->ki_complete(iocb, err ? err : (int)resultlen); -} -EXPORT_SYMBOL_GPL(af_alg_async_cb); - /** * af_alg_poll - poll system call handler * @file: file pointer @@ -1175,8 +1125,8 @@ struct af_alg_async_req *af_alg_alloc_areq(struct sock *sk, struct af_alg_ctx *ctx = alg_sk(sk)->private; struct af_alg_async_req *areq; - /* Only one AIO request can be in flight. */ - if (ctx->inflight) + /* Only one request can be in flight. */ + if (WARN_ON_ONCE(ctx->inflight)) return ERR_PTR(-EBUSY); areq = sock_kmalloc(sk, areqlen, GFP_KERNEL); diff --git a/crypto/ahash.c b/crypto/ahash.c index 7a730324c50e..dd56b0e45c0d 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -789,11 +789,9 @@ static void crypto_ahash_free_instance(struct crypto_instance *inst) static int __maybe_unused crypto_ahash_report( struct sk_buff *skb, struct crypto_alg *alg) { - struct crypto_report_hash rhash; - - memset(&rhash, 0, sizeof(rhash)); - - strscpy(rhash.type, "ahash", sizeof(rhash.type)); + struct crypto_report_hash rhash = { + .type = "ahash", + }; rhash.blocksize = alg->cra_blocksize; rhash.digestsize = __crypto_hash_alg_common(alg)->digestsize; diff --git a/crypto/akcipher.c b/crypto/akcipher.c index dfe87b3ce183..630bb19738be 100644 --- a/crypto/akcipher.c +++ b/crypto/akcipher.c @@ -36,11 +36,9 @@ struct crypto_akcipher_sync_data { static int __maybe_unused crypto_akcipher_report( struct sk_buff *skb, struct crypto_alg *alg) { - struct crypto_report_akcipher rakcipher; - - memset(&rakcipher, 0, sizeof(rakcipher)); - - strscpy(rakcipher.type, "akcipher", sizeof(rakcipher.type)); + struct crypto_report_akcipher rakcipher = { + .type = "akcipher", + }; return nla_put(skb, CRYPTOCFGA_REPORT_AKCIPHER, sizeof(rakcipher), &rakcipher); diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c index cb651ab58d62..787aac8aeb24 100644 --- a/crypto/algif_aead.c +++ b/crypto/algif_aead.c @@ -9,10 +9,10 @@ * The following concept of the memory management is used: * * The kernel maintains two SGLs, the TX SGL and the RX SGL. The TX SGL is - * filled by user space with the data submitted via sendmsg (maybe with - * MSG_SPLICE_PAGES). Filling up the TX SGL does not cause a crypto operation - * -- the data will only be tracked by the kernel. Upon receipt of one recvmsg - * call, the caller must provide a buffer which is tracked with the RX SGL. + * filled by user space with the data submitted via sendmsg. Filling up the TX + * SGL does not cause a crypto operation -- the data will only be tracked by the + * kernel. Upon receipt of one recvmsg call, the caller must provide a buffer + * which is tracked with the RX SGL. * * During the processing of the recvmsg operation, the cipher request is * allocated and prepared. As part of the recvmsg operation, the processed @@ -197,37 +197,14 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, aead_request_set_ad(&areq->cra_u.aead_req, ctx->aead_assoclen); aead_request_set_tfm(&areq->cra_u.aead_req, tfm); - if (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) { - /* AIO operation */ - sock_hold(sk); - areq->iocb = msg->msg_iocb; - - /* Remember output size that will be generated. */ - areq->outlen = outlen; - - aead_request_set_callback(&areq->cra_u.aead_req, - CRYPTO_TFM_REQ_MAY_SLEEP, - af_alg_async_cb, areq); - err = ctx->enc ? crypto_aead_encrypt(&areq->cra_u.aead_req) : - crypto_aead_decrypt(&areq->cra_u.aead_req); - - /* AIO operation in progress */ - if (err == -EINPROGRESS) - return -EIOCBQUEUED; - - sock_put(sk); - } else { - /* Synchronous operation */ - aead_request_set_callback(&areq->cra_u.aead_req, - CRYPTO_TFM_REQ_MAY_SLEEP | - CRYPTO_TFM_REQ_MAY_BACKLOG, - crypto_req_done, &ctx->wait); - err = crypto_wait_req(ctx->enc ? - crypto_aead_encrypt(&areq->cra_u.aead_req) : - crypto_aead_decrypt(&areq->cra_u.aead_req), - &ctx->wait); - } - + aead_request_set_callback(&areq->cra_u.aead_req, + CRYPTO_TFM_REQ_MAY_SLEEP | + CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &ctx->wait); + err = crypto_wait_req(ctx->enc ? + crypto_aead_encrypt(&areq->cra_u.aead_req) : + crypto_aead_decrypt(&areq->cra_u.aead_req), + &ctx->wait); free: af_alg_free_resources(areq); @@ -365,9 +342,9 @@ static struct proto_ops algif_aead_ops_nokey = { .poll = af_alg_poll, }; -static void *aead_bind(const char *name, u32 type, u32 mask) +static void *aead_bind(const char *name) { - return crypto_alloc_aead(name, type, mask); + return crypto_alloc_aead(name, 0, AF_ALG_CRYPTOAPI_MASK); } static void aead_release(void *private) diff --git a/crypto/algif_hash.c b/crypto/algif_hash.c index 4d3dfc60a16a..5452ad6c1506 100644 --- a/crypto/algif_hash.c +++ b/crypto/algif_hash.c @@ -380,9 +380,9 @@ static struct proto_ops algif_hash_ops_nokey = { .accept = hash_accept_nokey, }; -static void *hash_bind(const char *name, u32 type, u32 mask) +static void *hash_bind(const char *name) { - return crypto_alloc_ahash(name, type, mask); + return crypto_alloc_ahash(name, 0, AF_ALG_CRYPTOAPI_MASK); } static void hash_release(void *private) diff --git a/crypto/algif_rng.c b/crypto/algif_rng.c index a9fb492e929a..4dfe7899f8fa 100644 --- a/crypto/algif_rng.c +++ b/crypto/algif_rng.c @@ -197,7 +197,7 @@ static struct proto_ops __maybe_unused algif_rng_test_ops = { .sendmsg = rng_test_sendmsg, }; -static void *rng_bind(const char *name, u32 type, u32 mask) +static void *rng_bind(const char *name) { struct rng_parent_ctx *pctx; struct crypto_rng *rng; @@ -206,7 +206,7 @@ static void *rng_bind(const char *name, u32 type, u32 mask) if (!pctx) return ERR_PTR(-ENOMEM); - rng = crypto_alloc_rng(name, type, mask); + rng = crypto_alloc_rng(name, 0, AF_ALG_CRYPTOAPI_MASK); if (IS_ERR(rng)) { kfree(pctx); return ERR_CAST(rng); diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c index ba0a17fd95ac..df20bdfe1f1f 100644 --- a/crypto/algif_skcipher.c +++ b/crypto/algif_skcipher.c @@ -79,20 +79,6 @@ static int algif_skcipher_export(struct sock *sk, struct skcipher_request *req) return err; } -static void algif_skcipher_done(void *data, int err) -{ - struct af_alg_async_req *areq = data; - struct sock *sk = areq->sk; - - if (err) - goto out; - - err = algif_skcipher_export(sk, &areq->cra_u.skcipher_req); - -out: - af_alg_async_cb(data, err); -} - static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg, size_t ignored, int flags) { @@ -171,43 +157,19 @@ static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg, cflags |= CRYPTO_SKCIPHER_REQ_CONT; } - if (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) { - /* AIO operation */ - sock_hold(sk); - areq->iocb = msg->msg_iocb; + skcipher_request_set_callback(&areq->cra_u.skcipher_req, + cflags | + CRYPTO_TFM_REQ_MAY_SLEEP | + CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &ctx->wait); + err = crypto_wait_req(ctx->enc ? + crypto_skcipher_encrypt(&areq->cra_u.skcipher_req) : + crypto_skcipher_decrypt(&areq->cra_u.skcipher_req), + &ctx->wait); - /* Remember output size that will be generated. */ - areq->outlen = len; - - skcipher_request_set_callback(&areq->cra_u.skcipher_req, - cflags | - CRYPTO_TFM_REQ_MAY_SLEEP, - algif_skcipher_done, areq); - err = ctx->enc ? - crypto_skcipher_encrypt(&areq->cra_u.skcipher_req) : - crypto_skcipher_decrypt(&areq->cra_u.skcipher_req); - - /* AIO operation in progress */ - if (err == -EINPROGRESS) - return -EIOCBQUEUED; - - sock_put(sk); - } else { - /* Synchronous operation */ - skcipher_request_set_callback(&areq->cra_u.skcipher_req, - cflags | - CRYPTO_TFM_REQ_MAY_SLEEP | - CRYPTO_TFM_REQ_MAY_BACKLOG, - crypto_req_done, &ctx->wait); - err = crypto_wait_req(ctx->enc ? - crypto_skcipher_encrypt(&areq->cra_u.skcipher_req) : - crypto_skcipher_decrypt(&areq->cra_u.skcipher_req), - &ctx->wait); - - if (!err) - err = algif_skcipher_export( - sk, &areq->cra_u.skcipher_req); - } + if (!err) + err = algif_skcipher_export( + sk, &areq->cra_u.skcipher_req); free: af_alg_free_resources(areq); @@ -345,9 +307,9 @@ static struct proto_ops algif_skcipher_ops_nokey = { .poll = af_alg_poll, }; -static void *skcipher_bind(const char *name, u32 type, u32 mask) +static void *skcipher_bind(const char *name) { - return crypto_alloc_skcipher(name, type, mask); + return crypto_alloc_skcipher(name, 0, AF_ALG_CRYPTOAPI_MASK); } static void skcipher_release(void *private) diff --git a/crypto/asymmetric_keys/Kconfig b/crypto/asymmetric_keys/Kconfig index e50bd9b3e27b..6a2f664046ef 100644 --- a/crypto/asymmetric_keys/Kconfig +++ b/crypto/asymmetric_keys/Kconfig @@ -95,7 +95,6 @@ config FIPS_SIGNATURE_SELFTEST verification code, using some built in data. This is required for FIPS. depends on KEYS - depends on ASYMMETRIC_KEY_TYPE depends on PKCS7_MESSAGE_PARSER=X509_CERTIFICATE_PARSER depends on X509_CERTIFICATE_PARSER depends on CRYPTO_RSA diff --git a/crypto/asymmetric_keys/verify_pefile.c b/crypto/asymmetric_keys/verify_pefile.c index 1f3b227ba7f2..cec99db14129 100644 --- a/crypto/asymmetric_keys/verify_pefile.c +++ b/crypto/asymmetric_keys/verify_pefile.c @@ -305,6 +305,8 @@ static int pefile_digest_pe_contents(const void *pebuf, unsigned int pelen, if (pelen > hashed_bytes) { tmp = hashed_bytes + ctx->certs_size; + if (tmp <= hashed_bytes || pelen < tmp) + return -ELIBBAD; ret = crypto_shash_update(desc, pebuf + hashed_bytes, pelen - tmp); diff --git a/crypto/asymmetric_keys/x509_loader.c b/crypto/asymmetric_keys/x509_loader.c index a41741326998..0d516c77cc26 100644 --- a/crypto/asymmetric_keys/x509_loader.c +++ b/crypto/asymmetric_keys/x509_loader.c @@ -20,7 +20,7 @@ int x509_load_certificate_list(const u8 cert_list[], */ if (end - p < 4) goto dodgy_cert; - if (p[0] != 0x30 && + if (p[0] != 0x30 || p[1] != 0x82) goto dodgy_cert; plen = (p[2] << 8) | p[3]; diff --git a/crypto/authencesn.c b/crypto/authencesn.c index 522df41365d8..ca063576f670 100644 --- a/crypto/authencesn.c +++ b/crypto/authencesn.c @@ -94,11 +94,11 @@ static int crypto_authenc_esn_genicv_tail(struct aead_request *req, u32 tmp[2]; /* Move high-order bits of sequence number back. */ - scatterwalk_map_and_copy(tmp, dst, 4, 4, 0); - scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen, 4, 0); - scatterwalk_map_and_copy(tmp, dst, 0, 8, 1); + memcpy_from_sglist(tmp, dst, 4, 4); + memcpy_from_sglist(tmp + 1, dst, assoclen + cryptlen, 4); + memcpy_to_sglist(dst, 0, tmp, 8); - scatterwalk_map_and_copy(hash, dst, assoclen + cryptlen, authsize, 1); + memcpy_to_sglist(dst, assoclen + cryptlen, hash, authsize); return 0; } @@ -129,9 +129,9 @@ static int crypto_authenc_esn_genicv(struct aead_request *req, return 0; /* Move high-order bits of sequence number to the end. */ - scatterwalk_map_and_copy(tmp, dst, 0, 8, 0); - scatterwalk_map_and_copy(tmp, dst, 4, 4, 1); - scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen, 4, 1); + memcpy_from_sglist(tmp, dst, 0, 8); + memcpy_to_sglist(dst, 4, tmp, 4); + memcpy_to_sglist(dst, assoclen + cryptlen, tmp + 1, 4); sg_init_table(areq_ctx->dst, 2); dst = scatterwalk_ffwd(areq_ctx->dst, dst, 4); @@ -217,9 +217,9 @@ static int crypto_authenc_esn_decrypt_tail(struct aead_request *req, if (src == dst) { /* Move high-order bits of sequence number back. */ - scatterwalk_map_and_copy(tmp, dst, 4, 4, 0); - scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen, 4, 0); - scatterwalk_map_and_copy(tmp, dst, 0, 8, 1); + memcpy_from_sglist(tmp, dst, 4, 4); + memcpy_from_sglist(tmp + 1, dst, assoclen + cryptlen, 4); + memcpy_to_sglist(dst, 0, tmp, 8); } else memcpy_sglist(dst, src, assoclen); @@ -274,18 +274,17 @@ static int crypto_authenc_esn_decrypt(struct aead_request *req) goto tail; cryptlen -= authsize; - scatterwalk_map_and_copy(ihash, req->src, assoclen + cryptlen, - authsize, 0); + memcpy_from_sglist(ihash, req->src, assoclen + cryptlen, authsize); /* Move high-order bits of sequence number to the end. */ - scatterwalk_map_and_copy(tmp, src, 0, 8, 0); + memcpy_from_sglist(tmp, src, 0, 8); if (src == dst) { - scatterwalk_map_and_copy(tmp, dst, 4, 4, 1); - scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen, 4, 1); + memcpy_to_sglist(dst, 4, tmp, 4); + memcpy_to_sglist(dst, assoclen + cryptlen, tmp + 1, 4); dst = scatterwalk_ffwd(areq_ctx->dst, dst, 4); } else { - scatterwalk_map_and_copy(tmp, dst, 0, 4, 1); - scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen - 4, 4, 1); + memcpy_to_sglist(dst, 0, tmp, 4); + memcpy_to_sglist(dst, assoclen + cryptlen - 4, tmp + 1, 4); src = scatterwalk_ffwd(areq_ctx->src, src, 8); dst = scatterwalk_ffwd(areq_ctx->dst, dst, 4); diff --git a/crypto/chacha20poly1305.c b/crypto/chacha20poly1305.c index b4b5a7198d84..27df9e1eb058 100644 --- a/crypto/chacha20poly1305.c +++ b/crypto/chacha20poly1305.c @@ -375,6 +375,7 @@ static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb, struct aead_instance *inst; struct chachapoly_instance_ctx *ctx; struct skcipher_alg_common *chacha; + const char *poly_name; int err; if (ivsize > CHACHAPOLY_IV_SIZE) @@ -396,9 +397,15 @@ static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb, goto err_free_inst; chacha = crypto_spawn_skcipher_alg_common(&ctx->chacha); + poly_name = crypto_attr_alg_name(tb[2]); + if (IS_ERR(poly_name)) { + err = PTR_ERR(poly_name); + goto err_free_inst; + } + err = -EINVAL; - if (strcmp(crypto_attr_alg_name(tb[2]), "poly1305") && - strcmp(crypto_attr_alg_name(tb[2]), "poly1305-generic")) + if (strcmp(poly_name, "poly1305") && + strcmp(poly_name, "poly1305-generic")) goto err_free_inst; /* Need 16-byte IV size, including Initial Block Counter value */ if (chacha->ivsize != CHACHA_IV_SIZE) diff --git a/crypto/crypto_null.c b/crypto/crypto_null.c index 34588f39fdfc..fea151df1648 100644 --- a/crypto/crypto_null.c +++ b/crypto/crypto_null.c @@ -50,15 +50,6 @@ static int null_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { return 0; } -static int null_setkey(struct crypto_tfm *tfm, const u8 *key, - unsigned int keylen) -{ return 0; } - -static void null_crypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) -{ - memcpy(dst, src, NULL_BLOCK_SIZE); -} - static int null_skcipher_crypt(struct skcipher_request *req) { if (req->src != req->dst) @@ -97,35 +88,16 @@ static struct skcipher_alg skcipher_null = { .decrypt = null_skcipher_crypt, }; -static struct crypto_alg cipher_null = { - .cra_name = "cipher_null", - .cra_driver_name = "cipher_null-generic", - .cra_flags = CRYPTO_ALG_TYPE_CIPHER, - .cra_blocksize = NULL_BLOCK_SIZE, - .cra_ctxsize = 0, - .cra_module = THIS_MODULE, - .cra_u = { .cipher = { - .cia_min_keysize = NULL_KEY_SIZE, - .cia_max_keysize = NULL_KEY_SIZE, - .cia_setkey = null_setkey, - .cia_encrypt = null_crypt, - .cia_decrypt = null_crypt } } -}; - MODULE_ALIAS_CRYPTO("digest_null"); -MODULE_ALIAS_CRYPTO("cipher_null"); +MODULE_ALIAS_CRYPTO("ecb(cipher_null)"); static int __init crypto_null_mod_init(void) { int ret = 0; - ret = crypto_register_alg(&cipher_null); - if (ret < 0) - goto out; - ret = crypto_register_shash(&digest_null); if (ret < 0) - goto out_unregister_algs; + goto out; ret = crypto_register_skcipher(&skcipher_null); if (ret < 0) @@ -135,15 +107,12 @@ static int __init crypto_null_mod_init(void) out_unregister_shash: crypto_unregister_shash(&digest_null); -out_unregister_algs: - crypto_unregister_alg(&cipher_null); out: return ret; } static void __exit crypto_null_mod_fini(void) { - crypto_unregister_alg(&cipher_null); crypto_unregister_shash(&digest_null); crypto_unregister_skcipher(&skcipher_null); } diff --git a/crypto/crypto_user.c b/crypto/crypto_user.c index 3187e0d276f9..e8b6ae75f31f 100644 --- a/crypto/crypto_user.c +++ b/crypto/crypto_user.c @@ -70,11 +70,9 @@ static struct crypto_alg *crypto_alg_match(struct crypto_user_alg *p, int exact) static int crypto_report_cipher(struct sk_buff *skb, struct crypto_alg *alg) { - struct crypto_report_cipher rcipher; - - memset(&rcipher, 0, sizeof(rcipher)); - - strscpy(rcipher.type, "cipher", sizeof(rcipher.type)); + struct crypto_report_cipher rcipher = { + .type = "cipher", + }; rcipher.blocksize = alg->cra_blocksize; rcipher.min_keysize = alg->cra_cipher.cia_min_keysize; @@ -103,10 +101,10 @@ static int crypto_report_one(struct crypto_alg *alg, if (nla_put_u32(skb, CRYPTOCFGA_PRIORITY_VAL, alg->cra_priority)) goto nla_put_failure; if (alg->cra_flags & CRYPTO_ALG_LARVAL) { - struct crypto_report_larval rl; + struct crypto_report_larval rl = { + .type = "larval", + }; - memset(&rl, 0, sizeof(rl)); - strscpy(rl.type, "larval", sizeof(rl.type)); if (nla_put(skb, CRYPTOCFGA_REPORT_LARVAL, sizeof(rl), &rl)) goto nla_put_failure; goto out; diff --git a/crypto/df_sp80090a.c b/crypto/df_sp80090a.c deleted file mode 100644 index b8134be6f7ad..000000000000 --- a/crypto/df_sp80090a.c +++ /dev/null @@ -1,222 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -/* - * NIST SP800-90A DRBG derivation function - * - * Copyright (C) 2014, Stephan Mueller - */ - -#include -#include -#include -#include -#include -#include -#include - -static void drbg_kcapi_sym(struct aes_enckey *aeskey, unsigned char *outval, - const struct drbg_string *in, u8 blocklen_bytes) -{ - /* there is only component in *in */ - BUG_ON(in->len < blocklen_bytes); - aes_encrypt(aeskey, outval, in->buf); -} - -/* BCC function for CTR DRBG as defined in 10.4.3 */ - -static void drbg_ctr_bcc(struct aes_enckey *aeskey, - unsigned char *out, const unsigned char *key, - struct list_head *in, - u8 blocklen_bytes, - u8 keylen) -{ - struct drbg_string *curr = NULL; - struct drbg_string data; - short cnt = 0; - - drbg_string_fill(&data, out, blocklen_bytes); - - /* 10.4.3 step 2 / 4 */ - aes_prepareenckey(aeskey, key, keylen); - list_for_each_entry(curr, in, list) { - const unsigned char *pos = curr->buf; - size_t len = curr->len; - /* 10.4.3 step 4.1 */ - while (len) { - /* 10.4.3 step 4.2 */ - if (blocklen_bytes == cnt) { - cnt = 0; - drbg_kcapi_sym(aeskey, out, &data, blocklen_bytes); - } - out[cnt] ^= *pos; - pos++; - cnt++; - len--; - } - } - /* 10.4.3 step 4.2 for last block */ - if (cnt) - drbg_kcapi_sym(aeskey, out, &data, blocklen_bytes); -} - -/* - * scratchpad usage: drbg_ctr_update is interlinked with crypto_drbg_ctr_df - * (and drbg_ctr_bcc, but this function does not need any temporary buffers), - * the scratchpad is used as follows: - * drbg_ctr_update: - * temp - * start: drbg->scratchpad - * length: drbg_statelen(drbg) + drbg_blocklen(drbg) - * note: the cipher writing into this variable works - * blocklen-wise. Now, when the statelen is not a multiple - * of blocklen, the generateion loop below "spills over" - * by at most blocklen. Thus, we need to give sufficient - * memory. - * df_data - * start: drbg->scratchpad + - * drbg_statelen(drbg) + drbg_blocklen(drbg) - * length: drbg_statelen(drbg) - * - * crypto_drbg_ctr_df: - * pad - * start: df_data + drbg_statelen(drbg) - * length: drbg_blocklen(drbg) - * iv - * start: pad + drbg_blocklen(drbg) - * length: drbg_blocklen(drbg) - * temp - * start: iv + drbg_blocklen(drbg) - * length: drbg_satelen(drbg) + drbg_blocklen(drbg) - * note: temp is the buffer that the BCC function operates - * on. BCC operates blockwise. drbg_statelen(drbg) - * is sufficient when the DRBG state length is a multiple - * of the block size. For AES192 (and maybe other ciphers) - * this is not correct and the length for temp is - * insufficient (yes, that also means for such ciphers, - * the final output of all BCC rounds are truncated). - * Therefore, add drbg_blocklen(drbg) to cover all - * possibilities. - * refer to crypto_drbg_ctr_df_datalen() to get required length - */ - -/* Derivation Function for CTR DRBG as defined in 10.4.2 */ -int crypto_drbg_ctr_df(struct aes_enckey *aeskey, - unsigned char *df_data, size_t bytes_to_return, - struct list_head *seedlist, - u8 blocklen_bytes, - u8 statelen) -{ - unsigned char L_N[8]; - /* S3 is input */ - struct drbg_string S1, S2, S4, cipherin; - LIST_HEAD(bcc_list); - unsigned char *pad = df_data + statelen; - unsigned char *iv = pad + blocklen_bytes; - unsigned char *temp = iv + blocklen_bytes; - size_t padlen = 0; - unsigned int templen = 0; - /* 10.4.2 step 7 */ - unsigned int i = 0; - /* 10.4.2 step 8 */ - const unsigned char *K = (unsigned char *) - "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" - "\x10\x11\x12\x13\x14\x15\x16\x17" - "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"; - unsigned char *X; - size_t generated_len = 0; - size_t inputlen = 0; - struct drbg_string *seed = NULL; - u8 keylen; - - memset(pad, 0, blocklen_bytes); - memset(iv, 0, blocklen_bytes); - keylen = statelen - blocklen_bytes; - /* 10.4.2 step 1 is implicit as we work byte-wise */ - - /* 10.4.2 step 2 */ - if ((512 / 8) < bytes_to_return) - return -EINVAL; - - /* 10.4.2 step 2 -- calculate the entire length of all input data */ - list_for_each_entry(seed, seedlist, list) - inputlen += seed->len; - drbg_cpu_to_be32(inputlen, &L_N[0]); - - /* 10.4.2 step 3 */ - drbg_cpu_to_be32(bytes_to_return, &L_N[4]); - - /* 10.4.2 step 5: length is L_N, input_string, one byte, padding */ - padlen = (inputlen + sizeof(L_N) + 1) % (blocklen_bytes); - /* wrap the padlen appropriately */ - if (padlen) - padlen = blocklen_bytes - padlen; - /* - * pad / padlen contains the 0x80 byte and the following zero bytes. - * As the calculated padlen value only covers the number of zero - * bytes, this value has to be incremented by one for the 0x80 byte. - */ - padlen++; - pad[0] = 0x80; - - /* 10.4.2 step 4 -- first fill the linked list and then order it */ - drbg_string_fill(&S1, iv, blocklen_bytes); - list_add_tail(&S1.list, &bcc_list); - drbg_string_fill(&S2, L_N, sizeof(L_N)); - list_add_tail(&S2.list, &bcc_list); - list_splice_tail(seedlist, &bcc_list); - drbg_string_fill(&S4, pad, padlen); - list_add_tail(&S4.list, &bcc_list); - - /* 10.4.2 step 9 */ - while (templen < (keylen + (blocklen_bytes))) { - /* - * 10.4.2 step 9.1 - the padding is implicit as the buffer - * holds zeros after allocation -- even the increment of i - * is irrelevant as the increment remains within length of i - */ - drbg_cpu_to_be32(i, iv); - /* 10.4.2 step 9.2 -- BCC and concatenation with temp */ - drbg_ctr_bcc(aeskey, temp + templen, K, &bcc_list, - blocklen_bytes, keylen); - /* 10.4.2 step 9.3 */ - i++; - templen += blocklen_bytes; - } - - /* 10.4.2 step 11 */ - X = temp + (keylen); - drbg_string_fill(&cipherin, X, blocklen_bytes); - - /* 10.4.2 step 12: overwriting of outval is implemented in next step */ - - /* 10.4.2 step 13 */ - aes_prepareenckey(aeskey, temp, keylen); - while (generated_len < bytes_to_return) { - short blocklen = 0; - /* - * 10.4.2 step 13.1: the truncation of the key length is - * implicit as the key is only drbg_blocklen in size based on - * the implementation of the cipher function callback - */ - drbg_kcapi_sym(aeskey, X, &cipherin, blocklen_bytes); - blocklen = (blocklen_bytes < - (bytes_to_return - generated_len)) ? - blocklen_bytes : - (bytes_to_return - generated_len); - /* 10.4.2 step 13.2 and 14 */ - memcpy(df_data + generated_len, X, blocklen); - generated_len += blocklen; - } - - memset(iv, 0, blocklen_bytes); - memset(temp, 0, statelen + blocklen_bytes); - memset(pad, 0, blocklen_bytes); - return 0; -} -EXPORT_SYMBOL_GPL(crypto_drbg_ctr_df); - -MODULE_IMPORT_NS("CRYPTO_INTERNAL"); -MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Stephan Mueller "); -MODULE_DESCRIPTION("Derivation Function conformant to SP800-90A"); diff --git a/crypto/drbg.c b/crypto/drbg.c index 9204e6edb426..d66c7211d6bb 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -1,13 +1,9 @@ /* * DRBG: Deterministic Random Bits Generator - * Based on NIST Recommended DRBG from NIST SP800-90A with the following - * properties: - * * CTR DRBG with DF with AES-128, AES-192, AES-256 cores - * * Hash DRBG with DF with SHA-1, SHA-256, SHA-384, SHA-512 cores - * * HMAC DRBG with DF with SHA-1, SHA-256, SHA-384, SHA-512 cores - * * with and without prediction resistance + * Implementation of the HMAC SHA-512 DRBG from NIST SP800-90A * * Copyright Stephan Mueller , 2014 + * Copyright 2026 Google LLC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -55,7 +51,7 @@ * char data[DATALEN]; * * drng = crypto_alloc_rng(drng_name, 0, 0); - * err = crypto_rng_get_bytes(drng, &data, DATALEN); + * err = crypto_rng_get_bytes(drng, data, DATALEN); * crypto_free_rng(drng); * * @@ -64,15 +60,13 @@ * struct crypto_rng *drng; * int err; * char data[DATALEN]; - * struct drbg_string pers; * char personalization[11] = "some-string"; * - * drbg_string_fill(&pers, personalization, strlen(personalization)); * drng = crypto_alloc_rng(drng_name, 0, 0); * // The reset completely re-initializes the DRBG with the provided * // personalization string - * err = crypto_rng_reset(drng, &personalization, strlen(personalization)); - * err = crypto_rng_get_bytes(drng, &data, DATALEN); + * err = crypto_rng_reset(drng, personalization, strlen(personalization)); + * err = crypto_rng_get_bytes(drng, data, DATALEN); * crypto_free_rng(drng); * * @@ -82,13 +76,10 @@ * int err; * char data[DATALEN]; * char addtl_string[11] = "some-string"; - * string drbg_string addtl; * - * drbg_string_fill(&addtl, addtl_string, strlen(addtl_string)); * drng = crypto_alloc_rng(drng_name, 0, 0); - * // The following call is a wrapper to crypto_rng_get_bytes() and returns - * // the same error codes. - * err = crypto_drbg_get_bytes_addtl(drng, &data, DATALEN, &addtl); + * err = crypto_rng_generate(drng, addtl_string, strlen(addtl_string), + data, DATALEN); * crypto_free_rng(drng); * * @@ -97,803 +88,129 @@ * Just mix both scenarios above. */ -#include -#include -#include +#include +#include +#include #include -#include +#include +#include #include +#include -/*************************************************************** - * Backend cipher definitions available to DRBG - ***************************************************************/ +/* State length in bytes */ +#define DRBG_STATE_LEN SHA512_DIGEST_SIZE + +/* Security strength in bytes */ +#define DRBG_SEC_STRENGTH (SHA512_DIGEST_SIZE / 2) /* - * The order of the DRBG definitions here matter: every DRBG is registered - * as stdrng. Each DRBG receives an increasing cra_priority values the later - * they are defined in this array (see drbg_fill_array). - * - * HMAC DRBGs are favored over Hash DRBGs over CTR DRBGs, and the - * HMAC-SHA512 / SHA256 / AES 256 over other ciphers. Thus, the - * favored DRBGs are the latest entries in this array. + * Maximum number of requests before reseeding is forced. + * SP800-90A allows this to be up to 2**48. We use a lower value. */ -static const struct drbg_core drbg_cores[] = { -#ifdef CONFIG_CRYPTO_DRBG_CTR - { - .flags = DRBG_CTR | DRBG_STRENGTH128, - .statelen = 32, /* 256 bits as defined in 10.2.1 */ - .blocklen_bytes = 16, - .cra_name = "ctr_aes128", - .backend_cra_name = "aes", - }, { - .flags = DRBG_CTR | DRBG_STRENGTH192, - .statelen = 40, /* 320 bits as defined in 10.2.1 */ - .blocklen_bytes = 16, - .cra_name = "ctr_aes192", - .backend_cra_name = "aes", - }, { - .flags = DRBG_CTR | DRBG_STRENGTH256, - .statelen = 48, /* 384 bits as defined in 10.2.1 */ - .blocklen_bytes = 16, - .cra_name = "ctr_aes256", - .backend_cra_name = "aes", - }, -#endif /* CONFIG_CRYPTO_DRBG_CTR */ -#ifdef CONFIG_CRYPTO_DRBG_HASH - { - .flags = DRBG_HASH | DRBG_STRENGTH256, - .statelen = 111, /* 888 bits */ - .blocklen_bytes = 48, - .cra_name = "sha384", - .backend_cra_name = "sha384", - }, { - .flags = DRBG_HASH | DRBG_STRENGTH256, - .statelen = 111, /* 888 bits */ - .blocklen_bytes = 64, - .cra_name = "sha512", - .backend_cra_name = "sha512", - }, { - .flags = DRBG_HASH | DRBG_STRENGTH256, - .statelen = 55, /* 440 bits */ - .blocklen_bytes = 32, - .cra_name = "sha256", - .backend_cra_name = "sha256", - }, -#endif /* CONFIG_CRYPTO_DRBG_HASH */ -#ifdef CONFIG_CRYPTO_DRBG_HMAC - { - .flags = DRBG_HMAC | DRBG_STRENGTH256, - .statelen = 48, /* block length of cipher */ - .blocklen_bytes = 48, - .cra_name = "hmac_sha384", - .backend_cra_name = "hmac(sha384)", - }, { - .flags = DRBG_HMAC | DRBG_STRENGTH256, - .statelen = 32, /* block length of cipher */ - .blocklen_bytes = 32, - .cra_name = "hmac_sha256", - .backend_cra_name = "hmac(sha256)", - }, { - .flags = DRBG_HMAC | DRBG_STRENGTH256, - .statelen = 64, /* block length of cipher */ - .blocklen_bytes = 64, - .cra_name = "hmac_sha512", - .backend_cra_name = "hmac(sha512)", - }, -#endif /* CONFIG_CRYPTO_DRBG_HMAC */ +#define DRBG_MAX_REQUESTS 4096 + +/* + * Maximum number of random bytes that can be requested at once. + * SP800-90A allows up to 2**19 bits, which is 2**16 bytes. + */ +#define DRBG_MAX_REQUEST_BYTES (1 << 16) + +/* + * Maximum length of additional info and personalization strings, in bytes. + * SP800-90A allows up to 2**35 bits, i.e. 2**32 bytes. We use 2**32 - 2 bytes + * so that the value never quite completely fills the range of a size_t, + * allowing the health check to verify that larger values are rejected. + */ +#define DRBG_MAX_ADDTL_BYTES (U32_MAX - 1) + +struct drbg_state { + struct mutex drbg_mutex; /* lock around DRBG */ + u8 V[DRBG_STATE_LEN]; /* internal state -- 10.1.2.1 1a */ + struct hmac_sha512_key key; /* current key -- 10.1.2.1 1b */ + /* Number of RNG requests since last reseed -- 10.1.2.1 1c */ + size_t reseed_ctr; + bool instantiated; + struct crypto_rng *jent; + const u8 *test_entropy; + size_t test_entropylen; }; -static int drbg_uninstantiate(struct drbg_state *drbg); - /****************************************************************** - * Generic helper functions + * HMAC DRBG functions ******************************************************************/ -/* - * Return strength of DRBG according to SP800-90A section 8.4 - * - * @flags DRBG flags reference - * - * Return: normalized strength in *bytes* value or 32 as default - * to counter programming errors - */ -static inline unsigned short drbg_sec_strength(drbg_flag_t flags) -{ - switch (flags & DRBG_STRENGTH_MASK) { - case DRBG_STRENGTH128: - return 16; - case DRBG_STRENGTH192: - return 24; - case DRBG_STRENGTH256: - return 32; - default: - return 32; - } -} - -/* - * FIPS 140-2 continuous self test for the noise source - * The test is performed on the noise source input data. Thus, the function - * implicitly knows the size of the buffer to be equal to the security - * strength. - * - * Note, this function disregards the nonce trailing the entropy data during - * initial seeding. - * - * drbg->drbg_mutex must have been taken. - * - * @drbg DRBG handle - * @entropy buffer of seed data to be checked - * - * return: - * %true on success - * %false when the CTRNG is not yet primed - */ -static bool drbg_fips_continuous_test(struct drbg_state *drbg, - const unsigned char *entropy) - __must_hold(&drbg->drbg_mutex) -{ - unsigned short entropylen = drbg_sec_strength(drbg->core->flags); - - if (!IS_ENABLED(CONFIG_CRYPTO_FIPS)) - return true; - - /* skip test if we test the overall system */ - if (list_empty(&drbg->test_data.list)) - return true; - /* only perform test in FIPS mode */ - if (!fips_enabled) - return true; - - if (!drbg->fips_primed) { - /* Priming of FIPS test */ - memcpy(drbg->prev, entropy, entropylen); - drbg->fips_primed = true; - /* priming: another round is needed */ - return false; - } - if (!memcmp(drbg->prev, entropy, entropylen)) - panic("DRBG continuous self test failed\n"); - memcpy(drbg->prev, entropy, entropylen); - - /* the test shall pass when the two values are not equal */ - return true; -} - -/****************************************************************** - * CTR DRBG callback functions - ******************************************************************/ - -#ifdef CONFIG_CRYPTO_DRBG_CTR -#define CRYPTO_DRBG_CTR_STRING "CTR " -MODULE_ALIAS_CRYPTO("drbg_pr_ctr_aes256"); -MODULE_ALIAS_CRYPTO("drbg_nopr_ctr_aes256"); -MODULE_ALIAS_CRYPTO("drbg_pr_ctr_aes192"); -MODULE_ALIAS_CRYPTO("drbg_nopr_ctr_aes192"); -MODULE_ALIAS_CRYPTO("drbg_pr_ctr_aes128"); -MODULE_ALIAS_CRYPTO("drbg_nopr_ctr_aes128"); - -static int drbg_init_sym_kernel(struct drbg_state *drbg); -static int drbg_fini_sym_kernel(struct drbg_state *drbg); -static int drbg_kcapi_sym_ctr(struct drbg_state *drbg, - u8 *inbuf, u32 inbuflen, - u8 *outbuf, u32 outlen); -#define DRBG_OUTSCRATCHLEN 256 - -static int drbg_ctr_df(struct drbg_state *drbg, - unsigned char *df_data, size_t bytes_to_return, - struct list_head *seedlist) -{ - return crypto_drbg_ctr_df(drbg->priv_data, df_data, drbg_statelen(drbg), - seedlist, drbg_blocklen(drbg), drbg_statelen(drbg)); -} - -/* - * update function of CTR DRBG as defined in 10.2.1.2 - * - * The reseed variable has an enhanced meaning compared to the update - * functions of the other DRBGs as follows: - * 0 => initial seed from initialization - * 1 => reseed via drbg_seed - * 2 => first invocation from drbg_ctr_update when addtl is present. In - * this case, the df_data scratchpad is not deleted so that it is - * available for another calls to prevent calling the DF function - * again. - * 3 => second invocation from drbg_ctr_update. When the update function - * was called with addtl, the df_data memory already contains the - * DFed addtl information and we do not need to call DF again. - */ -static int drbg_ctr_update(struct drbg_state *drbg, struct list_head *seed, - int reseed) -{ - int ret = -EFAULT; - /* 10.2.1.2 step 1 */ - unsigned char *temp = drbg->scratchpad; - unsigned char *df_data = drbg->scratchpad + drbg_statelen(drbg) + - drbg_blocklen(drbg); - - if (3 > reseed) - memset(df_data, 0, drbg_statelen(drbg)); - - if (!reseed) { - /* - * The DRBG uses the CTR mode of the underlying AES cipher. The - * CTR mode increments the counter value after the AES operation - * but SP800-90A requires that the counter is incremented before - * the AES operation. Hence, we increment it at the time we set - * it by one. - */ - crypto_inc(drbg->V, drbg_blocklen(drbg)); - - ret = crypto_skcipher_setkey(drbg->ctr_handle, drbg->C, - drbg_keylen(drbg)); - if (ret) - goto out; - } - - /* 10.2.1.3.2 step 2 and 10.2.1.4.2 step 2 */ - if (seed) { - ret = drbg_ctr_df(drbg, df_data, drbg_statelen(drbg), seed); - if (ret) - goto out; - } - - ret = drbg_kcapi_sym_ctr(drbg, df_data, drbg_statelen(drbg), - temp, drbg_statelen(drbg)); - if (ret) - return ret; - - /* 10.2.1.2 step 5 */ - ret = crypto_skcipher_setkey(drbg->ctr_handle, temp, - drbg_keylen(drbg)); - if (ret) - goto out; - /* 10.2.1.2 step 6 */ - memcpy(drbg->V, temp + drbg_keylen(drbg), drbg_blocklen(drbg)); - /* See above: increment counter by one to compensate timing of CTR op */ - crypto_inc(drbg->V, drbg_blocklen(drbg)); - ret = 0; - -out: - memset(temp, 0, drbg_statelen(drbg) + drbg_blocklen(drbg)); - if (2 != reseed) - memset(df_data, 0, drbg_statelen(drbg)); - return ret; -} - -/* - * scratchpad use: drbg_ctr_update is called independently from - * drbg_ctr_extract_bytes. Therefore, the scratchpad is reused - */ -/* Generate function of CTR DRBG as defined in 10.2.1.5.2 */ -static int drbg_ctr_generate(struct drbg_state *drbg, - unsigned char *buf, unsigned int buflen, - struct list_head *addtl) -{ - int ret; - int len = min_t(int, buflen, INT_MAX); - - /* 10.2.1.5.2 step 2 */ - if (addtl && !list_empty(addtl)) { - ret = drbg_ctr_update(drbg, addtl, 2); - if (ret) - return 0; - } - - /* 10.2.1.5.2 step 4.1 */ - ret = drbg_kcapi_sym_ctr(drbg, NULL, 0, buf, len); - if (ret) - return ret; - - /* 10.2.1.5.2 step 6 */ - ret = drbg_ctr_update(drbg, NULL, 3); - if (ret) - len = ret; - - return len; -} - -static const struct drbg_state_ops drbg_ctr_ops = { - .update = drbg_ctr_update, - .generate = drbg_ctr_generate, - .crypto_init = drbg_init_sym_kernel, - .crypto_fini = drbg_fini_sym_kernel, -}; -#endif /* CONFIG_CRYPTO_DRBG_CTR */ - -/****************************************************************** - * HMAC DRBG callback functions - ******************************************************************/ - -#if defined(CONFIG_CRYPTO_DRBG_HASH) || defined(CONFIG_CRYPTO_DRBG_HMAC) -static int drbg_kcapi_hash(struct drbg_state *drbg, unsigned char *outval, - const struct list_head *in); -static void drbg_kcapi_hmacsetkey(struct drbg_state *drbg, - const unsigned char *key); -static int drbg_init_hash_kernel(struct drbg_state *drbg); -static int drbg_fini_hash_kernel(struct drbg_state *drbg); -#endif /* (CONFIG_CRYPTO_DRBG_HASH || CONFIG_CRYPTO_DRBG_HMAC) */ - -#ifdef CONFIG_CRYPTO_DRBG_HMAC -#define CRYPTO_DRBG_HMAC_STRING "HMAC " -MODULE_ALIAS_CRYPTO("drbg_pr_hmac_sha512"); -MODULE_ALIAS_CRYPTO("drbg_nopr_hmac_sha512"); -MODULE_ALIAS_CRYPTO("drbg_pr_hmac_sha384"); -MODULE_ALIAS_CRYPTO("drbg_nopr_hmac_sha384"); -MODULE_ALIAS_CRYPTO("drbg_pr_hmac_sha256"); -MODULE_ALIAS_CRYPTO("drbg_nopr_hmac_sha256"); - /* update function of HMAC DRBG as defined in 10.1.2.2 */ -static int drbg_hmac_update(struct drbg_state *drbg, struct list_head *seed, - int reseed) +static void drbg_hmac_update(struct drbg_state *drbg, + const u8 *data1, size_t data1_len, + const u8 *data2, size_t data2_len) { - int ret = -EFAULT; - int i = 0; - struct drbg_string seed1, seed2, vdata; - LIST_HEAD(seedlist); - LIST_HEAD(vdatalist); + struct hmac_sha512_ctx hmac_ctx; + u8 new_key[DRBG_STATE_LEN]; - if (!reseed) { - /* 10.1.2.3 step 2 -- memset(0) of C is implicit with kzalloc */ - memset(drbg->V, 1, drbg_statelen(drbg)); - drbg_kcapi_hmacsetkey(drbg, drbg->C); - } - - drbg_string_fill(&seed1, drbg->V, drbg_statelen(drbg)); - list_add_tail(&seed1.list, &seedlist); - /* buffer of seed2 will be filled in for loop below with one byte */ - drbg_string_fill(&seed2, NULL, 1); - list_add_tail(&seed2.list, &seedlist); - /* input data of seed is allowed to be NULL at this point */ - if (seed) - list_splice_tail(seed, &seedlist); - - drbg_string_fill(&vdata, drbg->V, drbg_statelen(drbg)); - list_add_tail(&vdata.list, &vdatalist); - for (i = 2; 0 < i; i--) { - /* first round uses 0x0, second 0x1 */ - unsigned char prefix = DRBG_PREFIX0; - if (1 == i) - prefix = DRBG_PREFIX1; + for (u8 i = 0; i < 2; i++) { /* 10.1.2.2 step 1 and 4 -- concatenation and HMAC for key */ - seed2.buf = &prefix; - ret = drbg_kcapi_hash(drbg, drbg->C, &seedlist); - if (ret) - return ret; - drbg_kcapi_hmacsetkey(drbg, drbg->C); + hmac_sha512_init(&hmac_ctx, &drbg->key); + hmac_sha512_update(&hmac_ctx, drbg->V, DRBG_STATE_LEN); + hmac_sha512_update(&hmac_ctx, &i, 1); + hmac_sha512_update(&hmac_ctx, data1, data1_len); + hmac_sha512_update(&hmac_ctx, data2, data2_len); + hmac_sha512_final(&hmac_ctx, new_key); + hmac_sha512_preparekey(&drbg->key, new_key, DRBG_STATE_LEN); /* 10.1.2.2 step 2 and 5 -- HMAC for V */ - ret = drbg_kcapi_hash(drbg, drbg->V, &vdatalist); - if (ret) - return ret; + hmac_sha512(&drbg->key, drbg->V, DRBG_STATE_LEN, drbg->V); /* 10.1.2.2 step 3 */ - if (!seed) - return ret; + if (data1_len == 0 && data2_len == 0) + break; } - - return 0; + memzero_explicit(new_key, sizeof(new_key)); } /* generate function of HMAC DRBG as defined in 10.1.2.5 */ -static int drbg_hmac_generate(struct drbg_state *drbg, - unsigned char *buf, - unsigned int buflen, - struct list_head *addtl) +static void drbg_hmac_generate(struct drbg_state *drbg, u8 *out, size_t outlen, + const u8 *addtl1, size_t addtl1_len) { - int len = 0; - int ret = 0; - struct drbg_string data; - LIST_HEAD(datalist); + u8 addtl2[32]; + size_t addtl2_len = 0; - /* 10.1.2.5 step 2 */ - if (addtl && !list_empty(addtl)) { - ret = drbg_hmac_update(drbg, addtl, 1); - if (ret) - return ret; + /* + * Append some bytes from get_random_bytes() to the additional input + * string, except when in test mode (as it would break the tests). + * Using a nonempty additional input string works around the forward + * secrecy bug in HMAC_DRBG described by Woodage & Shumow (2018) + * (https://eprint.iacr.org/2018/349.pdf). Filling the string with + * get_random_bytes() rather than a fixed value is safer still, and in + * particular makes random.c reseeds be immediately reflected. + * + * Note that there's no need to pull bytes from jitterentropy here too, + * since FIPS doesn't require any entropy in the additional input. + */ + if (drbg->test_entropylen == 0) { + get_random_bytes(addtl2, sizeof(addtl2)); + addtl2_len = sizeof(addtl2); } - drbg_string_fill(&data, drbg->V, drbg_statelen(drbg)); - list_add_tail(&data.list, &datalist); - while (len < buflen) { - unsigned int outlen = 0; + /* 10.1.2.5 step 2 */ + if (addtl1_len || addtl2_len) + drbg_hmac_update(drbg, addtl1, addtl1_len, addtl2, addtl2_len); + + while (outlen) { + size_t n = min(DRBG_STATE_LEN, outlen); + /* 10.1.2.5 step 4.1 */ - ret = drbg_kcapi_hash(drbg, drbg->V, &datalist); - if (ret) - return ret; - outlen = (drbg_blocklen(drbg) < (buflen - len)) ? - drbg_blocklen(drbg) : (buflen - len); + hmac_sha512(&drbg->key, drbg->V, DRBG_STATE_LEN, drbg->V); /* 10.1.2.5 step 4.2 */ - memcpy(buf + len, drbg->V, outlen); - len += outlen; + memcpy(out, drbg->V, n); + out += n; + outlen -= n; } /* 10.1.2.5 step 6 */ - if (addtl && !list_empty(addtl)) - ret = drbg_hmac_update(drbg, addtl, 1); - else - ret = drbg_hmac_update(drbg, NULL, 1); - if (ret) - return ret; + drbg_hmac_update(drbg, addtl1, addtl1_len, addtl2, addtl2_len); - return len; -} - -static const struct drbg_state_ops drbg_hmac_ops = { - .update = drbg_hmac_update, - .generate = drbg_hmac_generate, - .crypto_init = drbg_init_hash_kernel, - .crypto_fini = drbg_fini_hash_kernel, -}; -#endif /* CONFIG_CRYPTO_DRBG_HMAC */ - -/****************************************************************** - * Hash DRBG callback functions - ******************************************************************/ - -#ifdef CONFIG_CRYPTO_DRBG_HASH -#define CRYPTO_DRBG_HASH_STRING "HASH " -MODULE_ALIAS_CRYPTO("drbg_pr_sha512"); -MODULE_ALIAS_CRYPTO("drbg_nopr_sha512"); -MODULE_ALIAS_CRYPTO("drbg_pr_sha384"); -MODULE_ALIAS_CRYPTO("drbg_nopr_sha384"); -MODULE_ALIAS_CRYPTO("drbg_pr_sha256"); -MODULE_ALIAS_CRYPTO("drbg_nopr_sha256"); - -/* - * Increment buffer - * - * @dst buffer to increment - * @add value to add - */ -static inline void drbg_add_buf(unsigned char *dst, size_t dstlen, - const unsigned char *add, size_t addlen) -{ - /* implied: dstlen > addlen */ - unsigned char *dstptr; - const unsigned char *addptr; - unsigned int remainder = 0; - size_t len = addlen; - - dstptr = dst + (dstlen-1); - addptr = add + (addlen-1); - while (len) { - remainder += *dstptr + *addptr; - *dstptr = remainder & 0xff; - remainder >>= 8; - len--; dstptr--; addptr--; - } - len = dstlen - addlen; - while (len && remainder > 0) { - remainder = *dstptr + 1; - *dstptr = remainder & 0xff; - remainder >>= 8; - len--; dstptr--; - } -} - -/* - * scratchpad usage: as drbg_hash_update and drbg_hash_df are used - * interlinked, the scratchpad is used as follows: - * drbg_hash_update - * start: drbg->scratchpad - * length: drbg_statelen(drbg) - * drbg_hash_df: - * start: drbg->scratchpad + drbg_statelen(drbg) - * length: drbg_blocklen(drbg) - * - * drbg_hash_process_addtl uses the scratchpad, but fully completes - * before either of the functions mentioned before are invoked. Therefore, - * drbg_hash_process_addtl does not need to be specifically considered. - */ - -/* Derivation Function for Hash DRBG as defined in 10.4.1 */ -static int drbg_hash_df(struct drbg_state *drbg, - unsigned char *outval, size_t outlen, - struct list_head *entropylist) -{ - int ret = 0; - size_t len = 0; - unsigned char input[5]; - unsigned char *tmp = drbg->scratchpad + drbg_statelen(drbg); - struct drbg_string data; - - /* 10.4.1 step 3 */ - input[0] = 1; - drbg_cpu_to_be32((outlen * 8), &input[1]); - - /* 10.4.1 step 4.1 -- concatenation of data for input into hash */ - drbg_string_fill(&data, input, 5); - list_add(&data.list, entropylist); - - /* 10.4.1 step 4 */ - while (len < outlen) { - short blocklen = 0; - /* 10.4.1 step 4.1 */ - ret = drbg_kcapi_hash(drbg, tmp, entropylist); - if (ret) - goto out; - /* 10.4.1 step 4.2 */ - input[0]++; - blocklen = (drbg_blocklen(drbg) < (outlen - len)) ? - drbg_blocklen(drbg) : (outlen - len); - memcpy(outval + len, tmp, blocklen); - len += blocklen; - } - -out: - memset(tmp, 0, drbg_blocklen(drbg)); - return ret; -} - -/* update function for Hash DRBG as defined in 10.1.1.2 / 10.1.1.3 */ -static int drbg_hash_update(struct drbg_state *drbg, struct list_head *seed, - int reseed) -{ - int ret = 0; - struct drbg_string data1, data2; - LIST_HEAD(datalist); - LIST_HEAD(datalist2); - unsigned char *V = drbg->scratchpad; - unsigned char prefix = DRBG_PREFIX1; - - if (!seed) - return -EINVAL; - - if (reseed) { - /* 10.1.1.3 step 1 */ - memcpy(V, drbg->V, drbg_statelen(drbg)); - drbg_string_fill(&data1, &prefix, 1); - list_add_tail(&data1.list, &datalist); - drbg_string_fill(&data2, V, drbg_statelen(drbg)); - list_add_tail(&data2.list, &datalist); - } - list_splice_tail(seed, &datalist); - - /* 10.1.1.2 / 10.1.1.3 step 2 and 3 */ - ret = drbg_hash_df(drbg, drbg->V, drbg_statelen(drbg), &datalist); - if (ret) - goto out; - - /* 10.1.1.2 / 10.1.1.3 step 4 */ - prefix = DRBG_PREFIX0; - drbg_string_fill(&data1, &prefix, 1); - list_add_tail(&data1.list, &datalist2); - drbg_string_fill(&data2, drbg->V, drbg_statelen(drbg)); - list_add_tail(&data2.list, &datalist2); - /* 10.1.1.2 / 10.1.1.3 step 4 */ - ret = drbg_hash_df(drbg, drbg->C, drbg_statelen(drbg), &datalist2); - -out: - memset(drbg->scratchpad, 0, drbg_statelen(drbg)); - return ret; -} - -/* processing of additional information string for Hash DRBG */ -static int drbg_hash_process_addtl(struct drbg_state *drbg, - struct list_head *addtl) -{ - int ret = 0; - struct drbg_string data1, data2; - LIST_HEAD(datalist); - unsigned char prefix = DRBG_PREFIX2; - - /* 10.1.1.4 step 2 */ - if (!addtl || list_empty(addtl)) - return 0; - - /* 10.1.1.4 step 2a */ - drbg_string_fill(&data1, &prefix, 1); - drbg_string_fill(&data2, drbg->V, drbg_statelen(drbg)); - list_add_tail(&data1.list, &datalist); - list_add_tail(&data2.list, &datalist); - list_splice_tail(addtl, &datalist); - ret = drbg_kcapi_hash(drbg, drbg->scratchpad, &datalist); - if (ret) - goto out; - - /* 10.1.1.4 step 2b */ - drbg_add_buf(drbg->V, drbg_statelen(drbg), - drbg->scratchpad, drbg_blocklen(drbg)); - -out: - memset(drbg->scratchpad, 0, drbg_blocklen(drbg)); - return ret; -} - -/* Hashgen defined in 10.1.1.4 */ -static int drbg_hash_hashgen(struct drbg_state *drbg, - unsigned char *buf, - unsigned int buflen) -{ - int len = 0; - int ret = 0; - unsigned char *src = drbg->scratchpad; - unsigned char *dst = drbg->scratchpad + drbg_statelen(drbg); - struct drbg_string data; - LIST_HEAD(datalist); - - /* 10.1.1.4 step hashgen 2 */ - memcpy(src, drbg->V, drbg_statelen(drbg)); - - drbg_string_fill(&data, src, drbg_statelen(drbg)); - list_add_tail(&data.list, &datalist); - while (len < buflen) { - unsigned int outlen = 0; - /* 10.1.1.4 step hashgen 4.1 */ - ret = drbg_kcapi_hash(drbg, dst, &datalist); - if (ret) { - len = ret; - goto out; - } - outlen = (drbg_blocklen(drbg) < (buflen - len)) ? - drbg_blocklen(drbg) : (buflen - len); - /* 10.1.1.4 step hashgen 4.2 */ - memcpy(buf + len, dst, outlen); - len += outlen; - /* 10.1.1.4 hashgen step 4.3 */ - if (len < buflen) - crypto_inc(src, drbg_statelen(drbg)); - } - -out: - memset(drbg->scratchpad, 0, - (drbg_statelen(drbg) + drbg_blocklen(drbg))); - return len; -} - -/* generate function for Hash DRBG as defined in 10.1.1.4 */ -static int drbg_hash_generate(struct drbg_state *drbg, - unsigned char *buf, unsigned int buflen, - struct list_head *addtl) -{ - int len = 0; - int ret = 0; - union { - unsigned char req[8]; - __be64 req_int; - } u; - unsigned char prefix = DRBG_PREFIX3; - struct drbg_string data1, data2; - LIST_HEAD(datalist); - - /* 10.1.1.4 step 2 */ - ret = drbg_hash_process_addtl(drbg, addtl); - if (ret) - return ret; - /* 10.1.1.4 step 3 */ - len = drbg_hash_hashgen(drbg, buf, buflen); - - /* this is the value H as documented in 10.1.1.4 */ - /* 10.1.1.4 step 4 */ - drbg_string_fill(&data1, &prefix, 1); - list_add_tail(&data1.list, &datalist); - drbg_string_fill(&data2, drbg->V, drbg_statelen(drbg)); - list_add_tail(&data2.list, &datalist); - ret = drbg_kcapi_hash(drbg, drbg->scratchpad, &datalist); - if (ret) { - len = ret; - goto out; - } - - /* 10.1.1.4 step 5 */ - drbg_add_buf(drbg->V, drbg_statelen(drbg), - drbg->scratchpad, drbg_blocklen(drbg)); - drbg_add_buf(drbg->V, drbg_statelen(drbg), - drbg->C, drbg_statelen(drbg)); - u.req_int = cpu_to_be64(drbg->reseed_ctr); - drbg_add_buf(drbg->V, drbg_statelen(drbg), u.req, 8); - -out: - memset(drbg->scratchpad, 0, drbg_blocklen(drbg)); - return len; -} - -/* - * scratchpad usage: as update and generate are used isolated, both - * can use the scratchpad - */ -static const struct drbg_state_ops drbg_hash_ops = { - .update = drbg_hash_update, - .generate = drbg_hash_generate, - .crypto_init = drbg_init_hash_kernel, - .crypto_fini = drbg_fini_hash_kernel, -}; -#endif /* CONFIG_CRYPTO_DRBG_HASH */ - -/****************************************************************** - * Functions common for DRBG implementations - ******************************************************************/ - -static inline int __drbg_seed(struct drbg_state *drbg, struct list_head *seed, - int reseed, enum drbg_seed_state new_seed_state) -{ - int ret = drbg->d_ops->update(drbg, seed, reseed); - - if (ret) - return ret; - - drbg->seeded = new_seed_state; - drbg->last_seed_time = jiffies; - /* 10.1.1.2 / 10.1.1.3 step 5 */ - drbg->reseed_ctr = 1; - - switch (drbg->seeded) { - case DRBG_SEED_STATE_UNSEEDED: - /* Impossible, but handle it to silence compiler warnings. */ - fallthrough; - case DRBG_SEED_STATE_PARTIAL: - /* - * Require frequent reseeds until the seed source is - * fully initialized. - */ - drbg->reseed_threshold = 50; - break; - - case DRBG_SEED_STATE_FULL: - /* - * Seed source has become fully initialized, frequent - * reseeds no longer required. - */ - drbg->reseed_threshold = drbg_max_requests(drbg); - break; - } - - return ret; -} - -static inline void drbg_get_random_bytes(struct drbg_state *drbg, - unsigned char *entropy, - unsigned int entropylen) - __must_hold(&drbg->drbg_mutex) -{ - do - get_random_bytes(entropy, entropylen); - while (!drbg_fips_continuous_test(drbg, entropy)); -} - -static int drbg_seed_from_random(struct drbg_state *drbg) - __must_hold(&drbg->drbg_mutex) -{ - struct drbg_string data; - LIST_HEAD(seedlist); - unsigned int entropylen = drbg_sec_strength(drbg->core->flags); - unsigned char entropy[32]; - int ret; - - BUG_ON(!entropylen); - BUG_ON(entropylen > sizeof(entropy)); - - drbg_string_fill(&data, entropy, entropylen); - list_add_tail(&data.list, &seedlist); - - drbg_get_random_bytes(drbg, entropy, entropylen); - - ret = __drbg_seed(drbg, &seedlist, true, DRBG_SEED_STATE_FULL); - - memzero_explicit(entropy, entropylen); - return ret; -} - -static bool drbg_nopr_reseed_interval_elapsed(struct drbg_state *drbg) -{ - unsigned long next_reseed; - - /* Don't ever reseed from get_random_bytes() in test mode. */ - if (list_empty(&drbg->test_data.list)) - return false; - - /* - * Obtain fresh entropy for the nopr DRBGs after 300s have - * elapsed in order to still achieve sort of partial - * prediction resistance over the time domain at least. Note - * that the period of 300s has been chosen to match the - * CRNG_RESEED_INTERVAL of the get_random_bytes()' chacha - * rngs. - */ - next_reseed = drbg->last_seed_time + 300 * HZ; - return time_after(jiffies, next_reseed); + memzero_explicit(addtl2, sizeof(addtl2)); } /* @@ -901,33 +218,32 @@ static bool drbg_nopr_reseed_interval_elapsed(struct drbg_state *drbg) * * @drbg: DRBG state struct * @pers: personalization / additional information buffer - * @reseed: 0 for initial seed process, 1 for reseeding + * @pers_len: length of @pers in bytes + * @reseed: false for initial seeding (instantiation), true for reseeding * * return: * 0 on success * error value otherwise */ -static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers, +static int drbg_seed(struct drbg_state *drbg, const u8 *pers, size_t pers_len, bool reseed) __must_hold(&drbg->drbg_mutex) { int ret; - unsigned char entropy[((32 + 16) * 2)]; - unsigned int entropylen = drbg_sec_strength(drbg->core->flags); - struct drbg_string data1; - LIST_HEAD(seedlist); - enum drbg_seed_state new_seed_state = DRBG_SEED_STATE_FULL; + u8 entropy_buf[(32 + 16) * 2]; + size_t entropylen; + const u8 *entropy; /* 9.1 / 9.2 / 9.3.1 step 3 */ - if (pers && pers->len > (drbg_max_addtl(drbg))) { + if (pers_len > DRBG_MAX_ADDTL_BYTES) { pr_devel("DRBG: personalization string too long %zu\n", - pers->len); + pers_len); return -EINVAL; } - if (list_empty(&drbg->test_data.list)) { - drbg_string_fill(&data1, drbg->test_data.buf, - drbg->test_data.len); + if (drbg->test_entropylen) { + entropy = drbg->test_entropy; + entropylen = drbg->test_entropylen; pr_devel("DRBG: using test entropy\n"); } else { /* @@ -938,20 +254,18 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers, * of the strength. The consideration of a nonce is only * applicable during initial seeding. */ - BUG_ON(!entropylen); + entropy = entropy_buf; if (!reseed) - entropylen = ((entropylen + 1) / 2) * 3; - BUG_ON((entropylen * 2) > sizeof(entropy)); + entropylen = ((DRBG_SEC_STRENGTH + 1) / 2) * 3; + else + entropylen = DRBG_SEC_STRENGTH; + BUG_ON(entropylen * 2 > sizeof(entropy_buf)); /* Get seed from in-kernel /dev/urandom */ - if (!rng_is_initialized()) - new_seed_state = DRBG_SEED_STATE_PARTIAL; - - drbg_get_random_bytes(drbg, entropy, entropylen); + get_random_bytes(entropy_buf, entropylen); if (!drbg->jent) { - drbg_string_fill(&data1, entropy, entropylen); - pr_devel("DRBG: (re)seeding with %u bytes of entropy\n", + pr_devel("DRBG: (re)seeding with %zu bytes of entropy\n", entropylen); } else { /* @@ -959,7 +273,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers, * fatal only in FIPS mode. */ ret = crypto_rng_get_bytes(drbg->jent, - entropy + entropylen, + &entropy_buf[entropylen], entropylen); if (fips_enabled && ret) { pr_devel("DRBG: jent failed with %d\n", ret); @@ -982,238 +296,94 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers, goto out; } - drbg_string_fill(&data1, entropy, entropylen * 2); - pr_devel("DRBG: (re)seeding with %u bytes of entropy\n", - entropylen * 2); + entropylen *= 2; + pr_devel("DRBG: (re)seeding with %zu bytes of entropy\n", + entropylen); } } - list_add_tail(&data1.list, &seedlist); - /* - * concatenation of entropy with personalization str / addtl input) - * the variable pers is directly handed in by the caller, so check its - * contents whether it is appropriate - */ - if (pers && pers->buf && 0 < pers->len) { - list_add_tail(&pers->list, &seedlist); + if (pers_len) pr_devel("DRBG: using personalization string\n"); - } - - if (!reseed) { - memset(drbg->V, 0, drbg_statelen(drbg)); - memset(drbg->C, 0, drbg_statelen(drbg)); - } - - ret = __drbg_seed(drbg, &seedlist, reseed, new_seed_state); + drbg_hmac_update(drbg, entropy, entropylen, pers, pers_len); + drbg->reseed_ctr = 1; + ret = 0; out: - memzero_explicit(entropy, entropylen * 2); + memzero_explicit(entropy_buf, sizeof(entropy_buf)); return ret; } -/* Free all substructures in a DRBG state without the DRBG state structure */ -static inline void drbg_dealloc_state(struct drbg_state *drbg) -{ - if (!drbg) - return; - kfree_sensitive(drbg->Vbuf); - drbg->Vbuf = NULL; - drbg->V = NULL; - kfree_sensitive(drbg->Cbuf); - drbg->Cbuf = NULL; - drbg->C = NULL; - kfree_sensitive(drbg->scratchpadbuf); - drbg->scratchpadbuf = NULL; - drbg->reseed_ctr = 0; - drbg->d_ops = NULL; - drbg->core = NULL; - if (IS_ENABLED(CONFIG_CRYPTO_FIPS)) { - kfree_sensitive(drbg->prev); - drbg->prev = NULL; - drbg->fips_primed = false; - } -} - /* - * Allocate all sub-structures for a DRBG state. - * The DRBG state structure must already be allocated. - */ -static inline int drbg_alloc_state(struct drbg_state *drbg) -{ - int ret = -ENOMEM; - unsigned int sb_size = 0; - - switch (drbg->core->flags & DRBG_TYPE_MASK) { -#ifdef CONFIG_CRYPTO_DRBG_HMAC - case DRBG_HMAC: - drbg->d_ops = &drbg_hmac_ops; - break; -#endif /* CONFIG_CRYPTO_DRBG_HMAC */ -#ifdef CONFIG_CRYPTO_DRBG_HASH - case DRBG_HASH: - drbg->d_ops = &drbg_hash_ops; - break; -#endif /* CONFIG_CRYPTO_DRBG_HASH */ -#ifdef CONFIG_CRYPTO_DRBG_CTR - case DRBG_CTR: - drbg->d_ops = &drbg_ctr_ops; - break; -#endif /* CONFIG_CRYPTO_DRBG_CTR */ - default: - ret = -EOPNOTSUPP; - goto err; - } - - ret = drbg->d_ops->crypto_init(drbg); - if (ret < 0) - goto err; - - drbg->Vbuf = kmalloc(drbg_statelen(drbg) + ret, GFP_KERNEL); - if (!drbg->Vbuf) { - ret = -ENOMEM; - goto fini; - } - drbg->V = PTR_ALIGN(drbg->Vbuf, ret + 1); - drbg->Cbuf = kmalloc(drbg_statelen(drbg) + ret, GFP_KERNEL); - if (!drbg->Cbuf) { - ret = -ENOMEM; - goto fini; - } - drbg->C = PTR_ALIGN(drbg->Cbuf, ret + 1); - /* scratchpad is only generated for CTR and Hash */ - if (drbg->core->flags & DRBG_HMAC) - sb_size = 0; - else if (drbg->core->flags & DRBG_CTR) - sb_size = drbg_statelen(drbg) + drbg_blocklen(drbg) + /* temp */ - crypto_drbg_ctr_df_datalen(drbg_statelen(drbg), - drbg_blocklen(drbg)); - else - sb_size = drbg_statelen(drbg) + drbg_blocklen(drbg); - - if (0 < sb_size) { - drbg->scratchpadbuf = kzalloc(sb_size + ret, GFP_KERNEL); - if (!drbg->scratchpadbuf) { - ret = -ENOMEM; - goto fini; - } - drbg->scratchpad = PTR_ALIGN(drbg->scratchpadbuf, ret + 1); - } - - if (IS_ENABLED(CONFIG_CRYPTO_FIPS)) { - drbg->prev = kzalloc(drbg_sec_strength(drbg->core->flags), - GFP_KERNEL); - if (!drbg->prev) { - ret = -ENOMEM; - goto fini; - } - drbg->fips_primed = false; - } - - return 0; - -fini: - drbg->d_ops->crypto_fini(drbg); -err: - drbg_dealloc_state(drbg); - return ret; -} - -/************************************************************************* - * DRBG interface functions - *************************************************************************/ - -/* - * DRBG generate function as required by SP800-90A - this function - * generates random numbers + * Generate random bytes from an SP800-90A DRBG. * * @drbg DRBG state handle - * @buf Buffer where to store the random numbers -- the buffer must already - * be pre-allocated by caller - * @buflen Length of output buffer - this value defines the number of random - * bytes pulled from DRBG - * @addtl Additional input that is mixed into state, may be NULL -- note - * the entropy is pulled by the DRBG internally unconditionally - * as defined in SP800-90A. The additional input is mixed into - * the state in addition to the pulled entropy. + * @out Buffer where to store the random bytes + * @outlen Number of random bytes to generate + * @addtl Optional additional input that is mixed into state + * @addtl_len Length of @addtl in bytes, may be 0 * * return: 0 when all bytes are generated; < 0 in case of an error */ -static int drbg_generate(struct drbg_state *drbg, - unsigned char *buf, unsigned int buflen, - struct drbg_string *addtl) +static int drbg_generate(struct drbg_state *drbg, u8 *out, size_t outlen, + const u8 *addtl, size_t addtl_len) __must_hold(&drbg->drbg_mutex) { - int len = 0; - LIST_HEAD(addtllist); + int err; - if (!drbg->core) { - pr_devel("DRBG: not yet seeded\n"); + if (!drbg->instantiated) { + pr_devel("DRBG: not yet instantiated\n"); return -EINVAL; } - if (0 == buflen || !buf) { + if (out == NULL || outlen == 0) { pr_devel("DRBG: no output buffer provided\n"); return -EINVAL; } - if (addtl && NULL == addtl->buf && 0 < addtl->len) { + if (addtl == NULL && addtl_len != 0) { pr_devel("DRBG: wrong format of additional information\n"); return -EINVAL; } /* 9.3.1 step 2 */ - len = -EINVAL; - if (buflen > (drbg_max_request_bytes(drbg))) { - pr_devel("DRBG: requested random numbers too large %u\n", - buflen); - goto err; + if (outlen > DRBG_MAX_REQUEST_BYTES) { + pr_devel("DRBG: request length is too long %zu\n", outlen); + return -EINVAL; } /* 9.3.1 step 3 is implicit with the chosen DRBG */ /* 9.3.1 step 4 */ - if (addtl && addtl->len > (drbg_max_addtl(drbg))) { + if (addtl_len > DRBG_MAX_ADDTL_BYTES) { pr_devel("DRBG: additional information string too long %zu\n", - addtl->len); - goto err; + addtl_len); + return -EINVAL; } /* 9.3.1 step 5 is implicit with the chosen DRBG */ /* * 9.3.1 step 6 and 9 supplemented by 9.3.2 step c is implemented * here. The spec is a bit convoluted here, we make it simpler. + * + * We no longer try to detect when random.c has reseeded itself and call + * drbg_seed() then too, since drbg_hmac_generate() adds bytes from + * random.c to the additional input, which is a de facto reseed anyway. */ - if (drbg->reseed_threshold < drbg->reseed_ctr) - drbg->seeded = DRBG_SEED_STATE_UNSEEDED; - - if (drbg->pr || drbg->seeded == DRBG_SEED_STATE_UNSEEDED) { - pr_devel("DRBG: reseeding before generation (prediction " - "resistance: %s, state %s)\n", - str_true_false(drbg->pr), - (drbg->seeded == DRBG_SEED_STATE_FULL ? - "seeded" : "unseeded")); + if (drbg->reseed_ctr > DRBG_MAX_REQUESTS) { + pr_devel("DRBG: reseeding before generation\n"); /* 9.3.1 steps 7.1 through 7.3 */ - len = drbg_seed(drbg, addtl, true); - if (len) - goto err; + err = drbg_seed(drbg, addtl, addtl_len, true); + if (err) + return err; /* 9.3.1 step 7.4 */ addtl = NULL; - } else if (rng_is_initialized() && - (drbg->seeded == DRBG_SEED_STATE_PARTIAL || - drbg_nopr_reseed_interval_elapsed(drbg))) { - len = drbg_seed_from_random(drbg); - if (len) - goto err; + addtl_len = 0; } - if (addtl && 0 < addtl->len) - list_add_tail(&addtl->list, &addtllist); /* 9.3.1 step 8 and 10 */ - len = drbg->d_ops->generate(drbg, buf, buflen, &addtllist); + drbg_hmac_generate(drbg, out, outlen, addtl, addtl_len); - /* 10.1.1.4 step 6, 10.1.2.5 step 7, 10.2.1.5.2 step 7 */ + /* 10.1.2.5 step 7 */ drbg->reseed_ctr++; - if (0 >= len) - goto err; /* * Section 11.3.3 requires to re-perform self tests after some @@ -1226,437 +396,15 @@ static int drbg_generate(struct drbg_state *drbg, * In this case, the entire kernel operation is questionable and it * is unlikely that the integrity violation only affects the * correct operation of the DRBG. - * - * Albeit the following code is commented out, it is provided in - * case somebody has a need to implement the test of 11.3.3. - */ -#if 0 - if (drbg->reseed_ctr && !(drbg->reseed_ctr % 4096)) { - int err = 0; - pr_devel("DRBG: start to perform self test\n"); - if (drbg->core->flags & DRBG_HMAC) - err = alg_test("drbg_pr_hmac_sha512", - "drbg_pr_hmac_sha512", 0, 0); - else if (drbg->core->flags & DRBG_CTR) - err = alg_test("drbg_pr_ctr_aes256", - "drbg_pr_ctr_aes256", 0, 0); - else - err = alg_test("drbg_pr_sha256", - "drbg_pr_sha256", 0, 0); - if (err) { - pr_err("DRBG: periodical self test failed\n"); - /* - * uninstantiate implies that from now on, only errors - * are returned when reusing this DRBG cipher handle - */ - drbg_uninstantiate(drbg); - return 0; - } else { - pr_devel("DRBG: self test successful\n"); - } - } -#endif - - /* - * All operations were successful, return 0 as mandated by - * the kernel crypto API interface. - */ - len = 0; -err: - return len; -} - -/* - * Wrapper around drbg_generate which can pull arbitrary long strings - * from the DRBG without hitting the maximum request limitation. - * - * Parameters: see drbg_generate - * Return codes: see drbg_generate -- if one drbg_generate request fails, - * the entire drbg_generate_long request fails - */ -static int drbg_generate_long(struct drbg_state *drbg, - unsigned char *buf, unsigned int buflen, - struct drbg_string *addtl) -{ - unsigned int len = 0; - unsigned int slice = 0; - do { - int err = 0; - unsigned int chunk = 0; - slice = ((buflen - len) / drbg_max_request_bytes(drbg)); - chunk = slice ? drbg_max_request_bytes(drbg) : (buflen - len); - mutex_lock(&drbg->drbg_mutex); - err = drbg_generate(drbg, buf + len, chunk, addtl); - mutex_unlock(&drbg->drbg_mutex); - if (0 > err) - return err; - len += chunk; - } while (slice > 0 && (len < buflen)); - return 0; -} - -static int drbg_prepare_hrng(struct drbg_state *drbg) -{ - /* We do not need an HRNG in test mode. */ - if (list_empty(&drbg->test_data.list)) - return 0; - - drbg->jent = crypto_alloc_rng("jitterentropy_rng", 0, 0); - if (IS_ERR(drbg->jent)) { - const int err = PTR_ERR(drbg->jent); - - drbg->jent = NULL; - if (fips_enabled) - return err; - pr_info("DRBG: Continuing without Jitter RNG\n"); - } - - return 0; -} - -/* - * DRBG instantiation function as required by SP800-90A - this function - * sets up the DRBG handle, performs the initial seeding and all sanity - * checks required by SP800-90A - * - * @drbg memory of state -- if NULL, new memory is allocated - * @pers Personalization string that is mixed into state, may be NULL -- note - * the entropy is pulled by the DRBG internally unconditionally - * as defined in SP800-90A. The additional input is mixed into - * the state in addition to the pulled entropy. - * @coreref reference to core - * @pr prediction resistance enabled - * - * return - * 0 on success - * error value otherwise - */ -static int drbg_instantiate(struct drbg_state *drbg, struct drbg_string *pers, - int coreref, bool pr) -{ - int ret; - bool reseed = true; - - pr_devel("DRBG: Initializing DRBG core %d with prediction resistance " - "%s\n", coreref, str_enabled_disabled(pr)); - mutex_lock(&drbg->drbg_mutex); - - /* 9.1 step 1 is implicit with the selected DRBG type */ - - /* - * 9.1 step 2 is implicit as caller can select prediction resistance - * and the flag is copied into drbg->flags -- - * all DRBG types support prediction resistance */ - /* 9.1 step 4 is implicit in drbg_sec_strength */ - - if (!drbg->core) { - drbg->core = &drbg_cores[coreref]; - drbg->pr = pr; - drbg->seeded = DRBG_SEED_STATE_UNSEEDED; - drbg->last_seed_time = 0; - drbg->reseed_threshold = drbg_max_requests(drbg); - - ret = drbg_alloc_state(drbg); - if (ret) - goto unlock; - - ret = drbg_prepare_hrng(drbg); - if (ret) - goto free_everything; - - reseed = false; - } - - ret = drbg_seed(drbg, pers, reseed); - - if (ret && !reseed) - goto free_everything; - - mutex_unlock(&drbg->drbg_mutex); - return ret; - -unlock: - mutex_unlock(&drbg->drbg_mutex); - return ret; - -free_everything: - mutex_unlock(&drbg->drbg_mutex); - drbg_uninstantiate(drbg); - return ret; -} - -/* - * DRBG uninstantiate function as required by SP800-90A - this function - * frees all buffers and the DRBG handle - * - * @drbg DRBG state handle - * - * return - * 0 on success - */ -static int drbg_uninstantiate(struct drbg_state *drbg) -{ - if (!IS_ERR_OR_NULL(drbg->jent)) - crypto_free_rng(drbg->jent); - drbg->jent = NULL; - - if (drbg->d_ops) - drbg->d_ops->crypto_fini(drbg); - drbg_dealloc_state(drbg); - /* no scrubbing of test_data -- this shall survive an uninstantiate */ return 0; } -/* - * Helper function for setting the test data in the DRBG - * - * @drbg DRBG state handle - * @data test data - * @len test data length - */ -static void drbg_kcapi_set_entropy(struct crypto_rng *tfm, - const u8 *data, unsigned int len) -{ - struct drbg_state *drbg = crypto_rng_ctx(tfm); - - mutex_lock(&drbg->drbg_mutex); - drbg_string_fill(&drbg->test_data, data, len); - mutex_unlock(&drbg->drbg_mutex); -} - /*************************************************************** - * Kernel crypto API cipher invocations requested by DRBG + * Kernel crypto API interface to DRBG ***************************************************************/ -#if defined(CONFIG_CRYPTO_DRBG_HASH) || defined(CONFIG_CRYPTO_DRBG_HMAC) -struct sdesc { - struct shash_desc shash; -}; - -static int drbg_init_hash_kernel(struct drbg_state *drbg) -{ - struct sdesc *sdesc; - struct crypto_shash *tfm; - - tfm = crypto_alloc_shash(drbg->core->backend_cra_name, 0, 0); - if (IS_ERR(tfm)) { - pr_info("DRBG: could not allocate digest TFM handle: %s\n", - drbg->core->backend_cra_name); - return PTR_ERR(tfm); - } - BUG_ON(drbg_blocklen(drbg) != crypto_shash_digestsize(tfm)); - sdesc = kzalloc(sizeof(struct shash_desc) + crypto_shash_descsize(tfm), - GFP_KERNEL); - if (!sdesc) { - crypto_free_shash(tfm); - return -ENOMEM; - } - - sdesc->shash.tfm = tfm; - drbg->priv_data = sdesc; - - return 0; -} - -static int drbg_fini_hash_kernel(struct drbg_state *drbg) -{ - struct sdesc *sdesc = drbg->priv_data; - if (sdesc) { - crypto_free_shash(sdesc->shash.tfm); - kfree_sensitive(sdesc); - } - drbg->priv_data = NULL; - return 0; -} - -static void drbg_kcapi_hmacsetkey(struct drbg_state *drbg, - const unsigned char *key) -{ - struct sdesc *sdesc = drbg->priv_data; - - crypto_shash_setkey(sdesc->shash.tfm, key, drbg_statelen(drbg)); -} - -static int drbg_kcapi_hash(struct drbg_state *drbg, unsigned char *outval, - const struct list_head *in) -{ - struct sdesc *sdesc = drbg->priv_data; - struct drbg_string *input = NULL; - - crypto_shash_init(&sdesc->shash); - list_for_each_entry(input, in, list) - crypto_shash_update(&sdesc->shash, input->buf, input->len); - return crypto_shash_final(&sdesc->shash, outval); -} -#endif /* (CONFIG_CRYPTO_DRBG_HASH || CONFIG_CRYPTO_DRBG_HMAC) */ - -#ifdef CONFIG_CRYPTO_DRBG_CTR -static int drbg_fini_sym_kernel(struct drbg_state *drbg) -{ - struct aes_enckey *aeskey = drbg->priv_data; - - kfree(aeskey); - drbg->priv_data = NULL; - - if (drbg->ctr_handle) - crypto_free_skcipher(drbg->ctr_handle); - drbg->ctr_handle = NULL; - - if (drbg->ctr_req) - skcipher_request_free(drbg->ctr_req); - drbg->ctr_req = NULL; - - kfree(drbg->outscratchpadbuf); - drbg->outscratchpadbuf = NULL; - - return 0; -} - -static int drbg_init_sym_kernel(struct drbg_state *drbg) -{ - struct aes_enckey *aeskey; - struct crypto_skcipher *sk_tfm; - struct skcipher_request *req; - unsigned int alignmask; - char ctr_name[CRYPTO_MAX_ALG_NAME]; - - aeskey = kzalloc_obj(*aeskey); - if (!aeskey) - return -ENOMEM; - drbg->priv_data = aeskey; - - if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", - drbg->core->backend_cra_name) >= CRYPTO_MAX_ALG_NAME) { - drbg_fini_sym_kernel(drbg); - return -EINVAL; - } - sk_tfm = crypto_alloc_skcipher(ctr_name, 0, 0); - if (IS_ERR(sk_tfm)) { - pr_info("DRBG: could not allocate CTR cipher TFM handle: %s\n", - ctr_name); - drbg_fini_sym_kernel(drbg); - return PTR_ERR(sk_tfm); - } - drbg->ctr_handle = sk_tfm; - crypto_init_wait(&drbg->ctr_wait); - - req = skcipher_request_alloc(sk_tfm, GFP_KERNEL); - if (!req) { - pr_info("DRBG: could not allocate request queue\n"); - drbg_fini_sym_kernel(drbg); - return -ENOMEM; - } - drbg->ctr_req = req; - skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP, - crypto_req_done, &drbg->ctr_wait); - - alignmask = crypto_skcipher_alignmask(sk_tfm); - drbg->outscratchpadbuf = kmalloc(DRBG_OUTSCRATCHLEN + alignmask, - GFP_KERNEL); - if (!drbg->outscratchpadbuf) { - drbg_fini_sym_kernel(drbg); - return -ENOMEM; - } - drbg->outscratchpad = (u8 *)PTR_ALIGN(drbg->outscratchpadbuf, - alignmask + 1); - - sg_init_table(&drbg->sg_in, 1); - sg_init_one(&drbg->sg_out, drbg->outscratchpad, DRBG_OUTSCRATCHLEN); - - return alignmask; -} - -static int drbg_kcapi_sym_ctr(struct drbg_state *drbg, - u8 *inbuf, u32 inlen, - u8 *outbuf, u32 outlen) -{ - struct scatterlist *sg_in = &drbg->sg_in, *sg_out = &drbg->sg_out; - u32 scratchpad_use = min_t(u32, outlen, DRBG_OUTSCRATCHLEN); - int ret; - - if (inbuf) { - /* Use caller-provided input buffer */ - sg_set_buf(sg_in, inbuf, inlen); - } else { - /* Use scratchpad for in-place operation */ - inlen = scratchpad_use; - memset(drbg->outscratchpad, 0, scratchpad_use); - sg_set_buf(sg_in, drbg->outscratchpad, scratchpad_use); - } - - while (outlen) { - u32 cryptlen = min3(inlen, outlen, (u32)DRBG_OUTSCRATCHLEN); - - /* Output buffer may not be valid for SGL, use scratchpad */ - skcipher_request_set_crypt(drbg->ctr_req, sg_in, sg_out, - cryptlen, drbg->V); - ret = crypto_wait_req(crypto_skcipher_encrypt(drbg->ctr_req), - &drbg->ctr_wait); - if (ret) - goto out; - - crypto_init_wait(&drbg->ctr_wait); - - memcpy(outbuf, drbg->outscratchpad, cryptlen); - memzero_explicit(drbg->outscratchpad, cryptlen); - - outlen -= cryptlen; - outbuf += cryptlen; - } - ret = 0; - -out: - return ret; -} -#endif /* CONFIG_CRYPTO_DRBG_CTR */ - -/*************************************************************** - * Kernel crypto API interface to register DRBG - ***************************************************************/ - -/* - * Look up the DRBG flags by given kernel crypto API cra_name - * The code uses the drbg_cores definition to do this - * - * @cra_name kernel crypto API cra_name - * @coreref reference to integer which is filled with the pointer to - * the applicable core - * @pr reference for setting prediction resistance - * - * return: flags - */ -static inline void drbg_convert_tfm_core(const char *cra_driver_name, - int *coreref, bool *pr) -{ - int i = 0; - size_t start = 0; - int len = 0; - - *pr = true; - /* disassemble the names */ - if (!memcmp(cra_driver_name, "drbg_nopr_", 10)) { - start = 10; - *pr = false; - } else if (!memcmp(cra_driver_name, "drbg_pr_", 8)) { - start = 8; - } else { - return; - } - - /* remove the first part */ - len = strlen(cra_driver_name) - start; - for (i = 0; ARRAY_SIZE(drbg_cores) > i; i++) { - if (!memcmp(cra_driver_name + start, drbg_cores[i].cra_name, - len)) { - *coreref = i; - return; - } - } -} - static int drbg_kcapi_init(struct crypto_tfm *tfm) { struct drbg_state *drbg = crypto_tfm_ctx(tfm); @@ -1666,64 +414,108 @@ static int drbg_kcapi_init(struct crypto_tfm *tfm) return 0; } -static void drbg_kcapi_cleanup(struct crypto_tfm *tfm) +/* Set test entropy in the DRBG. */ +static void drbg_kcapi_set_entropy(struct crypto_rng *tfm, + const u8 *data, unsigned int len) { - drbg_uninstantiate(crypto_tfm_ctx(tfm)); + struct drbg_state *drbg = crypto_rng_ctx(tfm); + + mutex_lock(&drbg->drbg_mutex); + drbg->test_entropy = data; + drbg->test_entropylen = len; + mutex_unlock(&drbg->drbg_mutex); +} + +/* Seed (i.e. instantiate) or re-seed the DRBG. */ +static int drbg_kcapi_seed(struct crypto_rng *tfm, + const u8 *pers, unsigned int pers_len) +{ + static const u8 initial_key[DRBG_STATE_LEN]; /* all zeroes */ + struct drbg_state *drbg = crypto_rng_ctx(tfm); + int ret; + + pr_devel("DRBG: Initializing DRBG\n"); + guard(mutex)(&drbg->drbg_mutex); + + if (drbg->instantiated) + return drbg_seed(drbg, pers, pers_len, /* reseed= */ true); + + /* 9.1 step 1 is implicit with the selected DRBG type */ + + /* + * 9.1 step 2 is implicit, as this implementation doesn't support + * prediction resistance + */ + + /* 9.1 step 4 is implicit in DRBG_SEC_STRENGTH */ + + memset(drbg->V, 1, DRBG_STATE_LEN); + hmac_sha512_preparekey(&drbg->key, initial_key, DRBG_STATE_LEN); + + /* Allocate jitterentropy_rng if not in test mode. */ + if (drbg->test_entropylen == 0) { + drbg->jent = crypto_alloc_rng("jitterentropy_rng", 0, 0); + if (IS_ERR(drbg->jent)) { + ret = PTR_ERR(drbg->jent); + drbg->jent = NULL; + if (fips_enabled) + return ret; + pr_info("DRBG: Continuing without Jitter RNG\n"); + } + } + + ret = drbg_seed(drbg, pers, pers_len, /* reseed= */ false); + if (ret) { + crypto_free_rng(drbg->jent); + drbg->jent = NULL; + return ret; + } + drbg->instantiated = true; + return 0; } /* * Generate random numbers invoked by the kernel crypto API: - * The API of the kernel crypto API is extended as follows: * * src is additional input supplied to the RNG. * slen is the length of src. * dst is the output buffer where random data is to be stored. * dlen is the length of dst. */ -static int drbg_kcapi_random(struct crypto_rng *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int dlen) +static int drbg_kcapi_generate(struct crypto_rng *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int dlen) { struct drbg_state *drbg = crypto_rng_ctx(tfm); - struct drbg_string *addtl = NULL; - struct drbg_string string; - if (slen) { - /* linked list variable is now local to allow modification */ - drbg_string_fill(&string, src, slen); - addtl = &string; - } + /* + * Break the request into multiple requests if needed, to avoid + * exceeding the maximum request length of the core algorithm. + */ + do { + unsigned int n = min(dlen, DRBG_MAX_REQUEST_BYTES); + int err; - return drbg_generate_long(drbg, dst, dlen, addtl); + mutex_lock(&drbg->drbg_mutex); + err = drbg_generate(drbg, dst, n, src, slen); + mutex_unlock(&drbg->drbg_mutex); + if (err < 0) + return err; + dst += n; + dlen -= n; + } while (dlen); + return 0; } -/* - * Seed the DRBG invoked by the kernel crypto API - */ -static int drbg_kcapi_seed(struct crypto_rng *tfm, - const u8 *seed, unsigned int slen) +/* Uninstantiate the DRBG. */ +static void drbg_kcapi_exit(struct crypto_tfm *tfm) { - struct drbg_state *drbg = crypto_rng_ctx(tfm); - struct crypto_tfm *tfm_base = crypto_rng_tfm(tfm); - bool pr = false; - struct drbg_string string; - struct drbg_string *seed_string = NULL; - int coreref = 0; + struct drbg_state *drbg = crypto_tfm_ctx(tfm); - drbg_convert_tfm_core(crypto_tfm_alg_driver_name(tfm_base), &coreref, - &pr); - if (0 < slen) { - drbg_string_fill(&string, seed, slen); - seed_string = &string; - } - - return drbg_instantiate(drbg, seed_string, coreref, pr); + crypto_free_rng(drbg->jent); + memzero_explicit(drbg, sizeof(*drbg)); } -/*************************************************************** - * Kernel module: code to load the module - ***************************************************************/ - /* * Tests as defined in 11.3.2 in addition to the cipher tests: testing * of the error handling. @@ -1736,38 +528,21 @@ static int drbg_kcapi_seed(struct crypto_rng *tfm, */ static inline int __init drbg_healthcheck_sanity(void) { - int len = 0; #define OUTBUFLEN 16 - unsigned char buf[OUTBUFLEN]; + u8 buf[OUTBUFLEN]; struct drbg_state *drbg = NULL; int ret; - int rc = -EFAULT; - bool pr = false; - int coreref = 0; - struct drbg_string addtl; - size_t max_addtllen, max_request_bytes; /* only perform test in FIPS mode */ if (!fips_enabled) return 0; -#ifdef CONFIG_CRYPTO_DRBG_CTR - drbg_convert_tfm_core("drbg_nopr_ctr_aes256", &coreref, &pr); -#endif -#ifdef CONFIG_CRYPTO_DRBG_HASH - drbg_convert_tfm_core("drbg_nopr_sha256", &coreref, &pr); -#endif -#ifdef CONFIG_CRYPTO_DRBG_HMAC - drbg_convert_tfm_core("drbg_nopr_hmac_sha512", &coreref, &pr); -#endif - drbg = kzalloc_obj(struct drbg_state); if (!drbg) return -ENOMEM; guard(mutex_init)(&drbg->drbg_mutex); - drbg->core = &drbg_cores[coreref]; - drbg->reseed_threshold = drbg_max_requests(drbg); + drbg->instantiated = true; /* * if the following tests fail, it is likely that there is a buffer @@ -1777,128 +552,66 @@ static inline int __init drbg_healthcheck_sanity(void) * grave bug. */ - max_addtllen = drbg_max_addtl(drbg); - max_request_bytes = drbg_max_request_bytes(drbg); - drbg_string_fill(&addtl, buf, max_addtllen + 1); /* overflow addtllen with additional info string */ - len = drbg_generate(drbg, buf, OUTBUFLEN, &addtl); - BUG_ON(0 < len); + ret = drbg_generate(drbg, buf, OUTBUFLEN, buf, + DRBG_MAX_ADDTL_BYTES + 1); + BUG_ON(ret == 0); /* overflow max_bits */ - len = drbg_generate(drbg, buf, (max_request_bytes + 1), NULL); - BUG_ON(0 < len); + ret = drbg_generate(drbg, buf, DRBG_MAX_REQUEST_BYTES + 1, NULL, 0); + BUG_ON(ret == 0); /* overflow max addtllen with personalization string */ - ret = drbg_seed(drbg, &addtl, false); - BUG_ON(0 == ret); + ret = drbg_seed(drbg, buf, DRBG_MAX_ADDTL_BYTES + 1, false); + BUG_ON(ret == 0); /* all tests passed */ - rc = 0; pr_devel("DRBG: Sanity tests for failure code paths successfully " "completed\n"); kfree(drbg); - return rc; + return 0; } -static struct rng_alg drbg_algs[22]; - -/* - * Fill the array drbg_algs used to register the different DRBGs - * with the kernel crypto API. To fill the array, the information - * from drbg_cores[] is used. - */ -static inline void __init drbg_fill_array(struct rng_alg *alg, - const struct drbg_core *core, int pr) -{ - int pos = 0; - static int priority = 200; - - memcpy(alg->base.cra_name, "stdrng", 6); - if (pr) { - memcpy(alg->base.cra_driver_name, "drbg_pr_", 8); - pos = 8; - } else { - memcpy(alg->base.cra_driver_name, "drbg_nopr_", 10); - pos = 10; - } - memcpy(alg->base.cra_driver_name + pos, core->cra_name, - strlen(core->cra_name)); - - alg->base.cra_priority = priority; - priority++; - /* - * If FIPS mode enabled, the selected DRBG shall have the - * highest cra_priority over other stdrng instances to ensure - * it is selected. - */ - if (fips_enabled) - alg->base.cra_priority += 200; - - alg->base.cra_ctxsize = sizeof(struct drbg_state); - alg->base.cra_module = THIS_MODULE; - alg->base.cra_init = drbg_kcapi_init; - alg->base.cra_exit = drbg_kcapi_cleanup; - alg->generate = drbg_kcapi_random; - alg->seed = drbg_kcapi_seed; - alg->set_ent = drbg_kcapi_set_entropy; - alg->seedsize = 0; -} +static struct rng_alg drbg_alg = { + .base.cra_name = "stdrng", + .base.cra_driver_name = "drbg_nopr_hmac_sha512", + .base.cra_priority = 201, + .base.cra_ctxsize = sizeof(struct drbg_state), + .base.cra_module = THIS_MODULE, + .base.cra_init = drbg_kcapi_init, + .set_ent = drbg_kcapi_set_entropy, + .seed = drbg_kcapi_seed, + .generate = drbg_kcapi_generate, + .base.cra_exit = drbg_kcapi_exit, +}; static int __init drbg_init(void) { - unsigned int i = 0; /* pointer to drbg_algs */ - unsigned int j = 0; /* pointer to drbg_cores */ int ret; ret = drbg_healthcheck_sanity(); if (ret) return ret; - if (ARRAY_SIZE(drbg_cores) * 2 > ARRAY_SIZE(drbg_algs)) { - pr_info("DRBG: Cannot register all DRBG types" - "(slots needed: %zu, slots available: %zu)\n", - ARRAY_SIZE(drbg_cores) * 2, ARRAY_SIZE(drbg_algs)); - return -EFAULT; - } - /* - * each DRBG definition can be used with PR and without PR, thus - * we instantiate each DRBG in drbg_cores[] twice. - * - * As the order of placing them into the drbg_algs array matters - * (the later DRBGs receive a higher cra_priority) we register the - * prediction resistance DRBGs first as the should not be too - * interesting. + * In FIPS mode, boost the algorithm priority to ensure that when users + * request "stdrng", they really get the algorithm from here. */ - for (j = 0; ARRAY_SIZE(drbg_cores) > j; j++, i++) - drbg_fill_array(&drbg_algs[i], &drbg_cores[j], 1); - for (j = 0; ARRAY_SIZE(drbg_cores) > j; j++, i++) - drbg_fill_array(&drbg_algs[i], &drbg_cores[j], 0); - return crypto_register_rngs(drbg_algs, (ARRAY_SIZE(drbg_cores) * 2)); + if (fips_enabled) + drbg_alg.base.cra_priority += 2000; + + return crypto_register_rng(&drbg_alg); } static void __exit drbg_exit(void) { - crypto_unregister_rngs(drbg_algs, (ARRAY_SIZE(drbg_cores) * 2)); + crypto_unregister_rng(&drbg_alg); } module_init(drbg_init); module_exit(drbg_exit); -#ifndef CRYPTO_DRBG_HASH_STRING -#define CRYPTO_DRBG_HASH_STRING "" -#endif -#ifndef CRYPTO_DRBG_HMAC_STRING -#define CRYPTO_DRBG_HMAC_STRING "" -#endif -#ifndef CRYPTO_DRBG_CTR_STRING -#define CRYPTO_DRBG_CTR_STRING "" -#endif MODULE_LICENSE("GPL"); MODULE_AUTHOR("Stephan Mueller "); -MODULE_DESCRIPTION("NIST SP800-90A Deterministic Random Bit Generator (DRBG) " - "using following cores: " - CRYPTO_DRBG_HASH_STRING - CRYPTO_DRBG_HMAC_STRING - CRYPTO_DRBG_CTR_STRING); +MODULE_DESCRIPTION("NIST SP800-90A Deterministic Random Bit Generator (DRBG)"); MODULE_ALIAS_CRYPTO("stdrng"); -MODULE_IMPORT_NS("CRYPTO_INTERNAL"); +MODULE_ALIAS_CRYPTO("drbg_nopr_hmac_sha512"); diff --git a/crypto/ecc.c b/crypto/ecc.c index 43b0def3a225..6eb4d97a5f0d 100644 --- a/crypto/ecc.c +++ b/crypto/ecc.c @@ -393,14 +393,26 @@ static uint128_t mul_64_64(u64 left, u64 right) return result; } -static uint128_t add_128_128(uint128_t a, uint128_t b) +/* Calculate addition with overflow checking. Returns true on wrap-around, + * false otherwise. + */ +static bool check_add_128_128_overflow(uint128_t *result, uint128_t a, + uint128_t b) { - uint128_t result; + bool carry; - result.m_low = a.m_low + b.m_low; - result.m_high = a.m_high + b.m_high + (result.m_low < a.m_low); + result->m_low = a.m_low + b.m_low; + carry = (result->m_low < a.m_low); - return result; + result->m_high = a.m_high + b.m_high + carry; + + /* Using constant-time bitwise arithmetic to prevent timing + * side-channels. + */ + carry = (result->m_high < a.m_high) | + ((result->m_high == a.m_high) & carry); + + return carry; } static void vli_mult(u64 *result, const u64 *left, const u64 *right, @@ -425,9 +437,7 @@ static void vli_mult(u64 *result, const u64 *left, const u64 *right, uint128_t product; product = mul_64_64(left[i], right[k - i]); - - r01 = add_128_128(r01, product); - r2 += (r01.m_high < product.m_high); + r2 += check_add_128_128_overflow(&r01, r01, product); } result[k] = r01.m_low; @@ -450,7 +460,7 @@ static void vli_umult(u64 *result, const u64 *left, u32 right, uint128_t product; product = mul_64_64(left[k], right); - r01 = add_128_128(r01, product); + check_add_128_128_overflow(&r01, r01, product); /* no carry */ result[k] = r01.m_low; r01.m_low = r01.m_high; @@ -487,8 +497,7 @@ static void vli_square(u64 *result, const u64 *left, unsigned int ndigits) product.m_low <<= 1; } - r01 = add_128_128(r01, product); - r2 += (r01.m_high < product.m_high); + r2 += check_add_128_128_overflow(&r01, r01, product); } result[k] = r01.m_low; diff --git a/crypto/ecrdsa.c b/crypto/ecrdsa.c index 2c0602f0cd40..93c9e7d76792 100644 --- a/crypto/ecrdsa.c +++ b/crypto/ecrdsa.c @@ -145,7 +145,7 @@ int ecrdsa_param_curve(void *context, size_t hdrlen, unsigned char tag, struct ecrdsa_ctx *ctx = context; ctx->curve_oid = look_up_OID(value, vlen); - if (!ctx->curve_oid) + if (ctx->curve_oid == OID__NR) return -EINVAL; ctx->curve = get_curve_by_oid(ctx->curve_oid); return 0; @@ -259,16 +259,11 @@ static unsigned int ecrdsa_max_size(struct crypto_sig *tfm) return 2 * ctx->pub_key.ndigits * sizeof(u64); } -static void ecrdsa_exit_tfm(struct crypto_sig *tfm) -{ -} - static struct sig_alg ecrdsa_alg = { .verify = ecrdsa_verify, .set_pub_key = ecrdsa_set_pub_key, .key_size = ecrdsa_key_size, .max_size = ecrdsa_max_size, - .exit = ecrdsa_exit_tfm, .base = { .cra_name = "ecrdsa", .cra_driver_name = "ecrdsa-generic", diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c index d5832caa8ab3..b024bff7024a 100644 --- a/crypto/jitterentropy.c +++ b/crypto/jitterentropy.c @@ -7,7 +7,7 @@ * Design * ====== * - * See https://www.chronox.de/jent.html + * See https://www.chronox.de/jent/ * * License * ======= @@ -47,7 +47,7 @@ /* * This Jitterentropy RNG is based on the jitterentropy library - * version 3.4.0 provided at https://www.chronox.de/jent.html + * version 3.4.0 provided at https://www.chronox.de/jent/ */ #ifdef __OPTIMIZE__ @@ -775,7 +775,7 @@ int jent_entropy_init(unsigned int osr, unsigned int flags, * delta even when called shortly after each other -- this * implies that we also have a high resolution timer */ - if (!delta || (end_time == start_time)) { + if (!delta) { ret = JENT_ECOARSETIME; goto out; } diff --git a/crypto/kpp.c b/crypto/kpp.c index 7451d39a7ad8..522c352a03af 100644 --- a/crypto/kpp.c +++ b/crypto/kpp.c @@ -20,11 +20,9 @@ static int __maybe_unused crypto_kpp_report( struct sk_buff *skb, struct crypto_alg *alg) { - struct crypto_report_kpp rkpp; - - memset(&rkpp, 0, sizeof(rkpp)); - - strscpy(rkpp.type, "kpp", sizeof(rkpp.type)); + struct crypto_report_kpp rkpp = { + .type = "kpp", + }; return nla_put(skb, CRYPTOCFGA_REPORT_KPP, sizeof(rkpp), &rkpp); } diff --git a/crypto/krb5/krb5_api.c b/crypto/krb5/krb5_api.c index c7ea40f900a7..03395b89cc61 100644 --- a/crypto/krb5/krb5_api.c +++ b/crypto/krb5/krb5_api.c @@ -207,7 +207,7 @@ struct crypto_aead *krb5_prepare_encryption(const struct krb5_enctype *krb5, struct crypto_aead *ci = NULL; int ret = -ENOMEM; - ci = crypto_alloc_aead(krb5->encrypt_name, 0, 0); + ci = crypto_alloc_aead(krb5->encrypt_name, 0, CRYPTO_ALG_ASYNC); if (IS_ERR(ci)) { ret = PTR_ERR(ci); if (ret == -ENOENT) diff --git a/crypto/lskcipher.c b/crypto/lskcipher.c index bb166250b732..e4328df6e26c 100644 --- a/crypto/lskcipher.c +++ b/crypto/lskcipher.c @@ -264,12 +264,10 @@ static int __maybe_unused crypto_lskcipher_report( struct sk_buff *skb, struct crypto_alg *alg) { struct lskcipher_alg *skcipher = __crypto_lskcipher_alg(alg); - struct crypto_report_blkcipher rblkcipher; - - memset(&rblkcipher, 0, sizeof(rblkcipher)); - - strscpy(rblkcipher.type, "lskcipher", sizeof(rblkcipher.type)); - strscpy(rblkcipher.geniv, "", sizeof(rblkcipher.geniv)); + struct crypto_report_blkcipher rblkcipher = { + .type = "lskcipher", + .geniv = "", + }; rblkcipher.blocksize = alg->cra_blocksize; rblkcipher.min_keysize = skcipher->co.min_keysize; diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c index ed0feaba2383..9f372442981e 100644 --- a/crypto/pcrypt.c +++ b/crypto/pcrypt.c @@ -122,6 +122,8 @@ static int pcrypt_aead_encrypt(struct aead_request *req) return -EINPROGRESS; if (err == -EBUSY) { /* try non-parallel mode */ + aead_request_set_callback(creq, flags, req->base.complete, + req->base.data); return crypto_aead_encrypt(creq); } @@ -173,6 +175,8 @@ static int pcrypt_aead_decrypt(struct aead_request *req) return -EINPROGRESS; if (err == -EBUSY) { /* try non-parallel mode */ + aead_request_set_callback(creq, flags, req->base.complete, + req->base.data); return crypto_aead_decrypt(creq); } diff --git a/crypto/rng.c b/crypto/rng.c index 1d4b9177bad4..828b0d807473 100644 --- a/crypto/rng.c +++ b/crypto/rng.c @@ -65,11 +65,9 @@ static unsigned int seedsize(struct crypto_alg *alg) static int __maybe_unused crypto_rng_report( struct sk_buff *skb, struct crypto_alg *alg) { - struct crypto_report_rng rrng; - - memset(&rrng, 0, sizeof(rrng)); - - strscpy(rrng.type, "rng", sizeof(rrng.type)); + struct crypto_report_rng rrng = { + .type = "rng", + }; rrng.seedsize = seedsize(alg); @@ -232,5 +230,16 @@ void crypto_unregister_rngs(struct rng_alg *algs, int count) } EXPORT_SYMBOL_GPL(crypto_unregister_rngs); +static void __exit rng_exit(void) +{ + int err; + + err = crypto_del_default_rng(); + if (err) + pr_err("Failed delete default RNG: %d\n", err); +} + +module_exit(rng_exit); + MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Random Number Generator"); diff --git a/crypto/scompress.c b/crypto/scompress.c index 253655ece83f..de54227203ce 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -48,11 +48,9 @@ static DECLARE_WORK(scomp_scratch_work, scomp_scratch_workfn); static int __maybe_unused crypto_scomp_report( struct sk_buff *skb, struct crypto_alg *alg) { - struct crypto_report_comp rscomp; - - memset(&rscomp, 0, sizeof(rscomp)); - - strscpy(rscomp.type, "scomp", sizeof(rscomp.type)); + struct crypto_report_comp rscomp = { + .type = "scomp", + }; return nla_put(skb, CRYPTOCFGA_REPORT_COMPRESS, sizeof(rscomp), &rscomp); diff --git a/crypto/shash.c b/crypto/shash.c index 2f07d0bd1f61..d31a65570f69 100644 --- a/crypto/shash.c +++ b/crypto/shash.c @@ -333,12 +333,10 @@ static void crypto_shash_free_instance(struct crypto_instance *inst) static int __maybe_unused crypto_shash_report( struct sk_buff *skb, struct crypto_alg *alg) { - struct crypto_report_hash rhash; struct shash_alg *salg = __crypto_shash_alg(alg); - - memset(&rhash, 0, sizeof(rhash)); - - strscpy(rhash.type, "shash", sizeof(rhash.type)); + struct crypto_report_hash rhash = { + .type = "shash", + }; rhash.blocksize = alg->cra_blocksize; rhash.digestsize = salg->digestsize; diff --git a/crypto/sig.c b/crypto/sig.c index beba745b6405..7d2048da5c3a 100644 --- a/crypto/sig.c +++ b/crypto/sig.c @@ -53,9 +53,9 @@ static void __maybe_unused crypto_sig_show(struct seq_file *m, static int __maybe_unused crypto_sig_report(struct sk_buff *skb, struct crypto_alg *alg) { - struct crypto_report_sig rsig = {}; - - strscpy(rsig.type, "sig", sizeof(rsig.type)); + struct crypto_report_sig rsig = { + .type = "sig", + }; return nla_put(skb, CRYPTOCFGA_REPORT_SIG, sizeof(rsig), &rsig); } diff --git a/crypto/skcipher.c b/crypto/skcipher.c index 2b31d1d5d268..617e840432b1 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -591,12 +591,10 @@ static int __maybe_unused crypto_skcipher_report( struct sk_buff *skb, struct crypto_alg *alg) { struct skcipher_alg *skcipher = __crypto_skcipher_alg(alg); - struct crypto_report_blkcipher rblkcipher; - - memset(&rblkcipher, 0, sizeof(rblkcipher)); - - strscpy(rblkcipher.type, "skcipher", sizeof(rblkcipher.type)); - strscpy(rblkcipher.geniv, "", sizeof(rblkcipher.geniv)); + struct crypto_report_blkcipher rblkcipher = { + .type = "skcipher", + .geniv = "", + }; rblkcipher.blocksize = alg->cra_blocksize; rblkcipher.min_keysize = skcipher->min_keysize; diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 4d86efae65b2..b84d5fcbe107 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -28,13 +28,12 @@ #include #include #include -#include -#include #include #include #include #include #include +#include #include #include "internal.h" @@ -3482,13 +3481,11 @@ static int alg_test_comp(const struct alg_test_desc *desc, const char *driver, return err; } -static int drbg_cavs_test(const struct drbg_testvec *test, int pr, - const char *driver, u32 type, u32 mask) +static int drbg_cavs_test(const struct drbg_testvec *test, const char *driver, + u32 type, u32 mask) { int ret = -EAGAIN; struct crypto_rng *drng; - struct drbg_test_data test_data; - struct drbg_string addtl, pers, testentropy; unsigned char *buf = kzalloc(test->expectedlen, GFP_KERNEL); if (!buf) @@ -3504,39 +3501,34 @@ static int drbg_cavs_test(const struct drbg_testvec *test, int pr, return PTR_ERR(drng); } - test_data.testentropy = &testentropy; - drbg_string_fill(&testentropy, test->entropy, test->entropylen); - drbg_string_fill(&pers, test->pers, test->perslen); - ret = crypto_drbg_reset_test(drng, &pers, &test_data); + crypto_rng_set_entropy(drng, test->entropy, test->entropylen); + ret = crypto_rng_reset(drng, test->pers, test->perslen); if (ret) { - printk(KERN_ERR "alg: drbg: Failed to reset rng\n"); + printk(KERN_ERR "alg: drbg: Failed to instantiate rng\n"); goto outbuf; } - drbg_string_fill(&addtl, test->addtla, test->addtllen); - if (pr) { - drbg_string_fill(&testentropy, test->entpra, test->entprlen); - ret = crypto_drbg_get_bytes_addtl_test(drng, - buf, test->expectedlen, &addtl, &test_data); - } else { - ret = crypto_drbg_get_bytes_addtl(drng, - buf, test->expectedlen, &addtl); + if (test->ent_reseed_len) { + crypto_rng_set_entropy(drng, test->ent_reseed, + test->ent_reseed_len); + ret = crypto_rng_reset(drng, test->addtl_reseed, + test->addtl_reseed_len); + if (ret) { + printk(KERN_ERR "alg: drbg: Failed to reseed rng\n"); + goto outbuf; + } } + + ret = crypto_rng_generate(drng, test->addtla, test->addtllen, + buf, test->expectedlen); if (ret < 0) { printk(KERN_ERR "alg: drbg: could not obtain random data for " "driver %s\n", driver); goto outbuf; } - drbg_string_fill(&addtl, test->addtlb, test->addtllen); - if (pr) { - drbg_string_fill(&testentropy, test->entprb, test->entprlen); - ret = crypto_drbg_get_bytes_addtl_test(drng, - buf, test->expectedlen, &addtl, &test_data); - } else { - ret = crypto_drbg_get_bytes_addtl(drng, - buf, test->expectedlen, &addtl); - } + ret = crypto_rng_generate(drng, test->addtlb, test->addtllen, + buf, test->expectedlen); if (ret < 0) { printk(KERN_ERR "alg: drbg: could not obtain random data for " "driver %s\n", driver); @@ -3556,16 +3548,12 @@ static int alg_test_drbg(const struct alg_test_desc *desc, const char *driver, u32 type, u32 mask) { int err = 0; - int pr = 0; int i = 0; const struct drbg_testvec *template = desc->suite.drbg.vecs; unsigned int tcount = desc->suite.drbg.count; - if (0 == memcmp(driver, "drbg_pr_", 8)) - pr = 1; - for (i = 0; i < tcount; i++) { - err = drbg_cavs_test(&template[i], pr, driver, type, mask); + err = drbg_cavs_test(&template[i], driver, type, mask); if (err) { printk(KERN_ERR "alg: drbg: Test %d failed for %s\n", i, driver); @@ -4215,6 +4203,7 @@ static const struct alg_test_desc alg_test_descs[] = { .alg = "authenc(hmac(sha256),cts(cbc(aes)))", .generic_driver = "authenc(hmac-sha256-lib,cts(cbc(aes-lib)))", .test = alg_test_aead, + .fips_allowed = 1, .suite = { .aead = __VECS(krb5_test_aes128_cts_hmac_sha256_128) } @@ -4256,6 +4245,7 @@ static const struct alg_test_desc alg_test_descs[] = { .alg = "authenc(hmac(sha384),cts(cbc(aes)))", .generic_driver = "authenc(hmac-sha384-lib,cts(cbc(aes-lib)))", .test = alg_test_aead, + .fips_allowed = 1, .suite = { .aead = __VECS(krb5_test_aes256_cts_hmac_sha384_192) } @@ -4653,42 +4643,6 @@ static const struct alg_test_desc alg_test_descs[] = { }, { .alg = "digest_null", .test = alg_test_null, - }, { - .alg = "drbg_nopr_ctr_aes128", - .test = alg_test_drbg, - .fips_allowed = 1, - .suite = { - .drbg = __VECS(drbg_nopr_ctr_aes128_tv_template) - } - }, { - .alg = "drbg_nopr_ctr_aes192", - .test = alg_test_drbg, - .fips_allowed = 1, - .suite = { - .drbg = __VECS(drbg_nopr_ctr_aes192_tv_template) - } - }, { - .alg = "drbg_nopr_ctr_aes256", - .test = alg_test_drbg, - .fips_allowed = 1, - .suite = { - .drbg = __VECS(drbg_nopr_ctr_aes256_tv_template) - } - }, { - .alg = "drbg_nopr_hmac_sha256", - .test = alg_test_drbg, - .fips_allowed = 1, - .suite = { - .drbg = __VECS(drbg_nopr_hmac_sha256_tv_template) - } - }, { - /* - * There is no need to specifically test the DRBG with every - * backend cipher -- covered by drbg_nopr_hmac_sha512 test - */ - .alg = "drbg_nopr_hmac_sha384", - .test = alg_test_null, - .fips_allowed = 1 }, { .alg = "drbg_nopr_hmac_sha512", .test = alg_test_drbg, @@ -4696,70 +4650,6 @@ static const struct alg_test_desc alg_test_descs[] = { .suite = { .drbg = __VECS(drbg_nopr_hmac_sha512_tv_template) } - }, { - .alg = "drbg_nopr_sha256", - .test = alg_test_drbg, - .fips_allowed = 1, - .suite = { - .drbg = __VECS(drbg_nopr_sha256_tv_template) - } - }, { - /* covered by drbg_nopr_sha256 test */ - .alg = "drbg_nopr_sha384", - .test = alg_test_null, - .fips_allowed = 1 - }, { - .alg = "drbg_nopr_sha512", - .fips_allowed = 1, - .test = alg_test_null, - }, { - .alg = "drbg_pr_ctr_aes128", - .test = alg_test_drbg, - .fips_allowed = 1, - .suite = { - .drbg = __VECS(drbg_pr_ctr_aes128_tv_template) - } - }, { - /* covered by drbg_pr_ctr_aes128 test */ - .alg = "drbg_pr_ctr_aes192", - .fips_allowed = 1, - .test = alg_test_null, - }, { - .alg = "drbg_pr_ctr_aes256", - .fips_allowed = 1, - .test = alg_test_null, - }, { - .alg = "drbg_pr_hmac_sha256", - .test = alg_test_drbg, - .fips_allowed = 1, - .suite = { - .drbg = __VECS(drbg_pr_hmac_sha256_tv_template) - } - }, { - /* covered by drbg_pr_hmac_sha256 test */ - .alg = "drbg_pr_hmac_sha384", - .test = alg_test_null, - .fips_allowed = 1 - }, { - .alg = "drbg_pr_hmac_sha512", - .test = alg_test_null, - .fips_allowed = 1, - }, { - .alg = "drbg_pr_sha256", - .test = alg_test_drbg, - .fips_allowed = 1, - .suite = { - .drbg = __VECS(drbg_pr_sha256_tv_template) - } - }, { - /* covered by drbg_pr_sha256 test */ - .alg = "drbg_pr_sha384", - .test = alg_test_null, - .fips_allowed = 1 - }, { - .alg = "drbg_pr_sha512", - .fips_allowed = 1, - .test = alg_test_null, }, { .alg = "ecb(aes)", .generic_driver = "ecb(aes-lib)", @@ -5295,6 +5185,9 @@ static const struct alg_test_desc alg_test_descs[] = { .suite = { .sig = __VECS(pkcs1_rsa_none_tv_template) } + }, { + .alg = "pkcs1(rsa,sha1)", + .test = alg_test_null, }, { .alg = "pkcs1(rsa,sha224)", .test = alg_test_null, @@ -5330,6 +5223,9 @@ static const struct alg_test_desc alg_test_descs[] = { .alg = "pkcs1pad(rsa)", .test = alg_test_null, .fips_allowed = 1, + }, { + .alg = "pkcs1pad(rsa,sha1)", + .test = alg_test_null, }, { .alg = "rfc3686(ctr(aes))", .generic_driver = "rfc3686(ctr(aes-lib))", diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 9b4d7e11c9fd..b7dcf40af6db 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -120,16 +120,24 @@ struct aead_testvec { }; struct drbg_testvec { + /* Instantiate */ const unsigned char *entropy; size_t entropylen; - const unsigned char *entpra; - const unsigned char *entprb; - size_t entprlen; + const unsigned char *pers; + size_t perslen; + + /* Reseed (optional) */ + const unsigned char *ent_reseed; + size_t ent_reseed_len; + const unsigned char *addtl_reseed; + size_t addtl_reseed_len; + + /* Generate (twice) */ const unsigned char *addtla; const unsigned char *addtlb; size_t addtllen; - const unsigned char *pers; - size_t perslen; + + /* Expected output from last call to Generate */ const unsigned char *expected; size_t expectedlen; }; @@ -23422,883 +23430,201 @@ static const struct aead_testvec aegis128_tv_template[] = { }, }; -/* - * SP800-90A DRBG Test vectors from - * http://csrc.nist.gov/groups/STM/cavp/documents/drbg/drbgtestvectors.zip - * - * Test vectors for DRBG with prediction resistance. All types of DRBGs - * (Hash, HMAC, CTR) are tested with all permutations of use cases (w/ and - * w/o personalization string, w/ and w/o additional input string). - */ -static const struct drbg_testvec drbg_pr_sha256_tv_template[] = { - { - .entropy = (unsigned char *) - "\x72\x88\x4c\xcd\x6c\x85\x57\x70\xf7\x0b\x8b\x86" - "\xc1\xeb\xd2\x4e\x36\x14\xab\x18\xc4\x9c\xc9\xcf" - "\x1a\xe8\xf7\x7b\x02\x49\x73\xd7\xf1\x42\x7d\xc6" - "\x3f\x29\x2d\xec\xd3\x66\x51\x3f\x1d\x8d\x5b\x4e", - .entropylen = 48, - .entpra = (unsigned char *) - "\x38\x9c\x91\xfa\xc2\xa3\x46\x89\x56\x08\x3f\x62" - "\x73\xd5\x22\xa9\x29\x63\x3a\x1d\xe5\x5d\x5e\x4f" - "\x67\xb0\x67\x7a\x5e\x9e\x0c\x62", - .entprb = (unsigned char *) - "\xb2\x8f\x36\xb2\xf6\x8d\x39\x13\xfa\x6c\x66\xcf" - "\x62\x8a\x7e\x8c\x12\x33\x71\x9c\x69\xe4\xa5\xf0" - "\x8c\xee\xeb\x9c\xf5\x31\x98\x31", - .entprlen = 32, - .expected = (unsigned char *) - "\x52\x7b\xa3\xad\x71\x77\xa4\x49\x42\x04\x61\xc7" - "\xf0\xaf\xa5\xfd\xd3\xb3\x0d\x6a\x61\xba\x35\x49" - "\xbb\xaa\xaf\xe4\x25\x7d\xb5\x48\xaf\x5c\x18\x3d" - "\x33\x8d\x9d\x45\xdf\x98\xd5\x94\xa8\xda\x92\xfe" - "\xc4\x3c\x94\x2a\xcf\x7f\x7b\xf2\xeb\x28\xa9\xf1" - "\xe0\x86\x30\xa8\xfe\xf2\x48\x90\x91\x0c\x75\xb5" - "\x3c\x00\xf0\x4d\x09\x4f\x40\xa7\xa2\x8c\x52\xdf" - "\x52\xef\x17\xbf\x3d\xd1\xa2\x31\xb4\xb8\xdc\xe6" - "\x5b\x0d\x1f\x78\x36\xb4\xe6\x4b\xa7\x11\x25\xd5" - "\x94\xc6\x97\x36\xab\xf0\xe5\x31\x28\x6a\xbb\xce" - "\x30\x81\xa6\x8f\x27\x14\xf8\x1c", - .expectedlen = 128, - .addtla = NULL, - .addtlb = NULL, - .addtllen = 0, - .pers = NULL, - .perslen = 0, - }, { - .entropy = (unsigned char *) - "\x5d\xf2\x14\xbc\xf6\xb5\x4e\x0b\xf0\x0d\x6f\x2d" - "\xe2\x01\x66\x7b\xd0\xa4\x73\xa4\x21\xdd\xb0\xc0" - "\x51\x79\x09\xf4\xea\xa9\x08\xfa\xa6\x67\xe0\xe1" - "\xd1\x88\xa8\xad\xee\x69\x74\xb3\x55\x06\x9b\xf6", - .entropylen = 48, - .entpra = (unsigned char *) - "\xef\x48\x06\xa2\xc2\x45\xf1\x44\xfa\x34\x2c\xeb" - "\x8d\x78\x3c\x09\x8f\x34\x72\x20\xf2\xe7\xfd\x13" - "\x76\x0a\xf6\xdc\x3c\xf5\xc0\x15", - .entprb = (unsigned char *) - "\x4b\xbe\xe5\x24\xed\x6a\x2d\x0c\xdb\x73\x5e\x09" - "\xf9\xad\x67\x7c\x51\x47\x8b\x6b\x30\x2a\xc6\xde" - "\x76\xaa\x55\x04\x8b\x0a\x72\x95", - .entprlen = 32, - .expected = (unsigned char *) - "\x3b\x14\x71\x99\xa1\xda\xa0\x42\xe6\xc8\x85\x32" - "\x70\x20\x32\x53\x9a\xbe\xd1\x1e\x15\xef\xfb\x4c" - "\x25\x6e\x19\x3a\xf0\xb9\xcb\xde\xf0\x3b\xc6\x18" - "\x4d\x85\x5a\x9b\xf1\xe3\xc2\x23\x03\x93\x08\xdb" - "\xa7\x07\x4b\x33\x78\x40\x4d\xeb\x24\xf5\x6e\x81" - "\x4a\x1b\x6e\xa3\x94\x52\x43\xb0\xaf\x2e\x21\xf4" - "\x42\x46\x8e\x90\xed\x34\x21\x75\xea\xda\x67\xb6" - "\xe4\xf6\xff\xc6\x31\x6c\x9a\x5a\xdb\xb3\x97\x13" - "\x09\xd3\x20\x98\x33\x2d\x6d\xd7\xb5\x6a\xa8\xa9" - "\x9a\x5b\xd6\x87\x52\xa1\x89\x2b\x4b\x9c\x64\x60" - "\x50\x47\xa3\x63\x81\x16\xaf\x19", - .expectedlen = 128, - .addtla = (unsigned char *) - "\xbe\x13\xdb\x2a\xe9\xa8\xfe\x09\x97\xe1\xce\x5d" - "\xe8\xbb\xc0\x7c\x4f\xcb\x62\x19\x3f\x0f\xd2\xad" - "\xa9\xd0\x1d\x59\x02\xc4\xff\x70", - .addtlb = (unsigned char *) - "\x6f\x96\x13\xe2\xa7\xf5\x6c\xfe\xdf\x66\xe3\x31" - "\x63\x76\xbf\x20\x27\x06\x49\xf1\xf3\x01\x77\x41" - "\x9f\xeb\xe4\x38\xfe\x67\x00\xcd", - .addtllen = 32, - .pers = NULL, - .perslen = 0, - }, { - .entropy = (unsigned char *) - "\xc6\x1c\xaf\x83\xa2\x56\x38\xf9\xb0\xbc\xd9\x85" - "\xf5\x2e\xc4\x46\x9c\xe1\xb9\x40\x98\x70\x10\x72" - "\xd7\x7d\x15\x85\xa1\x83\x5a\x97\xdf\xc8\xa8\xe8" - "\x03\x4c\xcb\x70\x35\x8b\x90\x94\x46\x8a\x6e\xa1", - .entropylen = 48, - .entpra = (unsigned char *) - "\xc9\x05\xa4\xcf\x28\x80\x4b\x93\x0f\x8b\xc6\xf9" - "\x09\x41\x58\x74\xe9\xec\x28\xc7\x53\x0a\x73\x60" - "\xba\x0a\xde\x57\x5b\x4b\x9f\x29", - .entprb = (unsigned char *) - "\x4f\x31\xd2\xeb\xac\xfa\xa8\xe2\x01\x7d\xf3\xbd" - "\x42\xbd\x20\xa0\x30\x65\x74\xd5\x5d\xd2\xad\xa4" - "\xa9\xeb\x1f\x4d\xf6\xfd\xb8\x26", - .entprlen = 32, - .expected = (unsigned char *) - "\xf6\x13\x05\xcb\x83\x60\x16\x42\x49\x1d\xc6\x25" - "\x3b\x8c\x31\xa3\xbe\x8b\xbd\x1c\xe2\xec\x1d\xde" - "\xbb\xbf\xa1\xac\xa8\x9f\x50\xce\x69\xce\xef\xd5" - "\xd6\xf2\xef\x6a\xf7\x81\x38\xdf\xbc\xa7\x5a\xb9" - "\xb2\x42\x65\xab\xe4\x86\x8d\x2d\x9d\x59\x99\x2c" - "\x5a\x0d\x71\x55\x98\xa4\x45\xc2\x8d\xdb\x05\x5e" - "\x50\x21\xf7\xcd\xe8\x98\x43\xce\x57\x74\x63\x4c" - "\xf3\xb1\xa5\x14\x1e\x9e\x01\xeb\x54\xd9\x56\xae" - "\xbd\xb6\x6f\x1a\x47\x6b\x3b\x44\xe4\xa2\xe9\x3c" - "\x6c\x83\x12\x30\xb8\x78\x7f\x8e\x54\x82\xd4\xfe" - "\x90\x35\x0d\x4c\x4d\x85\xe7\x13", - .expectedlen = 128, - .addtla = NULL, - .addtlb = NULL, - .addtllen = 0, - .pers = (unsigned char *) - "\xa5\xbf\xac\x4f\x71\xa1\xbb\x67\x94\xc6\x50\xc7" - "\x2a\x45\x9e\x10\xa8\xed\xf7\x52\x4f\xfe\x21\x90" - "\xa4\x1b\xe1\xe2\x53\xcc\x61\x47", - .perslen = 32, - }, { - .entropy = (unsigned char *) - "\xb6\xc1\x8d\xdf\x99\x54\xbe\x95\x10\x48\xd9\xf6" - "\xd7\x48\xa8\x73\x2d\x74\xde\x1e\xde\x57\x7e\xf4" - "\x7b\x7b\x64\xef\x88\x7a\xa8\x10\x4b\xe1\xc1\x87" - "\xbb\x0b\xe1\x39\x39\x50\xaf\x68\x9c\xa2\xbf\x5e", - .entropylen = 48, - .entpra = (unsigned char *) - "\xdc\x81\x0a\x01\x58\xa7\x2e\xce\xee\x48\x8c\x7c" - "\x77\x9e\x3c\xf1\x17\x24\x7a\xbb\xab\x9f\xca\x12" - "\x19\xaf\x97\x2d\x5f\xf9\xff\xfc", - .entprb = (unsigned char *) - "\xaf\xfc\x4f\x98\x8b\x93\x95\xc1\xb5\x8b\x7f\x73" - "\x6d\xa6\xbe\x6d\x33\xeb\x2c\x82\xb1\xaf\xc1\xb6" - "\xb6\x05\xe2\x44\xaa\xfd\xe7\xdb", - .entprlen = 32, - .expected = (unsigned char *) - "\x51\x79\xde\x1c\x0f\x58\xf3\xf4\xc9\x57\x2e\x31" - "\xa7\x09\xa1\x53\x64\x63\xa2\xc5\x1d\x84\x88\x65" - "\x01\x1b\xc6\x16\x3c\x49\x5b\x42\x8e\x53\xf5\x18" - "\xad\x94\x12\x0d\x4f\x55\xcc\x45\x5c\x98\x0f\x42" - "\x28\x2f\x47\x11\xf9\xc4\x01\x97\x6b\xa0\x94\x50" - "\xa9\xd1\x5e\x06\x54\x3f\xdf\xbb\xc4\x98\xee\x8b" - "\xba\xa9\xfa\x49\xee\x1d\xdc\xfb\x50\xf6\x51\x9f" - "\x6c\x4a\x9a\x6f\x63\xa2\x7d\xad\xaf\x3a\x24\xa0" - "\xd9\x9f\x07\xeb\x15\xee\x26\xe0\xd5\x63\x39\xda" - "\x3c\x59\xd6\x33\x6c\x02\xe8\x05\x71\x46\x68\x44" - "\x63\x4a\x68\x72\xe9\xf5\x55\xfe", - .expectedlen = 128, - .addtla = (unsigned char *) - "\x15\x20\x2f\xf6\x98\x28\x63\xa2\xc4\x4e\xbb\x6c" - "\xb2\x25\x92\x61\x79\xc9\x22\xc4\x61\x54\x96\xff" - "\x4a\x85\xca\x80\xfe\x0d\x1c\xd0", - .addtlb = (unsigned char *) - "\xde\x29\x8e\x03\x42\x61\xa3\x28\x5e\xc8\x80\xc2" - "\x6d\xbf\xad\x13\xe1\x8d\x2a\xc7\xe8\xc7\x18\x89" - "\x42\x58\x9e\xd6\xcc\xad\x7b\x1e", - .addtllen = 32, - .pers = (unsigned char *) - "\x84\xc3\x73\x9e\xce\xb3\xbc\x89\xf7\x62\xb3\xe1" - "\xd7\x48\x45\x8a\xa9\xcc\xe9\xed\xd5\x81\x84\x52" - "\x82\x4c\xdc\x19\xb8\xf8\x92\x5c", - .perslen = 32, - }, -}; - -static const struct drbg_testvec drbg_pr_hmac_sha256_tv_template[] = { - { - .entropy = (unsigned char *) - "\x99\x69\xe5\x4b\x47\x03\xff\x31\x78\x5b\x87\x9a" - "\x7e\x5c\x0e\xae\x0d\x3e\x30\x95\x59\xe9\xfe\x96" - "\xb0\x67\x6d\x49\xd5\x91\xea\x4d\x07\xd2\x0d\x46" - "\xd0\x64\x75\x7d\x30\x23\xca\xc2\x37\x61\x27\xab", - .entropylen = 48, - .entpra = (unsigned char *) - "\xc6\x0f\x29\x99\x10\x0f\x73\x8c\x10\xf7\x47\x92" - "\x67\x6a\x3f\xc4\xa2\x62\xd1\x37\x21\x79\x80\x46" - "\xe2\x9a\x29\x51\x81\x56\x9f\x54", - .entprb = (unsigned char *) - "\xc1\x1d\x45\x24\xc9\x07\x1b\xd3\x09\x60\x15\xfc" - "\xf7\xbc\x24\xa6\x07\xf2\x2f\xa0\x65\xc9\x37\x65" - "\x8a\x2a\x77\xa8\x69\x90\x89\xf4", - .entprlen = 32, - .expected = (unsigned char *) - "\xab\xc0\x15\x85\x60\x94\x80\x3a\x93\x8d\xff\xd2" - "\x0d\xa9\x48\x43\x87\x0e\xf9\x35\xb8\x2c\xfe\xc1" - "\x77\x06\xb8\xf5\x51\xb8\x38\x50\x44\x23\x5d\xd4" - "\x4b\x59\x9f\x94\xb3\x9b\xe7\x8d\xd4\x76\xe0\xcf" - "\x11\x30\x9c\x99\x5a\x73\x34\xe0\xa7\x8b\x37\xbc" - "\x95\x86\x23\x50\x86\xfa\x3b\x63\x7b\xa9\x1c\xf8" - "\xfb\x65\xef\xa2\x2a\x58\x9c\x13\x75\x31\xaa\x7b" - "\x2d\x4e\x26\x07\xaa\xc2\x72\x92\xb0\x1c\x69\x8e" - "\x6e\x01\xae\x67\x9e\xb8\x7c\x01\xa8\x9c\x74\x22" - "\xd4\x37\x2d\x6d\x75\x4a\xba\xbb\x4b\xf8\x96\xfc" - "\xb1\xcd\x09\xd6\x92\xd0\x28\x3f", - .expectedlen = 128, - .addtla = NULL, - .addtlb = NULL, - .addtllen = 0, - .pers = NULL, - .perslen = 0, - }, { - .entropy = (unsigned char *) - "\xb9\x1f\xe9\xef\xdd\x9b\x7d\x20\xb6\xec\xe0\x2f" - "\xdb\x76\x24\xce\x41\xc8\x3a\x4a\x12\x7f\x3e\x2f" - "\xae\x05\x99\xea\xb5\x06\x71\x0d\x0c\x4c\xb4\x05" - "\x26\xc6\xbd\xf5\x7f\x2a\x3d\xf2\xb5\x49\x7b\xda", - .entropylen = 48, - .entpra = (unsigned char *) - "\xef\x67\x50\x9c\xa7\x7d\xdf\xb7\x2d\x81\x01\xa4" - "\x62\x81\x6a\x69\x5b\xb3\x37\x45\xa7\x34\x8e\x26" - "\x46\xd9\x26\xa2\x19\xd4\x94\x43", - .entprb = (unsigned char *) - "\x97\x75\x53\x53\xba\xb4\xa6\xb2\x91\x60\x71\x79" - "\xd1\x6b\x4a\x24\x9a\x34\x66\xcc\x33\xab\x07\x98" - "\x51\x78\x72\xb2\x79\xfd\x2c\xff", - .entprlen = 32, - .expected = (unsigned char *) - "\x9c\xdc\x63\x8a\x19\x23\x22\x66\x0c\xc5\xb9\xd7" - "\xfb\x2a\xb0\x31\xe3\x8a\x36\xa8\x5a\xa8\x14\xda" - "\x1e\xa9\xcc\xfe\xb8\x26\x44\x83\x9f\xf6\xff\xaa" - "\xc8\x98\xb8\x30\x35\x3b\x3d\x36\xd2\x49\xd4\x40" - "\x62\x0a\x65\x10\x76\x55\xef\xc0\x95\x9c\xa7\xda" - "\x3f\xcf\xb7\x7b\xc6\xe1\x28\x52\xfc\x0c\xe2\x37" - "\x0d\x83\xa7\x51\x4b\x31\x47\x3c\xe1\x3c\xae\x70" - "\x01\xc8\xa3\xd3\xc2\xac\x77\x9c\xd1\x68\x77\x9b" - "\x58\x27\x3b\xa5\x0f\xc2\x7a\x8b\x04\x65\x62\xd5" - "\xe8\xd6\xfe\x2a\xaf\xd3\xd3\xfe\xbd\x18\xfb\xcd" - "\xcd\x66\xb5\x01\x69\x66\xa0\x3c", - .expectedlen = 128, - .addtla = (unsigned char *) - "\x17\xc1\x56\xcb\xcc\x50\xd6\x03\x7d\x45\x76\xa3" - "\x75\x76\xc1\x4a\x66\x1b\x2e\xdf\xb0\x2e\x7d\x56" - "\x6d\x99\x3b\xc6\x58\xda\x03\xf6", - .addtlb = (unsigned char *) - "\x7c\x7b\x4a\x4b\x32\x5e\x6f\x67\x34\xf5\x21\x4c" - "\xf9\x96\xf9\xbf\x1c\x8c\x81\xd3\x9b\x60\x6a\x44" - "\xc6\x03\xa2\xfb\x13\x20\x19\xb7", - .addtllen = 32, - .pers = NULL, - .perslen = 0, - }, { - .entropy = (unsigned char *) - "\x13\x54\x96\xfc\x1b\x7d\x28\xf3\x18\xc9\xa7\x89" - "\xb6\xb3\xc8\x72\xac\x00\xd4\x59\x36\x25\x05\xaf" - "\xa5\xdb\x96\xcb\x3c\x58\x46\x87\xa5\xaa\xbf\x20" - "\x3b\xfe\x23\x0e\xd1\xc7\x41\x0f\x3f\xc9\xb3\x67", - .entropylen = 48, - .entpra = (unsigned char *) - "\xe2\xbd\xb7\x48\x08\x06\xf3\xe1\x93\x3c\xac\x79" - "\xa7\x2b\x11\xda\xe3\x2e\xe1\x91\xa5\x02\x19\x57" - "\x20\x28\xad\xf2\x60\xd7\xcd\x45", - .entprb = (unsigned char *) - "\x8b\xd4\x69\xfc\xff\x59\x95\x95\xc6\x51\xde\x71" - "\x68\x5f\xfc\xf9\x4a\xab\xec\x5a\xcb\xbe\xd3\x66" - "\x1f\xfa\x74\xd3\xac\xa6\x74\x60", - .entprlen = 32, - .expected = (unsigned char *) - "\x1f\x9e\xaf\xe4\xd2\x46\xb7\x47\x41\x4c\x65\x99" - "\x01\xe9\x3b\xbb\x83\x0c\x0a\xb0\xc1\x3a\xe2\xb3" - "\x31\x4e\xeb\x93\x73\xee\x0b\x26\xc2\x63\xa5\x75" - "\x45\x99\xd4\x5c\x9f\xa1\xd4\x45\x87\x6b\x20\x61" - "\x40\xea\x78\xa5\x32\xdf\x9e\x66\x17\xaf\xb1\x88" - "\x9e\x2e\x23\xdd\xc1\xda\x13\x97\x88\xa5\xb6\x5e" - "\x90\x14\x4e\xef\x13\xab\x5c\xd9\x2c\x97\x9e\x7c" - "\xd7\xf8\xce\xea\x81\xf5\xcd\x71\x15\x49\x44\xce" - "\x83\xb6\x05\xfb\x7d\x30\xb5\x57\x2c\x31\x4f\xfc" - "\xfe\x80\xb6\xc0\x13\x0c\x5b\x9b\x2e\x8f\x3d\xfc" - "\xc2\xa3\x0c\x11\x1b\x80\x5f\xf3", - .expectedlen = 128, - .addtla = NULL, - .addtlb = NULL, - .addtllen = 0, - .pers = (unsigned char *) - "\x64\xb6\xfc\x60\xbc\x61\x76\x23\x6d\x3f\x4a\x0f" - "\xe1\xb4\xd5\x20\x9e\x70\xdd\x03\x53\x6d\xbf\xce" - "\xcd\x56\x80\xbc\xb8\x15\xc8\xaa", - .perslen = 32, - }, { - .entropy = (unsigned char *) - "\xc7\xcc\xbc\x67\x7e\x21\x66\x1e\x27\x2b\x63\xdd" - "\x3a\x78\xdc\xdf\x66\x6d\x3f\x24\xae\xcf\x37\x01" - "\xa9\x0d\x89\x8a\xa7\xdc\x81\x58\xae\xb2\x10\x15" - "\x7e\x18\x44\x6d\x13\xea\xdf\x37\x85\xfe\x81\xfb", - .entropylen = 48, - .entpra = (unsigned char *) - "\x7b\xa1\x91\x5b\x3c\x04\xc4\x1b\x1d\x19\x2f\x1a" - "\x18\x81\x60\x3c\x6c\x62\x91\xb7\xe9\xf5\xcb\x96" - "\xbb\x81\x6a\xcc\xb5\xae\x55\xb6", - .entprb = (unsigned char *) - "\x99\x2c\xc7\x78\x7e\x3b\x88\x12\xef\xbe\xd3\xd2" - "\x7d\x2a\xa5\x86\xda\x8d\x58\x73\x4a\x0a\xb2\x2e" - "\xbb\x4c\x7e\xe3\x9a\xb6\x81\xc1", - .entprlen = 32, - .expected = (unsigned char *) - "\x95\x6f\x95\xfc\x3b\xb7\xfe\x3e\xd0\x4e\x1a\x14" - "\x6c\x34\x7f\x7b\x1d\x0d\x63\x5e\x48\x9c\x69\xe6" - "\x46\x07\xd2\x87\xf3\x86\x52\x3d\x98\x27\x5e\xd7" - "\x54\xe7\x75\x50\x4f\xfb\x4d\xfd\xac\x2f\x4b\x77" - "\xcf\x9e\x8e\xcc\x16\xa2\x24\xcd\x53\xde\x3e\xc5" - "\x55\x5d\xd5\x26\x3f\x89\xdf\xca\x8b\x4e\x1e\xb6" - "\x88\x78\x63\x5c\xa2\x63\x98\x4e\x6f\x25\x59\xb1" - "\x5f\x2b\x23\xb0\x4b\xa5\x18\x5d\xc2\x15\x74\x40" - "\x59\x4c\xb4\x1e\xcf\x9a\x36\xfd\x43\xe2\x03\xb8" - "\x59\x91\x30\x89\x2a\xc8\x5a\x43\x23\x7c\x73\x72" - "\xda\x3f\xad\x2b\xba\x00\x6b\xd1", - .expectedlen = 128, - .addtla = (unsigned char *) - "\x18\xe8\x17\xff\xef\x39\xc7\x41\x5c\x73\x03\x03" - "\xf6\x3d\xe8\x5f\xc8\xab\xe4\xab\x0f\xad\xe8\xd6" - "\x86\x88\x55\x28\xc1\x69\xdd\x76", - .addtlb = (unsigned char *) - "\xac\x07\xfc\xbe\x87\x0e\xd3\xea\x1f\x7e\xb8\xe7" - "\x9d\xec\xe8\xe7\xbc\xf3\x18\x25\x77\x35\x4a\xaa" - "\x00\x99\x2a\xdd\x0a\x00\x50\x82", - .addtllen = 32, - .pers = (unsigned char *) - "\xbc\x55\xab\x3c\xf6\x52\xb0\x11\x3d\x7b\x90\xb8" - "\x24\xc9\x26\x4e\x5a\x1e\x77\x0d\x3d\x58\x4a\xda" - "\xd1\x81\xe9\xf8\xeb\x30\x8f\x6f", - .perslen = 32, - }, -}; - -static const struct drbg_testvec drbg_pr_ctr_aes128_tv_template[] = { - { - .entropy = (unsigned char *) - "\xd1\x44\xc6\x61\x81\x6d\xca\x9d\x15\x28\x8a\x42" - "\x94\xd7\x28\x9c\x43\x77\x19\x29\x1a\x6d\xc3\xa2", - .entropylen = 24, - .entpra = (unsigned char *) - "\x96\xd8\x9e\x45\x32\xc9\xd2\x08\x7a\x6d\x97\x15" - "\xb4\xec\x80\xb1", - .entprb = (unsigned char *) - "\x8b\xb6\x72\xb5\x24\x0b\x98\x65\x95\x95\xe9\xc9" - "\x28\x07\xeb\xc2", - .entprlen = 16, - .expected = (unsigned char *) - "\x70\x19\xd0\x4c\x45\x78\xd6\x68\xa9\x9a\xaa\xfe" - "\xc1\xdf\x27\x9a\x1c\x0d\x0d\xf7\x24\x75\x46\xcc" - "\x77\x6b\xdf\x89\xc6\x94\xdc\x74\x50\x10\x70\x18" - "\x9b\xdc\x96\xb4\x89\x23\x40\x1a\xce\x09\x87\xce" - "\xd2\xf3\xd5\xe4\x51\x67\x74\x11\x5a\xcc\x8b\x3b" - "\x8a\xf1\x23\xa8", - .expectedlen = 64, - .addtla = NULL, - .addtlb = NULL, - .addtllen = 0, - .pers = NULL, - .perslen = 0, - }, { - .entropy = (unsigned char *) - "\x8e\x83\xe0\xeb\x37\xea\x3e\x53\x5e\x17\x6e\x77" - "\xbd\xb1\x53\x90\xfc\xdc\xc1\x3c\x9a\x88\x22\x94", - .entropylen = 24, - .entpra = (unsigned char *) - "\x6a\x85\xe7\x37\xc8\xf1\x04\x31\x98\x4f\xc8\x73" - "\x67\xd1\x08\xf8", - .entprb = (unsigned char *) - "\xd7\xa4\x68\xe2\x12\x74\xc3\xd9\xf1\xb7\x05\xbc" - "\xd4\xba\x04\x58", - .entprlen = 16, - .expected = (unsigned char *) - "\x78\xd6\xa6\x70\xff\xd1\x82\xf5\xa2\x88\x7f\x6d" - "\x3d\x8c\x39\xb1\xa8\xcb\x2c\x91\xab\x14\x7e\xbc" - "\x95\x45\x9f\x24\xb8\x20\xac\x21\x23\xdb\x72\xd7" - "\x12\x8d\x48\x95\xf3\x19\x0c\x43\xc6\x19\x45\xfc" - "\x8b\xac\x40\x29\x73\x00\x03\x45\x5e\x12\xff\x0c" - "\xc1\x02\x41\x82", - .expectedlen = 64, - .addtla = (unsigned char *) - "\xa2\xd9\x38\xcf\x8b\x29\x67\x5b\x65\x62\x6f\xe8" - "\xeb\xb3\x01\x76", - .addtlb = (unsigned char *) - "\x59\x63\x1e\x81\x8a\x14\xa8\xbb\xa1\xb8\x41\x25" - "\xd0\x7f\xcc\x43", - .addtllen = 16, - .pers = NULL, - .perslen = 0, - }, { - .entropy = (unsigned char *) - "\x04\xd9\x49\xa6\xdc\xe8\x6e\xbb\xf1\x08\x77\x2b" - "\x9e\x08\xca\x92\x65\x16\xda\x99\xa2\x59\xf3\xe8", - .entropylen = 24, - .entpra = (unsigned char *) - "\x38\x7e\x3f\x6b\x51\x70\x7b\x20\xec\x53\xd0\x66" - "\xc3\x0f\xe3\xb0", - .entprb = (unsigned char *) - "\xe0\x86\xa6\xaa\x5f\x72\x2f\xad\xf7\xef\x06\xb8" - "\xd6\x9c\x9d\xe8", - .entprlen = 16, - .expected = (unsigned char *) - "\xc9\x0a\xaf\x85\x89\x71\x44\x66\x4f\x25\x0b\x2b" - "\xde\xd8\xfa\xff\x52\x5a\x1b\x32\x5e\x41\x7a\x10" - "\x1f\xef\x1e\x62\x23\xe9\x20\x30\xc9\x0d\xad\x69" - "\xb4\x9c\x5b\xf4\x87\x42\xd5\xae\x5e\x5e\x43\xcc" - "\xd9\xfd\x0b\x93\x4a\xe3\xd4\x06\x37\x36\x0f\x3f" - "\x72\x82\x0c\xcf", - .expectedlen = 64, - .addtla = NULL, - .addtlb = NULL, - .addtllen = 0, - .pers = (unsigned char *) - "\xbf\xa4\x9a\x8f\x7b\xd8\xb1\x7a\x9d\xfa\x45\xed" - "\x21\x52\xb3\xad", - .perslen = 16, - }, { - .entropy = (unsigned char *) - "\x92\x89\x8f\x31\xfa\x1c\xff\x6d\x18\x2f\x26\x06" - "\x43\xdf\xf8\x18\xc2\xa4\xd9\x72\xc3\xb9\xb6\x97", - .entropylen = 24, - .entpra = (unsigned char *) - "\x20\x72\x8a\x06\xf8\x6f\x8d\xd4\x41\xe2\x72\xb7" - "\xc4\x2c\xe8\x10", - .entprb = (unsigned char *) - "\x3d\xb0\xf0\x94\xf3\x05\x50\x33\x17\x86\x3e\x22" - "\x08\xf7\xa5\x01", - .entprlen = 16, - .expected = (unsigned char *) - "\x5a\x35\x39\x87\x0f\x4d\x22\xa4\x09\x24\xee\x71" - "\xc9\x6f\xac\x72\x0a\xd6\xf0\x88\x82\xd0\x83\x28" - "\x73\xec\x3f\x93\xd8\xab\x45\x23\xf0\x7e\xac\x45" - "\x14\x5e\x93\x9f\xb1\xd6\x76\x43\x3d\xb6\xe8\x08" - "\x88\xf6\xda\x89\x08\x77\x42\xfe\x1a\xf4\x3f\xc4" - "\x23\xc5\x1f\x68", - .expectedlen = 64, - .addtla = (unsigned char *) - "\x1a\x40\xfa\xe3\xcc\x6c\x7c\xa0\xf8\xda\xba\x59" - "\x23\x6d\xad\x1d", - .addtlb = (unsigned char *) - "\x9f\x72\x76\x6c\xc7\x46\xe5\xed\x2e\x53\x20\x12" - "\xbc\x59\x31\x8c", - .addtllen = 16, - .pers = (unsigned char *) - "\xea\x65\xee\x60\x26\x4e\x7e\xb6\x0e\x82\x68\xc4" - "\x37\x3c\x5c\x0b", - .perslen = 16, - }, -}; - -/* - * SP800-90A DRBG Test vectors from - * http://csrc.nist.gov/groups/STM/cavp/documents/drbg/drbgtestvectors.zip - * - * Test vectors for DRBG without prediction resistance. All types of DRBGs - * (Hash, HMAC, CTR) are tested with all permutations of use cases (w/ and - * w/o personalization string, w/ and w/o additional input string). - */ -static const struct drbg_testvec drbg_nopr_sha256_tv_template[] = { - { - .entropy = (unsigned char *) - "\xa6\x5a\xd0\xf3\x45\xdb\x4e\x0e\xff\xe8\x75\xc3" - "\xa2\xe7\x1f\x42\xc7\x12\x9d\x62\x0f\xf5\xc1\x19" - "\xa9\xef\x55\xf0\x51\x85\xe0\xfb\x85\x81\xf9\x31" - "\x75\x17\x27\x6e\x06\xe9\x60\x7d\xdb\xcb\xcc\x2e", - .entropylen = 48, - .expected = (unsigned char *) - "\xd3\xe1\x60\xc3\x5b\x99\xf3\x40\xb2\x62\x82\x64" - "\xd1\x75\x10\x60\xe0\x04\x5d\xa3\x83\xff\x57\xa5" - "\x7d\x73\xa6\x73\xd2\xb8\xd8\x0d\xaa\xf6\xa6\xc3" - "\x5a\x91\xbb\x45\x79\xd7\x3f\xd0\xc8\xfe\xd1\x11" - "\xb0\x39\x13\x06\x82\x8a\xdf\xed\x52\x8f\x01\x81" - "\x21\xb3\xfe\xbd\xc3\x43\xe7\x97\xb8\x7d\xbb\x63" - "\xdb\x13\x33\xde\xd9\xd1\xec\xe1\x77\xcf\xa6\xb7" - "\x1f\xe8\xab\x1d\xa4\x66\x24\xed\x64\x15\xe5\x1c" - "\xcd\xe2\xc7\xca\x86\xe2\x83\x99\x0e\xea\xeb\x91" - "\x12\x04\x15\x52\x8b\x22\x95\x91\x02\x81\xb0\x2d" - "\xd4\x31\xf4\xc9\xf7\x04\x27\xdf", - .expectedlen = 128, - .addtla = NULL, - .addtlb = NULL, - .addtllen = 0, - .pers = NULL, - .perslen = 0, - }, { - .entropy = (unsigned char *) - "\x73\xd3\xfb\xa3\x94\x5f\x2b\x5f\xb9\x8f\xf6\x9c" - "\x8a\x93\x17\xae\x19\xc3\x4c\xc3\xd6\xca\xa3\x2d" - "\x16\xfc\x42\xd2\x2d\xd5\x6f\x56\xcc\x1d\x30\xff" - "\x9e\x06\x3e\x09\xce\x58\xe6\x9a\x35\xb3\xa6\x56", - .entropylen = 48, - .expected = (unsigned char *) - "\x71\x7b\x93\x46\x1a\x40\xaa\x35\xa4\xaa\xc5\xe7" - "\x6d\x5b\x5b\x8a\xa0\xdf\x39\x7d\xae\x71\x58\x5b" - "\x3c\x7c\xb4\xf0\x89\xfa\x4a\x8c\xa9\x5c\x54\xc0" - "\x40\xdf\xbc\xce\x26\x81\x34\xf8\xba\x7d\x1c\xe8" - "\xad\x21\xe0\x74\xcf\x48\x84\x30\x1f\xa1\xd5\x4f" - "\x81\x42\x2f\xf4\xdb\x0b\x23\xf8\x73\x27\xb8\x1d" - "\x42\xf8\x44\x58\xd8\x5b\x29\x27\x0a\xf8\x69\x59" - "\xb5\x78\x44\xeb\x9e\xe0\x68\x6f\x42\x9a\xb0\x5b" - "\xe0\x4e\xcb\x6a\xaa\xe2\xd2\xd5\x33\x25\x3e\xe0" - "\x6c\xc7\x6a\x07\xa5\x03\x83\x9f\xe2\x8b\xd1\x1c" - "\x70\xa8\x07\x59\x97\xeb\xf6\xbe", - .expectedlen = 128, - .addtla = (unsigned char *) - "\xf4\xd5\x98\x3d\xa8\xfc\xfa\x37\xb7\x54\x67\x73" - "\xc7\xc3\xdd\x47\x34\x71\x02\x5d\xc1\xa0\xd3\x10" - "\xc1\x8b\xbd\xf5\x66\x34\x6f\xdd", - .addtlb = (unsigned char *) - "\xf7\x9e\x6a\x56\x0e\x73\xe9\xd9\x7a\xd1\x69\xe0" - "\x6f\x8c\x55\x1c\x44\xd1\xce\x6f\x28\xcc\xa4\x4d" - "\xa8\xc0\x85\xd1\x5a\x0c\x59\x40", - .addtllen = 32, - .pers = NULL, - .perslen = 0, - }, { - .entropy = (unsigned char *) - "\x2a\x85\xa9\x8b\xd0\xda\x83\xd6\xad\xab\x9f\xbb" - "\x54\x31\x15\x95\x1c\x4d\x49\x9f\x6a\x15\xf6\xe4" - "\x15\x50\x88\x06\x29\x0d\xed\x8d\xb9\x6f\x96\xe1" - "\x83\x9f\xf7\x88\xda\x84\xbf\x44\x28\xd9\x1d\xaa", - .entropylen = 48, - .expected = (unsigned char *) - "\x2d\x55\xde\xc9\xed\x05\x47\x07\x3d\x04\xfc\x28" - "\x0f\x92\xf0\x4d\xd8\x00\x32\x47\x0a\x1b\x1c\x4b" - "\xef\xd9\x97\xa1\x17\x67\xda\x26\x6c\xfe\x76\x46" - "\x6f\xbc\x6d\x82\x4e\x83\x8a\x98\x66\x6c\x01\xb6" - "\xe6\x64\xe0\x08\x10\x6f\xd3\x5d\x90\xe7\x0d\x72" - "\xa6\xa7\xe3\xbb\x98\x11\x12\x56\x23\xc2\x6d\xd1" - "\xc8\xa8\x7a\x39\xf3\x34\xe3\xb8\xf8\x66\x00\x77" - "\x7d\xcf\x3c\x3e\xfa\xc9\x0f\xaf\xe0\x24\xfa\xe9" - "\x84\xf9\x6a\x01\xf6\x35\xdb\x5c\xab\x2a\xef\x4e" - "\xac\xab\x55\xb8\x9b\xef\x98\x68\xaf\x51\xd8\x16" - "\xa5\x5e\xae\xf9\x1e\xd2\xdb\xe6", - .expectedlen = 128, - .addtla = NULL, - .addtlb = NULL, - .addtllen = 0, - .pers = (unsigned char *) - "\xa8\x80\xec\x98\x30\x98\x15\xd2\xc6\xc4\x68\xf1" - "\x3a\x1c\xbf\xce\x6a\x40\x14\xeb\x36\x99\x53\xda" - "\x57\x6b\xce\xa4\x1c\x66\x3d\xbc", - .perslen = 32, - }, { - .entropy = (unsigned char *) - "\x69\xed\x82\xa9\xc5\x7b\xbf\xe5\x1d\x2f\xcb\x7a" - "\xd3\x50\x7d\x96\xb4\xb9\x2b\x50\x77\x51\x27\x74" - "\x33\x74\xba\xf1\x30\xdf\x8e\xdf\x87\x1d\x87\xbc" - "\x96\xb2\xc3\xa7\xed\x60\x5e\x61\x4e\x51\x29\x1a", - .entropylen = 48, - .expected = (unsigned char *) - "\xa5\x71\x24\x31\x11\xfe\x13\xe1\xa8\x24\x12\xfb" - "\x37\xa1\x27\xa5\xab\x77\xa1\x9f\xae\x8f\xaf\x13" - "\x93\xf7\x53\x85\x91\xb6\x1b\xab\xd4\x6b\xea\xb6" - "\xef\xda\x4c\x90\x6e\xef\x5f\xde\xe1\xc7\x10\x36" - "\xd5\x67\xbd\x14\xb6\x89\x21\x0c\xc9\x92\x65\x64" - "\xd0\xf3\x23\xe0\x7f\xd1\xe8\x75\xc2\x85\x06\xea" - "\xca\xc0\xcb\x79\x2d\x29\x82\xfc\xaa\x9a\xc6\x95" - "\x7e\xdc\x88\x65\xba\xec\x0e\x16\x87\xec\xa3\x9e" - "\xd8\x8c\x80\xab\x3a\x64\xe0\xcb\x0e\x45\x98\xdd" - "\x7c\x6c\x6c\x26\x11\x13\xc8\xce\xa9\x47\xa6\x06" - "\x57\xa2\x66\xbb\x2d\x7f\xf3\xc1", - .expectedlen = 128, - .addtla = (unsigned char *) - "\x74\xd3\x6d\xda\xe8\xd6\x86\x5f\x63\x01\xfd\xf2" - "\x7d\x06\x29\x6d\x94\xd1\x66\xf0\xd2\x72\x67\x4e" - "\x77\xc5\x3d\x9e\x03\xe3\xa5\x78", - .addtlb = (unsigned char *) - "\xf6\xb6\x3d\xf0\x7c\x26\x04\xc5\x8b\xcd\x3e\x6a" - "\x9f\x9c\x3a\x2e\xdb\x47\x87\xe5\x8e\x00\x5e\x2b" - "\x74\x7f\xa6\xf6\x80\xcd\x9b\x21", - .addtllen = 32, - .pers = (unsigned char *) - "\x74\xa6\xe0\x08\xf9\x27\xee\x1d\x6e\x3c\x28\x20" - "\x87\xdd\xd7\x54\x31\x47\x78\x4b\xe5\x6d\xa3\x73" - "\xa9\x65\xb1\x10\xc1\xdc\x77\x7c", - .perslen = 32, - }, -}; - -static const struct drbg_testvec drbg_nopr_hmac_sha256_tv_template[] = { - { - .entropy = (unsigned char *) - "\xca\x85\x19\x11\x34\x93\x84\xbf\xfe\x89\xde\x1c" - "\xbd\xc4\x6e\x68\x31\xe4\x4d\x34\xa4\xfb\x93\x5e" - "\xe2\x85\xdd\x14\xb7\x1a\x74\x88\x65\x9b\xa9\x6c" - "\x60\x1d\xc6\x9f\xc9\x02\x94\x08\x05\xec\x0c\xa8", - .entropylen = 48, - .expected = (unsigned char *) - "\xe5\x28\xe9\xab\xf2\xde\xce\x54\xd4\x7c\x7e\x75" - "\xe5\xfe\x30\x21\x49\xf8\x17\xea\x9f\xb4\xbe\xe6" - "\xf4\x19\x96\x97\xd0\x4d\x5b\x89\xd5\x4f\xbb\x97" - "\x8a\x15\xb5\xc4\x43\xc9\xec\x21\x03\x6d\x24\x60" - "\xb6\xf7\x3e\xba\xd0\xdc\x2a\xba\x6e\x62\x4a\xbf" - "\x07\x74\x5b\xc1\x07\x69\x4b\xb7\x54\x7b\xb0\x99" - "\x5f\x70\xde\x25\xd6\xb2\x9e\x2d\x30\x11\xbb\x19" - "\xd2\x76\x76\xc0\x71\x62\xc8\xb5\xcc\xde\x06\x68" - "\x96\x1d\xf8\x68\x03\x48\x2c\xb3\x7e\xd6\xd5\xc0" - "\xbb\x8d\x50\xcf\x1f\x50\xd4\x76\xaa\x04\x58\xbd" - "\xab\xa8\x06\xf4\x8b\xe9\xdc\xb8", - .expectedlen = 128, - .addtla = NULL, - .addtlb = NULL, - .addtllen = 0, - .pers = NULL, - .perslen = 0, - }, { - .entropy = (unsigned char *) - "\xf9\x7a\x3c\xfd\x91\xfa\xa0\x46\xb9\xe6\x1b\x94" - "\x93\xd4\x36\xc4\x93\x1f\x60\x4b\x22\xf1\x08\x15" - "\x21\xb3\x41\x91\x51\xe8\xff\x06\x11\xf3\xa7\xd4" - "\x35\x95\x35\x7d\x58\x12\x0b\xd1\xe2\xdd\x8a\xed", - .entropylen = 48, - .expected = (unsigned char *) - "\xc6\x87\x1c\xff\x08\x24\xfe\x55\xea\x76\x89\xa5" - "\x22\x29\x88\x67\x30\x45\x0e\x5d\x36\x2d\xa5\xbf" - "\x59\x0d\xcf\x9a\xcd\x67\xfe\xd4\xcb\x32\x10\x7d" - "\xf5\xd0\x39\x69\xa6\x6b\x1f\x64\x94\xfd\xf5\xd6" - "\x3d\x5b\x4d\x0d\x34\xea\x73\x99\xa0\x7d\x01\x16" - "\x12\x6d\x0d\x51\x8c\x7c\x55\xba\x46\xe1\x2f\x62" - "\xef\xc8\xfe\x28\xa5\x1c\x9d\x42\x8e\x6d\x37\x1d" - "\x73\x97\xab\x31\x9f\xc7\x3d\xed\x47\x22\xe5\xb4" - "\xf3\x00\x04\x03\x2a\x61\x28\xdf\x5e\x74\x97\xec" - "\xf8\x2c\xa7\xb0\xa5\x0e\x86\x7e\xf6\x72\x8a\x4f" - "\x50\x9a\x8c\x85\x90\x87\x03\x9c", - .expectedlen = 128, - .addtla = (unsigned char *) - "\x51\x72\x89\xaf\xe4\x44\xa0\xfe\x5e\xd1\xa4\x1d" - "\xbb\xb5\xeb\x17\x15\x00\x79\xbd\xd3\x1e\x29\xcf" - "\x2f\xf3\x00\x34\xd8\x26\x8e\x3b", - .addtlb = (unsigned char *) - "\x88\x02\x8d\x29\xef\x80\xb4\xe6\xf0\xfe\x12\xf9" - "\x1d\x74\x49\xfe\x75\x06\x26\x82\xe8\x9c\x57\x14" - "\x40\xc0\xc9\xb5\x2c\x42\xa6\xe0", - .addtllen = 32, - .pers = NULL, - .perslen = 0, - }, { - .entropy = (unsigned char *) - "\x8d\xf0\x13\xb4\xd1\x03\x52\x30\x73\x91\x7d\xdf" - "\x6a\x86\x97\x93\x05\x9e\x99\x43\xfc\x86\x54\x54" - "\x9e\x7a\xb2\x2f\x7c\x29\xf1\x22\xda\x26\x25\xaf" - "\x2d\xdd\x4a\xbc\xce\x3c\xf4\xfa\x46\x59\xd8\x4e", - .entropylen = 48, - .expected = (unsigned char *) - "\xb9\x1c\xba\x4c\xc8\x4f\xa2\x5d\xf8\x61\x0b\x81" - "\xb6\x41\x40\x27\x68\xa2\x09\x72\x34\x93\x2e\x37" - "\xd5\x90\xb1\x15\x4c\xbd\x23\xf9\x74\x52\xe3\x10" - "\xe2\x91\xc4\x51\x46\x14\x7f\x0d\xa2\xd8\x17\x61" - "\xfe\x90\xfb\xa6\x4f\x94\x41\x9c\x0f\x66\x2b\x28" - "\xc1\xed\x94\xda\x48\x7b\xb7\xe7\x3e\xec\x79\x8f" - "\xbc\xf9\x81\xb7\x91\xd1\xbe\x4f\x17\x7a\x89\x07" - "\xaa\x3c\x40\x16\x43\xa5\xb6\x2b\x87\xb8\x9d\x66" - "\xb3\xa6\x0e\x40\xd4\xa8\xe4\xe9\xd8\x2a\xf6\xd2" - "\x70\x0e\x6f\x53\x5c\xdb\x51\xf7\x5c\x32\x17\x29" - "\x10\x37\x41\x03\x0c\xcc\x3a\x56", - .expectedlen = 128, - .addtla = NULL, - .addtlb = NULL, - .addtllen = 0, - .pers = (unsigned char *) - "\xb5\x71\xe6\x6d\x7c\x33\x8b\xc0\x7b\x76\xad\x37" - "\x57\xbb\x2f\x94\x52\xbf\x7e\x07\x43\x7a\xe8\x58" - "\x1c\xe7\xbc\x7c\x3a\xc6\x51\xa9", - .perslen = 32, - }, { - .entropy = (unsigned char *) - "\xc2\xa5\x66\xa9\xa1\x81\x7b\x15\xc5\xc3\xb7\x78" - "\x17\x7a\xc8\x7c\x24\xe7\x97\xbe\x0a\x84\x5f\x11" - "\xc2\xfe\x39\x9d\xd3\x77\x32\xf2\xcb\x18\x94\xeb" - "\x2b\x97\xb3\xc5\x6e\x62\x83\x29\x51\x6f\x86\xec", - .entropylen = 48, - .expected = (unsigned char *) - "\xb3\xa3\x69\x8d\x77\x76\x99\xa0\xdd\x9f\xa3\xf0" - "\xa9\xfa\x57\x83\x2d\x3c\xef\xac\x5d\xf2\x44\x37" - "\xc6\xd7\x3a\x0f\xe4\x10\x40\xf1\x72\x90\x38\xae" - "\xf1\xe9\x26\x35\x2e\xa5\x9d\xe1\x20\xbf\xb7\xb0" - "\x73\x18\x3a\x34\x10\x6e\xfe\xd6\x27\x8f\xf8\xad" - "\x84\x4b\xa0\x44\x81\x15\xdf\xdd\xf3\x31\x9a\x82" - "\xde\x6b\xb1\x1d\x80\xbd\x87\x1a\x9a\xcd\x35\xc7" - "\x36\x45\xe1\x27\x0f\xb9\xfe\x4f\xa8\x8e\xc0\xe4" - "\x65\x40\x9e\xa0\xcb\xa8\x09\xfe\x2f\x45\xe0\x49" - "\x43\xa2\xe3\x96\xbb\xb7\xdd\x2f\x4e\x07\x95\x30" - "\x35\x24\xcc\x9c\xc5\xea\x54\xa1", - .expectedlen = 128, - .addtla = (unsigned char *) - "\x41\x3d\xd8\x3f\xe5\x68\x35\xab\xd4\x78\xcb\x96" - "\x93\xd6\x76\x35\x90\x1c\x40\x23\x9a\x26\x64\x62" - "\xd3\x13\x3b\x83\xe4\x9c\x82\x0b", - .addtlb = (unsigned char *) - "\xd5\xc4\xa7\x1f\x9d\x6d\x95\xa1\xbe\xdf\x0b\xd2" - "\x24\x7c\x27\x7d\x1f\x84\xa4\xe5\x7a\x4a\x88\x25" - "\xb8\x2a\x2d\x09\x7d\xe6\x3e\xf1", - .addtllen = 32, - .pers = (unsigned char *) - "\x13\xce\x4d\x8d\xd2\xdb\x97\x96\xf9\x41\x56\xc8" - "\xe8\xf0\x76\x9b\x0a\xa1\xc8\x2c\x13\x23\xb6\x15" - "\x36\x60\x3b\xca\x37\xc9\xee\x29", - .perslen = 32, - }, -}; - -/* Test vector obtained during NIST ACVP testing */ static const struct drbg_testvec drbg_nopr_hmac_sha512_tv_template[] = { + /* + * Borrowed from the first applicable test vector from ACVP: + * https://github.com/usnistgov/ACVP-Server/blob/v1.1.0.33/gen-val/json-files/hmacDRBG-1.0/prompt.json#L4596 + * https://github.com/usnistgov/ACVP-Server/blob/v1.1.0.33/gen-val/json-files/hmacDRBG-1.0/expectedResults.json#L986 + */ { - .entropy = (unsigned char *) - "\xDF\xB0\xF2\x18\xF0\x78\x07\x01\x29\xA4\x29\x26" - "\x2F\x8A\x34\xCB\x37\xEF\xEE\x41\xE6\x96\xF7\xFF" - "\x61\x47\xD3\xED\x41\x97\xEF\x64\x0C\x48\x56\x5A" - "\xE6\x40\x6E\x4A\x3B\x9E\x7F\xAC\x08\xEC\x25\xAE" - "\x0B\x51\x0E\x2C\x44\x2E\xBD\xDB\x57\xD0\x4A\x6D" - "\x80\x3E\x37\x0F", - .entropylen = 64, - .expected = (unsigned char *) - "\x48\xc6\xa8\xdb\x09\xae\xde\x5d\x8c\x77\xf3\x52" - "\x92\x71\xa7\xb9\x6d\x53\x6d\xa3\x73\xe3\x55\xb8" - "\x39\xd6\x44\x2b\xee\xcb\xe1\x32\x15\x30\xbe\x4e" - "\x9b\x1e\x06\xd1\x6b\xbf\xd5\x3e\xea\x7c\xf5\xaa" - "\x4b\x05\xb5\xd3\xa7\xb2\xc4\xfe\xe7\x1b\xda\x11" - "\x43\x98\x03\x70\x90\xbf\x6e\x43\x9b\xe4\x14\xef" - "\x71\xa3\x2a\xef\x9f\x0d\xb9\xe3\x52\xf2\x89\xc9" - "\x66\x9a\x60\x60\x99\x60\x62\x4c\xd6\x45\x52\x54" - "\xe6\x32\xb2\x1b\xd4\x48\xb5\xa6\xf9\xba\xd3\xff" - "\x29\xc5\x21\xe0\x91\x31\xe0\x38\x8c\x93\x0f\x3c" - "\x30\x7b\x53\xa3\xc0\x7f\x2d\xc1\x39\xec\x69\x0e" - "\xf2\x4a\x3c\x65\xcc\xed\x07\x2a\xf2\x33\x83\xdb" - "\x10\x74\x96\x40\xa7\xc5\x1b\xde\x81\xca\x0b\x8f" - "\x1e\x0a\x1a\x7a\xbf\x3c\x4a\xb8\x8c\xaf\x7b\x80" - "\xb7\xdc\x5d\x0f\xef\x1b\x97\x6e\x3d\x17\x23\x5a" - "\x31\xb9\x19\xcf\x5a\xc5\x00\x2a\xb6\xf3\x99\x34" - "\x65\xee\xe9\x1c\x55\xa0\x3b\x07\x60\xc9\xc4\xe4" - "\xf7\x57\x5c\x34\x9f\xc6\x31\x30\x3f\x23\xb2\x89" - "\xc0\xe7\x50\xf3\xde\x59\xd1\x0e\xb3\x0f\x78\xcc" - "\x7e\x54\x5e\x61\xf6\x86\x3d\xb3\x11\x94\x36\x3e" - "\x61\x5c\x48\x99\xf6\x7b\x02\x9a\xdc\x6a\x28\xe6" - "\xd1\xa7\xd1\xa3", - .expectedlen = 256, - .addtla = (unsigned char *) - "\x6B\x0F\x4A\x48\x0B\x12\x85\xE4\x72\x23\x7F\x7F" - "\x94\x7C\x24\x69\x14\x9F\xDC\x72\xA6\x33\xAD\x3C" - "\x8C\x72\xC1\x88\x49\x59\x82\xC5", - .addtlb = (unsigned char *) - "\xC4\xAF\x36\x3D\xB8\x5D\x9D\xFA\x92\xF5\xC3\x3C" - "\x2D\x1E\x22\x2A\xBD\x8B\x05\x6F\xA3\xFC\xBF\x16" - "\xED\xAA\x75\x8D\x73\x9A\xF6\xEC", - .addtllen = 32, - .pers = NULL, - .perslen = 0, + /* .entropy = ACVP entropyInput || nonce */ + .entropy = "\x47\x28\x33\x05\x82\xE8\xA7\x58\x05\xBD\x07\x9E" + "\xDB\x59\x87\x08\x3B\x82\x40\x70\xA1\xB7\x33\x4F" + "\x0F\x53\xBE\xF6\x98\x75\xFA\x21\x00\x04\xBC\x58" + "\x9B\xB1\xB3\xBC\x2A\xF6\xA4\x2A\xE1\xB3\x65\x24" + "\xA8\xEE\x56\x70\x5E\x28\x06\x04\x80\xCE\xB7\x36" + "\xA4\xD1\x4E\xAA\xE7\xE8\x51\x40\x7D\x30\xF7\x91" + "\xF2\x9E\x98\xA7\xBB\xB3\x1D\xC6\x8A\x62\xC7\xE7" + "\x1F\xE1\x0A\x52\x45\x39\x88\x92\x43\xA6\x5B\xC3" + "\x9D\x60\xDE\xFE\x79\xEF\x87\x6F\xD5\x76\x51\x69" + "\x7B\x4C\x3D\x08\x60\xFA\xDC\x00\x94\xEC\x0A\xC5" + "\xE7\xCB\x30\x05\x9F\xCC\x07\x44\x07\x14\xF4\xF9" + "\xBC\x7D\xC0\x66\x63\x5B\xD3\x2A\xA7\xF2\x1D\x75" + "\x3F\x28\x5F\x4A\xEB\x63\xD7\x92\x5A\x9A\x7C\x3A" + "\x56\xEA\xCB\xA9\xA0\xF9\x5B\x99\xC2\x7D\x31\xBF" + "\x44\xC7\x64\x6B\x7E\x64\x60\x64\x05\x86\xF0\x6B" + "\x76\xA7\x3E\x63\xAD\xB6\x76\xED\x3D\x05\x2E\xC7" + "\xC7\x52\x8F\x07\x56\x23\xFC\x2F\x93\xD5\xF2\x29" + "\xC6\x84\x48\xD1\x88\xB9\x3F\x84\xB2\x93\xFC\x2F" + "\x00\xB1\xBC\xBE\xAC\xC4\x94\x36\xF1\x2B\xF0\xC7" + "\x36\xE8\x1D\x45\x0A\xA2\xFE\x72\xD7\x3E\x15\x6A" + "\x74\x23\xB2\xD1\x3C\x20\xF2\xB7\x19\xC5\xF0\x74" + "\x69\x06\xD1\x92\x47\xC9\x68\xDC\xEE\xBC\xF4\x3A" + "\x8B\x7C\xA4\xA4\x4A\x15\xF4\xF0\x26\x26\xC5\x7D" + "\x57\xDC\x00\x59\x07\xDF\x84\x2A\x23\xA9\xC1\xAF" + "\x47\x49\x88\xCC\x2F\x3B\x90\x39\xA0\x09\xAE\x3C" + "\x79\xF9\x97\x9D\xEF\x13\x1D\x2B\xB6\x18\xDB\xCF" + "\x95\x98\x17\x7F\xF6\x95\x44\x3A\x40\x90\x63\x16" + "\x70\x38\x22\xC5\xD8\xE4\x57\xA0\x8E\x51\x6B\xED" + "\x03\x18\xC6\x74\xFA\xCA\xF2\xD5\xB2\x5A\x0C\x02" + "\x48\x9F\x70\x0B\x1C\x0A\xD1\xD6\x4F\x3F\x70\x3C" + "\x1A\x62\x29\x63\xD0\x92\x76\x30\x7C\xB9\x54\xEC" + "\xF6\x92\xA3\x11\x66\xB2\x56\xE0\x75\x3D\x27\x01", + .entropylen = 384, + .pers = "\x03\x75\x33\x7B\x6B\xE5\x14\xAF\x0E\x43\x33\xCC\xD3" + "\xF4\xF0\x8B\x8A\xCC\xFE\x95\xC1\x20\xB9\x56\x3D\x2C" + "\x34\xC7\xA1\x4C\xBC\xDC\x0A\x8D\xBE\x68\x28\x89\x2A" + "\x92\x10\x05\x4E\xF9\xE2\x51\xF7\x4F\xF4\x48\x76\xC9" + "\xCE\xA8\x6D\xC1\xDE\x5C\x05\x95\x77\x24\xB4\x5D\x3D" + "\x16\x1D\xCB\x43\xE5\xD3\x01\x96\xCC\x3E\x2E\xE1\x97" + "\xB3\x42\xC2\xAB\xEC\x56\xA9\xF9\xF2\x28\x59\x01\x4E" + "\xFC\x8A\xC0\x5E\x4E\x33\x30\x91\x96\x93\xBA\xFA\xE1" + "\x9F\x20\xC3\xC1\xD1\xC6\xC3\x2C\x59\x8F\x56\xD5\x73" + "\xE8\x8D\xCF\xAC\xB1\x85\x3E\xC7\x73\x21\x59\x8D\x0F" + "\xF4\xA4\xC4\x0C\xD2\x18\x81\x5F\x3E\x37\xC9\x9F\xD9" + "\xDE\x3F\x15\x6F\xAC\x1A\xEC\x5E\x28\xD5\x56\xA2\x65" + "\xD2\x7C\x41\xDE\x57\x8C\x60\xAB\xA5\x03\x42\xC8\x3C" + "\x70\xC2\xD1\x11\x62\x25\x4E\x17\x58\xBB\x08\x51\x79" + "\xE6\xED\xBA\x9A\xA7\x07\xD5\x0E\x89\x43\x31\x4F\x99" + "\xEE\xED\xF1\x36\xFA\x1E\xC0\x25\x01\xDA\x19\xC2\x7D" + "\x5D\x78\x74\x5C\xC0\x72\x2E\xF1\x7B\x4E\x9C\xDB\xFE" + "\x60\x89\x62\x71\x9C\xEA\xD3\xDF\x82\xC7\x8B\xB4\x7B" + "\x7B\x7F\x6A\x62\x3C\x57\x76\xA5\x67\x5F\x70\xFB\xE0" + "\x1B\x23\x79\x71\xB5\x7C\xEA\x76\x99", + .perslen = 256, + + .ent_reseed = "\xE0\x5B\x69\xC5\xF9\xD4\x5A\x1F\xC1\x71\x0C" + "\x37\x6F\xFE\x31\xE0\xEF\x3E\x9B\xD5\x46\x43" + "\xA2\x15\xFA\x86\x75\x1C\x95\x9F\xE6\xD6\x41" + "\x50\x36\x0F\x67\x68\x4A\x72\x32\x85\xB0\xCB" + "\xA4\x0D\x1F\xA2\x43\xCA\x30\xF1\x42\xF8\xC4" + "\x17\x6A\xD1\x2D\x43\x46\x8C\xA5\x72\x3C\xB1" + "\x44\x8E\xA1\x75\xAA\x20\x30\xDD\xCD\x0B\x5A" + "\x0D\xE6\x28\xCA\xBC\xA9\x3B\x11\x23\x88\x85" + "\xDF\xD8\x84\x11\x89\x7A\x4F\x00\x90\x0B\xD8" + "\x8A\x35\xE9\x5F\xEC\x05\x42\x15\xCA\x7E\xA3" + "\xEF\xEF\xE3\xB7\xBE\x96\x8C\x36\x7D\x15\x3C" + "\x7A\x78\x00\x61\xAF\x62\x45\xA7\x4B\xAE\x4A" + "\xFD\xC9\xE0\x89\x53\x36\x71\x8F\x38\xC4\x00" + "\x57\xD9\xE6\xC1\xEC\xBF\xF7\x4B\x3F\x55\xD6" + "\x54\x15\x72\xB2\x23\x76\xD1\x4E\x8E\x7E\xFB" + "\xC8\xD6\xA4\xA4\x7E\x8E\xD8\x76\xD3\xEB\x90" + "\x90\xEB\xB3\x84\x26\x34\x93\x36\xCA\xA9\x12" + "\xD3\x6D\x84\x07\xE1\x0E\xE7\x63\x24\x54\x90" + "\x42\x8C\x6B\xE7\x21\x53\x2D\x16\xE9\x68\xA9" + "\xC2\x9A\x7A\x93\x31\x5E\x35\x72\xE2\x0F\x3D" + "\x55\x41\x8C\xC5\xD1\xBB\xEE\x5F\x3C\x95\x52" + "\x2A\x65\x47\xB6\x84\x6A\x05\x19\xD9\xE7\x68" + "\x15\xFA\x49\xD9\x9D\xFD\x47\x9F\xB9\xA6\xBB" + "\xA8\xD4\x22\x6C\x8B\x59\x47\xEF\xD1\xD0\xBF" + "\xBC\xE4\x53\x31\x2F\xA5\xD4\x2C\x89\xCA\xE5" + "\x29\x64\xF8\x51\x2A\x02\xDE\x31\x0D\x54\x0E" + "\x0E\xB8\x78\xE6\x2A\x3E\x31\xA2\xA4\xCD\x91" + "\x79\xF2\x9B\x04\xA0\xDB\x78\xAC\x0D\xB5\xCA" + "\x60\x4C\x08\x96\xCA\x8A\xE9\x93\x9F\x62\x38" + "\xAE", + .ent_reseed_len = 320, + .addtl_reseed = "\xD5\x46\x71\x38\x46\xE0\xC1\x27\x2C\xF1\x07" + "\xDD\x29\x73\xD7\xF4\x5A\x3A\xE0\x99\x98\x0A" + "\xEB\xA5\x9E\x2E\x49\x21\x7B\x18\xC0\xD8\x3D" + "\xEE\x49\xF9\x52\x7C\x55\x5E\x5E\x06\xC2\x7E" + "\x34\x3A\xA5\xB7\x78\x21\xC1\xB8\x8B\x4E\x07" + "\x1F\x3F\xC4\xB4\x9B\x3F\x20\x2D\xEE\x70\x3B" + "\x18\xCF\x2B\x50\x15\x1B\xF3\x6C\xE7\x9F\x2A" + "\x2B\x69\x53\xCA\x88\x41\xA9\x07\x31\x14\x55" + "\xBD\x75\x49\x68\x98\x48\xCA\x46\xA5\xEC\xAA" + "\x9D\x8E\x6F\x40\xBA\xCA\xC8\xE9\x9F\xA4\xC5" + "\x94\x89\x2C\x4D\xE4\x0A\x66\xD3\xDB\x1A\xD9" + "\xA7\x3F\x6B\x0A\xE0\xE4\xDC\x0B\x8B\x64\xBF" + "\xB4\x95\xA1\xAE\x92\xA8\x7C\xE2\x4B\xED\xFD" + "\xFD\xE0\x54\x48\x7C\x25\xD6\xD1\x0A\xAB\x5E" + "\x33\x2F\x1D\xD3\x62\x47\xAB\x35\x8C\xA1\xC2" + "\x75\xDD\x75\xCA\x09\xE1\x38\xDE\x87\x79\x0C" + "\xD7\x5D\x3E\x6D\x28\x8F\x12\x98\x2C\x26\x34" + "\x91\x0B\x6B\xB7\x73", + .addtl_reseed_len = 192, + + .addtla = "\xC8\xE1\x03\x59\xEA\x9F\xD3\x1F\xCC\xCB\x5A\x3E" + "\x23\xDB\x47\x75\x1B\x3A\x6C\x0B\xED\xE6\xC0\xB6" + "\x4B\x82\x53\x5B\x6D\xEE\x2B\x97\x0F\x59\x07\xA4" + "\xFE\xE2\x26\x4F\xFF\xF7\x46\xAF\x62\xC4\x42\xE8" + "\x06\xFA\x1F\xAA\x66\xF3\xA4\xCF\x85\x02\x3C\x11" + "\xEF\xD2\x16\x13\x7A\xBD\x03\x66\xC3\x4D\x6F\x3A" + "\xE5\xD3\xFF\x92\xB6\x45\xB8\x88\x94\x42\x2E\x84" + "\x02\xA5\x54\x12\xBF\xCB\xF1\x6E\x1C\x0C\x23\xFE" + "\x8A\xFD\xF5\x1A\x4C\xD6\x2A\x34\x7F\x21\x50\xBB" + "\x06\x64\x9A\x77\x70\x59\x1D\x12\xD1\xA0\x10\x70" + "\xD7\x4F\x3C\xEE\x82\xEF\x5B\x35\x8F\x05\xF6\x51" + "\xBC\x28\xE1\xB1\x54\xFF\x25\x12\x16\x2C\x62\xFC" + "\xB9\x74\x5E\x78\x41\xB2\x04\x80\x4E\xB1\x43\x2E" + "\x88\x7F\x34\xA4\x33\x66\xE8\x68\x60\x54\xEB\xCE" + "\xD1\x5F\xC4\xD6\x21\x5E\x22\x1A\x7E\xC5\x6D\xB7" + "\x93\x8F\x9C\x77\x6E\x56\xA5\xA8\xF7\xB2\x10\x19", + .addtlb = "\xB8\x1E\x9E\xE2\xB3\xE4\x04\x43\x51\x7B\x96\x32" + "\xAC\x95\xF2\x1E\x5E\x7A\xBF\x68\xE3\x7A\xCE\xE5" + "\x2B\x3B\xB0\x34\x7F\xE1\xB5\x37\x3C\x5B\xD2\x6B" + "\x17\xF6\x18\xAF\x7C\x01\xA5\x84\xFE\x07\x0B\x46" + "\xCC\x96\x61\xAB\xCD\x81\x95\x8A\x8A\xB6\xD8\xCC" + "\x94\x80\x45\xF6\xFF\x95\x3A\xB8\xDC\xBF\xB4\xBF" + "\x26\x8C\xE9\x79\x05\x58\x9C\x03\x08\x45\xB5\xC4" + "\xD8\xB4\xCD\x76\x13\xAE\x2B\xCC\xB5\x3E\x3C\x65" + "\x7A\xAA\xDC\x9B\x0B\x13\x8B\x82\x5A\x39\x60\x9A" + "\xF7\xA8\x38\xB5\x83\x8E\x0B\xD1\xE6\xBF\xA1\xDC" + "\x30\x45\xCC\x67\x41\x0F\xF9\x86\x27\xE0\x22\x3E" + "\x0D\x1F\x92\x13\x16\x9B\x84\x79\x91\xFE\xBC\xDF" + "\x15\x6C\xDC\xAE\xC5\x64\x46\x9C\xE8\xEA\xBB\x9D" + "\x27\xA2\x1E\xDF\xB2\xF2\x98\xB4\x88\x4E\x22\x5F" + "\x3C\xD0\x35\x2C\x5C\x89\xC7\xEC\xE3\xF3\x09\x60" + "\xEF\xAC\x78\xBF\x94\xAA\x8C\x30\x7F\xCF\xE4\x4B", + .addtllen = 192, + + .expected = "\x93\xFF\x4F\x4A\xA7\xE0\xFC\x3B\xCF\xCE\x57\xCD" + "\xFD\xE9\x94\x6D\x0D\x45\x7E\xA3\x98\xBB\xE6\x54" + "\x36\xF9\x92\xE6\xBB\xC5\x93\x16\x69\x35\x01\xF0" + "\x5A\xE2\x43\xFC\x73\x7C\xA3\x5C\x5B\x49\x72\x69" + "\x6A\xC4\xB6\xD4\xB2\x48\x27\x72\xCA\xE3\xF3\xB8" + "\xD6\x95\x9F\x39\xC8\xD7\x72\xAF\xC2\x7C\x75\xE5" + "\xA4\xF7\x52\x4F\xDF\x80\x75\xCA\x4B\xFA\x75\x9A" + "\x7A\x8D\x48\x94\x30\x1C\x82\x5E\xD6\x3F\xF1\x78" + "\x84\xAA\x8D\x8B\x4F\x6A\x20\x12\xA2\x65\xC5\x02" + "\x32\xA1\xE4\x6B\x86\x2F\xCF\x92\xCD\x01\x55\xF7" + "\xB8\xAB\xC3\x2C\x42\xC3\xFD\x9B\x59\x2D\x47\x8E" + "\xB6\x8D\xE7\x04\x2E\x67\xD9\x07\x14\xA3\xF2\x74" + "\x79\x9C\xAB\xC6\xD6\x52\x3E\x91\x81\xA4\x3C\x9A" + "\x3E\x10\x1E\xD0\x28\xBC\x1C\x50\xF0\xB6\xEC\x01" + "\xE5\x40\x16\x08\xB5\x58\xE0\xC4\xC2\x9A\xBB\x13" + "\xAB\x10\x4B\x07\x24\x09\xEB\xF9\x5E\xAA\x3E\x3A" + "\x25\x34\x5B\x97\x92\xF9\x46\x8A\x8E\x5A\x06\x9C" + "\x2B\x1C\x8E\x47\x19\xC3\x3B\x14\x8A\xC1\xEB\xF7" + "\xB0\xA2\x05\xE0\x70\x12\x88\x21\xFF\xC0\x35\x6D" + "\xE9\x08\x7E\x11\x6E\xD7\x16\x48\x24\x43\xAF\x33" + "\xA3\x48\x44\x91\xC5\x32\x1D\x87\xF6\x9B\x10\x1C" + "\xA4\x7A\x74\x23\xB3\xFC\xB9\x87\x08\xE6\x9A\xEF" + "\xD5\x93\x82\x17\xD4\x78\xDA\x46\x34\xC2\x61\x8C" + "\xB7\x53\x89\xC4\x8A\xD8\xA0\x2E\xCE\xCA\x24\x16" + "\xF0\x18\x7F\xC3\x12\x3F\xD0\xB1\xEF\x37\x99\xA8" + "\x04\x1F\x76\x50\x81\x08\xCD\x4D\x43\xEF\x64\xB6" + "\x03\x57\xB7\x5F\xDD\xF6\xBA\x34\xE2\x88\xE7\x1F" + "\x62\xA5\xA4\x05\xEB\x1A\xFE\x0B\xFE\x74\x7D\x64" + "\x29\x83\xE8\x0B\x97\x7D\x94\x5B\xF9\x3C\x8E\xC8" + "\x6B\xCD\xAD\x63\x32\x6E\x00\xBF\x8E\x57\xF2\x32" + "\x88\x53\x17\x40\xE3\xF8\xF8\xD9\xFD\xCD\x76\x84" + "\xC6\xFD\xB1\x10\x40\xED\x75\xDD\xBD\x78\x8B\x0A" + "\x61\x1E\xCC\xC4\x4F\x95\xF1\x2F\x8F\x5A\xF0\x12" + "\x00\xCE\x75\xB8\xE9\xAB\x63\xC0\x7A\xF6\x9A\x47" + "\xD2\x3C\xB0\xAD\xAF\x03\xDF\x25\xDD\x0C\x0F\xB6" + "\x29\x09\x0F\x28\x5E\xA8\x67\xBB\xC9\x78\xBE\x12" + "\xD8\xD6\xD0\x98\x82\x29\xA3\x82\xFB\x8E\x1C\x79" + "\x28\x52\x22\x73\x3C\x6F\xB5\x33\xFF\x40\x26\x4C" + "\x30\xE4\xE0\x39\x59\x72\x65\x79\x9F\xB7\x89\x87" + "\x64\xAC\x45\x11\x0D\xDD\xC0\x44\x5E\x0D\xDF\xDB" + "\x91\x42\x5F\x52\x7F\xBB\x66\xEE\x32\x90\x8F\x6F" + "\xF8\xB9\xFE\xEA\x42\xFB\x60\x2B\xE3\xD9\xB2\xD1" + "\x74\x37\x14\xC6\xDA\x68\xE6\x09", + .expectedlen = 512, } }; -static const struct drbg_testvec drbg_nopr_ctr_aes192_tv_template[] = { - { - .entropy = (unsigned char *) - "\xc3\x5c\x2f\xa2\xa8\x9d\x52\xa1\x1f\xa3\x2a\xa9" - "\x6c\x95\xb8\xf1\xc9\xa8\xf9\xcb\x24\x5a\x8b\x40" - "\xf3\xa6\xe5\xa7\xfb\xd9\xd3\xc6\x8e\x27\x7b\xa9" - "\xac\x9b\xbb\x00", - .entropylen = 40, - .expected = (unsigned char *) - "\x8c\x2e\x72\xab\xfd\x9b\xb8\x28\x4d\xb7\x9e\x17" - "\xa4\x3a\x31\x46\xcd\x76\x94\xe3\x52\x49\xfc\x33" - "\x83\x91\x4a\x71\x17\xf4\x13\x68\xe6\xd4\xf1\x48" - "\xff\x49\xbf\x29\x07\x6b\x50\x15\xc5\x9f\x45\x79" - "\x45\x66\x2e\x3d\x35\x03\x84\x3f\x4a\xa5\xa3\xdf" - "\x9a\x9d\xf1\x0d", - .expectedlen = 64, - .addtla = NULL, - .addtlb = NULL, - .addtllen = 0, - .pers = NULL, - .perslen = 0, - }, -}; - -static const struct drbg_testvec drbg_nopr_ctr_aes256_tv_template[] = { - { - .entropy = (unsigned char *) - "\x36\x40\x19\x40\xfa\x8b\x1f\xba\x91\xa1\x66\x1f" - "\x21\x1d\x78\xa0\xb9\x38\x9a\x74\xe5\xbc\xcf\xec" - "\xe8\xd7\x66\xaf\x1a\x6d\x3b\x14\x49\x6f\x25\xb0" - "\xf1\x30\x1b\x4f\x50\x1b\xe3\x03\x80\xa1\x37\xeb", - .entropylen = 48, - .expected = (unsigned char *) - "\x58\x62\xeb\x38\xbd\x55\x8d\xd9\x78\xa6\x96\xe6" - "\xdf\x16\x47\x82\xdd\xd8\x87\xe7\xe9\xa6\xc9\xf3" - "\xf1\xfb\xaf\xb7\x89\x41\xb5\x35\xa6\x49\x12\xdf" - "\xd2\x24\xc6\xdc\x74\x54\xe5\x25\x0b\x3d\x97\x16" - "\x5e\x16\x26\x0c\x2f\xaf\x1c\xc7\x73\x5c\xb7\x5f" - "\xb4\xf0\x7e\x1d", - .expectedlen = 64, - .addtla = NULL, - .addtlb = NULL, - .addtllen = 0, - .pers = NULL, - .perslen = 0, - }, -}; - -static const struct drbg_testvec drbg_nopr_ctr_aes128_tv_template[] = { - { - .entropy = (unsigned char *) - "\x87\xe1\xc5\x32\x99\x7f\x57\xa3\x5c\x28\x6d\xe8" - "\x64\xbf\xf2\x64\xa3\x9e\x98\xdb\x6c\x10\x78\x7f", - .entropylen = 24, - .expected = (unsigned char *) - "\x2c\x14\x7e\x24\x11\x9a\xd8\xd4\xb2\xed\x61\xc1" - "\x53\xd0\x50\xc9\x24\xff\x59\x75\x15\xf1\x17\x3a" - "\x3d\xf4\x4b\x2c\x84\x28\xef\x89\x0e\xb9\xde\xf3" - "\xe4\x78\x04\xb2\xfd\x9b\x35\x7f\xe1\x3f\x8a\x3e" - "\x10\xc8\x67\x0a\xf9\xdf\x2d\x6c\x96\xfb\xb2\xb8" - "\xcb\x2d\xd6\xb0", - .expectedlen = 64, - .addtla = NULL, - .addtlb = NULL, - .addtllen = 0, - .pers = NULL, - .perslen = 0, - }, { - .entropy = (unsigned char *) - "\x71\xbd\xce\x35\x42\x7d\x20\xbf\x58\xcf\x17\x74" - "\xce\x72\xd8\x33\x34\x50\x2d\x8f\x5b\x14\xc4\xdd", - .entropylen = 24, - .expected = (unsigned char *) - "\x97\x33\xe8\x20\x12\xe2\x7b\xa1\x46\x8f\xf2\x34" - "\xb3\xc9\xb6\x6b\x20\xb2\x4f\xee\x27\xd8\x0b\x21" - "\x8c\xff\x63\x73\x69\x29\xfb\xf3\x85\xcd\x88\x8e" - "\x43\x2c\x71\x8b\xa2\x55\xd2\x0f\x1d\x7f\xe3\xe1" - "\x2a\xa3\xe9\x2c\x25\x89\xc7\x14\x52\x99\x56\xcc" - "\xc3\xdf\xb3\x81", - .expectedlen = 64, - .addtla = (unsigned char *) - "\x66\xef\x42\xd6\x9a\x8c\x3d\x6d\x4a\x9e\x95\xa6" - "\x91\x4d\x81\x56", - .addtlb = (unsigned char *) - "\xe3\x18\x83\xd9\x4b\x5e\xc4\xcc\xaa\x61\x2f\xbb" - "\x4a\x55\xd1\xc6", - .addtllen = 16, - .pers = NULL, - .perslen = 0, - }, { - .entropy = (unsigned char *) - "\xca\x4b\x1e\xfa\x75\xbd\x69\x36\x38\x73\xb8\xf9" - "\xdb\x4d\x35\x0e\x47\xbf\x6c\x37\x72\xfd\xf7\xa9", - .entropylen = 24, - .expected = (unsigned char *) - "\x59\xc3\x19\x79\x1b\xb1\xf3\x0e\xe9\x34\xae\x6e" - "\x8b\x1f\xad\x1f\x74\xca\x25\x45\x68\xb8\x7f\x75" - "\x12\xf8\xf2\xab\x4c\x23\x01\x03\x05\xe1\x70\xee" - "\x75\xd8\xcb\xeb\x23\x4c\x7a\x23\x6e\x12\x27\xdb" - "\x6f\x7a\xac\x3c\x44\xb7\x87\x4b\x65\x56\x74\x45" - "\x34\x30\x0c\x3d", - .expectedlen = 64, - .addtla = NULL, - .addtlb = NULL, - .addtllen = 0, - .pers = (unsigned char *) - "\xeb\xaa\x60\x2c\x4d\xbe\x33\xff\x1b\xef\xbf\x0a" - "\x0b\xc6\x97\x54", - .perslen = 16, - }, { - .entropy = (unsigned char *) - "\xc0\x70\x1f\x92\x50\x75\x8f\xcd\xf2\xbe\x73\x98" - "\x80\xdb\x66\xeb\x14\x68\xb4\xa5\x87\x9c\x2d\xa6", - .entropylen = 24, - .expected = (unsigned char *) - "\x97\xc0\xc0\xe5\xa0\xcc\xf2\x4f\x33\x63\x48\x8a" - "\xdb\x13\x0a\x35\x89\xbf\x80\x65\x62\xee\x13\x95" - "\x7c\x33\xd3\x7d\xf4\x07\x77\x7a\x2b\x65\x0b\x5f" - "\x45\x5c\x13\xf1\x90\x77\x7f\xc5\x04\x3f\xcc\x1a" - "\x38\xf8\xcd\x1b\xbb\xd5\x57\xd1\x4a\x4c\x2e\x8a" - "\x2b\x49\x1e\x5c", - .expectedlen = 64, - .addtla = (unsigned char *) - "\xf9\x01\xf8\x16\x7a\x1d\xff\xde\x8e\x3c\x83\xe2" - "\x44\x85\xe7\xfe", - .addtlb = (unsigned char *) - "\x17\x1c\x09\x38\xc2\x38\x9f\x97\x87\x60\x55\xb4" - "\x82\x16\x62\x7f", - .addtllen = 16, - .pers = (unsigned char *) - "\x80\x08\xae\xe8\xe9\x69\x40\xc5\x08\x73\xc7\x9f" - "\x8e\xcf\xe0\x02", - .perslen = 16, - }, -}; - /* Cast5 test vectors from RFC 2144 */ static const struct cipher_testvec cast5_tv_template[] = { { diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig index 492a2a61a65b..a5bcef4a54ee 100644 --- a/drivers/char/hw_random/Kconfig +++ b/drivers/char/hw_random/Kconfig @@ -373,6 +373,16 @@ config HW_RANDOM_HISTB To compile this driver as a module, choose M here: the module will be called histb-rng. +config HW_RANDOM_HISI_TRNG + tristate "HiSilicon True Random Number Generator support" + depends on ARM64 && ACPI + help + This driver provides kernel-side support for the True Random Number + Generator hardware found on some HiSilicon SoCs. + + To compile this driver as a module, choose M here: the module will be + called hisi-trng-v2. + config HW_RANDOM_ST tristate "ST Microelectronics HW Random Number Generator support" depends on ARCH_STI || COMPILE_TEST @@ -615,6 +625,17 @@ config HW_RANDOM_ROCKCHIP If unsure, say Y. +config HW_RANDOM_XILINX + tristate "Support for Xilinx True Random Generator" + depends on ZYNQMP_FIRMWARE || COMPILE_TEST + select CRYPTO_LIB_SHA512 + help + Xilinx Versal SoC driver provides kernel-side support for True Random Number + Generator and Pseudo random Number in CTR_DRBG mode as defined in NIST SP800-90A. + + To compile this driver as a module, choose M here: the module + will be called xilinx-trng. + endif # HW_RANDOM config UML_RANDOM diff --git a/drivers/char/hw_random/Makefile b/drivers/char/hw_random/Makefile index b9132b3f5d21..95b5adb49560 100644 --- a/drivers/char/hw_random/Makefile +++ b/drivers/char/hw_random/Makefile @@ -31,6 +31,7 @@ obj-$(CONFIG_HW_RANDOM_PSERIES) += pseries-rng.o obj-$(CONFIG_HW_RANDOM_POWERNV) += powernv-rng.o obj-$(CONFIG_HW_RANDOM_HISI) += hisi-rng.o obj-$(CONFIG_HW_RANDOM_HISTB) += histb-rng.o +obj-$(CONFIG_HW_RANDOM_HISI_TRNG) += hisi-trng-v2.o obj-$(CONFIG_HW_RANDOM_BCM2835) += bcm2835-rng.o obj-$(CONFIG_HW_RANDOM_BCM74110) += bcm74110-rng.o obj-$(CONFIG_HW_RANDOM_IPROC_RNG200) += iproc-rng200.o @@ -52,3 +53,4 @@ obj-$(CONFIG_HW_RANDOM_CN10K) += cn10k-rng.o obj-$(CONFIG_HW_RANDOM_POLARFIRE_SOC) += mpfs-rng.o obj-$(CONFIG_HW_RANDOM_ROCKCHIP) += rockchip-rng.o obj-$(CONFIG_HW_RANDOM_JH7110) += jh7110-trng.o +obj-$(CONFIG_HW_RANDOM_XILINX) += xilinx-trng.o diff --git a/drivers/char/hw_random/amd-rng.c b/drivers/char/hw_random/amd-rng.c index 332d594bdf70..dff80daae587 100644 --- a/drivers/char/hw_random/amd-rng.c +++ b/drivers/char/hw_random/amd-rng.c @@ -47,9 +47,9 @@ * want to register another driver on the same PCI id. */ static const struct pci_device_id pci_tbl[] = { - { PCI_VDEVICE(AMD, 0x7443), 0, }, - { PCI_VDEVICE(AMD, 0x746b), 0, }, - { 0, }, /* terminate list */ + { PCI_VDEVICE(AMD, 0x7443) }, + { PCI_VDEVICE(AMD, 0x746b) }, + { } /* terminate list */ }; MODULE_DEVICE_TABLE(pci, pci_tbl); diff --git a/drivers/char/hw_random/cavium-rng.c b/drivers/char/hw_random/cavium-rng.c index d9d7b6038c06..3e2c042009f6 100644 --- a/drivers/char/hw_random/cavium-rng.c +++ b/drivers/char/hw_random/cavium-rng.c @@ -73,8 +73,8 @@ static void cavium_rng_remove(struct pci_dev *pdev) } static const struct pci_device_id cavium_rng_pf_id_table[] = { - { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, 0xa018), 0, 0, 0}, /* Thunder RNM */ - {0,}, + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, 0xa018) }, /* Thunder RNM */ + { }, }; MODULE_DEVICE_TABLE(pci, cavium_rng_pf_id_table); diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index aba92d777f72..6931657ad2ca 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -25,12 +25,13 @@ #include #include #include +#include #include #include #define RNG_MODULE_NAME "hw_random" -#define RNG_BUFFER_SIZE (SMP_CACHE_BYTES < 32 ? 32 : SMP_CACHE_BYTES) +#define RNG_BUFFER_SIZE MAX(32, SMP_CACHE_BYTES) static struct hwrng __rcu *current_rng; /* the current rng has been explicitly chosen by user via sysfs */ @@ -54,13 +55,9 @@ module_param(default_quality, ushort, 0644); MODULE_PARM_DESC(default_quality, "default maximum entropy content of hwrng per 1024 bits of input"); -static void drop_current_rng(void); static int hwrng_init(struct hwrng *rng); static int hwrng_fillfn(void *unused); -static inline int rng_get_data(struct hwrng *rng, u8 *buffer, size_t size, - int wait); - static size_t rng_buffer_size(void) { return RNG_BUFFER_SIZE; @@ -214,8 +211,8 @@ static int rng_dev_open(struct inode *inode, struct file *filp) return 0; } -static inline int rng_get_data(struct hwrng *rng, u8 *buffer, size_t size, - int wait) { +static inline int rng_get_data(struct hwrng *rng, u8 *buffer, size_t size, bool wait) +{ int present; BUG_ON(!mutex_is_locked(&reading_mutex)); @@ -418,21 +415,17 @@ static ssize_t rng_available_show(struct device *dev, struct device_attribute *attr, char *buf) { - int err; struct hwrng *rng; + int len = 0; - err = mutex_lock_interruptible(&rng_mutex); - if (err) + if (mutex_lock_interruptible(&rng_mutex)) return -ERESTARTSYS; - buf[0] = '\0'; - list_for_each_entry(rng, &rng_list, list) { - strlcat(buf, rng->name, PAGE_SIZE); - strlcat(buf, " ", PAGE_SIZE); - } - strlcat(buf, "none\n", PAGE_SIZE); + list_for_each_entry(rng, &rng_list, list) + len += sysfs_emit_at(buf, len, "%s ", rng->name); + len += sysfs_emit_at(buf, len, "none\n"); mutex_unlock(&rng_mutex); - return strlen(buf); + return len; } static ssize_t rng_selected_show(struct device *dev, @@ -538,8 +531,7 @@ static int hwrng_fillfn(void *unused) } mutex_lock(&reading_mutex); - rc = rng_get_data(rng, rng_fillbuf, - rng_buffer_size(), 1); + rc = rng_get_data(rng, rng_fillbuf, rng_buffer_size(), true); if (current_quality != rng->quality) rng->quality = current_quality; /* obsolete */ quality = rng->quality; diff --git a/drivers/char/hw_random/geode-rng.c b/drivers/char/hw_random/geode-rng.c index 1b21d58e1768..ae63eff64344 100644 --- a/drivers/char/hw_random/geode-rng.c +++ b/drivers/char/hw_random/geode-rng.c @@ -46,8 +46,8 @@ * want to register another driver on the same PCI id. */ static const struct pci_device_id pci_tbl[] = { - { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_LX_AES), 0, }, - { 0, }, /* terminate list */ + { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_LX_AES) }, + { } /* terminate list */ }; MODULE_DEVICE_TABLE(pci, pci_tbl); diff --git a/drivers/char/hw_random/hisi-trng-v2.c b/drivers/char/hw_random/hisi-trng-v2.c new file mode 100644 index 000000000000..6584ed051e09 --- /dev/null +++ b/drivers/char/hw_random/hisi-trng-v2.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2019 HiSilicon Limited. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define HISI_TRNG_REG 0x00F0 +#define HISI_TRNG_BYTES 4 +#define HISI_TRNG_QUALITY 512 +#define SLEEP_US 10 +#define TIMEOUT_US 10000 + +struct hisi_trng { + void __iomem *base; + struct hwrng rng; +}; + +static int hisi_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait) +{ + struct hisi_trng *trng; + int currsize = 0; + u32 val = 0; + int ret; + + trng = container_of(rng, struct hisi_trng, rng); + + do { + ret = readl_poll_timeout(trng->base + HISI_TRNG_REG, val, + val, SLEEP_US, TIMEOUT_US); + if (ret) + return currsize; + + if (max - currsize >= HISI_TRNG_BYTES) { + memcpy(buf + currsize, &val, HISI_TRNG_BYTES); + currsize += HISI_TRNG_BYTES; + if (currsize == max) + return currsize; + continue; + } + + /* copy remaining bytes */ + memcpy(buf + currsize, &val, max - currsize); + currsize = max; + } while (currsize < max); + + return currsize; +} + +static int hisi_trng_probe(struct platform_device *pdev) +{ + struct hisi_trng *trng; + int ret; + + trng = devm_kzalloc(&pdev->dev, sizeof(*trng), GFP_KERNEL); + if (!trng) + return -ENOMEM; + + trng->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(trng->base)) + return PTR_ERR(trng->base); + + trng->rng.name = pdev->name; + trng->rng.read = hisi_trng_read; + trng->rng.quality = HISI_TRNG_QUALITY; + + ret = devm_hwrng_register(&pdev->dev, &trng->rng); + if (ret) + dev_err(&pdev->dev, "failed to register hwrng: %d!\n", ret); + return ret; +} + +static const struct acpi_device_id hisi_trng_acpi_match[] = { + { "HISI02B3", 0 }, + { } +}; +MODULE_DEVICE_TABLE(acpi, hisi_trng_acpi_match); + +static struct platform_driver hisi_trng_driver = { + .probe = hisi_trng_probe, + .driver = { + .name = "hisi-trng-v2", + .acpi_match_table = ACPI_PTR(hisi_trng_acpi_match), + }, +}; + +module_platform_driver(hisi_trng_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Weili Qian "); +MODULE_AUTHOR("Zaibo Xu "); +MODULE_DESCRIPTION("HiSilicon true random number generator V2 driver"); diff --git a/drivers/char/hw_random/jh7110-trng.c b/drivers/char/hw_random/jh7110-trng.c index 9776f4daa044..4712c3c530e4 100644 --- a/drivers/char/hw_random/jh7110-trng.c +++ b/drivers/char/hw_random/jh7110-trng.c @@ -256,19 +256,22 @@ static int starfive_trng_read(struct hwrng *rng, void *buf, size_t max, bool wai if (wait) { ret = starfive_trng_wait_idle(trng); - if (ret) - return -ETIMEDOUT; + if (ret) { + ret = -ETIMEDOUT; + goto out_put; + } } ret = starfive_trng_cmd(trng, STARFIVE_CTRL_GENE_RANDNUM, wait); if (ret) - return ret; + goto out_put; memcpy_fromio(buf, trng->base + STARFIVE_RAND0, max); + ret = max; +out_put: pm_runtime_put_sync_autosuspend(trng->dev); - - return max; + return ret; } static int starfive_trng_probe(struct platform_device *pdev) diff --git a/drivers/char/hw_random/mtk-rng.c b/drivers/char/hw_random/mtk-rng.c index 5808d09d12c4..1f9793010e6c 100644 --- a/drivers/char/hw_random/mtk-rng.c +++ b/drivers/char/hw_random/mtk-rng.c @@ -3,9 +3,11 @@ * Driver for Mediatek Hardware Random Number Generator * * Copyright (C) 2017 Sean Wang + * Copyright (C) 2026 Daniel Golle */ #define MTK_RNG_DEV KBUILD_MODNAME +#include #include #include #include @@ -17,6 +19,7 @@ #include #include #include +#include /* Runtime PM autosuspend timeout: */ #define RNG_AUTOSUSPEND_TIMEOUT 100 @@ -30,6 +33,11 @@ #define RNG_DATA 0x08 +/* Driver feature flags */ +#define MTK_RNG_SMC BIT(0) + +#define MTK_SIP_KERNEL_GET_RND MTK_SIP_SMC_CMD(0x550) + #define to_mtk_rng(p) container_of(p, struct mtk_rng, rng) struct mtk_rng { @@ -37,6 +45,7 @@ struct mtk_rng { struct clk *clk; struct hwrng rng; struct device *dev; + unsigned long flags; }; static int mtk_rng_init(struct hwrng *rng) @@ -103,6 +112,56 @@ static int mtk_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) return retval || !wait ? retval : -EIO; } +static int mtk_rng_read_smc(struct hwrng *rng, void *buf, size_t max, + bool wait) +{ + struct arm_smccc_res res; + int retval = 0; + + while (max >= sizeof(u32)) { + arm_smccc_smc(MTK_SIP_KERNEL_GET_RND, 0, 0, 0, 0, 0, 0, 0, + &res); + if (res.a0) + break; + + *(u32 *)buf = res.a1; + retval += sizeof(u32); + buf += sizeof(u32); + max -= sizeof(u32); + } + + return retval || !wait ? retval : -EIO; +} + +static bool mtk_rng_hw_accessible(struct mtk_rng *priv) +{ + u32 val; + int err; + + err = clk_prepare_enable(priv->clk); + if (err) + return false; + + val = readl(priv->base + RNG_CTRL); + val |= RNG_EN; + writel(val, priv->base + RNG_CTRL); + + val = readl(priv->base + RNG_CTRL); + + if (val & RNG_EN) { + /* HW is accessible, clean up: disable RNG and clock */ + writel(val & ~RNG_EN, priv->base + RNG_CTRL); + clk_disable_unprepare(priv->clk); + return true; + } + + /* + * If TF-A blocks direct access, the register reads back as 0. + * Leave the clock enabled as TF-A needs it. + */ + return false; +} + static int mtk_rng_probe(struct platform_device *pdev) { int ret; @@ -114,23 +173,42 @@ static int mtk_rng_probe(struct platform_device *pdev) priv->dev = &pdev->dev; priv->rng.name = pdev->name; -#ifndef CONFIG_PM - priv->rng.init = mtk_rng_init; - priv->rng.cleanup = mtk_rng_cleanup; -#endif - priv->rng.read = mtk_rng_read; priv->rng.quality = 900; + priv->flags = (unsigned long)device_get_match_data(&pdev->dev); - priv->clk = devm_clk_get(&pdev->dev, "rng"); - if (IS_ERR(priv->clk)) { - ret = PTR_ERR(priv->clk); - dev_err(&pdev->dev, "no clock for device: %d\n", ret); - return ret; + if (!(priv->flags & MTK_RNG_SMC)) { + priv->clk = devm_clk_get(&pdev->dev, "rng"); + if (IS_ERR(priv->clk)) { + ret = PTR_ERR(priv->clk); + dev_err(&pdev->dev, "no clock for device: %d\n", ret); + return ret; + } + + priv->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(priv->base)) + return PTR_ERR(priv->base); + + if (IS_ENABLED(CONFIG_HAVE_ARM_SMCCC) && + of_device_is_compatible(pdev->dev.of_node, + "mediatek,mt7986-rng") && + !mtk_rng_hw_accessible(priv)) { + priv->flags |= MTK_RNG_SMC; + dev_info(&pdev->dev, + "HW RNG not MMIO accessible, using SMC\n"); + } } - priv->base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(priv->base)) - return PTR_ERR(priv->base); + if (priv->flags & MTK_RNG_SMC) { + if (!IS_ENABLED(CONFIG_HAVE_ARM_SMCCC)) + return -ENODEV; + priv->rng.read = mtk_rng_read_smc; + } else { +#ifndef CONFIG_PM + priv->rng.init = mtk_rng_init; + priv->rng.cleanup = mtk_rng_cleanup; +#endif + priv->rng.read = mtk_rng_read; + } ret = devm_hwrng_register(&pdev->dev, &priv->rng); if (ret) { @@ -139,12 +217,15 @@ static int mtk_rng_probe(struct platform_device *pdev) return ret; } - dev_set_drvdata(&pdev->dev, priv); - pm_runtime_set_autosuspend_delay(&pdev->dev, RNG_AUTOSUSPEND_TIMEOUT); - pm_runtime_use_autosuspend(&pdev->dev); - ret = devm_pm_runtime_enable(&pdev->dev); - if (ret) - return ret; + if (!(priv->flags & MTK_RNG_SMC)) { + dev_set_drvdata(&pdev->dev, priv); + pm_runtime_set_autosuspend_delay(&pdev->dev, + RNG_AUTOSUSPEND_TIMEOUT); + pm_runtime_use_autosuspend(&pdev->dev); + ret = devm_pm_runtime_enable(&pdev->dev); + if (ret) + return ret; + } dev_info(&pdev->dev, "registered RNG driver\n"); @@ -181,8 +262,11 @@ static const struct dev_pm_ops mtk_rng_pm_ops = { #endif /* CONFIG_PM */ static const struct of_device_id mtk_rng_match[] = { - { .compatible = "mediatek,mt7986-rng" }, { .compatible = "mediatek,mt7623-rng" }, + { .compatible = "mediatek,mt7981-rng", .data = (void *)MTK_RNG_SMC }, + { .compatible = "mediatek,mt7986-rng" }, + { .compatible = "mediatek,mt7987-rng", .data = (void *)MTK_RNG_SMC }, + { .compatible = "mediatek,mt7988-rng", .data = (void *)MTK_RNG_SMC }, {}, }; MODULE_DEVICE_TABLE(of, mtk_rng_match); @@ -200,4 +284,5 @@ module_platform_driver(mtk_rng_driver); MODULE_DESCRIPTION("Mediatek Random Number Generator Driver"); MODULE_AUTHOR("Sean Wang "); +MODULE_AUTHOR("Daniel Golle "); MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/xilinx/xilinx-trng.c b/drivers/char/hw_random/xilinx-trng.c similarity index 75% rename from drivers/crypto/xilinx/xilinx-trng.c rename to drivers/char/hw_random/xilinx-trng.c index 5276ac2d82bb..f615d5adddde 100644 --- a/drivers/crypto/xilinx/xilinx-trng.c +++ b/drivers/char/hw_random/xilinx-trng.c @@ -4,9 +4,9 @@ * Copyright (c) 2024 - 2025 Advanced Micro Devices, Inc. */ +#include #include #include -#include #include #include #include @@ -14,14 +14,8 @@ #include #include #include -#include #include #include -#include -#include -#include -#include -#include /* TRNG Registers Offsets */ #define TRNG_STATUS_OFFSET 0x4U @@ -47,7 +41,6 @@ /* Sizes in bytes */ #define TRNG_SEED_LEN_BYTES 48U -#define TRNG_ENTROPY_SEED_LEN_BYTES 64U #define TRNG_SEC_STRENGTH_SHIFT 5U #define TRNG_SEC_STRENGTH_BYTES BIT(TRNG_SEC_STRENGTH_SHIFT) #define TRNG_BYTES_PER_REG 4U @@ -59,18 +52,9 @@ struct xilinx_rng { void __iomem *rng_base; struct device *dev; - unsigned char *scratchpadbuf; - struct aes_enckey *aeskey; - struct mutex lock; /* Protect access to TRNG device */ struct hwrng trng; }; -struct xilinx_rng_ctx { - struct xilinx_rng *rng; -}; - -static struct xilinx_rng *xilinx_rng_dev; - static void xtrng_readwrite32(void __iomem *addr, u32 mask, u8 value) { u32 val; @@ -183,29 +167,30 @@ static void xtrng_enable_entropy(struct xilinx_rng *rng) static int xtrng_reseed_internal(struct xilinx_rng *rng) { - u8 entropy[TRNG_ENTROPY_SEED_LEN_BYTES]; - struct drbg_string data; - LIST_HEAD(seedlist); + static const u8 default_salt[SHA512_DIGEST_SIZE]; + u8 entropy[SHA512_DIGEST_SIZE] __aligned(4); u32 val; int ret; - drbg_string_fill(&data, entropy, TRNG_SEED_LEN_BYTES); - list_add_tail(&data.list, &seedlist); - memset(entropy, 0, sizeof(entropy)); xtrng_enable_entropy(rng); - /* collect random data to use it as entropy (input for DF) */ + /* Collect some output from the TRNG. */ + static_assert(sizeof(entropy) >= TRNG_SEED_LEN_BYTES); ret = xtrng_collect_random_data(rng, entropy, TRNG_SEED_LEN_BYTES, true); if (ret != TRNG_SEED_LEN_BYTES) return -EINVAL; - ret = crypto_drbg_ctr_df(rng->aeskey, rng->scratchpadbuf, - TRNG_SEED_LEN_BYTES, &seedlist, AES_BLOCK_SIZE, - TRNG_SEED_LEN_BYTES); - if (ret) - return ret; + /* Extract entropy from the TRNG output using HKDF-SHA512-Extract. */ + hmac_sha512_usingrawkey(default_salt, sizeof(default_salt), entropy, + TRNG_SEED_LEN_BYTES, entropy); + + /* Write the extracted entropy to the hardware. */ xtrng_write_multiple_registers(rng->rng_base + TRNG_EXT_SEED_OFFSET, - (u32 *)rng->scratchpadbuf, TRNG_NUM_INIT_REGS); + (u32 *)entropy, TRNG_NUM_INIT_REGS); + + /* Clear the entropy from the stack. */ + memzero_explicit(entropy, sizeof(entropy)); + /* select reseed operation */ iowrite32(TRNG_CTRL_PRNGXS_MASK, rng->rng_base + TRNG_CTRL_OFFSET); @@ -246,71 +231,25 @@ static int xtrng_random_bytes_generate(struct xilinx_rng *rng, u8 *rand_buf_ptr, return nbytes; } -static int xtrng_trng_generate(struct crypto_rng *tfm, const u8 *src, u32 slen, - u8 *dst, u32 dlen) -{ - struct xilinx_rng_ctx *ctx = crypto_rng_ctx(tfm); - int ret; - - mutex_lock(&ctx->rng->lock); - ret = xtrng_random_bytes_generate(ctx->rng, dst, dlen, true); - mutex_unlock(&ctx->rng->lock); - - return ret < 0 ? ret : 0; -} - -static int xtrng_trng_seed(struct crypto_rng *tfm, const u8 *seed, unsigned int slen) -{ - return 0; -} - -static int xtrng_trng_init(struct crypto_tfm *rtfm) -{ - struct xilinx_rng_ctx *ctx = crypto_tfm_ctx(rtfm); - - ctx->rng = xilinx_rng_dev; - - return 0; -} - -static struct rng_alg xtrng_trng_alg = { - .generate = xtrng_trng_generate, - .seed = xtrng_trng_seed, - .seedsize = 0, - .base = { - .cra_name = "stdrng", - .cra_driver_name = "xilinx-trng", - .cra_priority = 300, - .cra_ctxsize = sizeof(struct xilinx_rng_ctx), - .cra_module = THIS_MODULE, - .cra_init = xtrng_trng_init, - }, -}; - static int xtrng_hwrng_trng_read(struct hwrng *hwrng, void *data, size_t max, bool wait) { u8 buf[TRNG_SEC_STRENGTH_BYTES]; struct xilinx_rng *rng; - int ret = -EINVAL, i = 0; + int ret = 0, i = 0; rng = container_of(hwrng, struct xilinx_rng, trng); - /* Return in case wait not set and lock not available. */ - if (!mutex_trylock(&rng->lock) && !wait) - return 0; - else if (!mutex_is_locked(&rng->lock) && wait) - mutex_lock(&rng->lock); - while (i < max) { ret = xtrng_random_bytes_generate(rng, buf, TRNG_SEC_STRENGTH_BYTES, wait); - if (ret < 0) + if (ret < 0) { + if (i == 0) + return ret; break; + } memcpy(data + i, buf, min_t(int, ret, (max - i))); i += min_t(int, ret, (max - i)); } - mutex_unlock(&rng->lock); - - return ret; + return i; } static int xtrng_hwrng_register(struct hwrng *trng) @@ -335,7 +274,6 @@ static void xtrng_hwrng_unregister(struct hwrng *trng) static int xtrng_probe(struct platform_device *pdev) { struct xilinx_rng *rng; - size_t sb_size; int ret; rng = devm_kzalloc(&pdev->dev, sizeof(*rng), GFP_KERNEL); @@ -349,46 +287,21 @@ static int xtrng_probe(struct platform_device *pdev) return PTR_ERR(rng->rng_base); } - rng->aeskey = devm_kzalloc(&pdev->dev, sizeof(*rng->aeskey), GFP_KERNEL); - if (!rng->aeskey) - return -ENOMEM; - - sb_size = crypto_drbg_ctr_df_datalen(TRNG_SEED_LEN_BYTES, AES_BLOCK_SIZE); - rng->scratchpadbuf = devm_kzalloc(&pdev->dev, sb_size, GFP_KERNEL); - if (!rng->scratchpadbuf) { - ret = -ENOMEM; - goto end; - } - xtrng_trng_reset(rng->rng_base); ret = xtrng_reseed_internal(rng); if (ret) { dev_err(&pdev->dev, "TRNG Seed fail\n"); - goto end; - } - - xilinx_rng_dev = rng; - mutex_init(&rng->lock); - ret = crypto_register_rng(&xtrng_trng_alg); - if (ret) { - dev_err(&pdev->dev, "Crypto Random device registration failed: %d\n", ret); - goto end; + return ret; } ret = xtrng_hwrng_register(&rng->trng); if (ret) { dev_err(&pdev->dev, "HWRNG device registration failed: %d\n", ret); - goto crypto_rng_free; + return ret; } platform_set_drvdata(pdev, rng); return 0; - -crypto_rng_free: - crypto_unregister_rng(&xtrng_trng_alg); - -end: - return ret; } static void xtrng_remove(struct platform_device *pdev) @@ -398,13 +311,11 @@ static void xtrng_remove(struct platform_device *pdev) rng = platform_get_drvdata(pdev); xtrng_hwrng_unregister(&rng->trng); - crypto_unregister_rng(&xtrng_trng_alg); xtrng_write_multiple_registers(rng->rng_base + TRNG_EXT_SEED_OFFSET, zero, TRNG_NUM_INIT_REGS); xtrng_write_multiple_registers(rng->rng_base + TRNG_PER_STRNG_OFFSET, zero, TRNG_NUM_INIT_REGS); xtrng_hold_reset(rng->rng_base); - xilinx_rng_dev = NULL; } static const struct of_device_id xtrng_of_match[] = { diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index d23b58b81ca3..216a00bad5d7 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -301,7 +301,6 @@ config CRYPTO_DEV_PPC4XX select CRYPTO_CCM select CRYPTO_CTR select CRYPTO_GCM - select CRYPTO_RNG select CRYPTO_SKCIPHER help This option allows you to have support for AMCC crypto acceleration. @@ -316,7 +315,7 @@ config HW_RANDOM_PPC4XX config CRYPTO_DEV_OMAP tristate "Support for OMAP crypto HW accelerators" - depends on ARCH_OMAP2PLUS + depends on ARCH_OMAP2PLUS || COMPILE_TEST help OMAP processors have various crypto HW accelerators. Select this if you want to use the OMAP modules for any of the crypto algorithms. @@ -352,7 +351,7 @@ config CRYPTO_DEV_OMAP_AES config CRYPTO_DEV_OMAP_DES tristate "Support for OMAP DES/3DES hw engine" - depends on ARCH_OMAP2PLUS + depends on ARCH_OMAP2PLUS || COMPILE_TEST select CRYPTO_LIB_DES select CRYPTO_SKCIPHER select CRYPTO_ENGINE @@ -375,21 +374,6 @@ config CRYPTO_DEV_SAHARA This option enables support for the SAHARA HW crypto accelerator found in some Freescale i.MX chips. -config CRYPTO_DEV_EXYNOS_RNG - tristate "Exynos HW pseudo random number generator support" - depends on ARCH_EXYNOS || COMPILE_TEST - depends on HAS_IOMEM - select CRYPTO_RNG - help - This driver provides kernel-side support through the - cryptographic API for the pseudo random number generator hardware - found on Exynos SoCs. - - To compile this driver as a module, choose M here: the - module will be called exynos-rng. - - If unsure, say Y. - config CRYPTO_DEV_S5P tristate "Support for Samsung S5PV210/Exynos crypto accelerator" depends on ARCH_S5PV210 || ARCH_EXYNOS || COMPILE_TEST @@ -404,7 +388,6 @@ config CRYPTO_DEV_S5P config CRYPTO_DEV_EXYNOS_HASH bool "Support for Samsung Exynos HASH accelerator" depends on CRYPTO_DEV_S5P - depends on !CRYPTO_DEV_EXYNOS_RNG && CRYPTO_DEV_EXYNOS_RNG!=m select CRYPTO_SHA1 select CRYPTO_MD5 select CRYPTO_SHA256 @@ -412,8 +395,6 @@ config CRYPTO_DEV_EXYNOS_HASH Select this to offload Exynos from HASH MD5/SHA1/SHA256. This will select software SHA1, MD5 and SHA256 as they are needed for small and zero-size messages. - HASH algorithms will be disabled if EXYNOS_RNG - is enabled due to hw conflict. config CRYPTO_DEV_NX bool "Support for IBM PowerPC Nest (NX) cryptographic acceleration" @@ -718,19 +699,6 @@ config CRYPTO_DEV_TEGRA Select this to enable Tegra Security Engine which accelerates various AES encryption/decryption and HASH algorithms. -config CRYPTO_DEV_XILINX_TRNG - tristate "Support for Xilinx True Random Generator" - depends on ZYNQMP_FIRMWARE || COMPILE_TEST - select CRYPTO_DF80090A - select CRYPTO_RNG - select HW_RANDOM - help - Xilinx Versal SoC driver provides kernel-side support for True Random Number - Generator and Pseudo random Number in CTR_DRBG mode as defined in NIST SP800-90A. - - To compile this driver as a module, choose M here: the module - will be called xilinx-trng. - config CRYPTO_DEV_ZYNQMP_AES tristate "Support for Xilinx ZynqMP AES hw accelerator" depends on ZYNQMP_FIRMWARE || COMPILE_TEST @@ -846,7 +814,6 @@ config CRYPTO_DEV_CCREE If unsure say Y. source "drivers/crypto/hisilicon/Kconfig" -source "drivers/crypto/loongson/Kconfig" source "drivers/crypto/amlogic/Kconfig" diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index 283bbc650b5b..5a950c7abc39 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -11,7 +11,6 @@ obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA204A) += atmel-sha204a.o obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/ obj-$(CONFIG_CRYPTO_DEV_CCREE) += ccree/ obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chelsio/ -obj-$(CONFIG_CRYPTO_DEV_EXYNOS_RNG) += exynos-rng.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_COMMON) += caam/ obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o @@ -43,7 +42,6 @@ obj-y += inside-secure/ obj-$(CONFIG_CRYPTO_DEV_ARTPEC6) += axis/ obj-y += xilinx/ obj-y += hisilicon/ -obj-y += loongson/ obj-$(CONFIG_CRYPTO_DEV_AMLOGIC_GXL) += amlogic/ obj-y += intel/ obj-y += starfive/ diff --git a/drivers/crypto/allwinner/Kconfig b/drivers/crypto/allwinner/Kconfig index 7270e5fbc573..06ea0e9fe6f2 100644 --- a/drivers/crypto/allwinner/Kconfig +++ b/drivers/crypto/allwinner/Kconfig @@ -14,7 +14,6 @@ config CRYPTO_DEV_SUN4I_SS select CRYPTO_SHA1 select CRYPTO_AES select CRYPTO_LIB_DES - select CRYPTO_RNG select CRYPTO_SKCIPHER help Some Allwinner SoC have a crypto accelerator named @@ -25,14 +24,6 @@ config CRYPTO_DEV_SUN4I_SS To compile this driver as a module, choose M here: the module will be called sun4i-ss. -config CRYPTO_DEV_SUN4I_SS_PRNG - bool "Support for Allwinner Security System PRNG" - depends on CRYPTO_DEV_SUN4I_SS - select CRYPTO_RNG - help - Select this option if you want to provide kernel-side support for - the Pseudo-Random Number Generator found in the Security System. - config CRYPTO_DEV_SUN4I_SS_DEBUG bool "Enable sun4i-ss stats" depends on CRYPTO_DEV_SUN4I_SS @@ -50,7 +41,6 @@ config CRYPTO_DEV_SUN8I_CE select CRYPTO_CBC select CRYPTO_AES select CRYPTO_DES - select CRYPTO_RNG depends on CRYPTO_DEV_ALLWINNER depends on PM help diff --git a/drivers/crypto/allwinner/sun4i-ss/Makefile b/drivers/crypto/allwinner/sun4i-ss/Makefile index c0a2797d3168..06a9ae81f9f8 100644 --- a/drivers/crypto/allwinner/sun4i-ss/Makefile +++ b/drivers/crypto/allwinner/sun4i-ss/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_CRYPTO_DEV_SUN4I_SS) += sun4i-ss.o sun4i-ss-y += sun4i-ss-core.o sun4i-ss-hash.o sun4i-ss-cipher.o -sun4i-ss-$(CONFIG_CRYPTO_DEV_SUN4I_SS_PRNG) += sun4i-ss-prng.o diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c index 58a76e2ba64e..35ef0930e77f 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c @@ -213,23 +213,6 @@ static struct sun4i_ss_alg_template ss_algs[] = { } } }, -#ifdef CONFIG_CRYPTO_DEV_SUN4I_SS_PRNG -{ - .type = CRYPTO_ALG_TYPE_RNG, - .alg.rng = { - .base = { - .cra_name = "stdrng", - .cra_driver_name = "sun4i_ss_rng", - .cra_priority = 300, - .cra_ctxsize = 0, - .cra_module = THIS_MODULE, - }, - .generate = sun4i_ss_prng_generate, - .seed = sun4i_ss_prng_seed, - .seedsize = SS_SEED_LEN / BITS_PER_BYTE, - } -}, -#endif }; static int sun4i_ss_debugfs_show(struct seq_file *seq, void *v) @@ -247,12 +230,6 @@ static int sun4i_ss_debugfs_show(struct seq_file *seq, void *v) ss_algs[i].stat_req, ss_algs[i].stat_opti, ss_algs[i].stat_fb, ss_algs[i].stat_bytes); break; - case CRYPTO_ALG_TYPE_RNG: - seq_printf(seq, "%s %s reqs=%lu tsize=%lu\n", - ss_algs[i].alg.rng.base.cra_driver_name, - ss_algs[i].alg.rng.base.cra_name, - ss_algs[i].stat_req, ss_algs[i].stat_bytes); - break; case CRYPTO_ALG_TYPE_AHASH: seq_printf(seq, "%s %s reqs=%lu\n", ss_algs[i].alg.hash.halg.base.cra_driver_name, @@ -471,13 +448,6 @@ static int sun4i_ss_probe(struct platform_device *pdev) goto error_alg; } break; - case CRYPTO_ALG_TYPE_RNG: - err = crypto_register_rng(&ss_algs[i].alg.rng); - if (err) { - dev_err(ss->dev, "Fail to register %s\n", - ss_algs[i].alg.rng.base.cra_name); - } - break; } } @@ -497,9 +467,6 @@ error_alg: case CRYPTO_ALG_TYPE_AHASH: crypto_unregister_ahash(&ss_algs[i].alg.hash); break; - case CRYPTO_ALG_TYPE_RNG: - crypto_unregister_rng(&ss_algs[i].alg.rng); - break; } } error_pm: @@ -520,9 +487,6 @@ static void sun4i_ss_remove(struct platform_device *pdev) case CRYPTO_ALG_TYPE_AHASH: crypto_unregister_ahash(&ss_algs[i].alg.hash); break; - case CRYPTO_ALG_TYPE_RNG: - crypto_unregister_rng(&ss_algs[i].alg.rng); - break; } } diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-prng.c b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-prng.c deleted file mode 100644 index 491fcb7b81b4..000000000000 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-prng.c +++ /dev/null @@ -1,69 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -#include "sun4i-ss.h" - -int sun4i_ss_prng_seed(struct crypto_rng *tfm, const u8 *seed, - unsigned int slen) -{ - struct sun4i_ss_alg_template *algt; - struct rng_alg *alg = crypto_rng_alg(tfm); - - algt = container_of(alg, struct sun4i_ss_alg_template, alg.rng); - memcpy(algt->ss->seed, seed, slen); - - return 0; -} - -int sun4i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int dlen) -{ - struct sun4i_ss_alg_template *algt; - struct rng_alg *alg = crypto_rng_alg(tfm); - int i, err; - u32 v; - u32 *data = (u32 *)dst; - const u32 mode = SS_OP_PRNG | SS_PRNG_CONTINUE | SS_ENABLED; - size_t len; - struct sun4i_ss_ctx *ss; - unsigned int todo = (dlen / 4) * 4; - - algt = container_of(alg, struct sun4i_ss_alg_template, alg.rng); - ss = algt->ss; - - err = pm_runtime_resume_and_get(ss->dev); - if (err < 0) - return err; - - if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN4I_SS_DEBUG)) { - algt->stat_req++; - algt->stat_bytes += todo; - } - - spin_lock_bh(&ss->slock); - - writel(mode, ss->base + SS_CTL); - - while (todo > 0) { - /* write the seed */ - for (i = 0; i < SS_SEED_LEN / BITS_PER_LONG; i++) - writel(ss->seed[i], ss->base + SS_KEY0 + i * 4); - - /* Read the random data */ - len = min_t(size_t, SS_DATA_LEN / BITS_PER_BYTE, todo); - readsl(ss->base + SS_TXFIFO, data, len / 4); - data += len / 4; - todo -= len; - - /* Update the seed */ - for (i = 0; i < SS_SEED_LEN / BITS_PER_LONG; i++) { - v = readl(ss->base + SS_KEY0 + i * 4); - ss->seed[i] = v; - } - } - - writel(0, ss->base + SS_CTL); - spin_unlock_bh(&ss->slock); - - pm_runtime_put(ss->dev); - - return 0; -} diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h index 6c5d4aa6453c..f7d1c79ac677 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h @@ -31,8 +31,6 @@ #include #include #include -#include -#include #define SS_CTL 0x00 #define SS_KEY0 0x04 @@ -62,10 +60,6 @@ /* SS_CTL configuration values */ -/* PRNG generator mode - bit 15 */ -#define SS_PRNG_ONESHOT (0 << 15) -#define SS_PRNG_CONTINUE (1 << 15) - /* IV mode for hash */ #define SS_IV_ARBITRARY (1 << 14) @@ -94,14 +88,10 @@ #define SS_OP_3DES (2 << 4) #define SS_OP_SHA1 (3 << 4) #define SS_OP_MD5 (4 << 4) -#define SS_OP_PRNG (5 << 4) /* Data end bit - bit 2 */ #define SS_DATA_END (1 << 2) -/* PRNG start bit - bit 1 */ -#define SS_PRNG_START (1 << 1) - /* SS Enable bit - bit 0 */ #define SS_DISABLED (0 << 0) #define SS_ENABLED (1 << 0) @@ -128,9 +118,6 @@ #define SS_RXFIFO_EMP_INT_ENABLE (1 << 2) #define SS_TXFIFO_AVA_INT_ENABLE (1 << 0) -#define SS_SEED_LEN 192 -#define SS_DATA_LEN 160 - /* * struct ss_variant - Describe SS hardware variant * @sha1_in_be: The SHA1 digest is given by SS in BE, and so need to be inverted. @@ -151,9 +138,6 @@ struct sun4i_ss_ctx { char buf[4 * SS_RX_MAX];/* buffer for linearize SG src */ char bufo[4 * SS_TX_MAX]; /* buffer for linearize SG dst */ spinlock_t slock; /* control the use of the device */ -#ifdef CONFIG_CRYPTO_DEV_SUN4I_SS_PRNG - u32 seed[SS_SEED_LEN / BITS_PER_LONG]; -#endif struct dentry *dbgfs_dir; struct dentry *dbgfs_stats; }; @@ -164,7 +148,6 @@ struct sun4i_ss_alg_template { union { struct skcipher_alg crypto; struct ahash_alg hash; - struct rng_alg rng; } alg; struct sun4i_ss_ctx *ss; unsigned long stat_req; @@ -231,6 +214,3 @@ int sun4i_ss_des_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen); int sun4i_ss_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen); -int sun4i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int dlen); -int sun4i_ss_prng_seed(struct crypto_rng *tfm, const u8 *seed, unsigned int slen); diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c index c16bb6ce6ee3..f3b58ed6aed0 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c @@ -676,6 +676,7 @@ static int sun8i_ce_debugfs_show(struct seq_file *seq, void *v) seq_printf(seq, "\tFallback due to SG numbers: %lu\n", ce_algs[i].stat_fb_maxsg); break; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_HASH case CRYPTO_ALG_TYPE_AHASH: seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n", ce_algs[i].alg.hash.base.halg.base.cra_driver_name, @@ -692,12 +693,15 @@ static int sun8i_ce_debugfs_show(struct seq_file *seq, void *v) seq_printf(seq, "\tFallback due to SG numbers: %lu\n", ce_algs[i].stat_fb_maxsg); break; +#endif +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG case CRYPTO_ALG_TYPE_RNG: seq_printf(seq, "%s %s reqs=%lu bytes=%lu\n", ce_algs[i].alg.rng.base.cra_driver_name, ce_algs[i].alg.rng.base.cra_name, ce_algs[i].stat_req, ce_algs[i].stat_bytes); break; +#endif } } #if defined(CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG) && \ @@ -905,6 +909,7 @@ static int sun8i_ce_register_algs(struct sun8i_ce_dev *ce) return err; } break; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_HASH case CRYPTO_ALG_TYPE_AHASH: id = ce_algs[i].ce_algo_id; ce_method = ce->variant->alg_hash[id]; @@ -925,6 +930,8 @@ static int sun8i_ce_register_algs(struct sun8i_ce_dev *ce) return err; } break; +#endif +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG case CRYPTO_ALG_TYPE_RNG: if (ce->variant->prng == CE_ID_NOTSUPP) { dev_info(ce->dev, @@ -942,6 +949,7 @@ static int sun8i_ce_register_algs(struct sun8i_ce_dev *ce) ce_algs[i].ce = NULL; } break; +#endif default: ce_algs[i].ce = NULL; dev_err(ce->dev, "ERROR: tried to register an unknown algo\n"); @@ -963,16 +971,20 @@ static void sun8i_ce_unregister_algs(struct sun8i_ce_dev *ce) ce_algs[i].alg.skcipher.base.base.cra_name); crypto_engine_unregister_skcipher(&ce_algs[i].alg.skcipher); break; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_HASH case CRYPTO_ALG_TYPE_AHASH: dev_info(ce->dev, "Unregister %d %s\n", i, ce_algs[i].alg.hash.base.halg.base.cra_name); crypto_engine_unregister_ahash(&ce_algs[i].alg.hash); break; +#endif +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG case CRYPTO_ALG_TYPE_RNG: dev_info(ce->dev, "Unregister %d %s\n", i, ce_algs[i].alg.rng.base.cra_name); crypto_unregister_rng(&ce_algs[i].alg.rng); break; +#endif } } } diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c index f45685707e0d..59c9bc45ec0f 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c @@ -501,12 +501,15 @@ static int sun8i_ss_debugfs_show(struct seq_file *seq, void *v) seq_printf(seq, "\tFallback due to SG numbers: %lu\n", ss_algs[i].stat_fb_sgnum); break; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_PRNG case CRYPTO_ALG_TYPE_RNG: seq_printf(seq, "%s %s reqs=%lu tsize=%lu\n", ss_algs[i].alg.rng.base.cra_driver_name, ss_algs[i].alg.rng.base.cra_name, ss_algs[i].stat_req, ss_algs[i].stat_bytes); break; +#endif +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_HASH case CRYPTO_ALG_TYPE_AHASH: seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n", ss_algs[i].alg.hash.base.halg.base.cra_driver_name, @@ -523,6 +526,7 @@ static int sun8i_ss_debugfs_show(struct seq_file *seq, void *v) seq_printf(seq, "\tFallback due to SG numbers: %lu\n", ss_algs[i].stat_fb_sgnum); break; +#endif } } return 0; @@ -707,6 +711,7 @@ static int sun8i_ss_register_algs(struct sun8i_ss_dev *ss) return err; } break; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_PRNG case CRYPTO_ALG_TYPE_RNG: err = crypto_register_rng(&ss_algs[i].alg.rng); if (err) { @@ -715,6 +720,8 @@ static int sun8i_ss_register_algs(struct sun8i_ss_dev *ss) ss_algs[i].ss = NULL; } break; +#endif +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_HASH case CRYPTO_ALG_TYPE_AHASH: id = ss_algs[i].ss_algo_id; ss_method = ss->variant->alg_hash[id]; @@ -735,6 +742,7 @@ static int sun8i_ss_register_algs(struct sun8i_ss_dev *ss) return err; } break; +#endif default: ss_algs[i].ss = NULL; dev_err(ss->dev, "ERROR: tried to register an unknown algo\n"); @@ -756,16 +764,20 @@ static void sun8i_ss_unregister_algs(struct sun8i_ss_dev *ss) ss_algs[i].alg.skcipher.base.base.cra_name); crypto_engine_unregister_skcipher(&ss_algs[i].alg.skcipher); break; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_PRNG case CRYPTO_ALG_TYPE_RNG: dev_info(ss->dev, "Unregister %d %s\n", i, ss_algs[i].alg.rng.base.cra_name); crypto_unregister_rng(&ss_algs[i].alg.rng); break; +#endif +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_HASH case CRYPTO_ALG_TYPE_AHASH: dev_info(ss->dev, "Unregister %d %s\n", i, ss_algs[i].alg.hash.base.halg.base.cra_name); crypto_engine_unregister_ahash(&ss_algs[i].alg.hash); break; +#endif } } } diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index b7b6c97d2147..001da785af07 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -31,11 +31,9 @@ #include #include #include -#include #include #include #include -#include #include #include "crypto4xx_reg_def.h" #include "crypto4xx_core.h" @@ -985,10 +983,6 @@ static int crypto4xx_register_alg(struct crypto4xx_device *sec_dev, rc = crypto_register_aead(&alg->alg.u.aead); break; - case CRYPTO_ALG_TYPE_RNG: - rc = crypto_register_rng(&alg->alg.u.rng); - break; - default: rc = crypto_register_skcipher(&alg->alg.u.cipher); break; @@ -1014,10 +1008,6 @@ static void crypto4xx_unregister_alg(struct crypto4xx_device *sec_dev) crypto_unregister_aead(&alg->alg.u.aead); break; - case CRYPTO_ALG_TYPE_RNG: - crypto_unregister_rng(&alg->alg.u.rng); - break; - default: crypto_unregister_skcipher(&alg->alg.u.cipher); } @@ -1076,69 +1066,6 @@ static irqreturn_t crypto4xx_ce_interrupt_handler_revb(int irq, void *data) PPC4XX_TMO_ERR_INT); } -static int ppc4xx_prng_data_read(struct crypto4xx_device *dev, - u8 *data, unsigned int max) -{ - unsigned int i, curr = 0; - u32 val[2]; - - do { - /* trigger PRN generation */ - writel(PPC4XX_PRNG_CTRL_AUTO_EN, - dev->ce_base + CRYPTO4XX_PRNG_CTRL); - - for (i = 0; i < 1024; i++) { - /* usually 19 iterations are enough */ - if ((readl(dev->ce_base + CRYPTO4XX_PRNG_STAT) & - CRYPTO4XX_PRNG_STAT_BUSY)) - continue; - - val[0] = readl_be(dev->ce_base + CRYPTO4XX_PRNG_RES_0); - val[1] = readl_be(dev->ce_base + CRYPTO4XX_PRNG_RES_1); - break; - } - if (i == 1024) - return -ETIMEDOUT; - - if ((max - curr) >= 8) { - memcpy(data, &val, 8); - data += 8; - curr += 8; - } else { - /* copy only remaining bytes */ - memcpy(data, &val, max - curr); - break; - } - } while (curr < max); - - return curr; -} - -static int crypto4xx_prng_generate(struct crypto_rng *tfm, - const u8 *src, unsigned int slen, - u8 *dstn, unsigned int dlen) -{ - struct rng_alg *alg = crypto_rng_alg(tfm); - struct crypto4xx_alg *amcc_alg; - struct crypto4xx_device *dev; - int ret; - - amcc_alg = container_of(alg, struct crypto4xx_alg, alg.u.rng); - dev = amcc_alg->dev; - - mutex_lock(&dev->core_dev->rng_lock); - ret = ppc4xx_prng_data_read(dev, dstn, dlen); - mutex_unlock(&dev->core_dev->rng_lock); - return ret; -} - - -static int crypto4xx_prng_seed(struct crypto_rng *tfm, const u8 *seed, - unsigned int slen) -{ - return 0; -} - /* * Supported Crypto Algorithms */ @@ -1268,18 +1195,6 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = { .cra_module = THIS_MODULE, }, } }, - { .type = CRYPTO_ALG_TYPE_RNG, .u.rng = { - .base = { - .cra_name = "stdrng", - .cra_driver_name = "crypto4xx_rng", - .cra_priority = 300, - .cra_ctxsize = 0, - .cra_module = THIS_MODULE, - }, - .generate = crypto4xx_prng_generate, - .seed = crypto4xx_prng_seed, - .seedsize = 0, - } }, }; /* @@ -1353,9 +1268,6 @@ static int crypto4xx_probe(struct platform_device *ofdev) core_dev->dev->core_dev = core_dev; core_dev->dev->is_revb = is_revb; core_dev->device = dev; - rc = devm_mutex_init(&ofdev->dev, &core_dev->rng_lock); - if (rc) - return rc; spin_lock_init(&core_dev->lock); INIT_LIST_HEAD(&core_dev->dev->alg_list); ratelimit_default_init(&core_dev->dev->aead_ratelimit); @@ -1382,7 +1294,11 @@ static int crypto4xx_probe(struct platform_device *ofdev) } /* Register for Crypto isr, Crypto Engine IRQ */ - core_dev->irq = irq_of_parse_and_map(ofdev->dev.of_node, 0); + core_dev->irq = platform_get_irq(ofdev, 0); + if (core_dev->irq < 0) { + rc = core_dev->irq; + goto err_iomap; + } rc = devm_request_irq(&ofdev->dev, core_dev->irq, is_revb ? crypto4xx_ce_interrupt_handler_revb : crypto4xx_ce_interrupt_handler, diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index ee36630c670f..66a95733c86d 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -14,10 +14,8 @@ #define __CRYPTO4XX_CORE_H__ #include -#include #include #include -#include #include #include "crypto4xx_reg_def.h" #include "crypto4xx_sa.h" @@ -108,10 +106,9 @@ struct crypto4xx_core_device { struct crypto4xx_device *dev; struct hwrng *trng; u32 int_status; - u32 irq; + int irq; struct tasklet_struct tasklet; spinlock_t lock; - struct mutex rng_lock; }; struct crypto4xx_ctx { @@ -135,7 +132,6 @@ struct crypto4xx_alg_common { union { struct skcipher_alg cipher; struct aead_alg aead; - struct rng_alg rng; } u; }; diff --git a/drivers/crypto/amcc/crypto4xx_reg_def.h b/drivers/crypto/amcc/crypto4xx_reg_def.h index 1038061224da..73d626308a84 100644 --- a/drivers/crypto/amcc/crypto4xx_reg_def.h +++ b/drivers/crypto/amcc/crypto4xx_reg_def.h @@ -90,20 +90,9 @@ #define CRYPTO4XX_BYTE_ORDER_CFG 0x000600d8 #define CRYPTO4XX_ENDIAN_CFG 0x000600d8 -#define CRYPTO4XX_PRNG_STAT 0x00070000 -#define CRYPTO4XX_PRNG_STAT_BUSY 0x1 #define CRYPTO4XX_PRNG_CTRL 0x00070004 #define CRYPTO4XX_PRNG_SEED_L 0x00070008 #define CRYPTO4XX_PRNG_SEED_H 0x0007000c - -#define CRYPTO4XX_PRNG_RES_0 0x00070020 -#define CRYPTO4XX_PRNG_RES_1 0x00070024 -#define CRYPTO4XX_PRNG_RES_2 0x00070028 -#define CRYPTO4XX_PRNG_RES_3 0x0007002C - -#define CRYPTO4XX_PRNG_LFSR_L 0x00070030 -#define CRYPTO4XX_PRNG_LFSR_H 0x00070034 - /* * Initialize CRYPTO ENGINE registers, and memory bases. */ diff --git a/drivers/crypto/amlogic/amlogic-gxl-core.c b/drivers/crypto/amlogic/amlogic-gxl-core.c index 1c18a5b8470e..6cb33949915f 100644 --- a/drivers/crypto/amlogic/amlogic-gxl-core.c +++ b/drivers/crypto/amlogic/amlogic-gxl-core.c @@ -291,8 +291,8 @@ static int meson_crypto_probe(struct platform_device *pdev) return 0; error_alg: meson_unregister_algs(mc); -error_flow: meson_free_chanlist(mc, MAXFLOW - 1); +error_flow: clk_disable_unprepare(mc->busclk); return err; } diff --git a/drivers/crypto/atmel-ecc.c b/drivers/crypto/atmel-ecc.c index 9c380351d2f9..9da9dd6585df 100644 --- a/drivers/crypto/atmel-ecc.c +++ b/drivers/crypto/atmel-ecc.c @@ -56,7 +56,7 @@ static void atmel_ecdh_done(struct atmel_i2c_work_data *work_data, void *areq, goto free_work_data; /* might want less than we've got */ - n_sz = min_t(size_t, ATMEL_ECC_NIST_P256_N_SIZE, req->dst_len); + n_sz = min(ATMEL_ECC_NIST_P256_N_SIZE, req->dst_len); /* copy the shared secret */ copied = sg_copy_from_buffer(req->dst, sg_nents_for_len(req->dst, n_sz), @@ -150,7 +150,7 @@ static int atmel_ecdh_generate_public_key(struct kpp_request *req) return -EINVAL; /* might want less than we've got */ - nbytes = min_t(size_t, ATMEL_ECC_PUBKEY_SIZE, req->dst_len); + nbytes = min(ATMEL_ECC_PUBKEY_SIZE, req->dst_len); /* public key was saved at private key generation */ copied = sg_copy_from_buffer(req->dst, @@ -284,15 +284,7 @@ static unsigned int atmel_ecdh_max_size(struct crypto_kpp *tfm) { struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm); - if (ctx->fallback) - return crypto_kpp_maxsize(ctx->fallback); - - /* - * The device only supports NIST P256 ECC keys. The public key size will - * always be the same. Use a macro for the key size to avoid unnecessary - * computations. - */ - return ATMEL_ECC_PUBKEY_SIZE; + return crypto_kpp_maxsize(ctx->fallback); } static struct kpp_alg atmel_ecdh_nist_p256 = { @@ -368,19 +360,16 @@ static void atmel_ecc_remove(struct i2c_client *client) spin_unlock(&driver_data.i2c_list_lock); } -#ifdef CONFIG_OF static const struct of_device_id atmel_ecc_dt_ids[] = { - { - .compatible = "atmel,atecc508a", - }, { - /* sentinel */ - } + { .compatible = "atmel,atecc508a", }, + { .compatible = "atmel,atecc608b", }, + { } }; MODULE_DEVICE_TABLE(of, atmel_ecc_dt_ids); -#endif static const struct i2c_device_id atmel_ecc_id[] = { - { "atecc508a" }, + { .name = "atecc508a" }, + { .name = "atecc608b" }, { } }; MODULE_DEVICE_TABLE(i2c, atmel_ecc_id); @@ -388,7 +377,7 @@ MODULE_DEVICE_TABLE(i2c, atmel_ecc_id); static struct i2c_driver atmel_ecc_driver = { .driver = { .name = "atmel-ecc", - .of_match_table = of_match_ptr(atmel_ecc_dt_ids), + .of_match_table = atmel_ecc_dt_ids, }, .probe = atmel_ecc_probe, .remove = atmel_ecc_remove, diff --git a/drivers/crypto/atmel-i2c.c b/drivers/crypto/atmel-i2c.c index 0e275dbdc8c5..ff19857894d0 100644 --- a/drivers/crypto/atmel-i2c.c +++ b/drivers/crypto/atmel-i2c.c @@ -294,7 +294,7 @@ void atmel_i2c_enqueue(struct atmel_i2c_work_data *work_data, void *areq, int status), void *areq) { - work_data->cbk = (void *)cbk; + work_data->cbk = cbk; work_data->areq = areq; INIT_WORK(&work_data->work, atmel_i2c_work_handler); diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c index 002b62902553..8e3b8efa8109 100644 --- a/drivers/crypto/atmel-sha.c +++ b/drivers/crypto/atmel-sha.c @@ -305,12 +305,12 @@ static size_t atmel_sha_append_sg(struct atmel_sha_reqctx *ctx) size_t count; while ((ctx->bufcnt < ctx->buflen) && ctx->total) { - count = min(ctx->sg->length - ctx->offset, ctx->total); - count = min(count, ctx->buflen - ctx->bufcnt); + count = min3(ctx->sg->length - ctx->offset, ctx->total, + ctx->buflen - ctx->bufcnt); - if (count <= 0) { + if (count == 0) { /* - * Check if count <= 0 because the buffer is full or + * Check if count == 0 because the buffer is full or * because the sg length is 0. In the latest case, * check if there is another sg in the list, a 0 length * sg doesn't necessarily mean the end of the sg list. @@ -1724,8 +1724,7 @@ static int atmel_sha_hmac_setup(struct atmel_sha_dev *dd, return atmel_sha_hmac_prehash_key(dd, key, keylen); /* Prepare ipad. */ - memcpy((u8 *)hmac->ipad, key, keylen); - memset((u8 *)hmac->ipad + keylen, 0, bs - keylen); + memcpy_and_pad(hmac->ipad, bs, key, keylen, 0); return atmel_sha_hmac_compute_ipad_hash(dd); } diff --git a/drivers/crypto/atmel-sha204a.c b/drivers/crypto/atmel-sha204a.c index dbb39ed0cea1..4c9af737b33a 100644 --- a/drivers/crypto/atmel-sha204a.c +++ b/drivers/crypto/atmel-sha204a.c @@ -19,6 +19,12 @@ #include #include "atmel-i2c.h" +/* + * According to review by Bill Cox [1], the ATSHA204 has very low entropy. + * [1] https://www.metzdowd.com/pipermail/cryptography/2014-December/023858.html + */ +static const unsigned short atsha204_quality = 1; + static void atmel_sha204a_rng_done(struct atmel_i2c_work_data *work_data, void *areq, int status) { @@ -48,8 +54,8 @@ static int atmel_sha204a_rng_read_nonblocking(struct hwrng *rng, void *data, if (rng->priv) { work_data = (struct atmel_i2c_work_data *)rng->priv; - max = min(sizeof(work_data->cmd.data), max); - memcpy(data, &work_data->cmd.data, max); + max = min(RANDOM_RSP_SIZE - CMD_OVERHEAD_SIZE, max); + memcpy(data, &work_data->cmd.data[RSP_DATA_IDX], max); rng->priv = 0; } else { work_data = kmalloc_obj(*work_data, GFP_ATOMIC); @@ -87,8 +93,8 @@ static int atmel_sha204a_rng_read(struct hwrng *rng, void *data, size_t max, if (ret) return ret; - max = min(sizeof(cmd.data), max); - memcpy(data, cmd.data, max); + max = min(RANDOM_RSP_SIZE - CMD_OVERHEAD_SIZE, max); + memcpy(data, &cmd.data[RSP_DATA_IDX], max); return max; } @@ -158,6 +164,7 @@ static const struct attribute_group atmel_sha204a_groups = { static int atmel_sha204a_probe(struct i2c_client *client) { struct atmel_i2c_client_priv *i2c_priv; + const unsigned short *quality; int ret; ret = atmel_i2c_probe(client); @@ -171,19 +178,19 @@ static int atmel_sha204a_probe(struct i2c_client *client) i2c_priv->hwrng.name = dev_name(&client->dev); i2c_priv->hwrng.read = atmel_sha204a_rng_read; - /* - * According to review by Bill Cox [1], this HWRNG has very low entropy. - * [1] https://www.metzdowd.com/pipermail/cryptography/2014-December/023858.html - */ - i2c_priv->hwrng.quality = 1; + quality = i2c_get_match_data(client); + if (quality) + i2c_priv->hwrng.quality = *quality; ret = devm_hwrng_register(&client->dev, &i2c_priv->hwrng); - if (ret) - dev_warn(&client->dev, "failed to register RNG (%d)\n", ret); + if (ret) { + dev_err(&client->dev, "failed to register RNG (%d)\n", ret); + return ret; + } ret = sysfs_create_group(&client->dev.kobj, &atmel_sha204a_groups); if (ret) { - dev_err(&client->dev, "failed to register sysfs entry\n"); + dev_err(&client->dev, "failed to create sysfs group (%d)\n", ret); return ret; } @@ -194,25 +201,24 @@ static void atmel_sha204a_remove(struct i2c_client *client) { struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); + sysfs_remove_group(&client->dev.kobj, &atmel_sha204a_groups); devm_hwrng_unregister(&client->dev, &i2c_priv->hwrng); atmel_i2c_flush_queue(); - sysfs_remove_group(&client->dev.kobj, &atmel_sha204a_groups); - kfree((void *)i2c_priv->hwrng.priv); } -static const struct of_device_id atmel_sha204a_dt_ids[] __maybe_unused = { - { .compatible = "atmel,atsha204", }, - { .compatible = "atmel,atsha204a", }, - { /* sentinel */ } +static const struct of_device_id atmel_sha204a_dt_ids[] = { + { .compatible = "atmel,atsha204" }, + { .compatible = "atmel,atsha204a" }, + { } }; MODULE_DEVICE_TABLE(of, atmel_sha204a_dt_ids); static const struct i2c_device_id atmel_sha204a_id[] = { - { "atsha204" }, - { "atsha204a" }, - { /* sentinel */ } + { .name = "atsha204", .driver_data = (kernel_ulong_t)&atsha204_quality }, + { .name = "atsha204a", .driver_data = (kernel_ulong_t)NULL }, + { } }; MODULE_DEVICE_TABLE(i2c, atmel_sha204a_id); @@ -222,7 +228,7 @@ static struct i2c_driver atmel_sha204a_driver = { .id_table = atmel_sha204a_id, .driver.name = "atmel-sha204a", - .driver.of_match_table = of_match_ptr(atmel_sha204a_dt_ids), + .driver.of_match_table = atmel_sha204a_dt_ids, }; static int __init atmel_sha204a_init(void) diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c index a4793b76300c..2bf8d71c86f4 100644 --- a/drivers/crypto/axis/artpec6_crypto.c +++ b/drivers/crypto/axis/artpec6_crypto.c @@ -706,22 +706,19 @@ artpec6_crypto_setup_out_descr(struct artpec6_crypto_req_common *common, void *dst, unsigned int len, bool eop, bool use_short) { - if (use_short && len < 7) { + dma_addr_t dma_addr; + int ret; + + if (use_short && len < 7) return artpec6_crypto_setup_out_descr_short(common, dst, len, eop); - } else { - int ret; - dma_addr_t dma_addr; - ret = artpec6_crypto_dma_map_single(common, dst, len, - DMA_TO_DEVICE, - &dma_addr); - if (ret) - return ret; + ret = artpec6_crypto_dma_map_single(common, dst, len, DMA_TO_DEVICE, + &dma_addr); + if (ret) + return ret; - return artpec6_crypto_setup_out_descr_phys(common, dma_addr, - len, eop); - } + return artpec6_crypto_setup_out_descr_phys(common, dma_addr, len, eop); } /** artpec6_crypto_setup_in_descr_phys - Setup an in channel with a diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c index 2bce15dc0aa8..240b40ae9cd6 100644 --- a/drivers/crypto/bcm/cipher.c +++ b/drivers/crypto/bcm/cipher.c @@ -4698,9 +4698,9 @@ static void bcm_spu_remove(struct platform_device *pdev) static struct platform_driver bcm_spu_pdriver = { .driver = { - .name = "brcm-spu-crypto", - .of_match_table = of_match_ptr(bcm_spu_dt_ids), - }, + .name = "brcm-spu-crypto", + .of_match_table = bcm_spu_dt_ids, + }, .probe = bcm_spu_probe, .remove = bcm_spu_remove, }; diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 32a6e6e15ee2..ddbd60cf3b3c 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -603,7 +603,7 @@ static int aead_setkey(struct crypto_aead *aead, dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n", keys.authkeylen + keys.enckeylen, keys.enckeylen, keys.authkeylen); - print_hex_dump_debug("key in @"__stringify(__LINE__)": ", + print_hex_dump_devel("key in @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); /* @@ -639,7 +639,7 @@ static int aead_setkey(struct crypto_aead *aead, dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + keys.enckeylen, ctx->dir); - print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ", + print_hex_dump_devel("ctx.key@"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, ctx->adata.keylen_pad + keys.enckeylen, 1); @@ -680,7 +680,7 @@ static int gcm_setkey(struct crypto_aead *aead, if (err) return err; - print_hex_dump_debug("key in @"__stringify(__LINE__)": ", + print_hex_dump_devel("key in @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); @@ -701,7 +701,7 @@ static int rfc4106_setkey(struct crypto_aead *aead, if (err) return err; - print_hex_dump_debug("key in @"__stringify(__LINE__)": ", + print_hex_dump_devel("key in @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); @@ -727,7 +727,7 @@ static int rfc4543_setkey(struct crypto_aead *aead, if (err) return err; - print_hex_dump_debug("key in @"__stringify(__LINE__)": ", + print_hex_dump_devel("key in @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); @@ -754,7 +754,7 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, u32 *desc; const bool is_rfc3686 = alg->caam.rfc3686; - print_hex_dump_debug("key in @"__stringify(__LINE__)": ", + print_hex_dump_devel("key in @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); /* Here keylen is actual key length */ diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index 65f6adb6c673..aa779caacfe5 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -212,7 +212,7 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key, dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n", keys.authkeylen + keys.enckeylen, keys.enckeylen, keys.authkeylen); - print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); /* @@ -248,7 +248,7 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key, ctx->adata.keylen_pad + keys.enckeylen, ctx->dir); - print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ", + print_hex_dump_devel("ctx.key@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, ctx->adata.keylen_pad + keys.enckeylen, 1); @@ -371,7 +371,7 @@ static int gcm_setkey(struct crypto_aead *aead, if (ret) return ret; - print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); @@ -475,7 +475,7 @@ static int rfc4106_setkey(struct crypto_aead *aead, if (ret) return ret; - print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); @@ -581,7 +581,7 @@ static int rfc4543_setkey(struct crypto_aead *aead, if (ret) return ret; - print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); @@ -631,7 +631,7 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, const bool is_rfc3686 = alg->caam.rfc3686; int ret = 0; - print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); ctx->cdata.keylen = keylen; diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c index bf10c3dda745..6b47bcc16a50 100644 --- a/drivers/crypto/caam/caamalg_qi2.c +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -301,7 +301,7 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key, dev_dbg(dev, "keylen %d enckeylen %d authkeylen %d\n", keys.authkeylen + keys.enckeylen, keys.enckeylen, keys.authkeylen); - print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); ctx->adata.keylen = keys.authkeylen; @@ -315,7 +315,7 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key, memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad + keys.enckeylen, ctx->dir); - print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ", + print_hex_dump_devel("ctx.key@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, ctx->adata.keylen_pad + keys.enckeylen, 1); @@ -732,7 +732,7 @@ static int gcm_setkey(struct crypto_aead *aead, ret = aes_check_keylen(keylen); if (ret) return ret; - print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); @@ -828,7 +828,7 @@ static int rfc4106_setkey(struct crypto_aead *aead, if (ret) return ret; - print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); @@ -927,7 +927,7 @@ static int rfc4543_setkey(struct crypto_aead *aead, if (ret) return ret; - print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); @@ -955,7 +955,7 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, u32 *desc; const bool is_rfc3686 = alg->caam.rfc3686; - print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); ctx->cdata.keylen = keylen; diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c index ddb2a35aec2d..3cd71810380f 100644 --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -505,7 +505,7 @@ static int axcbc_setkey(struct crypto_ahash *ahash, const u8 *key, DMA_TO_DEVICE); ctx->adata.keylen = keylen; - print_hex_dump_debug("axcbc ctx.key@" __stringify(__LINE__)" : ", + print_hex_dump_devel("axcbc ctx.key@" __stringify(__LINE__)" : ", DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, keylen, 1); return axcbc_set_sh_desc(ahash); @@ -525,7 +525,7 @@ static int acmac_setkey(struct crypto_ahash *ahash, const u8 *key, ctx->adata.key_virt = key; ctx->adata.keylen = keylen; - print_hex_dump_debug("acmac ctx.key@" __stringify(__LINE__)" : ", + print_hex_dump_devel("acmac ctx.key@" __stringify(__LINE__)" : ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); return acmac_set_sh_desc(ahash); diff --git a/drivers/crypto/caam/key_gen.c b/drivers/crypto/caam/key_gen.c index 88cc4fe2a585..de2fcc387477 100644 --- a/drivers/crypto/caam/key_gen.c +++ b/drivers/crypto/caam/key_gen.c @@ -58,7 +58,7 @@ int gen_split_key(struct device *jrdev, u8 *key_out, dev_dbg(jrdev, "split keylen %d split keylen padded %d\n", adata->keylen, adata->keylen_pad); - print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ", + print_hex_dump_devel("ctx.key@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key_in, keylen, 1); if (local_max > max_keylen) @@ -113,7 +113,7 @@ int gen_split_key(struct device *jrdev, u8 *key_out, wait_for_completion(&result.completion); ret = result.err; - print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ", + print_hex_dump_devel("ctx.key@"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key_out, adata->keylen_pad, 1); } diff --git a/drivers/crypto/cavium/cpt/cptpf_main.c b/drivers/crypto/cavium/cpt/cptpf_main.c index 54de869e5374..9358c1c041d4 100644 --- a/drivers/crypto/cavium/cpt/cptpf_main.c +++ b/drivers/crypto/cavium/cpt/cptpf_main.c @@ -651,6 +651,7 @@ static const struct pci_device_id cpt_id_table[] = { { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, CPT_81XX_PCI_PF_DEVICE_ID) }, { 0, } /* end of table */ }; +MODULE_DEVICE_TABLE(pci, cpt_id_table); static struct pci_driver cpt_pci_driver = { .name = DRV_NAME, @@ -666,4 +667,3 @@ MODULE_AUTHOR("George Cherian "); MODULE_DESCRIPTION("Cavium Thunder CPT Physical Function Driver"); MODULE_LICENSE("GPL v2"); MODULE_VERSION(DRV_VERSION); -MODULE_DEVICE_TABLE(pci, cpt_id_table); diff --git a/drivers/crypto/cavium/cpt/cptvf_main.c b/drivers/crypto/cavium/cpt/cptvf_main.c index 2c9a2af38876..db76df14c4a0 100644 --- a/drivers/crypto/cavium/cpt/cptvf_main.c +++ b/drivers/crypto/cavium/cpt/cptvf_main.c @@ -835,9 +835,10 @@ static void cptvf_shutdown(struct pci_dev *pdev) /* Supported devices */ static const struct pci_device_id cptvf_id_table[] = { - {PCI_VDEVICE(CAVIUM, CPT_81XX_PCI_VF_DEVICE_ID), 0}, - { 0, } /* end of table */ + { PCI_VDEVICE(CAVIUM, CPT_81XX_PCI_VF_DEVICE_ID) }, + { } /* end of table */ }; +MODULE_DEVICE_TABLE(pci, cptvf_id_table); static struct pci_driver cptvf_pci_driver = { .name = DRV_NAME, @@ -853,4 +854,3 @@ MODULE_AUTHOR("George Cherian "); MODULE_DESCRIPTION("Cavium Thunder CPT Virtual Function Driver"); MODULE_LICENSE("GPL v2"); MODULE_VERSION(DRV_VERSION); -MODULE_DEVICE_TABLE(pci, cptvf_id_table); diff --git a/drivers/crypto/cavium/cpt/cptvf_reqmanager.c b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c index e183b60277ff..de305cbeccbe 100644 --- a/drivers/crypto/cavium/cpt/cptvf_reqmanager.c +++ b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c @@ -108,8 +108,8 @@ static int setup_sgio_components(struct cpt_vf *cptvf, struct buf_ptr *list, sg_cleanup: for (j = 0; j < i; j++) { if (list[j].dma_addr) { - dma_unmap_single(&pdev->dev, list[i].dma_addr, - list[i].size, DMA_BIDIRECTIONAL); + dma_unmap_single(&pdev->dev, list[j].dma_addr, + list[j].size, DMA_BIDIRECTIONAL); } list[j].dma_addr = 0; diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c index 8664d97261fe..e474c84d8d38 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_main.c +++ b/drivers/crypto/cavium/nitrox/nitrox_main.c @@ -38,9 +38,9 @@ static unsigned int num_devices; * nitrox_pci_tbl - PCI Device ID Table */ static const struct pci_device_id nitrox_pci_tbl[] = { - {PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0}, + { PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID) }, /* required last entry */ - {0, } + { } }; MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl); diff --git a/drivers/crypto/ccp/psp-dev.c b/drivers/crypto/ccp/psp-dev.c index 5c7f7e02a7d8..b14ce51065d5 100644 --- a/drivers/crypto/ccp/psp-dev.c +++ b/drivers/crypto/ccp/psp-dev.c @@ -316,16 +316,16 @@ void psp_dev_destroy(struct sp_device *sp) if (!psp) return; - sev_dev_destroy(psp); - - tee_dev_destroy(psp); - - sfs_dev_destroy(psp); - dbc_dev_destroy(psp); platform_access_dev_destroy(psp); + sfs_dev_destroy(psp); + + tee_dev_destroy(psp); + + sev_dev_destroy(psp); + sp_free_psp_irq(sp, psp); if (sp->clear_psp_master_device) diff --git a/drivers/crypto/ccp/sev-dev-tsm.c b/drivers/crypto/ccp/sev-dev-tsm.c index b07ae529b591..46f2539d2d5a 100644 --- a/drivers/crypto/ccp/sev-dev-tsm.c +++ b/drivers/crypto/ccp/sev-dev-tsm.c @@ -58,13 +58,13 @@ static int stream_enable(struct pci_ide *ide) struct pci_dev *rp = pcie_find_root_port(ide->pdev); int ret; - ret = pci_ide_stream_enable(rp, ide); - if (ret) + ret = pci_ide_stream_enable(ide->pdev, ide); + if (ret && ret != -ENXIO) return ret; - ret = pci_ide_stream_enable(ide->pdev, ide); - if (ret) - pci_ide_stream_disable(rp, ide); + ret = pci_ide_stream_enable(rp, ide); + if (ret && ret != -ENXIO) + pci_ide_stream_disable(ide->pdev, ide); return ret; } @@ -248,12 +248,19 @@ static void dsm_remove(struct pci_tsm *tsm) static int dsm_create(struct tio_dsm *dsm) { struct pci_dev *pdev = dsm->tsm.base_tsm.pdev; - u8 segment_id = pdev->bus ? pci_domain_nr(pdev->bus) : 0; - struct pci_dev *rootport = pcie_find_root_port(pdev); - u16 device_id = pci_dev_id(pdev); + struct pci_dev *rootport; + u8 segment_id; + u16 device_id; u16 root_port_id; u32 lnkcap = 0; + if (!pdev->bus) + return -ENODEV; + + segment_id = pci_domain_nr(pdev->bus); + rootport = pcie_find_root_port(pdev); + device_id = pci_dev_id(pdev); + if (pci_read_config_dword(rootport, pci_pcie_cap(rootport) + PCI_EXP_LNKCAP, &lnkcap)) return -ENODEV; diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 78f98aee7a66..a8eb51ec0ee2 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -1328,10 +1328,11 @@ static int snp_filter_reserved_mem_regions(struct resource *rs, void *arg) size_t size; /* - * Ensure the list of HV_FIXED pages that will be passed to firmware - * do not exceed the page-sized argument buffer. + * Ensure the list of HV_FIXED pages passed to the firmware including + * the one about to be written to do not exceed the page-sized argument + * buffer. */ - if ((range_list->num_elements * sizeof(struct sev_data_range) + + if (((range_list->num_elements + 1) * sizeof(struct sev_data_range) + sizeof(struct sev_data_range_list)) > PAGE_SIZE) return -E2BIG; @@ -1355,7 +1356,7 @@ static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid) { struct sev_data_range_list *snp_range_list __free(kfree) = NULL; struct psp_device *psp = psp_master; - struct sev_data_snp_init_ex data; + struct sev_data_snp_init_ex data = {}; struct sev_device *sev; void *arg = &data; int cmd, rc = 0; @@ -1421,8 +1422,6 @@ static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid) */ snp_add_hv_fixed_pages(sev, snp_range_list); - memset(&data, 0, sizeof(data)); - if (max_snp_asid) { data.ciphertext_hiding_en = 1; data.max_snp_asid = max_snp_asid; @@ -1489,6 +1488,8 @@ static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid) &snp_panic_notifier); if (data.tio_en) { + struct page *page; + /* * This executes with the sev_cmd_mutex held so down the stack * snp_reclaim_pages(locked=false) might be needed (which is extremely @@ -1496,12 +1497,14 @@ static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid) * Instead of exporting __snp_alloc_firmware_pages(), allocate a page * for this one call here. */ - void *tio_status = page_address(__snp_alloc_firmware_pages( - GFP_KERNEL_ACCOUNT | __GFP_ZERO, 0, true)); + page = __snp_alloc_firmware_pages(GFP_KERNEL_ACCOUNT | __GFP_ZERO, + 0, true); + if (page) { + void *tio_status = page_address(page); - if (tio_status) { sev_tsm_init_locked(sev, tio_status); - __snp_free_firmware_pages(virt_to_page(tio_status), 0, true); + + __snp_free_firmware_pages(page, 0, true); } } @@ -1718,29 +1721,11 @@ static int sev_get_platform_state(int *state, int *error) static int sev_move_to_init_state(struct sev_issue_cmd *argp, bool *shutdown_required) { - struct sev_platform_init_args init_args = {0}; int rc; - rc = _sev_platform_init_locked(&init_args); - if (rc) { - argp->error = SEV_RET_INVALID_PLATFORM_STATE; + rc = __sev_platform_init_locked(&argp->error); + if (rc) return rc; - } - - *shutdown_required = true; - - return 0; -} - -static int snp_move_to_init_state(struct sev_issue_cmd *argp, bool *shutdown_required) -{ - int error, rc; - - rc = __sev_snp_init_locked(&error, 0); - if (rc) { - argp->error = SEV_RET_INVALID_PLATFORM_STATE; - return rc; - } *shutdown_required = true; @@ -2303,7 +2288,8 @@ static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable) /* Userspace wants to query the certificate length. */ if (!input.pdh_cert_address || !input.pdh_cert_len || - !input.cert_chain_address) + !input.cert_chain_address || + !input.cert_chain_len) goto cmd; /* Allocate a physically contiguous buffer to store the PDH blob. */ @@ -2442,24 +2428,13 @@ cleanup: static int sev_ioctl_do_snp_commit(struct sev_issue_cmd *argp) { - struct sev_device *sev = psp_master->sev_data; struct sev_data_snp_commit buf; - bool shutdown_required = false; - int ret, error; - - if (!sev->snp_initialized) { - ret = snp_move_to_init_state(argp, &shutdown_required); - if (ret) - return ret; - } + int ret; buf.len = sizeof(buf); ret = __sev_do_cmd_locked(SEV_CMD_SNP_COMMIT, &buf, &argp->error); - if (shutdown_required) - __sev_snp_shutdown_locked(&error, false); - return ret; } @@ -2467,8 +2442,6 @@ static int sev_ioctl_do_snp_set_config(struct sev_issue_cmd *argp, bool writable { struct sev_device *sev = psp_master->sev_data; struct sev_user_data_snp_config config; - bool shutdown_required = false; - int ret, error; if (!argp->data) return -EINVAL; @@ -2476,30 +2449,21 @@ static int sev_ioctl_do_snp_set_config(struct sev_issue_cmd *argp, bool writable if (!writable) return -EPERM; + if (!sev->snp_initialized) + return -ENODEV; + if (copy_from_user(&config, (void __user *)argp->data, sizeof(config))) return -EFAULT; - if (!sev->snp_initialized) { - ret = snp_move_to_init_state(argp, &shutdown_required); - if (ret) - return ret; - } - - ret = __sev_do_cmd_locked(SEV_CMD_SNP_CONFIG, &config, &argp->error); - - if (shutdown_required) - __sev_snp_shutdown_locked(&error, false); - - return ret; + return __sev_do_cmd_locked(SEV_CMD_SNP_CONFIG, &config, &argp->error); } static int sev_ioctl_do_snp_vlek_load(struct sev_issue_cmd *argp, bool writable) { struct sev_device *sev = psp_master->sev_data; struct sev_user_data_snp_vlek_load input; - bool shutdown_required = false; - int ret, error; void *blob; + int ret; if (!argp->data) return -EINVAL; @@ -2507,6 +2471,9 @@ static int sev_ioctl_do_snp_vlek_load(struct sev_issue_cmd *argp, bool writable) if (!writable) return -EPERM; + if (!sev->snp_initialized) + return -ENODEV; + if (copy_from_user(&input, u64_to_user_ptr(argp->data), sizeof(input))) return -EFAULT; @@ -2520,18 +2487,7 @@ static int sev_ioctl_do_snp_vlek_load(struct sev_issue_cmd *argp, bool writable) input.vlek_wrapped_address = __psp_pa(blob); - if (!sev->snp_initialized) { - ret = snp_move_to_init_state(argp, &shutdown_required); - if (ret) - goto cleanup; - } - ret = __sev_do_cmd_locked(SEV_CMD_SNP_VLEK_LOAD, &input, &argp->error); - - if (shutdown_required) - __sev_snp_shutdown_locked(&error, false); - -cleanup: kfree(blob); return ret; diff --git a/drivers/crypto/ccp/sp-pci.c b/drivers/crypto/ccp/sp-pci.c index 6ac805d99ccb..ede6ff9ad0c2 100644 --- a/drivers/crypto/ccp/sp-pci.c +++ b/drivers/crypto/ccp/sp-pci.c @@ -552,21 +552,21 @@ static const struct sp_dev_vdata dev_vdata[] = { }; static const struct pci_device_id sp_pci_table[] = { - { PCI_VDEVICE(AMD, 0x1537), (kernel_ulong_t)&dev_vdata[0] }, - { PCI_VDEVICE(AMD, 0x1456), (kernel_ulong_t)&dev_vdata[1] }, - { PCI_VDEVICE(AMD, 0x1468), (kernel_ulong_t)&dev_vdata[2] }, - { PCI_VDEVICE(AMD, 0x1486), (kernel_ulong_t)&dev_vdata[3] }, - { PCI_VDEVICE(AMD, 0x15DF), (kernel_ulong_t)&dev_vdata[4] }, - { PCI_VDEVICE(AMD, 0x14CA), (kernel_ulong_t)&dev_vdata[5] }, - { PCI_VDEVICE(AMD, 0x15C7), (kernel_ulong_t)&dev_vdata[6] }, - { PCI_VDEVICE(AMD, 0x1649), (kernel_ulong_t)&dev_vdata[6] }, - { PCI_VDEVICE(AMD, 0x1134), (kernel_ulong_t)&dev_vdata[7] }, - { PCI_VDEVICE(AMD, 0x17E0), (kernel_ulong_t)&dev_vdata[7] }, - { PCI_VDEVICE(AMD, 0x156E), (kernel_ulong_t)&dev_vdata[8] }, - { PCI_VDEVICE(AMD, 0x17D8), (kernel_ulong_t)&dev_vdata[8] }, - { PCI_VDEVICE(AMD, 0x115A), (kernel_ulong_t)&dev_vdata[9] }, + { PCI_VDEVICE(AMD, 0x1537), .driver_data = (kernel_ulong_t)&dev_vdata[0] }, + { PCI_VDEVICE(AMD, 0x1456), .driver_data = (kernel_ulong_t)&dev_vdata[1] }, + { PCI_VDEVICE(AMD, 0x1468), .driver_data = (kernel_ulong_t)&dev_vdata[2] }, + { PCI_VDEVICE(AMD, 0x1486), .driver_data = (kernel_ulong_t)&dev_vdata[3] }, + { PCI_VDEVICE(AMD, 0x15DF), .driver_data = (kernel_ulong_t)&dev_vdata[4] }, + { PCI_VDEVICE(AMD, 0x14CA), .driver_data = (kernel_ulong_t)&dev_vdata[5] }, + { PCI_VDEVICE(AMD, 0x15C7), .driver_data = (kernel_ulong_t)&dev_vdata[6] }, + { PCI_VDEVICE(AMD, 0x1649), .driver_data = (kernel_ulong_t)&dev_vdata[6] }, + { PCI_VDEVICE(AMD, 0x1134), .driver_data = (kernel_ulong_t)&dev_vdata[7] }, + { PCI_VDEVICE(AMD, 0x17E0), .driver_data = (kernel_ulong_t)&dev_vdata[7] }, + { PCI_VDEVICE(AMD, 0x156E), .driver_data = (kernel_ulong_t)&dev_vdata[8] }, + { PCI_VDEVICE(AMD, 0x17D8), .driver_data = (kernel_ulong_t)&dev_vdata[8] }, + { PCI_VDEVICE(AMD, 0x115A), .driver_data = (kernel_ulong_t)&dev_vdata[9] }, /* Last entry must be zero */ - { 0, } + { } }; MODULE_DEVICE_TABLE(pci, sp_pci_table); diff --git a/drivers/crypto/ccree/cc_aead.c b/drivers/crypto/ccree/cc_aead.c index 81533681f7fb..088c4603047f 100644 --- a/drivers/crypto/ccree/cc_aead.c +++ b/drivers/crypto/ccree/cc_aead.c @@ -3,11 +3,12 @@ #include #include +#include +#include #include #include #include #include -#include #include #include "cc_driver.h" #include "cc_buffer_mgr.h" @@ -2569,11 +2570,9 @@ static struct cc_crypto_alg *cc_create_aead_alg(struct cc_alg_template *tmpl, alg = &tmpl->template_aead; - if (snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", - tmpl->name) >= CRYPTO_MAX_ALG_NAME) + if (strscpy(alg->base.cra_name, tmpl->name) < 0) return ERR_PTR(-EINVAL); - if (snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", - tmpl->driver_name) >= CRYPTO_MAX_ALG_NAME) + if (strscpy(alg->base.cra_driver_name, tmpl->driver_name) < 0) return ERR_PTR(-EINVAL); alg->base.cra_module = THIS_MODULE; diff --git a/drivers/crypto/ccree/cc_cipher.c b/drivers/crypto/ccree/cc_cipher.c index e2cbfdf7a0e4..5339b3796c80 100644 --- a/drivers/crypto/ccree/cc_cipher.c +++ b/drivers/crypto/ccree/cc_cipher.c @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -1386,11 +1387,9 @@ static struct cc_crypto_alg *cc_create_alg(const struct cc_alg_template *tmpl, memcpy(alg, &tmpl->template_skcipher, sizeof(*alg)); - if (snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", - tmpl->name) >= CRYPTO_MAX_ALG_NAME) + if (strscpy(alg->base.cra_name, tmpl->name) < 0) return ERR_PTR(-EINVAL); - if (snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", - tmpl->driver_name) >= CRYPTO_MAX_ALG_NAME) + if (strscpy(alg->base.cra_driver_name, tmpl->driver_name) < 0) return ERR_PTR(-EINVAL); alg->base.cra_module = THIS_MODULE; diff --git a/drivers/crypto/ccree/cc_hash.c b/drivers/crypto/ccree/cc_hash.c index 73179bf725a7..5de721a2ca30 100644 --- a/drivers/crypto/ccree/cc_hash.c +++ b/drivers/crypto/ccree/cc_hash.c @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -1835,16 +1836,12 @@ static struct cc_hash_alg *cc_alloc_hash_alg(struct cc_hash_template *template, alg = &halg->halg.base; if (keyed) { - snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", - template->mac_name); - snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", - template->mac_driver_name); + strscpy(alg->cra_name, template->mac_name); + strscpy(alg->cra_driver_name, template->mac_driver_name); } else { halg->setkey = NULL; - snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", - template->name); - snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", - template->driver_name); + strscpy(alg->cra_name, template->name); + strscpy(alg->cra_driver_name, template->driver_name); } alg->cra_module = THIS_MODULE; alg->cra_ctxsize = sizeof(struct cc_hash_ctx) + crypto_dma_padding(); diff --git a/drivers/crypto/exynos-rng.c b/drivers/crypto/exynos-rng.c deleted file mode 100644 index 2aaa98f9b44e..000000000000 --- a/drivers/crypto/exynos-rng.c +++ /dev/null @@ -1,399 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * exynos-rng.c - Random Number Generator driver for the Exynos - * - * Copyright (c) 2017 Krzysztof Kozlowski - * - * Loosely based on old driver from drivers/char/hw_random/exynos-rng.c: - * Copyright (C) 2012 Samsung Electronics - * Jonghwa Lee - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#define EXYNOS_RNG_CONTROL 0x0 -#define EXYNOS_RNG_STATUS 0x10 - -#define EXYNOS_RNG_SEED_CONF 0x14 -#define EXYNOS_RNG_GEN_PRNG BIT(1) - -#define EXYNOS_RNG_SEED_BASE 0x140 -#define EXYNOS_RNG_SEED(n) (EXYNOS_RNG_SEED_BASE + (n * 0x4)) -#define EXYNOS_RNG_OUT_BASE 0x160 -#define EXYNOS_RNG_OUT(n) (EXYNOS_RNG_OUT_BASE + (n * 0x4)) - -/* EXYNOS_RNG_CONTROL bit fields */ -#define EXYNOS_RNG_CONTROL_START 0x18 -/* EXYNOS_RNG_STATUS bit fields */ -#define EXYNOS_RNG_STATUS_SEED_SETTING_DONE BIT(1) -#define EXYNOS_RNG_STATUS_RNG_DONE BIT(5) - -/* Five seed and output registers, each 4 bytes */ -#define EXYNOS_RNG_SEED_REGS 5 -#define EXYNOS_RNG_SEED_SIZE (EXYNOS_RNG_SEED_REGS * 4) - -enum exynos_prng_type { - EXYNOS_PRNG_UNKNOWN = 0, - EXYNOS_PRNG_EXYNOS4, - EXYNOS_PRNG_EXYNOS5, -}; - -/* - * Driver re-seeds itself with generated random numbers to hinder - * backtracking of the original seed. - * - * Time for next re-seed in ms. - */ -#define EXYNOS_RNG_RESEED_TIME 1000 -#define EXYNOS_RNG_RESEED_BYTES 65536 - -/* - * In polling mode, do not wait infinitely for the engine to finish the work. - */ -#define EXYNOS_RNG_WAIT_RETRIES 100 - -/* Context for crypto */ -struct exynos_rng_ctx { - struct exynos_rng_dev *rng; -}; - -/* Device associated memory */ -struct exynos_rng_dev { - struct device *dev; - enum exynos_prng_type type; - void __iomem *mem; - struct clk *clk; - struct mutex lock; - /* Generated numbers stored for seeding during resume */ - u8 seed_save[EXYNOS_RNG_SEED_SIZE]; - unsigned int seed_save_len; - /* Time of last seeding in jiffies */ - unsigned long last_seeding; - /* Bytes generated since last seeding */ - unsigned long bytes_seeding; -}; - -static struct exynos_rng_dev *exynos_rng_dev; - -static u32 exynos_rng_readl(struct exynos_rng_dev *rng, u32 offset) -{ - return readl_relaxed(rng->mem + offset); -} - -static void exynos_rng_writel(struct exynos_rng_dev *rng, u32 val, u32 offset) -{ - writel_relaxed(val, rng->mem + offset); -} - -static int exynos_rng_set_seed(struct exynos_rng_dev *rng, - const u8 *seed, unsigned int slen) -{ - u32 val; - int i; - - /* Round seed length because loop iterates over full register size */ - slen = ALIGN_DOWN(slen, 4); - - if (slen < EXYNOS_RNG_SEED_SIZE) - return -EINVAL; - - for (i = 0; i < slen ; i += 4) { - unsigned int seed_reg = (i / 4) % EXYNOS_RNG_SEED_REGS; - - val = seed[i] << 24; - val |= seed[i + 1] << 16; - val |= seed[i + 2] << 8; - val |= seed[i + 3] << 0; - - exynos_rng_writel(rng, val, EXYNOS_RNG_SEED(seed_reg)); - } - - val = exynos_rng_readl(rng, EXYNOS_RNG_STATUS); - if (!(val & EXYNOS_RNG_STATUS_SEED_SETTING_DONE)) { - dev_warn(rng->dev, "Seed setting not finished\n"); - return -EIO; - } - - rng->last_seeding = jiffies; - rng->bytes_seeding = 0; - - return 0; -} - -/* - * Start the engine and poll for finish. Then read from output registers - * filling the 'dst' buffer up to 'dlen' bytes or up to size of generated - * random data (EXYNOS_RNG_SEED_SIZE). - * - * On success: return 0 and store number of read bytes under 'read' address. - * On error: return -ERRNO. - */ -static int exynos_rng_get_random(struct exynos_rng_dev *rng, - u8 *dst, unsigned int dlen, - unsigned int *read) -{ - int retry = EXYNOS_RNG_WAIT_RETRIES; - - if (rng->type == EXYNOS_PRNG_EXYNOS4) { - exynos_rng_writel(rng, EXYNOS_RNG_CONTROL_START, - EXYNOS_RNG_CONTROL); - } else if (rng->type == EXYNOS_PRNG_EXYNOS5) { - exynos_rng_writel(rng, EXYNOS_RNG_GEN_PRNG, - EXYNOS_RNG_SEED_CONF); - } - - while (!(exynos_rng_readl(rng, - EXYNOS_RNG_STATUS) & EXYNOS_RNG_STATUS_RNG_DONE) && --retry) - cpu_relax(); - - if (!retry) - return -ETIMEDOUT; - - /* Clear status bit */ - exynos_rng_writel(rng, EXYNOS_RNG_STATUS_RNG_DONE, - EXYNOS_RNG_STATUS); - *read = min_t(size_t, dlen, EXYNOS_RNG_SEED_SIZE); - memcpy_fromio(dst, rng->mem + EXYNOS_RNG_OUT_BASE, *read); - rng->bytes_seeding += *read; - - return 0; -} - -/* Re-seed itself from time to time */ -static void exynos_rng_reseed(struct exynos_rng_dev *rng) -{ - unsigned long next_seeding = rng->last_seeding + \ - msecs_to_jiffies(EXYNOS_RNG_RESEED_TIME); - unsigned long now = jiffies; - unsigned int read = 0; - u8 seed[EXYNOS_RNG_SEED_SIZE]; - - if (time_before(now, next_seeding) && - rng->bytes_seeding < EXYNOS_RNG_RESEED_BYTES) - return; - - if (exynos_rng_get_random(rng, seed, sizeof(seed), &read)) - return; - - exynos_rng_set_seed(rng, seed, read); - - /* Let others do some of their job. */ - mutex_unlock(&rng->lock); - mutex_lock(&rng->lock); -} - -static int exynos_rng_generate(struct crypto_rng *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int dlen) -{ - struct exynos_rng_ctx *ctx = crypto_rng_ctx(tfm); - struct exynos_rng_dev *rng = ctx->rng; - unsigned int read = 0; - int ret; - - ret = clk_prepare_enable(rng->clk); - if (ret) - return ret; - - mutex_lock(&rng->lock); - do { - ret = exynos_rng_get_random(rng, dst, dlen, &read); - if (ret) - break; - - dlen -= read; - dst += read; - - exynos_rng_reseed(rng); - } while (dlen > 0); - mutex_unlock(&rng->lock); - - clk_disable_unprepare(rng->clk); - - return ret; -} - -static int exynos_rng_seed(struct crypto_rng *tfm, const u8 *seed, - unsigned int slen) -{ - struct exynos_rng_ctx *ctx = crypto_rng_ctx(tfm); - struct exynos_rng_dev *rng = ctx->rng; - int ret; - - ret = clk_prepare_enable(rng->clk); - if (ret) - return ret; - - mutex_lock(&rng->lock); - ret = exynos_rng_set_seed(ctx->rng, seed, slen); - mutex_unlock(&rng->lock); - - clk_disable_unprepare(rng->clk); - - return ret; -} - -static int exynos_rng_kcapi_init(struct crypto_tfm *tfm) -{ - struct exynos_rng_ctx *ctx = crypto_tfm_ctx(tfm); - - ctx->rng = exynos_rng_dev; - - return 0; -} - -static struct rng_alg exynos_rng_alg = { - .generate = exynos_rng_generate, - .seed = exynos_rng_seed, - .seedsize = EXYNOS_RNG_SEED_SIZE, - .base = { - .cra_name = "stdrng", - .cra_driver_name = "exynos_rng", - .cra_priority = 300, - .cra_ctxsize = sizeof(struct exynos_rng_ctx), - .cra_module = THIS_MODULE, - .cra_init = exynos_rng_kcapi_init, - } -}; - -static int exynos_rng_probe(struct platform_device *pdev) -{ - struct exynos_rng_dev *rng; - int ret; - - if (exynos_rng_dev) - return -EEXIST; - - rng = devm_kzalloc(&pdev->dev, sizeof(*rng), GFP_KERNEL); - if (!rng) - return -ENOMEM; - - rng->type = (uintptr_t)of_device_get_match_data(&pdev->dev); - - mutex_init(&rng->lock); - - rng->dev = &pdev->dev; - rng->clk = devm_clk_get(&pdev->dev, "secss"); - if (IS_ERR(rng->clk)) { - dev_err(&pdev->dev, "Couldn't get clock.\n"); - return PTR_ERR(rng->clk); - } - - rng->mem = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(rng->mem)) - return PTR_ERR(rng->mem); - - platform_set_drvdata(pdev, rng); - - exynos_rng_dev = rng; - - ret = crypto_register_rng(&exynos_rng_alg); - if (ret) { - dev_err(&pdev->dev, - "Couldn't register rng crypto alg: %d\n", ret); - exynos_rng_dev = NULL; - } - - return ret; -} - -static void exynos_rng_remove(struct platform_device *pdev) -{ - crypto_unregister_rng(&exynos_rng_alg); - - exynos_rng_dev = NULL; -} - -static int __maybe_unused exynos_rng_suspend(struct device *dev) -{ - struct exynos_rng_dev *rng = dev_get_drvdata(dev); - int ret; - - /* If we were never seeded then after resume it will be the same */ - if (!rng->last_seeding) - return 0; - - rng->seed_save_len = 0; - ret = clk_prepare_enable(rng->clk); - if (ret) - return ret; - - mutex_lock(&rng->lock); - - /* Get new random numbers and store them for seeding on resume. */ - exynos_rng_get_random(rng, rng->seed_save, sizeof(rng->seed_save), - &(rng->seed_save_len)); - - mutex_unlock(&rng->lock); - - dev_dbg(rng->dev, "Stored %u bytes for seeding on system resume\n", - rng->seed_save_len); - - clk_disable_unprepare(rng->clk); - - return 0; -} - -static int __maybe_unused exynos_rng_resume(struct device *dev) -{ - struct exynos_rng_dev *rng = dev_get_drvdata(dev); - int ret; - - /* Never seeded so nothing to do */ - if (!rng->last_seeding) - return 0; - - ret = clk_prepare_enable(rng->clk); - if (ret) - return ret; - - mutex_lock(&rng->lock); - - ret = exynos_rng_set_seed(rng, rng->seed_save, rng->seed_save_len); - - mutex_unlock(&rng->lock); - - clk_disable_unprepare(rng->clk); - - return ret; -} - -static SIMPLE_DEV_PM_OPS(exynos_rng_pm_ops, exynos_rng_suspend, - exynos_rng_resume); - -static const struct of_device_id exynos_rng_dt_match[] = { - { - .compatible = "samsung,exynos4-rng", - .data = (const void *)EXYNOS_PRNG_EXYNOS4, - }, { - .compatible = "samsung,exynos5250-prng", - .data = (const void *)EXYNOS_PRNG_EXYNOS5, - }, - { }, -}; -MODULE_DEVICE_TABLE(of, exynos_rng_dt_match); - -static struct platform_driver exynos_rng_driver = { - .driver = { - .name = "exynos-rng", - .pm = &exynos_rng_pm_ops, - .of_match_table = exynos_rng_dt_match, - }, - .probe = exynos_rng_probe, - .remove = exynos_rng_remove, -}; - -module_platform_driver(exynos_rng_driver); - -MODULE_DESCRIPTION("Exynos H/W Random Number Generator driver"); -MODULE_AUTHOR("Krzysztof Kozlowski "); -MODULE_LICENSE("GPL v2"); diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig index 1e6d772f4bb6..aeff08ccbadd 100644 --- a/drivers/crypto/hisilicon/Kconfig +++ b/drivers/crypto/hisilicon/Kconfig @@ -75,11 +75,3 @@ config CRYPTO_DEV_HISI_HPRE help Support for HiSilicon HPRE(High Performance RSA Engine) accelerator, which can accelerate RSA and DH algorithms. - -config CRYPTO_DEV_HISI_TRNG - tristate "Support for HISI TRNG Driver" - depends on ARM64 && ACPI - select HW_RANDOM - select CRYPTO_RNG - help - Support for HiSilicon TRNG Driver. diff --git a/drivers/crypto/hisilicon/Makefile b/drivers/crypto/hisilicon/Makefile index 8595a5a5d228..e1068ee9f973 100644 --- a/drivers/crypto/hisilicon/Makefile +++ b/drivers/crypto/hisilicon/Makefile @@ -5,4 +5,3 @@ obj-$(CONFIG_CRYPTO_DEV_HISI_SEC2) += sec2/ obj-$(CONFIG_CRYPTO_DEV_HISI_QM) += hisi_qm.o hisi_qm-objs = qm.o sgl.o debugfs.o obj-$(CONFIG_CRYPTO_DEV_HISI_ZIP) += zip/ -obj-$(CONFIG_CRYPTO_DEV_HISI_TRNG) += trng/ diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 357ab5e5887e..b6903fce6071 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -55,6 +55,8 @@ #define HPRE_RAS_FE_ENB 0x301418 #define HPRE_OOO_SHUTDOWN_SEL 0x301a3c #define HPRE_HAC_RAS_FE_ENABLE 0 +#define HPRE_RAS_MASK_ALL GENMASK(31, 0) +#define HPRE_RAS_CLEAR_ALL GENMASK(31, 0) #define HPRE_CORE_ENB (HPRE_CLSTR_BASE + HPRE_CORE_EN_OFFSET) #define HPRE_CORE_INI_CFG (HPRE_CLSTR_BASE + HPRE_CORE_INI_CFG_OFFSET) @@ -820,11 +822,8 @@ static void hpre_master_ooo_ctrl(struct hisi_qm *qm, bool enable) static void hpre_hw_error_disable(struct hisi_qm *qm) { - struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; - u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; - /* disable hpre hw error interrupts */ - writel(err_mask, qm->io_base + HPRE_INT_MASK); + writel(HPRE_RAS_MASK_ALL, qm->io_base + HPRE_INT_MASK); /* disable HPRE block master OOO when nfe occurs on Kunpeng930 */ hpre_master_ooo_ctrl(qm, false); } @@ -835,7 +834,7 @@ static void hpre_hw_error_enable(struct hisi_qm *qm) u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; /* clear HPRE hw error source if having */ - writel(err_mask, qm->io_base + HPRE_HAC_SOURCE_INT); + writel(HPRE_RAS_CLEAR_ALL, qm->io_base + HPRE_HAC_SOURCE_INT); /* configure error type */ writel(dev_err->ce, qm->io_base + HPRE_RAS_CE_ENB); @@ -1631,12 +1630,10 @@ static int hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_qm_del_list; } - if (qm->uacce) { - ret = uacce_register(qm->uacce); - if (ret) { - pci_err(pdev, "failed to register uacce (%d)!\n", ret); - goto err_with_alg_register; - } + ret = hisi_qm_register_uacce(qm); + if (ret) { + pci_err(pdev, "failed to register uacce (%d)!\n", ret); + goto err_with_alg_register; } if (qm->fun_type == QM_HW_PF && vfs_num) { diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 3ca47e2a9719..a951d2ef7833 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -128,7 +128,6 @@ #define QM_ABNORMAL_INT_SOURCE 0x100000 #define QM_ABNORMAL_INT_MASK 0x100004 -#define QM_ABNORMAL_INT_MASK_VALUE 0x7fff #define QM_ABNORMAL_INT_STATUS 0x100008 #define QM_ABNORMAL_INT_SET 0x10000c #define QM_ABNORMAL_INF00 0x100010 @@ -153,6 +152,8 @@ #define QM_DB_TIMEOUT BIT(10) #define QM_OF_FIFO_OF BIT(11) #define QM_RAS_AXI_ERROR (BIT(0) | BIT(1) | BIT(12)) +#define QM_RAS_MASK_ALL GENMASK(31, 0) +#define QM_RAS_CLEAR_ALL GENMASK(31, 0) #define QM_RESET_WAIT_TIMEOUT 400 #define QM_PEH_VENDOR_ID 0x1000d8 @@ -246,6 +247,20 @@ #define QM_QOS_MAX_CIR_U 6 #define QM_AUTOSUSPEND_DELAY 3000 +#define QM_DB_DROP_ALL_FUNC_ENABLE GENMASK(63, 0) +#define QM_DB_DROP_ALL_FUNC_DISABLE 0 +#define QM_DEV_DB_DROP 0x0100250 +#define QM_FUN_DB_DROP 0x0038 + +/* qm function err mask */ +#define QM_FUNC_AXI_ERR_ST0 0x100280 +#define QM_RAS_FUNC_ERROR (BIT(0) | BIT(1)) +#define QM_FUNC_RAS_CLEAR_ALL GENMASK(63, 0) + +/* qm isolation state mask */ +#define QM_ISOLATED_STATE BIT(31) +#define QM_ISOLATED_THRESHOLD_MASK GENMASK(15, 0) + /* abnormal status value for stopping queue */ #define QM_STOP_QUEUE_FAIL 1 #define QM_DUMP_SQC_FAIL 3 @@ -286,6 +301,20 @@ enum qm_alg_type { ALG_TYPE_1, }; +/* + * Message format for QM_VF_GET_ISOLATE and QM_PF_SET_ISOLATE commands + * + * These commands use a 32-bit command field (cmd) and 32-bit data field (data) + * + * Command behavior: + * - QM_VF_GET_ISOLATE: VF requests isolation status and threshold + * - QM_PF_SET_ISOLATE: PF sets isolation status and threshold + * + * Data field bit layout: + * - bit31 (MSB): Isolation status flag (1 = isolated, 0 = non-isolated) + * - bit15-0 (16 LSB): Isolation threshold value + * - bit30-16 (15 bits): Reserved + */ enum qm_ifc_cmd { QM_PF_FLR_PREPARE = 0x01, QM_PF_SRST_PREPARE, @@ -296,6 +325,9 @@ enum qm_ifc_cmd { QM_VF_START_FAIL, QM_PF_SET_QOS, QM_VF_GET_QOS, + QM_FUNCTION_RESET, + QM_VF_GET_ISOLATE, + QM_PF_SET_ISOLATE, }; enum qm_basic_type { @@ -473,6 +505,7 @@ static struct qm_typical_qos_table shaper_cbs_s[] = { static void qm_irqs_unregister(struct hisi_qm *qm); static int qm_reset_device(struct hisi_qm *qm); static void hisi_qm_stop_qp(struct hisi_qp *qp); +static int qm_restart(struct hisi_qm *qm); int hisi_qm_q_num_set(const char *val, const struct kernel_param *kp, unsigned int device) @@ -549,6 +582,29 @@ static int qm_wait_reset_finish(struct hisi_qm *qm) return 0; } +static void qm_fun_db_ctrl(struct hisi_qm *qm, bool enable) +{ + u32 val; + + if (qm->ver >= QM_HW_V5) { + val = readl(qm->io_base + QM_FUN_DB_DROP); + val = enable ? (val | BIT(0)) : (val & ~BIT(0)); + + writel(val, qm->io_base + QM_FUN_DB_DROP); + } +} + +static void qm_dev_db_ctrl(struct hisi_qm *qm, bool enable) +{ + u64 val; + + if (qm->ver >= QM_HW_V5 && qm->fun_type == QM_HW_PF) { + val = enable ? QM_DB_DROP_ALL_FUNC_ENABLE : QM_DB_DROP_ALL_FUNC_DISABLE; + + writeq(val, qm->io_base + QM_DEV_DB_DROP); + } +} + static int qm_reset_prepare_ready(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; @@ -1151,18 +1207,20 @@ static void qm_reset_function(struct hisi_qm *qm) return; } + dev_info(dev, "function reset start...\n"); ret = hisi_qm_stop(qm, QM_DOWN); if (ret) { dev_err(dev, "failed to stop qm when reset function\n"); goto clear_bit; } - ret = hisi_qm_start(qm); + ret = qm_restart(qm); if (ret) dev_err(dev, "failed to start qm when reset function\n"); clear_bit: qm_reset_bit_clear(qm); + dev_info(dev, "function reset end...\n"); } static irqreturn_t qm_aeq_thread(int irq, void *data) @@ -1175,6 +1233,11 @@ static irqreturn_t qm_aeq_thread(int irq, void *data) atomic64_inc(&qm->debug.dfx.aeq_irq_cnt); + if (qm_pm_get_sync(qm)) { + dev_err(&qm->pdev->dev, "failed to get runtime PM for aeq handle\n"); + return IRQ_HANDLED; + } + while (QM_AEQE_PHASE(dw0) == qm->status.aeqc_phase) { type = (dw0 >> QM_AEQE_TYPE_SHIFT) & QM_AEQE_TYPE_MASK; qp_id = dw0 & QM_AEQE_CQN_MASK; @@ -1210,6 +1273,8 @@ static irqreturn_t qm_aeq_thread(int irq, void *data) qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0); + qm_pm_put_sync(qm); + return IRQ_HANDLED; } @@ -1468,7 +1533,7 @@ static int qm_get_vft_v2(struct hisi_qm *qm, u32 *base, u32 *number) static void qm_hw_error_init_v1(struct hisi_qm *qm) { - writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK); + writel(QM_RAS_MASK_ALL, qm->io_base + QM_ABNORMAL_INT_MASK); } static void qm_hw_error_cfg(struct hisi_qm *qm) @@ -1477,7 +1542,9 @@ static void qm_hw_error_cfg(struct hisi_qm *qm) qm->error_mask = qm_err->nfe | qm_err->ce | qm_err->fe; /* clear QM hw residual error source */ - writel(qm->error_mask, qm->io_base + QM_ABNORMAL_INT_SOURCE); + writel(QM_RAS_CLEAR_ALL, qm->io_base + QM_ABNORMAL_INT_SOURCE); + if (qm->ver >= QM_HW_V5) + writeq(QM_FUNC_RAS_CLEAR_ALL, qm->io_base + QM_FUNC_AXI_ERR_ST0); /* configure error type */ writel(qm_err->ce, qm->io_base + QM_RAS_CE_ENABLE); @@ -1488,43 +1555,28 @@ static void qm_hw_error_cfg(struct hisi_qm *qm) static void qm_hw_error_init_v2(struct hisi_qm *qm) { - u32 irq_unmask; - qm_hw_error_cfg(qm); - irq_unmask = ~qm->error_mask; - irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK); - writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK); + writel(~qm->error_mask, qm->io_base + QM_ABNORMAL_INT_MASK); } static void qm_hw_error_uninit_v2(struct hisi_qm *qm) { - u32 irq_mask = qm->error_mask; - - irq_mask |= readl(qm->io_base + QM_ABNORMAL_INT_MASK); - writel(irq_mask, qm->io_base + QM_ABNORMAL_INT_MASK); + writel(QM_RAS_MASK_ALL, qm->io_base + QM_ABNORMAL_INT_MASK); } static void qm_hw_error_init_v3(struct hisi_qm *qm) { - u32 irq_unmask; - qm_hw_error_cfg(qm); /* enable close master ooo when hardware error happened */ writel(qm->err_info.qm_err.shutdown_mask, qm->io_base + QM_OOO_SHUTDOWN_SEL); - - irq_unmask = ~qm->error_mask; - irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK); - writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK); + writel(~qm->error_mask, qm->io_base + QM_ABNORMAL_INT_MASK); } static void qm_hw_error_uninit_v3(struct hisi_qm *qm) { - u32 irq_mask = qm->error_mask; - - irq_mask |= readl(qm->io_base + QM_ABNORMAL_INT_MASK); - writel(irq_mask, qm->io_base + QM_ABNORMAL_INT_MASK); + writel(QM_RAS_MASK_ALL, qm->io_base + QM_ABNORMAL_INT_MASK); /* disable close master ooo when hardware error happened */ writel(0x0, qm->io_base + QM_OOO_SHUTDOWN_SEL); @@ -1583,6 +1635,15 @@ static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm) qm->err_status.is_qm_ecc_mbit = true; qm_log_hw_error(qm, error_status); + /* Trigger func reset only when error is detected in bit 0 or bit 1. */ + if ((qm->ver >= QM_HW_V5) && + (error_status & QM_RAS_FUNC_ERROR) && + (error_status & qm_err->reset_mask) == 0) { + writel(error_status, qm->io_base + QM_ABNORMAL_INT_SOURCE); + writel(qm_err->nfe, qm->io_base + QM_RAS_NFE_ENABLE); + return ACC_ERR_NEED_FUNC_RESET; + } + if (error_status & qm_err->reset_mask) { /* Disable the same error reporting until device is recovered. */ writel(qm_err->nfe & (~error_status), qm->io_base + QM_RAS_NFE_ENABLE); @@ -1734,7 +1795,7 @@ err_unlock: return ret; } -static int qm_ping_all_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd) +static int qm_ping_all_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data) { struct device *dev = &qm->pdev->dev; u32 vfs_num = qm->vfs_num; @@ -1743,7 +1804,7 @@ static int qm_ping_all_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd) int ret; u32 i; - ret = qm->ops->set_ifc_begin(qm, cmd, 0, QM_MB_PING_ALL_VFS); + ret = qm->ops->set_ifc_begin(qm, cmd, data, QM_MB_PING_ALL_VFS); if (ret) { dev_err(dev, "failed to send command(0x%x) to all vfs!\n", cmd); qm->ops->set_ifc_end(qm); @@ -2779,6 +2840,7 @@ static enum uacce_dev_state hisi_qm_get_isolate_state(struct uacce_device *uacce static int hisi_qm_isolate_threshold_write(struct uacce_device *uacce, u32 num) { struct hisi_qm *qm = uacce->priv; + int ret; /* Must be set by PF */ if (uacce->is_vf) @@ -2792,6 +2854,18 @@ static int hisi_qm_isolate_threshold_write(struct uacce_device *uacce, u32 num) /* After the policy is updated, need to reset the hardware err list */ qm_hw_err_destroy(qm); + + if (!qm->vfs_num) { + mutex_unlock(&qm->isolate_data.isolate_lock); + return 0; + } + + /* Notify all VFs to update the isolation threshold. */ + if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) { + ret = qm_ping_all_vfs(qm, QM_PF_SET_ISOLATE, qm->isolate_data.err_threshold); + if (ret) + dev_err(&qm->pdev->dev, "failed to send command to all VFs set isolate!\n"); + } mutex_unlock(&qm->isolate_data.isolate_lock); return 0; @@ -2802,7 +2876,7 @@ static u32 hisi_qm_isolate_threshold_read(struct uacce_device *uacce) struct hisi_qm *qm = uacce->priv; struct hisi_qm *pf_qm; - if (uacce->is_vf) { + if (uacce->is_vf && !test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) { pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); return pf_qm->isolate_data.err_threshold; } @@ -2889,7 +2963,10 @@ static int qm_alloc_uacce(struct hisi_qm *qm) return -EINVAL; } - uacce->is_vf = pdev->is_virtfn; + if (qm->fun_type == QM_HW_PF) + uacce->is_vf = false; + else + uacce->is_vf = true; uacce->priv = qm; if (qm->ver == QM_HW_V1) @@ -2918,6 +2995,25 @@ static int qm_alloc_uacce(struct hisi_qm *qm) return 0; } +int hisi_qm_register_uacce(struct hisi_qm *qm) +{ + int ret; + + if (!qm->uacce) + return 0; + + dev_info(&qm->pdev->dev, "qm register to uacce\n"); + + if (qm->fun_type == QM_HW_VF && test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) { + ret = qm_ping_pf(qm, QM_VF_GET_ISOLATE); + if (ret) + dev_err(&qm->pdev->dev, "failed to send cmd to PF to get isolate!\n"); + } + + return uacce_register(qm->uacce); +} +EXPORT_SYMBOL_GPL(hisi_qm_register_uacce); + /** * qm_frozen() - Try to froze QM to cut continuous queue request. If * there is user on the QM, return failure without doing anything. @@ -2988,9 +3084,9 @@ void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list) msleep(WAIT_PERIOD); } - while (test_bit(QM_RST_SCHED, &qm->misc_ctl) || - test_bit(QM_RESETTING, &qm->misc_ctl)) - msleep(WAIT_PERIOD); + /* Cancel possible RAS reset process during the uninstallation procedure. */ + if (qm->fun_type == QM_HW_PF) + cancel_work_sync(&qm->rst_work); if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) flush_work(&qm->cmd_process); @@ -3366,6 +3462,9 @@ static int __hisi_qm_start(struct hisi_qm *qm) if (ret) return ret; + /* Enables the doorbell function when the device is enabled. */ + qm_dev_db_ctrl(qm, false); + qm_fun_db_ctrl(qm, false); qm_init_prefetch(qm); qm_enable_eq_aeq_interrupts(qm); @@ -3473,7 +3572,7 @@ static void qm_invalid_queues(struct hisi_qm *qm) if (qm->status.stop_reason == QM_NORMAL) return; - if (qm->status.stop_reason == QM_DOWN) + if (qm->status.stop_reason == QM_DOWN || qm->status.stop_reason == QM_SHUTDOWN) hisi_qm_cache_wb(qm); for (i = 0; i < qm->qp_num; i++) { @@ -3517,6 +3616,8 @@ int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r) if (qm->status.stop_reason != QM_NORMAL) { hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET); + if (qm->status.stop_reason != QM_SHUTDOWN) + qm_fun_db_ctrl(qm, true); /* * When performing soft reset, the hardware will no longer * do tasks, and the tasks in the device will be flushed @@ -4302,10 +4403,13 @@ static enum acc_err_result qm_process_dev_error(struct hisi_qm *qm) /* log device error */ dev_ret = qm_dev_err_handle(qm); + if (qm_ret == ACC_ERR_NEED_RESET || dev_ret == ACC_ERR_NEED_RESET) + return ACC_ERR_NEED_RESET; - return (qm_ret == ACC_ERR_NEED_RESET || - dev_ret == ACC_ERR_NEED_RESET) ? - ACC_ERR_NEED_RESET : ACC_ERR_RECOVERED; + if (qm_ret == ACC_ERR_NEED_FUNC_RESET) + return ACC_ERR_NEED_FUNC_RESET; + + return ACC_ERR_RECOVERED; } /** @@ -4330,7 +4434,7 @@ pci_ers_result_t hisi_qm_dev_err_detected(struct pci_dev *pdev, return PCI_ERS_RESULT_DISCONNECT; ret = qm_process_dev_error(qm); - if (ret == ACC_ERR_NEED_RESET) + if (ret == ACC_ERR_NEED_RESET || ret == ACC_ERR_NEED_FUNC_RESET) return PCI_ERS_RESULT_NEED_RESET; return PCI_ERS_RESULT_RECOVERED; @@ -4484,7 +4588,7 @@ static int qm_try_stop_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd, /* Kunpeng930 supports to notify VFs to stop before PF reset */ if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) { - ret = qm_ping_all_vfs(qm, cmd); + ret = qm_ping_all_vfs(qm, cmd, 0); if (ret) pci_err(pdev, "failed to send command to all VFs before PF reset!\n"); } else { @@ -4540,7 +4644,7 @@ static int qm_controller_reset_prepare(struct hisi_qm *qm) if (ret) pci_err(pdev, "failed to stop by vfs in soft reset!\n"); - clear_bit(QM_RST_SCHED, &qm->misc_ctl); + qm_dev_db_ctrl(qm, true); return 0; } @@ -4671,6 +4775,7 @@ restart_fail: static int qm_try_start_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd) { struct pci_dev *pdev = qm->pdev; + u32 data; int ret; if (!qm->vfs_num) @@ -4684,7 +4789,11 @@ static int qm_try_start_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd) /* Kunpeng930 supports to notify VFs to start after PF reset. */ if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) { - ret = qm_ping_all_vfs(qm, cmd); + data = qm->isolate_data.err_threshold; + if (qm->isolate_data.is_isolate) + data |= QM_ISOLATED_STATE; + /* Broadcasting isolate info via RAS to all VFs. */ + ret = qm_ping_all_vfs(qm, cmd, data); if (ret) pci_warn(pdev, "failed to send cmd to all VFs after PF reset!\n"); } else { @@ -4854,7 +4963,6 @@ static int qm_controller_reset(struct hisi_qm *qm) if (ret) { hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET); hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); - clear_bit(QM_RST_SCHED, &qm->misc_ctl); return ret; } @@ -4917,8 +5025,6 @@ void hisi_qm_reset_prepare(struct pci_dev *pdev) u32 delay = 0; int ret; - hisi_qm_dev_err_uninit(pf_qm); - /* * Check whether there is an ECC mbit error, If it occurs, need to * wait for soft reset to fix it. @@ -4935,6 +5041,8 @@ void hisi_qm_reset_prepare(struct pci_dev *pdev) return; } + hisi_qm_dev_err_uninit(pf_qm); + /* PF obtains the information of VF by querying the register. */ if (qm->fun_type == QM_HW_PF) qm_cmd_uninit(qm); @@ -4946,16 +5054,25 @@ void hisi_qm_reset_prepare(struct pci_dev *pdev) ret = hisi_qm_stop(qm, QM_DOWN); if (ret) { pci_err(pdev, "Failed to stop QM, ret = %d.\n", ret); - hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET); - hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); - return; + goto err_prepare; } ret = qm_wait_vf_prepare_finish(qm); if (ret) pci_err(pdev, "failed to stop by vfs in FLR!\n"); + qm_dev_db_ctrl(qm, true); + pci_info(pdev, "FLR resetting...\n"); + + return; + +err_prepare: + pci_info(pdev, "FLR resetting prepare failed!\n"); + atomic_set(&qm->status.flags, QM_STOP); + hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET); + hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); + qm_dev_db_ctrl(qm, true); } EXPORT_SYMBOL_GPL(hisi_qm_reset_prepare); @@ -5027,14 +5144,13 @@ static irqreturn_t qm_rsvd_irq(int irq, void *data) static irqreturn_t qm_abnormal_irq(int irq, void *data) { struct hisi_qm *qm = data; - enum acc_err_result ret; atomic64_inc(&qm->debug.dfx.abnormal_irq_cnt); - ret = qm_process_dev_error(qm); - if (ret == ACC_ERR_NEED_RESET && - !test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl) && - !test_and_set_bit(QM_RST_SCHED, &qm->misc_ctl)) + + if (!test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl)) schedule_work(&qm->rst_work); + else + pci_warn(qm->pdev, "Driver is down, need to reload driver!\n"); return IRQ_HANDLED; } @@ -5050,27 +5166,72 @@ void hisi_qm_dev_shutdown(struct pci_dev *pdev) struct hisi_qm *qm = pci_get_drvdata(pdev); int ret; - ret = hisi_qm_stop(qm, QM_DOWN); + ret = hisi_qm_stop(qm, QM_SHUTDOWN); if (ret) dev_err(&pdev->dev, "Fail to stop qm in shutdown!\n"); } EXPORT_SYMBOL_GPL(hisi_qm_dev_shutdown); +static u64 qm_get_function_mask(struct hisi_qm *qm) +{ + return readq(qm->io_base + QM_FUNC_AXI_ERR_ST0); +} + +static void qm_clear_function_mask(struct hisi_qm *qm, u64 func_mask) +{ + /* Register write 1 clear */ + writeq(func_mask, qm->io_base + QM_FUNC_AXI_ERR_ST0); +} + +static void qm_function_reset(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + u64 func_mask; + u32 fun_num; + int ret; + + func_mask = qm_get_function_mask(qm); + if (!func_mask) { + dev_info(dev, "no function need reset!\n"); + return; + } + + for (fun_num = 1; fun_num <= qm->vfs_num; fun_num++) { + if (func_mask & BIT(fun_num)) { + ret = qm_ping_single_vf(qm, QM_FUNCTION_RESET, 0, fun_num); + /* When function ping fail, user decides the VF reset method. */ + if (ret) + dev_err(dev, "failed to send command(0x%x) to VF(%u)!\n", + (unsigned int)QM_FUNCTION_RESET, fun_num); + } + } + + if (func_mask & BIT(0)) { + dev_info(dev, "function reset start...\n"); + qm_reset_function(qm); + dev_info(dev, "function reset end!\n"); + } + + qm_clear_function_mask(qm, func_mask); +} + static void hisi_qm_controller_reset(struct work_struct *rst_work) { struct hisi_qm *qm = container_of(rst_work, struct hisi_qm, rst_work); + enum acc_err_result err_result; int ret; ret = qm_pm_get_sync(qm); if (ret) { - clear_bit(QM_RST_SCHED, &qm->misc_ctl); + dev_err(&qm->pdev->dev, "failed to get runtime PM for controller\n"); return; } - /* reset pcie device controller */ - ret = qm_controller_reset(qm); - if (ret) - dev_err(&qm->pdev->dev, "controller reset failed (%d)\n", ret); + err_result = qm_process_dev_error(qm); + if (err_result == ACC_ERR_NEED_RESET) + (void)qm_controller_reset(qm); + else if (err_result == ACC_ERR_NEED_FUNC_RESET) + qm_function_reset(qm); qm_pm_put_sync(qm); } @@ -5117,7 +5278,7 @@ static void qm_pf_reset_vf_done(struct hisi_qm *qm) int ret; pci_restore_state(pdev); - ret = hisi_qm_start(qm); + ret = qm_restart(qm); if (ret) { dev_err(&pdev->dev, "failed to start QM, ret = %d.\n", ret); cmd = QM_VF_START_FAIL; @@ -5131,10 +5292,22 @@ static void qm_pf_reset_vf_done(struct hisi_qm *qm) qm_reset_bit_clear(qm); } -static int qm_wait_pf_reset_finish(struct hisi_qm *qm) +static void qm_vf_update_isolate_info(struct hisi_qm *qm, u32 data) +{ + /* Updating the local isolation status. */ + mutex_lock(&qm->isolate_data.isolate_lock); + if (data & QM_ISOLATED_STATE) + qm->isolate_data.is_isolate = true; + else + qm->isolate_data.is_isolate = false; + qm->isolate_data.err_threshold = data & QM_ISOLATED_THRESHOLD_MASK; + mutex_unlock(&qm->isolate_data.isolate_lock); +} + +static int qm_wait_pf_reset_finish(struct hisi_qm *qm, enum qm_stop_reason stop_reason) { struct device *dev = &qm->pdev->dev; - u32 val, cmd; + u32 val, cmd, data; int ret; /* Wait for reset to finish */ @@ -5151,7 +5324,7 @@ static int qm_wait_pf_reset_finish(struct hisi_qm *qm) * Whether message is got successfully, * VF needs to ack PF by clearing the interrupt. */ - ret = qm->ops->get_ifc(qm, &cmd, NULL, 0); + ret = qm->ops->get_ifc(qm, &cmd, &data, 0); qm_clear_cmd_interrupt(qm, 0); if (ret) { dev_err(dev, "failed to get command from PF in reset done!\n"); @@ -5160,10 +5333,14 @@ static int qm_wait_pf_reset_finish(struct hisi_qm *qm) if (cmd != QM_PF_RESET_DONE) { dev_err(dev, "the command(0x%x) is not reset done!\n", cmd); - ret = -EINVAL; + return -EINVAL; } - return ret; + /* The VF processes the device isolation information received from the RAS reset. */ + if (stop_reason == QM_SOFT_RESET) + qm_vf_update_isolate_info(qm, data); + + return 0; } static void qm_pf_reset_vf_process(struct hisi_qm *qm, @@ -5178,7 +5355,7 @@ static void qm_pf_reset_vf_process(struct hisi_qm *qm, qm_cmd_uninit(qm); qm_pf_reset_vf_prepare(qm, stop_reason); - ret = qm_wait_pf_reset_finish(qm); + ret = qm_wait_pf_reset_finish(qm, stop_reason); if (ret) goto err_get_status; @@ -5189,10 +5366,31 @@ static void qm_pf_reset_vf_process(struct hisi_qm *qm, return; err_get_status: + if (stop_reason == QM_SOFT_RESET) { + /* Update local isolation status on PF-VF reset failure. */ + mutex_lock(&qm->isolate_data.isolate_lock); + qm->isolate_data.is_isolate = true; + mutex_unlock(&qm->isolate_data.isolate_lock); + } qm_cmd_init(qm); qm_reset_bit_clear(qm); } +static void qm_vf_get_isolate_data(struct hisi_qm *qm, u32 fun_num) +{ + u32 data = qm->isolate_data.err_threshold; + struct device *dev = &qm->pdev->dev; + int ret; + + if (qm->isolate_data.is_isolate) + data |= QM_ISOLATED_STATE; + + ret = qm_ping_single_vf(qm, QM_PF_SET_ISOLATE, data, fun_num); + if (ret) + dev_err(dev, "failed to send command(0x%x) to VF(%u)!\n", + (unsigned int)QM_PF_SET_ISOLATE, fun_num); +} + static void qm_handle_cmd_msg(struct hisi_qm *qm, u32 fun_num) { struct device *dev = &qm->pdev->dev; @@ -5224,6 +5422,18 @@ static void qm_handle_cmd_msg(struct hisi_qm *qm, u32 fun_num) case QM_PF_SET_QOS: qm->mb_qos = data; break; + case QM_FUNCTION_RESET: + dev_info(dev, "function reset start...\n"); + qm_reset_function(qm); + dev_info(dev, "function reset end!\n"); + break; + case QM_VF_GET_ISOLATE: + /* Read the isolation policy of the PF during VF initialization. */ + qm_vf_get_isolate_data(qm, fun_num); + break; + case QM_PF_SET_ISOLATE: + qm_vf_update_isolate_info(qm, data); + break; default: dev_err(dev, "unsupported command(0x%x) sent by function(%u)!\n", cmd, fun_num); break; diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 2471a4dd0b50..77e0e03cbcab 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -20,7 +20,7 @@ #include "sec.h" #include "sec_crypto.h" -#define SEC_PRIORITY 4001 +#define SEC_PRIORITY 80 #define SEC_XTS_MIN_KEY_SIZE (2 * AES_MIN_KEY_SIZE) #define SEC_XTS_MID_KEY_SIZE (3 * AES_MIN_KEY_SIZE) #define SEC_XTS_MAX_KEY_SIZE (2 * AES_MAX_KEY_SIZE) diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 056bd8f4da5a..752565200c16 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -48,6 +48,7 @@ #define SEC_OOO_SHUTDOWN_SEL 0x301014 #define SEC_RAS_DISABLE 0x0 #define SEC_AXI_ERROR_MASK (BIT(0) | BIT(1)) +#define SEC_RAS_CLEAR_ALL GENMASK(31, 0) #define SEC_MEM_START_INIT_REG 0x301100 #define SEC_MEM_INIT_DONE_REG 0x301104 @@ -752,7 +753,7 @@ static void sec_hw_error_enable(struct hisi_qm *qm) } /* clear SEC hw error source if having */ - writel(err_mask, qm->io_base + SEC_CORE_INT_SOURCE); + writel(SEC_RAS_CLEAR_ALL, qm->io_base + SEC_CORE_INT_SOURCE); /* enable RAS int */ writel(dev_err->ce, qm->io_base + SEC_RAS_CE_REG); @@ -1449,12 +1450,10 @@ static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_qm_del_list; } - if (qm->uacce) { - ret = uacce_register(qm->uacce); - if (ret) { - pci_err(pdev, "failed to register uacce (%d)!\n", ret); - goto err_alg_unregister; - } + ret = hisi_qm_register_uacce(qm); + if (ret) { + pci_err(pdev, "failed to register uacce (%d)!\n", ret); + goto err_alg_unregister; } if (qm->fun_type == QM_HW_PF && vfs_num) { diff --git a/drivers/crypto/hisilicon/trng/Makefile b/drivers/crypto/hisilicon/trng/Makefile deleted file mode 100644 index d909079f351c..000000000000 --- a/drivers/crypto/hisilicon/trng/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -obj-$(CONFIG_CRYPTO_DEV_HISI_TRNG) += hisi-trng-v2.o -hisi-trng-v2-objs = trng.o diff --git a/drivers/crypto/hisilicon/trng/trng.c b/drivers/crypto/hisilicon/trng/trng.c deleted file mode 100644 index 5ca0b90859a8..000000000000 --- a/drivers/crypto/hisilicon/trng/trng.c +++ /dev/null @@ -1,390 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright (c) 2019 HiSilicon Limited. */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define HISI_TRNG_REG 0x00F0 -#define HISI_TRNG_BYTES 4 -#define HISI_TRNG_QUALITY 512 -#define HISI_TRNG_VERSION 0x01B8 -#define HISI_TRNG_VER_V1 GENMASK(31, 0) -#define SLEEP_US 10 -#define TIMEOUT_US 10000 -#define SW_DRBG_NUM_SHIFT 2 -#define SW_DRBG_KEY_BASE 0x082C -#define SW_DRBG_SEED(n) (SW_DRBG_KEY_BASE - ((n) << SW_DRBG_NUM_SHIFT)) -#define SW_DRBG_SEED_REGS_NUM 12 -#define SW_DRBG_SEED_SIZE 48 -#define SW_DRBG_BLOCKS 0x0830 -#define SW_DRBG_INIT 0x0834 -#define SW_DRBG_GEN 0x083c -#define SW_DRBG_STATUS 0x0840 -#define SW_DRBG_BLOCKS_NUM 4095 -#define SW_DRBG_DATA_BASE 0x0850 -#define SW_DRBG_DATA_NUM 4 -#define SW_DRBG_DATA(n) (SW_DRBG_DATA_BASE - ((n) << SW_DRBG_NUM_SHIFT)) -#define SW_DRBG_BYTES 16 -#define SW_DRBG_ENABLE_SHIFT 12 -#define SEED_SHIFT_24 24 -#define SEED_SHIFT_16 16 -#define SEED_SHIFT_8 8 -#define SW_MAX_RANDOM_BYTES 65520 - -struct hisi_trng_list { - struct mutex lock; - struct list_head list; - bool is_init; -}; - -struct hisi_trng { - void __iomem *base; - struct hisi_trng_list *trng_list; - struct list_head list; - struct hwrng rng; - u32 ver; - u32 ctx_num; - /* The bytes of the random number generated since the last seeding. */ - u32 random_bytes; - struct mutex lock; -}; - -struct hisi_trng_ctx { - struct hisi_trng *trng; -}; - -static atomic_t trng_active_devs; -static struct hisi_trng_list trng_devices; -static int hisi_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait); - -static int hisi_trng_set_seed(struct hisi_trng *trng, const u8 *seed) -{ - u32 val, seed_reg, i; - int ret; - - writel(0x0, trng->base + SW_DRBG_BLOCKS); - - for (i = 0; i < SW_DRBG_SEED_SIZE; - i += SW_DRBG_SEED_SIZE / SW_DRBG_SEED_REGS_NUM) { - val = seed[i] << SEED_SHIFT_24; - val |= seed[i + 1UL] << SEED_SHIFT_16; - val |= seed[i + 2UL] << SEED_SHIFT_8; - val |= seed[i + 3UL]; - - seed_reg = (i >> SW_DRBG_NUM_SHIFT) % SW_DRBG_SEED_REGS_NUM; - writel(val, trng->base + SW_DRBG_SEED(seed_reg)); - } - - writel(SW_DRBG_BLOCKS_NUM | (0x1 << SW_DRBG_ENABLE_SHIFT), - trng->base + SW_DRBG_BLOCKS); - writel(0x1, trng->base + SW_DRBG_INIT); - ret = readl_relaxed_poll_timeout(trng->base + SW_DRBG_STATUS, - val, val & BIT(0), SLEEP_US, TIMEOUT_US); - if (ret) { - pr_err("failed to init trng(%d)\n", ret); - return -EIO; - } - - trng->random_bytes = 0; - - return 0; -} - -static int hisi_trng_seed(struct crypto_rng *tfm, const u8 *seed, - unsigned int slen) -{ - struct hisi_trng_ctx *ctx = crypto_rng_ctx(tfm); - struct hisi_trng *trng = ctx->trng; - int ret; - - if (slen < SW_DRBG_SEED_SIZE) { - pr_err("slen(%u) is not matched with trng(%d)\n", slen, - SW_DRBG_SEED_SIZE); - return -EINVAL; - } - - mutex_lock(&trng->lock); - ret = hisi_trng_set_seed(trng, seed); - mutex_unlock(&trng->lock); - - return ret; -} - -static int hisi_trng_reseed(struct hisi_trng *trng) -{ - u8 seed[SW_DRBG_SEED_SIZE]; - int size; - - if (!trng->random_bytes) - return 0; - - size = hisi_trng_read(&trng->rng, seed, SW_DRBG_SEED_SIZE, false); - if (size != SW_DRBG_SEED_SIZE) - return -EIO; - - return hisi_trng_set_seed(trng, seed); -} - -static int hisi_trng_get_bytes(struct hisi_trng *trng, u8 *dstn, unsigned int dlen) -{ - u32 data[SW_DRBG_DATA_NUM]; - u32 currsize = 0; - u32 val = 0; - int ret; - u32 i; - - ret = hisi_trng_reseed(trng); - if (ret) - return ret; - - do { - ret = readl_relaxed_poll_timeout(trng->base + SW_DRBG_STATUS, - val, val & BIT(1), SLEEP_US, TIMEOUT_US); - if (ret) { - pr_err("failed to generate random number(%d)!\n", ret); - break; - } - - for (i = 0; i < SW_DRBG_DATA_NUM; i++) - data[i] = readl(trng->base + SW_DRBG_DATA(i)); - - if (dlen - currsize >= SW_DRBG_BYTES) { - memcpy(dstn + currsize, data, SW_DRBG_BYTES); - currsize += SW_DRBG_BYTES; - } else { - memcpy(dstn + currsize, data, dlen - currsize); - currsize = dlen; - } - - trng->random_bytes += SW_DRBG_BYTES; - writel(0x1, trng->base + SW_DRBG_GEN); - } while (currsize < dlen); - - return ret; -} - -static int hisi_trng_generate(struct crypto_rng *tfm, const u8 *src, - unsigned int slen, u8 *dstn, unsigned int dlen) -{ - struct hisi_trng_ctx *ctx = crypto_rng_ctx(tfm); - struct hisi_trng *trng = ctx->trng; - unsigned int currsize = 0; - unsigned int block_size; - int ret; - - if (!dstn || !dlen) { - pr_err("output is error, dlen %u!\n", dlen); - return -EINVAL; - } - - do { - block_size = min_t(unsigned int, dlen - currsize, SW_MAX_RANDOM_BYTES); - mutex_lock(&trng->lock); - ret = hisi_trng_get_bytes(trng, dstn + currsize, block_size); - mutex_unlock(&trng->lock); - if (ret) - return ret; - currsize += block_size; - } while (currsize < dlen); - - return 0; -} - -static int hisi_trng_init(struct crypto_tfm *tfm) -{ - struct hisi_trng_ctx *ctx = crypto_tfm_ctx(tfm); - struct hisi_trng *trng; - u32 ctx_num = ~0; - - mutex_lock(&trng_devices.lock); - list_for_each_entry(trng, &trng_devices.list, list) { - if (trng->ctx_num < ctx_num) { - ctx_num = trng->ctx_num; - ctx->trng = trng; - } - } - ctx->trng->ctx_num++; - mutex_unlock(&trng_devices.lock); - - return 0; -} - -static void hisi_trng_exit(struct crypto_tfm *tfm) -{ - struct hisi_trng_ctx *ctx = crypto_tfm_ctx(tfm); - - mutex_lock(&trng_devices.lock); - ctx->trng->ctx_num--; - mutex_unlock(&trng_devices.lock); -} - -static int hisi_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait) -{ - struct hisi_trng *trng; - int currsize = 0; - u32 val = 0; - int ret; - - trng = container_of(rng, struct hisi_trng, rng); - - do { - ret = readl_poll_timeout(trng->base + HISI_TRNG_REG, val, - val, SLEEP_US, TIMEOUT_US); - if (ret) - return currsize; - - if (max - currsize >= HISI_TRNG_BYTES) { - memcpy(buf + currsize, &val, HISI_TRNG_BYTES); - currsize += HISI_TRNG_BYTES; - if (currsize == max) - return currsize; - continue; - } - - /* copy remaining bytes */ - memcpy(buf + currsize, &val, max - currsize); - currsize = max; - } while (currsize < max); - - return currsize; -} - -static struct rng_alg hisi_trng_alg = { - .generate = hisi_trng_generate, - .seed = hisi_trng_seed, - .seedsize = SW_DRBG_SEED_SIZE, - .base = { - .cra_name = "stdrng", - .cra_driver_name = "hisi_stdrng", - .cra_priority = 300, - .cra_ctxsize = sizeof(struct hisi_trng_ctx), - .cra_module = THIS_MODULE, - .cra_init = hisi_trng_init, - .cra_exit = hisi_trng_exit, - }, -}; - -static void hisi_trng_add_to_list(struct hisi_trng *trng) -{ - mutex_lock(&trng_devices.lock); - list_add_tail(&trng->list, &trng_devices.list); - mutex_unlock(&trng_devices.lock); -} - -static int hisi_trng_del_from_list(struct hisi_trng *trng) -{ - int ret = -EBUSY; - - mutex_lock(&trng_devices.lock); - if (!trng->ctx_num) { - list_del(&trng->list); - ret = 0; - } - mutex_unlock(&trng_devices.lock); - - return ret; -} - -static int hisi_trng_probe(struct platform_device *pdev) -{ - struct hisi_trng *trng; - int ret; - - trng = devm_kzalloc(&pdev->dev, sizeof(*trng), GFP_KERNEL); - if (!trng) - return -ENOMEM; - - platform_set_drvdata(pdev, trng); - - trng->base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(trng->base)) - return PTR_ERR(trng->base); - - trng->ctx_num = 0; - trng->random_bytes = SW_MAX_RANDOM_BYTES; - mutex_init(&trng->lock); - trng->ver = readl(trng->base + HISI_TRNG_VERSION); - if (!trng_devices.is_init) { - INIT_LIST_HEAD(&trng_devices.list); - mutex_init(&trng_devices.lock); - trng_devices.is_init = true; - } - - hisi_trng_add_to_list(trng); - if (trng->ver != HISI_TRNG_VER_V1 && - atomic_inc_return(&trng_active_devs) == 1) { - ret = crypto_register_rng(&hisi_trng_alg); - if (ret) { - dev_err(&pdev->dev, - "failed to register crypto(%d)\n", ret); - atomic_dec_return(&trng_active_devs); - goto err_remove_from_list; - } - } - - trng->rng.name = pdev->name; - trng->rng.read = hisi_trng_read; - trng->rng.quality = HISI_TRNG_QUALITY; - ret = devm_hwrng_register(&pdev->dev, &trng->rng); - if (ret) { - dev_err(&pdev->dev, "failed to register hwrng: %d!\n", ret); - goto err_crypto_unregister; - } - - return ret; - -err_crypto_unregister: - if (trng->ver != HISI_TRNG_VER_V1 && - atomic_dec_return(&trng_active_devs) == 0) - crypto_unregister_rng(&hisi_trng_alg); - -err_remove_from_list: - hisi_trng_del_from_list(trng); - return ret; -} - -static void hisi_trng_remove(struct platform_device *pdev) -{ - struct hisi_trng *trng = platform_get_drvdata(pdev); - - /* Wait until the task is finished */ - while (hisi_trng_del_from_list(trng)) - ; - - if (trng->ver != HISI_TRNG_VER_V1 && - atomic_dec_return(&trng_active_devs) == 0) - crypto_unregister_rng(&hisi_trng_alg); -} - -static const struct acpi_device_id hisi_trng_acpi_match[] = { - { "HISI02B3", 0 }, - { } -}; -MODULE_DEVICE_TABLE(acpi, hisi_trng_acpi_match); - -static struct platform_driver hisi_trng_driver = { - .probe = hisi_trng_probe, - .remove = hisi_trng_remove, - .driver = { - .name = "hisi-trng-v2", - .acpi_match_table = ACPI_PTR(hisi_trng_acpi_match), - }, -}; - -module_platform_driver(hisi_trng_driver); - -MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Weili Qian "); -MODULE_AUTHOR("Zaibo Xu "); -MODULE_DESCRIPTION("HiSilicon true random number generator V2 driver"); diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 44df9c859bd8..706a73656977 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -64,7 +64,8 @@ #define HZIP_OOO_SHUTDOWN_SEL 0x30120C #define HZIP_SRAM_ECC_ERR_NUM_SHIFT 16 #define HZIP_SRAM_ECC_ERR_ADDR_SHIFT 24 -#define HZIP_CORE_INT_MASK_ALL GENMASK(12, 0) +#define HZIP_CORE_INT_MASK_ALL GENMASK(31, 0) +#define HZIP_CORE_RAS_CLEAR_ALL GENMASK(31, 0) #define HZIP_AXI_ERROR_MASK (BIT(2) | BIT(3)) #define HZIP_SQE_SIZE 128 #define HZIP_PF_DEF_Q_NUM 64 @@ -696,7 +697,7 @@ static void hisi_zip_hw_error_enable(struct hisi_qm *qm) } /* clear ZIP hw error source if having */ - writel(err_mask, qm->io_base + HZIP_CORE_INT_SOURCE); + writel(HZIP_CORE_RAS_CLEAR_ALL, qm->io_base + HZIP_CORE_INT_SOURCE); /* configure error type */ writel(dev_err->ce, qm->io_base + HZIP_CORE_INT_RAS_CE_ENB); @@ -713,11 +714,8 @@ static void hisi_zip_hw_error_enable(struct hisi_qm *qm) static void hisi_zip_hw_error_disable(struct hisi_qm *qm) { - struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; - u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; - /* disable ZIP hw error interrupts */ - writel(err_mask, qm->io_base + HZIP_CORE_INT_MASK_REG); + writel(HZIP_CORE_INT_MASK_ALL, qm->io_base + HZIP_CORE_INT_MASK_REG); hisi_zip_master_ooo_ctrl(qm, false); @@ -1559,12 +1557,10 @@ static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_qm_del_list; } - if (qm->uacce) { - ret = uacce_register(qm->uacce); - if (ret) { - pci_err(pdev, "failed to register uacce (%d)!\n", ret); - goto err_qm_alg_unregister; - } + ret = hisi_qm_register_uacce(qm); + if (ret) { + pci_err(pdev, "failed to register uacce (%d)!\n", ret); + goto err_qm_alg_unregister; } if (qm->fun_type == QM_HW_PF && vfs_num > 0) { diff --git a/drivers/crypto/inside-secure/eip93/eip93-main.c b/drivers/crypto/inside-secure/eip93/eip93-main.c index 7dccfdeb7b11..1a8dabc4ada4 100644 --- a/drivers/crypto/inside-secure/eip93/eip93-main.c +++ b/drivers/crypto/inside-secure/eip93/eip93-main.c @@ -433,13 +433,14 @@ static int eip93_crypto_probe(struct platform_device *pdev) ret = devm_request_threaded_irq(eip93->dev, eip93->irq, eip93_irq_handler, NULL, IRQF_ONESHOT, dev_name(eip93->dev), eip93); + if (ret) + return ret; eip93->ring = devm_kcalloc(eip93->dev, 1, sizeof(*eip93->ring), GFP_KERNEL); if (!eip93->ring) return -ENOMEM; ret = eip93_desc_init(eip93); - if (ret) return ret; diff --git a/drivers/crypto/inside-secure/eip93/eip93-regs.h b/drivers/crypto/inside-secure/eip93/eip93-regs.h index 96285ca6fbbe..96d28c6651bd 100644 --- a/drivers/crypto/inside-secure/eip93/eip93-regs.h +++ b/drivers/crypto/inside-secure/eip93/eip93-regs.h @@ -103,7 +103,7 @@ #define EIP93_PE_TARGET_COMMAND_NO_RDR_MODE FIELD_PREP(EIP93_PE_CONFIG_PE_MODE, 0x2) #define EIP93_PE_TARGET_COMMAND_WITH_RDR_MODE FIELD_PREP(EIP93_PE_CONFIG_PE_MODE, 0x1) #define EIP93_PE_DIRECT_HOST_MODE FIELD_PREP(EIP93_PE_CONFIG_PE_MODE, 0x0) -#define EIP93_PE_CONFIG_RST_RING BIT(2) +#define EIP93_PE_CONFIG_RST_RING BIT(1) #define EIP93_PE_CONFIG_RST_PE BIT(0) #define EIP93_REG_PE_STATUS 0x104 #define EIP93_REG_PE_BUF_THRESH 0x10c diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index fb4936e7afa2..52809e57361a 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1475,7 +1475,7 @@ static int safexcel_probe_generic(void *pdev, peid = version & 255; /* Detect EIP206 processing pipe */ - version = readl(EIP197_PE(priv) + + EIP197_PE_VERSION(0)); + version = readl(EIP197_PE(priv) + EIP197_PE_VERSION(0)); if (EIP197_REG_LO16(version) != EIP206_VERSION_LE) { dev_err(priv->dev, "EIP%d: EIP206 not detected\n", peid); return -ENODEV; @@ -1893,7 +1893,7 @@ static int safexcel_pci_probe(struct pci_dev *pdev, ent->vendor, ent->device, ent->subvendor, ent->subdevice, ent->driver_data); - priv = kzalloc_obj(*priv); + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); if (!priv) return -ENOMEM; diff --git a/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c b/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c index fcc0cf4df637..5b90cf0fb0e4 100644 --- a/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c +++ b/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c @@ -884,8 +884,9 @@ static struct buffer_desc *chainup_buffers(struct device *dev, ptr = sg_virt(sg); next_buf = dma_pool_alloc(buffer_pool, flags, &next_buf_phys); if (!next_buf) { - buf = NULL; - break; + buf->next = NULL; + buf->phys_next = 0; + return NULL; } sg_dma_address(sg) = dma_map_single(dev, ptr, len, dir); buf->next = next_buf; @@ -983,7 +984,7 @@ static int ablk_perform(struct skcipher_request *req, int encrypt) unsigned int nbytes = req->cryptlen; enum dma_data_direction src_direction = DMA_BIDIRECTIONAL; struct ablk_ctx *req_ctx = skcipher_request_ctx(req); - struct buffer_desc src_hook; + struct buffer_desc *buf, src_hook; struct device *dev = &pdev->dev; unsigned int offset; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? @@ -1025,22 +1026,24 @@ static int ablk_perform(struct skcipher_request *req, int encrypt) /* This was never tested by Intel * for more than one dst buffer, I think. */ req_ctx->dst = NULL; - if (!chainup_buffers(dev, req->dst, nbytes, &dst_hook, - flags, DMA_FROM_DEVICE)) - goto free_buf_dest; - src_direction = DMA_TO_DEVICE; + buf = chainup_buffers(dev, req->dst, nbytes, &dst_hook, + flags, DMA_FROM_DEVICE); req_ctx->dst = dst_hook.next; crypt->dst_buf = dst_hook.phys_next; + if (!buf) + goto free_buf_dest; + src_direction = DMA_TO_DEVICE; } else { req_ctx->dst = NULL; } req_ctx->src = NULL; - if (!chainup_buffers(dev, req->src, nbytes, &src_hook, flags, - src_direction)) - goto free_buf_src; - + buf = chainup_buffers(dev, req->src, nbytes, &src_hook, flags, + src_direction); req_ctx->src = src_hook.next; crypt->src_buf = src_hook.phys_next; + if (!buf) + goto free_buf_src; + crypt->ctl_flags |= CTL_FLAG_PERFORM_ABLK; qmgr_put_entry(send_qid, crypt_virt2phys(crypt)); BUG_ON(qmgr_stat_overflow(send_qid)); diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_drv.c b/drivers/crypto/intel/qat/qat_420xx/adf_drv.c index cfa00daeb4fb..0f0827e2b0bd 100644 --- a/drivers/crypto/intel/qat/qat_420xx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_420xx/adf_drv.c @@ -101,7 +101,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* Enable PCI device */ ret = pcim_enable_device(pdev); if (ret) { - dev_err(&pdev->dev, "Can't enable PCI device.\n"); + pci_err(pdev, "Can't enable PCI device.\n"); goto out_err; } @@ -131,7 +131,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) ret = pcim_request_all_regions(pdev, pci_name(pdev)); if (ret) { - dev_err(&pdev->dev, "Failed to request PCI regions.\n"); + pci_err(pdev, "Failed to request PCI regions.\n"); goto out_err; } @@ -140,16 +140,14 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) bar = &accel_pci_dev->pci_bars[i++]; bar->virt_addr = pcim_iomap(pdev, bar_nr, 0); if (!bar->virt_addr) { - dev_err(&pdev->dev, "Failed to ioremap PCI region.\n"); + pci_err(pdev, "Failed to ioremap PCI region.\n"); ret = -ENOMEM; goto out_err; } } - pci_set_master(pdev); - if (pci_save_state(pdev)) { - dev_err(&pdev->dev, "Failed to save pci state.\n"); + pci_err(pdev, "Failed to save pci state.\n"); ret = -ENOMEM; goto out_err; } @@ -210,6 +208,5 @@ MODULE_AUTHOR("Intel"); MODULE_FIRMWARE(ADF_420XX_FW); MODULE_FIRMWARE(ADF_420XX_MMP); MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); -MODULE_VERSION(ADF_DRV_VERSION); MODULE_SOFTDEP("pre: crypto-intel_qat"); MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c index c9be5dcddb27..aa95f762cb4b 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c @@ -103,7 +103,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* Enable PCI device */ ret = pcim_enable_device(pdev); if (ret) { - dev_err(&pdev->dev, "Can't enable PCI device.\n"); + pci_err(pdev, "Can't enable PCI device.\n"); goto out_err; } @@ -133,7 +133,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) ret = pcim_request_all_regions(pdev, pci_name(pdev)); if (ret) { - dev_err(&pdev->dev, "Failed to request PCI regions.\n"); + pci_err(pdev, "Failed to request PCI regions.\n"); goto out_err; } @@ -142,16 +142,14 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) bar = &accel_pci_dev->pci_bars[i++]; bar->virt_addr = pcim_iomap(pdev, bar_nr, 0); if (!bar->virt_addr) { - dev_err(&pdev->dev, "Failed to ioremap PCI region.\n"); + pci_err(pdev, "Failed to ioremap PCI region.\n"); ret = -ENOMEM; goto out_err; } } - pci_set_master(pdev); - if (pci_save_state(pdev)) { - dev_err(&pdev->dev, "Failed to save pci state.\n"); + pci_err(pdev, "Failed to save pci state.\n"); ret = -ENOMEM; goto out_err; } @@ -214,6 +212,5 @@ MODULE_FIRMWARE(ADF_402XX_FW); MODULE_FIRMWARE(ADF_4XXX_MMP); MODULE_FIRMWARE(ADF_402XX_MMP); MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); -MODULE_VERSION(ADF_DRV_VERSION); MODULE_SOFTDEP("pre: crypto-intel_qat"); MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c b/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c index 205680797e2c..388087e64540 100644 --- a/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c @@ -752,10 +752,12 @@ static u32 get_accel_cap(struct adf_accel_dev *accel_dev) unsigned long mask; u32 caps = 0; u32 fusectl1; + u32 fusectl0; if (adf_6xxx_is_wcy(GET_HW_DATA(accel_dev))) return get_accel_cap_wcy(accel_dev); + fusectl0 = GET_HW_DATA(accel_dev)->fuses[ADF_FUSECTL0]; fusectl1 = GET_HW_DATA(accel_dev)->fuses[ADF_FUSECTL1]; /* Read accelerator capabilities mask */ @@ -785,7 +787,8 @@ static u32 get_accel_cap(struct adf_accel_dev *accel_dev) capabilities_asym = ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | ICP_ACCEL_CAPABILITIES_SM2 | - ICP_ACCEL_CAPABILITIES_ECEDMONT; + ICP_ACCEL_CAPABILITIES_ECEDMONT | + ICP_ACCEL_CAPABILITIES_KPT; if (fusectl1 & ICP_ACCEL_GEN6_MASK_PKE_SLICE) { capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; @@ -793,6 +796,9 @@ static u32 get_accel_cap(struct adf_accel_dev *accel_dev) capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_ECEDMONT; } + if (fusectl0 & ADF_GEN6_KPT_FUSE_BIT) + capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_KPT; + capabilities_dc = ICP_ACCEL_CAPABILITIES_COMPRESSION | ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION | ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION | @@ -960,6 +966,18 @@ static int dev_config(struct adf_accel_dev *accel_dev) return ret; } +static void adf_gen6_init_kpt(struct adf_kpt_hw_data *kpt_data) +{ + kpt_data->max_swk_cnt_per_fn_pasid = ADF_6XXX_KPT_MAX_SWK_COUNT_PER_FNPASID; + kpt_data->max_swk_ttl = ADF_6XXX_KPT_MAX_SWK_TTL; + + kpt_data->user_input.enable = false; + kpt_data->user_input.swk_shared = ADF_6XXX_KPT_DEFAULT_SWK_SHARED_MODE; + kpt_data->user_input.swk_max_ttl = ADF_6XXX_KPT_DEFAULT_SWK_TTL; + kpt_data->user_input.swk_cnt_per_fn = ADF_6XXX_KPT_DEFAULT_SWK_CNT_PER_FN; + kpt_data->user_input.swk_cnt_per_pasid = ADF_6XXX_KPT_DEFAULT_SWK_CNT_PER_PASID; +} + static void adf_gen6_init_rl_data(struct adf_rl_hw_data *rl_data) { rl_data->pciout_tb_offset = ADF_GEN6_RL_TOKEN_PCIEOUT_BUCKET_OFFSET; @@ -1057,6 +1075,7 @@ void adf_init_hw_data_6xxx(struct adf_hw_device_data *hw_data) adf_gen6_init_tl_data(&hw_data->tl_data); adf_gen6_init_rl_data(&hw_data->rl_data); adf_gen6_init_anti_rb(&hw_data->anti_rb_data); + adf_gen6_init_kpt(&hw_data->kpt_data); } void adf_clean_hw_data_6xxx(struct adf_hw_device_data *hw_data) diff --git a/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.h b/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.h index e4d433bdd379..2719ddbc1e05 100644 --- a/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.h +++ b/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.h @@ -58,6 +58,7 @@ /* Fuse bits */ #define ADF_GEN6_ANTI_RB_FUSE_BIT BIT(24) +#define ADF_GEN6_KPT_FUSE_BIT BIT(16) /* * Watchdog timers @@ -164,6 +165,14 @@ /* Clock frequency */ #define ADF_6XXX_AE_FREQ (1000 * HZ_PER_MHZ) +/* KPT */ +#define ADF_6XXX_KPT_MAX_SWK_COUNT_PER_FNPASID 128 +#define ADF_6XXX_KPT_MAX_SWK_TTL 31536000 +#define ADF_6XXX_KPT_DEFAULT_SWK_SHARED_MODE 1 +#define ADF_6XXX_KPT_DEFAULT_SWK_TTL 0 +#define ADF_6XXX_KPT_DEFAULT_SWK_CNT_PER_FN 0 +#define ADF_6XXX_KPT_DEFAULT_SWK_CNT_PER_PASID 0 + enum icp_qat_gen6_slice_mask { ICP_ACCEL_GEN6_MASK_UCS_SLICE = BIT(0), ICP_ACCEL_GEN6_MASK_AUTH_SLICE = BIT(1), diff --git a/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c index c52462a48c34..319b4251ad46 100644 --- a/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c @@ -13,6 +13,7 @@ #include #include #include +#include #include "adf_gen6_shared.h" #include "adf_6xxx_hw_data.h" @@ -189,8 +190,6 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } } - pci_set_master(pdev); - /* * The PCI config space is saved at this point and will be restored * after a Function Level Reset (FLR) as the FLR does not completely @@ -219,6 +218,11 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) return ret; ret = adf_sysfs_init(accel_dev); + if (ret) + return ret; + + if (!(hw_data->fuses[ADF_FUSECTL0] & ADF_GEN6_KPT_FUSE_BIT)) + ret = adf_sysfs_init_kpt(accel_dev); return ret; } diff --git a/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c index bceb5dd8b148..e816cc00632f 100644 --- a/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c @@ -162,15 +162,14 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) bar->size = pci_resource_len(pdev, bar_nr); bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); if (!bar->virt_addr) { - dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr); + pci_err(pdev, "Failed to map BAR %d\n", bar_nr); ret = -EFAULT; goto out_err_free_reg; } } - pci_set_master(pdev); if (pci_save_state(pdev)) { - dev_err(&pdev->dev, "Failed to save pci state\n"); + pci_err(pdev, "Failed to save pci state\n"); ret = -ENOMEM; goto out_err_free_reg; } @@ -256,5 +255,4 @@ MODULE_AUTHOR("Intel"); MODULE_FIRMWARE(ADF_C3XXX_FW); MODULE_FIRMWARE(ADF_C3XXX_MMP); MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); -MODULE_VERSION(ADF_DRV_VERSION); MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/intel/qat/qat_c3xxxvf/adf_drv.c b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_drv.c index c622793e94a8..1c77f0a1882b 100644 --- a/drivers/crypto/intel/qat/qat_c3xxxvf/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_drv.c @@ -158,12 +158,11 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) bar->size = pci_resource_len(pdev, bar_nr); bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); if (!bar->virt_addr) { - dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr); + pci_err(pdev, "Failed to map BAR %d\n", bar_nr); ret = -EFAULT; goto out_err_free_reg; } } - pci_set_master(pdev); /* Completion for VF2PF request/response message exchange */ init_completion(&accel_dev->vf.msg_received); @@ -225,5 +224,4 @@ module_exit(adfdrv_release); MODULE_LICENSE("Dual BSD/GPL"); MODULE_AUTHOR("Intel"); MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); -MODULE_VERSION(ADF_DRV_VERSION); MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/intel/qat/qat_c62x/adf_drv.c b/drivers/crypto/intel/qat/qat_c62x/adf_drv.c index 23ccb72b6ea2..f48f3b437545 100644 --- a/drivers/crypto/intel/qat/qat_c62x/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_c62x/adf_drv.c @@ -162,15 +162,14 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) bar->size = pci_resource_len(pdev, bar_nr); bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); if (!bar->virt_addr) { - dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr); + pci_err(pdev, "Failed to map BAR %d\n", bar_nr); ret = -EFAULT; goto out_err_free_reg; } } - pci_set_master(pdev); if (pci_save_state(pdev)) { - dev_err(&pdev->dev, "Failed to save pci state\n"); + pci_err(pdev, "Failed to save pci state\n"); ret = -ENOMEM; goto out_err_free_reg; } @@ -256,5 +255,4 @@ MODULE_AUTHOR("Intel"); MODULE_FIRMWARE(ADF_C62X_FW); MODULE_FIRMWARE(ADF_C62X_MMP); MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); -MODULE_VERSION(ADF_DRV_VERSION); MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/intel/qat/qat_c62xvf/adf_drv.c b/drivers/crypto/intel/qat/qat_c62xvf/adf_drv.c index 4840d44bbd5b..b96f19e31d05 100644 --- a/drivers/crypto/intel/qat/qat_c62xvf/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_c62xvf/adf_drv.c @@ -158,12 +158,11 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) bar->size = pci_resource_len(pdev, bar_nr); bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); if (!bar->virt_addr) { - dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr); + pci_err(pdev, "Failed to map BAR %d\n", bar_nr); ret = -EFAULT; goto out_err_free_reg; } } - pci_set_master(pdev); /* Completion for VF2PF request/response message exchange */ init_completion(&accel_dev->vf.msg_received); @@ -225,5 +224,4 @@ module_exit(adfdrv_release); MODULE_LICENSE("Dual BSD/GPL"); MODULE_AUTHOR("Intel"); MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); -MODULE_VERSION(ADF_DRV_VERSION); MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/intel/qat/qat_common/Makefile b/drivers/crypto/intel/qat/qat_common/Makefile index 9478111c8437..306ec71bb6fa 100644 --- a/drivers/crypto/intel/qat/qat_common/Makefile +++ b/drivers/crypto/intel/qat/qat_common/Makefile @@ -9,7 +9,6 @@ intel_qat-y := adf_accel_engine.o \ adf_cfg.o \ adf_cfg_services.o \ adf_clock.o \ - adf_ctl_drv.o \ adf_dc.o \ adf_dev_mgr.o \ adf_gen2_config.o \ @@ -26,11 +25,14 @@ intel_qat-y := adf_accel_engine.o \ adf_hw_arbiter.o \ adf_init.o \ adf_isr.o \ + adf_kpt.o \ + adf_module.o \ adf_mstate_mgr.o \ adf_rl_admin.o \ adf_rl.o \ adf_sysfs.o \ adf_sysfs_anti_rb.o \ + adf_sysfs_kpt.o \ adf_sysfs_ras_counters.o \ adf_sysfs_rl.o \ adf_timer.o \ diff --git a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h index 03a4e9690208..f2d70641c377 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h @@ -14,6 +14,7 @@ #include "adf_anti_rb.h" #include "adf_cfg_common.h" #include "adf_dc.h" +#include "adf_kpt.h" #include "adf_rl.h" #include "adf_telemetry.h" #include "adf_pfvf_msg.h" @@ -335,6 +336,7 @@ struct adf_hw_device_data { struct adf_rl_hw_data rl_data; struct adf_tl_hw_data tl_data; struct adf_anti_rb_hw_data anti_rb_data; + struct adf_kpt_hw_data kpt_data; struct qat_migdev_ops vfmig_ops; const char *fw_name; const char *fw_mmp_name; @@ -480,6 +482,8 @@ struct adf_accel_dev { struct { /* protects VF2PF interrupts access */ spinlock_t vf2pf_ints_lock; + /* prevents VF2PF handling from racing with VF state teardown */ + bool vf2pf_disabled; /* vf_info is non-zero when SR-IOV is init'ed */ struct adf_accel_vf_info *vf_info; } pf; diff --git a/drivers/crypto/intel/qat/qat_common/adf_admin.c b/drivers/crypto/intel/qat/qat_common/adf_admin.c index 841aa802c79e..1eea74c2a8a5 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_admin.c +++ b/drivers/crypto/intel/qat/qat_common/adf_admin.c @@ -13,6 +13,7 @@ #include "adf_common_drv.h" #include "adf_cfg.h" #include "adf_heartbeat.h" +#include "adf_kpt.h" #include "icp_qat_fw_init_admin.h" #define ADF_ADMIN_MAILBOX_STRIDE 0x1000 @@ -606,6 +607,44 @@ int adf_send_admin_arb_commit(struct adf_accel_dev *accel_dev) return adf_send_admin_svn(accel_dev, ICP_QAT_FW_SVN_COMMIT, &resp); } +static_assert(sizeof(struct icp_qat_fw_init_admin_kpt_cfg) < PAGE_SIZE); + +static void adf_cfg_kpt_config(struct adf_accel_dev *accel_dev, + struct icp_qat_fw_init_admin_kpt_cfg *kpt_config) +{ + struct adf_kpt_interface_data *user_data = GET_KPT_USER_DATA(accel_dev); + + kpt_config->swk_cnt_per_fn = user_data->swk_cnt_per_fn; + kpt_config->swk_cnt_per_pasid = user_data->swk_cnt_per_pasid; + kpt_config->swk_ttl_in_secs = user_data->swk_max_ttl; + kpt_config->swk_shared_disable = !user_data->swk_shared; +} + +int adf_send_admin_kpt_init(struct adf_accel_dev *accel_dev, void *init_cfg, + size_t init_cfg_sz, dma_addr_t init_ptr) +{ + struct icp_qat_fw_init_admin_kpt_cfg *kpt_config = init_cfg; + u32 ae_mask = GET_HW_DATA(accel_dev)->admin_ae_mask; + struct icp_qat_fw_init_admin_resp resp = { }; + struct icp_qat_fw_init_admin_req req = { }; + int ret; + + if (!kpt_config || init_cfg_sz < sizeof(*kpt_config)) + return -EINVAL; + + adf_cfg_kpt_config(accel_dev, kpt_config); + + req.cmd_id = ICP_QAT_FW_KPT_ENABLE; + req.init_cfg_ptr = init_ptr; + req.init_cfg_sz = sizeof(*kpt_config); + + ret = adf_send_admin(accel_dev, &req, &resp, ae_mask); + if (ret) + dev_err(&GET_DEV(accel_dev), "Failed to send KPT init admin message\n"); + + return ret; +} + int adf_init_admin_comms(struct adf_accel_dev *accel_dev) { struct adf_admin_comms *admin; diff --git a/drivers/crypto/intel/qat/qat_common/adf_admin.h b/drivers/crypto/intel/qat/qat_common/adf_admin.h index 9704219f2eb7..44eb6dc92e45 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_admin.h +++ b/drivers/crypto/intel/qat/qat_common/adf_admin.h @@ -29,5 +29,7 @@ int adf_send_admin_tl_start(struct adf_accel_dev *accel_dev, int adf_send_admin_tl_stop(struct adf_accel_dev *accel_dev); int adf_send_admin_arb_query(struct adf_accel_dev *accel_dev, int cmd, u8 *svn); int adf_send_admin_arb_commit(struct adf_accel_dev *accel_dev); +int adf_send_admin_kpt_init(struct adf_accel_dev *accel_dev, void *init_cfg, + size_t init_cfg_sz, dma_addr_t init_ptr); #endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_aer.c b/drivers/crypto/intel/qat/qat_common/adf_aer.c index ed01fb9ad74e..d58cd7fbf707 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_aer.c +++ b/drivers/crypto/intel/qat/qat_common/adf_aer.c @@ -17,36 +17,80 @@ struct adf_fatal_error_data { static struct workqueue_struct *device_reset_wq; static struct workqueue_struct *device_sriov_wq; -static pci_ers_result_t adf_error_detected(struct pci_dev *pdev, - pci_channel_state_t state) +static pci_ers_result_t reset_prepare(struct pci_dev *pdev) { struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); - dev_info(&pdev->dev, "Acceleration driver hardware error detected.\n"); if (!accel_dev) { - dev_err(&pdev->dev, "Can't find acceleration device\n"); + pci_err(pdev, "Can't find acceleration device\n"); return PCI_ERS_RESULT_DISCONNECT; } - if (state == pci_channel_io_perm_failure) { - dev_err(&pdev->dev, "Can't recover from device error\n"); - return PCI_ERS_RESULT_DISCONNECT; - } + if (!adf_dev_started(accel_dev)) + return PCI_ERS_RESULT_CAN_RECOVER; set_bit(ADF_STATUS_RESTARTING, &accel_dev->status); if (accel_dev->hw_device->exit_arb) { dev_dbg(&pdev->dev, "Disabling arbitration\n"); accel_dev->hw_device->exit_arb(accel_dev); } - adf_error_notifier(accel_dev); - adf_pf2vf_notify_fatal_error(accel_dev); adf_dev_restarting_notify(accel_dev); - pci_clear_master(pdev); adf_dev_down(accel_dev); return PCI_ERS_RESULT_NEED_RESET; } +static pci_ers_result_t reset_done(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + int res; + + if (!accel_dev) { + pci_err(pdev, "Can't find acceleration device\n"); + return PCI_ERS_RESULT_DISCONNECT; + } + + if (!adf_devmgr_in_reset(accel_dev)) + goto reset_complete; + + pci_restore_state(pdev); + res = adf_dev_up(accel_dev, false); + if (res && res != -EALREADY) + return PCI_ERS_RESULT_DISCONNECT; + + adf_reenable_sriov(accel_dev); + adf_pf2vf_notify_restarted(accel_dev); + adf_dev_restarted_notify(accel_dev); + clear_bit(ADF_STATUS_RESTARTING, &accel_dev->status); + +reset_complete: + pci_info(pdev, "Device reset completed successfully\n"); + + return PCI_ERS_RESULT_RECOVERED; +} + +static pci_ers_result_t adf_error_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + pci_info(pdev, "Acceleration driver hardware error detected.\n"); + if (!accel_dev) { + pci_err(pdev, "Can't find acceleration device\n"); + return PCI_ERS_RESULT_DISCONNECT; + } + + if (state == pci_channel_io_perm_failure) { + pci_err(pdev, "Can't recover from device error\n"); + return PCI_ERS_RESULT_DISCONNECT; + } + + adf_error_notifier(accel_dev); + adf_pf2vf_notify_fatal_error(accel_dev); + + return reset_prepare(pdev); +} + /* reset dev data */ struct adf_reset_dev_data { int mode; @@ -72,10 +116,9 @@ void adf_reset_sbr(struct adf_accel_dev *accel_dev) parent = pdev; if (!pci_wait_for_pending_transaction(pdev)) - dev_info(&GET_DEV(accel_dev), - "Transaction still in progress. Proceeding\n"); + pci_info(pdev, "Transaction still in progress. Proceeding\n"); - dev_info(&GET_DEV(accel_dev), "Secondary bus reset\n"); + pci_info(pdev, "Secondary bus reset\n"); pci_read_config_word(parent, PCI_BRIDGE_CONTROL, &bridge_ctl); bridge_ctl |= PCI_BRIDGE_CTL_BUS_RESET; @@ -106,6 +149,13 @@ void adf_dev_restore(struct adf_accel_dev *accel_dev) } } +void adf_set_bme(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + + pci_set_master(pdev); +} + static void adf_device_sriov_worker(struct work_struct *work) { struct adf_sriov_dev_data *sriov_data = @@ -156,13 +206,14 @@ static int adf_dev_aer_schedule_reset(struct adf_accel_dev *accel_dev, struct adf_reset_dev_data *reset_data; if (!adf_dev_started(accel_dev) || - test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) + test_and_set_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) return 0; - set_bit(ADF_STATUS_RESTARTING, &accel_dev->status); reset_data = kzalloc_obj(*reset_data); - if (!reset_data) + if (!reset_data) { + clear_bit(ADF_STATUS_RESTARTING, &accel_dev->status); return -ENOMEM; + } reset_data->accel_dev = accel_dev; init_completion(&reset_data->compl); reset_data->mode = mode; @@ -190,38 +241,31 @@ static int adf_dev_aer_schedule_reset(struct adf_accel_dev *accel_dev, static pci_ers_result_t adf_slot_reset(struct pci_dev *pdev) { - struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); - int res = 0; - - if (!accel_dev) { - pr_err("QAT: Can't find acceleration device\n"); - return PCI_ERS_RESULT_DISCONNECT; - } - - if (!pdev->is_busmaster) - pci_set_master(pdev); - pci_restore_state(pdev); - res = adf_dev_up(accel_dev, false); - if (res && res != -EALREADY) - return PCI_ERS_RESULT_DISCONNECT; - - adf_reenable_sriov(accel_dev); - adf_pf2vf_notify_restarted(accel_dev); - adf_dev_restarted_notify(accel_dev); - clear_bit(ADF_STATUS_RESTARTING, &accel_dev->status); - return PCI_ERS_RESULT_RECOVERED; + return reset_done(pdev); } static void adf_resume(struct pci_dev *pdev) { - dev_info(&pdev->dev, "Acceleration driver reset completed\n"); - dev_info(&pdev->dev, "Device is up and running\n"); + pci_info(pdev, "Acceleration driver reset completed\n"); + pci_info(pdev, "Device is up and running\n"); +} + +static void adf_reset_prepare(struct pci_dev *pdev) +{ + reset_prepare(pdev); +} + +static void adf_reset_done(struct pci_dev *pdev) +{ + reset_done(pdev); } const struct pci_error_handlers adf_err_handler = { .error_detected = adf_error_detected, .slot_reset = adf_slot_reset, .resume = adf_resume, + .reset_prepare = adf_reset_prepare, + .reset_done = adf_reset_done, }; EXPORT_SYMBOL_GPL(adf_err_handler); diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg.c b/drivers/crypto/intel/qat/qat_common/adf_cfg.c index c202209f17d5..ea5d72d5090c 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg.c +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg.c @@ -103,16 +103,6 @@ static void adf_cfg_section_del_all(struct list_head *head); static void adf_cfg_section_del_all_except(struct list_head *head, const char *section_name); -void adf_cfg_del_all(struct adf_accel_dev *accel_dev) -{ - struct adf_cfg_device_data *dev_cfg_data = accel_dev->cfg; - - down_write(&dev_cfg_data->lock); - adf_cfg_section_del_all(&dev_cfg_data->sec_list); - up_write(&dev_cfg_data->lock); - clear_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); -} - void adf_cfg_del_all_except(struct adf_accel_dev *accel_dev, const char *section_name) { diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg.h b/drivers/crypto/intel/qat/qat_common/adf_cfg.h index 2afa6f0d15c5..108032b39242 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg.h +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg.h @@ -34,7 +34,6 @@ void adf_cfg_dev_remove(struct adf_accel_dev *accel_dev); void adf_cfg_dev_dbgfs_add(struct adf_accel_dev *accel_dev); void adf_cfg_dev_dbgfs_rm(struct adf_accel_dev *accel_dev); int adf_cfg_section_add(struct adf_accel_dev *accel_dev, const char *name); -void adf_cfg_del_all(struct adf_accel_dev *accel_dev); void adf_cfg_del_all_except(struct adf_accel_dev *accel_dev, const char *section_name); int adf_cfg_add_key_value_param(struct adf_accel_dev *accel_dev, diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h b/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h index 81e9e9d7eccd..d63f4dcccbb5 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h @@ -4,18 +4,11 @@ #define ADF_CFG_COMMON_H_ #include -#include #define ADF_CFG_MAX_STR_LEN 64 #define ADF_CFG_MAX_KEY_LEN_IN_BYTES ADF_CFG_MAX_STR_LEN #define ADF_CFG_MAX_VAL_LEN_IN_BYTES ADF_CFG_MAX_STR_LEN #define ADF_CFG_MAX_SECTION_LEN_IN_BYTES ADF_CFG_MAX_STR_LEN -#define ADF_CFG_BASE_DEC 10 -#define ADF_CFG_BASE_HEX 16 -#define ADF_CFG_ALL_DEVICES 0xFE -#define ADF_CFG_NO_DEVICE 0xFF -#define ADF_CFG_AFFINITY_WHATEVER 0xFF -#define MAX_DEVICE_NAME_SIZE 32 #define ADF_MAX_DEVICES (32 * 32) #define ADF_DEVS_ARRAY_SIZE BITS_TO_LONGS(ADF_MAX_DEVICES) @@ -51,29 +44,4 @@ enum adf_device_type { DEV_420XX, DEV_6XXX, }; - -struct adf_dev_status_info { - enum adf_device_type type; - __u32 accel_id; - __u32 instance_id; - __u8 num_ae; - __u8 num_accel; - __u8 num_logical_accel; - __u8 banks_per_accel; - __u8 state; - __u8 bus; - __u8 dev; - __u8 fun; - char name[MAX_DEVICE_NAME_SIZE]; -}; - -#define ADF_CTL_IOC_MAGIC 'a' -#define IOCTL_CONFIG_SYS_RESOURCE_PARAMETERS _IOW(ADF_CTL_IOC_MAGIC, 0, \ - struct adf_user_cfg_ctl_data) -#define IOCTL_STOP_ACCEL_DEV _IOW(ADF_CTL_IOC_MAGIC, 1, \ - struct adf_user_cfg_ctl_data) -#define IOCTL_START_ACCEL_DEV _IOW(ADF_CTL_IOC_MAGIC, 2, \ - struct adf_user_cfg_ctl_data) -#define IOCTL_STATUS_ACCEL_DEV _IOW(ADF_CTL_IOC_MAGIC, 3, __u32) -#define IOCTL_GET_NUM_DEVICES _IOW(ADF_CTL_IOC_MAGIC, 4, __s32) #endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c index 7d00bcb41ce7..1af6da8b263f 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c @@ -94,10 +94,9 @@ static int adf_service_mask_to_string(unsigned long mask, char *buf, size_t len) for_each_set_bit(bit, &mask, SVC_COUNT) { if (offset) offset += scnprintf(buf + offset, len - offset, - ADF_SERVICES_DELIMITER); - - offset += scnprintf(buf + offset, len - offset, "%s", - adf_cfg_services[bit]); + ADF_SERVICES_DELIMITER "%s", adf_cfg_services[bit]); + else + offset += scnprintf(buf, len, "%s", adf_cfg_services[bit]); } return 0; diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_user.h b/drivers/crypto/intel/qat/qat_common/adf_cfg_user.h deleted file mode 100644 index 421f4fb8b4dd..000000000000 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg_user.h +++ /dev/null @@ -1,38 +0,0 @@ -/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ -/* Copyright(c) 2014 - 2020 Intel Corporation */ -#ifndef ADF_CFG_USER_H_ -#define ADF_CFG_USER_H_ - -#include "adf_cfg_common.h" -#include "adf_cfg_strings.h" - -struct adf_user_cfg_key_val { - char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; - char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; - union { - struct adf_user_cfg_key_val *next; - __u64 padding3; - }; - enum adf_cfg_val_type type; -} __packed; - -struct adf_user_cfg_section { - char name[ADF_CFG_MAX_SECTION_LEN_IN_BYTES]; - union { - struct adf_user_cfg_key_val *params; - __u64 padding1; - }; - union { - struct adf_user_cfg_section *next; - __u64 padding3; - }; -} __packed; - -struct adf_user_cfg_ctl_data { - union { - struct adf_user_cfg_section *config_section; - __u64 padding; - }; - __u8 device_id; -} __packed; -#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_common_drv.h b/drivers/crypto/intel/qat/qat_common/adf_common_drv.h index 7b8b295ac459..762a0b5e774a 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/intel/qat/qat_common/adf_common_drv.h @@ -9,13 +9,6 @@ #include "icp_qat_fw_loader_handle.h" #include "icp_qat_hal.h" -#define ADF_MAJOR_VERSION 0 -#define ADF_MINOR_VERSION 6 -#define ADF_BUILD_VERSION 0 -#define ADF_DRV_VERSION __stringify(ADF_MAJOR_VERSION) "." \ - __stringify(ADF_MINOR_VERSION) "." \ - __stringify(ADF_BUILD_VERSION) - #define ADF_STATUS_RESTARTING 0 #define ADF_STATUS_STARTING 1 #define ADF_STATUS_CONFIGURED 2 @@ -68,10 +61,7 @@ int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev, void adf_devmgr_rm_dev(struct adf_accel_dev *accel_dev, struct adf_accel_dev *pf); struct list_head *adf_devmgr_get_head(void); -struct adf_accel_dev *adf_devmgr_get_dev_by_id(u32 id); struct adf_accel_dev *adf_devmgr_pci_to_accel_dev(struct pci_dev *pci_dev); -int adf_devmgr_verify_id(u32 id); -void adf_devmgr_get_num_dev(u32 *num); int adf_devmgr_in_reset(struct adf_accel_dev *accel_dev); int adf_dev_started(struct adf_accel_dev *accel_dev); int adf_dev_restarting_notify(struct adf_accel_dev *accel_dev); @@ -87,12 +77,12 @@ extern const struct pci_error_handlers adf_err_handler; void adf_reset_sbr(struct adf_accel_dev *accel_dev); void adf_reset_flr(struct adf_accel_dev *accel_dev); void adf_dev_restore(struct adf_accel_dev *accel_dev); +void adf_set_bme(struct adf_accel_dev *accel_dev); int adf_init_aer(void); void adf_exit_aer(void); int adf_init_arb(struct adf_accel_dev *accel_dev); void adf_exit_arb(struct adf_accel_dev *accel_dev); void adf_update_ring_arb(struct adf_etr_ring_data *ring); -int adf_disable_arb_thd(struct adf_accel_dev *accel_dev, u32 ae, u32 thr); int adf_dev_get(struct adf_accel_dev *accel_dev); void adf_dev_put(struct adf_accel_dev *accel_dev); @@ -121,6 +111,7 @@ void qat_comp_alg_callback(void *resp); int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev); void adf_isr_resource_free(struct adf_accel_dev *accel_dev); +void adf_isr_sync_ae_cluster(struct adf_accel_dev *accel_dev); int adf_vf_isr_resource_alloc(struct adf_accel_dev *accel_dev); void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev); @@ -194,6 +185,7 @@ int adf_sriov_configure(struct pci_dev *pdev, int numvfs); void adf_disable_sriov(struct adf_accel_dev *accel_dev); void adf_reenable_sriov(struct adf_accel_dev *accel_dev); void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask); +void adf_enable_all_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 num_vfs); void adf_disable_all_vf2pf_interrupts(struct adf_accel_dev *accel_dev); bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev); bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr); diff --git a/drivers/crypto/intel/qat/qat_common/adf_ctl_drv.c b/drivers/crypto/intel/qat/qat_common/adf_ctl_drv.c deleted file mode 100644 index c2e6f0cb7480..000000000000 --- a/drivers/crypto/intel/qat/qat_common/adf_ctl_drv.c +++ /dev/null @@ -1,466 +0,0 @@ -// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -/* Copyright(c) 2014 - 2020 Intel Corporation */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "adf_accel_devices.h" -#include "adf_common_drv.h" -#include "adf_cfg.h" -#include "adf_cfg_common.h" -#include "adf_cfg_user.h" - -#define ADF_CFG_MAX_SECTION 512 -#define ADF_CFG_MAX_KEY_VAL 256 - -#define DEVICE_NAME "qat_adf_ctl" - -static DEFINE_MUTEX(adf_ctl_lock); -static long adf_ctl_ioctl(struct file *fp, unsigned int cmd, unsigned long arg); - -static const struct file_operations adf_ctl_ops = { - .owner = THIS_MODULE, - .unlocked_ioctl = adf_ctl_ioctl, - .compat_ioctl = compat_ptr_ioctl, -}; - -static const struct class adf_ctl_class = { - .name = DEVICE_NAME, -}; - -struct adf_ctl_drv_info { - unsigned int major; - struct cdev drv_cdev; -}; - -static struct adf_ctl_drv_info adf_ctl_drv; - -static void adf_chr_drv_destroy(void) -{ - device_destroy(&adf_ctl_class, MKDEV(adf_ctl_drv.major, 0)); - cdev_del(&adf_ctl_drv.drv_cdev); - class_unregister(&adf_ctl_class); - unregister_chrdev_region(MKDEV(adf_ctl_drv.major, 0), 1); -} - -static int adf_chr_drv_create(void) -{ - dev_t dev_id; - struct device *drv_device; - int ret; - - if (alloc_chrdev_region(&dev_id, 0, 1, DEVICE_NAME)) { - pr_err("QAT: unable to allocate chrdev region\n"); - return -EFAULT; - } - - ret = class_register(&adf_ctl_class); - if (ret) - goto err_chrdev_unreg; - - adf_ctl_drv.major = MAJOR(dev_id); - cdev_init(&adf_ctl_drv.drv_cdev, &adf_ctl_ops); - if (cdev_add(&adf_ctl_drv.drv_cdev, dev_id, 1)) { - pr_err("QAT: cdev add failed\n"); - goto err_class_destr; - } - - drv_device = device_create(&adf_ctl_class, NULL, - MKDEV(adf_ctl_drv.major, 0), - NULL, DEVICE_NAME); - if (IS_ERR(drv_device)) { - pr_err("QAT: failed to create device\n"); - goto err_cdev_del; - } - return 0; -err_cdev_del: - cdev_del(&adf_ctl_drv.drv_cdev); -err_class_destr: - class_unregister(&adf_ctl_class); -err_chrdev_unreg: - unregister_chrdev_region(dev_id, 1); - return -EFAULT; -} - -static struct adf_user_cfg_ctl_data *adf_ctl_alloc_resources(unsigned long arg) -{ - struct adf_user_cfg_ctl_data *cfg_data; - - cfg_data = memdup_user((void __user *)arg, sizeof(*cfg_data)); - if (IS_ERR(cfg_data)) - pr_err("QAT: failed to copy from user cfg_data.\n"); - return cfg_data; -} - -static int adf_add_key_value_data(struct adf_accel_dev *accel_dev, - const char *section, - const struct adf_user_cfg_key_val *key_val) -{ - if (key_val->type == ADF_HEX) { - long *ptr = (long *)key_val->val; - long val = *ptr; - - if (adf_cfg_add_key_value_param(accel_dev, section, - key_val->key, (void *)val, - key_val->type)) { - dev_err(&GET_DEV(accel_dev), - "failed to add hex keyvalue.\n"); - return -EFAULT; - } - } else { - if (adf_cfg_add_key_value_param(accel_dev, section, - key_val->key, key_val->val, - key_val->type)) { - dev_err(&GET_DEV(accel_dev), - "failed to add keyvalue.\n"); - return -EFAULT; - } - } - return 0; -} - -static int adf_copy_key_value_data(struct adf_accel_dev *accel_dev, - struct adf_user_cfg_ctl_data *ctl_data) -{ - struct adf_user_cfg_key_val key_val; - struct adf_user_cfg_key_val *params_head; - struct adf_user_cfg_section section, *section_head; - int i, j; - - section_head = ctl_data->config_section; - - for (i = 0; section_head && i < ADF_CFG_MAX_SECTION; i++) { - if (copy_from_user(§ion, (void __user *)section_head, - sizeof(*section_head))) { - dev_err(&GET_DEV(accel_dev), - "failed to copy section info\n"); - goto out_err; - } - - if (adf_cfg_section_add(accel_dev, section.name)) { - dev_err(&GET_DEV(accel_dev), - "failed to add section.\n"); - goto out_err; - } - - params_head = section.params; - - for (j = 0; params_head && j < ADF_CFG_MAX_KEY_VAL; j++) { - if (copy_from_user(&key_val, (void __user *)params_head, - sizeof(key_val))) { - dev_err(&GET_DEV(accel_dev), - "Failed to copy keyvalue.\n"); - goto out_err; - } - if (adf_add_key_value_data(accel_dev, section.name, - &key_val)) { - goto out_err; - } - params_head = key_val.next; - } - section_head = section.next; - } - return 0; -out_err: - adf_cfg_del_all(accel_dev); - return -EFAULT; -} - -static int adf_ctl_ioctl_dev_config(struct file *fp, unsigned int cmd, - unsigned long arg) -{ - struct adf_user_cfg_ctl_data *ctl_data; - struct adf_accel_dev *accel_dev; - int ret = 0; - - ctl_data = adf_ctl_alloc_resources(arg); - if (IS_ERR(ctl_data)) - return PTR_ERR(ctl_data); - - accel_dev = adf_devmgr_get_dev_by_id(ctl_data->device_id); - if (!accel_dev) { - ret = -EFAULT; - goto out; - } - - if (adf_dev_started(accel_dev)) { - ret = -EFAULT; - goto out; - } - - if (adf_copy_key_value_data(accel_dev, ctl_data)) { - ret = -EFAULT; - goto out; - } - set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); -out: - kfree(ctl_data); - return ret; -} - -static int adf_ctl_is_device_in_use(int id) -{ - struct adf_accel_dev *dev; - - list_for_each_entry(dev, adf_devmgr_get_head(), list) { - if (id == dev->accel_id || id == ADF_CFG_ALL_DEVICES) { - if (adf_devmgr_in_reset(dev) || adf_dev_in_use(dev)) { - dev_info(&GET_DEV(dev), - "device qat_dev%d is busy\n", - dev->accel_id); - return -EBUSY; - } - } - } - return 0; -} - -static void adf_ctl_stop_devices(u32 id) -{ - struct adf_accel_dev *accel_dev; - - list_for_each_entry(accel_dev, adf_devmgr_get_head(), list) { - if (id == accel_dev->accel_id || id == ADF_CFG_ALL_DEVICES) { - if (!adf_dev_started(accel_dev)) - continue; - - /* First stop all VFs */ - if (!accel_dev->is_vf) - continue; - - adf_dev_down(accel_dev); - } - } - - list_for_each_entry(accel_dev, adf_devmgr_get_head(), list) { - if (id == accel_dev->accel_id || id == ADF_CFG_ALL_DEVICES) { - if (!adf_dev_started(accel_dev)) - continue; - - adf_dev_down(accel_dev); - } - } -} - -static int adf_ctl_ioctl_dev_stop(struct file *fp, unsigned int cmd, - unsigned long arg) -{ - int ret; - struct adf_user_cfg_ctl_data *ctl_data; - - ctl_data = adf_ctl_alloc_resources(arg); - if (IS_ERR(ctl_data)) - return PTR_ERR(ctl_data); - - if (adf_devmgr_verify_id(ctl_data->device_id)) { - pr_err("QAT: Device %d not found\n", ctl_data->device_id); - ret = -ENODEV; - goto out; - } - - ret = adf_ctl_is_device_in_use(ctl_data->device_id); - if (ret) - goto out; - - if (ctl_data->device_id == ADF_CFG_ALL_DEVICES) - pr_info("QAT: Stopping all acceleration devices.\n"); - else - pr_info("QAT: Stopping acceleration device qat_dev%d.\n", - ctl_data->device_id); - - adf_ctl_stop_devices(ctl_data->device_id); - -out: - kfree(ctl_data); - return ret; -} - -static int adf_ctl_ioctl_dev_start(struct file *fp, unsigned int cmd, - unsigned long arg) -{ - int ret; - struct adf_user_cfg_ctl_data *ctl_data; - struct adf_accel_dev *accel_dev; - - ctl_data = adf_ctl_alloc_resources(arg); - if (IS_ERR(ctl_data)) - return PTR_ERR(ctl_data); - - ret = -ENODEV; - accel_dev = adf_devmgr_get_dev_by_id(ctl_data->device_id); - if (!accel_dev) - goto out; - - dev_info(&GET_DEV(accel_dev), - "Starting acceleration device qat_dev%d.\n", - ctl_data->device_id); - - ret = adf_dev_up(accel_dev, false); - - if (ret) { - dev_err(&GET_DEV(accel_dev), "Failed to start qat_dev%d\n", - ctl_data->device_id); - adf_dev_down(accel_dev); - } -out: - kfree(ctl_data); - return ret; -} - -static int adf_ctl_ioctl_get_num_devices(struct file *fp, unsigned int cmd, - unsigned long arg) -{ - u32 num_devices = 0; - - adf_devmgr_get_num_dev(&num_devices); - if (copy_to_user((void __user *)arg, &num_devices, sizeof(num_devices))) - return -EFAULT; - - return 0; -} - -static int adf_ctl_ioctl_get_status(struct file *fp, unsigned int cmd, - unsigned long arg) -{ - struct adf_hw_device_data *hw_data; - struct adf_dev_status_info dev_info; - struct adf_accel_dev *accel_dev; - - if (copy_from_user(&dev_info, (void __user *)arg, - sizeof(struct adf_dev_status_info))) { - pr_err("QAT: failed to copy from user.\n"); - return -EFAULT; - } - - accel_dev = adf_devmgr_get_dev_by_id(dev_info.accel_id); - if (!accel_dev) - return -ENODEV; - - hw_data = accel_dev->hw_device; - dev_info.state = adf_dev_started(accel_dev) ? DEV_UP : DEV_DOWN; - dev_info.num_ae = hw_data->get_num_aes(hw_data); - dev_info.num_accel = hw_data->get_num_accels(hw_data); - dev_info.num_logical_accel = hw_data->num_logical_accel; - dev_info.banks_per_accel = hw_data->num_banks - / hw_data->num_logical_accel; - strscpy(dev_info.name, hw_data->dev_class->name, sizeof(dev_info.name)); - dev_info.instance_id = hw_data->instance_id; - dev_info.type = hw_data->dev_class->type; - dev_info.bus = accel_to_pci_dev(accel_dev)->bus->number; - dev_info.dev = PCI_SLOT(accel_to_pci_dev(accel_dev)->devfn); - dev_info.fun = PCI_FUNC(accel_to_pci_dev(accel_dev)->devfn); - - if (copy_to_user((void __user *)arg, &dev_info, - sizeof(struct adf_dev_status_info))) { - dev_err(&GET_DEV(accel_dev), "failed to copy status.\n"); - return -EFAULT; - } - return 0; -} - -static long adf_ctl_ioctl(struct file *fp, unsigned int cmd, unsigned long arg) -{ - int ret; - - if (mutex_lock_interruptible(&adf_ctl_lock)) - return -EFAULT; - - switch (cmd) { - case IOCTL_CONFIG_SYS_RESOURCE_PARAMETERS: - ret = adf_ctl_ioctl_dev_config(fp, cmd, arg); - break; - - case IOCTL_STOP_ACCEL_DEV: - ret = adf_ctl_ioctl_dev_stop(fp, cmd, arg); - break; - - case IOCTL_START_ACCEL_DEV: - ret = adf_ctl_ioctl_dev_start(fp, cmd, arg); - break; - - case IOCTL_GET_NUM_DEVICES: - ret = adf_ctl_ioctl_get_num_devices(fp, cmd, arg); - break; - - case IOCTL_STATUS_ACCEL_DEV: - ret = adf_ctl_ioctl_get_status(fp, cmd, arg); - break; - default: - pr_err_ratelimited("QAT: Invalid ioctl %d\n", cmd); - ret = -EFAULT; - break; - } - mutex_unlock(&adf_ctl_lock); - return ret; -} - -static int __init adf_register_ctl_device_driver(void) -{ - if (adf_chr_drv_create()) - goto err_chr_dev; - - if (adf_init_misc_wq()) - goto err_misc_wq; - - if (adf_init_aer()) - goto err_aer; - - if (adf_init_pf_wq()) - goto err_pf_wq; - - if (adf_init_vf_wq()) - goto err_vf_wq; - - if (qat_crypto_register()) - goto err_crypto_register; - - if (qat_compression_register()) - goto err_compression_register; - - return 0; - -err_compression_register: - qat_crypto_unregister(); -err_crypto_register: - adf_exit_vf_wq(); -err_vf_wq: - adf_exit_pf_wq(); -err_pf_wq: - adf_exit_aer(); -err_aer: - adf_exit_misc_wq(); -err_misc_wq: - adf_chr_drv_destroy(); -err_chr_dev: - mutex_destroy(&adf_ctl_lock); - return -EFAULT; -} - -static void __exit adf_unregister_ctl_device_driver(void) -{ - adf_chr_drv_destroy(); - adf_exit_misc_wq(); - adf_exit_aer(); - adf_exit_vf_wq(); - adf_exit_pf_wq(); - qat_crypto_unregister(); - qat_compression_unregister(); - adf_clean_vf_map(false); - mutex_destroy(&adf_ctl_lock); -} - -module_init(adf_register_ctl_device_driver); -module_exit(adf_unregister_ctl_device_driver); -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_AUTHOR("Intel"); -MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); -MODULE_ALIAS_CRYPTO("intel_qat"); -MODULE_VERSION(ADF_DRV_VERSION); -MODULE_IMPORT_NS("CRYPTO_INTERNAL"); diff --git a/drivers/crypto/intel/qat/qat_common/adf_dev_mgr.c b/drivers/crypto/intel/qat/qat_common/adf_dev_mgr.c index e050de16ab5d..161863841a52 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_dev_mgr.c +++ b/drivers/crypto/intel/qat/qat_common/adf_dev_mgr.c @@ -45,19 +45,6 @@ static struct vf_id_map *adf_find_vf(u32 bdf) return NULL; } -static int adf_get_vf_real_id(u32 fake) -{ - struct list_head *itr; - - list_for_each(itr, &vfs_table) { - struct vf_id_map *ptr = - list_entry(itr, struct vf_id_map, list); - if (ptr->fake_id == fake) - return ptr->id; - } - return -1; -} - /** * adf_clean_vf_map() - Cleans VF id mappings * @vf: flag indicating whether mappings is cleaned @@ -304,63 +291,6 @@ struct adf_accel_dev *adf_devmgr_pci_to_accel_dev(struct pci_dev *pci_dev) } EXPORT_SYMBOL_GPL(adf_devmgr_pci_to_accel_dev); -struct adf_accel_dev *adf_devmgr_get_dev_by_id(u32 id) -{ - struct list_head *itr; - int real_id; - - mutex_lock(&table_lock); - real_id = adf_get_vf_real_id(id); - if (real_id < 0) - goto unlock; - - id = real_id; - - list_for_each(itr, &accel_table) { - struct adf_accel_dev *ptr = - list_entry(itr, struct adf_accel_dev, list); - if (ptr->accel_id == id) { - mutex_unlock(&table_lock); - return ptr; - } - } -unlock: - mutex_unlock(&table_lock); - return NULL; -} - -int adf_devmgr_verify_id(u32 id) -{ - if (id == ADF_CFG_ALL_DEVICES) - return 0; - - if (adf_devmgr_get_dev_by_id(id)) - return 0; - - return -ENODEV; -} - -static int adf_get_num_dettached_vfs(void) -{ - struct list_head *itr; - int vfs = 0; - - mutex_lock(&table_lock); - list_for_each(itr, &vfs_table) { - struct vf_id_map *ptr = - list_entry(itr, struct vf_id_map, list); - if (ptr->bdf != ~0 && !ptr->attached) - vfs++; - } - mutex_unlock(&table_lock); - return vfs; -} - -void adf_devmgr_get_num_dev(u32 *num) -{ - *num = num_devices - adf_get_num_dettached_vfs(); -} - /** * adf_dev_in_use() - Check whether accel_dev is currently in use * @accel_dev: Pointer to acceleration device. diff --git a/drivers/crypto/intel/qat/qat_common/adf_heartbeat_inject.c b/drivers/crypto/intel/qat/qat_common/adf_heartbeat_inject.c index a3b474bdef6c..023c5f1e78b0 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_heartbeat_inject.c +++ b/drivers/crypto/intel/qat/qat_common/adf_heartbeat_inject.c @@ -64,10 +64,8 @@ int adf_heartbeat_inject_error(struct adf_accel_dev *accel_dev) if (ret) return ret; - /* Configure worker threads to stop processing any packet */ - ret = adf_disable_arb_thd(accel_dev, rand_ae, rand_thr); - if (ret) - return ret; + /* Disable arbiter to stop processing any packet */ + hw_device->exit_arb(accel_dev); /* Change HB counters memory to simulate a hang */ adf_set_hb_counters_fail(accel_dev, rand_ae, rand_thr); diff --git a/drivers/crypto/intel/qat/qat_common/adf_hw_arbiter.c b/drivers/crypto/intel/qat/qat_common/adf_hw_arbiter.c index f93d9cca70ce..dd9a31c20bc9 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_hw_arbiter.c +++ b/drivers/crypto/intel/qat/qat_common/adf_hw_arbiter.c @@ -99,28 +99,3 @@ void adf_exit_arb(struct adf_accel_dev *accel_dev) csr_ops->write_csr_ring_srv_arb_en(csr, i, 0); } EXPORT_SYMBOL_GPL(adf_exit_arb); - -int adf_disable_arb_thd(struct adf_accel_dev *accel_dev, u32 ae, u32 thr) -{ - void __iomem *csr = accel_dev->transport->banks[0].csr_addr; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - const u32 *thd_2_arb_cfg; - struct arb_info info; - u32 ae_thr_map; - - if (ADF_AE_STRAND0_THREAD == thr || ADF_AE_STRAND1_THREAD == thr) - thr = ADF_AE_ADMIN_THREAD; - - hw_data->get_arb_info(&info); - thd_2_arb_cfg = hw_data->get_arb_mapping(accel_dev); - if (!thd_2_arb_cfg) - return -EFAULT; - - /* Disable scheduling for this particular AE and thread */ - ae_thr_map = *(thd_2_arb_cfg + ae); - ae_thr_map &= ~(GENMASK(3, 0) << (thr * BIT(2))); - - WRITE_CSR_ARB_WT2SAM(csr, info.arb_offset, info.wt2sam_offset, ae, - ae_thr_map); - return 0; -} diff --git a/drivers/crypto/intel/qat/qat_common/adf_init.c b/drivers/crypto/intel/qat/qat_common/adf_init.c index f8088388cf12..3e39c53814de 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_init.c +++ b/drivers/crypto/intel/qat/qat_common/adf_init.c @@ -10,6 +10,7 @@ #include "adf_dbgfs.h" #include "adf_heartbeat.h" #include "adf_rl.h" +#include "adf_kpt.h" #include "adf_sysfs_anti_rb.h" #include "adf_sysfs_ras_counters.h" #include "adf_telemetry.h" @@ -74,6 +75,8 @@ static int adf_dev_init(struct adf_accel_dev *accel_dev) return -EFAULT; } + adf_set_bme(accel_dev); + if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status) && !accel_dev->is_vf) { dev_err(&GET_DEV(accel_dev), "Device not configured\n"); @@ -153,15 +156,18 @@ static int adf_dev_init(struct adf_accel_dev *accel_dev) * This is to facilitate any ordering dependencies between services * prior to starting any of the accelerators. */ + mutex_lock(&service_lock); list_for_each_entry(service, &service_table, list) { if (service->event_hld(accel_dev, ADF_EVENT_INIT)) { dev_err(&GET_DEV(accel_dev), "Failed to initialise service %s\n", service->name); + mutex_unlock(&service_lock); return -EFAULT; } set_bit(accel_dev->accel_id, service->init_status); } + mutex_unlock(&service_lock); return 0; } @@ -214,6 +220,13 @@ static int adf_dev_start(struct adf_accel_dev *accel_dev) return -EFAULT; } + /* Enable Key Protection Technology (KPT) */ + ret = adf_enable_kpt(accel_dev); + if (ret) { + dev_err(&GET_DEV(accel_dev), "Failed to enable KPT\n"); + return ret; + } + if (hw_data->start_timer) { ret = hw_data->start_timer(accel_dev); if (ret) { @@ -231,15 +244,18 @@ static int adf_dev_start(struct adf_accel_dev *accel_dev) if (ret && ret != -EOPNOTSUPP) return ret; + mutex_lock(&service_lock); list_for_each_entry(service, &service_table, list) { if (service->event_hld(accel_dev, ADF_EVENT_START)) { dev_err(&GET_DEV(accel_dev), "Failed to start service %s\n", service->name); + mutex_unlock(&service_lock); return -EFAULT; } set_bit(accel_dev->accel_id, service->start_status); } + mutex_unlock(&service_lock); clear_bit(ADF_STATUS_STARTING, &accel_dev->status); set_bit(ADF_STATUS_STARTED, &accel_dev->status); @@ -313,6 +329,7 @@ static void adf_dev_stop(struct adf_accel_dev *accel_dev) qat_comp_algs_unregister(hw_data->accel_capabilities_ext_mask); clear_bit(ADF_STATUS_COMP_ALGS_REGISTERED, &accel_dev->status); + mutex_lock(&service_lock); list_for_each_entry(service, &service_table, list) { if (!test_bit(accel_dev->accel_id, service->start_status)) continue; @@ -324,6 +341,7 @@ static void adf_dev_stop(struct adf_accel_dev *accel_dev) clear_bit(accel_dev->accel_id, service->start_status); } } + mutex_unlock(&service_lock); if (hw_data->stop_timer) hw_data->stop_timer(accel_dev); @@ -373,6 +391,7 @@ static void adf_dev_shutdown(struct adf_accel_dev *accel_dev) &accel_dev->status); } + mutex_lock(&service_lock); list_for_each_entry(service, &service_table, list) { if (!test_bit(accel_dev->accel_id, service->init_status)) continue; @@ -383,6 +402,7 @@ static void adf_dev_shutdown(struct adf_accel_dev *accel_dev) else clear_bit(accel_dev->accel_id, service->init_status); } + mutex_unlock(&service_lock); adf_rl_exit(accel_dev); @@ -417,12 +437,14 @@ int adf_dev_restarting_notify(struct adf_accel_dev *accel_dev) { struct service_hndl *service; + mutex_lock(&service_lock); list_for_each_entry(service, &service_table, list) { if (service->event_hld(accel_dev, ADF_EVENT_RESTARTING)) dev_err(&GET_DEV(accel_dev), "Failed to restart service %s.\n", service->name); } + mutex_unlock(&service_lock); return 0; } @@ -430,12 +452,14 @@ int adf_dev_restarted_notify(struct adf_accel_dev *accel_dev) { struct service_hndl *service; + mutex_lock(&service_lock); list_for_each_entry(service, &service_table, list) { if (service->event_hld(accel_dev, ADF_EVENT_RESTARTED)) dev_err(&GET_DEV(accel_dev), "Failed to restart service %s.\n", service->name); } + mutex_unlock(&service_lock); return 0; } @@ -443,12 +467,14 @@ void adf_error_notifier(struct adf_accel_dev *accel_dev) { struct service_hndl *service; + mutex_lock(&service_lock); list_for_each_entry(service, &service_table, list) { if (service->event_hld(accel_dev, ADF_EVENT_FATAL_ERROR)) dev_err(&GET_DEV(accel_dev), "Failed to send error event to %s.\n", service->name); } + mutex_unlock(&service_lock); } int adf_dev_down(struct adf_accel_dev *accel_dev) diff --git a/drivers/crypto/intel/qat/qat_common/adf_isr.c b/drivers/crypto/intel/qat/qat_common/adf_isr.c index 4639d7fd93e6..159e91a50106 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_isr.c +++ b/drivers/crypto/intel/qat/qat_common/adf_isr.c @@ -62,6 +62,23 @@ void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask) unsigned long flags; spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags); + if (!READ_ONCE(accel_dev->pf.vf2pf_disabled)) + GET_PFVF_OPS(accel_dev)->enable_vf2pf_interrupts(pmisc_addr, vf_mask); + spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags); +} + +void adf_enable_all_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 num_vfs) +{ + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + unsigned long flags; + u32 vf_mask; + + vf_mask = BIT_ULL(num_vfs) - 1; + if (!vf_mask) + return; + + spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags); + WRITE_ONCE(accel_dev->pf.vf2pf_disabled, false); GET_PFVF_OPS(accel_dev)->enable_vf2pf_interrupts(pmisc_addr, vf_mask); spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags); } @@ -72,6 +89,7 @@ void adf_disable_all_vf2pf_interrupts(struct adf_accel_dev *accel_dev) unsigned long flags; spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags); + WRITE_ONCE(accel_dev->pf.vf2pf_disabled, true); GET_PFVF_OPS(accel_dev)->disable_all_vf2pf_interrupts(pmisc_addr); spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags); } @@ -174,6 +192,27 @@ static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr) return IRQ_NONE; } +void adf_isr_sync_ae_cluster(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + u32 num_entries = pci_dev_info->msix_entries.num_entries; + struct adf_irq *irqs = pci_dev_info->msix_entries.irqs; + u32 irq_idx; + int irq; + + if (!test_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status) || !irqs) + return; + + irq_idx = num_entries > 1 ? hw_data->num_banks : 0; + if (irq_idx >= num_entries || !irqs[irq_idx].enabled) + return; + + irq = pci_irq_vector(pci_dev_info->pci_dev, hw_data->num_banks); + if (irq > 0) + synchronize_irq(irq); +} + static void adf_free_irqs(struct adf_accel_dev *accel_dev) { struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; diff --git a/drivers/crypto/intel/qat/qat_common/adf_kpt.c b/drivers/crypto/intel/qat/qat_common/adf_kpt.c new file mode 100644 index 000000000000..a0430141ea0e --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_kpt.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2026 Intel Corporation */ +#include + +#include "adf_admin.h" +#include "adf_cfg_services.h" +#include "adf_common_drv.h" +#include "adf_kpt.h" + +static bool adf_kpt_supported(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + + return hw_data->accel_capabilities_mask & ICP_ACCEL_CAPABILITIES_KPT; +} + +int adf_enable_kpt(struct adf_accel_dev *accel_dev) +{ + struct adf_kpt_interface_data *user_data = GET_KPT_USER_DATA(accel_dev); + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + dma_addr_t paddr; + void *vaddr; + int ret; + int svc; + + /* Return 0 if KPT is not supported by the hardware */ + if (!adf_kpt_supported(accel_dev)) + return 0; + + if (!user_data->enable) { + /* Disable KPT capability if user has not enabled it */ + hw_data->accel_capabilities_mask &= ~ICP_ACCEL_CAPABILITIES_KPT; + return 0; + } + + svc = adf_get_service_enabled(accel_dev); + if (svc < 0) + return svc; + + if (svc != SVC_ASYM) { + dev_err(&GET_DEV(accel_dev), + "KPT can only be enabled when service is configured as 'asym'\n"); + return -EINVAL; + } + + vaddr = dma_alloc_coherent(&GET_DEV(accel_dev), PAGE_SIZE, &paddr, + GFP_KERNEL); + if (!vaddr) + return -ENOMEM; + + ret = adf_send_admin_kpt_init(accel_dev, vaddr, PAGE_SIZE, paddr); + + dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE, vaddr, paddr); + + return ret; +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_kpt.h b/drivers/crypto/intel/qat/qat_common/adf_kpt.h new file mode 100644 index 000000000000..17d07d24a319 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_kpt.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2026 Intel Corporation */ +#ifndef ADF_KPT_H_ +#define ADF_KPT_H_ + +#include + +#define GET_KPT_CFG_DATA(accel_dev) (&(accel_dev)->hw_device->kpt_data) +#define GET_KPT_USER_DATA(accel_dev) (&(accel_dev)->hw_device->kpt_data.user_input) + +struct adf_accel_dev; + +struct adf_kpt_interface_data { + bool enable; + bool swk_shared; + unsigned int swk_cnt_per_fn; + unsigned int swk_cnt_per_pasid; + unsigned int swk_max_ttl; +}; + +struct adf_kpt_hw_data { + unsigned int max_swk_cnt_per_fn_pasid; + unsigned int max_swk_ttl; + struct adf_kpt_interface_data user_input; +}; + +int adf_enable_kpt(struct adf_accel_dev *accel_dev); + +#endif /* ADF_KPT_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_module.c b/drivers/crypto/intel/qat/qat_common/adf_module.c new file mode 100644 index 000000000000..30069feec3c1 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_module.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2020 Intel Corporation */ + +#include +#include +#include + +#include "adf_common_drv.h" + +static int __init adf_register_module(void) +{ + if (adf_init_misc_wq()) + goto err_misc_wq; + + if (adf_init_aer()) + goto err_aer; + + if (adf_init_pf_wq()) + goto err_pf_wq; + + if (adf_init_vf_wq()) + goto err_vf_wq; + + if (qat_crypto_register()) + goto err_crypto_register; + + if (qat_compression_register()) + goto err_compression_register; + + return 0; + +err_compression_register: + qat_crypto_unregister(); +err_crypto_register: + adf_exit_vf_wq(); +err_vf_wq: + adf_exit_pf_wq(); +err_pf_wq: + adf_exit_aer(); +err_aer: + adf_exit_misc_wq(); +err_misc_wq: + return -EFAULT; +} + +static void __exit adf_unregister_module(void) +{ + adf_exit_misc_wq(); + adf_exit_aer(); + adf_exit_vf_wq(); + adf_exit_pf_wq(); + qat_crypto_unregister(); + qat_compression_unregister(); + adf_clean_vf_map(false); +} + +module_init(adf_register_module); +module_exit(adf_unregister_module); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Intel"); +MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); +MODULE_ALIAS_CRYPTO("intel_qat"); +MODULE_IMPORT_NS("CRYPTO_INTERNAL"); diff --git a/drivers/crypto/intel/qat/qat_common/adf_sriov.c b/drivers/crypto/intel/qat/qat_common/adf_sriov.c index 8bf0fe1fcb4d..f45ca2eecc00 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/intel/qat/qat_common/adf_sriov.c @@ -26,6 +26,9 @@ static void adf_iov_send_resp(struct work_struct *work) u32 vf_nr = vf_info->vf_nr; bool ret; + if (READ_ONCE(accel_dev->pf.vf2pf_disabled)) + goto out; + mutex_lock(&vf_info->pfvf_mig_lock); ret = adf_recv_and_handle_vf2pf_msg(accel_dev, vf_nr); if (ret) @@ -33,13 +36,18 @@ static void adf_iov_send_resp(struct work_struct *work) adf_enable_vf2pf_interrupts(accel_dev, 1 << vf_nr); mutex_unlock(&vf_info->pfvf_mig_lock); +out: kfree(pf2vf_resp); } void adf_schedule_vf2pf_handler(struct adf_accel_vf_info *vf_info) { + struct adf_accel_dev *accel_dev = vf_info->accel_dev; struct adf_pf2vf_resp *pf2vf_resp; + if (READ_ONCE(accel_dev->pf.vf2pf_disabled)) + return; + pf2vf_resp = kzalloc_obj(*pf2vf_resp, GFP_ATOMIC); if (!pf2vf_resp) return; @@ -49,6 +57,12 @@ void adf_schedule_vf2pf_handler(struct adf_accel_vf_info *vf_info) queue_work(pf2vf_resp_wq, &pf2vf_resp->pf2vf_resp_work); } +static void adf_flush_pf2vf_resp_wq(void) +{ + if (pf2vf_resp_wq) + flush_workqueue(pf2vf_resp_wq); +} + static int adf_enable_sriov(struct adf_accel_dev *accel_dev) { struct pci_dev *pdev = accel_to_pci_dev(accel_dev); @@ -75,7 +89,11 @@ static int adf_enable_sriov(struct adf_accel_dev *accel_dev) hw_data->configure_iov_threads(accel_dev, true); /* Enable VF to PF interrupts for all VFs */ - adf_enable_vf2pf_interrupts(accel_dev, BIT_ULL(totalvfs) - 1); + adf_enable_all_vf2pf_interrupts(accel_dev, totalvfs); + + /* Do not enable SR-IOV if already enabled */ + if (pci_num_vf(pdev)) + return 0; /* * Due to the hardware design, when SR-IOV and the ring arbiter @@ -222,7 +240,7 @@ void adf_reenable_sriov(struct adf_accel_dev *accel_dev) if (adf_add_sriov_configuration(accel_dev)) return; - dev_dbg(&pdev->dev, "Re-enabling SRIOV\n"); + pci_dbg(pdev, "Re-enabling SRIOV\n"); adf_enable_sriov(accel_dev); } @@ -246,10 +264,18 @@ void adf_disable_sriov(struct adf_accel_dev *accel_dev) adf_pf2vf_notify_restarting(accel_dev); adf_pf2vf_wait_for_restarting_complete(accel_dev); - pci_disable_sriov(accel_to_pci_dev(accel_dev)); + /* + * When the device is restarting, preserve VF PCI devices across + * the reset by skipping pci_disable_sriov(). VFs are notified to + * quiesce regardless so the PF can safely shut down. + */ + if (!test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) + pci_disable_sriov(accel_to_pci_dev(accel_dev)); - /* Disable VF to PF interrupts */ + /* Block VF2PF work and disable VF to PF interrupts */ adf_disable_all_vf2pf_interrupts(accel_dev); + adf_isr_sync_ae_cluster(accel_dev); + adf_flush_pf2vf_resp_wq(); /* Clear Valid bits in AE Thread to PCIe Function Mapping */ if (hw_data->configure_iov_threads) diff --git a/drivers/crypto/intel/qat/qat_common/adf_sysfs_kpt.c b/drivers/crypto/intel/qat/qat_common/adf_sysfs_kpt.c new file mode 100644 index 000000000000..1f733ae80bdf --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_sysfs_kpt.c @@ -0,0 +1,296 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2026 Intel Corporation */ +#include +#include + +#include "adf_cfg.h" +#include "adf_cfg_services.h" +#include "adf_common_drv.h" +#include "adf_kpt.h" +#include "adf_sysfs_kpt.h" + +static ssize_t enable_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct adf_kpt_interface_data *user_data; + struct adf_accel_dev *accel_dev; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + user_data = GET_KPT_USER_DATA(accel_dev); + + return sysfs_emit(buf, "%d\n", user_data->enable); +} + +static ssize_t enable_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct adf_kpt_interface_data *user_data; + struct adf_hw_device_data *hw_data; + struct adf_accel_dev *accel_dev; + bool enable; + int ret; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + if (adf_dev_started(accel_dev)) { + dev_info(dev, "Device qat_dev%d must be down before enabling KPT\n", + accel_dev->accel_id); + return -EINVAL; + } + + if (adf_get_service_enabled(accel_dev) != SVC_ASYM) { + dev_info(dev, "KPT can only be enabled when the asymmetric service is enabled\n"); + return -EINVAL; + } + + hw_data = GET_HW_DATA(accel_dev); + + /* + * Restore the KPT capability bit in the device's capabilities mask + * before processing user input, as the bit may have been cleared if + * KPT was previously disabled by the user. + */ + hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); + if (!hw_data->accel_capabilities_mask) + return -EINVAL; + + ret = kstrtobool(buf, &enable); + if (ret) + return ret; + + user_data = GET_KPT_USER_DATA(accel_dev); + user_data->enable = enable; + + return count; +} +static DEVICE_ATTR_RW(enable); + +static ssize_t swk_shared_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct adf_kpt_interface_data *user_data; + struct adf_accel_dev *accel_dev; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + user_data = GET_KPT_USER_DATA(accel_dev); + + return sysfs_emit(buf, "%d\n", user_data->swk_shared); +} + +static ssize_t swk_shared_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct adf_kpt_interface_data *user_data; + struct adf_accel_dev *accel_dev; + bool swk_shared; + int ret; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + if (adf_dev_started(accel_dev)) { + dev_info(dev, "Device qat_dev%d must be down before setting swk_shared\n", + accel_dev->accel_id); + return -EINVAL; + } + + ret = kstrtobool(buf, &swk_shared); + if (ret) + return ret; + + user_data = GET_KPT_USER_DATA(accel_dev); + user_data->swk_shared = swk_shared; + + return count; +} +static DEVICE_ATTR_RW(swk_shared); + +static ssize_t swk_max_ttl_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct adf_kpt_interface_data *user_data; + struct adf_accel_dev *accel_dev; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + user_data = GET_KPT_USER_DATA(accel_dev); + + return sysfs_emit(buf, "%u\n", user_data->swk_max_ttl); +} + +static ssize_t swk_max_ttl_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct adf_kpt_hw_data *kpt_data; + struct adf_accel_dev *accel_dev; + unsigned int swk_max_ttl; + int ret; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + if (adf_dev_started(accel_dev)) { + dev_info(dev, "Device qat_dev%d must be down before setting swk_max_ttl\n", + accel_dev->accel_id); + return -EINVAL; + } + + ret = kstrtouint(buf, 10, &swk_max_ttl); + if (ret) + return ret; + + kpt_data = GET_KPT_CFG_DATA(accel_dev); + + if (swk_max_ttl > kpt_data->max_swk_ttl) { + dev_info(dev, "Configuration value is out of range (%u - %u)\n", + 0, kpt_data->max_swk_ttl); + return -EINVAL; + } + + kpt_data->user_input.swk_max_ttl = swk_max_ttl; + + return count; +} +static DEVICE_ATTR_RW(swk_max_ttl); + +static ssize_t swk_cnt_per_fn_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct adf_kpt_interface_data *user_data; + struct adf_accel_dev *accel_dev; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + user_data = GET_KPT_USER_DATA(accel_dev); + + return sysfs_emit(buf, "%u\n", user_data->swk_cnt_per_fn); +} + +static ssize_t swk_cnt_per_fn_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct adf_kpt_hw_data *kpt_data; + struct adf_accel_dev *accel_dev; + unsigned int swk_cnt_per_fn; + int ret; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + if (adf_dev_started(accel_dev)) { + dev_info(dev, "Device qat_dev%d must be down before setting swk_cnt_per_fn\n", + accel_dev->accel_id); + return -EINVAL; + } + + ret = kstrtouint(buf, 10, &swk_cnt_per_fn); + if (ret) + return ret; + + kpt_data = GET_KPT_CFG_DATA(accel_dev); + + if (swk_cnt_per_fn > kpt_data->max_swk_cnt_per_fn_pasid) { + dev_info(dev, "swk_cnt_per_fn: value out of range (0 - %u)\n", + kpt_data->max_swk_cnt_per_fn_pasid); + return -EINVAL; + } + + kpt_data->user_input.swk_cnt_per_fn = swk_cnt_per_fn; + + return count; +} +static DEVICE_ATTR_RW(swk_cnt_per_fn); + +static ssize_t swk_cnt_per_pasid_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct adf_kpt_interface_data *user_data; + struct adf_accel_dev *accel_dev; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + user_data = GET_KPT_USER_DATA(accel_dev); + + return sysfs_emit(buf, "%u\n", user_data->swk_cnt_per_pasid); +} + +static ssize_t swk_cnt_per_pasid_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct adf_kpt_hw_data *kpt_data; + struct adf_accel_dev *accel_dev; + unsigned int swk_cnt_per_pasid; + int ret; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + if (adf_dev_started(accel_dev)) { + dev_info(dev, "Device qat_dev%d must be down before setting swk_cnt_per_pasid\n", + accel_dev->accel_id); + return -EINVAL; + } + + ret = kstrtouint(buf, 10, &swk_cnt_per_pasid); + if (ret) + return ret; + + kpt_data = GET_KPT_CFG_DATA(accel_dev); + + if (swk_cnt_per_pasid > kpt_data->max_swk_cnt_per_fn_pasid) { + dev_info(dev, "swk_cnt_per_pasid: value out of range (0 - %u)\n", + kpt_data->max_swk_cnt_per_fn_pasid); + return -EINVAL; + } + + kpt_data->user_input.swk_cnt_per_pasid = swk_cnt_per_pasid; + + return count; +} +static DEVICE_ATTR_RW(swk_cnt_per_pasid); + +static struct attribute *qat_kpt_attrs[] = { + &dev_attr_enable.attr, + &dev_attr_swk_shared.attr, + &dev_attr_swk_max_ttl.attr, + &dev_attr_swk_cnt_per_fn.attr, + &dev_attr_swk_cnt_per_pasid.attr, + NULL, +}; + +static const struct attribute_group qat_kpt_group = { + .attrs = qat_kpt_attrs, + .name = "qat_kpt", +}; + +int adf_sysfs_init_kpt(struct adf_accel_dev *accel_dev) +{ + int ret; + + ret = devm_device_add_group(&GET_DEV(accel_dev), &qat_kpt_group); + if (ret) { + dev_err(&GET_DEV(accel_dev), "Failed to create qat_kpt attribute group\n"); + return ret; + } + + return 0; +} +EXPORT_SYMBOL_GPL(adf_sysfs_init_kpt); diff --git a/drivers/crypto/intel/qat/qat_common/adf_sysfs_kpt.h b/drivers/crypto/intel/qat/qat_common/adf_sysfs_kpt.h new file mode 100644 index 000000000000..d9d065f75114 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_sysfs_kpt.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2026 Intel Corporation */ +#ifndef ADF_SYSFS_KPT_H_ +#define ADF_SYSFS_KPT_H_ + +struct adf_accel_dev; + +int adf_sysfs_init_kpt(struct adf_accel_dev *accel_dev); + +#endif /* ADF_SYSFS_KPT_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_init_admin.h b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_init_admin.h index 6b0f0d100cb9..95f7f514cb63 100644 --- a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_init_admin.h +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_init_admin.h @@ -31,6 +31,7 @@ enum icp_qat_fw_init_admin_cmd_id { ICP_QAT_FW_RL_REMOVE = 136, ICP_QAT_FW_TL_START = 137, ICP_QAT_FW_TL_STOP = 138, + ICP_QAT_FW_KPT_ENABLE = 144, ICP_QAT_FW_SVN_READ = 146, ICP_QAT_FW_SVN_COMMIT = 147, }; @@ -212,4 +213,11 @@ struct icp_qat_fw_init_admin_pm_info { __u32 resvrd3[6]; }; +struct icp_qat_fw_init_admin_kpt_cfg { + __u32 swk_cnt_per_fn; + __u32 swk_cnt_per_pasid; + __u32 swk_ttl_in_secs; + __u32 swk_shared_disable; +}; + #endif diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_hw.h b/drivers/crypto/intel/qat/qat_common/icp_qat_hw.h index 16ef6d98fa42..e38115d3cd75 100644 --- a/drivers/crypto/intel/qat/qat_common/icp_qat_hw.h +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_hw.h @@ -114,7 +114,8 @@ enum icp_qat_capabilities_mask { ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION = BIT(24), ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION = BIT(25), ICP_ACCEL_CAPABILITIES_AES_V2 = BIT(26), - /* Bits 27-28 are currently reserved */ + ICP_ACCEL_CAPABILITIES_KPT = BIT(27), + /* Bit 28 is currently reserved */ ICP_ACCEL_CAPABILITIES_ZUC_256 = BIT(29), ICP_ACCEL_CAPABILITIES_WIRELESS_CRYPTO_EXT = BIT(30), }; diff --git a/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c b/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c index e09b9edfce42..75c15c8e41db 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c +++ b/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c @@ -1085,7 +1085,7 @@ static void qat_rsa_setkey_crt(struct qat_rsa_ctx *ctx, struct rsa_key *rsa_key) ptr = rsa_key->p; len = rsa_key->p_sz; qat_rsa_drop_leading_zeros(&ptr, &len); - if (!len) + if (!len || len > half_key_sz) goto err; ctx->p = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_p, GFP_KERNEL); if (!ctx->p) @@ -1096,7 +1096,7 @@ static void qat_rsa_setkey_crt(struct qat_rsa_ctx *ctx, struct rsa_key *rsa_key) ptr = rsa_key->q; len = rsa_key->q_sz; qat_rsa_drop_leading_zeros(&ptr, &len); - if (!len) + if (!len || len > half_key_sz) goto free_p; ctx->q = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_q, GFP_KERNEL); if (!ctx->q) @@ -1107,7 +1107,7 @@ static void qat_rsa_setkey_crt(struct qat_rsa_ctx *ctx, struct rsa_key *rsa_key) ptr = rsa_key->dp; len = rsa_key->dp_sz; qat_rsa_drop_leading_zeros(&ptr, &len); - if (!len) + if (!len || len > half_key_sz) goto free_q; ctx->dp = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_dp, GFP_KERNEL); @@ -1119,7 +1119,7 @@ static void qat_rsa_setkey_crt(struct qat_rsa_ctx *ctx, struct rsa_key *rsa_key) ptr = rsa_key->dq; len = rsa_key->dq_sz; qat_rsa_drop_leading_zeros(&ptr, &len); - if (!len) + if (!len || len > half_key_sz) goto free_dp; ctx->dq = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_dq, GFP_KERNEL); @@ -1131,7 +1131,7 @@ static void qat_rsa_setkey_crt(struct qat_rsa_ctx *ctx, struct rsa_key *rsa_key) ptr = rsa_key->qinv; len = rsa_key->qinv_sz; qat_rsa_drop_leading_zeros(&ptr, &len); - if (!len) + if (!len || len > half_key_sz) goto free_dq; ctx->qinv = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_qinv, GFP_KERNEL); diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c index b59e0cc49e52..571f302edea3 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c @@ -162,15 +162,14 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) bar->size = pci_resource_len(pdev, bar_nr); bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); if (!bar->virt_addr) { - dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr); + pci_err(pdev, "Failed to map BAR %d\n", bar_nr); ret = -EFAULT; goto out_err_free_reg; } } - pci_set_master(pdev); if (pci_save_state(pdev)) { - dev_err(&pdev->dev, "Failed to save pci state\n"); + pci_err(pdev, "Failed to save pci state\n"); ret = -ENOMEM; goto out_err_free_reg; } @@ -256,5 +255,4 @@ MODULE_AUTHOR("Intel"); MODULE_FIRMWARE(ADF_DH895XCC_FW); MODULE_FIRMWARE(ADF_DH895XCC_MMP); MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); -MODULE_VERSION(ADF_DRV_VERSION); MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/intel/qat/qat_dh895xccvf/adf_drv.c b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_drv.c index 7cd528ee31e7..481551a08708 100644 --- a/drivers/crypto/intel/qat/qat_dh895xccvf/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_drv.c @@ -158,12 +158,11 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) bar->size = pci_resource_len(pdev, bar_nr); bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); if (!bar->virt_addr) { - dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr); + pci_err(pdev, "Failed to map BAR %d\n", bar_nr); ret = -EFAULT; goto out_err_free_reg; } } - pci_set_master(pdev); /* Completion for VF2PF request/response message exchange */ init_completion(&accel_dev->vf.msg_received); @@ -225,5 +224,4 @@ module_exit(adfdrv_release); MODULE_LICENSE("Dual BSD/GPL"); MODULE_AUTHOR("Intel"); MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); -MODULE_VERSION(ADF_DRV_VERSION); MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/loongson/Kconfig b/drivers/crypto/loongson/Kconfig deleted file mode 100644 index 15475da8fc11..000000000000 --- a/drivers/crypto/loongson/Kconfig +++ /dev/null @@ -1,5 +0,0 @@ -config CRYPTO_DEV_LOONGSON_RNG - tristate "Support for Loongson RNG Driver" - depends on MFD_LOONGSON_SE - help - Support for Loongson RNG Driver. diff --git a/drivers/crypto/loongson/Makefile b/drivers/crypto/loongson/Makefile deleted file mode 100644 index 1ce5ec32b553..000000000000 --- a/drivers/crypto/loongson/Makefile +++ /dev/null @@ -1 +0,0 @@ -obj-$(CONFIG_CRYPTO_DEV_LOONGSON_RNG) += loongson-rng.o diff --git a/drivers/crypto/loongson/loongson-rng.c b/drivers/crypto/loongson/loongson-rng.c deleted file mode 100644 index 3a4940260f9e..000000000000 --- a/drivers/crypto/loongson/loongson-rng.c +++ /dev/null @@ -1,209 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright (c) 2019 HiSilicon Limited. */ -/* Copyright (c) 2025 Loongson Technology Corporation Limited. */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define SE_SEED_SIZE 32 - -struct loongson_rng_list { - struct mutex lock; - struct list_head list; - int registered; -}; - -struct loongson_rng { - u32 used; - struct loongson_se_engine *engine; - struct list_head list; - struct mutex lock; -}; - -struct loongson_rng_ctx { - struct loongson_rng *rng; -}; - -struct loongson_rng_cmd { - u32 cmd_id; - union { - u32 len; - u32 ret; - } u; - u32 seed_off; - u32 out_off; - u32 pad[4]; -}; - -static struct loongson_rng_list rng_devices = { - .lock = __MUTEX_INITIALIZER(rng_devices.lock), - .list = LIST_HEAD_INIT(rng_devices.list), -}; - -static int loongson_rng_generate(struct crypto_rng *tfm, const u8 *src, - unsigned int slen, u8 *dstn, unsigned int dlen) -{ - struct loongson_rng_ctx *ctx = crypto_rng_ctx(tfm); - struct loongson_rng *rng = ctx->rng; - struct loongson_rng_cmd *cmd = rng->engine->command; - int err, len; - - mutex_lock(&rng->lock); - cmd->seed_off = 0; - do { - len = min(dlen, rng->engine->buffer_size); - cmd = rng->engine->command; - cmd->u.len = len; - err = loongson_se_send_engine_cmd(rng->engine); - if (err) - break; - - cmd = rng->engine->command_ret; - if (cmd->u.ret) { - err = -EIO; - break; - } - - memcpy(dstn, rng->engine->data_buffer, len); - dlen -= len; - dstn += len; - } while (dlen > 0); - mutex_unlock(&rng->lock); - - return err; -} - -static int loongson_rng_init(struct crypto_tfm *tfm) -{ - struct loongson_rng_ctx *ctx = crypto_tfm_ctx(tfm); - struct loongson_rng *rng; - u32 min_used = U32_MAX; - - mutex_lock(&rng_devices.lock); - list_for_each_entry(rng, &rng_devices.list, list) { - if (rng->used < min_used) { - ctx->rng = rng; - min_used = rng->used; - } - } - ctx->rng->used++; - mutex_unlock(&rng_devices.lock); - - return 0; -} - -static void loongson_rng_exit(struct crypto_tfm *tfm) -{ - struct loongson_rng_ctx *ctx = crypto_tfm_ctx(tfm); - - mutex_lock(&rng_devices.lock); - ctx->rng->used--; - mutex_unlock(&rng_devices.lock); -} - -static int loongson_rng_seed(struct crypto_rng *tfm, const u8 *seed, - unsigned int slen) -{ - struct loongson_rng_ctx *ctx = crypto_rng_ctx(tfm); - struct loongson_rng *rng = ctx->rng; - struct loongson_rng_cmd *cmd; - int err; - - if (slen < SE_SEED_SIZE) - return -EINVAL; - - slen = min(slen, rng->engine->buffer_size); - - mutex_lock(&rng->lock); - cmd = rng->engine->command; - cmd->u.len = slen; - cmd->seed_off = rng->engine->buffer_off; - memcpy(rng->engine->data_buffer, seed, slen); - err = loongson_se_send_engine_cmd(rng->engine); - if (err) - goto out; - - cmd = rng->engine->command_ret; - if (cmd->u.ret) - err = -EIO; -out: - mutex_unlock(&rng->lock); - - return err; -} - -static struct rng_alg loongson_rng_alg = { - .generate = loongson_rng_generate, - .seed = loongson_rng_seed, - .seedsize = SE_SEED_SIZE, - .base = { - .cra_name = "stdrng", - .cra_driver_name = "loongson_stdrng", - .cra_priority = 300, - .cra_ctxsize = sizeof(struct loongson_rng_ctx), - .cra_module = THIS_MODULE, - .cra_init = loongson_rng_init, - .cra_exit = loongson_rng_exit, - }, -}; - -static int loongson_rng_probe(struct platform_device *pdev) -{ - struct loongson_rng_cmd *cmd; - struct loongson_rng *rng; - int ret = 0; - - rng = devm_kzalloc(&pdev->dev, sizeof(*rng), GFP_KERNEL); - if (!rng) - return -ENOMEM; - - rng->engine = loongson_se_init_engine(pdev->dev.parent, SE_ENGINE_RNG); - if (!rng->engine) - return -ENODEV; - cmd = rng->engine->command; - cmd->cmd_id = SE_CMD_RNG; - cmd->out_off = rng->engine->buffer_off; - mutex_init(&rng->lock); - - mutex_lock(&rng_devices.lock); - - if (!rng_devices.registered) { - ret = crypto_register_rng(&loongson_rng_alg); - if (ret) { - dev_err(&pdev->dev, "failed to register crypto(%d)\n", ret); - goto out; - } - rng_devices.registered = 1; - } - - list_add_tail(&rng->list, &rng_devices.list); -out: - mutex_unlock(&rng_devices.lock); - - return ret; -} - -static struct platform_driver loongson_rng_driver = { - .probe = loongson_rng_probe, - .driver = { - .name = "loongson-rng", - }, -}; -module_platform_driver(loongson_rng_driver); - -MODULE_ALIAS("platform:loongson-rng"); -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Yinggang Gu "); -MODULE_AUTHOR("Qunqin Zhao "); -MODULE_DESCRIPTION("Loongson Random Number Generator driver"); diff --git a/drivers/crypto/marvell/cesa/cesa.c b/drivers/crypto/marvell/cesa/cesa.c index 8afa3a87e38d..75d8ba23d9a2 100644 --- a/drivers/crypto/marvell/cesa/cesa.c +++ b/drivers/crypto/marvell/cesa/cesa.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -416,7 +417,7 @@ static int mv_cesa_probe(struct platform_device *pdev) const struct mbus_dram_target_info *dram; struct device *dev = &pdev->dev; struct mv_cesa_dev *cesa; - struct mv_cesa_engine *engines; + struct mv_cesa_engine *engine; int irq, ret, i, cpu; u32 sram_size; @@ -431,7 +432,8 @@ static int mv_cesa_probe(struct platform_device *pdev) return -ENOTSUPP; } - cesa = devm_kzalloc(dev, sizeof(*cesa), GFP_KERNEL); + cesa = devm_kzalloc(dev, struct_size(cesa, engines, caps->nengines), + GFP_KERNEL); if (!cesa) return -ENOMEM; @@ -441,14 +443,8 @@ static int mv_cesa_probe(struct platform_device *pdev) sram_size = CESA_SA_DEFAULT_SRAM_SIZE; of_property_read_u32(cesa->dev->of_node, "marvell,crypto-sram-size", &sram_size); - if (sram_size < CESA_SA_MIN_SRAM_SIZE) - sram_size = CESA_SA_MIN_SRAM_SIZE; - cesa->sram_size = sram_size; - cesa->engines = devm_kcalloc(dev, caps->nengines, sizeof(*engines), - GFP_KERNEL); - if (!cesa->engines) - return -ENOMEM; + cesa->sram_size = max(sram_size, CESA_SA_MIN_SRAM_SIZE); spin_lock_init(&cesa->lock); @@ -465,7 +461,7 @@ static int mv_cesa_probe(struct platform_device *pdev) platform_set_drvdata(pdev, cesa); for (i = 0; i < caps->nengines; i++) { - struct mv_cesa_engine *engine = &cesa->engines[i]; + engine = &cesa->engines[i]; char res_name[16]; engine->id = i; diff --git a/drivers/crypto/marvell/cesa/cesa.h b/drivers/crypto/marvell/cesa/cesa.h index 50ca1039fdaa..18f9f28040a6 100644 --- a/drivers/crypto/marvell/cesa/cesa.h +++ b/drivers/crypto/marvell/cesa/cesa.h @@ -402,27 +402,6 @@ struct mv_cesa_dev_dma { struct dma_pool *padding_pool; }; -/** - * struct mv_cesa_dev - CESA device - * @caps: device capabilities - * @regs: device registers - * @sram_size: usable SRAM size - * @lock: device lock - * @engines: array of engines - * @dma: dma pools - * - * Structure storing CESA device information. - */ -struct mv_cesa_dev { - const struct mv_cesa_caps *caps; - void __iomem *regs; - struct device *dev; - unsigned int sram_size; - spinlock_t lock; - struct mv_cesa_engine *engines; - struct mv_cesa_dev_dma *dma; -}; - /** * struct mv_cesa_engine - CESA engine * @id: engine id @@ -471,6 +450,27 @@ struct mv_cesa_engine { int irq; }; +/** + * struct mv_cesa_dev - CESA device + * @caps: device capabilities + * @regs: device registers + * @sram_size: usable SRAM size + * @lock: device lock + * @dma: dma pools + * @engines: array of engines + * + * Structure storing CESA device information. + */ +struct mv_cesa_dev { + const struct mv_cesa_caps *caps; + void __iomem *regs; + struct device *dev; + unsigned int sram_size; + spinlock_t lock; + struct mv_cesa_dev_dma *dma; + struct mv_cesa_engine engines[]; +}; + /** * struct mv_cesa_req_ops - CESA request operations * @process: process a request chunk result (should return 0 if the diff --git a/drivers/crypto/marvell/octeontx/otx_cptpf_main.c b/drivers/crypto/marvell/octeontx/otx_cptpf_main.c index 14a42559f81d..e4c828606a73 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptpf_main.c +++ b/drivers/crypto/marvell/octeontx/otx_cptpf_main.c @@ -283,6 +283,7 @@ static const struct pci_device_id otx_cpt_id_table[] = { { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OTX_CPT_PCI_PF_DEVICE_ID) }, { 0, } /* end of table */ }; +MODULE_DEVICE_TABLE(pci, otx_cpt_id_table); static struct pci_driver otx_cpt_pci_driver = { .name = DRV_NAME, @@ -298,4 +299,3 @@ MODULE_AUTHOR("Marvell International Ltd."); MODULE_DESCRIPTION("Marvell OcteonTX CPT Physical Function Driver"); MODULE_LICENSE("GPL v2"); MODULE_VERSION(DRV_VERSION); -MODULE_DEVICE_TABLE(pci, otx_cpt_id_table); diff --git a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c index e0f38d32bc93..205579a6ba2b 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c +++ b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c @@ -1318,7 +1318,7 @@ static ssize_t ucode_load_store(struct device *dev, { struct otx_cpt_engines engs[OTX_CPT_MAX_ETYPES_PER_GRP] = { {0} }; char *ucode_filename[OTX_CPT_MAX_ETYPES_PER_GRP]; - char tmp_buf[OTX_CPT_UCODE_NAME_LENGTH] = { 0 }; + char tmp_buf[OTX_CPT_UCODE_NAME_LENGTH]; char *start, *val, *err_msg, *tmp; struct otx_cpt_eng_grps *eng_grps; int grp_idx = 0, ret = -EINVAL; @@ -1326,12 +1326,11 @@ static ssize_t ucode_load_store(struct device *dev, int del_grp_idx = -1; int ucode_idx = 0; - if (count >= OTX_CPT_UCODE_NAME_LENGTH) + if (strscpy_pad(tmp_buf, buf) < 0) return -EINVAL; eng_grps = container_of(attr, struct otx_cpt_eng_grps, ucode_load_attr); err_msg = "Invalid engine group format"; - strscpy(tmp_buf, buf, OTX_CPT_UCODE_NAME_LENGTH); start = tmp_buf; has_se = has_ie = has_ae = false; diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c index 587609db6c69..159c1d290a36 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c @@ -957,9 +957,10 @@ static void otx_cptvf_remove(struct pci_dev *pdev) /* Supported devices */ static const struct pci_device_id otx_cptvf_id_table[] = { - {PCI_VDEVICE(CAVIUM, OTX_CPT_PCI_VF_DEVICE_ID), 0}, - { 0, } /* end of table */ + { PCI_VDEVICE(CAVIUM, OTX_CPT_PCI_VF_DEVICE_ID) }, + { } /* end of table */ }; +MODULE_DEVICE_TABLE(pci, otx_cptvf_id_table); static struct pci_driver otx_cptvf_pci_driver = { .name = DRV_NAME, @@ -974,4 +975,3 @@ MODULE_AUTHOR("Marvell International Ltd."); MODULE_DESCRIPTION("Marvell OcteonTX CPT Virtual Function Driver"); MODULE_LICENSE("GPL v2"); MODULE_VERSION(DRV_VERSION); -MODULE_DEVICE_TABLE(pci, otx_cptvf_id_table); diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_reqmgr.c b/drivers/crypto/marvell/octeontx/otx_cptvf_reqmgr.c index c80baf1ad90b..89030e2711ce 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptvf_reqmgr.c +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_reqmgr.c @@ -157,8 +157,8 @@ static inline int setup_sgio_components(struct pci_dev *pdev, sg_cleanup: for (j = 0; j < i; j++) { if (list[j].dma_addr) { - dma_unmap_single(&pdev->dev, list[i].dma_addr, - list[i].size, DMA_BIDIRECTIONAL); + dma_unmap_single(&pdev->dev, list[j].dma_addr, + list[j].size, DMA_BIDIRECTIONAL); } list[j].dma_addr = 0; diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c index 346d1345f11c..f6f47f4e5d83 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c @@ -867,6 +867,7 @@ static const struct pci_device_id otx2_cpt_id_table[] = { { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, CN10K_CPT_PCI_PF_DEVICE_ID) }, { 0, } /* end of table */ }; +MODULE_DEVICE_TABLE(pci, otx2_cpt_id_table); static struct pci_driver otx2_cpt_pci_driver = { .name = OTX2_CPT_DRV_NAME, @@ -883,4 +884,3 @@ MODULE_IMPORT_NS("CRYPTO_DEV_OCTEONTX2_CPT"); MODULE_AUTHOR("Marvell"); MODULE_DESCRIPTION(OTX2_CPT_DRV_STRING); MODULE_LICENSE("GPL v2"); -MODULE_DEVICE_TABLE(pci, otx2_cpt_id_table); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c index 858f851c9c8a..09ab85e6061c 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c @@ -460,10 +460,11 @@ static void otx2_cptvf_remove(struct pci_dev *pdev) /* Supported devices */ static const struct pci_device_id otx2_cptvf_id_table[] = { - {PCI_VDEVICE(CAVIUM, OTX2_CPT_PCI_VF_DEVICE_ID), 0}, - {PCI_VDEVICE(CAVIUM, CN10K_CPT_PCI_VF_DEVICE_ID), 0}, - { 0, } /* end of table */ + { PCI_VDEVICE(CAVIUM, OTX2_CPT_PCI_VF_DEVICE_ID) }, + { PCI_VDEVICE(CAVIUM, CN10K_CPT_PCI_VF_DEVICE_ID) }, + { } /* end of table */ }; +MODULE_DEVICE_TABLE(pci, otx2_cptvf_id_table); static struct pci_driver otx2_cptvf_pci_driver = { .name = OTX2_CPTVF_DRV_NAME, @@ -479,4 +480,3 @@ MODULE_IMPORT_NS("CRYPTO_DEV_OCTEONTX2_CPT"); MODULE_AUTHOR("Marvell"); MODULE_DESCRIPTION("Marvell RVU CPT Virtual Function Driver"); MODULE_LICENSE("GPL v2"); -MODULE_DEVICE_TABLE(pci, otx2_cptvf_id_table); diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c index 78135fb13f5c..1b7509e2ce44 100644 --- a/drivers/crypto/nx/nx.c +++ b/drivers/crypto/nx/nx.c @@ -714,15 +714,13 @@ int nx_crypto_ctx_aes_xcbc_init(struct crypto_shash *tfm) /** * nx_crypto_ctx_exit - destroy a crypto api context * - * @tfm: the crypto transform pointer for the context + * @nx_ctx: the crypto api context * * As crypto API contexts are destroyed, this exit hook is called to free the * memory associated with it. */ -void nx_crypto_ctx_exit(struct crypto_tfm *tfm) +void nx_crypto_ctx_exit(struct nx_crypto_ctx *nx_ctx) { - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(tfm); - kfree_sensitive(nx_ctx->kmem); nx_ctx->csbcpb = NULL; nx_ctx->csbcpb_aead = NULL; diff --git a/drivers/crypto/nx/nx.h b/drivers/crypto/nx/nx.h index 36974f08490a..6dfabfbf8192 100644 --- a/drivers/crypto/nx/nx.h +++ b/drivers/crypto/nx/nx.h @@ -153,7 +153,7 @@ int nx_crypto_ctx_aes_ctr_init(struct crypto_skcipher *tfm); int nx_crypto_ctx_aes_cbc_init(struct crypto_skcipher *tfm); int nx_crypto_ctx_aes_ecb_init(struct crypto_skcipher *tfm); int nx_crypto_ctx_sha_init(struct crypto_shash *tfm); -void nx_crypto_ctx_exit(struct crypto_tfm *tfm); +void nx_crypto_ctx_exit(struct nx_crypto_ctx *nx_ctx); void nx_crypto_ctx_skcipher_exit(struct crypto_skcipher *tfm); void nx_crypto_ctx_aead_exit(struct crypto_aead *tfm); void nx_crypto_ctx_shash_exit(struct crypto_shash *tfm); diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 3eadaf7a64fa..f31555c0d715 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -1089,6 +1089,20 @@ static struct attribute *omap_aes_attrs[] = { }; ATTRIBUTE_GROUPS(omap_aes); +static void omap_aes_unregister_algs(const struct omap_aes_pdata *pdata) +{ + struct omap_aes_algs_info *alg_info; + int i; + + for (i = pdata->algs_info_size - 1; i >= 0; i--) { + alg_info = &pdata->algs_info[i]; + + crypto_engine_unregister_skciphers(alg_info->algs_list, + alg_info->registered); + alg_info->registered = 0; + } +} + static int omap_aes_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1215,15 +1229,11 @@ static int omap_aes_probe(struct platform_device *pdev) return 0; err_aead_algs: - for (i = dd->pdata->aead_algs_info->registered - 1; i >= 0; i--) { - aalg = &dd->pdata->aead_algs_info->algs_list[i]; - crypto_engine_unregister_aead(aalg); - } + crypto_engine_unregister_aeads(dd->pdata->aead_algs_info->algs_list, + dd->pdata->aead_algs_info->registered); + dd->pdata->aead_algs_info->registered = 0; err_algs: - for (i = dd->pdata->algs_info_size - 1; i >= 0; i--) - for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) - crypto_engine_unregister_skcipher( - &dd->pdata->algs_info[i].algs_list[j]); + omap_aes_unregister_algs(dd->pdata); err_engine: if (dd->engine) @@ -1244,25 +1254,16 @@ err_data: static void omap_aes_remove(struct platform_device *pdev) { struct omap_aes_dev *dd = platform_get_drvdata(pdev); - struct aead_engine_alg *aalg; - int i, j; spin_lock_bh(&list_lock); list_del(&dd->list); spin_unlock_bh(&list_lock); - for (i = dd->pdata->algs_info_size - 1; i >= 0; i--) - for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) { - crypto_engine_unregister_skcipher( - &dd->pdata->algs_info[i].algs_list[j]); - dd->pdata->algs_info[i].registered--; - } + omap_aes_unregister_algs(dd->pdata); - for (i = dd->pdata->aead_algs_info->registered - 1; i >= 0; i--) { - aalg = &dd->pdata->aead_algs_info->algs_list[i]; - crypto_engine_unregister_aead(aalg); - dd->pdata->aead_algs_info->registered--; - } + crypto_engine_unregister_aeads(dd->pdata->aead_algs_info->algs_list, + dd->pdata->aead_algs_info->registered); + dd->pdata->aead_algs_info->registered = 0; crypto_engine_exit(dd->engine); diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c index 149ebd77710b..dfaf831dd8ec 100644 --- a/drivers/crypto/omap-des.c +++ b/drivers/crypto/omap-des.c @@ -800,7 +800,6 @@ static struct omap_des_algs_info omap_des_algs_info_ecb_cbc[] = { }, }; -#ifdef CONFIG_OF static const struct omap_des_pdata omap_des_pdata_omap4 = { .algs_info = omap_des_algs_info_ecb_cbc, .algs_info_size = ARRAY_SIZE(omap_des_algs_info_ecb_cbc), @@ -909,6 +908,7 @@ static const struct of_device_id omap_des_of_match[] = { }; MODULE_DEVICE_TABLE(of, omap_des_of_match); +#ifdef CONFIG_OF static int omap_des_get_of(struct omap_des_dev *dd, struct platform_device *pdev) { @@ -923,7 +923,7 @@ static int omap_des_get_of(struct omap_des_dev *dd, } #else static int omap_des_get_of(struct omap_des_dev *dd, - struct device *dev) + struct platform_device *pdev) { return -EINVAL; } @@ -938,6 +938,20 @@ static int omap_des_get_pdev(struct omap_des_dev *dd, return 0; } +static void omap_des_unregister_algs(const struct omap_des_pdata *pdata) +{ + struct omap_des_algs_info *alg_info; + int i; + + for (i = pdata->algs_info_size - 1; i >= 0; i--) { + alg_info = &pdata->algs_info[i]; + + crypto_engine_unregister_skciphers(alg_info->algs_list, + alg_info->registered); + alg_info->registered = 0; + } +} + static int omap_des_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1043,11 +1057,7 @@ static int omap_des_probe(struct platform_device *pdev) return 0; err_algs: - for (i = dd->pdata->algs_info_size - 1; i >= 0; i--) - for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) - crypto_engine_unregister_skcipher( - &dd->pdata->algs_info[i].algs_list[j]); - + omap_des_unregister_algs(dd->pdata); err_engine: if (dd->engine) crypto_engine_exit(dd->engine); @@ -1067,16 +1077,12 @@ err_data: static void omap_des_remove(struct platform_device *pdev) { struct omap_des_dev *dd = platform_get_drvdata(pdev); - int i, j; spin_lock_bh(&list_lock); list_del(&dd->list); spin_unlock_bh(&list_lock); - for (i = dd->pdata->algs_info_size - 1; i >= 0; i--) - for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) - crypto_engine_unregister_skcipher( - &dd->pdata->algs_info[i].algs_list[j]); + omap_des_unregister_algs(dd->pdata); cancel_work_sync(&dd->done_task); omap_des_dma_cleanup(dd); @@ -1111,7 +1117,7 @@ static struct platform_driver omap_des_driver = { .driver = { .name = "omap-des", .pm = &omap_des_pm_ops, - .of_match_table = of_match_ptr(omap_des_of_match), + .of_match_table = omap_des_of_match, }, }; diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index b8c416c5ee70..be1ac640ee59 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -2042,6 +2042,20 @@ static struct attribute *omap_sham_attrs[] = { }; ATTRIBUTE_GROUPS(omap_sham); +static void omap_sham_unregister_algs(const struct omap_sham_pdata *pdata) +{ + struct omap_sham_algs_info *alg_info; + int i; + + for (i = pdata->algs_info_size - 1; i >= 0; i--) { + alg_info = &pdata->algs_info[i]; + + crypto_engine_unregister_ahashes(alg_info->algs_list, + alg_info->registered); + alg_info->registered = 0; + } +} + static int omap_sham_probe(struct platform_device *pdev) { struct omap_sham_dev *dd; @@ -2158,10 +2172,7 @@ static int omap_sham_probe(struct platform_device *pdev) return 0; err_algs: - for (i = dd->pdata->algs_info_size - 1; i >= 0; i--) - for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) - crypto_engine_unregister_ahash( - &dd->pdata->algs_info[i].algs_list[j]); + omap_sham_unregister_algs(dd->pdata); err_engine_start: crypto_engine_exit(dd->engine); err_engine: @@ -2182,19 +2193,13 @@ data_err: static void omap_sham_remove(struct platform_device *pdev) { struct omap_sham_dev *dd; - int i, j; dd = platform_get_drvdata(pdev); spin_lock_bh(&sham.lock); list_del(&dd->list); spin_unlock_bh(&sham.lock); - for (i = dd->pdata->algs_info_size - 1; i >= 0; i--) - for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) { - crypto_engine_unregister_ahash( - &dd->pdata->algs_info[i].algs_list[j]); - dd->pdata->algs_info[i].registered--; - } + omap_sham_unregister_algs(dd->pdata); cancel_work_sync(&dd->done_task); pm_runtime_dont_use_autosuspend(&pdev->dev); pm_runtime_disable(&pdev->dev); diff --git a/drivers/crypto/qcom-rng.c b/drivers/crypto/qcom-rng.c index 0685ba122e8a..150e5802e351 100644 --- a/drivers/crypto/qcom-rng.c +++ b/drivers/crypto/qcom-rng.c @@ -265,7 +265,7 @@ static struct platform_driver qcom_rng_driver = { .remove = qcom_rng_remove, .driver = { .name = KBUILD_MODNAME, - .of_match_table = of_match_ptr(qcom_rng_of_match), + .of_match_table = qcom_rng_of_match, .acpi_match_table = ACPI_PTR(qcom_rng_acpi_match), } }; diff --git a/drivers/crypto/starfive/jh7110-cryp.c b/drivers/crypto/starfive/jh7110-cryp.c index 42114e9364f0..e19cd7945968 100644 --- a/drivers/crypto/starfive/jh7110-cryp.c +++ b/drivers/crypto/starfive/jh7110-cryp.c @@ -36,19 +36,14 @@ static struct starfive_dev_list dev_list = { struct starfive_cryp_dev *starfive_cryp_find_dev(struct starfive_cryp_ctx *ctx) { - struct starfive_cryp_dev *cryp = NULL, *tmp; + struct starfive_cryp_dev *cryp; spin_lock_bh(&dev_list.lock); - if (!ctx->cryp) { - list_for_each_entry(tmp, &dev_list.dev_list, list) { - cryp = tmp; - break; - } - ctx->cryp = cryp; - } else { - cryp = ctx->cryp; - } - + if (!ctx->cryp) + ctx->cryp = list_first_entry_or_null(&dev_list.dev_list, + struct starfive_cryp_dev, + list); + cryp = ctx->cryp; spin_unlock_bh(&dev_list.lock); return cryp; diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index bc61d0fe3514..584508963241 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -12,7 +12,6 @@ * All rights reserved. */ -#include #include #include #include @@ -255,6 +254,46 @@ static int init_device(struct device *dev) return 0; } +static void dma_map_request(struct device *dev, struct talitos_request *request, + struct talitos_desc *desc, bool is_sec1) +{ + struct talitos_edesc *edesc = + container_of(desc, struct talitos_edesc, desc); + dma_addr_t dma_desc, prev_dma_desc; + struct talitos_edesc *prev_edesc = NULL; + + if (is_sec1) { + while (edesc) { + edesc->desc.hdr1 = edesc->desc.hdr; + + dma_desc = dma_map_single(dev, &edesc->desc.hdr1, + TALITOS_DESC_SIZE, + DMA_BIDIRECTIONAL); + + if (!prev_edesc) { + request->dma_desc = dma_desc; + goto next; + } + + /* Chain in any previous descriptors. */ + + prev_edesc->desc.next_desc = cpu_to_be32(dma_desc); + + dma_sync_single_for_device(dev, prev_dma_desc, + TALITOS_DESC_SIZE, + DMA_TO_DEVICE); + +next: + prev_edesc = edesc; + prev_dma_desc = dma_desc; + edesc = edesc->next_desc; + } + } else { + request->dma_desc = dma_map_single(dev, desc, TALITOS_DESC_SIZE, + DMA_BIDIRECTIONAL); + } +} + /** * talitos_submit - submits a descriptor to the device for processing * @dev: the SEC device to be used @@ -291,16 +330,7 @@ static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, request = &priv->chan[ch].fifo[head]; /* map descriptor and save caller data */ - if (is_sec1) { - desc->hdr1 = desc->hdr; - request->dma_desc = dma_map_single(dev, &desc->hdr1, - TALITOS_DESC_SIZE, - DMA_BIDIRECTIONAL); - } else { - request->dma_desc = dma_map_single(dev, desc, - TALITOS_DESC_SIZE, - DMA_BIDIRECTIONAL); - } + dma_map_request(dev, request, desc, is_sec1); request->callback = callback; request->context = context; @@ -322,19 +352,51 @@ static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, return -EINPROGRESS; } -static __be32 get_request_hdr(struct talitos_request *request, bool is_sec1) +static __be32 get_request_hdr(struct device *dev, + struct talitos_request *request, bool is_sec1) +{ + struct talitos_edesc *edesc; + dma_addr_t dma_desc; + + if (!is_sec1) { + dma_sync_single_for_cpu(dev, request->dma_desc, + TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); + + return request->desc->hdr; + } + + edesc = container_of(request->desc, struct talitos_edesc, desc); + dma_desc = request->dma_desc; + while (edesc->next_desc) { + dma_desc = be32_to_cpu(edesc->desc.next_desc); + edesc = edesc->next_desc; + } + + dma_sync_single_for_cpu(dev, dma_desc, TALITOS_DESC_SIZE, + DMA_BIDIRECTIONAL); + + return edesc->desc.hdr1; +} + +static void dma_unmap_request(struct device *dev, + struct talitos_request *request, bool is_sec1) { struct talitos_edesc *edesc; - if (!is_sec1) - return request->desc->hdr; - - if (!request->desc->next_desc) - return request->desc->hdr1; - - edesc = container_of(request->desc, struct talitos_edesc, desc); - - return ((struct talitos_desc *)(edesc->buf + edesc->dma_len))->hdr1; + if (is_sec1) { + dma_unmap_single(dev, request->dma_desc, TALITOS_DESC_SIZE, + DMA_BIDIRECTIONAL); + edesc = container_of(request->desc, struct talitos_edesc, desc); + while (edesc->next_desc) { + dma_unmap_single(dev, + be32_to_cpu(edesc->desc.next_desc), + TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); + edesc = edesc->next_desc; + } + } else { + dma_unmap_single(dev, request->dma_desc, TALITOS_DESC_SIZE, + DMA_BIDIRECTIONAL); + } } /* @@ -358,7 +420,7 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch) /* descriptors with their done bits set don't get the error */ rmb(); - hdr = get_request_hdr(request, is_sec1); + hdr = get_request_hdr(dev, request, is_sec1); if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE) status = 0; @@ -368,9 +430,7 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch) else status = error; - dma_unmap_single(dev, request->dma_desc, - TALITOS_DESC_SIZE, - DMA_BIDIRECTIONAL); + dma_unmap_request(dev, request, is_sec1); /* copy entries so we can call callback outside lock */ saved_req.desc = request->desc; @@ -459,14 +519,35 @@ DEF_TALITOS2_DONE(ch0, TALITOS2_ISR_CH_0_DONE) DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE) DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE) +static __be32 search_desc_hdr_in_request(struct talitos_request *request, + dma_addr_t cur_desc, bool is_sec1) +{ + struct talitos_edesc *edesc; + + if (request->dma_desc == cur_desc) { + return request->desc->hdr; + } else if (is_sec1) { + edesc = container_of(request->desc, struct talitos_edesc, desc); + while (edesc->next_desc) { + if (edesc->desc.next_desc == cpu_to_be32(cur_desc)) + return edesc->next_desc->desc.hdr1; + edesc = edesc->next_desc; + } + } + return 0; +} + /* * locate current (offending) descriptor */ static __be32 current_desc_hdr(struct device *dev, int ch) { struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); + struct talitos_request *request; int tail, iter; dma_addr_t cur_desc; + __be32 hdr = 0; cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32; cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO); @@ -477,27 +558,21 @@ static __be32 current_desc_hdr(struct device *dev, int ch) } tail = priv->chan[ch].tail; - iter = tail; - while (priv->chan[ch].fifo[iter].dma_desc != cur_desc && - priv->chan[ch].fifo[iter].desc->next_desc != cpu_to_be32(cur_desc)) { + do { + request = &priv->chan[ch].fifo[iter]; + + hdr = search_desc_hdr_in_request(request, cur_desc, is_sec1); + if (hdr) + break; + iter = (iter + 1) & (priv->fifo_len - 1); - if (iter == tail) { - dev_err(dev, "couldn't locate current descriptor\n"); - return 0; - } - } + } while (iter != tail); - if (priv->chan[ch].fifo[iter].desc->next_desc == cpu_to_be32(cur_desc)) { - struct talitos_edesc *edesc; + if (!hdr) + dev_err(dev, "couldn't locate current descriptor\n"); - edesc = container_of(priv->chan[ch].fifo[iter].desc, - struct talitos_edesc, desc); - return ((struct talitos_desc *) - (edesc->buf + edesc->dma_len))->hdr; - } - - return priv->chan[ch].fifo[iter].desc->hdr; + return hdr; } /* @@ -869,28 +944,20 @@ struct talitos_ahash_req_ctx { u8 buf[2][HASH_MAX_BLOCK_SIZE]; int buf_idx; unsigned int swinit; - unsigned int first_desc; - unsigned int last_desc; + unsigned int first_request; unsigned int last_request; unsigned int to_hash_later; unsigned int nbuf; struct scatterlist bufsl[2]; struct scatterlist *psrc; - - struct scatterlist request_bufsl[2]; - struct ahash_request *areq; - struct scatterlist *request_sl; - unsigned int remaining_ahash_request_bytes; - unsigned int current_ahash_request_bytes; - struct work_struct sec1_ahash_process_remaining; }; struct talitos_export_state { u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)]; u8 buf[HASH_MAX_BLOCK_SIZE]; unsigned int swinit; - unsigned int first_desc; - unsigned int last_desc; + unsigned int first_request; + unsigned int last_request; unsigned int to_hash_later; unsigned int nbuf; }; @@ -1396,10 +1463,6 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, dma_len = 0; } alloc_len += icv_stashing ? authsize : 0; - - /* if its a ahash, add space for a second desc next to the first one */ - if (is_sec1 && !dst) - alloc_len += sizeof(struct talitos_desc); alloc_len += ivsize; edesc = kmalloc(ALIGN(alloc_len, dma_get_cache_alignment()), flags); @@ -1415,6 +1478,7 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, edesc->dst_nents = dst_nents; edesc->iv_dma = iv_dma; edesc->dma_len = dma_len; + edesc->next_desc = NULL; if (dma_len) edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0], edesc->dma_len, @@ -1715,39 +1779,37 @@ static void common_nonsnoop_hash_unmap(struct device *dev, struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); struct talitos_desc *desc = &edesc->desc; - struct talitos_desc *desc2 = (struct talitos_desc *) - (edesc->buf + edesc->dma_len); unmap_single_talitos_ptr(dev, &desc->ptr[5], DMA_FROM_DEVICE); - if (desc->next_desc && - desc->ptr[5].ptr != desc2->ptr[5].ptr) - unmap_single_talitos_ptr(dev, &desc2->ptr[5], DMA_FROM_DEVICE); - if (req_ctx->last_desc) + + if (edesc->last && req_ctx->last_request) memcpy(areq->result, req_ctx->hw_context, crypto_ahash_digestsize(tfm)); - if (req_ctx->psrc) - talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0); + if (edesc->src) + talitos_sg_unmap(dev, edesc, edesc->src, NULL, 0, 0); /* When using hashctx-in, must unmap it. */ if (from_talitos_ptr_len(&desc->ptr[1], is_sec1)) unmap_single_talitos_ptr(dev, &desc->ptr[1], DMA_TO_DEVICE); - else if (desc->next_desc) - unmap_single_talitos_ptr(dev, &desc2->ptr[1], - DMA_TO_DEVICE); - - if (is_sec1 && req_ctx->nbuf) - unmap_single_talitos_ptr(dev, &desc->ptr[3], - DMA_TO_DEVICE); if (edesc->dma_len) dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); +} - if (desc->next_desc) - dma_unmap_single(dev, be32_to_cpu(desc->next_desc), - TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); +static void free_edesc_list_from(struct ahash_request *areq, struct talitos_edesc *edesc) +{ + struct talitos_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct talitos_edesc *next; + + while (edesc) { + next = edesc->next_desc; + common_nonsnoop_hash_unmap(ctx->dev, edesc, areq); + kfree(edesc); + edesc = next; + } } static void ahash_done(struct device *dev, @@ -1758,30 +1820,46 @@ static void ahash_done(struct device *dev, struct talitos_edesc *edesc = container_of(desc, struct talitos_edesc, desc); struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + bool is_sec1 = has_ftr_sec1(dev_get_drvdata(dev)); + struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); + struct talitos_edesc *next; - if (!req_ctx->last_desc && req_ctx->to_hash_later) { - /* Position any partial block for next update/final/finup */ - req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1; - req_ctx->nbuf = req_ctx->to_hash_later; - } - common_nonsnoop_hash_unmap(dev, edesc, areq); + if (is_sec1) { + if (!req_ctx->last_request && req_ctx->to_hash_later) { + /* Position any partial block for next update/final/finup */ + req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1; + req_ctx->nbuf = req_ctx->to_hash_later; + } - kfree(edesc); + free_edesc_list_from(areq, edesc); + ahash_request_complete(areq, err); + } else { + next = edesc->next_desc; - if (err) { + common_nonsnoop_hash_unmap(dev, edesc, areq); + kfree(edesc); + + if (err) + goto out; + + if (next) { + err = talitos_submit(dev, ctx->ch, &next->desc, + ahash_done, areq); + if (err != -EINPROGRESS) + goto out; + return; + } +out: + if (!req_ctx->last_request && req_ctx->to_hash_later) { + /* Position any partial block for next update/final/finup */ + req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1; + req_ctx->nbuf = req_ctx->to_hash_later; + } + if (err && next) + free_edesc_list_from(areq, next); ahash_request_complete(areq, err); - return; } - - req_ctx->remaining_ahash_request_bytes -= - req_ctx->current_ahash_request_bytes; - - if (!req_ctx->remaining_ahash_request_bytes) { - ahash_request_complete(areq, 0); - return; - } - - schedule_work(&req_ctx->sec1_ahash_process_remaining); } /* @@ -1805,18 +1883,15 @@ static void talitos_handle_buggy_hash(struct talitos_ctx *ctx, (char *)padded_hash, DMA_TO_DEVICE); } -static int common_nonsnoop_hash(struct talitos_edesc *edesc, - struct ahash_request *areq, unsigned int length, - void (*callback) (struct device *dev, - struct talitos_desc *desc, - void *context, int error)) +static void common_nonsnoop_hash(struct talitos_edesc *edesc, + struct ahash_request *areq, + unsigned int length) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); struct device *dev = ctx->dev; struct talitos_desc *desc = &edesc->desc; - int ret; bool sync_needed = false; struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); @@ -1825,7 +1900,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, /* first DWORD empty */ /* hash context in */ - if (!req_ctx->first_desc || req_ctx->swinit) { + if (!edesc->first || !req_ctx->first_request || req_ctx->swinit) { map_single_talitos_ptr_nosync(dev, &desc->ptr[1], req_ctx->hw_context_size, req_ctx->hw_context, @@ -1833,40 +1908,31 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, req_ctx->swinit = 0; } /* Indicate next op is not the first. */ - req_ctx->first_desc = 0; + req_ctx->first_request = 0; /* HMAC key */ if (ctx->keylen) to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, is_sec1); - if (is_sec1 && req_ctx->nbuf) - length -= req_ctx->nbuf; - sg_count = edesc->src_nents ?: 1; if (is_sec1 && sg_count > 1) - sg_copy_to_buffer(req_ctx->psrc, sg_count, edesc->buf, length); + sg_copy_to_buffer(edesc->src, sg_count, edesc->buf, length); else if (length) - sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count, - DMA_TO_DEVICE); + sg_count = dma_map_sg(dev, edesc->src, sg_count, DMA_TO_DEVICE); + /* * data in */ - if (is_sec1 && req_ctx->nbuf) { - map_single_talitos_ptr(dev, &desc->ptr[3], req_ctx->nbuf, - req_ctx->buf[req_ctx->buf_idx], - DMA_TO_DEVICE); - } else { - sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc, - &desc->ptr[3], sg_count, 0, 0); - if (sg_count > 1) - sync_needed = true; - } + sg_count = talitos_sg_map(dev, edesc->src, length, edesc, &desc->ptr[3], + sg_count, 0, 0); + if (sg_count > 1) + sync_needed = true; /* fifth DWORD empty */ /* hash/HMAC out -or- hash context out */ - if (req_ctx->last_desc) + if (edesc->last && req_ctx->last_request) map_single_talitos_ptr(dev, &desc->ptr[5], crypto_ahash_digestsize(tfm), req_ctx->hw_context, DMA_FROM_DEVICE); @@ -1881,73 +1947,94 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0) talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]); - if (is_sec1 && req_ctx->nbuf && length) { - struct talitos_desc *desc2 = (struct talitos_desc *) - (edesc->buf + edesc->dma_len); - dma_addr_t next_desc; - - memset(desc2, 0, sizeof(*desc2)); - desc2->hdr = desc->hdr; - desc2->hdr &= ~DESC_HDR_MODE0_MDEU_INIT; - desc2->hdr1 = desc2->hdr; - desc->hdr &= ~DESC_HDR_MODE0_MDEU_PAD; - desc->hdr |= DESC_HDR_MODE0_MDEU_CONT; - desc->hdr &= ~DESC_HDR_DONE_NOTIFY; - - if (desc->ptr[1].ptr) - copy_talitos_ptr(&desc2->ptr[1], &desc->ptr[1], - is_sec1); - else - map_single_talitos_ptr_nosync(dev, &desc2->ptr[1], - req_ctx->hw_context_size, - req_ctx->hw_context, - DMA_TO_DEVICE); - copy_talitos_ptr(&desc2->ptr[2], &desc->ptr[2], is_sec1); - sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc, - &desc2->ptr[3], sg_count, 0, 0); - if (sg_count > 1) - sync_needed = true; - copy_talitos_ptr(&desc2->ptr[5], &desc->ptr[5], is_sec1); - if (req_ctx->last_desc) - map_single_talitos_ptr_nosync(dev, &desc->ptr[5], - req_ctx->hw_context_size, - req_ctx->hw_context, - DMA_FROM_DEVICE); - - next_desc = dma_map_single(dev, &desc2->hdr1, TALITOS_DESC_SIZE, - DMA_BIDIRECTIONAL); - desc->next_desc = cpu_to_be32(next_desc); - } - if (sync_needed) dma_sync_single_for_device(dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); - - ret = talitos_submit(dev, ctx->ch, desc, callback, areq); - if (ret != -EINPROGRESS) { - common_nonsnoop_hash_unmap(dev, edesc, areq); - kfree(edesc); - } - return ret; } static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq, + struct scatterlist *src, unsigned int nbytes) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); - struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); - struct talitos_private *priv = dev_get_drvdata(ctx->dev); - bool is_sec1 = has_ftr_sec1(priv); - if (is_sec1) - nbytes -= req_ctx->nbuf; - - return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0, + return talitos_edesc_alloc(ctx->dev, src, NULL, NULL, 0, nbytes, 0, 0, 0, areq->base.flags, false); } -static int ahash_process_req_one(struct ahash_request *areq, unsigned int nbytes) +static struct talitos_edesc * +ahash_process_req_prepare(struct ahash_request *areq, unsigned int nbytes, + unsigned int blocksize, bool is_sec1) +{ + struct talitos_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + struct talitos_edesc *first = NULL, *prev_edesc = NULL, *edesc; + size_t desc_max = is_sec1 ? TALITOS1_MAX_DATA_LEN : + TALITOS2_MAX_DATA_LEN; + struct scatterlist tmp[2]; + size_t to_hash_this_desc; + struct scatterlist *src; + size_t offset = 0; + + do { + src = scatterwalk_ffwd(tmp, req_ctx->psrc, offset); + + to_hash_this_desc = + min(nbytes, ALIGN_DOWN(desc_max, blocksize)); + + /* Allocate extended descriptor */ + edesc = ahash_edesc_alloc(areq, src, to_hash_this_desc); + if (IS_ERR(edesc)) { + if (first) + free_edesc_list_from(areq, first); + return edesc; + } + + edesc->src = + scatterwalk_ffwd(edesc->bufsl, req_ctx->psrc, offset); + edesc->desc.hdr = ctx->desc_hdr_template; + edesc->first = offset == 0; + edesc->last = nbytes - to_hash_this_desc == 0; + + /* On last one, request SEC to pad; otherwise continue */ + if (req_ctx->last_request && edesc->last) + edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD; + else + edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT; + + /* request SEC to INIT hash. */ + if (req_ctx->first_request && edesc->first && !req_ctx->swinit) + edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT; + + /* + * When the tfm context has a keylen, it's an HMAC. + * A first or last (ie. not middle) descriptor must request HMAC. + */ + if (ctx->keylen && ((req_ctx->first_request && edesc->first) || + (req_ctx->last_request && edesc->last))) + edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC; + + /* clear the DN bit */ + if (is_sec1 && !edesc->last) + edesc->desc.hdr &= ~DESC_HDR_DONE_NOTIFY; + + common_nonsnoop_hash(edesc, areq, to_hash_this_desc); + + offset += to_hash_this_desc; + nbytes -= to_hash_this_desc; + + if (!prev_edesc) + first = edesc; + else + prev_edesc->next_desc = edesc; + prev_edesc = edesc; + } while (nbytes); + + return first; +} + +static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); @@ -1955,23 +2042,23 @@ static int ahash_process_req_one(struct ahash_request *areq, unsigned int nbytes struct talitos_edesc *edesc; unsigned int blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + bool is_sec1 = has_ftr_sec1(dev_get_drvdata(ctx->dev)); unsigned int nbytes_to_hash; unsigned int to_hash_later; unsigned int nsg; int nents; struct device *dev = ctx->dev; - struct talitos_private *priv = dev_get_drvdata(dev); - bool is_sec1 = has_ftr_sec1(priv); u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx]; + int ret; - if (!req_ctx->last_desc && (nbytes + req_ctx->nbuf <= blocksize)) { + if (!req_ctx->last_request && (nbytes + req_ctx->nbuf <= blocksize)) { /* Buffer up to one whole block */ - nents = sg_nents_for_len(req_ctx->request_sl, nbytes); + nents = sg_nents_for_len(areq->src, nbytes); if (nents < 0) { dev_err(dev, "Invalid number of src SG.\n"); return nents; } - sg_copy_to_buffer(req_ctx->request_sl, nents, + sg_copy_to_buffer(areq->src, nents, ctx_buf + req_ctx->nbuf, nbytes); req_ctx->nbuf += nbytes; return 0; @@ -1981,7 +2068,7 @@ static int ahash_process_req_one(struct ahash_request *areq, unsigned int nbytes nbytes_to_hash = nbytes + req_ctx->nbuf; to_hash_later = nbytes_to_hash & (blocksize - 1); - if (req_ctx->last_desc) + if (req_ctx->last_request) to_hash_later = 0; else if (to_hash_later) /* There is a partial block. Hash the full block(s) now */ @@ -1993,123 +2080,39 @@ static int ahash_process_req_one(struct ahash_request *areq, unsigned int nbytes } /* Chain in any previously buffered data */ - if (!is_sec1 && req_ctx->nbuf) { + if (req_ctx->nbuf) { nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1; sg_init_table(req_ctx->bufsl, nsg); sg_set_buf(req_ctx->bufsl, ctx_buf, req_ctx->nbuf); if (nsg > 1) - sg_chain(req_ctx->bufsl, 2, req_ctx->request_sl); + sg_chain(req_ctx->bufsl, 2, areq->src); req_ctx->psrc = req_ctx->bufsl; - } else if (is_sec1 && req_ctx->nbuf && req_ctx->nbuf < blocksize) { - int offset; - - if (nbytes_to_hash > blocksize) - offset = blocksize - req_ctx->nbuf; - else - offset = nbytes_to_hash - req_ctx->nbuf; - nents = sg_nents_for_len(req_ctx->request_sl, offset); - if (nents < 0) { - dev_err(dev, "Invalid number of src SG.\n"); - return nents; - } - sg_copy_to_buffer(req_ctx->request_sl, nents, - ctx_buf + req_ctx->nbuf, offset); - req_ctx->nbuf += offset; - req_ctx->psrc = scatterwalk_ffwd(req_ctx->bufsl, req_ctx->request_sl, - offset); } else - req_ctx->psrc = req_ctx->request_sl; + req_ctx->psrc = areq->src; if (to_hash_later) { - nents = sg_nents_for_len(req_ctx->request_sl, nbytes); + nents = sg_nents_for_len(areq->src, nbytes); if (nents < 0) { dev_err(dev, "Invalid number of src SG.\n"); return nents; } - sg_pcopy_to_buffer(req_ctx->request_sl, nents, + sg_pcopy_to_buffer(areq->src, nents, req_ctx->buf[(req_ctx->buf_idx + 1) & 1], to_hash_later, nbytes - to_hash_later); } req_ctx->to_hash_later = to_hash_later; - /* Allocate extended descriptor */ - edesc = ahash_edesc_alloc(req_ctx->areq, nbytes_to_hash); + edesc = ahash_process_req_prepare(areq, nbytes_to_hash, blocksize, + is_sec1); if (IS_ERR(edesc)) return PTR_ERR(edesc); - edesc->desc.hdr = ctx->desc_hdr_template; + ret = talitos_submit(dev, ctx->ch, &edesc->desc, ahash_done, areq); + if (ret != -EINPROGRESS) + free_edesc_list_from(areq, edesc); - /* On last one, request SEC to pad; otherwise continue */ - if (req_ctx->last_desc) - edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD; - else - edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT; - - /* request SEC to INIT hash. */ - if (req_ctx->first_desc && !req_ctx->swinit) - edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT; - - /* When the tfm context has a keylen, it's an HMAC. - * A first or last (ie. not middle) descriptor must request HMAC. - */ - if (ctx->keylen && (req_ctx->first_desc || req_ctx->last_desc)) - edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC; - - return common_nonsnoop_hash(edesc, req_ctx->areq, nbytes_to_hash, ahash_done); -} - -static void sec1_ahash_process_remaining(struct work_struct *work) -{ - struct talitos_ahash_req_ctx *req_ctx = - container_of(work, struct talitos_ahash_req_ctx, - sec1_ahash_process_remaining); - int err = 0; - - req_ctx->request_sl = scatterwalk_ffwd(req_ctx->request_bufsl, - req_ctx->request_sl, TALITOS1_MAX_DATA_LEN); - - if (req_ctx->remaining_ahash_request_bytes > TALITOS1_MAX_DATA_LEN) - req_ctx->current_ahash_request_bytes = TALITOS1_MAX_DATA_LEN; - else { - req_ctx->current_ahash_request_bytes = - req_ctx->remaining_ahash_request_bytes; - - if (req_ctx->last_request) - req_ctx->last_desc = 1; - } - - err = ahash_process_req_one(req_ctx->areq, - req_ctx->current_ahash_request_bytes); - - if (err != -EINPROGRESS) - ahash_request_complete(req_ctx->areq, err); -} - -static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) -{ - struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); - struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); - struct device *dev = ctx->dev; - struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); - struct talitos_private *priv = dev_get_drvdata(dev); - bool is_sec1 = has_ftr_sec1(priv); - - req_ctx->areq = areq; - req_ctx->request_sl = areq->src; - req_ctx->remaining_ahash_request_bytes = nbytes; - - if (is_sec1) { - if (nbytes > TALITOS1_MAX_DATA_LEN) - nbytes = TALITOS1_MAX_DATA_LEN; - else if (req_ctx->last_request) - req_ctx->last_desc = 1; - } - - req_ctx->current_ahash_request_bytes = nbytes; - - return ahash_process_req_one(req_ctx->areq, - req_ctx->current_ahash_request_bytes); + return ret; } static int ahash_init(struct ahash_request *areq) @@ -2124,15 +2127,13 @@ static int ahash_init(struct ahash_request *areq) /* Initialize the context */ req_ctx->buf_idx = 0; req_ctx->nbuf = 0; - req_ctx->first_desc = 1; /* first_desc indicates h/w must init its context */ + req_ctx->first_request = 1; req_ctx->swinit = 0; /* assume h/w init of context */ size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; req_ctx->hw_context_size = size; req_ctx->last_request = 0; - req_ctx->last_desc = 0; - INIT_WORK(&req_ctx->sec1_ahash_process_remaining, sec1_ahash_process_remaining); dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size, DMA_TO_DEVICE); @@ -2224,8 +2225,8 @@ static int ahash_export(struct ahash_request *areq, void *out) req_ctx->hw_context_size); memcpy(export->buf, req_ctx->buf[req_ctx->buf_idx], req_ctx->nbuf); export->swinit = req_ctx->swinit; - export->first_desc = req_ctx->first_desc; - export->last_desc = req_ctx->last_desc; + export->first_request = req_ctx->first_request; + export->last_request = req_ctx->last_request; export->to_hash_later = req_ctx->to_hash_later; export->nbuf = req_ctx->nbuf; @@ -2250,8 +2251,8 @@ static int ahash_import(struct ahash_request *areq, const void *in) memcpy(req_ctx->hw_context, export->hw_context, size); memcpy(req_ctx->buf[0], export->buf, export->nbuf); req_ctx->swinit = export->swinit; - req_ctx->first_desc = export->first_desc; - req_ctx->last_desc = export->last_desc; + req_ctx->first_request = export->first_request; + req_ctx->last_request = export->last_request; req_ctx->to_hash_later = export->to_hash_later; req_ctx->nbuf = export->nbuf; @@ -3409,14 +3410,19 @@ static int talitos_probe(struct platform_device *ofdev) struct device *dev = &ofdev->dev; struct device_node *np = ofdev->dev.of_node; struct talitos_private *priv; + unsigned int num_channels; int i, err; int stride; - struct resource *res; - priv = devm_kzalloc(dev, sizeof(struct talitos_private), GFP_KERNEL); + if (of_property_read_u32(np, "fsl,num-channels", &num_channels)) + return -EINVAL; + + priv = devm_kzalloc(dev, struct_size(priv, chan, num_channels), GFP_KERNEL); if (!priv) return -ENOMEM; + priv->num_channels = num_channels; + INIT_LIST_HEAD(&priv->alg_list); dev_set_drvdata(dev, priv); @@ -3425,18 +3431,14 @@ static int talitos_probe(struct platform_device *ofdev) spin_lock_init(&priv->reg_lock); - res = platform_get_resource(ofdev, IORESOURCE_MEM, 0); - if (!res) - return -ENXIO; - priv->reg = devm_ioremap(dev, res->start, resource_size(res)); - if (!priv->reg) { + priv->reg = devm_platform_ioremap_resource(ofdev, 0); + if (IS_ERR(priv->reg)) { dev_err(dev, "failed to of_iomap\n"); - err = -ENOMEM; + err = PTR_ERR(priv->reg); goto err_out; } /* get SEC version capabilities from device tree */ - of_property_read_u32(np, "fsl,num-channels", &priv->num_channels); of_property_read_u32(np, "fsl,channel-fifo-len", &priv->chfifo_len); of_property_read_u32(np, "fsl,exec-units-mask", &priv->exec_units); of_property_read_u32(np, "fsl,descriptor-types-mask", @@ -3511,16 +3513,6 @@ static int talitos_probe(struct platform_device *ofdev) } } - priv->chan = devm_kcalloc(dev, - priv->num_channels, - sizeof(struct talitos_channel), - GFP_KERNEL); - if (!priv->chan) { - dev_err(dev, "failed to allocate channel management space\n"); - err = -ENOMEM; - goto err_out; - } - priv->fifo_len = roundup_pow_of_two(priv->chfifo_len); for (i = 0; i < priv->num_channels; i++) { diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h index 1a93ee355929..d4ff8d589f46 100644 --- a/drivers/crypto/talitos.h +++ b/drivers/crypto/talitos.h @@ -44,11 +44,17 @@ struct talitos_desc { /* * talitos_edesc - s/w-extended descriptor + * @bufsl: scatterlist buffer + * @src: pointer to input scatterlist + * @first: first descriptor of a chain + * @last: last descriptor of a chain + * * @src_nents: number of segments in input scatterlist * @dst_nents: number of segments in output scatterlist * @iv_dma: dma address of iv for checking continuity and link table * @dma_len: length of dma mapped link_tbl space * @dma_link_tbl: bus physical address of link_tbl/buf + * @next_desc: next descriptor * @desc: h/w descriptor * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2) * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1) @@ -58,11 +64,17 @@ struct talitos_desc { * of link_tbl data */ struct talitos_edesc { + struct scatterlist bufsl[2]; + struct scatterlist *src; + int first; + int last; + int src_nents; int dst_nents; dma_addr_t iv_dma; int dma_len; dma_addr_t dma_link_tbl; + struct talitos_edesc *next_desc; struct talitos_desc desc; union { DECLARE_FLEX_ARRAY(struct talitos_ptr, link_tbl); @@ -139,8 +151,6 @@ struct talitos_private { */ unsigned int fifo_len; - struct talitos_channel *chan; - /* next channel to be assigned next incoming descriptor */ atomic_t last_chan ____cacheline_aligned; @@ -153,6 +163,9 @@ struct talitos_private { /* hwrng device */ struct hwrng rng; bool rng_registered; + + struct talitos_channel chan[] __counted_by(num_channels); + }; /* .features flag */ diff --git a/drivers/crypto/tegra/tegra-se-aes.c b/drivers/crypto/tegra/tegra-se-aes.c index 30c78afe3dea..9094c03e991f 100644 --- a/drivers/crypto/tegra/tegra-se-aes.c +++ b/drivers/crypto/tegra/tegra-se-aes.c @@ -1201,6 +1201,7 @@ static int tegra_ccm_do_one_req(struct crypto_engine *engine, void *areq) struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm); struct tegra_se *se = ctx->se; + unsigned int bufsize; int ret; ret = tegra_ccm_crypt_init(req, se, rctx); @@ -1210,19 +1211,19 @@ static int tegra_ccm_do_one_req(struct crypto_engine *engine, void *areq) rctx->key_id = ctx->key_id; /* Allocate buffers required */ - rctx->inbuf.size = rctx->assoclen + rctx->authsize + rctx->cryptlen + 100; - rctx->inbuf.buf = dma_alloc_coherent(ctx->se->dev, rctx->inbuf.size, + bufsize = rctx->assoclen + rctx->authsize + rctx->cryptlen + 100; + rctx->inbuf.size = bufsize; + rctx->inbuf.buf = dma_alloc_coherent(ctx->se->dev, bufsize, &rctx->inbuf.addr, GFP_KERNEL); + ret = -ENOMEM; if (!rctx->inbuf.buf) goto out_finalize; - rctx->outbuf.size = rctx->assoclen + rctx->authsize + rctx->cryptlen + 100; - rctx->outbuf.buf = dma_alloc_coherent(ctx->se->dev, rctx->outbuf.size, + rctx->outbuf.size = bufsize; + rctx->outbuf.buf = dma_alloc_coherent(ctx->se->dev, bufsize, &rctx->outbuf.addr, GFP_KERNEL); - if (!rctx->outbuf.buf) { - ret = -ENOMEM; + if (!rctx->outbuf.buf) goto out_free_inbuf; - } if (!ctx->key_id) { ret = tegra_key_submit_reserved_aes(ctx->se, ctx->key, @@ -1254,11 +1255,11 @@ static int tegra_ccm_do_one_req(struct crypto_engine *engine, void *areq) } out: - dma_free_coherent(ctx->se->dev, rctx->inbuf.size, + dma_free_coherent(ctx->se->dev, bufsize, rctx->outbuf.buf, rctx->outbuf.addr); out_free_inbuf: - dma_free_coherent(ctx->se->dev, rctx->outbuf.size, + dma_free_coherent(ctx->se->dev, bufsize, rctx->inbuf.buf, rctx->inbuf.addr); if (tegra_key_is_reserved(rctx->key_id)) @@ -1278,6 +1279,7 @@ static int tegra_gcm_do_one_req(struct crypto_engine *engine, void *areq) struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm); struct tegra_aead_reqctx *rctx = aead_request_ctx(req); + unsigned int bufsize; int ret; rctx->src_sg = req->src; @@ -1296,16 +1298,17 @@ static int tegra_gcm_do_one_req(struct crypto_engine *engine, void *areq) rctx->key_id = ctx->key_id; /* Allocate buffers required */ - rctx->inbuf.size = rctx->assoclen + rctx->authsize + rctx->cryptlen; - rctx->inbuf.buf = dma_alloc_coherent(ctx->se->dev, rctx->inbuf.size, + bufsize = rctx->assoclen + rctx->authsize + rctx->cryptlen; + rctx->inbuf.size = bufsize; + rctx->inbuf.buf = dma_alloc_coherent(ctx->se->dev, bufsize, &rctx->inbuf.addr, GFP_KERNEL); if (!rctx->inbuf.buf) { ret = -ENOMEM; goto out_finalize; } - rctx->outbuf.size = rctx->assoclen + rctx->authsize + rctx->cryptlen; - rctx->outbuf.buf = dma_alloc_coherent(ctx->se->dev, rctx->outbuf.size, + rctx->outbuf.size = bufsize; + rctx->outbuf.buf = dma_alloc_coherent(ctx->se->dev, bufsize, &rctx->outbuf.addr, GFP_KERNEL); if (!rctx->outbuf.buf) { ret = -ENOMEM; @@ -1342,11 +1345,11 @@ static int tegra_gcm_do_one_req(struct crypto_engine *engine, void *areq) ret = tegra_gcm_do_verify(ctx->se, rctx); out: - dma_free_coherent(ctx->se->dev, rctx->outbuf.size, + dma_free_coherent(ctx->se->dev, bufsize, rctx->outbuf.buf, rctx->outbuf.addr); out_free_inbuf: - dma_free_coherent(ctx->se->dev, rctx->inbuf.size, + dma_free_coherent(ctx->se->dev, bufsize, rctx->inbuf.buf, rctx->inbuf.addr); if (tegra_key_is_reserved(rctx->key_id)) diff --git a/drivers/crypto/tegra/tegra-se-main.c b/drivers/crypto/tegra/tegra-se-main.c index eb71113ed146..d2f518ef9a10 100644 --- a/drivers/crypto/tegra/tegra-se-main.c +++ b/drivers/crypto/tegra/tegra-se-main.c @@ -52,7 +52,7 @@ tegra_se_cmdbuf_pin(struct device *dev, struct host1x_bo *bo, enum dma_data_dire return ERR_PTR(-ENOMEM); kref_init(&map->ref); - map->bo = host1x_bo_get(bo); + map->bo = bo; map->direction = direction; map->dev = dev; @@ -93,7 +93,6 @@ static void tegra_se_cmdbuf_unpin(struct host1x_bo_mapping *map) dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0); sg_free_table(map->sgt); kfree(map->sgt); - host1x_bo_put(map->bo); kfree(map); } @@ -180,7 +179,7 @@ int tegra_se_host1x_submit(struct tegra_se *se, struct tegra_se_cmdbuf *cmdbuf, MAX_SCHEDULE_TIMEOUT, NULL); if (ret) { dev_err(se->dev, "host1x job timed out\n"); - return ret; + goto job_put; } host1x_job_put(job); diff --git a/drivers/crypto/xilinx/Makefile b/drivers/crypto/xilinx/Makefile index 9b51636ef75e..730feff5b5f2 100644 --- a/drivers/crypto/xilinx/Makefile +++ b/drivers/crypto/xilinx/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_CRYPTO_DEV_XILINX_TRNG) += xilinx-trng.o obj-$(CONFIG_CRYPTO_DEV_ZYNQMP_AES) += zynqmp-aes-gcm.o obj-$(CONFIG_CRYPTO_DEV_ZYNQMP_SHA3) += zynqmp-sha.o diff --git a/include/crypto/df_sp80090a.h b/include/crypto/df_sp80090a.h deleted file mode 100644 index cb5d6fe15d40..000000000000 --- a/include/crypto/df_sp80090a.h +++ /dev/null @@ -1,28 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - -/* - * Copyright Stephan Mueller , 2014 - */ - -#ifndef _CRYPTO_DF80090A_H -#define _CRYPTO_DF80090A_H - -#include -#include - -static inline int crypto_drbg_ctr_df_datalen(u8 statelen, u8 blocklen) -{ - return statelen + /* df_data */ - blocklen + /* pad */ - blocklen + /* iv */ - statelen + blocklen; /* temp */ -} - -int crypto_drbg_ctr_df(struct aes_enckey *aes, - unsigned char *df_data, - size_t bytes_to_return, - struct list_head *seedlist, - u8 blocklen_bytes, - u8 statelen); - -#endif /* _CRYPTO_DF80090A_H */ diff --git a/include/crypto/drbg.h b/include/crypto/drbg.h deleted file mode 100644 index 2d42518cbdce..000000000000 --- a/include/crypto/drbg.h +++ /dev/null @@ -1,263 +0,0 @@ -/* - * DRBG based on NIST SP800-90A - * - * Copyright Stephan Mueller , 2014 - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, and the entire permission notice in its entirety, - * including the disclaimer of warranties. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The name of the author may not be used to endorse or promote - * products derived from this software without specific prior - * written permission. - * - * ALTERNATIVELY, this product may be distributed under the terms of - * the GNU General Public License, in which case the provisions of the GPL are - * required INSTEAD OF the above restrictions. (This clause is - * necessary due to a potential bad interaction between the GPL and - * the restrictions contained in a BSD-style copyright.) - * - * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF - * WHICH ARE HEREBY DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT - * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR - * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE - * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH - * DAMAGE. - */ - -#ifndef _DRBG_H -#define _DRBG_H - - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -struct drbg_state; -typedef uint32_t drbg_flag_t; - -struct drbg_core { - drbg_flag_t flags; /* flags for the cipher */ - __u8 statelen; /* maximum state length */ - __u8 blocklen_bytes; /* block size of output in bytes */ - char cra_name[CRYPTO_MAX_ALG_NAME]; /* mapping to kernel crypto API */ - /* kernel crypto API backend cipher name */ - char backend_cra_name[CRYPTO_MAX_ALG_NAME]; -}; - -struct drbg_state_ops { - int (*update)(struct drbg_state *drbg, struct list_head *seed, - int reseed); - int (*generate)(struct drbg_state *drbg, - unsigned char *buf, unsigned int buflen, - struct list_head *addtl); - int (*crypto_init)(struct drbg_state *drbg); - int (*crypto_fini)(struct drbg_state *drbg); - -}; - -struct drbg_test_data { - struct drbg_string *testentropy; /* TEST PARAMETER: test entropy */ -}; - -enum drbg_seed_state { - DRBG_SEED_STATE_UNSEEDED, - DRBG_SEED_STATE_PARTIAL, /* Seeded with !rng_is_initialized() */ - DRBG_SEED_STATE_FULL, -}; - -struct drbg_state { - struct mutex drbg_mutex; /* lock around DRBG */ - unsigned char *V; /* internal state 10.1.1.1 1a) */ - unsigned char *Vbuf; - /* hash: static value 10.1.1.1 1b) hmac / ctr: key */ - unsigned char *C; - unsigned char *Cbuf; - /* Number of RNG requests since last reseed -- 10.1.1.1 1c) */ - size_t reseed_ctr; - size_t reseed_threshold; - /* some memory the DRBG can use for its operation */ - unsigned char *scratchpad; - unsigned char *scratchpadbuf; - void *priv_data; /* Cipher handle */ - - struct crypto_skcipher *ctr_handle; /* CTR mode cipher handle */ - struct skcipher_request *ctr_req; /* CTR mode request handle */ - __u8 *outscratchpadbuf; /* CTR mode output scratchpad */ - __u8 *outscratchpad; /* CTR mode aligned outbuf */ - struct crypto_wait ctr_wait; /* CTR mode async wait obj */ - struct scatterlist sg_in, sg_out; /* CTR mode SGLs */ - - enum drbg_seed_state seeded; /* DRBG fully seeded? */ - unsigned long last_seed_time; - bool pr; /* Prediction resistance enabled? */ - bool fips_primed; /* Continuous test primed? */ - unsigned char *prev; /* FIPS 140-2 continuous test value */ - struct crypto_rng *jent; - const struct drbg_state_ops *d_ops; - const struct drbg_core *core; - struct drbg_string test_data; -}; - -static inline __u8 drbg_statelen(struct drbg_state *drbg) -{ - if (drbg && drbg->core) - return drbg->core->statelen; - return 0; -} - -static inline __u8 drbg_blocklen(struct drbg_state *drbg) -{ - if (drbg && drbg->core) - return drbg->core->blocklen_bytes; - return 0; -} - -static inline __u8 drbg_keylen(struct drbg_state *drbg) -{ - if (drbg && drbg->core) - return (drbg->core->statelen - drbg->core->blocklen_bytes); - return 0; -} - -static inline size_t drbg_max_request_bytes(struct drbg_state *drbg) -{ - /* SP800-90A requires the limit 2**19 bits, but we return bytes */ - return (1 << 16); -} - -static inline size_t drbg_max_addtl(struct drbg_state *drbg) -{ - /* SP800-90A requires 2**35 bytes additional info str / pers str */ -#if (__BITS_PER_LONG == 32) - /* - * SP800-90A allows smaller maximum numbers to be returned -- we - * return SIZE_MAX - 1 to allow the verification of the enforcement - * of this value in drbg_healthcheck_sanity. - */ - return (SIZE_MAX - 1); -#else - return (1UL<<35); -#endif -} - -static inline size_t drbg_max_requests(struct drbg_state *drbg) -{ - /* SP800-90A requires 2**48 maximum requests before reseeding */ - return (1<<20); -} - -/* - * This is a wrapper to the kernel crypto API function of - * crypto_rng_generate() to allow the caller to provide additional data. - * - * @drng DRBG handle -- see crypto_rng_get_bytes - * @outbuf output buffer -- see crypto_rng_get_bytes - * @outlen length of output buffer -- see crypto_rng_get_bytes - * @addtl_input additional information string input buffer - * @addtllen length of additional information string buffer - * - * return - * see crypto_rng_get_bytes - */ -static inline int crypto_drbg_get_bytes_addtl(struct crypto_rng *drng, - unsigned char *outbuf, unsigned int outlen, - struct drbg_string *addtl) -{ - return crypto_rng_generate(drng, addtl->buf, addtl->len, - outbuf, outlen); -} - -/* - * TEST code - * - * This is a wrapper to the kernel crypto API function of - * crypto_rng_generate() to allow the caller to provide additional data and - * allow furnishing of test_data - * - * @drng DRBG handle -- see crypto_rng_get_bytes - * @outbuf output buffer -- see crypto_rng_get_bytes - * @outlen length of output buffer -- see crypto_rng_get_bytes - * @addtl_input additional information string input buffer - * @addtllen length of additional information string buffer - * @test_data filled test data - * - * return - * see crypto_rng_get_bytes - */ -static inline int crypto_drbg_get_bytes_addtl_test(struct crypto_rng *drng, - unsigned char *outbuf, unsigned int outlen, - struct drbg_string *addtl, - struct drbg_test_data *test_data) -{ - crypto_rng_set_entropy(drng, test_data->testentropy->buf, - test_data->testentropy->len); - return crypto_rng_generate(drng, addtl->buf, addtl->len, - outbuf, outlen); -} - -/* - * TEST code - * - * This is a wrapper to the kernel crypto API function of - * crypto_rng_reset() to allow the caller to provide test_data - * - * @drng DRBG handle -- see crypto_rng_reset - * @pers personalization string input buffer - * @perslen length of additional information string buffer - * @test_data filled test data - * - * return - * see crypto_rng_reset - */ -static inline int crypto_drbg_reset_test(struct crypto_rng *drng, - struct drbg_string *pers, - struct drbg_test_data *test_data) -{ - crypto_rng_set_entropy(drng, test_data->testentropy->buf, - test_data->testentropy->len); - return crypto_rng_reset(drng, pers->buf, pers->len); -} - -/* DRBG type flags */ -#define DRBG_CTR ((drbg_flag_t)1<<0) -#define DRBG_HMAC ((drbg_flag_t)1<<1) -#define DRBG_HASH ((drbg_flag_t)1<<2) -#define DRBG_TYPE_MASK (DRBG_CTR | DRBG_HMAC | DRBG_HASH) -/* DRBG strength flags */ -#define DRBG_STRENGTH128 ((drbg_flag_t)1<<3) -#define DRBG_STRENGTH192 ((drbg_flag_t)1<<4) -#define DRBG_STRENGTH256 ((drbg_flag_t)1<<5) -#define DRBG_STRENGTH_MASK (DRBG_STRENGTH128 | DRBG_STRENGTH192 | \ - DRBG_STRENGTH256) - -enum drbg_prefixes { - DRBG_PREFIX0 = 0x00, - DRBG_PREFIX1, - DRBG_PREFIX2, - DRBG_PREFIX3 -}; - -#endif /* _DRBG_H */ diff --git a/include/crypto/if_alg.h b/include/crypto/if_alg.h index 0cc8fa749f68..7643ba954125 100644 --- a/include/crypto/if_alg.h +++ b/include/crypto/if_alg.h @@ -41,7 +41,7 @@ struct af_alg_control { }; struct af_alg_type { - void *(*bind)(const char *name, u32 type, u32 mask); + void *(*bind)(const char *name); void (*release)(void *private); int (*setkey)(void *private, const u8 *key, unsigned int keylen); int (*setentropy)(void *private, sockptr_t entropy, unsigned int len); @@ -80,7 +80,6 @@ struct af_alg_rsgl { /** * struct af_alg_async_req - definition of crypto request - * @iocb: IOCB for AIO operations * @sk: Socket the request is associated with * @first_rsgl: First RX SG * @last_rsgl: Pointer to last RX SG @@ -92,7 +91,6 @@ struct af_alg_rsgl { * @cra_u: Cipher request */ struct af_alg_async_req { - struct kiocb *iocb; struct sock *sk; struct af_alg_rsgl first_rsgl; @@ -138,7 +136,7 @@ struct af_alg_async_req { * @write: True if we are in the middle of a write. * @init: True if metadata has been sent. * @len: Length of memory allocated for this data structure. - * @inflight: Non-zero when AIO requests are in flight. + * @inflight: Non-zero when requests are in flight, for debugging only. */ struct af_alg_ctx { struct list_head tsgl_list; @@ -237,7 +235,6 @@ int af_alg_wait_for_data(struct sock *sk, unsigned flags, unsigned min); int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size, unsigned int ivsize); void af_alg_free_resources(struct af_alg_async_req *areq); -void af_alg_async_cb(void *data, int err); __poll_t af_alg_poll(struct file *file, struct socket *sock, poll_table *wait); struct af_alg_async_req *af_alg_alloc_areq(struct sock *sk, @@ -246,4 +243,16 @@ int af_alg_get_rsgl(struct sock *sk, struct msghdr *msg, int flags, struct af_alg_async_req *areq, size_t maxsize, size_t *outlen); +/* + * Mask used to disable unsupported algorithm implementations. + * + * This is the same as FSCRYPT_CRYPTOAPI_MASK in fs/crypto/fscrypt_private.h. + * In additions to the motivations there, this API is exposed to userspace + * that might not be fully trusted. + */ +#define AF_ALG_CRYPTOAPI_MASK \ + (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | \ + CRYPTO_ALG_KERN_DRIVER_ONLY) + + #endif /* _CRYPTO_IF_ALG_H */ diff --git a/include/crypto/internal/drbg.h b/include/crypto/internal/drbg.h deleted file mode 100644 index 371e52dcee6c..000000000000 --- a/include/crypto/internal/drbg.h +++ /dev/null @@ -1,54 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - -/* - * NIST SP800-90A DRBG derivation function - * - * Copyright (C) 2014, Stephan Mueller - */ - -#ifndef _INTERNAL_DRBG_H -#define _INTERNAL_DRBG_H - -/* - * Convert an integer into a byte representation of this integer. - * The byte representation is big-endian - * - * @val value to be converted - * @buf buffer holding the converted integer -- caller must ensure that - * buffer size is at least 32 bit - */ -static inline void drbg_cpu_to_be32(__u32 val, unsigned char *buf) -{ - struct s { - __be32 conv; - }; - struct s *conversion = (struct s *)buf; - - conversion->conv = cpu_to_be32(val); -} - -/* - * Concatenation Helper and string operation helper - * - * SP800-90A requires the concatenation of different data. To avoid copying - * buffers around or allocate additional memory, the following data structure - * is used to point to the original memory with its size. In addition, it - * is used to build a linked list. The linked list defines the concatenation - * of individual buffers. The order of memory block referenced in that - * linked list determines the order of concatenation. - */ -struct drbg_string { - const unsigned char *buf; - size_t len; - struct list_head list; -}; - -static inline void drbg_string_fill(struct drbg_string *string, - const unsigned char *buf, size_t len) -{ - string->buf = buf; - string->len = len; - INIT_LIST_HEAD(&string->list); -} - -#endif //_INTERNAL_DRBG_H diff --git a/include/linux/hisi_acc_qm.h b/include/linux/hisi_acc_qm.h index a6268dc4f7cb..f7570a409905 100644 --- a/include/linux/hisi_acc_qm.h +++ b/include/linux/hisi_acc_qm.h @@ -115,10 +115,22 @@ #define QM_ECC_MBIT BIT(2) +/** + * enum qm_stop_reason - Queue manager stop reasons + * @QM_NORMAL: Graceful stop. Used for device unbind, driver removal, + * or runtime power management (runtime_suspend). + * @QM_SOFT_RESET: Error recovery reset. Triggered by unrecoverable hardware + * errors (e.g., PCIe AER, timeout) to recover device state. + * @QM_DOWN: Function Level Reset. Used when the device needs to + * be reset at the function level without resetting the link. + * @QM_SHUTDOWN: System shutdown. Used during system poweroff, reboot, or + * kexec to ensure hardware is in a safe state. + */ enum qm_stop_reason { QM_NORMAL, QM_SOFT_RESET, QM_DOWN, + QM_SHUTDOWN, }; enum qm_state { @@ -158,7 +170,6 @@ enum qm_vf_state { enum qm_misc_ctl_bits { QM_DRIVER_REMOVING = 0x0, - QM_RST_SCHED, QM_RESETTING, QM_MODULE_PARAM, }; @@ -249,6 +260,7 @@ enum acc_err_result { ACC_ERR_NONE, ACC_ERR_NEED_RESET, ACC_ERR_RECOVERED, + ACC_ERR_NEED_FUNC_RESET, }; struct hisi_qm_err_mask { @@ -552,6 +564,7 @@ static inline void hisi_qm_del_list(struct hisi_qm *qm, struct hisi_qm_list *qm_ mutex_unlock(&qm_list->lock); } +int hisi_qm_register_uacce(struct hisi_qm *qm); int hisi_qm_q_num_set(const char *val, const struct kernel_param *kp, unsigned int device); int hisi_qm_init(struct hisi_qm *qm); diff --git a/include/linux/platform_data/crypto-ux500.h b/include/linux/platform_data/crypto-ux500.h deleted file mode 100644 index 5d43350e32cc..000000000000 --- a/include/linux/platform_data/crypto-ux500.h +++ /dev/null @@ -1,22 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) ST-Ericsson SA 2011 - * - * Author: Joakim Bech for ST-Ericsson - */ -#ifndef _CRYPTO_UX500_H -#define _CRYPTO_UX500_H -#include -#include - -struct hash_platform_data { - void *mem_to_engine; - bool (*dma_filter)(struct dma_chan *chan, void *filter_param); -}; - -struct cryp_platform_data { - struct stedma40_chan_cfg mem_to_engine; - struct stedma40_chan_cfg engine_to_mem; -}; - -#endif diff --git a/include/linux/socket.h b/include/linux/socket.h index ec4a0a025793..3ffdfe184b23 100644 --- a/include/linux/socket.h +++ b/include/linux/socket.h @@ -89,7 +89,6 @@ struct msghdr { bool msg_get_inq : 1;/* return INQ after receive */ unsigned int msg_flags; /* flags on received message */ __kernel_size_t msg_controllen; /* ancillary data buffer length */ - struct kiocb *msg_iocb; /* ptr to iocb for async requests */ struct ubuf_info *msg_ubuf; int (*sg_from_iter)(struct sk_buff *skb, struct iov_iter *from, size_t length); diff --git a/io_uring/net.c b/io_uring/net.c index ee848eb65ec9..34ab1f97cc5a 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -772,7 +772,6 @@ static int io_recvmsg_prep_setup(struct io_kiocb *req) kmsg->msg.msg_control = NULL; kmsg->msg.msg_get_inq = 1; kmsg->msg.msg_controllen = 0; - kmsg->msg.msg_iocb = NULL; kmsg->msg.msg_ubuf = NULL; if (req->flags & REQ_F_BUFFER_SELECT) diff --git a/net/compat.c b/net/compat.c index 2c9bd0edac99..d68cf9c3aad5 100644 --- a/net/compat.c +++ b/net/compat.c @@ -75,7 +75,6 @@ int __get_compat_msghdr(struct msghdr *kmsg, if (msg->msg_iovlen > UIO_MAXIOV) return -EMSGSIZE; - kmsg->msg_iocb = NULL; kmsg->msg_ubuf = NULL; return 0; } diff --git a/net/socket.c b/net/socket.c index f51bdcbaa43f..9e8dc769ff7a 100644 --- a/net/socket.c +++ b/net/socket.c @@ -1202,8 +1202,7 @@ static ssize_t sock_read_iter(struct kiocb *iocb, struct iov_iter *to) { struct file *file = iocb->ki_filp; struct socket *sock = file->private_data; - struct msghdr msg = {.msg_iter = *to, - .msg_iocb = iocb}; + struct msghdr msg = {.msg_iter = *to}; ssize_t res; if (file->f_flags & O_NONBLOCK || (iocb->ki_flags & IOCB_NOWAIT)) @@ -1224,8 +1223,7 @@ static ssize_t sock_write_iter(struct kiocb *iocb, struct iov_iter *from) { struct file *file = iocb->ki_filp; struct socket *sock = file->private_data; - struct msghdr msg = {.msg_iter = *from, - .msg_iocb = iocb}; + struct msghdr msg = {.msg_iter = *from}; ssize_t res; if (iocb->ki_pos != 0) @@ -2601,7 +2599,6 @@ int __copy_msghdr(struct msghdr *kmsg, if (msg->msg_iovlen > UIO_MAXIOV) return -EMSGSIZE; - kmsg->msg_iocb = NULL; kmsg->msg_ubuf = NULL; return 0; } diff --git a/tools/perf/trace/beauty/include/linux/socket.h b/tools/perf/trace/beauty/include/linux/socket.h index ec4a0a025793..3ffdfe184b23 100644 --- a/tools/perf/trace/beauty/include/linux/socket.h +++ b/tools/perf/trace/beauty/include/linux/socket.h @@ -89,7 +89,6 @@ struct msghdr { bool msg_get_inq : 1;/* return INQ after receive */ unsigned int msg_flags; /* flags on received message */ __kernel_size_t msg_controllen; /* ancillary data buffer length */ - struct kiocb *msg_iocb; /* ptr to iocb for async requests */ struct ubuf_info *msg_ubuf; int (*sg_from_iter)(struct sk_buff *skb, struct iov_iter *from, size_t length); diff --git a/tools/testing/crypto/chacha20-s390/test-cipher.c b/tools/testing/crypto/chacha20-s390/test-cipher.c index 827507844e8f..9f61454ed077 100644 --- a/tools/testing/crypto/chacha20-s390/test-cipher.c +++ b/tools/testing/crypto/chacha20-s390/test-cipher.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include