lib/crypto: arm64/sm3: Migrate optimized code into library
Instead of exposing the arm64-optimized SM3 code via arm64-specific crypto_shash algorithms, instead just implement the sm3_blocks() library function. This is much simpler, it makes the SM3 library functions be arm64-optimized, and it fixes the longstanding issue where the arm64-optimized SM3 code was disabled by default. SM3 still remains available through crypto_shash, but individual architectures no longer need to handle it. Tweak the SM3 assembly function prototypes to match what the library expects, including changing the block count from 'int' to 'size_t'. sm3_ce_transform() had to be updated to access 'x2' instead of 'w2', while sm3_neon_transform() already used 'x2'. Remove the CFI stubs which are no longer needed because the SM3 assembly functions are no longer ever indirectly called. Remove the dependency on KERNEL_MODE_NEON. It was unnecessary, because KERNEL_MODE_NEON is always enabled on arm64. Acked-by: Ard Biesheuvel <ardb@kernel.org> Link: https://lore.kernel.org/r/20260321040935.410034-8-ebiggers@kernel.org Signed-off-by: Eric Biggers <ebiggers@kernel.org>
This commit is contained in:
Eric Biggers committed
1 parent
ed065bd06e
commit
9f69f52b46
10 files changed
+62
-180
No files matched your search
@@ -279,6 +279,7 @@ config CRYPTO_LIB_SM3
|
||||
config CRYPTO_LIB_SM3_ARCH
|
||||
bool
|
||||
depends on CRYPTO_LIB_SM3 && !UML
|
||||
default y if ARM64
|
||||
|
||||
source "lib/crypto/tests/Kconfig"
|
||||
|
||||
|
||||
+10
-3
@@ -368,13 +368,20 @@ endif # CONFIG_CRYPTO_LIB_SHA3_ARCH
|
||||
|
||||
################################################################################
|
||||
|
||||
obj-$(CONFIG_CRYPTO_LIB_SM3) += libsm3.o
|
||||
libsm3-y := sm3.o
|
||||
ifeq ($(CONFIG_CRYPTO_LIB_SM3_ARCH),y)
|
||||
CFLAGS_sm3.o += -I$(src)/$(SRCARCH)
|
||||
libsm3-$(CONFIG_ARM64) += arm64/sm3-ce-core.o \
|
||||
arm64/sm3-neon-core.o
|
||||
endif # CONFIG_CRYPTO_LIB_SM3_ARCH
|
||||
|
||||
################################################################################
|
||||
|
||||
obj-$(CONFIG_MPILIB) += mpi/
|
||||
|
||||
obj-$(CONFIG_CRYPTO_SELFTESTS_FULL) += simd.o
|
||||
|
||||
obj-$(CONFIG_CRYPTO_LIB_SM3) += libsm3.o
|
||||
libsm3-y := sm3.o
|
||||
|
||||
# clean-files must be defined unconditionally
|
||||
clean-files += arm/sha256-core.S arm/sha512-core.S
|
||||
clean-files += arm64/sha256-core.S arm64/sha512-core.S
|
||||
@@ -0,0 +1,138 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* sm3-ce-core.S - SM3 secure hash using ARMv8.2 Crypto Extensions
|
||||
*
|
||||
* Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
|
||||
*/
|
||||
|
||||
#include <linux/linkage.h>
|
||||
#include <asm/assembler.h>
|
||||
|
||||
.irp b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12
|
||||
.set .Lv\b\().4s, \b
|
||||
.endr
|
||||
|
||||
.macro sm3partw1, rd, rn, rm
|
||||
.inst 0xce60c000 | .L\rd | (.L\rn << 5) | (.L\rm << 16)
|
||||
.endm
|
||||
|
||||
.macro sm3partw2, rd, rn, rm
|
||||
.inst 0xce60c400 | .L\rd | (.L\rn << 5) | (.L\rm << 16)
|
||||
.endm
|
||||
|
||||
.macro sm3ss1, rd, rn, rm, ra
|
||||
.inst 0xce400000 | .L\rd | (.L\rn << 5) | (.L\ra << 10) | (.L\rm << 16)
|
||||
.endm
|
||||
|
||||
.macro sm3tt1a, rd, rn, rm, imm2
|
||||
.inst 0xce408000 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
|
||||
.endm
|
||||
|
||||
.macro sm3tt1b, rd, rn, rm, imm2
|
||||
.inst 0xce408400 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
|
||||
.endm
|
||||
|
||||
.macro sm3tt2a, rd, rn, rm, imm2
|
||||
.inst 0xce408800 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
|
||||
.endm
|
||||
|
||||
.macro sm3tt2b, rd, rn, rm, imm2
|
||||
.inst 0xce408c00 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
|
||||
.endm
|
||||
|
||||
.macro round, ab, s0, t0, t1, i
|
||||
sm3ss1 v5.4s, v8.4s, \t0\().4s, v9.4s
|
||||
shl \t1\().4s, \t0\().4s, #1
|
||||
sri \t1\().4s, \t0\().4s, #31
|
||||
sm3tt1\ab v8.4s, v5.4s, v10.4s, \i
|
||||
sm3tt2\ab v9.4s, v5.4s, \s0\().4s, \i
|
||||
.endm
|
||||
|
||||
.macro qround, ab, s0, s1, s2, s3, s4
|
||||
.ifnb \s4
|
||||
ext \s4\().16b, \s1\().16b, \s2\().16b, #12
|
||||
ext v6.16b, \s0\().16b, \s1\().16b, #12
|
||||
ext v7.16b, \s2\().16b, \s3\().16b, #8
|
||||
sm3partw1 \s4\().4s, \s0\().4s, \s3\().4s
|
||||
.endif
|
||||
|
||||
eor v10.16b, \s0\().16b, \s1\().16b
|
||||
|
||||
round \ab, \s0, v11, v12, 0
|
||||
round \ab, \s0, v12, v11, 1
|
||||
round \ab, \s0, v11, v12, 2
|
||||
round \ab, \s0, v12, v11, 3
|
||||
|
||||
.ifnb \s4
|
||||
sm3partw2 \s4\().4s, v7.4s, v6.4s
|
||||
.endif
|
||||
.endm
|
||||
|
||||
/*
|
||||
* void sm3_ce_transform(struct sm3_block_state *state,
|
||||
* const u8 *data, size_t nblocks)
|
||||
*/
|
||||
.text
|
||||
SYM_FUNC_START(sm3_ce_transform)
|
||||
/* load state */
|
||||
ld1 {v8.4s-v9.4s}, [x0]
|
||||
rev64 v8.4s, v8.4s
|
||||
rev64 v9.4s, v9.4s
|
||||
ext v8.16b, v8.16b, v8.16b, #8
|
||||
ext v9.16b, v9.16b, v9.16b, #8
|
||||
|
||||
adr_l x8, .Lt
|
||||
ldp s13, s14, [x8]
|
||||
|
||||
/* load input */
|
||||
0: ld1 {v0.16b-v3.16b}, [x1], #64
|
||||
sub x2, x2, #1
|
||||
|
||||
mov v15.16b, v8.16b
|
||||
mov v16.16b, v9.16b
|
||||
|
||||
CPU_LE( rev32 v0.16b, v0.16b )
|
||||
CPU_LE( rev32 v1.16b, v1.16b )
|
||||
CPU_LE( rev32 v2.16b, v2.16b )
|
||||
CPU_LE( rev32 v3.16b, v3.16b )
|
||||
|
||||
ext v11.16b, v13.16b, v13.16b, #4
|
||||
|
||||
qround a, v0, v1, v2, v3, v4
|
||||
qround a, v1, v2, v3, v4, v0
|
||||
qround a, v2, v3, v4, v0, v1
|
||||
qround a, v3, v4, v0, v1, v2
|
||||
|
||||
ext v11.16b, v14.16b, v14.16b, #4
|
||||
|
||||
qround b, v4, v0, v1, v2, v3
|
||||
qround b, v0, v1, v2, v3, v4
|
||||
qround b, v1, v2, v3, v4, v0
|
||||
qround b, v2, v3, v4, v0, v1
|
||||
qround b, v3, v4, v0, v1, v2
|
||||
qround b, v4, v0, v1, v2, v3
|
||||
qround b, v0, v1, v2, v3, v4
|
||||
qround b, v1, v2, v3, v4, v0
|
||||
qround b, v2, v3, v4, v0, v1
|
||||
qround b, v3, v4
|
||||
qround b, v4, v0
|
||||
qround b, v0, v1
|
||||
|
||||
eor v8.16b, v8.16b, v15.16b
|
||||
eor v9.16b, v9.16b, v16.16b
|
||||
|
||||
/* handled all input blocks? */
|
||||
cbnz x2, 0b
|
||||
|
||||
/* save state */
|
||||
rev64 v8.4s, v8.4s
|
||||
rev64 v9.4s, v9.4s
|
||||
ext v8.16b, v8.16b, v8.16b, #8
|
||||
ext v9.16b, v9.16b, v9.16b, #8
|
||||
st1 {v8.4s-v9.4s}, [x0]
|
||||
ret
|
||||
SYM_FUNC_END(sm3_ce_transform)
|
||||
|
||||
.section ".rodata", "a"
|
||||
.align 3
|
||||
.Lt: .word 0x79cc4519, 0x9d8a7a87
|
||||
@@ -0,0 +1,600 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
/*
|
||||
* sm3-neon-core.S - SM3 secure hash using NEON instructions
|
||||
*
|
||||
* Linux/arm64 port of the libgcrypt SM3 implementation for AArch64
|
||||
*
|
||||
* Copyright (C) 2021 Jussi Kivilinna <jussi.kivilinna@iki.fi>
|
||||
* Copyright (c) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
|
||||
*/
|
||||
|
||||
#include <linux/linkage.h>
|
||||
#include <asm/assembler.h>
|
||||
|
||||
/* Context structure */
|
||||
|
||||
#define state_h0 0
|
||||
#define state_h1 4
|
||||
#define state_h2 8
|
||||
#define state_h3 12
|
||||
#define state_h4 16
|
||||
#define state_h5 20
|
||||
#define state_h6 24
|
||||
#define state_h7 28
|
||||
|
||||
/* Stack structure */
|
||||
|
||||
#define STACK_W_SIZE (32 * 2 * 3)
|
||||
|
||||
#define STACK_W (0)
|
||||
#define STACK_SIZE (STACK_W + STACK_W_SIZE)
|
||||
|
||||
/* Register macros */
|
||||
|
||||
#define RSTATE x0
|
||||
#define RDATA x1
|
||||
#define RNBLKS x2
|
||||
#define RKPTR x28
|
||||
#define RFRAME x29
|
||||
|
||||
#define ra w3
|
||||
#define rb w4
|
||||
#define rc w5
|
||||
#define rd w6
|
||||
#define re w7
|
||||
#define rf w8
|
||||
#define rg w9
|
||||
#define rh w10
|
||||
|
||||
#define t0 w11
|
||||
#define t1 w12
|
||||
#define t2 w13
|
||||
#define t3 w14
|
||||
#define t4 w15
|
||||
#define t5 w16
|
||||
#define t6 w17
|
||||
|
||||
#define k_even w19
|
||||
#define k_odd w20
|
||||
|
||||
#define addr0 x21
|
||||
#define addr1 x22
|
||||
|
||||
#define s0 w23
|
||||
#define s1 w24
|
||||
#define s2 w25
|
||||
#define s3 w26
|
||||
|
||||
#define W0 v0
|
||||
#define W1 v1
|
||||
#define W2 v2
|
||||
#define W3 v3
|
||||
#define W4 v4
|
||||
#define W5 v5
|
||||
|
||||
#define XTMP0 v6
|
||||
#define XTMP1 v7
|
||||
#define XTMP2 v16
|
||||
#define XTMP3 v17
|
||||
#define XTMP4 v18
|
||||
#define XTMP5 v19
|
||||
#define XTMP6 v20
|
||||
|
||||
/* Helper macros. */
|
||||
|
||||
#define _(...) /*_*/
|
||||
|
||||
#define clear_vec(x) \
|
||||
movi x.8h, #0;
|
||||
|
||||
#define rolw(o, a, n) \
|
||||
ror o, a, #(32 - n);
|
||||
|
||||
/* Round function macros. */
|
||||
|
||||
#define GG1_1(x, y, z, o, t) \
|
||||
eor o, x, y;
|
||||
#define GG1_2(x, y, z, o, t) \
|
||||
eor o, o, z;
|
||||
#define GG1_3(x, y, z, o, t)
|
||||
|
||||
#define FF1_1(x, y, z, o, t) GG1_1(x, y, z, o, t)
|
||||
#define FF1_2(x, y, z, o, t)
|
||||
#define FF1_3(x, y, z, o, t) GG1_2(x, y, z, o, t)
|
||||
|
||||
#define GG2_1(x, y, z, o, t) \
|
||||
bic o, z, x;
|
||||
#define GG2_2(x, y, z, o, t) \
|
||||
and t, y, x;
|
||||
#define GG2_3(x, y, z, o, t) \
|
||||
eor o, o, t;
|
||||
|
||||
#define FF2_1(x, y, z, o, t) \
|
||||
eor o, x, y;
|
||||
#define FF2_2(x, y, z, o, t) \
|
||||
and t, x, y; \
|
||||
and o, o, z;
|
||||
#define FF2_3(x, y, z, o, t) \
|
||||
eor o, o, t;
|
||||
|
||||
#define R(i, a, b, c, d, e, f, g, h, k, K_LOAD, round, widx, wtype, IOP, iop_param) \
|
||||
K_LOAD(round); \
|
||||
ldr t5, [sp, #(wtype##_W1_ADDR(round, widx))]; \
|
||||
rolw(t0, a, 12); /* rol(a, 12) => t0 */ \
|
||||
IOP(1, iop_param); \
|
||||
FF##i##_1(a, b, c, t1, t2); \
|
||||
ldr t6, [sp, #(wtype##_W1W2_ADDR(round, widx))]; \
|
||||
add k, k, e; \
|
||||
IOP(2, iop_param); \
|
||||
GG##i##_1(e, f, g, t3, t4); \
|
||||
FF##i##_2(a, b, c, t1, t2); \
|
||||
IOP(3, iop_param); \
|
||||
add k, k, t0; \
|
||||
add h, h, t5; \
|
||||
add d, d, t6; /* w1w2 + d => d */ \
|
||||
IOP(4, iop_param); \
|
||||
rolw(k, k, 7); /* rol (t0 + e + t), 7) => k */ \
|
||||
GG##i##_2(e, f, g, t3, t4); \
|
||||
add h, h, k; /* h + w1 + k => h */ \
|
||||
IOP(5, iop_param); \
|
||||
FF##i##_3(a, b, c, t1, t2); \
|
||||
eor t0, t0, k; /* k ^ t0 => t0 */ \
|
||||
GG##i##_3(e, f, g, t3, t4); \
|
||||
add d, d, t1; /* FF(a,b,c) + d => d */ \
|
||||
IOP(6, iop_param); \
|
||||
add t3, t3, h; /* GG(e,f,g) + h => t3 */ \
|
||||
rolw(b, b, 9); /* rol(b, 9) => b */ \
|
||||
eor h, t3, t3, ror #(32-9); \
|
||||
IOP(7, iop_param); \
|
||||
add d, d, t0; /* t0 + d => d */ \
|
||||
rolw(f, f, 19); /* rol(f, 19) => f */ \
|
||||
IOP(8, iop_param); \
|
||||
eor h, h, t3, ror #(32-17); /* P0(t3) => h */
|
||||
|
||||
#define R1(a, b, c, d, e, f, g, h, k, K_LOAD, round, widx, wtype, IOP, iop_param) \
|
||||
R(1, ##a, ##b, ##c, ##d, ##e, ##f, ##g, ##h, ##k, K_LOAD, round, widx, wtype, IOP, iop_param)
|
||||
|
||||
#define R2(a, b, c, d, e, f, g, h, k, K_LOAD, round, widx, wtype, IOP, iop_param) \
|
||||
R(2, ##a, ##b, ##c, ##d, ##e, ##f, ##g, ##h, ##k, K_LOAD, round, widx, wtype, IOP, iop_param)
|
||||
|
||||
#define KL(round) \
|
||||
ldp k_even, k_odd, [RKPTR, #(4*(round))];
|
||||
|
||||
/* Input expansion macros. */
|
||||
|
||||
/* Byte-swapped input address. */
|
||||
#define IW_W_ADDR(round, widx, offs) \
|
||||
(STACK_W + ((round) / 4) * 64 + (offs) + ((widx) * 4))
|
||||
|
||||
/* Expanded input address. */
|
||||
#define XW_W_ADDR(round, widx, offs) \
|
||||
(STACK_W + ((((round) / 3) - 4) % 2) * 64 + (offs) + ((widx) * 4))
|
||||
|
||||
/* Rounds 1-12, byte-swapped input block addresses. */
|
||||
#define IW_W1_ADDR(round, widx) IW_W_ADDR(round, widx, 32)
|
||||
#define IW_W1W2_ADDR(round, widx) IW_W_ADDR(round, widx, 48)
|
||||
|
||||
/* Rounds 1-12, expanded input block addresses. */
|
||||
#define XW_W1_ADDR(round, widx) XW_W_ADDR(round, widx, 0)
|
||||
#define XW_W1W2_ADDR(round, widx) XW_W_ADDR(round, widx, 16)
|
||||
|
||||
/* Input block loading.
|
||||
* Interleaving within round function needed for in-order CPUs. */
|
||||
#define LOAD_W_VEC_1_1() \
|
||||
add addr0, sp, #IW_W1_ADDR(0, 0);
|
||||
#define LOAD_W_VEC_1_2() \
|
||||
add addr1, sp, #IW_W1_ADDR(4, 0);
|
||||
#define LOAD_W_VEC_1_3() \
|
||||
ld1 {W0.16b}, [RDATA], #16;
|
||||
#define LOAD_W_VEC_1_4() \
|
||||
ld1 {W1.16b}, [RDATA], #16;
|
||||
#define LOAD_W_VEC_1_5() \
|
||||
ld1 {W2.16b}, [RDATA], #16;
|
||||
#define LOAD_W_VEC_1_6() \
|
||||
ld1 {W3.16b}, [RDATA], #16;
|
||||
#define LOAD_W_VEC_1_7() \
|
||||
rev32 XTMP0.16b, W0.16b;
|
||||
#define LOAD_W_VEC_1_8() \
|
||||
rev32 XTMP1.16b, W1.16b;
|
||||
#define LOAD_W_VEC_2_1() \
|
||||
rev32 XTMP2.16b, W2.16b;
|
||||
#define LOAD_W_VEC_2_2() \
|
||||
rev32 XTMP3.16b, W3.16b;
|
||||
#define LOAD_W_VEC_2_3() \
|
||||
eor XTMP4.16b, XTMP1.16b, XTMP0.16b;
|
||||
#define LOAD_W_VEC_2_4() \
|
||||
eor XTMP5.16b, XTMP2.16b, XTMP1.16b;
|
||||
#define LOAD_W_VEC_2_5() \
|
||||
st1 {XTMP0.16b}, [addr0], #16;
|
||||
#define LOAD_W_VEC_2_6() \
|
||||
st1 {XTMP4.16b}, [addr0]; \
|
||||
add addr0, sp, #IW_W1_ADDR(8, 0);
|
||||
#define LOAD_W_VEC_2_7() \
|
||||
eor XTMP6.16b, XTMP3.16b, XTMP2.16b;
|
||||
#define LOAD_W_VEC_2_8() \
|
||||
ext W0.16b, XTMP0.16b, XTMP0.16b, #8; /* W0: xx, w0, xx, xx */
|
||||
#define LOAD_W_VEC_3_1() \
|
||||
mov W2.16b, XTMP1.16b; /* W2: xx, w6, w5, w4 */
|
||||
#define LOAD_W_VEC_3_2() \
|
||||
st1 {XTMP1.16b}, [addr1], #16;
|
||||
#define LOAD_W_VEC_3_3() \
|
||||
st1 {XTMP5.16b}, [addr1]; \
|
||||
ext W1.16b, XTMP0.16b, XTMP0.16b, #4; /* W1: xx, w3, w2, w1 */
|
||||
#define LOAD_W_VEC_3_4() \
|
||||
ext W3.16b, XTMP1.16b, XTMP2.16b, #12; /* W3: xx, w9, w8, w7 */
|
||||
#define LOAD_W_VEC_3_5() \
|
||||
ext W4.16b, XTMP2.16b, XTMP3.16b, #8; /* W4: xx, w12, w11, w10 */
|
||||
#define LOAD_W_VEC_3_6() \
|
||||
st1 {XTMP2.16b}, [addr0], #16;
|
||||
#define LOAD_W_VEC_3_7() \
|
||||
st1 {XTMP6.16b}, [addr0];
|
||||
#define LOAD_W_VEC_3_8() \
|
||||
ext W5.16b, XTMP3.16b, XTMP3.16b, #4; /* W5: xx, w15, w14, w13 */
|
||||
|
||||
#define LOAD_W_VEC_1(iop_num, ...) \
|
||||
LOAD_W_VEC_1_##iop_num()
|
||||
#define LOAD_W_VEC_2(iop_num, ...) \
|
||||
LOAD_W_VEC_2_##iop_num()
|
||||
#define LOAD_W_VEC_3(iop_num, ...) \
|
||||
LOAD_W_VEC_3_##iop_num()
|
||||
|
||||
/* Message scheduling. Note: 3 words per vector register.
|
||||
* Interleaving within round function needed for in-order CPUs. */
|
||||
#define SCHED_W_1_1(round, w0, w1, w2, w3, w4, w5) \
|
||||
/* Load (w[i - 16]) => XTMP0 */ \
|
||||
/* Load (w[i - 13]) => XTMP5 */ \
|
||||
ext XTMP0.16b, w0.16b, w0.16b, #12; /* XTMP0: w0, xx, xx, xx */
|
||||
#define SCHED_W_1_2(round, w0, w1, w2, w3, w4, w5) \
|
||||
ext XTMP5.16b, w1.16b, w1.16b, #12;
|
||||
#define SCHED_W_1_3(round, w0, w1, w2, w3, w4, w5) \
|
||||
ext XTMP0.16b, XTMP0.16b, w1.16b, #12; /* XTMP0: xx, w2, w1, w0 */
|
||||
#define SCHED_W_1_4(round, w0, w1, w2, w3, w4, w5) \
|
||||
ext XTMP5.16b, XTMP5.16b, w2.16b, #12;
|
||||
#define SCHED_W_1_5(round, w0, w1, w2, w3, w4, w5) \
|
||||
/* w[i - 9] == w3 */ \
|
||||
/* W3 ^ XTMP0 => XTMP0 */ \
|
||||
eor XTMP0.16b, XTMP0.16b, w3.16b;
|
||||
#define SCHED_W_1_6(round, w0, w1, w2, w3, w4, w5) \
|
||||
/* w[i - 3] == w5 */ \
|
||||
/* rol(XMM5, 15) ^ XTMP0 => XTMP0 */ \
|
||||
/* rol(XTMP5, 7) => XTMP1 */ \
|
||||
add addr0, sp, #XW_W1_ADDR((round), 0); \
|
||||
shl XTMP2.4s, w5.4s, #15;
|
||||
#define SCHED_W_1_7(round, w0, w1, w2, w3, w4, w5) \
|
||||
shl XTMP1.4s, XTMP5.4s, #7;
|
||||
#define SCHED_W_1_8(round, w0, w1, w2, w3, w4, w5) \
|
||||
sri XTMP2.4s, w5.4s, #(32-15);
|
||||
#define SCHED_W_2_1(round, w0, w1, w2, w3, w4, w5) \
|
||||
sri XTMP1.4s, XTMP5.4s, #(32-7);
|
||||
#define SCHED_W_2_2(round, w0, w1, w2, w3, w4, w5) \
|
||||
eor XTMP0.16b, XTMP0.16b, XTMP2.16b;
|
||||
#define SCHED_W_2_3(round, w0, w1, w2, w3, w4, w5) \
|
||||
/* w[i - 6] == W4 */ \
|
||||
/* W4 ^ XTMP1 => XTMP1 */ \
|
||||
eor XTMP1.16b, XTMP1.16b, w4.16b;
|
||||
#define SCHED_W_2_4(round, w0, w1, w2, w3, w4, w5) \
|
||||
/* P1(XTMP0) ^ XTMP1 => W0 */ \
|
||||
shl XTMP3.4s, XTMP0.4s, #15;
|
||||
#define SCHED_W_2_5(round, w0, w1, w2, w3, w4, w5) \
|
||||
shl XTMP4.4s, XTMP0.4s, #23;
|
||||
#define SCHED_W_2_6(round, w0, w1, w2, w3, w4, w5) \
|
||||
eor w0.16b, XTMP1.16b, XTMP0.16b;
|
||||
#define SCHED_W_2_7(round, w0, w1, w2, w3, w4, w5) \
|
||||
sri XTMP3.4s, XTMP0.4s, #(32-15);
|
||||
#define SCHED_W_2_8(round, w0, w1, w2, w3, w4, w5) \
|
||||
sri XTMP4.4s, XTMP0.4s, #(32-23);
|
||||
#define SCHED_W_3_1(round, w0, w1, w2, w3, w4, w5) \
|
||||
eor w0.16b, w0.16b, XTMP3.16b;
|
||||
#define SCHED_W_3_2(round, w0, w1, w2, w3, w4, w5) \
|
||||
/* Load (w[i - 3]) => XTMP2 */ \
|
||||
ext XTMP2.16b, w4.16b, w4.16b, #12;
|
||||
#define SCHED_W_3_3(round, w0, w1, w2, w3, w4, w5) \
|
||||
eor w0.16b, w0.16b, XTMP4.16b;
|
||||
#define SCHED_W_3_4(round, w0, w1, w2, w3, w4, w5) \
|
||||
ext XTMP2.16b, XTMP2.16b, w5.16b, #12;
|
||||
#define SCHED_W_3_5(round, w0, w1, w2, w3, w4, w5) \
|
||||
/* W1 ^ W2 => XTMP3 */ \
|
||||
eor XTMP3.16b, XTMP2.16b, w0.16b;
|
||||
#define SCHED_W_3_6(round, w0, w1, w2, w3, w4, w5)
|
||||
#define SCHED_W_3_7(round, w0, w1, w2, w3, w4, w5) \
|
||||
st1 {XTMP2.16b-XTMP3.16b}, [addr0];
|
||||
#define SCHED_W_3_8(round, w0, w1, w2, w3, w4, w5)
|
||||
|
||||
#define SCHED_W_W0W1W2W3W4W5_1(iop_num, round) \
|
||||
SCHED_W_1_##iop_num(round, W0, W1, W2, W3, W4, W5)
|
||||
#define SCHED_W_W0W1W2W3W4W5_2(iop_num, round) \
|
||||
SCHED_W_2_##iop_num(round, W0, W1, W2, W3, W4, W5)
|
||||
#define SCHED_W_W0W1W2W3W4W5_3(iop_num, round) \
|
||||
SCHED_W_3_##iop_num(round, W0, W1, W2, W3, W4, W5)
|
||||
|
||||
#define SCHED_W_W1W2W3W4W5W0_1(iop_num, round) \
|
||||
SCHED_W_1_##iop_num(round, W1, W2, W3, W4, W5, W0)
|
||||
#define SCHED_W_W1W2W3W4W5W0_2(iop_num, round) \
|
||||
SCHED_W_2_##iop_num(round, W1, W2, W3, W4, W5, W0)
|
||||
#define SCHED_W_W1W2W3W4W5W0_3(iop_num, round) \
|
||||
SCHED_W_3_##iop_num(round, W1, W2, W3, W4, W5, W0)
|
||||
|
||||
#define SCHED_W_W2W3W4W5W0W1_1(iop_num, round) \
|
||||
SCHED_W_1_##iop_num(round, W2, W3, W4, W5, W0, W1)
|
||||
#define SCHED_W_W2W3W4W5W0W1_2(iop_num, round) \
|
||||
SCHED_W_2_##iop_num(round, W2, W3, W4, W5, W0, W1)
|
||||
#define SCHED_W_W2W3W4W5W0W1_3(iop_num, round) \
|
||||
SCHED_W_3_##iop_num(round, W2, W3, W4, W5, W0, W1)
|
||||
|
||||
#define SCHED_W_W3W4W5W0W1W2_1(iop_num, round) \
|
||||
SCHED_W_1_##iop_num(round, W3, W4, W5, W0, W1, W2)
|
||||
#define SCHED_W_W3W4W5W0W1W2_2(iop_num, round) \
|
||||
SCHED_W_2_##iop_num(round, W3, W4, W5, W0, W1, W2)
|
||||
#define SCHED_W_W3W4W5W0W1W2_3(iop_num, round) \
|
||||
SCHED_W_3_##iop_num(round, W3, W4, W5, W0, W1, W2)
|
||||
|
||||
#define SCHED_W_W4W5W0W1W2W3_1(iop_num, round) \
|
||||
SCHED_W_1_##iop_num(round, W4, W5, W0, W1, W2, W3)
|
||||
#define SCHED_W_W4W5W0W1W2W3_2(iop_num, round) \
|
||||
SCHED_W_2_##iop_num(round, W4, W5, W0, W1, W2, W3)
|
||||
#define SCHED_W_W4W5W0W1W2W3_3(iop_num, round) \
|
||||
SCHED_W_3_##iop_num(round, W4, W5, W0, W1, W2, W3)
|
||||
|
||||
#define SCHED_W_W5W0W1W2W3W4_1(iop_num, round) \
|
||||
SCHED_W_1_##iop_num(round, W5, W0, W1, W2, W3, W4)
|
||||
#define SCHED_W_W5W0W1W2W3W4_2(iop_num, round) \
|
||||
SCHED_W_2_##iop_num(round, W5, W0, W1, W2, W3, W4)
|
||||
#define SCHED_W_W5W0W1W2W3W4_3(iop_num, round) \
|
||||
SCHED_W_3_##iop_num(round, W5, W0, W1, W2, W3, W4)
|
||||
|
||||
|
||||
/*
|
||||
* Transform nblocks*64 bytes (nblocks*16 32-bit words) at 'data'.
|
||||
*
|
||||
* void sm3_neon_transform(struct sm3_block_state *state,
|
||||
* const u8 *data, size_t nblocks)
|
||||
*/
|
||||
.text
|
||||
.align 3
|
||||
SYM_FUNC_START(sm3_neon_transform)
|
||||
ldp ra, rb, [RSTATE, #0]
|
||||
ldp rc, rd, [RSTATE, #8]
|
||||
ldp re, rf, [RSTATE, #16]
|
||||
ldp rg, rh, [RSTATE, #24]
|
||||
|
||||
stp x28, x29, [sp, #-16]!
|
||||
stp x19, x20, [sp, #-16]!
|
||||
stp x21, x22, [sp, #-16]!
|
||||
stp x23, x24, [sp, #-16]!
|
||||
stp x25, x26, [sp, #-16]!
|
||||
mov RFRAME, sp
|
||||
|
||||
sub addr0, sp, #STACK_SIZE
|
||||
adr_l RKPTR, .LKtable
|
||||
and sp, addr0, #(~63)
|
||||
|
||||
/* Preload first block. */
|
||||
LOAD_W_VEC_1(1, 0)
|
||||
LOAD_W_VEC_1(2, 0)
|
||||
LOAD_W_VEC_1(3, 0)
|
||||
LOAD_W_VEC_1(4, 0)
|
||||
LOAD_W_VEC_1(5, 0)
|
||||
LOAD_W_VEC_1(6, 0)
|
||||
LOAD_W_VEC_1(7, 0)
|
||||
LOAD_W_VEC_1(8, 0)
|
||||
LOAD_W_VEC_2(1, 0)
|
||||
LOAD_W_VEC_2(2, 0)
|
||||
LOAD_W_VEC_2(3, 0)
|
||||
LOAD_W_VEC_2(4, 0)
|
||||
LOAD_W_VEC_2(5, 0)
|
||||
LOAD_W_VEC_2(6, 0)
|
||||
LOAD_W_VEC_2(7, 0)
|
||||
LOAD_W_VEC_2(8, 0)
|
||||
LOAD_W_VEC_3(1, 0)
|
||||
LOAD_W_VEC_3(2, 0)
|
||||
LOAD_W_VEC_3(3, 0)
|
||||
LOAD_W_VEC_3(4, 0)
|
||||
LOAD_W_VEC_3(5, 0)
|
||||
LOAD_W_VEC_3(6, 0)
|
||||
LOAD_W_VEC_3(7, 0)
|
||||
LOAD_W_VEC_3(8, 0)
|
||||
|
||||
.balign 16
|
||||
.Loop:
|
||||
/* Transform 0-3 */
|
||||
R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 0, 0, IW, _, 0)
|
||||
R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 1, 1, IW, _, 0)
|
||||
R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 2, 2, IW, _, 0)
|
||||
R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 3, 3, IW, _, 0)
|
||||
|
||||
/* Transform 4-7 + Precalc 12-14 */
|
||||
R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 4, 0, IW, _, 0)
|
||||
R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 5, 1, IW, _, 0)
|
||||
R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 6, 2, IW, SCHED_W_W0W1W2W3W4W5_1, 12)
|
||||
R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 7, 3, IW, SCHED_W_W0W1W2W3W4W5_2, 12)
|
||||
|
||||
/* Transform 8-11 + Precalc 12-17 */
|
||||
R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 8, 0, IW, SCHED_W_W0W1W2W3W4W5_3, 12)
|
||||
R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 9, 1, IW, SCHED_W_W1W2W3W4W5W0_1, 15)
|
||||
R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 10, 2, IW, SCHED_W_W1W2W3W4W5W0_2, 15)
|
||||
R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 11, 3, IW, SCHED_W_W1W2W3W4W5W0_3, 15)
|
||||
|
||||
/* Transform 12-14 + Precalc 18-20 */
|
||||
R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 12, 0, XW, SCHED_W_W2W3W4W5W0W1_1, 18)
|
||||
R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 13, 1, XW, SCHED_W_W2W3W4W5W0W1_2, 18)
|
||||
R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 14, 2, XW, SCHED_W_W2W3W4W5W0W1_3, 18)
|
||||
|
||||
/* Transform 15-17 + Precalc 21-23 */
|
||||
R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 15, 0, XW, SCHED_W_W3W4W5W0W1W2_1, 21)
|
||||
R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 16, 1, XW, SCHED_W_W3W4W5W0W1W2_2, 21)
|
||||
R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 17, 2, XW, SCHED_W_W3W4W5W0W1W2_3, 21)
|
||||
|
||||
/* Transform 18-20 + Precalc 24-26 */
|
||||
R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 18, 0, XW, SCHED_W_W4W5W0W1W2W3_1, 24)
|
||||
R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 19, 1, XW, SCHED_W_W4W5W0W1W2W3_2, 24)
|
||||
R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 20, 2, XW, SCHED_W_W4W5W0W1W2W3_3, 24)
|
||||
|
||||
/* Transform 21-23 + Precalc 27-29 */
|
||||
R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 21, 0, XW, SCHED_W_W5W0W1W2W3W4_1, 27)
|
||||
R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 22, 1, XW, SCHED_W_W5W0W1W2W3W4_2, 27)
|
||||
R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 23, 2, XW, SCHED_W_W5W0W1W2W3W4_3, 27)
|
||||
|
||||
/* Transform 24-26 + Precalc 30-32 */
|
||||
R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 24, 0, XW, SCHED_W_W0W1W2W3W4W5_1, 30)
|
||||
R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 25, 1, XW, SCHED_W_W0W1W2W3W4W5_2, 30)
|
||||
R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 26, 2, XW, SCHED_W_W0W1W2W3W4W5_3, 30)
|
||||
|
||||
/* Transform 27-29 + Precalc 33-35 */
|
||||
R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 27, 0, XW, SCHED_W_W1W2W3W4W5W0_1, 33)
|
||||
R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 28, 1, XW, SCHED_W_W1W2W3W4W5W0_2, 33)
|
||||
R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 29, 2, XW, SCHED_W_W1W2W3W4W5W0_3, 33)
|
||||
|
||||
/* Transform 30-32 + Precalc 36-38 */
|
||||
R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 30, 0, XW, SCHED_W_W2W3W4W5W0W1_1, 36)
|
||||
R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 31, 1, XW, SCHED_W_W2W3W4W5W0W1_2, 36)
|
||||
R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 32, 2, XW, SCHED_W_W2W3W4W5W0W1_3, 36)
|
||||
|
||||
/* Transform 33-35 + Precalc 39-41 */
|
||||
R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 33, 0, XW, SCHED_W_W3W4W5W0W1W2_1, 39)
|
||||
R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 34, 1, XW, SCHED_W_W3W4W5W0W1W2_2, 39)
|
||||
R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 35, 2, XW, SCHED_W_W3W4W5W0W1W2_3, 39)
|
||||
|
||||
/* Transform 36-38 + Precalc 42-44 */
|
||||
R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 36, 0, XW, SCHED_W_W4W5W0W1W2W3_1, 42)
|
||||
R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 37, 1, XW, SCHED_W_W4W5W0W1W2W3_2, 42)
|
||||
R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 38, 2, XW, SCHED_W_W4W5W0W1W2W3_3, 42)
|
||||
|
||||
/* Transform 39-41 + Precalc 45-47 */
|
||||
R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 39, 0, XW, SCHED_W_W5W0W1W2W3W4_1, 45)
|
||||
R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 40, 1, XW, SCHED_W_W5W0W1W2W3W4_2, 45)
|
||||
R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 41, 2, XW, SCHED_W_W5W0W1W2W3W4_3, 45)
|
||||
|
||||
/* Transform 42-44 + Precalc 48-50 */
|
||||
R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 42, 0, XW, SCHED_W_W0W1W2W3W4W5_1, 48)
|
||||
R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 43, 1, XW, SCHED_W_W0W1W2W3W4W5_2, 48)
|
||||
R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 44, 2, XW, SCHED_W_W0W1W2W3W4W5_3, 48)
|
||||
|
||||
/* Transform 45-47 + Precalc 51-53 */
|
||||
R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 45, 0, XW, SCHED_W_W1W2W3W4W5W0_1, 51)
|
||||
R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 46, 1, XW, SCHED_W_W1W2W3W4W5W0_2, 51)
|
||||
R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 47, 2, XW, SCHED_W_W1W2W3W4W5W0_3, 51)
|
||||
|
||||
/* Transform 48-50 + Precalc 54-56 */
|
||||
R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 48, 0, XW, SCHED_W_W2W3W4W5W0W1_1, 54)
|
||||
R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 49, 1, XW, SCHED_W_W2W3W4W5W0W1_2, 54)
|
||||
R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 50, 2, XW, SCHED_W_W2W3W4W5W0W1_3, 54)
|
||||
|
||||
/* Transform 51-53 + Precalc 57-59 */
|
||||
R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 51, 0, XW, SCHED_W_W3W4W5W0W1W2_1, 57)
|
||||
R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 52, 1, XW, SCHED_W_W3W4W5W0W1W2_2, 57)
|
||||
R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 53, 2, XW, SCHED_W_W3W4W5W0W1W2_3, 57)
|
||||
|
||||
/* Transform 54-56 + Precalc 60-62 */
|
||||
R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 54, 0, XW, SCHED_W_W4W5W0W1W2W3_1, 60)
|
||||
R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 55, 1, XW, SCHED_W_W4W5W0W1W2W3_2, 60)
|
||||
R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 56, 2, XW, SCHED_W_W4W5W0W1W2W3_3, 60)
|
||||
|
||||
/* Transform 57-59 + Precalc 63 */
|
||||
R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 57, 0, XW, SCHED_W_W5W0W1W2W3W4_1, 63)
|
||||
R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 58, 1, XW, SCHED_W_W5W0W1W2W3W4_2, 63)
|
||||
R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 59, 2, XW, SCHED_W_W5W0W1W2W3W4_3, 63)
|
||||
|
||||
/* Transform 60 */
|
||||
R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 60, 0, XW, _, _)
|
||||
subs RNBLKS, RNBLKS, #1
|
||||
b.eq .Lend
|
||||
|
||||
/* Transform 61-63 + Preload next block */
|
||||
R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 61, 1, XW, LOAD_W_VEC_1, _)
|
||||
ldp s0, s1, [RSTATE, #0]
|
||||
R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 62, 2, XW, LOAD_W_VEC_2, _)
|
||||
ldp s2, s3, [RSTATE, #8]
|
||||
R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 63, 0, XW, LOAD_W_VEC_3, _)
|
||||
|
||||
/* Update the chaining variables. */
|
||||
eor ra, ra, s0
|
||||
eor rb, rb, s1
|
||||
ldp s0, s1, [RSTATE, #16]
|
||||
eor rc, rc, s2
|
||||
ldp k_even, k_odd, [RSTATE, #24]
|
||||
eor rd, rd, s3
|
||||
eor re, re, s0
|
||||
stp ra, rb, [RSTATE, #0]
|
||||
eor rf, rf, s1
|
||||
stp rc, rd, [RSTATE, #8]
|
||||
eor rg, rg, k_even
|
||||
stp re, rf, [RSTATE, #16]
|
||||
eor rh, rh, k_odd
|
||||
stp rg, rh, [RSTATE, #24]
|
||||
b .Loop
|
||||
|
||||
.Lend:
|
||||
/* Transform 61-63 */
|
||||
R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 61, 1, XW, _, _)
|
||||
ldp s0, s1, [RSTATE, #0]
|
||||
R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 62, 2, XW, _, _)
|
||||
ldp s2, s3, [RSTATE, #8]
|
||||
R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 63, 0, XW, _, _)
|
||||
|
||||
/* Update the chaining variables. */
|
||||
eor ra, ra, s0
|
||||
clear_vec(W0)
|
||||
eor rb, rb, s1
|
||||
clear_vec(W1)
|
||||
ldp s0, s1, [RSTATE, #16]
|
||||
clear_vec(W2)
|
||||
eor rc, rc, s2
|
||||
clear_vec(W3)
|
||||
ldp k_even, k_odd, [RSTATE, #24]
|
||||
clear_vec(W4)
|
||||
eor rd, rd, s3
|
||||
clear_vec(W5)
|
||||
eor re, re, s0
|
||||
clear_vec(XTMP0)
|
||||
stp ra, rb, [RSTATE, #0]
|
||||
clear_vec(XTMP1)
|
||||
eor rf, rf, s1
|
||||
clear_vec(XTMP2)
|
||||
stp rc, rd, [RSTATE, #8]
|
||||
clear_vec(XTMP3)
|
||||
eor rg, rg, k_even
|
||||
clear_vec(XTMP4)
|
||||
stp re, rf, [RSTATE, #16]
|
||||
clear_vec(XTMP5)
|
||||
eor rh, rh, k_odd
|
||||
clear_vec(XTMP6)
|
||||
stp rg, rh, [RSTATE, #24]
|
||||
|
||||
/* Clear message expansion area */
|
||||
add addr0, sp, #STACK_W
|
||||
st1 {W0.16b-W3.16b}, [addr0], #64
|
||||
st1 {W0.16b-W3.16b}, [addr0], #64
|
||||
st1 {W0.16b-W3.16b}, [addr0]
|
||||
|
||||
mov sp, RFRAME
|
||||
|
||||
ldp x25, x26, [sp], #16
|
||||
ldp x23, x24, [sp], #16
|
||||
ldp x21, x22, [sp], #16
|
||||
ldp x19, x20, [sp], #16
|
||||
ldp x28, x29, [sp], #16
|
||||
|
||||
ret
|
||||
SYM_FUNC_END(sm3_neon_transform)
|
||||
|
||||
|
||||
.section ".rodata", "a"
|
||||
|
||||
.align 4
|
||||
.LKtable:
|
||||
.long 0x79cc4519, 0xf3988a32, 0xe7311465, 0xce6228cb
|
||||
.long 0x9cc45197, 0x3988a32f, 0x7311465e, 0xe6228cbc
|
||||
.long 0xcc451979, 0x988a32f3, 0x311465e7, 0x6228cbce
|
||||
.long 0xc451979c, 0x88a32f39, 0x11465e73, 0x228cbce6
|
||||
.long 0x9d8a7a87, 0x3b14f50f, 0x7629ea1e, 0xec53d43c
|
||||
.long 0xd8a7a879, 0xb14f50f3, 0x629ea1e7, 0xc53d43ce
|
||||
.long 0x8a7a879d, 0x14f50f3b, 0x29ea1e76, 0x53d43cec
|
||||
.long 0xa7a879d8, 0x4f50f3b1, 0x9ea1e762, 0x3d43cec5
|
||||
.long 0x7a879d8a, 0xf50f3b14, 0xea1e7629, 0xd43cec53
|
||||
.long 0xa879d8a7, 0x50f3b14f, 0xa1e7629e, 0x43cec53d
|
||||
.long 0x879d8a7a, 0x0f3b14f5, 0x1e7629ea, 0x3cec53d4
|
||||
.long 0x79d8a7a8, 0xf3b14f50, 0xe7629ea1, 0xcec53d43
|
||||
.long 0x9d8a7a87, 0x3b14f50f, 0x7629ea1e, 0xec53d43c
|
||||
.long 0xd8a7a879, 0xb14f50f3, 0x629ea1e7, 0xc53d43ce
|
||||
.long 0x8a7a879d, 0x14f50f3b, 0x29ea1e76, 0x53d43cec
|
||||
.long 0xa7a879d8, 0x4f50f3b1, 0x9ea1e762, 0x3d43cec5
|
||||
@@ -0,0 +1,41 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0-or-later */
|
||||
/*
|
||||
* SM3 optimized for ARM64
|
||||
*
|
||||
* Copyright 2026 Google LLC
|
||||
*/
|
||||
#include <asm/simd.h>
|
||||
#include <linux/cpufeature.h>
|
||||
|
||||
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
|
||||
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce);
|
||||
|
||||
asmlinkage void sm3_neon_transform(struct sm3_block_state *state,
|
||||
const u8 *data, size_t nblocks);
|
||||
asmlinkage void sm3_ce_transform(struct sm3_block_state *state,
|
||||
const u8 *data, size_t nblocks);
|
||||
|
||||
static void sm3_blocks(struct sm3_block_state *state,
|
||||
const u8 *data, size_t nblocks)
|
||||
{
|
||||
if (static_branch_likely(&have_neon) && likely(may_use_simd())) {
|
||||
scoped_ksimd() {
|
||||
if (static_branch_likely(&have_ce))
|
||||
sm3_ce_transform(state, data, nblocks);
|
||||
else
|
||||
sm3_neon_transform(state, data, nblocks);
|
||||
}
|
||||
} else {
|
||||
sm3_blocks_generic(state, data, nblocks);
|
||||
}
|
||||
}
|
||||
|
||||
#define sm3_mod_init_arch sm3_mod_init_arch
|
||||
static void sm3_mod_init_arch(void)
|
||||
{
|
||||
if (cpu_have_named_feature(ASIMD)) {
|
||||
static_branch_enable(&have_neon);
|
||||
if (cpu_have_named_feature(SM3))
|
||||
static_branch_enable(&have_ce);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user