From 5ee6cfdd11aaf5aee5cd48baf35b1710caa68a5c Mon Sep 17 00:00:00 2001 From: Shreenidhi Shedi Date: Sun, 15 May 2022 13:01:09 +0530 Subject: [PATCH 001/318] audit: remove redundant data_len check data_len is already getting checked if it's less than 2 earlier in this function. Signed-off-by: Shreenidhi Shedi Signed-off-by: Paul Moore --- kernel/audit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/audit.c b/kernel/audit.c index 7690c29d4ee4..0749211d5552 100644 --- a/kernel/audit.c +++ b/kernel/audit.c @@ -1390,7 +1390,7 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh) str); } else { audit_log_format(ab, " data="); - if (data_len > 0 && str[data_len - 1] == '\0') + if (str[data_len - 1] == '\0') data_len--; audit_log_n_untrustedstring(ab, str, data_len); } From 494688efdc5912b858d0a05563c9cf258c75d29a Mon Sep 17 00:00:00 2001 From: "GONG, Ruiqi" Date: Wed, 18 May 2022 09:21:37 +0000 Subject: [PATCH 002/318] selinux: add __randomize_layout to selinux_audit_data Randomize the layout of struct selinux_audit_data as suggested in [1], since it contains a pointer to struct selinux_state, an already randomized strucure. [1]: https://github.com/KSPP/linux/issues/188 Signed-off-by: GONG, Ruiqi Signed-off-by: Paul Moore --- security/selinux/include/avc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/security/selinux/include/avc.h b/security/selinux/include/avc.h index 2b372f98f2d7..5525b94fd266 100644 --- a/security/selinux/include/avc.h +++ b/security/selinux/include/avc.h @@ -53,7 +53,7 @@ struct selinux_audit_data { u32 denied; int result; struct selinux_state *state; -}; +} __randomize_layout; /* * AVC operations From 4d3d0ed60ee0d2da4c541e525c132dc374464624 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20G=C3=B6ttsche?= Date: Thu, 17 Feb 2022 15:21:28 +0100 Subject: [PATCH 003/318] selinux: drop unnecessary NULL check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit e3489f8974e1 ("selinux: kill selinux_sb_get_mnt_opts()") introduced a NULL check on the context after a successful call to security_sid_to_context(). This is on the one hand redundant after checking for success and on the other hand insufficient on an actual NULL pointer, since the context is passed to seq_escape() leading to a call of strlen() on it. Reported by Clang analyzer: In file included from security/selinux/hooks.c:28: In file included from ./include/linux/tracehook.h:50: In file included from ./include/linux/memcontrol.h:13: In file included from ./include/linux/cgroup.h:18: ./include/linux/seq_file.h:136:25: warning: Null pointer passed as 1st argument to string length function [unix.cstring.NullArg] seq_escape_mem(m, src, strlen(src), flags, esc); ^~~~~~~~~~~ Signed-off-by: Christian Göttsche Signed-off-by: Paul Moore --- security/selinux/hooks.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index beceb89f68d9..4af4986d3893 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -1019,7 +1019,7 @@ static int show_sid(struct seq_file *m, u32 sid) rc = security_sid_to_context(&selinux_state, sid, &context, &len); if (!rc) { - bool has_comma = context && strchr(context, ','); + bool has_comma = strchr(context, ','); seq_putc(m, '='); if (has_comma) From 13dc15a3f5fd7f884e4bfa8c011a0ae868df12ae Mon Sep 17 00:00:00 2001 From: John Allen Date: Wed, 18 May 2022 15:31:26 +0000 Subject: [PATCH 004/318] crypto: ccp - Use kzalloc for sev ioctl interfaces to prevent kernel memory leak For some sev ioctl interfaces, input may be passed that is less than or equal to SEV_FW_BLOB_MAX_SIZE, but larger than the data that PSP firmware returns. In this case, kmalloc will allocate memory that is the size of the input rather than the size of the data. Since PSP firmware doesn't fully overwrite the buffer, the sev ioctl interfaces with the issue may return uninitialized slab memory. Currently, all of the ioctl interfaces in the ccp driver are safe, but to prevent future problems, change all ioctl interfaces that allocate memory with kmalloc to use kzalloc and memset the data buffer to zero in sev_ioctl_do_platform_status. Fixes: 38103671aad3 ("crypto: ccp: Use the stack and common buffer for status commands") Fixes: e799035609e15 ("crypto: ccp: Implement SEV_PEK_CSR ioctl command") Fixes: 76a2b524a4b1d ("crypto: ccp: Implement SEV_PDH_CERT_EXPORT ioctl command") Fixes: d6112ea0cb344 ("crypto: ccp - introduce SEV_GET_ID2 command") Cc: stable@vger.kernel.org Reported-by: Andy Nguyen Suggested-by: David Rientjes Suggested-by: Peter Gonda Signed-off-by: John Allen Reviewed-by: Peter Gonda Acked-by: David Rientjes Signed-off-by: Herbert Xu --- drivers/crypto/ccp/sev-dev.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 799b476fc3e8..0c92d940ac4e 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -577,6 +577,8 @@ static int sev_ioctl_do_platform_status(struct sev_issue_cmd *argp) struct sev_user_data_status data; int ret; + memset(&data, 0, sizeof(data)); + ret = __sev_do_cmd_locked(SEV_CMD_PLATFORM_STATUS, &data, &argp->error); if (ret) return ret; @@ -630,7 +632,7 @@ static int sev_ioctl_do_pek_csr(struct sev_issue_cmd *argp, bool writable) if (input.length > SEV_FW_BLOB_MAX_SIZE) return -EFAULT; - blob = kmalloc(input.length, GFP_KERNEL); + blob = kzalloc(input.length, GFP_KERNEL); if (!blob) return -ENOMEM; @@ -854,7 +856,7 @@ static int sev_ioctl_do_get_id2(struct sev_issue_cmd *argp) input_address = (void __user *)input.address; if (input.address && input.length) { - id_blob = kmalloc(input.length, GFP_KERNEL); + id_blob = kzalloc(input.length, GFP_KERNEL); if (!id_blob) return -ENOMEM; @@ -973,14 +975,14 @@ static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable) if (input.cert_chain_len > SEV_FW_BLOB_MAX_SIZE) return -EFAULT; - pdh_blob = kmalloc(input.pdh_cert_len, GFP_KERNEL); + pdh_blob = kzalloc(input.pdh_cert_len, GFP_KERNEL); if (!pdh_blob) return -ENOMEM; data.pdh_cert_address = __psp_pa(pdh_blob); data.pdh_cert_len = input.pdh_cert_len; - cert_blob = kmalloc(input.cert_chain_len, GFP_KERNEL); + cert_blob = kzalloc(input.cert_chain_len, GFP_KERNEL); if (!cert_blob) { ret = -ENOMEM; goto e_free_pdh; From d2765e1b9ac4b2d5a5d5bf17f468c9b3566c3770 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Wed, 18 May 2022 20:33:44 +0300 Subject: [PATCH 005/318] crypto: sun8i-ss - fix error codes in allocate_flows() These failure paths should return -ENOMEM. Currently they return success. Fixes: 359e893e8af4 ("crypto: sun8i-ss - rework handling of IV") Fixes: 8eec4563f152 ("crypto: sun8i-ss - do not allocate memory when handling hash requests") Signed-off-by: Dan Carpenter Acked-by: Corentin Labbe Tested-by: Corentin Labbe Signed-off-by: Herbert Xu --- .../crypto/allwinner/sun8i-ss/sun8i-ss-core.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c index 98593a0cff69..ac2329e2b0e5 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c @@ -528,25 +528,33 @@ static int allocate_flows(struct sun8i_ss_dev *ss) ss->flows[i].biv = devm_kmalloc(ss->dev, AES_BLOCK_SIZE, GFP_KERNEL | GFP_DMA); - if (!ss->flows[i].biv) + if (!ss->flows[i].biv) { + err = -ENOMEM; goto error_engine; + } for (j = 0; j < MAX_SG; j++) { ss->flows[i].iv[j] = devm_kmalloc(ss->dev, AES_BLOCK_SIZE, GFP_KERNEL | GFP_DMA); - if (!ss->flows[i].iv[j]) + if (!ss->flows[i].iv[j]) { + err = -ENOMEM; goto error_engine; + } } /* the padding could be up to two block. */ ss->flows[i].pad = devm_kmalloc(ss->dev, MAX_PAD_SIZE, GFP_KERNEL | GFP_DMA); - if (!ss->flows[i].pad) + if (!ss->flows[i].pad) { + err = -ENOMEM; goto error_engine; + } ss->flows[i].result = devm_kmalloc(ss->dev, SHA256_DIGEST_SIZE, GFP_KERNEL | GFP_DMA); - if (!ss->flows[i].result) + if (!ss->flows[i].result) { + err = -ENOMEM; goto error_engine; + } ss->flows[i].engine = crypto_engine_alloc_init(ss->dev, true); if (!ss->flows[i].engine) { From 6cb3f9b25c55928b95a02b9ed8e87ed653b3cce8 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Wed, 18 May 2022 20:33:54 +0300 Subject: [PATCH 006/318] crypto: sun8i-ss - Fix error codes for dma_mapping_error() If there is a dma_mapping_error() then return negative error codes. Currently this code returns success. Fixes: 801b7d572c0a ("crypto: sun8i-ss - add hmac(sha1)") Signed-off-by: Dan Carpenter Acked-by: Corentin Labbe Tested-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c index ac417a6b39e5..845019bd9591 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c @@ -586,7 +586,8 @@ retry: rctx->t_dst[k + 1].len = rctx->t_dst[k].len; } addr_xpad = dma_map_single(ss->dev, tfmctx->ipad, bs, DMA_TO_DEVICE); - if (dma_mapping_error(ss->dev, addr_xpad)) { + err = dma_mapping_error(ss->dev, addr_xpad); + if (err) { dev_err(ss->dev, "Fail to create DMA mapping of ipad\n"); goto err_dma_xpad; } @@ -612,7 +613,8 @@ retry: goto err_dma_result; } addr_xpad = dma_map_single(ss->dev, tfmctx->opad, bs, DMA_TO_DEVICE); - if (dma_mapping_error(ss->dev, addr_xpad)) { + err = dma_mapping_error(ss->dev, addr_xpad); + if (err) { dev_err(ss->dev, "Fail to create DMA mapping of opad\n"); goto err_dma_xpad; } From 7df7563b16aa0281cb811785e4bb3681b46e2a28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Fri, 20 May 2022 19:21:00 +0200 Subject: [PATCH 007/318] crypto: atmel-ecc - Remove duplicated error reporting in .remove() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Returning an error value in an i2c remove callback results in an error message being emitted by the i2c core, but otherwise it doesn't make a difference. The device goes away anyhow and the devm cleanups are called. As atmel_ecc_remove() already emits an error message on failure and the additional error message by the i2c core doesn't add any useful information, change the return value to zero to suppress this message. Also make the error message a bit more drastical because when the device is still busy on remove, it's likely that it will access freed memory soon. This patch is a preparation for making i2c remove callbacks return void. Signed-off-by: Uwe Kleine-König Reviewed-by: Tudor Ambarus Signed-off-by: Herbert Xu --- drivers/crypto/atmel-ecc.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/atmel-ecc.c b/drivers/crypto/atmel-ecc.c index 59a57279e77b..a4b13d326cfc 100644 --- a/drivers/crypto/atmel-ecc.c +++ b/drivers/crypto/atmel-ecc.c @@ -349,8 +349,16 @@ static int atmel_ecc_remove(struct i2c_client *client) /* Return EBUSY if i2c client already allocated. */ if (atomic_read(&i2c_priv->tfm_count)) { - dev_err(&client->dev, "Device is busy\n"); - return -EBUSY; + /* + * After we return here, the memory backing the device is freed. + * That happens no matter what the return value of this function + * is because in the Linux device model there is no error + * handling for unbinding a driver. + * If there is still some action pending, it probably involves + * accessing the freed memory. + */ + dev_emerg(&client->dev, "Device is busy, expect memory corruption.\n"); + return 0; } crypto_unregister_kpp(&atmel_ecdh_nist_p256); From 17fee07a2ac577da18b44dd658a9c3c864281c49 Mon Sep 17 00:00:00 2001 From: Nathan Huckleberry Date: Fri, 20 May 2022 18:14:53 +0000 Subject: [PATCH 008/318] crypto: xctr - Add XCTR support Add a generic implementation of XCTR mode as a template. XCTR is a blockcipher mode similar to CTR mode. XCTR uses XORs and little-endian addition rather than big-endian arithmetic which has two advantages: It is slightly faster on little-endian CPUs and it is less likely to be implemented incorrect since integer overflows are not possible on practical input sizes. XCTR is used as a component to implement HCTR2. More information on XCTR mode can be found in the HCTR2 paper: https://eprint.iacr.org/2021/1441.pdf Signed-off-by: Nathan Huckleberry Reviewed-by: Eric Biggers Reviewed-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/Kconfig | 9 + crypto/Makefile | 1 + crypto/tcrypt.c | 1 + crypto/testmgr.c | 6 + crypto/testmgr.h | 693 +++++++++++++++++++++++++++++++++++++++++++++++ crypto/xctr.c | 191 +++++++++++++ 6 files changed, 901 insertions(+) create mode 100644 crypto/xctr.c diff --git a/crypto/Kconfig b/crypto/Kconfig index 19197469cfab..b9e4d511bf5a 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -460,6 +460,15 @@ config CRYPTO_PCBC PCBC: Propagating Cipher Block Chaining mode This block cipher algorithm is required for RxRPC. +config CRYPTO_XCTR + tristate + select CRYPTO_SKCIPHER + select CRYPTO_MANAGER + help + XCTR: XOR Counter mode. This blockcipher mode is a variant of CTR mode + using XORs and little-endian addition rather than big-endian arithmetic. + XCTR mode is used to implement HCTR2. + config CRYPTO_XTS tristate "XTS support" select CRYPTO_SKCIPHER diff --git a/crypto/Makefile b/crypto/Makefile index 43bc33e247d1..93d0afeb3a77 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -94,6 +94,7 @@ obj-$(CONFIG_CRYPTO_CTS) += cts.o obj-$(CONFIG_CRYPTO_LRW) += lrw.o obj-$(CONFIG_CRYPTO_XTS) += xts.o obj-$(CONFIG_CRYPTO_CTR) += ctr.o +obj-$(CONFIG_CRYPTO_XCTR) += xctr.o obj-$(CONFIG_CRYPTO_KEYWRAP) += keywrap.o obj-$(CONFIG_CRYPTO_ADIANTUM) += adiantum.o obj-$(CONFIG_CRYPTO_NHPOLY1305) += nhpoly1305.o diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 2bacf8384f59..fd671d0e2012 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -1556,6 +1556,7 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) ret += tcrypt_test("rfc3686(ctr(aes))"); ret += tcrypt_test("ofb(aes)"); ret += tcrypt_test("cfb(aes)"); + ret += tcrypt_test("xctr(aes)"); break; case 11: diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 5801a8f9f713..0ea77dcdc6c0 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -5548,6 +5548,12 @@ static const struct alg_test_desc alg_test_descs[] = { .suite = { .cipher = __VECS(xchacha20_tv_template) }, + }, { + .alg = "xctr(aes)", + .test = alg_test_skcipher, + .suite = { + .cipher = __VECS(aes_xctr_tv_template) + } }, { .alg = "xts(aes)", .generic_driver = "xts(ecb(aes-generic))", diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 4d7449fc6a65..7179df0a39b6 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -34251,4 +34251,697 @@ static const struct hash_testvec blakes2s_256_tv_template[] = {{ 0xd5, 0x06, 0xb5, 0x3a, 0x7c, 0x7a, 0x65, 0x1d, }, }}; +/* + * Test vectors generated using https://github.com/google/hctr2 + */ +static const struct cipher_testvec aes_xctr_tv_template[] = { + { + .key = "\x9c\x8d\xc4\xbd\x71\x36\xdc\x82" + "\x7c\xa1\xca\xa3\x23\x5a\xdb\xa4", + .iv = "\x8d\xe7\xa5\x6a\x95\x86\x42\xde" + "\xba\xea\x6e\x69\x03\x33\x86\x0f", + .ptext = "\xbd", + .ctext = "\xb9", + .klen = 16, + .len = 1, + }, + { + .key = "\xbc\x1b\x12\x0c\x3f\x18\xcc\x1f" + "\x5a\x1d\xab\x81\xa8\x68\x7c\x63", + .iv = "\x22\xc1\xdd\x25\x0b\x18\xcb\xa5" + "\x4a\xda\x15\x07\x73\xd9\x88\x10", + .ptext = "\x24\x6e\x64\xc6\x15\x26\x9c\xda" + "\x2a\x4b\x57\x12\xff\x7c\xd6\xb5", + .ctext = "\xd6\x47\x8d\x58\x92\xb2\x84\xf9" + "\xb7\xee\x0d\x98\xa1\x39\x4d\x8f", + .klen = 16, + .len = 16, + }, + { + .key = "\x44\x03\xbf\x4c\x30\xf0\xa7\xd6" + "\xbd\x54\xbb\x66\x8e\xa6\x0e\x8a", + .iv = "\xe6\xf7\x26\xdf\x8c\x3c\xaa\x88" + "\xce\xc1\xbd\x43\x3b\x09\x62\xad", + .ptext = "\x3c\xe3\x46\xb9\x8f\x9d\x3f\x8d" + "\xef\xf2\x53\xab\x24\xe2\x29\x08" + "\xf8\x7e\x1d\xa6\x6d\x86\x7d\x60" + "\x97\x63\x93\x29\x71\x94\xb4", + .ctext = "\xd4\xa3\xc6\xb8\xc1\x6f\x70\x1a" + "\x52\x0c\xed\x4c\xaf\x51\x56\x23" + "\x48\x45\x07\x10\x34\xc5\xba\x71" + "\xe5\xf8\x1e\xd8\xcb\xa6\xe7", + .klen = 16, + .len = 31, + }, + { + .key = "\x5b\x17\x30\x94\x19\x31\xa1\xae" + "\x24\x8e\x42\x1e\x82\xe6\xec\xb8", + .iv = "\xd1\x2e\xb9\xb8\xf8\x49\xeb\x68" + "\x06\xeb\x65\x33\x34\xa2\xeb\xf0", + .ptext = "\x19\x75\xec\x59\x60\x1b\x7a\x3e" + "\x62\x46\x87\xf0\xde\xab\x81\x36" + "\x63\x53\x11\xa0\x1f\xce\x25\x85" + "\x49\x6b\x28\xfa\x1c\x92\xe5\x18" + "\x38\x14\x00\x79\xf2\x9e\xeb\xfc" + "\x36\xa7\x6b\xe1\xe5\xcf\x04\x48" + "\x44\x6d\xbd\x64\xb3\xcb\x78\x05" + "\x8d\x7f\x9a\xaf\x3c\xcf\x6c\x45" + "\x6c\x7c\x46\x4c\xa8\xc0\x1e\xe4" + "\x33\xa5\x7b\xbb\x26\xd9\xc0\x32" + "\x9d\x8a\xb3\xf3\x3d\x52\xe6\x48" + "\x4c\x9b\x4c\x6e\xa4\xa3\xad\x66" + "\x56\x48\xd5\x98\x3a\x93\xc4\x85" + "\xe9\x89\xca\xa6\xc1\xc8\xe7\xf8" + "\xc3\xe9\xef\xbe\x77\xe6\xd1\x3a" + "\xa6\x99\xc8\x2d\xdf\x40\x0f\x44", + .ctext = "\xc6\x1a\x01\x1a\x00\xba\x04\xff" + "\x10\xd1\x7e\x5d\xad\x91\xde\x8c" + "\x08\x55\x95\xae\xd7\x22\x77\x40" + "\xf0\x33\x1b\x51\xef\xfe\x3d\x67" + "\xdf\xc4\x9f\x39\x47\x67\x93\xab" + "\xaa\x37\x55\xfe\x41\xe0\xba\xcd" + "\x25\x02\x7c\x61\x51\xa1\xcc\x72" + "\x7a\x20\x26\xb9\x06\x68\xbd\x19" + "\xc5\x2e\x1b\x75\x4a\x40\xb2\xd2" + "\xc4\xee\xd8\x5b\xa4\x55\x7d\x25" + "\xfc\x01\x4d\x6f\x0a\xfd\x37\x5d" + "\x3e\x67\xc0\x35\x72\x53\x7b\xe2" + "\xd6\x19\x5b\x92\x6c\x3a\x8c\x2a" + "\xe2\xc2\xa2\x4f\x2a\xf2\xb5\x15" + "\x65\xc5\x8d\x97\xf9\xbf\x8c\x98" + "\xe4\x50\x1a\xf2\x76\x55\x07\x49", + .klen = 16, + .len = 128, + }, + { + .key = "\x17\xa6\x01\x3d\x5d\xd6\xef\x2d" + "\x69\x8f\x4c\x54\x5b\xae\x43\xf0", + .iv = "\xa9\x1b\x47\x60\x26\x82\xf7\x1c" + "\x80\xf8\x88\xdd\xfb\x44\xd9\xda", + .ptext = "\xf7\x67\xcd\xa6\x04\x65\x53\x99" + "\x90\x5c\xa2\x56\x74\xd7\x9d\xf2" + "\x0b\x03\x7f\x4e\xa7\x84\x72\x2b" + "\xf0\xa5\xbf\xe6\x9a\x62\x3a\xfe" + "\x69\x5c\x93\x79\x23\x86\x64\x85" + "\xeb\x13\xb1\x5a\xd5\x48\x39\xa0" + "\x70\xfb\x06\x9a\xd7\x12\x5a\xb9" + "\xbe\xed\x2c\x81\x64\xf7\xcf\x80" + "\xee\xe6\x28\x32\x2d\x37\x4c\x32" + "\xf4\x1f\x23\x21\xe9\xc8\xc9\xbf" + "\x54\xbc\xcf\xb4\xc2\x65\x39\xdf" + "\xa5\xfb\x14\x11\xed\x62\x38\xcf" + "\x9b\x58\x11\xdd\xe9\xbd\x37\x57" + "\x75\x4c\x9e\xd5\x67\x0a\x48\xc6" + "\x0d\x05\x4e\xb1\x06\xd7\xec\x2e" + "\x9e\x59\xde\x4f\xab\x38\xbb\xe5" + "\x87\x04\x5a\x2c\x2a\xa2\x8f\x3c" + "\xe7\xe1\x46\xa9\x49\x9f\x24\xad" + "\x2d\xb0\x55\x40\x64\xd5\xda\x7e" + "\x1e\x77\xb8\x29\x72\x73\xc3\x84" + "\xcd\xf3\x94\x90\x58\x76\xc9\x2c" + "\x2a\xad\x56\xde\x33\x18\xb6\x3b" + "\x10\xe9\xe9\x8d\xf0\xa9\x7f\x05" + "\xf7\xb5\x8c\x13\x7e\x11\x3d\x1e" + "\x02\xbb\x5b\xea\x69\xff\x85\xcf" + "\x6a\x18\x97\x45\xe3\x96\xba\x4d" + "\x2d\x7a\x70\x78\x15\x2c\xe9\xdc" + "\x4e\x09\x92\x57\x04\xd8\x0b\xa6" + "\x20\x71\x76\x47\x76\x96\x89\xa0" + "\xd9\x29\xa2\x5a\x06\xdb\x56\x39" + "\x60\x33\x59\x04\x95\x89\xf6\x18" + "\x1d\x70\x75\x85\x3a\xb7\x6e", + .ctext = "\xe1\xe7\x3f\xd3\x6a\xb9\x2f\x64" + "\x37\xc5\xa4\xe9\xca\x0a\xa1\xd6" + "\xea\x7d\x39\xe5\xe6\xcc\x80\x54" + "\x74\x31\x2a\x04\x33\x79\x8c\x8e" + "\x4d\x47\x84\x28\x27\x9b\x3c\x58" + "\x54\x58\x20\x4f\x70\x01\x52\x5b" + "\xac\x95\x61\x49\x5f\xef\xba\xce" + "\xd7\x74\x56\xe7\xbb\xe0\x3c\xd0" + "\x7f\xa9\x23\x57\x33\x2a\xf6\xcb" + "\xbe\x42\x14\x95\xa8\xf9\x7a\x7e" + "\x12\x53\x3a\xe2\x13\xfe\x2d\x89" + "\xeb\xac\xd7\xa8\xa5\xf8\x27\xf3" + "\x74\x9a\x65\x63\xd1\x98\x3a\x7e" + "\x27\x7b\xc0\x20\x00\x4d\xf4\xe5" + "\x7b\x69\xa6\xa8\x06\x50\x85\xb6" + "\x7f\xac\x7f\xda\x1f\xf5\x37\x56" + "\x9b\x2f\xd3\x86\x6b\x70\xbd\x0e" + "\x55\x9a\x9d\x4b\x08\xb5\x5b\x7b" + "\xd4\x7c\xb4\x71\x49\x92\x4a\x1e" + "\xed\x6d\x11\x09\x47\x72\x32\x6a" + "\x97\x53\x36\xaf\xf3\x06\x06\x2c" + "\x69\xf1\x59\x00\x36\x95\x28\x2a" + "\xb6\xcd\x10\x21\x84\x73\x5c\x96" + "\x86\x14\x2c\x3d\x02\xdb\x53\x9a" + "\x61\xde\xea\x99\x84\x7a\x27\xf6" + "\xf7\xc8\x49\x73\x4b\xb8\xeb\xd3" + "\x41\x33\xdd\x09\x68\xe2\x64\xb8" + "\x5f\x75\x74\x97\x91\x54\xda\xc2" + "\x73\x2c\x1e\x5a\x84\x48\x01\x1a" + "\x0d\x8b\x0a\xdf\x07\x2e\xee\x77" + "\x1d\x17\x41\x7a\xc9\x33\x63\xfa" + "\x9f\xc3\x74\x57\x5f\x03\x4c", + .klen = 16, + .len = 255, + }, + { + .key = "\xe5\xf1\x48\x2e\x88\xdb\xc7\x28" + "\xa2\x55\x5d\x2f\x90\x02\xdc\xd3" + "\xf5\xd3\x9e\x87\xd5\x58\x30\x4a", + .iv = "\xa6\x40\x39\xf9\x63\x6c\x2d\xd4" + "\x1b\x71\x05\xa4\x88\x86\x11\xd3", + .ptext = "\xb6\x06\xae\x15\x11\x96\xc1\x44" + "\x44\xc2\x98\xf9\xa8\x0a\x0b", + .ctext = "\x27\x3b\x68\x40\xa9\x5e\x74\x6b" + "\x74\x67\x18\xf9\x37\xed\xed", + .klen = 24, + .len = 15, + }, + { + .key = "\xc8\xa0\x27\x67\x04\x3f\xed\xa5" + "\xb4\x0c\x51\x91\x2d\x27\x77\x33" + "\xa5\xfc\x2a\x9f\x78\xd8\x1c\x68", + .iv = "\x83\x99\x1a\xe2\x84\xca\xa9\x16" + "\x8d\xc4\x2d\x1b\x67\xc8\x86\x21", + .ptext = "\xd6\x22\x85\xb8\x5d\x7e\x26\x2e" + "\xbe\x04\x9d\x0c\x03\x91\x45\x4a" + "\x36", + .ctext = "\x0f\x44\xa9\x62\x72\xec\x12\x26" + "\x3a\xc6\x83\x26\x62\x5e\xb7\x13" + "\x05", + .klen = 24, + .len = 17, + }, + { + .key = "\xc5\x87\x18\x09\x0a\x4e\x66\x3e" + "\x50\x90\x19\x93\xc0\x33\xcf\x80" + "\x3a\x36\x6b\x6c\x43\xd7\xe4\x93", + .iv = "\xdd\x0b\x75\x1f\xee\x2f\xb4\x52" + "\x10\x82\x1f\x79\x8a\xa4\x9b\x87", + .ptext = "\x56\xf9\x13\xce\x9f\x30\x10\x11" + "\x1b\x59\xfd\x39\x5a\x29\xa3\x44" + "\x78\x97\x8c\xf6\x99\x6d\x26\xf1" + "\x32\x60\x6a\xeb\x04\x47\x29\x4c" + "\x7e\x14\xef\x4d\x55\x29\xfe\x36" + "\x37\xcf\x0b\x6e\xf3\xce\x15\xd2", + .ctext = "\x8f\x98\xe1\x5a\x7f\xfe\xc7\x05" + "\x76\xb0\xd5\xde\x90\x52\x2b\xa8" + "\xf3\x6e\x3c\x77\xa5\x33\x63\xdd" + "\x6f\x62\x12\xb0\x80\x10\xc1\x28" + "\x58\xe5\xd6\x24\x44\x04\x55\xf3" + "\x6d\x94\xcb\x2c\x7e\x7a\x85\x79", + .klen = 24, + .len = 48, + }, + { + .key = "\x84\x9b\xe8\x10\x4c\xb3\xd1\x7a" + "\xb3\xab\x4e\x6f\x90\x12\x07\xf8" + "\xef\xde\x42\x09\xbf\x34\x95\xb2", + .iv = "\x66\x62\xf9\x48\x9d\x17\xf7\xdf" + "\x06\x67\xf4\x6d\xf2\xbc\xa2\xe5", + .ptext = "\x2f\xd6\x16\x6b\xf9\x4b\x44\x14" + "\x90\x93\xe5\xfd\x05\xaa\x00\x26" + "\xbd\xab\x11\xb8\xf0\xcb\x11\x72" + "\xdd\xc5\x15\x4f\x4e\x1b\xf8\xc9" + "\x8f\x4a\xd5\x69\xf8\x9e\xfb\x05" + "\x8a\x37\x46\xfe\xfa\x58\x9b\x0e" + "\x72\x90\x9a\x06\xa5\x42\xf4\x7c" + "\x35\xd5\x64\x70\x72\x67\xfc\x8b" + "\xab\x5a\x2f\x64\x9b\xa1\xec\xe7" + "\xe6\x92\x69\xdb\x62\xa4\xe7\x44" + "\x88\x28\xd4\x52\x64\x19\xa9\xd7" + "\x0c\x00\xe6\xe7\xc1\x28\xc1\xf5" + "\x72\xc5\xfa\x09\x22\x2e\xf4\x82" + "\xa3\xdc\xc1\x68\xf9\x29\x55\x8d" + "\x04\x67\x13\xa6\x52\x04\x3c\x0c" + "\x14\xf2\x87\x23\x61\xab\x82\xcb" + "\x49\x5b\x6b\xd4\x4f\x0d\xd4\x95" + "\x82\xcd\xe3\x69\x47\x1b\x31\x73" + "\x73\x77\xc1\x53\x7d\x43\x5e\x4a" + "\x80\x3a\xca\x9c\xc7\x04\x1a\x31" + "\x8e\xe6\x76\x7f\xe1\xb3\xd0\x57" + "\xa2\xb2\xf6\x09\x51\xc9\x6d\xbc" + "\x79\xed\x57\x50\x36\xd2\x93\xa4" + "\x40\x5d\xac\x3a\x3b\xb6\x2d\x89" + "\x78\xa2\xbd\x23\xec\x35\x06\xf0" + "\xa8\xc8\xc9\xb0\xe3\x28\x2b\xba" + "\x70\xa0\xfe\xed\x13\xc4\xd7\x90" + "\xb1\x6a\xe0\xe1\x30\x71\x15\xd0" + "\xe2\xb3\xa6\x4e\xb0\x01\xf9\xe7" + "\x59\xc6\x1e\xed\x46\x2b\xe3\xa8" + "\x22\xeb\x7f\x1c\xd9\xcd\xe0\xa6" + "\x72\x42\x2c\x06\x75\xbb\xb7\x6b" + "\xca\x49\x5e\xa1\x47\x8d\x9e\xfe" + "\x60\xcc\x34\x95\x8e\xfa\x1e\x3e" + "\x85\x4b\x03\x54\xea\x34\x1c\x41" + "\x90\x45\xa6\xbe\xcf\x58\x4f\xca" + "\x2c\x79\xc0\x3e\x8f\xd7\x3b\xd4" + "\x55\x74\xa8\xe1\x57\x09\xbf\xab" + "\x2c\xf9\xe4\xdd\x17\x99\x57\x60" + "\x4b\x88\x2a\x7f\x43\x86\xb9\x9a" + "\x60\xbf\x4c\xcf\x9b\x41\xb8\x99" + "\x69\x15\x4f\x91\x4d\xeb\xdf\x6f" + "\xcc\x4c\xf9\x6f\xf2\x33\x23\xe7" + "\x02\x44\xaa\xa2\xfa\xb1\x39\xa5" + "\xff\x88\xf5\x37\x02\x33\x24\xfc" + "\x79\x11\x4c\x94\xc2\x31\x87\x9c" + "\x53\x19\x99\x32\xe4\xde\x18\xf4" + "\x8f\xe2\xe8\xa3\xfb\x0b\xaa\x7c" + "\xdb\x83\x0f\xf6\xc0\x8a\x9b\xcd" + "\x7b\x16\x05\x5b\xe4\xb4\x34\x03" + "\xe3\x8f\xc9\x4b\x56\x84\x2a\x4c" + "\x36\x72\x3c\x84\x4f\xba\xa2\x7f" + "\xf7\x1b\xba\x4d\x8a\xb8\x5d\x51" + "\x36\xfb\xef\x23\x18\x6f\x33\x2d" + "\xbb\x06\x24\x8e\x33\x98\x6e\xcd" + "\x63\x11\x18\x6b\xcc\x1b\x66\xb9" + "\x38\x8d\x06\x8d\x98\x1a\xef\xaa" + "\x35\x4a\x90\xfa\xb1\xd3\xcc\x11" + "\x50\x4c\x54\x18\x60\x5d\xe4\x11" + "\xfc\x19\xe1\x53\x20\x5c\xe7\xef" + "\x8a\x2b\xa8\x82\x51\x5f\x5d\x43" + "\x34\xe5\xcf\x7b\x1b\x6f\x81\x19" + "\xb7\xdf\xa8\x9e\x81\x89\x5f\x33" + "\x69\xaf\xde\x89\x68\x88\xf0\x71", + .ctext = "\xab\x15\x46\x5b\xed\x4f\xa8\xac" + "\xbf\x31\x30\x84\x55\xa4\xb8\x98" + "\x79\xba\xa0\x15\xa4\x55\x20\xec" + "\xf9\x94\x71\xe6\x6a\x6f\xee\x87" + "\x2e\x3a\xa2\x95\xae\x6e\x56\x09" + "\xe9\xc0\x0f\xe2\xc6\xb7\x30\xa9" + "\x73\x8e\x59\x7c\xfd\xe3\x71\xf7" + "\xae\x8b\x91\xab\x5e\x36\xe9\xa8" + "\xff\x17\xfa\xa2\x94\x93\x11\x42" + "\x67\x96\x99\xc5\xf0\xad\x2a\x57" + "\xf9\xa6\x70\x4a\xdf\x71\xff\xc0" + "\xe2\xaf\x9a\xae\x57\x58\x13\x3b" + "\x2d\xf1\xc7\x8f\xdb\x8a\xcc\xce" + "\x53\x1a\x69\x55\x39\xc8\xbe\xc3" + "\x2d\xb1\x03\xd9\xa3\x99\xf4\x8d" + "\xd9\x2d\x27\xae\xa5\xe7\x77\x7f" + "\xbb\x88\x84\xea\xfa\x19\x3f\x44" + "\x61\x21\x8a\x1f\xbe\xac\x60\xb4" + "\xaf\xe9\x00\xab\xef\x3c\x53\x56" + "\xcd\x4b\x53\xd8\x9b\xfe\x88\x23" + "\x5b\x85\x76\x08\xec\xd1\x6e\x4a" + "\x87\xa4\x7d\x29\x4e\x4f\x3f\xc9" + "\xa4\xab\x63\xea\xdd\xef\x9f\x79" + "\x38\x18\x7d\x90\x90\xf9\x12\x57" + "\x1d\x89\xea\xfe\xd4\x47\x45\x32" + "\x6a\xf6\xe7\xde\x22\x7e\xee\xc1" + "\xbc\x2d\xc3\xbb\xe5\xd4\x13\xac" + "\x63\xff\x5b\xb1\x05\x96\xd5\xf3" + "\x07\x9a\x62\xb6\x30\xea\x7d\x1e" + "\xee\x75\x0a\x1b\xcc\x6e\x4d\xa7" + "\xf7\x4d\x74\xd8\x60\x32\x5e\xd0" + "\x93\xd7\x19\x90\x4e\x26\xdb\xe4" + "\x5e\xd4\xa8\xb9\x76\xba\x56\x91" + "\xc4\x75\x04\x1e\xc2\x77\x24\x6f" + "\xf9\xe8\x4a\xec\x7f\x86\x95\xb3" + "\x5c\x2c\x97\xab\xf0\xf7\x74\x5b" + "\x0b\xc2\xda\x42\x40\x34\x16\xed" + "\x06\xc1\x25\x53\x17\x0d\x81\x4e" + "\xe6\xf2\x0f\x6d\x94\x3c\x90\x7a" + "\xae\x20\xe9\x3f\xf8\x18\x67\x6a" + "\x49\x1e\x41\xb6\x46\xab\xc8\xa7" + "\xcb\x19\x96\xf5\x99\xc0\x66\x3e" + "\x77\xcf\x73\x52\x83\x2a\xe2\x48" + "\x27\x6c\xeb\xe7\xe7\xc4\xd5\x6a" + "\x40\x67\xbc\xbf\x6b\x3c\xf3\xbb" + "\x51\x5e\x31\xac\x03\x81\xab\x61" + "\xfa\xa5\xa6\x7d\x8b\xc3\x8a\x75" + "\x28\x7a\x71\x9c\xac\x8f\x76\xfc" + "\xf9\x6c\x5d\x9b\xd7\xf6\x36\x2d" + "\x61\xd5\x61\xaa\xdd\x01\xfc\x57" + "\x91\x10\xcd\xcd\x6d\x27\x63\x24" + "\x67\x46\x7a\xbb\x61\x56\x39\xb1" + "\xd6\x79\xfe\x77\xca\xd6\x73\x59" + "\x6e\x58\x11\x90\x03\x26\x74\x2a" + "\xfa\x52\x12\x47\xfb\x12\xeb\x3e" + "\x88\xf0\x52\x6c\xc0\x54\x7a\x88" + "\x8c\xe5\xde\x9e\xba\xb9\xf2\xe1" + "\x97\x2e\x5c\xbd\xf4\x13\x7e\xf3" + "\xc4\xe1\x87\xa5\x35\xfa\x7c\x71" + "\x1a\xc9\xf4\xa8\x57\xe2\x5a\x6b" + "\x14\xe0\x73\xaf\x56\x6b\xa0\x00" + "\x9e\x5f\x64\xac\x00\xfb\xc4\x92" + "\xe5\xe2\x8a\xb2\x9e\x75\x49\x85" + "\x25\x66\xa5\x1a\xf9\x7d\x1d\x60", + .klen = 24, + .len = 512, + }, + { + .key = "\x05\x60\x3a\x7e\x60\x90\x46\x18" + "\x6c\x60\xba\xeb\x12\xd7\xbe\xd1" + "\xd3\xf6\x10\x46\x9d\xf1\x0c\xb4" + "\x73\xe3\x93\x27\xa8\x2c\x13\xaa", + .iv = "\xf5\x96\xd1\xb6\xcb\x44\xd8\xd0" + "\x3e\xdb\x92\x80\x08\x94\xcd\xd3", + .ptext = "\x78", + .ctext = "\xc5", + .klen = 32, + .len = 1, + }, + { + .key = "\x35\xca\x38\xf3\xd9\xd6\x34\xef" + "\xcd\xee\xa3\x26\x86\xba\xfb\x45" + "\x01\xfa\x52\x67\xff\xc5\x9d\xaa" + "\x64\x9a\x05\xbb\x85\x20\xa7\xf2", + .iv = "\xe3\xda\xf5\xff\x42\x59\x87\x86" + "\xee\x7b\xd6\xb4\x6a\x25\x44\xff", + .ptext = "\x44\x67\x1e\x04\x53\xd2\x4b\xd9" + "\x96\x33\x07\x54\xe4\x8e\x20", + .ctext = "\xcc\x55\x40\x79\x47\x5c\x8b\xa6" + "\xca\x7b\x9f\x50\xe3\x21\xea", + .klen = 32, + .len = 15, + }, + { + .key = "\xaf\xd9\x14\x14\xd5\xdb\xc9\xce" + "\x76\x5c\x5a\xbf\x43\x05\x29\x24" + "\xc4\x13\x68\xcc\xe8\x37\xbd\xb9" + "\x41\x20\xf5\x53\x48\xd0\xa2\xd6", + .iv = "\xa7\xb4\x00\x08\x79\x10\xae\xf5" + "\x02\xbf\x85\xb2\x69\x4c\xc6\x04", + .ptext = "\xac\x6a\xa8\x0c\xb0\x84\xbf\x4c" + "\xae\x94\x20\x58\x7e\x00\x93\x89", + .ctext = "\xd5\xaa\xe2\xe9\x86\x4c\x95\x4e" + "\xde\xb6\x15\xcb\xdc\x1f\x13\x38", + .klen = 32, + .len = 16, + }, + { + .key = "\xed\xe3\x8b\xe7\x1c\x17\xbf\x4a" + "\x02\xe2\xfc\x76\xac\xf5\x3c\x00" + "\x5d\xdc\xfc\x83\xeb\x45\xb4\xcb" + "\x59\x62\x60\xec\x69\x9c\x16\x45", + .iv = "\xe4\x0e\x2b\x90\xd2\xfa\x94\x2e" + "\x10\xe5\x64\x2b\x97\x28\x15\xc7", + .ptext = "\xe6\x53\xff\x60\x0e\xc4\x51\xe4" + "\x93\x4d\xe5\x55\xc5\xd9\xad\x48" + "\x52", + .ctext = "\xba\x25\x28\xf5\xcf\x31\x91\x80" + "\xda\x2b\x95\x5f\x20\xcb\xfb\x9f" + "\xc6", + .klen = 32, + .len = 17, + }, + { + .key = "\x77\x5c\xc0\x73\x9a\x64\x97\x91" + "\x2f\xee\xe0\x20\xc2\x04\x59\x2e" + "\x97\xd2\xa7\x70\xb3\xb0\x21\x6b" + "\x8f\xbf\xb8\x51\xa8\xea\x0f\x62", + .iv = "\x31\x8e\x1f\xcd\xfd\x23\xeb\x7f" + "\x8a\x1f\x1b\x23\x53\x27\x44\xe5", + .ptext = "\xcd\xff\x8c\x9b\x94\x5a\x51\x3f" + "\x40\x93\x56\x93\x66\x39\x63\x1f" + "\xbf\xe6\xa4\xfa\xbe\x79\x93\x03" + "\xf5\x66\x74\x16\xfc\xe4\xce", + .ctext = "\x8b\xd3\xc3\xce\x66\xf8\x66\x4c" + "\xad\xd6\xf5\x0f\xd8\x99\x5a\x75" + "\xa1\x3c\xab\x0b\x21\x36\x57\x72" + "\x88\x29\xe9\xea\x4a\x8d\xe9", + .klen = 32, + .len = 31, + }, + { + .key = "\xa1\x2f\x4d\xde\xfe\xa1\xff\xa8" + "\x73\xdd\xe3\xe2\x95\xfc\xea\x9c" + "\xd0\x80\x42\x0c\xb8\x43\x3e\x99" + "\x39\x38\x0a\x8c\xe8\x45\x3a\x7b", + .iv = "\x32\xc4\x6f\xb1\x14\x43\xd1\x87" + "\xe2\x6f\x5a\x58\x02\x36\x7e\x2a", + .ptext = "\x9e\x5c\x1e\xf1\xd6\x7d\x09\x57" + "\x18\x48\x55\xda\x7d\x44\xf9\x6d" + "\xac\xcd\x59\xbb\x10\xa2\x94\x67" + "\xd1\x6f\xfe\x6b\x4a\x11\xe8\x04" + "\x09\x26\x4f\x8d\x5d\xa1\x7b\x42" + "\xf9\x4b\x66\x76\x38\x12\xfe\xfe", + .ctext = "\x42\xbc\xa7\x64\x15\x9a\x04\x71" + "\x2c\x5f\x94\xba\x89\x3a\xad\xbc" + "\x87\xb3\xf4\x09\x4f\x57\x06\x18" + "\xdc\x84\x20\xf7\x64\x85\xca\x3b" + "\xab\xe6\x33\x56\x34\x60\x5d\x4b" + "\x2e\x16\x13\xd4\x77\xde\x2d\x2b", + .klen = 32, + .len = 48, + }, + { + .key = "\xfb\xf5\xb7\x3d\xa6\x95\x42\xbf" + "\xd2\x94\x6c\x74\x0f\xbc\x5a\x28" + "\x35\x3c\x51\x58\x84\xfb\x7d\x11" + "\x16\x1e\x00\x97\x37\x08\xb7\x16", + .iv = "\x9b\x53\x57\x40\xe6\xd9\xa7\x27" + "\x78\xd4\x9b\xd2\x29\x1d\x24\xa9", + .ptext = "\x8b\x02\x60\x0a\x3e\xb7\x10\x59" + "\xc3\xac\xd5\x2a\x75\x81\xf2\xdb" + "\x55\xca\x65\x86\x44\xfb\xfe\x91" + "\x26\xbb\x45\xb2\x46\x22\x3e\x08" + "\xa2\xbf\x46\xcb\x68\x7d\x45\x7b" + "\xa1\x6a\x3c\x6e\x25\xeb\xed\x31" + "\x7a\x8b\x47\xf9\xde\xec\x3d\x87" + "\x09\x20\x2e\xfa\xba\x8b\x9b\xc5" + "\x6c\x25\x9c\x9d\x2a\xe8\xab\x90" + "\x3f\x86\xee\x61\x13\x21\xd4\xde" + "\xe1\x0c\x95\xfc\x5c\x8a\x6e\x0a" + "\x73\xcf\x08\x69\x44\x4e\xde\x25" + "\xaf\xaa\x56\x04\xc4\xb3\x60\x44" + "\x3b\x8b\x3d\xee\xae\x42\x4b\xd2" + "\x9a\x6c\xa0\x8e\x52\x06\xb2\xd1" + "\x5d\x38\x30\x6d\x27\x9b\x1a\xd8", + .ctext = "\xa3\x78\x33\x78\x95\x95\x97\x07" + "\x53\xa3\xa1\x5b\x18\x32\x27\xf7" + "\x09\x12\x53\x70\x83\xb5\x6a\x9f" + "\x26\x6d\x10\x0d\xe0\x1c\xe6\x2b" + "\x70\x00\xdc\xa1\x60\xef\x1b\xee" + "\xc5\xa5\x51\x17\xae\xcc\xf2\xed" + "\xc4\x60\x07\xdf\xd5\x7a\xe9\x90" + "\x3c\x9f\x96\x5d\x72\x65\x5d\xef" + "\xd0\x94\x32\xc4\x85\x90\x78\xa1" + "\x2e\x64\xf6\xee\x8e\x74\x3f\x20" + "\x2f\x12\x3b\x3d\xd5\x39\x8e\x5a" + "\xf9\x8f\xce\x94\x5d\x82\x18\x66" + "\x14\xaf\x4c\xfe\xe0\x91\xc3\x4a" + "\x85\xcf\xe7\xe8\xf7\xcb\xf0\x31" + "\x88\x7d\xc9\x5b\x71\x9d\x5f\xd2" + "\xfa\xed\xa6\x24\xda\xbb\xb1\x84", + .klen = 32, + .len = 128, + }, + { + .key = "\x32\x37\x2b\x8f\x7b\xb1\x23\x79" + "\x05\x52\xde\x05\xf1\x68\x3f\x6c" + "\xa4\xae\xbc\x21\xc2\xc6\xf0\xbd" + "\x0f\x20\xb7\xa4\xc5\x05\x7b\x64", + .iv = "\xff\x26\x4e\x67\x48\xdd\xcf\xfe" + "\x42\x09\x04\x98\x5f\x1e\xfa\x80", + .ptext = "\x99\xdc\x3b\x19\x41\xf9\xff\x6e" + "\x76\xb5\x03\xfa\x61\xed\xf8\x44" + "\x70\xb9\xf0\x83\x80\x6e\x31\x77" + "\x77\xe4\xc7\xb4\x77\x02\xab\x91" + "\x82\xc6\xf8\x7c\x46\x61\x03\x69" + "\x09\xa0\xf7\x12\xb7\x81\x6c\xa9" + "\x10\x5c\xbb\x55\xb3\x44\xed\xb5" + "\xa2\x52\x48\x71\x90\x5d\xda\x40" + "\x0b\x7f\x4a\x11\x6d\xa7\x3d\x8e" + "\x1b\xcd\x9d\x4e\x75\x8b\x7d\x87" + "\xe5\x39\x34\x32\x1e\xe6\x8d\x51" + "\xd4\x1f\xe3\x1d\x50\xa0\x22\x37" + "\x7c\xb0\xd9\xfb\xb6\xb2\x16\xf6" + "\x6d\x26\xa0\x4e\x8c\x6a\xe6\xb6" + "\xbe\x4c\x7c\xe3\x88\x10\x18\x90" + "\x11\x50\x19\x90\xe7\x19\x3f\xd0" + "\x31\x15\x0f\x06\x96\xfe\xa7\x7b" + "\xc3\x32\x88\x69\xa4\x12\xe3\x64" + "\x02\x30\x17\x74\x6c\x88\x7c\x9b" + "\xd6\x6d\x75\xdf\x11\x86\x70\x79" + "\x48\x7d\x34\x3e\x33\x58\x07\x8b" + "\xd2\x50\xac\x35\x15\x45\x05\xb4" + "\x4d\x31\x97\x19\x87\x23\x4b\x87" + "\x53\xdc\xa9\x19\x78\xf1\xbf\x35" + "\x30\x04\x14\xd4\xcf\xb2\x8c\x87" + "\x7d\xdb\x69\xc9\xcd\xfe\x40\x3e" + "\x8d\x66\x5b\x61\xe5\xf0\x2d\x87" + "\x93\x3a\x0c\x2b\x04\x98\x05\xc2" + "\x56\x4d\xc4\x6c\xcd\x7a\x98\x7e" + "\xe2\x2d\x79\x07\x91\x9f\xdf\x2f" + "\x72\xc9\x8f\xcb\x0b\x87\x1b\xb7" + "\x04\x86\xcb\x47\xfa\x5d\x03", + .ctext = "\x0b\x00\xf7\xf2\xc8\x6a\xba\x9a" + "\x0a\x97\x18\x7a\x00\xa0\xdb\xf4" + "\x5e\x8e\x4a\xb7\xe0\x51\xf1\x75" + "\x17\x8b\xb4\xf1\x56\x11\x05\x9f" + "\x2f\x2e\xba\x67\x04\xe1\xb4\xa5" + "\xfc\x7c\x8c\xad\xc6\xb9\xd1\x64" + "\xca\xbd\x5d\xaf\xdb\x65\x48\x4f" + "\x1b\xb3\x94\x5c\x0b\xd0\xee\xcd" + "\xb5\x7f\x43\x8a\xd8\x8b\x66\xde" + "\xd2\x9c\x13\x65\xa4\x47\xa7\x03" + "\xc5\xa1\x46\x8f\x2f\x84\xbc\xef" + "\x48\x9d\x9d\xb5\xbd\x43\xff\xd2" + "\xd2\x7a\x5a\x13\xbf\xb4\xf6\x05" + "\x17\xcd\x01\x12\xf0\x35\x27\x96" + "\xf4\xc1\x65\xf7\x69\xef\x64\x1b" + "\x6e\x4a\xe8\x77\xce\x83\x01\xb7" + "\x60\xe6\x45\x2a\xcd\x41\x4a\xb5" + "\x8e\xcc\x45\x93\xf1\xd6\x64\x5f" + "\x32\x60\xe4\x29\x4a\x82\x6c\x86" + "\x16\xe4\xcc\xdb\x5f\xc8\x11\xa6" + "\xfe\x88\xd6\xc3\xe5\x5c\xbb\x67" + "\xec\xa5\x7b\xf5\xa8\x4f\x77\x25" + "\x5d\x0c\x2a\x99\xf9\xb9\xd1\xae" + "\x3c\x83\x2a\x93\x9b\x66\xec\x68" + "\x2c\x93\x02\x8a\x8a\x1e\x2f\x50" + "\x09\x37\x19\x5c\x2a\x3a\xc2\xcb" + "\xcb\x89\x82\x81\xb7\xbb\xef\x73" + "\x8b\xc9\xae\x42\x96\xef\x70\xc0" + "\x89\xc7\x3e\x6a\x26\xc3\xe4\x39" + "\x53\xa9\xcf\x63\x7d\x05\xf3\xff" + "\x52\x04\xf6\x7f\x23\x96\xe9\xf7" + "\xff\xd6\x50\xa3\x0e\x20\x71", + .klen = 32, + .len = 255, + }, + { + .key = "\x39\x5f\xf4\x9c\x90\x3a\x9a\x25" + "\x15\x11\x79\x39\xed\x26\x5e\xf6" + "\xda\xcf\x33\x4f\x82\x97\xab\x10" + "\xc1\x55\x48\x82\x80\xa8\x02\xb2", + .iv = "\x82\x60\xd9\x06\xeb\x40\x99\x76" + "\x08\xc5\xa4\x83\x45\xb8\x38\x5a", + .ptext = "\xa1\xa8\xac\xac\x08\xaf\x8f\x84" + "\xbf\xcc\x79\x31\x5e\x61\x01\xd1" + "\x4d\x5f\x9b\xcd\x91\x92\x9a\xa1" + "\x99\x0d\x49\xb2\xd7\xfd\x25\x93" + "\x51\x96\xbd\x91\x8b\x08\xf1\xc6" + "\x0d\x17\xf6\xef\xfd\xd2\x78\x16" + "\xc8\x08\x27\x7b\xca\x98\xc6\x12" + "\x86\x11\xdb\xd5\x08\x3d\x5a\x2c" + "\xcf\x15\x0e\x9b\x42\x78\xeb\x1f" + "\x52\xbc\xd7\x5a\x8a\x33\x6c\x14" + "\xfc\x61\xad\x2e\x1e\x03\x66\xea" + "\x79\x0e\x88\x88\xde\x93\xe3\x81" + "\xb5\xc4\x1c\xe6\x9c\x08\x18\x8e" + "\xa0\x87\xda\xe6\xf8\xcb\x30\x44" + "\x2d\x4e\xc0\xa3\x60\xf9\x62\x7b" + "\x4b\xd5\x61\x6d\xe2\x67\x95\x54" + "\x10\xd1\xca\x22\xe8\xb6\xb1\x3a" + "\x2d\xd7\x35\x5b\x22\x88\x55\x67" + "\x3d\x83\x8f\x07\x98\xa8\xf2\xcf" + "\x04\xb7\x9e\x52\xca\xe0\x98\x72" + "\x5c\xc1\x00\xd4\x1f\x2c\x61\xf3" + "\xe8\x40\xaf\x4a\xee\x66\x41\xa0" + "\x02\x77\x29\x30\x65\x59\x4b\x20" + "\x7b\x0d\x80\x97\x27\x7f\xd5\x90" + "\xbb\x9d\x76\x90\xe5\x43\x43\x72" + "\xd0\xd4\x14\x75\x66\xb3\xb6\xaf" + "\x09\xe4\x23\xb0\x62\xad\x17\x28" + "\x39\x26\xab\xf5\xf7\x5c\xb6\x33" + "\xbd\x27\x09\x5b\x29\xe4\x40\x0b" + "\xc1\x26\x32\xdb\x9a\xdf\xf9\x5a" + "\xae\x03\x2c\xa4\x40\x84\x9a\xb7" + "\x4e\x47\xa8\x0f\x23\xc7\xbb\xcf" + "\x2b\xf2\x32\x6c\x35\x6a\x91\xba" + "\x0e\xea\xa2\x8b\x2f\xbd\xb5\xea" + "\x6e\xbc\xb5\x4b\x03\xb3\x86\xe0" + "\x86\xcf\xba\xcb\x38\x2c\x32\xa6" + "\x6d\xe5\x28\xa6\xad\xd2\x7f\x73" + "\x43\x14\xf8\xb1\x99\x12\x2d\x2b" + "\xdf\xcd\xf2\x81\x43\x94\xdf\xb1" + "\x17\xc9\x33\xa6\x3d\xef\x96\xb8" + "\xd6\x0d\x00\xec\x49\x66\x85\x5d" + "\x44\x62\x12\x04\x55\x5c\x48\xd3" + "\xbd\x73\xac\x54\x8f\xbf\x97\x8e" + "\x85\xfd\xc2\xa1\x25\x32\x38\x6a" + "\x1f\xac\x57\x3c\x4f\x56\x73\xf2" + "\x1d\xb6\x48\x68\xc7\x0c\xe7\x60" + "\xd2\x8e\x4d\xfb\xc7\x20\x7b\xb7" + "\x45\x28\x12\xc6\x26\xae\xea\x7c" + "\x5d\xe2\x46\xb5\xae\xe1\xc3\x98" + "\x6f\x72\xd5\xa2\xfd\xed\x40\xfd" + "\xf9\xdf\x61\xec\x45\x2c\x15\xe0" + "\x1e\xbb\xde\x71\x37\x5f\x73\xc2" + "\x11\xcc\x6e\x6d\xe1\xb5\x1b\xd2" + "\x2a\xdd\x19\x8a\xc2\xe1\xa0\xa4" + "\x26\xeb\xb2\x2c\x4f\x77\x52\xf1" + "\x42\x72\x6c\xad\xd7\x78\x5d\x72" + "\xc9\x16\x26\x25\x1b\x4c\xe6\x58" + "\x79\x57\xb5\x06\x15\x4f\xe5\xba" + "\xa2\x7f\x2d\x5b\x87\x8a\x44\x70" + "\xec\xc7\xef\x84\xae\x60\xa2\x61" + "\x86\xe9\x18\xcd\x28\xc4\xa4\xf5" + "\xbc\x84\xb8\x86\xa0\xba\xf1\xf1" + "\x08\x3b\x32\x75\x35\x22\x7a\x65" + "\xca\x48\xe8\xef\x6e\xe2\x8e\x00", + .ctext = "\x2f\xae\xd8\x67\xeb\x15\xde\x75" + "\x53\xa3\x0e\x5a\xcf\x1c\xbe\xea" + "\xde\xf9\xcf\xc2\x9f\xfd\x0f\x44" + "\xc0\xe0\x7a\x76\x1d\xcb\x4a\xf8" + "\x35\xd6\xe3\x95\x98\x6b\x3f\x89" + "\xc4\xe6\xb6\x6f\xe1\x8b\x39\x4b" + "\x1c\x6c\x77\xe4\xe1\x8a\xbc\x61" + "\x00\x6a\xb1\x37\x2f\x45\xe6\x04" + "\x52\x0b\xfc\x1e\x32\xc1\xd8\x9d" + "\xfa\xdd\x67\x5c\xe0\x75\x83\xd0" + "\x21\x9e\x02\xea\xc0\x7f\xc0\x29" + "\xb3\x6c\xa5\x97\xb3\x29\x82\x1a" + "\x94\xa5\xb4\xb6\x49\xe5\xa5\xad" + "\x95\x40\x52\x7c\x84\x88\xa4\xa8" + "\x26\xe4\xd9\x5d\x41\xf2\x93\x7b" + "\xa4\x48\x1b\x66\x91\xb9\x7c\xc2" + "\x99\x29\xdf\xd8\x30\xac\xd4\x47" + "\x42\xa0\x14\x87\x67\xb8\xfd\x0b" + "\x1e\xcb\x5e\x5c\x9a\xc2\x04\x8b" + "\x17\x29\x9d\x99\x7f\x86\x4c\xe2" + "\x5c\x96\xa6\x0f\xb6\x47\x33\x5c" + "\xe4\x50\x49\xd5\x4f\x92\x0b\x9a" + "\xbc\x52\x4c\x41\xf5\xc9\x3e\x76" + "\x55\x55\xd4\xdc\x71\x14\x23\xfc" + "\x5f\xd5\x08\xde\xa0\xf7\x28\xc0" + "\xe1\x61\xac\x64\x66\xf6\xd1\x31" + "\xe4\xa4\xa9\xed\xbc\xad\x4f\x3b" + "\x59\xb9\x48\x1b\xe7\xb1\x6f\xc6" + "\xba\x40\x1c\x0b\xe7\x2f\x31\x65" + "\x85\xf5\xe9\x14\x0a\x31\xf5\xf3" + "\xc0\x1c\x20\x35\x73\x38\x0f\x8e" + "\x39\xf0\x68\xae\x08\x9c\x87\x4b" + "\x42\xfc\x22\x17\xee\x96\x51\x2a" + "\xd8\x57\x5a\x35\xea\x72\x74\xfc" + "\xb3\x0e\x69\x9a\xe1\x4f\x24\x90" + "\xc5\x4b\xe5\xd7\xe3\x82\x2f\xc5" + "\x62\x46\x3e\xab\x72\x4e\xe0\xf3" + "\x90\x09\x4c\xb2\xe1\xe8\xa0\xf5" + "\x46\x40\x2b\x47\x85\x3c\x21\x90" + "\x3d\xad\x25\x5a\x36\xdf\xe5\xbc" + "\x7e\x80\x4d\x53\x77\xf1\x79\xa6" + "\xec\x22\x80\x88\x68\xd6\x2d\x8b" + "\x3e\xf7\x52\xc7\x2a\x20\x42\x5c" + "\xed\x99\x4f\x32\x80\x00\x7e\x73" + "\xd7\x6d\x7f\x7d\x42\x54\x4a\xfe" + "\xff\x6f\x61\xca\x2a\xbb\x4f\xeb" + "\x4f\xe4\x4e\xaf\x2c\x4f\x82\xcd" + "\xa1\xa7\x11\xb3\x34\x33\xcf\x32" + "\x63\x0e\x24\x3a\x35\xbe\x06\xd5" + "\x17\xcb\x02\x30\x33\x6e\x8c\x49" + "\x40\x6e\x34\x8c\x07\xd4\x3e\xe6" + "\xaf\x78\x6d\x8c\x10\x5f\x21\x58" + "\x49\x26\xc5\xaf\x0d\x7d\xd4\xaf" + "\xcd\x5b\xa1\xe3\xf6\x39\x1c\x9b" + "\x8e\x00\xa1\xa7\x9e\x17\x4a\xc0" + "\x54\x56\x9e\xcf\xcf\x88\x79\x8d" + "\x50\xf7\x56\x8e\x0a\x73\x46\x6b" + "\xc3\xb9\x9b\x6c\x7d\xc4\xc8\xb6" + "\x03\x5f\x30\x62\x7d\xe6\xdb\x15" + "\xe1\x39\x02\x8c\xff\xda\xc8\x43" + "\xf2\xa9\xbf\x00\xe7\x3a\x61\x89" + "\xdf\xb0\xca\x7d\x8c\x8a\x6a\x9f" + "\x18\x89\x3d\x39\xac\x36\x6f\x05" + "\x1f\xb5\xda\x00\xea\xe1\x51\x21", + .klen = 32, + .len = 512, + }, + +}; + #endif /* _CRYPTO_TESTMGR_H */ diff --git a/crypto/xctr.c b/crypto/xctr.c new file mode 100644 index 000000000000..5c00147e8ec4 --- /dev/null +++ b/crypto/xctr.c @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * XCTR: XOR Counter mode - Adapted from ctr.c + * + * (C) Copyright IBM Corp. 2007 - Joy Latten + * Copyright 2021 Google LLC + */ + +/* + * XCTR mode is a blockcipher mode of operation used to implement HCTR2. XCTR is + * closely related to the CTR mode of operation; the main difference is that CTR + * generates the keystream using E(CTR + IV) whereas XCTR generates the + * keystream using E(CTR ^ IV). This allows implementations to avoid dealing + * with multi-limb integers (as is required in CTR mode). XCTR is also specified + * using little-endian arithmetic which makes it slightly faster on LE machines. + * + * See the HCTR2 paper for more details: + * Length-preserving encryption with HCTR2 + * (https://eprint.iacr.org/2021/1441.pdf) + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* For now this implementation is limited to 16-byte blocks for simplicity */ +#define XCTR_BLOCKSIZE 16 + +static void crypto_xctr_crypt_final(struct skcipher_walk *walk, + struct crypto_cipher *tfm, u32 byte_ctr) +{ + u8 keystream[XCTR_BLOCKSIZE]; + const u8 *src = walk->src.virt.addr; + u8 *dst = walk->dst.virt.addr; + unsigned int nbytes = walk->nbytes; + __le32 ctr32 = cpu_to_le32(byte_ctr / XCTR_BLOCKSIZE + 1); + + crypto_xor(walk->iv, (u8 *)&ctr32, sizeof(ctr32)); + crypto_cipher_encrypt_one(tfm, keystream, walk->iv); + crypto_xor_cpy(dst, keystream, src, nbytes); + crypto_xor(walk->iv, (u8 *)&ctr32, sizeof(ctr32)); +} + +static int crypto_xctr_crypt_segment(struct skcipher_walk *walk, + struct crypto_cipher *tfm, u32 byte_ctr) +{ + void (*fn)(struct crypto_tfm *, u8 *, const u8 *) = + crypto_cipher_alg(tfm)->cia_encrypt; + const u8 *src = walk->src.virt.addr; + u8 *dst = walk->dst.virt.addr; + unsigned int nbytes = walk->nbytes; + __le32 ctr32 = cpu_to_le32(byte_ctr / XCTR_BLOCKSIZE + 1); + + do { + crypto_xor(walk->iv, (u8 *)&ctr32, sizeof(ctr32)); + fn(crypto_cipher_tfm(tfm), dst, walk->iv); + crypto_xor(dst, src, XCTR_BLOCKSIZE); + crypto_xor(walk->iv, (u8 *)&ctr32, sizeof(ctr32)); + + le32_add_cpu(&ctr32, 1); + + src += XCTR_BLOCKSIZE; + dst += XCTR_BLOCKSIZE; + } while ((nbytes -= XCTR_BLOCKSIZE) >= XCTR_BLOCKSIZE); + + return nbytes; +} + +static int crypto_xctr_crypt_inplace(struct skcipher_walk *walk, + struct crypto_cipher *tfm, u32 byte_ctr) +{ + void (*fn)(struct crypto_tfm *, u8 *, const u8 *) = + crypto_cipher_alg(tfm)->cia_encrypt; + unsigned long alignmask = crypto_cipher_alignmask(tfm); + unsigned int nbytes = walk->nbytes; + u8 *data = walk->src.virt.addr; + u8 tmp[XCTR_BLOCKSIZE + MAX_CIPHER_ALIGNMASK]; + u8 *keystream = PTR_ALIGN(tmp + 0, alignmask + 1); + __le32 ctr32 = cpu_to_le32(byte_ctr / XCTR_BLOCKSIZE + 1); + + do { + crypto_xor(walk->iv, (u8 *)&ctr32, sizeof(ctr32)); + fn(crypto_cipher_tfm(tfm), keystream, walk->iv); + crypto_xor(data, keystream, XCTR_BLOCKSIZE); + crypto_xor(walk->iv, (u8 *)&ctr32, sizeof(ctr32)); + + le32_add_cpu(&ctr32, 1); + + data += XCTR_BLOCKSIZE; + } while ((nbytes -= XCTR_BLOCKSIZE) >= XCTR_BLOCKSIZE); + + return nbytes; +} + +static int crypto_xctr_crypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct crypto_cipher *cipher = skcipher_cipher_simple(tfm); + struct skcipher_walk walk; + unsigned int nbytes; + int err; + u32 byte_ctr = 0; + + err = skcipher_walk_virt(&walk, req, false); + + while (walk.nbytes >= XCTR_BLOCKSIZE) { + if (walk.src.virt.addr == walk.dst.virt.addr) + nbytes = crypto_xctr_crypt_inplace(&walk, cipher, + byte_ctr); + else + nbytes = crypto_xctr_crypt_segment(&walk, cipher, + byte_ctr); + + byte_ctr += walk.nbytes - nbytes; + err = skcipher_walk_done(&walk, nbytes); + } + + if (walk.nbytes) { + crypto_xctr_crypt_final(&walk, cipher, byte_ctr); + err = skcipher_walk_done(&walk, 0); + } + + return err; +} + +static int crypto_xctr_create(struct crypto_template *tmpl, struct rtattr **tb) +{ + struct skcipher_instance *inst; + struct crypto_alg *alg; + int err; + + inst = skcipher_alloc_instance_simple(tmpl, tb); + if (IS_ERR(inst)) + return PTR_ERR(inst); + + alg = skcipher_ialg_simple(inst); + + /* Block size must be 16 bytes. */ + err = -EINVAL; + if (alg->cra_blocksize != XCTR_BLOCKSIZE) + goto out_free_inst; + + /* XCTR mode is a stream cipher. */ + inst->alg.base.cra_blocksize = 1; + + /* + * To simplify the implementation, configure the skcipher walk to only + * give a partial block at the very end, never earlier. + */ + inst->alg.chunksize = alg->cra_blocksize; + + inst->alg.encrypt = crypto_xctr_crypt; + inst->alg.decrypt = crypto_xctr_crypt; + + err = skcipher_register_instance(tmpl, inst); + if (err) { +out_free_inst: + inst->free(inst); + } + + return err; +} + +static struct crypto_template crypto_xctr_tmpl = { + .name = "xctr", + .create = crypto_xctr_create, + .module = THIS_MODULE, +}; + +static int __init crypto_xctr_module_init(void) +{ + return crypto_register_template(&crypto_xctr_tmpl); +} + +static void __exit crypto_xctr_module_exit(void) +{ + crypto_unregister_template(&crypto_xctr_tmpl); +} + +subsys_initcall(crypto_xctr_module_init); +module_exit(crypto_xctr_module_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("XCTR block cipher mode of operation"); +MODULE_ALIAS_CRYPTO("xctr"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); From f3c923a09c4c4f5861b1ed53cf75673992a6ba68 Mon Sep 17 00:00:00 2001 From: Nathan Huckleberry Date: Fri, 20 May 2022 18:14:54 +0000 Subject: [PATCH 009/318] crypto: polyval - Add POLYVAL support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support for POLYVAL, an ε-Δ-universal hash function similar to GHASH. This patch only uses POLYVAL as a component to implement HCTR2 mode. It should be noted that POLYVAL was originally specified for use in AES-GCM-SIV (RFC 8452), but the kernel does not currently support this mode. POLYVAL is implemented as an shash algorithm. The implementation is modified from ghash-generic.c. For more information on POLYVAL see: Length-preserving encryption with HCTR2: https://eprint.iacr.org/2021/1441.pdf AES-GCM-SIV: Nonce Misuse-Resistant Authenticated Encryption: https://datatracker.ietf.org/doc/html/rfc8452 Signed-off-by: Nathan Huckleberry Reviewed-by: Eric Biggers Reviewed-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/Kconfig | 8 ++ crypto/Makefile | 1 + crypto/polyval-generic.c | 205 +++++++++++++++++++++++++++++++++++++++ crypto/tcrypt.c | 4 + crypto/testmgr.c | 6 ++ crypto/testmgr.h | 171 ++++++++++++++++++++++++++++++++ include/crypto/polyval.h | 17 ++++ 7 files changed, 412 insertions(+) create mode 100644 crypto/polyval-generic.c create mode 100644 include/crypto/polyval.h diff --git a/crypto/Kconfig b/crypto/Kconfig index b9e4d511bf5a..d59e70dca197 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -773,6 +773,14 @@ config CRYPTO_GHASH GHASH is the hash function used in GCM (Galois/Counter Mode). It is not a general-purpose cryptographic hash function. +config CRYPTO_POLYVAL + tristate + select CRYPTO_GF128MUL + select CRYPTO_HASH + help + POLYVAL is the hash function used in HCTR2. It is not a general-purpose + cryptographic hash function. + config CRYPTO_POLY1305 tristate "Poly1305 authenticator algorithm" select CRYPTO_HASH diff --git a/crypto/Makefile b/crypto/Makefile index 93d0afeb3a77..7694ed0a44d5 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -172,6 +172,7 @@ UBSAN_SANITIZE_jitterentropy.o = n jitterentropy_rng-y := jitterentropy.o jitterentropy-kcapi.o obj-$(CONFIG_CRYPTO_TEST) += tcrypt.o obj-$(CONFIG_CRYPTO_GHASH) += ghash-generic.o +obj-$(CONFIG_CRYPTO_POLYVAL) += polyval-generic.o obj-$(CONFIG_CRYPTO_USER_API) += af_alg.o obj-$(CONFIG_CRYPTO_USER_API_HASH) += algif_hash.o obj-$(CONFIG_CRYPTO_USER_API_SKCIPHER) += algif_skcipher.o diff --git a/crypto/polyval-generic.c b/crypto/polyval-generic.c new file mode 100644 index 000000000000..bf2b03b7bfc0 --- /dev/null +++ b/crypto/polyval-generic.c @@ -0,0 +1,205 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * POLYVAL: hash function for HCTR2. + * + * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen + * Copyright (c) 2009 Intel Corp. + * Author: Huang Ying + * Copyright 2021 Google LLC + */ + +/* + * Code based on crypto/ghash-generic.c + * + * POLYVAL is a keyed hash function similar to GHASH. POLYVAL uses a different + * modulus for finite field multiplication which makes hardware accelerated + * implementations on little-endian machines faster. POLYVAL is used in the + * kernel to implement HCTR2, but was originally specified for AES-GCM-SIV + * (RFC 8452). + * + * For more information see: + * Length-preserving encryption with HCTR2: + * https://eprint.iacr.org/2021/1441.pdf + * AES-GCM-SIV: Nonce Misuse-Resistant Authenticated Encryption: + * https://datatracker.ietf.org/doc/html/rfc8452 + * + * Like GHASH, POLYVAL is not a cryptographic hash function and should + * not be used outside of crypto modes explicitly designed to use POLYVAL. + * + * This implementation uses a convenient trick involving the GHASH and POLYVAL + * fields. This trick allows multiplication in the POLYVAL field to be + * implemented by using multiplication in the GHASH field as a subroutine. An + * element of the POLYVAL field can be converted to an element of the GHASH + * field by computing x*REVERSE(a), where REVERSE reverses the byte-ordering of + * a. Similarly, an element of the GHASH field can be converted back to the + * POLYVAL field by computing REVERSE(x^{-1}*a). For more information, see: + * https://datatracker.ietf.org/doc/html/rfc8452#appendix-A + * + * By using this trick, we do not need to implement the POLYVAL field for the + * generic implementation. + * + * Warning: this generic implementation is not intended to be used in practice + * and is not constant time. For practical use, a hardware accelerated + * implementation of POLYVAL should be used instead. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct polyval_tfm_ctx { + struct gf128mul_4k *gf128; +}; + +struct polyval_desc_ctx { + union { + u8 buffer[POLYVAL_BLOCK_SIZE]; + be128 buffer128; + }; + u32 bytes; +}; + +static void copy_and_reverse(u8 dst[POLYVAL_BLOCK_SIZE], + const u8 src[POLYVAL_BLOCK_SIZE]) +{ + u64 a = get_unaligned((const u64 *)&src[0]); + u64 b = get_unaligned((const u64 *)&src[8]); + + put_unaligned(swab64(a), (u64 *)&dst[8]); + put_unaligned(swab64(b), (u64 *)&dst[0]); +} + +static int polyval_setkey(struct crypto_shash *tfm, + const u8 *key, unsigned int keylen) +{ + struct polyval_tfm_ctx *ctx = crypto_shash_ctx(tfm); + be128 k; + + if (keylen != POLYVAL_BLOCK_SIZE) + return -EINVAL; + + gf128mul_free_4k(ctx->gf128); + + BUILD_BUG_ON(sizeof(k) != POLYVAL_BLOCK_SIZE); + copy_and_reverse((u8 *)&k, key); + gf128mul_x_lle(&k, &k); + + ctx->gf128 = gf128mul_init_4k_lle(&k); + memzero_explicit(&k, POLYVAL_BLOCK_SIZE); + + if (!ctx->gf128) + return -ENOMEM; + + return 0; +} + +static int polyval_init(struct shash_desc *desc) +{ + struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); + + memset(dctx, 0, sizeof(*dctx)); + + return 0; +} + +static int polyval_update(struct shash_desc *desc, + const u8 *src, unsigned int srclen) +{ + struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); + const struct polyval_tfm_ctx *ctx = crypto_shash_ctx(desc->tfm); + u8 *pos; + u8 tmp[POLYVAL_BLOCK_SIZE]; + int n; + + if (dctx->bytes) { + n = min(srclen, dctx->bytes); + pos = dctx->buffer + dctx->bytes - 1; + + dctx->bytes -= n; + srclen -= n; + + while (n--) + *pos-- ^= *src++; + + if (!dctx->bytes) + gf128mul_4k_lle(&dctx->buffer128, ctx->gf128); + } + + while (srclen >= POLYVAL_BLOCK_SIZE) { + copy_and_reverse(tmp, src); + crypto_xor(dctx->buffer, tmp, POLYVAL_BLOCK_SIZE); + gf128mul_4k_lle(&dctx->buffer128, ctx->gf128); + src += POLYVAL_BLOCK_SIZE; + srclen -= POLYVAL_BLOCK_SIZE; + } + + if (srclen) { + dctx->bytes = POLYVAL_BLOCK_SIZE - srclen; + pos = dctx->buffer + POLYVAL_BLOCK_SIZE - 1; + while (srclen--) + *pos-- ^= *src++; + } + + return 0; +} + +static int polyval_final(struct shash_desc *desc, u8 *dst) +{ + struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); + const struct polyval_tfm_ctx *ctx = crypto_shash_ctx(desc->tfm); + + if (dctx->bytes) + gf128mul_4k_lle(&dctx->buffer128, ctx->gf128); + copy_and_reverse(dst, dctx->buffer); + return 0; +} + +static void polyval_exit_tfm(struct crypto_tfm *tfm) +{ + struct polyval_tfm_ctx *ctx = crypto_tfm_ctx(tfm); + + gf128mul_free_4k(ctx->gf128); +} + +static struct shash_alg polyval_alg = { + .digestsize = POLYVAL_DIGEST_SIZE, + .init = polyval_init, + .update = polyval_update, + .final = polyval_final, + .setkey = polyval_setkey, + .descsize = sizeof(struct polyval_desc_ctx), + .base = { + .cra_name = "polyval", + .cra_driver_name = "polyval-generic", + .cra_priority = 100, + .cra_blocksize = POLYVAL_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct polyval_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_exit = polyval_exit_tfm, + }, +}; + +static int __init polyval_mod_init(void) +{ + return crypto_register_shash(&polyval_alg); +} + +static void __exit polyval_mod_exit(void) +{ + crypto_unregister_shash(&polyval_alg); +} + +subsys_initcall(polyval_mod_init); +module_exit(polyval_mod_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("POLYVAL hash function"); +MODULE_ALIAS_CRYPTO("polyval"); +MODULE_ALIAS_CRYPTO("polyval-generic"); diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index fd671d0e2012..dd9cf216029b 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -1730,6 +1730,10 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) ret += tcrypt_test("ccm(sm4)"); break; + case 57: + ret += tcrypt_test("polyval"); + break; + case 100: ret += tcrypt_test("hmac(md5)"); break; diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 0ea77dcdc6c0..0f40e260b5a9 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -5342,6 +5342,12 @@ static const struct alg_test_desc alg_test_descs[] = { .suite = { .hash = __VECS(poly1305_tv_template) } + }, { + .alg = "polyval", + .test = alg_test_hash, + .suite = { + .hash = __VECS(polyval_tv_template) + } }, { .alg = "rfc3686(ctr(aes))", .test = alg_test_skcipher, diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 7179df0a39b6..b58e13930307 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -34944,4 +34944,175 @@ static const struct cipher_testvec aes_xctr_tv_template[] = { }; +/* + * Test vectors generated using https://github.com/google/hctr2 + * + * To ensure compatibility with RFC 8452, some tests were sourced from + * https://datatracker.ietf.org/doc/html/rfc8452 + */ +static const struct hash_testvec polyval_tv_template[] = { + { // From RFC 8452 + .key = "\x31\x07\x28\xd9\x91\x1f\x1f\x38" + "\x37\xb2\x43\x16\xc3\xfa\xb9\xa0", + .plaintext = "\x65\x78\x61\x6d\x70\x6c\x65\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x48\x65\x6c\x6c\x6f\x20\x77\x6f" + "\x72\x6c\x64\x00\x00\x00\x00\x00" + "\x38\x00\x00\x00\x00\x00\x00\x00" + "\x58\x00\x00\x00\x00\x00\x00\x00", + .digest = "\xad\x7f\xcf\x0b\x51\x69\x85\x16" + "\x62\x67\x2f\x3c\x5f\x95\x13\x8f", + .psize = 48, + .ksize = 16, + }, + { // From RFC 8452 + .key = "\xd9\xb3\x60\x27\x96\x94\x94\x1a" + "\xc5\xdb\xc6\x98\x7a\xda\x73\x77", + .plaintext = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .digest = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .psize = 16, + .ksize = 16, + }, + { // From RFC 8452 + .key = "\xd9\xb3\x60\x27\x96\x94\x94\x1a" + "\xc5\xdb\xc6\x98\x7a\xda\x73\x77", + .plaintext = "\x01\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x40\x00\x00\x00\x00\x00\x00\x00", + .digest = "\xeb\x93\xb7\x74\x09\x62\xc5\xe4" + "\x9d\x2a\x90\xa7\xdc\x5c\xec\x74", + .psize = 32, + .ksize = 16, + }, + { // From RFC 8452 + .key = "\xd9\xb3\x60\x27\x96\x94\x94\x1a" + "\xc5\xdb\xc6\x98\x7a\xda\x73\x77", + .plaintext = "\x01\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x02\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x03\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x80\x01\x00\x00\x00\x00\x00\x00", + .digest = "\x81\x38\x87\x46\xbc\x22\xd2\x6b" + "\x2a\xbc\x3d\xcb\x15\x75\x42\x22", + .psize = 64, + .ksize = 16, + }, + { // From RFC 8452 + .key = "\xd9\xb3\x60\x27\x96\x94\x94\x1a" + "\xc5\xdb\xc6\x98\x7a\xda\x73\x77", + .plaintext = "\x01\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x02\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x03\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x04\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x02\x00\x00\x00\x00\x00\x00", + .digest = "\x1e\x39\xb6\xd3\x34\x4d\x34\x8f" + "\x60\x44\xf8\x99\x35\xd1\xcf\x78", + .psize = 80, + .ksize = 16, + }, + { // From RFC 8452 + .key = "\xd9\xb3\x60\x27\x96\x94\x94\x1a" + "\xc5\xdb\xc6\x98\x7a\xda\x73\x77", + .plaintext = "\x01\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x02\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x03\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x04\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x05\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x08\x00\x00\x00\x00\x00\x00\x00" + "\x00\x02\x00\x00\x00\x00\x00\x00", + .digest = "\xff\xcd\x05\xd5\x77\x0f\x34\xad" + "\x92\x67\xf0\xa5\x99\x94\xb1\x5a", + .psize = 96, + .ksize = 16, + }, + { // Random ( 1) + .key = "\x90\xcc\xac\xee\xba\xd7\xd4\x68" + "\x98\xa6\x79\x70\xdf\x66\x15\x6c", + .plaintext = "", + .digest = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .psize = 0, + .ksize = 16, + }, + { // Random ( 1) + .key = "\xc1\x45\x71\xf0\x30\x07\x94\xe7" + "\x3a\xdd\xe4\xc6\x19\x2d\x02\xa2", + .plaintext = "\xc1\x5d\x47\xc7\x4c\x7c\x5e\x07" + "\x85\x14\x8f\x79\xcc\x73\x83\xf7" + "\x35\xb8\xcb\x73\x61\xf0\x53\x31" + "\xbf\x84\xde\xb6\xde\xaf\xb0\xb8" + "\xb7\xd9\x11\x91\x89\xfd\x1e\x4c" + "\x84\x4a\x1f\x2a\x87\xa4\xaf\x62" + "\x8d\x7d\x58\xf6\x43\x35\xfc\x53" + "\x8f\x1a\xf6\x12\xe1\x13\x3f\x66" + "\x91\x4b\x13\xd6\x45\xfb\xb0\x7a" + "\xe0\x8b\x8e\x99\xf7\x86\x46\x37" + "\xd1\x22\x9e\x52\xf3\x3f\xd9\x75" + "\x2c\x2c\xc6\xbb\x0e\x08\x14\x29" + "\xe8\x50\x2f\xd8\xbe\xf4\xe9\x69" + "\x4a\xee\xf7\xae\x15\x65\x35\x1e", + .digest = "\x00\x4f\x5d\xe9\x3b\xc0\xd6\x50" + "\x3e\x38\x73\x86\xc6\xda\xca\x7f", + .psize = 112, + .ksize = 16, + }, + { // Random ( 1) + .key = "\x37\xbe\x68\x16\x50\xb9\x4e\xb0" + "\x47\xde\xe2\xbd\xde\xe4\x48\x09", + .plaintext = "\x87\xfc\x68\x9f\xff\xf2\x4a\x1e" + "\x82\x3b\x73\x8f\xc1\xb2\x1b\x7a" + "\x6c\x4f\x81\xbc\x88\x9b\x6c\xa3" + "\x9c\xc2\xa5\xbc\x14\x70\x4c\x9b" + "\x0c\x9f\x59\x92\x16\x4b\x91\x3d" + "\x18\x55\x22\x68\x12\x8c\x63\xb2" + "\x51\xcb\x85\x4b\xd2\xae\x0b\x1c" + "\x5d\x28\x9d\x1d\xb1\xc8\xf0\x77" + "\xe9\xb5\x07\x4e\x06\xc8\xee\xf8" + "\x1b\xed\x72\x2a\x55\x7d\x16\xc9" + "\xf2\x54\xe7\xe9\xe0\x44\x5b\x33" + "\xb1\x49\xee\xff\x43\xfb\x82\xcd" + "\x4a\x70\x78\x81\xa4\x34\x36\xe8" + "\x4c\x28\x54\xa6\x6c\xc3\x6b\x78" + "\xe7\xc0\x5d\xc6\x5d\x81\xab\x70" + "\x08\x86\xa1\xfd\xf4\x77\x55\xfd" + "\xa3\xe9\xe2\x1b\xdf\x99\xb7\x80" + "\xf9\x0a\x4f\x72\x4a\xd3\xaf\xbb" + "\xb3\x3b\xeb\x08\x58\x0f\x79\xce" + "\xa5\x99\x05\x12\x34\xd4\xf4\x86" + "\x37\x23\x1d\xc8\x49\xc0\x92\xae" + "\xa6\xac\x9b\x31\x55\xed\x15\xc6" + "\x05\x17\x37\x8d\x90\x42\xe4\x87" + "\x89\x62\x88\x69\x1c\x6a\xfd\xe3" + "\x00\x2b\x47\x1a\x73\xc1\x51\xc2" + "\xc0\x62\x74\x6a\x9e\xb2\xe5\x21" + "\xbe\x90\xb5\xb0\x50\xca\x88\x68" + "\xe1\x9d\x7a\xdf\x6c\xb7\xb9\x98" + "\xee\x28\x62\x61\x8b\xd1\x47\xf9" + "\x04\x7a\x0b\x5d\xcd\x2b\x65\xf5" + "\x12\xa3\xfe\x1a\xaa\x2c\x78\x42" + "\xb8\xbe\x7d\x74\xeb\x59\xba\xba", + .digest = "\xae\x11\xd4\x60\x2a\x5f\x9e\x42" + "\x89\x04\xc2\x34\x8d\x55\x94\x0a", + .psize = 256, + .ksize = 16, + }, + +}; + #endif /* _CRYPTO_TESTMGR_H */ diff --git a/include/crypto/polyval.h b/include/crypto/polyval.h new file mode 100644 index 000000000000..b14c38aa9166 --- /dev/null +++ b/include/crypto/polyval.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Common values for the Polyval hash algorithm + * + * Copyright 2021 Google LLC + */ + +#ifndef _CRYPTO_POLYVAL_H +#define _CRYPTO_POLYVAL_H + +#include +#include + +#define POLYVAL_BLOCK_SIZE 16 +#define POLYVAL_DIGEST_SIZE 16 + +#endif From 7ff554ced7c7d7cf77586e07474e8633e011e2d0 Mon Sep 17 00:00:00 2001 From: Nathan Huckleberry Date: Fri, 20 May 2022 18:14:55 +0000 Subject: [PATCH 010/318] crypto: hctr2 - Add HCTR2 support Add support for HCTR2 as a template. HCTR2 is a length-preserving encryption mode that is efficient on processors with instructions to accelerate AES and carryless multiplication, e.g. x86 processors with AES-NI and CLMUL, and ARM processors with the ARMv8 Crypto Extensions. As a length-preserving encryption mode, HCTR2 is suitable for applications such as storage encryption where ciphertext expansion is not possible, and thus authenticated encryption cannot be used. Currently, such applications usually use XTS, or in some cases Adiantum. XTS has the disadvantage that it is a narrow-block mode: a bitflip will only change 16 bytes in the resulting ciphertext or plaintext. This reveals more information to an attacker than necessary. HCTR2 is a wide-block mode, so it provides a stronger security property: a bitflip will change the entire message. HCTR2 is somewhat similar to Adiantum, which is also a wide-block mode. However, HCTR2 is designed to take advantage of existing crypto instructions, while Adiantum targets devices without such hardware support. Adiantum is also designed with longer messages in mind, while HCTR2 is designed to be efficient even on short messages. HCTR2 requires POLYVAL and XCTR as components. More information on HCTR2 can be found here: "Length-preserving encryption with HCTR2": https://eprint.iacr.org/2021/1441.pdf Signed-off-by: Nathan Huckleberry Reviewed-by: Ard Biesheuvel Reviewed-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/Kconfig | 11 + crypto/Makefile | 1 + crypto/hctr2.c | 581 ++++++++++++++++++++++++++++++++++++++++ crypto/tcrypt.c | 5 + crypto/testmgr.c | 8 + crypto/testmgr.h | 672 +++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 1278 insertions(+) create mode 100644 crypto/hctr2.c diff --git a/crypto/Kconfig b/crypto/Kconfig index d59e70dca197..0601a2d2feef 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -532,6 +532,17 @@ config CRYPTO_ADIANTUM If unsure, say N. +config CRYPTO_HCTR2 + tristate "HCTR2 support" + select CRYPTO_XCTR + select CRYPTO_POLYVAL + select CRYPTO_MANAGER + help + HCTR2 is a length-preserving encryption mode for storage encryption that + is efficient on processors with instructions to accelerate AES and + carryless multiplication, e.g. x86 processors with AES-NI and CLMUL, and + ARM processors with the ARMv8 crypto extensions. + config CRYPTO_ESSIV tristate "ESSIV support for block encryption" select CRYPTO_AUTHENC diff --git a/crypto/Makefile b/crypto/Makefile index 7694ed0a44d5..3bbc0dd49160 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -95,6 +95,7 @@ obj-$(CONFIG_CRYPTO_LRW) += lrw.o obj-$(CONFIG_CRYPTO_XTS) += xts.o obj-$(CONFIG_CRYPTO_CTR) += ctr.o obj-$(CONFIG_CRYPTO_XCTR) += xctr.o +obj-$(CONFIG_CRYPTO_HCTR2) += hctr2.o obj-$(CONFIG_CRYPTO_KEYWRAP) += keywrap.o obj-$(CONFIG_CRYPTO_ADIANTUM) += adiantum.o obj-$(CONFIG_CRYPTO_NHPOLY1305) += nhpoly1305.o diff --git a/crypto/hctr2.c b/crypto/hctr2.c new file mode 100644 index 000000000000..7d00a3bcb667 --- /dev/null +++ b/crypto/hctr2.c @@ -0,0 +1,581 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * HCTR2 length-preserving encryption mode + * + * Copyright 2021 Google LLC + */ + + +/* + * HCTR2 is a length-preserving encryption mode that is efficient on + * processors with instructions to accelerate AES and carryless + * multiplication, e.g. x86 processors with AES-NI and CLMUL, and ARM + * processors with the ARMv8 crypto extensions. + * + * For more details, see the paper: "Length-preserving encryption with HCTR2" + * (https://eprint.iacr.org/2021/1441.pdf) + */ + +#include +#include +#include +#include +#include +#include + +#define BLOCKCIPHER_BLOCK_SIZE 16 + +/* + * The specification allows variable-length tweaks, but Linux's crypto API + * currently only allows algorithms to support a single length. The "natural" + * tweak length for HCTR2 is 16, since that fits into one POLYVAL block for + * the best performance. But longer tweaks are useful for fscrypt, to avoid + * needing to derive per-file keys. So instead we use two blocks, or 32 bytes. + */ +#define TWEAK_SIZE 32 + +struct hctr2_instance_ctx { + struct crypto_cipher_spawn blockcipher_spawn; + struct crypto_skcipher_spawn xctr_spawn; + struct crypto_shash_spawn polyval_spawn; +}; + +struct hctr2_tfm_ctx { + struct crypto_cipher *blockcipher; + struct crypto_skcipher *xctr; + struct crypto_shash *polyval; + u8 L[BLOCKCIPHER_BLOCK_SIZE]; + int hashed_tweak_offset; + /* + * This struct is allocated with extra space for two exported hash + * states. Since the hash state size is not known at compile-time, we + * can't add these to the struct directly. + * + * hashed_tweaklen_divisible; + * hashed_tweaklen_remainder; + */ +}; + +struct hctr2_request_ctx { + u8 first_block[BLOCKCIPHER_BLOCK_SIZE]; + u8 xctr_iv[BLOCKCIPHER_BLOCK_SIZE]; + struct scatterlist *bulk_part_dst; + struct scatterlist *bulk_part_src; + struct scatterlist sg_src[2]; + struct scatterlist sg_dst[2]; + /* + * Sub-request sizes are unknown at compile-time, so they need to go + * after the members with known sizes. + */ + union { + struct shash_desc hash_desc; + struct skcipher_request xctr_req; + } u; + /* + * This struct is allocated with extra space for one exported hash + * state. Since the hash state size is not known at compile-time, we + * can't add it to the struct directly. + * + * hashed_tweak; + */ +}; + +static inline u8 *hctr2_hashed_tweaklen(const struct hctr2_tfm_ctx *tctx, + bool has_remainder) +{ + u8 *p = (u8 *)tctx + sizeof(*tctx); + + if (has_remainder) /* For messages not a multiple of block length */ + p += crypto_shash_statesize(tctx->polyval); + return p; +} + +static inline u8 *hctr2_hashed_tweak(const struct hctr2_tfm_ctx *tctx, + struct hctr2_request_ctx *rctx) +{ + return (u8 *)rctx + tctx->hashed_tweak_offset; +} + +/* + * The input data for each HCTR2 hash step begins with a 16-byte block that + * contains the tweak length and a flag that indicates whether the input is evenly + * divisible into blocks. Since this implementation only supports one tweak + * length, we precompute the two hash states resulting from hashing the two + * possible values of this initial block. This reduces by one block the amount of + * data that needs to be hashed for each encryption/decryption + * + * These precomputed hashes are stored in hctr2_tfm_ctx. + */ +static int hctr2_hash_tweaklen(struct hctr2_tfm_ctx *tctx, bool has_remainder) +{ + SHASH_DESC_ON_STACK(shash, tfm->polyval); + __le64 tweak_length_block[2]; + int err; + + shash->tfm = tctx->polyval; + memset(tweak_length_block, 0, sizeof(tweak_length_block)); + + tweak_length_block[0] = cpu_to_le64(TWEAK_SIZE * 8 * 2 + 2 + has_remainder); + err = crypto_shash_init(shash); + if (err) + return err; + err = crypto_shash_update(shash, (u8 *)tweak_length_block, + POLYVAL_BLOCK_SIZE); + if (err) + return err; + return crypto_shash_export(shash, hctr2_hashed_tweaklen(tctx, has_remainder)); +} + +static int hctr2_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct hctr2_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + u8 hbar[BLOCKCIPHER_BLOCK_SIZE]; + int err; + + crypto_cipher_clear_flags(tctx->blockcipher, CRYPTO_TFM_REQ_MASK); + crypto_cipher_set_flags(tctx->blockcipher, + crypto_skcipher_get_flags(tfm) & + CRYPTO_TFM_REQ_MASK); + err = crypto_cipher_setkey(tctx->blockcipher, key, keylen); + if (err) + return err; + + crypto_skcipher_clear_flags(tctx->xctr, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(tctx->xctr, + crypto_skcipher_get_flags(tfm) & + CRYPTO_TFM_REQ_MASK); + err = crypto_skcipher_setkey(tctx->xctr, key, keylen); + if (err) + return err; + + memset(hbar, 0, sizeof(hbar)); + crypto_cipher_encrypt_one(tctx->blockcipher, hbar, hbar); + + memset(tctx->L, 0, sizeof(tctx->L)); + tctx->L[0] = 0x01; + crypto_cipher_encrypt_one(tctx->blockcipher, tctx->L, tctx->L); + + crypto_shash_clear_flags(tctx->polyval, CRYPTO_TFM_REQ_MASK); + crypto_shash_set_flags(tctx->polyval, crypto_skcipher_get_flags(tfm) & + CRYPTO_TFM_REQ_MASK); + err = crypto_shash_setkey(tctx->polyval, hbar, BLOCKCIPHER_BLOCK_SIZE); + if (err) + return err; + memzero_explicit(hbar, sizeof(hbar)); + + return hctr2_hash_tweaklen(tctx, true) ?: hctr2_hash_tweaklen(tctx, false); +} + +static int hctr2_hash_tweak(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct hctr2_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + struct hctr2_request_ctx *rctx = skcipher_request_ctx(req); + struct shash_desc *hash_desc = &rctx->u.hash_desc; + int err; + bool has_remainder = req->cryptlen % POLYVAL_BLOCK_SIZE; + + hash_desc->tfm = tctx->polyval; + err = crypto_shash_import(hash_desc, hctr2_hashed_tweaklen(tctx, has_remainder)); + if (err) + return err; + err = crypto_shash_update(hash_desc, req->iv, TWEAK_SIZE); + if (err) + return err; + + // Store the hashed tweak, since we need it when computing both + // H(T || N) and H(T || V). + return crypto_shash_export(hash_desc, hctr2_hashed_tweak(tctx, rctx)); +} + +static int hctr2_hash_message(struct skcipher_request *req, + struct scatterlist *sgl, + u8 digest[POLYVAL_DIGEST_SIZE]) +{ + static const u8 padding[BLOCKCIPHER_BLOCK_SIZE] = { 0x1 }; + struct hctr2_request_ctx *rctx = skcipher_request_ctx(req); + struct shash_desc *hash_desc = &rctx->u.hash_desc; + const unsigned int bulk_len = req->cryptlen - BLOCKCIPHER_BLOCK_SIZE; + struct sg_mapping_iter miter; + unsigned int remainder = bulk_len % BLOCKCIPHER_BLOCK_SIZE; + int i; + int err = 0; + int n = 0; + + sg_miter_start(&miter, sgl, sg_nents(sgl), + SG_MITER_FROM_SG | SG_MITER_ATOMIC); + for (i = 0; i < bulk_len; i += n) { + sg_miter_next(&miter); + n = min_t(unsigned int, miter.length, bulk_len - i); + err = crypto_shash_update(hash_desc, miter.addr, n); + if (err) + break; + } + sg_miter_stop(&miter); + + if (err) + return err; + + if (remainder) { + err = crypto_shash_update(hash_desc, padding, + BLOCKCIPHER_BLOCK_SIZE - remainder); + if (err) + return err; + } + return crypto_shash_final(hash_desc, digest); +} + +static int hctr2_finish(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct hctr2_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + struct hctr2_request_ctx *rctx = skcipher_request_ctx(req); + u8 digest[POLYVAL_DIGEST_SIZE]; + struct shash_desc *hash_desc = &rctx->u.hash_desc; + int err; + + // U = UU ^ H(T || V) + // or M = MM ^ H(T || N) + hash_desc->tfm = tctx->polyval; + err = crypto_shash_import(hash_desc, hctr2_hashed_tweak(tctx, rctx)); + if (err) + return err; + err = hctr2_hash_message(req, rctx->bulk_part_dst, digest); + if (err) + return err; + crypto_xor(rctx->first_block, digest, BLOCKCIPHER_BLOCK_SIZE); + + // Copy U (or M) into dst scatterlist + scatterwalk_map_and_copy(rctx->first_block, req->dst, + 0, BLOCKCIPHER_BLOCK_SIZE, 1); + return 0; +} + +static void hctr2_xctr_done(struct crypto_async_request *areq, + int err) +{ + struct skcipher_request *req = areq->data; + + if (!err) + err = hctr2_finish(req); + + skcipher_request_complete(req, err); +} + +static int hctr2_crypt(struct skcipher_request *req, bool enc) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct hctr2_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + struct hctr2_request_ctx *rctx = skcipher_request_ctx(req); + u8 digest[POLYVAL_DIGEST_SIZE]; + int bulk_len = req->cryptlen - BLOCKCIPHER_BLOCK_SIZE; + int err; + + // Requests must be at least one block + if (req->cryptlen < BLOCKCIPHER_BLOCK_SIZE) + return -EINVAL; + + // Copy M (or U) into a temporary buffer + scatterwalk_map_and_copy(rctx->first_block, req->src, + 0, BLOCKCIPHER_BLOCK_SIZE, 0); + + // Create scatterlists for N and V + rctx->bulk_part_src = scatterwalk_ffwd(rctx->sg_src, req->src, + BLOCKCIPHER_BLOCK_SIZE); + rctx->bulk_part_dst = scatterwalk_ffwd(rctx->sg_dst, req->dst, + BLOCKCIPHER_BLOCK_SIZE); + + // MM = M ^ H(T || N) + // or UU = U ^ H(T || V) + err = hctr2_hash_tweak(req); + if (err) + return err; + err = hctr2_hash_message(req, rctx->bulk_part_src, digest); + if (err) + return err; + crypto_xor(digest, rctx->first_block, BLOCKCIPHER_BLOCK_SIZE); + + // UU = E(MM) + // or MM = D(UU) + if (enc) + crypto_cipher_encrypt_one(tctx->blockcipher, rctx->first_block, + digest); + else + crypto_cipher_decrypt_one(tctx->blockcipher, rctx->first_block, + digest); + + // S = MM ^ UU ^ L + crypto_xor(digest, rctx->first_block, BLOCKCIPHER_BLOCK_SIZE); + crypto_xor_cpy(rctx->xctr_iv, digest, tctx->L, BLOCKCIPHER_BLOCK_SIZE); + + // V = XCTR(S, N) + // or N = XCTR(S, V) + skcipher_request_set_tfm(&rctx->u.xctr_req, tctx->xctr); + skcipher_request_set_crypt(&rctx->u.xctr_req, rctx->bulk_part_src, + rctx->bulk_part_dst, bulk_len, + rctx->xctr_iv); + skcipher_request_set_callback(&rctx->u.xctr_req, + req->base.flags, + hctr2_xctr_done, req); + return crypto_skcipher_encrypt(&rctx->u.xctr_req) ?: + hctr2_finish(req); +} + +static int hctr2_encrypt(struct skcipher_request *req) +{ + return hctr2_crypt(req, true); +} + +static int hctr2_decrypt(struct skcipher_request *req) +{ + return hctr2_crypt(req, false); +} + +static int hctr2_init_tfm(struct crypto_skcipher *tfm) +{ + struct skcipher_instance *inst = skcipher_alg_instance(tfm); + struct hctr2_instance_ctx *ictx = skcipher_instance_ctx(inst); + struct hctr2_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + struct crypto_skcipher *xctr; + struct crypto_cipher *blockcipher; + struct crypto_shash *polyval; + unsigned int subreq_size; + int err; + + xctr = crypto_spawn_skcipher(&ictx->xctr_spawn); + if (IS_ERR(xctr)) + return PTR_ERR(xctr); + + blockcipher = crypto_spawn_cipher(&ictx->blockcipher_spawn); + if (IS_ERR(blockcipher)) { + err = PTR_ERR(blockcipher); + goto err_free_xctr; + } + + polyval = crypto_spawn_shash(&ictx->polyval_spawn); + if (IS_ERR(polyval)) { + err = PTR_ERR(polyval); + goto err_free_blockcipher; + } + + tctx->xctr = xctr; + tctx->blockcipher = blockcipher; + tctx->polyval = polyval; + + BUILD_BUG_ON(offsetofend(struct hctr2_request_ctx, u) != + sizeof(struct hctr2_request_ctx)); + subreq_size = max(sizeof_field(struct hctr2_request_ctx, u.hash_desc) + + crypto_shash_descsize(polyval), + sizeof_field(struct hctr2_request_ctx, u.xctr_req) + + crypto_skcipher_reqsize(xctr)); + + tctx->hashed_tweak_offset = offsetof(struct hctr2_request_ctx, u) + + subreq_size; + crypto_skcipher_set_reqsize(tfm, tctx->hashed_tweak_offset + + crypto_shash_statesize(polyval)); + return 0; + +err_free_blockcipher: + crypto_free_cipher(blockcipher); +err_free_xctr: + crypto_free_skcipher(xctr); + return err; +} + +static void hctr2_exit_tfm(struct crypto_skcipher *tfm) +{ + struct hctr2_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + + crypto_free_cipher(tctx->blockcipher); + crypto_free_skcipher(tctx->xctr); + crypto_free_shash(tctx->polyval); +} + +static void hctr2_free_instance(struct skcipher_instance *inst) +{ + struct hctr2_instance_ctx *ictx = skcipher_instance_ctx(inst); + + crypto_drop_cipher(&ictx->blockcipher_spawn); + crypto_drop_skcipher(&ictx->xctr_spawn); + crypto_drop_shash(&ictx->polyval_spawn); + kfree(inst); +} + +static int hctr2_create_common(struct crypto_template *tmpl, + struct rtattr **tb, + const char *xctr_name, + const char *polyval_name) +{ + u32 mask; + struct skcipher_instance *inst; + struct hctr2_instance_ctx *ictx; + struct skcipher_alg *xctr_alg; + struct crypto_alg *blockcipher_alg; + struct shash_alg *polyval_alg; + char blockcipher_name[CRYPTO_MAX_ALG_NAME]; + int len; + int err; + + err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SKCIPHER, &mask); + if (err) + return err; + + inst = kzalloc(sizeof(*inst) + sizeof(*ictx), GFP_KERNEL); + if (!inst) + return -ENOMEM; + ictx = skcipher_instance_ctx(inst); + + /* Stream cipher, xctr(block_cipher) */ + err = crypto_grab_skcipher(&ictx->xctr_spawn, + skcipher_crypto_instance(inst), + xctr_name, 0, mask); + if (err) + goto err_free_inst; + xctr_alg = crypto_spawn_skcipher_alg(&ictx->xctr_spawn); + + err = -EINVAL; + if (strncmp(xctr_alg->base.cra_name, "xctr(", 5)) + goto err_free_inst; + len = strscpy(blockcipher_name, xctr_alg->base.cra_name + 5, + sizeof(blockcipher_name)); + if (len < 1) + goto err_free_inst; + if (blockcipher_name[len - 1] != ')') + goto err_free_inst; + blockcipher_name[len - 1] = 0; + + /* Block cipher, e.g. "aes" */ + err = crypto_grab_cipher(&ictx->blockcipher_spawn, + skcipher_crypto_instance(inst), + blockcipher_name, 0, mask); + if (err) + goto err_free_inst; + blockcipher_alg = crypto_spawn_cipher_alg(&ictx->blockcipher_spawn); + + /* Require blocksize of 16 bytes */ + err = -EINVAL; + if (blockcipher_alg->cra_blocksize != BLOCKCIPHER_BLOCK_SIZE) + goto err_free_inst; + + /* Polyval ε-∆U hash function */ + err = crypto_grab_shash(&ictx->polyval_spawn, + skcipher_crypto_instance(inst), + polyval_name, 0, mask); + if (err) + goto err_free_inst; + polyval_alg = crypto_spawn_shash_alg(&ictx->polyval_spawn); + + /* Ensure Polyval is being used */ + err = -EINVAL; + if (strcmp(polyval_alg->base.cra_name, "polyval") != 0) + goto err_free_inst; + + /* Instance fields */ + + err = -ENAMETOOLONG; + if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, "hctr2(%s)", + blockcipher_alg->cra_name) >= CRYPTO_MAX_ALG_NAME) + goto err_free_inst; + if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, + "hctr2_base(%s,%s)", + xctr_alg->base.cra_driver_name, + polyval_alg->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME) + goto err_free_inst; + + inst->alg.base.cra_blocksize = BLOCKCIPHER_BLOCK_SIZE; + inst->alg.base.cra_ctxsize = sizeof(struct hctr2_tfm_ctx) + + polyval_alg->statesize * 2; + inst->alg.base.cra_alignmask = xctr_alg->base.cra_alignmask | + polyval_alg->base.cra_alignmask; + /* + * The hash function is called twice, so it is weighted higher than the + * xctr and blockcipher. + */ + inst->alg.base.cra_priority = (2 * xctr_alg->base.cra_priority + + 4 * polyval_alg->base.cra_priority + + blockcipher_alg->cra_priority) / 7; + + inst->alg.setkey = hctr2_setkey; + inst->alg.encrypt = hctr2_encrypt; + inst->alg.decrypt = hctr2_decrypt; + inst->alg.init = hctr2_init_tfm; + inst->alg.exit = hctr2_exit_tfm; + inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(xctr_alg); + inst->alg.max_keysize = crypto_skcipher_alg_max_keysize(xctr_alg); + inst->alg.ivsize = TWEAK_SIZE; + + inst->free = hctr2_free_instance; + + err = skcipher_register_instance(tmpl, inst); + if (err) { +err_free_inst: + hctr2_free_instance(inst); + } + return err; +} + +static int hctr2_create_base(struct crypto_template *tmpl, struct rtattr **tb) +{ + const char *xctr_name; + const char *polyval_name; + + xctr_name = crypto_attr_alg_name(tb[1]); + if (IS_ERR(xctr_name)) + return PTR_ERR(xctr_name); + + polyval_name = crypto_attr_alg_name(tb[2]); + if (IS_ERR(polyval_name)) + return PTR_ERR(polyval_name); + + return hctr2_create_common(tmpl, tb, xctr_name, polyval_name); +} + +static int hctr2_create(struct crypto_template *tmpl, struct rtattr **tb) +{ + const char *blockcipher_name; + char xctr_name[CRYPTO_MAX_ALG_NAME]; + + blockcipher_name = crypto_attr_alg_name(tb[1]); + if (IS_ERR(blockcipher_name)) + return PTR_ERR(blockcipher_name); + + if (snprintf(xctr_name, CRYPTO_MAX_ALG_NAME, "xctr(%s)", + blockcipher_name) >= CRYPTO_MAX_ALG_NAME) + return -ENAMETOOLONG; + + return hctr2_create_common(tmpl, tb, xctr_name, "polyval"); +} + +static struct crypto_template hctr2_tmpls[] = { + { + /* hctr2_base(xctr_name, polyval_name) */ + .name = "hctr2_base", + .create = hctr2_create_base, + .module = THIS_MODULE, + }, { + /* hctr2(blockcipher_name) */ + .name = "hctr2", + .create = hctr2_create, + .module = THIS_MODULE, + } +}; + +static int __init hctr2_module_init(void) +{ + return crypto_register_templates(hctr2_tmpls, ARRAY_SIZE(hctr2_tmpls)); +} + +static void __exit hctr2_module_exit(void) +{ + return crypto_unregister_templates(hctr2_tmpls, + ARRAY_SIZE(hctr2_tmpls)); +} + +subsys_initcall(hctr2_module_init); +module_exit(hctr2_module_exit); + +MODULE_DESCRIPTION("HCTR2 length-preserving encryption mode"); +MODULE_LICENSE("GPL v2"); +MODULE_ALIAS_CRYPTO("hctr2"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index dd9cf216029b..336598da8eac 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -2191,6 +2191,11 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) 16, 16, aead_speed_template_19, num_mb); break; + case 226: + test_cipher_speed("hctr2(aes)", ENCRYPT, sec, NULL, + 0, speed_template_32); + break; + case 300: if (alg) { test_hash_speed(alg, sec, generic_hash_speed_template); diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 0f40e260b5a9..485097461068 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -5088,6 +5088,14 @@ static const struct alg_test_desc alg_test_descs[] = { .suite = { .hash = __VECS(ghash_tv_template) } + }, { + .alg = "hctr2(aes)", + .generic_driver = + "hctr2_base(xctr(aes-generic),polyval-generic)", + .test = alg_test_skcipher, + .suite = { + .cipher = __VECS(aes_hctr2_tv_template) + } }, { .alg = "hmac(md5)", .test = alg_test_hash, diff --git a/crypto/testmgr.h b/crypto/testmgr.h index b58e13930307..808ba07baa04 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -35115,4 +35115,676 @@ static const struct hash_testvec polyval_tv_template[] = { }; +/* + * Test vectors generated using https://github.com/google/hctr2 + */ +static const struct cipher_testvec aes_hctr2_tv_template[] = { + { + .key = "\xe1\x15\x66\x3c\x8d\xc6\x3a\xff" + "\xef\x41\xd7\x47\xa2\xcc\x8a\xba", + .iv = "\xc3\xbe\x2a\xcb\xb5\x39\x86\xf1" + "\x91\xad\x6c\xf4\xde\x74\x45\x63" + "\x5c\x7a\xd5\xcc\x8b\x76\xef\x0e" + "\xcf\x2c\x60\x69\x37\xfd\x07\x96", + .ptext = "\x65\x75\xae\xd3\xe2\xbc\x43\x5c" + "\xb3\x1a\xd8\x05\xc3\xd0\x56\x29", + .ctext = "\x11\x91\xea\x74\x58\xcc\xd5\xa2" + "\xd0\x55\x9e\x3d\xfe\x7f\xc8\xfe", + .klen = 16, + .len = 16, + }, + { + .key = "\xe7\xd1\x77\x48\x76\x0b\xcd\x34" + "\x2a\x2d\xe7\x74\xca\x11\x9c\xae", + .iv = "\x71\x1c\x49\x62\xd9\x5b\x50\x5e" + "\x68\x87\xbc\xf6\x89\xff\xed\x30" + "\xe4\xe5\xbd\xb6\x10\x4f\x9f\x66" + "\x28\x06\x5a\xf4\x27\x35\xcd\xe5", + .ptext = "\x87\x03\x8f\x06\xa8\x61\x54\xda" + "\x01\x45\xd4\x01\xef\x4a\x22\xcf" + "\x78\x15\x9f\xbd\x64\xbd\x2c\xb9" + "\x40\x1d\x72\xae\x53\x63\xa5", + .ctext = "\x4e\xa1\x05\x27\xb8\x45\xe4\xa1" + "\xbb\x30\xb4\xa6\x12\x74\x63\xd6" + "\x17\xc9\xcc\x2f\x18\x64\xe0\x06" + "\x0a\xa0\xff\x72\x10\x7b\x22", + .klen = 16, + .len = 31, + }, + { + .key = "\x59\x65\x3b\x1d\x43\x5e\xc0\xae" + "\xb8\x9d\x9b\xdd\x22\x03\xbf\xca", + .iv = "\xec\x95\xfa\x5a\xcf\x5e\xd2\x93" + "\xa3\xb5\xe5\xbe\xf3\x01\x7b\x01" + "\xd1\xca\x6c\x06\x82\xf0\xbd\x67" + "\xd9\x6c\xa4\xdc\xb4\x38\x0f\x74", + .ptext = "\x45\xdf\x75\x87\xbc\x72\xce\x55" + "\xc9\xfa\xcb\xfc\x9f\x40\x82\x2b" + "\xc6\x4f\x4f\x5b\x8b\x3b\x6d\x67" + "\xa6\x93\x62\x89\x8c\x19\xf4\xe3" + "\x08\x92\x9c\xc9\x47\x2c\x6e\xd0" + "\xa3\x02\x2b\xdb\x2c\xf2\x8d\x46" + "\xcd\xb0\x9d\x26\x63\x4c\x40\x6b" + "\x79\x43\xe5\xce\x42\xa8\xec\x3b" + "\x5b\xd0\xea\xa4\xe6\xdb\x66\x55" + "\x7a\x76\xec\xab\x7d\x2a\x2b\xbd" + "\xa9\xab\x22\x64\x1a\xa1\xae\x84" + "\x86\x79\x67\xe9\xb2\x50\xbe\x12" + "\x2f\xb2\x14\xf0\xdb\x71\xd8\xa7" + "\x41\x8a\x88\xa0\x6a\x6e\x9d\x2a" + "\xfa\x11\x37\x40\x32\x09\x4c\x47" + "\x41\x07\x31\x85\x3d\xa8\xf7\x64", + .ctext = "\x2d\x4b\x9f\x93\xca\x5a\x48\x26" + "\x01\xcc\x54\xe4\x31\x50\x12\xf0" + "\x49\xff\x59\x42\x68\xbd\x87\x8f" + "\x9e\x62\x96\xcd\xb9\x24\x57\xa4" + "\x0b\x7b\xf5\x2e\x0e\xa8\x65\x07" + "\xab\x05\xd5\xca\xe7\x9c\x6c\x34" + "\x5d\x42\x34\xa4\x62\xe9\x75\x48" + "\x3d\x9e\x8f\xfa\x42\xe9\x75\x08" + "\x4e\x54\x91\x2b\xbd\x11\x0f\x8e" + "\xf0\x82\xf5\x24\xf1\xc4\xfc\xae" + "\x42\x54\x7f\xce\x15\xa8\xb2\x33" + "\xc0\x86\xb6\x2b\xe8\x44\xce\x1f" + "\x68\x57\x66\x94\x6e\xad\xeb\xf3" + "\x30\xf8\x11\xbd\x60\x00\xc6\xd5" + "\x4c\x81\xf1\x20\x2b\x4a\x5b\x99" + "\x79\x3b\xc9\x5c\x74\x23\xe6\x5d", + .klen = 16, + .len = 128, + }, + { + .key = "\x3e\x08\x5d\x64\x6c\x98\xec\xec" + "\x70\x0e\x0d\xa1\x41\x20\x99\x82", + .iv = "\x11\xb7\x77\x91\x0d\x99\xd9\x8d" + "\x35\x3a\xf7\x14\x6b\x09\x37\xe5" + "\xad\x51\xf6\xc3\x96\x4b\x64\x56" + "\xa8\xbd\x81\xcc\xbe\x94\xaf\xe4", + .ptext = "\xff\x8d\xb9\xc0\xe3\x69\xb3\xb2" + "\x8b\x11\x26\xb3\x11\xec\xfb\xb9" + "\x9c\xc1\x71\xd6\xe3\x26\x0e\xe0" + "\x68\x40\x60\xb9\x3a\x63\x56\x8a" + "\x9e\xc1\xf0\x10\xb1\x64\x32\x70" + "\xf8\xcd\xc6\xc4\x49\x4c\xe1\xce" + "\xf3\xe1\x03\xf8\x35\xae\xe0\x5e" + "\xef\x5f\xbc\x41\x75\x26\x13\xcc" + "\x37\x85\xdf\xc0\x5d\xa6\x47\x98" + "\xf1\x97\x52\x58\x04\xe6\xb5\x01" + "\xc0\xb8\x17\x6d\x74\xbd\x9a\xdf" + "\xa4\x37\x94\x86\xb0\x13\x83\x28" + "\xc9\xa2\x07\x3f\xb5\xb2\x72\x40" + "\x0e\x60\xdf\x57\x07\xb7\x2c\x66" + "\x10\x3f\x8d\xdd\x30\x0a\x47\xd5" + "\xe8\x9d\xfb\xa1\xaf\x53\xd7\x05" + "\xc7\xd2\xba\xe7\x2c\xa0\xbf\xb8" + "\xd1\x93\xe7\x41\x82\xa3\x41\x3a" + "\xaf\x12\xd6\xf8\x34\xda\x92\x46" + "\xad\xa2\x2f\xf6\x7e\x46\x96\xd8" + "\x03\xf3\x49\x64\xde\xd8\x06\x8b" + "\xa0\xbc\x63\x35\x38\xb6\x6b\xda" + "\x5b\x50\x3f\x13\xa5\x84\x1b\x1b" + "\x66\x89\x95\xb7\xc2\x16\x3c\xe9" + "\x24\xb0\x8c\x6f\x49\xef\xf7\x28" + "\x6a\x24\xfd\xbe\x25\xe2\xb4\x90" + "\x77\x44\x08\xb8\xda\xd2\xde\x2c" + "\xa0\x57\x45\x57\x29\x47\x6b\x89" + "\x4a\xf6\xa7\x2a\xc3\x9e\x7b\xc8" + "\xfd\x9f\x89\xab\xee\x6d\xa3\xb4" + "\x23\x90\x7a\xe9\x89\xa0\xc7\xb3" + "\x17\x41\x87\x91\xfc\x97\x42", + .ctext = "\xfc\x9b\x96\x66\xc4\x82\x2a\x4a" + "\xb1\x24\xba\xc7\x78\x5f\x79\xc1" + "\x57\x2e\x47\x29\x4d\x7b\xd2\x9a" + "\xbd\xc6\xc1\x26\x7b\x8e\x3f\x5d" + "\xd4\xb4\x9f\x6a\x02\x24\x4a\xad" + "\x0c\x00\x1b\xdf\x92\xc5\x8a\xe1" + "\x77\x79\xcc\xd5\x20\xbf\x83\xf4" + "\x4b\xad\x11\xbf\xdb\x47\x65\x70" + "\x43\xf3\x65\xdf\xb7\xdc\xb2\xb9" + "\xaa\x3f\xb3\xdf\x79\x69\x0d\xa0" + "\x86\x1c\xba\x48\x0b\x01\xc1\x88" + "\xdf\x03\xb1\x06\x3c\x1d\x56\xa1" + "\x8e\x98\xc1\xa6\x95\xa2\x5b\x72" + "\x76\x59\xd2\x26\x25\xcd\xef\x7c" + "\xc9\x60\xea\x43\xd1\x12\x8a\x8a" + "\x63\x12\x78\xcb\x2f\x88\x1e\x88" + "\x78\x59\xde\xba\x4d\x2c\x78\x61" + "\x75\x37\x54\xfd\x80\xc7\x5e\x98" + "\xcf\x14\x62\x8e\xfb\x72\xee\x4d" + "\x9f\xaf\x8b\x09\xe5\x21\x0a\x91" + "\x8f\x88\x87\xd5\xb1\x84\xab\x18" + "\x08\x57\xed\x72\x35\xa6\x0e\xc6" + "\xff\xcb\xfe\x2c\x48\x39\x14\x44" + "\xba\x59\x32\x3a\x2d\xc4\x5f\xcb" + "\xbe\x68\x8e\x7b\xee\x21\xa4\x32" + "\x11\xa0\x99\xfd\x90\xde\x59\x43" + "\xeb\xed\xd5\x87\x68\x46\xc6\xde" + "\x0b\x07\x17\x59\x6a\xab\xca\x15" + "\x65\x02\x01\xb6\x71\x8c\x3b\xaa" + "\x18\x3b\x30\xae\x38\x5b\x2c\x74" + "\xd4\xee\x4a\xfc\xf7\x1b\x09\xd4" + "\xda\x8b\x1d\x5d\x6f\x21\x6c", + .klen = 16, + .len = 255, + }, + { + .key = "\x24\xf6\xe1\x62\xe5\xaf\x99\xda" + "\x84\xec\x41\xb0\xa3\x0b\xd5\xa8" + "\xa0\x3e\x7b\xa6\xdd\x6c\x8f\xa8", + .iv = "\x7f\x80\x24\x62\x32\xdd\xab\x66" + "\xf2\x87\x29\x24\xec\xd2\x4b\x9f" + "\x0c\x33\x52\xd9\xe0\xcc\x6e\xe4" + "\x90\x85\x43\x97\xc4\x62\x14\x33", + .ptext = "\xef\x58\xe7\x7f\xa9\xd9\xb8\xd7" + "\xa2\x91\x97\x07\x27\x9e\xba\xe8" + "\xaa", + .ctext = "\xd7\xc3\x81\x91\xf2\x40\x17\x73" + "\x3e\x3b\x1c\x2a\x8e\x11\x9c\x17" + "\xf1", + .klen = 24, + .len = 17, + }, + { + .key = "\xbf\xaf\xd7\x67\x8c\x47\xcf\x21" + "\x8a\xa5\xdd\x32\x25\x47\xbe\x4f" + "\xf1\x3a\x0b\xa6\xaa\x2d\xcf\x09", + .iv = "\xd9\xe8\xf0\x92\x4e\xfc\x1d\xf2" + "\x81\x37\x7c\x8f\xf1\x59\x09\x20" + "\xf4\x46\x51\x86\x4f\x54\x8b\x32" + "\x58\xd1\x99\x8b\x8c\x03\xeb\x5d", + .ptext = "\xcd\x64\x90\xf9\x7c\xe5\x0e\x5a" + "\x75\xe7\x8e\x39\x86\xec\x20\x43" + "\x8a\x49\x09\x15\x47\xf4\x3c\x89" + "\x21\xeb\xcf\x4e\xcf\x91\xb5\x40" + "\xcd\xe5\x4d\x5c\x6f\xf2\xd2\x80" + "\xfa\xab\xb3\x76\x9f\x7f\x84\x0a", + .ctext = "\x44\x98\x64\x15\xb7\x0b\x80\xa3" + "\xb9\xca\x23\xff\x3b\x0b\x68\x74" + "\xbb\x3e\x20\x19\x9f\x28\x71\x2a" + "\x48\x3c\x7c\xe2\xef\xb5\x10\xac" + "\x82\x9f\xcd\x08\x8f\x6b\x16\x6f" + "\xc3\xbb\x07\xfb\x3c\xb0\x1b\x27", + .klen = 24, + .len = 48, + }, + { + .key = "\xb8\x35\xa2\x5f\x86\xbb\x82\x99" + "\x27\xeb\x01\x3f\x92\xaf\x80\x24" + "\x4c\x66\xa2\x89\xff\x2e\xa2\x25", + .iv = "\x0a\x1d\x96\xd3\xe0\xe8\x0c\x9b" + "\x9d\x6f\x21\x97\xc2\x17\xdb\x39" + "\x3f\xd8\x64\x48\x80\x04\xee\x43" + "\x02\xce\x88\xe2\x81\x81\x5f\x81", + .ptext = "\xb8\xf9\x16\x8b\x25\x68\xd0\x9c" + "\xd2\x28\xac\xa8\x79\xc2\x30\xc1" + "\x31\xde\x1c\x37\x1b\xa2\xb5\xe6" + "\xf0\xd0\xf8\x9c\x7f\xc6\x46\x07" + "\x5c\xc3\x06\xe4\xf0\x02\xec\xf8" + "\x59\x7c\xc2\x5d\xf8\x0c\x21\xae" + "\x9e\x82\xb1\x1a\x5f\x78\x44\x15" + "\x00\xa7\x2e\x52\xc5\x98\x98\x35" + "\x03\xae\xd0\x8e\x07\x57\xe2\x5a" + "\x17\xbf\x52\x40\x54\x5b\x74\xe5" + "\x2d\x35\xaf\x9e\x37\xf7\x7e\x4a" + "\x8c\x9e\xa1\xdc\x40\xb4\x5b\x36" + "\xdc\x3a\x68\xe6\xb7\x35\x0b\x8a" + "\x90\xec\x74\x8f\x09\x9a\x7f\x02" + "\x4d\x03\x46\x35\x62\xb1\xbd\x08" + "\x3f\x54\x2a\x10\x0b\xdc\x69\xaf" + "\x25\x3a\x0c\x5f\xe0\x51\xe7\x11" + "\xb7\x00\xab\xbb\x9a\xb0\xdc\x4d" + "\xc3\x7d\x1a\x6e\xd1\x09\x52\xbd" + "\x6b\x43\x55\x22\x3a\x78\x14\x7d" + "\x79\xfd\x8d\xfc\x9b\x1d\x0f\xa2" + "\xc7\xb9\xf8\x87\xd5\x96\x50\x61" + "\xa7\x5e\x1e\x57\x97\xe0\xad\x2f" + "\x93\xe6\xe8\x83\xec\x85\x26\x5e" + "\xd9\x2a\x15\xe0\xe9\x09\x25\xa1" + "\x77\x2b\x88\xdc\xa4\xa5\x48\xb6" + "\xf7\xcc\xa6\xa9\xba\xf3\x42\x5c" + "\x70\x9d\xe9\x29\xc1\xf1\x33\xdd" + "\x56\x48\x17\x86\x14\x51\x5c\x10" + "\xab\xfd\xd3\x26\x8c\x21\xf5\x93" + "\x1b\xeb\x47\x97\x73\xbb\x88\x10" + "\xf3\xfe\xf5\xde\xf3\x2e\x05\x46" + "\x1c\x0d\xa3\x10\x48\x9c\x71\x16" + "\x78\x33\x4d\x0a\x74\x3b\xe9\x34" + "\x0b\xa7\x0e\x9e\x61\xe9\xe9\xfd" + "\x85\xa0\xcb\x19\xfd\x7c\x33\xe3" + "\x0e\xce\xc2\x6f\x9d\xa4\x2d\x77" + "\xfd\xad\xee\x5e\x08\x3e\xd7\xf5" + "\xfb\xc3\xd7\x93\x96\x08\x96\xca" + "\x58\x81\x16\x9b\x98\x0a\xe2\xef" + "\x7f\xda\x40\xe4\x1f\x46\x9e\x67" + "\x2b\x84\xcb\x42\xc4\xd6\x6a\xcf" + "\x2d\xb2\x33\xc0\x56\xb3\x35\x6f" + "\x29\x36\x8f\x6a\x5b\xec\xd5\x4f" + "\xa0\x70\xff\xb6\x5b\xde\x6a\x93" + "\x20\x3c\xe2\x76\x7a\xef\x3c\x79" + "\x31\x65\xce\x3a\x0e\xd0\xbe\xa8" + "\x21\x95\xc7\x2b\x62\x8e\x67\xdd" + "\x20\x79\xe4\xe5\x01\x15\xc0\xec" + "\x0f\xd9\x23\xc8\xca\xdf\xd4\x7d" + "\x1d\xf8\x64\x4f\x56\xb1\x83\xa7" + "\x43\xbe\xfc\xcf\xc2\x8c\x33\xda" + "\x36\xd0\x52\xef\x9e\x9e\x88\xf4" + "\xa8\x21\x0f\xaa\xee\x8d\xa0\x24" + "\x4d\xcb\xb1\x72\x07\xf0\xc2\x06" + "\x60\x65\x85\x84\x2c\x60\xcf\x61" + "\xe7\x56\x43\x5b\x2b\x50\x74\xfa" + "\xdb\x4e\xea\x88\xd4\xb3\x83\x8f" + "\x6f\x97\x4b\x57\x7a\x64\x64\xae" + "\x0a\x37\x66\xc5\x03\xad\xb5\xf9" + "\x08\xb0\x3a\x74\xde\x97\x51\xff" + "\x48\x4f\x5c\xa4\xf8\x7a\xb4\x05" + "\x27\x70\x52\x86\x1b\x78\xfc\x18" + "\x06\x27\xa9\x62\xf7\xda\xd2\x8e", + .ctext = "\x3b\xe1\xdb\xb3\xc5\x9a\xde\x69" + "\x58\x05\xcc\xeb\x02\x51\x78\x4a" + "\xac\x28\xe9\xed\xd1\xc9\x15\x7d" + "\x33\x7d\xc1\x47\x12\x41\x11\xf8" + "\x4a\x2c\xb7\xa3\x41\xbe\x59\xf7" + "\x22\xdb\x2c\xda\x9c\x00\x61\x9b" + "\x73\xb3\x0b\x84\x2b\xc1\xf3\x80" + "\x84\xeb\x19\x60\x80\x09\xe1\xcd" + "\x16\x3a\x20\x23\xc4\x82\x4f\xba" + "\x3b\x8e\x55\xd7\xa9\x0b\x75\xd0" + "\xda\xce\xd2\xee\x7e\x4b\x7f\x65" + "\x4d\x28\xc5\xd3\x15\x2c\x40\x96" + "\x52\xd4\x18\x61\x2b\xe7\x83\xec" + "\x89\x62\x9c\x4c\x50\xe6\xe2\xbb" + "\x25\xa1\x0f\xa7\xb0\xb4\xb2\xde" + "\x54\x20\xae\xa3\x56\xa5\x26\x4c" + "\xd5\xcc\xe5\xcb\x28\x44\xb1\xef" + "\x67\x2e\x93\x6d\x00\x88\x83\x9a" + "\xf2\x1c\x48\x38\xec\x1a\x24\x90" + "\x73\x0a\xdb\xe8\xce\x95\x7a\x2c" + "\x8c\xe9\xb7\x07\x1d\xb3\xa3\x20" + "\xbe\xad\x61\x84\xac\xde\x76\xb5" + "\xa6\x28\x29\x47\x63\xc4\xfc\x13" + "\x3f\x71\xfb\x58\x37\x34\x82\xed" + "\x9e\x05\x19\x1f\xc1\x67\xc1\xab" + "\xf5\xfd\x7c\xea\xfa\xa4\xf8\x0a" + "\xac\x4c\x92\xdf\x65\x73\xd7\xdb" + "\xed\x2c\xe0\x84\x5f\x57\x8c\x76" + "\x3e\x05\xc0\xc3\x68\x96\x95\x0b" + "\x88\x97\xfe\x2e\x99\xd5\xc2\xb9" + "\x53\x9f\xf3\x32\x10\x1f\x1f\x5d" + "\xdf\x21\x95\x70\x91\xe8\xa1\x3e" + "\x19\x3e\xb6\x0b\xa8\xdb\xf8\xd4" + "\x54\x27\xb8\xab\x5d\x78\x0c\xe6" + "\xb7\x08\xee\xa4\xb6\x6b\xeb\x5a" + "\x89\x69\x2b\xbd\xd4\x21\x5b\xbf" + "\x79\xbb\x0f\xff\xdb\x23\x9a\xeb" + "\x8d\xf2\xc4\x39\xb4\x90\x77\x6f" + "\x68\xe2\xb8\xf3\xf1\x65\x4f\xd5" + "\x24\x80\x06\xaf\x7c\x8d\x15\x0c" + "\xfd\x56\xe5\xe3\x01\xa5\xf7\x1c" + "\x31\xd6\xa2\x01\x1e\x59\xf9\xa9" + "\x42\xd5\xc2\x34\xda\x25\xde\xc6" + "\x5d\x38\xef\xd1\x4c\xc1\xd9\x1b" + "\x98\xfd\xcd\x57\x6f\xfd\x46\x91" + "\x90\x3d\x52\x2b\x2c\x7d\xcf\x71" + "\xcf\xd1\x77\x23\x71\x36\xb1\xce" + "\xc7\x5d\xf0\x5b\x44\x3d\x43\x71" + "\xac\xb8\xa0\x6a\xea\x89\x5c\xff" + "\x81\x73\xd4\x83\xd1\xc9\xe9\xe2" + "\xa8\xa6\x0f\x36\xe6\xaa\x57\xd4" + "\x27\xd2\xc9\xda\x94\x02\x1f\xfb" + "\xe1\xa1\x07\xbe\xe1\x1b\x15\x94" + "\x1e\xac\x2f\x57\xbb\x41\x22\xaf" + "\x60\x5e\xcc\x66\xcb\x16\x62\xab" + "\xb8\x7c\x99\xf4\x84\x93\x0c\xc2" + "\xa2\x49\xe4\xfd\x17\x55\xe1\xa6" + "\x8d\x5b\xc6\x1b\xc8\xac\xec\x11" + "\x33\xcf\xb0\xe8\xc7\x28\x4f\xb2" + "\x5c\xa6\xe2\x71\xab\x80\x0a\xa7" + "\x5c\x59\x50\x9f\x7a\x32\xb7\xe5" + "\x24\x9a\x8e\x25\x21\x2e\xb7\x18" + "\xd0\xf2\xe7\x27\x6f\xda\xc1\x00" + "\xd9\xa6\x03\x59\xac\x4b\xcb\xba", + .klen = 24, + .len = 512, + }, + { + .key = "\x9e\xeb\xb2\x49\x3c\x1c\xf5\xf4" + "\x6a\x99\xc2\xc4\xdf\xb1\xf4\xdd" + "\x75\x20\x57\xea\x2c\x4f\xcd\xb2" + "\xa5\x3d\x7b\x49\x1e\xab\xfd\x0f", + .iv = "\xdf\x63\xd4\xab\xd2\x49\xf3\xd8" + "\x33\x81\x37\x60\x7d\xfa\x73\x08" + "\xd8\x49\x6d\x80\xe8\x2f\x62\x54" + "\xeb\x0e\xa9\x39\x5b\x45\x7f\x8a", + .ptext = "\x67\xc9\xf2\x30\x84\x41\x8e\x43" + "\xfb\xf3\xb3\x3e\x79\x36\x7f\xe8", + .ctext = "\x27\x38\x78\x47\x16\xd9\x71\x35" + "\x2e\x7e\xdd\x7e\x43\x3c\xb8\x40", + .klen = 32, + .len = 16, + }, + { + .key = "\x93\xfa\x7e\xe2\x0e\x67\xc4\x39" + "\xe7\xca\x47\x95\x68\x9d\x5e\x5a" + "\x7c\x26\x19\xab\xc6\xca\x6a\x4c" + "\x45\xa6\x96\x42\xae\x6c\xff\xe7", + .iv = "\xea\x82\x47\x95\x3b\x22\xa1\x3a" + "\x6a\xca\x24\x4c\x50\x7e\x23\xcd" + "\x0e\x50\xe5\x41\xb6\x65\x29\xd8" + "\x30\x23\x00\xd2\x54\xa7\xd6\x56", + .ptext = "\xdb\x1f\x1f\xec\xad\x83\x6e\x5d" + "\x19\xa5\xf6\x3b\xb4\x93\x5a\x57" + "\x6f", + .ctext = "\xf1\x46\x6e\x9d\xb3\x01\xf0\x6b" + "\xc2\xac\x57\x88\x48\x6d\x40\x72" + "\x68", + .klen = 32, + .len = 17, + }, + { + .key = "\x36\x2b\x57\x97\xf8\x5d\xcd\x99" + "\x5f\x1a\x5a\x44\x1d\x92\x0f\x27" + "\xcc\x16\xd7\x2b\x85\x63\x99\xd3" + "\xba\x96\xa1\xdb\xd2\x60\x68\xda", + .iv = "\xef\x58\x69\xb1\x2c\x5e\x9a\x47" + "\x24\xc1\xb1\x69\xe1\x12\x93\x8f" + "\x43\x3d\x6d\x00\xdb\x5e\xd8\xd9" + "\x12\x9a\xfe\xd9\xff\x2d\xaa\xc4", + .ptext = "\x5e\xa8\x68\x19\x85\x98\x12\x23" + "\x26\x0a\xcc\xdb\x0a\x04\xb9\xdf" + "\x4d\xb3\x48\x7b\xb0\xe3\xc8\x19" + "\x43\x5a\x46\x06\x94\x2d\xf2", + .ctext = "\xdb\xfd\xc8\x03\xd0\xec\xc1\xfe" + "\xbd\x64\x37\xb8\x82\x43\x62\x4e" + "\x7e\x54\xa3\xe2\x24\xa7\x27\xe8" + "\xa4\xd5\xb3\x6c\xb2\x26\xb4", + .klen = 32, + .len = 31, + }, + { + .key = "\x03\x65\x03\x6e\x4d\xe6\xe8\x4e" + "\x8b\xbe\x22\x19\x48\x31\xee\xd9" + "\xa0\x91\x21\xbe\x62\x89\xde\x78" + "\xd9\xb0\x36\xa3\x3c\xce\x43\xd5", + .iv = "\xa9\xc3\x4b\xe7\x0f\xfc\x6d\xbf" + "\x56\x27\x21\x1c\xfc\xd6\x04\x10" + "\x5f\x43\xe2\x30\x35\x29\x6c\x10" + "\x90\xf1\xbf\x61\xed\x0f\x8a\x91", + .ptext = "\x07\xaa\x02\x26\xb4\x98\x11\x5e" + "\x33\x41\x21\x51\x51\x63\x2c\x72" + "\x00\xab\x32\xa7\x1c\xc8\x3c\x9c" + "\x25\x0e\x8b\x9a\xdf\x85\xed\x2d" + "\xf4\xf2\xbc\x55\xca\x92\x6d\x22" + "\xfd\x22\x3b\x42\x4c\x0b\x74\xec", + .ctext = "\x7b\xb1\x43\x6d\xd8\x72\x6c\xf6" + "\x67\x6a\x00\xc4\xf1\xf0\xf5\xa4" + "\xfc\x60\x91\xab\x46\x0b\x15\xfc" + "\xd7\xc1\x28\x15\xa1\xfc\xf7\x68" + "\x8e\xcc\x27\x62\x00\x64\x56\x72" + "\xa6\x17\xd7\x3f\x67\x80\x10\x58", + .klen = 32, + .len = 48, + }, + { + .key = "\xa5\x28\x24\x34\x1a\x3c\xd8\xf7" + "\x05\x91\x8f\xee\x85\x1f\x35\x7f" + "\x80\x3d\xfc\x9b\x94\xf6\xfc\x9e" + "\x19\x09\x00\xa9\x04\x31\x4f\x11", + .iv = "\xa1\xba\x49\x95\xff\x34\x6d\xb8" + "\xcd\x87\x5d\x5e\xfd\xea\x85\xdb" + "\x8a\x7b\x5e\xb2\x5d\x57\xdd\x62" + "\xac\xa9\x8c\x41\x42\x94\x75\xb7", + .ptext = "\x69\xb4\xe8\x8c\x37\xe8\x67\x82" + "\xf1\xec\x5d\x04\xe5\x14\x91\x13" + "\xdf\xf2\x87\x1b\x69\x81\x1d\x71" + "\x70\x9e\x9c\x3b\xde\x49\x70\x11" + "\xa0\xa3\xdb\x0d\x54\x4f\x66\x69" + "\xd7\xdb\x80\xa7\x70\x92\x68\xce" + "\x81\x04\x2c\xc6\xab\xae\xe5\x60" + "\x15\xe9\x6f\xef\xaa\x8f\xa7\xa7" + "\x63\x8f\xf2\xf0\x77\xf1\xa8\xea" + "\xe1\xb7\x1f\x9e\xab\x9e\x4b\x3f" + "\x07\x87\x5b\x6f\xcd\xa8\xaf\xb9" + "\xfa\x70\x0b\x52\xb8\xa8\xa7\x9e" + "\x07\x5f\xa6\x0e\xb3\x9b\x79\x13" + "\x79\xc3\x3e\x8d\x1c\x2c\x68\xc8" + "\x51\x1d\x3c\x7b\x7d\x79\x77\x2a" + "\x56\x65\xc5\x54\x23\x28\xb0\x03", + .ctext = "\xeb\xf9\x98\x86\x3c\x40\x9f\x16" + "\x84\x01\xf9\x06\x0f\xeb\x3c\xa9" + "\x4c\xa4\x8e\x5d\xc3\x8d\xe5\xd3" + "\xae\xa6\xe6\xcc\xd6\x2d\x37\x4f" + "\x99\xc8\xa3\x21\x46\xb8\x69\xf2" + "\xe3\x14\x89\xd7\xb9\xf5\x9e\x4e" + "\x07\x93\x6f\x78\x8e\x6b\xea\x8f" + "\xfb\x43\xb8\x3e\x9b\x4c\x1d\x7e" + "\x20\x9a\xc5\x87\xee\xaf\xf6\xf9" + "\x46\xc5\x18\x8a\xe8\x69\xe7\x96" + "\x52\x55\x5f\x00\x1e\x1a\xdc\xcc" + "\x13\xa5\xee\xff\x4b\x27\xca\xdc" + "\x10\xa6\x48\x76\x98\x43\x94\xa3" + "\xc7\xe2\xc9\x65\x9b\x08\x14\x26" + "\x1d\x68\xfb\x15\x0a\x33\x49\x84" + "\x84\x33\x5a\x1b\x24\x46\x31\x92", + .klen = 32, + .len = 128, + }, + { + .key = "\x36\x45\x11\xa2\x98\x5f\x96\x7c" + "\xc6\xb4\x94\x31\x0a\x67\x09\x32" + "\x6c\x6f\x6f\x00\xf0\x17\xcb\xac" + "\xa5\xa9\x47\x9e\x2e\x85\x2f\xfa", + .iv = "\x28\x88\xaa\x9b\x59\x3b\x1e\x97" + "\x82\xe5\x5c\x9e\x6d\x14\x11\x19" + "\x6e\x38\x8f\xd5\x40\x2b\xca\xf9" + "\x7b\x4c\xe4\xa3\xd0\xd2\x8a\x13", + .ptext = "\x95\xd2\xf7\x71\x1b\xca\xa5\x86" + "\xd9\x48\x01\x93\x2f\x79\x55\x29" + "\x71\x13\x15\x0e\xe6\x12\xbc\x4d" + "\x8a\x31\xe3\x40\x2a\xc6\x5e\x0d" + "\x68\xbb\x4a\x62\x8d\xc7\x45\x77" + "\xd2\xb8\xc7\x1d\xf1\xd2\x5d\x97" + "\xcf\xac\x52\xe5\x32\x77\xb6\xda" + "\x30\x85\xcf\x2b\x98\xe9\xaa\x34" + "\x62\xb5\x23\x9e\xb7\xa6\xd4\xe0" + "\xb4\x58\x18\x8c\x4d\xde\x4d\x01" + "\x83\x89\x24\xca\xfb\x11\xd4\x82" + "\x30\x7a\x81\x35\xa0\xb4\xd4\xb6" + "\x84\xea\x47\x91\x8c\x19\x86\x25" + "\xa6\x06\x8d\x78\xe6\xed\x87\xeb" + "\xda\xea\x73\x7c\xbf\x66\xb8\x72" + "\xe3\x0a\xb8\x0c\xcb\x1a\x73\xf1" + "\xa7\xca\x0a\xde\x57\x2b\xbd\x2b" + "\xeb\x8b\x24\x38\x22\xd3\x0e\x1f" + "\x17\xa0\x84\x98\x31\x77\xfd\x34" + "\x6a\x4e\x3d\x84\x4c\x0e\xfb\xed" + "\xc8\x2a\x51\xfa\xd8\x73\x21\x8a" + "\xdb\xb5\xfe\x1f\xee\xc4\xe8\x65" + "\x54\x84\xdd\x96\x6d\xfd\xd3\x31" + "\x77\x36\x52\x6b\x80\x4f\x9e\xb4" + "\xa2\x55\xbf\x66\x41\x49\x4e\x87" + "\xa7\x0c\xca\xe7\xa5\xc5\xf6\x6f" + "\x27\x56\xe2\x48\x22\xdd\x5f\x59" + "\x3c\xf1\x9f\x83\xe5\x2d\xfb\x71" + "\xad\xd1\xae\x1b\x20\x5c\x47\xb7" + "\x3b\xd3\x14\xce\x81\x42\xb1\x0a" + "\xf0\x49\xfa\xc2\xe7\x86\xbf\xcd" + "\xb0\x95\x9f\x8f\x79\x41\x54", + .ctext = "\xf6\x57\x51\xc4\x25\x61\x2d\xfa" + "\xd6\xd9\x3f\x9a\x81\x51\xdd\x8e" + "\x3d\xe7\xaa\x2d\xb1\xda\xc8\xa6" + "\x9d\xaa\x3c\xab\x62\xf2\x80\xc3" + "\x2c\xe7\x58\x72\x1d\x44\xc5\x28" + "\x7f\xb4\xf9\xbc\x9c\xb2\xab\x8e" + "\xfa\xd1\x4d\x72\xd9\x79\xf5\xa0" + "\x24\x3e\x90\x25\x31\x14\x38\x45" + "\x59\xc8\xf6\xe2\xc6\xf6\xc1\xa7" + "\xb2\xf8\xa7\xa9\x2b\x6f\x12\x3a" + "\xb0\x81\xa4\x08\x57\x59\xb1\x56" + "\x4c\x8f\x18\x55\x33\x5f\xd6\x6a" + "\xc6\xa0\x4b\xd6\x6b\x64\x3e\x9e" + "\xfd\x66\x16\xe2\xdb\xeb\x5f\xb3" + "\x50\x50\x3e\xde\x8d\x72\x76\x01" + "\xbe\xcc\xc9\x52\x09\x2d\x8d\xe7" + "\xd6\xc3\x66\xdb\x36\x08\xd1\x77" + "\xc8\x73\x46\x26\x24\x29\xbf\x68" + "\x2d\x2a\x99\x43\x56\x55\xe4\x93" + "\xaf\xae\x4d\xe7\x55\x4a\xc0\x45" + "\x26\xeb\x3b\x12\x90\x7c\xdc\xd1" + "\xd5\x6f\x0a\xd0\xa9\xd7\x4b\x89" + "\x0b\x07\xd8\x86\xad\xa1\xc4\x69" + "\x1f\x5e\x8b\xc4\x9e\x91\x41\x25" + "\x56\x98\x69\x78\x3a\x9e\xae\x91" + "\xd8\xd9\xfa\xfb\xff\x81\x25\x09" + "\xfc\xed\x2d\x87\xbc\x04\x62\x97" + "\x35\xe1\x26\xc2\x46\x1c\xcf\xd7" + "\x14\xed\x02\x09\xa5\xb2\xb6\xaa" + "\x27\x4e\x61\xb3\x71\x6b\x47\x16" + "\xb7\xe8\xd4\xaf\x52\xeb\x6a\x6b" + "\xdb\x4c\x65\x21\x9e\x1c\x36", + .klen = 32, + .len = 255, + }, + { + .key = "\xd3\x81\x72\x18\x23\xff\x6f\x4a" + "\x25\x74\x29\x0d\x51\x8a\x0e\x13" + "\xc1\x53\x5d\x30\x8d\xee\x75\x0d" + "\x14\xd6\x69\xc9\x15\xa9\x0c\x60", + .iv = "\x65\x9b\xd4\xa8\x7d\x29\x1d\xf4" + "\xc4\xd6\x9b\x6a\x28\xab\x64\xe2" + "\x62\x81\x97\xc5\x81\xaa\xf9\x44" + "\xc1\x72\x59\x82\xaf\x16\xc8\x2c", + .ptext = "\xc7\x6b\x52\x6a\x10\xf0\xcc\x09" + "\xc1\x12\x1d\x6d\x21\xa6\x78\xf5" + "\x05\xa3\x69\x60\x91\x36\x98\x57" + "\xba\x0c\x14\xcc\xf3\x2d\x73\x03" + "\xc6\xb2\x5f\xc8\x16\x27\x37\x5d" + "\xd0\x0b\x87\xb2\x50\x94\x7b\x58" + "\x04\xf4\xe0\x7f\x6e\x57\x8e\xc9" + "\x41\x84\xc1\xb1\x7e\x4b\x91\x12" + "\x3a\x8b\x5d\x50\x82\x7b\xcb\xd9" + "\x9a\xd9\x4e\x18\x06\x23\x9e\xd4" + "\xa5\x20\x98\xef\xb5\xda\xe5\xc0" + "\x8a\x6a\x83\x77\x15\x84\x1e\xae" + "\x78\x94\x9d\xdf\xb7\xd1\xea\x67" + "\xaa\xb0\x14\x15\xfa\x67\x21\x84" + "\xd3\x41\x2a\xce\xba\x4b\x4a\xe8" + "\x95\x62\xa9\x55\xf0\x80\xad\xbd" + "\xab\xaf\xdd\x4f\xa5\x7c\x13\x36" + "\xed\x5e\x4f\x72\xad\x4b\xf1\xd0" + "\x88\x4e\xec\x2c\x88\x10\x5e\xea" + "\x12\xc0\x16\x01\x29\xa3\xa0\x55" + "\xaa\x68\xf3\xe9\x9d\x3b\x0d\x3b" + "\x6d\xec\xf8\xa0\x2d\xf0\x90\x8d" + "\x1c\xe2\x88\xd4\x24\x71\xf9\xb3" + "\xc1\x9f\xc5\xd6\x76\x70\xc5\x2e" + "\x9c\xac\xdb\x90\xbd\x83\x72\xba" + "\x6e\xb5\xa5\x53\x83\xa9\xa5\xbf" + "\x7d\x06\x0e\x3c\x2a\xd2\x04\xb5" + "\x1e\x19\x38\x09\x16\xd2\x82\x1f" + "\x75\x18\x56\xb8\x96\x0b\xa6\xf9" + "\xcf\x62\xd9\x32\x5d\xa9\xd7\x1d" + "\xec\xe4\xdf\x1b\xbe\xf1\x36\xee" + "\xe3\x7b\xb5\x2f\xee\xf8\x53\x3d" + "\x6a\xb7\x70\xa9\xfc\x9c\x57\x25" + "\xf2\x89\x10\xd3\xb8\xa8\x8c\x30" + "\xae\x23\x4f\x0e\x13\x66\x4f\xe1" + "\xb6\xc0\xe4\xf8\xef\x93\xbd\x6e" + "\x15\x85\x6b\xe3\x60\x81\x1d\x68" + "\xd7\x31\x87\x89\x09\xab\xd5\x96" + "\x1d\xf3\x6d\x67\x80\xca\x07\x31" + "\x5d\xa7\xe4\xfb\x3e\xf2\x9b\x33" + "\x52\x18\xc8\x30\xfe\x2d\xca\x1e" + "\x79\x92\x7a\x60\x5c\xb6\x58\x87" + "\xa4\x36\xa2\x67\x92\x8b\xa4\xb7" + "\xf1\x86\xdf\xdc\xc0\x7e\x8f\x63" + "\xd2\xa2\xdc\x78\xeb\x4f\xd8\x96" + "\x47\xca\xb8\x91\xf9\xf7\x94\x21" + "\x5f\x9a\x9f\x5b\xb8\x40\x41\x4b" + "\x66\x69\x6a\x72\xd0\xcb\x70\xb7" + "\x93\xb5\x37\x96\x05\x37\x4f\xe5" + "\x8c\xa7\x5a\x4e\x8b\xb7\x84\xea" + "\xc7\xfc\x19\x6e\x1f\x5a\xa1\xac" + "\x18\x7d\x52\x3b\xb3\x34\x62\x99" + "\xe4\x9e\x31\x04\x3f\xc0\x8d\x84" + "\x17\x7c\x25\x48\x52\x67\x11\x27" + "\x67\xbb\x5a\x85\xca\x56\xb2\x5c" + "\xe6\xec\xd5\x96\x3d\x15\xfc\xfb" + "\x22\x25\xf4\x13\xe5\x93\x4b\x9a" + "\x77\xf1\x52\x18\xfa\x16\x5e\x49" + "\x03\x45\xa8\x08\xfa\xb3\x41\x92" + "\x79\x50\x33\xca\xd0\xd7\x42\x55" + "\xc3\x9a\x0c\x4e\xd9\xa4\x3c\x86" + "\x80\x9f\x53\xd1\xa4\x2e\xd1\xbc" + "\xf1\x54\x6e\x93\xa4\x65\x99\x8e" + "\xdf\x29\xc0\x64\x63\x07\xbb\xea", + .ctext = "\x9f\x72\x87\xc7\x17\xfb\x20\x15" + "\x65\xb3\x55\xa8\x1c\x8e\x52\x32" + "\xb1\x82\x8d\xbf\xb5\x9f\x10\x0a" + "\xe8\x0c\x70\x62\xef\x89\xb6\x1f" + "\x73\xcc\xe4\xcc\x7a\x3a\x75\x4a" + "\x26\xe7\xf5\xd7\x7b\x17\x39\x2d" + "\xd2\x27\x6e\xf9\x2f\x9e\xe2\xf6" + "\xfa\x16\xc2\xf2\x49\x26\xa7\x5b" + "\xe7\xca\x25\x0e\x45\xa0\x34\xc2" + "\x9a\x37\x79\x7e\x7c\x58\x18\x94" + "\x10\xa8\x7c\x48\xa9\xd7\x63\x89" + "\x9e\x61\x4d\x26\x34\xd9\xf0\xb1" + "\x2d\x17\x2c\x6f\x7c\x35\x0e\xbe" + "\x77\x71\x7c\x17\x5b\xab\x70\xdb" + "\x2f\x54\x0f\xa9\xc8\xf4\xf5\xab" + "\x52\x04\x3a\xb8\x03\xa7\xfd\x57" + "\x45\x5e\xbc\x77\xe1\xee\x79\x8c" + "\x58\x7b\x1f\xf7\x75\xde\x68\x17" + "\x98\x85\x8a\x18\x5c\xd2\x39\x78" + "\x7a\x6f\x26\x6e\xe1\x13\x91\xdd" + "\xdf\x0e\x6e\x67\xcc\x51\x53\xd8" + "\x17\x5e\xce\xa7\xe4\xaf\xfa\xf3" + "\x4f\x9f\x01\x9b\x04\xe7\xfc\xf9" + "\x6a\xdc\x1d\x0c\x9a\xaa\x3a\x7a" + "\x73\x03\xdf\xbf\x3b\x82\xbe\xb0" + "\xb4\xa4\xcf\x07\xd7\xde\x71\x25" + "\xc5\x10\xee\x0a\x15\x96\x8b\x4f" + "\xfe\xb8\x28\xbd\x4a\xcd\xeb\x9f" + "\x5d\x00\xc1\xee\xe8\x16\x44\xec" + "\xe9\x7b\xd6\x85\x17\x29\xcf\x58" + "\x20\xab\xf7\xce\x6b\xe7\x71\x7d" + "\x4f\xa8\xb0\xe9\x7d\x70\xd6\x0b" + "\x2e\x20\xb1\x1a\x63\x37\xaa\x2c" + "\x94\xee\xd5\xf6\x58\x2a\xf4\x7a" + "\x4c\xba\xf5\xe9\x3c\x6f\x95\x13" + "\x5f\x96\x81\x5b\xb5\x62\xf2\xd7" + "\x8d\xbe\xa1\x31\x51\xe6\xfe\xc9" + "\x07\x7d\x0f\x00\x3a\x66\x8c\x4b" + "\x94\xaa\xe5\x56\xde\xcd\x74\xa7" + "\x48\x67\x6f\xed\xc9\x6a\xef\xaf" + "\x9a\xb7\xae\x60\xfa\xc0\x37\x39" + "\xa5\x25\xe5\x22\xea\x82\x55\x68" + "\x3e\x30\xc3\x5a\xb6\x29\x73\x7a" + "\xb6\xfb\x34\xee\x51\x7c\x54\xe5" + "\x01\x4d\x72\x25\x32\x4a\xa3\x68" + "\x80\x9a\x89\xc5\x11\x66\x4c\x8c" + "\x44\x50\xbe\xd7\xa0\xee\xa6\xbb" + "\x92\x0c\xe6\xd7\x83\x51\xb1\x69" + "\x63\x40\xf3\xf4\x92\x84\xc4\x38" + "\x29\xfb\xb4\x84\xa0\x19\x75\x16" + "\x60\xbf\x0a\x9c\x89\xee\xad\xb4" + "\x43\xf9\x71\x39\x45\x7c\x24\x83" + "\x30\xbb\xee\x28\xb0\x86\x7b\xec" + "\x93\xc1\xbf\xb9\x97\x1b\x96\xef" + "\xee\x58\x35\x61\x12\x19\xda\x25" + "\x77\xe5\x80\x1a\x31\x27\x9b\xe4" + "\xda\x8b\x7e\x51\x4d\xcb\x01\x19" + "\x4f\xdc\x92\x1a\x17\xd5\x6b\xf4" + "\x50\xe3\x06\xe4\x76\x9f\x65\x00" + "\xbd\x7a\xe2\x64\x26\xf2\xe4\x7e" + "\x40\xf2\x80\xab\x62\xd5\xef\x23" + "\x8b\xfb\x6f\x24\x6e\x9b\x66\x0e" + "\xf4\x1c\x24\x1e\x1d\x26\x95\x09" + "\x94\x3c\xb2\xb6\x02\xa7\xd9\x9a", + .klen = 32, + .len = 512, + }, + +}; + #endif /* _CRYPTO_TESTMGR_H */ From fd94fcf09957a75e25941f7dbfc84d30a63817ac Mon Sep 17 00:00:00 2001 From: Nathan Huckleberry Date: Fri, 20 May 2022 18:14:56 +0000 Subject: [PATCH 011/318] crypto: x86/aesni-xctr - Add accelerated implementation of XCTR Add hardware accelerated version of XCTR for x86-64 CPUs with AESNI support. More information on XCTR can be found in the HCTR2 paper: "Length-preserving encryption with HCTR2": https://eprint.iacr.org/2021/1441.pdf Signed-off-by: Nathan Huckleberry Reviewed-by: Ard Biesheuvel Reviewed-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/aes_ctrby8_avx-x86_64.S | 232 ++++++++++++++++-------- arch/x86/crypto/aesni-intel_glue.c | 114 +++++++++++- crypto/Kconfig | 2 +- 3 files changed, 266 insertions(+), 82 deletions(-) diff --git a/arch/x86/crypto/aes_ctrby8_avx-x86_64.S b/arch/x86/crypto/aes_ctrby8_avx-x86_64.S index 43852ba6e19c..2402b9418cd7 100644 --- a/arch/x86/crypto/aes_ctrby8_avx-x86_64.S +++ b/arch/x86/crypto/aes_ctrby8_avx-x86_64.S @@ -23,6 +23,11 @@ #define VMOVDQ vmovdqu +/* + * Note: the "x" prefix in these aliases means "this is an xmm register". The + * alias prefixes have no relation to XCTR where the "X" prefix means "XOR + * counter". + */ #define xdata0 %xmm0 #define xdata1 %xmm1 #define xdata2 %xmm2 @@ -31,8 +36,10 @@ #define xdata5 %xmm5 #define xdata6 %xmm6 #define xdata7 %xmm7 -#define xcounter %xmm8 -#define xbyteswap %xmm9 +#define xcounter %xmm8 // CTR mode only +#define xiv %xmm8 // XCTR mode only +#define xbyteswap %xmm9 // CTR mode only +#define xtmp %xmm9 // XCTR mode only #define xkey0 %xmm10 #define xkey4 %xmm11 #define xkey8 %xmm12 @@ -45,7 +52,7 @@ #define p_keys %rdx #define p_out %rcx #define num_bytes %r8 - +#define counter %r9 // XCTR mode only #define tmp %r10 #define DDQ_DATA 0 #define XDATA 1 @@ -102,7 +109,7 @@ ddq_add_8: * do_aes num_in_par load_keys key_len * This increments p_in, but not p_out */ -.macro do_aes b, k, key_len +.macro do_aes b, k, key_len, xctr .set by, \b .set load_keys, \k .set klen, \key_len @@ -111,29 +118,48 @@ ddq_add_8: vmovdqa 0*16(p_keys), xkey0 .endif - vpshufb xbyteswap, xcounter, xdata0 - - .set i, 1 - .rept (by - 1) - club XDATA, i - vpaddq (ddq_add_1 + 16 * (i - 1))(%rip), xcounter, var_xdata - vptest ddq_low_msk(%rip), var_xdata - jnz 1f - vpaddq ddq_high_add_1(%rip), var_xdata, var_xdata - vpaddq ddq_high_add_1(%rip), xcounter, xcounter - 1: - vpshufb xbyteswap, var_xdata, var_xdata - .set i, (i +1) - .endr + .if \xctr + movq counter, xtmp + .set i, 0 + .rept (by) + club XDATA, i + vpaddq (ddq_add_1 + 16 * i)(%rip), xtmp, var_xdata + .set i, (i +1) + .endr + .set i, 0 + .rept (by) + club XDATA, i + vpxor xiv, var_xdata, var_xdata + .set i, (i +1) + .endr + .else + vpshufb xbyteswap, xcounter, xdata0 + .set i, 1 + .rept (by - 1) + club XDATA, i + vpaddq (ddq_add_1 + 16 * (i - 1))(%rip), xcounter, var_xdata + vptest ddq_low_msk(%rip), var_xdata + jnz 1f + vpaddq ddq_high_add_1(%rip), var_xdata, var_xdata + vpaddq ddq_high_add_1(%rip), xcounter, xcounter + 1: + vpshufb xbyteswap, var_xdata, var_xdata + .set i, (i +1) + .endr + .endif vmovdqa 1*16(p_keys), xkeyA vpxor xkey0, xdata0, xdata0 - vpaddq (ddq_add_1 + 16 * (by - 1))(%rip), xcounter, xcounter - vptest ddq_low_msk(%rip), xcounter - jnz 1f - vpaddq ddq_high_add_1(%rip), xcounter, xcounter - 1: + .if \xctr + add $by, counter + .else + vpaddq (ddq_add_1 + 16 * (by - 1))(%rip), xcounter, xcounter + vptest ddq_low_msk(%rip), xcounter + jnz 1f + vpaddq ddq_high_add_1(%rip), xcounter, xcounter + 1: + .endif .set i, 1 .rept (by - 1) @@ -371,94 +397,99 @@ ddq_add_8: .endr .endm -.macro do_aes_load val, key_len - do_aes \val, 1, \key_len +.macro do_aes_load val, key_len, xctr + do_aes \val, 1, \key_len, \xctr .endm -.macro do_aes_noload val, key_len - do_aes \val, 0, \key_len +.macro do_aes_noload val, key_len, xctr + do_aes \val, 0, \key_len, \xctr .endm /* main body of aes ctr load */ -.macro do_aes_ctrmain key_len +.macro do_aes_ctrmain key_len, xctr cmp $16, num_bytes - jb .Ldo_return2\key_len + jb .Ldo_return2\xctr\key_len - vmovdqa byteswap_const(%rip), xbyteswap - vmovdqu (p_iv), xcounter - vpshufb xbyteswap, xcounter, xcounter + .if \xctr + shr $4, counter + vmovdqu (p_iv), xiv + .else + vmovdqa byteswap_const(%rip), xbyteswap + vmovdqu (p_iv), xcounter + vpshufb xbyteswap, xcounter, xcounter + .endif mov num_bytes, tmp and $(7*16), tmp - jz .Lmult_of_8_blks\key_len + jz .Lmult_of_8_blks\xctr\key_len /* 1 <= tmp <= 7 */ cmp $(4*16), tmp - jg .Lgt4\key_len - je .Leq4\key_len + jg .Lgt4\xctr\key_len + je .Leq4\xctr\key_len -.Llt4\key_len: +.Llt4\xctr\key_len: cmp $(2*16), tmp - jg .Leq3\key_len - je .Leq2\key_len + jg .Leq3\xctr\key_len + je .Leq2\xctr\key_len -.Leq1\key_len: - do_aes_load 1, \key_len +.Leq1\xctr\key_len: + do_aes_load 1, \key_len, \xctr add $(1*16), p_out and $(~7*16), num_bytes - jz .Ldo_return2\key_len - jmp .Lmain_loop2\key_len + jz .Ldo_return2\xctr\key_len + jmp .Lmain_loop2\xctr\key_len -.Leq2\key_len: - do_aes_load 2, \key_len +.Leq2\xctr\key_len: + do_aes_load 2, \key_len, \xctr add $(2*16), p_out and $(~7*16), num_bytes - jz .Ldo_return2\key_len - jmp .Lmain_loop2\key_len + jz .Ldo_return2\xctr\key_len + jmp .Lmain_loop2\xctr\key_len -.Leq3\key_len: - do_aes_load 3, \key_len +.Leq3\xctr\key_len: + do_aes_load 3, \key_len, \xctr add $(3*16), p_out and $(~7*16), num_bytes - jz .Ldo_return2\key_len - jmp .Lmain_loop2\key_len + jz .Ldo_return2\xctr\key_len + jmp .Lmain_loop2\xctr\key_len -.Leq4\key_len: - do_aes_load 4, \key_len +.Leq4\xctr\key_len: + do_aes_load 4, \key_len, \xctr add $(4*16), p_out and $(~7*16), num_bytes - jz .Ldo_return2\key_len - jmp .Lmain_loop2\key_len + jz .Ldo_return2\xctr\key_len + jmp .Lmain_loop2\xctr\key_len -.Lgt4\key_len: +.Lgt4\xctr\key_len: cmp $(6*16), tmp - jg .Leq7\key_len - je .Leq6\key_len + jg .Leq7\xctr\key_len + je .Leq6\xctr\key_len -.Leq5\key_len: - do_aes_load 5, \key_len +.Leq5\xctr\key_len: + do_aes_load 5, \key_len, \xctr add $(5*16), p_out and $(~7*16), num_bytes - jz .Ldo_return2\key_len - jmp .Lmain_loop2\key_len + jz .Ldo_return2\xctr\key_len + jmp .Lmain_loop2\xctr\key_len -.Leq6\key_len: - do_aes_load 6, \key_len +.Leq6\xctr\key_len: + do_aes_load 6, \key_len, \xctr add $(6*16), p_out and $(~7*16), num_bytes - jz .Ldo_return2\key_len - jmp .Lmain_loop2\key_len + jz .Ldo_return2\xctr\key_len + jmp .Lmain_loop2\xctr\key_len -.Leq7\key_len: - do_aes_load 7, \key_len +.Leq7\xctr\key_len: + do_aes_load 7, \key_len, \xctr add $(7*16), p_out and $(~7*16), num_bytes - jz .Ldo_return2\key_len - jmp .Lmain_loop2\key_len + jz .Ldo_return2\xctr\key_len + jmp .Lmain_loop2\xctr\key_len -.Lmult_of_8_blks\key_len: +.Lmult_of_8_blks\xctr\key_len: .if (\key_len != KEY_128) vmovdqa 0*16(p_keys), xkey0 vmovdqa 4*16(p_keys), xkey4 @@ -471,17 +502,19 @@ ddq_add_8: vmovdqa 9*16(p_keys), xkey12 .endif .align 16 -.Lmain_loop2\key_len: +.Lmain_loop2\xctr\key_len: /* num_bytes is a multiple of 8 and >0 */ - do_aes_noload 8, \key_len + do_aes_noload 8, \key_len, \xctr add $(8*16), p_out sub $(8*16), num_bytes - jne .Lmain_loop2\key_len + jne .Lmain_loop2\xctr\key_len -.Ldo_return2\key_len: - /* return updated IV */ - vpshufb xbyteswap, xcounter, xcounter - vmovdqu xcounter, (p_iv) +.Ldo_return2\xctr\key_len: + .if !\xctr + /* return updated IV */ + vpshufb xbyteswap, xcounter, xcounter + vmovdqu xcounter, (p_iv) + .endif RET .endm @@ -494,7 +527,7 @@ ddq_add_8: */ SYM_FUNC_START(aes_ctr_enc_128_avx_by8) /* call the aes main loop */ - do_aes_ctrmain KEY_128 + do_aes_ctrmain KEY_128 0 SYM_FUNC_END(aes_ctr_enc_128_avx_by8) @@ -507,7 +540,7 @@ SYM_FUNC_END(aes_ctr_enc_128_avx_by8) */ SYM_FUNC_START(aes_ctr_enc_192_avx_by8) /* call the aes main loop */ - do_aes_ctrmain KEY_192 + do_aes_ctrmain KEY_192 0 SYM_FUNC_END(aes_ctr_enc_192_avx_by8) @@ -520,6 +553,45 @@ SYM_FUNC_END(aes_ctr_enc_192_avx_by8) */ SYM_FUNC_START(aes_ctr_enc_256_avx_by8) /* call the aes main loop */ - do_aes_ctrmain KEY_256 + do_aes_ctrmain KEY_256 0 SYM_FUNC_END(aes_ctr_enc_256_avx_by8) + +/* + * routine to do AES128 XCTR enc/decrypt "by8" + * XMM registers are clobbered. + * Saving/restoring must be done at a higher level + * aes_xctr_enc_128_avx_by8(const u8 *in, const u8 *iv, const void *keys, + * u8* out, unsigned int num_bytes, unsigned int byte_ctr) + */ +SYM_FUNC_START(aes_xctr_enc_128_avx_by8) + /* call the aes main loop */ + do_aes_ctrmain KEY_128 1 + +SYM_FUNC_END(aes_xctr_enc_128_avx_by8) + +/* + * routine to do AES192 XCTR enc/decrypt "by8" + * XMM registers are clobbered. + * Saving/restoring must be done at a higher level + * aes_xctr_enc_192_avx_by8(const u8 *in, const u8 *iv, const void *keys, + * u8* out, unsigned int num_bytes, unsigned int byte_ctr) + */ +SYM_FUNC_START(aes_xctr_enc_192_avx_by8) + /* call the aes main loop */ + do_aes_ctrmain KEY_192 1 + +SYM_FUNC_END(aes_xctr_enc_192_avx_by8) + +/* + * routine to do AES256 XCTR enc/decrypt "by8" + * XMM registers are clobbered. + * Saving/restoring must be done at a higher level + * aes_xctr_enc_256_avx_by8(const u8 *in, const u8 *iv, const void *keys, + * u8* out, unsigned int num_bytes, unsigned int byte_ctr) + */ +SYM_FUNC_START(aes_xctr_enc_256_avx_by8) + /* call the aes main loop */ + do_aes_ctrmain KEY_256 1 + +SYM_FUNC_END(aes_xctr_enc_256_avx_by8) diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index 41901ba9d3a2..a5b0cb3efeba 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -135,6 +135,20 @@ asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv, void *keys, u8 *out, unsigned int num_bytes); asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv, void *keys, u8 *out, unsigned int num_bytes); + + +asmlinkage void aes_xctr_enc_128_avx_by8(const u8 *in, const u8 *iv, + const void *keys, u8 *out, unsigned int num_bytes, + unsigned int byte_ctr); + +asmlinkage void aes_xctr_enc_192_avx_by8(const u8 *in, const u8 *iv, + const void *keys, u8 *out, unsigned int num_bytes, + unsigned int byte_ctr); + +asmlinkage void aes_xctr_enc_256_avx_by8(const u8 *in, const u8 *iv, + const void *keys, u8 *out, unsigned int num_bytes, + unsigned int byte_ctr); + /* * asmlinkage void aesni_gcm_init_avx_gen2() * gcm_data *my_ctx_data, context data @@ -527,6 +541,59 @@ static int ctr_crypt(struct skcipher_request *req) return err; } +static void aesni_xctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out, + const u8 *in, unsigned int len, u8 *iv, + unsigned int byte_ctr) +{ + if (ctx->key_length == AES_KEYSIZE_128) + aes_xctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len, + byte_ctr); + else if (ctx->key_length == AES_KEYSIZE_192) + aes_xctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len, + byte_ctr); + else + aes_xctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len, + byte_ctr); +} + +static int xctr_crypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); + u8 keystream[AES_BLOCK_SIZE]; + struct skcipher_walk walk; + unsigned int nbytes; + unsigned int byte_ctr = 0; + int err; + __le32 block[AES_BLOCK_SIZE / sizeof(__le32)]; + + err = skcipher_walk_virt(&walk, req, false); + + while ((nbytes = walk.nbytes) > 0) { + kernel_fpu_begin(); + if (nbytes & AES_BLOCK_MASK) + aesni_xctr_enc_avx_tfm(ctx, walk.dst.virt.addr, + walk.src.virt.addr, nbytes & AES_BLOCK_MASK, + walk.iv, byte_ctr); + nbytes &= ~AES_BLOCK_MASK; + byte_ctr += walk.nbytes - nbytes; + + if (walk.nbytes == walk.total && nbytes > 0) { + memcpy(block, walk.iv, AES_BLOCK_SIZE); + block[0] ^= cpu_to_le32(1 + byte_ctr / AES_BLOCK_SIZE); + aesni_enc(ctx, keystream, (u8 *)block); + crypto_xor_cpy(walk.dst.virt.addr + walk.nbytes - + nbytes, walk.src.virt.addr + walk.nbytes + - nbytes, keystream, nbytes); + byte_ctr += nbytes; + nbytes = 0; + } + kernel_fpu_end(); + err = skcipher_walk_done(&walk, nbytes); + } + return err; +} + static int rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len) { @@ -1050,6 +1117,33 @@ static struct skcipher_alg aesni_skciphers[] = { static struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)]; +#ifdef CONFIG_X86_64 +/* + * XCTR does not have a non-AVX implementation, so it must be enabled + * conditionally. + */ +static struct skcipher_alg aesni_xctr = { + .base = { + .cra_name = "__xctr(aes)", + .cra_driver_name = "__xctr-aes-aesni", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_INTERNAL, + .cra_blocksize = 1, + .cra_ctxsize = CRYPTO_AES_CTX_SIZE, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .chunksize = AES_BLOCK_SIZE, + .setkey = aesni_skcipher_setkey, + .encrypt = xctr_crypt, + .decrypt = xctr_crypt, +}; + +static struct simd_skcipher_alg *aesni_simd_xctr; +#endif /* CONFIG_X86_64 */ + #ifdef CONFIG_X86_64 static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key, unsigned int key_len) @@ -1163,7 +1257,7 @@ static int __init aesni_init(void) static_call_update(aesni_ctr_enc_tfm, aesni_ctr_enc_avx_tfm); pr_info("AES CTR mode by8 optimization enabled\n"); } -#endif +#endif /* CONFIG_X86_64 */ err = crypto_register_alg(&aesni_cipher_alg); if (err) @@ -1180,8 +1274,22 @@ static int __init aesni_init(void) if (err) goto unregister_skciphers; +#ifdef CONFIG_X86_64 + if (boot_cpu_has(X86_FEATURE_AVX)) + err = simd_register_skciphers_compat(&aesni_xctr, 1, + &aesni_simd_xctr); + if (err) + goto unregister_aeads; +#endif /* CONFIG_X86_64 */ + return 0; +#ifdef CONFIG_X86_64 +unregister_aeads: + simd_unregister_aeads(aesni_aeads, ARRAY_SIZE(aesni_aeads), + aesni_simd_aeads); +#endif /* CONFIG_X86_64 */ + unregister_skciphers: simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers), aesni_simd_skciphers); @@ -1197,6 +1305,10 @@ static void __exit aesni_exit(void) simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers), aesni_simd_skciphers); crypto_unregister_alg(&aesni_cipher_alg); +#ifdef CONFIG_X86_64 + if (boot_cpu_has(X86_FEATURE_AVX)) + simd_unregister_skciphers(&aesni_xctr, 1, &aesni_simd_xctr); +#endif /* CONFIG_X86_64 */ } late_initcall(aesni_init); diff --git a/crypto/Kconfig b/crypto/Kconfig index 0601a2d2feef..dfcc3235e918 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1169,7 +1169,7 @@ config CRYPTO_AES_NI_INTEL In addition to AES cipher algorithm support, the acceleration for some popular block cipher mode is supported too, including ECB, CBC, LRW, XTS. The 64 bit version has additional - acceleration for CTR. + acceleration for CTR and XCTR. config CRYPTO_AES_SPARC64 tristate "AES cipher algorithms (SPARC64)" From 23a251cc1696e1bf68df1dbba569d2fe12469d22 Mon Sep 17 00:00:00 2001 From: Nathan Huckleberry Date: Fri, 20 May 2022 18:14:57 +0000 Subject: [PATCH 012/318] crypto: arm64/aes-xctr - Add accelerated implementation of XCTR Add hardware accelerated version of XCTR for ARM64 CPUs with ARMv8 Crypto Extension support. This XCTR implementation is based on the CTR implementation in aes-modes.S. More information on XCTR can be found in the HCTR2 paper: "Length-preserving encryption with HCTR2": https://eprint.iacr.org/2021/1441.pdf Signed-off-by: Nathan Huckleberry Reviewed-by: Ard Biesheuvel Reviewed-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm64/crypto/Kconfig | 4 +- arch/arm64/crypto/aes-glue.c | 64 +++++++++++++- arch/arm64/crypto/aes-modes.S | 162 +++++++++++++++++++++------------- 3 files changed, 166 insertions(+), 64 deletions(-) diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index ac85682c013c..74d5bed9394f 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -96,13 +96,13 @@ config CRYPTO_AES_ARM64_CE_CCM select CRYPTO_LIB_AES config CRYPTO_AES_ARM64_CE_BLK - tristate "AES in ECB/CBC/CTR/XTS modes using ARMv8 Crypto Extensions" + tristate "AES in ECB/CBC/CTR/XTS/XCTR modes using ARMv8 Crypto Extensions" depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER select CRYPTO_AES_ARM64_CE config CRYPTO_AES_ARM64_NEON_BLK - tristate "AES in ECB/CBC/CTR/XTS modes using NEON instructions" + tristate "AES in ECB/CBC/CTR/XTS/XCTR modes using NEON instructions" depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER select CRYPTO_LIB_AES diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c index 561dd2332571..b6883288234c 100644 --- a/arch/arm64/crypto/aes-glue.c +++ b/arch/arm64/crypto/aes-glue.c @@ -34,10 +34,11 @@ #define aes_essiv_cbc_encrypt ce_aes_essiv_cbc_encrypt #define aes_essiv_cbc_decrypt ce_aes_essiv_cbc_decrypt #define aes_ctr_encrypt ce_aes_ctr_encrypt +#define aes_xctr_encrypt ce_aes_xctr_encrypt #define aes_xts_encrypt ce_aes_xts_encrypt #define aes_xts_decrypt ce_aes_xts_decrypt #define aes_mac_update ce_aes_mac_update -MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions"); +MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS/XCTR using ARMv8 Crypto Extensions"); #else #define MODE "neon" #define PRIO 200 @@ -50,16 +51,18 @@ MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions"); #define aes_essiv_cbc_encrypt neon_aes_essiv_cbc_encrypt #define aes_essiv_cbc_decrypt neon_aes_essiv_cbc_decrypt #define aes_ctr_encrypt neon_aes_ctr_encrypt +#define aes_xctr_encrypt neon_aes_xctr_encrypt #define aes_xts_encrypt neon_aes_xts_encrypt #define aes_xts_decrypt neon_aes_xts_decrypt #define aes_mac_update neon_aes_mac_update -MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 NEON"); +MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS/XCTR using ARMv8 NEON"); #endif #if defined(USE_V8_CRYPTO_EXTENSIONS) || !IS_ENABLED(CONFIG_CRYPTO_AES_ARM64_BS) MODULE_ALIAS_CRYPTO("ecb(aes)"); MODULE_ALIAS_CRYPTO("cbc(aes)"); MODULE_ALIAS_CRYPTO("ctr(aes)"); MODULE_ALIAS_CRYPTO("xts(aes)"); +MODULE_ALIAS_CRYPTO("xctr(aes)"); #endif MODULE_ALIAS_CRYPTO("cts(cbc(aes))"); MODULE_ALIAS_CRYPTO("essiv(cbc(aes),sha256)"); @@ -89,6 +92,9 @@ asmlinkage void aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[], asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[], int rounds, int bytes, u8 ctr[]); +asmlinkage void aes_xctr_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int bytes, u8 ctr[], int byte_ctr); + asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[], int rounds, int bytes, u32 const rk2[], u8 iv[], int first); @@ -442,6 +448,44 @@ static int __maybe_unused essiv_cbc_decrypt(struct skcipher_request *req) return err ?: cbc_decrypt_walk(req, &walk); } +static int __maybe_unused xctr_encrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + int err, rounds = 6 + ctx->key_length / 4; + struct skcipher_walk walk; + unsigned int byte_ctr = 0; + + err = skcipher_walk_virt(&walk, req, false); + + while (walk.nbytes > 0) { + const u8 *src = walk.src.virt.addr; + unsigned int nbytes = walk.nbytes; + u8 *dst = walk.dst.virt.addr; + u8 buf[AES_BLOCK_SIZE]; + + if (unlikely(nbytes < AES_BLOCK_SIZE)) + src = dst = memcpy(buf + sizeof(buf) - nbytes, + src, nbytes); + else if (nbytes < walk.total) + nbytes &= ~(AES_BLOCK_SIZE - 1); + + kernel_neon_begin(); + aes_xctr_encrypt(dst, src, ctx->key_enc, rounds, nbytes, + walk.iv, byte_ctr); + kernel_neon_end(); + + if (unlikely(nbytes < AES_BLOCK_SIZE)) + memcpy(walk.dst.virt.addr, + buf + sizeof(buf) - nbytes, nbytes); + byte_ctr += nbytes; + + err = skcipher_walk_done(&walk, walk.nbytes - nbytes); + } + + return err; +} + static int __maybe_unused ctr_encrypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); @@ -669,6 +713,22 @@ static struct skcipher_alg aes_algs[] = { { .setkey = skcipher_aes_setkey, .encrypt = ctr_encrypt, .decrypt = ctr_encrypt, +}, { + .base = { + .cra_name = "xctr(aes)", + .cra_driver_name = "xctr-aes-" MODE, + .cra_priority = PRIO, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct crypto_aes_ctx), + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .chunksize = AES_BLOCK_SIZE, + .setkey = skcipher_aes_setkey, + .encrypt = xctr_encrypt, + .decrypt = xctr_encrypt, }, { .base = { .cra_name = "xts(aes)", diff --git a/arch/arm64/crypto/aes-modes.S b/arch/arm64/crypto/aes-modes.S index dc35eb0245c5..6c36a3b0ed7d 100644 --- a/arch/arm64/crypto/aes-modes.S +++ b/arch/arm64/crypto/aes-modes.S @@ -318,79 +318,102 @@ AES_FUNC_END(aes_cbc_cts_decrypt) .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff .previous - /* - * aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, - * int bytes, u8 ctr[]) + * This macro generates the code for CTR and XCTR mode. */ - -AES_FUNC_START(aes_ctr_encrypt) +.macro ctr_encrypt xctr stp x29, x30, [sp, #-16]! mov x29, sp enc_prepare w3, x2, x12 ld1 {vctr.16b}, [x5] - umov x12, vctr.d[1] /* keep swabbed ctr in reg */ - rev x12, x12 + .if \xctr + umov x12, vctr.d[0] + lsr w11, w6, #4 + .else + umov x12, vctr.d[1] /* keep swabbed ctr in reg */ + rev x12, x12 + .endif -.LctrloopNx: +.LctrloopNx\xctr: add w7, w4, #15 sub w4, w4, #MAX_STRIDE << 4 lsr w7, w7, #4 mov w8, #MAX_STRIDE cmp w7, w8 csel w7, w7, w8, lt - adds x12, x12, x7 + .if \xctr + add x11, x11, x7 + .else + adds x12, x12, x7 + .endif mov v0.16b, vctr.16b mov v1.16b, vctr.16b mov v2.16b, vctr.16b mov v3.16b, vctr.16b ST5( mov v4.16b, vctr.16b ) - bcs 0f + .if \xctr + sub x6, x11, #MAX_STRIDE - 1 + sub x7, x11, #MAX_STRIDE - 2 + sub x8, x11, #MAX_STRIDE - 3 + sub x9, x11, #MAX_STRIDE - 4 +ST5( sub x10, x11, #MAX_STRIDE - 5 ) + eor x6, x6, x12 + eor x7, x7, x12 + eor x8, x8, x12 + eor x9, x9, x12 +ST5( eor x10, x10, x12 ) + mov v0.d[0], x6 + mov v1.d[0], x7 + mov v2.d[0], x8 + mov v3.d[0], x9 +ST5( mov v4.d[0], x10 ) + .else + bcs 0f + .subsection 1 + /* apply carry to outgoing counter */ +0: umov x8, vctr.d[0] + rev x8, x8 + add x8, x8, #1 + rev x8, x8 + ins vctr.d[0], x8 - .subsection 1 - /* apply carry to outgoing counter */ -0: umov x8, vctr.d[0] - rev x8, x8 - add x8, x8, #1 - rev x8, x8 - ins vctr.d[0], x8 + /* apply carry to N counter blocks for N := x12 */ + cbz x12, 2f + adr x16, 1f + sub x16, x16, x12, lsl #3 + br x16 + bti c + mov v0.d[0], vctr.d[0] + bti c + mov v1.d[0], vctr.d[0] + bti c + mov v2.d[0], vctr.d[0] + bti c + mov v3.d[0], vctr.d[0] +ST5( bti c ) +ST5( mov v4.d[0], vctr.d[0] ) +1: b 2f + .previous - /* apply carry to N counter blocks for N := x12 */ - cbz x12, 2f - adr x16, 1f - sub x16, x16, x12, lsl #3 - br x16 - bti c - mov v0.d[0], vctr.d[0] - bti c - mov v1.d[0], vctr.d[0] - bti c - mov v2.d[0], vctr.d[0] - bti c - mov v3.d[0], vctr.d[0] -ST5( bti c ) -ST5( mov v4.d[0], vctr.d[0] ) -1: b 2f - .previous - -2: rev x7, x12 - ins vctr.d[1], x7 - sub x7, x12, #MAX_STRIDE - 1 - sub x8, x12, #MAX_STRIDE - 2 - sub x9, x12, #MAX_STRIDE - 3 - rev x7, x7 - rev x8, x8 - mov v1.d[1], x7 - rev x9, x9 -ST5( sub x10, x12, #MAX_STRIDE - 4 ) - mov v2.d[1], x8 -ST5( rev x10, x10 ) - mov v3.d[1], x9 -ST5( mov v4.d[1], x10 ) - tbnz w4, #31, .Lctrtail +2: rev x7, x12 + ins vctr.d[1], x7 + sub x7, x12, #MAX_STRIDE - 1 + sub x8, x12, #MAX_STRIDE - 2 + sub x9, x12, #MAX_STRIDE - 3 + rev x7, x7 + rev x8, x8 + mov v1.d[1], x7 + rev x9, x9 +ST5( sub x10, x12, #MAX_STRIDE - 4 ) + mov v2.d[1], x8 +ST5( rev x10, x10 ) + mov v3.d[1], x9 +ST5( mov v4.d[1], x10 ) + .endif + tbnz w4, #31, .Lctrtail\xctr ld1 {v5.16b-v7.16b}, [x1], #48 ST4( bl aes_encrypt_block4x ) ST5( bl aes_encrypt_block5x ) @@ -403,16 +426,17 @@ ST5( ld1 {v5.16b-v6.16b}, [x1], #32 ) ST5( eor v4.16b, v6.16b, v4.16b ) st1 {v0.16b-v3.16b}, [x0], #64 ST5( st1 {v4.16b}, [x0], #16 ) - cbz w4, .Lctrout - b .LctrloopNx + cbz w4, .Lctrout\xctr + b .LctrloopNx\xctr -.Lctrout: - st1 {vctr.16b}, [x5] /* return next CTR value */ +.Lctrout\xctr: + .if !\xctr + st1 {vctr.16b}, [x5] /* return next CTR value */ + .endif ldp x29, x30, [sp], #16 ret -.Lctrtail: - /* XOR up to MAX_STRIDE * 16 - 1 bytes of in/output with v0 ... v3/v4 */ +.Lctrtail\xctr: mov x16, #16 ands x6, x4, #0xf csel x13, x6, x16, ne @@ -427,7 +451,7 @@ ST5( csel x14, x16, xzr, gt ) adr_l x12, .Lcts_permute_table add x12, x12, x13 - ble .Lctrtail1x + ble .Lctrtail1x\xctr ST5( ld1 {v5.16b}, [x1], x14 ) ld1 {v6.16b}, [x1], x15 @@ -459,9 +483,9 @@ ST5( st1 {v5.16b}, [x0], x14 ) add x13, x13, x0 st1 {v9.16b}, [x13] // overlapping stores st1 {v8.16b}, [x0] - b .Lctrout + b .Lctrout\xctr -.Lctrtail1x: +.Lctrtail1x\xctr: sub x7, x6, #16 csel x6, x6, x7, eq add x1, x1, x6 @@ -476,9 +500,27 @@ ST5( mov v3.16b, v4.16b ) eor v5.16b, v5.16b, v3.16b bif v5.16b, v6.16b, v11.16b st1 {v5.16b}, [x0] - b .Lctrout + b .Lctrout\xctr +.endm + + /* + * aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, + * int bytes, u8 ctr[]) + */ + +AES_FUNC_START(aes_ctr_encrypt) + ctr_encrypt 0 AES_FUNC_END(aes_ctr_encrypt) + /* + * aes_xctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, + * int bytes, u8 const iv[], int byte_ctr) + */ + +AES_FUNC_START(aes_xctr_encrypt) + ctr_encrypt 1 +AES_FUNC_END(aes_xctr_encrypt) + /* * aes_xts_encrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds, From c0eb7591c1ed9cbdb0ad796bb56aed13748b55fa Mon Sep 17 00:00:00 2001 From: Nathan Huckleberry Date: Fri, 20 May 2022 18:14:58 +0000 Subject: [PATCH 013/318] crypto: arm64/aes-xctr - Improve readability of XCTR and CTR modes Added some clarifying comments, changed the register allocations to make the code clearer, and added register aliases. Signed-off-by: Nathan Huckleberry Reviewed-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm64/crypto/aes-glue.c | 16 +++ arch/arm64/crypto/aes-modes.S | 237 ++++++++++++++++++++++++---------- 2 files changed, 185 insertions(+), 68 deletions(-) diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c index b6883288234c..162787c7aa86 100644 --- a/arch/arm64/crypto/aes-glue.c +++ b/arch/arm64/crypto/aes-glue.c @@ -464,6 +464,14 @@ static int __maybe_unused xctr_encrypt(struct skcipher_request *req) u8 *dst = walk.dst.virt.addr; u8 buf[AES_BLOCK_SIZE]; + /* + * If given less than 16 bytes, we must copy the partial block + * into a temporary buffer of 16 bytes to avoid out of bounds + * reads and writes. Furthermore, this code is somewhat unusual + * in that it expects the end of the data to be at the end of + * the temporary buffer, rather than the start of the data at + * the start of the temporary buffer. + */ if (unlikely(nbytes < AES_BLOCK_SIZE)) src = dst = memcpy(buf + sizeof(buf) - nbytes, src, nbytes); @@ -501,6 +509,14 @@ static int __maybe_unused ctr_encrypt(struct skcipher_request *req) u8 *dst = walk.dst.virt.addr; u8 buf[AES_BLOCK_SIZE]; + /* + * If given less than 16 bytes, we must copy the partial block + * into a temporary buffer of 16 bytes to avoid out of bounds + * reads and writes. Furthermore, this code is somewhat unusual + * in that it expects the end of the data to be at the end of + * the temporary buffer, rather than the start of the data at + * the start of the temporary buffer. + */ if (unlikely(nbytes < AES_BLOCK_SIZE)) src = dst = memcpy(buf + sizeof(buf) - nbytes, src, nbytes); diff --git a/arch/arm64/crypto/aes-modes.S b/arch/arm64/crypto/aes-modes.S index 6c36a3b0ed7d..5abc834271f4 100644 --- a/arch/arm64/crypto/aes-modes.S +++ b/arch/arm64/crypto/aes-modes.S @@ -322,32 +322,60 @@ AES_FUNC_END(aes_cbc_cts_decrypt) * This macro generates the code for CTR and XCTR mode. */ .macro ctr_encrypt xctr + // Arguments + OUT .req x0 + IN .req x1 + KEY .req x2 + ROUNDS_W .req w3 + BYTES_W .req w4 + IV .req x5 + BYTE_CTR_W .req w6 // XCTR only + // Intermediate values + CTR_W .req w11 // XCTR only + CTR .req x11 // XCTR only + IV_PART .req x12 + BLOCKS .req x13 + BLOCKS_W .req w13 + stp x29, x30, [sp, #-16]! mov x29, sp - enc_prepare w3, x2, x12 - ld1 {vctr.16b}, [x5] + enc_prepare ROUNDS_W, KEY, IV_PART + ld1 {vctr.16b}, [IV] + /* + * Keep 64 bits of the IV in a register. For CTR mode this lets us + * easily increment the IV. For XCTR mode this lets us efficiently XOR + * the 64-bit counter with the IV. + */ .if \xctr - umov x12, vctr.d[0] - lsr w11, w6, #4 + umov IV_PART, vctr.d[0] + lsr CTR_W, BYTE_CTR_W, #4 .else - umov x12, vctr.d[1] /* keep swabbed ctr in reg */ - rev x12, x12 + umov IV_PART, vctr.d[1] + rev IV_PART, IV_PART .endif .LctrloopNx\xctr: - add w7, w4, #15 - sub w4, w4, #MAX_STRIDE << 4 - lsr w7, w7, #4 + add BLOCKS_W, BYTES_W, #15 + sub BYTES_W, BYTES_W, #MAX_STRIDE << 4 + lsr BLOCKS_W, BLOCKS_W, #4 mov w8, #MAX_STRIDE - cmp w7, w8 - csel w7, w7, w8, lt + cmp BLOCKS_W, w8 + csel BLOCKS_W, BLOCKS_W, w8, lt + /* + * Set up the counter values in v0-v{MAX_STRIDE-1}. + * + * If we are encrypting less than MAX_STRIDE blocks, the tail block + * handling code expects the last keystream block to be in + * v{MAX_STRIDE-1}. For example: if encrypting two blocks with + * MAX_STRIDE=5, then v3 and v4 should have the next two counter blocks. + */ .if \xctr - add x11, x11, x7 + add CTR, CTR, BLOCKS .else - adds x12, x12, x7 + adds IV_PART, IV_PART, BLOCKS .endif mov v0.16b, vctr.16b mov v1.16b, vctr.16b @@ -355,16 +383,16 @@ AES_FUNC_END(aes_cbc_cts_decrypt) mov v3.16b, vctr.16b ST5( mov v4.16b, vctr.16b ) .if \xctr - sub x6, x11, #MAX_STRIDE - 1 - sub x7, x11, #MAX_STRIDE - 2 - sub x8, x11, #MAX_STRIDE - 3 - sub x9, x11, #MAX_STRIDE - 4 -ST5( sub x10, x11, #MAX_STRIDE - 5 ) - eor x6, x6, x12 - eor x7, x7, x12 - eor x8, x8, x12 - eor x9, x9, x12 -ST5( eor x10, x10, x12 ) + sub x6, CTR, #MAX_STRIDE - 1 + sub x7, CTR, #MAX_STRIDE - 2 + sub x8, CTR, #MAX_STRIDE - 3 + sub x9, CTR, #MAX_STRIDE - 4 +ST5( sub x10, CTR, #MAX_STRIDE - 5 ) + eor x6, x6, IV_PART + eor x7, x7, IV_PART + eor x8, x8, IV_PART + eor x9, x9, IV_PART +ST5( eor x10, x10, IV_PART ) mov v0.d[0], x6 mov v1.d[0], x7 mov v2.d[0], x8 @@ -373,17 +401,32 @@ ST5( mov v4.d[0], x10 ) .else bcs 0f .subsection 1 - /* apply carry to outgoing counter */ + /* + * This subsection handles carries. + * + * Conditional branching here is allowed with respect to time + * invariance since the branches are dependent on the IV instead + * of the plaintext or key. This code is rarely executed in + * practice anyway. + */ + + /* Apply carry to outgoing counter. */ 0: umov x8, vctr.d[0] rev x8, x8 add x8, x8, #1 rev x8, x8 ins vctr.d[0], x8 - /* apply carry to N counter blocks for N := x12 */ - cbz x12, 2f + /* + * Apply carry to counter blocks if needed. + * + * Since the carry flag was set, we know 0 <= IV_PART < + * MAX_STRIDE. Using the value of IV_PART we can determine how + * many counter blocks need to be updated. + */ + cbz IV_PART, 2f adr x16, 1f - sub x16, x16, x12, lsl #3 + sub x16, x16, IV_PART, lsl #3 br x16 bti c mov v0.d[0], vctr.d[0] @@ -398,71 +441,88 @@ ST5( mov v4.d[0], vctr.d[0] ) 1: b 2f .previous -2: rev x7, x12 +2: rev x7, IV_PART ins vctr.d[1], x7 - sub x7, x12, #MAX_STRIDE - 1 - sub x8, x12, #MAX_STRIDE - 2 - sub x9, x12, #MAX_STRIDE - 3 + sub x7, IV_PART, #MAX_STRIDE - 1 + sub x8, IV_PART, #MAX_STRIDE - 2 + sub x9, IV_PART, #MAX_STRIDE - 3 rev x7, x7 rev x8, x8 mov v1.d[1], x7 rev x9, x9 -ST5( sub x10, x12, #MAX_STRIDE - 4 ) +ST5( sub x10, IV_PART, #MAX_STRIDE - 4 ) mov v2.d[1], x8 ST5( rev x10, x10 ) mov v3.d[1], x9 ST5( mov v4.d[1], x10 ) .endif - tbnz w4, #31, .Lctrtail\xctr - ld1 {v5.16b-v7.16b}, [x1], #48 + + /* + * If there are at least MAX_STRIDE blocks left, XOR the data with + * keystream and store. Otherwise jump to tail handling. + */ + tbnz BYTES_W, #31, .Lctrtail\xctr + ld1 {v5.16b-v7.16b}, [IN], #48 ST4( bl aes_encrypt_block4x ) ST5( bl aes_encrypt_block5x ) eor v0.16b, v5.16b, v0.16b -ST4( ld1 {v5.16b}, [x1], #16 ) +ST4( ld1 {v5.16b}, [IN], #16 ) eor v1.16b, v6.16b, v1.16b -ST5( ld1 {v5.16b-v6.16b}, [x1], #32 ) +ST5( ld1 {v5.16b-v6.16b}, [IN], #32 ) eor v2.16b, v7.16b, v2.16b eor v3.16b, v5.16b, v3.16b ST5( eor v4.16b, v6.16b, v4.16b ) - st1 {v0.16b-v3.16b}, [x0], #64 -ST5( st1 {v4.16b}, [x0], #16 ) - cbz w4, .Lctrout\xctr + st1 {v0.16b-v3.16b}, [OUT], #64 +ST5( st1 {v4.16b}, [OUT], #16 ) + cbz BYTES_W, .Lctrout\xctr b .LctrloopNx\xctr .Lctrout\xctr: .if !\xctr - st1 {vctr.16b}, [x5] /* return next CTR value */ + st1 {vctr.16b}, [IV] /* return next CTR value */ .endif ldp x29, x30, [sp], #16 ret .Lctrtail\xctr: + /* + * Handle up to MAX_STRIDE * 16 - 1 bytes of plaintext + * + * This code expects the last keystream block to be in v{MAX_STRIDE-1}. + * For example: if encrypting two blocks with MAX_STRIDE=5, then v3 and + * v4 should have the next two counter blocks. + * + * This allows us to store the ciphertext by writing to overlapping + * regions of memory. Any invalid ciphertext blocks get overwritten by + * correctly computed blocks. This approach greatly simplifies the + * logic for storing the ciphertext. + */ mov x16, #16 - ands x6, x4, #0xf - csel x13, x6, x16, ne + ands w7, BYTES_W, #0xf + csel x13, x7, x16, ne -ST5( cmp w4, #64 - (MAX_STRIDE << 4) ) +ST5( cmp BYTES_W, #64 - (MAX_STRIDE << 4)) ST5( csel x14, x16, xzr, gt ) - cmp w4, #48 - (MAX_STRIDE << 4) + cmp BYTES_W, #48 - (MAX_STRIDE << 4) csel x15, x16, xzr, gt - cmp w4, #32 - (MAX_STRIDE << 4) + cmp BYTES_W, #32 - (MAX_STRIDE << 4) csel x16, x16, xzr, gt - cmp w4, #16 - (MAX_STRIDE << 4) + cmp BYTES_W, #16 - (MAX_STRIDE << 4) - adr_l x12, .Lcts_permute_table - add x12, x12, x13 + adr_l x9, .Lcts_permute_table + add x9, x9, x13 ble .Lctrtail1x\xctr -ST5( ld1 {v5.16b}, [x1], x14 ) - ld1 {v6.16b}, [x1], x15 - ld1 {v7.16b}, [x1], x16 +ST5( ld1 {v5.16b}, [IN], x14 ) + ld1 {v6.16b}, [IN], x15 + ld1 {v7.16b}, [IN], x16 ST4( bl aes_encrypt_block4x ) ST5( bl aes_encrypt_block5x ) - ld1 {v8.16b}, [x1], x13 - ld1 {v9.16b}, [x1] - ld1 {v10.16b}, [x12] + ld1 {v8.16b}, [IN], x13 + ld1 {v9.16b}, [IN] + ld1 {v10.16b}, [x9] ST4( eor v6.16b, v6.16b, v0.16b ) ST4( eor v7.16b, v7.16b, v1.16b ) @@ -477,35 +537,70 @@ ST5( eor v7.16b, v7.16b, v2.16b ) ST5( eor v8.16b, v8.16b, v3.16b ) ST5( eor v9.16b, v9.16b, v4.16b ) -ST5( st1 {v5.16b}, [x0], x14 ) - st1 {v6.16b}, [x0], x15 - st1 {v7.16b}, [x0], x16 - add x13, x13, x0 +ST5( st1 {v5.16b}, [OUT], x14 ) + st1 {v6.16b}, [OUT], x15 + st1 {v7.16b}, [OUT], x16 + add x13, x13, OUT st1 {v9.16b}, [x13] // overlapping stores - st1 {v8.16b}, [x0] + st1 {v8.16b}, [OUT] b .Lctrout\xctr .Lctrtail1x\xctr: - sub x7, x6, #16 - csel x6, x6, x7, eq - add x1, x1, x6 - add x0, x0, x6 - ld1 {v5.16b}, [x1] - ld1 {v6.16b}, [x0] + /* + * Handle <= 16 bytes of plaintext + * + * This code always reads and writes 16 bytes. To avoid out of bounds + * accesses, XCTR and CTR modes must use a temporary buffer when + * encrypting/decrypting less than 16 bytes. + * + * This code is unusual in that it loads the input and stores the output + * relative to the end of the buffers rather than relative to the start. + * This causes unusual behaviour when encrypting/decrypting less than 16 + * bytes; the end of the data is expected to be at the end of the + * temporary buffer rather than the start of the data being at the start + * of the temporary buffer. + */ + sub x8, x7, #16 + csel x7, x7, x8, eq + add IN, IN, x7 + add OUT, OUT, x7 + ld1 {v5.16b}, [IN] + ld1 {v6.16b}, [OUT] ST5( mov v3.16b, v4.16b ) - encrypt_block v3, w3, x2, x8, w7 - ld1 {v10.16b-v11.16b}, [x12] + encrypt_block v3, ROUNDS_W, KEY, x8, w7 + ld1 {v10.16b-v11.16b}, [x9] tbl v3.16b, {v3.16b}, v10.16b sshr v11.16b, v11.16b, #7 eor v5.16b, v5.16b, v3.16b bif v5.16b, v6.16b, v11.16b - st1 {v5.16b}, [x0] + st1 {v5.16b}, [OUT] b .Lctrout\xctr + + // Arguments + .unreq OUT + .unreq IN + .unreq KEY + .unreq ROUNDS_W + .unreq BYTES_W + .unreq IV + .unreq BYTE_CTR_W // XCTR only + // Intermediate values + .unreq CTR_W // XCTR only + .unreq CTR // XCTR only + .unreq IV_PART + .unreq BLOCKS + .unreq BLOCKS_W .endm /* * aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, * int bytes, u8 ctr[]) + * + * The input and output buffers must always be at least 16 bytes even if + * encrypting/decrypting less than 16 bytes. Otherwise out of bounds + * accesses will occur. The data to be encrypted/decrypted is expected + * to be at the end of this 16-byte temporary buffer rather than the + * start. */ AES_FUNC_START(aes_ctr_encrypt) @@ -515,6 +610,12 @@ AES_FUNC_END(aes_ctr_encrypt) /* * aes_xctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, * int bytes, u8 const iv[], int byte_ctr) + * + * The input and output buffers must always be at least 16 bytes even if + * encrypting/decrypting less than 16 bytes. Otherwise out of bounds + * accesses will occur. The data to be encrypted/decrypted is expected + * to be at the end of this 16-byte temporary buffer rather than the + * start. */ AES_FUNC_START(aes_xctr_encrypt) From 34f7f6c3011276313383099156be287ac745bcea Mon Sep 17 00:00:00 2001 From: Nathan Huckleberry Date: Fri, 20 May 2022 18:14:59 +0000 Subject: [PATCH 014/318] crypto: x86/polyval - Add PCLMULQDQ accelerated implementation of POLYVAL Add hardware accelerated version of POLYVAL for x86-64 CPUs with PCLMULQDQ support. This implementation is accelerated using PCLMULQDQ instructions to perform the finite field computations. For added efficiency, 8 blocks of the message are processed simultaneously by precomputing the first 8 powers of the key. Schoolbook multiplication is used instead of Karatsuba multiplication because it was found to be slightly faster on x86-64 machines. Montgomery reduction must be used instead of Barrett reduction due to the difference in modulus between POLYVAL's field and other finite fields. More information on POLYVAL can be found in the HCTR2 paper: "Length-preserving encryption with HCTR2": https://eprint.iacr.org/2021/1441.pdf Signed-off-by: Nathan Huckleberry Reviewed-by: Ard Biesheuvel Reviewed-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/Makefile | 3 + arch/x86/crypto/polyval-clmulni_asm.S | 321 +++++++++++++++++++++++++ arch/x86/crypto/polyval-clmulni_glue.c | 203 ++++++++++++++++ crypto/Kconfig | 9 + crypto/polyval-generic.c | 40 +++ include/crypto/polyval.h | 5 + 6 files changed, 581 insertions(+) create mode 100644 arch/x86/crypto/polyval-clmulni_asm.S create mode 100644 arch/x86/crypto/polyval-clmulni_glue.c diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index 2831685adf6f..b9847152acd8 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -69,6 +69,9 @@ libblake2s-x86_64-y := blake2s-core.o blake2s-glue.o obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o +obj-$(CONFIG_CRYPTO_POLYVAL_CLMUL_NI) += polyval-clmulni.o +polyval-clmulni-y := polyval-clmulni_asm.o polyval-clmulni_glue.o + obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o crc32c-intel-y := crc32c-intel_glue.o crc32c-intel-$(CONFIG_64BIT) += crc32c-pcl-intel-asm_64.o diff --git a/arch/x86/crypto/polyval-clmulni_asm.S b/arch/x86/crypto/polyval-clmulni_asm.S new file mode 100644 index 000000000000..a6ebe4e7dd2b --- /dev/null +++ b/arch/x86/crypto/polyval-clmulni_asm.S @@ -0,0 +1,321 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright 2021 Google LLC + */ +/* + * This is an efficient implementation of POLYVAL using intel PCLMULQDQ-NI + * instructions. It works on 8 blocks at a time, by precomputing the first 8 + * keys powers h^8, ..., h^1 in the POLYVAL finite field. This precomputation + * allows us to split finite field multiplication into two steps. + * + * In the first step, we consider h^i, m_i as normal polynomials of degree less + * than 128. We then compute p(x) = h^8m_0 + ... + h^1m_7 where multiplication + * is simply polynomial multiplication. + * + * In the second step, we compute the reduction of p(x) modulo the finite field + * modulus g(x) = x^128 + x^127 + x^126 + x^121 + 1. + * + * This two step process is equivalent to computing h^8m_0 + ... + h^1m_7 where + * multiplication is finite field multiplication. The advantage is that the + * two-step process only requires 1 finite field reduction for every 8 + * polynomial multiplications. Further parallelism is gained by interleaving the + * multiplications and polynomial reductions. + */ + +#include +#include + +#define STRIDE_BLOCKS 8 + +#define GSTAR %xmm7 +#define PL %xmm8 +#define PH %xmm9 +#define TMP_XMM %xmm11 +#define LO %xmm12 +#define HI %xmm13 +#define MI %xmm14 +#define SUM %xmm15 + +#define KEY_POWERS %rdi +#define MSG %rsi +#define BLOCKS_LEFT %rdx +#define ACCUMULATOR %rcx +#define TMP %rax + +.section .rodata.cst16.gstar, "aM", @progbits, 16 +.align 16 + +.Lgstar: + .quad 0xc200000000000000, 0xc200000000000000 + +.text + +/* + * Performs schoolbook1_iteration on two lists of 128-bit polynomials of length + * count pointed to by MSG and KEY_POWERS. + */ +.macro schoolbook1 count + .set i, 0 + .rept (\count) + schoolbook1_iteration i 0 + .set i, (i +1) + .endr +.endm + +/* + * Computes the product of two 128-bit polynomials at the memory locations + * specified by (MSG + 16*i) and (KEY_POWERS + 16*i) and XORs the components of + * the 256-bit product into LO, MI, HI. + * + * Given: + * X = [X_1 : X_0] + * Y = [Y_1 : Y_0] + * + * We compute: + * LO += X_0 * Y_0 + * MI += X_0 * Y_1 + X_1 * Y_0 + * HI += X_1 * Y_1 + * + * Later, the 256-bit result can be extracted as: + * [HI_1 : HI_0 + MI_1 : LO_1 + MI_0 : LO_0] + * This step is done when computing the polynomial reduction for efficiency + * reasons. + * + * If xor_sum == 1, then also XOR the value of SUM into m_0. This avoids an + * extra multiplication of SUM and h^8. + */ +.macro schoolbook1_iteration i xor_sum + movups (16*\i)(MSG), %xmm0 + .if (\i == 0 && \xor_sum == 1) + pxor SUM, %xmm0 + .endif + vpclmulqdq $0x01, (16*\i)(KEY_POWERS), %xmm0, %xmm2 + vpclmulqdq $0x00, (16*\i)(KEY_POWERS), %xmm0, %xmm1 + vpclmulqdq $0x10, (16*\i)(KEY_POWERS), %xmm0, %xmm3 + vpclmulqdq $0x11, (16*\i)(KEY_POWERS), %xmm0, %xmm4 + vpxor %xmm2, MI, MI + vpxor %xmm1, LO, LO + vpxor %xmm4, HI, HI + vpxor %xmm3, MI, MI +.endm + +/* + * Performs the same computation as schoolbook1_iteration, except we expect the + * arguments to already be loaded into xmm0 and xmm1 and we set the result + * registers LO, MI, and HI directly rather than XOR'ing into them. + */ +.macro schoolbook1_noload + vpclmulqdq $0x01, %xmm0, %xmm1, MI + vpclmulqdq $0x10, %xmm0, %xmm1, %xmm2 + vpclmulqdq $0x00, %xmm0, %xmm1, LO + vpclmulqdq $0x11, %xmm0, %xmm1, HI + vpxor %xmm2, MI, MI +.endm + +/* + * Computes the 256-bit polynomial represented by LO, HI, MI. Stores + * the result in PL, PH. + * [PH : PL] = [HI_1 : HI_0 + MI_1 : LO_1 + MI_0 : LO_0] + */ +.macro schoolbook2 + vpslldq $8, MI, PL + vpsrldq $8, MI, PH + pxor LO, PL + pxor HI, PH +.endm + +/* + * Computes the 128-bit reduction of PH : PL. Stores the result in dest. + * + * This macro computes p(x) mod g(x) where p(x) is in montgomery form and g(x) = + * x^128 + x^127 + x^126 + x^121 + 1. + * + * We have a 256-bit polynomial PH : PL = P_3 : P_2 : P_1 : P_0 that is the + * product of two 128-bit polynomials in Montgomery form. We need to reduce it + * mod g(x). Also, since polynomials in Montgomery form have an "extra" factor + * of x^128, this product has two extra factors of x^128. To get it back into + * Montgomery form, we need to remove one of these factors by dividing by x^128. + * + * To accomplish both of these goals, we add multiples of g(x) that cancel out + * the low 128 bits P_1 : P_0, leaving just the high 128 bits. Since the low + * bits are zero, the polynomial division by x^128 can be done by right shifting. + * + * Since the only nonzero term in the low 64 bits of g(x) is the constant term, + * the multiple of g(x) needed to cancel out P_0 is P_0 * g(x). The CPU can + * only do 64x64 bit multiplications, so split P_0 * g(x) into x^128 * P_0 + + * x^64 * g*(x) * P_0 + P_0, where g*(x) is bits 64-127 of g(x). Adding this to + * the original polynomial gives P_3 : P_2 + P_0 + T_1 : P_1 + T_0 : 0, where T + * = T_1 : T_0 = g*(x) * P_0. Thus, bits 0-63 got "folded" into bits 64-191. + * + * Repeating this same process on the next 64 bits "folds" bits 64-127 into bits + * 128-255, giving the answer in bits 128-255. This time, we need to cancel P_1 + * + T_0 in bits 64-127. The multiple of g(x) required is (P_1 + T_0) * g(x) * + * x^64. Adding this to our previous computation gives P_3 + P_1 + T_0 + V_1 : + * P_2 + P_0 + T_1 + V_0 : 0 : 0, where V = V_1 : V_0 = g*(x) * (P_1 + T_0). + * + * So our final computation is: + * T = T_1 : T_0 = g*(x) * P_0 + * V = V_1 : V_0 = g*(x) * (P_1 + T_0) + * p(x) / x^{128} mod g(x) = P_3 + P_1 + T_0 + V_1 : P_2 + P_0 + T_1 + V_0 + * + * The implementation below saves a XOR instruction by computing P_1 + T_0 : P_0 + * + T_1 and XORing into dest, rather than separately XORing P_1 : P_0 and T_0 : + * T_1 into dest. This allows us to reuse P_1 + T_0 when computing V. + */ +.macro montgomery_reduction dest + vpclmulqdq $0x00, PL, GSTAR, TMP_XMM # TMP_XMM = T_1 : T_0 = P_0 * g*(x) + pshufd $0b01001110, TMP_XMM, TMP_XMM # TMP_XMM = T_0 : T_1 + pxor PL, TMP_XMM # TMP_XMM = P_1 + T_0 : P_0 + T_1 + pxor TMP_XMM, PH # PH = P_3 + P_1 + T_0 : P_2 + P_0 + T_1 + pclmulqdq $0x11, GSTAR, TMP_XMM # TMP_XMM = V_1 : V_0 = V = [(P_1 + T_0) * g*(x)] + vpxor TMP_XMM, PH, \dest +.endm + +/* + * Compute schoolbook multiplication for 8 blocks + * m_0h^8 + ... + m_7h^1 + * + * If reduce is set, also computes the montgomery reduction of the + * previous full_stride call and XORs with the first message block. + * (m_0 + REDUCE(PL, PH))h^8 + ... + m_7h^1. + * I.e., the first multiplication uses m_0 + REDUCE(PL, PH) instead of m_0. + */ +.macro full_stride reduce + pxor LO, LO + pxor HI, HI + pxor MI, MI + + schoolbook1_iteration 7 0 + .if \reduce + vpclmulqdq $0x00, PL, GSTAR, TMP_XMM + .endif + + schoolbook1_iteration 6 0 + .if \reduce + pshufd $0b01001110, TMP_XMM, TMP_XMM + .endif + + schoolbook1_iteration 5 0 + .if \reduce + pxor PL, TMP_XMM + .endif + + schoolbook1_iteration 4 0 + .if \reduce + pxor TMP_XMM, PH + .endif + + schoolbook1_iteration 3 0 + .if \reduce + pclmulqdq $0x11, GSTAR, TMP_XMM + .endif + + schoolbook1_iteration 2 0 + .if \reduce + vpxor TMP_XMM, PH, SUM + .endif + + schoolbook1_iteration 1 0 + + schoolbook1_iteration 0 1 + + addq $(8*16), MSG + schoolbook2 +.endm + +/* + * Process BLOCKS_LEFT blocks, where 0 < BLOCKS_LEFT < STRIDE_BLOCKS + */ +.macro partial_stride + mov BLOCKS_LEFT, TMP + shlq $4, TMP + addq $(16*STRIDE_BLOCKS), KEY_POWERS + subq TMP, KEY_POWERS + + movups (MSG), %xmm0 + pxor SUM, %xmm0 + movaps (KEY_POWERS), %xmm1 + schoolbook1_noload + dec BLOCKS_LEFT + addq $16, MSG + addq $16, KEY_POWERS + + test $4, BLOCKS_LEFT + jz .Lpartial4BlocksDone + schoolbook1 4 + addq $(4*16), MSG + addq $(4*16), KEY_POWERS +.Lpartial4BlocksDone: + test $2, BLOCKS_LEFT + jz .Lpartial2BlocksDone + schoolbook1 2 + addq $(2*16), MSG + addq $(2*16), KEY_POWERS +.Lpartial2BlocksDone: + test $1, BLOCKS_LEFT + jz .LpartialDone + schoolbook1 1 +.LpartialDone: + schoolbook2 + montgomery_reduction SUM +.endm + +/* + * Perform montgomery multiplication in GF(2^128) and store result in op1. + * + * Computes op1*op2*x^{-128} mod x^128 + x^127 + x^126 + x^121 + 1 + * If op1, op2 are in montgomery form, this computes the montgomery + * form of op1*op2. + * + * void clmul_polyval_mul(u8 *op1, const u8 *op2); + */ +SYM_FUNC_START(clmul_polyval_mul) + FRAME_BEGIN + vmovdqa .Lgstar(%rip), GSTAR + movups (%rdi), %xmm0 + movups (%rsi), %xmm1 + schoolbook1_noload + schoolbook2 + montgomery_reduction SUM + movups SUM, (%rdi) + FRAME_END + RET +SYM_FUNC_END(clmul_polyval_mul) + +/* + * Perform polynomial evaluation as specified by POLYVAL. This computes: + * h^n * accumulator + h^n * m_0 + ... + h^1 * m_{n-1} + * where n=nblocks, h is the hash key, and m_i are the message blocks. + * + * rdi - pointer to precomputed key powers h^8 ... h^1 + * rsi - pointer to message blocks + * rdx - number of blocks to hash + * rcx - pointer to the accumulator + * + * void clmul_polyval_update(const struct polyval_tfm_ctx *keys, + * const u8 *in, size_t nblocks, u8 *accumulator); + */ +SYM_FUNC_START(clmul_polyval_update) + FRAME_BEGIN + vmovdqa .Lgstar(%rip), GSTAR + movups (ACCUMULATOR), SUM + subq $STRIDE_BLOCKS, BLOCKS_LEFT + js .LstrideLoopExit + full_stride 0 + subq $STRIDE_BLOCKS, BLOCKS_LEFT + js .LstrideLoopExitReduce +.LstrideLoop: + full_stride 1 + subq $STRIDE_BLOCKS, BLOCKS_LEFT + jns .LstrideLoop +.LstrideLoopExitReduce: + montgomery_reduction SUM +.LstrideLoopExit: + add $STRIDE_BLOCKS, BLOCKS_LEFT + jz .LskipPartial + partial_stride +.LskipPartial: + movups SUM, (ACCUMULATOR) + FRAME_END + RET +SYM_FUNC_END(clmul_polyval_update) diff --git a/arch/x86/crypto/polyval-clmulni_glue.c b/arch/x86/crypto/polyval-clmulni_glue.c new file mode 100644 index 000000000000..b7664d018851 --- /dev/null +++ b/arch/x86/crypto/polyval-clmulni_glue.c @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Glue code for POLYVAL using PCMULQDQ-NI + * + * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen + * Copyright (c) 2009 Intel Corp. + * Author: Huang Ying + * Copyright 2021 Google LLC + */ + +/* + * Glue code based on ghash-clmulni-intel_glue.c. + * + * This implementation of POLYVAL uses montgomery multiplication + * accelerated by PCLMULQDQ-NI to implement the finite field + * operations. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define NUM_KEY_POWERS 8 + +struct polyval_tfm_ctx { + /* + * These powers must be in the order h^8, ..., h^1. + */ + u8 key_powers[NUM_KEY_POWERS][POLYVAL_BLOCK_SIZE]; +}; + +struct polyval_desc_ctx { + u8 buffer[POLYVAL_BLOCK_SIZE]; + u32 bytes; +}; + +asmlinkage void clmul_polyval_update(const struct polyval_tfm_ctx *keys, + const u8 *in, size_t nblocks, u8 *accumulator); +asmlinkage void clmul_polyval_mul(u8 *op1, const u8 *op2); + +static void internal_polyval_update(const struct polyval_tfm_ctx *keys, + const u8 *in, size_t nblocks, u8 *accumulator) +{ + if (likely(crypto_simd_usable())) { + kernel_fpu_begin(); + clmul_polyval_update(keys, in, nblocks, accumulator); + kernel_fpu_end(); + } else { + polyval_update_non4k(keys->key_powers[NUM_KEY_POWERS-1], in, + nblocks, accumulator); + } +} + +static void internal_polyval_mul(u8 *op1, const u8 *op2) +{ + if (likely(crypto_simd_usable())) { + kernel_fpu_begin(); + clmul_polyval_mul(op1, op2); + kernel_fpu_end(); + } else { + polyval_mul_non4k(op1, op2); + } +} + +static int polyval_x86_setkey(struct crypto_shash *tfm, + const u8 *key, unsigned int keylen) +{ + struct polyval_tfm_ctx *tctx = crypto_shash_ctx(tfm); + int i; + + if (keylen != POLYVAL_BLOCK_SIZE) + return -EINVAL; + + memcpy(tctx->key_powers[NUM_KEY_POWERS-1], key, POLYVAL_BLOCK_SIZE); + + for (i = NUM_KEY_POWERS-2; i >= 0; i--) { + memcpy(tctx->key_powers[i], key, POLYVAL_BLOCK_SIZE); + internal_polyval_mul(tctx->key_powers[i], + tctx->key_powers[i+1]); + } + + return 0; +} + +static int polyval_x86_init(struct shash_desc *desc) +{ + struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); + + memset(dctx, 0, sizeof(*dctx)); + + return 0; +} + +static int polyval_x86_update(struct shash_desc *desc, + const u8 *src, unsigned int srclen) +{ + struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); + const struct polyval_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); + u8 *pos; + unsigned int nblocks; + unsigned int n; + + if (dctx->bytes) { + n = min(srclen, dctx->bytes); + pos = dctx->buffer + POLYVAL_BLOCK_SIZE - dctx->bytes; + + dctx->bytes -= n; + srclen -= n; + + while (n--) + *pos++ ^= *src++; + + if (!dctx->bytes) + internal_polyval_mul(dctx->buffer, + tctx->key_powers[NUM_KEY_POWERS-1]); + } + + while (srclen >= POLYVAL_BLOCK_SIZE) { + /* Allow rescheduling every 4K bytes. */ + nblocks = min(srclen, 4096U) / POLYVAL_BLOCK_SIZE; + internal_polyval_update(tctx, src, nblocks, dctx->buffer); + srclen -= nblocks * POLYVAL_BLOCK_SIZE; + src += nblocks * POLYVAL_BLOCK_SIZE; + } + + if (srclen) { + dctx->bytes = POLYVAL_BLOCK_SIZE - srclen; + pos = dctx->buffer; + while (srclen--) + *pos++ ^= *src++; + } + + return 0; +} + +static int polyval_x86_final(struct shash_desc *desc, u8 *dst) +{ + struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); + const struct polyval_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); + + if (dctx->bytes) { + internal_polyval_mul(dctx->buffer, + tctx->key_powers[NUM_KEY_POWERS-1]); + } + + memcpy(dst, dctx->buffer, POLYVAL_BLOCK_SIZE); + + return 0; +} + +static struct shash_alg polyval_alg = { + .digestsize = POLYVAL_DIGEST_SIZE, + .init = polyval_x86_init, + .update = polyval_x86_update, + .final = polyval_x86_final, + .setkey = polyval_x86_setkey, + .descsize = sizeof(struct polyval_desc_ctx), + .base = { + .cra_name = "polyval", + .cra_driver_name = "polyval-clmulni", + .cra_priority = 200, + .cra_blocksize = POLYVAL_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct polyval_tfm_ctx), + .cra_module = THIS_MODULE, + }, +}; + +__maybe_unused static const struct x86_cpu_id pcmul_cpu_id[] = { + X86_MATCH_FEATURE(X86_FEATURE_PCLMULQDQ, NULL), + {} +}; +MODULE_DEVICE_TABLE(x86cpu, pcmul_cpu_id); + +static int __init polyval_clmulni_mod_init(void) +{ + if (!x86_match_cpu(pcmul_cpu_id)) + return -ENODEV; + + if (!boot_cpu_has(X86_FEATURE_AVX)) + return -ENODEV; + + return crypto_register_shash(&polyval_alg); +} + +static void __exit polyval_clmulni_mod_exit(void) +{ + crypto_unregister_shash(&polyval_alg); +} + +module_init(polyval_clmulni_mod_init); +module_exit(polyval_clmulni_mod_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("POLYVAL hash function accelerated by PCLMULQDQ-NI"); +MODULE_ALIAS_CRYPTO("polyval"); +MODULE_ALIAS_CRYPTO("polyval-clmulni"); diff --git a/crypto/Kconfig b/crypto/Kconfig index dfcc3235e918..9b654984de79 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -792,6 +792,15 @@ config CRYPTO_POLYVAL POLYVAL is the hash function used in HCTR2. It is not a general-purpose cryptographic hash function. +config CRYPTO_POLYVAL_CLMUL_NI + tristate "POLYVAL hash function (CLMUL-NI accelerated)" + depends on X86 && 64BIT + select CRYPTO_POLYVAL + help + This is the x86_64 CLMUL-NI accelerated implementation of POLYVAL. It is + used to efficiently implement HCTR2 on x86-64 processors that support + carry-less multiplication instructions. + config CRYPTO_POLY1305 tristate "Poly1305 authenticator algorithm" select CRYPTO_HASH diff --git a/crypto/polyval-generic.c b/crypto/polyval-generic.c index bf2b03b7bfc0..16bfa6925b31 100644 --- a/crypto/polyval-generic.c +++ b/crypto/polyval-generic.c @@ -76,6 +76,46 @@ static void copy_and_reverse(u8 dst[POLYVAL_BLOCK_SIZE], put_unaligned(swab64(b), (u64 *)&dst[0]); } +/* + * Performs multiplication in the POLYVAL field using the GHASH field as a + * subroutine. This function is used as a fallback for hardware accelerated + * implementations when simd registers are unavailable. + * + * Note: This function is not used for polyval-generic, instead we use the 4k + * lookup table implementation for finite field multiplication. + */ +void polyval_mul_non4k(u8 *op1, const u8 *op2) +{ + be128 a, b; + + // Assume one argument is in Montgomery form and one is not. + copy_and_reverse((u8 *)&a, op1); + copy_and_reverse((u8 *)&b, op2); + gf128mul_x_lle(&a, &a); + gf128mul_lle(&a, &b); + copy_and_reverse(op1, (u8 *)&a); +} +EXPORT_SYMBOL_GPL(polyval_mul_non4k); + +/* + * Perform a POLYVAL update using non4k multiplication. This function is used + * as a fallback for hardware accelerated implementations when simd registers + * are unavailable. + * + * Note: This function is not used for polyval-generic, instead we use the 4k + * lookup table implementation of finite field multiplication. + */ +void polyval_update_non4k(const u8 *key, const u8 *in, + size_t nblocks, u8 *accumulator) +{ + while (nblocks--) { + crypto_xor(accumulator, in, POLYVAL_BLOCK_SIZE); + polyval_mul_non4k(accumulator, key); + in += POLYVAL_BLOCK_SIZE; + } +} +EXPORT_SYMBOL_GPL(polyval_update_non4k); + static int polyval_setkey(struct crypto_shash *tfm, const u8 *key, unsigned int keylen) { diff --git a/include/crypto/polyval.h b/include/crypto/polyval.h index b14c38aa9166..1d630f371f77 100644 --- a/include/crypto/polyval.h +++ b/include/crypto/polyval.h @@ -14,4 +14,9 @@ #define POLYVAL_BLOCK_SIZE 16 #define POLYVAL_DIGEST_SIZE 16 +void polyval_mul_non4k(u8 *op1, const u8 *op2); + +void polyval_update_non4k(const u8 *key, const u8 *in, + size_t nblocks, u8 *accumulator); + #endif From 9d2c0b485c46c7c5f781067c60300def5d1365cb Mon Sep 17 00:00:00 2001 From: Nathan Huckleberry Date: Fri, 20 May 2022 18:15:00 +0000 Subject: [PATCH 015/318] crypto: arm64/polyval - Add PMULL accelerated implementation of POLYVAL Add hardware accelerated version of POLYVAL for ARM64 CPUs with Crypto Extensions support. This implementation is accelerated using PMULL instructions to perform the finite field computations. For added efficiency, 8 blocks of the message are processed simultaneously by precomputing the first 8 powers of the key. Karatsuba multiplication is used instead of Schoolbook multiplication because it was found to be slightly faster on ARM64 CPUs. Montgomery reduction must be used instead of Barrett reduction due to the difference in modulus between POLYVAL's field and other finite fields. More information on POLYVAL can be found in the HCTR2 paper: "Length-preserving encryption with HCTR2": https://eprint.iacr.org/2021/1441.pdf Signed-off-by: Nathan Huckleberry Reviewed-by: Ard Biesheuvel Reviewed-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm64/crypto/Kconfig | 5 + arch/arm64/crypto/Makefile | 3 + arch/arm64/crypto/polyval-ce-core.S | 361 ++++++++++++++++++++++++++++ arch/arm64/crypto/polyval-ce-glue.c | 191 +++++++++++++++ 4 files changed, 560 insertions(+) create mode 100644 arch/arm64/crypto/polyval-ce-core.S create mode 100644 arch/arm64/crypto/polyval-ce-glue.c diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index 74d5bed9394f..4391a463abd7 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -72,6 +72,11 @@ config CRYPTO_GHASH_ARM64_CE select CRYPTO_GF128MUL select CRYPTO_LIB_AES +config CRYPTO_POLYVAL_ARM64_CE + tristate "POLYVAL using ARMv8 Crypto Extensions (for HCTR2)" + depends on KERNEL_MODE_NEON + select CRYPTO_POLYVAL + config CRYPTO_CRCT10DIF_ARM64_CE tristate "CRCT10DIF digest algorithm using PMULL instructions" depends on KERNEL_MODE_NEON && CRC_T10DIF diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile index bea8995133b1..24bb0c4610de 100644 --- a/arch/arm64/crypto/Makefile +++ b/arch/arm64/crypto/Makefile @@ -32,6 +32,9 @@ sm4-neon-y := sm4-neon-glue.o sm4-neon-core.o obj-$(CONFIG_CRYPTO_GHASH_ARM64_CE) += ghash-ce.o ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o +obj-$(CONFIG_CRYPTO_POLYVAL_ARM64_CE) += polyval-ce.o +polyval-ce-y := polyval-ce-glue.o polyval-ce-core.o + obj-$(CONFIG_CRYPTO_CRCT10DIF_ARM64_CE) += crct10dif-ce.o crct10dif-ce-y := crct10dif-ce-core.o crct10dif-ce-glue.o diff --git a/arch/arm64/crypto/polyval-ce-core.S b/arch/arm64/crypto/polyval-ce-core.S new file mode 100644 index 000000000000..b5326540d2e3 --- /dev/null +++ b/arch/arm64/crypto/polyval-ce-core.S @@ -0,0 +1,361 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Implementation of POLYVAL using ARMv8 Crypto Extensions. + * + * Copyright 2021 Google LLC + */ +/* + * This is an efficient implementation of POLYVAL using ARMv8 Crypto Extensions + * It works on 8 blocks at a time, by precomputing the first 8 keys powers h^8, + * ..., h^1 in the POLYVAL finite field. This precomputation allows us to split + * finite field multiplication into two steps. + * + * In the first step, we consider h^i, m_i as normal polynomials of degree less + * than 128. We then compute p(x) = h^8m_0 + ... + h^1m_7 where multiplication + * is simply polynomial multiplication. + * + * In the second step, we compute the reduction of p(x) modulo the finite field + * modulus g(x) = x^128 + x^127 + x^126 + x^121 + 1. + * + * This two step process is equivalent to computing h^8m_0 + ... + h^1m_7 where + * multiplication is finite field multiplication. The advantage is that the + * two-step process only requires 1 finite field reduction for every 8 + * polynomial multiplications. Further parallelism is gained by interleaving the + * multiplications and polynomial reductions. + */ + +#include +#define STRIDE_BLOCKS 8 + +KEY_POWERS .req x0 +MSG .req x1 +BLOCKS_LEFT .req x2 +ACCUMULATOR .req x3 +KEY_START .req x10 +EXTRA_BYTES .req x11 +TMP .req x13 + +M0 .req v0 +M1 .req v1 +M2 .req v2 +M3 .req v3 +M4 .req v4 +M5 .req v5 +M6 .req v6 +M7 .req v7 +KEY8 .req v8 +KEY7 .req v9 +KEY6 .req v10 +KEY5 .req v11 +KEY4 .req v12 +KEY3 .req v13 +KEY2 .req v14 +KEY1 .req v15 +PL .req v16 +PH .req v17 +TMP_V .req v18 +LO .req v20 +MI .req v21 +HI .req v22 +SUM .req v23 +GSTAR .req v24 + + .text + + .arch armv8-a+crypto + .align 4 + +.Lgstar: + .quad 0xc200000000000000, 0xc200000000000000 + +/* + * Computes the product of two 128-bit polynomials in X and Y and XORs the + * components of the 256-bit product into LO, MI, HI. + * + * Given: + * X = [X_1 : X_0] + * Y = [Y_1 : Y_0] + * + * We compute: + * LO += X_0 * Y_0 + * MI += (X_0 + X_1) * (Y_0 + Y_1) + * HI += X_1 * Y_1 + * + * Later, the 256-bit result can be extracted as: + * [HI_1 : HI_0 + HI_1 + MI_1 + LO_1 : LO_1 + HI_0 + MI_0 + LO_0 : LO_0] + * This step is done when computing the polynomial reduction for efficiency + * reasons. + * + * Karatsuba multiplication is used instead of Schoolbook multiplication because + * it was found to be slightly faster on ARM64 CPUs. + * + */ +.macro karatsuba1 X Y + X .req \X + Y .req \Y + ext v25.16b, X.16b, X.16b, #8 + ext v26.16b, Y.16b, Y.16b, #8 + eor v25.16b, v25.16b, X.16b + eor v26.16b, v26.16b, Y.16b + pmull2 v28.1q, X.2d, Y.2d + pmull v29.1q, X.1d, Y.1d + pmull v27.1q, v25.1d, v26.1d + eor HI.16b, HI.16b, v28.16b + eor LO.16b, LO.16b, v29.16b + eor MI.16b, MI.16b, v27.16b + .unreq X + .unreq Y +.endm + +/* + * Same as karatsuba1, except overwrites HI, LO, MI rather than XORing into + * them. + */ +.macro karatsuba1_store X Y + X .req \X + Y .req \Y + ext v25.16b, X.16b, X.16b, #8 + ext v26.16b, Y.16b, Y.16b, #8 + eor v25.16b, v25.16b, X.16b + eor v26.16b, v26.16b, Y.16b + pmull2 HI.1q, X.2d, Y.2d + pmull LO.1q, X.1d, Y.1d + pmull MI.1q, v25.1d, v26.1d + .unreq X + .unreq Y +.endm + +/* + * Computes the 256-bit polynomial represented by LO, HI, MI. Stores + * the result in PL, PH. + * [PH : PL] = + * [HI_1 : HI_1 + HI_0 + MI_1 + LO_1 : HI_0 + MI_0 + LO_1 + LO_0 : LO_0] + */ +.macro karatsuba2 + // v4 = [HI_1 + MI_1 : HI_0 + MI_0] + eor v4.16b, HI.16b, MI.16b + // v4 = [HI_1 + MI_1 + LO_1 : HI_0 + MI_0 + LO_0] + eor v4.16b, v4.16b, LO.16b + // v5 = [HI_0 : LO_1] + ext v5.16b, LO.16b, HI.16b, #8 + // v4 = [HI_1 + HI_0 + MI_1 + LO_1 : HI_0 + MI_0 + LO_1 + LO_0] + eor v4.16b, v4.16b, v5.16b + // HI = [HI_0 : HI_1] + ext HI.16b, HI.16b, HI.16b, #8 + // LO = [LO_0 : LO_1] + ext LO.16b, LO.16b, LO.16b, #8 + // PH = [HI_1 : HI_1 + HI_0 + MI_1 + LO_1] + ext PH.16b, v4.16b, HI.16b, #8 + // PL = [HI_0 + MI_0 + LO_1 + LO_0 : LO_0] + ext PL.16b, LO.16b, v4.16b, #8 +.endm + +/* + * Computes the 128-bit reduction of PH : PL. Stores the result in dest. + * + * This macro computes p(x) mod g(x) where p(x) is in montgomery form and g(x) = + * x^128 + x^127 + x^126 + x^121 + 1. + * + * We have a 256-bit polynomial PH : PL = P_3 : P_2 : P_1 : P_0 that is the + * product of two 128-bit polynomials in Montgomery form. We need to reduce it + * mod g(x). Also, since polynomials in Montgomery form have an "extra" factor + * of x^128, this product has two extra factors of x^128. To get it back into + * Montgomery form, we need to remove one of these factors by dividing by x^128. + * + * To accomplish both of these goals, we add multiples of g(x) that cancel out + * the low 128 bits P_1 : P_0, leaving just the high 128 bits. Since the low + * bits are zero, the polynomial division by x^128 can be done by right + * shifting. + * + * Since the only nonzero term in the low 64 bits of g(x) is the constant term, + * the multiple of g(x) needed to cancel out P_0 is P_0 * g(x). The CPU can + * only do 64x64 bit multiplications, so split P_0 * g(x) into x^128 * P_0 + + * x^64 * g*(x) * P_0 + P_0, where g*(x) is bits 64-127 of g(x). Adding this to + * the original polynomial gives P_3 : P_2 + P_0 + T_1 : P_1 + T_0 : 0, where T + * = T_1 : T_0 = g*(x) * P_0. Thus, bits 0-63 got "folded" into bits 64-191. + * + * Repeating this same process on the next 64 bits "folds" bits 64-127 into bits + * 128-255, giving the answer in bits 128-255. This time, we need to cancel P_1 + * + T_0 in bits 64-127. The multiple of g(x) required is (P_1 + T_0) * g(x) * + * x^64. Adding this to our previous computation gives P_3 + P_1 + T_0 + V_1 : + * P_2 + P_0 + T_1 + V_0 : 0 : 0, where V = V_1 : V_0 = g*(x) * (P_1 + T_0). + * + * So our final computation is: + * T = T_1 : T_0 = g*(x) * P_0 + * V = V_1 : V_0 = g*(x) * (P_1 + T_0) + * p(x) / x^{128} mod g(x) = P_3 + P_1 + T_0 + V_1 : P_2 + P_0 + T_1 + V_0 + * + * The implementation below saves a XOR instruction by computing P_1 + T_0 : P_0 + * + T_1 and XORing into dest, rather than separately XORing P_1 : P_0 and T_0 : + * T_1 into dest. This allows us to reuse P_1 + T_0 when computing V. + */ +.macro montgomery_reduction dest + DEST .req \dest + // TMP_V = T_1 : T_0 = P_0 * g*(x) + pmull TMP_V.1q, PL.1d, GSTAR.1d + // TMP_V = T_0 : T_1 + ext TMP_V.16b, TMP_V.16b, TMP_V.16b, #8 + // TMP_V = P_1 + T_0 : P_0 + T_1 + eor TMP_V.16b, PL.16b, TMP_V.16b + // PH = P_3 + P_1 + T_0 : P_2 + P_0 + T_1 + eor PH.16b, PH.16b, TMP_V.16b + // TMP_V = V_1 : V_0 = (P_1 + T_0) * g*(x) + pmull2 TMP_V.1q, TMP_V.2d, GSTAR.2d + eor DEST.16b, PH.16b, TMP_V.16b + .unreq DEST +.endm + +/* + * Compute Polyval on 8 blocks. + * + * If reduce is set, also computes the montgomery reduction of the + * previous full_stride call and XORs with the first message block. + * (m_0 + REDUCE(PL, PH))h^8 + ... + m_7h^1. + * I.e., the first multiplication uses m_0 + REDUCE(PL, PH) instead of m_0. + * + * Sets PL, PH. + */ +.macro full_stride reduce + eor LO.16b, LO.16b, LO.16b + eor MI.16b, MI.16b, MI.16b + eor HI.16b, HI.16b, HI.16b + + ld1 {M0.16b, M1.16b, M2.16b, M3.16b}, [MSG], #64 + ld1 {M4.16b, M5.16b, M6.16b, M7.16b}, [MSG], #64 + + karatsuba1 M7 KEY1 + .if \reduce + pmull TMP_V.1q, PL.1d, GSTAR.1d + .endif + + karatsuba1 M6 KEY2 + .if \reduce + ext TMP_V.16b, TMP_V.16b, TMP_V.16b, #8 + .endif + + karatsuba1 M5 KEY3 + .if \reduce + eor TMP_V.16b, PL.16b, TMP_V.16b + .endif + + karatsuba1 M4 KEY4 + .if \reduce + eor PH.16b, PH.16b, TMP_V.16b + .endif + + karatsuba1 M3 KEY5 + .if \reduce + pmull2 TMP_V.1q, TMP_V.2d, GSTAR.2d + .endif + + karatsuba1 M2 KEY6 + .if \reduce + eor SUM.16b, PH.16b, TMP_V.16b + .endif + + karatsuba1 M1 KEY7 + eor M0.16b, M0.16b, SUM.16b + + karatsuba1 M0 KEY8 + karatsuba2 +.endm + +/* + * Handle any extra blocks after full_stride loop. + */ +.macro partial_stride + add KEY_POWERS, KEY_START, #(STRIDE_BLOCKS << 4) + sub KEY_POWERS, KEY_POWERS, BLOCKS_LEFT, lsl #4 + ld1 {KEY1.16b}, [KEY_POWERS], #16 + + ld1 {TMP_V.16b}, [MSG], #16 + eor SUM.16b, SUM.16b, TMP_V.16b + karatsuba1_store KEY1 SUM + sub BLOCKS_LEFT, BLOCKS_LEFT, #1 + + tst BLOCKS_LEFT, #4 + beq .Lpartial4BlocksDone + ld1 {M0.16b, M1.16b, M2.16b, M3.16b}, [MSG], #64 + ld1 {KEY8.16b, KEY7.16b, KEY6.16b, KEY5.16b}, [KEY_POWERS], #64 + karatsuba1 M0 KEY8 + karatsuba1 M1 KEY7 + karatsuba1 M2 KEY6 + karatsuba1 M3 KEY5 +.Lpartial4BlocksDone: + tst BLOCKS_LEFT, #2 + beq .Lpartial2BlocksDone + ld1 {M0.16b, M1.16b}, [MSG], #32 + ld1 {KEY8.16b, KEY7.16b}, [KEY_POWERS], #32 + karatsuba1 M0 KEY8 + karatsuba1 M1 KEY7 +.Lpartial2BlocksDone: + tst BLOCKS_LEFT, #1 + beq .LpartialDone + ld1 {M0.16b}, [MSG], #16 + ld1 {KEY8.16b}, [KEY_POWERS], #16 + karatsuba1 M0 KEY8 +.LpartialDone: + karatsuba2 + montgomery_reduction SUM +.endm + +/* + * Perform montgomery multiplication in GF(2^128) and store result in op1. + * + * Computes op1*op2*x^{-128} mod x^128 + x^127 + x^126 + x^121 + 1 + * If op1, op2 are in montgomery form, this computes the montgomery + * form of op1*op2. + * + * void pmull_polyval_mul(u8 *op1, const u8 *op2); + */ +SYM_FUNC_START(pmull_polyval_mul) + adr TMP, .Lgstar + ld1 {GSTAR.2d}, [TMP] + ld1 {v0.16b}, [x0] + ld1 {v1.16b}, [x1] + karatsuba1_store v0 v1 + karatsuba2 + montgomery_reduction SUM + st1 {SUM.16b}, [x0] + ret +SYM_FUNC_END(pmull_polyval_mul) + +/* + * Perform polynomial evaluation as specified by POLYVAL. This computes: + * h^n * accumulator + h^n * m_0 + ... + h^1 * m_{n-1} + * where n=nblocks, h is the hash key, and m_i are the message blocks. + * + * x0 - pointer to precomputed key powers h^8 ... h^1 + * x1 - pointer to message blocks + * x2 - number of blocks to hash + * x3 - pointer to accumulator + * + * void pmull_polyval_update(const struct polyval_ctx *ctx, const u8 *in, + * size_t nblocks, u8 *accumulator); + */ +SYM_FUNC_START(pmull_polyval_update) + adr TMP, .Lgstar + mov KEY_START, KEY_POWERS + ld1 {GSTAR.2d}, [TMP] + ld1 {SUM.16b}, [ACCUMULATOR] + subs BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS + blt .LstrideLoopExit + ld1 {KEY8.16b, KEY7.16b, KEY6.16b, KEY5.16b}, [KEY_POWERS], #64 + ld1 {KEY4.16b, KEY3.16b, KEY2.16b, KEY1.16b}, [KEY_POWERS], #64 + full_stride 0 + subs BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS + blt .LstrideLoopExitReduce +.LstrideLoop: + full_stride 1 + subs BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS + bge .LstrideLoop +.LstrideLoopExitReduce: + montgomery_reduction SUM +.LstrideLoopExit: + adds BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS + beq .LskipPartial + partial_stride +.LskipPartial: + st1 {SUM.16b}, [ACCUMULATOR] + ret +SYM_FUNC_END(pmull_polyval_update) diff --git a/arch/arm64/crypto/polyval-ce-glue.c b/arch/arm64/crypto/polyval-ce-glue.c new file mode 100644 index 000000000000..0a3b5718df85 --- /dev/null +++ b/arch/arm64/crypto/polyval-ce-glue.c @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Glue code for POLYVAL using ARMv8 Crypto Extensions + * + * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen + * Copyright (c) 2009 Intel Corp. + * Author: Huang Ying + * Copyright 2021 Google LLC + */ + +/* + * Glue code based on ghash-clmulni-intel_glue.c. + * + * This implementation of POLYVAL uses montgomery multiplication accelerated by + * ARMv8 Crypto Extensions instructions to implement the finite field operations. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define NUM_KEY_POWERS 8 + +struct polyval_tfm_ctx { + /* + * These powers must be in the order h^8, ..., h^1. + */ + u8 key_powers[NUM_KEY_POWERS][POLYVAL_BLOCK_SIZE]; +}; + +struct polyval_desc_ctx { + u8 buffer[POLYVAL_BLOCK_SIZE]; + u32 bytes; +}; + +asmlinkage void pmull_polyval_update(const struct polyval_tfm_ctx *keys, + const u8 *in, size_t nblocks, u8 *accumulator); +asmlinkage void pmull_polyval_mul(u8 *op1, const u8 *op2); + +static void internal_polyval_update(const struct polyval_tfm_ctx *keys, + const u8 *in, size_t nblocks, u8 *accumulator) +{ + if (likely(crypto_simd_usable())) { + kernel_neon_begin(); + pmull_polyval_update(keys, in, nblocks, accumulator); + kernel_neon_end(); + } else { + polyval_update_non4k(keys->key_powers[NUM_KEY_POWERS-1], in, + nblocks, accumulator); + } +} + +static void internal_polyval_mul(u8 *op1, const u8 *op2) +{ + if (likely(crypto_simd_usable())) { + kernel_neon_begin(); + pmull_polyval_mul(op1, op2); + kernel_neon_end(); + } else { + polyval_mul_non4k(op1, op2); + } +} + +static int polyval_arm64_setkey(struct crypto_shash *tfm, + const u8 *key, unsigned int keylen) +{ + struct polyval_tfm_ctx *tctx = crypto_shash_ctx(tfm); + int i; + + if (keylen != POLYVAL_BLOCK_SIZE) + return -EINVAL; + + memcpy(tctx->key_powers[NUM_KEY_POWERS-1], key, POLYVAL_BLOCK_SIZE); + + for (i = NUM_KEY_POWERS-2; i >= 0; i--) { + memcpy(tctx->key_powers[i], key, POLYVAL_BLOCK_SIZE); + internal_polyval_mul(tctx->key_powers[i], + tctx->key_powers[i+1]); + } + + return 0; +} + +static int polyval_arm64_init(struct shash_desc *desc) +{ + struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); + + memset(dctx, 0, sizeof(*dctx)); + + return 0; +} + +static int polyval_arm64_update(struct shash_desc *desc, + const u8 *src, unsigned int srclen) +{ + struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); + const struct polyval_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); + u8 *pos; + unsigned int nblocks; + unsigned int n; + + if (dctx->bytes) { + n = min(srclen, dctx->bytes); + pos = dctx->buffer + POLYVAL_BLOCK_SIZE - dctx->bytes; + + dctx->bytes -= n; + srclen -= n; + + while (n--) + *pos++ ^= *src++; + + if (!dctx->bytes) + internal_polyval_mul(dctx->buffer, + tctx->key_powers[NUM_KEY_POWERS-1]); + } + + while (srclen >= POLYVAL_BLOCK_SIZE) { + /* allow rescheduling every 4K bytes */ + nblocks = min(srclen, 4096U) / POLYVAL_BLOCK_SIZE; + internal_polyval_update(tctx, src, nblocks, dctx->buffer); + srclen -= nblocks * POLYVAL_BLOCK_SIZE; + src += nblocks * POLYVAL_BLOCK_SIZE; + } + + if (srclen) { + dctx->bytes = POLYVAL_BLOCK_SIZE - srclen; + pos = dctx->buffer; + while (srclen--) + *pos++ ^= *src++; + } + + return 0; +} + +static int polyval_arm64_final(struct shash_desc *desc, u8 *dst) +{ + struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); + const struct polyval_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); + + if (dctx->bytes) { + internal_polyval_mul(dctx->buffer, + tctx->key_powers[NUM_KEY_POWERS-1]); + } + + memcpy(dst, dctx->buffer, POLYVAL_BLOCK_SIZE); + + return 0; +} + +static struct shash_alg polyval_alg = { + .digestsize = POLYVAL_DIGEST_SIZE, + .init = polyval_arm64_init, + .update = polyval_arm64_update, + .final = polyval_arm64_final, + .setkey = polyval_arm64_setkey, + .descsize = sizeof(struct polyval_desc_ctx), + .base = { + .cra_name = "polyval", + .cra_driver_name = "polyval-ce", + .cra_priority = 200, + .cra_blocksize = POLYVAL_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct polyval_tfm_ctx), + .cra_module = THIS_MODULE, + }, +}; + +static int __init polyval_ce_mod_init(void) +{ + return crypto_register_shash(&polyval_alg); +} + +static void __exit polyval_ce_mod_exit(void) +{ + crypto_unregister_shash(&polyval_alg); +} + +module_cpu_feature_match(PMULL, polyval_ce_mod_init) +module_exit(polyval_ce_mod_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("POLYVAL hash function accelerated by ARMv8 Crypto Extensions"); +MODULE_ALIAS_CRYPTO("polyval"); +MODULE_ALIAS_CRYPTO("polyval-ce"); From 6b2a51ff03bf0c54cbc699ee85a9a49eb203ebfc Mon Sep 17 00:00:00 2001 From: Nathan Huckleberry Date: Fri, 20 May 2022 18:15:01 +0000 Subject: [PATCH 016/318] fscrypt: Add HCTR2 support for filename encryption HCTR2 is a tweakable, length-preserving encryption mode that is intended for use on CPUs with dedicated crypto instructions. HCTR2 has the property that a bitflip in the plaintext changes the entire ciphertext. This property fixes a known weakness with filename encryption: when two filenames in the same directory share a prefix of >= 16 bytes, with AES-CTS-CBC their encrypted filenames share a common substring, leaking information. HCTR2 does not have this problem. More information on HCTR2 can be found here: "Length-preserving encryption with HCTR2": https://eprint.iacr.org/2021/1441.pdf Signed-off-by: Nathan Huckleberry Reviewed-by: Ard Biesheuvel Acked-by: Eric Biggers Signed-off-by: Herbert Xu --- Documentation/filesystems/fscrypt.rst | 22 +++++++++++++++++----- fs/crypto/fscrypt_private.h | 2 +- fs/crypto/keysetup.c | 7 +++++++ fs/crypto/policy.c | 14 +++++++++++--- include/uapi/linux/fscrypt.h | 3 ++- 5 files changed, 38 insertions(+), 10 deletions(-) diff --git a/Documentation/filesystems/fscrypt.rst b/Documentation/filesystems/fscrypt.rst index 2e9aaa295125..5ba5817c17c2 100644 --- a/Documentation/filesystems/fscrypt.rst +++ b/Documentation/filesystems/fscrypt.rst @@ -337,6 +337,7 @@ Currently, the following pairs of encryption modes are supported: - AES-256-XTS for contents and AES-256-CTS-CBC for filenames - AES-128-CBC for contents and AES-128-CTS-CBC for filenames - Adiantum for both contents and filenames +- AES-256-XTS for contents and AES-256-HCTR2 for filenames (v2 policies only) If unsure, you should use the (AES-256-XTS, AES-256-CTS-CBC) pair. @@ -357,6 +358,17 @@ To use Adiantum, CONFIG_CRYPTO_ADIANTUM must be enabled. Also, fast implementations of ChaCha and NHPoly1305 should be enabled, e.g. CONFIG_CRYPTO_CHACHA20_NEON and CONFIG_CRYPTO_NHPOLY1305_NEON for ARM. +AES-256-HCTR2 is another true wide-block encryption mode that is intended for +use on CPUs with dedicated crypto instructions. AES-256-HCTR2 has the property +that a bitflip in the plaintext changes the entire ciphertext. This property +makes it desirable for filename encryption since initialization vectors are +reused within a directory. For more details on AES-256-HCTR2, see the paper +"Length-preserving encryption with HCTR2" +(https://eprint.iacr.org/2021/1441.pdf). To use AES-256-HCTR2, +CONFIG_CRYPTO_HCTR2 must be enabled. Also, fast implementations of XCTR and +POLYVAL should be enabled, e.g. CRYPTO_POLYVAL_ARM64_CE and +CRYPTO_AES_ARM64_CE_BLK for ARM64. + New encryption modes can be added relatively easily, without changes to individual filesystems. However, authenticated encryption (AE) modes are not currently supported because of the difficulty of dealing @@ -404,11 +416,11 @@ alternatively has the file's nonce (for `DIRECT_KEY policies`_) or inode number (for `IV_INO_LBLK_64 policies`_) included in the IVs. Thus, IV reuse is limited to within a single directory. -With CTS-CBC, the IV reuse means that when the plaintext filenames -share a common prefix at least as long as the cipher block size (16 -bytes for AES), the corresponding encrypted filenames will also share -a common prefix. This is undesirable. Adiantum does not have this -weakness, as it is a wide-block encryption mode. +With CTS-CBC, the IV reuse means that when the plaintext filenames share a +common prefix at least as long as the cipher block size (16 bytes for AES), the +corresponding encrypted filenames will also share a common prefix. This is +undesirable. Adiantum and HCTR2 do not have this weakness, as they are +wide-block encryption modes. All supported filenames encryption modes accept any plaintext length >= 16 bytes; cipher block alignment is not required. However, diff --git a/fs/crypto/fscrypt_private.h b/fs/crypto/fscrypt_private.h index 6b4c8094cc7b..f5be777d8279 100644 --- a/fs/crypto/fscrypt_private.h +++ b/fs/crypto/fscrypt_private.h @@ -31,7 +31,7 @@ #define FSCRYPT_CONTEXT_V2 2 /* Keep this in sync with include/uapi/linux/fscrypt.h */ -#define FSCRYPT_MODE_MAX FSCRYPT_MODE_ADIANTUM +#define FSCRYPT_MODE_MAX FSCRYPT_MODE_AES_256_HCTR2 struct fscrypt_context_v1 { u8 version; /* FSCRYPT_CONTEXT_V1 */ diff --git a/fs/crypto/keysetup.c b/fs/crypto/keysetup.c index c35711896bd4..fbc71abdabe3 100644 --- a/fs/crypto/keysetup.c +++ b/fs/crypto/keysetup.c @@ -53,6 +53,13 @@ struct fscrypt_mode fscrypt_modes[] = { .ivsize = 32, .blk_crypto_mode = BLK_ENCRYPTION_MODE_ADIANTUM, }, + [FSCRYPT_MODE_AES_256_HCTR2] = { + .friendly_name = "AES-256-HCTR2", + .cipher_str = "hctr2(aes)", + .keysize = 32, + .security_strength = 32, + .ivsize = 32, + }, }; static DEFINE_MUTEX(fscrypt_mode_key_setup_mutex); diff --git a/fs/crypto/policy.c b/fs/crypto/policy.c index 5f858cee1e3b..8a054e6d1e68 100644 --- a/fs/crypto/policy.c +++ b/fs/crypto/policy.c @@ -61,7 +61,7 @@ fscrypt_get_dummy_policy(struct super_block *sb) return sb->s_cop->get_dummy_policy(sb); } -static bool fscrypt_valid_enc_modes(u32 contents_mode, u32 filenames_mode) +static bool fscrypt_valid_enc_modes_v1(u32 contents_mode, u32 filenames_mode) { if (contents_mode == FSCRYPT_MODE_AES_256_XTS && filenames_mode == FSCRYPT_MODE_AES_256_CTS) @@ -78,6 +78,14 @@ static bool fscrypt_valid_enc_modes(u32 contents_mode, u32 filenames_mode) return false; } +static bool fscrypt_valid_enc_modes_v2(u32 contents_mode, u32 filenames_mode) +{ + if (contents_mode == FSCRYPT_MODE_AES_256_XTS && + filenames_mode == FSCRYPT_MODE_AES_256_HCTR2) + return true; + return fscrypt_valid_enc_modes_v1(contents_mode, filenames_mode); +} + static bool supported_direct_key_modes(const struct inode *inode, u32 contents_mode, u32 filenames_mode) { @@ -151,7 +159,7 @@ static bool supported_iv_ino_lblk_policy(const struct fscrypt_policy_v2 *policy, static bool fscrypt_supported_v1_policy(const struct fscrypt_policy_v1 *policy, const struct inode *inode) { - if (!fscrypt_valid_enc_modes(policy->contents_encryption_mode, + if (!fscrypt_valid_enc_modes_v1(policy->contents_encryption_mode, policy->filenames_encryption_mode)) { fscrypt_warn(inode, "Unsupported encryption modes (contents %d, filenames %d)", @@ -187,7 +195,7 @@ static bool fscrypt_supported_v2_policy(const struct fscrypt_policy_v2 *policy, { int count = 0; - if (!fscrypt_valid_enc_modes(policy->contents_encryption_mode, + if (!fscrypt_valid_enc_modes_v2(policy->contents_encryption_mode, policy->filenames_encryption_mode)) { fscrypt_warn(inode, "Unsupported encryption modes (contents %d, filenames %d)", diff --git a/include/uapi/linux/fscrypt.h b/include/uapi/linux/fscrypt.h index 9f4428be3e36..a756b29afcc2 100644 --- a/include/uapi/linux/fscrypt.h +++ b/include/uapi/linux/fscrypt.h @@ -27,7 +27,8 @@ #define FSCRYPT_MODE_AES_128_CBC 5 #define FSCRYPT_MODE_AES_128_CTS 6 #define FSCRYPT_MODE_ADIANTUM 9 -/* If adding a mode number > 9, update FSCRYPT_MODE_MAX in fscrypt_private.h */ +#define FSCRYPT_MODE_AES_256_HCTR2 10 +/* If adding a mode number > 10, update FSCRYPT_MODE_MAX in fscrypt_private.h */ /* * Legacy policy version; ad-hoc KDF and no key verification. From 3f3bbf22a592e8bd15d2f6d8e9a4a34e2b5028cd Mon Sep 17 00:00:00 2001 From: Julia Lawall Date: Sat, 21 May 2022 13:10:45 +0200 Subject: [PATCH 017/318] crypto: hisilicon/sec - fix typos in comment Spelling mistakes (triple letters) in comment. Detected with the help of Coccinelle. Signed-off-by: Julia Lawall Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sec2/sec.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index c2e9b01187a7..42bb486f3b6d 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -143,10 +143,10 @@ struct sec_ctx { /* Threshold for fake busy, trigger to return -EBUSY to user */ u32 fake_req_limit; - /* Currrent cyclic index to select a queue for encipher */ + /* Current cyclic index to select a queue for encipher */ atomic_t enc_qcyclic; - /* Currrent cyclic index to select a queue for decipher */ + /* Current cyclic index to select a queue for decipher */ atomic_t dec_qcyclic; enum sec_alg_type alg_type; From cd81775a56bce2ad480ff8444e6f44f3980ceb7d Mon Sep 17 00:00:00 2001 From: Julia Lawall Date: Sat, 21 May 2022 13:10:48 +0200 Subject: [PATCH 018/318] crypto: ccp - fix typo in comment Spelling mistake (triple letters) in comment. Detected with the help of Coccinelle. Signed-off-by: Julia Lawall Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-dev.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h index a5d9123a22ea..83350e2d9821 100644 --- a/drivers/crypto/ccp/ccp-dev.h +++ b/drivers/crypto/ccp/ccp-dev.h @@ -366,7 +366,7 @@ struct ccp_device { /* Master lists that all cmds are queued on. Because there can be * more than one CCP command queue that can process a cmd a separate - * backlog list is neeeded so that the backlog completion call + * backlog list is needed so that the backlog completion call * completes before the cmd is available for execution. */ spinlock_t cmd_lock ____cacheline_aligned; From 4ad28689df853db3ee7b2656a225f1d722ee7639 Mon Sep 17 00:00:00 2001 From: Shijith Thotton Date: Fri, 27 May 2022 13:24:48 +0530 Subject: [PATCH 019/318] crypto: octeontx2 - add firmware version in devlink info Added running firmware version information of AE, SE and IE components in devlink info. Signed-off-by: Shijith Thotton Signed-off-by: Herbert Xu --- .../marvell/octeontx2/otx2_cpt_devlink.c | 40 ++++++++++++++++++- .../marvell/octeontx2/otx2_cptpf_ucode.c | 2 +- .../marvell/octeontx2/otx2_cptpf_ucode.h | 3 ++ 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c b/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c index bb02e0db3615..7503f6b18ac5 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c @@ -51,11 +51,47 @@ static const struct devlink_param otx2_cpt_dl_params[] = { NULL), }; -static int otx2_cpt_devlink_info_get(struct devlink *devlink, +static int otx2_cpt_dl_info_firmware_version_put(struct devlink_info_req *req, + struct otx2_cpt_eng_grp_info grp[], + const char *ver_name, int eng_type) +{ + struct otx2_cpt_engs_rsvd *eng; + int i; + + for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) { + eng = find_engines_by_type(&grp[i], eng_type); + if (eng) + return devlink_info_version_running_put(req, ver_name, + eng->ucode->ver_str); + } + + return 0; +} + +static int otx2_cpt_devlink_info_get(struct devlink *dl, struct devlink_info_req *req, struct netlink_ext_ack *extack) { - return devlink_info_driver_name_put(req, "rvu_cptpf"); + struct otx2_cpt_devlink *cpt_dl = devlink_priv(dl); + struct otx2_cptpf_dev *cptpf = cpt_dl->cptpf; + int err; + + err = devlink_info_driver_name_put(req, "rvu_cptpf"); + if (err) + return err; + + err = otx2_cpt_dl_info_firmware_version_put(req, cptpf->eng_grps.grp, + "fw.ae", OTX2_CPT_AE_TYPES); + if (err) + return err; + + err = otx2_cpt_dl_info_firmware_version_put(req, cptpf->eng_grps.grp, + "fw.se", OTX2_CPT_SE_TYPES); + if (err) + return err; + + return otx2_cpt_dl_info_firmware_version_put(req, cptpf->eng_grps.grp, + "fw.ie", OTX2_CPT_IE_TYPES); } static const struct devlink_ops otx2_cpt_devlink_ops = { diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c index 9cba2f714c7e..46ffb7ae982c 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c @@ -476,7 +476,7 @@ release_fw: return ret; } -static struct otx2_cpt_engs_rsvd *find_engines_by_type( +struct otx2_cpt_engs_rsvd *find_engines_by_type( struct otx2_cpt_eng_grp_info *eng_grp, int eng_type) { diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h index 8f4d4e5f531a..e69320a54b5d 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h @@ -166,4 +166,7 @@ int otx2_cpt_dl_custom_egrp_create(struct otx2_cptpf_dev *cptpf, int otx2_cpt_dl_custom_egrp_delete(struct otx2_cptpf_dev *cptpf, struct devlink_param_gset_ctx *ctx); void otx2_cpt_print_uc_dbg_info(struct otx2_cptpf_dev *cptpf); +struct otx2_cpt_engs_rsvd *find_engines_by_type( + struct otx2_cpt_eng_grp_info *eng_grp, + int eng_type); #endif /* __OTX2_CPTPF_UCODE_H */ From 920b0442b9f884f55f4745b53430c80e71e90275 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Sat, 28 May 2022 12:24:29 +0200 Subject: [PATCH 020/318] crypto: memneq - move into lib/ This is used by code that doesn't need CONFIG_CRYPTO, so move this into lib/ with a Kconfig option so that it can be selected by whatever needs it. This fixes a linker error Zheng pointed out when CRYPTO_MANAGER_DISABLE_TESTS!=y and CRYPTO=m: lib/crypto/curve25519-selftest.o: In function `curve25519_selftest': curve25519-selftest.c:(.init.text+0x60): undefined reference to `__crypto_memneq' curve25519-selftest.c:(.init.text+0xec): undefined reference to `__crypto_memneq' curve25519-selftest.c:(.init.text+0x114): undefined reference to `__crypto_memneq' curve25519-selftest.c:(.init.text+0x154): undefined reference to `__crypto_memneq' Reported-by: Zheng Bin Cc: Eric Biggers Cc: stable@vger.kernel.org Fixes: aa127963f1ca ("crypto: lib/curve25519 - re-add selftests") Signed-off-by: Jason A. Donenfeld Reviewed-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/Kconfig | 1 + crypto/Makefile | 2 +- lib/Kconfig | 3 +++ lib/Makefile | 1 + lib/crypto/Kconfig | 1 + {crypto => lib}/memneq.c | 0 6 files changed, 7 insertions(+), 1 deletion(-) rename {crypto => lib}/memneq.c (100%) diff --git a/crypto/Kconfig b/crypto/Kconfig index 9b654984de79..6e30e8138057 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -15,6 +15,7 @@ source "crypto/async_tx/Kconfig" # menuconfig CRYPTO tristate "Cryptographic API" + select LIB_MEMNEQ help This option provides the core Cryptographic API. diff --git a/crypto/Makefile b/crypto/Makefile index 3bbc0dd49160..1f529704fe80 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -4,7 +4,7 @@ # obj-$(CONFIG_CRYPTO) += crypto.o -crypto-y := api.o cipher.o compress.o memneq.o +crypto-y := api.o cipher.o compress.o obj-$(CONFIG_CRYPTO_ENGINE) += crypto_engine.o obj-$(CONFIG_CRYPTO_FIPS) += fips.o diff --git a/lib/Kconfig b/lib/Kconfig index 6a843639814f..eaaad4d85bf2 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -120,6 +120,9 @@ config INDIRECT_IOMEM_FALLBACK source "lib/crypto/Kconfig" +config LIB_MEMNEQ + bool + config CRC_CCITT tristate "CRC-CCITT functions" help diff --git a/lib/Makefile b/lib/Makefile index ea54294d73bf..f99bf61f8bbc 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -251,6 +251,7 @@ obj-$(CONFIG_DIMLIB) += dim/ obj-$(CONFIG_SIGNATURE) += digsig.o lib-$(CONFIG_CLZ_TAB) += clz_tab.o +lib-$(CONFIG_LIB_MEMNEQ) += memneq.o obj-$(CONFIG_GENERIC_STRNCPY_FROM_USER) += strncpy_from_user.o obj-$(CONFIG_GENERIC_STRNLEN_USER) += strnlen_user.o diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 9856e291f414..2082af43d51f 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -71,6 +71,7 @@ config CRYPTO_LIB_CURVE25519 tristate "Curve25519 scalar multiplication library" depends on CRYPTO_ARCH_HAVE_LIB_CURVE25519 || !CRYPTO_ARCH_HAVE_LIB_CURVE25519 select CRYPTO_LIB_CURVE25519_GENERIC if CRYPTO_ARCH_HAVE_LIB_CURVE25519=n + select LIB_MEMNEQ help Enable the Curve25519 library interface. This interface may be fulfilled by either the generic implementation or an arch-specific diff --git a/crypto/memneq.c b/lib/memneq.c similarity index 100% rename from crypto/memneq.c rename to lib/memneq.c From 2d16803c562ecc644803d42ba98a8e0aef9c014e Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Sat, 28 May 2022 21:44:07 +0200 Subject: [PATCH 021/318] crypto: blake2s - remove shash module BLAKE2s has no currently known use as an shash. Just remove all of this unnecessary plumbing. Removing this shash was something we talked about back when we were making BLAKE2s a built-in, but I simply never got around to doing it. So this completes that project. Importantly, this fixs a bug in which the lib code depends on crypto_simd_disabled_for_test, causing linker errors. Also add more alignment tests to the selftests and compare SIMD and non-SIMD compression functions, to make up for what we lose from testmgr.c. Reported-by: gaochao Cc: Eric Biggers Cc: Ard Biesheuvel Cc: stable@vger.kernel.org Fixes: 6048fdcc5f26 ("lib/crypto: blake2s: include as built-in") Signed-off-by: Jason A. Donenfeld Signed-off-by: Herbert Xu --- arch/arm/crypto/Kconfig | 2 +- arch/arm/crypto/Makefile | 4 +- arch/arm/crypto/blake2s-shash.c | 75 ----------- arch/x86/crypto/Makefile | 4 +- arch/x86/crypto/blake2s-glue.c | 3 +- arch/x86/crypto/blake2s-shash.c | 77 ----------- crypto/Kconfig | 20 +-- crypto/Makefile | 1 - crypto/blake2s_generic.c | 75 ----------- crypto/tcrypt.c | 12 -- crypto/testmgr.c | 24 ---- crypto/testmgr.h | 217 ------------------------------ include/crypto/internal/blake2s.h | 108 --------------- lib/crypto/blake2s-selftest.c | 41 ++++++ lib/crypto/blake2s.c | 37 ++++- 15 files changed, 76 insertions(+), 624 deletions(-) delete mode 100644 arch/arm/crypto/blake2s-shash.c delete mode 100644 arch/x86/crypto/blake2s-shash.c delete mode 100644 crypto/blake2s_generic.c diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index e4dba5461cb3..149a5bd6b88c 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -63,7 +63,7 @@ config CRYPTO_SHA512_ARM using optimized ARM assembler and NEON, when available. config CRYPTO_BLAKE2S_ARM - tristate "BLAKE2s digest algorithm (ARM)" + bool "BLAKE2s digest algorithm (ARM)" select CRYPTO_ARCH_HAVE_LIB_BLAKE2S help BLAKE2s digest algorithm optimized with ARM scalar instructions. This diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile index 0274f81cc8ea..971e74546fb1 100644 --- a/arch/arm/crypto/Makefile +++ b/arch/arm/crypto/Makefile @@ -9,8 +9,7 @@ obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) += sha1-arm-neon.o obj-$(CONFIG_CRYPTO_SHA256_ARM) += sha256-arm.o obj-$(CONFIG_CRYPTO_SHA512_ARM) += sha512-arm.o -obj-$(CONFIG_CRYPTO_BLAKE2S_ARM) += blake2s-arm.o -obj-$(if $(CONFIG_CRYPTO_BLAKE2S_ARM),y) += libblake2s-arm.o +obj-$(CONFIG_CRYPTO_BLAKE2S_ARM) += libblake2s-arm.o obj-$(CONFIG_CRYPTO_BLAKE2B_NEON) += blake2b-neon.o obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha-neon.o obj-$(CONFIG_CRYPTO_POLY1305_ARM) += poly1305-arm.o @@ -32,7 +31,6 @@ sha256-arm-neon-$(CONFIG_KERNEL_MODE_NEON) := sha256_neon_glue.o sha256-arm-y := sha256-core.o sha256_glue.o $(sha256-arm-neon-y) sha512-arm-neon-$(CONFIG_KERNEL_MODE_NEON) := sha512-neon-glue.o sha512-arm-y := sha512-core.o sha512-glue.o $(sha512-arm-neon-y) -blake2s-arm-y := blake2s-shash.o libblake2s-arm-y:= blake2s-core.o blake2s-glue.o blake2b-neon-y := blake2b-neon-core.o blake2b-neon-glue.o sha1-arm-ce-y := sha1-ce-core.o sha1-ce-glue.o diff --git a/arch/arm/crypto/blake2s-shash.c b/arch/arm/crypto/blake2s-shash.c deleted file mode 100644 index 763c73beea2d..000000000000 --- a/arch/arm/crypto/blake2s-shash.c +++ /dev/null @@ -1,75 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * BLAKE2s digest algorithm, ARM scalar implementation - * - * Copyright 2020 Google LLC - */ - -#include -#include - -#include - -static int crypto_blake2s_update_arm(struct shash_desc *desc, - const u8 *in, unsigned int inlen) -{ - return crypto_blake2s_update(desc, in, inlen, false); -} - -static int crypto_blake2s_final_arm(struct shash_desc *desc, u8 *out) -{ - return crypto_blake2s_final(desc, out, false); -} - -#define BLAKE2S_ALG(name, driver_name, digest_size) \ - { \ - .base.cra_name = name, \ - .base.cra_driver_name = driver_name, \ - .base.cra_priority = 200, \ - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, \ - .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, \ - .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), \ - .base.cra_module = THIS_MODULE, \ - .digestsize = digest_size, \ - .setkey = crypto_blake2s_setkey, \ - .init = crypto_blake2s_init, \ - .update = crypto_blake2s_update_arm, \ - .final = crypto_blake2s_final_arm, \ - .descsize = sizeof(struct blake2s_state), \ - } - -static struct shash_alg blake2s_arm_algs[] = { - BLAKE2S_ALG("blake2s-128", "blake2s-128-arm", BLAKE2S_128_HASH_SIZE), - BLAKE2S_ALG("blake2s-160", "blake2s-160-arm", BLAKE2S_160_HASH_SIZE), - BLAKE2S_ALG("blake2s-224", "blake2s-224-arm", BLAKE2S_224_HASH_SIZE), - BLAKE2S_ALG("blake2s-256", "blake2s-256-arm", BLAKE2S_256_HASH_SIZE), -}; - -static int __init blake2s_arm_mod_init(void) -{ - return IS_REACHABLE(CONFIG_CRYPTO_HASH) ? - crypto_register_shashes(blake2s_arm_algs, - ARRAY_SIZE(blake2s_arm_algs)) : 0; -} - -static void __exit blake2s_arm_mod_exit(void) -{ - if (IS_REACHABLE(CONFIG_CRYPTO_HASH)) - crypto_unregister_shashes(blake2s_arm_algs, - ARRAY_SIZE(blake2s_arm_algs)); -} - -module_init(blake2s_arm_mod_init); -module_exit(blake2s_arm_mod_exit); - -MODULE_DESCRIPTION("BLAKE2s digest algorithm, ARM scalar implementation"); -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Eric Biggers "); -MODULE_ALIAS_CRYPTO("blake2s-128"); -MODULE_ALIAS_CRYPTO("blake2s-128-arm"); -MODULE_ALIAS_CRYPTO("blake2s-160"); -MODULE_ALIAS_CRYPTO("blake2s-160-arm"); -MODULE_ALIAS_CRYPTO("blake2s-224"); -MODULE_ALIAS_CRYPTO("blake2s-224-arm"); -MODULE_ALIAS_CRYPTO("blake2s-256"); -MODULE_ALIAS_CRYPTO("blake2s-256-arm"); diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index b9847152acd8..04d07ab744b2 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -61,9 +61,7 @@ sha256-ssse3-$(CONFIG_AS_SHA256_NI) += sha256_ni_asm.o obj-$(CONFIG_CRYPTO_SHA512_SSSE3) += sha512-ssse3.o sha512-ssse3-y := sha512-ssse3-asm.o sha512-avx-asm.o sha512-avx2-asm.o sha512_ssse3_glue.o -obj-$(CONFIG_CRYPTO_BLAKE2S_X86) += blake2s-x86_64.o -blake2s-x86_64-y := blake2s-shash.o -obj-$(if $(CONFIG_CRYPTO_BLAKE2S_X86),y) += libblake2s-x86_64.o +obj-$(CONFIG_CRYPTO_BLAKE2S_X86) += libblake2s-x86_64.o libblake2s-x86_64-y := blake2s-core.o blake2s-glue.o obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o diff --git a/arch/x86/crypto/blake2s-glue.c b/arch/x86/crypto/blake2s-glue.c index 69853c13e8fb..aaba21230528 100644 --- a/arch/x86/crypto/blake2s-glue.c +++ b/arch/x86/crypto/blake2s-glue.c @@ -4,7 +4,6 @@ */ #include -#include #include #include @@ -33,7 +32,7 @@ void blake2s_compress(struct blake2s_state *state, const u8 *block, /* SIMD disables preemption, so relax after processing each page. */ BUILD_BUG_ON(SZ_4K / BLAKE2S_BLOCK_SIZE < 8); - if (!static_branch_likely(&blake2s_use_ssse3) || !crypto_simd_usable()) { + if (!static_branch_likely(&blake2s_use_ssse3) || !may_use_simd()) { blake2s_compress_generic(state, block, nblocks, inc); return; } diff --git a/arch/x86/crypto/blake2s-shash.c b/arch/x86/crypto/blake2s-shash.c deleted file mode 100644 index 59ae28abe35c..000000000000 --- a/arch/x86/crypto/blake2s-shash.c +++ /dev/null @@ -1,77 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 OR MIT -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#include -#include -#include - -#include -#include -#include -#include - -#include -#include - -static int crypto_blake2s_update_x86(struct shash_desc *desc, - const u8 *in, unsigned int inlen) -{ - return crypto_blake2s_update(desc, in, inlen, false); -} - -static int crypto_blake2s_final_x86(struct shash_desc *desc, u8 *out) -{ - return crypto_blake2s_final(desc, out, false); -} - -#define BLAKE2S_ALG(name, driver_name, digest_size) \ - { \ - .base.cra_name = name, \ - .base.cra_driver_name = driver_name, \ - .base.cra_priority = 200, \ - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, \ - .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, \ - .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), \ - .base.cra_module = THIS_MODULE, \ - .digestsize = digest_size, \ - .setkey = crypto_blake2s_setkey, \ - .init = crypto_blake2s_init, \ - .update = crypto_blake2s_update_x86, \ - .final = crypto_blake2s_final_x86, \ - .descsize = sizeof(struct blake2s_state), \ - } - -static struct shash_alg blake2s_algs[] = { - BLAKE2S_ALG("blake2s-128", "blake2s-128-x86", BLAKE2S_128_HASH_SIZE), - BLAKE2S_ALG("blake2s-160", "blake2s-160-x86", BLAKE2S_160_HASH_SIZE), - BLAKE2S_ALG("blake2s-224", "blake2s-224-x86", BLAKE2S_224_HASH_SIZE), - BLAKE2S_ALG("blake2s-256", "blake2s-256-x86", BLAKE2S_256_HASH_SIZE), -}; - -static int __init blake2s_mod_init(void) -{ - if (IS_REACHABLE(CONFIG_CRYPTO_HASH) && boot_cpu_has(X86_FEATURE_SSSE3)) - return crypto_register_shashes(blake2s_algs, ARRAY_SIZE(blake2s_algs)); - return 0; -} - -static void __exit blake2s_mod_exit(void) -{ - if (IS_REACHABLE(CONFIG_CRYPTO_HASH) && boot_cpu_has(X86_FEATURE_SSSE3)) - crypto_unregister_shashes(blake2s_algs, ARRAY_SIZE(blake2s_algs)); -} - -module_init(blake2s_mod_init); -module_exit(blake2s_mod_exit); - -MODULE_ALIAS_CRYPTO("blake2s-128"); -MODULE_ALIAS_CRYPTO("blake2s-128-x86"); -MODULE_ALIAS_CRYPTO("blake2s-160"); -MODULE_ALIAS_CRYPTO("blake2s-160-x86"); -MODULE_ALIAS_CRYPTO("blake2s-224"); -MODULE_ALIAS_CRYPTO("blake2s-224-x86"); -MODULE_ALIAS_CRYPTO("blake2s-256"); -MODULE_ALIAS_CRYPTO("blake2s-256-x86"); -MODULE_LICENSE("GPL v2"); diff --git a/crypto/Kconfig b/crypto/Kconfig index 6e30e8138057..59489a300cd1 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -712,26 +712,8 @@ config CRYPTO_BLAKE2B See https://blake2.net for further information. -config CRYPTO_BLAKE2S - tristate "BLAKE2s digest algorithm" - select CRYPTO_LIB_BLAKE2S_GENERIC - select CRYPTO_HASH - help - Implementation of cryptographic hash function BLAKE2s - optimized for 8-32bit platforms and can produce digests of any size - between 1 to 32. The keyed hash is also implemented. - - This module provides the following algorithms: - - - blake2s-128 - - blake2s-160 - - blake2s-224 - - blake2s-256 - - See https://blake2.net for further information. - config CRYPTO_BLAKE2S_X86 - tristate "BLAKE2s digest algorithm (x86 accelerated version)" + bool "BLAKE2s digest algorithm (x86 accelerated version)" depends on X86 && 64BIT select CRYPTO_LIB_BLAKE2S_GENERIC select CRYPTO_ARCH_HAVE_LIB_BLAKE2S diff --git a/crypto/Makefile b/crypto/Makefile index 1f529704fe80..a4a84860fe43 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -84,7 +84,6 @@ obj-$(CONFIG_CRYPTO_STREEBOG) += streebog_generic.o obj-$(CONFIG_CRYPTO_WP512) += wp512.o CFLAGS_wp512.o := $(call cc-option,-fno-schedule-insns) # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=79149 obj-$(CONFIG_CRYPTO_BLAKE2B) += blake2b_generic.o -obj-$(CONFIG_CRYPTO_BLAKE2S) += blake2s_generic.o obj-$(CONFIG_CRYPTO_GF128MUL) += gf128mul.o obj-$(CONFIG_CRYPTO_ECB) += ecb.o obj-$(CONFIG_CRYPTO_CBC) += cbc.o diff --git a/crypto/blake2s_generic.c b/crypto/blake2s_generic.c deleted file mode 100644 index 5f96a21f8788..000000000000 --- a/crypto/blake2s_generic.c +++ /dev/null @@ -1,75 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 OR MIT -/* - * shash interface to the generic implementation of BLAKE2s - * - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#include -#include - -#include -#include -#include - -static int crypto_blake2s_update_generic(struct shash_desc *desc, - const u8 *in, unsigned int inlen) -{ - return crypto_blake2s_update(desc, in, inlen, true); -} - -static int crypto_blake2s_final_generic(struct shash_desc *desc, u8 *out) -{ - return crypto_blake2s_final(desc, out, true); -} - -#define BLAKE2S_ALG(name, driver_name, digest_size) \ - { \ - .base.cra_name = name, \ - .base.cra_driver_name = driver_name, \ - .base.cra_priority = 100, \ - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, \ - .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, \ - .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), \ - .base.cra_module = THIS_MODULE, \ - .digestsize = digest_size, \ - .setkey = crypto_blake2s_setkey, \ - .init = crypto_blake2s_init, \ - .update = crypto_blake2s_update_generic, \ - .final = crypto_blake2s_final_generic, \ - .descsize = sizeof(struct blake2s_state), \ - } - -static struct shash_alg blake2s_algs[] = { - BLAKE2S_ALG("blake2s-128", "blake2s-128-generic", - BLAKE2S_128_HASH_SIZE), - BLAKE2S_ALG("blake2s-160", "blake2s-160-generic", - BLAKE2S_160_HASH_SIZE), - BLAKE2S_ALG("blake2s-224", "blake2s-224-generic", - BLAKE2S_224_HASH_SIZE), - BLAKE2S_ALG("blake2s-256", "blake2s-256-generic", - BLAKE2S_256_HASH_SIZE), -}; - -static int __init blake2s_mod_init(void) -{ - return crypto_register_shashes(blake2s_algs, ARRAY_SIZE(blake2s_algs)); -} - -static void __exit blake2s_mod_exit(void) -{ - crypto_unregister_shashes(blake2s_algs, ARRAY_SIZE(blake2s_algs)); -} - -subsys_initcall(blake2s_mod_init); -module_exit(blake2s_mod_exit); - -MODULE_ALIAS_CRYPTO("blake2s-128"); -MODULE_ALIAS_CRYPTO("blake2s-128-generic"); -MODULE_ALIAS_CRYPTO("blake2s-160"); -MODULE_ALIAS_CRYPTO("blake2s-160-generic"); -MODULE_ALIAS_CRYPTO("blake2s-224"); -MODULE_ALIAS_CRYPTO("blake2s-224-generic"); -MODULE_ALIAS_CRYPTO("blake2s-256"); -MODULE_ALIAS_CRYPTO("blake2s-256-generic"); -MODULE_LICENSE("GPL v2"); diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 336598da8eac..a8831060c4ce 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -1670,10 +1670,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) ret += tcrypt_test("rmd160"); break; - case 41: - ret += tcrypt_test("blake2s-256"); - break; - case 42: ret += tcrypt_test("blake2b-512"); break; @@ -2250,10 +2246,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) test_hash_speed("rmd160", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; fallthrough; - case 316: - test_hash_speed("blake2s-256", sec, generic_hash_speed_template); - if (mode > 300 && mode < 400) break; - fallthrough; case 317: test_hash_speed("blake2b-512", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; @@ -2362,10 +2354,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) test_ahash_speed("rmd160", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; fallthrough; - case 416: - test_ahash_speed("blake2s-256", sec, generic_hash_speed_template); - if (mode > 400 && mode < 500) break; - fallthrough; case 417: test_ahash_speed("blake2b-512", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 485097461068..7a8a56749960 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -4375,30 +4375,6 @@ static const struct alg_test_desc alg_test_descs[] = { .suite = { .hash = __VECS(blake2b_512_tv_template) } - }, { - .alg = "blake2s-128", - .test = alg_test_hash, - .suite = { - .hash = __VECS(blakes2s_128_tv_template) - } - }, { - .alg = "blake2s-160", - .test = alg_test_hash, - .suite = { - .hash = __VECS(blakes2s_160_tv_template) - } - }, { - .alg = "blake2s-224", - .test = alg_test_hash, - .suite = { - .hash = __VECS(blakes2s_224_tv_template) - } - }, { - .alg = "blake2s-256", - .test = alg_test_hash, - .suite = { - .hash = __VECS(blakes2s_256_tv_template) - } }, { .alg = "cbc(aes)", .test = alg_test_skcipher, diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 808ba07baa04..4f3955ea40bf 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -34034,223 +34034,6 @@ static const struct hash_testvec blake2b_512_tv_template[] = {{ 0xae, 0x15, 0x81, 0x15, 0xd0, 0x88, 0xa0, 0x3c, }, }}; -static const struct hash_testvec blakes2s_128_tv_template[] = {{ - .digest = (u8[]){ 0x64, 0x55, 0x0d, 0x6f, 0xfe, 0x2c, 0x0a, 0x01, - 0xa1, 0x4a, 0xba, 0x1e, 0xad, 0xe0, 0x20, 0x0c, }, -}, { - .plaintext = blake2_ordered_sequence, - .psize = 64, - .digest = (u8[]){ 0xdc, 0x66, 0xca, 0x8f, 0x03, 0x86, 0x58, 0x01, - 0xb0, 0xff, 0xe0, 0x6e, 0xd8, 0xa1, 0xa9, 0x0e, }, -}, { - .ksize = 16, - .key = blake2_ordered_sequence, - .plaintext = blake2_ordered_sequence, - .psize = 1, - .digest = (u8[]){ 0x88, 0x1e, 0x42, 0xe7, 0xbb, 0x35, 0x80, 0x82, - 0x63, 0x7c, 0x0a, 0x0f, 0xd7, 0xec, 0x6c, 0x2f, }, -}, { - .ksize = 32, - .key = blake2_ordered_sequence, - .plaintext = blake2_ordered_sequence, - .psize = 7, - .digest = (u8[]){ 0xcf, 0x9e, 0x07, 0x2a, 0xd5, 0x22, 0xf2, 0xcd, - 0xa2, 0xd8, 0x25, 0x21, 0x80, 0x86, 0x73, 0x1c, }, -}, { - .ksize = 1, - .key = "B", - .plaintext = blake2_ordered_sequence, - .psize = 15, - .digest = (u8[]){ 0xf6, 0x33, 0x5a, 0x2c, 0x22, 0xa0, 0x64, 0xb2, - 0xb6, 0x3f, 0xeb, 0xbc, 0xd1, 0xc3, 0xe5, 0xb2, }, -}, { - .ksize = 16, - .key = blake2_ordered_sequence, - .plaintext = blake2_ordered_sequence, - .psize = 247, - .digest = (u8[]){ 0x72, 0x66, 0x49, 0x60, 0xf9, 0x4a, 0xea, 0xbe, - 0x1f, 0xf4, 0x60, 0xce, 0xb7, 0x81, 0xcb, 0x09, }, -}, { - .ksize = 32, - .key = blake2_ordered_sequence, - .plaintext = blake2_ordered_sequence, - .psize = 256, - .digest = (u8[]){ 0xd5, 0xa4, 0x0e, 0xc3, 0x16, 0xc7, 0x51, 0xa6, - 0x3c, 0xd0, 0xd9, 0x11, 0x57, 0xfa, 0x1e, 0xbb, }, -}}; - -static const struct hash_testvec blakes2s_160_tv_template[] = {{ - .plaintext = blake2_ordered_sequence, - .psize = 7, - .digest = (u8[]){ 0xb4, 0xf2, 0x03, 0x49, 0x37, 0xed, 0xb1, 0x3e, - 0x5b, 0x2a, 0xca, 0x64, 0x82, 0x74, 0xf6, 0x62, - 0xe3, 0xf2, 0x84, 0xff, }, -}, { - .plaintext = blake2_ordered_sequence, - .psize = 256, - .digest = (u8[]){ 0xaa, 0x56, 0x9b, 0xdc, 0x98, 0x17, 0x75, 0xf2, - 0xb3, 0x68, 0x83, 0xb7, 0x9b, 0x8d, 0x48, 0xb1, - 0x9b, 0x2d, 0x35, 0x05, }, -}, { - .ksize = 1, - .key = "B", - .digest = (u8[]){ 0x50, 0x16, 0xe7, 0x0c, 0x01, 0xd0, 0xd3, 0xc3, - 0xf4, 0x3e, 0xb1, 0x6e, 0x97, 0xa9, 0x4e, 0xd1, - 0x79, 0x65, 0x32, 0x93, }, -}, { - .ksize = 32, - .key = blake2_ordered_sequence, - .plaintext = blake2_ordered_sequence, - .psize = 1, - .digest = (u8[]){ 0x1c, 0x2b, 0xcd, 0x9a, 0x68, 0xca, 0x8c, 0x71, - 0x90, 0x29, 0x6c, 0x54, 0xfa, 0x56, 0x4a, 0xef, - 0xa2, 0x3a, 0x56, 0x9c, }, -}, { - .ksize = 16, - .key = blake2_ordered_sequence, - .plaintext = blake2_ordered_sequence, - .psize = 15, - .digest = (u8[]){ 0x36, 0xc3, 0x5f, 0x9a, 0xdc, 0x7e, 0xbf, 0x19, - 0x68, 0xaa, 0xca, 0xd8, 0x81, 0xbf, 0x09, 0x34, - 0x83, 0x39, 0x0f, 0x30, }, -}, { - .ksize = 1, - .key = "B", - .plaintext = blake2_ordered_sequence, - .psize = 64, - .digest = (u8[]){ 0x86, 0x80, 0x78, 0xa4, 0x14, 0xec, 0x03, 0xe5, - 0xb6, 0x9a, 0x52, 0x0e, 0x42, 0xee, 0x39, 0x9d, - 0xac, 0xa6, 0x81, 0x63, }, -}, { - .ksize = 32, - .key = blake2_ordered_sequence, - .plaintext = blake2_ordered_sequence, - .psize = 247, - .digest = (u8[]){ 0x2d, 0xd8, 0xd2, 0x53, 0x66, 0xfa, 0xa9, 0x01, - 0x1c, 0x9c, 0xaf, 0xa3, 0xe2, 0x9d, 0x9b, 0x10, - 0x0a, 0xf6, 0x73, 0xe8, }, -}}; - -static const struct hash_testvec blakes2s_224_tv_template[] = {{ - .plaintext = blake2_ordered_sequence, - .psize = 1, - .digest = (u8[]){ 0x61, 0xb9, 0x4e, 0xc9, 0x46, 0x22, 0xa3, 0x91, - 0xd2, 0xae, 0x42, 0xe6, 0x45, 0x6c, 0x90, 0x12, - 0xd5, 0x80, 0x07, 0x97, 0xb8, 0x86, 0x5a, 0xfc, - 0x48, 0x21, 0x97, 0xbb, }, -}, { - .plaintext = blake2_ordered_sequence, - .psize = 247, - .digest = (u8[]){ 0x9e, 0xda, 0xc7, 0x20, 0x2c, 0xd8, 0x48, 0x2e, - 0x31, 0x94, 0xab, 0x46, 0x6d, 0x94, 0xd8, 0xb4, - 0x69, 0xcd, 0xae, 0x19, 0x6d, 0x9e, 0x41, 0xcc, - 0x2b, 0xa4, 0xd5, 0xf6, }, -}, { - .ksize = 16, - .key = blake2_ordered_sequence, - .digest = (u8[]){ 0x32, 0xc0, 0xac, 0xf4, 0x3b, 0xd3, 0x07, 0x9f, - 0xbe, 0xfb, 0xfa, 0x4d, 0x6b, 0x4e, 0x56, 0xb3, - 0xaa, 0xd3, 0x27, 0xf6, 0x14, 0xbf, 0xb9, 0x32, - 0xa7, 0x19, 0xfc, 0xb8, }, -}, { - .ksize = 1, - .key = "B", - .plaintext = blake2_ordered_sequence, - .psize = 7, - .digest = (u8[]){ 0x73, 0xad, 0x5e, 0x6d, 0xb9, 0x02, 0x8e, 0x76, - 0xf2, 0x66, 0x42, 0x4b, 0x4c, 0xfa, 0x1f, 0xe6, - 0x2e, 0x56, 0x40, 0xe5, 0xa2, 0xb0, 0x3c, 0xe8, - 0x7b, 0x45, 0xfe, 0x05, }, -}, { - .ksize = 32, - .key = blake2_ordered_sequence, - .plaintext = blake2_ordered_sequence, - .psize = 15, - .digest = (u8[]){ 0x16, 0x60, 0xfb, 0x92, 0x54, 0xb3, 0x6e, 0x36, - 0x81, 0xf4, 0x16, 0x41, 0xc3, 0x3d, 0xd3, 0x43, - 0x84, 0xed, 0x10, 0x6f, 0x65, 0x80, 0x7a, 0x3e, - 0x25, 0xab, 0xc5, 0x02, }, -}, { - .ksize = 16, - .key = blake2_ordered_sequence, - .plaintext = blake2_ordered_sequence, - .psize = 64, - .digest = (u8[]){ 0xca, 0xaa, 0x39, 0x67, 0x9c, 0xf7, 0x6b, 0xc7, - 0xb6, 0x82, 0xca, 0x0e, 0x65, 0x36, 0x5b, 0x7c, - 0x24, 0x00, 0xfa, 0x5f, 0xda, 0x06, 0x91, 0x93, - 0x6a, 0x31, 0x83, 0xb5, }, -}, { - .ksize = 1, - .key = "B", - .plaintext = blake2_ordered_sequence, - .psize = 256, - .digest = (u8[]){ 0x90, 0x02, 0x26, 0xb5, 0x06, 0x9c, 0x36, 0x86, - 0x94, 0x91, 0x90, 0x1e, 0x7d, 0x2a, 0x71, 0xb2, - 0x48, 0xb5, 0xe8, 0x16, 0xfd, 0x64, 0x33, 0x45, - 0xb3, 0xd7, 0xec, 0xcc, }, -}}; - -static const struct hash_testvec blakes2s_256_tv_template[] = {{ - .plaintext = blake2_ordered_sequence, - .psize = 15, - .digest = (u8[]){ 0xd9, 0x7c, 0x82, 0x8d, 0x81, 0x82, 0xa7, 0x21, - 0x80, 0xa0, 0x6a, 0x78, 0x26, 0x83, 0x30, 0x67, - 0x3f, 0x7c, 0x4e, 0x06, 0x35, 0x94, 0x7c, 0x04, - 0xc0, 0x23, 0x23, 0xfd, 0x45, 0xc0, 0xa5, 0x2d, }, -}, { - .ksize = 32, - .key = blake2_ordered_sequence, - .digest = (u8[]){ 0x48, 0xa8, 0x99, 0x7d, 0xa4, 0x07, 0x87, 0x6b, - 0x3d, 0x79, 0xc0, 0xd9, 0x23, 0x25, 0xad, 0x3b, - 0x89, 0xcb, 0xb7, 0x54, 0xd8, 0x6a, 0xb7, 0x1a, - 0xee, 0x04, 0x7a, 0xd3, 0x45, 0xfd, 0x2c, 0x49, }, -}, { - .ksize = 1, - .key = "B", - .plaintext = blake2_ordered_sequence, - .psize = 1, - .digest = (u8[]){ 0x22, 0x27, 0xae, 0xaa, 0x6e, 0x81, 0x56, 0x03, - 0xa7, 0xe3, 0xa1, 0x18, 0xa5, 0x9a, 0x2c, 0x18, - 0xf4, 0x63, 0xbc, 0x16, 0x70, 0xf1, 0xe7, 0x4b, - 0x00, 0x6d, 0x66, 0x16, 0xae, 0x9e, 0x74, 0x4e, }, -}, { - .ksize = 16, - .key = blake2_ordered_sequence, - .plaintext = blake2_ordered_sequence, - .psize = 7, - .digest = (u8[]){ 0x58, 0x5d, 0xa8, 0x60, 0x1c, 0xa4, 0xd8, 0x03, - 0x86, 0x86, 0x84, 0x64, 0xd7, 0xa0, 0x8e, 0x15, - 0x2f, 0x05, 0xa2, 0x1b, 0xbc, 0xef, 0x7a, 0x34, - 0xb3, 0xc5, 0xbc, 0x4b, 0xf0, 0x32, 0xeb, 0x12, }, -}, { - .ksize = 32, - .key = blake2_ordered_sequence, - .plaintext = blake2_ordered_sequence, - .psize = 64, - .digest = (u8[]){ 0x89, 0x75, 0xb0, 0x57, 0x7f, 0xd3, 0x55, 0x66, - 0xd7, 0x50, 0xb3, 0x62, 0xb0, 0x89, 0x7a, 0x26, - 0xc3, 0x99, 0x13, 0x6d, 0xf0, 0x7b, 0xab, 0xab, - 0xbd, 0xe6, 0x20, 0x3f, 0xf2, 0x95, 0x4e, 0xd4, }, -}, { - .ksize = 1, - .key = "B", - .plaintext = blake2_ordered_sequence, - .psize = 247, - .digest = (u8[]){ 0x2e, 0x74, 0x1c, 0x1d, 0x03, 0xf4, 0x9d, 0x84, - 0x6f, 0xfc, 0x86, 0x32, 0x92, 0x49, 0x7e, 0x66, - 0xd7, 0xc3, 0x10, 0x88, 0xfe, 0x28, 0xb3, 0xe0, - 0xbf, 0x50, 0x75, 0xad, 0x8e, 0xa4, 0xe6, 0xb2, }, -}, { - .ksize = 16, - .key = blake2_ordered_sequence, - .plaintext = blake2_ordered_sequence, - .psize = 256, - .digest = (u8[]){ 0xb9, 0xd2, 0x81, 0x0e, 0x3a, 0xb1, 0x62, 0x9b, - 0xad, 0x44, 0x05, 0xf4, 0x92, 0x2e, 0x99, 0xc1, - 0x4a, 0x47, 0xbb, 0x5b, 0x6f, 0xb2, 0x96, 0xed, - 0xd5, 0x06, 0xb5, 0x3a, 0x7c, 0x7a, 0x65, 0x1d, }, -}}; - /* * Test vectors generated using https://github.com/google/hctr2 */ diff --git a/include/crypto/internal/blake2s.h b/include/crypto/internal/blake2s.h index 52363eee2b20..506d56530ca9 100644 --- a/include/crypto/internal/blake2s.h +++ b/include/crypto/internal/blake2s.h @@ -8,7 +8,6 @@ #define _CRYPTO_INTERNAL_BLAKE2S_H #include -#include #include void blake2s_compress_generic(struct blake2s_state *state, const u8 *block, @@ -19,111 +18,4 @@ void blake2s_compress(struct blake2s_state *state, const u8 *block, bool blake2s_selftest(void); -static inline void blake2s_set_lastblock(struct blake2s_state *state) -{ - state->f[0] = -1; -} - -/* Helper functions for BLAKE2s shared by the library and shash APIs */ - -static __always_inline void -__blake2s_update(struct blake2s_state *state, const u8 *in, size_t inlen, - bool force_generic) -{ - const size_t fill = BLAKE2S_BLOCK_SIZE - state->buflen; - - if (unlikely(!inlen)) - return; - if (inlen > fill) { - memcpy(state->buf + state->buflen, in, fill); - if (force_generic) - blake2s_compress_generic(state, state->buf, 1, - BLAKE2S_BLOCK_SIZE); - else - blake2s_compress(state, state->buf, 1, - BLAKE2S_BLOCK_SIZE); - state->buflen = 0; - in += fill; - inlen -= fill; - } - if (inlen > BLAKE2S_BLOCK_SIZE) { - const size_t nblocks = DIV_ROUND_UP(inlen, BLAKE2S_BLOCK_SIZE); - /* Hash one less (full) block than strictly possible */ - if (force_generic) - blake2s_compress_generic(state, in, nblocks - 1, - BLAKE2S_BLOCK_SIZE); - else - blake2s_compress(state, in, nblocks - 1, - BLAKE2S_BLOCK_SIZE); - in += BLAKE2S_BLOCK_SIZE * (nblocks - 1); - inlen -= BLAKE2S_BLOCK_SIZE * (nblocks - 1); - } - memcpy(state->buf + state->buflen, in, inlen); - state->buflen += inlen; -} - -static __always_inline void -__blake2s_final(struct blake2s_state *state, u8 *out, bool force_generic) -{ - blake2s_set_lastblock(state); - memset(state->buf + state->buflen, 0, - BLAKE2S_BLOCK_SIZE - state->buflen); /* Padding */ - if (force_generic) - blake2s_compress_generic(state, state->buf, 1, state->buflen); - else - blake2s_compress(state, state->buf, 1, state->buflen); - cpu_to_le32_array(state->h, ARRAY_SIZE(state->h)); - memcpy(out, state->h, state->outlen); -} - -/* Helper functions for shash implementations of BLAKE2s */ - -struct blake2s_tfm_ctx { - u8 key[BLAKE2S_KEY_SIZE]; - unsigned int keylen; -}; - -static inline int crypto_blake2s_setkey(struct crypto_shash *tfm, - const u8 *key, unsigned int keylen) -{ - struct blake2s_tfm_ctx *tctx = crypto_shash_ctx(tfm); - - if (keylen == 0 || keylen > BLAKE2S_KEY_SIZE) - return -EINVAL; - - memcpy(tctx->key, key, keylen); - tctx->keylen = keylen; - - return 0; -} - -static inline int crypto_blake2s_init(struct shash_desc *desc) -{ - const struct blake2s_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - struct blake2s_state *state = shash_desc_ctx(desc); - unsigned int outlen = crypto_shash_digestsize(desc->tfm); - - __blake2s_init(state, outlen, tctx->key, tctx->keylen); - return 0; -} - -static inline int crypto_blake2s_update(struct shash_desc *desc, - const u8 *in, unsigned int inlen, - bool force_generic) -{ - struct blake2s_state *state = shash_desc_ctx(desc); - - __blake2s_update(state, in, inlen, force_generic); - return 0; -} - -static inline int crypto_blake2s_final(struct shash_desc *desc, u8 *out, - bool force_generic) -{ - struct blake2s_state *state = shash_desc_ctx(desc); - - __blake2s_final(state, out, force_generic); - return 0; -} - #endif /* _CRYPTO_INTERNAL_BLAKE2S_H */ diff --git a/lib/crypto/blake2s-selftest.c b/lib/crypto/blake2s-selftest.c index 409e4b728770..66f505220f43 100644 --- a/lib/crypto/blake2s-selftest.c +++ b/lib/crypto/blake2s-selftest.c @@ -4,6 +4,8 @@ */ #include +#include +#include #include /* @@ -587,5 +589,44 @@ bool __init blake2s_selftest(void) } } + for (i = 0; i < 32; ++i) { + enum { TEST_ALIGNMENT = 16 }; + u8 unaligned_block[BLAKE2S_BLOCK_SIZE + TEST_ALIGNMENT - 1] + __aligned(TEST_ALIGNMENT); + u8 blocks[BLAKE2S_BLOCK_SIZE * 3]; + struct blake2s_state state1, state2; + + get_random_bytes(blocks, sizeof(blocks)); + get_random_bytes(&state, sizeof(state)); + +#if defined(CONFIG_CRYPTO_LIB_BLAKE2S_GENERIC) && \ + defined(CONFIG_CRYPTO_ARCH_HAVE_LIB_BLAKE2S) + memcpy(&state1, &state, sizeof(state1)); + memcpy(&state2, &state, sizeof(state2)); + blake2s_compress(&state1, blocks, 3, BLAKE2S_BLOCK_SIZE); + blake2s_compress_generic(&state2, blocks, 3, BLAKE2S_BLOCK_SIZE); + if (memcmp(&state1, &state2, sizeof(state1))) { + pr_err("blake2s random compress self-test %d: FAIL\n", + i + 1); + success = false; + } +#endif + + memcpy(&state1, &state, sizeof(state1)); + blake2s_compress(&state1, blocks, 1, BLAKE2S_BLOCK_SIZE); + for (l = 1; l < TEST_ALIGNMENT; ++l) { + memcpy(unaligned_block + l, blocks, + BLAKE2S_BLOCK_SIZE); + memcpy(&state2, &state, sizeof(state2)); + blake2s_compress(&state2, unaligned_block + l, 1, + BLAKE2S_BLOCK_SIZE); + if (memcmp(&state1, &state2, sizeof(state1))) { + pr_err("blake2s random compress align %d self-test %d: FAIL\n", + l, i + 1); + success = false; + } + } + } + return success; } diff --git a/lib/crypto/blake2s.c b/lib/crypto/blake2s.c index c71c09621c09..98e688c6d891 100644 --- a/lib/crypto/blake2s.c +++ b/lib/crypto/blake2s.c @@ -16,16 +16,44 @@ #include #include +static inline void blake2s_set_lastblock(struct blake2s_state *state) +{ + state->f[0] = -1; +} + void blake2s_update(struct blake2s_state *state, const u8 *in, size_t inlen) { - __blake2s_update(state, in, inlen, false); + const size_t fill = BLAKE2S_BLOCK_SIZE - state->buflen; + + if (unlikely(!inlen)) + return; + if (inlen > fill) { + memcpy(state->buf + state->buflen, in, fill); + blake2s_compress(state, state->buf, 1, BLAKE2S_BLOCK_SIZE); + state->buflen = 0; + in += fill; + inlen -= fill; + } + if (inlen > BLAKE2S_BLOCK_SIZE) { + const size_t nblocks = DIV_ROUND_UP(inlen, BLAKE2S_BLOCK_SIZE); + blake2s_compress(state, in, nblocks - 1, BLAKE2S_BLOCK_SIZE); + in += BLAKE2S_BLOCK_SIZE * (nblocks - 1); + inlen -= BLAKE2S_BLOCK_SIZE * (nblocks - 1); + } + memcpy(state->buf + state->buflen, in, inlen); + state->buflen += inlen; } EXPORT_SYMBOL(blake2s_update); void blake2s_final(struct blake2s_state *state, u8 *out) { WARN_ON(IS_ENABLED(DEBUG) && !out); - __blake2s_final(state, out, false); + blake2s_set_lastblock(state); + memset(state->buf + state->buflen, 0, + BLAKE2S_BLOCK_SIZE - state->buflen); /* Padding */ + blake2s_compress(state, state->buf, 1, state->buflen); + cpu_to_le32_array(state->h, ARRAY_SIZE(state->h)); + memcpy(out, state->h, state->outlen); memzero_explicit(state, sizeof(*state)); } EXPORT_SYMBOL(blake2s_final); @@ -38,12 +66,7 @@ static int __init blake2s_mod_init(void) return 0; } -static void __exit blake2s_mod_exit(void) -{ -} - module_init(blake2s_mod_init); -module_exit(blake2s_mod_exit); MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("BLAKE2s hash function"); MODULE_AUTHOR("Jason A. Donenfeld "); From b03c0dc0788abccc7a25ef7dff5818f4123bb992 Mon Sep 17 00:00:00 2001 From: Shijith Thotton Date: Wed, 1 Jun 2022 13:38:59 +0530 Subject: [PATCH 022/318] crypto: octeontx2 - fix potential null pointer access Added missing checks to avoid null pointer dereference. The patch fixes below issue reported by klocwork tool: . Pointer 'strsep( &val, ":" )' returned from call to function 'strsep' at line 1608 may be NULL and will be dereferenced at line 1608. Also there are 2 similar errors on lines 1620, 1632 in otx2_cptpf_ucode.c. Signed-off-by: Shijith Thotton Signed-off-by: Herbert Xu --- .../crypto/marvell/octeontx2/otx2_cptpf_ucode.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c index 46ffb7ae982c..f10050fead16 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c @@ -1605,7 +1605,10 @@ int otx2_cpt_dl_custom_egrp_create(struct otx2_cptpf_dev *cptpf, if (!strncasecmp(val, "se", 2) && strchr(val, ':')) { if (has_se || ucode_idx) goto err_print; - tmp = strim(strsep(&val, ":")); + tmp = strsep(&val, ":"); + if (!tmp) + goto err_print; + tmp = strim(tmp); if (!val) goto err_print; if (strlen(tmp) != 2) @@ -1617,7 +1620,10 @@ int otx2_cpt_dl_custom_egrp_create(struct otx2_cptpf_dev *cptpf, } else if (!strncasecmp(val, "ae", 2) && strchr(val, ':')) { if (has_ae || ucode_idx) goto err_print; - tmp = strim(strsep(&val, ":")); + tmp = strsep(&val, ":"); + if (!tmp) + goto err_print; + tmp = strim(tmp); if (!val) goto err_print; if (strlen(tmp) != 2) @@ -1629,7 +1635,10 @@ int otx2_cpt_dl_custom_egrp_create(struct otx2_cptpf_dev *cptpf, } else if (!strncasecmp(val, "ie", 2) && strchr(val, ':')) { if (has_ie || ucode_idx) goto err_print; - tmp = strim(strsep(&val, ":")); + tmp = strsep(&val, ":"); + if (!tmp) + goto err_print; + tmp = strim(tmp); if (!val) goto err_print; if (strlen(tmp) != 2) From 7e8df1fc2d669d04c1f8a9e2d61d7afba1b43df4 Mon Sep 17 00:00:00 2001 From: Peng Wu Date: Thu, 2 Jun 2022 07:22:34 +0000 Subject: [PATCH 023/318] crypto: sun8i-ss - fix a NULL vs IS_ERR() check in sun8i_ss_hashkey The crypto_alloc_shash() function never returns NULL. It returns error pointers. Fixes: 801b7d572c0a ("crypto: sun8i-ss - add hmac(sha1)") Signed-off-by: Peng Wu Reported-by: Hulk Robot Signed-off-by: Herbert Xu --- drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c index 845019bd9591..36a82b22953c 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c @@ -30,8 +30,8 @@ static int sun8i_ss_hashkey(struct sun8i_ss_hash_tfm_ctx *tfmctx, const u8 *key, int ret = 0; xtfm = crypto_alloc_shash("sha1", 0, CRYPTO_ALG_NEED_FALLBACK); - if (!xtfm) - return -ENOMEM; + if (IS_ERR(xtfm)) + return PTR_ERR(xtfm); len = sizeof(*sdesc) + crypto_shash_descsize(xtfm); sdesc = kmalloc(len, GFP_KERNEL); From 9691e4f9ba6c7dc6af07b8a4feba6279d76f0003 Mon Sep 17 00:00:00 2001 From: Jonas Lindner Date: Thu, 9 Jun 2022 00:36:16 +0200 Subject: [PATCH 024/318] selinux: fix typos in comments Signed-off-by: Jonas Lindner [PM: fixed duplicated subject line] Signed-off-by: Paul Moore --- security/selinux/hooks.c | 4 ++-- security/selinux/include/audit.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 4af4986d3893..4d20a139a86d 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -640,7 +640,7 @@ static int selinux_set_mnt_opts(struct super_block *sb, * we need to skip the double mount verification. * * This does open a hole in which we will not notice if the first - * mount using this sb set explict options and a second mount using + * mount using this sb set explicit options and a second mount using * this sb does not set any security options. (The first options * will be used for both mounts) */ @@ -6795,7 +6795,7 @@ static u32 bpf_map_fmode_to_av(fmode_t fmode) } /* This function will check the file pass through unix socket or binder to see - * if it is a bpf related object. And apply correspinding checks on the bpf + * if it is a bpf related object. And apply corresponding checks on the bpf * object based on the type. The bpf maps and programs, not like other files and * socket, are using a shared anonymous inode inside the kernel as their inode. * So checking that inode cannot identify if the process have privilege to diff --git a/security/selinux/include/audit.h b/security/selinux/include/audit.h index 1cba83d17f41..406bceb90c6c 100644 --- a/security/selinux/include/audit.h +++ b/security/selinux/include/audit.h @@ -18,7 +18,7 @@ /** * selinux_audit_rule_init - alloc/init an selinux audit rule structure. * @field: the field this rule refers to - * @op: the operater the rule uses + * @op: the operator the rule uses * @rulestr: the text "target" of the rule * @rule: pointer to the new rule structure returned via this * From df4bf98ec2d199ae32f215d083774ebb6424bf1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20S=C3=B6derlund?= Date: Mon, 13 Jun 2022 11:05:10 +0200 Subject: [PATCH 025/318] scripts: kernel-doc: Always increment warnings counter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some warnings do not increment the warnings counter making the behavior of running kernel-doc with -Werror unlogical as some warnings will be generated but not treated as errors. Fix this by creating a helper function that always incrementing the warnings counter every time a warning is emitted. There is one location in get_sphinx_version() where a warning is not touched as it concerns the execution environment of the kernel-doc and not the documentation being processed. Incrementing the counter only have effect when running kernel-doc in either verbose mode (-v or environment variable KBUILD_VERBOSE) or when treating warnings as errors (-Werror or environment variable KDOC_WERROR). In both cases the number of warnings printed is printed to stderr and for the later the exit code of kernel-doc is non-zero if warnings where encountered. Simple test case to demo one of the warnings, $ cat test.c /** * foo() - Description */ int bar(); # Without this change $ ./scripts/kernel-doc -Werror -none test.c test.c:4: warning: expecting prototype for foo(). Prototype was for bar() instead # With this change $ ./scripts/kernel-doc -Werror -none test.c test.c:4: warning: expecting prototype for foo(). Prototype was for bar() instead 1 warnings as Errors Signed-off-by: Niklas Söderlund Link: https://lore.kernel.org/r/20220613090510.3088294-1-niklas.soderlund@corigine.com Signed-off-by: Jonathan Corbet --- scripts/kernel-doc | 82 ++++++++++++++++++++-------------------------- 1 file changed, 35 insertions(+), 47 deletions(-) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index 7516949bb049..aea04365bc69 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -427,6 +427,13 @@ sub print_lineno { print ".. LINENO " . $lineno . "\n"; } } + +sub emit_warning { + my $location = shift; + my $msg = shift; + print STDERR "$location: warning: $msg"; + ++$warnings; +} ## # dumps section contents to arrays/hashes intended for that purpose. # @@ -451,8 +458,7 @@ sub dump_section { if (defined($sections{$name}) && ($sections{$name} ne "")) { # Only warn on user specified duplicate section names. if ($name ne $section_default) { - print STDERR "${file}:$.: warning: duplicate section name '$name'\n"; - ++$warnings; + emit_warning("${file}:$.", "duplicate section name '$name'\n"); } $sections{$name} .= $contents; } else { @@ -1094,7 +1100,7 @@ sub dump_struct($$) { if ($members) { if ($identifier ne $declaration_name) { - print STDERR "${file}:$.: warning: expecting prototype for $decl_type $identifier. Prototype was for $decl_type $declaration_name instead\n"; + emit_warning("${file}:$.", "expecting prototype for $decl_type $identifier. Prototype was for $decl_type $declaration_name instead\n"); return; } @@ -1298,9 +1304,9 @@ sub dump_enum($$) { if ($members) { if ($identifier ne $declaration_name) { if ($identifier eq "") { - print STDERR "${file}:$.: warning: wrong kernel-doc identifier on line:\n"; + emit_warning("${file}:$.", "wrong kernel-doc identifier on line:\n"); } else { - print STDERR "${file}:$.: warning: expecting prototype for enum $identifier. Prototype was for enum $declaration_name instead\n"; + emit_warning("${file}:$.", "expecting prototype for enum $identifier. Prototype was for enum $declaration_name instead\n"); } return; } @@ -1316,7 +1322,7 @@ sub dump_enum($$) { if (!$parameterdescs{$arg}) { $parameterdescs{$arg} = $undescribed; if (show_warnings("enum", $declaration_name)) { - print STDERR "${file}:$.: warning: Enum value '$arg' not described in enum '$declaration_name'\n"; + emit_warning("${file}:$.", "Enum value '$arg' not described in enum '$declaration_name'\n"); } } $_members{$arg} = 1; @@ -1325,7 +1331,7 @@ sub dump_enum($$) { while (my ($k, $v) = each %parameterdescs) { if (!exists($_members{$k})) { if (show_warnings("enum", $declaration_name)) { - print STDERR "${file}:$.: warning: Excess enum value '$k' description in '$declaration_name'\n"; + emit_warning("${file}:$.", "Excess enum value '$k' description in '$declaration_name'\n"); } } } @@ -1367,7 +1373,7 @@ sub dump_typedef($$) { $return_type =~ s/^\s+//; if ($identifier ne $declaration_name) { - print STDERR "${file}:$.: warning: expecting prototype for typedef $identifier. Prototype was for typedef $declaration_name instead\n"; + emit_warning("${file}:$.", "expecting prototype for typedef $identifier. Prototype was for typedef $declaration_name instead\n"); return; } @@ -1398,7 +1404,7 @@ sub dump_typedef($$) { $declaration_name = $1; if ($identifier ne $declaration_name) { - print STDERR "${file}:$.: warning: expecting prototype for typedef $identifier. Prototype was for typedef $declaration_name instead\n"; + emit_warning("${file}:$.", "expecting prototype for typedef $identifier. Prototype was for typedef $declaration_name instead\n"); return; } @@ -1554,9 +1560,7 @@ sub push_parameter($$$$$) { $parameterdescs{$param} = $undescribed; if (show_warnings($type, $declaration_name) && $param !~ /\./) { - print STDERR - "${file}:$.: warning: Function parameter or member '$param' not described in '$declaration_name'\n"; - ++$warnings; + emit_warning("${file}:$.", "Function parameter or member '$param' not described in '$declaration_name'\n"); } } @@ -1604,11 +1608,10 @@ sub check_sections($$$$$) { } if ($err) { if ($decl_type eq "function") { - print STDERR "${file}:$.: warning: " . + emit_warning("${file}:$.", "Excess function parameter " . "'$sects[$sx]' " . - "description in '$decl_name'\n"; - ++$warnings; + "description in '$decl_name'\n"); } } } @@ -1629,10 +1632,9 @@ sub check_return_section { if (!defined($sections{$section_return}) || $sections{$section_return} eq "") { - print STDERR "${file}:$.: warning: " . + emit_warning("${file}:$.", "No description found for return value of " . - "'$declaration_name'\n"; - ++$warnings; + "'$declaration_name'\n"); } } @@ -1714,12 +1716,12 @@ sub dump_function($$) { create_parameterlist($args, ',', $file, $declaration_name); } else { - print STDERR "${file}:$.: warning: cannot understand function prototype: '$prototype'\n"; + emit_warning("${file}:$.", "cannot understand function prototype: '$prototype'\n"); return; } if ($identifier ne $declaration_name) { - print STDERR "${file}:$.: warning: expecting prototype for $identifier(). Prototype was for $declaration_name() instead\n"; + emit_warning("${file}:$.", "expecting prototype for $identifier(). Prototype was for $declaration_name() instead\n"); return; } @@ -1801,8 +1803,8 @@ sub tracepoint_munge($) { $tracepointargs = $1; } if (($tracepointname eq 0) || ($tracepointargs eq 0)) { - print STDERR "${file}:$.: warning: Unrecognized tracepoint format: \n". - "$prototype\n"; + emit_warning("${file}:$.", "Unrecognized tracepoint format: \n". + "$prototype\n"); } else { $prototype = "static inline void trace_$tracepointname($tracepointargs)"; $identifier = "trace_$identifier"; @@ -2027,22 +2029,16 @@ sub process_name($$) { } if (!$is_kernel_comment) { - print STDERR "${file}:$.: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst\n"; - print STDERR $_; - ++$warnings; + emit_warning("${file}:$.", "This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst\n$_"); $state = STATE_NORMAL; } if (($declaration_purpose eq "") && $verbose) { - print STDERR "${file}:$.: warning: missing initial short description on line:\n"; - print STDERR $_; - ++$warnings; + emit_warning("${file}:$.", "missing initial short description on line:\n$_"); } if ($identifier eq "" && $decl_type ne "enum") { - print STDERR "${file}:$.: warning: wrong kernel-doc identifier on line:\n"; - print STDERR $_; - ++$warnings; + emit_warning("${file}:$.", "wrong kernel-doc identifier on line:\n$_"); $state = STATE_NORMAL; } @@ -2050,9 +2046,7 @@ sub process_name($$) { print STDERR "${file}:$.: info: Scanning doc for $decl_type $identifier\n"; } } else { - print STDERR "${file}:$.: warning: Cannot understand $_ on line $.", - " - I thought it was a doc line\n"; - ++$warnings; + emit_warning("${file}:$.", "Cannot understand $_ on line $. - I thought it was a doc line\n"); $state = STATE_NORMAL; } } @@ -2071,8 +2065,7 @@ sub process_body($$) { $section =~ s/\.\.\.$//; if ($verbose) { - print STDERR "${file}:$.: warning: Variable macro arguments should be documented without dots\n"; - ++$warnings; + emit_warning("${file}:$.", "Variable macro arguments should be documented without dots\n"); } } @@ -2101,8 +2094,7 @@ sub process_body($$) { if (($contents ne "") && ($contents ne "\n")) { if (!$in_doc_sect && $verbose) { - print STDERR "${file}:$.: warning: contents before sections\n"; - ++$warnings; + emit_warning("${file}:$.", "contents before sections\n"); } dump_section($file, $section, $contents); $section = $section_default; @@ -2128,8 +2120,7 @@ sub process_body($$) { } # look for doc_com + + doc_end: if ($_ =~ m'\s*\*\s*[a-zA-Z_0-9:\.]+\*/') { - print STDERR "${file}:$.: warning: suspicious ending line: $_"; - ++$warnings; + emit_warning("${file}:$.", "suspicious ending line: $_"); } $prototype = ""; @@ -2173,8 +2164,7 @@ sub process_body($$) { } } else { # i dont know - bad line? ignore. - print STDERR "${file}:$.: warning: bad line: $_"; - ++$warnings; + emit_warning("${file}:$.", "bad line: $_"); } } @@ -2268,9 +2258,7 @@ sub process_inline($$) { } } elsif ($inline_doc_state == STATE_INLINE_NAME) { $inline_doc_state = STATE_INLINE_ERROR; - print STDERR "${file}:$.: warning: "; - print STDERR "Incorrect use of kernel-doc format: $_"; - ++$warnings; + emit_warning("${file}:$.", "Incorrect use of kernel-doc format: $_"); } } } @@ -2319,11 +2307,11 @@ sub process_file($) { if ($initial_section_counter == $section_counter && $ output_mode ne "none") { if ($output_selection == OUTPUT_INCLUDE) { - print STDERR "${file}:1: warning: '$_' not found\n" + emit_warning("${file}:1", "'$_' not found\n") for keys %function_table; } else { - print STDERR "${file}:1: warning: no structured comments found\n"; + emit_warning("${file}:1", "no structured comments found\n"); } } close IN_FILE; From 94bdaa7d9a9d778d893c670d494c06d8e007c1cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Li=C5=A1ka?= Date: Mon, 23 May 2022 14:18:06 +0200 Subject: [PATCH 026/318] docs/arm64: elf_hwcaps: Unify HWCAP lists as description lists Add a series of blank lines to this document to get consistend rendering as an RST description list. Signed-off-by: Martin Liska Link: https://lore.kernel.org/r/d0e576ab-6121-b7d7-da5b-7750f05ca7f4@suse.cz Signed-off-by: Jonathan Corbet --- Documentation/arm64/elf_hwcaps.rst | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/Documentation/arm64/elf_hwcaps.rst b/Documentation/arm64/elf_hwcaps.rst index 3d116fb536c5..37f0116c39fa 100644 --- a/Documentation/arm64/elf_hwcaps.rst +++ b/Documentation/arm64/elf_hwcaps.rst @@ -171,96 +171,73 @@ HWCAP_PACG Documentation/arm64/pointer-authentication.rst. HWCAP2_DCPODP - Functionality implied by ID_AA64ISAR1_EL1.DPB == 0b0010. HWCAP2_SVE2 - Functionality implied by ID_AA64ZFR0_EL1.SVEVer == 0b0001. HWCAP2_SVEAES - Functionality implied by ID_AA64ZFR0_EL1.AES == 0b0001. HWCAP2_SVEPMULL - Functionality implied by ID_AA64ZFR0_EL1.AES == 0b0010. HWCAP2_SVEBITPERM - Functionality implied by ID_AA64ZFR0_EL1.BitPerm == 0b0001. HWCAP2_SVESHA3 - Functionality implied by ID_AA64ZFR0_EL1.SHA3 == 0b0001. HWCAP2_SVESM4 - Functionality implied by ID_AA64ZFR0_EL1.SM4 == 0b0001. HWCAP2_FLAGM2 - Functionality implied by ID_AA64ISAR0_EL1.TS == 0b0010. HWCAP2_FRINT - Functionality implied by ID_AA64ISAR1_EL1.FRINTTS == 0b0001. HWCAP2_SVEI8MM - Functionality implied by ID_AA64ZFR0_EL1.I8MM == 0b0001. HWCAP2_SVEF32MM - Functionality implied by ID_AA64ZFR0_EL1.F32MM == 0b0001. HWCAP2_SVEF64MM - Functionality implied by ID_AA64ZFR0_EL1.F64MM == 0b0001. HWCAP2_SVEBF16 - Functionality implied by ID_AA64ZFR0_EL1.BF16 == 0b0001. HWCAP2_I8MM - Functionality implied by ID_AA64ISAR1_EL1.I8MM == 0b0001. HWCAP2_BF16 - Functionality implied by ID_AA64ISAR1_EL1.BF16 == 0b0001. HWCAP2_DGH - Functionality implied by ID_AA64ISAR1_EL1.DGH == 0b0001. HWCAP2_RNG - Functionality implied by ID_AA64ISAR0_EL1.RNDR == 0b0001. HWCAP2_BTI - Functionality implied by ID_AA64PFR0_EL1.BT == 0b0001. HWCAP2_MTE - Functionality implied by ID_AA64PFR1_EL1.MTE == 0b0010, as described by Documentation/arm64/memory-tagging-extension.rst. HWCAP2_ECV - Functionality implied by ID_AA64MMFR0_EL1.ECV == 0b0001. HWCAP2_AFP - Functionality implied by ID_AA64MFR1_EL1.AFP == 0b0001. HWCAP2_RPRES - Functionality implied by ID_AA64ISAR2_EL1.RPRES == 0b0001. HWCAP2_MTE3 - Functionality implied by ID_AA64PFR1_EL1.MTE == 0b0011, as described by Documentation/arm64/memory-tagging-extension.rst. From d5b1d57b425c53e1e7e743265bbd4c71093bbc12 Mon Sep 17 00:00:00 2001 From: Wu XiangCheng Date: Wed, 18 May 2022 11:16:28 +0800 Subject: [PATCH 027/318] docs/zh_CN: Update translation of reporting-issues.rst to 5.18 Update zh_CN/admin-guide/reporting-issues.rst to newest English version commit 247097e2bbff4 ("docs: reporting-issues.rst: link new document about regressions") Signed-off-by: Wu XiangCheng Reviewed-by: Alex Shi Link: https://lore.kernel.org/r/YoRlDOUl/eg+h/Cc@bobwxc.mipc Signed-off-by: Jonathan Corbet --- .../zh_CN/admin-guide/reporting-issues.rst | 125 +++++++++++------- 1 file changed, 79 insertions(+), 46 deletions(-) diff --git a/Documentation/translations/zh_CN/admin-guide/reporting-issues.rst b/Documentation/translations/zh_CN/admin-guide/reporting-issues.rst index 6b4988da2c5a..59e51e3539b4 100644 --- a/Documentation/translations/zh_CN/admin-guide/reporting-issues.rst +++ b/Documentation/translations/zh_CN/admin-guide/reporting-issues.rst @@ -1,13 +1,6 @@ .. SPDX-License-Identifier: (GPL-2.0+ OR CC-BY-4.0) -.. - If you want to distribute this text under CC-BY-4.0 only, please use 'The - Linux kernel developers' for author attribution and link this as source: - https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/plain/Documentation/admin-guide/reporting-issues.rst -.. - Note: Only the content of this RST file as found in the Linux kernel sources - is available under CC-BY-4.0, as versions of this text that were processed - (for example by the kernel's build system) might contain content taken from - files which use a more restrictive license. +.. See the bottom of this file for additional redistribution information. + .. include:: ../disclaimer-zh_CN.rst @@ -29,7 +22,9 @@ 请搜索 `LKML内核邮件列表 `_ 和 `Linux稳定版邮件列表 `_ 存档中匹配的报告并 加入讨论。如果找不到匹配的报告，请安装该系列的最新版本。如果它仍然出现问题， -报告给稳定版邮件列表（stable@vger.kernel.org）。 +请报告给稳定版邮件列表（stable@vger.kernel.org）并抄送回归邮件列表 +（regressions@lists.linux.dev）；理想情况下，还可以抄送维护者和相关子系统的 +邮件列表。 在所有其他情况下，请尽可能猜测是哪个内核部分导致了问题。查看MAINTAINERS文件， 了解开发人员希望如何得知问题，大多数情况下，报告问题都是通过电子邮件和抄送 @@ -46,9 +41,10 @@ 有使用附加模块）。还要确保它是在一个正常的环境中构建和运行，并且在问题发生 之前没有被污染（tainted）。 -在编写报告时，要涵盖与问题相关的所有信息，如使用的内核和发行版。在碰见回归时， -尝试给出引入它的更改的提交ID，二分可以找到它。如果您同时面临Linux内核的多个 -问题，请分别报告每个问题。 +当你同时面临Linux内核的多个问题时，请分别报告。在编写报告时，要涵盖与问题 +相关的所有信息，如使用的内核和发行版。如果碰见回归，请把报告抄送回归邮件列表 +（regressions@lists.linux.dev）。也请试试用二分法找出源头；如果成功找到，请 +在报告中写上它的提交ID并抄送sign-off-by链中的所有人。 一旦报告发出，请回答任何出现的问题，并尽可能地提供帮助。这包括通过不时重新 测试新版本并发送状态更新来推动进展。 @@ -156,9 +152,10 @@ 存在问题，因为问题可能已经在那里被修复了。如果您第一次发现供应商内核的问题， 请检查已知最新版本的普通构建是否可以正常运行。 - * 向Linux稳定版邮件列表发送一个简短的问题报告(stable@vger.kernel.org)。大致 - 描述问题，并解释如何复现。讲清楚首个出现问题的版本和最后一个工作正常的版本。 - 然后等待进一步的指示。 + * 向Linux稳定版邮件列表发送一个简短的问题报告（stable@vger.kernel.org）并抄送 + Linux回归邮件列表（regressions@lists.linux.dev）；如果你怀疑是由某子系统 + 引起的，请抄送其维护人员和子系统邮件列表。大致描述问题，并解释如何复现。 + 讲清楚首个出现问题的版本和最后一个工作正常的版本。然后等待进一步的指示。 下面的参考章节部分详细解释了这些步骤中的每一步。 @@ -296,17 +293,14 @@ Linus Torvalds和主要的Linux内核开发人员希望看到一些问题尽快 报告过程中有一些“高优先级问题”的处理略有不同。有三种情况符合条件:回归、安全 问题和非常严重的问题。 -如果在旧版本的Linux内核中工作的东西不能在新版本的Linux内核中工作，或者某种 -程度上在新版本的Linux内核中工作得更差，那么你就需要处理“回归”。因此，当一个 -在Linux 5.7中表现良好的WiFi驱动程序在5.8中表现不佳或根本不能工作时，这是一 -种回归。如果应用程序在新的内核中出现不稳定的现象，这也是一种回归，这可能是 -由于内核和用户空间之间的接口（如procfs和sysfs）发生不兼容的更改造成的。显著 -的性能降低或功耗增加也可以称为回归。但是请记住:新内核需要使用与旧内核相似的 -配置来构建（参见下面如何实现这一点）。这是因为内核开发人员在实现新特性时有 -时无法避免不兼容性；但是为了避免回归，这些特性必须在构建配置期间显式地启用。 +如果某个应用程序或实际用例在原先的Linux内核上运行良好，但在使用类似配置编译的 +较新版本上效果更差、或者根本不能用，那么你就需要处理回归问题。 +Documentation/admin-guide/reporting-regressions.rst 对此进行了更详细的解释。 +它还提供了很多你可能想知道的关于回归的其他信息；例如，它解释了如何将您的问题 +添加到回归跟踪列表中，以确保它不会被忽略。 什么是安全问题留给您自己判断。在继续之前，请考虑阅读 -“Documentation/translations/zh_CN/admin-guide/security-bugs.rst”， +Documentation/translations/zh_CN/admin-guide/security-bugs.rst ， 因为它提供了如何最恰当地处理安全问题的额外细节。 当发生了完全无法接受的糟糕事情时，此问题就是一个“非常严重的问题”。例如， @@ -390,7 +384,7 @@ Linux内核破坏了它处理的数据或损坏了它运行的硬件。当内核 核未被污染，那么它应该以“Not infected”结束；如果你看到“Tainted:”且后跟一些 空格和字母，那就被污染了。 -如果你的内核被污染了，请阅读“Documentation/translations/zh_CN/admin-guide/tainted-kernels.rst” +如果你的内核被污染了，请阅读 Documentation/translations/zh_CN/admin-guide/tainted-kernels.rst 以找出原因。设法消除污染因素。通常是由以下三种因素之一引起的： 1. 发生了一个可恢复的错误（“kernel Oops”），内核污染了自己，因为内核知道在 @@ -591,7 +585,8 @@ ath10k@lists.infradead.org”，将引导您到ath10k邮件列表的信息页， 搜索引擎，并添加类似“site:lists.infadead.org/pipermail/ath10k/”这 样的搜索条件，这会把结果限制在该链接中的档案。 -也请进一步搜索网络、LKML和bugzilla.kernel.org网站。 +也请进一步搜索网络、LKML和bugzilla.kernel.org网站。如果你的报告需要发送到缺陷 +跟踪器中，那么您可能还需要检查子系统的邮件列表存档，因为可能有人只在那里报告了它。 有关如何搜索以及在找到匹配报告时如何操作的详细信息，请参阅上面的“搜索现有报告 （第一部分）”。 @@ -802,10 +797,10 @@ Linux 首席开发者 Linus Torvalds 认为 Linux 内核永远不应恶化，这 重现它。 有一个叫做“二分”的过程可以来寻找变化，这在 -“Documentation/translations/zh_CN/admin-guide/bug-bisect.rst”文档中进行了详细 +Documentation/translations/zh_CN/admin-guide/bug-bisect.rst 文档中进行了详细 的描述，这个过程通常需要你构建十到二十个内核镜像，每次都尝试在构建下一个镜像 之前重现问题。是的，这需要花费一些时间，但不用担心，它比大多数人想象的要快得多。 -多亏了“binary search二进制搜索”，这将引导你在源代码管理系统中找到导致回归的提交。 +多亏了“binary search二分搜索”，这将引导你在源代码管理系统中找到导致回归的提交。 一旦你找到它，就在网上搜索其主题、提交ID和缩短的提交ID（提交ID的前12个字符）。 如果有的话，这将引导您找到关于它的现有报告。 @@ -823,10 +818,10 @@ Linux 首席开发者 Linus Torvalds 认为 Linux 内核永远不应恶化，这 当处理回归问题时，请确保你所面临的问题真的是由内核引起的，而不是由其他东西 引起的，如上文所述。 -在整个过程中，请记住：只有当旧内核和新内核的配置相似时，问题才算回归。最好 -的方法是：把配置文件（``.config``）从旧的工作内核直接复制到你尝试的每个新内 -核版本。之后运行 ``make oldnoconfig`` 来调整它以适应新版本的需要，而不启用 -任何新的功能，因为那些功能也可能导致回归。 +在整个过程中，请记住：只有当旧内核和新内核的配置相似时，问题才算回归。这可以 +通过 ``make olddefconfig`` 来实现，详细解释参见 +Documentation/admin-guide/reporting-regressions.rst ；它还提供了大量其他您 +可能希望了解的有关回归的信息。 撰写并发送报告 @@ -959,11 +954,19 @@ Linux 首席开发者 Linus Torvalds 认为 Linux 内核永远不应恶化，这 **非常严重的缺陷** ：确保在主题或工单标题以及第一段中明显标出 severeness （非常严重的）。 -**回归** ：如果问题是一个回归，请在邮件的主题或缺陷跟踪器的标题中添加 -[REGRESSION]。如果您没有进行二分，请至少注明您测试的最新主线版本（比如 5.7） -和出现问题的最新版本（比如 5.8）。如果您成功地进行了二分，请注明导致回归 -的提交ID和主题。也请添加该变更的作者到你的报告中；如果您需要将您的缺陷提交 -到缺陷跟踪器中，请将报告以私人邮件的形式转发给他，并注明报告提交地点。 +**回归** ：报告的主题应以“[REGRESSION]”开头。 + +如果您成功用二分法定位了问题，请使用引入回归之更改的标题作为主题的第二部分。 +请在报告中写明“罪魁祸首”的提交ID。如果未能成功二分，请在报告中讲明最后一个 +正常工作的版本（例如5.7）和最先发生问题的版本（例如5.8-rc1）。 + +通过邮件发送报告时，请抄送Linux回归邮件列表（regressions@lists.linux.dev）。 +如果报告需要提交到某个web追踪器，请继续提交；并在提交后，通过邮件将报告转发 +至回归列表；抄送相关子系统的维护人员和邮件列表。请确保报告是内联转发的，不要 +把它作为附件。另外请在顶部添加一个简短的说明，在那里写上工单的网址。 + +在邮寄或转发报告时，如果成功二分，需要将“罪魁祸首”的作者添加到收件人中；同时 +抄送signed-off-by链中的每个人，您可以在提交消息的末尾找到。 **安全问题** ：对于这种问题，你将必须评估：如果细节被公开披露，是否会对其他 用户产生短期风险。如果不会，只需按照所述继续报告问题。如果有此风险，你需要 @@ -980,7 +983,7 @@ Linux 首席开发者 Linus Torvalds 认为 Linux 内核永远不应恶化，这 报告，请将报告的文本转发到这些地址；但请在报告的顶部加上注释，表明您提交了 报告，并附上工单链接。 -更多信息请参见“Documentation/translations/zh_CN/admin-guide/security-bugs.rst”。 +更多信息请参见 Documentation/translations/zh_CN/admin-guide/security-bugs.rst 。 发布报告后的责任 @@ -1173,14 +1176,18 @@ FLOSS 问题报告的人看，询问他们的意见。同时征求他们关于 报告回归 ~~~~~~~~~~ - *向Linux稳定版邮件列表发送一个简短的问题报告(stable@vger.kernel.org)。 - 大致描述问题，并解释如何复现。讲清楚首个出现问题的版本和最后一个工作正常 - 的版本。然后等待进一步的指示。* + *向Linux稳定版邮件列表发送一个简短的问题报告(stable@vger.kernel.org)并 + 抄送Linux回归邮件列表（regressions@lists.linux.dev）；如果你怀疑是由某 + 子系统引起的，请抄送其维护人员和子系统邮件列表。大致描述问题，并解释如 + 何复现。讲清楚首个出现问题的版本和最后一个工作正常的版本。然后等待进一 + 步的指示。* 当报告在稳定版或长期支持内核线内发生的回归（例如在从5.10.4更新到5.10.5时）， -一份简短的报告足以快速报告问题。因此只需要粗略的描述。 +一份简短的报告足以快速报告问题。因此只需向稳定版和回归邮件列表发送粗略的描述； +不过如果你怀疑某子系统导致此问题的话，请一并抄送其维护人员和子系统邮件列表， +这会加快进程。 -但是请注意，如果您能够指明引入问题的确切版本，这将对开发人员有很大帮助。因此 +请注意，如果您能够指明引入问题的确切版本，这将对开发人员有很大帮助。因此 如果有时间的话，请尝试使用普通内核找到该版本。让我们假设发行版发布Linux内核 5.10.5到5.10.8的更新时发生了故障。那么按照上面的指示，去检查该版本线中的最新 内核，比如5.10.9。如果问题出现，请尝试普通5.10.5，以确保供应商应用的补丁不会 @@ -1190,7 +1197,9 @@ FLOSS 问题报告的人看，询问他们的意见。同时征求他们关于 前一段基本粗略地概述了“二分”方法。一旦报告出来，您可能会被要求做一个正确的 报告，因为它允许精确地定位导致问题的确切更改（然后很容易被恢复以快速修复问题）。 因此如果时间允许，考虑立即进行适当的二分。有关如何详细信息，请参阅“对回归的 -特别关照”部分和文档“Documentation/translations/zh_CN/admin-guide/bug-bisect.rst”。 +特别关照”部分和文档 Documentation/translations/zh_CN/admin-guide/bug-bisect.rst 。 +如果成功二分的话，请将“罪魁祸首”的作者添加到收件人中；同时抄送所有在 +signed-off-by链中的人，您可以在提交消息的末尾找到。 “报告仅在旧内核版本线中发生的问题”的参考 @@ -1207,7 +1216,7 @@ FLOSS 问题报告的人看，询问他们的意见。同时征求他们关于 即使是微小的、看似明显的代码变化，有时也会带来新的、完全意想不到的问题。稳 定版和长期支持内核的维护者非常清楚这一点，因此他们只对这些内核进行符合 -“Documentation/translations/zh_CN/process/stable-kernel-rules.rst”中所列出的 +Documentation/translations/zh_CN/process/stable-kernel-rules.rst 中所列出的 规则的修改。 复杂或有风险的修改不符合条件，因此只能应用于主线。其他的修复很容易被回溯到 @@ -1333,3 +1342,27 @@ FLOSS 问题报告的人看，询问他们的意见。同时征求他们关于 向 Linux 内核开发者报告问题是很难的：这个文档的长度和复杂性以及字里行间的内 涵都说明了这一点。但目前就是这样了。这篇文字的主要作者希望通过记录现状来为 以后改善这种状况打下一些基础。 + + +.. + end-of-content +.. + This English version of this document is maintained by Thorsten Leemhuis + . If you spot a typo or small mistake, feel free to + let him know directly and he'll fix it. For translation problems, please + contact with translators. You are free to do the same in a mostly informal + way if you want to contribute changes to the text, but for copyright + reasons please CC linux-doc@vger.kernel.org and "sign-off" your + contribution as Documentation/process/submitting-patches.rst outlines in + the section "Sign your work - the Developer's Certificate of Origin". +.. + This text is available under GPL-2.0+ or CC-BY-4.0, as stated at the top + of the file. If you want to distribute this text under CC-BY-4.0 only, + please use "The Linux kernel developers" for author attribution and link + this as source: + https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/plain/Documentation/admin-guide/reporting-issues.rst +.. + Note: Only the content of this RST file as found in the Linux kernel sources + is available under CC-BY-4.0, as versions of this text that were processed + (for example by the kernel's build system) might contain content taken from + files which use a more restrictive license. From 2adb5bb1b860705b9d4f3ac272f37b2d68649f7b Mon Sep 17 00:00:00 2001 From: Juerg Haefliger Date: Tue, 17 May 2022 16:08:37 +0200 Subject: [PATCH 028/318] docs: Kconfig: Fix help text indentation The convention for help text indentation seems to be a single tab followed by two whitespaces. Do that. While at it, remove a stray empty line. Signed-off-by: Juerg Haefliger Link: https://lore.kernel.org/r/20220517140837.331298-1-juergh@canonical.com Signed-off-by: Jonathan Corbet --- Documentation/Kconfig | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/Documentation/Kconfig b/Documentation/Kconfig index e549a61f4d96..252bfc164dbd 100644 --- a/Documentation/Kconfig +++ b/Documentation/Kconfig @@ -1,23 +1,22 @@ config WARN_MISSING_DOCUMENTS - bool "Warn if there's a missing documentation file" depends on COMPILE_TEST help - It is not uncommon that a document gets renamed. - This option makes the Kernel to check for missing dependencies, - warning when something is missing. Works only if the Kernel - is built from a git tree. + It is not uncommon that a document gets renamed. + This option makes the Kernel to check for missing dependencies, + warning when something is missing. Works only if the Kernel + is built from a git tree. - If unsure, select 'N'. + If unsure, select 'N'. config WARN_ABI_ERRORS bool "Warn if there are errors at ABI files" depends on COMPILE_TEST help - The files under Documentation/ABI should follow what's - described at Documentation/ABI/README. Yet, as they're manually - written, it would be possible that some of those files would - have errors that would break them for being parsed by - scripts/get_abi.pl. Add a check to verify them. + The files under Documentation/ABI should follow what's + described at Documentation/ABI/README. Yet, as they're manually + written, it would be possible that some of those files would + have errors that would break them for being parsed by + scripts/get_abi.pl. Add a check to verify them. - If unsure, select 'N'. + If unsure, select 'N'. From 546093206ba16623c18e344630dbfdd71a4327e0 Mon Sep 17 00:00:00 2001 From: Xiu Jianfeng Date: Sat, 11 Jun 2022 17:23:04 +0800 Subject: [PATCH 029/318] audit: make is_audit_feature_set() static Currently nobody use is_audit_feature_set() outside this file, so make it static. Signed-off-by: Xiu Jianfeng Signed-off-by: Paul Moore --- include/linux/audit.h | 2 -- kernel/audit.c | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/include/linux/audit.h b/include/linux/audit.h index cece70231138..00f7a80f1a3e 100644 --- a/include/linux/audit.h +++ b/include/linux/audit.h @@ -119,8 +119,6 @@ enum audit_nfcfgop { AUDIT_NFT_OP_INVALID, }; -extern int is_audit_feature_set(int which); - extern int __init audit_register_class(int class, unsigned *list); extern int audit_classify_syscall(int abi, unsigned syscall); extern int audit_classify_arch(int arch); diff --git a/kernel/audit.c b/kernel/audit.c index 0749211d5552..a75978ae38ad 100644 --- a/kernel/audit.c +++ b/kernel/audit.c @@ -1100,7 +1100,7 @@ static inline void audit_log_user_recv_msg(struct audit_buffer **ab, audit_log_common_recv_msg(NULL, ab, msg_type); } -int is_audit_feature_set(int i) +static int is_audit_feature_set(int i) { return af.features & AUDIT_FEATURE_TO_MASK(i); } From 2bfe15c5261212130f1a71f32a300bcf426443d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20G=C3=B6ttsche?= Date: Tue, 25 Jan 2022 15:33:04 +0100 Subject: [PATCH 030/318] mm: create security context for memfd_secret inodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Create a security context for the inodes created by memfd_secret(2) via the LSM hook inode_init_security_anon to allow a fine grained control. As secret memory areas can affect hibernation and have a global shared limit access control might be desirable. Signed-off-by: Christian Göttsche Signed-off-by: Paul Moore --- mm/secretmem.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mm/secretmem.c b/mm/secretmem.c index 206ed6b40c1d..f544ec66ebaf 100644 --- a/mm/secretmem.c +++ b/mm/secretmem.c @@ -180,11 +180,20 @@ static struct file *secretmem_file_create(unsigned long flags) { struct file *file = ERR_PTR(-ENOMEM); struct inode *inode; + const char *anon_name = "[secretmem]"; + const struct qstr qname = QSTR_INIT(anon_name, strlen(anon_name)); + int err; inode = alloc_anon_inode(secretmem_mnt->mnt_sb); if (IS_ERR(inode)) return ERR_CAST(inode); + err = security_inode_init_security_anon(inode, &qname, NULL); + if (err) { + file = ERR_PTR(err); + goto err_free_inode; + } + file = alloc_file_pseudo(inode, secretmem_mnt, "secretmem", O_RDWR, &secretmem_fops); if (IS_ERR(file)) From 8d6d51edcb79b0906288170df165c1b86e278218 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Mon, 28 Feb 2022 20:14:54 -0800 Subject: [PATCH 031/318] docs: selinux: add '=' signs to kernel boot options Provide the full kernel boot option string (with ending '=' sign). They won't work without that and that is how other boot options are listed. If used without an '=' sign (as listed here), they cause an "Unknown parameters" message and are added to init's argument strings, polluting them. Unknown kernel command line parameters "enforcing checkreqprot BOOT_IMAGE=/boot/bzImage-517rc6", will be passed to user space. Run /sbin/init as init process with arguments: /sbin/init enforcing checkreqprot with environment: HOME=/ TERM=linux BOOT_IMAGE=/boot/bzImage-517rc6 Signed-off-by: Randy Dunlap Cc: Paul Moore Cc: Stephen Smalley Cc: Eric Paris Cc: selinux@vger.kernel.org Cc: Jonathan Corbet [PM: removed bogus 'Fixes' line] Signed-off-by: Paul Moore --- Documentation/admin-guide/kernel-parameters.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 8090130b544b..815af6d7a199 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -550,7 +550,7 @@ nosocket -- Disable socket memory accounting. nokmem -- Disable kernel memory accounting. - checkreqprot [SELINUX] Set initial checkreqprot flag value. + checkreqprot= [SELINUX] Set initial checkreqprot flag value. Format: { "0" | "1" } See security/selinux/Kconfig help text. 0 -- check protection applied by kernel (includes @@ -1439,7 +1439,7 @@ (in particular on some ATI chipsets). The kernel tries to set a reasonable default. - enforcing [SELINUX] Set initial enforcing status. + enforcing= [SELINUX] Set initial enforcing status. Format: {"0" | "1"} See security/selinux/Kconfig help text. 0 -- permissive (log only, no denials). From 73de1befcc53a7c68b0c5e76b9b5ac41c517760f Mon Sep 17 00:00:00 2001 From: Xiu Jianfeng Date: Mon, 13 Jun 2022 21:59:53 +0800 Subject: [PATCH 032/318] selinux: fix memleak in security_read_state_kernel() In this function, it directly returns the result of __security_read_policy without freeing the allocated memory in *data, cause memory leak issue, so free the memory if __security_read_policy failed. Signed-off-by: Xiu Jianfeng [PM: subject line tweak] Signed-off-by: Paul Moore --- security/selinux/ss/services.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/security/selinux/ss/services.c b/security/selinux/ss/services.c index 69b2734311a6..fe5fcf571c56 100644 --- a/security/selinux/ss/services.c +++ b/security/selinux/ss/services.c @@ -4048,6 +4048,7 @@ int security_read_policy(struct selinux_state *state, int security_read_state_kernel(struct selinux_state *state, void **data, size_t *len) { + int err; struct selinux_policy *policy; policy = rcu_dereference_protected( @@ -4060,5 +4061,11 @@ int security_read_state_kernel(struct selinux_state *state, if (!*data) return -ENOMEM; - return __security_read_policy(policy, *data, len); + err = __security_read_policy(policy, *data, len); + if (err) { + vfree(*data); + *data = NULL; + *len = 0; + } + return err; } From 15ec76fb29be31df2bccb30fc09875274cba2776 Mon Sep 17 00:00:00 2001 From: Xiu Jianfeng Date: Tue, 14 Jun 2022 10:14:49 +0800 Subject: [PATCH 033/318] selinux: Add boundary check in put_entry() Just like next_entry(), boundary check is necessary to prevent memory out-of-bound access. Signed-off-by: Xiu Jianfeng Signed-off-by: Paul Moore --- security/selinux/ss/policydb.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/security/selinux/ss/policydb.h b/security/selinux/ss/policydb.h index c24d4e1063ea..ffc4e7bad205 100644 --- a/security/selinux/ss/policydb.h +++ b/security/selinux/ss/policydb.h @@ -370,6 +370,8 @@ static inline int put_entry(const void *buf, size_t bytes, int num, struct polic { size_t len = bytes * num; + if (len > fp->len) + return -EINVAL; memcpy(fp->data, buf, len); fp->data += len; fp->len -= len; From ce6330f74b08c8cbb2e3c04bd31cbace3de20660 Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Thu, 9 Jun 2022 02:56:39 +0000 Subject: [PATCH 034/318] MAINTAINERS: update HiSilicon ZIP and QM maintainers This patch splits QM and ZIP in MAINTAINERS, then add Weili Qian for QM driver and Yang Shen for ZIP driver. This patch adds missing Kconfig and Makefile files as well. Signed-off-by: Zhou Wang Signed-off-by: Herbert Xu --- MAINTAINERS | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index a6d3bd9d2a8d..b04fb3324837 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -8936,15 +8936,24 @@ F: Documentation/admin-guide/perf/hisi-pcie-pmu.rst F: Documentation/admin-guide/perf/hisi-pmu.rst F: drivers/perf/hisilicon -HISILICON QM AND ZIP Controller DRIVER +HISILICON QM DRIVER +M: Weili Qian +M: Zhou Wang +L: linux-crypto@vger.kernel.org +S: Maintained +F: drivers/crypto/hisilicon/Kconfig +F: drivers/crypto/hisilicon/Makefile +F: drivers/crypto/hisilicon/qm.c +F: drivers/crypto/hisilicon/sgl.c +F: include/linux/hisi_acc_qm.h + +HISILICON ZIP Controller DRIVER +M: Yang Shen M: Zhou Wang L: linux-crypto@vger.kernel.org S: Maintained F: Documentation/ABI/testing/debugfs-hisi-zip -F: drivers/crypto/hisilicon/qm.c -F: drivers/crypto/hisilicon/sgl.c F: drivers/crypto/hisilicon/zip/ -F: include/linux/hisi_acc_qm.h HISILICON ROCE DRIVER M: Wenpeng Liang From 00856e5391fbfbe00bf641a39f1bf8d1a144367a Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Thu, 9 Jun 2022 19:18:19 +0800 Subject: [PATCH 035/318] crypto: hisilicon/trng - fix local variable type The return value of 'readl_poll_timeout' is '0' or '-ETIMEDOUT'. Therefore, change the local variable 'ret' type from 'u32' to 'int'. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/trng/trng.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/hisilicon/trng/trng.c b/drivers/crypto/hisilicon/trng/trng.c index 829f2caf0f67..97e500db0a82 100644 --- a/drivers/crypto/hisilicon/trng/trng.c +++ b/drivers/crypto/hisilicon/trng/trng.c @@ -185,7 +185,7 @@ static int hisi_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait) struct hisi_trng *trng; int currsize = 0; u32 val = 0; - u32 ret; + int ret; trng = container_of(rng, struct hisi_trng, rng); From bf081d6fa8e90aefe991e34a29eff7aa22deb3ff Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Thu, 9 Jun 2022 20:31:17 +0800 Subject: [PATCH 036/318] crypto: hisilicon/qm - add functions for releasing resources The resources allocated by hisi_qm_memory_init() are released by hisi_qm_uninit(). Add hisi_qm_memory_uninit() to release resources, no functional change. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index b4ca2eb034d7..903896ab5be5 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -3672,6 +3672,21 @@ static void qm_last_regs_uninit(struct hisi_qm *qm) debug->qm_last_words = NULL; } +static void hisi_qm_memory_uninit(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + + hisi_qp_memory_uninit(qm, qm->qp_num); + if (qm->qdma.va) { + hisi_qm_cache_wb(qm); + dma_free_coherent(dev, qm->qdma.size, + qm->qdma.va, qm->qdma.dma); + } + + idr_destroy(&qm->qp_idr); + kfree(qm->factor); +} + /** * hisi_qm_uninit() - Uninitialize qm. * @qm: The qm needed uninit. @@ -3680,13 +3695,9 @@ static void qm_last_regs_uninit(struct hisi_qm *qm) */ void hisi_qm_uninit(struct hisi_qm *qm) { - struct pci_dev *pdev = qm->pdev; - struct device *dev = &pdev->dev; - qm_last_regs_uninit(qm); qm_cmd_uninit(qm); - kfree(qm->factor); down_write(&qm->qps_lock); if (!qm_avail_state(qm, QM_CLOSE)) { @@ -3694,14 +3705,7 @@ void hisi_qm_uninit(struct hisi_qm *qm) return; } - hisi_qp_memory_uninit(qm, qm->qp_num); - idr_destroy(&qm->qp_idr); - - if (qm->qdma.va) { - hisi_qm_cache_wb(qm); - dma_free_coherent(dev, qm->qdma.size, - qm->qdma.va, qm->qdma.dma); - } + hisi_qm_memory_uninit(qm); hisi_qm_set_state(qm, QM_NOT_READY); up_write(&qm->qps_lock); From 3099fc9c2b3aaace80947d07d13b40da2dd79fd4 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Thu, 9 Jun 2022 20:31:18 +0800 Subject: [PATCH 037/318] crypto: hisilicon/qm - move alloc qm->wq to qm.c Before stopping the function, the driver needs to flush all the remaining work about event irq. Therefore, accelerator drivers use a private workqueue(qm->wq) to handle event irq instead of the system workqueue. This patch moves alloc workqueue from sec_main.c and zip_main.c to qm.c. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 35 +++++++++++++++++------- drivers/crypto/hisilicon/sec2/sec_main.c | 24 +--------------- drivers/crypto/hisilicon/zip/zip_main.c | 17 +----------- 3 files changed, 27 insertions(+), 49 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 903896ab5be5..f8d36b68494e 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -959,10 +959,7 @@ static irqreturn_t do_qm_irq(int irq, void *data) struct hisi_qm *qm = (struct hisi_qm *)data; /* the workqueue created by device driver of QM */ - if (qm->wq) - queue_work(qm->wq, &qm->work); - else - schedule_work(&qm->work); + queue_work(qm->wq, &qm->work); return IRQ_HANDLED; } @@ -3134,11 +3131,8 @@ static int qm_stop_qp_nolock(struct hisi_qp *qp) if (ret) dev_err(dev, "Failed to drain out data for stopping!\n"); - if (qp->qm->wq) - flush_workqueue(qp->qm->wq); - else - flush_work(&qp->qm->work); + flush_workqueue(qp->qm->wq); if (unlikely(qp->is_resetting && atomic_read(&qp->qp_status.used))) qp_stop_fail_cb(qp); @@ -3672,6 +3666,11 @@ static void qm_last_regs_uninit(struct hisi_qm *qm) debug->qm_last_words = NULL; } +static void hisi_qm_unint_work(struct hisi_qm *qm) +{ + destroy_workqueue(qm->wq); +} + static void hisi_qm_memory_uninit(struct hisi_qm *qm) { struct device *dev = &qm->pdev->dev; @@ -3698,6 +3697,7 @@ void hisi_qm_uninit(struct hisi_qm *qm) qm_last_regs_uninit(qm); qm_cmd_uninit(qm); + hisi_qm_unint_work(qm); down_write(&qm->qps_lock); if (!qm_avail_state(qm, QM_CLOSE)) { @@ -6022,7 +6022,7 @@ err_disable_pcidev: return ret; } -static void hisi_qm_init_work(struct hisi_qm *qm) +static int hisi_qm_init_work(struct hisi_qm *qm) { INIT_WORK(&qm->work, qm_work_process); if (qm->fun_type == QM_HW_PF) @@ -6030,6 +6030,16 @@ static void hisi_qm_init_work(struct hisi_qm *qm) if (qm->ver > QM_HW_V2) INIT_WORK(&qm->cmd_process, qm_cmd_process); + + qm->wq = alloc_workqueue("%s", WQ_HIGHPRI | WQ_MEM_RECLAIM | + WQ_UNBOUND, num_online_cpus(), + pci_name(qm->pdev)); + if (!qm->wq) { + pci_err(qm->pdev, "failed to alloc workqueue!\n"); + return -ENOMEM; + } + + return 0; } static int hisi_qp_alloc_memory(struct hisi_qm *qm) @@ -6180,7 +6190,10 @@ int hisi_qm_init(struct hisi_qm *qm) if (ret) goto err_alloc_uacce; - hisi_qm_init_work(qm); + ret = hisi_qm_init_work(qm); + if (ret) + goto err_free_qm_memory; + qm_cmd_init(qm); atomic_set(&qm->status.flags, QM_INIT); @@ -6188,6 +6201,8 @@ int hisi_qm_init(struct hisi_qm *qm) return 0; +err_free_qm_memory: + hisi_qm_memory_uninit(qm); err_alloc_uacce: if (qm->use_sva) { uacce_remove(qm->uacce); diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 4d85d2cbf376..bdb690aaed12 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -1002,8 +1002,6 @@ static int sec_pf_probe_init(struct sec_dev *sec) static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) { - int ret; - qm->pdev = pdev; qm->ver = pdev->revision; qm->algs = "cipher\ndigest\naead"; @@ -1029,25 +1027,7 @@ static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) qm->qp_num = SEC_QUEUE_NUM_V1 - SEC_PF_DEF_Q_NUM; } - /* - * WQ_HIGHPRI: SEC request must be low delayed, - * so need a high priority workqueue. - * WQ_UNBOUND: SEC task is likely with long - * running CPU intensive workloads. - */ - qm->wq = alloc_workqueue("%s", WQ_HIGHPRI | WQ_MEM_RECLAIM | - WQ_UNBOUND, num_online_cpus(), - pci_name(qm->pdev)); - if (!qm->wq) { - pci_err(qm->pdev, "fail to alloc workqueue\n"); - return -ENOMEM; - } - - ret = hisi_qm_init(qm); - if (ret) - destroy_workqueue(qm->wq); - - return ret; + return hisi_qm_init(qm); } static void sec_qm_uninit(struct hisi_qm *qm) @@ -1078,8 +1058,6 @@ static int sec_probe_init(struct sec_dev *sec) static void sec_probe_uninit(struct hisi_qm *qm) { hisi_qm_dev_err_uninit(qm); - - destroy_workqueue(qm->wq); } static void sec_iommu_used_check(struct sec_dev *sec) diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 9c925e9c0a2d..c3303d99acac 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -990,8 +990,6 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) { - int ret; - qm->pdev = pdev; qm->ver = pdev->revision; if (pdev->revision >= QM_HW_V3) @@ -1021,25 +1019,12 @@ static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) qm->qp_num = HZIP_QUEUE_NUM_V1 - HZIP_PF_DEF_Q_NUM; } - qm->wq = alloc_workqueue("%s", WQ_HIGHPRI | WQ_MEM_RECLAIM | - WQ_UNBOUND, num_online_cpus(), - pci_name(qm->pdev)); - if (!qm->wq) { - pci_err(qm->pdev, "fail to alloc workqueue\n"); - return -ENOMEM; - } - - ret = hisi_qm_init(qm); - if (ret) - destroy_workqueue(qm->wq); - - return ret; + return hisi_qm_init(qm); } static void hisi_zip_qm_uninit(struct hisi_qm *qm) { hisi_qm_uninit(qm); - destroy_workqueue(qm->wq); } static int hisi_zip_probe_init(struct hisi_zip *hisi_zip) From d64de9773c18409d2161228242968ff3ebe3707e Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Thu, 9 Jun 2022 20:31:19 +0800 Subject: [PATCH 038/318] crypto: hisilicon/qm - modify event irq processing When the driver receives an event interrupt, the driver will enable the event interrupt after handling all completed tasks on the function, tasks on the function are parsed through only one thread. If the task's user callback takes time, other tasks on the function will be blocked. Therefore, the event irq processing is modified as follows: 1. Obtain the ID of the queue that completes the task. 2. Enable event interrupt. 3. Parse the completed tasks in the queue and call the user callback. Enabling event interrupt in advance can quickly report pending event interrupts and process tasks in multiple threads. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 142 ++++++++++++++++++++++------------ include/linux/hisi_acc_qm.h | 8 +- 2 files changed, 99 insertions(+), 51 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index f8d36b68494e..ad83c194d664 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -877,13 +877,6 @@ static void qm_pm_put_sync(struct hisi_qm *qm) pm_runtime_put_autosuspend(dev); } -static struct hisi_qp *qm_to_hisi_qp(struct hisi_qm *qm, struct qm_eqe *eqe) -{ - u16 cqn = le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK; - - return &qm->qp_array[cqn]; -} - static void qm_cq_head_update(struct hisi_qp *qp) { if (qp->qp_status.cq_head == QM_Q_DEPTH - 1) { @@ -894,47 +887,37 @@ static void qm_cq_head_update(struct hisi_qp *qp) } } -static void qm_poll_qp(struct hisi_qp *qp, struct hisi_qm *qm) +static void qm_poll_req_cb(struct hisi_qp *qp) { - if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP)) - return; + struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head; + struct hisi_qm *qm = qp->qm; - if (qp->event_cb) { - qp->event_cb(qp); - return; - } - - if (qp->req_cb) { - struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head; - - while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) { - dma_rmb(); - qp->req_cb(qp, qp->sqe + qm->sqe_size * - le16_to_cpu(cqe->sq_head)); - qm_cq_head_update(qp); - cqe = qp->cqe + qp->qp_status.cq_head; - qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ, - qp->qp_status.cq_head, 0); - atomic_dec(&qp->qp_status.used); - } - - /* set c_flag */ + while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) { + dma_rmb(); + qp->req_cb(qp, qp->sqe + qm->sqe_size * + le16_to_cpu(cqe->sq_head)); + qm_cq_head_update(qp); + cqe = qp->cqe + qp->qp_status.cq_head; qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ, - qp->qp_status.cq_head, 1); + qp->qp_status.cq_head, 0); + atomic_dec(&qp->qp_status.used); } + + /* set c_flag */ + qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ, qp->qp_status.cq_head, 1); } -static void qm_work_process(struct work_struct *work) +static int qm_get_complete_eqe_num(struct hisi_qm_poll_data *poll_data) { - struct hisi_qm *qm = container_of(work, struct hisi_qm, work); + struct hisi_qm *qm = poll_data->qm; struct qm_eqe *eqe = qm->eqe + qm->status.eq_head; - struct hisi_qp *qp; int eqe_num = 0; + u16 cqn; while (QM_EQE_PHASE(eqe) == qm->status.eqc_phase) { + cqn = le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK; + poll_data->qp_finish_id[eqe_num] = cqn; eqe_num++; - qp = qm_to_hisi_qp(qm, eqe); - qm_poll_qp(qp, qm); if (qm->status.eq_head == QM_EQ_DEPTH - 1) { qm->status.eqc_phase = !qm->status.eqc_phase; @@ -945,34 +928,70 @@ static void qm_work_process(struct work_struct *work) qm->status.eq_head++; } - if (eqe_num == QM_EQ_DEPTH / 2 - 1) { - eqe_num = 0; - qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); - } + if (eqe_num == (QM_EQ_DEPTH >> 1) - 1) + break; } qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); + + return eqe_num; } -static irqreturn_t do_qm_irq(int irq, void *data) +static void qm_work_process(struct work_struct *work) { - struct hisi_qm *qm = (struct hisi_qm *)data; + struct hisi_qm_poll_data *poll_data = + container_of(work, struct hisi_qm_poll_data, work); + struct hisi_qm *qm = poll_data->qm; + struct hisi_qp *qp; + int eqe_num, i; - /* the workqueue created by device driver of QM */ - queue_work(qm->wq, &qm->work); + /* Get qp id of completed tasks and re-enable the interrupt. */ + eqe_num = qm_get_complete_eqe_num(poll_data); + for (i = eqe_num - 1; i >= 0; i--) { + qp = &qm->qp_array[poll_data->qp_finish_id[i]]; + if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP)) + continue; - return IRQ_HANDLED; + if (qp->event_cb) { + qp->event_cb(qp); + continue; + } + + if (likely(qp->req_cb)) + qm_poll_req_cb(qp); + } +} + +static bool do_qm_irq(struct hisi_qm *qm) +{ + struct qm_eqe *eqe = qm->eqe + qm->status.eq_head; + struct hisi_qm_poll_data *poll_data; + u16 cqn; + + if (!readl(qm->io_base + QM_VF_EQ_INT_SOURCE)) + return false; + + if (QM_EQE_PHASE(eqe) == qm->status.eqc_phase) { + cqn = le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK; + poll_data = &qm->poll_data[cqn]; + queue_work(qm->wq, &poll_data->work); + + return true; + } + + return false; } static irqreturn_t qm_irq(int irq, void *data) { struct hisi_qm *qm = data; + bool ret; - if (readl(qm->io_base + QM_VF_EQ_INT_SOURCE)) - return do_qm_irq(irq, data); + ret = do_qm_irq(qm); + if (ret) + return IRQ_HANDLED; atomic64_inc(&qm->debug.dfx.err_irq_cnt); - dev_err(&qm->pdev->dev, "invalid int source\n"); qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); return IRQ_NONE; @@ -3551,8 +3570,10 @@ static void hisi_qp_memory_uninit(struct hisi_qm *qm, int num) for (i = num - 1; i >= 0; i--) { qdma = &qm->qp_array[i].qdma; dma_free_coherent(dev, qdma->size, qdma->va, qdma->dma); + kfree(qm->poll_data[i].qp_finish_id); } + kfree(qm->poll_data); kfree(qm->qp_array); } @@ -3561,12 +3582,18 @@ static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id) struct device *dev = &qm->pdev->dev; size_t off = qm->sqe_size * QM_Q_DEPTH; struct hisi_qp *qp; + int ret = -ENOMEM; + + qm->poll_data[id].qp_finish_id = kcalloc(qm->qp_num, sizeof(u16), + GFP_KERNEL); + if (!qm->poll_data[id].qp_finish_id) + return -ENOMEM; qp = &qm->qp_array[id]; qp->qdma.va = dma_alloc_coherent(dev, dma_size, &qp->qdma.dma, GFP_KERNEL); if (!qp->qdma.va) - return -ENOMEM; + goto err_free_qp_finish_id; qp->sqe = qp->qdma.va; qp->sqe_dma = qp->qdma.dma; @@ -3577,6 +3604,10 @@ static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id) qp->qp_id = id; return 0; + +err_free_qp_finish_id: + kfree(qm->poll_data[id].qp_finish_id); + return ret; } static void hisi_qm_pre_init(struct hisi_qm *qm) @@ -6024,7 +6055,11 @@ err_disable_pcidev: static int hisi_qm_init_work(struct hisi_qm *qm) { - INIT_WORK(&qm->work, qm_work_process); + int i; + + for (i = 0; i < qm->qp_num; i++) + INIT_WORK(&qm->poll_data[i].work, qm_work_process); + if (qm->fun_type == QM_HW_PF) INIT_WORK(&qm->rst_work, hisi_qm_controller_reset); @@ -6052,11 +6087,18 @@ static int hisi_qp_alloc_memory(struct hisi_qm *qm) if (!qm->qp_array) return -ENOMEM; + qm->poll_data = kcalloc(qm->qp_num, sizeof(struct hisi_qm_poll_data), GFP_KERNEL); + if (!qm->poll_data) { + kfree(qm->qp_array); + return -ENOMEM; + } + /* one more page for device or qp statuses */ qp_dma_size = qm->sqe_size * QM_Q_DEPTH + sizeof(struct qm_cqe) * QM_Q_DEPTH; qp_dma_size = PAGE_ALIGN(qp_dma_size) + PAGE_SIZE; for (i = 0; i < qm->qp_num; i++) { + qm->poll_data[i].qm = qm; ret = hisi_qp_memory_init(qm, qp_dma_size, i); if (ret) goto err_init_qp_mem; diff --git a/include/linux/hisi_acc_qm.h b/include/linux/hisi_acc_qm.h index 6cabafffd0dd..116e8bd68c99 100644 --- a/include/linux/hisi_acc_qm.h +++ b/include/linux/hisi_acc_qm.h @@ -265,6 +265,12 @@ struct hisi_qm_list { void (*unregister_from_crypto)(struct hisi_qm *qm); }; +struct hisi_qm_poll_data { + struct hisi_qm *qm; + struct work_struct work; + u16 *qp_finish_id; +}; + struct hisi_qm { enum qm_hw_ver ver; enum qm_fun_type fun_type; @@ -302,6 +308,7 @@ struct hisi_qm { struct rw_semaphore qps_lock; struct idr qp_idr; struct hisi_qp *qp_array; + struct hisi_qm_poll_data *poll_data; struct mutex mailbox_lock; @@ -312,7 +319,6 @@ struct hisi_qm { u32 error_mask; struct workqueue_struct *wq; - struct work_struct work; struct work_struct rst_work; struct work_struct cmd_process; From d61a7b3decf7f0cf4121a7204303deefd2c7151b Mon Sep 17 00:00:00 2001 From: Alexey Khoroshilov Date: Fri, 10 Jun 2022 21:27:15 +0300 Subject: [PATCH 039/318] crypto: sun8i-ss - fix infinite loop in sun8i_ss_setup_ivs() There is no i decrement in while (i >= 0) loop. Found by Linux Verification Center (linuxtesting.org) with SVACE. Signed-off-by: Alexey Khoroshilov Fixes: 359e893e8af4 ("crypto: sun8i-ss - rework handling of IV") Acked-by: Corentin Labbe Tested-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c index 5bb950182026..910d6751644c 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c @@ -170,6 +170,7 @@ dma_iv_error: while (i >= 0) { dma_unmap_single(ss->dev, rctx->p_iv[i], ivsize, DMA_TO_DEVICE); memzero_explicit(sf->iv[i], ivsize); + i--; } return err; } From bffa1fc065893a14703545efba7d69bb4082b18a Mon Sep 17 00:00:00 2001 From: Kai Ye Date: Sat, 11 Jun 2022 15:38:08 +0800 Subject: [PATCH 040/318] crypto: hisilicon/sec - only HW V2 needs to change the BD err detection The base register address of V2 and V3 are different. HW V3 not needs to change the BD err detection. Signed-off-by: Kai Ye Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sec2/sec_main.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index bdb690aaed12..2c0be91c0b09 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -508,16 +508,17 @@ static int sec_engine_init(struct hisi_qm *qm) writel(SEC_SAA_ENABLE, qm->io_base + SEC_SAA_EN_REG); - /* HW V2 enable sm4 extra mode, as ctr/ecb */ - if (qm->ver < QM_HW_V3) + if (qm->ver < QM_HW_V3) { + /* HW V2 enable sm4 extra mode, as ctr/ecb */ writel_relaxed(SEC_BD_ERR_CHK_EN0, qm->io_base + SEC_BD_ERR_CHK_EN_REG0); - /* Enable sm4 xts mode multiple iv */ - writel_relaxed(SEC_BD_ERR_CHK_EN1, - qm->io_base + SEC_BD_ERR_CHK_EN_REG1); - writel_relaxed(SEC_BD_ERR_CHK_EN3, - qm->io_base + SEC_BD_ERR_CHK_EN_REG3); + /* HW V2 enable sm4 xts mode multiple iv */ + writel_relaxed(SEC_BD_ERR_CHK_EN1, + qm->io_base + SEC_BD_ERR_CHK_EN_REG1); + writel_relaxed(SEC_BD_ERR_CHK_EN3, + qm->io_base + SEC_BD_ERR_CHK_EN_REG3); + } /* config endian */ sec_set_endian(qm); From 2403e8044f222e7c816fb2416661f5f469662973 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 21 Mar 2022 18:41:46 -0700 Subject: [PATCH 041/318] rcu: Make normal polling GP be more precise about sequence numbers Currently, poll_state_synchronize_rcu() uses rcu_seq_done() to check whether the specified grace period has completed. However, rcu_seq_done() does a simple comparison that reserves have of the sequence-number space for uncompleted grace periods. This has the unfortunate side-effect of not handling sequence-number wrap gracefully. Of course, one can argue that if someone has already waited for half of the full range of grace periods, they can wait for the other half, but why wait at all in this case? This commit therefore creates a rcu_seq_done_exact() that counts as uncompleted only the two grace periods during which the sequence number might have been handed out, while still being uncompleted. This way, if sequence-number wrap happens to hit that range, at most two additional grace periods need be waited for. This commit is in preparation for polled expedited grace periods. Link: https://lore.kernel.org/all/20220121142454.1994916-1-bfoster@redhat.com/ Link: https://docs.google.com/document/d/1RNKWW9jQyfjxw2E8dsXVTdvZYh0HnYeSHDKog9jhdN8/edit?usp=sharing Cc: Brian Foster Cc: Dave Chinner Cc: Al Viro Cc: Ian Kent Signed-off-by: Paul E. McKenney --- kernel/rcu/rcu.h | 12 ++++++++++++ kernel/rcu/tree.c | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index 4916077119f3..0adb55941aeb 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -119,6 +119,18 @@ static inline bool rcu_seq_done(unsigned long *sp, unsigned long s) return ULONG_CMP_GE(READ_ONCE(*sp), s); } +/* + * Given a snapshot from rcu_seq_snap(), determine whether or not a + * full update-side operation has occurred, but do not allow the + * (ULONG_MAX / 2) safety-factor/guard-band. + */ +static inline bool rcu_seq_done_exact(unsigned long *sp, unsigned long s) +{ + unsigned long cur_s = READ_ONCE(*sp); + + return ULONG_CMP_GE(cur_s, s) || ULONG_CMP_LT(cur_s, s - (2 * RCU_SEQ_STATE_MASK + 1)); +} + /* * Has a grace period completed since the time the old gp_seq was collected? */ diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index c25ba442044a..ec28e259774e 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -3911,7 +3911,7 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu); * * Yes, this function does not take counter wrap into account. * But counter wrap is harmless. If the counter wraps, we have waited for - * more than 2 billion grace periods (and way more on a 64-bit system!). + * more than a billion grace periods (and way more on a 64-bit system!). * Those needing to keep oldstate values for very long time periods * (many hours even on 32-bit systems) should check them occasionally * and either refresh them or set a flag indicating that the grace period @@ -3924,7 +3924,7 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu); */ bool poll_state_synchronize_rcu(unsigned long oldstate) { - if (rcu_seq_done(&rcu_state.gp_seq, oldstate)) { + if (rcu_seq_done_exact(&rcu_state.gp_seq, oldstate)) { smp_mb(); /* Ensure GP ends before subsequent accesses. */ return true; } From 414c12385d4741e35d88670c6cc2f40a77809734 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 13 Apr 2022 15:17:25 -0700 Subject: [PATCH 042/318] rcu: Provide a get_completed_synchronize_rcu() function It is currently up to the caller to handle stale return values from get_state_synchronize_rcu(). If poll_state_synchronize_rcu() returned true once, a grace period has elapsed, regardless of the fact that counter wrap might cause some future poll_state_synchronize_rcu() invocation to return false. For example, the caller might store a separate flag that indicates whether some previous call to poll_state_synchronize_rcu() determined that the relevant grace period had already ended. This approach works, but it requires extra storage and is easy to get wrong. This commit therefore introduces a get_completed_synchronize_rcu() that returns a cookie that causes poll_state_synchronize_rcu() to always return true. This already-completed cookie can be stored in place of the cookie that previously caused poll_state_synchronize_rcu() to return true. It can also be used to flag a given structure as not having been exposed to readers, and thus not requiring a grace period to elapse. This commit is in preparation for polled expedited grace periods. Link: https://lore.kernel.org/all/20220121142454.1994916-1-bfoster@redhat.com/ Link: https://docs.google.com/document/d/1RNKWW9jQyfjxw2E8dsXVTdvZYh0HnYeSHDKog9jhdN8/edit?usp=sharing Cc: Brian Foster Cc: Dave Chinner Cc: Al Viro Cc: Ian Kent Signed-off-by: Paul E. McKenney --- include/linux/rcupdate.h | 1 + kernel/rcu/rcu.h | 3 +++ kernel/rcu/tiny.c | 4 ++-- kernel/rcu/tree.c | 3 ++- kernel/rcu/update.c | 13 +++++++++++++ 5 files changed, 21 insertions(+), 3 deletions(-) diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 1a32036c918c..7f12daa4708b 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -41,6 +41,7 @@ void call_rcu(struct rcu_head *head, rcu_callback_t func); void rcu_barrier_tasks(void); void rcu_barrier_tasks_rude(void); void synchronize_rcu(void); +unsigned long get_completed_synchronize_rcu(void); #ifdef CONFIG_PREEMPT_RCU diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index 0adb55941aeb..32291f4eefde 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -23,6 +23,9 @@ #define RCU_SEQ_CTR_SHIFT 2 #define RCU_SEQ_STATE_MASK ((1 << RCU_SEQ_CTR_SHIFT) - 1) +/* Low-order bit definition for polled grace-period APIs. */ +#define RCU_GET_STATE_COMPLETED 0x1 + extern int sysctl_sched_rt_runtime; /* diff --git a/kernel/rcu/tiny.c b/kernel/rcu/tiny.c index 340b3f8b090d..dbee6bea6726 100644 --- a/kernel/rcu/tiny.c +++ b/kernel/rcu/tiny.c @@ -58,7 +58,7 @@ void rcu_qs(void) rcu_ctrlblk.donetail = rcu_ctrlblk.curtail; raise_softirq_irqoff(RCU_SOFTIRQ); } - WRITE_ONCE(rcu_ctrlblk.gp_seq, rcu_ctrlblk.gp_seq + 1); + WRITE_ONCE(rcu_ctrlblk.gp_seq, rcu_ctrlblk.gp_seq + 2); local_irq_restore(flags); } @@ -213,7 +213,7 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu); */ bool poll_state_synchronize_rcu(unsigned long oldstate) { - return READ_ONCE(rcu_ctrlblk.gp_seq) != oldstate; + return oldstate == RCU_GET_STATE_COMPLETED || READ_ONCE(rcu_ctrlblk.gp_seq) != oldstate; } EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu); diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index ec28e259774e..46cfceea8784 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -3924,7 +3924,8 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu); */ bool poll_state_synchronize_rcu(unsigned long oldstate) { - if (rcu_seq_done_exact(&rcu_state.gp_seq, oldstate)) { + if (oldstate == RCU_GET_STATE_COMPLETED || + rcu_seq_done_exact(&rcu_state.gp_seq, oldstate)) { smp_mb(); /* Ensure GP ends before subsequent accesses. */ return true; } diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c index fc7fef575606..2e93acad1e31 100644 --- a/kernel/rcu/update.c +++ b/kernel/rcu/update.c @@ -516,6 +516,19 @@ int rcu_cpu_stall_suppress_at_boot __read_mostly; // !0 = suppress boot stalls. EXPORT_SYMBOL_GPL(rcu_cpu_stall_suppress_at_boot); module_param(rcu_cpu_stall_suppress_at_boot, int, 0444); +/** + * get_completed_synchronize_rcu - Return a pre-completed polled state cookie + * + * Returns a value that will always be treated by functions like + * poll_state_synchronize_rcu() as a cookie whose grace period has already + * completed. + */ +unsigned long get_completed_synchronize_rcu(void) +{ + return RCU_GET_STATE_COMPLETED; +} +EXPORT_SYMBOL_GPL(get_completed_synchronize_rcu); + #ifdef CONFIG_PROVE_RCU /* From d0eac20f9909de044fbeb958960b08ff235cf8a2 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 13 Apr 2022 16:14:02 -0700 Subject: [PATCH 043/318] rcutorture: Validate get_completed_synchronize_rcu() This commit verifies that the RCU grace-period state cookie returned from get_completed_synchronize_rcu() causes poll_state_synchronize_rcu() to return true, as required. This commit is in preparation for polled expedited grace periods. Link: https://lore.kernel.org/all/20220121142454.1994916-1-bfoster@redhat.com/ Link: https://docs.google.com/document/d/1RNKWW9jQyfjxw2E8dsXVTdvZYh0HnYeSHDKog9jhdN8/edit?usp=sharing Cc: Brian Foster Cc: Dave Chinner Cc: Al Viro Cc: Ian Kent Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 7120165a9342..4ceec9f4169c 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -338,6 +338,7 @@ struct rcu_torture_ops { void (*sync)(void); void (*exp_sync)(void); unsigned long (*get_gp_state)(void); + unsigned long (*get_gp_completed)(void); unsigned long (*start_gp_poll)(void); bool (*poll_gp_state)(unsigned long oldstate); void (*cond_sync)(unsigned long oldstate); @@ -504,6 +505,7 @@ static struct rcu_torture_ops rcu_ops = { .sync = synchronize_rcu, .exp_sync = synchronize_rcu_expedited, .get_gp_state = get_state_synchronize_rcu, + .get_gp_completed = get_completed_synchronize_rcu, .start_gp_poll = start_poll_synchronize_rcu, .poll_gp_state = poll_state_synchronize_rcu, .cond_sync = cond_synchronize_rcu, @@ -1254,6 +1256,10 @@ rcu_torture_writer(void *arg) rcu_torture_writer_state_getname(), rcu_torture_writer_state, cookie, cur_ops->get_gp_state()); + if (cur_ops->get_gp_completed) { + cookie = cur_ops->get_gp_completed(); + WARN_ON_ONCE(!cur_ops->poll_gp_state(cookie)); + } cur_ops->readunlock(idx); } switch (synctype[torture_random(&rand) % nsynctypes]) { From 4cf0585c4d663654fefa0b359f1908b5cd72802b Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 6 Dec 2021 16:19:40 -0800 Subject: [PATCH 044/318] rcu-tasks: Check for abandoned callbacks This commit adds a debugging scan for callbacks that got lost during a callback-queueing transition. Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 3925e32159b5..b8690a412c5b 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -439,6 +439,11 @@ static int rcu_tasks_need_gpcb(struct rcu_tasks *rtp) WRITE_ONCE(rtp->percpu_dequeue_lim, 1); pr_info("Completing switch %s to CPU-0 callback queuing.\n", rtp->name); } + for (cpu = rtp->percpu_dequeue_lim; cpu < nr_cpu_ids; cpu++) { + struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); + + WARN_ON_ONCE(rcu_segcblist_n_cbs(&rtpcp->cblist)); + } raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags); } From d96225fd09ffb3a6424cebe01c8b1e1bddc30f07 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 19 Apr 2022 10:47:28 -0700 Subject: [PATCH 045/318] rcu-tasks: Split rcu_tasks_one_gp() from rcu_tasks_kthread() This commit abstracts most of the rcu_tasks_kthread() function's loop body into a new rcu_tasks_one_gp() function. It also introduces a new ->tasks_gp_mutex to synchronize concurrent calls to this new rcu_tasks_one_gp() function. This commit is preparation for allowing RCU tasks grace periods to be driven by the calling task during the mid-boot dead zone. Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 58 ++++++++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index b8690a412c5b..d7b12f524e81 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -48,6 +48,7 @@ struct rcu_tasks_percpu { * struct rcu_tasks - Definition for a Tasks-RCU-like mechanism. * @cbs_wait: RCU wait allowing a new callback to get kthread's attention. * @cbs_gbl_lock: Lock protecting callback list. + * @tasks_gp_mutex: Mutex protecting grace period, needed during mid-boot dead zone. * @kthread_ptr: This flavor's grace-period/callback-invocation kthread. * @gp_func: This flavor's grace-period-wait function. * @gp_state: Grace period's most recent state transition (debugging). @@ -79,6 +80,7 @@ struct rcu_tasks_percpu { struct rcu_tasks { struct rcuwait cbs_wait; raw_spinlock_t cbs_gbl_lock; + struct mutex tasks_gp_mutex; int gp_state; int gp_sleep; int init_fract; @@ -119,6 +121,7 @@ static struct rcu_tasks rt_name = \ { \ .cbs_wait = __RCUWAIT_INITIALIZER(rt_name.wait), \ .cbs_gbl_lock = __RAW_SPIN_LOCK_UNLOCKED(rt_name.cbs_gbl_lock), \ + .tasks_gp_mutex = __MUTEX_INITIALIZER(rt_name.tasks_gp_mutex), \ .gp_func = gp, \ .call_func = call, \ .rtpcpu = &rt_name ## __percpu, \ @@ -502,10 +505,37 @@ static void rcu_tasks_invoke_cbs_wq(struct work_struct *wp) rcu_tasks_invoke_cbs(rtp, rtpcp); } -/* RCU-tasks kthread that detects grace periods and invokes callbacks. */ -static int __noreturn rcu_tasks_kthread(void *arg) +// Wait for one grace period. +static void rcu_tasks_one_gp(struct rcu_tasks *rtp) { int needgpcb; + + mutex_lock(&rtp->tasks_gp_mutex); + set_tasks_gp_state(rtp, RTGS_WAIT_CBS); + + // If there were none, wait a bit and start over. + rcuwait_wait_event(&rtp->cbs_wait, + (needgpcb = rcu_tasks_need_gpcb(rtp)), + TASK_IDLE); + + if (needgpcb & 0x2) { + // Wait for one grace period. + set_tasks_gp_state(rtp, RTGS_WAIT_GP); + rtp->gp_start = jiffies; + rcu_seq_start(&rtp->tasks_gp_seq); + rtp->gp_func(rtp); + rcu_seq_end(&rtp->tasks_gp_seq); + } + + // Invoke callbacks. + set_tasks_gp_state(rtp, RTGS_INVOKE_CBS); + rcu_tasks_invoke_cbs(rtp, per_cpu_ptr(rtp->rtpcpu, 0)); + mutex_unlock(&rtp->tasks_gp_mutex); +} + +// RCU-tasks kthread that detects grace periods and invokes callbacks. +static int __noreturn rcu_tasks_kthread(void *arg) +{ struct rcu_tasks *rtp = arg; /* Run on housekeeping CPUs by default. Sysadm can move if desired. */ @@ -519,27 +549,11 @@ static int __noreturn rcu_tasks_kthread(void *arg) * This loop is terminated by the system going down. ;-) */ for (;;) { - set_tasks_gp_state(rtp, RTGS_WAIT_CBS); + // Wait for one grace period and invoke any callbacks + // that are ready. + rcu_tasks_one_gp(rtp); - /* If there were none, wait a bit and start over. */ - rcuwait_wait_event(&rtp->cbs_wait, - (needgpcb = rcu_tasks_need_gpcb(rtp)), - TASK_IDLE); - - if (needgpcb & 0x2) { - // Wait for one grace period. - set_tasks_gp_state(rtp, RTGS_WAIT_GP); - rtp->gp_start = jiffies; - rcu_seq_start(&rtp->tasks_gp_seq); - rtp->gp_func(rtp); - rcu_seq_end(&rtp->tasks_gp_seq); - } - - /* Invoke callbacks. */ - set_tasks_gp_state(rtp, RTGS_INVOKE_CBS); - rcu_tasks_invoke_cbs(rtp, per_cpu_ptr(rtp->rtpcpu, 0)); - - /* Paranoid sleep to keep this from entering a tight loop */ + // Paranoid sleep to keep this from entering a tight loop. schedule_timeout_idle(rtp->gp_sleep); } } From 68cb47204db478302a37adb2aaa26a05882dbaa1 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 19 Apr 2022 11:06:03 -0700 Subject: [PATCH 046/318] rcu-tasks: Move synchronize_rcu_tasks_generic() down This is strictly a code-motion commit that moves the synchronize_rcu_tasks_generic() down to where it can invoke rcu_tasks_one_gp() without the need for a forward declaration. Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index d7b12f524e81..ad993c4ed924 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -326,17 +326,6 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, irq_work_queue(&rtpcp->rtp_irq_work); } -// Wait for a grace period for the specified flavor of Tasks RCU. -static void synchronize_rcu_tasks_generic(struct rcu_tasks *rtp) -{ - /* Complain if the scheduler has not started. */ - RCU_LOCKDEP_WARN(rcu_scheduler_active == RCU_SCHEDULER_INACTIVE, - "synchronize_rcu_tasks called too soon"); - - /* Wait for the grace period. */ - wait_rcu_gp(rtp->call_func); -} - // RCU callback function for rcu_barrier_tasks_generic(). static void rcu_barrier_tasks_generic_cb(struct rcu_head *rhp) { @@ -558,6 +547,17 @@ static int __noreturn rcu_tasks_kthread(void *arg) } } +// Wait for a grace period for the specified flavor of Tasks RCU. +static void synchronize_rcu_tasks_generic(struct rcu_tasks *rtp) +{ + /* Complain if the scheduler has not started. */ + RCU_LOCKDEP_WARN(rcu_scheduler_active == RCU_SCHEDULER_INACTIVE, + "synchronize_rcu_tasks called too soon"); + + /* Wait for the grace period. */ + wait_rcu_gp(rtp->call_func); +} + /* Spawn RCU-tasks grace-period kthread. */ static void __init rcu_spawn_tasks_kthread_generic(struct rcu_tasks *rtp) { From 4a8cc433b8bf3106cf7b1173a936c62d77b40b40 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 19 Apr 2022 15:41:38 -0700 Subject: [PATCH 047/318] rcu-tasks: Drive synchronous grace periods from calling task This commit causes synchronous grace periods to be driven from the task invoking synchronize_rcu_*(), allowing these functions to be invoked from the mid-boot dead zone extending from when the scheduler was initialized to to point that the various RCU tasks grace-period kthreads are spawned. This change will allow the self-tests to run in a consistent manner. Reported-by: Matthew Wilcox Reported-by: Zhouyi Zhou Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index ad993c4ed924..bd9f2e24f5c7 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -495,17 +495,21 @@ static void rcu_tasks_invoke_cbs_wq(struct work_struct *wp) } // Wait for one grace period. -static void rcu_tasks_one_gp(struct rcu_tasks *rtp) +static void rcu_tasks_one_gp(struct rcu_tasks *rtp, bool midboot) { int needgpcb; mutex_lock(&rtp->tasks_gp_mutex); - set_tasks_gp_state(rtp, RTGS_WAIT_CBS); // If there were none, wait a bit and start over. - rcuwait_wait_event(&rtp->cbs_wait, - (needgpcb = rcu_tasks_need_gpcb(rtp)), - TASK_IDLE); + if (unlikely(midboot)) { + needgpcb = 0x2; + } else { + set_tasks_gp_state(rtp, RTGS_WAIT_CBS); + rcuwait_wait_event(&rtp->cbs_wait, + (needgpcb = rcu_tasks_need_gpcb(rtp)), + TASK_IDLE); + } if (needgpcb & 0x2) { // Wait for one grace period. @@ -540,7 +544,7 @@ static int __noreturn rcu_tasks_kthread(void *arg) for (;;) { // Wait for one grace period and invoke any callbacks // that are ready. - rcu_tasks_one_gp(rtp); + rcu_tasks_one_gp(rtp, false); // Paranoid sleep to keep this from entering a tight loop. schedule_timeout_idle(rtp->gp_sleep); @@ -554,8 +558,12 @@ static void synchronize_rcu_tasks_generic(struct rcu_tasks *rtp) RCU_LOCKDEP_WARN(rcu_scheduler_active == RCU_SCHEDULER_INACTIVE, "synchronize_rcu_tasks called too soon"); - /* Wait for the grace period. */ - wait_rcu_gp(rtp->call_func); + // If the grace-period kthread is running, use it. + if (READ_ONCE(rtp->kthread_ptr)) { + wait_rcu_gp(rtp->call_func); + return; + } + rcu_tasks_one_gp(rtp, true); } /* Spawn RCU-tasks grace-period kthread. */ From 3847b64570b1753e9863a4106aec03f4d68e7b17 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 23 May 2022 20:50:11 -0700 Subject: [PATCH 048/318] rcu-tasks: Merge state into .b.need_qs and atomically update This commit gets rid of the task_struct structure's ->trc_reader_checked field, making it instead be a bit within the task_struct structure's existing ->trc_reader_special.b.need_qs field. This commit also atomically loads, stores, and checks the resulting combination of the reader-checked and need-quiescent state flags. This will in turn allow significant simplification of the rcu_tasks_trace_postgp() function as well as elimination of the trc_n_readers_need_end counter in later commits. These changes will in turn simplify later elimination of the RCU Tasks Trace scan of the task list, which will make RCU Tasks Trace grace periods less CPU-intensive. [ paulmck: Apply kernel test robot feedback. ] Signed-off-by: Paul E. McKenney Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- include/linux/rcupdate.h | 18 ++++--- include/linux/sched.h | 1 - kernel/rcu/tasks.h | 103 +++++++++++++++++++++++++++------------ 3 files changed, 82 insertions(+), 40 deletions(-) diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 1a32036c918c..1e728d544fc1 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -169,13 +169,17 @@ void synchronize_rcu_tasks(void); # endif # ifdef CONFIG_TASKS_TRACE_RCU -# define rcu_tasks_trace_qs(t) \ - do { \ - if (!likely(READ_ONCE((t)->trc_reader_checked)) && \ - !unlikely(READ_ONCE((t)->trc_reader_nesting))) { \ - smp_store_release(&(t)->trc_reader_checked, true); \ - smp_mb(); /* Readers partitioned by store. */ \ - } \ +// Bits for ->trc_reader_special.b.need_qs field. +#define TRC_NEED_QS 0x1 // Task needs a quiescent state. +#define TRC_NEED_QS_CHECKED 0x2 // Task has been checked for needing quiescent state. + +u8 rcu_trc_cmpxchg_need_qs(struct task_struct *t, u8 old, u8 new); + +# define rcu_tasks_trace_qs(t) \ + do { \ + if (likely(!READ_ONCE((t)->trc_reader_special.b.need_qs)) && \ + likely(!READ_ONCE((t)->trc_reader_nesting))) \ + rcu_trc_cmpxchg_need_qs((t), 0, TRC_NEED_QS_CHECKED); \ } while (0) # else # define rcu_tasks_trace_qs(t) do { } while (0) diff --git a/include/linux/sched.h b/include/linux/sched.h index c46f3a63b758..e6eb5871593e 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -843,7 +843,6 @@ struct task_struct { int trc_reader_nesting; int trc_ipi_to_cpu; union rcu_special trc_reader_special; - bool trc_reader_checked; struct list_head trc_holdout_list; #endif /* #ifdef CONFIG_TASKS_TRACE_RCU */ diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index bd9f2e24f5c7..7bdc62606816 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1208,6 +1208,39 @@ void call_rcu_tasks_trace(struct rcu_head *rhp, rcu_callback_t func); DEFINE_RCU_TASKS(rcu_tasks_trace, rcu_tasks_wait_gp, call_rcu_tasks_trace, "RCU Tasks Trace"); +/* Load from ->trc_reader_special.b.need_qs with proper ordering. */ +static u8 rcu_ld_need_qs(struct task_struct *t) +{ + smp_mb(); // Enforce full grace-period ordering. + return smp_load_acquire(&t->trc_reader_special.b.need_qs); +} + +/* Store to ->trc_reader_special.b.need_qs with proper ordering. */ +static void rcu_st_need_qs(struct task_struct *t, u8 v) +{ + smp_store_release(&t->trc_reader_special.b.need_qs, v); + smp_mb(); // Enforce full grace-period ordering. +} + +/* + * Do a cmpxchg() on ->trc_reader_special.b.need_qs, allowing for + * the four-byte operand-size restriction of some platforms. + * Returns the old value, which is often ignored. + */ +u8 rcu_trc_cmpxchg_need_qs(struct task_struct *t, u8 old, u8 new) +{ + union rcu_special ret; + union rcu_special trs_old = READ_ONCE(t->trc_reader_special); + union rcu_special trs_new = trs_old; + + if (trs_old.b.need_qs != old) + return trs_old.b.need_qs; + trs_new.b.need_qs = new; + ret.s = cmpxchg(&t->trc_reader_special.s, trs_old.s, trs_new.s); + return ret.b.need_qs; +} +EXPORT_SYMBOL_GPL(rcu_trc_cmpxchg_need_qs); + /* * This irq_work handler allows rcu_read_unlock_trace() to be invoked * while the scheduler locks are held. @@ -1221,16 +1254,20 @@ static DEFINE_IRQ_WORK(rcu_tasks_trace_iw, rcu_read_unlock_iw); /* If we are the last reader, wake up the grace-period kthread. */ void rcu_read_unlock_trace_special(struct task_struct *t) { - int nq = READ_ONCE(t->trc_reader_special.b.need_qs); + int nqs = (rcu_ld_need_qs(t) == (TRC_NEED_QS_CHECKED | TRC_NEED_QS)); - if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB) && - t->trc_reader_special.b.need_mb) + if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB) && t->trc_reader_special.b.need_mb) smp_mb(); // Pairs with update-side barriers. // Update .need_qs before ->trc_reader_nesting for irq/NMI handlers. - if (nq) - WRITE_ONCE(t->trc_reader_special.b.need_qs, false); + if (nqs) { + u8 result = rcu_trc_cmpxchg_need_qs(t, TRC_NEED_QS_CHECKED | TRC_NEED_QS, + TRC_NEED_QS_CHECKED); + + WARN_ONCE(result != (TRC_NEED_QS_CHECKED | TRC_NEED_QS), + "%s: result = %d", __func__, result); + } WRITE_ONCE(t->trc_reader_nesting, 0); - if (nq && atomic_dec_and_test(&trc_n_readers_need_end)) + if (nqs && atomic_dec_and_test(&trc_n_readers_need_end)) irq_work_queue(&rcu_tasks_trace_iw); } EXPORT_SYMBOL_GPL(rcu_read_unlock_trace_special); @@ -1260,27 +1297,24 @@ static void trc_read_check_handler(void *t_in) struct task_struct *texp = t_in; // If the task is no longer running on this CPU, leave. - if (unlikely(texp != t)) { + if (unlikely(texp != t)) goto reset_ipi; // Already on holdout list, so will check later. - } // If the task is not in a read-side critical section, and // if this is the last reader, awaken the grace-period kthread. if (likely(!READ_ONCE(t->trc_reader_nesting))) { - WRITE_ONCE(t->trc_reader_checked, true); + rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS_CHECKED); goto reset_ipi; } // If we are racing with an rcu_read_unlock_trace(), try again later. if (unlikely(READ_ONCE(t->trc_reader_nesting) < 0)) goto reset_ipi; - WRITE_ONCE(t->trc_reader_checked, true); // Get here if the task is in a read-side critical section. Set // its state so that it will awaken the grace-period kthread upon // exit from that critical section. - atomic_inc(&trc_n_readers_need_end); // One more to wait on. - WARN_ON_ONCE(READ_ONCE(t->trc_reader_special.b.need_qs)); - WRITE_ONCE(t->trc_reader_special.b.need_qs, true); + if (!rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS | TRC_NEED_QS_CHECKED)) + atomic_inc(&trc_n_readers_need_end); // One more to wait on. reset_ipi: // Allow future IPIs to be sent on CPU and for task. @@ -1291,8 +1325,9 @@ reset_ipi: } /* Callback function for scheduler to check locked-down task. */ -static int trc_inspect_reader(struct task_struct *t, void *arg) +static int trc_inspect_reader(struct task_struct *t, void *bhp_in) { + struct list_head *bhp = bhp_in; int cpu = task_cpu(t); int nesting; bool ofl = cpu_is_offline(cpu); @@ -1323,16 +1358,19 @@ static int trc_inspect_reader(struct task_struct *t, void *arg) // If not exiting a read-side critical section, mark as checked // so that the grace-period kthread will remove it from the // holdout list. - t->trc_reader_checked = nesting >= 0; - if (nesting <= 0) + if (nesting <= 0) { + if (!nesting) + rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS_CHECKED); return nesting ? -EINVAL : 0; // If in QS, done, otherwise try again later. + } // The task is in a read-side critical section, so set up its // state so that it will awaken the grace-period kthread upon exit // from that critical section. - atomic_inc(&trc_n_readers_need_end); // One more to wait on. - WARN_ON_ONCE(READ_ONCE(t->trc_reader_special.b.need_qs)); - WRITE_ONCE(t->trc_reader_special.b.need_qs, true); + if (!rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS | TRC_NEED_QS_CHECKED)) { + atomic_inc(&trc_n_readers_need_end); // One more to wait on. + trc_add_holdout(t, bhp); + } return 0; } @@ -1348,14 +1386,14 @@ static void trc_wait_for_one_reader(struct task_struct *t, // The current task had better be in a quiescent state. if (t == current) { - t->trc_reader_checked = true; + rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS_CHECKED); WARN_ON_ONCE(READ_ONCE(t->trc_reader_nesting)); return; } // Attempt to nail down the task for inspection. get_task_struct(t); - if (!task_call_func(t, trc_inspect_reader, NULL)) { + if (!task_call_func(t, trc_inspect_reader, bhp)) { put_task_struct(t); return; } @@ -1419,8 +1457,7 @@ static void rcu_tasks_trace_pertask(struct task_struct *t, if (unlikely(t == NULL)) return; - WRITE_ONCE(t->trc_reader_special.b.need_qs, false); - WRITE_ONCE(t->trc_reader_checked, false); + rcu_st_need_qs(t, 0); t->trc_ipi_to_cpu = -1; trc_wait_for_one_reader(t, hop); } @@ -1442,7 +1479,8 @@ static void rcu_tasks_trace_postscan(struct list_head *hop) // Wait for late-stage exiting tasks to finish exiting. // These might have passed the call to exit_tasks_rcu_finish(). synchronize_rcu(); - // Any tasks that exit after this point will set ->trc_reader_checked. + // Any tasks that exit after this point will set + // TRC_NEED_QS_CHECKED in ->trc_reader_special.b.need_qs. } /* Communicate task state back to the RCU tasks trace stall warning request. */ @@ -1460,7 +1498,7 @@ static int trc_check_slow_task(struct task_struct *t, void *arg) return false; // It is running, so decline to inspect it. trc_rdrp->nesting = READ_ONCE(t->trc_reader_nesting); trc_rdrp->ipi_to_cpu = READ_ONCE(t->trc_ipi_to_cpu); - trc_rdrp->needqs = READ_ONCE(t->trc_reader_special.b.need_qs); + trc_rdrp->needqs = rcu_ld_need_qs(t); return true; } @@ -1514,12 +1552,12 @@ static void check_all_holdout_tasks_trace(struct list_head *hop, list_for_each_entry_safe(t, g, hop, trc_holdout_list) { // If safe and needed, try to check the current task. if (READ_ONCE(t->trc_ipi_to_cpu) == -1 && - !READ_ONCE(t->trc_reader_checked)) + !(rcu_ld_need_qs(t) & TRC_NEED_QS_CHECKED)) trc_wait_for_one_reader(t, hop); // If check succeeded, remove this task from the list. if (smp_load_acquire(&t->trc_ipi_to_cpu) == -1 && - READ_ONCE(t->trc_reader_checked)) + rcu_ld_need_qs(t) == TRC_NEED_QS_CHECKED) trc_del_holdout(t); else if (needreport) show_stalled_task_trace(t, firstreport); @@ -1574,12 +1612,12 @@ static void rcu_tasks_trace_postgp(struct rcu_tasks *rtp) // Stall warning time, so make a list of the offenders. rcu_read_lock(); for_each_process_thread(g, t) - if (READ_ONCE(t->trc_reader_special.b.need_qs)) + if (rcu_ld_need_qs(t) & TRC_NEED_QS) trc_add_holdout(t, &holdouts); rcu_read_unlock(); firstreport = true; list_for_each_entry_safe(t, g, &holdouts, trc_holdout_list) { - if (READ_ONCE(t->trc_reader_special.b.need_qs)) + if (rcu_ld_need_qs(t) & TRC_NEED_QS) show_stalled_task_trace(t, &firstreport); trc_del_holdout(t); // Release task_struct reference. } @@ -1595,11 +1633,12 @@ static void rcu_tasks_trace_postgp(struct rcu_tasks *rtp) /* Report any needed quiescent state for this exiting task. */ static void exit_tasks_rcu_finish_trace(struct task_struct *t) { - WRITE_ONCE(t->trc_reader_checked, true); + rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS_CHECKED); WARN_ON_ONCE(READ_ONCE(t->trc_reader_nesting)); - WRITE_ONCE(t->trc_reader_nesting, 0); - if (WARN_ON_ONCE(READ_ONCE(t->trc_reader_special.b.need_qs))) + if (WARN_ON_ONCE(rcu_ld_need_qs(t) & TRC_NEED_QS)) rcu_read_unlock_trace_special(t); + else + WRITE_ONCE(t->trc_reader_nesting, 0); } /** From 550611269b153dc17b44fa2d692c30f628d1c65c Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 25 May 2022 13:37:36 -0700 Subject: [PATCH 049/318] rcu-tasks: Remove rcu_tasks_trace_postgp() wait for counter Now that tasks are not removed from the list until they have responded to any needed request for a quiescent state, it is no longer necessary to wait for the trc_n_readers_need_end counter to go to zero. This commit therefore removes that waiting code. It is therefore also no longer necessary for rcu_tasks_trace_postgp() to do the final decrement of this counter, so that code is also removed. This in turn means that trc_n_readers_need_end counter itself can be removed, as can the rcu_tasks_trace_iw irq_work structure and the rcu_read_unlock_iw() function. [ paulmck: Apply feedback from Zqiang. ] Signed-off-by: Paul E. McKenney Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- kernel/rcu/tasks.h | 62 +++------------------------------------------- 1 file changed, 3 insertions(+), 59 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 7bdc62606816..561d24f7f73c 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1192,9 +1192,6 @@ EXPORT_SYMBOL_GPL(rcu_trace_lock_map); #ifdef CONFIG_TASKS_TRACE_RCU -static atomic_t trc_n_readers_need_end; // Number of waited-for readers. -static DECLARE_WAIT_QUEUE_HEAD(trc_wait); // List of holdout tasks. - // Record outstanding IPIs to each CPU. No point in sending two... static DEFINE_PER_CPU(bool, trc_ipi_to_cpu); @@ -1241,16 +1238,6 @@ u8 rcu_trc_cmpxchg_need_qs(struct task_struct *t, u8 old, u8 new) } EXPORT_SYMBOL_GPL(rcu_trc_cmpxchg_need_qs); -/* - * This irq_work handler allows rcu_read_unlock_trace() to be invoked - * while the scheduler locks are held. - */ -static void rcu_read_unlock_iw(struct irq_work *iwp) -{ - wake_up(&trc_wait); -} -static DEFINE_IRQ_WORK(rcu_tasks_trace_iw, rcu_read_unlock_iw); - /* If we are the last reader, wake up the grace-period kthread. */ void rcu_read_unlock_trace_special(struct task_struct *t) { @@ -1267,8 +1254,6 @@ void rcu_read_unlock_trace_special(struct task_struct *t) "%s: result = %d", __func__, result); } WRITE_ONCE(t->trc_reader_nesting, 0); - if (nqs && atomic_dec_and_test(&trc_n_readers_need_end)) - irq_work_queue(&rcu_tasks_trace_iw); } EXPORT_SYMBOL_GPL(rcu_read_unlock_trace_special); @@ -1313,8 +1298,7 @@ static void trc_read_check_handler(void *t_in) // Get here if the task is in a read-side critical section. Set // its state so that it will awaken the grace-period kthread upon // exit from that critical section. - if (!rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS | TRC_NEED_QS_CHECKED)) - atomic_inc(&trc_n_readers_need_end); // One more to wait on. + rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS | TRC_NEED_QS_CHECKED); reset_ipi: // Allow future IPIs to be sent on CPU and for task. @@ -1367,10 +1351,8 @@ static int trc_inspect_reader(struct task_struct *t, void *bhp_in) // The task is in a read-side critical section, so set up its // state so that it will awaken the grace-period kthread upon exit // from that critical section. - if (!rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS | TRC_NEED_QS_CHECKED)) { - atomic_inc(&trc_n_readers_need_end); // One more to wait on. + if (!rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS | TRC_NEED_QS_CHECKED)) trc_add_holdout(t, bhp); - } return 0; } @@ -1436,9 +1418,6 @@ static void rcu_tasks_trace_pregp_step(void) { int cpu; - // Allow for fast-acting IPIs. - atomic_set(&trc_n_readers_need_end, 1); - // There shouldn't be any old IPIs, but... for_each_possible_cpu(cpu) WARN_ON_ONCE(per_cpu(trc_ipi_to_cpu, cpu)); @@ -1581,10 +1560,6 @@ static void rcu_tasks_trace_empty_fn(void *unused) static void rcu_tasks_trace_postgp(struct rcu_tasks *rtp) { int cpu; - bool firstreport; - struct task_struct *g, *t; - LIST_HEAD(holdouts); - long ret; // Wait for any lingering IPI handlers to complete. Note that // if a CPU has gone offline or transitioned to userspace in the @@ -1595,37 +1570,6 @@ static void rcu_tasks_trace_postgp(struct rcu_tasks *rtp) if (WARN_ON_ONCE(smp_load_acquire(per_cpu_ptr(&trc_ipi_to_cpu, cpu)))) smp_call_function_single(cpu, rcu_tasks_trace_empty_fn, NULL, 1); - // Remove the safety count. - smp_mb__before_atomic(); // Order vs. earlier atomics - atomic_dec(&trc_n_readers_need_end); - smp_mb__after_atomic(); // Order vs. later atomics - - // Wait for readers. - set_tasks_gp_state(rtp, RTGS_WAIT_READERS); - for (;;) { - ret = wait_event_idle_exclusive_timeout( - trc_wait, - atomic_read(&trc_n_readers_need_end) == 0, - READ_ONCE(rcu_task_stall_timeout)); - if (ret) - break; // Count reached zero. - // Stall warning time, so make a list of the offenders. - rcu_read_lock(); - for_each_process_thread(g, t) - if (rcu_ld_need_qs(t) & TRC_NEED_QS) - trc_add_holdout(t, &holdouts); - rcu_read_unlock(); - firstreport = true; - list_for_each_entry_safe(t, g, &holdouts, trc_holdout_list) { - if (rcu_ld_need_qs(t) & TRC_NEED_QS) - show_stalled_task_trace(t, &firstreport); - trc_del_holdout(t); // Release task_struct reference. - } - if (firstreport) - pr_err("INFO: rcu_tasks_trace detected stalls? (Counter/taskslist mismatch?)\n"); - show_stalled_ipi_trace(); - pr_err("\t%d holdouts\n", atomic_read(&trc_n_readers_need_end)); - } smp_mb(); // Caller's code must be ordered after wakeup. // Pairs with pretty much every ordering primitive. } @@ -1725,7 +1669,7 @@ void show_rcu_tasks_trace_gp_kthread(void) { char buf[64]; - sprintf(buf, "N%d h:%lu/%lu/%lu", atomic_read(&trc_n_readers_need_end), + sprintf(buf, "h:%lu/%lu/%lu", data_race(n_heavy_reader_ofl_updates), data_race(n_heavy_reader_updates), data_race(n_heavy_reader_attempts)); From 9ff86b4c443cc93bf3a9b624e3385e4114388e85 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 26 May 2022 16:12:51 -0700 Subject: [PATCH 050/318] rcu-tasks: Make trc_read_check_handler() fetch ->trc_reader_nesting only once This commit replaces the pair of READ_ONCE(t->trc_reader_nesting) calls with a single such call and a local variable. This makes the code's intent more clear. Signed-off-by: Paul E. McKenney Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- kernel/rcu/tasks.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 561d24f7f73c..8fe78a7fecaf 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1278,6 +1278,7 @@ static void trc_del_holdout(struct task_struct *t) /* IPI handler to check task state. */ static void trc_read_check_handler(void *t_in) { + int nesting; struct task_struct *t = current; struct task_struct *texp = t_in; @@ -1287,12 +1288,13 @@ static void trc_read_check_handler(void *t_in) // If the task is not in a read-side critical section, and // if this is the last reader, awaken the grace-period kthread. - if (likely(!READ_ONCE(t->trc_reader_nesting))) { + nesting = READ_ONCE(t->trc_reader_nesting); + if (likely(!nesting)) { rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS_CHECKED); goto reset_ipi; } // If we are racing with an rcu_read_unlock_trace(), try again later. - if (unlikely(READ_ONCE(t->trc_reader_nesting) < 0)) + if (unlikely(nesting < 0)) goto reset_ipi; // Get here if the task is in a read-side critical section. Set From 5c9a9ca44fda41c5e82f50efced5297a9c19760d Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 24 May 2022 16:59:52 -0700 Subject: [PATCH 051/318] rcu-tasks: Idle tasks on offline CPUs are in quiescent states Any idle task corresponding to an offline CPU is in an RCU Tasks Trace quiescent state. This commit causes rcu_tasks_trace_postscan() to ignore idle tasks for offline CPUs, which it can do safely due to CPU-hotplug operations being disabled. Signed-off-by: Paul E. McKenney Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- kernel/rcu/tasks.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 8fe78a7fecaf..ec68bfe98c95 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1451,7 +1451,7 @@ static void rcu_tasks_trace_postscan(struct list_head *hop) { int cpu; - for_each_possible_cpu(cpu) + for_each_online_cpu(cpu) rcu_tasks_trace_pertask(idle_task(cpu), hop); // Re-enable CPU hotplug now that the tasklist scan has completed. From 897ba84dc5aa7a5518e19da180d2985790723d30 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 24 May 2022 18:02:40 -0700 Subject: [PATCH 052/318] rcu-tasks: Handle idle tasks for recently offlined CPUs This commit identifies idle tasks for recently offlined CPUs as residing in a quiescent state. This is safe only because CPU-hotplug operations are excluded during these checks. Signed-off-by: Paul E. McKenney Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- kernel/rcu/tasks.h | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index ec68bfe98c95..414861d65196 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1318,27 +1318,26 @@ static int trc_inspect_reader(struct task_struct *t, void *bhp_in) int nesting; bool ofl = cpu_is_offline(cpu); - if (task_curr(t)) { - WARN_ON_ONCE(ofl && !is_idle_task(t)); - + if (task_curr(t) && !ofl) { // If no chance of heavyweight readers, do it the hard way. - if (!ofl && !IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB)) + if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB)) return -EINVAL; // If heavyweight readers are enabled on the remote task, // we can inspect its state despite its currently running. // However, we cannot safely change its state. n_heavy_reader_attempts++; - if (!ofl && // Check for "running" idle tasks on offline CPUs. - !rcu_dynticks_zero_in_eqs(cpu, &t->trc_reader_nesting)) + // Check for "running" idle tasks on offline CPUs. + if (!rcu_dynticks_zero_in_eqs(cpu, &t->trc_reader_nesting)) return -EINVAL; // No quiescent state, do it the hard way. n_heavy_reader_updates++; - if (ofl) - n_heavy_reader_ofl_updates++; nesting = 0; } else { // The task is not running, so C-language access is safe. nesting = t->trc_reader_nesting; + WARN_ON_ONCE(ofl && task_curr(t) && !is_idle_task(t)); + if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB) && ofl) + n_heavy_reader_ofl_updates++; } // If not exiting a read-side critical section, mark as checked From 5d4c90d755d5703d5a74fac0870c0a697250ec33 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 24 May 2022 18:19:17 -0700 Subject: [PATCH 053/318] rcu-tasks: RCU Tasks Trace grace-period kthread has implicit QS Because the task driving the grace-period kthread is in quiescent state throughout, this commit excludes it from the list of tasks from which a quiescent state is needed. This does mean that attaching a sleepable BPF program to function in kernel/rcu/tasks.h is a bad idea, by the way. Signed-off-by: Paul E. McKenney Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- kernel/rcu/tasks.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 414861d65196..554b2e59a1d5 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1433,8 +1433,9 @@ static void rcu_tasks_trace_pertask(struct task_struct *t, struct list_head *hop) { // During early boot when there is only the one boot CPU, there - // is no idle task for the other CPUs. Just return. - if (unlikely(t == NULL)) + // is no idle task for the other CPUs. Also, the grace-period + // kthread is always in a quiescent state. Either way, just return. + if (unlikely(t == NULL) || t == current) return; rcu_st_need_qs(t, 0); From 6a694411977a6d57ff76a896a745c2f717372dac Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 24 May 2022 20:33:17 -0700 Subject: [PATCH 054/318] rcu-tasks: Make rcu_note_context_switch() unconditionally call rcu_tasks_qs() This commit makes rcu_note_context_switch() unconditionally invoke the rcu_tasks_qs() function, as opposed to doing so only when RCU (as opposed to RCU Tasks Trace) urgently needs a grace period to end. Signed-off-by: Paul E. McKenney Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- kernel/rcu/tree_plugin.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index c8ba0fe17267..c966d680b789 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -899,8 +899,8 @@ void rcu_note_context_switch(bool preempt) this_cpu_write(rcu_data.rcu_urgent_qs, false); if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs))) rcu_momentary_dyntick_idle(); - rcu_tasks_qs(current, preempt); out: + rcu_tasks_qs(current, preempt); trace_rcu_utilization(TPS("End context switch")); } EXPORT_SYMBOL_GPL(rcu_note_context_switch); From 0968e8920b5b11b7a33982890ad9150e09e1cb1f Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 31 May 2022 21:38:15 -0700 Subject: [PATCH 055/318] rcu-tasks: Simplify trc_inspect_reader() QS logic Currently, trc_inspect_reader() does one check for nesting less than or equal to zero, then sorts out the distinctions within this single "if" statement. This commit simplifies the logic by providing one "if" statement for quiescent states (nesting of zero) and another "if" statement for transitioning from one nesting level to another or the outermost rcu_read_unlock_trace() (negative nesting). Signed-off-by: Paul E. McKenney Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- kernel/rcu/tasks.h | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 554b2e59a1d5..6b44c69eeca8 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1343,15 +1343,16 @@ static int trc_inspect_reader(struct task_struct *t, void *bhp_in) // If not exiting a read-side critical section, mark as checked // so that the grace-period kthread will remove it from the // holdout list. - if (nesting <= 0) { - if (!nesting) - rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS_CHECKED); - return nesting ? -EINVAL : 0; // If in QS, done, otherwise try again later. + if (!nesting) { + rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS_CHECKED); + return 0; // In QS, so done. } + if (nesting < 0) + return -EINVAL; // QS transitioning, try again later. // The task is in a read-side critical section, so set up its - // state so that it will awaken the grace-period kthread upon exit - // from that critical section. + // state so that it will update state upon exit from that critical + // section. if (!rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS | TRC_NEED_QS_CHECKED)) trc_add_holdout(t, bhp); return 0; From 9f3eb5fb8e468d3c3a6073d0c816405fa73c8038 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 24 May 2022 16:05:15 -0700 Subject: [PATCH 056/318] rcu-tasks: Add slow-IPI indicator to RCU Tasks Trace stall warnings This commit adds a "I" indicator to the RCU Tasks Trace CPU stall warning when an IPI directed to a task has thus far failed to arrive. This serves as a debugging aid. Signed-off-by: Paul E. McKenney Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- kernel/rcu/tasks.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 6b44c69eeca8..1cfbebf2b597 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1497,8 +1497,9 @@ static void show_stalled_task_trace(struct task_struct *t, bool *firstreport) } cpu = task_cpu(t); if (!task_call_func(t, trc_check_slow_task, &trc_rdr)) - pr_alert("P%d: %c\n", + pr_alert("P%d: %c%c\n", t->pid, + ".I"[t->trc_ipi_to_cpu >= 0], ".i"[is_idle_tsk]); else pr_alert("P%d: %c%c%c nesting: %d%c cpu: %d\n", From c8c03ad9d7cd694b88ab8db4199b4e4d3c6cc5aa Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 24 May 2022 20:36:08 -0700 Subject: [PATCH 057/318] rcu-tasks: Flag offline CPUs in RCU Tasks Trace stall warnings This commit tags offline CPUs with "(offline)" in RCU Tasks Trace CPU stall warnings. This is a debugging aid. Signed-off-by: Paul E. McKenney Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- kernel/rcu/tasks.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 1cfbebf2b597..93096188d363 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1502,14 +1502,14 @@ static void show_stalled_task_trace(struct task_struct *t, bool *firstreport) ".I"[t->trc_ipi_to_cpu >= 0], ".i"[is_idle_tsk]); else - pr_alert("P%d: %c%c%c nesting: %d%c cpu: %d\n", + pr_alert("P%d: %c%c%c nesting: %d%c cpu: %d%s\n", t->pid, ".I"[trc_rdr.ipi_to_cpu >= 0], ".i"[is_idle_tsk], ".N"[cpu >= 0 && tick_nohz_full_cpu(cpu)], trc_rdr.nesting, " N"[!!trc_rdr.needqs], - cpu); + cpu, cpu_online(cpu) ? "" : "(offline)"); sched_show_task(t); } From be15a16486dd6513ad801ea320eb21e10eec2b55 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 25 May 2022 09:20:59 -0700 Subject: [PATCH 058/318] rcu-tasks: Make RCU Tasks Trace stall warnings print full .b.need_qs field Currently, the RCU Tasks Trace CPU stall warning simply indicates whether or not the .b.need_qs field is zero. This commit shows the three permitted values and flags other values with either "!" or "?". This is a debugging aid. Signed-off-by: Paul E. McKenney Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- kernel/rcu/tasks.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 93096188d363..5eefbab7f2ed 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1502,13 +1502,14 @@ static void show_stalled_task_trace(struct task_struct *t, bool *firstreport) ".I"[t->trc_ipi_to_cpu >= 0], ".i"[is_idle_tsk]); else - pr_alert("P%d: %c%c%c nesting: %d%c cpu: %d%s\n", + pr_alert("P%d: %c%c%c nesting: %d%c%c cpu: %d%s\n", t->pid, ".I"[trc_rdr.ipi_to_cpu >= 0], ".i"[is_idle_tsk], ".N"[cpu >= 0 && tick_nohz_full_cpu(cpu)], trc_rdr.nesting, - " N"[!!trc_rdr.needqs], + " !CN"[trc_rdr.needqs & 0x3], + " ?"[trc_rdr.needqs > 0x3], cpu, cpu_online(cpu) ? "" : "(offline)"); sched_show_task(t); } From f90f19da88bfe32dd1fdfd104de4c0526a3be701 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 25 May 2022 09:49:26 -0700 Subject: [PATCH 059/318] rcu-tasks: Make RCU Tasks Trace stall warning handle idle offline tasks When a CPU is offline, its idle task can appear to be running, but it cannot be doing anything while CPU-hotplug operations are excluded. This commit takes advantage of that fact by making trc_check_slow_task() check for task_curr(t) && cpu_online(task_cpu(t)), and recording full information in that case. Signed-off-by: Paul E. McKenney Cc: Peter Zijlstra Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- kernel/rcu/tasks.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 5eefbab7f2ed..64eb4d7b142e 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1476,7 +1476,7 @@ static int trc_check_slow_task(struct task_struct *t, void *arg) { struct trc_stall_chk_rdr *trc_rdrp = arg; - if (task_curr(t)) + if (task_curr(t) && cpu_online(task_cpu(t))) return false; // It is running, so decline to inspect it. trc_rdrp->nesting = READ_ONCE(t->trc_reader_nesting); trc_rdrp->ipi_to_cpu = READ_ONCE(t->trc_ipi_to_cpu); From 434c9eefb959c36331a93617ea95df903469b99f Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 16 May 2022 17:56:16 -0700 Subject: [PATCH 060/318] rcu-tasks: Add data structures for lightweight grace periods This commit adds fields to task_struct and to rcu_tasks_percpu that will be used to avoid the task-list scan for RCU Tasks Trace grace periods, and also initializes these fields. Signed-off-by: Paul E. McKenney Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- include/linux/sched.h | 2 ++ init/init_task.c | 1 + kernel/fork.c | 1 + kernel/rcu/tasks.h | 4 ++++ 4 files changed, 8 insertions(+) diff --git a/include/linux/sched.h b/include/linux/sched.h index e6eb5871593e..b88caf54e168 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -844,6 +844,8 @@ struct task_struct { int trc_ipi_to_cpu; union rcu_special trc_reader_special; struct list_head trc_holdout_list; + struct list_head trc_blkd_node; + int trc_blkd_cpu; #endif /* #ifdef CONFIG_TASKS_TRACE_RCU */ struct sched_info sched_info; diff --git a/init/init_task.c b/init/init_task.c index 73cc8f03511a..ff6c4b9bfe6b 100644 --- a/init/init_task.c +++ b/init/init_task.c @@ -157,6 +157,7 @@ struct task_struct init_task .trc_reader_nesting = 0, .trc_reader_special.s = 0, .trc_holdout_list = LIST_HEAD_INIT(init_task.trc_holdout_list), + .trc_blkd_node = LIST_HEAD_INIT(init_task.trc_blkd_node), #endif #ifdef CONFIG_CPUSETS .mems_allowed_seq = SEQCNT_SPINLOCK_ZERO(init_task.mems_allowed_seq, diff --git a/kernel/fork.c b/kernel/fork.c index 9d44f2d46c69..1950eb870244 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1814,6 +1814,7 @@ static inline void rcu_copy_process(struct task_struct *p) p->trc_reader_nesting = 0; p->trc_reader_special.s = 0; INIT_LIST_HEAD(&p->trc_holdout_list); + INIT_LIST_HEAD(&p->trc_blkd_node); #endif /* #ifdef CONFIG_TASKS_TRACE_RCU */ } diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 64eb4d7b142e..fd4508af055e 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -29,6 +29,7 @@ typedef void (*postgp_func_t)(struct rcu_tasks *rtp); * @rtp_work: Work queue for invoking callbacks. * @rtp_irq_work: IRQ work queue for deferred wakeups. * @barrier_q_head: RCU callback for barrier operation. + * @rtp_blkd_tasks: List of tasks blocked as readers. * @cpu: CPU number corresponding to this entry. * @rtpp: Pointer to the rcu_tasks structure. */ @@ -40,6 +41,7 @@ struct rcu_tasks_percpu { struct work_struct rtp_work; struct irq_work rtp_irq_work; struct rcu_head barrier_q_head; + struct list_head rtp_blkd_tasks; int cpu; struct rcu_tasks *rtpp; }; @@ -256,6 +258,8 @@ static void cblist_init_generic(struct rcu_tasks *rtp) INIT_WORK(&rtpcp->rtp_work, rcu_tasks_invoke_cbs_wq); rtpcp->cpu = cpu; rtpcp->rtpp = rtp; + if (!rtpcp->rtp_blkd_tasks.next) + INIT_LIST_HEAD(&rtpcp->rtp_blkd_tasks); raw_spin_unlock_rcu_node(rtpcp); // irqs remain disabled. } raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags); From 0356d4e66214569de674ab2684f2e0b440a466ab Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 17 May 2022 11:30:32 -0700 Subject: [PATCH 061/318] rcu-tasks: Track blocked RCU Tasks Trace readers This commit places any task that has ever blocked within its current RCU Tasks Trace read-side critical section on a per-CPU list within the rcu_tasks_percpu structure. Tasks are removed from this list when they exit by the exit_tasks_rcu_finish_trace() function. The purpose of this commit is to provide the information needed to eliminate the current scan of the full task list. This commit offsets the INT_MIN value for ->trc_reader_nesting with the new nesting level in order to avoid queueing tasks that are exiting their read-side critical sections. [ paulmck: Apply kernel test robot feedback. ] [ paulmck: Apply feedback from syzbot+9bb26e7c5e8e4fa7e641@syzkaller.appspotmail.com ] Signed-off-by: Paul E. McKenney Tested-by: syzbot Tested-by: "Zhang, Qiang1" Cc: Peter Zijlstra Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- include/linux/rcupdate.h | 11 +++++++++-- include/linux/rcupdate_trace.h | 2 +- kernel/rcu/tasks.h | 22 +++++++++++++++++++++- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 1e728d544fc1..ebdfeead44e5 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -174,12 +174,19 @@ void synchronize_rcu_tasks(void); #define TRC_NEED_QS_CHECKED 0x2 // Task has been checked for needing quiescent state. u8 rcu_trc_cmpxchg_need_qs(struct task_struct *t, u8 old, u8 new); +void rcu_tasks_trace_qs_blkd(struct task_struct *t); # define rcu_tasks_trace_qs(t) \ do { \ + int ___rttq_nesting = READ_ONCE((t)->trc_reader_nesting); \ + \ if (likely(!READ_ONCE((t)->trc_reader_special.b.need_qs)) && \ - likely(!READ_ONCE((t)->trc_reader_nesting))) \ + likely(!___rttq_nesting)) { \ rcu_trc_cmpxchg_need_qs((t), 0, TRC_NEED_QS_CHECKED); \ + } else if (___rttq_nesting && ___rttq_nesting != INT_MIN && \ + !READ_ONCE((t)->trc_reader_special.b.blocked)) { \ + rcu_tasks_trace_qs_blkd(t); \ + } \ } while (0) # else # define rcu_tasks_trace_qs(t) do { } while (0) @@ -188,7 +195,7 @@ u8 rcu_trc_cmpxchg_need_qs(struct task_struct *t, u8 old, u8 new); #define rcu_tasks_qs(t, preempt) \ do { \ rcu_tasks_classic_qs((t), (preempt)); \ - rcu_tasks_trace_qs((t)); \ + rcu_tasks_trace_qs(t); \ } while (0) # ifdef CONFIG_TASKS_RUDE_RCU diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h index 6f9c35817398..9bc8cbb33340 100644 --- a/include/linux/rcupdate_trace.h +++ b/include/linux/rcupdate_trace.h @@ -75,7 +75,7 @@ static inline void rcu_read_unlock_trace(void) nesting = READ_ONCE(t->trc_reader_nesting) - 1; barrier(); // Critical section before disabling. // Disable IPI-based setting of .need_qs. - WRITE_ONCE(t->trc_reader_nesting, INT_MIN); + WRITE_ONCE(t->trc_reader_nesting, INT_MIN + nesting); if (likely(!READ_ONCE(t->trc_reader_special.s)) || nesting) { WRITE_ONCE(t->trc_reader_nesting, nesting); return; // We assume shallow reader nesting. diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index fd4508af055e..bab75ec26bdb 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1261,6 +1261,24 @@ void rcu_read_unlock_trace_special(struct task_struct *t) } EXPORT_SYMBOL_GPL(rcu_read_unlock_trace_special); +/* Add a newly blocked reader task to its CPU's list. */ +void rcu_tasks_trace_qs_blkd(struct task_struct *t) +{ + unsigned long flags; + struct rcu_tasks_percpu *rtpcp; + + local_irq_save(flags); + rtpcp = this_cpu_ptr(rcu_tasks_trace.rtpcpu); + raw_spin_lock_rcu_node(rtpcp); // irqs already disabled + t->trc_blkd_cpu = smp_processor_id(); + if (!rtpcp->rtp_blkd_tasks.next) + INIT_LIST_HEAD(&rtpcp->rtp_blkd_tasks); + list_add(&t->trc_blkd_node, &rtpcp->rtp_blkd_tasks); + t->trc_reader_special.b.blocked = true; + raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); +} +EXPORT_SYMBOL_GPL(rcu_tasks_trace_qs_blkd); + /* Add a task to the holdout list, if it is not already on the list. */ static void trc_add_holdout(struct task_struct *t, struct list_head *bhp) { @@ -1586,9 +1604,11 @@ static void rcu_tasks_trace_postgp(struct rcu_tasks *rtp) /* Report any needed quiescent state for this exiting task. */ static void exit_tasks_rcu_finish_trace(struct task_struct *t) { + union rcu_special trs = READ_ONCE(t->trc_reader_special); + rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS_CHECKED); WARN_ON_ONCE(READ_ONCE(t->trc_reader_nesting)); - if (WARN_ON_ONCE(rcu_ld_need_qs(t) & TRC_NEED_QS)) + if (WARN_ON_ONCE(rcu_ld_need_qs(t) & TRC_NEED_QS) || trs.b.blocked) rcu_read_unlock_trace_special(t); else WRITE_ONCE(t->trc_reader_nesting, 0); From 0bcb386857376224b5fd3f38b6e3173ec74d8d36 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 17 May 2022 15:01:14 -0700 Subject: [PATCH 062/318] rcu-tasks: Untrack blocked RCU Tasks Trace at reader end This commit causes rcu_read_unlock_trace() to check for the current task being on a per-CPU list within the rcu_tasks_percpu structure, and removes it from that list if so. This has the effect of curtailing tracking of a task that blocked within an RCU Tasks Trace read-side critical section once it exits that critical section. Signed-off-by: Paul E. McKenney Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- kernel/rcu/tasks.h | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index bab75ec26bdb..eb87a759ef0b 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1245,17 +1245,29 @@ EXPORT_SYMBOL_GPL(rcu_trc_cmpxchg_need_qs); /* If we are the last reader, wake up the grace-period kthread. */ void rcu_read_unlock_trace_special(struct task_struct *t) { - int nqs = (rcu_ld_need_qs(t) == (TRC_NEED_QS_CHECKED | TRC_NEED_QS)); + unsigned long flags; + struct rcu_tasks_percpu *rtpcp; + union rcu_special trs; + + // Open-coded full-word version of rcu_ld_need_qs(). + smp_mb(); // Enforce full grace-period ordering. + trs = smp_load_acquire(&t->trc_reader_special); if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB) && t->trc_reader_special.b.need_mb) smp_mb(); // Pairs with update-side barriers. // Update .need_qs before ->trc_reader_nesting for irq/NMI handlers. - if (nqs) { + if (trs.b.need_qs == (TRC_NEED_QS_CHECKED | TRC_NEED_QS)) { u8 result = rcu_trc_cmpxchg_need_qs(t, TRC_NEED_QS_CHECKED | TRC_NEED_QS, TRC_NEED_QS_CHECKED); - WARN_ONCE(result != (TRC_NEED_QS_CHECKED | TRC_NEED_QS), - "%s: result = %d", __func__, result); + WARN_ONCE(result != trs.b.need_qs, "%s: result = %d", __func__, result); + } + if (trs.b.blocked) { + rtpcp = per_cpu_ptr(rcu_tasks_trace.rtpcpu, t->trc_blkd_cpu); + raw_spin_lock_irqsave_rcu_node(rtpcp, flags); + list_del_init(&t->trc_blkd_node); + WRITE_ONCE(t->trc_reader_special.b.blocked, false); + raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); } WRITE_ONCE(t->trc_reader_nesting, 0); } @@ -1274,7 +1286,7 @@ void rcu_tasks_trace_qs_blkd(struct task_struct *t) if (!rtpcp->rtp_blkd_tasks.next) INIT_LIST_HEAD(&rtpcp->rtp_blkd_tasks); list_add(&t->trc_blkd_node, &rtpcp->rtp_blkd_tasks); - t->trc_reader_special.b.blocked = true; + WRITE_ONCE(t->trc_reader_special.b.blocked, true); raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); } EXPORT_SYMBOL_GPL(rcu_tasks_trace_qs_blkd); @@ -1608,7 +1620,7 @@ static void exit_tasks_rcu_finish_trace(struct task_struct *t) rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS_CHECKED); WARN_ON_ONCE(READ_ONCE(t->trc_reader_nesting)); - if (WARN_ON_ONCE(rcu_ld_need_qs(t) & TRC_NEED_QS) || trs.b.blocked) + if (WARN_ON_ONCE(rcu_ld_need_qs(t) & TRC_NEED_QS || trs.b.blocked)) rcu_read_unlock_trace_special(t); else WRITE_ONCE(t->trc_reader_nesting, 0); From 387c0ad702296c4eb7b95322c8af0e4391bf146f Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 17 May 2022 16:47:40 -0700 Subject: [PATCH 063/318] rcu-tasks: Add blocked-task indicator to RCU Tasks Trace stall warnings This commit adds a "B" indicator to the RCU Tasks Trace CPU stall warning when the task has blocked within its current read-side critical section. This serves as a debugging aid. Signed-off-by: Paul E. McKenney Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- kernel/rcu/tasks.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index eb87a759ef0b..6f4b89f9517e 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1536,11 +1536,12 @@ static void show_stalled_task_trace(struct task_struct *t, bool *firstreport) ".I"[t->trc_ipi_to_cpu >= 0], ".i"[is_idle_tsk]); else - pr_alert("P%d: %c%c%c nesting: %d%c%c cpu: %d%s\n", + pr_alert("P%d: %c%c%c%c nesting: %d%c%c cpu: %d%s\n", t->pid, ".I"[trc_rdr.ipi_to_cpu >= 0], ".i"[is_idle_tsk], ".N"[cpu >= 0 && tick_nohz_full_cpu(cpu)], + ".B"[!!data_race(t->trc_reader_special.b.blocked)], trc_rdr.nesting, " !CN"[trc_rdr.needqs & 0x3], " ?"[trc_rdr.needqs > 0x3], From 1fa98e2e40e5a46fa528b6ab256ba00dfb319dde Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 17 May 2022 17:11:37 -0700 Subject: [PATCH 064/318] rcu-tasks: Move rcu_tasks_trace_pertask() before rcu_tasks_trace_pregp_step() This is a code-motion-only commit that moves rcu_tasks_trace_pertask() to precede rcu_tasks_trace_pregp_step(), so that the latter will be able to invoke the other without forward references. Signed-off-by: Paul E. McKenney Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- kernel/rcu/tasks.h | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 6f4b89f9517e..66d8473f1bda 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1449,20 +1449,6 @@ static void trc_wait_for_one_reader(struct task_struct *t, } } -/* Initialize for a new RCU-tasks-trace grace period. */ -static void rcu_tasks_trace_pregp_step(void) -{ - int cpu; - - // There shouldn't be any old IPIs, but... - for_each_possible_cpu(cpu) - WARN_ON_ONCE(per_cpu(trc_ipi_to_cpu, cpu)); - - // Disable CPU hotplug across the tasklist scan. - // This also waits for all readers in CPU-hotplug code paths. - cpus_read_lock(); -} - /* Do first-round processing for the specified task. */ static void rcu_tasks_trace_pertask(struct task_struct *t, struct list_head *hop) @@ -1478,6 +1464,20 @@ static void rcu_tasks_trace_pertask(struct task_struct *t, trc_wait_for_one_reader(t, hop); } +/* Initialize for a new RCU-tasks-trace grace period. */ +static void rcu_tasks_trace_pregp_step(void) +{ + int cpu; + + // There shouldn't be any old IPIs, but... + for_each_possible_cpu(cpu) + WARN_ON_ONCE(per_cpu(trc_ipi_to_cpu, cpu)); + + // Disable CPU hotplug across the tasklist scan. + // This also waits for all readers in CPU-hotplug code paths. + cpus_read_lock(); +} + /* * Do intermediate processing between task and holdout scans and * pick up the idle tasks. From 19415004d5221ba59be6ef566fdbb52c44808f7e Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 17 May 2022 17:38:02 -0700 Subject: [PATCH 065/318] rcu-tasks: Avoid rcu_tasks_trace_pertask() duplicate list additions This commit adds checks within rcu_tasks_trace_pertask() to avoid duplicate (and destructive) additions to the holdouts list. These checks will be required later due to the possibility of a given task having blocked while in an RCU Tasks Trace read-side critical section, but now running on a CPU. Signed-off-by: Paul E. McKenney Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- kernel/rcu/tasks.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 66d8473f1bda..1aa6a24a9bc2 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1454,9 +1454,10 @@ static void rcu_tasks_trace_pertask(struct task_struct *t, struct list_head *hop) { // During early boot when there is only the one boot CPU, there - // is no idle task for the other CPUs. Also, the grace-period - // kthread is always in a quiescent state. Either way, just return. - if (unlikely(t == NULL) || t == current) + // is no idle task for the other CPUs. Also, the grace-period + // kthread is always in a quiescent state. In addition, just return + // if this task is already on the list. + if (unlikely(t == NULL) || t == current || !list_empty(&t->trc_holdout_list)) return; rcu_st_need_qs(t, 0); From 7460ade1fc6e3f62d5c0006c972755f0aefd41b2 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 18 May 2022 16:06:55 -0700 Subject: [PATCH 066/318] rcu-tasks: Scan running tasks for RCU Tasks Trace readers A running task might be within an RCU Tasks Trace read-side critical section for any length of time, but will not be placed on any of the per-CPU rcu_tasks_percpu structure's ->rtp_blkd_tasks lists. Therefore any RCU Tasks Trace grace-period processing that does not scan the full task list must interact with the running tasks. This commit therefore causes the rcu_tasks_trace_pregp_step() function to IPI each CPU in order to place the corresponding task on the holdouts list and to record whether or not it was in an RCU Tasks Trace read-side critical section. Yes, it is possible to avoid adding it to that list if it is not a reader, but that would prevent the system from remembering that this task was in a quiescent state. Which is why the running tasks are unconditionally added to the holdout list. Signed-off-by: Paul E. McKenney Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- kernel/rcu/tasks.h | 51 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 1aa6a24a9bc2..a8f95864c921 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -14,7 +14,7 @@ struct rcu_tasks; typedef void (*rcu_tasks_gp_func_t)(struct rcu_tasks *rtp); -typedef void (*pregp_func_t)(void); +typedef void (*pregp_func_t)(struct list_head *hop); typedef void (*pertask_func_t)(struct task_struct *t, struct list_head *hop); typedef void (*postscan_func_t)(struct list_head *hop); typedef void (*holdouts_func_t)(struct list_head *hop, bool ndrpt, bool *frptp); @@ -661,7 +661,7 @@ static void rcu_tasks_wait_gp(struct rcu_tasks *rtp) struct task_struct *t; set_tasks_gp_state(rtp, RTGS_PRE_WAIT_GP); - rtp->pregp_func(); + rtp->pregp_func(&holdouts); /* * There were callbacks, so we need to wait for an RCU-tasks @@ -791,7 +791,7 @@ static void rcu_tasks_wait_gp(struct rcu_tasks *rtp) // disabling. /* Pre-grace-period preparation. */ -static void rcu_tasks_pregp_step(void) +static void rcu_tasks_pregp_step(struct list_head *hop) { /* * Wait for all pre-existing t->on_rq and t->nvcsw transitions @@ -1449,24 +1449,48 @@ static void trc_wait_for_one_reader(struct task_struct *t, } } -/* Do first-round processing for the specified task. */ -static void rcu_tasks_trace_pertask(struct task_struct *t, - struct list_head *hop) +/* + * Initialize for first-round processing for the specified task. + * Return false if task is NULL or already taken care of, true otherwise. + */ +static bool rcu_tasks_trace_pertask_prep(struct task_struct *t, bool notself) { // During early boot when there is only the one boot CPU, there // is no idle task for the other CPUs. Also, the grace-period // kthread is always in a quiescent state. In addition, just return // if this task is already on the list. - if (unlikely(t == NULL) || t == current || !list_empty(&t->trc_holdout_list)) - return; + if (unlikely(t == NULL) || (t == current && notself) || !list_empty(&t->trc_holdout_list)) + return false; rcu_st_need_qs(t, 0); t->trc_ipi_to_cpu = -1; - trc_wait_for_one_reader(t, hop); + return true; +} + +/* Do first-round processing for the specified task. */ +static void rcu_tasks_trace_pertask(struct task_struct *t, struct list_head *hop) +{ + if (rcu_tasks_trace_pertask_prep(t, true)) + trc_wait_for_one_reader(t, hop); +} + +/* + * Get the current CPU's current task on the holdout list. + * Calls to this function must be serialized. + */ +static void rcu_tasks_trace_pertask_handler(void *hop_in) +{ + struct list_head *hop = hop_in; + struct task_struct *t = current; + + // Pull in the currently running task, but only if it is currently + // in an RCU tasks trace read-side critical section. + if (rcu_tasks_trace_pertask_prep(t, false)) + trc_add_holdout(t, hop); } /* Initialize for a new RCU-tasks-trace grace period. */ -static void rcu_tasks_trace_pregp_step(void) +static void rcu_tasks_trace_pregp_step(struct list_head *hop) { int cpu; @@ -1474,9 +1498,14 @@ static void rcu_tasks_trace_pregp_step(void) for_each_possible_cpu(cpu) WARN_ON_ONCE(per_cpu(trc_ipi_to_cpu, cpu)); - // Disable CPU hotplug across the tasklist scan. + // Disable CPU hotplug across the CPU scan. // This also waits for all readers in CPU-hotplug code paths. cpus_read_lock(); + + // These smp_call_function_single() calls are serialized to + // allow safe access to the hop list. + for_each_online_cpu(cpu) + smp_call_function_single(cpu, rcu_tasks_trace_pertask_handler, hop, 1); } /* From dc7d54b45170e1e3ced9f86718aa4274fd727790 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 18 May 2022 17:19:27 -0700 Subject: [PATCH 067/318] rcu-tasks: Pull in tasks blocked within RCU Tasks Trace readers This commit scans each CPU's ->rtp_blkd_tasks list, adding them to the list of holdout tasks. This will cause the current RCU Tasks Trace grace period to wait until these tasks exit their RCU Tasks Trace read-side critical sections. This commit will enable later work omitting the scan of the full task list. Signed-off-by: Paul E. McKenney Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- kernel/rcu/tasks.h | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index a8f95864c921..d318cdfd2309 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1492,7 +1492,11 @@ static void rcu_tasks_trace_pertask_handler(void *hop_in) /* Initialize for a new RCU-tasks-trace grace period. */ static void rcu_tasks_trace_pregp_step(struct list_head *hop) { + LIST_HEAD(blkd_tasks); int cpu; + unsigned long flags; + struct rcu_tasks_percpu *rtpcp; + struct task_struct *t; // There shouldn't be any old IPIs, but... for_each_possible_cpu(cpu) @@ -1506,6 +1510,26 @@ static void rcu_tasks_trace_pregp_step(struct list_head *hop) // allow safe access to the hop list. for_each_online_cpu(cpu) smp_call_function_single(cpu, rcu_tasks_trace_pertask_handler, hop, 1); + + // Only after all running tasks have been accounted for is it + // safe to take care of the tasks that have blocked within their + // current RCU tasks trace read-side critical section. + for_each_possible_cpu(cpu) { + rtpcp = per_cpu_ptr(rcu_tasks_trace.rtpcpu, cpu); + raw_spin_lock_irqsave_rcu_node(rtpcp, flags); + list_splice_init(&rtpcp->rtp_blkd_tasks, &blkd_tasks); + while (!list_empty(&blkd_tasks)) { + rcu_read_lock(); + t = list_first_entry(&blkd_tasks, struct task_struct, trc_blkd_node); + list_del_init(&t->trc_blkd_node); + list_add(&t->trc_blkd_node, &rtpcp->rtp_blkd_tasks); + raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); + rcu_tasks_trace_pertask(t, hop); + rcu_read_unlock(); + raw_spin_lock_irqsave_rcu_node(rtpcp, flags); + } + raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); + } } /* From 955a0192082023bf08f1be279182090264cb2557 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 18 May 2022 17:19:27 -0700 Subject: [PATCH 068/318] rcu-tasks: Stop RCU Tasks Trace from scanning idle tasks Now that RCU scans both running tasks and tasks that have blocked within their current RCU Tasks Trace read-side critical section, there is no need for it to scan the idle tasks. After all, an idle loop should not be remain within an RCU Tasks Trace read-side critical section across exit from idle, and from a BPF viewpoint, functions invoked from the idle loop should not sleep. So only running idle tasks can be within RCU Tasks Trace read-side critical sections. This commit therefore removes the scan of the idle tasks from the rcu_tasks_trace_postscan() function. Signed-off-by: Paul E. McKenney Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- kernel/rcu/tasks.h | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index d318cdfd2309..272c905995e5 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1533,16 +1533,10 @@ static void rcu_tasks_trace_pregp_step(struct list_head *hop) } /* - * Do intermediate processing between task and holdout scans and - * pick up the idle tasks. + * Do intermediate processing between task and holdout scans. */ static void rcu_tasks_trace_postscan(struct list_head *hop) { - int cpu; - - for_each_online_cpu(cpu) - rcu_tasks_trace_pertask(idle_task(cpu), hop); - // Re-enable CPU hotplug now that the tasklist scan has completed. cpus_read_unlock(); From ab69d3c8b99435ebd2ac4a241a5d931024c83c02 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 12 Apr 2022 15:17:37 -0700 Subject: [PATCH 069/318] torture: Make kvm-remote.sh announce which system is being waited on If a remote system fails in certain ways, for example, if it is rebooted without removing the contents of the /tmp directory, its remote.run file never will be removed and the kvm-remote.sh script will loop waiting forever. The manual workaround for this (hopefully!) rare event is to manually remove the file, which will cause the results up to the reboot to be collected and evaluated. Unfortunately, to work out which system is holding things up, the user must refer to the name of the last system whose results were collected, then look up the name of the next system in sequence, then manually remove the remote.run file. Even more unfortunately, this procedure can be fooled in runs where each system handles more than one batch should a given system take longer than expected, causing the systems to be handled out of order. This commit therefore causes kvm-remote.sh to print out the name of the system it will wait on next, allowing the user to refer directly to that name. Making the kvm-remote.sh script automatically handle unscheduled termination of the qemu processes is left as future work. Quite possibly deep future work. Signed-off-by: Paul E. McKenney --- tools/testing/selftests/rcutorture/bin/kvm-remote.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/testing/selftests/rcutorture/bin/kvm-remote.sh b/tools/testing/selftests/rcutorture/bin/kvm-remote.sh index 0ff59bd8b640..9f0a5d5ff2dd 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm-remote.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm-remote.sh @@ -262,6 +262,7 @@ echo All batches started. `date` | tee -a "$oldrun/remote-log" # Wait for all remaining scenarios to complete and collect results. for i in $systems do + echo " ---" Waiting for $i `date` | tee -a "$oldrun/remote-log" while checkremotefile "$i" "$resdir/$ds/remote.run" do sleep 30 From 245a62982502255314b63dd2c4daaedd1cd595a6 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Mon, 25 Apr 2022 09:04:04 +0800 Subject: [PATCH 070/318] rcu: Dump rcuc kthread status for CPUs not reporting quiescent state If the rcutree.use_softirq kernel boot parameter is disabled, then it is possible that a RCU CPU stall is due to the rcuc kthreads being starved of CPU time. There is currently no easy way to infer this from the RCU CPU stall warning output. This commit therefore adds a string of the form " rcuc=%ld jiffies(starved)" to a given CPU's output if the corresponding rcuc kthread has been starved for more than two seconds. [ paulmck: Eliminate extraneous space characters. ] Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/tree_stall.h | 49 ++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index 4995c078cff9..3556637768fd 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -409,7 +409,19 @@ static bool rcu_is_gp_kthread_starving(unsigned long *jp) static bool rcu_is_rcuc_kthread_starving(struct rcu_data *rdp, unsigned long *jp) { - unsigned long j = jiffies - READ_ONCE(rdp->rcuc_activity); + int cpu; + struct task_struct *rcuc; + unsigned long j; + + rcuc = rdp->rcu_cpu_kthread_task; + if (!rcuc) + return false; + + cpu = task_cpu(rcuc); + if (cpu_is_offline(cpu) || idle_cpu(cpu)) + return false; + + j = jiffies - READ_ONCE(rdp->rcuc_activity); if (jp) *jp = j; @@ -434,6 +446,9 @@ static void print_cpu_stall_info(int cpu) struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); char *ticks_title; unsigned long ticks_value; + bool rcuc_starved; + unsigned long j; + char buf[32]; /* * We could be printing a lot while holding a spinlock. Avoid @@ -451,7 +466,10 @@ static void print_cpu_stall_info(int cpu) delta = rcu_seq_ctr(rdp->mynode->gp_seq - rdp->rcu_iw_gp_seq); falsepositive = rcu_is_gp_kthread_starving(NULL) && rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp)); - pr_err("\t%d-%c%c%c%c: (%lu %s) idle=%03x/%ld/%#lx softirq=%u/%u fqs=%ld %s\n", + rcuc_starved = rcu_is_rcuc_kthread_starving(rdp, &j); + if (rcuc_starved) + sprintf(buf, " rcuc=%ld jiffies(starved)", j); + pr_err("\t%d-%c%c%c%c: (%lu %s) idle=%03x/%ld/%#lx softirq=%u/%u fqs=%ld%s%s\n", cpu, "O."[!!cpu_online(cpu)], "o."[!!(rdp->grpmask & rdp->mynode->qsmaskinit)], @@ -464,32 +482,10 @@ static void print_cpu_stall_info(int cpu) rdp->dynticks_nesting, rdp->dynticks_nmi_nesting, rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu), data_race(rcu_state.n_force_qs) - rcu_state.n_force_qs_gpstart, + rcuc_starved ? buf : "", falsepositive ? " (false positive?)" : ""); } -static void rcuc_kthread_dump(struct rcu_data *rdp) -{ - int cpu; - unsigned long j; - struct task_struct *rcuc; - - rcuc = rdp->rcu_cpu_kthread_task; - if (!rcuc) - return; - - cpu = task_cpu(rcuc); - if (cpu_is_offline(cpu) || idle_cpu(cpu)) - return; - - if (!rcu_is_rcuc_kthread_starving(rdp, &j)) - return; - - pr_err("%s kthread starved for %ld jiffies\n", rcuc->comm, j); - sched_show_task(rcuc); - if (!trigger_single_cpu_backtrace(cpu)) - dump_cpu_task(cpu); -} - /* Complain about starvation of grace-period kthread. */ static void rcu_check_gp_kthread_starvation(void) { @@ -662,9 +658,6 @@ static void print_cpu_stall(unsigned long gps) rcu_check_gp_kthread_expired_fqs_timer(); rcu_check_gp_kthread_starvation(); - if (!use_softirq) - rcuc_kthread_dump(rdp); - rcu_dump_cpu_stacks(); raw_spin_lock_irqsave_rcu_node(rnp, flags); From ed4ae5eff4b38797607cbdd80da394149110fb37 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 17 May 2022 21:00:04 -0700 Subject: [PATCH 071/318] rcu: Apply noinstr to rcu_idle_enter() and rcu_idle_exit() This commit applies the "noinstr" tag to the rcu_idle_enter() and rcu_idle_exit() functions, which are invoked from portions of the idle loop that cannot be instrumented. These tags require reworking the rcu_eqs_enter() and rcu_eqs_exit() functions that these two functions invoke in order to cause them to use normal assertions rather than lockdep. In addition, within rcu_idle_exit(), the raw versions of local_irq_save() and local_irq_restore() are used, again to avoid issues with lockdep in uninstrumented code. This patch is based in part on an earlier patch by Jiri Olsa, discussions with Peter Zijlstra and Frederic Weisbecker, earlier changes by Thomas Gleixner, and off-list discussions with Yonghong Song. Link: https://lore.kernel.org/lkml/20220515203653.4039075-1-jolsa@kernel.org/ Reported-by: Jiri Olsa Reported-by: Alexei Starovoitov Reported-by: Andrii Nakryiko Signed-off-by: Paul E. McKenney Reviewed-by: Yonghong Song --- kernel/rcu/tree.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index c25ba442044a..9a5edab5558c 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -631,8 +631,8 @@ static noinstr void rcu_eqs_enter(bool user) return; } - lockdep_assert_irqs_disabled(); instrumentation_begin(); + lockdep_assert_irqs_disabled(); trace_rcu_dyntick(TPS("Start"), rdp->dynticks_nesting, 0, atomic_read(&rdp->dynticks)); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); rcu_preempt_deferred_qs(current); @@ -659,9 +659,9 @@ static noinstr void rcu_eqs_enter(bool user) * If you add or remove a call to rcu_idle_enter(), be sure to test with * CONFIG_RCU_EQS_DEBUG=y. */ -void rcu_idle_enter(void) +void noinstr rcu_idle_enter(void) { - lockdep_assert_irqs_disabled(); + WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !raw_irqs_disabled()); rcu_eqs_enter(false); } EXPORT_SYMBOL_GPL(rcu_idle_enter); @@ -861,7 +861,7 @@ static void noinstr rcu_eqs_exit(bool user) struct rcu_data *rdp; long oldval; - lockdep_assert_irqs_disabled(); + WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !raw_irqs_disabled()); rdp = this_cpu_ptr(&rcu_data); oldval = rdp->dynticks_nesting; WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0); @@ -896,13 +896,13 @@ static void noinstr rcu_eqs_exit(bool user) * If you add or remove a call to rcu_idle_exit(), be sure to test with * CONFIG_RCU_EQS_DEBUG=y. */ -void rcu_idle_exit(void) +void noinstr rcu_idle_exit(void) { unsigned long flags; - local_irq_save(flags); + raw_local_irq_save(flags); rcu_eqs_exit(false); - local_irq_restore(flags); + raw_local_irq_restore(flags); } EXPORT_SYMBOL_GPL(rcu_idle_exit); From 5f8a62af527ad8a615d68889d816b72c3f5f227f Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:18 +0200 Subject: [PATCH 072/318] context_tracking: Remove unused context_tracking_in_user() This function is not used and CT_WARN_ON() coupled with ct_state() is the preferred way to assert context tracking state values. Reported-by: Nicolas Saenz Julienne Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- include/linux/context_tracking_state.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/include/linux/context_tracking_state.h b/include/linux/context_tracking_state.h index ae1e63e26947..edc7b46376a6 100644 --- a/include/linux/context_tracking_state.h +++ b/include/linux/context_tracking_state.h @@ -41,12 +41,7 @@ static inline bool context_tracking_enabled_this_cpu(void) return context_tracking_enabled() && __this_cpu_read(context_tracking.active); } -static __always_inline bool context_tracking_in_user(void) -{ - return __this_cpu_read(context_tracking.state) == CONTEXT_USER; -} #else -static __always_inline bool context_tracking_in_user(void) { return false; } static __always_inline bool context_tracking_enabled(void) { return false; } static __always_inline bool context_tracking_enabled_cpu(int cpu) { return false; } static __always_inline bool context_tracking_enabled_this_cpu(void) { return false; } From 1ef150cf40be986747c3fa0c0e75acaec412e85e Mon Sep 17 00:00:00 2001 From: Ammar Faizi Date: Fri, 20 May 2022 00:21:15 +0700 Subject: [PATCH 073/318] tools/nolibc/stdlib: Support overflow checking for older compiler versions Previously, we used __builtin_mul_overflow() to check for overflow in the multiplication operation in the calloc() function. However, older compiler versions don't support this built-in. This patch changes the overflow checking mechanism to make it work on any compiler version by using a division method to check for overflow. No functional change intended. While in there, remove the unused variable `void *orig`. Link: https://lore.kernel.org/lkml/20220330024114.GA18892@1wt.eu Suggested-by: Willy Tarreau Cc: Alviro Iskandar Setiawan Signed-off-by: Ammar Faizi Acked-by: Willy Tarreau Reviewed-by: Alviro Iskandar Setiawan Signed-off-by: Paul E. McKenney --- tools/include/nolibc/stdlib.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/include/nolibc/stdlib.h b/tools/include/nolibc/stdlib.h index 8fd32eaf8037..92378c4b9660 100644 --- a/tools/include/nolibc/stdlib.h +++ b/tools/include/nolibc/stdlib.h @@ -128,10 +128,9 @@ void *malloc(size_t len) static __attribute__((unused)) void *calloc(size_t size, size_t nmemb) { - void *orig; - size_t res = 0; + size_t x = size * nmemb; - if (__builtin_expect(__builtin_mul_overflow(nmemb, size, &res), 0)) { + if (__builtin_expect(size && ((x / size) != nmemb), 0)) { SET_ERRNO(ENOMEM); return NULL; } @@ -140,7 +139,7 @@ void *calloc(size_t size, size_t nmemb) * No need to zero the heap, the MAP_ANONYMOUS in malloc() * already does it. */ - return malloc(res); + return malloc(x); } static __attribute__((unused)) From 4f2c9703a128d21db7d0bf2ea14c5fe50d0dfe7e Mon Sep 17 00:00:00 2001 From: Alviro Iskandar Setiawan Date: Fri, 20 May 2022 00:21:16 +0700 Subject: [PATCH 074/318] tools/nolibc/stdio: Add format attribute to enable printf warnings When we use printf and fprintf functions from the nolibc, we don't get any warning from the compiler if we have the wrong arguments. For example, the following calls will compile silently: ``` printf("%s %s\n", "aaa"); fprintf(stdout, "%s %s\n", "xxx", 1); ``` (Note the wrong arguments). Those calls are undefined behavior. The compiler can help us warn about the above mistakes by adding a `printf` format attribute to those functions declaration. This patch adds it, and now it yields these warnings for those mistakes: ``` warning: format `%s` expects a matching `char *` argument [-Wformat=] warning: format `%s` expects argument of type `char *`, but argument 4 has type `int` [-Wformat=] ``` [ ammarfaizi2: Simplify the attribute placement. ] Signed-off-by: Alviro Iskandar Setiawan Signed-off-by: Ammar Faizi Acked-by: Willy Tarreau Signed-off-by: Paul E. McKenney --- tools/include/nolibc/stdio.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/include/nolibc/stdio.h b/tools/include/nolibc/stdio.h index 15dedf8d0902..a3cebc4bc3ac 100644 --- a/tools/include/nolibc/stdio.h +++ b/tools/include/nolibc/stdio.h @@ -273,7 +273,7 @@ int vfprintf(FILE *stream, const char *fmt, va_list args) return written; } -static __attribute__((unused)) +static __attribute__((unused, format(printf, 2, 3))) int fprintf(FILE *stream, const char *fmt, ...) { va_list args; @@ -285,7 +285,7 @@ int fprintf(FILE *stream, const char *fmt, ...) return ret; } -static __attribute__((unused)) +static __attribute__((unused, format(printf, 1, 2))) int printf(const char *fmt, ...) { va_list args; From 6a3ad243b29be01d0c30f8c3b35a1149e1a0139a Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Sat, 28 May 2022 17:45:44 +0200 Subject: [PATCH 075/318] tools/nolibc: fix the makefile to also work as "make -C tools ..." As reported by Linus, the nolibc's makefile is currently broken when invoked as per the documented method (make -C tools nolibc_), because it now relies on the ARCH and OUTPUT variables that are not set in this case. This patch addresses this by sourcing subarch.include, and by presetting OUTPUT to the current directory if not set. This is sufficient to make the commands work both as a standalone target and as a tools/ sub-target. Signed-off-by: Willy Tarreau Signed-off-by: Paul E. McKenney --- tools/include/nolibc/Makefile | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tools/include/nolibc/Makefile b/tools/include/nolibc/Makefile index 7a16d917c185..e8bac6ef3653 100644 --- a/tools/include/nolibc/Makefile +++ b/tools/include/nolibc/Makefile @@ -7,6 +7,22 @@ ifeq ($(srctree),) srctree := $(patsubst %/tools/include/,%,$(dir $(CURDIR))) endif +# when run as make -C tools/ nolibc_ the arch is not set +ifeq ($(ARCH),) +include $(srctree)/scripts/subarch.include +ARCH = $(SUBARCH) +endif + +# OUTPUT is only set when run from the main makefile, otherwise +# it defaults to this nolibc directory. +OUTPUT ?= $(CURDIR)/ + +ifeq ($(V),1) +Q= +else +Q=@ +endif + nolibc_arch := $(patsubst arm64,aarch64,$(ARCH)) arch_file := arch-$(nolibc_arch).h all_files := ctype.h errno.h nolibc.h signal.h std.h stdio.h stdlib.h string.h \ @@ -36,7 +52,7 @@ headers: headers_standalone: headers $(Q)$(MAKE) -C $(srctree) headers - $(Q)$(MAKE) -C $(srctree) headers_install INSTALL_HDR_PATH=$(OUTPUT)/sysroot + $(Q)$(MAKE) -C $(srctree) headers_install INSTALL_HDR_PATH=$(OUTPUT)sysroot clean: $(call QUIET_CLEAN, nolibc) rm -rf "$(OUTPUT)sysroot" From fe20cad47e6c79febd323c8db0a1f85896acbcdd Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Sat, 28 May 2022 17:45:45 +0200 Subject: [PATCH 076/318] tools/nolibc: make the default target build the headers The help in "make -C tools" enumerates nolibc as a valid target so we must at least make it do something. Let's make it do the equivalent of "make headers" in that it will prepare a sysroot with the arch's headers, but will not install the kernel's headers. This is the minimum some tools will need when built with a full-blown toolchain anyway. Signed-off-by: Willy Tarreau Signed-off-by: Paul E. McKenney --- tools/Makefile | 3 +++ tools/include/nolibc/Makefile | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/Makefile b/tools/Makefile index c074e42fd92f..e497875fc7e3 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -78,6 +78,9 @@ bpf/%: FORCE libapi: FORCE $(call descend,lib/api) +nolibc: FORCE + $(call descend,include/nolibc) + nolibc_%: FORCE $(call descend,include/nolibc,$(patsubst nolibc_%,%,$@)) diff --git a/tools/include/nolibc/Makefile b/tools/include/nolibc/Makefile index e8bac6ef3653..9768819abd55 100644 --- a/tools/include/nolibc/Makefile +++ b/tools/include/nolibc/Makefile @@ -29,7 +29,7 @@ all_files := ctype.h errno.h nolibc.h signal.h std.h stdio.h stdlib.h string.h \ sys.h time.h types.h unistd.h # install all headers needed to support a bare-metal compiler -all: +all: headers # Note: when ARCH is "x86" we concatenate both x86_64 and i386 headers: From 4f8126f3a665456b68ae923cd7a7e9b9eb98547d Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Sat, 28 May 2022 17:45:46 +0200 Subject: [PATCH 077/318] tools/nolibc: add a help target to list supported targets The "help" target simply presents the list of supported targets and the current set of variables being used to build the sysroot. Since the help in tools/ suggests to use "install", which is supported by most tools while such a target is not really relevant here, an "install" target was also added, redirecting to "help". Signed-off-by: Willy Tarreau Signed-off-by: Paul E. McKenney --- tools/include/nolibc/Makefile | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tools/include/nolibc/Makefile b/tools/include/nolibc/Makefile index 9768819abd55..cfd06764b5ae 100644 --- a/tools/include/nolibc/Makefile +++ b/tools/include/nolibc/Makefile @@ -31,6 +31,23 @@ all_files := ctype.h errno.h nolibc.h signal.h std.h stdio.h stdlib.h string.h \ # install all headers needed to support a bare-metal compiler all: headers +install: help + +help: + @echo "Supported targets under nolibc:" + @echo " all call \"headers\"" + @echo " clean clean the sysroot" + @echo " headers prepare a sysroot in tools/include/nolibc/sysroot" + @echo " headers_standalone like \"headers\", and also install kernel headers" + @echo " help this help" + @echo "" + @echo "These targets may also be called from tools as \"make nolibc_\"." + @echo "" + @echo "Currently using the following variables:" + @echo " ARCH = $(ARCH)" + @echo " OUTPUT = $(OUTPUT)" + @echo "" + # Note: when ARCH is "x86" we concatenate both x86_64 and i386 headers: $(Q)mkdir -p $(OUTPUT)sysroot From f67671baadf6dbae8eca46cb95e85b762e462b2b Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:19 +0200 Subject: [PATCH 078/318] context_tracking: Add a note about noinstr VS unsafe context tracking functions Some context tracking functions enter or exit into/from RCU idle mode while using trace-able and lockdep-aware IRQs (un-)masking. As a result those functions can't get tagged as noinstr. This is unlikely to be fixed since these are obsolete APIs. Drop a note about this matter. [ paulmck: Apply Peter Zijlstra feedback. ] Reported-by: Peter Zijlstra Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- kernel/context_tracking.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index 36a98c48aedc..3082332f6476 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -103,6 +103,15 @@ void noinstr __context_tracking_enter(enum ctx_state state) } EXPORT_SYMBOL_GPL(__context_tracking_enter); +/* + * OBSOLETE: + * This function should be noinstr but the below local_irq_restore() is + * unsafe because it involves illegal RCU uses through tracing and lockdep. + * This is unlikely to be fixed as this function is obsolete. The preferred + * way is to call __context_tracking_enter() through user_enter_irqoff() + * or context_tracking_guest_enter(). It should be the arch entry code + * responsibility to call into context tracking with IRQs disabled. + */ void context_tracking_enter(enum ctx_state state) { unsigned long flags; @@ -125,6 +134,14 @@ void context_tracking_enter(enum ctx_state state) NOKPROBE_SYMBOL(context_tracking_enter); EXPORT_SYMBOL_GPL(context_tracking_enter); +/* + * OBSOLETE: + * This function should be noinstr but it unsafely calls local_irq_restore(), + * involving illegal RCU uses through tracing and lockdep. + * This is unlikely to be fixed as this function is obsolete. The preferred + * way is to call user_enter_irqoff(). It should be the arch entry code + * responsibility to call into context tracking with IRQs disabled. + */ void context_tracking_user_enter(void) { user_enter(); @@ -168,6 +185,15 @@ void noinstr __context_tracking_exit(enum ctx_state state) } EXPORT_SYMBOL_GPL(__context_tracking_exit); +/* + * OBSOLETE: + * This function should be noinstr but the below local_irq_save() is + * unsafe because it involves illegal RCU uses through tracing and lockdep. + * This is unlikely to be fixed as this function is obsolete. The preferred + * way is to call __context_tracking_exit() through user_exit_irqoff() + * or context_tracking_guest_exit(). It should be the arch entry code + * responsibility to call into context tracking with IRQs disabled. + */ void context_tracking_exit(enum ctx_state state) { unsigned long flags; @@ -182,6 +208,14 @@ void context_tracking_exit(enum ctx_state state) NOKPROBE_SYMBOL(context_tracking_exit); EXPORT_SYMBOL_GPL(context_tracking_exit); +/* + * OBSOLETE: + * This function should be noinstr but it unsafely calls local_irq_save(), + * involving illegal RCU uses through tracing and lockdep. This is unlikely + * to be fixed as this function is obsolete. The preferred way is to call + * user_exit_irqoff(). It should be the arch entry code responsibility to + * call into context tracking with IRQs disabled. + */ void context_tracking_user_exit(void) { user_exit(); From ef54ccb61616d8293bc68220d88a8e74271141b5 Mon Sep 17 00:00:00 2001 From: Xiu Jianfeng Date: Fri, 17 Jun 2022 17:44:12 +0800 Subject: [PATCH 079/318] selinux: selinux_add_opt() callers free memory The selinux_add_opt() function may need to allocate memory for the mount options if none has already been allocated, but there is no need to free that memory on error as the callers handle that. Drop the existing kfree() on error to help increase consistency in the selinux_add_opt() error handling. This patch also changes selinux_add_opt() to return -EINVAL when the mount option value, @s, is NULL. It currently return -ENOMEM. Link: https://lore.kernel.org/lkml/20220611090550.135674-1-xiujianfeng@huawei.com/T/ Suggested-by: Paul Moore Signed-off-by: Xiu Jianfeng [PM: fix subject, rework commit description language] Signed-off-by: Paul Moore --- security/selinux/hooks.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 4d20a139a86d..9d08b91e05a2 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -944,10 +944,12 @@ out: return rc; } +/* + * NOTE: the caller is resposible for freeing the memory even if on error. + */ static int selinux_add_opt(int token, const char *s, void **mnt_opts) { struct selinux_mnt_opts *opts = *mnt_opts; - bool is_alloc_opts = false; u32 *dst_sid; int rc; @@ -955,7 +957,7 @@ static int selinux_add_opt(int token, const char *s, void **mnt_opts) /* eaten and completely ignored */ return 0; if (!s) - return -ENOMEM; + return -EINVAL; if (!selinux_initialized(&selinux_state)) { pr_warn("SELinux: Unable to set superblock options before the security server is initialized\n"); @@ -967,7 +969,6 @@ static int selinux_add_opt(int token, const char *s, void **mnt_opts) if (!opts) return -ENOMEM; *mnt_opts = opts; - is_alloc_opts = true; } switch (token) { @@ -1002,10 +1003,6 @@ static int selinux_add_opt(int token, const char *s, void **mnt_opts) return rc; err: - if (is_alloc_opts) { - kfree(opts); - *mnt_opts = NULL; - } pr_warn(SEL_MOUNT_FAIL_MSG); return -EINVAL; } From cb506e130e02da24d98103d83b8de859e26d1860 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sun, 17 Apr 2022 20:43:13 -0700 Subject: [PATCH 080/318] rcutorture: Update rcutorture.fwd_progress help text This commit updates the rcutorture.fwd_progress help text to say that it is the number of forward-progress kthreads to spawn rather than the old enable/disable functionality. While in the area, make the list of torture-test parameters easier to read by taking advantage of 100 columns. Signed-off-by: Paul E. McKenney Reviewed-by: Neeraj Upadhyay --- kernel/rcu/rcutorture.c | 53 ++++++++++++++--------------------------- 1 file changed, 18 insertions(+), 35 deletions(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 7120165a9342..6f47d1490c4f 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -75,62 +75,45 @@ MODULE_AUTHOR("Paul E. McKenney and Josh Triplett Date: Wed, 20 Apr 2022 08:59:46 -0700 Subject: [PATCH 081/318] doc: Document the rcutree.rcu_divisor kernel boot parameter This commit adds kernel-parameters.txt documentation for the rcutree.rcu_divisor kernel boot parameter, which controls the softirq callback-invocation batch limit. Cc: Eric Dumazet Signed-off-by: Paul E. McKenney Reviewed-by: Neeraj Upadhyay --- Documentation/admin-guide/kernel-parameters.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 2522b11e593f..bdf431bdbfdc 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -4666,6 +4666,21 @@ When RCU_NOCB_CPU is set, also adjust the priority of NOCB callback kthreads. + rcutree.rcu_divisor= [KNL] + Set the shift-right count to use to compute + the callback-invocation batch limit bl from + the number of callbacks queued on this CPU. + The result will be bounded below by the value of + the rcutree.blimit kernel parameter. Every bl + callbacks, the softirq handler will exit in + order to allow the CPU to do other work. + + Please note that this callback-invocation batch + limit applies only to non-offloaded callback + invocation. Offloaded callbacks are instead + invoked in the context of an rcuoc kthread, which + scheduler will preempt as it does any other task. + rcutree.rcu_nocb_gp_stride= [KNL] Set the number of NOCB callback kthreads in each group, which defaults to the square root From 89f7f29140da767f4675efbbe7892f38786451ec Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 27 Apr 2022 09:24:31 -0700 Subject: [PATCH 082/318] doc: Document rcutree.nocb_nobypass_lim_per_jiffy kernel parameter This commit provides documentation for the kernel parameter controlling RCU's handling of callback floods on offloaded (rcu_nocbs) CPUs. This parameter might be obscure, but it is always there when you need it. Reported-by: Frederic Weisbecker Reported-by: Uladzislau Rezki Signed-off-by: Paul E. McKenney Reviewed-by: Neeraj Upadhyay --- Documentation/admin-guide/kernel-parameters.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index bdf431bdbfdc..40a05df05580 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -4681,6 +4681,19 @@ invoked in the context of an rcuoc kthread, which scheduler will preempt as it does any other task. + rcutree.nocb_nobypass_lim_per_jiffy= [KNL] + On callback-offloaded (rcu_nocbs) CPUs, + RCU reduces the lock contention that would + otherwise be caused by callback floods through + use of the ->nocb_bypass list. However, in the + common non-flooded case, RCU queues directly to + the main ->cblist in order to avoid the extra + overhead of the ->nocb_bypass list and its lock. + But if there are too many callbacks queued during + a single jiffy, RCU pre-queues the callbacks into + the ->nocb_bypass queue. The definition of "too + many" is supplied by this kernel boot parameter. + rcutree.rcu_nocb_gp_stride= [KNL] Set the number of NOCB callback kthreads in each group, which defaults to the square root From 1a4a8153e0df1705397aa3ff561dd8fdc2e6fc23 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 20 May 2022 10:21:00 -0700 Subject: [PATCH 083/318] rcu-tasks: Stop RCU Tasks Trace from scanning full tasks list This commit takes off the training wheels and relies only on scanning currently running tasks and tasks that have blocked or been preempted within their current RCU Tasks Trace read-side critical section. Before this commit, the time complexity of an RCU Tasks Trace grace period is O(T), where T is the number of tasks. After this commit, this time complexity is O(C+B), where C is the number of CPUs and B is the number of tasks that have blocked (or been preempted) at least once during their current RCU Tasks Trace read-side critical sections. Of course, if all tasks have blocked (or been preempted) at least once during their current RCU Tasks Trace read-side critical sections, this is still O(T), but current expectations are that RCU Tasks Trace read-side critical section will be short and that there will normally not be large numbers of tasks blocked within such a critical section. Dave Marchevsky kindly measured the effects of this commit on the RCU Tasks Trace grace-period latency and the rcu_tasks_trace_kthread task's CPU consumption per RCU Tasks Trace grace period over the course of a fixed test, all in milliseconds: Before After GP latency 22.3 ms stddev > 0.1 17.0 ms stddev < 0.1 GP CPU 2.3 ms stddev 0.3 1.1 ms stddev 0.2 This was on a system with 15,000 tasks, so it is reasonable to expect much larger savings on the systems on which this issue was first noted, given that they sport well in excess of 100,000 tasks. CPU consumption was measured using profiling techniques. Signed-off-by: Paul E. McKenney Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh Tested-by: Dave Marchevsky --- kernel/rcu/tasks.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 272c905995e5..fe0552086ccf 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -670,10 +670,12 @@ static void rcu_tasks_wait_gp(struct rcu_tasks *rtp) * and make a list of them in holdouts. */ set_tasks_gp_state(rtp, RTGS_SCAN_TASKLIST); - rcu_read_lock(); - for_each_process_thread(g, t) - rtp->pertask_func(t, &holdouts); - rcu_read_unlock(); + if (rtp->pertask_func) { + rcu_read_lock(); + for_each_process_thread(g, t) + rtp->pertask_func(t, &holdouts); + rcu_read_unlock(); + } set_tasks_gp_state(rtp, RTGS_POST_SCAN_TASKLIST); rtp->postscan_func(&holdouts); @@ -1746,7 +1748,6 @@ static int __init rcu_spawn_tasks_trace_kthread(void) rcu_tasks_trace.init_fract = 1; } rcu_tasks_trace.pregp_func = rcu_tasks_trace_pregp_step; - rcu_tasks_trace.pertask_func = rcu_tasks_trace_pertask; rcu_tasks_trace.postscan_func = rcu_tasks_trace_postscan; rcu_tasks_trace.holdouts_func = check_all_holdout_tasks_trace; rcu_tasks_trace.postgp_func = rcu_tasks_trace_postgp; From ffcc21a315e1ebafad51b318e8ac0cb884df0cdc Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 1 Jun 2022 21:26:57 -0700 Subject: [PATCH 084/318] rcu-tasks: Maintain a count of tasks blocking RCU Tasks Trace grace period This commit maintains a new n_trc_holdouts counter that tracks the number of tasks blocking the RCU Tasks grace period. This counter is useful for debugging, and its value has been added to a diagostic message. Signed-off-by: Paul E. McKenney Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- kernel/rcu/tasks.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index fe0552086ccf..9d7d6fd4b8a7 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1206,6 +1206,7 @@ static DEFINE_PER_CPU(bool, trc_ipi_to_cpu); static unsigned long n_heavy_reader_attempts; static unsigned long n_heavy_reader_updates; static unsigned long n_heavy_reader_ofl_updates; +static unsigned long n_trc_holdouts; void call_rcu_tasks_trace(struct rcu_head *rhp, rcu_callback_t func); DEFINE_RCU_TASKS(rcu_tasks_trace, rcu_tasks_wait_gp, call_rcu_tasks_trace, @@ -1299,6 +1300,7 @@ static void trc_add_holdout(struct task_struct *t, struct list_head *bhp) if (list_empty(&t->trc_holdout_list)) { get_task_struct(t); list_add(&t->trc_holdout_list, bhp); + n_trc_holdouts++; } } @@ -1308,6 +1310,7 @@ static void trc_del_holdout(struct task_struct *t) if (!list_empty(&t->trc_holdout_list)) { list_del_init(&t->trc_holdout_list); put_task_struct(t); + n_trc_holdouts--; } } @@ -1760,7 +1763,8 @@ void show_rcu_tasks_trace_gp_kthread(void) { char buf[64]; - sprintf(buf, "h:%lu/%lu/%lu", + sprintf(buf, "N%lu h:%lu/%lu/%lu", + data_race(n_trc_holdouts), data_race(n_heavy_reader_ofl_updates), data_race(n_heavy_reader_updates), data_race(n_heavy_reader_attempts)); From e386b6725798eec07facedf4d4bb710c079fd25c Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 2 Jun 2022 17:30:01 -0700 Subject: [PATCH 085/318] rcu-tasks: Eliminate RCU Tasks Trace IPIs to online CPUs Currently, the RCU Tasks Trace grace-period kthread IPIs each online CPU using smp_call_function_single() in order to track any tasks currently in RCU Tasks Trace read-side critical sections during which the corresponding task has neither blocked nor been preempted. These IPIs are annoying and are also not strictly necessary because any task that blocks or is preempted within its current RCU Tasks Trace read-side critical section will be tracked on one of the per-CPU rcu_tasks_percpu structure's ->rtp_blkd_tasks list. So the only time that this is a problem is if one of the CPUs runs through a long-duration RCU Tasks Trace read-side critical section without a context switch. Note that the task_call_func() function cannot help here because there is no safe way to identify the target task. Of course, the task_call_func() function will be very useful later, when processing the list of tasks, but it needs to know the task. This commit therefore creates a cpu_curr_snapshot() function that returns a pointer the task_struct structure of some task that happened to be running on the specified CPU more or less during the time that the cpu_curr_snapshot() function was executing. If there was no context switch during this time, this function will return a pointer to the task_struct structure of the task that was running throughout. If there was a context switch, then the outgoing task will be taken care of by RCU's context-switch hook, and the incoming task was either already taken care during some previous context switch, or it is not currently within an RCU Tasks Trace read-side critical section. And in this latter case, the grace period already started, so there is no need to wait on this task. This new cpu_curr_snapshot() function is invoked on each CPU early in the RCU Tasks Trace grace-period processing, and the resulting tasks are queued for later quiescent-state inspection. Signed-off-by: Paul E. McKenney Cc: Peter Zijlstra Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- include/linux/sched.h | 1 + kernel/rcu/tasks.h | 24 +++++++----------------- kernel/sched/core.c | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 17 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index b88caf54e168..72242bc73d85 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2224,6 +2224,7 @@ static inline void set_task_cpu(struct task_struct *p, unsigned int cpu) extern bool sched_task_on_rq(struct task_struct *p); extern unsigned long get_wchan(struct task_struct *p); +extern struct task_struct *cpu_curr_snapshot(int cpu); /* * In order to reduce various lock holder preemption latencies provide an diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 9d7d6fd4b8a7..c2aae2643a0b 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1479,21 +1479,6 @@ static void rcu_tasks_trace_pertask(struct task_struct *t, struct list_head *hop trc_wait_for_one_reader(t, hop); } -/* - * Get the current CPU's current task on the holdout list. - * Calls to this function must be serialized. - */ -static void rcu_tasks_trace_pertask_handler(void *hop_in) -{ - struct list_head *hop = hop_in; - struct task_struct *t = current; - - // Pull in the currently running task, but only if it is currently - // in an RCU tasks trace read-side critical section. - if (rcu_tasks_trace_pertask_prep(t, false)) - trc_add_holdout(t, hop); -} - /* Initialize for a new RCU-tasks-trace grace period. */ static void rcu_tasks_trace_pregp_step(struct list_head *hop) { @@ -1513,8 +1498,13 @@ static void rcu_tasks_trace_pregp_step(struct list_head *hop) // These smp_call_function_single() calls are serialized to // allow safe access to the hop list. - for_each_online_cpu(cpu) - smp_call_function_single(cpu, rcu_tasks_trace_pertask_handler, hop, 1); + for_each_online_cpu(cpu) { + rcu_read_lock(); + t = cpu_curr_snapshot(cpu); + if (rcu_tasks_trace_pertask_prep(t, true)) + trc_add_holdout(t, hop); + rcu_read_unlock(); + } // Only after all running tasks have been accounted for is it // safe to take care of the tasks that have blocked within their diff --git a/kernel/sched/core.c b/kernel/sched/core.c index da0bf6fe9ecd..9568019be124 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4263,6 +4263,38 @@ int task_call_func(struct task_struct *p, task_call_f func, void *arg) return ret; } +/** + * cpu_curr_snapshot - Return a snapshot of the currently running task + * @cpu: The CPU on which to snapshot the task. + * + * Returns the task_struct pointer of the task "currently" running on + * the specified CPU. If the same task is running on that CPU throughout, + * the return value will be a pointer to that task's task_struct structure. + * If the CPU did any context switches even vaguely concurrently with the + * execution of this function, the return value will be a pointer to the + * task_struct structure of a randomly chosen task that was running on + * that CPU somewhere around the time that this function was executing. + * + * If the specified CPU was offline, the return value is whatever it + * is, perhaps a pointer to the task_struct structure of that CPU's idle + * task, but there is no guarantee. Callers wishing a useful return + * value must take some action to ensure that the specified CPU remains + * online throughout. + * + * This function executes full memory barriers before and after fetching + * the pointer, which permits the caller to confine this function's fetch + * with respect to the caller's accesses to other shared variables. + */ +struct task_struct *cpu_curr_snapshot(int cpu) +{ + struct task_struct *t; + + smp_mb(); /* Pairing determined by caller's synchronization design. */ + t = rcu_dereference(cpu_curr(cpu)); + smp_mb(); /* Pairing determined by caller's synchronization design. */ + return t; +} + /** * wake_up_process - Wake up a specific process * @p: The process to be woken up. From 56096ecd5b04148b6d292e3847c23d4a2a454e94 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 7 Jun 2022 17:25:03 -0700 Subject: [PATCH 086/318] rcu-tasks: Disable and enable CPU hotplug in same function The rcu_tasks_trace_pregp_step() function invokes cpus_read_lock() to disable CPU hotplug, and a later call to the rcu_tasks_trace_postscan() function invokes cpus_read_unlock() to re-enable it. This was absolutely necessary in the past in order to protect the intervening scan of the full tasks list, but there is no longer such a scan. This commit therefore improves readability by moving the cpus_read_unlock() call to the end of the rcu_tasks_trace_pregp_step() function. This commit is a pure code-motion commit without any (intended) change in functionality. Signed-off-by: Paul E. McKenney Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- kernel/rcu/tasks.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index c2aae2643a0b..bf9cc5bc4ae5 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1525,6 +1525,9 @@ static void rcu_tasks_trace_pregp_step(struct list_head *hop) } raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); } + + // Re-enable CPU hotplug now that the holdout list is populated. + cpus_read_unlock(); } /* @@ -1532,9 +1535,6 @@ static void rcu_tasks_trace_pregp_step(struct list_head *hop) */ static void rcu_tasks_trace_postscan(struct list_head *hop) { - // Re-enable CPU hotplug now that the tasklist scan has completed. - cpus_read_unlock(); - // Wait for late-stage exiting tasks to finish exiting. // These might have passed the call to exit_tasks_rcu_finish(). synchronize_rcu(); From eea3423b162d5d5cdc08af23e8ee2c2d1134fd07 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 6 Jun 2022 21:30:38 -0700 Subject: [PATCH 087/318] rcu-tasks: Update comments This commit updates comments to reflect the changes in the series of commits that eliminated the full task-list scan. Signed-off-by: Paul E. McKenney Cc: Neeraj Upadhyay Cc: Eric Dumazet Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Martin KaFai Lau Cc: KP Singh --- kernel/rcu/tasks.h | 71 +++++++++++++++++++++------------------------- 1 file changed, 33 insertions(+), 38 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index bf9cc5bc4ae5..df6b2cb2f205 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1138,11 +1138,10 @@ EXPORT_SYMBOL_GPL(show_rcu_tasks_rude_gp_kthread); // 3. Avoids expensive read-side instructions, having overhead similar // to that of Preemptible RCU. // -// There are of course downsides. The grace-period code can send IPIs to -// CPUs, even when those CPUs are in the idle loop or in nohz_full userspace. -// It is necessary to scan the full tasklist, much as for Tasks RCU. There -// is a single callback queue guarded by a single lock, again, much as for -// Tasks RCU. If needed, these downsides can be at least partially remedied. +// There are of course downsides. For example, the grace-period code +// can send IPIs to CPUs, even when those CPUs are in the idle loop or +// in nohz_full userspace. If needed, these downsides can be at least +// partially remedied. // // Perhaps most important, this variant of RCU does not affect the vanilla // flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace @@ -1155,38 +1154,30 @@ EXPORT_SYMBOL_GPL(show_rcu_tasks_rude_gp_kthread); // invokes these functions in this order: // // rcu_tasks_trace_pregp_step(): -// Initialize the count of readers and block CPU-hotplug operations. -// rcu_tasks_trace_pertask(), invoked on every non-idle task: -// Initialize per-task state and attempt to identify an immediate -// quiescent state for that task, or, failing that, attempt to -// set that task's .need_qs flag so that task's next outermost -// rcu_read_unlock_trace() will report the quiescent state (in which -// case the count of readers is incremented). If both attempts fail, -// the task is added to a "holdout" list. Note that IPIs are used -// to invoke trc_read_check_handler() in the context of running tasks -// in order to avoid ordering overhead on common-case shared-variable -// accessses. +// Disables CPU hotplug, adds all currently executing tasks to the +// holdout list, then checks the state of all tasks that blocked +// or were preempted within their current RCU Tasks Trace read-side +// critical section, adding them to the holdout list if appropriate. +// Finally, this function re-enables CPU hotplug. +// The ->pertask_func() pointer is NULL, so there is no per-task processing. // rcu_tasks_trace_postscan(): -// Initialize state and attempt to identify an immediate quiescent -// state as above (but only for idle tasks), unblock CPU-hotplug -// operations, and wait for an RCU grace period to avoid races with -// tasks that are in the process of exiting. +// Invokes synchronize_rcu() to wait for late-stage exiting tasks +// to finish exiting. // check_all_holdout_tasks_trace(), repeatedly until holdout list is empty: // Scans the holdout list, attempting to identify a quiescent state // for each task on the list. If there is a quiescent state, the -// corresponding task is removed from the holdout list. +// corresponding task is removed from the holdout list. Once this +// list is empty, the grace period has completed. // rcu_tasks_trace_postgp(): -// Wait for the count of readers do drop to zero, reporting any stalls. -// Also execute full memory barriers to maintain ordering with code -// executing after the grace period. +// Provides the needed full memory barrier and does debug checks. // // The exit_tasks_rcu_finish_trace() synchronizes with exiting tasks. // -// Pre-grace-period update-side code is ordered before the grace -// period via the ->cbs_lock and barriers in rcu_tasks_kthread(). -// Pre-grace-period read-side code is ordered before the grace period by -// atomic_dec_and_test() of the count of readers (for IPIed readers) and by -// scheduler context-switch ordering (for locked-down non-running readers). +// Pre-grace-period update-side code is ordered before the grace period +// via the ->cbs_lock and barriers in rcu_tasks_kthread(). Pre-grace-period +// read-side code is ordered before the grace period by atomic operations +// on .b.need_qs flag of each task involved in this process, or by scheduler +// context-switch ordering (for locked-down non-running readers). // The lockdep state must be outside of #ifdef to be useful. #ifdef CONFIG_DEBUG_LOCK_ALLOC @@ -1245,7 +1236,10 @@ u8 rcu_trc_cmpxchg_need_qs(struct task_struct *t, u8 old, u8 new) } EXPORT_SYMBOL_GPL(rcu_trc_cmpxchg_need_qs); -/* If we are the last reader, wake up the grace-period kthread. */ +/* + * If we are the last reader, signal the grace-period kthread. + * Also remove from the per-CPU list of blocked tasks. + */ void rcu_read_unlock_trace_special(struct task_struct *t) { unsigned long flags; @@ -1336,9 +1330,9 @@ static void trc_read_check_handler(void *t_in) if (unlikely(nesting < 0)) goto reset_ipi; - // Get here if the task is in a read-side critical section. Set - // its state so that it will awaken the grace-period kthread upon - // exit from that critical section. + // Get here if the task is in a read-side critical section. + // Set its state so that it will update state for the grace-period + // kthread upon exit from that critical section. rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS | TRC_NEED_QS_CHECKED); reset_ipi: @@ -1387,7 +1381,7 @@ static int trc_inspect_reader(struct task_struct *t, void *bhp_in) return 0; // In QS, so done. } if (nesting < 0) - return -EINVAL; // QS transitioning, try again later. + return -EINVAL; // Reader transitioning, try again later. // The task is in a read-side critical section, so set up its // state so that it will update state upon exit from that critical @@ -1492,11 +1486,12 @@ static void rcu_tasks_trace_pregp_step(struct list_head *hop) for_each_possible_cpu(cpu) WARN_ON_ONCE(per_cpu(trc_ipi_to_cpu, cpu)); - // Disable CPU hotplug across the CPU scan. - // This also waits for all readers in CPU-hotplug code paths. + // Disable CPU hotplug across the CPU scan for the benefit of + // any IPIs that might be needed. This also waits for all readers + // in CPU-hotplug code paths. cpus_read_lock(); - // These smp_call_function_single() calls are serialized to + // These rcu_tasks_trace_pertask_prep() calls are serialized to // allow safe access to the hop list. for_each_online_cpu(cpu) { rcu_read_lock(); @@ -1608,7 +1603,7 @@ static void check_all_holdout_tasks_trace(struct list_head *hop, { struct task_struct *g, *t; - // Disable CPU hotplug across the holdout list scan. + // Disable CPU hotplug across the holdout list scan for IPIs. cpus_read_lock(); list_for_each_entry_safe(t, g, hop, trc_holdout_list) { From 1cf1144e8473e8c3180ac8b91309e29b6acfd95f Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 7 Jun 2022 15:23:52 -0700 Subject: [PATCH 088/318] rcu-tasks: Be more patient for RCU Tasks boot-time testing The RCU-Tasks family of grace-period primitives can take some time to complete, and the amount of time can depend on the exact hardware and software configuration. Some configurations boot up fast enough that the RCU-Tasks verification process gets false-positive failures. This commit therefore allows up to 30 seconds for the grace periods to complete, with this value adjustable downwards using the rcupdate.rcu_task_stall_timeout kernel boot parameter. Reported-by: Matthew Wilcox Reported-by: Zhouyi Zhou Signed-off-by: Paul E. McKenney Tested-by: Zhouyi Zhou Tested-by: Mark Rutland --- kernel/rcu/tasks.h | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index df6b2cb2f205..fcbd0ec33c86 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -145,6 +145,7 @@ static int rcu_task_ipi_delay __read_mostly = RCU_TASK_IPI_DELAY; module_param(rcu_task_ipi_delay, int, 0644); /* Control stall timeouts. Disable with <= 0, otherwise jiffies till stall. */ +#define RCU_TASK_BOOT_STALL_TIMEOUT (HZ * 30) #define RCU_TASK_STALL_TIMEOUT (HZ * 60 * 10) static int rcu_task_stall_timeout __read_mostly = RCU_TASK_STALL_TIMEOUT; module_param(rcu_task_stall_timeout, int, 0644); @@ -1776,23 +1777,24 @@ struct rcu_tasks_test_desc { struct rcu_head rh; const char *name; bool notrun; + unsigned long runstart; }; static struct rcu_tasks_test_desc tests[] = { { .name = "call_rcu_tasks()", /* If not defined, the test is skipped. */ - .notrun = !IS_ENABLED(CONFIG_TASKS_RCU), + .notrun = IS_ENABLED(CONFIG_TASKS_RCU), }, { .name = "call_rcu_tasks_rude()", /* If not defined, the test is skipped. */ - .notrun = !IS_ENABLED(CONFIG_TASKS_RUDE_RCU), + .notrun = IS_ENABLED(CONFIG_TASKS_RUDE_RCU), }, { .name = "call_rcu_tasks_trace()", /* If not defined, the test is skipped. */ - .notrun = !IS_ENABLED(CONFIG_TASKS_TRACE_RCU) + .notrun = IS_ENABLED(CONFIG_TASKS_TRACE_RCU) } }; @@ -1803,23 +1805,28 @@ static void test_rcu_tasks_callback(struct rcu_head *rhp) pr_info("Callback from %s invoked.\n", rttd->name); - rttd->notrun = true; + rttd->notrun = false; } static void rcu_tasks_initiate_self_tests(void) { + unsigned long j = jiffies; + pr_info("Running RCU-tasks wait API self tests\n"); #ifdef CONFIG_TASKS_RCU + tests[0].runstart = j; synchronize_rcu_tasks(); call_rcu_tasks(&tests[0].rh, test_rcu_tasks_callback); #endif #ifdef CONFIG_TASKS_RUDE_RCU + tests[1].runstart = j; synchronize_rcu_tasks_rude(); call_rcu_tasks_rude(&tests[1].rh, test_rcu_tasks_callback); #endif #ifdef CONFIG_TASKS_TRACE_RCU + tests[2].runstart = j; synchronize_rcu_tasks_trace(); call_rcu_tasks_trace(&tests[2].rh, test_rcu_tasks_callback); #endif @@ -1829,11 +1836,18 @@ static int rcu_tasks_verify_self_tests(void) { int ret = 0; int i; + unsigned long bst = rcu_task_stall_timeout; + if (bst <= 0 || bst > RCU_TASK_BOOT_STALL_TIMEOUT) + bst = RCU_TASK_BOOT_STALL_TIMEOUT; for (i = 0; i < ARRAY_SIZE(tests); i++) { - if (!tests[i].notrun) { // still hanging. - pr_err("%s has been failed.\n", tests[i].name); - ret = -1; + while (tests[i].notrun) { // still hanging. + if (time_after(jiffies, tests[i].runstart + bst)) { + pr_err("%s has failed boot-time tests.\n", tests[i].name); + ret = -1; + break; + } + schedule_timeout_uninterruptible(1); } } From e72ee5e1a866b85cb6c3d4c80a1125976020a7e8 Mon Sep 17 00:00:00 2001 From: Waiman Long Date: Tue, 14 Jun 2022 08:06:20 -0400 Subject: [PATCH 089/318] rcu-tasks: Use delayed_work to delay rcu_tasks_verify_self_tests() Commit 2585014188d5 ("rcu-tasks: Be more patient for RCU Tasks boot-time testing") fixes false positive rcu_tasks verification check failure by repeating the test once every second until timeout using schedule_timeout_uninterruptible(). Since rcu_tasks_verify_selft_tests() is called from do_initcalls() as a late_initcall, this has the undesirable side effect of delaying other late_initcall's queued after it by a second or more. Fix this by instead using delayed_work to repeat the verification check. Fixes: 2585014188d5 ("rcu-tasks: Be more patient for RCU Tasks boot-time testing") Signed-off-by: Waiman Long Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index fcbd0ec33c86..83c7e6620d40 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1832,6 +1832,11 @@ static void rcu_tasks_initiate_self_tests(void) #endif } +/* + * Return: 0 - test passed + * 1 - test failed, but have not timed out yet + * -1 - test failed and timed out + */ static int rcu_tasks_verify_self_tests(void) { int ret = 0; @@ -1847,16 +1852,38 @@ static int rcu_tasks_verify_self_tests(void) ret = -1; break; } - schedule_timeout_uninterruptible(1); + ret = 1; + break; } } - - if (ret) - WARN_ON(1); + WARN_ON(ret < 0); return ret; } -late_initcall(rcu_tasks_verify_self_tests); + +/* + * Repeat the rcu_tasks_verify_self_tests() call once every second until the + * test passes or has timed out. + */ +static struct delayed_work rcu_tasks_verify_work; +static void rcu_tasks_verify_work_fn(struct work_struct *work __maybe_unused) +{ + int ret = rcu_tasks_verify_self_tests(); + + if (ret <= 0) + return; + + /* Test fails but not timed out yet, reschedule another check */ + schedule_delayed_work(&rcu_tasks_verify_work, HZ); +} + +static int rcu_tasks_verify_schedule_work(void) +{ + INIT_DELAYED_WORK(&rcu_tasks_verify_work, rcu_tasks_verify_work_fn); + rcu_tasks_verify_work_fn(NULL); + return 0; +} +late_initcall(rcu_tasks_verify_schedule_work); #else /* #ifdef CONFIG_PROVE_RCU */ static void rcu_tasks_initiate_self_tests(void) { } #endif /* #else #ifdef CONFIG_PROVE_RCU */ From 14c0017c19ead104511dbcf6a59b83b2456d09af Mon Sep 17 00:00:00 2001 From: Anna-Maria Behnsen Date: Mon, 11 Apr 2022 17:19:03 +0200 Subject: [PATCH 090/318] rcu/torture: Change order of warning and trace dump Dumping a big ftrace buffer could lead to a RCU stall. So there is the ftrace buffer and the stall information which needs to be printed. When there is additionally a WARN_ON() which describes the reason for the ftrace buffer dump and the WARN_ON() is executed _after_ ftrace buffer dump, the information get lost in the middle of the RCU stall information. Therefore print WARN_ON() message before dumping the ftrace buffer in rcu_torture_writer(). [ paulmck: Add tracing_off() to avoid cruft from WARN(). ] Signed-off-by: Anna-Maria Behnsen Reviewed-by: Benedikt Spranger Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 7120165a9342..3032dd7c7ad3 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -1321,8 +1321,9 @@ rcu_torture_writer(void *arg) if (list_empty(&rcu_tortures[i].rtort_free) && rcu_access_pointer(rcu_torture_current) != &rcu_tortures[i]) { - rcu_ftrace_dump(DUMP_ALL); + tracing_off(); WARN(1, "%s: rtort_pipe_count: %d\n", __func__, rcu_tortures[i].rtort_pipe_count); + rcu_ftrace_dump(DUMP_ALL); } if (stutter_waited) sched_set_normal(current, oldnice); From d984114ec23818670c8873939eac81ba6e104ff5 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 27 Apr 2022 11:46:02 -0700 Subject: [PATCH 091/318] rcutorture: Simplify rcu_torture_read_exit_child() loop The existing loop has an implicit manual loop that obscures the flow and requires an extra control variable. This commit makes this implicit loop explicit, thus saving several lines of code. Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 47 ++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 3032dd7c7ad3..eed32cea938b 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -2874,7 +2874,6 @@ static int rcu_torture_read_exit_child(void *trsp_in) // Parent kthread which creates and destroys read-exit child kthreads. static int rcu_torture_read_exit(void *unused) { - int count = 0; bool errexit = false; int i; struct task_struct *tsp; @@ -2886,34 +2885,28 @@ static int rcu_torture_read_exit(void *unused) // Each pass through this loop does one read-exit episode. do { - if (++count > read_exit_burst) { - VERBOSE_TOROUT_STRING("rcu_torture_read_exit: End of episode"); - rcu_barrier(); // Wait for task_struct free, avoid OOM. - for (i = 0; i < read_exit_delay; i++) { - schedule_timeout_uninterruptible(HZ); - if (READ_ONCE(read_exit_child_stop)) - break; + VERBOSE_TOROUT_STRING("rcu_torture_read_exit: Start of episode"); + for (i = 0; i < read_exit_burst; i++) { + if (READ_ONCE(read_exit_child_stop)) + break; + stutter_wait("rcu_torture_read_exit"); + // Spawn child. + tsp = kthread_run(rcu_torture_read_exit_child, + &trs, "%s", "rcu_torture_read_exit_child"); + if (IS_ERR(tsp)) { + TOROUT_ERRSTRING("out of memory"); + errexit = true; + break; } - if (!READ_ONCE(read_exit_child_stop)) - VERBOSE_TOROUT_STRING("rcu_torture_read_exit: Start of episode"); - count = 0; + cond_resched(); + kthread_stop(tsp); + n_read_exits++; } - if (READ_ONCE(read_exit_child_stop)) - break; - // Spawn child. - tsp = kthread_run(rcu_torture_read_exit_child, - &trs, "%s", - "rcu_torture_read_exit_child"); - if (IS_ERR(tsp)) { - TOROUT_ERRSTRING("out of memory"); - errexit = true; - tsp = NULL; - break; - } - cond_resched(); - kthread_stop(tsp); - n_read_exits ++; - stutter_wait("rcu_torture_read_exit"); + VERBOSE_TOROUT_STRING("rcu_torture_read_exit: End of episode"); + rcu_barrier(); // Wait for task_struct free, avoid OOM. + i = 0; + for (; !errexit && !READ_ONCE(read_exit_child_stop) && i < read_exit_delay; i++) + schedule_timeout_uninterruptible(HZ); } while (!errexit && !READ_ONCE(read_exit_child_stop)); // Clean up and exit. From 98ea20328786372cbbc90c601be168f5fe1f8845 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Wed, 27 Apr 2022 15:15:20 +0800 Subject: [PATCH 092/318] rcutorture: Fix memory leak in rcu_test_debug_objects() The kernel memory leak detector located the following: unreferenced object 0xffff95d941135b50 (size 16): comm "swapper/0", pid 1, jiffies 4294667610 (age 1367.451s) hex dump (first 16 bytes): f0 c6 c2 bd d9 95 ff ff 00 00 00 00 00 00 00 00 ................ backtrace: [<00000000bc81d9b1>] kmem_cache_alloc_trace+0x2f6/0x500 [<00000000d28be229>] rcu_torture_init+0x1235/0x1354 [<0000000032c3acd9>] do_one_initcall+0x51/0x210 [<000000003c117727>] kernel_init_freeable+0x205/0x259 [<000000003961f965>] kernel_init+0x1a/0x120 [<000000001998f890>] ret_from_fork+0x22/0x30 This is caused by the rcu_test_debug_objects() function allocating an rcu_head structure, then failing to free it. This commit therefore adds the needed kfree() after the last use of this structure. Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index eed32cea938b..62841e9cd268 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -3116,6 +3116,7 @@ static void rcu_test_debug_objects(void) pr_alert("%s: WARN: Duplicate call_rcu() test complete.\n", KBUILD_MODNAME); destroy_rcu_head_on_stack(&rh1); destroy_rcu_head_on_stack(&rh2); + kfree(rhp); #else /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ pr_alert("%s: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_rcu()\n", KBUILD_MODNAME); #endif /* #else #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ From 5c92d7501699a5deb72a579f808500db5bb6f92a Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 17 May 2022 13:22:28 -0700 Subject: [PATCH 093/318] torture: Adjust to again produce debugging information A recent change to the DEBUG_INFO Kconfig option means that simply adding CONFIG_DEBUG_INFO=y to the .config file and running "make oldconfig" no longer works. It is instead necessary to add CONFIG_DEBUG_INFO_NONE=n and (for example) CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y. This combination will then result in CONFIG_DEBUG_INFO being selected. This commit therefore updates the Kconfig options produced in response to the kvm.sh --gdb, --kasan, and --kcsan Kconfig options. Fixes: f9b3cd245784 ("Kconfig.debug: make DEBUG_INFO selectable from a choice") Signed-off-by: Paul E. McKenney --- tools/testing/selftests/rcutorture/bin/kvm.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/rcutorture/bin/kvm.sh b/tools/testing/selftests/rcutorture/bin/kvm.sh index 263e16aeca0e..6c734818a875 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm.sh @@ -164,7 +164,7 @@ do shift ;; --gdb) - TORTURE_KCONFIG_GDB_ARG="CONFIG_DEBUG_INFO=y"; export TORTURE_KCONFIG_GDB_ARG + TORTURE_KCONFIG_GDB_ARG="CONFIG_DEBUG_INFO_NONE=n CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y"; export TORTURE_KCONFIG_GDB_ARG TORTURE_BOOT_GDB_ARG="nokaslr"; export TORTURE_BOOT_GDB_ARG TORTURE_QEMU_GDB_ARG="-s -S"; export TORTURE_QEMU_GDB_ARG ;; @@ -180,7 +180,7 @@ do shift ;; --kasan) - TORTURE_KCONFIG_KASAN_ARG="CONFIG_DEBUG_INFO=y CONFIG_KASAN=y"; export TORTURE_KCONFIG_KASAN_ARG + TORTURE_KCONFIG_KASAN_ARG="CONFIG_DEBUG_INFO_NONE=n CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_KASAN=y"; export TORTURE_KCONFIG_KASAN_ARG if test -n "$torture_qemu_mem_default" then TORTURE_QEMU_MEM=2G @@ -192,7 +192,7 @@ do shift ;; --kcsan) - TORTURE_KCONFIG_KCSAN_ARG="CONFIG_DEBUG_INFO=y CONFIG_KCSAN=y CONFIG_KCSAN_STRICT=y CONFIG_KCSAN_REPORT_ONCE_IN_MS=100000 CONFIG_KCSAN_VERBOSE=y CONFIG_DEBUG_LOCK_ALLOC=y CONFIG_PROVE_LOCKING=y"; export TORTURE_KCONFIG_KCSAN_ARG + TORTURE_KCONFIG_KCSAN_ARG="CONFIG_DEBUG_INFO_NONE=n CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_KCSAN=y CONFIG_KCSAN_STRICT=y CONFIG_KCSAN_REPORT_ONCE_IN_MS=100000 CONFIG_KCSAN_VERBOSE=y CONFIG_DEBUG_LOCK_ALLOC=y CONFIG_PROVE_LOCKING=y"; export TORTURE_KCONFIG_KCSAN_ARG ;; --kmake-arg|--kmake-args) checkarg --kmake-arg "(kernel make arguments)" $# "$2" '.*' '^error$' From 8c0666d320f2fff6bc7cf76422bfbe90c20f53cc Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 20 May 2022 13:18:16 -0700 Subject: [PATCH 094/318] rcutorture: Make failure indication note reader-batch overflow The loop scanning the pipesummary[] array currently skips the last element, which means that the diagnostics ignore those rarest of situations, namely where some readers persist across more than ten grace periods, but all other readers avoid spanning a full grace period. This commit therefore adjusts the scan to include the last element of this array. Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 62841e9cd268..7e7c3518ab06 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -1869,7 +1869,7 @@ rcu_torture_stats_print(void) batchsummary[i] += READ_ONCE(per_cpu(rcu_torture_batch, cpu)[i]); } } - for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) { + for (i = RCU_TORTURE_PIPE_LEN; i >= 0; i--) { if (pipesummary[i] != 0) break; } From 92366810644d5675043c792abb70eaf974a77384 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Sat, 21 May 2022 14:56:26 +0800 Subject: [PATCH 095/318] rcuscale: Fix smp_processor_id()-in-preemptible warnings Systems built with CONFIG_DEBUG_PREEMPT=y can trigger the following BUG while running the rcuscale performance test: BUG: using smp_processor_id() in preemptible [00000000] code: rcu_scale_write/69 CPU: 0 PID: 66 Comm: rcu_scale_write Not tainted 5.18.0-rc7-next-20220517-yoctodev-standard+ caller is debug_smp_processor_id+0x17/0x20 Call Trace: dump_stack_lvl+0x49/0x5e dump_stack+0x10/0x12 check_preemption_disabled+0xdf/0xf0 debug_smp_processor_id+0x17/0x20 rcu_scale_writer+0x2b5/0x580 kthread+0x177/0x1b0 ret_from_fork+0x22/0x30 Reproduction method: runqemu kvm slirp nographic qemuparams="-m 4096 -smp 8" bootparams="isolcpus=2,3 nohz_full=2,3 rcu_nocbs=2,3 rcutree.dump_tree=1 rcuscale.shutdown=false rcuscale.gp_async=true" -d The problem is that the rcu_scale_writer() kthreads fail to set the PF_NO_SETAFFINITY flags, which causes is_percpu_thread() to assume that the kthread's affinity might change at any time, thus the BUG noted above. This commit therefore causes rcu_scale_writer() to set PF_NO_SETAFFINITY in its kthread's ->flags field, thus preventing this BUG. Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/rcuscale.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/rcu/rcuscale.c b/kernel/rcu/rcuscale.c index 277a5bfb37d4..3ef02d4a8108 100644 --- a/kernel/rcu/rcuscale.c +++ b/kernel/rcu/rcuscale.c @@ -419,6 +419,7 @@ rcu_scale_writer(void *arg) VERBOSE_SCALEOUT_STRING("rcu_scale_writer task started"); WARN_ON(!wdpp); set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids)); + current->flags |= PF_NO_SETAFFINITY; sched_set_fifo_low(current); if (holdoff) From 148df92fb14e5aab1c84b8ca4d2181e7b117290d Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 2 Jun 2022 19:53:52 -0700 Subject: [PATCH 096/318] torture: Create kvm-check-branches.sh output in proper location Currently, kvm-check-branches.sh causes each kvm.sh invocation create a separate date-stamped directory, then after that invocation completes, moves it into the *-group/NNNN directory. This works, but makes it more difficult to monitor an ongoing run. This commit therefore uses the kvm.sh --datestamp argument to make kvm.sh put the output in the right place to start with, and also dispenses with the additional level of datestamping. (Those wanting datestamps can find them in the log files.) Signed-off-by: Paul E. McKenney --- .../selftests/rcutorture/bin/kvm-check-branches.sh | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tools/testing/selftests/rcutorture/bin/kvm-check-branches.sh b/tools/testing/selftests/rcutorture/bin/kvm-check-branches.sh index f17000a2ccf1..ed0ec7f0927e 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm-check-branches.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm-check-branches.sh @@ -35,7 +35,7 @@ then exit 1 fi -# Remember where we started so that we can get back and the end. +# Remember where we started so that we can get back at the end. curcommit="`git status | head -1 | awk '{ print $NF }'`" nfail=0 @@ -73,15 +73,10 @@ do # Test the specified commit. git checkout $i > $resdir/$ds/$idir/git-checkout.out 2>&1 echo git checkout return code: $? "(Commit $ntry: $i)" - kvm.sh --allcpus --duration 3 --trust-make > $resdir/$ds/$idir/kvm.sh.out 2>&1 + kvm.sh --allcpus --duration 3 --trust-make --datestamp "$ds/$idir" > $resdir/$ds/$idir/kvm.sh.out 2>&1 ret=$? echo kvm.sh return code $ret for commit $i from branch $gitbr - - # Move the build products to their resting place. - runresdir="`grep -m 1 '^Results directory:' < $resdir/$ds/$idir/kvm.sh.out | sed -e 's/^Results directory://'`" - mv $runresdir $resdir/$ds/$idir - rrd="`echo $runresdir | sed -e 's,^.*/,,'`" - echo Run results: $resdir/$ds/$idir/$rrd + echo Run results: $resdir/$ds/$idir if test "$ret" -ne 0 then # Failure, so leave all evidence intact. From 3002153a91a9732a6d1d0bb95138593c7da15743 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 10 Jun 2022 15:03:57 +0200 Subject: [PATCH 097/318] rcutorture: Fix ksoftirqd boosting timing and iteration The RCU priority boosting can fail in two situations: 1) If (nr_cpus= > maxcpus=), which means if the total number of CPUs is higher than those brought online at boot, then torture_onoff() may later bring up CPUs that weren't online on boot. Now since rcutorture initialization only boosts the ksoftirqds of the CPUs that have been set online on boot, the CPUs later set online by torture_onoff won't benefit from the boost, making RCU priority boosting fail. 2) The ksoftirqd kthreads are boosted after the creation of rcu_torture_boost() kthreads, which opens a window large enough for these rcu_torture_boost() kthreads to wait (despite running at FIFO priority) for ksoftirqds that are still running at SCHED_NORMAL priority. The issues can trigger for example with: ./kvm.sh --configs TREE01 --kconfig "CONFIG_RCU_BOOST=y" [ 34.968561] rcu-torture: !!! [ 34.968627] ------------[ cut here ]------------ [ 35.014054] WARNING: CPU: 4 PID: 114 at kernel/rcu/rcutorture.c:1979 rcu_torture_stats_print+0x5ad/0x610 [ 35.052043] Modules linked in: [ 35.069138] CPU: 4 PID: 114 Comm: rcu_torture_sta Not tainted 5.18.0-rc1 #1 [ 35.096424] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.14.0-0-g155821a-rebuilt.opensuse.org 04/01/2014 [ 35.154570] RIP: 0010:rcu_torture_stats_print+0x5ad/0x610 [ 35.198527] Code: 63 1b 02 00 74 02 0f 0b 48 83 3d 35 63 1b 02 00 74 02 0f 0b 48 83 3d 21 63 1b 02 00 74 02 0f 0b 48 83 3d 0d 63 1b 02 00 74 02 <0f> 0b 83 eb 01 0f 8e ba fc ff ff 0f 0b e9 b3 fc ff f82 [ 37.251049] RSP: 0000:ffffa92a0050bdf8 EFLAGS: 00010202 [ 37.277320] rcu: De-offloading 8 [ 37.290367] RAX: 0000000000000000 RBX: 0000000000000001 RCX: 0000000000000001 [ 37.290387] RDX: 0000000000000000 RSI: 00000000ffffbfff RDI: 00000000ffffffff [ 37.290398] RBP: 000000000000007b R08: 0000000000000000 R09: c0000000ffffbfff [ 37.290407] R10: 000000000000002a R11: ffffa92a0050bc18 R12: ffffa92a0050be20 [ 37.290417] R13: ffffa92a0050be78 R14: 0000000000000000 R15: 000000000001bea0 [ 37.290427] FS: 0000000000000000(0000) GS:ffff96045eb00000(0000) knlGS:0000000000000000 [ 37.290448] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 37.290460] CR2: 0000000000000000 CR3: 000000001dc0c000 CR4: 00000000000006e0 [ 37.290470] Call Trace: [ 37.295049] [ 37.295065] ? preempt_count_add+0x63/0x90 [ 37.295095] ? _raw_spin_lock_irqsave+0x12/0x40 [ 37.295125] ? rcu_torture_stats_print+0x610/0x610 [ 37.295143] rcu_torture_stats+0x29/0x70 [ 37.295160] kthread+0xe3/0x110 [ 37.295176] ? kthread_complete_and_exit+0x20/0x20 [ 37.295193] ret_from_fork+0x22/0x30 [ 37.295218] Fix this with boosting the ksoftirqds kthreads from the boosting hotplug callback itself and before the boosting kthreads are created. Fixes: ea6d962e80b6 ("rcutorture: Judge RCU priority boosting on grace periods, not callbacks") Signed-off-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 7e7c3518ab06..6ba1cf8b3b71 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -2076,6 +2076,19 @@ static int rcutorture_booster_init(unsigned int cpu) if (boost_tasks[cpu] != NULL) return 0; /* Already created, nothing more to do. */ + // Testing RCU priority boosting requires rcutorture do + // some serious abuse. Counter this by running ksoftirqd + // at higher priority. + if (IS_BUILTIN(CONFIG_RCU_TORTURE_TEST)) { + struct sched_param sp; + struct task_struct *t; + + t = per_cpu(ksoftirqd, cpu); + WARN_ON_ONCE(!t); + sp.sched_priority = 2; + sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); + } + /* Don't allow time recalculation while creating a new task. */ mutex_lock(&boost_mutex); rcu_torture_disable_rt_throttle(); @@ -3324,21 +3337,6 @@ rcu_torture_init(void) rcutor_hp = firsterr; if (torture_init_error(firsterr)) goto unwind; - - // Testing RCU priority boosting requires rcutorture do - // some serious abuse. Counter this by running ksoftirqd - // at higher priority. - if (IS_BUILTIN(CONFIG_RCU_TORTURE_TEST)) { - for_each_online_cpu(cpu) { - struct sched_param sp; - struct task_struct *t; - - t = per_cpu(ksoftirqd, cpu); - WARN_ON_ONCE(!t); - sp.sched_priority = 2; - sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); - } - } } shutdown_jiffies = jiffies + shutdown_secs * HZ; firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup); From 1a5ca5e09811dec9472f50b98eeb3e3b2442d050 Mon Sep 17 00:00:00 2001 From: Li Qiong Date: Sun, 12 Jun 2022 14:48:25 +0800 Subject: [PATCH 098/318] rcutorture: Handle failure of memory allocation functions This commit adds warnings for allocation failure during the mem_dump_obj() tests. It also terminates these tests upon such failure. Signed-off-by: Li Qiong Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 6ba1cf8b3b71..2ba74498d36d 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -1991,7 +1991,13 @@ static void rcu_torture_mem_dump_obj(void) static int z; kcp = kmem_cache_create("rcuscale", 136, 8, SLAB_STORE_USER, NULL); + if (WARN_ON_ONCE(!kcp)) + return; rhp = kmem_cache_alloc(kcp, GFP_KERNEL); + if (WARN_ON_ONCE(!rhp)) { + kmem_cache_destroy(kcp); + return; + } pr_alert("mem_dump_obj() slab test: rcu_torture_stats = %px, &rhp = %px, rhp = %px, &z = %px\n", stats_task, &rhp, rhp, &z); pr_alert("mem_dump_obj(ZERO_SIZE_PTR):"); mem_dump_obj(ZERO_SIZE_PTR); @@ -2008,6 +2014,8 @@ static void rcu_torture_mem_dump_obj(void) kmem_cache_free(kcp, rhp); kmem_cache_destroy(kcp); rhp = kmalloc(sizeof(*rhp), GFP_KERNEL); + if (WARN_ON_ONCE(!rhp)) + return; pr_alert("mem_dump_obj() kmalloc test: rcu_torture_stats = %px, &rhp = %px, rhp = %px\n", stats_task, &rhp, rhp); pr_alert("mem_dump_obj(kmalloc %px):", rhp); mem_dump_obj(rhp); @@ -2015,6 +2023,8 @@ static void rcu_torture_mem_dump_obj(void) mem_dump_obj(&rhp->func); kfree(rhp); rhp = vmalloc(4096); + if (WARN_ON_ONCE(!rhp)) + return; pr_alert("mem_dump_obj() vmalloc test: rcu_torture_stats = %px, &rhp = %px, rhp = %px\n", stats_task, &rhp, rhp); pr_alert("mem_dump_obj(vmalloc %px):", rhp); mem_dump_obj(rhp); From 7bf336fb8dac718febb7bf4fe79e1be0c5e4a631 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Sun, 12 Jun 2022 10:02:25 +0800 Subject: [PATCH 099/318] refscale: Convert test_lock spinlock to raw_spinlock In kernels built with CONFIG_PREEMPT_RT=y, spinlocks are replaced by rt_mutex, which can sleep. This means that acquiring a non-raw spinlock in a critical section where preemption is disabled can trigger the following BUG: BUG: scheduling while atomic: ref_scale_reade/76/0x00000002 Preemption disabled at: ref_lock_section+0x16/0x80 Call Trace: dump_stack_lvl+0x5b/0x82 dump_stack+0x10/0x12 __schedule_bug.cold+0x9c/0xad __schedule+0x839/0xc00 schedule_rtlock+0x22/0x40 rtlock_slowlock_locked+0x460/0x1350 rt_spin_lock+0x61/0xe0 ref_lock_section+0x29/0x80 rcu_scale_one_reader+0x52/0x60 ref_scale_reader+0x28d/0x490 kthread+0x128/0x150 ret_from_fork+0x22/0x30 This commit therefore converts spinlock to raw_spinlock. Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/refscale.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c index 909644abee67..435c884c02b5 100644 --- a/kernel/rcu/refscale.c +++ b/kernel/rcu/refscale.c @@ -385,7 +385,7 @@ static struct ref_scale_ops rwsem_ops = { }; // Definitions for global spinlock -static DEFINE_SPINLOCK(test_lock); +static DEFINE_RAW_SPINLOCK(test_lock); static void ref_lock_section(const int nloops) { @@ -393,8 +393,8 @@ static void ref_lock_section(const int nloops) preempt_disable(); for (i = nloops; i >= 0; i--) { - spin_lock(&test_lock); - spin_unlock(&test_lock); + raw_spin_lock(&test_lock); + raw_spin_unlock(&test_lock); } preempt_enable(); } @@ -405,9 +405,9 @@ static void ref_lock_delay_section(const int nloops, const int udl, const int nd preempt_disable(); for (i = nloops; i >= 0; i--) { - spin_lock(&test_lock); + raw_spin_lock(&test_lock); un_delay(udl, ndl); - spin_unlock(&test_lock); + raw_spin_unlock(&test_lock); } preempt_enable(); } @@ -427,8 +427,8 @@ static void ref_lock_irq_section(const int nloops) preempt_disable(); for (i = nloops; i >= 0; i--) { - spin_lock_irqsave(&test_lock, flags); - spin_unlock_irqrestore(&test_lock, flags); + raw_spin_lock_irqsave(&test_lock, flags); + raw_spin_unlock_irqrestore(&test_lock, flags); } preempt_enable(); } @@ -440,9 +440,9 @@ static void ref_lock_irq_delay_section(const int nloops, const int udl, const in preempt_disable(); for (i = nloops; i >= 0; i--) { - spin_lock_irqsave(&test_lock, flags); + raw_spin_lock_irqsave(&test_lock, flags); un_delay(udl, ndl); - spin_unlock_irqrestore(&test_lock, flags); + raw_spin_unlock_irqrestore(&test_lock, flags); } preempt_enable(); } From 0ffc781a19ed3030c792ad1a0e44e6e047bb9adc Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:20 +0200 Subject: [PATCH 100/318] context_tracking: Rename __context_tracking_enter/exit() to __ct_user_enter/exit() The context tracking namespace is going to expand and some new functions will require even longer names. Start shrinking the context_tracking prefix to "ct" as is already the case for some existing macros, this will make the introduction of new functions easier. Acked-by: Paul E. McKenney Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- include/linux/context_tracking.h | 12 ++++++------ kernel/context_tracking.c | 20 ++++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h index 7a14807c9d1a..773035124bad 100644 --- a/include/linux/context_tracking.h +++ b/include/linux/context_tracking.h @@ -14,8 +14,8 @@ extern void context_tracking_cpu_set(int cpu); /* Called with interrupts disabled. */ -extern void __context_tracking_enter(enum ctx_state state); -extern void __context_tracking_exit(enum ctx_state state); +extern void __ct_user_enter(enum ctx_state state); +extern void __ct_user_exit(enum ctx_state state); extern void context_tracking_enter(enum ctx_state state); extern void context_tracking_exit(enum ctx_state state); @@ -38,13 +38,13 @@ static inline void user_exit(void) static __always_inline void user_enter_irqoff(void) { if (context_tracking_enabled()) - __context_tracking_enter(CONTEXT_USER); + __ct_user_enter(CONTEXT_USER); } static __always_inline void user_exit_irqoff(void) { if (context_tracking_enabled()) - __context_tracking_exit(CONTEXT_USER); + __ct_user_exit(CONTEXT_USER); } static inline enum ctx_state exception_enter(void) @@ -74,7 +74,7 @@ static inline void exception_exit(enum ctx_state prev_ctx) static __always_inline bool context_tracking_guest_enter(void) { if (context_tracking_enabled()) - __context_tracking_enter(CONTEXT_GUEST); + __ct_user_enter(CONTEXT_GUEST); return context_tracking_enabled_this_cpu(); } @@ -82,7 +82,7 @@ static __always_inline bool context_tracking_guest_enter(void) static __always_inline void context_tracking_guest_exit(void) { if (context_tracking_enabled()) - __context_tracking_exit(CONTEXT_GUEST); + __ct_user_exit(CONTEXT_GUEST); } /** diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index 3082332f6476..88c60ab39fbb 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -51,15 +51,15 @@ static __always_inline void context_tracking_recursion_exit(void) } /** - * context_tracking_enter - Inform the context tracking that the CPU is going - * enter user or guest space mode. + * __ct_user_enter - Inform the context tracking that the CPU is going + * to enter user or guest space mode. * * This function must be called right before we switch from the kernel * to user or guest space, when it's guaranteed the remaining kernel * instructions to execute won't use any RCU read side critical section * because this function sets RCU in extended quiescent state. */ -void noinstr __context_tracking_enter(enum ctx_state state) +void noinstr __ct_user_enter(enum ctx_state state) { /* Kernel threads aren't supposed to go to userspace */ WARN_ON_ONCE(!current->mm); @@ -101,7 +101,7 @@ void noinstr __context_tracking_enter(enum ctx_state state) } context_tracking_recursion_exit(); } -EXPORT_SYMBOL_GPL(__context_tracking_enter); +EXPORT_SYMBOL_GPL(__ct_user_enter); /* * OBSOLETE: @@ -128,7 +128,7 @@ void context_tracking_enter(enum ctx_state state) return; local_irq_save(flags); - __context_tracking_enter(state); + __ct_user_enter(state); local_irq_restore(flags); } NOKPROBE_SYMBOL(context_tracking_enter); @@ -149,8 +149,8 @@ void context_tracking_user_enter(void) NOKPROBE_SYMBOL(context_tracking_user_enter); /** - * context_tracking_exit - Inform the context tracking that the CPU is - * exiting user or guest mode and entering the kernel. + * __ct_user_exit - Inform the context tracking that the CPU is + * exiting user or guest mode and entering the kernel. * * This function must be called after we entered the kernel from user or * guest space before any use of RCU read side critical section. This @@ -160,7 +160,7 @@ NOKPROBE_SYMBOL(context_tracking_user_enter); * This call supports re-entrancy. This way it can be called from any exception * handler without needing to know if we came from userspace or not. */ -void noinstr __context_tracking_exit(enum ctx_state state) +void noinstr __ct_user_exit(enum ctx_state state) { if (!context_tracking_recursion_enter()) return; @@ -183,7 +183,7 @@ void noinstr __context_tracking_exit(enum ctx_state state) } context_tracking_recursion_exit(); } -EXPORT_SYMBOL_GPL(__context_tracking_exit); +EXPORT_SYMBOL_GPL(__ct_user_exit); /* * OBSOLETE: @@ -202,7 +202,7 @@ void context_tracking_exit(enum ctx_state state) return; local_irq_save(flags); - __context_tracking_exit(state); + __ct_user_exit(state); local_irq_restore(flags); } NOKPROBE_SYMBOL(context_tracking_exit); From 1b05ece0c931536c0a38a9385e243a7962e933f6 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Thu, 16 Jun 2022 10:26:18 -0500 Subject: [PATCH 101/318] crypto: ccp - During shutdown, check SEV data pointer before using On shutdown, each CCP device instance performs shutdown processing. However, __sev_platform_shutdown_locked() uses the controlling psp structure to obtain the pointer to the sev_device structure. However, during driver initialization, it is possible that an error can be received from the firmware that results in the sev_data pointer being cleared from the controlling psp structure. The __sev_platform_shutdown_locked() function does not check for this situation and will segfault. While not common, this scenario should be accounted for. Add a check for a NULL sev_device structure before attempting to use it. Fixes: 5441a07a127f ("crypto: ccp - shutdown SEV firmware on kexec") Signed-off-by: Tom Lendacky Signed-off-by: Herbert Xu --- drivers/crypto/ccp/sev-dev.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 0c92d940ac4e..9f588c9728f8 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -503,7 +503,7 @@ static int __sev_platform_shutdown_locked(int *error) struct sev_device *sev = psp_master->sev_data; int ret; - if (sev->state == SEV_STATE_UNINIT) + if (!sev || sev->state == SEV_STATE_UNINIT) return 0; ret = __sev_do_cmd_locked(SEV_CMD_SHUTDOWN, NULL, error); From f145d411a67efacc0731fc3f9c7b2d89fb62523a Mon Sep 17 00:00:00 2001 From: Ignat Korchagin Date: Fri, 17 Jun 2022 09:42:10 +0100 Subject: [PATCH 102/318] crypto: rsa - implement Chinese Remainder Theorem for faster private key operations Changes from v1: * exported mpi_sub and mpi_mul, otherwise the build fails when RSA is a module The kernel RSA ASN.1 private key parser already supports only private keys with additional values to be used with the Chinese Remainder Theorem [1], but these values are currently not used. This rudimentary CRT implementation speeds up RSA private key operations for the following Go benchmark up to ~3x. This implementation also tries to minimise the allocation of additional MPIs, so existing MPIs are reused as much as possible (hence the variable names are a bit weird). The benchmark used: ``` package keyring_test import ( "crypto" "crypto/rand" "crypto/rsa" "crypto/x509" "io" "syscall" "testing" "unsafe" ) type KeySerial int32 type Keyring int32 const ( KEY_SPEC_PROCESS_KEYRING Keyring = -2 KEYCTL_PKEY_SIGN = 27 ) var ( keyTypeAsym = []byte("asymmetric\x00") sha256pkcs1 = []byte("enc=pkcs1 hash=sha256\x00") ) func (keyring Keyring) LoadAsym(desc string, payload []byte) (KeySerial, error) { cdesc := []byte(desc + "\x00") serial, _, errno := syscall.Syscall6(syscall.SYS_ADD_KEY, uintptr(unsafe.Pointer(&keyTypeAsym[0])), uintptr(unsafe.Pointer(&cdesc[0])), uintptr(unsafe.Pointer(&payload[0])), uintptr(len(payload)), uintptr(keyring), uintptr(0)) if errno == 0 { return KeySerial(serial), nil } return KeySerial(serial), errno } type pkeyParams struct { key_id KeySerial in_len uint32 out_or_in2_len uint32 __spare [7]uint32 } // the output signature buffer is an input parameter here, because we want to // avoid Go buffer allocation leaking into our benchmarks func (key KeySerial) Sign(info, digest, out []byte) error { var params pkeyParams params.key_id = key params.in_len = uint32(len(digest)) params.out_or_in2_len = uint32(len(out)) _, _, errno := syscall.Syscall6(syscall.SYS_KEYCTL, KEYCTL_PKEY_SIGN, uintptr(unsafe.Pointer(¶ms)), uintptr(unsafe.Pointer(&info[0])), uintptr(unsafe.Pointer(&digest[0])), uintptr(unsafe.Pointer(&out[0])), uintptr(0)) if errno == 0 { return nil } return errno } func BenchmarkSign(b *testing.B) { priv, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { b.Fatalf("failed to generate private key: %v", err) } pkcs8, err := x509.MarshalPKCS8PrivateKey(priv) if err != nil { b.Fatalf("failed to serialize the private key to PKCS8 blob: %v", err) } serial, err := KEY_SPEC_PROCESS_KEYRING.LoadAsym("test rsa key", pkcs8) if err != nil { b.Fatalf("failed to load the private key into the keyring: %v", err) } b.Logf("loaded test rsa key: %v", serial) digest := make([]byte, 32) _, err = io.ReadFull(rand.Reader, digest) if err != nil { b.Fatalf("failed to generate a random digest: %v", err) } sig := make([]byte, 256) for n := 0; n < b.N; n++ { err = serial.Sign(sha256pkcs1, digest, sig) if err != nil { b.Fatalf("failed to sign the digest: %v", err) } } err = rsa.VerifyPKCS1v15(&priv.PublicKey, crypto.SHA256, digest, sig) if err != nil { b.Fatalf("failed to verify the signature: %v", err) } } ``` [1]: https://en.wikipedia.org/wiki/RSA_(cryptosystem)#Using_the_Chinese_remainder_algorithm Signed-off-by: Ignat Korchagin Reported-by: kernel test robot Signed-off-by: Herbert Xu --- crypto/rsa.c | 78 ++++++++++++++++++++++++++++++++++++++++++++--- lib/mpi/mpi-add.c | 2 +- lib/mpi/mpi-mul.c | 1 + 3 files changed, 75 insertions(+), 6 deletions(-) diff --git a/crypto/rsa.c b/crypto/rsa.c index 39e04176b04b..0e555ee4addb 100644 --- a/crypto/rsa.c +++ b/crypto/rsa.c @@ -17,6 +17,11 @@ struct rsa_mpi_key { MPI n; MPI e; MPI d; + MPI p; + MPI q; + MPI dp; + MPI dq; + MPI qinv; }; /* @@ -35,16 +40,49 @@ static int _rsa_enc(const struct rsa_mpi_key *key, MPI c, MPI m) /* * RSADP function [RFC3447 sec 5.1.2] - * m = c^d mod n; + * m_1 = c^dP mod p; + * m_2 = c^dQ mod q; + * h = (m_1 - m_2) * qInv mod p; + * m = m_2 + q * h; */ -static int _rsa_dec(const struct rsa_mpi_key *key, MPI m, MPI c) +static int _rsa_dec_crt(const struct rsa_mpi_key *key, MPI m_or_m1_or_h, MPI c) { + MPI m2, m12_or_qh; + int ret = -ENOMEM; + /* (1) Validate 0 <= c < n */ if (mpi_cmp_ui(c, 0) < 0 || mpi_cmp(c, key->n) >= 0) return -EINVAL; - /* (2) m = c^d mod n */ - return mpi_powm(m, c, key->d, key->n); + m2 = mpi_alloc(0); + m12_or_qh = mpi_alloc(0); + if (!m2 || !m12_or_qh) + goto err_free_mpi; + + /* (2i) m_1 = c^dP mod p */ + ret = mpi_powm(m_or_m1_or_h, c, key->dp, key->p); + if (ret) + goto err_free_mpi; + + /* (2i) m_2 = c^dQ mod q */ + ret = mpi_powm(m2, c, key->dq, key->q); + if (ret) + goto err_free_mpi; + + /* (2iii) h = (m_1 - m_2) * qInv mod p */ + mpi_sub(m12_or_qh, m_or_m1_or_h, m2); + mpi_mulm(m_or_m1_or_h, m12_or_qh, key->qinv, key->p); + + /* (2iv) m = m_2 + q * h */ + mpi_mul(m12_or_qh, key->q, m_or_m1_or_h); + mpi_addm(m_or_m1_or_h, m2, m12_or_qh, key->n); + + ret = 0; + +err_free_mpi: + mpi_free(m12_or_qh); + mpi_free(m2); + return ret; } static inline struct rsa_mpi_key *rsa_get_key(struct crypto_akcipher *tfm) @@ -112,7 +150,7 @@ static int rsa_dec(struct akcipher_request *req) if (!c) goto err_free_m; - ret = _rsa_dec(pkey, m, c); + ret = _rsa_dec_crt(pkey, m, c); if (ret) goto err_free_c; @@ -134,9 +172,19 @@ static void rsa_free_mpi_key(struct rsa_mpi_key *key) mpi_free(key->d); mpi_free(key->e); mpi_free(key->n); + mpi_free(key->p); + mpi_free(key->q); + mpi_free(key->dp); + mpi_free(key->dq); + mpi_free(key->qinv); key->d = NULL; key->e = NULL; key->n = NULL; + key->p = NULL; + key->q = NULL; + key->dp = NULL; + key->dq = NULL; + key->qinv = NULL; } static int rsa_check_key_length(unsigned int len) @@ -217,6 +265,26 @@ static int rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key, if (!mpi_key->n) goto err; + mpi_key->p = mpi_read_raw_data(raw_key.p, raw_key.p_sz); + if (!mpi_key->p) + goto err; + + mpi_key->q = mpi_read_raw_data(raw_key.q, raw_key.q_sz); + if (!mpi_key->q) + goto err; + + mpi_key->dp = mpi_read_raw_data(raw_key.dp, raw_key.dp_sz); + if (!mpi_key->dp) + goto err; + + mpi_key->dq = mpi_read_raw_data(raw_key.dq, raw_key.dq_sz); + if (!mpi_key->dq) + goto err; + + mpi_key->qinv = mpi_read_raw_data(raw_key.qinv, raw_key.qinv_sz); + if (!mpi_key->qinv) + goto err; + if (rsa_check_key_length(mpi_get_size(mpi_key->n) << 3)) { rsa_free_mpi_key(mpi_key); return -EINVAL; diff --git a/lib/mpi/mpi-add.c b/lib/mpi/mpi-add.c index 2cdae54c1bd0..9056fc5167fc 100644 --- a/lib/mpi/mpi-add.c +++ b/lib/mpi/mpi-add.c @@ -138,7 +138,7 @@ void mpi_sub(MPI w, MPI u, MPI v) mpi_add(w, u, vv); mpi_free(vv); } - +EXPORT_SYMBOL_GPL(mpi_sub); void mpi_addm(MPI w, MPI u, MPI v, MPI m) { diff --git a/lib/mpi/mpi-mul.c b/lib/mpi/mpi-mul.c index 8f5fa200f297..7f4eda8560dc 100644 --- a/lib/mpi/mpi-mul.c +++ b/lib/mpi/mpi-mul.c @@ -82,6 +82,7 @@ void mpi_mul(MPI w, MPI u, MPI v) if (tmp_limb) mpi_free_limb_space(tmp_limb); } +EXPORT_SYMBOL_GPL(mpi_mul); void mpi_mulm(MPI w, MPI u, MPI v, MPI m) { From c2a1b91e47984e477298912ffd55570095d67318 Mon Sep 17 00:00:00 2001 From: Andre Przywara Date: Fri, 17 Jun 2022 09:59:44 +0000 Subject: [PATCH 103/318] crypto: qat - replace get_current_node() with numa_node_id() Currently the QAT driver code uses a self-defined wrapper function called get_current_node() when it wants to learn the current NUMA node. This implementation references the topology_physical_package_id[] array, which more or less coincidentally contains the NUMA node id, at least on x86. Because this is not universal, and Linux offers a direct function to learn the NUMA node ID, replace that function with a call to numa_node_id(), which would work everywhere. This fixes the QAT driver operation on arm64 machines. Reported-by: Yoan Picchi Signed-off-by: Andre Przywara Signed-off-by: Yoan Picchi Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_common_drv.h | 5 ----- drivers/crypto/qat/qat_common/qat_algs.c | 4 ++-- drivers/crypto/qat/qat_common/qat_asym_algs.c | 4 ++-- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index 0464fa257929..b364bc06c732 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -49,11 +49,6 @@ struct service_hndl { struct list_head list; }; -static inline int get_current_node(void) -{ - return topology_physical_package_id(raw_smp_processor_id()); -} - int adf_service_register(struct service_hndl *service); int adf_service_unregister(struct service_hndl *service); diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index 148edbe379e3..fb45fa83841c 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -605,7 +605,7 @@ static int qat_alg_aead_newkey(struct crypto_aead *tfm, const u8 *key, { struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); struct qat_crypto_instance *inst = NULL; - int node = get_current_node(); + int node = numa_node_id(); struct device *dev; int ret; @@ -1065,7 +1065,7 @@ static int qat_alg_skcipher_newkey(struct qat_alg_skcipher_ctx *ctx, { struct qat_crypto_instance *inst = NULL; struct device *dev; - int node = get_current_node(); + int node = numa_node_id(); int ret; inst = qat_crypto_get_instance_node(node); diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c index 16d97db9ea15..095ed2a404d2 100644 --- a/drivers/crypto/qat/qat_common/qat_asym_algs.c +++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c @@ -489,7 +489,7 @@ static int qat_dh_init_tfm(struct crypto_kpp *tfm) { struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); struct qat_crypto_instance *inst = - qat_crypto_get_instance_node(get_current_node()); + qat_crypto_get_instance_node(numa_node_id()); if (!inst) return -EINVAL; @@ -1225,7 +1225,7 @@ static int qat_rsa_init_tfm(struct crypto_akcipher *tfm) { struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); struct qat_crypto_instance *inst = - qat_crypto_get_instance_node(get_current_node()); + qat_crypto_get_instance_node(numa_node_id()); if (!inst) return -EINVAL; From 9c846c5d2d4e63d75b2cb172625087cadadbe065 Mon Sep 17 00:00:00 2001 From: Yoan Picchi Date: Fri, 17 Jun 2022 09:59:45 +0000 Subject: [PATCH 104/318] crypto: qat - Removes the x86 dependency on the QAT drivers This dependency looks outdated. After the previous patch, we have been able to use this driver to encrypt some data and to create working VF on arm64. We have not tested it yet on any big endian machine, hence the new dependency Signed-off-by: Yoan Picchi Signed-off-by: Herbert Xu --- drivers/crypto/qat/Kconfig | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/crypto/qat/Kconfig b/drivers/crypto/qat/Kconfig index 4b90c0f22b03..1220cc86f910 100644 --- a/drivers/crypto/qat/Kconfig +++ b/drivers/crypto/qat/Kconfig @@ -17,7 +17,7 @@ config CRYPTO_DEV_QAT config CRYPTO_DEV_QAT_DH895xCC tristate "Support for Intel(R) DH895xCC" - depends on X86 && PCI + depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST) select CRYPTO_DEV_QAT help Support for Intel(R) DH895xcc with Intel(R) QuickAssist Technology @@ -28,7 +28,7 @@ config CRYPTO_DEV_QAT_DH895xCC config CRYPTO_DEV_QAT_C3XXX tristate "Support for Intel(R) C3XXX" - depends on X86 && PCI + depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST) select CRYPTO_DEV_QAT help Support for Intel(R) C3xxx with Intel(R) QuickAssist Technology @@ -39,7 +39,7 @@ config CRYPTO_DEV_QAT_C3XXX config CRYPTO_DEV_QAT_C62X tristate "Support for Intel(R) C62X" - depends on X86 && PCI + depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST) select CRYPTO_DEV_QAT help Support for Intel(R) C62x with Intel(R) QuickAssist Technology @@ -50,7 +50,7 @@ config CRYPTO_DEV_QAT_C62X config CRYPTO_DEV_QAT_4XXX tristate "Support for Intel(R) QAT_4XXX" - depends on X86 && PCI + depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST) select CRYPTO_DEV_QAT help Support for Intel(R) QuickAssist Technology QAT_4xxx @@ -61,7 +61,7 @@ config CRYPTO_DEV_QAT_4XXX config CRYPTO_DEV_QAT_DH895xCCVF tristate "Support for Intel(R) DH895xCC Virtual Function" - depends on X86 && PCI + depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST) select PCI_IOV select CRYPTO_DEV_QAT @@ -74,7 +74,7 @@ config CRYPTO_DEV_QAT_DH895xCCVF config CRYPTO_DEV_QAT_C3XXXVF tristate "Support for Intel(R) C3XXX Virtual Function" - depends on X86 && PCI + depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST) select PCI_IOV select CRYPTO_DEV_QAT help @@ -86,7 +86,7 @@ config CRYPTO_DEV_QAT_C3XXXVF config CRYPTO_DEV_QAT_C62XVF tristate "Support for Intel(R) C62X Virtual Function" - depends on X86 && PCI + depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST) select PCI_IOV select CRYPTO_DEV_QAT help From 30fb8761419ebf1f26492a0bfaa5d795aa819179 Mon Sep 17 00:00:00 2001 From: Stephen Kitt Date: Fri, 24 Jun 2022 13:02:30 +0200 Subject: [PATCH 105/318] docs: admin-guide/sysctl: Fix rendering error Text in ``literal`` markup must be separated by word separators, so text like ``lowwater``% renders incorrectly. Add the suggested "\ " after two problematic occurrences. Signed-off-by: Stephen Kitt Link: https://lore.kernel.org/r/20220624110230.595740-1-steve@sk2.org [jc: tweaked to use "\ "] Signed-off-by: Jonathan Corbet --- Documentation/admin-guide/sysctl/kernel.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/sysctl/kernel.rst b/Documentation/admin-guide/sysctl/kernel.rst index ddccd1077462..8ab042beeb76 100644 --- a/Documentation/admin-guide/sysctl/kernel.rst +++ b/Documentation/admin-guide/sysctl/kernel.rst @@ -38,8 +38,8 @@ acct If BSD-style process accounting is enabled these values control its behaviour. If free space on filesystem where the log lives -goes below ``lowwater``% accounting suspends. If free space gets -above ``highwater``% accounting resumes. ``frequency`` determines +goes below ``lowwater``\ % accounting suspends. If free space gets +above ``highwater``\ % accounting resumes. ``frequency`` determines how often do we check the amount of free space (value is in seconds). Default: From df6725651f6f4b0f4f475cf7f539dfb8c9535da5 Mon Sep 17 00:00:00 2001 From: Deming Wang Date: Thu, 23 Jun 2022 21:46:05 -0400 Subject: [PATCH 106/318] docs: Remove duplicate word Delete duplicate words of "the". Signed-off-by: Deming Wang Link: https://lore.kernel.org/r/20220624014605.2007-1-wangdeming@inspur.com Signed-off-by: Jonathan Corbet --- Documentation/filesystems/overlayfs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/filesystems/overlayfs.rst b/Documentation/filesystems/overlayfs.rst index 7da6c30ed596..4c76fda07645 100644 --- a/Documentation/filesystems/overlayfs.rst +++ b/Documentation/filesystems/overlayfs.rst @@ -607,7 +607,7 @@ can be removed. User xattr ---------- -The the "-o userxattr" mount option forces overlayfs to use the +The "-o userxattr" mount option forces overlayfs to use the "user.overlay." xattr namespace instead of "trusted.overlay.". This is useful for unprivileged mounting of overlayfs. From f9b21cd086070a111eebc107341d822a37796c89 Mon Sep 17 00:00:00 2001 From: Tom Schwindl Date: Thu, 23 Jun 2022 21:37:06 +0000 Subject: [PATCH 107/318] docs: driver-api: gpio: Fix some typos Correct some simple spelling mistakes in consumer.rst, driver.rst and using-gpio.rst. Signed-off-by: Tom Schwindl Reviewed-by: Randy Dunlap Link: https://lore.kernel.org/r/YrTdAv3YPlCiDr2u@posteo.de Signed-off-by: Jonathan Corbet --- Documentation/driver-api/gpio/consumer.rst | 2 +- Documentation/driver-api/gpio/driver.rst | 6 +++--- Documentation/driver-api/gpio/using-gpio.rst | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Documentation/driver-api/gpio/consumer.rst b/Documentation/driver-api/gpio/consumer.rst index 47869ca8ccf0..083afa1a4189 100644 --- a/Documentation/driver-api/gpio/consumer.rst +++ b/Documentation/driver-api/gpio/consumer.rst @@ -114,7 +114,7 @@ For a function using multiple GPIOs all of those can be obtained with one call:: This function returns a struct gpio_descs which contains an array of descriptors. It also contains a pointer to a gpiolib private structure which, -if passed back to get/set array functions, may speed up I/O proocessing:: +if passed back to get/set array functions, may speed up I/O processing:: struct gpio_descs { struct gpio_array *info; diff --git a/Documentation/driver-api/gpio/driver.rst b/Documentation/driver-api/gpio/driver.rst index 70ff43ac4fcc..6baaeab79534 100644 --- a/Documentation/driver-api/gpio/driver.rst +++ b/Documentation/driver-api/gpio/driver.rst @@ -119,7 +119,7 @@ GPIO lines with debounce support Debouncing is a configuration set to a pin indicating that it is connected to a mechanical switch or button, or similar that may bounce. Bouncing means the line is pulled high/low quickly at very short intervals for mechanical -reasons. This can result in the value being unstable or irqs fireing repeatedly +reasons. This can result in the value being unstable or irqs firing repeatedly unless the line is debounced. Debouncing in practice involves setting up a timer when something happens on @@ -219,7 +219,7 @@ use a trick: when a line is set as output, if the line is flagged as open drain, and the IN output value is low, it will be driven low as usual. But if the IN output value is set to high, it will instead *NOT* be driven high, instead it will be switched to input, as input mode is high impedance, thus -achieveing an "open drain emulation" of sorts: electrically the behaviour will +achieving an "open drain emulation" of sorts: electrically the behaviour will be identical, with the exception of possible hardware glitches when switching the mode of the line. @@ -642,7 +642,7 @@ In this case the typical set-up will look like this: As you can see pretty similar, but you do not supply a parent handler for the IRQ, instead a parent irqdomain, an fwnode for the hardware and -a funcion .child_to_parent_hwirq() that has the purpose of looking up +a function .child_to_parent_hwirq() that has the purpose of looking up the parent hardware irq from a child (i.e. this gpio chip) hardware irq. As always it is good to look at examples in the kernel tree for advice on how to find the required pieces. diff --git a/Documentation/driver-api/gpio/using-gpio.rst b/Documentation/driver-api/gpio/using-gpio.rst index 64c8d3f76c3a..894d88855d73 100644 --- a/Documentation/driver-api/gpio/using-gpio.rst +++ b/Documentation/driver-api/gpio/using-gpio.rst @@ -44,7 +44,7 @@ These devices will appear on the system as ``/dev/gpiochip0`` thru found in the kernel tree ``tools/gpio`` subdirectory. For structured and managed applications, we recommend that you make use of the -libgpiod_ library. This provides helper abstractions, command line utlities +libgpiod_ library. This provides helper abstractions, command line utilities and arbitration for multiple simultaneous consumers on the same GPIO chip. .. _libgpiod: https://git.kernel.org/pub/scm/libs/libgpiod/libgpiod.git/ From 417c434aa1b4426aff242c46a562e9e71ab14e25 Mon Sep 17 00:00:00 2001 From: Binbin Zhou Date: Wed, 22 Jun 2022 18:18:47 +0800 Subject: [PATCH 108/318] docs/zh_CN: core-api: Update the translation of cachetlb.rst to 5.19-rc3 Synchronous translation from upstream commit 08b0b0059bf1 ("mm: Add flush_dcache_folio()") Signed-off-by: Binbin Zhou Reviewed-by: Yanteng Si Reviewed-by: Wu XiangCheng Link: https://lore.kernel.org/r/d1fe8dfa3cfc3fe38c7caf8b5cf3b3482b5bddf8.1655863769.git.zhoubinbin@loongson.cn Signed-off-by: Jonathan Corbet --- Documentation/translations/zh_CN/core-api/cachetlb.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Documentation/translations/zh_CN/core-api/cachetlb.rst b/Documentation/translations/zh_CN/core-api/cachetlb.rst index 6fee45fe5e80..b4a76ec75daa 100644 --- a/Documentation/translations/zh_CN/core-api/cachetlb.rst +++ b/Documentation/translations/zh_CN/core-api/cachetlb.rst @@ -5,6 +5,7 @@ :翻译: 司延腾 Yanteng Si + 周彬彬 Binbin Zhou :校译: @@ -278,6 +279,11 @@ HyperSparc cpu就是这样一个具有这种属性的cpu。 CPU上，因为它将cpu存储到页面上，使其变脏。同样，请看 sparc64关于如何处理这个问题的例子。 + ``void flush_dcache_folio(struct folio *folio)`` + + 该函数的调用情形与flush_dcache_page()相同。它允许架构针对刷新整个 + folio页面进行优化，而不是一次刷新一页。 + ``void copy_to_user_page(struct vm_area_struct *vma, struct page *page, unsigned long user_vaddr, void *dst, void *src, int len)`` ``void copy_from_user_page(struct vm_area_struct *vma, struct page *page, From 1ebfae49fd441686b805cb340808697b45159bf9 Mon Sep 17 00:00:00 2001 From: Binbin Zhou Date: Wed, 22 Jun 2022 18:18:48 +0800 Subject: [PATCH 109/318] docs/zh_CN: core-api: Update the translation of cpu_hotplug.rst to 5.19-rc3 Synchronous translation from upstream commit c9871c800f65 ("Documentation: core-api/cpuhotplug: Rewrite the API section") Signed-off-by: Binbin Zhou Reviewed-by: Yanteng Si Reviewed-by: Wu XiangCheng Link: https://lore.kernel.org/r/9a123a1938a09f883db0b429396954e0792c84c1.1655863769.git.zhoubinbin@loongson.cn Signed-off-by: Jonathan Corbet --- .../zh_CN/core-api/cpu_hotplug.rst | 435 +++++++++++++++--- 1 file changed, 377 insertions(+), 58 deletions(-) diff --git a/Documentation/translations/zh_CN/core-api/cpu_hotplug.rst b/Documentation/translations/zh_CN/core-api/cpu_hotplug.rst index 85a264287426..4772a900c37a 100644 --- a/Documentation/translations/zh_CN/core-api/cpu_hotplug.rst +++ b/Documentation/translations/zh_CN/core-api/cpu_hotplug.rst @@ -4,6 +4,7 @@ :翻译: 司延腾 Yanteng Si + 周彬彬 Binbin Zhou :校译: @@ -15,12 +16,13 @@ 内核中的CPU热拔插 ================= -:时间: 2016年12月 +:时间: 2021年9月 :作者: Sebastian Andrzej Siewior , - Rusty Russell , - Srivatsa Vaddagiri , - Ashok Raj , - Joel Schopp + Rusty Russell , + Srivatsa Vaddagiri , + Ashok Raj , + Joel Schopp , + Thomas Gleixner 简介 ==== @@ -139,7 +141,7 @@ CPU的热拔插协作 下线情况 -------- -一旦CPU被逻辑关闭，注册的热插拔状态的清除回调将被调用，从 ``CPUHP_ONLINE`` 开始，在 +一旦CPU被逻辑关闭，注册的热插拔状态的清除回调将被调用，从 ``CPUHP_ONLINE`` 开始，到 ``CPUHP_OFFLINE`` 状态结束。这包括: * 如果任务因暂停操作而被冻结，那么 *cpuhp_tasks_frozen* 将被设置为true。 @@ -154,82 +156,399 @@ CPU的热拔插协作 * 一旦所有的服务被迁移，内核会调用一个特定的例程 ``__cpu_disable()`` 来进行特定的清 理。 -使用热插拔API -------------- +CPU热插拔API +============ -一旦一个CPU下线或上线，就有可能收到通知。这对某些需要根据可用CPU数量执行某种设置或清 -理功能的驱动程序来说可能很重要:: +CPU热拔插状态机 +--------------- - #include +CPU热插拔使用一个从CPUHP_OFFLINE到CPUHP_ONLINE的线性状态空间的普通状态机。每个状态都 +有一个startup和teardown的回调。 - ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "X/Y:online", - Y_online, Y_prepare_down); +当一个CPU上线时，将按顺序调用startup回调，直到达到CPUHP_ONLINE状态。当设置状态的回调 +或将实例添加到多实例状态时，也可以调用它们。 -*X* 是子系统， *Y* 是特定的驱动程序。 *Y_online* 回调将在所有在线CPU的注册过程中被调用。 -如果在线回调期间发生错误， *Y_prepare_down* 回调将在所有之前调用过在线回调的CPU上调 -用。注册完成后，一旦有CPU上线， *Y_online* 回调将被调用，当CPU关闭时， *Y_prepare_down* -将被调用。所有之前在 *Y_online* 中分配的资源都应该在 *Y_prepare_down* 中释放。如果在 -注册过程中发生错误，返回值 *ret* 为负值。否则会返回一个正值，其中包含动态分配状态 -（ *CPUHP_AP_ONLINE_DYN* ）的分配热拔插。对于预定义的状态，它将返回0。 +当一个CPU下线时，将按相反的顺序依次调用teardown回调，直到达到CPUHP_OFFLINE状态。当删 +除状态的回调或从多实例状态中删除实例时，也可以调用它们。 -该回调可以通过调用 ``cpuhp_remove_state()`` 来删除。如果是动态分配的状态 -（ *CPUHP_AP_ONLINE_DYN* ），则使用返回的状态。在移除热插拔状态的过程中，将调用拆解回调。 +如果某个使用场景只需要一个方向的热插拔操作回调（CPU上线或CPU下线），则在设置状态时， +可以将另一个不需要的回调设置为NULL。 -多个实例 -~~~~~~~~ +状态空间被划分成三个阶段: -如果一个驱动程序有多个实例，并且每个实例都需要独立执行回调，那么很可能应该使用 -``multi-state`` 。首先需要注册一个多状态的状态:: +* PREPARE阶段 - ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "X/Y:online, - Y_online, Y_prepare_down); - Y_hp_online = ret; + PREPARE阶段涵盖了从CPUHP_OFFLINE到CPUHP_BRINGUP_CPU之间的状态空间。 -``cpuhp_setup_state_multi()`` 的行为与 ``cpuhp_setup_state()`` 类似，只是它 -为多状态准备了回调，但不调用回调。这是一个一次性的设置。 -一旦分配了一个新的实例，你需要注册这个新实例:: + 在该阶段中，startup回调在CPU上线操作启动CPU之前被调用，teardown回调在CPU下线操作使 + CPU功能失效之后被调用。 - ret = cpuhp_state_add_instance(Y_hp_online, &d->node); + 这些回调是在控制CPU上调用的，因为它们显然不能在热插拔的CPU上运行，此时热插拔的CPU要 + 么还没有启动，要么已经功能失效。 -这个函数将把这个实例添加到你先前分配的 ``Y_hp_online`` 状态，并在所有在线的 -CPU上调用先前注册的回调（ ``Y_online`` ）。 *node* 元素是你的每个实例数据结构 -中的一个 ``struct hlist_node`` 成员。 + startup回调用于设置CPU成功上线所需要的资源。teardown回调用于释放资源或在热插拔的CPU + 功能失效后，将待处理的工作转移到在线的CPU上。 -在移除该实例时:: + 允许startup回调失败。如果回调失败，CPU上线操作被中止，CPU将再次被降到之前的状态（通 + 常是CPUHP_OFFLINE）。 - cpuhp_state_remove_instance(Y_hp_online, &d->node) + 本阶段中的teardown回调不允许失败。 -应该被调用，这将在所有在线CPU上调用拆分回调。 +* STARTING阶段 -手动设置 -~~~~~~~~ + STARTING阶段涵盖了CPUHP_BRINGUP_CPU + 1到CPUHP_AP_ONLINE之间的状态空间。 -通常情况下，在注册或移除状态时调用setup和teamdown回调是很方便的，因为通常在CPU上线 -（下线）和驱动的初始设置（关闭）时需要执行该操作。然而，每个注册和删除功能也有一个 -_nocalls的后缀，如果不希望调用回调，则不调用所提供的回调。在手动设置（或关闭）期间， -应该使用 ``get_online_cpus()`` 和 ``put_online_cpus()`` 函数来抑制CPU热插拔操作。 + 该阶段中的startup回调是在早期CPU设置代码中的CPU上线操作期间，禁用中断的情况下在热拔 + 插的CPU上被调用。teardown回调是在CPU完全关闭前不久的CPU下线操作期间，禁用中断的情况 + 下在热拔插的CPU上被调用。 + 该阶段中的回调不允许失败。 -事件的顺序 ----------- + 回调用于低级别的硬件初始化/关机和核心子系统。 -热插拔状态被定义在 ``include/linux/cpuhotplug.h``: +* ONLINE阶段 -* ``CPUHP_OFFLINE`` ... ``CPUHP_AP_OFFLINE`` 状态是在CPU启动前调用的。 + ONLINE阶段涵盖了CPUHP_AP_ONLINE + 1到CPUHP_ONLINE之间的状态空间。 -* ``CPUHP_AP_OFFLINE`` ... ``CPUHP_AP_ONLINE`` 状态是在CPU被启动后被调用的。 - 中断是关闭的，调度程序还没有在这个CPU上活动。从 ``CPUHP_AP_OFFLINE`` 开始， - 回调被调用到目标CPU上。 + 该阶段中的startup回调是在CPU上线时在热插拔的CPU上调用的。teardown回调是在CPU下线操 + 作时在热插拔CPU上调用的。 -* ``CPUHP_AP_ONLINE_DYN`` 和 ``CPUHP_AP_ONLINE_DYN_END`` 之间的状态被保留 - 给动态分配。 + 回调是在每个CPU热插拔线程的上下文中调用的，该线程绑定在热插拔的CPU上。回调是在启用 + 中断和抢占的情况下调用的。 -* 这些状态在CPU关闭时以相反的顺序调用，从 ``CPUHP_ONLINE`` 开始，在 ``CPUHP_OFFLINE`` - 停止。这里的回调是在将被关闭的CPU上调用的，直到 ``CPUHP_AP_OFFLINE`` 。 + 允许回调失败。如果回调失败，CPU热插拔操作被中止，CPU将恢复到之前的状态。 -通过 ``CPUHP_AP_ONLINE_DYN`` 动态分配的状态通常已经足够了。然而，如果在启动或关闭 -期间需要更早的调用，那么应该获得一个显式状态。如果热拔插事件需要相对于另一个热拔插事 -件的特定排序，也可能需要一个显式状态。 +CPU 上线/下线操作 +----------------- + +一个成功的上线操作如下:: + + [CPUHP_OFFLINE] + [CPUHP_OFFLINE + 1]->startup() -> 成功 + [CPUHP_OFFLINE + 2]->startup() -> 成功 + [CPUHP_OFFLINE + 3] -> 略过，因为startup == NULL + ... + [CPUHP_BRINGUP_CPU]->startup() -> 成功 + === PREPARE阶段结束 + [CPUHP_BRINGUP_CPU + 1]->startup() -> 成功 + ... + [CPUHP_AP_ONLINE]->startup() -> 成功 + === STARTUP阶段结束 + [CPUHP_AP_ONLINE + 1]->startup() -> 成功 + ... + [CPUHP_ONLINE - 1]->startup() -> 成功 + [CPUHP_ONLINE] + +一个成功的下线操作如下:: + + [CPUHP_ONLINE] + [CPUHP_ONLINE - 1]->teardown() -> 成功 + ... + [CPUHP_AP_ONLINE + 1]->teardown() -> 成功 + === STARTUP阶段开始 + [CPUHP_AP_ONLINE]->teardown() -> 成功 + ... + [CPUHP_BRINGUP_ONLINE - 1]->teardown() + ... + === PREPARE阶段开始 + [CPUHP_BRINGUP_CPU]->teardown() + [CPUHP_OFFLINE + 3]->teardown() + [CPUHP_OFFLINE + 2] -> 略过，因为teardown == NULL + [CPUHP_OFFLINE + 1]->teardown() + [CPUHP_OFFLINE] + +一个失败的上线操作如下:: + + [CPUHP_OFFLINE] + [CPUHP_OFFLINE + 1]->startup() -> 成功 + [CPUHP_OFFLINE + 2]->startup() -> 成功 + [CPUHP_OFFLINE + 3] -> 略过，因为startup == NULL + ... + [CPUHP_BRINGUP_CPU]->startup() -> 成功 + === PREPARE阶段结束 + [CPUHP_BRINGUP_CPU + 1]->startup() -> 成功 + ... + [CPUHP_AP_ONLINE]->startup() -> 成功 + === STARTUP阶段结束 + [CPUHP_AP_ONLINE + 1]->startup() -> 成功 + --- + [CPUHP_AP_ONLINE + N]->startup() -> 失败 + [CPUHP_AP_ONLINE + (N - 1)]->teardown() + ... + [CPUHP_AP_ONLINE + 1]->teardown() + === STARTUP阶段开始 + [CPUHP_AP_ONLINE]->teardown() + ... + [CPUHP_BRINGUP_ONLINE - 1]->teardown() + ... + === PREPARE阶段开始 + [CPUHP_BRINGUP_CPU]->teardown() + [CPUHP_OFFLINE + 3]->teardown() + [CPUHP_OFFLINE + 2] -> 略过，因为teardown == NULL + [CPUHP_OFFLINE + 1]->teardown() + [CPUHP_OFFLINE] + +一个失败的下线操作如下:: + + [CPUHP_ONLINE] + [CPUHP_ONLINE - 1]->teardown() -> 成功 + ... + [CPUHP_ONLINE - N]->teardown() -> 失败 + [CPUHP_ONLINE - (N - 1)]->startup() + ... + [CPUHP_ONLINE - 1]->startup() + [CPUHP_ONLINE] + +递归失败不能被合理地处理。 +请看下面的例子，由于下线操作失败而导致的递归失败:: + + [CPUHP_ONLINE] + [CPUHP_ONLINE - 1]->teardown() -> 成功 + ... + [CPUHP_ONLINE - N]->teardown() -> 失败 + [CPUHP_ONLINE - (N - 1)]->startup() -> 成功 + [CPUHP_ONLINE - (N - 2)]->startup() -> 失败 + +CPU热插拔状态机在此停止，且不再尝试回滚，因为这可能会导致死循环:: + + [CPUHP_ONLINE - (N - 1)]->teardown() -> 成功 + [CPUHP_ONLINE - N]->teardown() -> 失败 + [CPUHP_ONLINE - (N - 1)]->startup() -> 成功 + [CPUHP_ONLINE - (N - 2)]->startup() -> 失败 + [CPUHP_ONLINE - (N - 1)]->teardown() -> 成功 + [CPUHP_ONLINE - N]->teardown() -> 失败 + +周而复始，不断重复。在这种情况下，CPU留在该状态中:: + + [CPUHP_ONLINE - (N - 1)] + +这至少可以让系统取得进展，让用户有机会进行调试，甚至解决这个问题。 + +分配一个状态 +------------ + +有两种方式分配一个CPU热插拔状态: + +* 静态分配 + + 当子系统或驱动程序有相对于其他CPU热插拔状态的排序要求时，必须使用静态分配。例如， + 在CPU上线操作期间，PERF核心startup回调必须在PERF驱动startup回调之前被调用。在CPU + 下线操作中，驱动teardown回调必须在核心teardown回调之前调用。静态分配的状态由 + cpuhp_state枚举中的常量描述，可以在include/linux/cpuhotplug.h中找到。 + + 在适当的位置将状态插入枚举中，这样就满足了排序要求。状态常量必须被用于状态的设置 + 和移除。 + + 当状态回调不是在运行时设置的，并且是kernel/cpu.c中CPU热插拔状态数组初始化的一部分 + 时，也需要静态分配。 + +* 动态分配 + + 当对状态回调没有排序要求时，动态分配是首选方法。状态编号由setup函数分配，并在成功 + 后返回给调用者。 + + 只有PREPARE和ONLINE阶段提供了一个动态分配范围。STARTING阶段则没有，因为该部分的大多 + 数回调都有明确的排序要求。 + +CPU热插拔状态的设置 +------------------- + +核心代码提供了以下函数用来设置状态： + +* cpuhp_setup_state(state, name, startup, teardown) +* cpuhp_setup_state_nocalls(state, name, startup, teardown) +* cpuhp_setup_state_cpuslocked(state, name, startup, teardown) +* cpuhp_setup_state_nocalls_cpuslocked(state, name, startup, teardown) + +对于一个驱动程序或子系统有多个实例，并且每个实例都需要调用相同的CPU hotplug状态回 +调的情况，CPU hotplug核心提供多实例支持。与驱动程序特定的实例列表相比，其优势在于 +与实例相关的函数完全针对CPU hotplug操作进行序列化，并在添加和删除时提供状态回调的 +自动调用。要设置这样一个多实例状态，可以使用以下函数： + +* cpuhp_setup_state_multi(state, name, startup, teardown) + +@state参数要么是静态分配的状态，要么是动态分配状态（PUHP_PREPARE_DYN，CPUHP_ONLINE_DYN） +的常量之一， 具体取决于应该分配动态状态的状态阶段（PREPARE，ONLINE）。 + +@name参数用于sysfs输出和检测。命名惯例是"subsys:mode"或"subsys/driver:mode"， +例如 "perf:mode"或"perf/x86:mode"。常见的mode名称有： + +======== ============================================ +prepare 对应PREPARE阶段中的状态 + +dead 对应PREPARE阶段中不提供startup回调的状态 + +starting 对应STARTING阶段中的状态 + +dying 对应STARTING阶段中不提供startup回调的状态 + +online 对应ONLINE阶段中的状态 + +offline 对应ONLINE阶段中不提供startup回调的状态 +======== ============================================ + +由于@name参数只用于sysfs和检测，如果其他mode描述符比常见的描述符更好地描述状态的性质， +也可以使用。 + +@name参数的示例："perf/online", "perf/x86:prepare", "RCU/tree:dying", "sched/waitempty" + +@startup参数是一个指向回调的函数指针，在CPU上线操作时被调用。若应用不需要startup +回调，则将该指针设为NULL。 + +@teardown参数是一个指向回调的函数指针，在CPU下线操作时调用。若应用不需要teardown +回调，则将该指针设为NULL。 + +这些函数在处理已注册回调的方式上有所不同: + + * cpuhp_setup_state_nocalls(), cpuhp_setup_state_nocalls_cpuslocked()和 + cpuhp_setup_state_multi()只注册回调。 + + * cpuhp_setup_state()和cpuhp_setup_state_cpuslocked()注册回调，并对当前状态大于新 + 安装状态的所有在线CPU调用@startup回调（如果不是NULL）。根据状态阶段，回调要么在 + 当前的CPU上调用（PREPARE阶段），要么在CPU的热插拔线程中调用每个在线CPU（ONLINE阶段）。 + + 如果CPU N的回调失败，那么CPU 0...N-1的teardown回调被调用以回滚操作。状态设置失败， + 状态的回调没有被注册，在动态分配的情况下，分配的状态被释放。 + +状态设置和回调调用是针对CPU热拔插操作进行序列化的。如果设置函数必须从CPU热插拔的读 +锁定区域调用，那么必须使用_cpuslocked()变体。这些函数不能在CPU热拔插回调中使用。 + +函数返回值： + ======== ========================================================== + 0 静态分配的状态设置成功 + + >0 动态分配的状态设置成功 + + 返回的数值是被分配的状态编号。如果状态回调后来必须被移除， + 例如模块移除，那么这个数值必须由调用者保存，并作为状态移 + 除函数的@state参数。对于多实例状态，动态分配的状态编号也 + 需要作为实例添加/删除操作的@state参数。 + + <0 操作失败 + ======== ========================================================== + +移除CPU热拔插状态 +----------------- + +为了移除一个之前设置好的状态，提供了如下函数： + +* cpuhp_remove_state(state) +* cpuhp_remove_state_nocalls(state) +* cpuhp_remove_state_nocalls_cpuslocked(state) +* cpuhp_remove_multi_state(state) + +@state参数要么是静态分配的状态，要么是由cpuhp_setup_state*()在动态范围内分配 +的状态编号。如果状态在动态范围内，则状态编号被释放，可再次进行动态分配。 + +这些函数在处理已注册回调的方式上有所不同: + + * cpuhp_remove_state_nocalls(), cpuhp_remove_state_nocalls_cpuslocked() + 和 cpuhp_remove_multi_state()只删除回调。 + + * cpuhp_remove_state()删除回调，并调用所有当前状态大于被删除状态的在线CPU的 + teardown回调（如果不是NULL）。根据状态阶段，回调要么在当前的CPU上调用 + （PREPARE阶段），要么在CPU的热插拔线程中调用每个在线CPU（ONLINE阶段）。 + + 为了完成移除工作，teardown回调不能失败。 + +状态移除和回调调用是针对CPU热拔插操作进行序列化的。如果移除函数必须从CPU hotplug +读取锁定区域调用，那么必须使用_cpuslocked()变体。这些函数不能从CPU热插拔的回调中使用。 + +如果一个多实例的状态被移除，那么调用者必须先移除所有的实例。 + +多实例状态实例管理 +------------------ + +一旦多实例状态被建立，实例就可以被添加到状态中： + + * cpuhp_state_add_instance(state, node) + * cpuhp_state_add_instance_nocalls(state, node) + +@state参数是一个静态分配的状态或由cpuhp_setup_state_multi()在动态范围内分配的状 +态编号。 + +@node参数是一个指向hlist_node的指针，它被嵌入到实例的数据结构中。这个指针被交给 +多实例状态的回调，可以被回调用来通过container_of()检索到实例。 + +这些函数在处理已注册回调的方式上有所不同: + + * cpuhp_state_add_instance_nocalls()只将实例添加到多实例状态的节点列表中。 + + * cpuhp_state_add_instance()为所有当前状态大于@state的在线CPU添加实例并调用与 + @state相关的startup回调（如果不是NULL）。该回调只对将要添加的实例进行调用。 + 根据状态阶段，回调要么在当前的CPU上调用（PREPARE阶段），要么在CPU的热插拔线 + 程中调用每个在线CPU（ONLINE阶段）。 + + 如果CPU N的回调失败，那么CPU 0 ... N-1的teardown回调被调用以回滚操作，该函数 + 失败，实例不会被添加到多实例状态的节点列表中。 + +要从状态的节点列表中删除一个实例，可以使用这些函数: + + * cpuhp_state_remove_instance(state, node) + * cpuhp_state_remove_instance_nocalls(state, node) + +参数与上述cpuhp_state_add_instance*()变体相同。 + +这些函数在处理已注册回调的方式上有所不同: + + * cpuhp_state_remove_instance_nocalls()只从状态的节点列表中删除实例。 + + * cpuhp_state_remove_instance()删除实例并调用与@state相关的回调（如果不是NULL）， + 用于所有当前状态大于@state的在线CPU。 该回调只对将要被移除的实例进行调用。 + 根据状态阶段，回调要么在当前的CPU上调用（PREPARE阶段），要么在CPU的热插拔 + 线程中调用每个在线CPU（ONLINE阶段）。 + + 为了完成移除工作，teardown回调不能失败。 + +节点列表的添加/删除操作和回调调用是针对CPU热拔插操作进行序列化。这些函数不能在 +CPU hotplug回调和CPU hotplug读取锁定区域内使用。 + +样例 +---- + +在STARTING阶段设置和取消静态分配的状态，以获取上线和下线操作的通知:: + + ret = cpuhp_setup_state(CPUHP_SUBSYS_STARTING, "subsys:starting", subsys_cpu_starting, subsys_cpu_dying); + if (ret < 0) + return ret; + .... + cpuhp_remove_state(CPUHP_SUBSYS_STARTING); + +在ONLINE阶段设置和取消动态分配的状态，以获取下线操作的通知:: + + state = cpuhp_setup_state(CPUHP_ONLINE_DYN, "subsys:offline", NULL, subsys_cpu_offline); + if (state < 0) + return state; + .... + cpuhp_remove_state(state); + +在ONLINE阶段设置和取消动态分配的状态，以获取有关上线操作的通知，而无需调用回调:: + + state = cpuhp_setup_state_nocalls(CPUHP_ONLINE_DYN, "subsys:online", subsys_cpu_online, NULL); + if (state < 0) + return state; + .... + cpuhp_remove_state_nocalls(state); + +在ONLINE阶段设置、使用和取消动态分配的多实例状态，以获得上线和下线操作的通知:: + + state = cpuhp_setup_state_multi(CPUHP_ONLINE_DYN, "subsys:online", subsys_cpu_online, subsys_cpu_offline); + if (state < 0) + return state; + .... + ret = cpuhp_state_add_instance(state, &inst1->node); + if (ret) + return ret; + .... + ret = cpuhp_state_add_instance(state, &inst2->node); + if (ret) + return ret; + .... + cpuhp_remove_instance(state, &inst1->node); + .... + cpuhp_remove_instance(state, &inst2->node); + .... + remove_multi_state(state); 测试热拔插状态 ============== From 722ecdbce68a87de2d9296f91308f44ea900a039 Mon Sep 17 00:00:00 2001 From: Binbin Zhou Date: Wed, 22 Jun 2022 18:18:49 +0800 Subject: [PATCH 110/318] docs/zh_CN: core-api: Update the translation of irq/irq-domain.rst to 5.19-rc3 Synchronous translation from the following commits(Latest in front): [1]: commit 0953fb263714("irq: remove handle_domain_{irq,nmi}()") [2]: commit 0ddc5e55e6f1("Documentation: Fix irq-domain.rst build warning") [3]: commit 991007ba6cca("Documentation: Update irq_domain.rst with new lookup APIs") [4]: commit 405e94e9aed2("irqdomain: Kill irq_domain_add_legacy_isa") Signed-off-by: Binbin Zhou Reviewed-by: Yanteng Si Reviewed-by: Wu XiangCheng Link: https://lore.kernel.org/r/eb82606f323c4c25e497a01a060e1e76f18be72f.1655863769.git.zhoubinbin@loongson.cn Signed-off-by: Jonathan Corbet --- .../zh_CN/core-api/irq/irq-domain.rst | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/Documentation/translations/zh_CN/core-api/irq/irq-domain.rst b/Documentation/translations/zh_CN/core-api/irq/irq-domain.rst index 7d077742f758..9174fce12c1b 100644 --- a/Documentation/translations/zh_CN/core-api/irq/irq-domain.rst +++ b/Documentation/translations/zh_CN/core-api/irq/irq-domain.rst @@ -5,6 +5,7 @@ :翻译: 司延腾 Yanteng Si + 周彬彬 Binbin Zhou .. _cn_irq-domain.rst: @@ -52,8 +53,18 @@ irq_domain和一个hwirq号作为参数。 如果hwirq的映射还不存在， 一个新的Linux irq_desc，将其与hwirq关联起来，并调用.map()回调，这样驱动 程序就可以执行任何必要的硬件设置。 -当接收到一个中断时，应该使用irq_find_mapping()函数从hwirq号中找到 -Linux IRQ号。 +一旦建立了映射，可以通过多种方法检索或使用它： + +- irq_resolve_mapping()返回一个指向给定域和hwirq号的irq_desc结构指针， + 如果没有映射则返回NULL。 + +- irq_find_mapping()返回给定域和hwirq的Linux IRQ号，如果没有映射则返回0。 + +- irq_linear_revmap()现与irq_find_mapping()相同，已被废弃。 + +- generic_handle_domain_irq()处理一个由域和hwirq号描述的中断。 + +请注意，irq域的查找必须发生在与RCU读临界区兼容的上下文中。 在调用irq_find_mapping()之前，至少要调用一次irq_create_mapping()函数， 以免描述符不能被分配。 @@ -119,7 +130,8 @@ irq_domain_add_tree()和irq_domain_create_tree()在功能上是等价的，除 Linux IRQ号编入硬件本身，这样就不需要映射了。 调用irq_create_direct_mapping() 会分配一个Linux IRQ号，并调用.map()回调，这样驱动就可以将Linux IRQ号编入硬件中。 -大多数驱动程序不能使用这个映射。 +大多数驱动程序无法使用此映射，现在它由CONFIG_IRQ_DOMAIN_NOMAP选项控制。 +请不要引入此API的新用户。 传统映射类型 ------------ @@ -128,7 +140,6 @@ Linux IRQ号编入硬件本身，这样就不需要映射了。 调用irq_create irq_domain_add_simple() irq_domain_add_legacy() - irq_domain_add_legacy_isa() irq_domain_create_simple() irq_domain_create_legacy() @@ -137,6 +148,9 @@ Linux IRQ号编入硬件本身，这样就不需要映射了。 调用irq_create 一组用于IRQ号的定义（#define），这些定义被传递给struct设备注册。 在这种情况下， 不能动态分配Linux IRQ号，应该使用传统映射。 +顾名思义，\*_legacy()系列函数已被废弃，只是为了方便对古老平台的支持而存在。 +不应该增加新的用户。当\*_simple()系列函数的使用导致遗留行为时，他们也是如此。 + 传统映射假设已经为控制器分配了一个连续的IRQ号范围，并且可以通过向hwirq号添加一 个固定的偏移来计算IRQ号，反之亦然。 缺点是需要中断控制器管理IRQ分配，并且需要为每 个hwirq分配一个irq_desc，即使它没有被使用。 From b2fdf7f080b4fbc8b22e006c2b78153253518741 Mon Sep 17 00:00:00 2001 From: Binbin Zhou Date: Wed, 22 Jun 2022 18:18:50 +0800 Subject: [PATCH 111/318] docs/zh_CN: core-api: Update the translation of kernel-api.rst to 5.19-rc3 Synchronous translation from upstream commit 640d1930bef4 ("block: Add bio_for_each_folio_all()") Signed-off-by: Binbin Zhou Reviewed-by: Wu XiangCheng Reviewed-by: Yanteng Si Link: https://lore.kernel.org/r/9173878562404f4518fecc7c55032fe815f87fd9.1655863769.git.zhoubinbin@loongson.cn Signed-off-by: Jonathan Corbet --- Documentation/translations/zh_CN/core-api/kernel-api.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Documentation/translations/zh_CN/core-api/kernel-api.rst b/Documentation/translations/zh_CN/core-api/kernel-api.rst index e45fe80d1cd8..120c051f6fcb 100644 --- a/Documentation/translations/zh_CN/core-api/kernel-api.rst +++ b/Documentation/translations/zh_CN/core-api/kernel-api.rst @@ -5,6 +5,7 @@ :翻译: 司延腾 Yanteng Si + 周彬彬 Binbin Zhou .. _cn_kernel-api.rst: @@ -282,6 +283,8 @@ kernel/acct.c 该API在以下内核代码中: +include/linux/bio.h + block/blk-core.c block/blk-core.c From f5158bfe8777d1c42694cd53c59a7b0297e57cc6 Mon Sep 17 00:00:00 2001 From: Binbin Zhou Date: Wed, 22 Jun 2022 18:18:51 +0800 Subject: [PATCH 112/318] docs/zh_CN: core-api: Update the translation of mm-api.rst to 5.19-rc3 Synchronous translation from the following commits(Latest in front): [1]: commit 84dacdbd5352("mm: document and polish read-ahead code") [2]: commit 2f52578f9c64("mm/util: Add folio_mapping() and folio_file_mapping()") [3]: commit 889a3747b3b7("mm/lru: Add folio LRU functions") [4]: commit c24016ac3a62("mm: Add folio reference count functions") Signed-off-by: Binbin Zhou Reviewed-by: Yanteng Si Reviewed-by: Wu XiangCheng Link: https://lore.kernel.org/r/0bd7b03a5d38dd1599715343360bef4289b7c0fb.1655863769.git.zhoubinbin@loongson.cn Signed-off-by: Jonathan Corbet --- .../translations/zh_CN/core-api/mm-api.rst | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/Documentation/translations/zh_CN/core-api/mm-api.rst b/Documentation/translations/zh_CN/core-api/mm-api.rst index 0ea43dc67953..a732b0eebf16 100644 --- a/Documentation/translations/zh_CN/core-api/mm-api.rst +++ b/Documentation/translations/zh_CN/core-api/mm-api.rst @@ -5,6 +5,7 @@ :翻译: 司延腾 Yanteng Si + 周彬彬 Binbin Zhou :校译: @@ -66,12 +67,24 @@ mm/vmalloc.c 该API在以下内核代码中: -mm/readahead.c +文件映射 +-------- mm/filemap.c +预读 +---- + +mm/readahead.c + +回写 +---- + mm/page-writeback.c +截断 +---- + mm/truncate.c include/linux/pagemap.h @@ -105,6 +118,14 @@ mm/mempolicy.c include/linux/mm_types.h +include/linux/mm_inline.h + +include/linux/page-flags.h + include/linux/mm.h +include/linux/page_ref.h + include/linux/mmzone.h + +mm/util.c From 4faa99e38c3e45ff031400839783ab0843740c35 Mon Sep 17 00:00:00 2001 From: Binbin Zhou Date: Wed, 22 Jun 2022 18:18:52 +0800 Subject: [PATCH 113/318] docs/zh_CN: core-api: Update the translation of printk-basics.rst to 5.19-rc3 Synchronous translation from upstream commit 7d9e2661f268 ("printk: Move the printk() kerneldoc comment to its new home") Signed-off-by: Binbin Zhou Reviewed-by: Yanteng Si Reviewed-by: Wu XiangCheng Link: https://lore.kernel.org/r/56c6d212d6835029b4adf19cf78eec49c7a126d6.1655863769.git.zhoubinbin@loongson.cn Signed-off-by: Jonathan Corbet --- Documentation/translations/zh_CN/core-api/printk-basics.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Documentation/translations/zh_CN/core-api/printk-basics.rst b/Documentation/translations/zh_CN/core-api/printk-basics.rst index d574de3167c8..59c6efb3fc41 100644 --- a/Documentation/translations/zh_CN/core-api/printk-basics.rst +++ b/Documentation/translations/zh_CN/core-api/printk-basics.rst @@ -6,6 +6,7 @@ :翻译: 司延腾 Yanteng Si + 周彬彬 Binbin Zhou .. _cn_printk-basics.rst: @@ -107,6 +108,4 @@ pr_debug()和pr_devel()，除非定义了 ``DEBUG`` (或者在pr_debug()的情 该API在以下内核代码中: -kernel/printk/printk.c - include/linux/printk.h From e86a0e297f0b5e7c0f233c2e2a4a6c403b27afa6 Mon Sep 17 00:00:00 2001 From: Binbin Zhou Date: Wed, 22 Jun 2022 18:18:53 +0800 Subject: [PATCH 114/318] docs/zh_CN: core-api: Update the translation of printk-format.rst to 5.19-rc3 Synchronous translation from upstream commit 6a7ca80f4033 ("vsprintf: Update %pGp documentation about that it prints hex value") Signed-off-by: Binbin Zhou Reviewed-by: Yanteng Si Reviewed-by: Wu XiangCheng Link: https://lore.kernel.org/r/2ba15ab2aa46bfdd400d1c043eecc12bc6154836.1655863769.git.zhoubinbin@loongson.cn Signed-off-by: Jonathan Corbet --- Documentation/translations/zh_CN/core-api/printk-formats.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation/translations/zh_CN/core-api/printk-formats.rst b/Documentation/translations/zh_CN/core-api/printk-formats.rst index ce39c788cf5a..bd36d35eba4e 100644 --- a/Documentation/translations/zh_CN/core-api/printk-formats.rst +++ b/Documentation/translations/zh_CN/core-api/printk-formats.rst @@ -5,6 +5,7 @@ :翻译: 司延腾 Yanteng Si + 周彬彬 Binbin Zhou .. _cn_printk-formats.rst: @@ -548,7 +549,7 @@ nodemask_pr_args()来方便打印cpumask和nodemask。 :: - %pGp referenced|uptodate|lru|active|private|node=0|zone=2|lastcpupid=0x1fffff + %pGp 0x17ffffc0002036(referenced|uptodate|lru|active|private|node=0|zone=2|lastcpupid=0x1fffff) %pGg GFP_USER|GFP_DMA32|GFP_NOWARN %pGv read|exec|mayread|maywrite|mayexec|denywrite From c290f175e73fe04feba492e08a92093a4763914c Mon Sep 17 00:00:00 2001 From: Binbin Zhou Date: Wed, 22 Jun 2022 18:19:42 +0800 Subject: [PATCH 115/318] docs/zh_CN: core-api: Update the translation of workqueue.rst to 5.19-rc3 Synchronous translation from upstream commit f9eaaa82b474 ("workqueue: doc: Call out the non-reentrance conditions") Signed-off-by: Binbin Zhou Reviewed-by: Wu XiangCheng Reviewed-by: Yanteng Si Link: https://lore.kernel.org/r/d045cb5623b5ae703c2296d3caa05017d5d225bd.1655863769.git.zhoubinbin@loongson.cn Signed-off-by: Jonathan Corbet --- .../translations/zh_CN/core-api/workqueue.rst | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/Documentation/translations/zh_CN/core-api/workqueue.rst b/Documentation/translations/zh_CN/core-api/workqueue.rst index e372fa5cf101..f6567cf9d3fb 100644 --- a/Documentation/translations/zh_CN/core-api/workqueue.rst +++ b/Documentation/translations/zh_CN/core-api/workqueue.rst @@ -6,6 +6,7 @@ :翻译: 司延腾 Yanteng Si + 周彬彬 Binbin Zhou .. _cn_workqueue.rst: @@ -178,10 +179,6 @@ workqueue将自动创建与属性相匹配的后备工作者池。调节并发 这个标志对于未绑定的wq来说是没有意义的。 -请注意，标志 ``WQ_NON_REENTRANT`` 不再存在，因为现在所有的工作 -队列都是不可逆的——任何工作项都保证在任何时间内最多被整个系统的一 -个工作者执行。 - ``max_active`` -------------- @@ -328,6 +325,22 @@ And with cmwq with ``@max_active`` >= 3, :: 工作项函数在堆栈追踪中应该是微不足道的。 +不可重入条件 +============ + +工作队列保证，如果在工作项排队后满足以下条件，则工作项不能重入： + + + 1. 工作函数没有被改变。 + 2. 没有人将该工作项排到另一个工作队列中。 + 3. 该工作项尚未被重新启动。 + +换言之，如果上述条件成立，则保证在任何给定时间最多由一个系统范围内的工作程序执行 +该工作项。 + +请注意，在self函数中将工作项重新排队（到同一队列）不会破坏这些条件，因此可以安全 +地执行此操作。否则在破坏工作函数内部的条件时需要小心。 + 内核内联文档参考 ================ From 4a6d00a43ef7c1d1449f2601f7195e4e5d40d432 Mon Sep 17 00:00:00 2001 From: Binbin Zhou Date: Wed, 22 Jun 2022 18:19:49 +0800 Subject: [PATCH 116/318] docs/zh_CN: core-api: Update the translation of xarray.rst to 5.19-rc3 Synchronous translation from upstream commit ac23d1a96460 ("XArray: Document the locking requirement for the xa_state") Signed-off-by: Binbin Zhou Reviewed-by: Wu XiangCheng Reviewed-by: Yanteng Si Link: https://lore.kernel.org/r/b7d5f3ca914667ec14ba35a7c0db3ec3506e670d.1655863769.git.zhoubinbin@loongson.cn Signed-off-by: Jonathan Corbet --- Documentation/translations/zh_CN/core-api/xarray.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Documentation/translations/zh_CN/core-api/xarray.rst b/Documentation/translations/zh_CN/core-api/xarray.rst index ff2d9bcb7c34..fb19324966ce 100644 --- a/Documentation/translations/zh_CN/core-api/xarray.rst +++ b/Documentation/translations/zh_CN/core-api/xarray.rst @@ -6,6 +6,7 @@ :翻译: 司延腾 Yanteng Si + 周彬彬 Binbin Zhou :校译: @@ -254,7 +255,8 @@ __xa_set_mark() 和 __xa_clear_mark() 函数也适用于你查找一个条目并 高级API是基于xa_state的。这是一个不透明的数据结构，你使用XA_STATE()宏在堆栈中声明。这个宏初始化了 xa_state，准备开始在XArray上移动。它被用作一个游标来保持在XArray中的位置，并让你把各种操作组合在一 -起，而不必每次都从头开始。 +起，而不必每次都从头开始。xa_state的内容受rcu_read_lock()或xas_lock()的保护。如果需要删除保护状态 +和树的这些锁中的任何一个，你必须调用xas_pause()以便将来的调用不会依赖于状态中未受保护的部分。 xa_state也被用来存储错误(store errors)。你可以调用xas_error()来检索错误。所有的操作在进行之前都 会检查xa_state是否处于错误状态，所以你没有必要在每次调用之后检查错误；你可以连续进行多次调用，只在 From 5b8d9ee0034576062460255d8bab3b3e61a2a25f Mon Sep 17 00:00:00 2001 From: Steven Lung <1030steven@gmail.com> Date: Tue, 21 Jun 2022 15:29:10 +0800 Subject: [PATCH 117/318] docs: UML: fix typo Replace 'absense' with 'absence'. Signed-off-by: Steven Lung <1030steven@gmail.com> Link: https://lore.kernel.org/r/20220621072910.4704-1-1030steven@gmail.com Signed-off-by: Jonathan Corbet --- Documentation/virt/uml/user_mode_linux_howto_v2.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/virt/uml/user_mode_linux_howto_v2.rst b/Documentation/virt/uml/user_mode_linux_howto_v2.rst index 863f67b72c05..af2a97429692 100644 --- a/Documentation/virt/uml/user_mode_linux_howto_v2.rst +++ b/Documentation/virt/uml/user_mode_linux_howto_v2.rst @@ -322,7 +322,7 @@ Shared Options * ``v6=[0,1]`` to specify if a v6 connection is desired for all transports which operate over IP. Additionally, for transports that have some differences in the way they operate over v4 and v6 (for example - EoL2TPv3), sets the correct mode of operation. In the absense of this + EoL2TPv3), sets the correct mode of operation. In the absence of this option, the socket type is determined based on what do the src and dst arguments resolve/parse to. From 882946793000debbd83253216b7f4ee7d2496b5e Mon Sep 17 00:00:00 2001 From: Yanteng Si Date: Thu, 16 Jun 2022 15:44:56 +0800 Subject: [PATCH 118/318] docs/zh_CN: Update the translation of highmem to 5.19-rc1 update to commit 110bf7a52307 ("Documentation/vm: rework "Temporary Virtual Mappings" section") Signed-off-by: Yanteng Si Reviewed-by: Wu XiangCheng Reviewed-by: Alex Shi Link: https://lore.kernel.org/r/76c418554d9b8d395f7e8331e8e3b89d1b7beff0.1655362610.git.siyanteng@loongson.cn Signed-off-by: Jonathan Corbet --- .../translations/zh_CN/vm/highmem.rst | 83 ++++++++++--------- 1 file changed, 46 insertions(+), 37 deletions(-) diff --git a/Documentation/translations/zh_CN/vm/highmem.rst b/Documentation/translations/zh_CN/vm/highmem.rst index 018838e58c3e..200321774646 100644 --- a/Documentation/translations/zh_CN/vm/highmem.rst +++ b/Documentation/translations/zh_CN/vm/highmem.rst @@ -50,55 +50,55 @@ 临时虚拟映射 ============ -内核包含几种创建临时映射的方法。: +内核包含几种创建临时映射的方法。下面的列表按照使用的优先顺序显示了它们。 -* vmap(). 这可以用来将多个物理页长期映射到一个连续的虚拟空间。它需要synchronization - 来解除映射。 +* kmap_local_page()。这个函数是用来要求短期映射的。它可以从任何上下文（包括中断）中调用， + 但是映射只能在获取它们的上下文中使用。 -* kmap(). 这允许对单个页面进行短期映射。它需要synchronization，但在一定程度上被摊销。 - 当以嵌套方式使用时，它也很容易出现死锁，因此不建议在新代码中使用它。 + 在可行的情况下，这个函数应该比其他所有的函数优先使用。 + + 这些映射是线程本地和CPU本地的，这意味着映射只能从这个线程中访问，并且当映射处于活动状 + 态时，该线程与CPU绑定。即使线程被抢占了（因为抢占永远不会被函数禁用），CPU也不能通过 + CPU-hotplug从系统中拔出，直到映射被处理掉。 + + 在本地的kmap区域中采取pagefaults是有效的，除非获取本地映射的上下文由于其他原因不允许 + 这样做。 + + kmap_local_page()总是返回一个有效的虚拟地址，并且假定kunmap_local()不会失败。 + + 嵌套kmap_local_page()和kmap_atomic()映射在一定程度上是允许的（最多到KMAP_TYPE_NR）， + 但是它们的调用必须严格排序，因为映射的实现是基于堆栈的。关于如何管理嵌套映射的细节， + 请参见kmap_local_page() kdocs（包含在 "函数 "部分）。 * kmap_atomic(). 这允许对单个页面进行非常短的时间映射。由于映射被限制在发布它的CPU上， - 它表现得很好，但发布任务因此被要求留在该CPU上直到它完成，以免其他任务取代它的映射。 + 它表现得很好，但发布的任务因此被要求留在该CPU上直到它完成，以免其他任务取代它的映射。 - kmap_atomic() 也可以由中断上下文使用，因为它不睡眠，而且调用者可能在调用kunmap_atomic() - 之后才睡眠。 + kmap_atomic()也可以被中断上下文使用，因为它不睡眠，调用者也可能在调用kunmap_atomic() + 后才睡眠。 - 可以假设k[un]map_atomic()不会失败。 + 内核中对kmap_atomic()的每次调用都会创建一个不可抢占的段，并禁用缺页异常。这可能是 + 未预期延迟的来源之一。因此用户应该选择kmap_local_page()而不是kmap_atomic()。 + 假设k[un]map_atomic()不会失败。 -使用kmap_atomic -=============== +* kmap()。这应该被用来对单个页面进行短时间的映射，对抢占或迁移没有限制。它会带来开销， + 因为映射空间是受限制的，并且受到全局锁的保护，以实现同步。当不再需要映射时，必须用 + kunmap()释放该页被映射的地址。 -何时何地使用 kmap_atomic() 是很直接的。当代码想要访问一个可能从高内存（见__GFP_HIGHMEM） -分配的页面的内容时，例如在页缓存中的页面，就会使用它。该API有两个函数，它们的使用方式与 -下面类似:: + 映射变化必须广播到所有CPU（核）上，kmap()还需要在kmap的池被回绕（TLB项用光了，需要从第 + 一项复用）时进行全局TLB无效化，当映射空间被完全利用时，它可能会阻塞，直到有一个可用的 + 槽出现。因此，kmap()只能从可抢占的上下文中调用。 - /* 找到感兴趣的页面。 */ - struct page *page = find_get_page(mapping, offset); + 如果一个映射必须持续相对较长的时间，上述所有的工作都是必要的，但是内核中大部分的 + 高内存映射都是短暂的，而且只在一个地方使用。这意味着在这种情况下，kmap()的成本大 + 多被浪费了。kmap()并不是为长期映射而设计的，但是它已经朝着这个方向发展了，在较新 + 的代码中强烈不鼓励使用它，前面的函数集应该是首选。 - /* 获得对该页内容的访问权。 */ - void *vaddr = kmap_atomic(page); - - /* 对该页的内容做一些处理。 */ - memset(vaddr, 0, PAGE_SIZE); - - /* 解除该页面的映射。 */ - kunmap_atomic(vaddr); - -注意，kunmap_atomic()调用的是kmap_atomic()调用的结果而不是参数。 - -如果你需要映射两个页面，因为你想从一个页面复制到另一个页面，你需要保持kmap_atomic调用严 -格嵌套，如:: - - vaddr1 = kmap_atomic(page1); - vaddr2 = kmap_atomic(page2); - - memcpy(vaddr1, vaddr2, PAGE_SIZE); - - kunmap_atomic(vaddr2); - kunmap_atomic(vaddr1); + 在64位系统中，调用kmap_local_page()、kmap_atomic()和kmap()没有实际作用，因为64位 + 地址空间足以永久映射所有物理内存页面。 +* vmap()。这可以用来将多个物理页长期映射到一个连续的虚拟空间。它需要全局同步来解除 + 映射。 临时映射的成本 ============== @@ -126,3 +126,12 @@ i386 PAE 一般的建议是，你不要在32位机器上使用超过8GiB的空间--尽管更多的空间可能对你和你的工作 量有用，但你几乎是靠你自己--不要指望内核开发者真的会很关心事情的进展情况。 + +函数 +==== + +该API在以下内核代码中: + +include/linux/highmem.h + +include/linux/highmem-internal.h From 34191e04cd49c1b5b7a6a558b328c0b0d7b4b404 Mon Sep 17 00:00:00 2001 From: Yanteng Si Date: Thu, 16 Jun 2022 15:44:57 +0800 Subject: [PATCH 119/318] docs/zh_CN: Update the translation of page_owner to 5.19-rc1 update to commit d1ed51fcdbd6 ("docs: vm/page_owner: tweak literal block in STANDARD FORMAT SPECIFIERS") Signed-off-by: Yanteng Si Reviewed-by: Wu XiangCheng Reviewed-by: Alex Shi Link: https://lore.kernel.org/r/e96557b73b036c33a9fa5abdab0c541704235e92.1655362610.git.siyanteng@loongson.cn Signed-off-by: Jonathan Corbet --- .../translations/zh_CN/vm/page_owner.rst | 73 +++++++++++++++++-- 1 file changed, 67 insertions(+), 6 deletions(-) diff --git a/Documentation/translations/zh_CN/vm/page_owner.rst b/Documentation/translations/zh_CN/vm/page_owner.rst index 9e951fabba9d..0034d2ae1cd5 100644 --- a/Documentation/translations/zh_CN/vm/page_owner.rst +++ b/Documentation/translations/zh_CN/vm/page_owner.rst @@ -96,21 +96,82 @@ page owner在默认情况下是禁用的。所以，如果你想使用它，你 默认情况下， ``page_owner_sort`` 是根据buf的时间来排序的。如果你想 按buf的页数排序，请使用-m参数。详细的参数是: - 基本函数: + 基本函数:: - Sort: + 排序: -a 按内存分配时间排序 -m 按总内存排序 -p 按pid排序。 -P 按tgid排序。 + -n 按任务命令名称排序。 -r 按内存释放时间排序。 -s 按堆栈跟踪排序。 -t 按时间排序（默认）。 + --sort 指定排序顺序。排序的语法是[+|-]key[,[+|-]key[,...]]。从 + **标准格式指定器**那一节选择一个键。"+"是可选的，因为默认的方向是数字或 + 词法的增加。允许混合使用缩写和完整格式的键。 - 其它函数: + 例子: + ./page_owner_sort --sort=n,+pid,-tgid + ./page_owner_sort --sort=at - Cull: - -c 通过比较堆栈跟踪而不是总块来进行剔除。 + 其它函数:: - Filter: + 剔除: + --cull + 指定剔除规则。剔除的语法是key[,key[,...]]。从**标准格式指定器** + 部分选择一个多字母键。 + 是一个以逗号分隔的列表形式的单一参数，它提供了一种指定单个剔除规则的 + 方法。 识别的关键字在下面的**标准格式指定器**部分有描述。<规则>可以通过键的 + 序列k1,k2,...来指定，在下面的标准排序键部分有描述。允许混合使用简写和完整形 + 式的键。 + + Examples: + ./page_owner_sort --cull=stacktrace + ./page_owner_sort --cull=st,pid,name + ./page_owner_sort --cull=n,f + + 过滤: -f 过滤掉内存已被释放的块的信息。 + + 选择: + --pid 按pid选择。这将选择进程ID号出现在中的块。 + --tgid 按tgid选择。这将选择其线程组ID号出现在 + 中的块。 + --name 按任务命令名称选择。这将选择其任务命令名称出现在 + 中的区块。 + + , , 是以逗号分隔的列表形式的单个参数， + 它提供了一种指定单个选择规则的方法。 + + + 例子: + ./page_owner_sort --pid=1 + ./page_owner_sort --tgid=1,2,3 + ./page_owner_sort --name name1,name2 + +标准格式指定器 +============== +:: + + --sort的选项: + + 键 键长 描述 + p pid 进程ID + tg tgid 线程组ID + n name 任务命令名称 + st stacktrace 页面分配的堆栈跟踪 + T txt 块的全文 + ft free_ts 页面发布时的时间戳 + at alloc_ts 页面被分配时的时间戳 + ator allocator 页面的内存分配器 + + --curl的选项: + + 键 键长 描述 + p pid 进程ID + tg tgid 线程组ID + n name 任务命令名称 + f free 该页是否已经发布 + st stacktrace 页面分配的堆栈跟踪 + ator allocator 页面的内存分配器 From 910cb3137c076dc908e267bad034364cd9e4af6e Mon Sep 17 00:00:00 2001 From: Yanteng Si Date: Thu, 16 Jun 2022 15:44:58 +0800 Subject: [PATCH 120/318] docs/zh_CN: Update the translation of vm index to 5.19-rc1 update to commit 481cc97349d6 ("mm,doc: Add new documentation structure") Note: There are some empty files that do not need to be updated for now, they are: bootmem, oom, page_allocation, page_cache, page_reclaim, page_tables, physical_memory process_addrs, shmfs, slab, vmalloc, swap. Signed-off-by: Yanteng Si Reviewed-by: Wu XiangCheng Reviewed-by: Alex Shi Link: https://lore.kernel.org/r/35d10864fb9b57bf288d4efa5642f93a60079029.1655362610.git.siyanteng@loongson.cn Signed-off-by: Jonathan Corbet --- Documentation/translations/zh_CN/vm/index.rst | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/Documentation/translations/zh_CN/vm/index.rst b/Documentation/translations/zh_CN/vm/index.rst index a1c6d529b6ff..e02976f438ab 100644 --- a/Documentation/translations/zh_CN/vm/index.rst +++ b/Documentation/translations/zh_CN/vm/index.rst @@ -12,11 +12,27 @@ Linux内存管理文档 ================= -这是一个关于Linux内存管理（mm）子系统内部的文档集，其中有不同层次的细节，包括注释 -和邮件列表的回复，用于阐述数据结构和算法的基本情况。如果你正在寻找关于简单分配内存的建 -议，请参阅(Documentation/translations/zh_CN/core-api/memory-allocation.rst)。 -对于控制和调整指南，请参阅(Documentation/admin-guide/mm/index)。 -TODO:待引用文档集被翻译完毕后请及时修改此处） +这是一份关于了解Linux的内存管理子系统的指南。如果你正在寻找关于简单分配内存的 +建议，请参阅内存分配指南 +(Documentation/translations/zh_CN/core-api/memory-allocation.rst)。 +关于控制和调整的指南，请看管理指南 +(Documentation/translations/zh_CN/admin-guide/mm/index.rst)。 + + +.. toctree:: + :maxdepth: 1 + + highmem + +该处剩余文档待原始文档有内容后翻译。 + + +遗留文档 +======== + +这是一个关于Linux内存管理（MM）子系统内部的旧文档的集合，其中有不同层次的细节， +包括注释和邮件列表的回复，用于阐述数据结构和算法的描述。它应该被很好地整合到上述 +结构化的文档中，如果它已经完成了它的使命，可以删除。 .. toctree:: :maxdepth: 1 @@ -25,7 +41,6 @@ TODO:待引用文档集被翻译完毕后请及时修改此处） balance damon/index free_page_reporting - highmem ksm frontswap hmm From d218bee86aa570daae9b3c8dc375ac6110813782 Mon Sep 17 00:00:00 2001 From: Chao Liu Date: Mon, 13 Jun 2022 10:08:00 +0800 Subject: [PATCH 121/318] docs: filesystems: f2fs: fix description about compress ioctl Since commit c61404153eb6 ("f2fs: introduce FI_COMPRESS_RELEASED instead of using IMMUTABLE bit"), we no longer use the IMMUTABLE bit to prevent writing data for compression. Let's correct the corresponding documentation. BTW, this patch fixes some alignment issues in the compress metadata layout. Signed-off-by: Chao Liu Link: https://lore.kernel.org/r/20220613020800.3379482-1-chaoliu719@gmail.com Signed-off-by: Jonathan Corbet --- Documentation/filesystems/f2fs.rst | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Documentation/filesystems/f2fs.rst b/Documentation/filesystems/f2fs.rst index ad8dc8c040a2..98dc24f5c6f0 100644 --- a/Documentation/filesystems/f2fs.rst +++ b/Documentation/filesystems/f2fs.rst @@ -818,10 +818,11 @@ Compression implementation Instead, the main goal is to reduce data writes to flash disk as much as possible, resulting in extending disk life time as well as relaxing IO congestion. Alternatively, we've added ioctl(F2FS_IOC_RELEASE_COMPRESS_BLOCKS) - interface to reclaim compressed space and show it to user after putting the - immutable bit. Immutable bit, after release, it doesn't allow writing/mmaping - on the file, until reserving compressed space via - ioctl(F2FS_IOC_RESERVE_COMPRESS_BLOCKS) or truncating filesize to zero. + interface to reclaim compressed space and show it to user after setting a + special flag to the inode. Once the compressed space is released, the flag + will block writing data to the file until either the compressed space is + reserved via ioctl(F2FS_IOC_RESERVE_COMPRESS_BLOCKS) or the file size is + truncated to zero. Compress metadata layout:: @@ -830,12 +831,12 @@ Compress metadata layout:: | cluster 1 | cluster 2 | ......... | cluster N | +-----------------------------------------------+ . . . . - . . . . + . . . . . Compressed Cluster . . Normal Cluster . +----------+---------+---------+---------+ +---------+---------+---------+---------+ |compr flag| block 1 | block 2 | block 3 | | block 1 | block 2 | block 3 | block 4 | +----------+---------+---------+---------+ +---------+---------+---------+---------+ - . . + . . . . . . +-------------+-------------+----------+----------------------------+ From 87444fdce5bb3249f740f5e7f66443169b3003a8 Mon Sep 17 00:00:00 2001 From: Dongliang Mu Date: Mon, 13 Jun 2022 15:12:42 +0800 Subject: [PATCH 122/318] docs: dev-tools: modify SPL reference URL to actual SPL doc entry Fix the invalid url about Semantic Patch Language Signed-off-by: Dongliang Mu Link: https://lore.kernel.org/r/20220613071243.12961-1-dzm91@hust.edu.cn Signed-off-by: Jonathan Corbet --- Documentation/dev-tools/coccinelle.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/dev-tools/coccinelle.rst b/Documentation/dev-tools/coccinelle.rst index 9c454de5a7f7..d9976069ed12 100644 --- a/Documentation/dev-tools/coccinelle.rst +++ b/Documentation/dev-tools/coccinelle.rst @@ -66,7 +66,7 @@ The wiki documentation always refers to the linux-next version of the script. For Semantic Patch Language(SmPL) grammar documentation refer to: -http://coccinelle.lip6.fr/documentation.php +https://coccinelle.gitlabpages.inria.fr/website/docs/main_grammar.html Using Coccinelle on the Linux kernel ------------------------------------ From 1a5b8aa25f153318efecc1fb5b9048a748f49ef2 Mon Sep 17 00:00:00 2001 From: Binbin Zhou Date: Wed, 8 Jun 2022 19:08:20 +0800 Subject: [PATCH 123/318] docs/zh_CN: riscv: Remove the translation of pmu.rst Synchronous translation from the following: [1]: commit 23b1f18326ec("Documentation: riscv: Remove the old documentation") [2]: commit 8933e7f2e375 ("Documentation: riscv: remove non-existent directory from table of contents") Signed-off-by: Binbin Zhou Reviewed-by: Yanteng Si Reviewed-by: Wu XiangCheng Link: https://lore.kernel.org/r/6fe45d69210300a6c065262470bce963f8dc0ec3.1654685338.git.zhoubinbin@loongson.cn Signed-off-by: Jonathan Corbet --- .../translations/zh_CN/riscv/index.rst | 1 - .../translations/zh_CN/riscv/pmu.rst | 235 ------------------ 2 files changed, 236 deletions(-) delete mode 100644 Documentation/translations/zh_CN/riscv/pmu.rst diff --git a/Documentation/translations/zh_CN/riscv/index.rst b/Documentation/translations/zh_CN/riscv/index.rst index 614cde0c0997..131e405aa857 100644 --- a/Documentation/translations/zh_CN/riscv/index.rst +++ b/Documentation/translations/zh_CN/riscv/index.rst @@ -19,7 +19,6 @@ RISC-V 体系结构 boot-image-header vm-layout - pmu patch-acceptance diff --git a/Documentation/translations/zh_CN/riscv/pmu.rst b/Documentation/translations/zh_CN/riscv/pmu.rst deleted file mode 100644 index 7ec801026c4d..000000000000 --- a/Documentation/translations/zh_CN/riscv/pmu.rst +++ /dev/null @@ -1,235 +0,0 @@ -.. include:: ../disclaimer-zh_CN.rst - -:Original: Documentation/riscv/pmu.rst - -:翻译: - - 司延腾 Yanteng Si - -.. _cn_riscv_pmu: - -======================== -RISC-V平台上对PMUs的支持 -======================== - -Alan Kao , Mar 2018 - -简介 ------------- - -截止本文撰写时，在The RISC-V ISA Privileged Version 1.10中提到的 perf_event -相关特性如下: -（详情请查阅手册） - -* [m|s]counteren -* mcycle[h], cycle[h] -* minstret[h], instret[h] -* mhpeventx, mhpcounterx[h] - -仅有以上这些功能，移植perf需要做很多工作，究其原因是缺少以下通用架构的性能 -监测特性: - -* 启用/停用计数器 - 在我们这里，计数器一直在自由运行。 -* 计数器溢出引起的中断 - 规范中没有这种功能。 -* 中断指示器 - 不可能所有的计数器都有很多的中断端口，所以需要一个中断指示器让软件来判断 - 哪个计数器刚好溢出。 -* 写入计数器 - 由于内核不能修改计数器，所以会有一个SBI来支持这个功能[1]。 另外，一些厂商 - 考虑实现M-S-U型号机器的硬件扩展来直接写入计数器。 - -这篇文档旨在为开发者提供一个在内核中支持PMU的简要指南。下面的章节简要解释了 -perf' 机制和待办事项。 - -你可以在这里查看以前的讨论[1][2]。 另外，查看附录中的相关内核结构体可能会有 -帮助。 - - -1. 初始化 ---------- - -*riscv_pmu* 是一个类型为 *struct riscv_pmu* 的全局指针，它包含了根据perf内部 -约定的各种方法和PMU-specific参数。人们应该声明这样的实例来代表PMU。 默认情况 -下， *riscv_pmu* 指向一个常量结构体 *riscv_base_pmu* ，它对基准QEMU模型有非常 -基础的支持。 - - -然后他/她可以将实例的指针分配给 *riscv_pmu* ，这样就可以利用已经实现的最小逻 -辑，或者创建他/她自己的 *riscv_init_platform_pmu* 实现。 - -换句话说，现有的 *riscv_base_pmu* 源只是提供了一个参考实现。 开发者可以灵活地 -决定多少部分可用，在最极端的情况下，他们可以根据自己的需要定制每一个函数。 - - -2. Event Initialization ------------------------ - -当用户启动perf命令来监控一些事件时，首先会被用户空间的perf工具解释为多个 -*perf_event_open* 系统调用，然后进一步调用上一步分配的 *event_init* 成员函数 -的主体。 在 *riscv_base_pmu* 的情况下，就是 *riscv_event_init* 。 - -该功能的主要目的是将用户提供的事件翻译成映射图，从而可以直接对HW-related的控 -制寄存器或计数器进行操作。该翻译基于 *riscv_pmu* 中提供的映射和方法。 - -注意，有些功能也可以在这个阶段完成: - -(1) 中断设置，这个在下一节说； -(2) 特限级设置(仅用户空间、仅内核空间、两者都有)； -(3) 析构函数设置。 通常应用 *riscv_destroy_event* 即可； -(4) 对非采样事件的调整，这将被函数应用，如 *perf_adjust_period* ，通常如下:: - - if (!is_sampling_event(event)) { - hwc->sample_period = x86_pmu.max_period; - hwc->last_period = hwc->sample_period; - local64_set(&hwc->period_left, hwc->sample_period); - } - - -在 *riscv_base_pmu* 的情况下，目前只提供了（3）。 - - -3. 中断 -------- - -3.1. 中断初始化 - -这种情况经常出现在 *event_init* 方案的开头。通常情况下，这应该是一个代码段，如:: - - int x86_reserve_hardware(void) - { - int err = 0; - - if (!atomic_inc_not_zero(&pmc_refcount)) { - mutex_lock(&pmc_reserve_mutex); - if (atomic_read(&pmc_refcount) == 0) { - if (!reserve_pmc_hardware()) - err = -EBUSY; - else - reserve_ds_buffers(); - } - if (!err) - atomic_inc(&pmc_refcount); - mutex_unlock(&pmc_reserve_mutex); - } - - return err; - } - -而神奇的是 *reserve_pmc_hardware* ，它通常做原子操作，使实现的IRQ可以从某个全局函 -数指针访问。 而 *release_pmc_hardware* 的作用正好相反，它用在上一节提到的事件分配 -器中。 - - (注：从所有架构的实现来看，*reserve/release* 对总是IRQ设置，所以 *pmc_hardware* - 似乎有些误导。 它并不处理事件和物理计数器之间的绑定，这一点将在下一节介绍。) - -3.2. IRQ结构体 - -基本上，一个IRQ运行以下伪代码:: - - for each hardware counter that triggered this overflow - - get the event of this counter - - // following two steps are defined as *read()*, - // check the section Reading/Writing Counters for details. - count the delta value since previous interrupt - update the event->count (# event occurs) by adding delta, and - event->hw.period_left by subtracting delta - - if the event overflows - sample data - set the counter appropriately for the next overflow - - if the event overflows again - too frequently, throttle this event - fi - fi - - end for - - 然而截至目前，没有一个RISC-V的实现为perf设计了中断，所以具体的实现要在未来完成。 - -4. Reading/Writing 计数 ------------------------ - -它们看似差不多，但perf对待它们的态度却截然不同。 对于读，在 *struct pmu* 中有一个 -*read* 接口，但它的作用不仅仅是读。 根据上下文，*read* 函数不仅要读取计数器的内容 -（event->count），还要更新左周期到下一个中断（event->hw.period_left）。 - - 但 perf 的核心不需要直接写计数器。 写计数器隐藏在以下两点的抽象化之后， - 1） *pmu->start* ，从字面上看就是开始计数，所以必须把计数器设置成一个合适的值，以 - 便下一次中断； - 2）在IRQ里面，应该把计数器设置成同样的合理值。 - -在RISC-V中，读操作不是问题，但写操作就需要费些力气了，因为S模式不允许写计数器。 - - -5. add()/del()/start()/stop() ------------------------------ - -基本思想: add()/del() 向PMU添加/删除事件，start()/stop() 启动/停止PMU中某个事件 -的计数器。 所有这些函数都使用相同的参数: *struct perf_event *event* 和 *int flag* 。 - -把 perf 看作一个状态机，那么你会发现这些函数作为这些状态之间的状态转换过程。 -定义了三种状态（event->hw.state）: - -* PERF_HES_STOPPED: 计数停止 -* PERF_HES_UPTODATE: event->count是最新的 -* PERF_HES_ARCH: 依赖于体系结构的用法，。。。我们现在并不需要它。 - -这些状态转换的正常流程如下: - -* 用户启动一个 perf 事件，导致调用 *event_init* 。 -* 当被上下文切换进来的时候，*add* 会被 perf core 调用，并带有一个标志 PERF_EF_START， - 也就是说事件被添加后应该被启动。 在这个阶段，如果有的话，一般事件会被绑定到一个物 - 理计数器上。当状态变为PERF_HES_STOPPED和PERF_HES_UPTODATE，因为现在已经停止了, - （软件）事件计数不需要更新。 - - - 然后调用 *start* ，并启用计数器。 - 通过PERF_EF_RELOAD标志，它向计数器写入一个适当的值（详细情况请参考上一节）。 - 如果标志不包含PERF_EF_RELOAD，则不会写入任何内容。 - 现在状态被重置为none，因为它既没有停止也没有更新（计数已经开始）。 - -*当被上下文切换出来时被调用。 然后，它检查出PMU中的所有事件，并调用 *stop* 来更新它们 - 的计数。 - - - *stop* 被 *del* 和perf核心调用，标志为PERF_EF_UPDATE，它经常以相同的逻辑和 *read* - 共用同一个子程序。 - 状态又一次变为PERF_HES_STOPPED和PERF_HES_UPTODATE。 - - - 这两对程序的生命周期: *add* 和 *del* 在任务切换时被反复调用；*start* 和 *stop* 在 - perf核心需要快速停止和启动时也会被调用，比如在调整中断周期时。 - -目前的实现已经足够了，将来可以很容易地扩展到功能。 - -A. 相关结构体 -------------- - -* struct pmu: include/linux/perf_event.h -* struct riscv_pmu: arch/riscv/include/asm/perf_event.h - - 两个结构体都被设计为只读。 - - *struct pmu* 定义了一些函数指针接口，它们大多以 *struct perf_event* 作为主参数，根据 - perf的内部状态机处理perf事件（详情请查看kernel/events/core.c）。 - - *struct riscv_pmu* 定义了PMU的具体参数。 命名遵循所有其它架构的惯例。 - -* struct perf_event: include/linux/perf_event.h -* struct hw_perf_event - - 表示 perf 事件的通用结构体，以及硬件相关的细节。 - -* struct riscv_hw_events: arch/riscv/include/asm/perf_event.h - - 保存事件状态的结构有两个固定成员。 - 事件的数量和事件的数组。 - -参考文献 --------- - -[1] https://github.com/riscv/riscv-linux/pull/124 - -[2] https://groups.google.com/a/groups.riscv.org/forum/#!topic/sw-dev/f19TmCNP6yA From f41c74af10583fa440deed2713488176e7efa8b7 Mon Sep 17 00:00:00 2001 From: Binbin Zhou Date: Wed, 8 Jun 2022 19:08:21 +0800 Subject: [PATCH 124/318] docs/zh_CN: riscv: Update the translation of vm-layout.rst to 5.19-rc1 Synchronous translation from upstream commit 9d19f2b5256e ("Documentation: riscv: Add sv48 description to VM layout") Signed-off-by: Binbin Zhou Reviewed-by: Yanteng Si Reviewed-by: Wu XiangCheng Link: https://lore.kernel.org/r/10cd6a241a0266d1d19ee5f0b2177d7d14d17ed2.1654685338.git.zhoubinbin@loongson.cn Signed-off-by: Jonathan Corbet --- .../translations/zh_CN/riscv/vm-layout.rst | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/Documentation/translations/zh_CN/riscv/vm-layout.rst b/Documentation/translations/zh_CN/riscv/vm-layout.rst index 585cb89317a3..91884e2dfff8 100644 --- a/Documentation/translations/zh_CN/riscv/vm-layout.rst +++ b/Documentation/translations/zh_CN/riscv/vm-layout.rst @@ -6,6 +6,7 @@ :翻译: 司延腾 Yanteng Si + Binbin Zhou ============================ RISC-V Linux上的虚拟内存布局 @@ -65,3 +66,39 @@ RISC-V Linux Kernel SV39 ffffffff00000000 | -4 GB | ffffffff7fffffff | 2 GB | modules, BPF ffffffff80000000 | -2 GB | ffffffffffffffff | 2 GB | kernel __________________|____________|__________________|_________|____________________________________________________________ + + +RISC-V Linux Kernel SV48 +------------------------ + +:: + + ======================================================================================================================== + 开始地址 | 偏移 | 结束地址 | 大小 | 虚拟内存区域描述 + ======================================================================================================================== + | | | | + 0000000000000000 | 0 | 00007fffffffffff | 128 TB | 用户空间虚拟内存，每个内存管理器不同 + __________________|____________|__________________|_________|___________________________________________________________ + | | | | + 0000800000000000 | +128 TB | ffff7fffffffffff | ~16M TB | ... 巨大的、几乎64位宽的直到内核映射的-128TB地方 + | | | | 开始偏移的非经典虚拟内存地址空洞。 + | | | | + __________________|____________|__________________|_________|___________________________________________________________ + | + | 内核空间的虚拟内存，在所有进程之间共享: + ____________________________________________________________|___________________________________________________________ + | | | | + ffff8d7ffee00000 | -114.5 TB | ffff8d7ffeffffff | 2 MB | fixmap + ffff8d7fff000000 | -114.5 TB | ffff8d7fffffffff | 16 MB | PCI io + ffff8d8000000000 | -114.5 TB | ffff8f7fffffffff | 2 TB | vmemmap + ffff8f8000000000 | -112.5 TB | ffffaf7fffffffff | 32 TB | vmalloc/ioremap space + ffffaf8000000000 | -80.5 TB | ffffef7fffffffff | 64 TB | 直接映射所有物理内存 + ffffef8000000000 | -16.5 TB | fffffffeffffffff | 16.5 TB | kasan + __________________|____________|__________________|_________|____________________________________________________________ + | + | 从此处开始，与39-bit布局相同： + ____________________________________________________________|____________________________________________________________ + | | | | + ffffffff00000000 | -4 GB | ffffffff7fffffff | 2 GB | modules, BPF + ffffffff80000000 | -2 GB | ffffffffffffffff | 2 GB | kernel + __________________|____________|__________________|_________|____________________________________________________________ From 3f39b3d70fcdea68e8e9c7b9d1e763fe90418ca0 Mon Sep 17 00:00:00 2001 From: Yanteng Si Date: Tue, 7 Jun 2022 15:13:00 +0800 Subject: [PATCH 125/318] docs/zh_CN: add vm page_migration translation Translate .../vm/page_migration.rst into Chinese. Signed-off-by: Yanteng Si Reviewed-by: Wu XiangCheng Link: https://lore.kernel.org/r/44bc4486e5019f689bcbec20a629a9f7800ece33.1654585011.git.siyanteng@loongson.cn Signed-off-by: Jonathan Corbet --- Documentation/translations/zh_CN/vm/index.rst | 2 +- .../translations/zh_CN/vm/page_migration.rst | 228 ++++++++++++++++++ 2 files changed, 229 insertions(+), 1 deletion(-) create mode 100644 Documentation/translations/zh_CN/vm/page_migration.rst diff --git a/Documentation/translations/zh_CN/vm/index.rst b/Documentation/translations/zh_CN/vm/index.rst index e02976f438ab..07d4bdf0c629 100644 --- a/Documentation/translations/zh_CN/vm/index.rst +++ b/Documentation/translations/zh_CN/vm/index.rst @@ -51,6 +51,7 @@ Linux内存管理文档 numa overcommit-accounting page_frags + page_migration page_owner page_table_check remap_file_pages @@ -62,7 +63,6 @@ TODOLIST: * arch_pgtable_helpers * free_page_reporting * hugetlbfs_reserv -* page_migration * slub * transhuge * unevictable-lru diff --git a/Documentation/translations/zh_CN/vm/page_migration.rst b/Documentation/translations/zh_CN/vm/page_migration.rst new file mode 100644 index 000000000000..566880a41ea0 --- /dev/null +++ b/Documentation/translations/zh_CN/vm/page_migration.rst @@ -0,0 +1,228 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/vm/index.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + +======== +页面迁移 +======== + +页面迁移允许在进程运行时在NUMA系统的节点之间移动页面的物理位置。这意味着进程所看到的虚拟地 +址并没有改变。然而，系统会重新安排这些页面的物理位置。 + +也可以参见 :ref: `<异构内存管理 (HMM)>` 以了解将页面迁移到设备私有内存或从设备私有内存中迁移。 + +页面迁移的主要目的是通过将页面移到访问该内存的进程所运行的处理器附近来减少内存访问的延迟。 + +页面迁移允许进程通过MF_MOVE和MF_MOVE_ALL选项手动重新定位其页面所在的节点，同时通过 +mbind()设置一个新的内存策略。一个进程的页面也可以通过sys_migrate_pages()函数调用从另 +一个进程重新定位。migrate_pages()函数调用接收两组节点，并将一个进程位于旧节点上的页面移 +动到目标节点上。页面迁移功能由Andi Kleen的numactl包提供(需要0.9.3以上的版本，其仓库 +地址https://github.com/numactl/numactl.git)。numactl提供了libnuma，它为页面迁移 +提供了与其他NUMA功能类似的接口。执行 cat ``/proc//numa_maps`` 允许轻松查看进 +程的页面位置。参见proc(5)手册中的numa_maps文档。 + +如果调度程序将一个进程重新安置到一个遥远的节点上的处理器，手动迁移是很有用的。批量调度程序 +或管理员可以检测到这种情况，并将进程的页面移到新处理器附近。内核本身只提供手动的页迁移支持。 +自动的页面迁移可以通过用户空间的进程移动页面来实现。一个特殊的函数调用 "move_pages" 允许 +在一个进程中移动单个页面。例如，NUMA分析器可以获得一个显示频繁的节点外访问的日志，并可以使 +用这个结果将页面移动到更有利的位置。 + +较大型的设备通常使用cpusets将系统分割成若干个节点。Paul Jackson为cpusets配备了当任务被 +转移到另一个cpuset时移动页面的能力(见:ref:`CPUSETS `)。Cpusets允许进程定 +位的自动化。如果一个任务被移到一个新的cpuset上，那么它的所有页面也会随之移动，这样进程的 +性能就不会急剧下降。如果cpuset允许的内存节点发生变化，cpuset中的进程页也会被移动。 + +页面迁移允许为所有迁移技术保留一组节点中页面的相对位置，这将保留生成的特定内存分配模式即使 +进程已被迁移。为了保留内存延迟，这一点是必要的。迁移后的进程将以类似的性能运行。 + +页面迁移分几个步骤进行。首先为那些试图从内核中使用migrate_pages()的进程做一个高层次的 +描述（对于用户空间的使用，可以参考上面提到的Andi Kleen的numactl包），然后对低水平的细 +节工作做一个低水平描述。 + +在内核中使用 migrate_pages() +============================ + +1. 从LRU中移除页面。 + + 要迁移的页面列表是通过扫描页面并把它们移到列表中来生成的。这是通过调用 isolate_lru_page() + 来完成的。调用isolate_lru_page()增加了对该页的引用，这样在页面迁移发生时它就不会 + 消失。它还可以防止交换器或其他扫描器遇到该页。 + + +2. 我们需要有一个new_page_t类型的函数，可以传递给migrate_pages()。这个函数应该计算 + 出如何在给定的旧页面中分配正确的新页面。 + +3. migrate_pages()函数被调用，它试图进行迁移。它将调用该函数为每个被考虑迁移的页面分 + 配新的页面。 + +migrate_pages()如何工作 +======================= + +migrate_pages()对它的页面列表进行了多次处理。如果当时对一个页面的所有引用都可以被移除， +那么这个页面就会被移动。该页已经通过isolate_lru_page()从LRU中移除，并且refcount被 +增加，以便在页面迁移发生时不释放该页。 + +步骤: + +1. 锁定要迁移的页面。 + +2. 确保回写已经完成。 + +3. 锁定我们要迁移到的新页面。锁定它是为了在迁移过程中立即阻止对这个（尚未更新的）页面的 + 访问。 + +4. 所有对该页的页表引用都被转换为迁移条目。这就减少了一个页面的mapcount。如果产生的 + mapcount不是零，那么我们就不迁移该页。所有试图访问该页的用户空间进程现在将等待页 + 面锁或者等待迁移页表项被移除。 + +5. i_pages的锁被持有。这将导致所有试图通过映射访问该页的进程在自旋锁上阻塞。 + +6. 检查该页的Refcount，如果还有引用，我们就退出。否则，我们知道我们是唯一引用这个页 + 面的人。 + +7. 检查基数树，如果它不包含指向这个页面的指针，那么我们就退出，因为其他人修改了基数树。 + +8. 新的页面要用旧的页面的一些设置进行预处理，这样访问新的页面就会发现一个具有正确设置 + 的页面。 + +9. 基数树被改变以指向新的页面。 + +10. 旧页的引用计数被删除，因为地址空间的引用已经消失。对新页的引用被建立，因为新页被 + 地址空间引用。 + +11. i_pages锁被放弃。这样一来，在映射中的查找又变得可能了。进程将从在锁上自旋到在 + 被锁的新页上睡眠。 + +12. 页面内容被复制到新的页面上。 + +13. 剩余的页面标志被复制到新的页面上。 + +14. 旧的页面标志被清除，以表明该页面不再提供任何信息。 + +15. 新页面上的回写队列被触发了。 + +16. 如果迁移条目被插入到页表中，那么就用真正的ptes替换它们。这样做将使那些尚未等待页 + 锁的用户空间进程能够访问。 + +17. 页面锁从新旧页面上被撤销。等待页锁的进程将重做他们的缺页异常，并将到达新的页面。 + +18. 新的页面被移到LRU中，可以被交换器等再次扫描。 + +非LRU页面迁移 +============= + +尽管迁移最初的目的是为了减少NUMA的内存访问延迟，但压缩也使用迁移来创建高阶页面。 + +目前实现的问题是，它被设计为只迁移*LRU*页。然而，有一些潜在的非LRU页面可以在驱动中 +被迁移，例如，zsmalloc，virtio-balloon页面。 + +对于virtio-balloon页面，迁移代码路径的某些部分已经被钩住，并添加了virtio-balloon +的特定函数来拦截迁移逻辑。这对一个驱动来说太特殊了，所以其他想让自己的页面可移动的驱 +动就必须在迁移路径中添加自己的特定钩子。 + +为了克服这个问题，VM支持非LRU页面迁移，它为非LRU可移动页面提供了通用函数，而在迁移 +路径中没有特定的驱动程序钩子。 + +如果一个驱动程序想让它的页面可移动，它应该定义三个函数，这些函数是 +struct address_space_operations的函数指针。 + +1. ``bool (*isolate_page) (struct page *page, isolate_mode_t mode);`` + + VM对驱动的isolate_page()函数的期望是，如果驱动成功隔离了该页，则返回*true*。 + 返回true后，VM会将该页标记为PG_isolated，这样多个CPU的并发隔离就会跳过该 + 页进行隔离。如果驱动程序不能隔离该页，它应该返回*false*。 + + 一旦页面被成功隔离，VM就会使用page.lru字段，因此驱动程序不应期望保留这些字段的值。 + +2. ``int (*migratepage) (struct address_space *mapping,`` +| ``struct page *newpage, struct page *oldpage, enum migrate_mode);`` + + 隔离后，虚拟机用隔离的页面调用驱动的migratepage()。migratepage()的功能是将旧页 + 的内容移动到新页，并设置struct page newpage的字段。请记住，如果你成功迁移了旧页 + 并返回MIGRATEPAGE_SUCCESS，你应该通过page_lock下的__ClearPageMovable()向虚 + 拟机表明旧页不再可移动。如果驱动暂时不能迁移该页，驱动可以返回-EAGAIN。在-EAGAIN + 时，VM会在短时间内重试页面迁移，因为VM将-EAGAIN理解为 "临时迁移失败"。在返回除 + -EAGAIN以外的任何错误时，VM将放弃页面迁移而不重试。 + + 在migratepage()函数中，驱动程序不应该接触page.lru字段。 + +3. ``void (*putback_page)(struct page *);`` + + 如果在隔离页上迁移失败，VM应该将隔离页返回给驱动，因此VM用隔离页调用驱动的 + putback_page()。在这个函数中，驱动应该把隔离页放回自己的数据结构中。 + +非LRU可移动页标志 + + 有两个页面标志用于支持非LRU可移动页面。 + + * PG_movable + + 驱动应该使用下面的函数来使页面在page_lock下可移动。:: + + void __SetPageMovable(struct page *page, struct address_space *mapping) + + 它需要address_space的参数来注册将被VM调用的migration family函数。确切地说， + PG_movable不是struct page的一个真正的标志。相反，VM复用了page->mapping的低 + 位来表示它:: + + #define PAGE_MAPPING_MOVABLE 0x2 + page->mapping = page->mapping | PAGE_MAPPING_MOVABLE; + + 所以驱动不应该直接访问page->mapping。相反，驱动应该使用page_mapping()，它可 + 以在页面锁下屏蔽掉page->mapping的低2位，从而获得正确的struct address_space。 + + 对于非LRU可移动页面的测试，VM支持__PageMovable()函数。然而，它并不能保证识别 + 非LRU可移动页面，因为page->mapping字段与struct page中的其他变量是统一的。如 + 果驱动程序在被虚拟机隔离后释放了页面，尽管page->mapping设置了PAGE_MAPPING_MOVABLE， + 但它并没有一个稳定的值（看看__ClearPageMovable）。但是__PageMovable()在页 + 面被隔离后，无论页面是LRU还是非LRU可移动的，调用它开销都很低，因为LRU页面在 + page->mapping中不可能有PAGE_MAPPING_MOVABLE设置。在用pfn扫描中的lock_page() + 进行更大开销的检查来选择受害者之前，它也很适合只是瞥一眼来测试非LRU可移动的页面。 + + 为了保证非LRU的可移动页面，VM提供了PageMovable()函数。与__PageMovable()不 + 同，PageMovable()在lock_page()下验证page->mapping和 + mapping->a_ops->isolate_page。lock_page()可以防止突然破坏page->mapping。 + + 使用__SetPageMovable()的驱动应该在释放页面之前通过page_lock()下的 + __ClearMovablePage()清除该标志。 + + * PG_isolated + + 为了防止几个CPU同时进行隔离，VM在lock_page()下将隔离的页面标记为PG_isolated。 + 因此，如果一个CPU遇到PG_isolated非LRU可移动页面，它可以跳过它。驱动程序不需要 + 操作这个标志，因为VM会自动设置/清除它。请记住，如果驱动程序看到PG_isolated页， + 这意味着该页已经被VM隔离，所以它不应该碰page.lru字段。PG_isolated标志与 + PG_reclaim标志是同义的，所以驱动程序不应该为自己的目的使用PG_isolated。 + +监测迁移 +======== + +以下事件（计数器）可用于监控页面迁移。 + +1. PGMIGRATE_SUCCESS: 正常的页面迁移成功。每个计数器意味着一个页面被迁移了。如果该 + 页是一个非THP和非hugetlb页，那么这个计数器会增加1。如果该页面是一个THP或hugetlb + 页面，那么这个计数器会随着THP或hugetlb子页面的数量而增加。例如，迁移一个有4KB大小 + 的基础页（子页）的2MB THP，将导致这个计数器增加512。 + +2. PGMIGRATE_FAIL: 正常的页面迁移失败。与上面PGMIGRATE_SUCCESS的计数规则相同：如 + 果是THP或hugetlb，这个计数将被子页的数量增加。 + +3. THP_MIGRATION_SUCCESS: 一个THP被迁移而没有被分割。 + +4. THP_MIGRATION_FAIL: 一个THP不能被迁移，也不能被分割。 + +5. THP_MIGRATION_SPLIT: 一个THP被迁移了，但不是这样的：首先，这个THP必须被分割。 + 在拆分之后，对它的子页面进行了迁移重试。 + +THP_MIGRATION_* 事件也会更新相应的PGMIGRATE_SUCCESS或PGMIGRATE_FAIL事件。 +例如，一个THP迁移失败将导致THP_MIGRATION_FAIL和PGMIGRATE_FAIL增加。 + +Christoph Lameter，2006年5月8日。 + +Minchan Kim，2016年3月28日。 From 17c260504f6fb3d7da096fdad1774ac262aaac7d Mon Sep 17 00:00:00 2001 From: Yanteng Si Date: Tue, 7 Jun 2022 15:13:18 +0800 Subject: [PATCH 126/318] docs/zh_CN: add vm vmalloced-kernel-stacks translation Translate .../vm/vmalloced-kernel-stacks.rst into Chinese. Signed-off-by: Yanteng Si Reviewed-by: Wu XiangCheng Link: https://lore.kernel.org/r/01de54c7ecfd1999a636ee59ac40b79917251af2.1654585011.git.siyanteng@loongson.cn Signed-off-by: Jonathan Corbet --- Documentation/translations/zh_CN/vm/index.rst | 2 +- .../zh_CN/vm/vmalloced-kernel-stacks.rst | 133 ++++++++++++++++++ 2 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 Documentation/translations/zh_CN/vm/vmalloced-kernel-stacks.rst diff --git a/Documentation/translations/zh_CN/vm/index.rst b/Documentation/translations/zh_CN/vm/index.rst index 07d4bdf0c629..c77a56553845 100644 --- a/Documentation/translations/zh_CN/vm/index.rst +++ b/Documentation/translations/zh_CN/vm/index.rst @@ -56,6 +56,7 @@ Linux内存管理文档 page_table_check remap_file_pages split_page_table_lock + vmalloced-kernel-stacks z3fold zsmalloc @@ -66,4 +67,3 @@ TODOLIST: * slub * transhuge * unevictable-lru -* vmalloced-kernel-stacks diff --git a/Documentation/translations/zh_CN/vm/vmalloced-kernel-stacks.rst b/Documentation/translations/zh_CN/vm/vmalloced-kernel-stacks.rst new file mode 100644 index 000000000000..ad23f274f6d7 --- /dev/null +++ b/Documentation/translations/zh_CN/vm/vmalloced-kernel-stacks.rst @@ -0,0 +1,133 @@ +.. SPDX-License-Identifier: GPL-2.0 +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/vm/vmalloced-kernel-stacks.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + +==================== +支持虚拟映射的内核栈 +==================== + +:作者: Shuah Khan + +.. contents:: :local: + +概览 +---- + +这是介绍 `虚拟映射内核栈功能 ` 的代码 +和原始补丁系列的信息汇总。 + +简介 +---- + +内核堆栈溢出通常难以调试，并使内核容易被（恶意）利用。问题可能在稍后的时间出现，使其难以 +隔离和究其根本原因。 + +带有保护页的虚拟映射内核堆栈如果溢出，会被立即捕获，而不会放任其导致难以诊断的损 +坏。 + +HAVE_ARCH_VMAP_STACK和VMAP_STACK配置选项能够支持带有保护页的虚拟映射堆栈。 +当堆栈溢出时，这个特性会引发可靠的异常。溢出后堆栈跟踪的可用性以及对溢出本身的 +响应取决于架构。 + +.. note:: + 截至本文撰写时， arm64, powerpc, riscv, s390, um, 和 x86 支持VMAP_STACK。 + +HAVE_ARCH_VMAP_STACK +-------------------- + +能够支持虚拟映射内核栈的架构应该启用这个bool配置选项。要求是: + +- vmalloc空间必须大到足以容纳许多内核堆栈。这可能排除了许多32位架构。 +- vmalloc空间的堆栈需要可靠地工作。例如，如果vmap页表是按需创建的，当堆栈指向 + 具有未填充页表的虚拟地址时，这种机制需要工作，或者架构代码（switch_to()和 + switch_mm()，很可能）需要确保堆栈的页表项在可能未填充的堆栈上运行之前已经填 + 充。 +- 如果堆栈溢出到一个保护页，就应该发生一些合理的事情。“合理”的定义是灵活的，但 + 在没有记录任何东西的情况下立即重启是不友好的。 + +VMAP_STACK +---------- + +VMAP_STACK bool配置选项在启用时分配虚拟映射的任务栈。这个选项依赖于 +HAVE_ARCH_VMAP_STACK。 + +- 如果你想使用带有保护页的虚拟映射的内核堆栈，请启用该选项。这将导致内核栈溢出 + 被立即捕获，而不是难以诊断的损坏。 + +.. note:: + + 使用KASAN的这个功能需要架构支持用真实的影子内存来支持虚拟映射，并且 + 必须启用KASAN_VMALLOC。 + +.. note:: + + 启用VMAP_STACK时，无法在堆栈分配的数据上运行DMA。 + +内核配置选项和依赖性不断变化。请参考最新的代码库: + +`Kconfig ` + +分配方法 +-------- + +当一个新的内核线程被创建时，线程堆栈是由页级分配器分配的虚拟连续的内存页组成。这 +些页面被映射到有PAGE_KERNEL保护的连续的内核虚拟空间。 + +alloc_thread_stack_node()调用__vmalloc_node_range()来分配带有PAGE_KERNEL +保护的栈。 + +- 分配的堆栈被缓存起来，以后会被新的线程重用，所以在分配/释放堆栈给任务时，要手动 + 进行memcg核算。因此，__vmalloc_node_range被调用时没有__GFP_ACCOUNT。 +- vm_struct被缓存起来，以便能够找到在中断上下文中启动的空闲线程。 free_thread_stack() + 可以在中断上下文中调用。 +- 在arm64上，所有VMAP的堆栈都需要有相同的对齐方式，以确保VMAP的堆栈溢出检测正常 + 工作。架构特定的vmap堆栈分配器照顾到了这个细节。 +- 这并不涉及中断堆栈--参考原始补丁 + +线程栈分配是由clone()、fork()、vfork()、kernel_thread()通过kernel_clone() +启动的。留点提示在这，以便搜索代码库，了解线程栈何时以及如何分配。 + +大量的代码是在: +`kernel/fork.c `. + +task_struct中的stack_vm_area指针可以跟踪虚拟分配的堆栈，一个非空的stack_vm_area +指针可以表明虚拟映射的内核堆栈已经启用。 + +:: + + struct vm_struct *stack_vm_area; + +堆栈溢出处理 +------------ + +前守护页和后守护页有助于检测堆栈溢出。当堆栈溢出到守护页时，处理程序必须小心不要再 +次溢出堆栈。当处理程序被调用时，很可能只留下很少的堆栈空间。 + +在x86上，这是通过处理表明内核堆栈溢出的双异常堆栈的缺页异常来实现的。 + +用守护页测试VMAP分配 +-------------------- + +我们如何确保VMAP_STACK在分配时确实有前守护页和后守护页的保护？下面的 lkdtm 测试 +可以帮助检测任何回归。 + +:: + + void lkdtm_STACK_GUARD_PAGE_LEADING() + void lkdtm_STACK_GUARD_PAGE_TRAILING() + +结论 +---- + +- vmalloced堆栈的percpu缓存似乎比高阶堆栈分配要快一些，至少在缓存命中时是这样。 +- THREAD_INFO_IN_TASK完全摆脱了arch-specific thread_info，并简单地将 + thread_info（仅包含标志）和'int cpu'嵌入task_struct中。 +- 一旦任务死亡，线程栈就可以被释放（无需等待RCU），然后，如果使用vmapped栈，就 + 可以将整个栈缓存起来，以便在同一cpu上重复使用。 From 6de8d58cf98dd93f054291e9ac354d4becfc8c7e Mon Sep 17 00:00:00 2001 From: Wu XiangCheng Date: Tue, 7 Jun 2022 01:14:33 +0800 Subject: [PATCH 127/318] docs/zh_CN: Update zh_CN/kernel-hacking/hacking.rst to 5.19-rc1 * update to commit f35cf1a59e9a ("Documentation: kernel-hacking: minor edits for style") * fix a homophone typo reported by Li Feng Reported-by: Li Feng Signed-off-by: Wu XiangCheng Reviewed-by: Yanteng Si Link: https://lore.kernel.org/r/Yp41+eTjoPRa4hrl@bobwxc.mipc Signed-off-by: Jonathan Corbet --- .../zh_CN/kernel-hacking/hacking.rst | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Documentation/translations/zh_CN/kernel-hacking/hacking.rst b/Documentation/translations/zh_CN/kernel-hacking/hacking.rst index f2bc154c5bcc..bda79646bb1e 100644 --- a/Documentation/translations/zh_CN/kernel-hacking/hacking.rst +++ b/Documentation/translations/zh_CN/kernel-hacking/hacking.rst @@ -81,7 +81,7 @@ 过硬件中断）的“软件中断”将运行（ ``kernel/softirq.c`` ）。 此处完成了许多真正的中断处理工作。在向SMP过渡的早期，只有“bottom halves下半 -部”（BHs）机制，无法利用多个CPU的优势。在从那些一团糟的就电脑切换过来后不久， +部”（BHs）机制，无法利用多个CPU的优势。在从那些一团糟的旧电脑切换过来后不久， 我们放弃了这个限制，转而使用“软中断”。 ``include/linux/interrupt.h`` 列出了不同的软中断。定时器软中断是一个非常重要 @@ -95,8 +95,7 @@ .. warning:: - “tasklet”这个名字是误导性的：它们与“任务”无关，可能更多与当时 - 阿列克谢·库兹涅佐夫享用的糟糕伏特加有关。 + “tasklet”这个名字是误导性的：它们与“任务”无关。 你可以使用 :c:func:`in_softirq()` 宏（ ``include/linux/preempt.h`` ）来确认 是否处于软中断（或子任务）中。 @@ -247,7 +246,7 @@ Provide mechanism not policy”。 与 :c:func:`put_user()` 和 :c:func:`get_user()` 不同，它们返回未复制的 数据量（即0仍然意味着成功）。 -【是的，这个愚蠢的接口真心让我尴尬。火爆的口水仗大概每年都会发生。 +【是的，这个讨厌的接口真心让我尴尬。火爆的口水仗大概每年都会发生。 —— Rusty Russell】 这些函数可以隐式睡眠。它不应该在用户上下文之外调用（没有意义）、调用时禁用中断 @@ -538,9 +537,9 @@ Documentation/core-api/symbol-namespaces.rst 。 Linus和其他开发人员有时会更改开发内核中的函数或结构体名称；这样做不仅是为了 让每个人都保持警惕，还反映了一个重大的更改（例如，不能再在打开中断的情况下 -调用，或者执行额外的检查，或者不执行以前捕获的检查）。通常这会附带一个linux -内核邮件列表中相当全面的注释；请搜索存档以查看。简单地对文件进行全局替换通常 -会让事情变得 **更糟** 。 +调用，或者执行额外的检查，或者不执行以前捕获的检查）。通常这会附带发送一个 +相当全面的注释到相应的内核邮件列表中；请搜索存档以查看。简单地对文件进行全局 +替换通常只会让事情变得 **更糟** 。 初始化结构体成员 ------------------ @@ -610,7 +609,7 @@ C++ 为了让你的东西更正式、补丁更整洁，还有一些工作要做： -- 搞清楚你在谁的地界儿上干活。查看源文件的顶部、 ``MAINTAINERS`` 文件以及 +- 搞清楚你修改的代码属于谁。查看源文件的根目录、 ``MAINTAINERS`` 文件以及 ``CREDITS`` 文件的最后一部分。你应该和此人协调，确保你没有重新发明轮子， 或者尝试一些已经被拒绝的东西。 @@ -629,9 +628,10 @@ C++ “obj-$(CONFIG_xxx) += xxx.o”。语法记录在 Documentation/kbuild/makefiles.rst 。 -- 如果你做了一些有意义的事情，那可以把自己放进 ``CREDITS`` ，通常不止一个 - 文件（无论如何你的名字都应该在源文件的顶部）。维护人员意味着您希望在对 - 子系统进行更改时得到询问，并了解缺陷；这意味着对某部分代码做出更多承诺。 +- 如果你认为自己做了一些有意义的事情，可以把自己放进 ``CREDITS`` ，通常不 + 止一个文件（无论如何你的名字都应该在源文件的顶部）。 ``MAINTAINERS`` + 意味着您希望在对子系统进行更改时得到询问，并了解缺陷；这意味着对某部分 + 代码做出更多承诺。 - 最后，别忘记去阅读 Documentation/process/submitting-patches.rst ， 也许还有 Documentation/process/submitting-drivers.rst 。 From 820636106342a6c5cfbb8279a4bf029ea2e00acd Mon Sep 17 00:00:00 2001 From: Muhammad Usama Anjum Date: Sat, 21 May 2022 12:36:51 +0500 Subject: [PATCH 128/318] docs/kselftest: add more guidelines for adding new tests Improve and add instructions to add new tests. Add build commands to test before sending the new test patch. Signed-off-by: Muhammad Usama Anjum Reviewed-by: Shuah Khan Link: https://lore.kernel.org/r/20220521073651.4191910-1-usama.anjum@collabora.com Signed-off-by: Jonathan Corbet --- Documentation/dev-tools/kselftest.rst | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/Documentation/dev-tools/kselftest.rst b/Documentation/dev-tools/kselftest.rst index a833ecf12fbc..ee6467ca8293 100644 --- a/Documentation/dev-tools/kselftest.rst +++ b/Documentation/dev-tools/kselftest.rst @@ -208,6 +208,14 @@ In general, the rules for selftests are Contributing new tests (details) ================================ + * In your Makefile, use facilities from lib.mk by including it instead of + reinventing the wheel. Specify flags and binaries generation flags on + need basis before including lib.mk. :: + + CFLAGS = $(KHDR_INCLUDES) + TEST_GEN_PROGS := close_range_test + include ../lib.mk + * Use TEST_GEN_XXX if such binaries or files are generated during compiling. @@ -230,13 +238,30 @@ Contributing new tests (details) * First use the headers inside the kernel source and/or git repo, and then the system headers. Headers for the kernel release as opposed to headers installed by the distro on the system should be the primary focus to be able - to find regressions. + to find regressions. Use KHDR_INCLUDES in Makefile to include headers from + the kernel source. * If a test needs specific kernel config options enabled, add a config file in the test directory to enable them. e.g: tools/testing/selftests/android/config + * Create a .gitignore file inside test directory and add all generated objects + in it. + + * Add new test name in TARGETS in selftests/Makefile:: + + TARGETS += android + + * All changes should pass:: + + kselftest-{all,install,clean,gen_tar} + kselftest-{all,install,clean,gen_tar} O=abo_path + kselftest-{all,install,clean,gen_tar} O=rel_path + make -C tools/testing/selftests {all,install,clean,gen_tar} + make -C tools/testing/selftests {all,install,clean,gen_tar} O=abs_path + make -C tools/testing/selftests {all,install,clean,gen_tar} O=rel_path + Test Module =========== From 7c43214dddfddbb7fd2dac9bf543a056b34ce58c Mon Sep 17 00:00:00 2001 From: Akira Yokosawa Date: Thu, 9 Jun 2022 22:23:27 +0900 Subject: [PATCH 129/318] docs/doc-guide: Add footnote on Inkscape for better images in PDF documents With kernel releases v5.18 and later, if you have Inkscape, images embedded in PDF documents are of vector graphics, not the raster images as are the case with pre-v5.18 releases. Even with pre-5.18 releases, having Inkscape would improve images converted from some of SVG files which are not fully covered by the limited capability of rsvg-convert(1) [1]. Add a footnote mentioning the expected improvements of such images. Link: https://gitlab.gnome.org/GNOME/librsvg#non-goals-of-librsvg [1] Signed-off-by: Akira Yokosawa Link: https://lore.kernel.org/r/81cddbde-bc28-bec1-fca4-3c8fe2df502f@gmail.com Signed-off-by: Jonathan Corbet --- Documentation/doc-guide/sphinx.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Documentation/doc-guide/sphinx.rst b/Documentation/doc-guide/sphinx.rst index 2ff1ab4158d4..edc4fa023986 100644 --- a/Documentation/doc-guide/sphinx.rst +++ b/Documentation/doc-guide/sphinx.rst @@ -132,7 +132,8 @@ format-specific subdirectories under ``Documentation/output``. To generate documentation, Sphinx (``sphinx-build``) must obviously be installed. For prettier HTML output, the Read the Docs Sphinx theme (``sphinx_rtd_theme``) is used if available. For PDF output you'll also need -``XeLaTeX`` and ``convert(1)`` from ImageMagick (https://www.imagemagick.org). +``XeLaTeX`` and ``convert(1)`` from ImageMagick +(https://www.imagemagick.org).\ [#ink]_ All of these are widely available and packaged in distributions. To pass extra options to Sphinx, you can use the ``SPHINXOPTS`` make @@ -152,6 +153,10 @@ The Sphinx theme can be overridden by using the ``DOCS_THEME`` make variable. To remove the generated documentation, run ``make cleandocs``. +.. [#ink] Having ``inkscape(1)`` from Inkscape (https://inkscape.org) + as well would improve the quality of images embedded in PDF + documents, especially for kernel releases 5.18 and later. + Writing Documentation ===================== From 5372de4e4545dead978c53d9c03b7aee12d0f94d Mon Sep 17 00:00:00 2001 From: Akira Yokosawa Date: Thu, 9 Jun 2022 22:28:40 +0900 Subject: [PATCH 130/318] docs/doc-guide: Put meta title for kernel-doc HTML page kernel-doc.rst has two 1st level section titles of "Writing kernel-doc comments" and "Including kernel-doc comments". Therefore, rather than using the first one, put a meta title of "Kernel-doc comments" for the title of the resulting HTML page by using the "title" directive. Signed-off-by: Akira Yokosawa Link: https://lore.kernel.org/r/15636c13-7fa2-f973-6d3d-361222b839ef@gmail.com Signed-off-by: Jonathan Corbet --- Documentation/doc-guide/kernel-doc.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/doc-guide/kernel-doc.rst b/Documentation/doc-guide/kernel-doc.rst index a7cb2afd7990..9c779bd7a751 100644 --- a/Documentation/doc-guide/kernel-doc.rst +++ b/Documentation/doc-guide/kernel-doc.rst @@ -1,3 +1,5 @@ +.. title:: Kernel-doc comments + =========================== Writing kernel-doc comments =========================== From f163f0302ab69722c052519f4014814bf10026a9 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:21 +0200 Subject: [PATCH 131/318] context_tracking: Rename context_tracking_user_enter/exit() to user_enter/exit_callable() context_tracking_user_enter() and context_tracking_user_exit() are ASM callable versions of user_enter() and user_exit() for architectures that didn't manage to check the context tracking static key from ASM. Change those function names to better reflect their purpose. [ frederic: Apply Max Filippov feedback. ] Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- arch/arm/kernel/entry-header.S | 8 ++++---- arch/csky/kernel/entry.S | 4 ++-- arch/riscv/kernel/entry.S | 6 +++--- arch/xtensa/kernel/entry.S | 4 ++-- include/linux/context_tracking.h | 4 ++-- kernel/context_tracking.c | 28 +++++++++++++++++----------- 6 files changed, 30 insertions(+), 24 deletions(-) diff --git a/arch/arm/kernel/entry-header.S b/arch/arm/kernel/entry-header.S index 5865621bf691..95def2b38d1c 100644 --- a/arch/arm/kernel/entry-header.S +++ b/arch/arm/kernel/entry-header.S @@ -369,10 +369,10 @@ ALT_UP_B(.L1_\@) #ifdef CONFIG_CONTEXT_TRACKING .if \save stmdb sp!, {r0-r3, ip, lr} - bl context_tracking_user_exit + bl user_exit_callable ldmia sp!, {r0-r3, ip, lr} .else - bl context_tracking_user_exit + bl user_exit_callable .endif #endif .endm @@ -381,10 +381,10 @@ ALT_UP_B(.L1_\@) #ifdef CONFIG_CONTEXT_TRACKING .if \save stmdb sp!, {r0-r3, ip, lr} - bl context_tracking_user_enter + bl user_enter_callable ldmia sp!, {r0-r3, ip, lr} .else - bl context_tracking_user_enter + bl user_enter_callable .endif #endif .endm diff --git a/arch/csky/kernel/entry.S b/arch/csky/kernel/entry.S index a4ababf25e24..bc734d17c16f 100644 --- a/arch/csky/kernel/entry.S +++ b/arch/csky/kernel/entry.S @@ -23,7 +23,7 @@ mfcr a0, epsr btsti a0, 31 bt 1f - jbsr context_tracking_user_exit + jbsr user_exit_callable ldw a0, (sp, LSAVE_A0) ldw a1, (sp, LSAVE_A1) ldw a2, (sp, LSAVE_A2) @@ -160,7 +160,7 @@ ret_from_exception: cmpnei r10, 0 bt exit_work #ifdef CONFIG_CONTEXT_TRACKING - jbsr context_tracking_user_enter + jbsr user_enter_callable #endif 1: #ifdef CONFIG_PREEMPTION diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index 2e5b88ca11ce..12f6bba57e33 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -112,11 +112,11 @@ _save_context: #endif #ifdef CONFIG_CONTEXT_TRACKING - /* If previous state is in user mode, call context_tracking_user_exit. */ + /* If previous state is in user mode, call user_exit_callable(). */ li a0, SR_PP and a0, s1, a0 bnez a0, skip_context_tracking - call context_tracking_user_exit + call user_exit_callable skip_context_tracking: #endif @@ -270,7 +270,7 @@ resume_userspace: bnez s1, work_pending #ifdef CONFIG_CONTEXT_TRACKING - call context_tracking_user_enter + call user_enter_callable #endif /* Save unwound kernel stack pointer in thread_info */ diff --git a/arch/xtensa/kernel/entry.S b/arch/xtensa/kernel/entry.S index e3eae648ba2e..d72bcafae90c 100644 --- a/arch/xtensa/kernel/entry.S +++ b/arch/xtensa/kernel/entry.S @@ -458,7 +458,7 @@ KABI_W or a3, a3, a2 #ifdef CONFIG_CONTEXT_TRACKING l32i abi_tmp0, a1, PT_PS bbci.l abi_tmp0, PS_UM_BIT, 1f - abi_call context_tracking_user_exit + abi_call user_exit_callable 1: #endif @@ -545,7 +545,7 @@ common_exception_return: .Lexit_tif_loop_user: #ifdef CONFIG_CONTEXT_TRACKING - abi_call context_tracking_user_enter + abi_call user_enter_callable #endif #ifdef CONFIG_HAVE_HW_BREAKPOINT _bbci.l abi_saved0, TIF_DB_DISABLED, 1f diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h index 773035124bad..69532cd18f72 100644 --- a/include/linux/context_tracking.h +++ b/include/linux/context_tracking.h @@ -19,8 +19,8 @@ extern void __ct_user_exit(enum ctx_state state); extern void context_tracking_enter(enum ctx_state state); extern void context_tracking_exit(enum ctx_state state); -extern void context_tracking_user_enter(void); -extern void context_tracking_user_exit(void); +extern void user_enter_callable(void); +extern void user_exit_callable(void); static inline void user_enter(void) { diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index 88c60ab39fbb..8f7dd5799bda 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -134,19 +134,22 @@ void context_tracking_enter(enum ctx_state state) NOKPROBE_SYMBOL(context_tracking_enter); EXPORT_SYMBOL_GPL(context_tracking_enter); -/* - * OBSOLETE: - * This function should be noinstr but it unsafely calls local_irq_restore(), - * involving illegal RCU uses through tracing and lockdep. +/** + * user_enter_callable() - Unfortunate ASM callable version of user_enter() for + * archs that didn't manage to check the context tracking + * static key from low level code. + * + * This OBSOLETE function should be noinstr but it unsafely calls + * local_irq_restore(), involving illegal RCU uses through tracing and lockdep. * This is unlikely to be fixed as this function is obsolete. The preferred * way is to call user_enter_irqoff(). It should be the arch entry code * responsibility to call into context tracking with IRQs disabled. */ -void context_tracking_user_enter(void) +void user_enter_callable(void) { user_enter(); } -NOKPROBE_SYMBOL(context_tracking_user_enter); +NOKPROBE_SYMBOL(user_enter_callable); /** * __ct_user_exit - Inform the context tracking that the CPU is @@ -208,19 +211,22 @@ void context_tracking_exit(enum ctx_state state) NOKPROBE_SYMBOL(context_tracking_exit); EXPORT_SYMBOL_GPL(context_tracking_exit); -/* - * OBSOLETE: - * This function should be noinstr but it unsafely calls local_irq_save(), +/** + * user_exit_callable() - Unfortunate ASM callable version of user_exit() for + * archs that didn't manage to check the context tracking + * static key from low level code. + * + * This OBSOLETE function should be noinstr but it unsafely calls local_irq_save(), * involving illegal RCU uses through tracing and lockdep. This is unlikely * to be fixed as this function is obsolete. The preferred way is to call * user_exit_irqoff(). It should be the arch entry code responsibility to * call into context tracking with IRQs disabled. */ -void context_tracking_user_exit(void) +void user_exit_callable(void) { user_exit(); } -NOKPROBE_SYMBOL(context_tracking_user_exit); +NOKPROBE_SYMBOL(user_exit_callable); void __init context_tracking_cpu_set(int cpu) { From fe98db1c6d1ad7349e61a5a2766ad64975bc9ae4 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:22 +0200 Subject: [PATCH 132/318] context_tracking: Rename context_tracking_enter/exit() to ct_user_enter/exit() context_tracking_enter() and context_tracking_exit() have confusing names that don't explain the fact they are referring to user/guest state. Use more self-explanatory names and shrink to the new context tracking prefix instead. Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- include/linux/context_tracking.h | 13 +++++++------ kernel/context_tracking.c | 12 ++++++------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h index 69532cd18f72..7a5f04ae1758 100644 --- a/include/linux/context_tracking.h +++ b/include/linux/context_tracking.h @@ -17,21 +17,22 @@ extern void context_tracking_cpu_set(int cpu); extern void __ct_user_enter(enum ctx_state state); extern void __ct_user_exit(enum ctx_state state); -extern void context_tracking_enter(enum ctx_state state); -extern void context_tracking_exit(enum ctx_state state); +extern void ct_user_enter(enum ctx_state state); +extern void ct_user_exit(enum ctx_state state); + extern void user_enter_callable(void); extern void user_exit_callable(void); static inline void user_enter(void) { if (context_tracking_enabled()) - context_tracking_enter(CONTEXT_USER); + ct_user_enter(CONTEXT_USER); } static inline void user_exit(void) { if (context_tracking_enabled()) - context_tracking_exit(CONTEXT_USER); + ct_user_exit(CONTEXT_USER); } /* Called with interrupts disabled. */ @@ -57,7 +58,7 @@ static inline enum ctx_state exception_enter(void) prev_ctx = this_cpu_read(context_tracking.state); if (prev_ctx != CONTEXT_KERNEL) - context_tracking_exit(prev_ctx); + ct_user_exit(prev_ctx); return prev_ctx; } @@ -67,7 +68,7 @@ static inline void exception_exit(enum ctx_state prev_ctx) if (!IS_ENABLED(CONFIG_HAVE_CONTEXT_TRACKING_OFFSTACK) && context_tracking_enabled()) { if (prev_ctx != CONTEXT_KERNEL) - context_tracking_enter(prev_ctx); + ct_user_enter(prev_ctx); } } diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index 8f7dd5799bda..590c920ad57f 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -112,7 +112,7 @@ EXPORT_SYMBOL_GPL(__ct_user_enter); * or context_tracking_guest_enter(). It should be the arch entry code * responsibility to call into context tracking with IRQs disabled. */ -void context_tracking_enter(enum ctx_state state) +void ct_user_enter(enum ctx_state state) { unsigned long flags; @@ -131,8 +131,8 @@ void context_tracking_enter(enum ctx_state state) __ct_user_enter(state); local_irq_restore(flags); } -NOKPROBE_SYMBOL(context_tracking_enter); -EXPORT_SYMBOL_GPL(context_tracking_enter); +NOKPROBE_SYMBOL(ct_user_enter); +EXPORT_SYMBOL_GPL(ct_user_enter); /** * user_enter_callable() - Unfortunate ASM callable version of user_enter() for @@ -197,7 +197,7 @@ EXPORT_SYMBOL_GPL(__ct_user_exit); * or context_tracking_guest_exit(). It should be the arch entry code * responsibility to call into context tracking with IRQs disabled. */ -void context_tracking_exit(enum ctx_state state) +void ct_user_exit(enum ctx_state state) { unsigned long flags; @@ -208,8 +208,8 @@ void context_tracking_exit(enum ctx_state state) __ct_user_exit(state); local_irq_restore(flags); } -NOKPROBE_SYMBOL(context_tracking_exit); -EXPORT_SYMBOL_GPL(context_tracking_exit); +NOKPROBE_SYMBOL(ct_user_exit); +EXPORT_SYMBOL_GPL(ct_user_exit); /** * user_exit_callable() - Unfortunate ASM callable version of user_exit() for From 2a0aafce963dab996b843f6cdb49237b0ae4a118 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:23 +0200 Subject: [PATCH 133/318] context_tracking: Rename context_tracking_cpu_set() to ct_cpu_track_user() context_tracking_cpu_set() is called in order to tell a CPU to track user/kernel transitions. Since context tracking is going to expand in to also track transitions from/to idle/IRQ/NMIs, the scope of this function name becomes too broad and needs to be made more specific. Also shorten the prefix to align with the new namespace. Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- include/linux/context_tracking.h | 2 +- kernel/context_tracking.c | 4 ++-- kernel/time/tick-sched.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h index 7a5f04ae1758..63259fece7c7 100644 --- a/include/linux/context_tracking.h +++ b/include/linux/context_tracking.h @@ -11,7 +11,7 @@ #ifdef CONFIG_CONTEXT_TRACKING -extern void context_tracking_cpu_set(int cpu); +extern void ct_cpu_track_user(int cpu); /* Called with interrupts disabled. */ extern void __ct_user_enter(enum ctx_state state); diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index 590c920ad57f..d361bd52e4e1 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -228,7 +228,7 @@ void user_exit_callable(void) } NOKPROBE_SYMBOL(user_exit_callable); -void __init context_tracking_cpu_set(int cpu) +void __init ct_cpu_track_user(int cpu) { static __initdata bool initialized = false; @@ -258,6 +258,6 @@ void __init context_tracking_init(void) int cpu; for_each_possible_cpu(cpu) - context_tracking_cpu_set(cpu); + ct_cpu_track_user(cpu); } #endif diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 58a11f859ac7..de192dcff828 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -571,7 +571,7 @@ void __init tick_nohz_init(void) } for_each_cpu(cpu, tick_nohz_full_mask) - context_tracking_cpu_set(cpu); + ct_cpu_track_user(cpu); ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "kernel/nohz:predown", NULL, From 24a9c54182b3758801b8ca6c8c237cc2ff654732 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:24 +0200 Subject: [PATCH 134/318] context_tracking: Split user tracking Kconfig Context tracking is going to be used not only to track user transitions but also idle/IRQs/NMIs. The user tracking part will then become a separate feature. Prepare Kconfig for that. [ frederic: Apply Max Filippov feedback. ] Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- .../time/context-tracking/arch-support.txt | 6 ++-- arch/Kconfig | 4 +-- arch/arm/Kconfig | 2 +- arch/arm/kernel/entry-common.S | 4 +-- arch/arm/kernel/entry-header.S | 4 +-- arch/arm64/Kconfig | 2 +- arch/csky/Kconfig | 2 +- arch/csky/kernel/entry.S | 4 +-- arch/loongarch/Kconfig | 2 +- arch/mips/Kconfig | 2 +- arch/powerpc/Kconfig | 2 +- arch/powerpc/include/asm/context_tracking.h | 2 +- arch/riscv/Kconfig | 2 +- arch/riscv/kernel/entry.S | 6 ++-- arch/sparc/Kconfig | 2 +- arch/sparc/kernel/rtrap_64.S | 2 +- arch/x86/Kconfig | 4 +-- arch/xtensa/Kconfig | 2 +- arch/xtensa/kernel/entry.S | 4 +-- include/linux/context_tracking.h | 12 +++---- include/linux/context_tracking_state.h | 4 +-- init/Kconfig | 4 +-- kernel/context_tracking.c | 6 +++- kernel/sched/core.c | 2 +- kernel/time/Kconfig | 31 ++++++++++++------- 25 files changed, 65 insertions(+), 52 deletions(-) diff --git a/Documentation/features/time/context-tracking/arch-support.txt b/Documentation/features/time/context-tracking/arch-support.txt index c9e0a16290e6..e59071a49090 100644 --- a/Documentation/features/time/context-tracking/arch-support.txt +++ b/Documentation/features/time/context-tracking/arch-support.txt @@ -1,7 +1,7 @@ # -# Feature name: context-tracking -# Kconfig: HAVE_CONTEXT_TRACKING -# description: arch supports context tracking for NO_HZ_FULL +# Feature name: user-context-tracking +# Kconfig: HAVE_CONTEXT_TRACKING_USER +# description: arch supports user context tracking for NO_HZ_FULL # ----------------------- | arch |status| diff --git a/arch/Kconfig b/arch/Kconfig index fcf9a41a4ef5..154b7b78da09 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -774,7 +774,7 @@ config HAVE_ARCH_WITHIN_STACK_FRAMES and similar) by implementing an inline arch_within_stack_frames(), which is used by CONFIG_HARDENED_USERCOPY. -config HAVE_CONTEXT_TRACKING +config HAVE_CONTEXT_TRACKING_USER bool help Provide kernel/user boundaries probes necessary for subsystems @@ -785,7 +785,7 @@ config HAVE_CONTEXT_TRACKING protected inside rcu_irq_enter/rcu_irq_exit() but preemption or signal handling on irq exit still need to be protected. -config HAVE_CONTEXT_TRACKING_OFFSTACK +config HAVE_CONTEXT_TRACKING_USER_OFFSTACK bool help Architecture neither relies on exception_enter()/exception_exit() diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 7630ba9cb6cc..9acc6aac5912 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -84,7 +84,7 @@ config ARM select HAVE_ARCH_TRANSPARENT_HUGEPAGE if ARM_LPAE select HAVE_ARM_SMCCC if CPU_V7 select HAVE_EBPF_JIT if !CPU_ENDIAN_BE32 - select HAVE_CONTEXT_TRACKING + select HAVE_CONTEXT_TRACKING_USER select HAVE_C_RECORDMCOUNT select HAVE_BUILDTIME_MCOUNT_SORT select HAVE_DEBUG_KMEMLEAK if !XIP_KERNEL diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S index 7aa3ded4af92..37a0125fc926 100644 --- a/arch/arm/kernel/entry-common.S +++ b/arch/arm/kernel/entry-common.S @@ -28,7 +28,7 @@ #include "entry-header.S" saved_psr .req r8 -#if defined(CONFIG_TRACE_IRQFLAGS) || defined(CONFIG_CONTEXT_TRACKING) +#if defined(CONFIG_TRACE_IRQFLAGS) || defined(CONFIG_CONTEXT_TRACKING_USER) saved_pc .req r9 #define TRACE(x...) x #else @@ -38,7 +38,7 @@ saved_pc .req lr .section .entry.text,"ax",%progbits .align 5 -#if !(IS_ENABLED(CONFIG_TRACE_IRQFLAGS) || IS_ENABLED(CONFIG_CONTEXT_TRACKING) || \ +#if !(IS_ENABLED(CONFIG_TRACE_IRQFLAGS) || IS_ENABLED(CONFIG_CONTEXT_TRACKING_USER) || \ IS_ENABLED(CONFIG_DEBUG_RSEQ)) /* * This is the fast syscall return path. We do as little as possible here, diff --git a/arch/arm/kernel/entry-header.S b/arch/arm/kernel/entry-header.S index 95def2b38d1c..99411fa91350 100644 --- a/arch/arm/kernel/entry-header.S +++ b/arch/arm/kernel/entry-header.S @@ -366,7 +366,7 @@ ALT_UP_B(.L1_\@) * between user and kernel mode. */ .macro ct_user_exit, save = 1 -#ifdef CONFIG_CONTEXT_TRACKING +#ifdef CONFIG_CONTEXT_TRACKING_USER .if \save stmdb sp!, {r0-r3, ip, lr} bl user_exit_callable @@ -378,7 +378,7 @@ ALT_UP_B(.L1_\@) .endm .macro ct_user_enter, save = 1 -#ifdef CONFIG_CONTEXT_TRACKING +#ifdef CONFIG_CONTEXT_TRACKING_USER .if \save stmdb sp!, {r0-r3, ip, lr} bl user_enter_callable diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 1652a9800ebe..7c5dd2af9ca9 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -174,7 +174,7 @@ config ARM64 select HAVE_C_RECORDMCOUNT select HAVE_CMPXCHG_DOUBLE select HAVE_CMPXCHG_LOCAL - select HAVE_CONTEXT_TRACKING + select HAVE_CONTEXT_TRACKING_USER select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_CONTIGUOUS select HAVE_DYNAMIC_FTRACE diff --git a/arch/csky/Kconfig b/arch/csky/Kconfig index 21d72b078eef..f55ba1745f7b 100644 --- a/arch/csky/Kconfig +++ b/arch/csky/Kconfig @@ -42,7 +42,7 @@ config CSKY select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_MMAP_RND_BITS select HAVE_ARCH_SECCOMP_FILTER - select HAVE_CONTEXT_TRACKING + select HAVE_CONTEXT_TRACKING_USER select HAVE_VIRT_CPU_ACCOUNTING_GEN select HAVE_DEBUG_BUGVERBOSE select HAVE_DEBUG_KMEMLEAK diff --git a/arch/csky/kernel/entry.S b/arch/csky/kernel/entry.S index bc734d17c16f..547b4cd1b24b 100644 --- a/arch/csky/kernel/entry.S +++ b/arch/csky/kernel/entry.S @@ -19,7 +19,7 @@ .endm .macro context_tracking -#ifdef CONFIG_CONTEXT_TRACKING +#ifdef CONFIG_CONTEXT_TRACKING_USER mfcr a0, epsr btsti a0, 31 bt 1f @@ -159,7 +159,7 @@ ret_from_exception: and r10, r9 cmpnei r10, 0 bt exit_work -#ifdef CONFIG_CONTEXT_TRACKING +#ifdef CONFIG_CONTEXT_TRACKING_USER jbsr user_enter_callable #endif 1: diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 1920d52653b4..130dc65f3c85 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -76,7 +76,7 @@ config LOONGARCH select HAVE_ARCH_TRACEHOOK select HAVE_ARCH_TRANSPARENT_HUGEPAGE select HAVE_ASM_MODVERSIONS - select HAVE_CONTEXT_TRACKING + select HAVE_CONTEXT_TRACKING_USER select HAVE_COPY_THREAD_TLS select HAVE_DEBUG_STACKOVERFLOW select HAVE_DMA_CONTIGUOUS diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index db09d45d59ec..9457894db237 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -56,7 +56,7 @@ config MIPS select HAVE_ARCH_TRACEHOOK select HAVE_ARCH_TRANSPARENT_HUGEPAGE if CPU_SUPPORTS_HUGEPAGES select HAVE_ASM_MODVERSIONS - select HAVE_CONTEXT_TRACKING + select HAVE_CONTEXT_TRACKING_USER select HAVE_TIF_NOHZ select HAVE_C_RECORDMCOUNT select HAVE_DEBUG_KMEMLEAK diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index c2ce2e60c8f0..874c8d81284a 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -202,7 +202,7 @@ config PPC select HAVE_ARCH_SECCOMP_FILTER select HAVE_ARCH_TRACEHOOK select HAVE_ASM_MODVERSIONS - select HAVE_CONTEXT_TRACKING if PPC64 + select HAVE_CONTEXT_TRACKING_USER if PPC64 select HAVE_C_RECORDMCOUNT select HAVE_DEBUG_KMEMLEAK select HAVE_DEBUG_STACKOVERFLOW diff --git a/arch/powerpc/include/asm/context_tracking.h b/arch/powerpc/include/asm/context_tracking.h index f2682b28b050..4b63931c49e0 100644 --- a/arch/powerpc/include/asm/context_tracking.h +++ b/arch/powerpc/include/asm/context_tracking.h @@ -2,7 +2,7 @@ #ifndef _ASM_POWERPC_CONTEXT_TRACKING_H #define _ASM_POWERPC_CONTEXT_TRACKING_H -#ifdef CONFIG_CONTEXT_TRACKING +#ifdef CONFIG_CONTEXT_TRACKING_USER #define SCHEDULE_USER bl schedule_user #else #define SCHEDULE_USER bl schedule diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 32ffef9f6e5b..29b46f217345 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -86,7 +86,7 @@ config RISCV select HAVE_ARCH_THREAD_STRUCT_WHITELIST select HAVE_ARCH_VMAP_STACK if MMU && 64BIT select HAVE_ASM_MODVERSIONS - select HAVE_CONTEXT_TRACKING + select HAVE_CONTEXT_TRACKING_USER select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_CONTIGUOUS if MMU select HAVE_EBPF_JIT if MMU diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index 12f6bba57e33..b9eda3fcbd6d 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -111,7 +111,7 @@ _save_context: call __trace_hardirqs_off #endif -#ifdef CONFIG_CONTEXT_TRACKING +#ifdef CONFIG_CONTEXT_TRACKING_USER /* If previous state is in user mode, call user_exit_callable(). */ li a0, SR_PP and a0, s1, a0 @@ -176,7 +176,7 @@ handle_syscall: */ csrs CSR_STATUS, SR_IE #endif -#if defined(CONFIG_TRACE_IRQFLAGS) || defined(CONFIG_CONTEXT_TRACKING) +#if defined(CONFIG_TRACE_IRQFLAGS) || defined(CONFIG_CONTEXT_TRACKING_USER) /* Recover a0 - a7 for system calls */ REG_L a0, PT_A0(sp) REG_L a1, PT_A1(sp) @@ -269,7 +269,7 @@ resume_userspace: andi s1, s0, _TIF_WORK_MASK bnez s1, work_pending -#ifdef CONFIG_CONTEXT_TRACKING +#ifdef CONFIG_CONTEXT_TRACKING_USER call user_enter_callable #endif diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index ba449c47effd..9232411a8821 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -71,7 +71,7 @@ config SPARC64 select HAVE_DYNAMIC_FTRACE select HAVE_FTRACE_MCOUNT_RECORD select HAVE_SYSCALL_TRACEPOINTS - select HAVE_CONTEXT_TRACKING + select HAVE_CONTEXT_TRACKING_USER select HAVE_TIF_NOHZ select HAVE_DEBUG_KMEMLEAK select IOMMU_HELPER diff --git a/arch/sparc/kernel/rtrap_64.S b/arch/sparc/kernel/rtrap_64.S index c5fd4b450d9b..eef102765a7e 100644 --- a/arch/sparc/kernel/rtrap_64.S +++ b/arch/sparc/kernel/rtrap_64.S @@ -15,7 +15,7 @@ #include #include -#ifdef CONFIG_CONTEXT_TRACKING +#ifdef CONFIG_CONTEXT_TRACKING_USER # define SCHEDULE_USER schedule_user #else # define SCHEDULE_USER schedule diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index be0b95e51df6..b0a6dbbb760b 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -186,8 +186,8 @@ config X86 select HAVE_ASM_MODVERSIONS select HAVE_CMPXCHG_DOUBLE select HAVE_CMPXCHG_LOCAL - select HAVE_CONTEXT_TRACKING if X86_64 - select HAVE_CONTEXT_TRACKING_OFFSTACK if HAVE_CONTEXT_TRACKING + select HAVE_CONTEXT_TRACKING_USER if X86_64 + select HAVE_CONTEXT_TRACKING_USER_OFFSTACK if HAVE_CONTEXT_TRACKING_USER select HAVE_C_RECORDMCOUNT select HAVE_OBJTOOL_MCOUNT if HAVE_OBJTOOL select HAVE_BUILDTIME_MCOUNT_SORT diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig index 0b0f0172cced..7927fed7bc83 100644 --- a/arch/xtensa/Kconfig +++ b/arch/xtensa/Kconfig @@ -33,7 +33,7 @@ config XTENSA select HAVE_ARCH_KCSAN select HAVE_ARCH_SECCOMP_FILTER select HAVE_ARCH_TRACEHOOK - select HAVE_CONTEXT_TRACKING + select HAVE_CONTEXT_TRACKING_USER select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_CONTIGUOUS select HAVE_EXIT_THREAD diff --git a/arch/xtensa/kernel/entry.S b/arch/xtensa/kernel/entry.S index d72bcafae90c..fb67d85116e4 100644 --- a/arch/xtensa/kernel/entry.S +++ b/arch/xtensa/kernel/entry.S @@ -455,7 +455,7 @@ KABI_W or a3, a3, a2 abi_call trace_hardirqs_off 1: #endif -#ifdef CONFIG_CONTEXT_TRACKING +#ifdef CONFIG_CONTEXT_TRACKING_USER l32i abi_tmp0, a1, PT_PS bbci.l abi_tmp0, PS_UM_BIT, 1f abi_call user_exit_callable @@ -544,7 +544,7 @@ common_exception_return: j .Lrestore_state .Lexit_tif_loop_user: -#ifdef CONFIG_CONTEXT_TRACKING +#ifdef CONFIG_CONTEXT_TRACKING_USER abi_call user_enter_callable #endif #ifdef CONFIG_HAVE_HW_BREAKPOINT diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h index 63259fece7c7..e35ae66b4794 100644 --- a/include/linux/context_tracking.h +++ b/include/linux/context_tracking.h @@ -10,7 +10,7 @@ #include -#ifdef CONFIG_CONTEXT_TRACKING +#ifdef CONFIG_CONTEXT_TRACKING_USER extern void ct_cpu_track_user(int cpu); /* Called with interrupts disabled. */ @@ -52,7 +52,7 @@ static inline enum ctx_state exception_enter(void) { enum ctx_state prev_ctx; - if (IS_ENABLED(CONFIG_HAVE_CONTEXT_TRACKING_OFFSTACK) || + if (IS_ENABLED(CONFIG_HAVE_CONTEXT_TRACKING_USER_OFFSTACK) || !context_tracking_enabled()) return 0; @@ -65,7 +65,7 @@ static inline enum ctx_state exception_enter(void) static inline void exception_exit(enum ctx_state prev_ctx) { - if (!IS_ENABLED(CONFIG_HAVE_CONTEXT_TRACKING_OFFSTACK) && + if (!IS_ENABLED(CONFIG_HAVE_CONTEXT_TRACKING_USER_OFFSTACK) && context_tracking_enabled()) { if (prev_ctx != CONTEXT_KERNEL) ct_user_enter(prev_ctx); @@ -109,14 +109,14 @@ static inline enum ctx_state ct_state(void) { return CONTEXT_DISABLED; } static __always_inline bool context_tracking_guest_enter(void) { return false; } static inline void context_tracking_guest_exit(void) { } -#endif /* !CONFIG_CONTEXT_TRACKING */ +#endif /* !CONFIG_CONTEXT_TRACKING_USER */ #define CT_WARN_ON(cond) WARN_ON(context_tracking_enabled() && (cond)) -#ifdef CONFIG_CONTEXT_TRACKING_FORCE +#ifdef CONFIG_CONTEXT_TRACKING_USER_FORCE extern void context_tracking_init(void); #else static inline void context_tracking_init(void) { } -#endif /* CONFIG_CONTEXT_TRACKING_FORCE */ +#endif /* CONFIG_CONTEXT_TRACKING_USER_FORCE */ #endif diff --git a/include/linux/context_tracking_state.h b/include/linux/context_tracking_state.h index edc7b46376a6..2b46afe105a9 100644 --- a/include/linux/context_tracking_state.h +++ b/include/linux/context_tracking_state.h @@ -22,7 +22,7 @@ struct context_tracking { } state; }; -#ifdef CONFIG_CONTEXT_TRACKING +#ifdef CONFIG_CONTEXT_TRACKING_USER extern struct static_key_false context_tracking_key; DECLARE_PER_CPU(struct context_tracking, context_tracking); @@ -45,6 +45,6 @@ static inline bool context_tracking_enabled_this_cpu(void) static __always_inline bool context_tracking_enabled(void) { return false; } static __always_inline bool context_tracking_enabled_cpu(int cpu) { return false; } static __always_inline bool context_tracking_enabled_this_cpu(void) { return false; } -#endif /* CONFIG_CONTEXT_TRACKING */ +#endif /* CONFIG_CONTEXT_TRACKING_USER */ #endif diff --git a/init/Kconfig b/init/Kconfig index c7900e8975f1..06454d19e2f0 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -494,11 +494,11 @@ config VIRT_CPU_ACCOUNTING_NATIVE config VIRT_CPU_ACCOUNTING_GEN bool "Full dynticks CPU time accounting" - depends on HAVE_CONTEXT_TRACKING + depends on HAVE_CONTEXT_TRACKING_USER depends on HAVE_VIRT_CPU_ACCOUNTING_GEN depends on GENERIC_CLOCKEVENTS select VIRT_CPU_ACCOUNTING - select CONTEXT_TRACKING + select CONTEXT_TRACKING_USER help Select this option to enable task and CPU time accounting on full dynticks systems. This accounting is implemented by watching every diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index d361bd52e4e1..f3dec1be2bf6 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -22,6 +22,8 @@ #include #include +#ifdef CONFIG_CONTEXT_TRACKING_USER + #define CREATE_TRACE_POINTS #include @@ -252,7 +254,7 @@ void __init ct_cpu_track_user(int cpu) initialized = true; } -#ifdef CONFIG_CONTEXT_TRACKING_FORCE +#ifdef CONFIG_CONTEXT_TRACKING_USER_FORCE void __init context_tracking_init(void) { int cpu; @@ -261,3 +263,5 @@ void __init context_tracking_init(void) ct_cpu_track_user(cpu); } #endif + +#endif /* #ifdef CONFIG_CONTEXT_TRACKING_USER */ diff --git a/kernel/sched/core.c b/kernel/sched/core.c index da0bf6fe9ecd..883167a57bf9 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6559,7 +6559,7 @@ void __sched schedule_idle(void) } while (need_resched()); } -#if defined(CONFIG_CONTEXT_TRACKING) && !defined(CONFIG_HAVE_CONTEXT_TRACKING_OFFSTACK) +#if defined(CONFIG_CONTEXT_TRACKING_USER) && !defined(CONFIG_HAVE_CONTEXT_TRACKING_USER_OFFSTACK) asmlinkage __visible void __sched schedule_user(void) { /* diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig index 27b7868b5c30..41f99bcfe9e6 100644 --- a/kernel/time/Kconfig +++ b/kernel/time/Kconfig @@ -73,6 +73,9 @@ config TIME_KUNIT_TEST If unsure, say N. +config CONTEXT_TRACKING + bool + if GENERIC_CLOCKEVENTS menu "Timers subsystem" @@ -111,7 +114,7 @@ config NO_HZ_FULL # NO_HZ_COMMON dependency # We need at least one periodic CPU for timekeeping depends on SMP - depends on HAVE_CONTEXT_TRACKING + depends on HAVE_CONTEXT_TRACKING_USER # VIRT_CPU_ACCOUNTING_GEN dependency depends on HAVE_VIRT_CPU_ACCOUNTING_GEN select NO_HZ_COMMON @@ -137,31 +140,37 @@ config NO_HZ_FULL endchoice -config CONTEXT_TRACKING - bool +config CONTEXT_TRACKING_USER + bool + depends on HAVE_CONTEXT_TRACKING_USER + select CONTEXT_TRACKING + help + Track transitions between kernel and user on behalf of RCU and + tickless cputime accounting. The former case relies on context + tracking to enter/exit RCU extended quiescent states. -config CONTEXT_TRACKING_FORCE - bool "Force context tracking" - depends on CONTEXT_TRACKING +config CONTEXT_TRACKING_USER_FORCE + bool "Force user context tracking" + depends on CONTEXT_TRACKING_USER default y if !NO_HZ_FULL help The major pre-requirement for full dynticks to work is to - support the context tracking subsystem. But there are also + support the user context tracking subsystem. But there are also other dependencies to provide in order to make the full dynticks working. This option stands for testing when an arch implements the - context tracking backend but doesn't yet fulfill all the + user context tracking backend but doesn't yet fulfill all the requirements to make the full dynticks feature working. Without the full dynticks, there is no way to test the support - for context tracking and the subsystems that rely on it: RCU + for user context tracking and the subsystems that rely on it: RCU userspace extended quiescent state and tickless cputime accounting. This option copes with the absence of the full - dynticks subsystem by forcing the context tracking on all + dynticks subsystem by forcing the user context tracking on all CPUs in the system. Say Y only if you're working on the development of an - architecture backend for the context tracking. + architecture backend for the user context tracking. Say N otherwise, this option brings an overhead that you don't want in production. From d6c14da474bf260d73953fbf7992c98d9112aec7 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Mon, 20 Jun 2022 09:52:43 +0200 Subject: [PATCH 135/318] crypto: lib/blake2s - reduce stack frame usage in self test Using 3 blocks here doesn't give us much more than using 2, and it causes a stack frame size warning on certain compiler/config/arch combinations: lib/crypto/blake2s-selftest.c: In function 'blake2s_selftest': >> lib/crypto/blake2s-selftest.c:632:1: warning: the frame size of 1088 bytes is larger than 1024 bytes [-Wframe-larger-than=] 632 | } | ^ So this patch just reduces the block from 3 to 2, which makes the warning go away. Reported-by: kernel test robot Link: https://lore.kernel.org/linux-crypto/202206200851.gE3MHCgd-lkp@intel.com Fixes: 2d16803c562e ("crypto: blake2s - remove shash module") Signed-off-by: Jason A. Donenfeld Signed-off-by: Herbert Xu --- lib/crypto/blake2s-selftest.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/crypto/blake2s-selftest.c b/lib/crypto/blake2s-selftest.c index 66f505220f43..7d77dea15587 100644 --- a/lib/crypto/blake2s-selftest.c +++ b/lib/crypto/blake2s-selftest.c @@ -593,7 +593,7 @@ bool __init blake2s_selftest(void) enum { TEST_ALIGNMENT = 16 }; u8 unaligned_block[BLAKE2S_BLOCK_SIZE + TEST_ALIGNMENT - 1] __aligned(TEST_ALIGNMENT); - u8 blocks[BLAKE2S_BLOCK_SIZE * 3]; + u8 blocks[BLAKE2S_BLOCK_SIZE * 2]; struct blake2s_state state1, state2; get_random_bytes(blocks, sizeof(blocks)); @@ -603,8 +603,8 @@ bool __init blake2s_selftest(void) defined(CONFIG_CRYPTO_ARCH_HAVE_LIB_BLAKE2S) memcpy(&state1, &state, sizeof(state1)); memcpy(&state2, &state, sizeof(state2)); - blake2s_compress(&state1, blocks, 3, BLAKE2S_BLOCK_SIZE); - blake2s_compress_generic(&state2, blocks, 3, BLAKE2S_BLOCK_SIZE); + blake2s_compress(&state1, blocks, 2, BLAKE2S_BLOCK_SIZE); + blake2s_compress_generic(&state2, blocks, 2, BLAKE2S_BLOCK_SIZE); if (memcmp(&state1, &state2, sizeof(state1))) { pr_err("blake2s random compress self-test %d: FAIL\n", i + 1); From 54a8b6802f03b6d0fa6ecad67f51466f8fa2ad6d Mon Sep 17 00:00:00 2001 From: Jiang Jian Date: Tue, 21 Jun 2022 18:22:54 +0800 Subject: [PATCH 136/318] crypto: nx - drop unexpected word "the" there is an unexpected word "the" in the comments that need to be dropped >- * The DDE is setup with the the DDE count, byte count, and address of >+ * The DDE is setup with the DDE count, byte count, and address of Signed-off-by: Jiang Jian Signed-off-by: Herbert Xu --- drivers/crypto/nx/nx-common-powernv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/nx/nx-common-powernv.c b/drivers/crypto/nx/nx-common-powernv.c index f418817c0f43..f34c75a862f2 100644 --- a/drivers/crypto/nx/nx-common-powernv.c +++ b/drivers/crypto/nx/nx-common-powernv.c @@ -75,7 +75,7 @@ static int (*nx842_powernv_exec)(const unsigned char *in, /** * setup_indirect_dde - Setup an indirect DDE * - * The DDE is setup with the the DDE count, byte count, and address of + * The DDE is setup with the DDE count, byte count, and address of * first direct DDE in the list. */ static void setup_indirect_dde(struct data_descriptor_entry *dde, From 882f0a59377da3fc5153c06790227930e5c00f88 Mon Sep 17 00:00:00 2001 From: Jiang Jian Date: Tue, 21 Jun 2022 18:27:33 +0800 Subject: [PATCH 137/318] crypto: ux500/hash - drop unexpected word "the" there is an unexpected word "the" in the comments that need to be dropped >- * specified in the the hw design spec. Either due to incorrect info in the >+ * specified in the hw design spec. Either due to incorrect info in the Signed-off-by: Jiang Jian Signed-off-by: Herbert Xu --- drivers/crypto/ux500/hash/hash_core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/ux500/hash/hash_core.c b/drivers/crypto/ux500/hash/hash_core.c index 265ef3e96fdd..f104e8a43036 100644 --- a/drivers/crypto/ux500/hash/hash_core.c +++ b/drivers/crypto/ux500/hash/hash_core.c @@ -421,7 +421,7 @@ static int hash_get_device_data(struct hash_ctx *ctx, * @keylen: The lengt of the key. * * Note! This function DOES NOT write to the NBLW registry, even though - * specified in the the hw design spec. Either due to incorrect info in the + * specified in the hw design spec. Either due to incorrect info in the * spec or due to a bug in the hw. */ static void hash_hw_write_key(struct hash_device_data *device_data, From 1b069597c254a2969cf21142c3881b2d5e80c658 Mon Sep 17 00:00:00 2001 From: Jilin Yuan Date: Wed, 22 Jun 2022 17:07:30 +0800 Subject: [PATCH 138/318] crypto: arm64/aes-neon - Fix typo in comment Delete the redundant word 'the'. Signed-off-by: Jilin Yuan Signed-off-by: Herbert Xu --- arch/arm64/crypto/aes-neon.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/crypto/aes-neon.S b/arch/arm64/crypto/aes-neon.S index e47d3ec2cfb4..9de7fbc797af 100644 --- a/arch/arm64/crypto/aes-neon.S +++ b/arch/arm64/crypto/aes-neon.S @@ -66,7 +66,7 @@ prepare crypto_aes_inv_sbox, .LReverse_ShiftRows, \temp .endm - /* apply SubBytes transformation using the the preloaded Sbox */ + /* apply SubBytes transformation using the preloaded Sbox */ .macro sub_bytes, in sub v9.16b, \in\().16b, v15.16b tbl \in\().16b, {v16.16b-v19.16b}, \in\().16b From e34525c3975832971451cb0908594c310fc83cd5 Mon Sep 17 00:00:00 2001 From: Jiang Jian Date: Thu, 23 Jun 2022 00:02:22 +0800 Subject: [PATCH 139/318] crypto: cavium - fix typos in comments Replace 'is' with 'it' file: drivers/crypto/cavium/cpt/cpt_hw_types.h line: 268 * which if the line hits and is is dirty will cause the line to be changed to: * which if the line hits and it is dirty will cause the line to be Signed-off-by: Jiang Jian Signed-off-by: Herbert Xu --- drivers/crypto/cavium/cpt/cpt_hw_types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/cavium/cpt/cpt_hw_types.h b/drivers/crypto/cavium/cpt/cpt_hw_types.h index 96bc963bb804..8ec6edc69f3f 100644 --- a/drivers/crypto/cavium/cpt/cpt_hw_types.h +++ b/drivers/crypto/cavium/cpt/cpt_hw_types.h @@ -265,7 +265,7 @@ union cptx_pf_exe_bist_status { * big-endian format in memory. * iqb_ldwb:1 [7:7](R/W) Instruction load don't write back. * 0 = The hardware issues NCB transient load (LDT) towards the cache, - * which if the line hits and is is dirty will cause the line to be + * which if the line hits and it is dirty will cause the line to be * written back before being replaced. * 1 = The hardware issues NCB LDWB read-and-invalidate command towards * the cache when fetching the last word of instructions; as a result the From dac230179fdc9ffa074fcb1533a0dd56f162a00a Mon Sep 17 00:00:00 2001 From: Jiang Jian Date: Thu, 23 Jun 2022 15:11:23 +0800 Subject: [PATCH 140/318] crypto: caam - drop unexpected word 'a' in comments Drop the unexpected word 'a' in the comments that need to be dropped * This is a a cache of buffers, from which the users of CAAM QI driver --> * This is a cache of buffers, from which the users of CAAM QI driver Signed-off-by: Jiang Jian Reviewed-by: Gaurav Jain Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg_qi2.c | 2 +- drivers/crypto/caam/qi.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c index 6753f0e6e55d..4b81fb33f199 100644 --- a/drivers/crypto/caam/caamalg_qi2.c +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -29,7 +29,7 @@ SHA512_DIGEST_SIZE * 2) /* - * This is a a cache of buffers, from which the users of CAAM QI driver + * This is a cache of buffers, from which the users of CAAM QI driver * can allocate short buffers. It's speedier than doing kmalloc on the hotpath. * NOTE: A more elegant solution would be to have some headroom in the frames * being processed. This can be added by the dpaa2-eth driver. This would diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c index 8163f5df8ebf..49439d0d1b3c 100644 --- a/drivers/crypto/caam/qi.c +++ b/drivers/crypto/caam/qi.c @@ -75,7 +75,7 @@ bool caam_congested __read_mostly; EXPORT_SYMBOL(caam_congested); /* - * This is a a cache of buffers, from which the users of CAAM QI driver + * This is a cache of buffers, from which the users of CAAM QI driver * can allocate short (CAAM_QI_MEMCACHE_SIZE) buffers. It's faster than * doing malloc on the hotpath. * NOTE: A more elegant solution would be to have some headroom in the frames From a89db0595f52ca4d05c9301b4b0bae4cce2ecaf7 Mon Sep 17 00:00:00 2001 From: Jiang Jian Date: Thu, 23 Jun 2022 18:25:36 +0800 Subject: [PATCH 141/318] crypto: caam - drop unexpected word 'for' in comments there is an unexpected word 'for' in the comments that need to be dropped file - drivers/crypto/caam/caamhash_desc.c line - 25 * must be false for for ahash first and digest changed to: * must be false for ahash first and digest Signed-off-by: Jiang Jian Reviewed-by: Gaurav Jain Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamhash_desc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/caam/caamhash_desc.c b/drivers/crypto/caam/caamhash_desc.c index 78383d77da99..619564509936 100644 --- a/drivers/crypto/caam/caamhash_desc.c +++ b/drivers/crypto/caam/caamhash_desc.c @@ -22,7 +22,7 @@ * @ctx_len: size of Context Register * @import_ctx: true if previous Context Register needs to be restored * must be true for ahash update and final - * must be false for for ahash first and digest + * must be false for ahash first and digest * @era: SEC Era */ void cnstr_shdsc_ahash(u32 * const desc, struct alginfo *adata, u32 state, From 8e7ae8bafe98a288727d647dace03fed2bdcc5fc Mon Sep 17 00:00:00 2001 From: Jiang Jian Date: Thu, 23 Jun 2022 18:30:14 +0800 Subject: [PATCH 142/318] crypto: vmx - drop unexpected word 'for' in comments there is an unexpected word 'for' in the comments that need to be dropped file - drivers/crypto/vmx/ghashp8-ppc.pl line - 19 "# GHASH for for PowerISA v2.07." changed to: "# GHASH for PowerISA v2.07." Signed-off-by: Jiang Jian Signed-off-by: Herbert Xu --- drivers/crypto/vmx/ghashp8-ppc.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/vmx/ghashp8-ppc.pl b/drivers/crypto/vmx/ghashp8-ppc.pl index 09bba1852eec..041e633c214f 100644 --- a/drivers/crypto/vmx/ghashp8-ppc.pl +++ b/drivers/crypto/vmx/ghashp8-ppc.pl @@ -16,7 +16,7 @@ # details see https://www.openssl.org/~appro/cryptogams/. # ==================================================================== # -# GHASH for for PowerISA v2.07. +# GHASH for PowerISA v2.07. # # July 2014 # From 0bb8f125253843c445b70fc6ef4fb21aa7b25625 Mon Sep 17 00:00:00 2001 From: lei he Date: Fri, 24 Jun 2022 18:06:25 +0800 Subject: [PATCH 143/318] crypto: testmgr - fix version number of RSA tests According to PKCS#1 standard, the 'otherPrimeInfos' field contains the information for the additional primes r_3, ..., r_u, in order. It shall be omitted if the version is 0 and shall contain at least one instance of OtherPrimeInfo if the version is 1, see: https://www.rfc-editor.org/rfc/rfc3447#page-44 Replace the version number '1' with 0, otherwise, some drivers may not pass the run-time tests. Signed-off-by: lei he Signed-off-by: Herbert Xu --- crypto/testmgr.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 4f3955ea40bf..8e2dce86dd48 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -186,7 +186,7 @@ static const struct akcipher_testvec rsa_tv_template[] = { #ifndef CONFIG_CRYPTO_FIPS .key = "\x30\x81\x9A" /* sequence of 154 bytes */ - "\x02\x01\x01" /* version - integer of 1 byte */ + "\x02\x01\x00" /* version - integer of 1 byte */ "\x02\x41" /* modulus - integer of 65 bytes */ "\x00\xAA\x36\xAB\xCE\x88\xAC\xFD\xFF\x55\x52\x3C\x7F\xC4\x52\x3F" "\x90\xEF\xA0\x0D\xF3\x77\x4A\x25\x9F\x2E\x62\xB4\xC5\xD9\x9C\xB5" @@ -216,7 +216,7 @@ static const struct akcipher_testvec rsa_tv_template[] = { }, { .key = "\x30\x82\x01\x1D" /* sequence of 285 bytes */ - "\x02\x01\x01" /* version - integer of 1 byte */ + "\x02\x01\x00" /* version - integer of 1 byte */ "\x02\x81\x81" /* modulus - integer of 129 bytes */ "\x00\xBB\xF8\x2F\x09\x06\x82\xCE\x9C\x23\x38\xAC\x2B\x9D\xA8\x71" "\xF7\x36\x8D\x07\xEE\xD4\x10\x43\xA4\x40\xD6\xB6\xF0\x74\x54\xF5" @@ -260,7 +260,7 @@ static const struct akcipher_testvec rsa_tv_template[] = { #endif .key = "\x30\x82\x02\x20" /* sequence of 544 bytes */ - "\x02\x01\x01" /* version - integer of 1 byte */ + "\x02\x01\x00" /* version - integer of 1 byte */ "\x02\x82\x01\x01\x00" /* modulus - integer of 256 bytes */ "\xDB\x10\x1A\xC2\xA3\xF1\xDC\xFF\x13\x6B\xED\x44\xDF\xF0\x02\x6D" "\x13\xC7\x88\xDA\x70\x6B\x54\xF1\xE8\x27\xDC\xC3\x0F\x99\x6A\xFA" From fac76f2260893dde5aa05bb693b4c13e8ed0454b Mon Sep 17 00:00:00 2001 From: Qian Cai Date: Fri, 24 Jun 2022 07:13:38 -0400 Subject: [PATCH 144/318] crypto: arm64/gcm - Select AEAD for GHASH_ARM64_CE Otherwise, we could fail to compile. ld: arch/arm64/crypto/ghash-ce-glue.o: in function 'ghash_ce_mod_exit': ghash-ce-glue.c:(.exit.text+0x24): undefined reference to 'crypto_unregister_aead' ld: arch/arm64/crypto/ghash-ce-glue.o: in function 'ghash_ce_mod_init': ghash-ce-glue.c:(.init.text+0x34): undefined reference to 'crypto_register_aead' Fixes: 537c1445ab0b ("crypto: arm64/gcm - implement native driver using v8 Crypto Extensions") Signed-off-by: Qian Cai Signed-off-by: Herbert Xu --- arch/arm64/crypto/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index 4391a463abd7..60db5bb2ddda 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -71,6 +71,7 @@ config CRYPTO_GHASH_ARM64_CE select CRYPTO_HASH select CRYPTO_GF128MUL select CRYPTO_LIB_AES + select CRYPTO_AEAD config CRYPTO_POLYVAL_ARM64_CE tristate "POLYVAL using ARMv8 Crypto Extensions (for HCTR2)" From bf02fb07c79af46483bee539f12635d998963462 Mon Sep 17 00:00:00 2001 From: Tang Yizhou Date: Wed, 29 Jun 2022 09:32:39 +0800 Subject: [PATCH 145/318] docs/zh_CN: Show the Chinese translation of spinlocks Last time I forgot to edit locking/index.rst so the Chinese translation of spinlocks hasn't been shown on web yet. Signed-off-by: Tang Yizhou Reviewed-by: Yanteng Si Reviewed-by: Alex Shi Link: https://lore.kernel.org/r/20220629013240.65386-2-yizhou.tang@shopee.com Signed-off-by: Jonathan Corbet --- Documentation/translations/zh_CN/locking/index.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation/translations/zh_CN/locking/index.rst b/Documentation/translations/zh_CN/locking/index.rst index 700df8a2bb70..76a8be9bf78b 100644 --- a/Documentation/translations/zh_CN/locking/index.rst +++ b/Documentation/translations/zh_CN/locking/index.rst @@ -14,6 +14,8 @@ .. toctree:: :maxdepth: 1 + spinlocks + TODOList: * locktypes @@ -24,7 +26,6 @@ TODOList: * rt-mutex-design * rt-mutex * seqlock - * spinlocks * ww-mutex-design * preempt-locking * pi-futex From ee5956bb9ce637d7743b93ce6749cac883339e67 Mon Sep 17 00:00:00 2001 From: Tang Yizhou Date: Wed, 29 Jun 2022 09:32:40 +0800 Subject: [PATCH 146/318] docs/zh_CN: Add mutex-design Chinese translation Translate locking/mutex-design.rst into Chinese. Signed-off-by: Tang Yizhou Reviewed-by: Alex Shi Reviewed-by: Yanteng Si Link: https://lore.kernel.org/r/20220629013240.65386-3-yizhou.tang@shopee.com Signed-off-by: Jonathan Corbet --- .../translations/zh_CN/locking/index.rst | 2 +- .../zh_CN/locking/mutex-design.rst | 145 ++++++++++++++++++ 2 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 Documentation/translations/zh_CN/locking/mutex-design.rst diff --git a/Documentation/translations/zh_CN/locking/index.rst b/Documentation/translations/zh_CN/locking/index.rst index 76a8be9bf78b..f0b10707668d 100644 --- a/Documentation/translations/zh_CN/locking/index.rst +++ b/Documentation/translations/zh_CN/locking/index.rst @@ -14,6 +14,7 @@ .. toctree:: :maxdepth: 1 + mutex-design spinlocks TODOList: @@ -22,7 +23,6 @@ TODOList: * lockdep-design * lockstat * locktorture - * mutex-design * rt-mutex-design * rt-mutex * seqlock diff --git a/Documentation/translations/zh_CN/locking/mutex-design.rst b/Documentation/translations/zh_CN/locking/mutex-design.rst new file mode 100644 index 000000000000..6aad54372a6c --- /dev/null +++ b/Documentation/translations/zh_CN/locking/mutex-design.rst @@ -0,0 +1,145 @@ +.. SPDX-License-Identifier: GPL-2.0 +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/locking/mutex-design.rst + +:翻译: + + 唐艺舟 Tang Yizhou + +================ +通用互斥锁子系统 +================ + +:初稿: + + Ingo Molnar + +:更新: + + Davidlohr Bueso + +什么是互斥锁？ +-------------- + +在Linux内核中，互斥锁（mutex）指的是一个特殊的加锁原语，它在共享内存系统上 +强制保证序列化，而不仅仅是指在学术界或类似的理论教科书中出现的通用术语“相互 +排斥”。互斥锁是一种睡眠锁，它的行为类似于二进制信号量（semaphores），在 +2006年被引入时[1]，作为后者的替代品。这种新的数据结构提供了许多优点，包括更 +简单的接口，以及在当时更少的代码量（见缺陷）。 + +[1] https://lwn.net/Articles/164802/ + +实现 +---- + +互斥锁由“struct mutex”表示，在include/linux/mutex.h中定义，并在 +kernel/locking/mutex.c中实现。这些锁使用一个原子变量（->owner）来跟踪 +它们生命周期内的锁状态。字段owner实际上包含的是指向当前锁所有者的 +`struct task_struct *` 指针，因此如果无人持有锁，则它的值为空（NULL）。 +由于task_struct的指针至少按L1_CACHE_BYTES对齐，低位（3）被用来存储额外 +的状态（例如，等待者列表非空）。在其最基本的形式中，它还包括一个等待队列和 +一个确保对其序列化访问的自旋锁。此外，CONFIG_MUTEX_SPIN_ON_OWNER=y的 +系统使用一个自旋MCS锁（->osq，译注：MCS是两个人名的合并缩写），在下文的 +（ii）中描述。 + +准备获得一把自旋锁时，有三种可能经过的路径，取决于锁的状态： + +(i) 快速路径：试图通过调用cmpxchg()修改锁的所有者为当前任务，以此原子化地 + 获取锁。这只在无竞争的情况下有效（cmpxchg()检查值是否为0，所以3个状态 + 比特必须为0）。如果锁处在竞争状态，代码进入下一个可能的路径。 + +(ii) 中速路径：也就是乐观自旋，当锁的所有者正在运行并且没有其它优先级更高的 + 任务（need_resched，需要重新调度）准备运行时，当前任务试图自旋来获得 + 锁。原理是，如果锁的所有者正在运行，它很可能不久就会释放锁。互斥锁自旋体 + 使用MCS锁排队，这样只有一个自旋体可以竞争互斥锁。 + + MCS锁（由Mellor-Crummey和Scott提出）是一个简单的自旋锁，它具有一些 + 理想的特性，比如公平，以及每个CPU在试图获得锁时在一个本地变量上自旋。 + 它避免了常见的“检测-设置”自旋锁实现导致的（CPU核间）缓存行回弹 + （cacheline bouncing）这种昂贵的开销。一个类MCS锁是为实现睡眠锁的 + 乐观自旋而专门定制的。这种定制MCS锁的一个重要特性是，它有一个额外的属性， + 当自旋体需要重新调度时，它们能够退出MCS自旋锁队列。这进一步有助于避免 + 以下场景：需要重新调度的MCS自旋体将继续自旋等待自旋体所有者，即将获得 + MCS锁时却直接进入慢速路径。 + +(iii) 慢速路径：最后的手段，如果仍然无法获得锁，该任务会被添加到等待队列中， + 休眠直到被解锁路径唤醒。在通常情况下，它以TASK_UNINTERRUPTIBLE状态 + 阻塞。 + +虽然从形式上看，内核互斥锁是可睡眠的锁，路径(ii)使它实际上成为混合类型。通过 +简单地不中断一个任务并忙着等待几个周期，而不是立即睡眠，这种锁已经被认为显著 +改善一些工作负载的性能。注意，这种技术也被用于读写信号量（rw-semaphores）。 + +语义 +---- + +互斥锁子系统检查并强制执行以下规则: + + - 每次只有一个任务可以持有该互斥锁。 + - 只有锁的所有者可以解锁该互斥锁。 + - 不允许多次解锁。 + - 不允许递归加锁/解锁。 + - 互斥锁只能通过API进行初始化（见下文）。 + - 一个任务不能在持有互斥锁的情况下退出。 + - 持有锁的内存区域不得被释放。 + - 被持有的锁不能被重新初始化。 + - 互斥锁不能用于硬件或软件中断上下文，如小任务（tasklet）和定时器。 + +当CONFIG DEBUG_MUTEXES被启用时，这些语义将被完全强制执行。此外，互斥锁 +调试代码还实现了一些其它特性，使锁的调试更容易、更快速： + + - 当打印到调试输出时，总是使用互斥锁的符号名称。 + - 加锁点跟踪，函数名符号化查找，系统持有的全部锁的列表，打印出它们。 + - 所有者跟踪。 + - 检测自我递归的锁并打印所有相关信息。 + - 检测多任务环形依赖死锁，并打印所有受影响的锁和任务（并且只限于这些任务）。 + + +接口 +---- +静态定义互斥锁:: + + DEFINE_MUTEX(name); + +动态初始化互斥锁:: + + mutex_init(mutex); + +以不可中断方式（uninterruptible）获取互斥锁:: + + void mutex_lock(struct mutex *lock); + void mutex_lock_nested(struct mutex *lock, unsigned int subclass); + int mutex_trylock(struct mutex *lock); + +以可中断方式（interruptible）获取互斥锁:: + + int mutex_lock_interruptible_nested(struct mutex *lock, + unsigned int subclass); + int mutex_lock_interruptible(struct mutex *lock); + +当原子变量减为0时，以可中断方式（interruptible）获取互斥锁:: + + int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock); + +释放互斥锁:: + + void mutex_unlock(struct mutex *lock); + +检测是否已经获取互斥锁:: + + int mutex_is_locked(struct mutex *lock); + +缺陷 +---- + +与它最初的设计和目的不同，'struct mutex' 是内核中最大的锁之一。例如：在 +x86-64上它是32字节，而 'struct semaphore' 是24字节，rw_semaphore是 +40字节。更大的结构体大小意味着更多的CPU缓存和内存占用。 + + +何时使用互斥锁 +-------------- + +总是优先选择互斥锁而不是任何其它锁原语，除非互斥锁的严格语义不合适，和/或临界区 +阻止锁被共享。 From 2bc6430884d5ee0e30ae18652d31f821d8e9ec32 Mon Sep 17 00:00:00 2001 From: David Reaver Date: Sat, 25 Jun 2022 14:15:48 -0700 Subject: [PATCH 147/318] scripts: get_feat.pl: use /usr/bin/env to find perl I tried running `make pdfdocs` on NixOS, but it failed because get_feat.pl uses a shebang line with /usr/bin/perl, and that file path doesn't exist on NixOS. Using the more portable /usr/bin/env perl fixes the problem. Signed-off-by: David Reaver Link: https://lore.kernel.org/r/20220625211548.1200198-1-me@davidreaver.com Signed-off-by: Jonathan Corbet --- scripts/get_feat.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/get_feat.pl b/scripts/get_feat.pl index 76cfb96b59b6..5c5397eeb237 100755 --- a/scripts/get_feat.pl +++ b/scripts/get_feat.pl @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/usr/bin/env perl # SPDX-License-Identifier: GPL-2.0 use strict; From 4d627ef12b409fd149226617180f1cc6088bbf12 Mon Sep 17 00:00:00 2001 From: Akira Yokosawa Date: Sat, 25 Jun 2022 06:58:09 +0900 Subject: [PATCH 148/318] docs/doc-guide: Mention make variable SPHINXDIRS SPHINXDIRS is useful when you want to test build only those documents affected by your changes. Mention it in the "Sphinx Build" section. Signed-off-by: Akira Yokosawa Cc: Maciej W. Rozycki Link: https://lore.kernel.org/r/50d6f0bc-030d-9529-0665-e2b3e7c792d8@gmail.com Signed-off-by: Jonathan Corbet --- Documentation/doc-guide/sphinx.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Documentation/doc-guide/sphinx.rst b/Documentation/doc-guide/sphinx.rst index edc4fa023986..1228b85f6f77 100644 --- a/Documentation/doc-guide/sphinx.rst +++ b/Documentation/doc-guide/sphinx.rst @@ -151,6 +151,13 @@ If the theme is not available, it will fall-back to the classic one. The Sphinx theme can be overridden by using the ``DOCS_THEME`` make variable. +There is another make variable ``SPHINXDIRS``, which is useful when test +building a subset of documentation. For example, you can build documents +under ``Documentation/doc-guide`` by running +``make SPHINXDIRS=doc-guide htmldocs``. +The documentation section of ``make help`` will show you the list of +subdirectories you can specify. + To remove the generated documentation, run ``make cleandocs``. .. [#ink] Having ``inkscape(1)`` from Inkscape (https://inkscape.org) From e67198cc05b8ecbb7b8e2d8ef9fb5c8d26821873 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:25 +0200 Subject: [PATCH 149/318] context_tracking: Take idle eqs entrypoints over RCU The RCU dynticks counter is going to be merged into the context tracking subsystem. Start with moving the idle extended quiescent states entrypoints to context tracking. For now those are dumb redirections to existing RCU calls. [ paulmck: Apply kernel test robot feedback. ] Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- Documentation/RCU/stallwarn.rst | 4 ++-- arch/arm/mach-imx/cpuidle-imx6q.c | 5 +++-- drivers/acpi/processor_idle.c | 5 +++-- drivers/cpuidle/cpuidle.c | 9 +++++---- include/linux/context_tracking.h | 8 ++++++++ include/linux/rcupdate.h | 2 +- kernel/context_tracking.c | 15 +++++++++++++++ kernel/locking/lockdep.c | 2 +- kernel/rcu/Kconfig | 2 ++ kernel/rcu/tree.c | 2 -- kernel/rcu/update.c | 2 +- kernel/sched/idle.c | 10 +++++----- kernel/sched/sched.h | 1 + kernel/time/Kconfig | 6 ++++++ 14 files changed, 53 insertions(+), 20 deletions(-) diff --git a/Documentation/RCU/stallwarn.rst b/Documentation/RCU/stallwarn.rst index 794837eb519b..b95bda7755fa 100644 --- a/Documentation/RCU/stallwarn.rst +++ b/Documentation/RCU/stallwarn.rst @@ -97,8 +97,8 @@ warnings: which will include additional debugging information. - A low-level kernel issue that either fails to invoke one of the - variants of rcu_user_enter(), rcu_user_exit(), rcu_idle_enter(), - rcu_idle_exit(), rcu_irq_enter(), or rcu_irq_exit() on the one + variants of rcu_user_enter(), rcu_user_exit(), ct_idle_enter(), + ct_idle_exit(), rcu_irq_enter(), or rcu_irq_exit() on the one hand, or that invokes one of them too many times on the other. Historically, the most frequent issue has been an omission of either irq_enter() or irq_exit(), which in turn invoke diff --git a/arch/arm/mach-imx/cpuidle-imx6q.c b/arch/arm/mach-imx/cpuidle-imx6q.c index 094337dc1bc7..d086cbae09c3 100644 --- a/arch/arm/mach-imx/cpuidle-imx6q.c +++ b/arch/arm/mach-imx/cpuidle-imx6q.c @@ -3,6 +3,7 @@ * Copyright (C) 2012 Freescale Semiconductor, Inc. */ +#include #include #include #include @@ -24,9 +25,9 @@ static int imx6q_enter_wait(struct cpuidle_device *dev, imx6_set_lpm(WAIT_UNCLOCKED); raw_spin_unlock(&cpuidle_lock); - rcu_idle_enter(); + ct_idle_enter(); cpu_do_idle(); - rcu_idle_exit(); + ct_idle_exit(); raw_spin_lock(&cpuidle_lock); if (num_idle_cpus-- == num_online_cpus()) diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c index 6a5572a1a80c..1401d193a2df 100644 --- a/drivers/acpi/processor_idle.c +++ b/drivers/acpi/processor_idle.c @@ -23,6 +23,7 @@ #include #include #include +#include /* * Include the apic definitions for x86 to have the APIC timer related defines @@ -647,11 +648,11 @@ static int acpi_idle_enter_bm(struct cpuidle_driver *drv, raw_spin_unlock(&c3_lock); } - rcu_idle_enter(); + ct_idle_enter(); acpi_idle_do_entry(cx); - rcu_idle_exit(); + ct_idle_exit(); /* Re-enable bus master arbitration */ if (dis_bm) { diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c index ef2ea1b12cd8..62dd956025f3 100644 --- a/drivers/cpuidle/cpuidle.c +++ b/drivers/cpuidle/cpuidle.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include "cpuidle.h" @@ -150,12 +151,12 @@ static void enter_s2idle_proper(struct cpuidle_driver *drv, */ stop_critical_timings(); if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE)) - rcu_idle_enter(); + ct_idle_enter(); target_state->enter_s2idle(dev, drv, index); if (WARN_ON_ONCE(!irqs_disabled())) local_irq_disable(); if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE)) - rcu_idle_exit(); + ct_idle_exit(); tick_unfreeze(); start_critical_timings(); @@ -233,10 +234,10 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv, stop_critical_timings(); if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE)) - rcu_idle_enter(); + ct_idle_enter(); entered_state = target_state->enter(dev, drv, index); if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE)) - rcu_idle_exit(); + ct_idle_exit(); start_critical_timings(); sched_clock_idle_wakeup_event(); diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h index e35ae66b4794..01abadb2f993 100644 --- a/include/linux/context_tracking.h +++ b/include/linux/context_tracking.h @@ -119,4 +119,12 @@ extern void context_tracking_init(void); static inline void context_tracking_init(void) { } #endif /* CONFIG_CONTEXT_TRACKING_USER_FORCE */ +#ifdef CONFIG_CONTEXT_TRACKING_IDLE +extern void ct_idle_enter(void); +extern void ct_idle_exit(void); +#else +static inline void ct_idle_enter(void) { } +static inline void ct_idle_exit(void) { } +#endif /* !CONFIG_CONTEXT_TRACKING_IDLE */ + #endif diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 1a32036c918c..6ebe754501c3 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -128,7 +128,7 @@ static inline void rcu_nocb_flush_deferred_wakeup(void) { } * @a: Code that RCU needs to pay attention to. * * RCU read-side critical sections are forbidden in the inner idle loop, - * that is, between the rcu_idle_enter() and the rcu_idle_exit() -- RCU + * that is, between the ct_idle_enter() and the ct_idle_exit() -- RCU * will happily ignore any such read-side critical sections. However, * things like powertop need tracepoints in the inner idle loop. * diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index f3dec1be2bf6..c0b3798d4e94 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -22,6 +22,21 @@ #include #include + +#ifdef CONFIG_CONTEXT_TRACKING_IDLE +noinstr void ct_idle_enter(void) +{ + rcu_idle_enter(); +} +EXPORT_SYMBOL_GPL(ct_idle_enter); + +void ct_idle_exit(void) +{ + rcu_idle_exit(); +} +EXPORT_SYMBOL_GPL(ct_idle_exit); +#endif /* #ifdef CONFIG_CONTEXT_TRACKING_IDLE */ + #ifdef CONFIG_CONTEXT_TRACKING_USER #define CREATE_TRACE_POINTS diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index f06b91ca6482..5ea690cb4f7a 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -6570,7 +6570,7 @@ void lockdep_rcu_suspicious(const char *file, const int line, const char *s) /* * If a CPU is in the RCU-free window in idle (ie: in the section - * between rcu_idle_enter() and rcu_idle_exit(), then RCU + * between ct_idle_enter() and ct_idle_exit(), then RCU * considers that CPU to be in an "extended quiescent state", * which means that RCU will be completely ignoring that CPU. * Therefore, rcu_read_lock() and friends have absolutely no diff --git a/kernel/rcu/Kconfig b/kernel/rcu/Kconfig index 1c630e573548..3fa24e63d6f9 100644 --- a/kernel/rcu/Kconfig +++ b/kernel/rcu/Kconfig @@ -8,6 +8,8 @@ menu "RCU Subsystem" config TREE_RCU bool default y if SMP + # Dynticks-idle tracking + select CONTEXT_TRACKING_IDLE help This option selects the RCU implementation that is designed for very large SMP system with hundreds or diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 9a5edab5558c..051fed0844b6 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -664,7 +664,6 @@ void noinstr rcu_idle_enter(void) WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !raw_irqs_disabled()); rcu_eqs_enter(false); } -EXPORT_SYMBOL_GPL(rcu_idle_enter); #ifdef CONFIG_NO_HZ_FULL @@ -904,7 +903,6 @@ void noinstr rcu_idle_exit(void) rcu_eqs_exit(false); raw_local_irq_restore(flags); } -EXPORT_SYMBOL_GPL(rcu_idle_exit); #ifdef CONFIG_NO_HZ_FULL /** diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c index fc7fef575606..147214b2cd68 100644 --- a/kernel/rcu/update.c +++ b/kernel/rcu/update.c @@ -85,7 +85,7 @@ module_param(rcu_normal_after_boot, int, 0444); * and while lockdep is disabled. * * Note that if the CPU is in the idle loop from an RCU point of view (ie: - * that we are in the section between rcu_idle_enter() and rcu_idle_exit()) + * that we are in the section between ct_idle_enter() and ct_idle_exit()) * then rcu_read_lock_held() sets ``*ret`` to false even if the CPU did an * rcu_read_lock(). The reason for this is that RCU ignores CPUs that are * in such a section, considering these as in extended quiescent state, diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index 328cccbee444..f26ab2675f7d 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -53,14 +53,14 @@ static noinline int __cpuidle cpu_idle_poll(void) { trace_cpu_idle(0, smp_processor_id()); stop_critical_timings(); - rcu_idle_enter(); + ct_idle_enter(); local_irq_enable(); while (!tif_need_resched() && (cpu_idle_force_poll || tick_check_broadcast_expired())) cpu_relax(); - rcu_idle_exit(); + ct_idle_exit(); start_critical_timings(); trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id()); @@ -98,12 +98,12 @@ void __cpuidle default_idle_call(void) * * Trace IRQs enable here, then switch off RCU, and have * arch_cpu_idle() use raw_local_irq_enable(). Note that - * rcu_idle_enter() relies on lockdep IRQ state, so switch that + * ct_idle_enter() relies on lockdep IRQ state, so switch that * last -- this is very similar to the entry code. */ trace_hardirqs_on_prepare(); lockdep_hardirqs_on_prepare(); - rcu_idle_enter(); + ct_idle_enter(); lockdep_hardirqs_on(_THIS_IP_); arch_cpu_idle(); @@ -116,7 +116,7 @@ void __cpuidle default_idle_call(void) */ raw_local_irq_disable(); lockdep_hardirqs_off(_THIS_IP_); - rcu_idle_exit(); + ct_idle_exit(); lockdep_hardirqs_on(_THIS_IP_); raw_local_irq_enable(); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 47b89a0fc6e5..0cfe2d0af294 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig index 41f99bcfe9e6..a41753be1a2b 100644 --- a/kernel/time/Kconfig +++ b/kernel/time/Kconfig @@ -76,6 +76,12 @@ config TIME_KUNIT_TEST config CONTEXT_TRACKING bool +config CONTEXT_TRACKING_IDLE + bool + select CONTEXT_TRACKING + help + Tracks idle state on behalf of RCU. + if GENERIC_CLOCKEVENTS menu "Timers subsystem" From 6f0e6c1598b1a3d19fc30db86b6e26d6f881b43d Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:26 +0200 Subject: [PATCH 150/318] context_tracking: Take IRQ eqs entrypoints over RCU The RCU dynticks counter is going to be merged into the context tracking subsystem. Prepare with moving the IRQ extended quiescent states entrypoints to context tracking. For now those are dumb redirection to existing RCU calls. [ paulmck: Apply Stephen Rothwell feedback from -next. ] [ paulmck: Apply Nathan Chancellor feedback. ] Acked-by: Paul E. McKenney Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- .../RCU/Design/Requirements/Requirements.rst | 10 ++++---- Documentation/RCU/stallwarn.rst | 4 ++-- arch/Kconfig | 2 +- arch/arm64/kernel/entry-common.c | 6 ++--- arch/x86/mm/fault.c | 2 +- drivers/cpuidle/cpuidle-psci.c | 8 +++---- drivers/cpuidle/cpuidle-riscv-sbi.c | 8 +++---- include/linux/context_tracking_irq.h | 17 +++++++++++++ include/linux/context_tracking_state.h | 1 + include/linux/entry-common.h | 10 ++++---- include/linux/rcupdate.h | 5 ++-- include/linux/tracepoint.h | 4 ++-- kernel/cfi.c | 4 ++-- kernel/context_tracking.c | 24 +++++++++++++++++-- kernel/cpu_pm.c | 8 +++---- kernel/entry/common.c | 12 +++++----- kernel/softirq.c | 4 ++-- kernel/trace/trace.c | 6 ++--- 18 files changed, 87 insertions(+), 48 deletions(-) create mode 100644 include/linux/context_tracking_irq.h diff --git a/Documentation/RCU/Design/Requirements/Requirements.rst b/Documentation/RCU/Design/Requirements/Requirements.rst index 04ed8bf27a0e..074810c73936 100644 --- a/Documentation/RCU/Design/Requirements/Requirements.rst +++ b/Documentation/RCU/Design/Requirements/Requirements.rst @@ -1844,10 +1844,10 @@ that meets this requirement. Furthermore, NMI handlers can be interrupted by what appear to RCU to be normal interrupts. One way that this can happen is for code that -directly invokes rcu_irq_enter() and rcu_irq_exit() to be called +directly invokes ct_irq_enter() and ct_irq_exit() to be called from an NMI handler. This astonishing fact of life prompted the current -code structure, which has rcu_irq_enter() invoking -rcu_nmi_enter() and rcu_irq_exit() invoking rcu_nmi_exit(). +code structure, which has ct_irq_enter() invoking +rcu_nmi_enter() and ct_irq_exit() invoking rcu_nmi_exit(). And yes, I also learned of this requirement the hard way. Loadable Modules @@ -2195,7 +2195,7 @@ scheduling-clock interrupt be enabled when RCU needs it to be: sections, and RCU believes this CPU to be idle, no problem. This sort of thing is used by some architectures for light-weight exception handlers, which can then avoid the overhead of - rcu_irq_enter() and rcu_irq_exit() at exception entry and + ct_irq_enter() and ct_irq_exit() at exception entry and exit, respectively. Some go further and avoid the entireties of irq_enter() and irq_exit(). Just make very sure you are running some of your tests with @@ -2226,7 +2226,7 @@ scheduling-clock interrupt be enabled when RCU needs it to be: +-----------------------------------------------------------------------+ | **Answer**: | +-----------------------------------------------------------------------+ -| One approach is to do ``rcu_irq_exit();rcu_irq_enter();`` every so | +| One approach is to do ``ct_irq_exit();ct_irq_enter();`` every so | | often. But given that long-running interrupt handlers can cause other | | problems, not least for response time, shouldn't you work to keep | | your interrupt handler's runtime within reasonable bounds? | diff --git a/Documentation/RCU/stallwarn.rst b/Documentation/RCU/stallwarn.rst index b95bda7755fa..ce1f58a9d954 100644 --- a/Documentation/RCU/stallwarn.rst +++ b/Documentation/RCU/stallwarn.rst @@ -98,11 +98,11 @@ warnings: - A low-level kernel issue that either fails to invoke one of the variants of rcu_user_enter(), rcu_user_exit(), ct_idle_enter(), - ct_idle_exit(), rcu_irq_enter(), or rcu_irq_exit() on the one + ct_idle_exit(), ct_irq_enter(), or ct_irq_exit() on the one hand, or that invokes one of them too many times on the other. Historically, the most frequent issue has been an omission of either irq_enter() or irq_exit(), which in turn invoke - rcu_irq_enter() or rcu_irq_exit(), respectively. Building your + ct_irq_enter() or ct_irq_exit(), respectively. Building your kernel with CONFIG_RCU_EQS_DEBUG=y can help track down these types of issues, which sometimes arise in architecture-specific code. diff --git a/arch/Kconfig b/arch/Kconfig index 154b7b78da09..342642be105f 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -782,7 +782,7 @@ config HAVE_CONTEXT_TRACKING_USER Syscalls need to be wrapped inside user_exit()-user_enter(), either optimized behind static key or through the slow path using TIF_NOHZ flag. Exceptions handlers must be wrapped as well. Irqs are already - protected inside rcu_irq_enter/rcu_irq_exit() but preemption or signal + protected inside ct_irq_enter/ct_irq_exit() but preemption or signal handling on irq exit still need to be protected. config HAVE_CONTEXT_TRACKING_USER_OFFSTACK diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index 56cefd33eb8e..8dabe9ec10f1 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -41,7 +41,7 @@ static __always_inline void __enter_from_kernel_mode(struct pt_regs *regs) if (!IS_ENABLED(CONFIG_TINY_RCU) && is_idle_task(current)) { lockdep_hardirqs_off(CALLER_ADDR0); - rcu_irq_enter(); + ct_irq_enter(); trace_hardirqs_off_finish(); regs->exit_rcu = true; @@ -76,7 +76,7 @@ static __always_inline void __exit_to_kernel_mode(struct pt_regs *regs) if (regs->exit_rcu) { trace_hardirqs_on_prepare(); lockdep_hardirqs_on_prepare(); - rcu_irq_exit(); + ct_irq_exit(); lockdep_hardirqs_on(CALLER_ADDR0); return; } @@ -84,7 +84,7 @@ static __always_inline void __exit_to_kernel_mode(struct pt_regs *regs) trace_hardirqs_on(); } else { if (regs->exit_rcu) - rcu_irq_exit(); + ct_irq_exit(); } } diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index fad8faa29d04..971977c438fc 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -1526,7 +1526,7 @@ DEFINE_IDTENTRY_RAW_ERRORCODE(exc_page_fault) /* * Entry handling for valid #PF from kernel mode is slightly - * different: RCU is already watching and rcu_irq_enter() must not + * different: RCU is already watching and ct_irq_enter() must not * be invoked because a kernel fault on a user space address might * sleep. * diff --git a/drivers/cpuidle/cpuidle-psci.c b/drivers/cpuidle/cpuidle-psci.c index 540105ca0781..57bc3e3ae391 100644 --- a/drivers/cpuidle/cpuidle-psci.c +++ b/drivers/cpuidle/cpuidle-psci.c @@ -69,12 +69,12 @@ static int __psci_enter_domain_idle_state(struct cpuidle_device *dev, return -1; /* Do runtime PM to manage a hierarchical CPU toplogy. */ - rcu_irq_enter_irqson(); + ct_irq_enter_irqson(); if (s2idle) dev_pm_genpd_suspend(pd_dev); else pm_runtime_put_sync_suspend(pd_dev); - rcu_irq_exit_irqson(); + ct_irq_exit_irqson(); state = psci_get_domain_state(); if (!state) @@ -82,12 +82,12 @@ static int __psci_enter_domain_idle_state(struct cpuidle_device *dev, ret = psci_cpu_suspend_enter(state) ? -1 : idx; - rcu_irq_enter_irqson(); + ct_irq_enter_irqson(); if (s2idle) dev_pm_genpd_resume(pd_dev); else pm_runtime_get_sync(pd_dev); - rcu_irq_exit_irqson(); + ct_irq_exit_irqson(); cpu_pm_exit(); diff --git a/drivers/cpuidle/cpuidle-riscv-sbi.c b/drivers/cpuidle/cpuidle-riscv-sbi.c index 1151e5e2ba82..862a2876f1c9 100644 --- a/drivers/cpuidle/cpuidle-riscv-sbi.c +++ b/drivers/cpuidle/cpuidle-riscv-sbi.c @@ -116,12 +116,12 @@ static int __sbi_enter_domain_idle_state(struct cpuidle_device *dev, return -1; /* Do runtime PM to manage a hierarchical CPU toplogy. */ - rcu_irq_enter_irqson(); + ct_irq_enter_irqson(); if (s2idle) dev_pm_genpd_suspend(pd_dev); else pm_runtime_put_sync_suspend(pd_dev); - rcu_irq_exit_irqson(); + ct_irq_exit_irqson(); if (sbi_is_domain_state_available()) state = sbi_get_domain_state(); @@ -130,12 +130,12 @@ static int __sbi_enter_domain_idle_state(struct cpuidle_device *dev, ret = sbi_suspend(state) ? -1 : idx; - rcu_irq_enter_irqson(); + ct_irq_enter_irqson(); if (s2idle) dev_pm_genpd_resume(pd_dev); else pm_runtime_get_sync(pd_dev); - rcu_irq_exit_irqson(); + ct_irq_exit_irqson(); cpu_pm_exit(); diff --git a/include/linux/context_tracking_irq.h b/include/linux/context_tracking_irq.h new file mode 100644 index 000000000000..62f62bbd1a50 --- /dev/null +++ b/include/linux/context_tracking_irq.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_CONTEXT_TRACKING_IRQ_H +#define _LINUX_CONTEXT_TRACKING_IRQ_H + +#ifdef CONFIG_CONTEXT_TRACKING_IDLE +void ct_irq_enter(void); +void ct_irq_exit(void); +void ct_irq_enter_irqson(void); +void ct_irq_exit_irqson(void); +#else +static inline void ct_irq_enter(void) { } +static inline void ct_irq_exit(void) { } +static inline void ct_irq_enter_irqson(void) { } +static inline void ct_irq_exit_irqson(void) { } +#endif + +#endif diff --git a/include/linux/context_tracking_state.h b/include/linux/context_tracking_state.h index 2b46afe105a9..9c16a8b2c194 100644 --- a/include/linux/context_tracking_state.h +++ b/include/linux/context_tracking_state.h @@ -4,6 +4,7 @@ #include #include +#include struct context_tracking { /* diff --git a/include/linux/entry-common.h b/include/linux/entry-common.h index c92ac75d6556..84a466b176cf 100644 --- a/include/linux/entry-common.h +++ b/include/linux/entry-common.h @@ -357,7 +357,7 @@ void irqentry_exit_to_user_mode(struct pt_regs *regs); /** * struct irqentry_state - Opaque object for exception state storage * @exit_rcu: Used exclusively in the irqentry_*() calls; signals whether the - * exit path has to invoke rcu_irq_exit(). + * exit path has to invoke ct_irq_exit(). * @lockdep: Used exclusively in the irqentry_nmi_*() calls; ensures that * lockdep state is restored correctly on exit from nmi. * @@ -395,12 +395,12 @@ typedef struct irqentry_state { * * For kernel mode entries RCU handling is done conditional. If RCU is * watching then the only RCU requirement is to check whether the tick has - * to be restarted. If RCU is not watching then rcu_irq_enter() has to be - * invoked on entry and rcu_irq_exit() on exit. + * to be restarted. If RCU is not watching then ct_irq_enter() has to be + * invoked on entry and ct_irq_exit() on exit. * - * Avoiding the rcu_irq_enter/exit() calls is an optimization but also + * Avoiding the ct_irq_enter/exit() calls is an optimization but also * solves the problem of kernel mode pagefaults which can schedule, which - * is not possible after invoking rcu_irq_enter() without undoing it. + * is not possible after invoking ct_irq_enter() without undoing it. * * For user mode entries irqentry_enter_from_user_mode() is invoked to * establish the proper context for NOHZ_FULL. Otherwise scheduling on exit diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 6ebe754501c3..f1562d91c67d 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -29,6 +29,7 @@ #include #include #include +#include #define ULONG_CMP_GE(a, b) (ULONG_MAX / 2 >= (a) - (b)) #define ULONG_CMP_LT(a, b) (ULONG_MAX / 2 < (a) - (b)) @@ -143,9 +144,9 @@ static inline void rcu_nocb_flush_deferred_wakeup(void) { } */ #define RCU_NONIDLE(a) \ do { \ - rcu_irq_enter_irqson(); \ + ct_irq_enter_irqson(); \ do { a; } while (0); \ - rcu_irq_exit_irqson(); \ + ct_irq_exit_irqson(); \ } while (0) /* diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h index 28031b15f878..55717a2eda08 100644 --- a/include/linux/tracepoint.h +++ b/include/linux/tracepoint.h @@ -200,13 +200,13 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p) */ \ if (rcuidle) { \ __idx = srcu_read_lock_notrace(&tracepoint_srcu);\ - rcu_irq_enter_irqson(); \ + ct_irq_enter_irqson(); \ } \ \ __DO_TRACE_CALL(name, TP_ARGS(args)); \ \ if (rcuidle) { \ - rcu_irq_exit_irqson(); \ + ct_irq_exit_irqson(); \ srcu_read_unlock_notrace(&tracepoint_srcu, __idx);\ } \ \ diff --git a/kernel/cfi.c b/kernel/cfi.c index 08102d19ec15..2046276ee234 100644 --- a/kernel/cfi.c +++ b/kernel/cfi.c @@ -295,7 +295,7 @@ static inline cfi_check_fn find_check_fn(unsigned long ptr) rcu_idle = !rcu_is_watching(); if (rcu_idle) { local_irq_save(flags); - rcu_irq_enter(); + ct_irq_enter(); } if (IS_ENABLED(CONFIG_CFI_CLANG_SHADOW)) @@ -304,7 +304,7 @@ static inline cfi_check_fn find_check_fn(unsigned long ptr) fn = find_module_check_fn(ptr); if (rcu_idle) { - rcu_irq_exit(); + ct_irq_exit(); local_irq_restore(flags); } diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index c0b3798d4e94..72bd71a02c44 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -35,6 +35,26 @@ void ct_idle_exit(void) rcu_idle_exit(); } EXPORT_SYMBOL_GPL(ct_idle_exit); + +noinstr void ct_irq_enter(void) +{ + rcu_irq_enter(); +} + +noinstr void ct_irq_exit(void) +{ + rcu_irq_exit(); +} + +void ct_irq_enter_irqson(void) +{ + rcu_irq_enter_irqson(); +} + +void ct_irq_exit_irqson(void) +{ + rcu_irq_exit_irqson(); +} #endif /* #ifdef CONFIG_CONTEXT_TRACKING_IDLE */ #ifdef CONFIG_CONTEXT_TRACKING_USER @@ -90,7 +110,7 @@ void noinstr __ct_user_enter(enum ctx_state state) * At this stage, only low level arch entry code remains and * then we'll run in userspace. We can assume there won't be * any RCU read-side critical section until the next call to - * user_exit() or rcu_irq_enter(). Let's remove RCU's dependency + * user_exit() or ct_irq_enter(). Let's remove RCU's dependency * on the tick. */ if (state == CONTEXT_USER) { @@ -136,7 +156,7 @@ void ct_user_enter(enum ctx_state state) /* * Some contexts may involve an exception occuring in an irq, * leading to that nesting: - * rcu_irq_enter() rcu_user_exit() rcu_user_exit() rcu_irq_exit() + * ct_irq_enter() rcu_user_exit() rcu_user_exit() ct_irq_exit() * This would mess up the dyntick_nesting count though. And rcu_irq_*() * helpers are enough to protect RCU uses inside the exception. So * just return immediately if we detect we are in an IRQ. diff --git a/kernel/cpu_pm.c b/kernel/cpu_pm.c index 246efc74e3f3..ba4ba71facf9 100644 --- a/kernel/cpu_pm.c +++ b/kernel/cpu_pm.c @@ -35,11 +35,11 @@ static int cpu_pm_notify(enum cpu_pm_event event) * disfunctional in cpu idle. Copy RCU_NONIDLE code to let RCU know * this. */ - rcu_irq_enter_irqson(); + ct_irq_enter_irqson(); rcu_read_lock(); ret = raw_notifier_call_chain(&cpu_pm_notifier.chain, event, NULL); rcu_read_unlock(); - rcu_irq_exit_irqson(); + ct_irq_exit_irqson(); return notifier_to_errno(ret); } @@ -49,11 +49,11 @@ static int cpu_pm_notify_robust(enum cpu_pm_event event_up, enum cpu_pm_event ev unsigned long flags; int ret; - rcu_irq_enter_irqson(); + ct_irq_enter_irqson(); raw_spin_lock_irqsave(&cpu_pm_notifier.lock, flags); ret = raw_notifier_call_chain_robust(&cpu_pm_notifier.chain, event_up, event_down, NULL); raw_spin_unlock_irqrestore(&cpu_pm_notifier.lock, flags); - rcu_irq_exit_irqson(); + ct_irq_exit_irqson(); return notifier_to_errno(ret); } diff --git a/kernel/entry/common.c b/kernel/entry/common.c index 032f164abe7c..667ba5d581ff 100644 --- a/kernel/entry/common.c +++ b/kernel/entry/common.c @@ -321,7 +321,7 @@ noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs) } /* - * If this entry hit the idle task invoke rcu_irq_enter() whether + * If this entry hit the idle task invoke ct_irq_enter() whether * RCU is watching or not. * * Interrupts can nest when the first interrupt invokes softirq @@ -332,12 +332,12 @@ noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs) * not nested into another interrupt. * * Checking for rcu_is_watching() here would prevent the nesting - * interrupt to invoke rcu_irq_enter(). If that nested interrupt is + * interrupt to invoke ct_irq_enter(). If that nested interrupt is * the tick then rcu_flavor_sched_clock_irq() would wrongfully * assume that it is the first interrupt and eventually claim * quiescent state and end grace periods prematurely. * - * Unconditionally invoke rcu_irq_enter() so RCU state stays + * Unconditionally invoke ct_irq_enter() so RCU state stays * consistent. * * TINY_RCU does not support EQS, so let the compiler eliminate @@ -350,7 +350,7 @@ noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs) * as in irqentry_enter_from_user_mode(). */ lockdep_hardirqs_off(CALLER_ADDR0); - rcu_irq_enter(); + ct_irq_enter(); instrumentation_begin(); trace_hardirqs_off_finish(); instrumentation_end(); @@ -418,7 +418,7 @@ noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state) trace_hardirqs_on_prepare(); lockdep_hardirqs_on_prepare(); instrumentation_end(); - rcu_irq_exit(); + ct_irq_exit(); lockdep_hardirqs_on(CALLER_ADDR0); return; } @@ -436,7 +436,7 @@ noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state) * was not watching on entry. */ if (state.exit_rcu) - rcu_irq_exit(); + ct_irq_exit(); } } diff --git a/kernel/softirq.c b/kernel/softirq.c index 9f0aef8aa9ff..c8a6913c067d 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -620,7 +620,7 @@ void irq_enter_rcu(void) */ void irq_enter(void) { - rcu_irq_enter(); + ct_irq_enter(); irq_enter_rcu(); } @@ -672,7 +672,7 @@ void irq_exit_rcu(void) void irq_exit(void) { __irq_exit_rcu(); - rcu_irq_exit(); + ct_irq_exit(); /* must be last! */ lockdep_hardirq_exit(); } diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 2c95992e2c71..fe78a6818126 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -3107,15 +3107,15 @@ void __trace_stack(struct trace_array *tr, unsigned int trace_ctx, /* * When an NMI triggers, RCU is enabled via rcu_nmi_enter(), * but if the above rcu_is_watching() failed, then the NMI - * triggered someplace critical, and rcu_irq_enter() should + * triggered someplace critical, and ct_irq_enter() should * not be called from NMI. */ if (unlikely(in_nmi())) return; - rcu_irq_enter_irqson(); + ct_irq_enter_irqson(); __ftrace_trace_stack(buffer, trace_ctx, skip, NULL); - rcu_irq_exit_irqson(); + ct_irq_exit_irqson(); } /** From 493c1822825f00025d6754ec0632990a27edc6f8 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:27 +0200 Subject: [PATCH 151/318] context_tracking: Take NMI eqs entrypoints over RCU The RCU dynticks counter is going to be merged into the context tracking subsystem. Prepare with moving the NMI extended quiescent states entrypoints to context tracking. For now those are dumb redirection to existing RCU calls. Acked-by: Paul E. McKenney Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- Documentation/RCU/Design/Requirements/Requirements.rst | 2 +- arch/Kconfig | 2 +- arch/arm64/kernel/entry-common.c | 8 ++++---- include/linux/context_tracking_irq.h | 4 ++++ include/linux/hardirq.h | 4 ++-- kernel/context_tracking.c | 10 ++++++++++ kernel/entry/common.c | 4 ++-- kernel/extable.c | 4 ++-- kernel/trace/trace.c | 2 +- 9 files changed, 27 insertions(+), 13 deletions(-) diff --git a/Documentation/RCU/Design/Requirements/Requirements.rst b/Documentation/RCU/Design/Requirements/Requirements.rst index 074810c73936..a0f8164c8513 100644 --- a/Documentation/RCU/Design/Requirements/Requirements.rst +++ b/Documentation/RCU/Design/Requirements/Requirements.rst @@ -1847,7 +1847,7 @@ normal interrupts. One way that this can happen is for code that directly invokes ct_irq_enter() and ct_irq_exit() to be called from an NMI handler. This astonishing fact of life prompted the current code structure, which has ct_irq_enter() invoking -rcu_nmi_enter() and ct_irq_exit() invoking rcu_nmi_exit(). +ct_nmi_enter() and ct_irq_exit() invoking ct_nmi_exit(). And yes, I also learned of this requirement the hard way. Loadable Modules diff --git a/arch/Kconfig b/arch/Kconfig index 342642be105f..f56f7c0e924d 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -797,7 +797,7 @@ config HAVE_CONTEXT_TRACKING_USER_OFFSTACK - Critical entry code isn't preemptible (or better yet: not interruptible). - - No use of RCU read side critical sections, unless rcu_nmi_enter() + - No use of RCU read side critical sections, unless ct_nmi_enter() got called. - No use of instrumentation, unless instrumentation_begin() got called. diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index 8dabe9ec10f1..c75ca36b4a49 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -161,7 +161,7 @@ static void noinstr arm64_enter_nmi(struct pt_regs *regs) __nmi_enter(); lockdep_hardirqs_off(CALLER_ADDR0); lockdep_hardirq_enter(); - rcu_nmi_enter(); + ct_nmi_enter(); trace_hardirqs_off_finish(); ftrace_nmi_enter(); @@ -182,7 +182,7 @@ static void noinstr arm64_exit_nmi(struct pt_regs *regs) lockdep_hardirqs_on_prepare(); } - rcu_nmi_exit(); + ct_nmi_exit(); lockdep_hardirq_exit(); if (restore) lockdep_hardirqs_on(CALLER_ADDR0); @@ -199,7 +199,7 @@ static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs) regs->lockdep_hardirqs = lockdep_hardirqs_enabled(); lockdep_hardirqs_off(CALLER_ADDR0); - rcu_nmi_enter(); + ct_nmi_enter(); trace_hardirqs_off_finish(); } @@ -218,7 +218,7 @@ static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs) lockdep_hardirqs_on_prepare(); } - rcu_nmi_exit(); + ct_nmi_exit(); if (restore) lockdep_hardirqs_on(CALLER_ADDR0); } diff --git a/include/linux/context_tracking_irq.h b/include/linux/context_tracking_irq.h index 62f62bbd1a50..c50b5670c4a5 100644 --- a/include/linux/context_tracking_irq.h +++ b/include/linux/context_tracking_irq.h @@ -7,11 +7,15 @@ void ct_irq_enter(void); void ct_irq_exit(void); void ct_irq_enter_irqson(void); void ct_irq_exit_irqson(void); +void ct_nmi_enter(void); +void ct_nmi_exit(void); #else static inline void ct_irq_enter(void) { } static inline void ct_irq_exit(void) { } static inline void ct_irq_enter_irqson(void) { } static inline void ct_irq_exit_irqson(void) { } +static inline void ct_nmi_enter(void) { } +static inline void ct_nmi_exit(void) { } #endif #endif diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h index 76878b357ffa..345cdbe9c1b7 100644 --- a/include/linux/hardirq.h +++ b/include/linux/hardirq.h @@ -124,7 +124,7 @@ extern void rcu_nmi_exit(void); do { \ __nmi_enter(); \ lockdep_hardirq_enter(); \ - rcu_nmi_enter(); \ + ct_nmi_enter(); \ instrumentation_begin(); \ ftrace_nmi_enter(); \ instrumentation_end(); \ @@ -143,7 +143,7 @@ extern void rcu_nmi_exit(void); instrumentation_begin(); \ ftrace_nmi_exit(); \ instrumentation_end(); \ - rcu_nmi_exit(); \ + ct_nmi_exit(); \ lockdep_hardirq_exit(); \ __nmi_exit(); \ } while (0) diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index 72bd71a02c44..b8a731f20778 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -55,6 +55,16 @@ void ct_irq_exit_irqson(void) { rcu_irq_exit_irqson(); } + +noinstr void ct_nmi_enter(void) +{ + rcu_nmi_enter(); +} + +noinstr void ct_nmi_exit(void) +{ + rcu_nmi_exit(); +} #endif /* #ifdef CONFIG_CONTEXT_TRACKING_IDLE */ #ifdef CONFIG_CONTEXT_TRACKING_USER diff --git a/kernel/entry/common.c b/kernel/entry/common.c index 667ba5d581ff..063068a9ea9b 100644 --- a/kernel/entry/common.c +++ b/kernel/entry/common.c @@ -449,7 +449,7 @@ irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs) __nmi_enter(); lockdep_hardirqs_off(CALLER_ADDR0); lockdep_hardirq_enter(); - rcu_nmi_enter(); + ct_nmi_enter(); instrumentation_begin(); trace_hardirqs_off_finish(); @@ -469,7 +469,7 @@ void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t irq_state) } instrumentation_end(); - rcu_nmi_exit(); + ct_nmi_exit(); lockdep_hardirq_exit(); if (irq_state.lockdep) lockdep_hardirqs_on(CALLER_ADDR0); diff --git a/kernel/extable.c b/kernel/extable.c index bda5e9761541..71f482581cab 100644 --- a/kernel/extable.c +++ b/kernel/extable.c @@ -114,7 +114,7 @@ int kernel_text_address(unsigned long addr) /* Treat this like an NMI as it can happen anywhere */ if (no_rcu) - rcu_nmi_enter(); + ct_nmi_enter(); if (is_module_text_address(addr)) goto out; @@ -127,7 +127,7 @@ int kernel_text_address(unsigned long addr) ret = 0; out: if (no_rcu) - rcu_nmi_exit(); + ct_nmi_exit(); return ret; } diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index fe78a6818126..5fc7f17f5ec7 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -3105,7 +3105,7 @@ void __trace_stack(struct trace_array *tr, unsigned int trace_ctx, } /* - * When an NMI triggers, RCU is enabled via rcu_nmi_enter(), + * When an NMI triggers, RCU is enabled via ct_nmi_enter(), * but if the above rcu_is_watching() failed, then the NMI * triggered someplace critical, and ct_irq_enter() should * not be called from NMI. From 3864caafe7c66f01b188ffccb6a4215f3bf56292 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:28 +0200 Subject: [PATCH 152/318] rcu/context-tracking: Remove rcu_irq_enter/exit() Now rcu_irq_enter/exit() is an unnecessary middle call between ct_irq_enter/exit() and nmi_irq_enter/exit(). Take this opportunity to remove the former functions and move the comments above them to the new entrypoints. Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- include/linux/rcutiny.h | 4 -- include/linux/rcutree.h | 4 -- kernel/context_tracking.c | 71 +++++++++++++++++++++++++++++++-- kernel/rcu/tree.c | 83 --------------------------------------- 4 files changed, 67 insertions(+), 95 deletions(-) diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h index 5fed476f977f..591119413cf1 100644 --- a/include/linux/rcutiny.h +++ b/include/linux/rcutiny.h @@ -78,10 +78,6 @@ static inline void rcu_cpu_stall_reset(void) { } static inline int rcu_jiffies_till_stall_check(void) { return 21 * HZ; } static inline void rcu_idle_enter(void) { } static inline void rcu_idle_exit(void) { } -static inline void rcu_irq_enter(void) { } -static inline void rcu_irq_exit_irqson(void) { } -static inline void rcu_irq_enter_irqson(void) { } -static inline void rcu_irq_exit(void) { } static inline void rcu_irq_exit_check_preempt(void) { } #define rcu_is_idle_cpu(cpu) \ (is_idle_task(current) && !in_nmi() && !in_hardirq() && !in_serving_softirq()) diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index 9c6cfb742504..4522b6a7cc42 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h @@ -47,10 +47,6 @@ void cond_synchronize_rcu(unsigned long oldstate); void rcu_idle_enter(void); void rcu_idle_exit(void); -void rcu_irq_enter(void); -void rcu_irq_exit(void); -void rcu_irq_enter_irqson(void); -void rcu_irq_exit_irqson(void); bool rcu_is_idle_cpu(int cpu); #ifdef CONFIG_PROVE_RCU diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index b8a731f20778..c0d86dac98f1 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -36,24 +36,87 @@ void ct_idle_exit(void) } EXPORT_SYMBOL_GPL(ct_idle_exit); +/** + * ct_irq_enter - inform RCU that current CPU is entering irq away from idle + * + * Enter an interrupt handler, which might possibly result in exiting + * idle mode, in other words, entering the mode in which read-side critical + * sections can occur. The caller must have disabled interrupts. + * + * Note that the Linux kernel is fully capable of entering an interrupt + * handler that it never exits, for example when doing upcalls to user mode! + * This code assumes that the idle loop never does upcalls to user mode. + * If your architecture's idle loop does do upcalls to user mode (or does + * anything else that results in unbalanced calls to the irq_enter() and + * irq_exit() functions), RCU will give you what you deserve, good and hard. + * But very infrequently and irreproducibly. + * + * Use things like work queues to work around this limitation. + * + * You have been warned. + * + * If you add or remove a call to ct_irq_enter(), be sure to test with + * CONFIG_RCU_EQS_DEBUG=y. + */ noinstr void ct_irq_enter(void) { - rcu_irq_enter(); + lockdep_assert_irqs_disabled(); + ct_nmi_enter(); } +/** + * ct_irq_exit - inform RCU that current CPU is exiting irq towards idle + * + * Exit from an interrupt handler, which might possibly result in entering + * idle mode, in other words, leaving the mode in which read-side critical + * sections can occur. The caller must have disabled interrupts. + * + * This code assumes that the idle loop never does anything that might + * result in unbalanced calls to irq_enter() and irq_exit(). If your + * architecture's idle loop violates this assumption, RCU will give you what + * you deserve, good and hard. But very infrequently and irreproducibly. + * + * Use things like work queues to work around this limitation. + * + * You have been warned. + * + * If you add or remove a call to ct_irq_exit(), be sure to test with + * CONFIG_RCU_EQS_DEBUG=y. + */ noinstr void ct_irq_exit(void) { - rcu_irq_exit(); + lockdep_assert_irqs_disabled(); + ct_nmi_exit(); } +/* + * Wrapper for ct_irq_enter() where interrupts are enabled. + * + * If you add or remove a call to ct_irq_enter_irqson(), be sure to test + * with CONFIG_RCU_EQS_DEBUG=y. + */ void ct_irq_enter_irqson(void) { - rcu_irq_enter_irqson(); + unsigned long flags; + + local_irq_save(flags); + ct_irq_enter(); + local_irq_restore(flags); } +/* + * Wrapper for ct_irq_exit() where interrupts are enabled. + * + * If you add or remove a call to ct_irq_exit_irqson(), be sure to test + * with CONFIG_RCU_EQS_DEBUG=y. + */ void ct_irq_exit_irqson(void) { - rcu_irq_exit_irqson(); + unsigned long flags; + + local_irq_save(flags); + ct_irq_exit(); + local_irq_restore(flags); } noinstr void ct_nmi_enter(void) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 051fed0844b6..75b433dba427 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -789,31 +789,6 @@ noinstr void rcu_nmi_exit(void) rcu_dynticks_task_enter(); } -/** - * rcu_irq_exit - inform RCU that current CPU is exiting irq towards idle - * - * Exit from an interrupt handler, which might possibly result in entering - * idle mode, in other words, leaving the mode in which read-side critical - * sections can occur. The caller must have disabled interrupts. - * - * This code assumes that the idle loop never does anything that might - * result in unbalanced calls to irq_enter() and irq_exit(). If your - * architecture's idle loop violates this assumption, RCU will give you what - * you deserve, good and hard. But very infrequently and irreproducibly. - * - * Use things like work queues to work around this limitation. - * - * You have been warned. - * - * If you add or remove a call to rcu_irq_exit(), be sure to test with - * CONFIG_RCU_EQS_DEBUG=y. - */ -void noinstr rcu_irq_exit(void) -{ - lockdep_assert_irqs_disabled(); - rcu_nmi_exit(); -} - #ifdef CONFIG_PROVE_RCU /** * rcu_irq_exit_check_preempt - Validate that scheduling is possible @@ -832,21 +807,6 @@ void rcu_irq_exit_check_preempt(void) } #endif /* #ifdef CONFIG_PROVE_RCU */ -/* - * Wrapper for rcu_irq_exit() where interrupts are enabled. - * - * If you add or remove a call to rcu_irq_exit_irqson(), be sure to test - * with CONFIG_RCU_EQS_DEBUG=y. - */ -void rcu_irq_exit_irqson(void) -{ - unsigned long flags; - - local_irq_save(flags); - rcu_irq_exit(); - local_irq_restore(flags); -} - /* * Exit an RCU extended quiescent state, which can be either the * idle loop or adaptive-tickless usermode execution. @@ -1041,49 +1001,6 @@ noinstr void rcu_nmi_enter(void) barrier(); } -/** - * rcu_irq_enter - inform RCU that current CPU is entering irq away from idle - * - * Enter an interrupt handler, which might possibly result in exiting - * idle mode, in other words, entering the mode in which read-side critical - * sections can occur. The caller must have disabled interrupts. - * - * Note that the Linux kernel is fully capable of entering an interrupt - * handler that it never exits, for example when doing upcalls to user mode! - * This code assumes that the idle loop never does upcalls to user mode. - * If your architecture's idle loop does do upcalls to user mode (or does - * anything else that results in unbalanced calls to the irq_enter() and - * irq_exit() functions), RCU will give you what you deserve, good and hard. - * But very infrequently and irreproducibly. - * - * Use things like work queues to work around this limitation. - * - * You have been warned. - * - * If you add or remove a call to rcu_irq_enter(), be sure to test with - * CONFIG_RCU_EQS_DEBUG=y. - */ -noinstr void rcu_irq_enter(void) -{ - lockdep_assert_irqs_disabled(); - rcu_nmi_enter(); -} - -/* - * Wrapper for rcu_irq_enter() where interrupts are enabled. - * - * If you add or remove a call to rcu_irq_enter_irqson(), be sure to test - * with CONFIG_RCU_EQS_DEBUG=y. - */ -void rcu_irq_enter_irqson(void) -{ - unsigned long flags; - - local_irq_save(flags); - rcu_irq_enter(); - local_irq_restore(flags); -} - /* * Check to see if any future non-offloaded RCU-related work will need * to be done by the current CPU, even if none need be done immediately, From 62e2412df4b90ae6706ce1f1a9649b789b2e44ef Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:29 +0200 Subject: [PATCH 153/318] rcu/context_tracking: Move dynticks counter to context tracking In order to prepare for merging RCU dynticks counter into the context tracking state, move the rcu_data's dynticks field to the context tracking structure. It will later be mixed within the context tracking state itself. [ paulmck: Move enum ctx_state into global scope. ] Acked-by: Paul E. McKenney Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- include/linux/context_tracking_state.h | 45 +++++++++++++++++---- kernel/context_tracking.c | 10 +++-- kernel/rcu/tree.c | 56 +++++++++++++------------- kernel/rcu/tree.h | 1 - kernel/rcu/tree_exp.h | 2 +- kernel/rcu/tree_stall.h | 4 +- 6 files changed, 75 insertions(+), 43 deletions(-) diff --git a/include/linux/context_tracking_state.h b/include/linux/context_tracking_state.h index 9c16a8b2c194..5a8da2787287 100644 --- a/include/linux/context_tracking_state.h +++ b/include/linux/context_tracking_state.h @@ -6,7 +6,15 @@ #include #include +enum ctx_state { + CONTEXT_DISABLED = -1, /* returned by ct_state() if unknown */ + CONTEXT_KERNEL = 0, + CONTEXT_USER, + CONTEXT_GUEST, +}; + struct context_tracking { +#ifdef CONFIG_CONTEXT_TRACKING_USER /* * When active is false, probes are unset in order * to minimize overhead: TIF flags are cleared @@ -15,17 +23,40 @@ struct context_tracking { */ bool active; int recursion; - enum ctx_state { - CONTEXT_DISABLED = -1, /* returned by ct_state() if unknown */ - CONTEXT_KERNEL = 0, - CONTEXT_USER, - CONTEXT_GUEST, - } state; + enum ctx_state state; +#endif +#ifdef CONFIG_CONTEXT_TRACKING_IDLE + atomic_t dynticks; /* Even value for idle, else odd. */ +#endif }; +#ifdef CONFIG_CONTEXT_TRACKING +DECLARE_PER_CPU(struct context_tracking, context_tracking); +#endif + +#ifdef CONFIG_CONTEXT_TRACKING_IDLE +static __always_inline int ct_dynticks(void) +{ + return atomic_read(this_cpu_ptr(&context_tracking.dynticks)); +} + +static __always_inline int ct_dynticks_cpu(int cpu) +{ + struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu); + + return atomic_read(&ct->dynticks); +} + +static __always_inline int ct_dynticks_cpu_acquire(int cpu) +{ + struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu); + + return atomic_read_acquire(&ct->dynticks); +} +#endif /* #ifdef CONFIG_CONTEXT_TRACKING_IDLE */ + #ifdef CONFIG_CONTEXT_TRACKING_USER extern struct static_key_false context_tracking_key; -DECLARE_PER_CPU(struct context_tracking, context_tracking); static __always_inline bool context_tracking_enabled(void) { diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index c0d86dac98f1..01abbcec52f7 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -23,6 +23,13 @@ #include +DEFINE_PER_CPU(struct context_tracking, context_tracking) = { +#ifdef CONFIG_CONTEXT_TRACKING_IDLE + .dynticks = ATOMIC_INIT(1), +#endif +}; +EXPORT_SYMBOL_GPL(context_tracking); + #ifdef CONFIG_CONTEXT_TRACKING_IDLE noinstr void ct_idle_enter(void) { @@ -138,9 +145,6 @@ noinstr void ct_nmi_exit(void) DEFINE_STATIC_KEY_FALSE(context_tracking_key); EXPORT_SYMBOL_GPL(context_tracking_key); -DEFINE_PER_CPU(struct context_tracking, context_tracking); -EXPORT_SYMBOL_GPL(context_tracking); - static noinstr bool context_tracking_recursion_enter(void) { int recursion; diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 75b433dba427..a471edc3d893 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -77,7 +77,6 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = { .dynticks_nesting = 1, .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE, - .dynticks = ATOMIC_INIT(1), #ifdef CONFIG_RCU_NOCB_CPU .cblist.flags = SEGCBLIST_RCU_CORE, #endif @@ -268,7 +267,7 @@ void rcu_softirq_qs(void) */ static noinline noinstr unsigned long rcu_dynticks_inc(int incby) { - return arch_atomic_add_return(incby, this_cpu_ptr(&rcu_data.dynticks)); + return arch_atomic_add_return(incby, this_cpu_ptr(&context_tracking.dynticks)); } /* @@ -324,9 +323,7 @@ static noinstr void rcu_dynticks_eqs_exit(void) */ static void rcu_dynticks_eqs_online(void) { - struct rcu_data *rdp = this_cpu_ptr(&rcu_data); - - if (atomic_read(&rdp->dynticks) & 0x1) + if (ct_dynticks() & 0x1) return; rcu_dynticks_inc(1); } @@ -338,17 +335,17 @@ static void rcu_dynticks_eqs_online(void) */ static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void) { - return !(arch_atomic_read(this_cpu_ptr(&rcu_data.dynticks)) & 0x1); + return !(arch_atomic_read(this_cpu_ptr(&context_tracking.dynticks)) & 0x1); } /* * Snapshot the ->dynticks counter with full ordering so as to allow * stable comparison of this counter with past and future snapshots. */ -static int rcu_dynticks_snap(struct rcu_data *rdp) +static int rcu_dynticks_snap(int cpu) { smp_mb(); // Fundamental RCU ordering guarantee. - return atomic_read_acquire(&rdp->dynticks); + return ct_dynticks_cpu_acquire(cpu); } /* @@ -363,9 +360,7 @@ static bool rcu_dynticks_in_eqs(int snap) /* Return true if the specified CPU is currently idle from an RCU viewpoint. */ bool rcu_is_idle_cpu(int cpu) { - struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); - - return rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp)); + return rcu_dynticks_in_eqs(rcu_dynticks_snap(cpu)); } /* @@ -375,7 +370,7 @@ bool rcu_is_idle_cpu(int cpu) */ static bool rcu_dynticks_in_eqs_since(struct rcu_data *rdp, int snap) { - return snap != rcu_dynticks_snap(rdp); + return snap != rcu_dynticks_snap(rdp->cpu); } /* @@ -384,11 +379,10 @@ static bool rcu_dynticks_in_eqs_since(struct rcu_data *rdp, int snap) */ bool rcu_dynticks_zero_in_eqs(int cpu, int *vp) { - struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); int snap; // If not quiescent, force back to earlier extended quiescent state. - snap = atomic_read(&rdp->dynticks) & ~0x1; + snap = ct_dynticks_cpu(cpu) & ~0x1; smp_rmb(); // Order ->dynticks and *vp reads. if (READ_ONCE(*vp)) @@ -396,7 +390,7 @@ bool rcu_dynticks_zero_in_eqs(int cpu, int *vp) smp_rmb(); // Order *vp read and ->dynticks re-read. // If still in the same extended quiescent state, we are good! - return snap == atomic_read(&rdp->dynticks); + return snap == ct_dynticks_cpu(cpu); } /* @@ -620,6 +614,7 @@ EXPORT_SYMBOL_GPL(rcutorture_get_gp_data); static noinstr void rcu_eqs_enter(bool user) { struct rcu_data *rdp = this_cpu_ptr(&rcu_data); + struct context_tracking *ct = this_cpu_ptr(&context_tracking); WARN_ON_ONCE(rdp->dynticks_nmi_nesting != DYNTICK_IRQ_NONIDLE); WRITE_ONCE(rdp->dynticks_nmi_nesting, 0); @@ -633,12 +628,12 @@ static noinstr void rcu_eqs_enter(bool user) instrumentation_begin(); lockdep_assert_irqs_disabled(); - trace_rcu_dyntick(TPS("Start"), rdp->dynticks_nesting, 0, atomic_read(&rdp->dynticks)); + trace_rcu_dyntick(TPS("Start"), rdp->dynticks_nesting, 0, ct_dynticks()); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); rcu_preempt_deferred_qs(current); // instrumentation for the noinstr rcu_dynticks_eqs_enter() - instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks)); + instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); instrumentation_end(); WRITE_ONCE(rdp->dynticks_nesting, 0); /* Avoid irq-access tearing. */ @@ -740,7 +735,7 @@ noinstr void rcu_user_enter(void) * rcu_nmi_exit - inform RCU of exit from NMI context * * If we are returning from the outermost NMI handler that interrupted an - * RCU-idle period, update rdp->dynticks and rdp->dynticks_nmi_nesting + * RCU-idle period, update ct->dynticks and rdp->dynticks_nmi_nesting * to let the RCU grace-period handling know that the CPU is back to * being RCU-idle. * @@ -749,6 +744,7 @@ noinstr void rcu_user_enter(void) */ noinstr void rcu_nmi_exit(void) { + struct context_tracking *ct = this_cpu_ptr(&context_tracking); struct rcu_data *rdp = this_cpu_ptr(&rcu_data); instrumentation_begin(); @@ -766,7 +762,7 @@ noinstr void rcu_nmi_exit(void) */ if (rdp->dynticks_nmi_nesting != 1) { trace_rcu_dyntick(TPS("--="), rdp->dynticks_nmi_nesting, rdp->dynticks_nmi_nesting - 2, - atomic_read(&rdp->dynticks)); + ct_dynticks()); WRITE_ONCE(rdp->dynticks_nmi_nesting, /* No store tearing. */ rdp->dynticks_nmi_nesting - 2); instrumentation_end(); @@ -774,11 +770,11 @@ noinstr void rcu_nmi_exit(void) } /* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */ - trace_rcu_dyntick(TPS("Startirq"), rdp->dynticks_nmi_nesting, 0, atomic_read(&rdp->dynticks)); + trace_rcu_dyntick(TPS("Startirq"), rdp->dynticks_nmi_nesting, 0, ct_dynticks()); WRITE_ONCE(rdp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */ // instrumentation for the noinstr rcu_dynticks_eqs_enter() - instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks)); + instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); instrumentation_end(); // RCU is watching here ... @@ -817,6 +813,7 @@ void rcu_irq_exit_check_preempt(void) */ static void noinstr rcu_eqs_exit(bool user) { + struct context_tracking *ct = this_cpu_ptr(&context_tracking); struct rcu_data *rdp; long oldval; @@ -836,9 +833,9 @@ static void noinstr rcu_eqs_exit(bool user) instrumentation_begin(); // instrumentation for the noinstr rcu_dynticks_eqs_exit() - instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks)); + instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); - trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, atomic_read(&rdp->dynticks)); + trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, ct_dynticks()); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); WRITE_ONCE(rdp->dynticks_nesting, 1); WARN_ON_ONCE(rdp->dynticks_nmi_nesting); @@ -944,7 +941,7 @@ void __rcu_irq_enter_check_tick(void) /** * rcu_nmi_enter - inform RCU of entry to NMI context * - * If the CPU was idle from RCU's viewpoint, update rdp->dynticks and + * If the CPU was idle from RCU's viewpoint, update ct->dynticks and * rdp->dynticks_nmi_nesting to let the RCU grace-period handling know * that the CPU is active. This implementation permits nested NMIs, as * long as the nesting level does not overflow an int. (You will probably @@ -957,6 +954,7 @@ noinstr void rcu_nmi_enter(void) { long incby = 2; struct rcu_data *rdp = this_cpu_ptr(&rcu_data); + struct context_tracking *ct = this_cpu_ptr(&context_tracking); /* Complain about underflow. */ WARN_ON_ONCE(rdp->dynticks_nmi_nesting < 0); @@ -980,9 +978,9 @@ noinstr void rcu_nmi_enter(void) instrumentation_begin(); // instrumentation for the noinstr rcu_dynticks_curr_cpu_in_eqs() - instrument_atomic_read(&rdp->dynticks, sizeof(rdp->dynticks)); + instrument_atomic_read(&ct->dynticks, sizeof(ct->dynticks)); // instrumentation for the noinstr rcu_dynticks_eqs_exit() - instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks)); + instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); incby = 1; } else if (!in_nmi()) { @@ -994,7 +992,7 @@ noinstr void rcu_nmi_enter(void) trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="), rdp->dynticks_nmi_nesting, - rdp->dynticks_nmi_nesting + incby, atomic_read(&rdp->dynticks)); + rdp->dynticks_nmi_nesting + incby, ct_dynticks()); instrumentation_end(); WRITE_ONCE(rdp->dynticks_nmi_nesting, /* Prevent store tearing. */ rdp->dynticks_nmi_nesting + incby); @@ -1138,7 +1136,7 @@ static void rcu_gpnum_ovf(struct rcu_node *rnp, struct rcu_data *rdp) */ static int dyntick_save_progress_counter(struct rcu_data *rdp) { - rdp->dynticks_snap = rcu_dynticks_snap(rdp); + rdp->dynticks_snap = rcu_dynticks_snap(rdp->cpu); if (rcu_dynticks_in_eqs(rdp->dynticks_snap)) { trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti")); rcu_gpnum_ovf(rdp->mynode, rdp); @@ -4142,7 +4140,7 @@ rcu_boot_init_percpu_data(int cpu) rdp->grpmask = leaf_node_cpu_bit(rdp->mynode, cpu); INIT_WORK(&rdp->strict_work, strict_work_handler); WARN_ON_ONCE(rdp->dynticks_nesting != 1); - WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp))); + WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(cpu))); rdp->barrier_seq_snap = rcu_state.barrier_sequence; rdp->rcu_ofl_gp_seq = rcu_state.gp_seq; rdp->rcu_ofl_gp_flags = RCU_GP_CLEANED; diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 2ccf5845957d..ebb973f5b190 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -189,7 +189,6 @@ struct rcu_data { int dynticks_snap; /* Per-GP tracking for dynticks. */ long dynticks_nesting; /* Track process nesting level. */ long dynticks_nmi_nesting; /* Track irq/NMI nesting level. */ - atomic_t dynticks; /* Even value for idle, else odd. */ bool rcu_need_heavy_qs; /* GP old, so heavy quiescent state! */ bool rcu_urgent_qs; /* GP old need light quiescent state. */ bool rcu_forced_tick; /* Forced tick to provide QS. */ diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index 0f70f62039a9..75c22d1034c1 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -356,7 +356,7 @@ static void __sync_rcu_exp_select_node_cpus(struct rcu_exp_work *rewp) !(rnp->qsmaskinitnext & mask)) { mask_ofl_test |= mask; } else { - snap = rcu_dynticks_snap(rdp); + snap = rcu_dynticks_snap(cpu); if (rcu_dynticks_in_eqs(snap)) mask_ofl_test |= mask; else diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index 3556637768fd..250fbf2e8522 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -465,7 +465,7 @@ static void print_cpu_stall_info(int cpu) } delta = rcu_seq_ctr(rdp->mynode->gp_seq - rdp->rcu_iw_gp_seq); falsepositive = rcu_is_gp_kthread_starving(NULL) && - rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp)); + rcu_dynticks_in_eqs(rcu_dynticks_snap(cpu)); rcuc_starved = rcu_is_rcuc_kthread_starving(rdp, &j); if (rcuc_starved) sprintf(buf, " rcuc=%ld jiffies(starved)", j); @@ -478,7 +478,7 @@ static void print_cpu_stall_info(int cpu) rdp->rcu_iw_pending ? (int)min(delta, 9UL) + '0' : "!."[!delta], ticks_value, ticks_title, - rcu_dynticks_snap(rdp) & 0xfff, + rcu_dynticks_snap(cpu) & 0xfff, rdp->dynticks_nesting, rdp->dynticks_nmi_nesting, rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu), data_race(rcu_state.n_force_qs) - rcu_state.n_force_qs_gpstart, From 904e600e60f46f92eb4bcfb95788b1fedf7e8237 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:30 +0200 Subject: [PATCH 154/318] rcu/context_tracking: Move dynticks_nesting to context tracking The RCU eqs tracking is going to be performed by the context tracking subsystem. The related nesting counters thus need to be moved to the context tracking structure. Acked-by: Paul E. McKenney Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- include/linux/context_tracking_state.h | 13 +++++++++++ kernel/context_tracking.c | 1 + kernel/rcu/tree.c | 31 +++++++++++++------------- kernel/rcu/tree.h | 1 - kernel/rcu/tree_stall.h | 2 +- 5 files changed, 31 insertions(+), 17 deletions(-) diff --git a/include/linux/context_tracking_state.h b/include/linux/context_tracking_state.h index 5a8da2787287..13a4a9d1ec7e 100644 --- a/include/linux/context_tracking_state.h +++ b/include/linux/context_tracking_state.h @@ -27,6 +27,7 @@ struct context_tracking { #endif #ifdef CONFIG_CONTEXT_TRACKING_IDLE atomic_t dynticks; /* Even value for idle, else odd. */ + long dynticks_nesting; /* Track process nesting level. */ #endif }; @@ -53,6 +54,18 @@ static __always_inline int ct_dynticks_cpu_acquire(int cpu) return atomic_read_acquire(&ct->dynticks); } + +static __always_inline long ct_dynticks_nesting(void) +{ + return __this_cpu_read(context_tracking.dynticks_nesting); +} + +static __always_inline long ct_dynticks_nesting_cpu(int cpu) +{ + struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu); + + return ct->dynticks_nesting; +} #endif /* #ifdef CONFIG_CONTEXT_TRACKING_IDLE */ #ifdef CONFIG_CONTEXT_TRACKING_USER diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index 01abbcec52f7..dfefe04400f8 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -25,6 +25,7 @@ DEFINE_PER_CPU(struct context_tracking, context_tracking) = { #ifdef CONFIG_CONTEXT_TRACKING_IDLE + .dynticks_nesting = 1, .dynticks = ATOMIC_INIT(1), #endif }; diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index a471edc3d893..f6bf328bb9cf 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -75,7 +75,6 @@ /* Data structures. */ static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = { - .dynticks_nesting = 1, .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE, #ifdef CONFIG_RCU_NOCB_CPU .cblist.flags = SEGCBLIST_RCU_CORE, @@ -436,7 +435,7 @@ static int rcu_is_cpu_rrupt_from_idle(void) lockdep_assert_irqs_disabled(); /* Check for counter underflows */ - RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) < 0, + RCU_LOCKDEP_WARN(ct_dynticks_nesting() < 0, "RCU dynticks_nesting counter underflow!"); RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) <= 0, "RCU dynticks_nmi_nesting counter underflow/zero!"); @@ -452,7 +451,7 @@ static int rcu_is_cpu_rrupt_from_idle(void) WARN_ON_ONCE(!nesting && !is_idle_task(current)); /* Does CPU appear to be idle from an RCU standpoint? */ - return __this_cpu_read(rcu_data.dynticks_nesting) == 0; + return ct_dynticks_nesting() == 0; } #define DEFAULT_RCU_BLIMIT (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ? 1000 : 10) @@ -619,16 +618,16 @@ static noinstr void rcu_eqs_enter(bool user) WARN_ON_ONCE(rdp->dynticks_nmi_nesting != DYNTICK_IRQ_NONIDLE); WRITE_ONCE(rdp->dynticks_nmi_nesting, 0); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && - rdp->dynticks_nesting == 0); - if (rdp->dynticks_nesting != 1) { + ct_dynticks_nesting() == 0); + if (ct_dynticks_nesting() != 1) { // RCU will still be watching, so just do accounting and leave. - rdp->dynticks_nesting--; + ct->dynticks_nesting--; return; } instrumentation_begin(); lockdep_assert_irqs_disabled(); - trace_rcu_dyntick(TPS("Start"), rdp->dynticks_nesting, 0, ct_dynticks()); + trace_rcu_dyntick(TPS("Start"), ct_dynticks_nesting(), 0, ct_dynticks()); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); rcu_preempt_deferred_qs(current); @@ -636,7 +635,7 @@ static noinstr void rcu_eqs_enter(bool user) instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); instrumentation_end(); - WRITE_ONCE(rdp->dynticks_nesting, 0); /* Avoid irq-access tearing. */ + WRITE_ONCE(ct->dynticks_nesting, 0); /* Avoid irq-access tearing. */ // RCU is watching here ... rcu_dynticks_eqs_enter(); // ... but is no longer watching here. @@ -793,7 +792,7 @@ void rcu_irq_exit_check_preempt(void) { lockdep_assert_irqs_disabled(); - RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) <= 0, + RCU_LOCKDEP_WARN(ct_dynticks_nesting() <= 0, "RCU dynticks_nesting counter underflow/zero!"); RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) != DYNTICK_IRQ_NONIDLE, @@ -819,11 +818,11 @@ static void noinstr rcu_eqs_exit(bool user) WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !raw_irqs_disabled()); rdp = this_cpu_ptr(&rcu_data); - oldval = rdp->dynticks_nesting; + oldval = ct_dynticks_nesting(); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0); if (oldval) { // RCU was already watching, so just do accounting and leave. - rdp->dynticks_nesting++; + ct->dynticks_nesting++; return; } rcu_dynticks_task_exit(); @@ -835,9 +834,9 @@ static void noinstr rcu_eqs_exit(bool user) // instrumentation for the noinstr rcu_dynticks_eqs_exit() instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); - trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, ct_dynticks()); + trace_rcu_dyntick(TPS("End"), ct_dynticks_nesting(), 1, ct_dynticks()); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); - WRITE_ONCE(rdp->dynticks_nesting, 1); + WRITE_ONCE(ct->dynticks_nesting, 1); WARN_ON_ONCE(rdp->dynticks_nmi_nesting); WRITE_ONCE(rdp->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE); instrumentation_end(); @@ -4134,12 +4133,13 @@ static void rcu_init_new_rnp(struct rcu_node *rnp_leaf) static void __init rcu_boot_init_percpu_data(int cpu) { + struct context_tracking *ct = this_cpu_ptr(&context_tracking); struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); /* Set up local state, ensuring consistent view of global state. */ rdp->grpmask = leaf_node_cpu_bit(rdp->mynode, cpu); INIT_WORK(&rdp->strict_work, strict_work_handler); - WARN_ON_ONCE(rdp->dynticks_nesting != 1); + WARN_ON_ONCE(ct->dynticks_nesting != 1); WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(cpu))); rdp->barrier_seq_snap = rcu_state.barrier_sequence; rdp->rcu_ofl_gp_seq = rcu_state.gp_seq; @@ -4164,6 +4164,7 @@ rcu_boot_init_percpu_data(int cpu) int rcutree_prepare_cpu(unsigned int cpu) { unsigned long flags; + struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu); struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); struct rcu_node *rnp = rcu_get_root(); @@ -4172,7 +4173,7 @@ int rcutree_prepare_cpu(unsigned int cpu) rdp->qlen_last_fqs_check = 0; rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); rdp->blimit = blimit; - rdp->dynticks_nesting = 1; /* CPU not up, no tearing. */ + ct->dynticks_nesting = 1; /* CPU not up, no tearing. */ raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ /* diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index ebb973f5b190..650ff3cf0121 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -187,7 +187,6 @@ struct rcu_data { /* 3) dynticks interface. */ int dynticks_snap; /* Per-GP tracking for dynticks. */ - long dynticks_nesting; /* Track process nesting level. */ long dynticks_nmi_nesting; /* Track irq/NMI nesting level. */ bool rcu_need_heavy_qs; /* GP old, so heavy quiescent state! */ bool rcu_urgent_qs; /* GP old need light quiescent state. */ diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index 250fbf2e8522..a9c82254b6c6 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -479,7 +479,7 @@ static void print_cpu_stall_info(int cpu) "!."[!delta], ticks_value, ticks_title, rcu_dynticks_snap(cpu) & 0xfff, - rdp->dynticks_nesting, rdp->dynticks_nmi_nesting, + ct_dynticks_nesting_cpu(cpu), rdp->dynticks_nmi_nesting, rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu), data_race(rcu_state.n_force_qs) - rcu_state.n_force_qs_gpstart, rcuc_starved ? buf : "", From 95e04f48ec0a634e2f221081f5fa1a904755f326 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:31 +0200 Subject: [PATCH 155/318] rcu/context_tracking: Move dynticks_nmi_nesting to context tracking The RCU eqs tracking is going to be performed by the context tracking subsystem. The related nesting counters thus need to be moved to the context tracking structure. Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- include/linux/context_tracking_state.h | 16 +++++++++ kernel/context_tracking.c | 1 + kernel/rcu/rcu.h | 4 --- kernel/rcu/tree.c | 48 +++++++++++--------------- kernel/rcu/tree.h | 1 - kernel/rcu/tree_stall.h | 2 +- 6 files changed, 39 insertions(+), 33 deletions(-) diff --git a/include/linux/context_tracking_state.h b/include/linux/context_tracking_state.h index 13a4a9d1ec7e..5f11e3d2d85a 100644 --- a/include/linux/context_tracking_state.h +++ b/include/linux/context_tracking_state.h @@ -13,6 +13,9 @@ enum ctx_state { CONTEXT_GUEST, }; +/* Offset to allow distinguishing irq vs. task-based idle entry/exit. */ +#define DYNTICK_IRQ_NONIDLE ((LONG_MAX / 2) + 1) + struct context_tracking { #ifdef CONFIG_CONTEXT_TRACKING_USER /* @@ -28,6 +31,7 @@ struct context_tracking { #ifdef CONFIG_CONTEXT_TRACKING_IDLE atomic_t dynticks; /* Even value for idle, else odd. */ long dynticks_nesting; /* Track process nesting level. */ + long dynticks_nmi_nesting; /* Track irq/NMI nesting level. */ #endif }; @@ -66,6 +70,18 @@ static __always_inline long ct_dynticks_nesting_cpu(int cpu) return ct->dynticks_nesting; } + +static __always_inline long ct_dynticks_nmi_nesting(void) +{ + return __this_cpu_read(context_tracking.dynticks_nmi_nesting); +} + +static __always_inline long ct_dynticks_nmi_nesting_cpu(int cpu) +{ + struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu); + + return ct->dynticks_nmi_nesting; +} #endif /* #ifdef CONFIG_CONTEXT_TRACKING_IDLE */ #ifdef CONFIG_CONTEXT_TRACKING_USER diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index dfefe04400f8..7c3033e9a518 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -26,6 +26,7 @@ DEFINE_PER_CPU(struct context_tracking, context_tracking) = { #ifdef CONFIG_CONTEXT_TRACKING_IDLE .dynticks_nesting = 1, + .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE, .dynticks = ATOMIC_INIT(1), #endif }; diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index 4916077119f3..7b4a88deff9a 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -12,10 +12,6 @@ #include -/* Offset to allow distinguishing irq vs. task-based idle entry/exit. */ -#define DYNTICK_IRQ_NONIDLE ((LONG_MAX / 2) + 1) - - /* * Grace-period counter management. */ diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index f6bf328bb9cf..006939b29e82 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -75,7 +75,6 @@ /* Data structures. */ static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = { - .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE, #ifdef CONFIG_RCU_NOCB_CPU .cblist.flags = SEGCBLIST_RCU_CORE, #endif @@ -437,11 +436,11 @@ static int rcu_is_cpu_rrupt_from_idle(void) /* Check for counter underflows */ RCU_LOCKDEP_WARN(ct_dynticks_nesting() < 0, "RCU dynticks_nesting counter underflow!"); - RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) <= 0, + RCU_LOCKDEP_WARN(ct_dynticks_nmi_nesting() <= 0, "RCU dynticks_nmi_nesting counter underflow/zero!"); /* Are we at first interrupt nesting level? */ - nesting = __this_cpu_read(rcu_data.dynticks_nmi_nesting); + nesting = ct_dynticks_nmi_nesting(); if (nesting > 1) return false; @@ -612,11 +611,10 @@ EXPORT_SYMBOL_GPL(rcutorture_get_gp_data); */ static noinstr void rcu_eqs_enter(bool user) { - struct rcu_data *rdp = this_cpu_ptr(&rcu_data); struct context_tracking *ct = this_cpu_ptr(&context_tracking); - WARN_ON_ONCE(rdp->dynticks_nmi_nesting != DYNTICK_IRQ_NONIDLE); - WRITE_ONCE(rdp->dynticks_nmi_nesting, 0); + WARN_ON_ONCE(ct_dynticks_nmi_nesting() != DYNTICK_IRQ_NONIDLE); + WRITE_ONCE(ct->dynticks_nmi_nesting, 0); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && ct_dynticks_nesting() == 0); if (ct_dynticks_nesting() != 1) { @@ -734,7 +732,7 @@ noinstr void rcu_user_enter(void) * rcu_nmi_exit - inform RCU of exit from NMI context * * If we are returning from the outermost NMI handler that interrupted an - * RCU-idle period, update ct->dynticks and rdp->dynticks_nmi_nesting + * RCU-idle period, update ct->dynticks and ct->dynticks_nmi_nesting * to let the RCU grace-period handling know that the CPU is back to * being RCU-idle. * @@ -744,7 +742,6 @@ noinstr void rcu_user_enter(void) noinstr void rcu_nmi_exit(void) { struct context_tracking *ct = this_cpu_ptr(&context_tracking); - struct rcu_data *rdp = this_cpu_ptr(&rcu_data); instrumentation_begin(); /* @@ -752,25 +749,25 @@ noinstr void rcu_nmi_exit(void) * (We are exiting an NMI handler, so RCU better be paying attention * to us!) */ - WARN_ON_ONCE(rdp->dynticks_nmi_nesting <= 0); + WARN_ON_ONCE(ct_dynticks_nmi_nesting() <= 0); WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs()); /* * If the nesting level is not 1, the CPU wasn't RCU-idle, so * leave it in non-RCU-idle state. */ - if (rdp->dynticks_nmi_nesting != 1) { - trace_rcu_dyntick(TPS("--="), rdp->dynticks_nmi_nesting, rdp->dynticks_nmi_nesting - 2, + if (ct_dynticks_nmi_nesting() != 1) { + trace_rcu_dyntick(TPS("--="), ct_dynticks_nmi_nesting(), ct_dynticks_nmi_nesting() - 2, ct_dynticks()); - WRITE_ONCE(rdp->dynticks_nmi_nesting, /* No store tearing. */ - rdp->dynticks_nmi_nesting - 2); + WRITE_ONCE(ct->dynticks_nmi_nesting, /* No store tearing. */ + ct_dynticks_nmi_nesting() - 2); instrumentation_end(); return; } /* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */ - trace_rcu_dyntick(TPS("Startirq"), rdp->dynticks_nmi_nesting, 0, ct_dynticks()); - WRITE_ONCE(rdp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */ + trace_rcu_dyntick(TPS("Startirq"), ct_dynticks_nmi_nesting(), 0, ct_dynticks()); + WRITE_ONCE(ct->dynticks_nmi_nesting, 0); /* Avoid store tearing. */ // instrumentation for the noinstr rcu_dynticks_eqs_enter() instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); @@ -794,7 +791,7 @@ void rcu_irq_exit_check_preempt(void) RCU_LOCKDEP_WARN(ct_dynticks_nesting() <= 0, "RCU dynticks_nesting counter underflow/zero!"); - RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) != + RCU_LOCKDEP_WARN(ct_dynticks_nmi_nesting() != DYNTICK_IRQ_NONIDLE, "Bad RCU dynticks_nmi_nesting counter\n"); RCU_LOCKDEP_WARN(rcu_dynticks_curr_cpu_in_eqs(), @@ -813,11 +810,9 @@ void rcu_irq_exit_check_preempt(void) static void noinstr rcu_eqs_exit(bool user) { struct context_tracking *ct = this_cpu_ptr(&context_tracking); - struct rcu_data *rdp; long oldval; WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !raw_irqs_disabled()); - rdp = this_cpu_ptr(&rcu_data); oldval = ct_dynticks_nesting(); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0); if (oldval) { @@ -837,8 +832,8 @@ static void noinstr rcu_eqs_exit(bool user) trace_rcu_dyntick(TPS("End"), ct_dynticks_nesting(), 1, ct_dynticks()); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); WRITE_ONCE(ct->dynticks_nesting, 1); - WARN_ON_ONCE(rdp->dynticks_nmi_nesting); - WRITE_ONCE(rdp->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE); + WARN_ON_ONCE(ct_dynticks_nmi_nesting()); + WRITE_ONCE(ct->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE); instrumentation_end(); } @@ -941,7 +936,7 @@ void __rcu_irq_enter_check_tick(void) * rcu_nmi_enter - inform RCU of entry to NMI context * * If the CPU was idle from RCU's viewpoint, update ct->dynticks and - * rdp->dynticks_nmi_nesting to let the RCU grace-period handling know + * ct->dynticks_nmi_nesting to let the RCU grace-period handling know * that the CPU is active. This implementation permits nested NMIs, as * long as the nesting level does not overflow an int. (You will probably * run out of stack space first.) @@ -952,11 +947,10 @@ void __rcu_irq_enter_check_tick(void) noinstr void rcu_nmi_enter(void) { long incby = 2; - struct rcu_data *rdp = this_cpu_ptr(&rcu_data); struct context_tracking *ct = this_cpu_ptr(&context_tracking); /* Complain about underflow. */ - WARN_ON_ONCE(rdp->dynticks_nmi_nesting < 0); + WARN_ON_ONCE(ct_dynticks_nmi_nesting() < 0); /* * If idle from RCU viewpoint, atomically increment ->dynticks @@ -990,11 +984,11 @@ noinstr void rcu_nmi_enter(void) } trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="), - rdp->dynticks_nmi_nesting, - rdp->dynticks_nmi_nesting + incby, ct_dynticks()); + ct_dynticks_nmi_nesting(), + ct_dynticks_nmi_nesting() + incby, ct_dynticks()); instrumentation_end(); - WRITE_ONCE(rdp->dynticks_nmi_nesting, /* Prevent store tearing. */ - rdp->dynticks_nmi_nesting + incby); + WRITE_ONCE(ct->dynticks_nmi_nesting, /* Prevent store tearing. */ + ct_dynticks_nmi_nesting() + incby); barrier(); } diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 650ff3cf0121..72dbf8512ce7 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -187,7 +187,6 @@ struct rcu_data { /* 3) dynticks interface. */ int dynticks_snap; /* Per-GP tracking for dynticks. */ - long dynticks_nmi_nesting; /* Track irq/NMI nesting level. */ bool rcu_need_heavy_qs; /* GP old, so heavy quiescent state! */ bool rcu_urgent_qs; /* GP old need light quiescent state. */ bool rcu_forced_tick; /* Forced tick to provide QS. */ diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index a9c82254b6c6..2683ce0a7c72 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -479,7 +479,7 @@ static void print_cpu_stall_info(int cpu) "!."[!delta], ticks_value, ticks_title, rcu_dynticks_snap(cpu) & 0xfff, - ct_dynticks_nesting_cpu(cpu), rdp->dynticks_nmi_nesting, + ct_dynticks_nesting_cpu(cpu), ct_dynticks_nmi_nesting_cpu(cpu), rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu), data_race(rcu_state.n_force_qs) - rcu_state.n_force_qs_gpstart, rcuc_starved ? buf : "", From 564506495ca96a6e66d077d3d5b9f02d4b9b0f45 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:32 +0200 Subject: [PATCH 156/318] rcu/context-tracking: Move deferred nocb resched to context tracking To prepare for migrating the RCU eqs accounting code to context tracking, split the last-resort deferred nocb resched from rcu_user_enter() and move it into a separate call from context tracking. Acked-by: Paul E. McKenney Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- include/linux/rcupdate.h | 6 ++++++ kernel/context_tracking.c | 8 ++++++++ kernel/rcu/tree.c | 15 ++------------- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index f1562d91c67d..3717cad983a6 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -112,6 +112,12 @@ static inline void rcu_user_enter(void) { } static inline void rcu_user_exit(void) { } #endif /* CONFIG_NO_HZ_FULL */ +#if defined(CONFIG_NO_HZ_FULL) && (!defined(CONFIG_GENERIC_ENTRY) || !defined(CONFIG_KVM_XFER_TO_GUEST_WORK)) +void rcu_irq_work_resched(void); +#else +static inline void rcu_irq_work_resched(void) { } +#endif + #ifdef CONFIG_RCU_NOCB_CPU void rcu_init_nohz(void); int rcu_nocb_cpu_offload(int cpu); diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index 7c3033e9a518..8cf59d8a6af6 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -177,6 +177,8 @@ static __always_inline void context_tracking_recursion_exit(void) */ void noinstr __ct_user_enter(enum ctx_state state) { + lockdep_assert_irqs_disabled(); + /* Kernel threads aren't supposed to go to userspace */ WARN_ON_ONCE(!current->mm); @@ -198,6 +200,12 @@ void noinstr __ct_user_enter(enum ctx_state state) vtime_user_enter(current); instrumentation_end(); } + /* + * Other than generic entry implementation, we may be past the last + * rescheduling opportunity in the entry code. Trigger a self IPI + * that will fire and reschedule once we resume in user/guest mode. + */ + rcu_irq_work_resched(); rcu_user_enter(); } /* diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 006939b29e82..8c0c3490532e 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -681,7 +681,7 @@ static DEFINE_PER_CPU(struct irq_work, late_wakeup_work) = * last resort is to fire a local irq_work that will trigger a reschedule once IRQs * get re-enabled again. */ -noinstr static void rcu_irq_work_resched(void) +noinstr void rcu_irq_work_resched(void) { struct rcu_data *rdp = this_cpu_ptr(&rcu_data); @@ -697,10 +697,7 @@ noinstr static void rcu_irq_work_resched(void) } instrumentation_end(); } - -#else -static inline void rcu_irq_work_resched(void) { } -#endif +#endif /* #if !defined(CONFIG_GENERIC_ENTRY) || !defined(CONFIG_KVM_XFER_TO_GUEST_WORK) */ /** * rcu_user_enter - inform RCU that we are resuming userspace. @@ -715,14 +712,6 @@ static inline void rcu_irq_work_resched(void) { } */ noinstr void rcu_user_enter(void) { - lockdep_assert_irqs_disabled(); - - /* - * Other than generic entry implementation, we may be past the last - * rescheduling opportunity in the entry code. Trigger a self IPI - * that will fire and reschedule once we resume in user/guest mode. - */ - rcu_irq_work_resched(); rcu_eqs_enter(true); } From 172114552701b85d5c3b1a089a73ee85d0d7786b Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:33 +0200 Subject: [PATCH 157/318] rcu/context-tracking: Move RCU-dynticks internal functions to context_tracking Move the core RCU eqs/dynticks functions to context tracking so that we can later merge all that code within context tracking. Acked-by: Paul E. McKenney Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- include/linux/context_tracking.h | 20 ++ include/linux/rcutree.h | 3 + kernel/context_tracking.c | 336 +++++++++++++++++++++++++++++++ kernel/rcu/tree.c | 324 +---------------------------- kernel/rcu/tree.h | 5 - kernel/rcu/tree_plugin.h | 38 +--- 6 files changed, 364 insertions(+), 362 deletions(-) diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h index 01abadb2f993..1f568676bc1d 100644 --- a/include/linux/context_tracking.h +++ b/include/linux/context_tracking.h @@ -122,6 +122,26 @@ static inline void context_tracking_init(void) { } #ifdef CONFIG_CONTEXT_TRACKING_IDLE extern void ct_idle_enter(void); extern void ct_idle_exit(void); + +/* + * Is the current CPU in an extended quiescent state? + * + * No ordering, as we are sampling CPU-local information. + */ +static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void) +{ + return !(arch_atomic_read(this_cpu_ptr(&context_tracking.dynticks)) & 0x1); +} + +/* + * Increment the current CPU's context_tracking structure's ->dynticks field + * with ordering. Return the new value. + */ +static __always_inline unsigned long rcu_dynticks_inc(int incby) +{ + return arch_atomic_add_return(incby, this_cpu_ptr(&context_tracking.dynticks)); +} + #else static inline void ct_idle_enter(void) { } static inline void ct_idle_exit(void) { } diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index 4522b6a7cc42..24db1e41695c 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h @@ -55,6 +55,9 @@ void rcu_irq_exit_check_preempt(void); static inline void rcu_irq_exit_check_preempt(void) { } #endif +struct task_struct; +void rcu_preempt_deferred_qs(struct task_struct *t); + void exit_rcu(void); void rcu_scheduler_starting(void); diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index 8cf59d8a6af6..072c4b6044b3 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -21,6 +21,7 @@ #include #include #include +#include DEFINE_PER_CPU(struct context_tracking, context_tracking) = { @@ -33,6 +34,309 @@ DEFINE_PER_CPU(struct context_tracking, context_tracking) = { EXPORT_SYMBOL_GPL(context_tracking); #ifdef CONFIG_CONTEXT_TRACKING_IDLE +#define TPS(x) tracepoint_string(x) + +/* Record the current task on dyntick-idle entry. */ +static __always_inline void rcu_dynticks_task_enter(void) +{ +#if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) + WRITE_ONCE(current->rcu_tasks_idle_cpu, smp_processor_id()); +#endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */ +} + +/* Record no current task on dyntick-idle exit. */ +static __always_inline void rcu_dynticks_task_exit(void) +{ +#if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) + WRITE_ONCE(current->rcu_tasks_idle_cpu, -1); +#endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */ +} + +/* Turn on heavyweight RCU tasks trace readers on idle/user entry. */ +static __always_inline void rcu_dynticks_task_trace_enter(void) +{ +#ifdef CONFIG_TASKS_TRACE_RCU + if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB)) + current->trc_reader_special.b.need_mb = true; +#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */ +} + +/* Turn off heavyweight RCU tasks trace readers on idle/user exit. */ +static __always_inline void rcu_dynticks_task_trace_exit(void) +{ +#ifdef CONFIG_TASKS_TRACE_RCU + if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB)) + current->trc_reader_special.b.need_mb = false; +#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */ +} + +/* + * Record entry into an extended quiescent state. This is only to be + * called when not already in an extended quiescent state, that is, + * RCU is watching prior to the call to this function and is no longer + * watching upon return. + */ +static noinstr void rcu_dynticks_eqs_enter(void) +{ + int seq; + + /* + * CPUs seeing atomic_add_return() must see prior RCU read-side + * critical sections, and we also must force ordering with the + * next idle sojourn. + */ + rcu_dynticks_task_trace_enter(); // Before ->dynticks update! + seq = rcu_dynticks_inc(1); + // RCU is no longer watching. Better be in extended quiescent state! + WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && (seq & 0x1)); +} + +/* + * Record exit from an extended quiescent state. This is only to be + * called from an extended quiescent state, that is, RCU is not watching + * prior to the call to this function and is watching upon return. + */ +static noinstr void rcu_dynticks_eqs_exit(void) +{ + int seq; + + /* + * CPUs seeing atomic_add_return() must see prior idle sojourns, + * and we also must force ordering with the next RCU read-side + * critical section. + */ + seq = rcu_dynticks_inc(1); + // RCU is now watching. Better not be in an extended quiescent state! + rcu_dynticks_task_trace_exit(); // After ->dynticks update! + WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !(seq & 0x1)); +} + +/* + * Enter an RCU extended quiescent state, which can be either the + * idle loop or adaptive-tickless usermode execution. + * + * We crowbar the ->dynticks_nmi_nesting field to zero to allow for + * the possibility of usermode upcalls having messed up our count + * of interrupt nesting level during the prior busy period. + */ +static void noinstr rcu_eqs_enter(bool user) +{ + struct context_tracking *ct = this_cpu_ptr(&context_tracking); + + WARN_ON_ONCE(ct_dynticks_nmi_nesting() != DYNTICK_IRQ_NONIDLE); + WRITE_ONCE(ct->dynticks_nmi_nesting, 0); + WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && + ct_dynticks_nesting() == 0); + if (ct_dynticks_nesting() != 1) { + // RCU will still be watching, so just do accounting and leave. + ct->dynticks_nesting--; + return; + } + + instrumentation_begin(); + lockdep_assert_irqs_disabled(); + trace_rcu_dyntick(TPS("Start"), ct_dynticks_nesting(), 0, ct_dynticks()); + WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); + rcu_preempt_deferred_qs(current); + + // instrumentation for the noinstr rcu_dynticks_eqs_enter() + instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); + + instrumentation_end(); + WRITE_ONCE(ct->dynticks_nesting, 0); /* Avoid irq-access tearing. */ + // RCU is watching here ... + rcu_dynticks_eqs_enter(); + // ... but is no longer watching here. + rcu_dynticks_task_enter(); +} + +/* + * Exit an RCU extended quiescent state, which can be either the + * idle loop or adaptive-tickless usermode execution. + * + * We crowbar the ->dynticks_nmi_nesting field to DYNTICK_IRQ_NONIDLE to + * allow for the possibility of usermode upcalls messing up our count of + * interrupt nesting level during the busy period that is just now starting. + */ +static void noinstr rcu_eqs_exit(bool user) +{ + struct context_tracking *ct = this_cpu_ptr(&context_tracking); + long oldval; + + WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !raw_irqs_disabled()); + oldval = ct_dynticks_nesting(); + WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0); + if (oldval) { + // RCU was already watching, so just do accounting and leave. + ct->dynticks_nesting++; + return; + } + rcu_dynticks_task_exit(); + // RCU is not watching here ... + rcu_dynticks_eqs_exit(); + // ... but is watching here. + instrumentation_begin(); + + // instrumentation for the noinstr rcu_dynticks_eqs_exit() + instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); + + trace_rcu_dyntick(TPS("End"), ct_dynticks_nesting(), 1, ct_dynticks()); + WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); + WRITE_ONCE(ct->dynticks_nesting, 1); + WARN_ON_ONCE(ct_dynticks_nmi_nesting()); + WRITE_ONCE(ct->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE); + instrumentation_end(); +} + +/** + * rcu_nmi_exit - inform RCU of exit from NMI context + * + * If we are returning from the outermost NMI handler that interrupted an + * RCU-idle period, update ct->dynticks and ct->dynticks_nmi_nesting + * to let the RCU grace-period handling know that the CPU is back to + * being RCU-idle. + * + * If you add or remove a call to rcu_nmi_exit(), be sure to test + * with CONFIG_RCU_EQS_DEBUG=y. + */ +void noinstr rcu_nmi_exit(void) +{ + struct context_tracking *ct = this_cpu_ptr(&context_tracking); + + instrumentation_begin(); + /* + * Check for ->dynticks_nmi_nesting underflow and bad ->dynticks. + * (We are exiting an NMI handler, so RCU better be paying attention + * to us!) + */ + WARN_ON_ONCE(ct_dynticks_nmi_nesting() <= 0); + WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs()); + + /* + * If the nesting level is not 1, the CPU wasn't RCU-idle, so + * leave it in non-RCU-idle state. + */ + if (ct_dynticks_nmi_nesting() != 1) { + trace_rcu_dyntick(TPS("--="), ct_dynticks_nmi_nesting(), ct_dynticks_nmi_nesting() - 2, + ct_dynticks()); + WRITE_ONCE(ct->dynticks_nmi_nesting, /* No store tearing. */ + ct_dynticks_nmi_nesting() - 2); + instrumentation_end(); + return; + } + + /* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */ + trace_rcu_dyntick(TPS("Startirq"), ct_dynticks_nmi_nesting(), 0, ct_dynticks()); + WRITE_ONCE(ct->dynticks_nmi_nesting, 0); /* Avoid store tearing. */ + + // instrumentation for the noinstr rcu_dynticks_eqs_enter() + instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); + instrumentation_end(); + + // RCU is watching here ... + rcu_dynticks_eqs_enter(); + // ... but is no longer watching here. + + if (!in_nmi()) + rcu_dynticks_task_enter(); +} + +/** + * rcu_nmi_enter - inform RCU of entry to NMI context + * + * If the CPU was idle from RCU's viewpoint, update ct->dynticks and + * ct->dynticks_nmi_nesting to let the RCU grace-period handling know + * that the CPU is active. This implementation permits nested NMIs, as + * long as the nesting level does not overflow an int. (You will probably + * run out of stack space first.) + * + * If you add or remove a call to rcu_nmi_enter(), be sure to test + * with CONFIG_RCU_EQS_DEBUG=y. + */ +void noinstr rcu_nmi_enter(void) +{ + long incby = 2; + struct context_tracking *ct = this_cpu_ptr(&context_tracking); + + /* Complain about underflow. */ + WARN_ON_ONCE(ct_dynticks_nmi_nesting() < 0); + + /* + * If idle from RCU viewpoint, atomically increment ->dynticks + * to mark non-idle and increment ->dynticks_nmi_nesting by one. + * Otherwise, increment ->dynticks_nmi_nesting by two. This means + * if ->dynticks_nmi_nesting is equal to one, we are guaranteed + * to be in the outermost NMI handler that interrupted an RCU-idle + * period (observation due to Andy Lutomirski). + */ + if (rcu_dynticks_curr_cpu_in_eqs()) { + + if (!in_nmi()) + rcu_dynticks_task_exit(); + + // RCU is not watching here ... + rcu_dynticks_eqs_exit(); + // ... but is watching here. + + instrumentation_begin(); + // instrumentation for the noinstr rcu_dynticks_curr_cpu_in_eqs() + instrument_atomic_read(&ct->dynticks, sizeof(ct->dynticks)); + // instrumentation for the noinstr rcu_dynticks_eqs_exit() + instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); + + incby = 1; + } else if (!in_nmi()) { + instrumentation_begin(); + rcu_irq_enter_check_tick(); + } else { + instrumentation_begin(); + } + + trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="), + ct_dynticks_nmi_nesting(), + ct_dynticks_nmi_nesting() + incby, ct_dynticks()); + instrumentation_end(); + WRITE_ONCE(ct->dynticks_nmi_nesting, /* Prevent store tearing. */ + ct_dynticks_nmi_nesting() + incby); + barrier(); +} + +/** + * rcu_idle_enter - inform RCU that current CPU is entering idle + * + * Enter idle mode, in other words, -leave- the mode in which RCU + * read-side critical sections can occur. (Though RCU read-side + * critical sections can occur in irq handlers in idle, a possibility + * handled by irq_enter() and irq_exit().) + * + * If you add or remove a call to rcu_idle_enter(), be sure to test with + * CONFIG_RCU_EQS_DEBUG=y. + */ +void noinstr rcu_idle_enter(void) +{ + WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !raw_irqs_disabled()); + rcu_eqs_enter(false); +} + +/** + * rcu_idle_exit - inform RCU that current CPU is leaving idle + * + * Exit idle mode, in other words, -enter- the mode in which RCU + * read-side critical sections can occur. + * + * If you add or remove a call to rcu_idle_exit(), be sure to test with + * CONFIG_RCU_EQS_DEBUG=y. + */ +void noinstr rcu_idle_exit(void) +{ + unsigned long flags; + + raw_local_irq_save(flags); + rcu_eqs_exit(false); + raw_local_irq_restore(flags); +} +EXPORT_SYMBOL_GPL(rcu_idle_exit); + noinstr void ct_idle_enter(void) { rcu_idle_enter(); @@ -139,6 +443,38 @@ noinstr void ct_nmi_exit(void) } #endif /* #ifdef CONFIG_CONTEXT_TRACKING_IDLE */ +#ifdef CONFIG_NO_HZ_FULL +/** + * rcu_user_enter - inform RCU that we are resuming userspace. + * + * Enter RCU idle mode right before resuming userspace. No use of RCU + * is permitted between this call and rcu_user_exit(). This way the + * CPU doesn't need to maintain the tick for RCU maintenance purposes + * when the CPU runs in userspace. + * + * If you add or remove a call to rcu_user_enter(), be sure to test with + * CONFIG_RCU_EQS_DEBUG=y. + */ +noinstr void rcu_user_enter(void) +{ + rcu_eqs_enter(true); +} + +/** + * rcu_user_exit - inform RCU that we are exiting userspace. + * + * Exit RCU idle mode while entering the kernel because it can + * run a RCU read side critical section anytime. + * + * If you add or remove a call to rcu_user_exit(), be sure to test with + * CONFIG_RCU_EQS_DEBUG=y. + */ +void noinstr rcu_user_exit(void) +{ + rcu_eqs_exit(true); +} +#endif /* #ifdef CONFIG_NO_HZ_FULL */ + #ifdef CONFIG_CONTEXT_TRACKING_USER #define CREATE_TRACE_POINTS diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 8c0c3490532e..e2a2083079a2 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -62,6 +62,7 @@ #include #include #include +#include #include "../time/tick-internal.h" #include "tree.h" @@ -259,56 +260,6 @@ void rcu_softirq_qs(void) rcu_tasks_qs(current, false); } -/* - * Increment the current CPU's rcu_data structure's ->dynticks field - * with ordering. Return the new value. - */ -static noinline noinstr unsigned long rcu_dynticks_inc(int incby) -{ - return arch_atomic_add_return(incby, this_cpu_ptr(&context_tracking.dynticks)); -} - -/* - * Record entry into an extended quiescent state. This is only to be - * called when not already in an extended quiescent state, that is, - * RCU is watching prior to the call to this function and is no longer - * watching upon return. - */ -static noinstr void rcu_dynticks_eqs_enter(void) -{ - int seq; - - /* - * CPUs seeing atomic_add_return() must see prior RCU read-side - * critical sections, and we also must force ordering with the - * next idle sojourn. - */ - rcu_dynticks_task_trace_enter(); // Before ->dynticks update! - seq = rcu_dynticks_inc(1); - // RCU is no longer watching. Better be in extended quiescent state! - WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && (seq & 0x1)); -} - -/* - * Record exit from an extended quiescent state. This is only to be - * called from an extended quiescent state, that is, RCU is not watching - * prior to the call to this function and is watching upon return. - */ -static noinstr void rcu_dynticks_eqs_exit(void) -{ - int seq; - - /* - * CPUs seeing atomic_add_return() must see prior idle sojourns, - * and we also must force ordering with the next RCU read-side - * critical section. - */ - seq = rcu_dynticks_inc(1); - // RCU is now watching. Better not be in an extended quiescent state! - rcu_dynticks_task_trace_exit(); // After ->dynticks update! - WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !(seq & 0x1)); -} - /* * Reset the current CPU's ->dynticks counter to indicate that the * newly onlined CPU is no longer in an extended quiescent state. @@ -326,16 +277,6 @@ static void rcu_dynticks_eqs_online(void) rcu_dynticks_inc(1); } -/* - * Is the current CPU in an extended quiescent state? - * - * No ordering, as we are sampling CPU-local information. - */ -static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void) -{ - return !(arch_atomic_read(this_cpu_ptr(&context_tracking.dynticks)) & 0x1); -} - /* * Snapshot the ->dynticks counter with full ordering so as to allow * stable comparison of this counter with past and future snapshots. @@ -601,65 +542,7 @@ void rcutorture_get_gp_data(enum rcutorture_type test_type, int *flags, } EXPORT_SYMBOL_GPL(rcutorture_get_gp_data); -/* - * Enter an RCU extended quiescent state, which can be either the - * idle loop or adaptive-tickless usermode execution. - * - * We crowbar the ->dynticks_nmi_nesting field to zero to allow for - * the possibility of usermode upcalls having messed up our count - * of interrupt nesting level during the prior busy period. - */ -static noinstr void rcu_eqs_enter(bool user) -{ - struct context_tracking *ct = this_cpu_ptr(&context_tracking); - - WARN_ON_ONCE(ct_dynticks_nmi_nesting() != DYNTICK_IRQ_NONIDLE); - WRITE_ONCE(ct->dynticks_nmi_nesting, 0); - WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && - ct_dynticks_nesting() == 0); - if (ct_dynticks_nesting() != 1) { - // RCU will still be watching, so just do accounting and leave. - ct->dynticks_nesting--; - return; - } - - instrumentation_begin(); - lockdep_assert_irqs_disabled(); - trace_rcu_dyntick(TPS("Start"), ct_dynticks_nesting(), 0, ct_dynticks()); - WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); - rcu_preempt_deferred_qs(current); - - // instrumentation for the noinstr rcu_dynticks_eqs_enter() - instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); - - instrumentation_end(); - WRITE_ONCE(ct->dynticks_nesting, 0); /* Avoid irq-access tearing. */ - // RCU is watching here ... - rcu_dynticks_eqs_enter(); - // ... but is no longer watching here. - rcu_dynticks_task_enter(); -} - -/** - * rcu_idle_enter - inform RCU that current CPU is entering idle - * - * Enter idle mode, in other words, -leave- the mode in which RCU - * read-side critical sections can occur. (Though RCU read-side - * critical sections can occur in irq handlers in idle, a possibility - * handled by irq_enter() and irq_exit().) - * - * If you add or remove a call to rcu_idle_enter(), be sure to test with - * CONFIG_RCU_EQS_DEBUG=y. - */ -void noinstr rcu_idle_enter(void) -{ - WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !raw_irqs_disabled()); - rcu_eqs_enter(false); -} - -#ifdef CONFIG_NO_HZ_FULL - -#if !defined(CONFIG_GENERIC_ENTRY) || !defined(CONFIG_KVM_XFER_TO_GUEST_WORK) +#if defined(CONFIG_NO_HZ_FULL) && (!defined(CONFIG_GENERIC_ENTRY) || !defined(CONFIG_KVM_XFER_TO_GUEST_WORK)) /* * An empty function that will trigger a reschedule on * IRQ tail once IRQs get re-enabled on userspace/guest resume. @@ -697,78 +580,7 @@ noinstr void rcu_irq_work_resched(void) } instrumentation_end(); } -#endif /* #if !defined(CONFIG_GENERIC_ENTRY) || !defined(CONFIG_KVM_XFER_TO_GUEST_WORK) */ - -/** - * rcu_user_enter - inform RCU that we are resuming userspace. - * - * Enter RCU idle mode right before resuming userspace. No use of RCU - * is permitted between this call and rcu_user_exit(). This way the - * CPU doesn't need to maintain the tick for RCU maintenance purposes - * when the CPU runs in userspace. - * - * If you add or remove a call to rcu_user_enter(), be sure to test with - * CONFIG_RCU_EQS_DEBUG=y. - */ -noinstr void rcu_user_enter(void) -{ - rcu_eqs_enter(true); -} - -#endif /* CONFIG_NO_HZ_FULL */ - -/** - * rcu_nmi_exit - inform RCU of exit from NMI context - * - * If we are returning from the outermost NMI handler that interrupted an - * RCU-idle period, update ct->dynticks and ct->dynticks_nmi_nesting - * to let the RCU grace-period handling know that the CPU is back to - * being RCU-idle. - * - * If you add or remove a call to rcu_nmi_exit(), be sure to test - * with CONFIG_RCU_EQS_DEBUG=y. - */ -noinstr void rcu_nmi_exit(void) -{ - struct context_tracking *ct = this_cpu_ptr(&context_tracking); - - instrumentation_begin(); - /* - * Check for ->dynticks_nmi_nesting underflow and bad ->dynticks. - * (We are exiting an NMI handler, so RCU better be paying attention - * to us!) - */ - WARN_ON_ONCE(ct_dynticks_nmi_nesting() <= 0); - WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs()); - - /* - * If the nesting level is not 1, the CPU wasn't RCU-idle, so - * leave it in non-RCU-idle state. - */ - if (ct_dynticks_nmi_nesting() != 1) { - trace_rcu_dyntick(TPS("--="), ct_dynticks_nmi_nesting(), ct_dynticks_nmi_nesting() - 2, - ct_dynticks()); - WRITE_ONCE(ct->dynticks_nmi_nesting, /* No store tearing. */ - ct_dynticks_nmi_nesting() - 2); - instrumentation_end(); - return; - } - - /* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */ - trace_rcu_dyntick(TPS("Startirq"), ct_dynticks_nmi_nesting(), 0, ct_dynticks()); - WRITE_ONCE(ct->dynticks_nmi_nesting, 0); /* Avoid store tearing. */ - - // instrumentation for the noinstr rcu_dynticks_eqs_enter() - instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); - instrumentation_end(); - - // RCU is watching here ... - rcu_dynticks_eqs_enter(); - // ... but is no longer watching here. - - if (!in_nmi()) - rcu_dynticks_task_enter(); -} +#endif /* #if defined(CONFIG_NO_HZ_FULL) && (!defined(CONFIG_GENERIC_ENTRY) || !defined(CONFIG_KVM_XFER_TO_GUEST_WORK)) */ #ifdef CONFIG_PROVE_RCU /** @@ -788,77 +600,7 @@ void rcu_irq_exit_check_preempt(void) } #endif /* #ifdef CONFIG_PROVE_RCU */ -/* - * Exit an RCU extended quiescent state, which can be either the - * idle loop or adaptive-tickless usermode execution. - * - * We crowbar the ->dynticks_nmi_nesting field to DYNTICK_IRQ_NONIDLE to - * allow for the possibility of usermode upcalls messing up our count of - * interrupt nesting level during the busy period that is just now starting. - */ -static void noinstr rcu_eqs_exit(bool user) -{ - struct context_tracking *ct = this_cpu_ptr(&context_tracking); - long oldval; - - WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !raw_irqs_disabled()); - oldval = ct_dynticks_nesting(); - WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0); - if (oldval) { - // RCU was already watching, so just do accounting and leave. - ct->dynticks_nesting++; - return; - } - rcu_dynticks_task_exit(); - // RCU is not watching here ... - rcu_dynticks_eqs_exit(); - // ... but is watching here. - instrumentation_begin(); - - // instrumentation for the noinstr rcu_dynticks_eqs_exit() - instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); - - trace_rcu_dyntick(TPS("End"), ct_dynticks_nesting(), 1, ct_dynticks()); - WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); - WRITE_ONCE(ct->dynticks_nesting, 1); - WARN_ON_ONCE(ct_dynticks_nmi_nesting()); - WRITE_ONCE(ct->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE); - instrumentation_end(); -} - -/** - * rcu_idle_exit - inform RCU that current CPU is leaving idle - * - * Exit idle mode, in other words, -enter- the mode in which RCU - * read-side critical sections can occur. - * - * If you add or remove a call to rcu_idle_exit(), be sure to test with - * CONFIG_RCU_EQS_DEBUG=y. - */ -void noinstr rcu_idle_exit(void) -{ - unsigned long flags; - - raw_local_irq_save(flags); - rcu_eqs_exit(false); - raw_local_irq_restore(flags); -} - #ifdef CONFIG_NO_HZ_FULL -/** - * rcu_user_exit - inform RCU that we are exiting userspace. - * - * Exit RCU idle mode while entering the kernel because it can - * run a RCU read side critical section anytime. - * - * If you add or remove a call to rcu_user_exit(), be sure to test with - * CONFIG_RCU_EQS_DEBUG=y. - */ -void noinstr rcu_user_exit(void) -{ - rcu_eqs_exit(true); -} - /** * __rcu_irq_enter_check_tick - Enable scheduler tick on CPU if RCU needs it. * @@ -921,66 +663,6 @@ void __rcu_irq_enter_check_tick(void) } #endif /* CONFIG_NO_HZ_FULL */ -/** - * rcu_nmi_enter - inform RCU of entry to NMI context - * - * If the CPU was idle from RCU's viewpoint, update ct->dynticks and - * ct->dynticks_nmi_nesting to let the RCU grace-period handling know - * that the CPU is active. This implementation permits nested NMIs, as - * long as the nesting level does not overflow an int. (You will probably - * run out of stack space first.) - * - * If you add or remove a call to rcu_nmi_enter(), be sure to test - * with CONFIG_RCU_EQS_DEBUG=y. - */ -noinstr void rcu_nmi_enter(void) -{ - long incby = 2; - struct context_tracking *ct = this_cpu_ptr(&context_tracking); - - /* Complain about underflow. */ - WARN_ON_ONCE(ct_dynticks_nmi_nesting() < 0); - - /* - * If idle from RCU viewpoint, atomically increment ->dynticks - * to mark non-idle and increment ->dynticks_nmi_nesting by one. - * Otherwise, increment ->dynticks_nmi_nesting by two. This means - * if ->dynticks_nmi_nesting is equal to one, we are guaranteed - * to be in the outermost NMI handler that interrupted an RCU-idle - * period (observation due to Andy Lutomirski). - */ - if (rcu_dynticks_curr_cpu_in_eqs()) { - - if (!in_nmi()) - rcu_dynticks_task_exit(); - - // RCU is not watching here ... - rcu_dynticks_eqs_exit(); - // ... but is watching here. - - instrumentation_begin(); - // instrumentation for the noinstr rcu_dynticks_curr_cpu_in_eqs() - instrument_atomic_read(&ct->dynticks, sizeof(ct->dynticks)); - // instrumentation for the noinstr rcu_dynticks_eqs_exit() - instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); - - incby = 1; - } else if (!in_nmi()) { - instrumentation_begin(); - rcu_irq_enter_check_tick(); - } else { - instrumentation_begin(); - } - - trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="), - ct_dynticks_nmi_nesting(), - ct_dynticks_nmi_nesting() + incby, ct_dynticks()); - instrumentation_end(); - WRITE_ONCE(ct->dynticks_nmi_nesting, /* Prevent store tearing. */ - ct_dynticks_nmi_nesting() + incby); - barrier(); -} - /* * Check to see if any future non-offloaded RCU-related work will need * to be done by the current CPU, even if none need be done immediately, diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 72dbf8512ce7..0d5d1de327e4 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -427,7 +427,6 @@ static void rcu_cpu_kthread_setup(unsigned int cpu); static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp); static bool rcu_preempt_has_tasks(struct rcu_node *rnp); static bool rcu_preempt_need_deferred_qs(struct task_struct *t); -static void rcu_preempt_deferred_qs(struct task_struct *t); static void zero_cpu_stall_ticks(struct rcu_data *rdp); static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp); static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq); @@ -467,10 +466,6 @@ do { \ static void rcu_bind_gp_kthread(void); static bool rcu_nohz_full_cpu(void); -static void rcu_dynticks_task_enter(void); -static void rcu_dynticks_task_exit(void); -static void rcu_dynticks_task_trace_enter(void); -static void rcu_dynticks_task_trace_exit(void); /* Forward declarations for tree_stall.h */ static void record_gp_stall_check_time(void); diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index c8ba0fe17267..4a53aa013f82 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -595,7 +595,7 @@ static bool rcu_preempt_need_deferred_qs(struct task_struct *t) * evaluate safety in terms of interrupt, softirq, and preemption * disabling. */ -static void rcu_preempt_deferred_qs(struct task_struct *t) +void rcu_preempt_deferred_qs(struct task_struct *t) { unsigned long flags; @@ -935,7 +935,7 @@ static bool rcu_preempt_need_deferred_qs(struct task_struct *t) // period for a quiescent state from this CPU. Note that requests from // tasks are handled when removing the task from the blocked-tasks list // below. -static void rcu_preempt_deferred_qs(struct task_struct *t) +void rcu_preempt_deferred_qs(struct task_struct *t) { struct rcu_data *rdp = this_cpu_ptr(&rcu_data); @@ -1290,37 +1290,3 @@ static void rcu_bind_gp_kthread(void) return; housekeeping_affine(current, HK_TYPE_RCU); } - -/* Record the current task on dyntick-idle entry. */ -static __always_inline void rcu_dynticks_task_enter(void) -{ -#if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) - WRITE_ONCE(current->rcu_tasks_idle_cpu, smp_processor_id()); -#endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */ -} - -/* Record no current task on dyntick-idle exit. */ -static __always_inline void rcu_dynticks_task_exit(void) -{ -#if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) - WRITE_ONCE(current->rcu_tasks_idle_cpu, -1); -#endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */ -} - -/* Turn on heavyweight RCU tasks trace readers on idle/user entry. */ -static __always_inline void rcu_dynticks_task_trace_enter(void) -{ -#ifdef CONFIG_TASKS_TRACE_RCU - if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB)) - current->trc_reader_special.b.need_mb = true; -#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */ -} - -/* Turn off heavyweight RCU tasks trace readers on idle/user exit. */ -static __always_inline void rcu_dynticks_task_trace_exit(void) -{ -#ifdef CONFIG_TASKS_TRACE_RCU - if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB)) - current->trc_reader_special.b.need_mb = false; -#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */ -} From c33ef43a359001415032665dfcd433979c462b71 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:34 +0200 Subject: [PATCH 158/318] rcu/context-tracking: Remove unused and/or unecessary middle functions Some eqs functions are now only used internally by context tracking, so their public declarations can be removed. Also middle functions such as rcu_user_*() and rcu_idle_*() which now directly call to rcu_eqs_enter() and rcu_eqs_exit() can be wiped out as well. Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- Documentation/RCU/stallwarn.rst | 2 +- include/linux/hardirq.h | 8 --- include/linux/rcupdate.h | 8 --- include/linux/rcutiny.h | 2 - include/linux/rcutree.h | 2 - kernel/context_tracking.c | 98 +++++++++------------------------ 6 files changed, 28 insertions(+), 92 deletions(-) diff --git a/Documentation/RCU/stallwarn.rst b/Documentation/RCU/stallwarn.rst index ce1f58a9d954..e38c587067fc 100644 --- a/Documentation/RCU/stallwarn.rst +++ b/Documentation/RCU/stallwarn.rst @@ -97,7 +97,7 @@ warnings: which will include additional debugging information. - A low-level kernel issue that either fails to invoke one of the - variants of rcu_user_enter(), rcu_user_exit(), ct_idle_enter(), + variants of rcu_eqs_enter(true), rcu_eqs_exit(true), ct_idle_enter(), ct_idle_exit(), ct_irq_enter(), or ct_irq_exit() on the one hand, or that invokes one of them too many times on the other. Historically, the most frequent issue has been an omission diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h index 345cdbe9c1b7..d57cab4d4c06 100644 --- a/include/linux/hardirq.h +++ b/include/linux/hardirq.h @@ -92,14 +92,6 @@ void irq_exit_rcu(void); #define arch_nmi_exit() do { } while (0) #endif -#ifdef CONFIG_TINY_RCU -static inline void rcu_nmi_enter(void) { } -static inline void rcu_nmi_exit(void) { } -#else -extern void rcu_nmi_enter(void); -extern void rcu_nmi_exit(void); -#endif - /* * NMI vs Tracing * -------------- diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 3717cad983a6..434da1eb88cd 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -104,14 +104,6 @@ static inline void rcu_sysrq_start(void) { } static inline void rcu_sysrq_end(void) { } #endif /* #else #ifdef CONFIG_RCU_STALL_COMMON */ -#ifdef CONFIG_NO_HZ_FULL -void rcu_user_enter(void); -void rcu_user_exit(void); -#else -static inline void rcu_user_enter(void) { } -static inline void rcu_user_exit(void) { } -#endif /* CONFIG_NO_HZ_FULL */ - #if defined(CONFIG_NO_HZ_FULL) && (!defined(CONFIG_GENERIC_ENTRY) || !defined(CONFIG_KVM_XFER_TO_GUEST_WORK)) void rcu_irq_work_resched(void); #else diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h index 591119413cf1..900ba35c3582 100644 --- a/include/linux/rcutiny.h +++ b/include/linux/rcutiny.h @@ -76,8 +76,6 @@ static inline int rcu_needs_cpu(void) static inline void rcu_virt_note_context_switch(int cpu) { } static inline void rcu_cpu_stall_reset(void) { } static inline int rcu_jiffies_till_stall_check(void) { return 21 * HZ; } -static inline void rcu_idle_enter(void) { } -static inline void rcu_idle_exit(void) { } static inline void rcu_irq_exit_check_preempt(void) { } #define rcu_is_idle_cpu(cpu) \ (is_idle_task(current) && !in_nmi() && !in_hardirq() && !in_serving_softirq()) diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index 24db1e41695c..9cca00ed9bc9 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h @@ -45,8 +45,6 @@ unsigned long start_poll_synchronize_rcu(void); bool poll_state_synchronize_rcu(unsigned long oldstate); void cond_synchronize_rcu(unsigned long oldstate); -void rcu_idle_enter(void); -void rcu_idle_exit(void); bool rcu_is_idle_cpu(int cpu); #ifdef CONFIG_PROVE_RCU diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index 072c4b6044b3..e485b6b01537 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -189,17 +189,17 @@ static void noinstr rcu_eqs_exit(bool user) } /** - * rcu_nmi_exit - inform RCU of exit from NMI context + * ct_nmi_exit - inform RCU of exit from NMI context * * If we are returning from the outermost NMI handler that interrupted an * RCU-idle period, update ct->dynticks and ct->dynticks_nmi_nesting * to let the RCU grace-period handling know that the CPU is back to * being RCU-idle. * - * If you add or remove a call to rcu_nmi_exit(), be sure to test + * If you add or remove a call to ct_nmi_exit(), be sure to test * with CONFIG_RCU_EQS_DEBUG=y. */ -void noinstr rcu_nmi_exit(void) +void noinstr ct_nmi_exit(void) { struct context_tracking *ct = this_cpu_ptr(&context_tracking); @@ -242,7 +242,7 @@ void noinstr rcu_nmi_exit(void) } /** - * rcu_nmi_enter - inform RCU of entry to NMI context + * ct_nmi_enter - inform RCU of entry to NMI context * * If the CPU was idle from RCU's viewpoint, update ct->dynticks and * ct->dynticks_nmi_nesting to let the RCU grace-period handling know @@ -250,10 +250,10 @@ void noinstr rcu_nmi_exit(void) * long as the nesting level does not overflow an int. (You will probably * run out of stack space first.) * - * If you add or remove a call to rcu_nmi_enter(), be sure to test + * If you add or remove a call to ct_nmi_enter(), be sure to test * with CONFIG_RCU_EQS_DEBUG=y. */ -void noinstr rcu_nmi_enter(void) +void noinstr ct_nmi_enter(void) { long incby = 2; struct context_tracking *ct = this_cpu_ptr(&context_tracking); @@ -302,32 +302,33 @@ void noinstr rcu_nmi_enter(void) } /** - * rcu_idle_enter - inform RCU that current CPU is entering idle + * ct_idle_enter - inform RCU that current CPU is entering idle * * Enter idle mode, in other words, -leave- the mode in which RCU * read-side critical sections can occur. (Though RCU read-side * critical sections can occur in irq handlers in idle, a possibility * handled by irq_enter() and irq_exit().) * - * If you add or remove a call to rcu_idle_enter(), be sure to test with + * If you add or remove a call to ct_idle_enter(), be sure to test with * CONFIG_RCU_EQS_DEBUG=y. */ -void noinstr rcu_idle_enter(void) +void noinstr ct_idle_enter(void) { WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !raw_irqs_disabled()); rcu_eqs_enter(false); } +EXPORT_SYMBOL_GPL(ct_idle_enter); /** - * rcu_idle_exit - inform RCU that current CPU is leaving idle + * ct_idle_exit - inform RCU that current CPU is leaving idle * * Exit idle mode, in other words, -enter- the mode in which RCU * read-side critical sections can occur. * - * If you add or remove a call to rcu_idle_exit(), be sure to test with + * If you add or remove a call to ct_idle_exit(), be sure to test with * CONFIG_RCU_EQS_DEBUG=y. */ -void noinstr rcu_idle_exit(void) +void noinstr ct_idle_exit(void) { unsigned long flags; @@ -335,18 +336,6 @@ void noinstr rcu_idle_exit(void) rcu_eqs_exit(false); raw_local_irq_restore(flags); } -EXPORT_SYMBOL_GPL(rcu_idle_exit); - -noinstr void ct_idle_enter(void) -{ - rcu_idle_enter(); -} -EXPORT_SYMBOL_GPL(ct_idle_enter); - -void ct_idle_exit(void) -{ - rcu_idle_exit(); -} EXPORT_SYMBOL_GPL(ct_idle_exit); /** @@ -431,50 +420,11 @@ void ct_irq_exit_irqson(void) ct_irq_exit(); local_irq_restore(flags); } - -noinstr void ct_nmi_enter(void) -{ - rcu_nmi_enter(); -} - -noinstr void ct_nmi_exit(void) -{ - rcu_nmi_exit(); -} +#else +static __always_inline void rcu_eqs_enter(bool user) { } +static __always_inline void rcu_eqs_exit(bool user) { } #endif /* #ifdef CONFIG_CONTEXT_TRACKING_IDLE */ -#ifdef CONFIG_NO_HZ_FULL -/** - * rcu_user_enter - inform RCU that we are resuming userspace. - * - * Enter RCU idle mode right before resuming userspace. No use of RCU - * is permitted between this call and rcu_user_exit(). This way the - * CPU doesn't need to maintain the tick for RCU maintenance purposes - * when the CPU runs in userspace. - * - * If you add or remove a call to rcu_user_enter(), be sure to test with - * CONFIG_RCU_EQS_DEBUG=y. - */ -noinstr void rcu_user_enter(void) -{ - rcu_eqs_enter(true); -} - -/** - * rcu_user_exit - inform RCU that we are exiting userspace. - * - * Exit RCU idle mode while entering the kernel because it can - * run a RCU read side critical section anytime. - * - * If you add or remove a call to rcu_user_exit(), be sure to test with - * CONFIG_RCU_EQS_DEBUG=y. - */ -void noinstr rcu_user_exit(void) -{ - rcu_eqs_exit(true); -} -#endif /* #ifdef CONFIG_NO_HZ_FULL */ - #ifdef CONFIG_CONTEXT_TRACKING_USER #define CREATE_TRACE_POINTS @@ -542,7 +492,13 @@ void noinstr __ct_user_enter(enum ctx_state state) * that will fire and reschedule once we resume in user/guest mode. */ rcu_irq_work_resched(); - rcu_user_enter(); + /* + * Enter RCU idle mode right before resuming userspace. No use of RCU + * is permitted between this call and rcu_eqs_exit(). This way the + * CPU doesn't need to maintain the tick for RCU maintenance purposes + * when the CPU runs in userspace. + */ + rcu_eqs_enter(true); } /* * Even if context tracking is disabled on this CPU, because it's outside @@ -579,7 +535,7 @@ void ct_user_enter(enum ctx_state state) /* * Some contexts may involve an exception occuring in an irq, * leading to that nesting: - * ct_irq_enter() rcu_user_exit() rcu_user_exit() ct_irq_exit() + * ct_irq_enter() rcu_eqs_exit(true) rcu_eqs_enter(true) ct_irq_exit() * This would mess up the dyntick_nesting count though. And rcu_irq_*() * helpers are enough to protect RCU uses inside the exception. So * just return immediately if we detect we are in an IRQ. @@ -631,10 +587,10 @@ void noinstr __ct_user_exit(enum ctx_state state) if (__this_cpu_read(context_tracking.state) == state) { if (__this_cpu_read(context_tracking.active)) { /* - * We are going to run code that may use RCU. Inform - * RCU core about that (ie: we may need the tick again). + * Exit RCU idle mode while entering the kernel because it can + * run a RCU read side critical section anytime. */ - rcu_user_exit(); + rcu_eqs_exit(true); if (state == CONTEXT_USER) { instrumentation_begin(); vtime_user_exit(current); From 171476775d32a40bfebf83250136c19b2e842672 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:35 +0200 Subject: [PATCH 159/318] context_tracking: Convert state to atomic_t Context tracking's state and dynticks counter are going to be merged in a single field so that both updates can happen atomically and at the same time. Prepare for that with converting the state into an atomic_t. [ paulmck: Apply kernel test robot feedback. ] Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- include/linux/context_tracking.h | 32 ++---- include/linux/context_tracking_state.h | 59 +++++++--- kernel/context_tracking.c | 143 ++++++++++++++++--------- kernel/rcu/tree.c | 13 ++- kernel/rcu/tree_stall.h | 4 +- 5 files changed, 159 insertions(+), 92 deletions(-) diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h index 1f568676bc1d..dcef4a9e4d63 100644 --- a/include/linux/context_tracking.h +++ b/include/linux/context_tracking.h @@ -56,7 +56,7 @@ static inline enum ctx_state exception_enter(void) !context_tracking_enabled()) return 0; - prev_ctx = this_cpu_read(context_tracking.state); + prev_ctx = __ct_state(); if (prev_ctx != CONTEXT_KERNEL) ct_user_exit(prev_ctx); @@ -86,33 +86,21 @@ static __always_inline void context_tracking_guest_exit(void) __ct_user_exit(CONTEXT_GUEST); } -/** - * ct_state() - return the current context tracking state if known - * - * Returns the current cpu's context tracking state if context tracking - * is enabled. If context tracking is disabled, returns - * CONTEXT_DISABLED. This should be used primarily for debugging. - */ -static __always_inline enum ctx_state ct_state(void) -{ - return context_tracking_enabled() ? - this_cpu_read(context_tracking.state) : CONTEXT_DISABLED; -} +#define CT_WARN_ON(cond) WARN_ON(context_tracking_enabled() && (cond)) + #else static inline void user_enter(void) { } static inline void user_exit(void) { } static inline void user_enter_irqoff(void) { } static inline void user_exit_irqoff(void) { } -static inline enum ctx_state exception_enter(void) { return 0; } +static inline int exception_enter(void) { return 0; } static inline void exception_exit(enum ctx_state prev_ctx) { } -static inline enum ctx_state ct_state(void) { return CONTEXT_DISABLED; } +static inline int ct_state(void) { return -1; } static __always_inline bool context_tracking_guest_enter(void) { return false; } static inline void context_tracking_guest_exit(void) { } - +#define CT_WARN_ON(cond) do { } while (0) #endif /* !CONFIG_CONTEXT_TRACKING_USER */ -#define CT_WARN_ON(cond) WARN_ON(context_tracking_enabled() && (cond)) - #ifdef CONFIG_CONTEXT_TRACKING_USER_FORCE extern void context_tracking_init(void); #else @@ -130,16 +118,16 @@ extern void ct_idle_exit(void); */ static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void) { - return !(arch_atomic_read(this_cpu_ptr(&context_tracking.dynticks)) & 0x1); + return !(arch_atomic_read(this_cpu_ptr(&context_tracking.state)) & RCU_DYNTICKS_IDX); } /* - * Increment the current CPU's context_tracking structure's ->dynticks field + * Increment the current CPU's context_tracking structure's ->state field * with ordering. Return the new value. */ -static __always_inline unsigned long rcu_dynticks_inc(int incby) +static __always_inline unsigned long ct_state_inc(int incby) { - return arch_atomic_add_return(incby, this_cpu_ptr(&context_tracking.dynticks)); + return arch_atomic_add_return(incby, this_cpu_ptr(&context_tracking.state)); } #else diff --git a/include/linux/context_tracking_state.h b/include/linux/context_tracking_state.h index 5f11e3d2d85a..e20a74bc0597 100644 --- a/include/linux/context_tracking_state.h +++ b/include/linux/context_tracking_state.h @@ -6,16 +6,24 @@ #include #include -enum ctx_state { - CONTEXT_DISABLED = -1, /* returned by ct_state() if unknown */ - CONTEXT_KERNEL = 0, - CONTEXT_USER, - CONTEXT_GUEST, -}; - /* Offset to allow distinguishing irq vs. task-based idle entry/exit. */ #define DYNTICK_IRQ_NONIDLE ((LONG_MAX / 2) + 1) +enum ctx_state { + CONTEXT_DISABLED = -1, /* returned by ct_state() if unknown */ + CONTEXT_KERNEL = 0, + CONTEXT_IDLE = 1, + CONTEXT_USER = 2, + CONTEXT_GUEST = 3, + CONTEXT_MAX = 4, +}; + +/* Even value for idle, else odd. */ +#define RCU_DYNTICKS_IDX CONTEXT_MAX + +#define CT_STATE_MASK (CONTEXT_MAX - 1) +#define CT_DYNTICKS_MASK (~CT_STATE_MASK) + struct context_tracking { #ifdef CONFIG_CONTEXT_TRACKING_USER /* @@ -26,10 +34,11 @@ struct context_tracking { */ bool active; int recursion; - enum ctx_state state; +#endif +#ifdef CONFIG_CONTEXT_TRACKING + atomic_t state; #endif #ifdef CONFIG_CONTEXT_TRACKING_IDLE - atomic_t dynticks; /* Even value for idle, else odd. */ long dynticks_nesting; /* Track process nesting level. */ long dynticks_nmi_nesting; /* Track irq/NMI nesting level. */ #endif @@ -37,26 +46,31 @@ struct context_tracking { #ifdef CONFIG_CONTEXT_TRACKING DECLARE_PER_CPU(struct context_tracking, context_tracking); + +static __always_inline int __ct_state(void) +{ + return atomic_read(this_cpu_ptr(&context_tracking.state)) & CT_STATE_MASK; +} #endif #ifdef CONFIG_CONTEXT_TRACKING_IDLE static __always_inline int ct_dynticks(void) { - return atomic_read(this_cpu_ptr(&context_tracking.dynticks)); + return atomic_read(this_cpu_ptr(&context_tracking.state)) & CT_DYNTICKS_MASK; } static __always_inline int ct_dynticks_cpu(int cpu) { struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu); - return atomic_read(&ct->dynticks); + return atomic_read(&ct->state) & CT_DYNTICKS_MASK; } static __always_inline int ct_dynticks_cpu_acquire(int cpu) { struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu); - return atomic_read_acquire(&ct->dynticks); + return atomic_read_acquire(&ct->state) & CT_DYNTICKS_MASK; } static __always_inline long ct_dynticks_nesting(void) @@ -102,6 +116,27 @@ static inline bool context_tracking_enabled_this_cpu(void) return context_tracking_enabled() && __this_cpu_read(context_tracking.active); } +/** + * ct_state() - return the current context tracking state if known + * + * Returns the current cpu's context tracking state if context tracking + * is enabled. If context tracking is disabled, returns + * CONTEXT_DISABLED. This should be used primarily for debugging. + */ +static __always_inline int ct_state(void) +{ + int ret; + + if (!context_tracking_enabled()) + return CONTEXT_DISABLED; + + preempt_disable(); + ret = __ct_state(); + preempt_enable(); + + return ret; +} + #else static __always_inline bool context_tracking_enabled(void) { return false; } static __always_inline bool context_tracking_enabled_cpu(int cpu) { return false; } diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index e485b6b01537..ca78ff27dc53 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -28,8 +28,8 @@ DEFINE_PER_CPU(struct context_tracking, context_tracking) = { #ifdef CONFIG_CONTEXT_TRACKING_IDLE .dynticks_nesting = 1, .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE, - .dynticks = ATOMIC_INIT(1), #endif + .state = ATOMIC_INIT(RCU_DYNTICKS_IDX), }; EXPORT_SYMBOL_GPL(context_tracking); @@ -76,7 +76,7 @@ static __always_inline void rcu_dynticks_task_trace_exit(void) * RCU is watching prior to the call to this function and is no longer * watching upon return. */ -static noinstr void rcu_dynticks_eqs_enter(void) +static noinstr void ct_kernel_exit_state(int offset) { int seq; @@ -86,9 +86,9 @@ static noinstr void rcu_dynticks_eqs_enter(void) * next idle sojourn. */ rcu_dynticks_task_trace_enter(); // Before ->dynticks update! - seq = rcu_dynticks_inc(1); + seq = ct_state_inc(offset); // RCU is no longer watching. Better be in extended quiescent state! - WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && (seq & 0x1)); + WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && (seq & RCU_DYNTICKS_IDX)); } /* @@ -96,7 +96,7 @@ static noinstr void rcu_dynticks_eqs_enter(void) * called from an extended quiescent state, that is, RCU is not watching * prior to the call to this function and is watching upon return. */ -static noinstr void rcu_dynticks_eqs_exit(void) +static noinstr void ct_kernel_enter_state(int offset) { int seq; @@ -105,10 +105,10 @@ static noinstr void rcu_dynticks_eqs_exit(void) * and we also must force ordering with the next RCU read-side * critical section. */ - seq = rcu_dynticks_inc(1); + seq = ct_state_inc(offset); // RCU is now watching. Better not be in an extended quiescent state! rcu_dynticks_task_trace_exit(); // After ->dynticks update! - WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !(seq & 0x1)); + WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !(seq & RCU_DYNTICKS_IDX)); } /* @@ -119,7 +119,7 @@ static noinstr void rcu_dynticks_eqs_exit(void) * the possibility of usermode upcalls having messed up our count * of interrupt nesting level during the prior busy period. */ -static void noinstr rcu_eqs_enter(bool user) +static void noinstr ct_kernel_exit(bool user, int offset) { struct context_tracking *ct = this_cpu_ptr(&context_tracking); @@ -139,13 +139,13 @@ static void noinstr rcu_eqs_enter(bool user) WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); rcu_preempt_deferred_qs(current); - // instrumentation for the noinstr rcu_dynticks_eqs_enter() - instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); + // instrumentation for the noinstr ct_kernel_exit_state() + instrument_atomic_write(&ct->state, sizeof(ct->state)); instrumentation_end(); WRITE_ONCE(ct->dynticks_nesting, 0); /* Avoid irq-access tearing. */ // RCU is watching here ... - rcu_dynticks_eqs_enter(); + ct_kernel_exit_state(offset); // ... but is no longer watching here. rcu_dynticks_task_enter(); } @@ -158,7 +158,7 @@ static void noinstr rcu_eqs_enter(bool user) * allow for the possibility of usermode upcalls messing up our count of * interrupt nesting level during the busy period that is just now starting. */ -static void noinstr rcu_eqs_exit(bool user) +static void noinstr ct_kernel_enter(bool user, int offset) { struct context_tracking *ct = this_cpu_ptr(&context_tracking); long oldval; @@ -173,12 +173,12 @@ static void noinstr rcu_eqs_exit(bool user) } rcu_dynticks_task_exit(); // RCU is not watching here ... - rcu_dynticks_eqs_exit(); + ct_kernel_enter_state(offset); // ... but is watching here. instrumentation_begin(); - // instrumentation for the noinstr rcu_dynticks_eqs_exit() - instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); + // instrumentation for the noinstr ct_kernel_enter_state() + instrument_atomic_write(&ct->state, sizeof(ct->state)); trace_rcu_dyntick(TPS("End"), ct_dynticks_nesting(), 1, ct_dynticks()); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); @@ -192,7 +192,7 @@ static void noinstr rcu_eqs_exit(bool user) * ct_nmi_exit - inform RCU of exit from NMI context * * If we are returning from the outermost NMI handler that interrupted an - * RCU-idle period, update ct->dynticks and ct->dynticks_nmi_nesting + * RCU-idle period, update ct->state and ct->dynticks_nmi_nesting * to let the RCU grace-period handling know that the CPU is back to * being RCU-idle. * @@ -229,12 +229,12 @@ void noinstr ct_nmi_exit(void) trace_rcu_dyntick(TPS("Startirq"), ct_dynticks_nmi_nesting(), 0, ct_dynticks()); WRITE_ONCE(ct->dynticks_nmi_nesting, 0); /* Avoid store tearing. */ - // instrumentation for the noinstr rcu_dynticks_eqs_enter() - instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); + // instrumentation for the noinstr ct_kernel_exit_state() + instrument_atomic_write(&ct->state, sizeof(ct->state)); instrumentation_end(); // RCU is watching here ... - rcu_dynticks_eqs_enter(); + ct_kernel_exit_state(RCU_DYNTICKS_IDX); // ... but is no longer watching here. if (!in_nmi()) @@ -244,7 +244,7 @@ void noinstr ct_nmi_exit(void) /** * ct_nmi_enter - inform RCU of entry to NMI context * - * If the CPU was idle from RCU's viewpoint, update ct->dynticks and + * If the CPU was idle from RCU's viewpoint, update ct->state and * ct->dynticks_nmi_nesting to let the RCU grace-period handling know * that the CPU is active. This implementation permits nested NMIs, as * long as the nesting level does not overflow an int. (You will probably @@ -275,14 +275,14 @@ void noinstr ct_nmi_enter(void) rcu_dynticks_task_exit(); // RCU is not watching here ... - rcu_dynticks_eqs_exit(); + ct_kernel_enter_state(RCU_DYNTICKS_IDX); // ... but is watching here. instrumentation_begin(); // instrumentation for the noinstr rcu_dynticks_curr_cpu_in_eqs() - instrument_atomic_read(&ct->dynticks, sizeof(ct->dynticks)); - // instrumentation for the noinstr rcu_dynticks_eqs_exit() - instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks)); + instrument_atomic_read(&ct->state, sizeof(ct->state)); + // instrumentation for the noinstr ct_kernel_enter_state() + instrument_atomic_write(&ct->state, sizeof(ct->state)); incby = 1; } else if (!in_nmi()) { @@ -315,7 +315,7 @@ void noinstr ct_nmi_enter(void) void noinstr ct_idle_enter(void) { WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !raw_irqs_disabled()); - rcu_eqs_enter(false); + ct_kernel_exit(false, RCU_DYNTICKS_IDX + CONTEXT_IDLE); } EXPORT_SYMBOL_GPL(ct_idle_enter); @@ -333,7 +333,7 @@ void noinstr ct_idle_exit(void) unsigned long flags; raw_local_irq_save(flags); - rcu_eqs_exit(false); + ct_kernel_enter(false, RCU_DYNTICKS_IDX - CONTEXT_IDLE); raw_local_irq_restore(flags); } EXPORT_SYMBOL_GPL(ct_idle_exit); @@ -421,8 +421,8 @@ void ct_irq_exit_irqson(void) local_irq_restore(flags); } #else -static __always_inline void rcu_eqs_enter(bool user) { } -static __always_inline void rcu_eqs_exit(bool user) { } +static __always_inline void ct_kernel_exit(bool user, int offset) { } +static __always_inline void ct_kernel_enter(bool user, int offset) { } #endif /* #ifdef CONFIG_CONTEXT_TRACKING_IDLE */ #ifdef CONFIG_CONTEXT_TRACKING_USER @@ -463,6 +463,7 @@ static __always_inline void context_tracking_recursion_exit(void) */ void noinstr __ct_user_enter(enum ctx_state state) { + struct context_tracking *ct = this_cpu_ptr(&context_tracking); lockdep_assert_irqs_disabled(); /* Kernel threads aren't supposed to go to userspace */ @@ -471,8 +472,8 @@ void noinstr __ct_user_enter(enum ctx_state state) if (!context_tracking_recursion_enter()) return; - if ( __this_cpu_read(context_tracking.state) != state) { - if (__this_cpu_read(context_tracking.active)) { + if (__ct_state() != state) { + if (ct->active) { /* * At this stage, only low level arch entry code remains and * then we'll run in userspace. We can assume there won't be @@ -492,28 +493,49 @@ void noinstr __ct_user_enter(enum ctx_state state) * that will fire and reschedule once we resume in user/guest mode. */ rcu_irq_work_resched(); + /* * Enter RCU idle mode right before resuming userspace. No use of RCU * is permitted between this call and rcu_eqs_exit(). This way the * CPU doesn't need to maintain the tick for RCU maintenance purposes * when the CPU runs in userspace. */ - rcu_eqs_enter(true); + ct_kernel_exit(true, RCU_DYNTICKS_IDX + state); + + /* + * Special case if we only track user <-> kernel transitions for tickless + * cputime accounting but we don't support RCU extended quiescent state. + * In this we case we don't care about any concurrency/ordering. + */ + if (!IS_ENABLED(CONFIG_CONTEXT_TRACKING_IDLE)) + atomic_set(&ct->state, state); + } else { + /* + * Even if context tracking is disabled on this CPU, because it's outside + * the full dynticks mask for example, we still have to keep track of the + * context transitions and states to prevent inconsistency on those of + * other CPUs. + * If a task triggers an exception in userspace, sleep on the exception + * handler and then migrate to another CPU, that new CPU must know where + * the exception returns by the time we call exception_exit(). + * This information can only be provided by the previous CPU when it called + * exception_enter(). + * OTOH we can spare the calls to vtime and RCU when context_tracking.active + * is false because we know that CPU is not tickless. + */ + if (!IS_ENABLED(CONFIG_CONTEXT_TRACKING_IDLE)) { + /* Tracking for vtime only, no concurrent RCU EQS accounting */ + atomic_set(&ct->state, state); + } else { + /* + * Tracking for vtime and RCU EQS. Make sure we don't race + * with NMIs. OTOH we don't care about ordering here since + * RCU only requires RCU_DYNTICKS_IDX increments to be fully + * ordered. + */ + atomic_add(state, &ct->state); + } } - /* - * Even if context tracking is disabled on this CPU, because it's outside - * the full dynticks mask for example, we still have to keep track of the - * context transitions and states to prevent inconsistency on those of - * other CPUs. - * If a task triggers an exception in userspace, sleep on the exception - * handler and then migrate to another CPU, that new CPU must know where - * the exception returns by the time we call exception_exit(). - * This information can only be provided by the previous CPU when it called - * exception_enter(). - * OTOH we can spare the calls to vtime and RCU when context_tracking.active - * is false because we know that CPU is not tickless. - */ - __this_cpu_write(context_tracking.state, state); } context_tracking_recursion_exit(); } @@ -581,24 +603,47 @@ NOKPROBE_SYMBOL(user_enter_callable); */ void noinstr __ct_user_exit(enum ctx_state state) { + struct context_tracking *ct = this_cpu_ptr(&context_tracking); + if (!context_tracking_recursion_enter()) return; - if (__this_cpu_read(context_tracking.state) == state) { - if (__this_cpu_read(context_tracking.active)) { + if (__ct_state() == state) { + if (ct->active) { /* * Exit RCU idle mode while entering the kernel because it can * run a RCU read side critical section anytime. */ - rcu_eqs_exit(true); + ct_kernel_enter(true, RCU_DYNTICKS_IDX - state); if (state == CONTEXT_USER) { instrumentation_begin(); vtime_user_exit(current); trace_user_exit(0); instrumentation_end(); } + + /* + * Special case if we only track user <-> kernel transitions for tickless + * cputime accounting but we don't support RCU extended quiescent state. + * In this we case we don't care about any concurrency/ordering. + */ + if (!IS_ENABLED(CONFIG_CONTEXT_TRACKING_IDLE)) + atomic_set(&ct->state, CONTEXT_KERNEL); + + } else { + if (!IS_ENABLED(CONFIG_CONTEXT_TRACKING_IDLE)) { + /* Tracking for vtime only, no concurrent RCU EQS accounting */ + atomic_set(&ct->state, CONTEXT_KERNEL); + } else { + /* + * Tracking for vtime and RCU EQS. Make sure we don't race + * with NMIs. OTOH we don't care about ordering here since + * RCU only requires RCU_DYNTICKS_IDX increments to be fully + * ordered. + */ + atomic_sub(state, &ct->state); + } } - __this_cpu_write(context_tracking.state, CONTEXT_KERNEL); } context_tracking_recursion_exit(); } diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index e2a2083079a2..f9d20b40071f 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -272,9 +272,9 @@ void rcu_softirq_qs(void) */ static void rcu_dynticks_eqs_online(void) { - if (ct_dynticks() & 0x1) + if (ct_dynticks() & RCU_DYNTICKS_IDX) return; - rcu_dynticks_inc(1); + ct_state_inc(RCU_DYNTICKS_IDX); } /* @@ -293,7 +293,7 @@ static int rcu_dynticks_snap(int cpu) */ static bool rcu_dynticks_in_eqs(int snap) { - return !(snap & 0x1); + return !(snap & RCU_DYNTICKS_IDX); } /* Return true if the specified CPU is currently idle from an RCU viewpoint. */ @@ -321,8 +321,7 @@ bool rcu_dynticks_zero_in_eqs(int cpu, int *vp) int snap; // If not quiescent, force back to earlier extended quiescent state. - snap = ct_dynticks_cpu(cpu) & ~0x1; - + snap = ct_dynticks_cpu(cpu) & ~RCU_DYNTICKS_IDX; smp_rmb(); // Order ->dynticks and *vp reads. if (READ_ONCE(*vp)) return false; // Non-zero, so report failure; @@ -348,9 +347,9 @@ notrace void rcu_momentary_dyntick_idle(void) int seq; raw_cpu_write(rcu_data.rcu_need_heavy_qs, false); - seq = rcu_dynticks_inc(2); + seq = ct_state_inc(2 * RCU_DYNTICKS_IDX); /* It is illegal to call this from idle state. */ - WARN_ON_ONCE(!(seq & 0x1)); + WARN_ON_ONCE(!(seq & RCU_DYNTICKS_IDX)); rcu_preempt_deferred_qs(current); } EXPORT_SYMBOL_GPL(rcu_momentary_dyntick_idle); diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index 2683ce0a7c72..195cad14742d 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -469,7 +469,7 @@ static void print_cpu_stall_info(int cpu) rcuc_starved = rcu_is_rcuc_kthread_starving(rdp, &j); if (rcuc_starved) sprintf(buf, " rcuc=%ld jiffies(starved)", j); - pr_err("\t%d-%c%c%c%c: (%lu %s) idle=%03x/%ld/%#lx softirq=%u/%u fqs=%ld%s%s\n", + pr_err("\t%d-%c%c%c%c: (%lu %s) idle=%04x/%ld/%#lx softirq=%u/%u fqs=%ld%s%s\n", cpu, "O."[!!cpu_online(cpu)], "o."[!!(rdp->grpmask & rdp->mynode->qsmaskinit)], @@ -478,7 +478,7 @@ static void print_cpu_stall_info(int cpu) rdp->rcu_iw_pending ? (int)min(delta, 9UL) + '0' : "!."[!delta], ticks_value, ticks_title, - rcu_dynticks_snap(cpu) & 0xfff, + rcu_dynticks_snap(cpu) & 0xffff, ct_dynticks_nesting_cpu(cpu), ct_dynticks_nmi_nesting_cpu(cpu), rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu), data_race(rcu_state.n_force_qs) - rcu_state.n_force_qs_gpstart, From 08ab707dfc83d6ab7829c1c0f39b0d4530fa42a8 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 8 Jun 2022 16:40:37 +0200 Subject: [PATCH 160/318] MAINTAINERS: Add Paul as context tracking maintainer Since most of the bits have been imported from kernel/rcu/tree.c and now that the context tracking code is tightly linked to RCU, add Paul as a context tracking maintainer. Also update the context tracking file header accordingly. Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Boqun Feng Cc: Nicolas Saenz Julienne Cc: Marcelo Tosatti Cc: Xiongfeng Wang Cc: Yu Liao Cc: Phil Auld Cc: Paul Gortmaker Cc: Alex Belits Signed-off-by: Paul E. McKenney Reviewed-by: Nicolas Saenz Julienne Tested-by: Nicolas Saenz Julienne --- MAINTAINERS | 1 + kernel/context_tracking.c | 12 +++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 3cf9842d9233..4e38d7533cbe 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5039,6 +5039,7 @@ F: include/linux/console* CONTEXT TRACKING M: Frederic Weisbecker +M: "Paul E. McKenney" S: Maintained F: kernel/context_tracking.c F: include/linux/context_tracking* diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index ca78ff27dc53..77978e372377 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -1,18 +1,20 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Context tracking: Probe on high level context boundaries such as kernel - * and userspace. This includes syscalls and exceptions entry/exit. + * Context tracking: Probe on high level context boundaries such as kernel, + * userspace, guest or idle. * * This is used by RCU to remove its dependency on the timer tick while a CPU - * runs in userspace. + * runs in idle, userspace or guest mode. * - * Started by Frederic Weisbecker: + * User/guest tracking started by Frederic Weisbecker: * - * Copyright (C) 2012 Red Hat, Inc., Frederic Weisbecker + * Copyright (C) 2012 Red Hat, Inc., Frederic Weisbecker * * Many thanks to Gilad Ben-Yossef, Paul McKenney, Ingo Molnar, Andrew Morton, * Steven Rostedt, Peter Zijlstra for suggestions and improvements. * + * RCU extended quiescent state bits imported from kernel/rcu/tree.c + * where the relevant authorship may be found. */ #include From 1dcaa3b462265f688613163a1562a65ee53a3311 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 16 Jun 2022 09:30:37 -0700 Subject: [PATCH 161/318] context_tracking: Use arch_atomic_read() in __ct_state for KASAN Context tracking's __ct_state() function can be invoked from noinstr state where RCU is not watching. This means that its use of atomic_read() causes KASAN to invoke the non-noinstr __kasan_check_read() function from the noinstr function __ct_state(). This is problematic because someone tracing the __kasan_check_read() function could get a nasty surprise because of RCU not watching. This commit therefore replaces the __ct_state() function's use of atomic_read() with arch_atomic_read(), which KASAN does not attempt to add instrumention to. Reported-by: kernel test robot Signed-off-by: Paul E. McKenney Cc: Frederic Weisbecker Cc: Marco Elver Reviewed-by: Marco Elver --- include/linux/context_tracking_state.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/context_tracking_state.h b/include/linux/context_tracking_state.h index e20a74bc0597..4a4d56f77180 100644 --- a/include/linux/context_tracking_state.h +++ b/include/linux/context_tracking_state.h @@ -49,7 +49,7 @@ DECLARE_PER_CPU(struct context_tracking, context_tracking); static __always_inline int __ct_state(void) { - return atomic_read(this_cpu_ptr(&context_tracking.state)) & CT_STATE_MASK; + return arch_atomic_read(this_cpu_ptr(&context_tracking.state)) & CT_STATE_MASK; } #endif From 7c2d45a347c7933cbe0efff14fe96adeb13fd761 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sat, 2 Jul 2022 11:11:25 +0100 Subject: [PATCH 162/318] scripts: sphinx-pre-install: fix venv version check logic The logic which checks if the venv version is good enough but was not activated is broken: it is checking against the wrong val, making it to recommend to re-create a venv every time. Fix it. Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/afe01b7863fd655986d84ace8948f3d7aede796d.1656756450.git.mchehab@kernel.org Signed-off-by: Jonathan Corbet --- scripts/sphinx-pre-install | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index f126ecbb0494..ae8c49734899 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -741,7 +741,7 @@ sub recommend_sphinx_upgrade() # Get the highest version from sphinx_*/bin/sphinx-build and the # corresponding command to activate the venv/virtenv - $activate_cmd = get_virtenv(); + ($activate_cmd, $venv_ver) = get_virtenv(); # Store the highest version from Sphinx existing virtualenvs if (($activate_cmd ne "") && ($venv_ver gt $cur_version)) { @@ -759,10 +759,14 @@ sub recommend_sphinx_upgrade() # Either there are already a virtual env or a new one should be created $need_pip = 1; + return if (!$latest_avail_ver); + # Return if the reason is due to an upgrade or not if ($latest_avail_ver lt $rec_version) { $rec_sphinx_upgrade = 1; } + + return $latest_avail_ver; } # @@ -820,7 +824,7 @@ sub recommend_sphinx_version($) } # Suggest newer versions if current ones are too old - if ($latest_avail_ver && $cur_version ge $min_version) { + if ($latest_avail_ver && $latest_avail_ver ge $min_version) { # If there's a good enough version, ask the user to enable it if ($latest_avail_ver ge $rec_version) { printf "\nNeed to activate Sphinx (version $latest_avail_ver) on virtualenv with:\n"; @@ -897,7 +901,7 @@ sub check_needs() } } - recommend_sphinx_upgrade(); + my $venv_ver = recommend_sphinx_upgrade(); my $virtualenv_cmd; From ed2133b7b5859bc1de86ed44d5cd93096cadc446 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sat, 2 Jul 2022 11:11:26 +0100 Subject: [PATCH 163/318] scripts: sphinx-pre-install: report broken venv After distro upgrades, the directory names for python may change. On such case, the previously-created venv will be broken, and sphinx-build won't run. Add a logic to report it. Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/f117f03555b0636d2be0b68f3a23b1d3f03ccf1d.1656756450.git.mchehab@kernel.org [jc: fixed typo in warning message] Signed-off-by: Jonathan Corbet --- scripts/sphinx-pre-install | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index ae8c49734899..cfb82c548fb3 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -720,6 +720,12 @@ sub get_virtenv() next if (! -f $sphinx_cmd); my $ver = get_sphinx_version($sphinx_cmd); + + if (!$ver) { + $f =~ s#/bin/activate##; + print("Warning: virtual environment $f is not working.\nPython version upgrade? Remove it with:\n\n\trm -rf $f\n\n"); + } + if ($need_sphinx && ($ver ge $min_version)) { return ($f, $ver); } elsif ($ver gt $cur_version) { From 5b27d2551c199c3077b7104b1417b6aecff15d34 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sat, 2 Jul 2022 11:11:27 +0100 Subject: [PATCH 164/318] scripts: sphinx-pre-install: check for PDF min version later on Better to add the PDF note late for venv recommendation. Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/8e117aabe6dfa1b1ec92dccd20e801393c977667.1656756450.git.mchehab@kernel.org Signed-off-by: Jonathan Corbet --- scripts/sphinx-pre-install | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index cfb82c548fb3..2ef2c561fdc6 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -785,12 +785,13 @@ sub recommend_sphinx_version($) { my $virtualenv_cmd = shift; - if ($latest_avail_ver lt $min_pdf_version) { - print "note: If you want pdf, you need at least Sphinx $min_pdf_version.\n"; - } - # Version is OK. Nothing to do. - return if ($cur_version && ($cur_version ge $rec_version)); + if ($cur_version && ($cur_version ge $rec_version)) { + if ($cur_version lt $min_pdf_version) { + print "note: If you want pdf, you need at least Sphinx $min_pdf_version.\n"; + } + return; + }; if (!$need_sphinx) { # sphinx-build is present and its version is >= $min_version @@ -837,6 +838,10 @@ sub recommend_sphinx_version($) printf "\t. $activate_cmd\n"; deactivate_help(); + if ($latest_avail_ver lt $min_pdf_version) { + print "note: If you want pdf, you need at least Sphinx $min_pdf_version.\n"; + } + return; } From 6a5d7320fb75af071cde628e82ebc938575a2fd6 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sat, 2 Jul 2022 11:11:28 +0100 Subject: [PATCH 165/318] scripts: sphinx-pre-install: provide both venv and package installs As it is not a consensus about installing sphinx using venv, and modern distributions are now shipping with Sphinx versions above the minimal requirements to build the docs, provide both venv and package install commands by default. Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/7c99d985df204c73e3daafd5fd2f30a31269405d.1656756450.git.mchehab@kernel.org Signed-off-by: Jonathan Corbet --- scripts/sphinx-pre-install | 55 +++++++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 13 deletions(-) diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index 2ef2c561fdc6..b9b7d5e301f4 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -25,6 +25,7 @@ my $need_sphinx = 0; my $need_pip = 0; my $need_virtualenv = 0; my $rec_sphinx_upgrade = 0; +my $verbose_warn_install = 1; my $install = ""; my $virtenv_dir = ""; my $python_cmd = ""; @@ -103,10 +104,12 @@ sub check_missing(%) next; } - if ($is_optional) { - print "Warning: better to also install \"$prog\".\n"; - } else { - print "ERROR: please install \"$prog\", otherwise, build won't work.\n"; + if ($verbose_warn_install) { + if ($is_optional) { + print "Warning: better to also install \"$prog\".\n"; + } else { + print "ERROR: please install \"$prog\", otherwise, build won't work.\n"; + } } if (defined($map{$prog})) { $install .= " " . $map{$prog}; @@ -386,7 +389,8 @@ sub give_debian_hints() check_missing(\%map); return if (!$need && !$optional); - printf("You should run:\n\n\tsudo apt-get install $install\n"); + printf("You should run:\n") if ($verbose_warn_install); + printf("\n\tsudo apt-get install $install\n"); } sub give_redhat_hints() @@ -458,10 +462,12 @@ sub give_redhat_hints() if (!$old) { # dnf, for Fedora 18+ - printf("You should run:\n\n\tsudo dnf install -y $install\n"); + printf("You should run:\n") if ($verbose_warn_install); + printf("\n\tsudo dnf install -y $install\n"); } else { # yum, for RHEL (and clones) or Fedora version < 18 - printf("You should run:\n\n\tsudo yum install -y $install\n"); + printf("You should run:\n") if ($verbose_warn_install); + printf("\n\tsudo yum install -y $install\n"); } } @@ -509,7 +515,8 @@ sub give_opensuse_hints() check_missing(\%map); return if (!$need && !$optional); - printf("You should run:\n\n\tsudo zypper install --no-recommends $install\n"); + printf("You should run:\n") if ($verbose_warn_install); + printf("\n\tsudo zypper install --no-recommends $install\n"); } sub give_mageia_hints() @@ -553,7 +560,8 @@ sub give_mageia_hints() check_missing(\%map); return if (!$need && !$optional); - printf("You should run:\n\n\tsudo $packager_cmd $install\n"); + printf("You should run:\n") if ($verbose_warn_install); + printf("\n\tsudo $packager_cmd $install\n"); } sub give_arch_linux_hints() @@ -583,7 +591,8 @@ sub give_arch_linux_hints() check_missing(\%map); return if (!$need && !$optional); - printf("You should run:\n\n\tsudo pacman -S $install\n"); + printf("You should run:\n") if ($verbose_warn_install); + printf("\n\tsudo pacman -S $install\n"); } sub give_gentoo_hints() @@ -610,7 +619,8 @@ sub give_gentoo_hints() return if (!$need && !$optional); - printf("You should run:\n\n"); + printf("You should run:\n") if ($verbose_warn_install); + printf("\n"); my $imagemagick = "media-gfx/imagemagick svg png"; my $cairo = "media-gfx/graphviz cairo pdf"; @@ -700,7 +710,7 @@ sub check_distros() sub deactivate_help() { - printf "\nIf you want to exit the virtualenv, you can use:\n"; + printf "\n If you want to exit the virtualenv, you can use:\n"; printf "\tdeactivate\n"; } @@ -863,7 +873,7 @@ sub recommend_sphinx_version($) print "To upgrade Sphinx, use:\n\n"; } } else { - print "Sphinx needs to be installed either as a package or via pip/pypi with:\n"; + print "\nSphinx needs to be installed either:\n1) via pip/pypi with:\n\n"; } $python_cmd = find_python_no_venv(); @@ -873,6 +883,25 @@ sub recommend_sphinx_version($) printf "\t. $virtenv_dir/bin/activate\n"; printf "\tpip install -r $requirement_file\n"; deactivate_help(); + + printf "\n2) As a package with:\n"; + + my $old_need = $need; + my $old_optional = $optional; + %missing = (); + $pdf = 0; + $optional = 0; + $install = ""; + $verbose_warn_install = 0; + + add_package("python-sphinx", 0); + check_python_module("sphinx_rtd_theme", 1); + + check_distros(); + + $need = $old_need; + $optional = $old_optional; + } sub check_needs() From 02bba8ca90b1877ae47400c31002ceb498b0dd9f Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sat, 2 Jul 2022 11:11:29 +0100 Subject: [PATCH 166/318] scripts: sphinx-pre-install: place a warning for Sphinx >= 3.0 Newer versions of Sphinx have a known bug: https://github.com/sphinx-doc/sphinx/pull/8313 Those currently produces 11 false-positive warnings On Sphinx version 3.1+. While Sphinx fix is not applied, let's warn adventurers that would be using Sphinx installed via distro packages. Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/085e0ada65c11da9303d07e70c510dc45f21315b.1656756450.git.mchehab@kernel.org Signed-off-by: Jonathan Corbet --- scripts/sphinx-pre-install | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index b9b7d5e301f4..ec84fc62774e 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -902,6 +902,10 @@ sub recommend_sphinx_version($) $need = $old_need; $optional = $old_optional; + printf "\n Please note that Sphinx >= 3.0 will currently produce false-positive\n"; + printf " warning when the same name is used for more than one type (functions,\n"; + printf " structs, enums,...). This is known Sphinx bug. For more details, see:\n"; + printf "\thttps://github.com/sphinx-doc/sphinx/pull/8313\n"; } sub check_needs() From f2c1ff0d3be2670d04afba792ec0c79d37a9cfc8 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sat, 2 Jul 2022 12:07:34 +0100 Subject: [PATCH 167/318] docs: tegra194-hte.rst: don't include gpiolib.c twice All extern functions of drivers/gpio/gpiolib.c are already inside the Kernel documentation, as driver-api/gpio/index.rst already includes it. Placing a kernel-doc here will only cause mess, as the same symbol will be placed on two parts of the document, causing breakages in cross-references. So, instead, add a cross-reference there. This solves those Sphinx 3.1+ warnings: .../Documentation/driver-api/hte/tegra194-hte:28: ./drivers/gpio/gpiolib.c:2464: WARNING: Duplicate C declaration, also defined at driver-api/gpio/index:2464. .../Documentation/driver-api/hte/tegra194-hte:28: ./drivers/gpio/gpiolib.c:2493: WARNING: Duplicate C declaration, also defined at driver-api/gpio/index:2493. .../Documentation/driver-api/hte/tegra194-hte.rst:2464: WARNING: Duplicate C declaration, also defined at driver-api/gpio/index:2464. .../Documentation/driver-api/hte/tegra194-hte.rst:2464: WARNING: Duplicate C declaration, also defined at driver-api/gpio/index:2464. .../Documentation/driver-api/hte/tegra194-hte.rst:2464: WARNING: Duplicate C declaration, also defined at driver-api/gpio/index:2464. .../Documentation/driver-api/hte/tegra194-hte.rst:2493: WARNING: Duplicate C declaration, also defined at driver-api/gpio/index:2493. .../Documentation/driver-api/hte/tegra194-hte.rst:2493: WARNING: Duplicate C declaration, also defined at driver-api/gpio/index:2493. .../Documentation/driver-api/hte/tegra194-hte.rst:2493: WARNING: Duplicate C declaration, also defined at driver-api/gpio/index:2493. Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/de81b472f552bd651f140f0aa779a29652fffa62.1656759989.git.mchehab@kernel.org Acked-by: Dipen Patel Signed-off-by: Jonathan Corbet --- Documentation/driver-api/hte/tegra194-hte.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Documentation/driver-api/hte/tegra194-hte.rst b/Documentation/driver-api/hte/tegra194-hte.rst index 41983e04d2a0..d16956256c85 100644 --- a/Documentation/driver-api/hte/tegra194-hte.rst +++ b/Documentation/driver-api/hte/tegra194-hte.rst @@ -25,8 +25,7 @@ and userspace consumers. The kernel space consumers can directly talk to HTE subsystem while userspace consumers timestamp requests go through GPIOLIB CDEV framework to HTE subsystem. -.. kernel-doc:: drivers/gpio/gpiolib.c - :functions: gpiod_enable_hw_timestamp_ns gpiod_disable_hw_timestamp_ns +See gpiod_enable_hw_timestamp_ns() and gpiod_disable_hw_timestamp_ns(). For userspace consumers, GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE flag must be specified during IOCTL calls. Refer to ``tools/gpio/gpio-event-mon.c``, which From 26c82972f20f6b3fdbd77b2802693808da3c0c09 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 30 Jun 2022 10:36:29 -0600 Subject: [PATCH 168/318] docs: automarkup: track failed cross-reference attempts The automarkup code tries to create a lot of cross-references that don't exist. Cross-reference lookups are expensive, especially in later versions of Sphinx, so there is value in avoiding unnecessary ones. Remember attempts that failed and do not retry them. This improves the htmldocs build time by 5-10% depending on the phase of the moon and other factors. Tested-by: Akira Yokosawa Link: https://lore.kernel.org/r/20220630163630.714673-2-corbet@lwn.net Signed-off-by: Jonathan Corbet --- Documentation/sphinx/automarkup.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Documentation/sphinx/automarkup.py b/Documentation/sphinx/automarkup.py index cc348b219fca..5b1f83e6192f 100644 --- a/Documentation/sphinx/automarkup.py +++ b/Documentation/sphinx/automarkup.py @@ -120,6 +120,16 @@ def markup_refs(docname, app, node): repl.append(nodes.Text(t[done:])) return repl +# +# Keep track of cross-reference lookups that failed so we don't have to +# do them again. +# +failed_lookups = { } +def failure_seen(target, reftype): + return (target + '::' + reftype) in failed_lookups +def note_failure(target, reftype): + failed_lookups[target + '::' + reftype] = True + # # In sphinx3 we can cross-reference to C macro and function, each one with its # own C role, but both match the same regex, so we try both. @@ -145,6 +155,8 @@ def markup_func_ref_sphinx3(docname, app, match): for target in possible_targets: if target not in Skipfuncs: for class_s, reftype_s in zip(class_str, reftype_str): + if failure_seen(target, reftype_s): + continue lit_text = nodes.literal(classes=['xref', 'c', class_s]) lit_text += target_text pxref = addnodes.pending_xref('', refdomain = 'c', @@ -164,6 +176,7 @@ def markup_func_ref_sphinx3(docname, app, match): if xref: return xref + note_failure(target, reftype_s) return target_text From 309027b57c61d4d168679d49fd11add0dfdce900 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Thu, 30 Jun 2022 10:36:30 -0600 Subject: [PATCH 169/318] docs: automarkup: do not look up symbols twice The automarkup code tries to look up symbols once as a function, and once as a macro. The Sphinx C domain code, though, totally ignores that distinction and will return the same results either way. So just look things up once and be done with it; the resulting output does not change, but htmldocs build time drops by about 5%. Tested-by: Akira Yokosawa Link: https://lore.kernel.org/r/20220630163630.714673-3-corbet@lwn.net Signed-off-by: Jonathan Corbet --- Documentation/sphinx/automarkup.py | 55 ++++++++++++++---------------- 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/Documentation/sphinx/automarkup.py b/Documentation/sphinx/automarkup.py index 5b1f83e6192f..06b34740bf90 100644 --- a/Documentation/sphinx/automarkup.py +++ b/Documentation/sphinx/automarkup.py @@ -125,19 +125,16 @@ def markup_refs(docname, app, node): # do them again. # failed_lookups = { } -def failure_seen(target, reftype): - return (target + '::' + reftype) in failed_lookups -def note_failure(target, reftype): - failed_lookups[target + '::' + reftype] = True +def failure_seen(target): + return (target) in failed_lookups +def note_failure(target): + failed_lookups[target] = True # # In sphinx3 we can cross-reference to C macro and function, each one with its # own C role, but both match the same regex, so we try both. # def markup_func_ref_sphinx3(docname, app, match): - class_str = ['c-func', 'c-macro'] - reftype_str = ['function', 'macro'] - cdom = app.env.domains['c'] # # Go through the dance of getting an xref out of the C domain @@ -153,30 +150,28 @@ def markup_func_ref_sphinx3(docname, app, match): if base_target not in Skipnames: for target in possible_targets: - if target not in Skipfuncs: - for class_s, reftype_s in zip(class_str, reftype_str): - if failure_seen(target, reftype_s): - continue - lit_text = nodes.literal(classes=['xref', 'c', class_s]) - lit_text += target_text - pxref = addnodes.pending_xref('', refdomain = 'c', - reftype = reftype_s, - reftarget = target, modname = None, - classname = None) - # - # XXX The Latex builder will throw NoUri exceptions here, - # work around that by ignoring them. - # - try: - xref = cdom.resolve_xref(app.env, docname, app.builder, - reftype_s, target, pxref, - lit_text) - except NoUri: - xref = None + if (target not in Skipfuncs) and not failure_seen(target): + lit_text = nodes.literal(classes=['xref', 'c', 'c-func']) + lit_text += target_text + pxref = addnodes.pending_xref('', refdomain = 'c', + reftype = 'function', + reftarget = target, + modname = None, + classname = None) + # + # XXX The Latex builder will throw NoUri exceptions here, + # work around that by ignoring them. + # + try: + xref = cdom.resolve_xref(app.env, docname, app.builder, + 'function', target, pxref, + lit_text) + except NoUri: + xref = None - if xref: - return xref - note_failure(target, reftype_s) + if xref: + return xref + note_failure(target) return target_text From 335220ab017d5f727fbc557c157fda0e10e19c8c Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sun, 26 Jun 2022 10:10:47 +0100 Subject: [PATCH 170/318] docs: netdev: update maintainer-netdev.rst reference Changeset 8df0136376dc ("docs: netdev: move the netdev-FAQ to the process pages") renamed: Documentation/networking/netdev-FAQ.rst to: Documentation/process/maintainer-netdev.rst. Update its cross-reference accordingly. Fixes: 8df0136376dc ("docs: netdev: move the netdev-FAQ to the process pages") Acked-by: Jakub Kicinski Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/aed0482b3d931bfcfe84f38197526edcbb8fd830.1656234456.git.mchehab@kernel.org Signed-off-by: Jonathan Corbet --- Documentation/translations/it_IT/networking/netdev-FAQ.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/translations/it_IT/networking/netdev-FAQ.rst b/Documentation/translations/it_IT/networking/netdev-FAQ.rst index 7e2456bb7d92..8a1e049585c0 100644 --- a/Documentation/translations/it_IT/networking/netdev-FAQ.rst +++ b/Documentation/translations/it_IT/networking/netdev-FAQ.rst @@ -1,6 +1,6 @@ .. include:: ../disclaimer-ita.rst -:Original: :ref:`Documentation/networking/netdev-FAQ.rst ` +:Original: :ref:`Documentation/process/maintainer-netdev.rst ` .. _it_netdev-FAQ: From 752f596371289cb3b45e99453b5c09d96ed36614 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sun, 26 Jun 2022 10:10:48 +0100 Subject: [PATCH 171/318] docs: filesystems: update netfs-api.rst reference Changeset efc930fa1d84 ("docs: filesystems: caching/netfs-api.txt: convert it to ReST") renamed: Documentation/filesystems/caching/netfs-api.txt to: Documentation/filesystems/caching/netfs-api.rst. Update its cross-reference accordingly. Fixes: efc930fa1d84 ("docs: filesystems: caching/netfs-api.txt: convert it to ReST") Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/5f867f01d42c3e65e111167739ed1a41a26623f9.1656234456.git.mchehab@kernel.org Signed-off-by: Jonathan Corbet --- include/linux/fscache.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/fscache.h b/include/linux/fscache.h index 72585c9729a2..2f82ea31d4c1 100644 --- a/include/linux/fscache.h +++ b/include/linux/fscache.h @@ -378,7 +378,7 @@ void fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data, * * Request that the size of an object be changed. * - * See Documentation/filesystems/caching/netfs-api.txt for a complete + * See Documentation/filesystems/caching/netfs-api.rst for a complete * description. */ static inline From 5fe84fe446d8309bf9a704741850c30ab9a95ac2 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sun, 26 Jun 2022 10:10:49 +0100 Subject: [PATCH 172/318] docs: zh_CN: page_frags.rst: fix a broken reference typo: page_frag.rst -> page_frags.rst Fixes: f51debc256f8 ("docs/zh_CN: add vm page_frags translation") Signed-off-by: Mauro Carvalho Chehab Acked-by: Yanteng Si Link: https://lore.kernel.org/r/a12e18044ddb2ca16a1fed10823d59558d6de405.1656234456.git.mchehab@kernel.org Signed-off-by: Jonathan Corbet --- Documentation/translations/zh_CN/vm/page_frags.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/translations/zh_CN/vm/page_frags.rst b/Documentation/translations/zh_CN/vm/page_frags.rst index ad27fed33634..38ecddb9e1c0 100644 --- a/Documentation/translations/zh_CN/vm/page_frags.rst +++ b/Documentation/translations/zh_CN/vm/page_frags.rst @@ -1,4 +1,4 @@ -:Original: Documentation/vm/page_frag.rst +:Original: Documentation/vm/page_frags.rst :翻译: From e13ce769a00658c9ccc0ead9954c43558056a7d8 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sun, 26 Jun 2022 10:10:51 +0100 Subject: [PATCH 173/318] docs: zh_CN/devicetree: fix typos The devicetree directory was using wrong case: Devicetree/ -> devicetree/ Fixes: a17b0169f29b ("docs/zh_CN: add devicetree index translation") Fixes: c56481299df3 ("docs/zh_CN: add devicetree usage-model translation") Fixes: 09d4466d3f3c ("docs/zh_CN: add devicetree of_unittest translation") Signed-off-by: Mauro Carvalho Chehab Acked-by: Yanteng Si Link: https://lore.kernel.org/r/b4ca010ef223ab9cb76e8caa0a3cde734e0b6b2f.1656234456.git.mchehab@kernel.org Signed-off-by: Jonathan Corbet --- Documentation/translations/zh_CN/devicetree/index.rst | 2 +- Documentation/translations/zh_CN/devicetree/of_unittest.rst | 2 +- Documentation/translations/zh_CN/devicetree/usage-model.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/translations/zh_CN/devicetree/index.rst b/Documentation/translations/zh_CN/devicetree/index.rst index 23d0b6fccd58..3fc355fe0037 100644 --- a/Documentation/translations/zh_CN/devicetree/index.rst +++ b/Documentation/translations/zh_CN/devicetree/index.rst @@ -1,7 +1,7 @@ .. SPDX-License-Identifier: GPL-2.0 .. include:: ../disclaimer-zh_CN.rst -:Original: Documentation/Devicetree/index.rst +:Original: Documentation/devicetree/index.rst :翻译: diff --git a/Documentation/translations/zh_CN/devicetree/of_unittest.rst b/Documentation/translations/zh_CN/devicetree/of_unittest.rst index abd94e771ef8..11eb08ca8866 100644 --- a/Documentation/translations/zh_CN/devicetree/of_unittest.rst +++ b/Documentation/translations/zh_CN/devicetree/of_unittest.rst @@ -1,7 +1,7 @@ .. SPDX-License-Identifier: GPL-2.0 .. include:: ../disclaimer-zh_CN.rst -:Original: Documentation/Devicetree/of_unittest.rst +:Original: Documentation/devicetree/of_unittest.rst :翻译: diff --git a/Documentation/translations/zh_CN/devicetree/usage-model.rst b/Documentation/translations/zh_CN/devicetree/usage-model.rst index accdc33475a0..c6aee82c7e6e 100644 --- a/Documentation/translations/zh_CN/devicetree/usage-model.rst +++ b/Documentation/translations/zh_CN/devicetree/usage-model.rst @@ -1,7 +1,7 @@ .. SPDX-License-Identifier: GPL-2.0 .. include:: ../disclaimer-zh_CN.rst -:Original: Documentation/Devicetree/usage-model.rst +:Original: Documentation/devicetree/usage-model.rst :翻译: From 695c62a823a5798245a5a9c99ea62b65d6f50d6e Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sun, 26 Jun 2022 10:10:52 +0100 Subject: [PATCH 174/318] docs: zh_CN: fix a broken reference The location of the rust doc is wrong (probably a typo). Fix it. Fixes: 3a5915156429 ("docs/zh_CN: Add translation zh_CN/doc-guide/kernel-doc.rst") Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/94fe771609d32583fee41f47fe29433cbce5b04e.1656234456.git.mchehab@kernel.org Signed-off-by: Jonathan Corbet --- Documentation/translations/zh_CN/doc-guide/kernel-doc.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/translations/zh_CN/doc-guide/kernel-doc.rst b/Documentation/translations/zh_CN/doc-guide/kernel-doc.rst index 82ec84470c0b..ccfb9b8329c2 100644 --- a/Documentation/translations/zh_CN/doc-guide/kernel-doc.rst +++ b/Documentation/translations/zh_CN/doc-guide/kernel-doc.rst @@ -14,7 +14,7 @@ Linux内核源文件可以包含kernel-doc格式的结构化文档注释，用 实际有着明显的不同。内核源包含成千上万个kernel-doc注释。请坚持遵循 此处描述的风格。 -.. note:: kernel-doc无法包含Rust代码：请参考 Documentation/rust/docs.rst 。 +.. note:: kernel-doc无法包含Rust代码：请参考 Documentation/rust/general-information.rst 。 从注释中提取kernel-doc结构，并从中生成适当的 `Sphinx C 域`_ 函数和带有锚点的 类型描述。这些注释将被过滤以生成特殊kernel-doc高亮和交叉引用。详见下文。 From 36395fb365df3c216b6bda1fd55cf9a39907852b Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sun, 26 Jun 2022 10:10:53 +0100 Subject: [PATCH 175/318] docs: zh_CN/vm: fix a typo for page reporting ReST file typo: _free_page_reporting.rst -> free_page_reporting.rst Fixes: 9859b277bffe ("docs/zh_CN: add vm frontswap translation") Fixes: adeacecbd36c ("docs/zh_CN: add free_page_reporting translation") Signed-off-by: Mauro Carvalho Chehab Acked-by: Yanteng Si Link: https://lore.kernel.org/r/c7db8f975f461eaebd7ea7489ff9666d89e121c6.1656234456.git.mchehab@kernel.org Signed-off-by: Jonathan Corbet --- Documentation/translations/zh_CN/vm/free_page_reporting.rst | 2 +- Documentation/translations/zh_CN/vm/frontswap.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/translations/zh_CN/vm/free_page_reporting.rst b/Documentation/translations/zh_CN/vm/free_page_reporting.rst index 31d6c34b956b..14336a3aa5f4 100644 --- a/Documentation/translations/zh_CN/vm/free_page_reporting.rst +++ b/Documentation/translations/zh_CN/vm/free_page_reporting.rst @@ -1,6 +1,6 @@ .. include:: ../disclaimer-zh_CN.rst -:Original: Documentation/vm/_free_page_reporting.rst +:Original: Documentation/vm/free_page_reporting.rst :翻译: diff --git a/Documentation/translations/zh_CN/vm/frontswap.rst b/Documentation/translations/zh_CN/vm/frontswap.rst index 3eb07870e2ef..98aa6f581ea7 100644 --- a/Documentation/translations/zh_CN/vm/frontswap.rst +++ b/Documentation/translations/zh_CN/vm/frontswap.rst @@ -1,4 +1,4 @@ -:Original: Documentation/vm/_free_page_reporting.rst +:Original: Documentation/vm/free_page_reporting.rst :翻译: From 97e937dec70d0b78b8d11091273e818158b56070 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sun, 26 Jun 2022 10:10:54 +0100 Subject: [PATCH 176/318] docs: zh_CN/vm/zsmalloc.rst: fix a typo typo: zs_malloc.rst -> zsmalloc.rst Fixes: 6dbc3a5bdb46 ("docs/zh_CN: add vm zsmalloc translation") Signed-off-by: Mauro Carvalho Chehab Acked-by: Yanteng Si Link: https://lore.kernel.org/r/25a9fff20d39a6638b8a7c0fefa0767e77d96dec.1656234456.git.mchehab@kernel.org Signed-off-by: Jonathan Corbet --- Documentation/translations/zh_CN/vm/zsmalloc.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/translations/zh_CN/vm/zsmalloc.rst b/Documentation/translations/zh_CN/vm/zsmalloc.rst index 29e9c70a8eb6..45a9b7ab2a51 100644 --- a/Documentation/translations/zh_CN/vm/zsmalloc.rst +++ b/Documentation/translations/zh_CN/vm/zsmalloc.rst @@ -1,4 +1,4 @@ -:Original: Documentation/vm/zs_malloc.rst +:Original: Documentation/vm/zsmalloc.rst :翻译: From c02b872a7ca7842e4cdbbf621f77607d0a655f83 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sun, 26 Jun 2022 10:10:56 +0100 Subject: [PATCH 177/318] Documentation: update watch_queue.rst references Changeset f5461124d59b ("Documentation: move watch_queue to core-api") renamed: Documentation/watch_queue.rst to: Documentation/core-api/watch_queue.rst. Update the cross-references accordingly. Fixes: f5461124d59b ("Documentation: move watch_queue to core-api") Reviewed-by: Randy Dunlap Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/1c220de9c58f35e815a3df9458ac2bea323c8bfb.1656234456.git.mchehab@kernel.org Signed-off-by: Jonathan Corbet --- Documentation/security/keys/core.rst | 2 +- include/linux/watch_queue.h | 2 +- init/Kconfig | 2 +- kernel/watch_queue.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/security/keys/core.rst b/Documentation/security/keys/core.rst index b3ed5c581034..811b905b56bf 100644 --- a/Documentation/security/keys/core.rst +++ b/Documentation/security/keys/core.rst @@ -1046,7 +1046,7 @@ The keyctl syscall functions are: "filter" is either NULL to remove a watch or a filter specification to indicate what events are required from the key. - See Documentation/watch_queue.rst for more information. + See Documentation/core-api/watch_queue.rst for more information. Note that only one watch may be emplaced for any particular { key, queue_fd } combination. diff --git a/include/linux/watch_queue.h b/include/linux/watch_queue.h index 3b9a40ae8bdb..fc6bba20273b 100644 --- a/include/linux/watch_queue.h +++ b/include/linux/watch_queue.h @@ -4,7 +4,7 @@ * Copyright (C) 2020 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) * - * See Documentation/watch_queue.rst + * See Documentation/core-api/watch_queue.rst */ #ifndef _LINUX_WATCH_QUEUE_H diff --git a/init/Kconfig b/init/Kconfig index c7900e8975f1..6f1a5355dc61 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -414,7 +414,7 @@ config WATCH_QUEUE with watches for key/keyring change notifications and device notifications. - See Documentation/watch_queue.rst + See Documentation/core-api/watch_queue.rst config CROSS_MEMORY_ATTACH bool "Enable process_vm_readv/writev syscalls" diff --git a/kernel/watch_queue.c b/kernel/watch_queue.c index 230038d4f908..869fea4fe26b 100644 --- a/kernel/watch_queue.c +++ b/kernel/watch_queue.c @@ -4,7 +4,7 @@ * Copyright (C) 2020 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) * - * See Documentation/watch_queue.rst + * See Documentation/core-api/watch_queue.rst */ #define pr_fmt(fmt) "watchq: " fmt From e38fd63749eb5debc9e6b030a813f45f6141dee0 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sun, 26 Jun 2022 10:10:57 +0100 Subject: [PATCH 178/318] Documentation: KVM: update s390-pv.rst reference Changesets: daec8d408308 ("Documentation: KVM: add separate directories for architecture-specific documentation") and: daec8d408308 ("Documentation: KVM: add separate directories for architecture-specific documentation") renamed: Documentation/virt/kvm/s390-pv.rst to: Documentation/virt/kvm/s390/s390-pv.rst. Update its cross-reference accordingly. Fixes: daec8d408308 ("Documentation: KVM: add separate directories for architecture-specific documentation") Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/e2676f087d287db0bc31ae7c05c80ce5adf93333.1656234456.git.mchehab@kernel.org Signed-off-by: Jonathan Corbet --- Documentation/virt/kvm/s390/s390-pv-boot.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/virt/kvm/s390/s390-pv-boot.rst b/Documentation/virt/kvm/s390/s390-pv-boot.rst index 73a6083cb5e7..96c48480a360 100644 --- a/Documentation/virt/kvm/s390/s390-pv-boot.rst +++ b/Documentation/virt/kvm/s390/s390-pv-boot.rst @@ -10,7 +10,7 @@ The memory of Protected Virtual Machines (PVMs) is not accessible to I/O or the hypervisor. In those cases where the hypervisor needs to access the memory of a PVM, that memory must be made accessible. Memory made accessible to the hypervisor will be encrypted. See -Documentation/virt/kvm/s390-pv.rst for details." +Documentation/virt/kvm/s390/s390-pv.rst for details." On IPL (boot) a small plaintext bootloader is started, which provides information about the encrypted components and necessary metadata to From 7ac3945d8e22cfa4f32ec5b7f81a68d2afe4e38e Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sun, 26 Jun 2022 10:10:58 +0100 Subject: [PATCH 179/318] Documentation: KVM: update amd-memory-encryption.rst references Changeset daec8d408308 ("Documentation: KVM: add separate directories for architecture-specific documentation") renamed: Documentation/virt/kvm/amd-memory-encryption.rst to: Documentation/virt/kvm/x86/amd-memory-encryption.rst. Update the cross-references accordingly. Fixes: daec8d408308 ("Documentation: KVM: add separate directories for architecture-specific documentation") Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/fd80db889e34aae87a4ca88cad94f650723668f4.1656234456.git.mchehab@kernel.org Signed-off-by: Jonathan Corbet --- Documentation/admin-guide/kernel-parameters.txt | 2 +- Documentation/security/secrets/coco.rst | 2 +- Documentation/virt/kvm/api.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 8090130b544b..5ef0b2998af7 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3104,7 +3104,7 @@ mem_encrypt=on: Activate SME mem_encrypt=off: Do not activate SME - Refer to Documentation/virt/kvm/amd-memory-encryption.rst + Refer to Documentation/virt/kvm/x86/amd-memory-encryption.rst for details on when memory encryption can be activated. mem_sleep_default= [SUSPEND] Default system suspend mode: diff --git a/Documentation/security/secrets/coco.rst b/Documentation/security/secrets/coco.rst index 262e7abb1b24..087e2d1ae38b 100644 --- a/Documentation/security/secrets/coco.rst +++ b/Documentation/security/secrets/coco.rst @@ -98,6 +98,6 @@ References See [sev-api-spec]_ for more info regarding SEV ``LAUNCH_SECRET`` operation. -.. [sev] Documentation/virt/kvm/amd-memory-encryption.rst +.. [sev] Documentation/virt/kvm/x86/amd-memory-encryption.rst .. [secrets-coco-abi] Documentation/ABI/testing/securityfs-secrets-coco .. [sev-api-spec] https://www.amd.com/system/files/TechDocs/55766_SEV-KM_API_Specification.pdf diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 11e00a46c610..b71e1d778e28 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -4667,7 +4667,7 @@ encrypted VMs. Currently, this ioctl is used for issuing Secure Encrypted Virtualization (SEV) commands on AMD Processors. The SEV commands are defined in -Documentation/virt/kvm/amd-memory-encryption.rst. +Documentation/virt/kvm/x86/amd-memory-encryption.rst. 4.111 KVM_MEMORY_ENCRYPT_REG_REGION ----------------------------------- From 48b36e59ac213d534b79df79739f9ed2da7b12df Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sun, 26 Jun 2022 10:10:59 +0100 Subject: [PATCH 180/318] Documentation: KVM: update msr.rst reference Changeset daec8d408308 ("Documentation: KVM: add separate directories for architecture-specific documentation") renamed: Documentation/virt/kvm/msr.rst to: Documentation/virt/kvm/x86/msr.rst. Update its cross-reference accordingly. Fixes: daec8d408308 ("Documentation: KVM: add separate directories for architecture-specific documentation") Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/5652b7f5caff3b817a660b75f1f319a2f8962380.1656234456.git.mchehab@kernel.org Signed-off-by: Jonathan Corbet --- Documentation/virt/kvm/api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index b71e1d778e28..9cc6981f5a34 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -7670,7 +7670,7 @@ architecture-specific interfaces. This capability and the architecture- specific interfaces must be consistent, i.e. if one says the feature is supported, than the other should as well and vice versa. For arm64 see Documentation/virt/kvm/devices/vcpu.rst "KVM_ARM_VCPU_PVTIME_CTRL". -For x86 see Documentation/virt/kvm/msr.rst "MSR_KVM_STEAL_TIME". +For x86 see Documentation/virt/kvm/x86/msr.rst "MSR_KVM_STEAL_TIME". 8.25 KVM_CAP_S390_DIAG318 ------------------------- From 8a5d192166726cd106a94d3e3bab47fb9843c14c Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sun, 26 Jun 2022 10:11:00 +0100 Subject: [PATCH 181/318] Documentation: KVM: update s390-diag.rst reference Changeset daec8d408308 ("Documentation: KVM: add separate directories for architecture-specific documentation") renamed: Documentation/virt/kvm/s390-diag.rst to: Documentation/virt/kvm/s390/s390-diag.rst. Update its cross-reference accordingly. Fixes: daec8d408308 ("Documentation: KVM: add separate directories for architecture-specific documentation") Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/85b81e4678bbe23d0e9692616798762a6465f0a3.1656234456.git.mchehab@kernel.org Signed-off-by: Jonathan Corbet --- Documentation/virt/kvm/x86/hypercalls.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/virt/kvm/x86/hypercalls.rst b/Documentation/virt/kvm/x86/hypercalls.rst index e56fa8b9cfca..10db7924720f 100644 --- a/Documentation/virt/kvm/x86/hypercalls.rst +++ b/Documentation/virt/kvm/x86/hypercalls.rst @@ -22,7 +22,7 @@ S390: number in R1. For further information on the S390 diagnose call as supported by KVM, - refer to Documentation/virt/kvm/s390-diag.rst. + refer to Documentation/virt/kvm/s390/s390-diag.rst. PowerPC: It uses R3-R10 and hypercall number in R11. R4-R11 are used as output registers. From d6a21f2d73258f2a4cd2e7806f5755ee73fddced Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sun, 26 Jun 2022 10:11:01 +0100 Subject: [PATCH 182/318] objtool: update objtool.txt references Changeset a8e35fece49b ("objtool: Update documentation") renamed: tools/objtool/Documentation/stack-validation.txt to: tools/objtool/Documentation/objtool.txt. Update the cross-references accordingly. Fixes: a8e35fece49b ("objtool: Update documentation") Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/ec285ece6348a5be191aebe45f78d06b3319056b.1656234456.git.mchehab@kernel.org Signed-off-by: Jonathan Corbet --- Documentation/x86/orc-unwinder.rst | 2 +- include/linux/objtool.h | 2 +- lib/Kconfig.debug | 2 +- tools/include/linux/objtool.h | 2 +- tools/objtool/check.c | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Documentation/x86/orc-unwinder.rst b/Documentation/x86/orc-unwinder.rst index 9a66a88be765..cdb257015bd9 100644 --- a/Documentation/x86/orc-unwinder.rst +++ b/Documentation/x86/orc-unwinder.rst @@ -140,7 +140,7 @@ Unwinder implementation details Objtool generates the ORC data by integrating with the compile-time stack metadata validation feature, which is described in detail in -tools/objtool/Documentation/stack-validation.txt. After analyzing all +tools/objtool/Documentation/objtool.txt. After analyzing all the code paths of a .o file, it creates an array of orc_entry structs, and a parallel array of instruction addresses associated with those structs, and writes them to the .orc_unwind and .orc_unwind_ip sections diff --git a/include/linux/objtool.h b/include/linux/objtool.h index 6491fa8fba6d..6f89471005ee 100644 --- a/include/linux/objtool.h +++ b/include/linux/objtool.h @@ -62,7 +62,7 @@ struct unwind_hint { * It should only be used in special cases where you're 100% sure it won't * affect the reliability of frame pointers and kernel stack traces. * - * For more information, see tools/objtool/Documentation/stack-validation.txt. + * For more information, see tools/objtool/Documentation/objtool.txt. */ #define STACK_FRAME_NON_STANDARD(func) \ static void __used __section(".discard.func_stack_frame_non_standard") \ diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 2e24db4bff19..79a71eb96111 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -498,7 +498,7 @@ config STACK_VALIDATION runtime stack traces are more reliable. For more information, see - tools/objtool/Documentation/stack-validation.txt. + tools/objtool/Documentation/objtool.txt. config NOINSTR_VALIDATION bool diff --git a/tools/include/linux/objtool.h b/tools/include/linux/objtool.h index 6491fa8fba6d..6f89471005ee 100644 --- a/tools/include/linux/objtool.h +++ b/tools/include/linux/objtool.h @@ -62,7 +62,7 @@ struct unwind_hint { * It should only be used in special cases where you're 100% sure it won't * affect the reliability of frame pointers and kernel stack traces. * - * For more information, see tools/objtool/Documentation/stack-validation.txt. + * For more information, see tools/objtool/Documentation/objtool.txt. */ #define STACK_FRAME_NON_STANDARD(func) \ static void __used __section(".discard.func_stack_frame_non_standard") \ diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 864bb9dd3584..970844ceecdc 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -3190,7 +3190,7 @@ static struct instruction *next_insn_to_validate(struct objtool_file *file, * Follow the branch starting at the given instruction, and recursively follow * any other branches (jumps). Meanwhile, track the frame pointer state at * each instruction and validate all the rules described in - * tools/objtool/Documentation/stack-validation.txt. + * tools/objtool/Documentation/objtool.txt. */ static int validate_branch(struct objtool_file *file, struct symbol *func, struct instruction *insn, struct insn_state state) From 78b02317310ee30e922328e332b937d2607727e1 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sun, 26 Jun 2022 10:11:02 +0100 Subject: [PATCH 183/318] arch: m68k: q40: README: drop references to IDE driver Since IDE support was deleted by Christoph Hellwig , Jun 16 2021, drop the left-over from README file, updating the documentation to point to ata/pata_falcon.c. Fixes: 44b1fbc0f5f3 ("m68k/q40: Replace q40ide driver with pata_falcon and falconide") Fixes: b7fb14d3ac63 ("ide: remove the legacy ide driver") Signed-off-by: Mauro Carvalho Chehab Reviewed-by: Geert Uytterhoeven Acked-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/d6e3482ed622b6953db69cddb70f20c55c96e4da.1656234456.git.mchehab@kernel.org Signed-off-by: Jonathan Corbet --- arch/m68k/q40/README | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/arch/m68k/q40/README b/arch/m68k/q40/README index a4991d2d8af6..9760e9081f6f 100644 --- a/arch/m68k/q40/README +++ b/arch/m68k/q40/README @@ -30,11 +30,10 @@ drivers used by the Q40, apart from the very obvious (console etc.): genrtc.c # RTC char/joystick/* # most of this should work, not # in default config.in - block/q40ide.c # startup for ide - ide* # see Documentation/ide/ide.rst - floppy.c # normal PC driver, DMA emu in asm/floppy.h + block/floppy.c # normal PC driver, DMA emu in asm/floppy.h # and arch/m68k/kernel/entry.S # see drivers/block/README.fd + ata/pata_falcon.c net/ne.c video/q40fb.c parport/* From b510592b76d80e83f06db1a810b7ea3348aa3823 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sun, 26 Jun 2022 10:11:03 +0100 Subject: [PATCH 184/318] tegra194-hte.rst: fix reference to its binding The binding directory for hte was renamed. Update references accordingly. Fixes: af583852d2ef ("dt-bindings: Renamed hte directory to timestamp") Signed-off-by: Mauro Carvalho Chehab Reviewed-by: Dipen Patel Link: https://lore.kernel.org/r/2e2c86485cb0642455cee01796f9a74de21403e6.1656234456.git.mchehab@kernel.org Signed-off-by: Jonathan Corbet --- Documentation/driver-api/hte/tegra194-hte.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/driver-api/hte/tegra194-hte.rst b/Documentation/driver-api/hte/tegra194-hte.rst index d16956256c85..f2d617265546 100644 --- a/Documentation/driver-api/hte/tegra194-hte.rst +++ b/Documentation/driver-api/hte/tegra194-hte.rst @@ -36,7 +36,7 @@ LIC (Legacy Interrupt Controller) IRQ GTE This GTE instance timestamps LIC IRQ lines in real time. There are 352 IRQ lines which this instance can add timestamps to in real time. The hte -devicetree binding described at ``Documentation/devicetree/bindings/hte/`` +devicetree binding described at ``Documentation/devicetree/bindings/timestamp`` provides an example of how a consumer can request an IRQ line. Since it is a one-to-one mapping with IRQ GTE provider, consumers can simply specify the IRQ number that they are interested in. There is no userspace consumer support for From 932e778b294cd7a94072a59be2047a32288f860c Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sun, 26 Jun 2022 10:11:04 +0100 Subject: [PATCH 185/318] dt-bindings: mfd: update dlg,da9063.yaml reference Changeset 441613662db7 ("dt-bindings: mfd: Convert da9063 to yaml") renamed: Documentation/devicetree/bindings/mfd/da9063.txt to: Documentation/devicetree/bindings/mfd/dlg,da9063.yaml. Update its cross-reference accordingly. Fixes: 441613662db7 ("dt-bindings: mfd: Convert da9063 to yaml") Signed-off-by: Mauro Carvalho Chehab Acked-by: Rob Herring Acked-by: Conor Dooley Link: https://lore.kernel.org/r/da459164d166d994b06f9ce8bf3a2108ccd0abd5.1656234456.git.mchehab@kernel.org Signed-off-by: Jonathan Corbet --- Documentation/devicetree/bindings/input/da9062-onkey.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/input/da9062-onkey.txt b/Documentation/devicetree/bindings/input/da9062-onkey.txt index 5f9fbc68e58a..e5eef59a93dc 100644 --- a/Documentation/devicetree/bindings/input/da9062-onkey.txt +++ b/Documentation/devicetree/bindings/input/da9062-onkey.txt @@ -2,7 +2,7 @@ This module is part of the DA9061/DA9062/DA9063. For more details about entire DA9062 and DA9061 chips see Documentation/devicetree/bindings/mfd/da9062.txt -For DA9063 see Documentation/devicetree/bindings/mfd/da9063.txt +For DA9063 see Documentation/devicetree/bindings/mfd/dlg,da9063.yaml This module provides the KEY_POWER event. From 75c654242f2947013cf50823f5a8f3bc814d9590 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sun, 26 Jun 2022 10:11:05 +0100 Subject: [PATCH 186/318] MAINTAINERS: update nvidia,tegra20-host1x.yaml reference Changeset fd27de58b0ad ("dt-bindings: display: tegra: Convert to json-schema") renamed: Documentation/devicetree/bindings/display/tegra/nvidia,tegra20-host1x.txt to: Documentation/devicetree/bindings/display/tegra/nvidia,tegra20-host1x.yaml. Update its cross-reference accordingly. Fixes: fd27de58b0ad ("dt-bindings: display: tegra: Convert to json-schema") Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/e0adee01fe8064f981eb854e04fb64c4a0408b66.1656234456.git.mchehab@kernel.org Signed-off-by: Jonathan Corbet --- MAINTAINERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 1fc9ead83d2a..3320dd8bd198 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -6714,7 +6714,7 @@ L: dri-devel@lists.freedesktop.org L: linux-tegra@vger.kernel.org S: Supported T: git git://anongit.freedesktop.org/tegra/linux.git -F: Documentation/devicetree/bindings/display/tegra/nvidia,tegra20-host1x.txt +F: Documentation/devicetree/bindings/display/tegra/nvidia,tegra20-host1x.yaml F: Documentation/devicetree/bindings/gpu/host1x/ F: drivers/gpu/drm/tegra/ F: drivers/gpu/host1x/ @@ -19647,7 +19647,7 @@ M: Sowjanya Komatineni L: linux-media@vger.kernel.org L: linux-tegra@vger.kernel.org S: Maintained -F: Documentation/devicetree/bindings/display/tegra/nvidia,tegra20-host1x.txt +F: Documentation/devicetree/bindings/display/tegra/nvidia,tegra20-host1x.yaml F: drivers/staging/media/tegra-video/ TEGRA XUSB PADCTL DRIVER From 5ee52118ac1481dd8a8f7e6a9bfe6ee05ac6ec92 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 27 Jun 2022 09:36:49 +0100 Subject: [PATCH 187/318] crypto: qat - expose device state through sysfs for 4xxx Expose the device state through an attribute in sysfs and allow to change it. This is to stop and shutdown a QAT device in order to change its configuration. The state attribute has been added to a newly created `qat` attribute group which will contain all _QAT specific_ attributes. The logic that implements the sysfs entries is part of a new file, adf_sysfs.c. This exposes an entry point to allow the driver to create attributes. The function that creates the sysfs attributes is called from the probe function of the driver and not in the state machine init function to allow the change of states even if the device is in the down state. In order to restore the device configuration between a transition from down to up, the function that configures the devices has been abstracted into the HW data structure. The `state` attribute is only exposed for qat_4xxx devices. Signed-off-by: Giovanni Cabiddu Co-developed-by: Tomasz Kowallik Signed-off-by: Tomasz Kowallik Reviewed-by: Adam Guerin Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Vladis Dronov Signed-off-by: Herbert Xu --- Documentation/ABI/testing/sysfs-driver-qat | 21 ++++ .../crypto/qat/qat_4xxx/adf_4xxx_hw_data.c | 1 + .../crypto/qat/qat_4xxx/adf_4xxx_hw_data.h | 1 + drivers/crypto/qat/qat_4xxx/adf_drv.c | 6 +- drivers/crypto/qat/qat_common/Makefile | 1 + .../crypto/qat/qat_common/adf_accel_devices.h | 1 + .../crypto/qat/qat_common/adf_common_drv.h | 2 + drivers/crypto/qat/qat_common/adf_sysfs.c | 119 ++++++++++++++++++ 8 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 Documentation/ABI/testing/sysfs-driver-qat create mode 100644 drivers/crypto/qat/qat_common/adf_sysfs.c diff --git a/Documentation/ABI/testing/sysfs-driver-qat b/Documentation/ABI/testing/sysfs-driver-qat new file mode 100644 index 000000000000..769b09cefa89 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-driver-qat @@ -0,0 +1,21 @@ +What: /sys/bus/pci/devices//qat/state +Date: June 2022 +KernelVersion: 5.20 +Contact: qat-linux@intel.com +Description: Reports the current state of the QAT device and allows to + change it. + + This attribute is RW. + + Returned values: + up: the device is up and running + down: the device is down + + Allowed values: + up: initialize and start the device + down: stop the device and bring it down + + It is possible to transition the device from up to down only + if the device is up and vice versa. + + This attribute is only available for qat_4xxx devices. diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c index fb5970a68484..fda5f699ff57 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -357,6 +357,7 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data) hw_data->ring_pair_reset = adf_gen4_ring_pair_reset; hw_data->enable_pm = adf_gen4_enable_pm; hw_data->handle_pm_interrupt = adf_gen4_handle_pm_interrupt; + hw_data->dev_config = adf_crypto_dev_config; adf_gen4_init_hw_csr_ops(&hw_data->csr_ops); adf_gen4_init_pf_pfvf_ops(&hw_data->pfvf_ops); diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h index 1034752845ca..9d49248931f6 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h @@ -70,5 +70,6 @@ enum icp_qat_4xxx_slice_mask { void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data); void adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data); +int adf_crypto_dev_config(struct adf_accel_dev *accel_dev); #endif diff --git a/drivers/crypto/qat/qat_4xxx/adf_drv.c b/drivers/crypto/qat/qat_4xxx/adf_drv.c index 181fa1c8b3c7..2f212561acc4 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_drv.c +++ b/drivers/crypto/qat/qat_4xxx/adf_drv.c @@ -53,7 +53,7 @@ static int adf_cfg_dev_init(struct adf_accel_dev *accel_dev) return 0; } -static int adf_crypto_dev_config(struct adf_accel_dev *accel_dev) +int adf_crypto_dev_config(struct adf_accel_dev *accel_dev) { char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; int banks = GET_MAX_BANKS(accel_dev); @@ -289,6 +289,10 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_err_disable_aer; } + ret = adf_sysfs_init(accel_dev); + if (ret) + goto out_err_disable_aer; + ret = adf_crypto_dev_config(accel_dev); if (ret) goto out_err_disable_aer; diff --git a/drivers/crypto/qat/qat_common/Makefile b/drivers/crypto/qat/qat_common/Makefile index 04f058acc4d3..80919cfcc29d 100644 --- a/drivers/crypto/qat/qat_common/Makefile +++ b/drivers/crypto/qat/qat_common/Makefile @@ -10,6 +10,7 @@ intel_qat-objs := adf_cfg.o \ adf_transport.o \ adf_admin.o \ adf_hw_arbiter.o \ + adf_sysfs.o \ adf_gen2_hw_data.o \ adf_gen4_hw_data.o \ adf_gen4_pm.o \ diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index ede6458c9dbf..0a55a4f34dcf 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -199,6 +199,7 @@ struct adf_hw_device_data { char *(*uof_get_name)(struct adf_accel_dev *accel_dev, u32 obj_num); u32 (*uof_get_num_objs)(void); u32 (*uof_get_ae_mask)(struct adf_accel_dev *accel_dev, u32 obj_num); + int (*dev_config)(struct adf_accel_dev *accel_dev); struct adf_pfvf_ops pfvf_ops; struct adf_hw_csr_ops csr_ops; const char *fw_name; diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index b364bc06c732..b841e85c8f5d 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -127,6 +127,8 @@ void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev); int adf_pfvf_comms_disabled(struct adf_accel_dev *accel_dev); +int adf_sysfs_init(struct adf_accel_dev *accel_dev); + int qat_hal_init(struct adf_accel_dev *accel_dev); void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle); int qat_hal_start(struct icp_qat_fw_loader_handle *handle); diff --git a/drivers/crypto/qat/qat_common/adf_sysfs.c b/drivers/crypto/qat/qat_common/adf_sysfs.c new file mode 100644 index 000000000000..8f47a5694dd7 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_sysfs.c @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2022 Intel Corporation */ +#include +#include +#include +#include "adf_accel_devices.h" +#include "adf_cfg.h" +#include "adf_common_drv.h" + +static const char * const state_operations[] = { + [DEV_DOWN] = "down", + [DEV_UP] = "up", +}; + +static ssize_t state_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct adf_accel_dev *accel_dev; + char *state; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + state = adf_dev_started(accel_dev) ? "up" : "down"; + return sysfs_emit(buf, "%s\n", state); +} + +static ssize_t state_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct adf_accel_dev *accel_dev; + u32 accel_id; + int ret; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + accel_id = accel_dev->accel_id; + + if (adf_devmgr_in_reset(accel_dev) || adf_dev_in_use(accel_dev)) { + dev_info(dev, "Device qat_dev%d is busy\n", accel_id); + return -EBUSY; + } + + ret = sysfs_match_string(state_operations, buf); + if (ret < 0) + return ret; + + switch (ret) { + case DEV_DOWN: + if (!adf_dev_started(accel_dev)) { + dev_info(dev, "Device qat_dev%d already down\n", + accel_id); + return -EINVAL; + } + + dev_info(dev, "Stopping device qat_dev%d\n", accel_id); + + adf_dev_stop(accel_dev); + adf_dev_shutdown(accel_dev); + + break; + case DEV_UP: + if (adf_dev_started(accel_dev)) { + dev_info(dev, "Device qat_dev%d already up\n", + accel_id); + return -EINVAL; + } + + dev_info(dev, "Starting device qat_dev%d\n", accel_id); + + ret = GET_HW_DATA(accel_dev)->dev_config(accel_dev); + if (!ret) + ret = adf_dev_init(accel_dev); + if (!ret) + ret = adf_dev_start(accel_dev); + + if (ret < 0) { + dev_err(dev, "Failed to start device qat_dev%d\n", + accel_id); + adf_dev_stop(accel_dev); + adf_dev_shutdown(accel_dev); + return ret; + } + break; + default: + return -EINVAL; + } + + return count; +} + +static DEVICE_ATTR_RW(state); + +static struct attribute *qat_attrs[] = { + &dev_attr_state.attr, + NULL, +}; + +static struct attribute_group qat_group = { + .attrs = qat_attrs, + .name = "qat", +}; + +int adf_sysfs_init(struct adf_accel_dev *accel_dev) +{ + int ret; + + ret = devm_device_add_group(&GET_DEV(accel_dev), &qat_group); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to create qat attribute group: %d\n", ret); + } + + return ret; +} +EXPORT_SYMBOL_GPL(adf_sysfs_init); From 92bf269fbfe94018f15405e1644049de7c7b46dd Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 27 Jun 2022 09:36:50 +0100 Subject: [PATCH 188/318] crypto: qat - change behaviour of adf_cfg_add_key_value_param() The function adf_cfg_add_key_value_param() allows to insert duplicates entries in the key value store of the driver. Change the behaviour of that function to the following policy: - if the key doesn't exist, add it; - if the key already exists with a different value, then delete it and replace it with a new one containing the new value; - if the key exists with the same value, then return without doing anything. The behaviour of this function has been changed in order to easily update key-values in the driver database. In particular this is required to update the value of the ServiceEnables key used to change the service loaded on a device. Signed-off-by: Giovanni Cabiddu Reviewed-by: Adam Guerin Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Vladis Dronov Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_cfg.c | 41 ++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/qat/qat_common/adf_cfg.c b/drivers/crypto/qat/qat_common/adf_cfg.c index b5b208cbe5a1..e61b3e13db3b 100644 --- a/drivers/crypto/qat/qat_common/adf_cfg.c +++ b/drivers/crypto/qat/qat_common/adf_cfg.c @@ -128,6 +128,24 @@ static void adf_cfg_keyval_add(struct adf_cfg_key_val *new, list_add_tail(&new->list, &sec->param_head); } +static void adf_cfg_keyval_remove(const char *key, struct adf_cfg_section *sec) +{ + struct list_head *head = &sec->param_head; + struct list_head *list_ptr, *tmp; + + list_for_each_prev_safe(list_ptr, tmp, head) { + struct adf_cfg_key_val *ptr = + list_entry(list_ptr, struct adf_cfg_key_val, list); + + if (strncmp(ptr->key, key, sizeof(ptr->key))) + continue; + + list_del(list_ptr); + kfree(ptr); + break; + } +} + static void adf_cfg_keyval_del_all(struct list_head *head) { struct list_head *list_ptr, *tmp; @@ -208,7 +226,8 @@ static int adf_cfg_key_val_get(struct adf_accel_dev *accel_dev, * @type: Type - string, int or address * * Function adds configuration key - value entry in the appropriate section - * in the given acceleration device + * in the given acceleration device. If the key exists already, the value + * is updated. * To be used by QAT device specific drivers. * * Return: 0 on success, error code otherwise. @@ -222,6 +241,8 @@ int adf_cfg_add_key_value_param(struct adf_accel_dev *accel_dev, struct adf_cfg_key_val *key_val; struct adf_cfg_section *section = adf_cfg_sec_find(accel_dev, section_name); + char temp_val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; + if (!section) return -EFAULT; @@ -246,6 +267,24 @@ int adf_cfg_add_key_value_param(struct adf_accel_dev *accel_dev, return -EINVAL; } key_val->type = type; + + /* Add the key-value pair as below policy: + * 1. if the key doesn't exist, add it; + * 2. if the key already exists with a different value then update it + * to the new value (the key is deleted and the newly created + * key_val containing the new value is added to the database); + * 3. if the key exists with the same value, then return without doing + * anything (the newly created key_val is freed). + */ + if (!adf_cfg_key_val_get(accel_dev, section_name, key, temp_val)) { + if (strncmp(temp_val, key_val->val, sizeof(temp_val))) { + adf_cfg_keyval_remove(key, section); + } else { + kfree(key_val); + return 0; + } + } + down_write(&cfg->lock); adf_cfg_keyval_add(key_val, section); up_write(&cfg->lock); From 16c1ed95d1c4956e428c8daa2783bcc7fa7f6fb9 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 27 Jun 2022 09:36:51 +0100 Subject: [PATCH 189/318] crypto: qat - relocate and rename adf_sriov_prepare_restart() The function adf_sriov_prepare_restart() is used in adf_sriov.c to stop and shutdown a device preserving its configuration. Since this function will be re-used by the logic that allows to reconfigure the device through sysfs, move it to adf_init.c and rename it as adf_dev_shutdown_cache_cfg(); Signed-off-by: Giovanni Cabiddu Reviewed-by: Adam Guerin Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Vladis Dronov Signed-off-by: Herbert Xu --- .../crypto/qat/qat_common/adf_common_drv.h | 1 + drivers/crypto/qat/qat_common/adf_init.c | 26 +++++++++++++++++ drivers/crypto/qat/qat_common/adf_sriov.c | 28 +------------------ 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index b841e85c8f5d..7bb477c3ce25 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -56,6 +56,7 @@ int adf_dev_init(struct adf_accel_dev *accel_dev); int adf_dev_start(struct adf_accel_dev *accel_dev); void adf_dev_stop(struct adf_accel_dev *accel_dev); void adf_dev_shutdown(struct adf_accel_dev *accel_dev); +int adf_dev_shutdown_cache_cfg(struct adf_accel_dev *accel_dev); void adf_devmgr_update_class_index(struct adf_hw_device_data *hw_data); void adf_clean_vf_map(bool); diff --git a/drivers/crypto/qat/qat_common/adf_init.c b/drivers/crypto/qat/qat_common/adf_init.c index c2c718f1b489..33a9a46d6949 100644 --- a/drivers/crypto/qat/qat_common/adf_init.c +++ b/drivers/crypto/qat/qat_common/adf_init.c @@ -363,3 +363,29 @@ int adf_dev_restarted_notify(struct adf_accel_dev *accel_dev) } return 0; } + +int adf_dev_shutdown_cache_cfg(struct adf_accel_dev *accel_dev) +{ + char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0}; + int ret; + + ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, + ADF_SERVICES_ENABLED, services); + + adf_dev_stop(accel_dev); + adf_dev_shutdown(accel_dev); + + if (!ret) { + ret = adf_cfg_section_add(accel_dev, ADF_GENERAL_SEC); + if (ret) + return ret; + + ret = adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC, + ADF_SERVICES_ENABLED, + services, ADF_STR); + if (ret) + return ret; + } + + return 0; +} diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c index f38b2ffde146..b2db1d70d71f 100644 --- a/drivers/crypto/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/qat/qat_common/adf_sriov.c @@ -120,32 +120,6 @@ void adf_disable_sriov(struct adf_accel_dev *accel_dev) } EXPORT_SYMBOL_GPL(adf_disable_sriov); -static int adf_sriov_prepare_restart(struct adf_accel_dev *accel_dev) -{ - char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0}; - int ret; - - ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, - ADF_SERVICES_ENABLED, services); - - adf_dev_stop(accel_dev); - adf_dev_shutdown(accel_dev); - - if (!ret) { - ret = adf_cfg_section_add(accel_dev, ADF_GENERAL_SEC); - if (ret) - return ret; - - ret = adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC, - ADF_SERVICES_ENABLED, - services, ADF_STR); - if (ret) - return ret; - } - - return 0; -} - /** * adf_sriov_configure() - Enable SRIOV for the device * @pdev: Pointer to PCI device. @@ -185,7 +159,7 @@ int adf_sriov_configure(struct pci_dev *pdev, int numvfs) return -EBUSY; } - ret = adf_sriov_prepare_restart(accel_dev); + ret = adf_dev_shutdown_cache_cfg(accel_dev); if (ret) return ret; } From d4cfb144f60551d80732c93c892fe76fc8df860d Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 27 Jun 2022 09:36:52 +0100 Subject: [PATCH 190/318] crypto: qat - expose device config through sysfs for 4xxx qat_4xxx devices can be configured to allow either crypto or compression operations. At the moment, devices are configured statically according to the following rule: - odd numbered devices assigned to compression services - even numbered devices assigned to crypto services Expose the sysfs attribute /sys/bus/pci/devices//qat/cfg_services to allow to detect the configuration of a device and to change it. The `cfg_service` attribute is only exposed for qat_4xxx devices and it is limited to two configurations: (1) "sym;asym" for crypto services and "dc" for compression services. Signed-off-by: Giovanni Cabiddu Co-developed-by: Tomasz Kowallik Signed-off-by: Tomasz Kowallik Reviewed-by: Adam Guerin Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Vladis Dronov Signed-off-by: Herbert Xu --- Documentation/ABI/testing/sysfs-driver-qat | 39 +++++++++++ drivers/crypto/qat/qat_common/adf_sysfs.c | 80 ++++++++++++++++++++-- 2 files changed, 115 insertions(+), 4 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-driver-qat b/Documentation/ABI/testing/sysfs-driver-qat index 769b09cefa89..a600531e9562 100644 --- a/Documentation/ABI/testing/sysfs-driver-qat +++ b/Documentation/ABI/testing/sysfs-driver-qat @@ -19,3 +19,42 @@ Description: Reports the current state of the QAT device and allows to if the device is up and vice versa. This attribute is only available for qat_4xxx devices. + +What: /sys/bus/pci/devices//qat/cfg_services +Date: June 2022 +KernelVersion: 5.20 +Contact: qat-linux@intel.com +Description: Reports the current configuration of the QAT device and allows + to change it. + + This attribute is RW. + + Returned values: + sym;asym: the device is configured for running + crypto services + dc: the device is configured for running + compression services + + Allowed values: + sym;asym: configure the device for running + crypto services + dc: configure the device for running + compression services + + It is possible to set the configuration only if the device + is in the `down` state (see /sys/bus/pci/devices//qat/state) + + The following example shows how to change the configuration of + a device configured for running crypto services in order to + run data compression: + # cat /sys/bus/pci/devices//qat/state + up + # cat /sys/bus/pci/devices//qat/cfg_services + sym;asym + # echo down > /sys/bus/pci/devices//qat/state + # echo dc > /sys/bus/pci/devices//qat/cfg_services + # echo up > /sys/bus/pci/devices//qat/state + # cat /sys/bus/pci/devices//qat/cfg_services + dc + + This attribute is only available for qat_4xxx devices. diff --git a/drivers/crypto/qat/qat_common/adf_sysfs.c b/drivers/crypto/qat/qat_common/adf_sysfs.c index 8f47a5694dd7..e8b078e719c2 100644 --- a/drivers/crypto/qat/qat_common/adf_sysfs.c +++ b/drivers/crypto/qat/qat_common/adf_sysfs.c @@ -58,8 +58,9 @@ static ssize_t state_store(struct device *dev, struct device_attribute *attr, dev_info(dev, "Stopping device qat_dev%d\n", accel_id); - adf_dev_stop(accel_dev); - adf_dev_shutdown(accel_dev); + ret = adf_dev_shutdown_cache_cfg(accel_dev); + if (ret < 0) + return -EINVAL; break; case DEV_UP: @@ -80,8 +81,7 @@ static ssize_t state_store(struct device *dev, struct device_attribute *attr, if (ret < 0) { dev_err(dev, "Failed to start device qat_dev%d\n", accel_id); - adf_dev_stop(accel_dev); - adf_dev_shutdown(accel_dev); + adf_dev_shutdown_cache_cfg(accel_dev); return ret; } break; @@ -92,10 +92,82 @@ static ssize_t state_store(struct device *dev, struct device_attribute *attr, return count; } +static const char * const services_operations[] = { + ADF_CFG_CY, + ADF_CFG_DC, +}; + +static ssize_t cfg_services_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0}; + struct adf_accel_dev *accel_dev; + int ret; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, + ADF_SERVICES_ENABLED, services); + if (ret) + return ret; + + return sysfs_emit(buf, "%s\n", services); +} + +static int adf_sysfs_update_dev_config(struct adf_accel_dev *accel_dev, + const char *services) +{ + return adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC, + ADF_SERVICES_ENABLED, services, + ADF_STR); +} + +static ssize_t cfg_services_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct adf_hw_device_data *hw_data; + struct adf_accel_dev *accel_dev; + int ret; + + ret = sysfs_match_string(services_operations, buf); + if (ret < 0) + return ret; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + if (adf_dev_started(accel_dev)) { + dev_info(dev, "Device qat_dev%d must be down to reconfigure the service.\n", + accel_dev->accel_id); + return -EINVAL; + } + + ret = adf_sysfs_update_dev_config(accel_dev, services_operations[ret]); + if (ret < 0) + return ret; + + hw_data = GET_HW_DATA(accel_dev); + + /* Update capabilities mask after change in configuration. + * A call to this function is required as capabilities are, at the + * moment, tied to configuration + */ + hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); + if (!hw_data->accel_capabilities_mask) + return -EINVAL; + + return count; +} + static DEVICE_ATTR_RW(state); +static DEVICE_ATTR_RW(cfg_services); static struct attribute *qat_attrs[] = { &dev_attr_state.attr, + &dev_attr_cfg_services.attr, NULL, }; From cc8166bfc829043020b5cc3b7cdba02a17d03b6d Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 30 Jun 2022 16:11:49 +0800 Subject: [PATCH 191/318] crypto: vmx - Fix warning on p8_ghash_alg The compiler complains that p8_ghash_alg isn't declared which is because the header file aesp8-ppc.h isn't included in ghash.c. This patch fixes the warning. Signed-off-by: Herbert Xu Acked-by: Breno Leitao Signed-off-by: Herbert Xu --- drivers/crypto/vmx/ghash.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/crypto/vmx/ghash.c b/drivers/crypto/vmx/ghash.c index 5bc5710a6de0..77eca20bc7ac 100644 --- a/drivers/crypto/vmx/ghash.c +++ b/drivers/crypto/vmx/ghash.c @@ -23,6 +23,7 @@ #include #include #include +#include "aesp8-ppc.h" void gcm_init_p8(u128 htable[16], const u64 Xi[2]); void gcm_gmult_p8(u64 Xi[2], const u128 htable[16]); From 9984a6447389a01cc5e8501b2ead3024b2d99c19 Mon Sep 17 00:00:00 2001 From: Liang He Date: Thu, 30 Jun 2022 16:36:56 +0800 Subject: [PATCH 192/318] crypto: amcc - Hold the reference returned by of_find_compatible_node In crypto4xx_probe(), we should hold the reference returned by of_find_compatible_node() and use it to call of_node_put to keep refcount balance. Signed-off-by: Liang He Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_core.c | 40 +++++++++++++++++----------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 8278d98074e9..280f4b0e7133 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -1378,6 +1378,7 @@ static int crypto4xx_probe(struct platform_device *ofdev) struct resource res; struct device *dev = &ofdev->dev; struct crypto4xx_core_device *core_dev; + struct device_node *np; u32 pvr; bool is_revb = true; @@ -1385,29 +1386,36 @@ static int crypto4xx_probe(struct platform_device *ofdev) if (rc) return -ENODEV; - if (of_find_compatible_node(NULL, NULL, "amcc,ppc460ex-crypto")) { + np = of_find_compatible_node(NULL, NULL, "amcc,ppc460ex-crypto"); + if (np) { mtdcri(SDR0, PPC460EX_SDR0_SRST, mfdcri(SDR0, PPC460EX_SDR0_SRST) | PPC460EX_CE_RESET); mtdcri(SDR0, PPC460EX_SDR0_SRST, mfdcri(SDR0, PPC460EX_SDR0_SRST) & ~PPC460EX_CE_RESET); - } else if (of_find_compatible_node(NULL, NULL, - "amcc,ppc405ex-crypto")) { - mtdcri(SDR0, PPC405EX_SDR0_SRST, - mfdcri(SDR0, PPC405EX_SDR0_SRST) | PPC405EX_CE_RESET); - mtdcri(SDR0, PPC405EX_SDR0_SRST, - mfdcri(SDR0, PPC405EX_SDR0_SRST) & ~PPC405EX_CE_RESET); - is_revb = false; - } else if (of_find_compatible_node(NULL, NULL, - "amcc,ppc460sx-crypto")) { - mtdcri(SDR0, PPC460SX_SDR0_SRST, - mfdcri(SDR0, PPC460SX_SDR0_SRST) | PPC460SX_CE_RESET); - mtdcri(SDR0, PPC460SX_SDR0_SRST, - mfdcri(SDR0, PPC460SX_SDR0_SRST) & ~PPC460SX_CE_RESET); } else { - printk(KERN_ERR "Crypto Function Not supported!\n"); - return -EINVAL; + np = of_find_compatible_node(NULL, NULL, "amcc,ppc405ex-crypto"); + if (np) { + mtdcri(SDR0, PPC405EX_SDR0_SRST, + mfdcri(SDR0, PPC405EX_SDR0_SRST) | PPC405EX_CE_RESET); + mtdcri(SDR0, PPC405EX_SDR0_SRST, + mfdcri(SDR0, PPC405EX_SDR0_SRST) & ~PPC405EX_CE_RESET); + is_revb = false; + } else { + np = of_find_compatible_node(NULL, NULL, "amcc,ppc460sx-crypto"); + if (np) { + mtdcri(SDR0, PPC460SX_SDR0_SRST, + mfdcri(SDR0, PPC460SX_SDR0_SRST) | PPC460SX_CE_RESET); + mtdcri(SDR0, PPC460SX_SDR0_SRST, + mfdcri(SDR0, PPC460SX_SDR0_SRST) & ~PPC460SX_CE_RESET); + } else { + printk(KERN_ERR "Crypto Function Not supported!\n"); + return -EINVAL; + } + } } + of_node_put(np); + core_dev = kzalloc(sizeof(struct crypto4xx_core_device), GFP_KERNEL); if (!core_dev) return -ENOMEM; From 8ccc9cc47c8c9e69c7bd9a7694a1d8c33108dd28 Mon Sep 17 00:00:00 2001 From: Liang He Date: Thu, 30 Jun 2022 16:36:57 +0800 Subject: [PATCH 193/318] crypto: nx - Hold the reference returned by of_find_compatible_node In nx842_pseries_init(), we should hold the reference returned by of_find_compatible_node() and use it to call of_node_put to keep refcount balance. Signed-off-by: Liang He Signed-off-by: Herbert Xu --- drivers/crypto/nx/nx-common-pseries.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/nx/nx-common-pseries.c b/drivers/crypto/nx/nx-common-pseries.c index 7584a34ba88c..3ea334b7f820 100644 --- a/drivers/crypto/nx/nx-common-pseries.c +++ b/drivers/crypto/nx/nx-common-pseries.c @@ -1208,10 +1208,13 @@ static struct vio_driver nx842_vio_driver = { static int __init nx842_pseries_init(void) { struct nx842_devdata *new_devdata; + struct device_node *np; int ret; - if (!of_find_compatible_node(NULL, NULL, "ibm,compression")) + np = of_find_compatible_node(NULL, NULL, "ibm,compression"); + if (!np) return -ENODEV; + of_node_put(np); RCU_INIT_POINTER(devdata, NULL); new_devdata = kzalloc(sizeof(*new_devdata), GFP_KERNEL); From 32c0f7d4194c4352f902f07d2ab990994800596e Mon Sep 17 00:00:00 2001 From: Jianglei Nie Date: Thu, 30 Jun 2022 22:31:32 +0800 Subject: [PATCH 194/318] crypto: hisilicon/sec - fix inconsistent argument The argument passed to sec_queue_aw_alloc() should be SEC_QUEUE_AW_FROCE_NOALLOC instead of SEC_QUEUE_AR_FROCE_NOALLOC. Signed-off-by: Jianglei Nie Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sec/sec_drv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/hisilicon/sec/sec_drv.c b/drivers/crypto/hisilicon/sec/sec_drv.c index c8de1b51c843..e75851326c1e 100644 --- a/drivers/crypto/hisilicon/sec/sec_drv.c +++ b/drivers/crypto/hisilicon/sec/sec_drv.c @@ -892,7 +892,7 @@ bool sec_queue_can_enqueue(struct sec_queue *queue, int num) static void sec_queue_hw_init(struct sec_queue *queue) { sec_queue_ar_alloc(queue, SEC_QUEUE_AR_FROCE_NOALLOC); - sec_queue_aw_alloc(queue, SEC_QUEUE_AR_FROCE_NOALLOC); + sec_queue_aw_alloc(queue, SEC_QUEUE_AW_FROCE_NOALLOC); sec_queue_ar_pkgattr(queue, 1); sec_queue_aw_pkgattr(queue, 1); From 02884a4f12de11f54d4ca67a07dd1f111d96fdbd Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Fri, 1 Jul 2022 09:55:11 +0800 Subject: [PATCH 195/318] crypto: hisilicon/sec - don't sleep when in softirq When kunpeng920 encryption driver is used to deencrypt and decrypt packets during the softirq, it is not allowed to use mutex lock. The kernel will report the following error: BUG: scheduling while atomic: swapper/57/0/0x00000300 Call trace: dump_backtrace+0x0/0x1e4 show_stack+0x20/0x2c dump_stack+0xd8/0x140 __schedule_bug+0x68/0x80 __schedule+0x728/0x840 schedule+0x50/0xe0 schedule_preempt_disabled+0x18/0x24 __mutex_lock.constprop.0+0x594/0x5dc __mutex_lock_slowpath+0x1c/0x30 mutex_lock+0x50/0x60 sec_request_init+0x8c/0x1a0 [hisi_sec2] sec_process+0x28/0x1ac [hisi_sec2] sec_skcipher_crypto+0xf4/0x1d4 [hisi_sec2] sec_skcipher_encrypt+0x1c/0x30 [hisi_sec2] crypto_skcipher_encrypt+0x2c/0x40 crypto_authenc_encrypt+0xc8/0xfc [authenc] crypto_aead_encrypt+0x2c/0x40 echainiv_encrypt+0x144/0x1a0 [echainiv] crypto_aead_encrypt+0x2c/0x40 esp_output_tail+0x348/0x5c0 [esp4] esp_output+0x120/0x19c [esp4] xfrm_output_one+0x25c/0x4d4 xfrm_output_resume+0x6c/0x1fc xfrm_output+0xac/0x3c0 xfrm4_output+0x64/0x130 ip_build_and_send_pkt+0x158/0x20c tcp_v4_send_synack+0xdc/0x1f0 tcp_conn_request+0x7d0/0x994 tcp_v4_conn_request+0x58/0x6c tcp_v6_conn_request+0xf0/0x100 tcp_rcv_state_process+0x1cc/0xd60 tcp_v4_do_rcv+0x10c/0x250 tcp_v4_rcv+0xfc4/0x10a4 ip_protocol_deliver_rcu+0xf4/0x200 ip_local_deliver_finish+0x58/0x70 ip_local_deliver+0x68/0x120 ip_sublist_rcv_finish+0x70/0x94 ip_list_rcv_finish.constprop.0+0x17c/0x1d0 ip_sublist_rcv+0x40/0xb0 ip_list_rcv+0x140/0x1dc __netif_receive_skb_list_core+0x154/0x28c __netif_receive_skb_list+0x120/0x1a0 netif_receive_skb_list_internal+0xe4/0x1f0 napi_complete_done+0x70/0x1f0 gro_cell_poll+0x9c/0xb0 napi_poll+0xcc/0x264 net_rx_action+0xd4/0x21c __do_softirq+0x130/0x358 irq_exit+0x11c/0x13c __handle_domain_irq+0x88/0xf0 gic_handle_irq+0x78/0x2c0 el1_irq+0xb8/0x140 arch_cpu_idle+0x18/0x40 default_idle_call+0x5c/0x1c0 cpuidle_idle_call+0x174/0x1b0 do_idle+0xc8/0x160 cpu_startup_entry+0x30/0x11c secondary_start_kernel+0x158/0x1e4 softirq: huh, entered softirq 3 NET_RX 0000000093774ee4 with preempt_count 00000100, exited with fffffe00? Fixes: 416d82204df4 ("crypto: hisilicon - add HiSilicon SEC V2 driver") Signed-off-by: Zhengchao Shao Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sec2/sec.h | 2 +- drivers/crypto/hisilicon/sec2/sec_crypto.c | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index 42bb486f3b6d..d2a0bc93e752 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -119,7 +119,7 @@ struct sec_qp_ctx { struct idr req_idr; struct sec_alg_res res[QM_Q_DEPTH]; struct sec_ctx *ctx; - struct mutex req_lock; + spinlock_t req_lock; struct list_head backlog; struct hisi_acc_sgl_pool *c_in_pool; struct hisi_acc_sgl_pool *c_out_pool; diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 6eebe739893c..71dfa7db6394 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -127,11 +127,11 @@ static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx) { int req_id; - mutex_lock(&qp_ctx->req_lock); + spin_lock_bh(&qp_ctx->req_lock); req_id = idr_alloc_cyclic(&qp_ctx->req_idr, NULL, 0, QM_Q_DEPTH, GFP_ATOMIC); - mutex_unlock(&qp_ctx->req_lock); + spin_unlock_bh(&qp_ctx->req_lock); if (unlikely(req_id < 0)) { dev_err(req->ctx->dev, "alloc req id fail!\n"); return req_id; @@ -156,9 +156,9 @@ static void sec_free_req_id(struct sec_req *req) qp_ctx->req_list[req_id] = NULL; req->qp_ctx = NULL; - mutex_lock(&qp_ctx->req_lock); + spin_lock_bh(&qp_ctx->req_lock); idr_remove(&qp_ctx->req_idr, req_id); - mutex_unlock(&qp_ctx->req_lock); + spin_unlock_bh(&qp_ctx->req_lock); } static u8 pre_parse_finished_bd(struct bd_status *status, void *resp) @@ -273,7 +273,7 @@ static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req) !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; - mutex_lock(&qp_ctx->req_lock); + spin_lock_bh(&qp_ctx->req_lock); ret = hisi_qp_send(qp_ctx->qp, &req->sec_sqe); if (ctx->fake_req_limit <= @@ -281,10 +281,10 @@ static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req) list_add_tail(&req->backlog_head, &qp_ctx->backlog); atomic64_inc(&ctx->sec->debug.dfx.send_cnt); atomic64_inc(&ctx->sec->debug.dfx.send_busy_cnt); - mutex_unlock(&qp_ctx->req_lock); + spin_unlock_bh(&qp_ctx->req_lock); return -EBUSY; } - mutex_unlock(&qp_ctx->req_lock); + spin_unlock_bh(&qp_ctx->req_lock); if (unlikely(ret == -EBUSY)) return -ENOBUFS; @@ -487,7 +487,7 @@ static int sec_create_qp_ctx(struct hisi_qm *qm, struct sec_ctx *ctx, qp->req_cb = sec_req_cb; - mutex_init(&qp_ctx->req_lock); + spin_lock_init(&qp_ctx->req_lock); idr_init(&qp_ctx->req_idr); INIT_LIST_HEAD(&qp_ctx->backlog); @@ -1382,7 +1382,7 @@ static struct sec_req *sec_back_req_clear(struct sec_ctx *ctx, { struct sec_req *backlog_req = NULL; - mutex_lock(&qp_ctx->req_lock); + spin_lock_bh(&qp_ctx->req_lock); if (ctx->fake_req_limit >= atomic_read(&qp_ctx->qp->qp_status.used) && !list_empty(&qp_ctx->backlog)) { @@ -1390,7 +1390,7 @@ static struct sec_req *sec_back_req_clear(struct sec_ctx *ctx, typeof(*backlog_req), backlog_head); list_del(&backlog_req->backlog_head); } - mutex_unlock(&qp_ctx->req_lock); + spin_unlock_bh(&qp_ctx->req_lock); return backlog_req; } From 68740ab505431f268dc1ee26a54b871e75f0ddaa Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Fri, 1 Jul 2022 09:59:54 +0800 Subject: [PATCH 196/318] crypto: hisilicon - Kunpeng916 crypto driver don't sleep when in softirq When kunpeng916 encryption driver is used to deencrypt and decrypt packets during the softirq, it is not allowed to use mutex lock. Fixes: 915e4e8413da ("crypto: hisilicon - SEC security accelerator driver") Signed-off-by: Zhengchao Shao Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sec/sec_algs.c | 14 +++++++------- drivers/crypto/hisilicon/sec/sec_drv.h | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/crypto/hisilicon/sec/sec_algs.c b/drivers/crypto/hisilicon/sec/sec_algs.c index 0a3c8f019b02..490e1542305e 100644 --- a/drivers/crypto/hisilicon/sec/sec_algs.c +++ b/drivers/crypto/hisilicon/sec/sec_algs.c @@ -449,7 +449,7 @@ static void sec_skcipher_alg_callback(struct sec_bd_info *sec_resp, */ } - mutex_lock(&ctx->queue->queuelock); + spin_lock_bh(&ctx->queue->queuelock); /* Put the IV in place for chained cases */ switch (ctx->cipher_alg) { case SEC_C_AES_CBC_128: @@ -509,7 +509,7 @@ static void sec_skcipher_alg_callback(struct sec_bd_info *sec_resp, list_del(&backlog_req->backlog_head); } } - mutex_unlock(&ctx->queue->queuelock); + spin_unlock_bh(&ctx->queue->queuelock); mutex_lock(&sec_req->lock); list_del(&sec_req_el->head); @@ -798,7 +798,7 @@ static int sec_alg_skcipher_crypto(struct skcipher_request *skreq, */ /* Grab a big lock for a long time to avoid concurrency issues */ - mutex_lock(&queue->queuelock); + spin_lock_bh(&queue->queuelock); /* * Can go on to queue if we have space in either: @@ -814,15 +814,15 @@ static int sec_alg_skcipher_crypto(struct skcipher_request *skreq, ret = -EBUSY; if ((skreq->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { list_add_tail(&sec_req->backlog_head, &ctx->backlog); - mutex_unlock(&queue->queuelock); + spin_unlock_bh(&queue->queuelock); goto out; } - mutex_unlock(&queue->queuelock); + spin_unlock_bh(&queue->queuelock); goto err_free_elements; } ret = sec_send_request(sec_req, queue); - mutex_unlock(&queue->queuelock); + spin_unlock_bh(&queue->queuelock); if (ret) goto err_free_elements; @@ -881,7 +881,7 @@ static int sec_alg_skcipher_init(struct crypto_skcipher *tfm) if (IS_ERR(ctx->queue)) return PTR_ERR(ctx->queue); - mutex_init(&ctx->queue->queuelock); + spin_lock_init(&ctx->queue->queuelock); ctx->queue->havesoftqueue = false; return 0; diff --git a/drivers/crypto/hisilicon/sec/sec_drv.h b/drivers/crypto/hisilicon/sec/sec_drv.h index 179a8250d691..e2a50bf2234b 100644 --- a/drivers/crypto/hisilicon/sec/sec_drv.h +++ b/drivers/crypto/hisilicon/sec/sec_drv.h @@ -347,7 +347,7 @@ struct sec_queue { DECLARE_BITMAP(unprocessed, SEC_QUEUE_LEN); DECLARE_KFIFO_PTR(softqueue, typeof(struct sec_request_el *)); bool havesoftqueue; - struct mutex queuelock; + spinlock_t queuelock; void *shadow[SEC_QUEUE_LEN]; }; From 932be3e95b228d13cd1b8aaf5c0578c76e3f7dc6 Mon Sep 17 00:00:00 2001 From: Ofer Heifetz Date: Sat, 2 Jul 2022 10:14:26 +0300 Subject: [PATCH 197/318] crypto: inside-secure - fix packed bit-field result descriptor When mixing bit-field and none bit-filed in packed struct the none bit-field starts at a distinct memory location, thus adding an additional byte to the overall structure which is used in memory zero-ing and other configuration calculations. Fix this by removing the none bit-field that has a following bit-field. Signed-off-by: Ofer Heifetz Acked-by: Antoine Tenart Acked-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index ce1e611a163e..797ff91512e0 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -497,15 +497,15 @@ struct result_data_desc { u32 packet_length:17; u32 error_code:15; - u8 bypass_length:4; - u8 e15:1; - u16 rsvd0; - u8 hash_bytes:1; - u8 hash_length:6; - u8 generic_bytes:1; - u8 checksum:1; - u8 next_header:1; - u8 length:1; + u32 bypass_length:4; + u32 e15:1; + u32 rsvd0:16; + u32 hash_bytes:1; + u32 hash_length:6; + u32 generic_bytes:1; + u32 checksum:1; + u32 next_header:1; + u32 length:1; u16 application_id; u16 rsvd1; From 79e6e2f3f3ff345947075341781e900e4f70db81 Mon Sep 17 00:00:00 2001 From: Ignat Korchagin Date: Mon, 4 Jul 2022 11:38:40 +0100 Subject: [PATCH 198/318] crypto: testmgr - populate RSA CRT parameters in RSA test vectors Changes from v1: * replace some accidental spaces with tabs In commit f145d411a67e ("crypto: rsa - implement Chinese Remainder Theorem for faster private key operations") we have started to use the additional primes and coefficients for RSA private key operations. However, these additional parameters are not present (defined as 0 integers) in the RSA test vectors. Some parameters were borrowed from OpenSSL, so I was able to find the source. I could not find the public source for 1 vector though, so had to recover the parameters by implementing Appendix C from [1]. [1]: https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-56Br1.pdf Fixes: f145d411a67e ("crypto: rsa - implement Chinese Remainder Theorem for faster private key operations") Reported-by: Tasmiya Nalatwad Signed-off-by: Ignat Korchagin Signed-off-by: Herbert Xu --- crypto/testmgr.h | 121 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 100 insertions(+), 21 deletions(-) diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 8e2dce86dd48..f1dffdace219 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -185,7 +185,7 @@ static const struct akcipher_testvec rsa_tv_template[] = { { #ifndef CONFIG_CRYPTO_FIPS .key = - "\x30\x81\x9A" /* sequence of 154 bytes */ + "\x30\x82\x01\x38" /* sequence of 312 bytes */ "\x02\x01\x00" /* version - integer of 1 byte */ "\x02\x41" /* modulus - integer of 65 bytes */ "\x00\xAA\x36\xAB\xCE\x88\xAC\xFD\xFF\x55\x52\x3C\x7F\xC4\x52\x3F" @@ -199,23 +199,36 @@ static const struct akcipher_testvec rsa_tv_template[] = { "\xC2\xCD\x2D\xFF\x43\x40\x98\xCD\x20\xD8\xA1\x38\xD0\x90\xBF\x64" "\x79\x7C\x3F\xA7\xA2\xCD\xCB\x3C\xD1\xE0\xBD\xBA\x26\x54\xB4\xF9" "\xDF\x8E\x8A\xE5\x9D\x73\x3D\x9F\x33\xB3\x01\x62\x4A\xFD\x1D\x51" - "\x02\x01\x00" /* prime1 - integer of 1 byte */ - "\x02\x01\x00" /* prime2 - integer of 1 byte */ - "\x02\x01\x00" /* exponent1 - integer of 1 byte */ - "\x02\x01\x00" /* exponent2 - integer of 1 byte */ - "\x02\x01\x00", /* coefficient - integer of 1 byte */ + "\x02\x21" /* prime1 - integer of 33 bytes */ + "\x00\xD8\x40\xB4\x16\x66\xB4\x2E\x92\xEA\x0D\xA3\xB4\x32\x04\xB5" + "\xCF\xCE\x33\x52\x52\x4D\x04\x16\xA5\xA4\x41\xE7\x00\xAF\x46\x12" + "\x0D" + "\x02\x21" /* prime2 - integer of 33 bytes */ + "\x00\xC9\x7F\xB1\xF0\x27\xF4\x53\xF6\x34\x12\x33\xEA\xAA\xD1\xD9" + "\x35\x3F\x6C\x42\xD0\x88\x66\xB1\xD0\x5A\x0F\x20\x35\x02\x8B\x9D" + "\x89" + "\x02\x20" /* exponent1 - integer of 32 bytes */ + "\x59\x0B\x95\x72\xA2\xC2\xA9\xC4\x06\x05\x9D\xC2\xAB\x2F\x1D\xAF" + "\xEB\x7E\x8B\x4F\x10\xA7\x54\x9E\x8E\xED\xF5\xB4\xFC\xE0\x9E\x05" + "\x02\x21" /* exponent2 - integer of 33 bytes */ + "\x00\x8E\x3C\x05\x21\xFE\x15\xE0\xEA\x06\xA3\x6F\xF0\xF1\x0C\x99" + "\x52\xC3\x5B\x7A\x75\x14\xFD\x32\x38\xB8\x0A\xAD\x52\x98\x62\x8D" + "\x51" + "\x02\x20" /* coefficient - integer of 32 bytes */ + "\x36\x3F\xF7\x18\x9D\xA8\xE9\x0B\x1D\x34\x1F\x71\xD0\x9B\x76\xA8" + "\xA9\x43\xE1\x1D\x10\xB2\x4D\x24\x9F\x2D\xEA\xFE\xF8\x0C\x18\x26", .m = "\x54\x85\x9b\x34\x2c\x49\xea\x2a", .c = "\x63\x1c\xcd\x7b\xe1\x7e\xe4\xde\xc9\xa8\x89\xa1\x74\xcb\x3c\x63" "\x7d\x24\xec\x83\xc3\x15\xe4\x7f\x73\x05\x34\xd1\xec\x22\xbb\x8a" "\x5e\x32\x39\x6d\xc1\x1d\x7d\x50\x3b\x9f\x7a\xad\xf0\x2e\x25\x53" "\x9f\x6e\xbd\x4c\x55\x84\x0c\x9b\xcf\x1a\x4b\x51\x1e\x9e\x0c\x06", - .key_len = 157, + .key_len = 316, .m_size = 8, .c_size = 64, }, { .key = - "\x30\x82\x01\x1D" /* sequence of 285 bytes */ + "\x30\x82\x02\x5B" /* sequence of 603 bytes */ "\x02\x01\x00" /* version - integer of 1 byte */ "\x02\x81\x81" /* modulus - integer of 129 bytes */ "\x00\xBB\xF8\x2F\x09\x06\x82\xCE\x9C\x23\x38\xAC\x2B\x9D\xA8\x71" @@ -238,12 +251,35 @@ static const struct akcipher_testvec rsa_tv_template[] = { "\x93\x99\x26\xED\x4F\x74\xA1\x3E\xDD\xFB\xE1\xA1\xCE\xCC\x48\x94" "\xAF\x94\x28\xC2\xB7\xB8\x88\x3F\xE4\x46\x3A\x4B\xC8\x5B\x1C\xB3" "\xC1" - "\x02\x01\x00" /* prime1 - integer of 1 byte */ - "\x02\x01\x00" /* prime2 - integer of 1 byte */ - "\x02\x01\x00" /* exponent1 - integer of 1 byte */ - "\x02\x01\x00" /* exponent2 - integer of 1 byte */ - "\x02\x01\x00", /* coefficient - integer of 1 byte */ - .key_len = 289, + "\x02\x41" /* prime1 - integer of 65 bytes */ + "\x00\xEE\xCF\xAE\x81\xB1\xB9\xB3\xC9\x08\x81\x0B\x10\xA1\xB5\x60" + "\x01\x99\xEB\x9F\x44\xAE\xF4\xFD\xA4\x93\xB8\x1A\x9E\x3D\x84\xF6" + "\x32\x12\x4E\xF0\x23\x6E\x5D\x1E\x3B\x7E\x28\xFA\xE7\xAA\x04\x0A" + "\x2D\x5B\x25\x21\x76\x45\x9D\x1F\x39\x75\x41\xBA\x2A\x58\xFB\x65" + "\x99" + "\x02\x41" /* prime2 - integer of 65 bytes */ + "\x00\xC9\x7F\xB1\xF0\x27\xF4\x53\xF6\x34\x12\x33\xEA\xAA\xD1\xD9" + "\x35\x3F\x6C\x42\xD0\x88\x66\xB1\xD0\x5A\x0F\x20\x35\x02\x8B\x9D" + "\x86\x98\x40\xB4\x16\x66\xB4\x2E\x92\xEA\x0D\xA3\xB4\x32\x04\xB5" + "\xCF\xCE\x33\x52\x52\x4D\x04\x16\xA5\xA4\x41\xE7\x00\xAF\x46\x15" + "\x03" + "\x02\x40" /* exponent1 - integer of 64 bytes */ + "\x54\x49\x4C\xA6\x3E\xBA\x03\x37\xE4\xE2\x40\x23\xFC\xD6\x9A\x5A" + "\xEB\x07\xDD\xDC\x01\x83\xA4\xD0\xAC\x9B\x54\xB0\x51\xF2\xB1\x3E" + "\xD9\x49\x09\x75\xEA\xB7\x74\x14\xFF\x59\xC1\xF7\x69\x2E\x9A\x2E" + "\x20\x2B\x38\xFC\x91\x0A\x47\x41\x74\xAD\xC9\x3C\x1F\x67\xC9\x81" + "\x02\x40" /* exponent2 - integer of 64 bytes */ + "\x47\x1E\x02\x90\xFF\x0A\xF0\x75\x03\x51\xB7\xF8\x78\x86\x4C\xA9" + "\x61\xAD\xBD\x3A\x8A\x7E\x99\x1C\x5C\x05\x56\xA9\x4C\x31\x46\xA7" + "\xF9\x80\x3F\x8F\x6F\x8A\xE3\x42\xE9\x31\xFD\x8A\xE4\x7A\x22\x0D" + "\x1B\x99\xA4\x95\x84\x98\x07\xFE\x39\xF9\x24\x5A\x98\x36\xDA\x3D" + "\x02\x41", /* coefficient - integer of 65 bytes */ + "\x00\xB0\x6C\x4F\xDA\xBB\x63\x01\x19\x8D\x26\x5B\xDB\xAE\x94\x23" + "\xB3\x80\xF2\x71\xF7\x34\x53\x88\x50\x93\x07\x7F\xCD\x39\xE2\x11" + "\x9F\xC9\x86\x32\x15\x4F\x58\x83\xB1\x67\xA9\x67\xBF\x40\x2B\x4E" + "\x9E\x2E\x0F\x96\x56\xE6\x98\xEA\x36\x66\xED\xFB\x25\x79\x80\x39" + "\xF7", + .key_len = 607, .m = "\x54\x85\x9b\x34\x2c\x49\xea\x2a", .c = "\x74\x1b\x55\xac\x47\xb5\x08\x0a\x6e\x2b\x2d\xf7\x94\xb8\x8a\x95" @@ -259,7 +295,7 @@ static const struct akcipher_testvec rsa_tv_template[] = { }, { #endif .key = - "\x30\x82\x02\x20" /* sequence of 544 bytes */ + "\x30\x82\x04\xA3" /* sequence of 1187 bytes */ "\x02\x01\x00" /* version - integer of 1 byte */ "\x02\x82\x01\x01\x00" /* modulus - integer of 256 bytes */ "\xDB\x10\x1A\xC2\xA3\xF1\xDC\xFF\x13\x6B\xED\x44\xDF\xF0\x02\x6D" @@ -296,12 +332,55 @@ static const struct akcipher_testvec rsa_tv_template[] = { "\x62\xFF\xE9\x46\xB8\xD8\x44\xDB\xA5\xCC\x31\x54\x34\xCE\x3E\x82" "\xD6\xBF\x7A\x0B\x64\x21\x6D\x88\x7E\x5B\x45\x12\x1E\x63\x8D\x49" "\xA7\x1D\xD9\x1E\x06\xCD\xE8\xBA\x2C\x8C\x69\x32\xEA\xBE\x60\x71" - "\x02\x01\x00" /* prime1 - integer of 1 byte */ - "\x02\x01\x00" /* prime2 - integer of 1 byte */ - "\x02\x01\x00" /* exponent1 - integer of 1 byte */ - "\x02\x01\x00" /* exponent2 - integer of 1 byte */ - "\x02\x01\x00", /* coefficient - integer of 1 byte */ - .key_len = 548, + "\x02\x81\x81" /* prime1 - integer of 129 bytes */ + "\x00\xFA\xAC\xE1\x37\x5E\x32\x11\x34\xC6\x72\x58\x2D\x91\x06\x3E" + "\x77\xE7\x11\x21\xCD\x4A\xF8\xA4\x3F\x0F\xEF\x31\xE3\xF3\x55\xA0" + "\xB9\xAC\xB6\xCB\xBB\x41\xD0\x32\x81\x9A\x8F\x7A\x99\x30\x77\x6C" + "\x68\x27\xE2\x96\xB5\x72\xC9\xC3\xD4\x42\xAA\xAA\xCA\x95\x8F\xFF" + "\xC9\x9B\x52\x34\x30\x1D\xCF\xFE\xCF\x3C\x56\x68\x6E\xEF\xE7\x6C" + "\xD7\xFB\x99\xF5\x4A\xA5\x21\x1F\x2B\xEA\x93\xE8\x98\x26\xC4\x6E" + "\x42\x21\x5E\xA0\xA1\x2A\x58\x35\xBB\x10\xE7\xBA\x27\x0A\x3B\xB3" + "\xAF\xE2\x75\x36\x04\xAC\x56\xA0\xAB\x52\xDE\xCE\xDD\x2C\x28\x77" + "\x03" + "\x02\x81\x81" /* prime2 - integer of 129 bytes */ + "\x00\xDF\xB7\x52\xB6\xD7\xC0\xE2\x96\xE7\xC9\xFE\x5D\x71\x5A\xC4" + "\x40\x96\x2F\xE5\x87\xEA\xF3\xA5\x77\x11\x67\x3C\x8D\x56\x08\xA7" + "\xB5\x67\xFA\x37\xA8\xB8\xCF\x61\xE8\x63\xD8\x38\x06\x21\x2B\x92" + "\x09\xA6\x39\x3A\xEA\xA8\xB4\x45\x4B\x36\x10\x4C\xE4\x00\x66\x71" + "\x65\xF8\x0B\x94\x59\x4F\x8C\xFD\xD5\x34\xA2\xE7\x62\x84\x0A\xA7" + "\xBB\xDB\xD9\x8A\xCD\x05\xE1\xCC\x57\x7B\xF1\xF1\x1F\x11\x9D\xBA" + "\x3E\x45\x18\x99\x1B\x41\x64\x43\xEE\x97\x5D\x77\x13\x5B\x74\x69" + "\x73\x87\x95\x05\x07\xBE\x45\x07\x17\x7E\x4A\x69\x22\xF3\xDB\x05" + "\x39" + "\x02\x81\x80" /* exponent1 - integer of 128 bytes */ + "\x5E\xD8\xDC\xDA\x53\x44\xC4\x67\xE0\x92\x51\x34\xE4\x83\xA5\x4D" + "\x3E\xDB\xA7\x9B\x82\xBB\x73\x81\xFC\xE8\x77\x4B\x15\xBE\x17\x73" + "\x49\x9B\x5C\x98\xBC\xBD\x26\xEF\x0C\xE9\x2E\xED\x19\x7E\x86\x41" + "\x1E\x9E\x48\x81\xDD\x2D\xE4\x6F\xC2\xCD\xCA\x93\x9E\x65\x7E\xD5" + "\xEC\x73\xFD\x15\x1B\xA2\xA0\x7A\x0F\x0D\x6E\xB4\x53\x07\x90\x92" + "\x64\x3B\x8B\xA9\x33\xB3\xC5\x94\x9B\x4C\x5D\x9C\x7C\x46\xA4\xA5" + "\x56\xF4\xF3\xF8\x27\x0A\x7B\x42\x0D\x92\x70\x47\xE7\x42\x51\xA9" + "\xC2\x18\xB1\x58\xB1\x50\x91\xB8\x61\x41\xB6\xA9\xCE\xD4\x7C\xBB" + "\x02\x81\x80" /* exponent2 - integer of 128 bytes */ + "\x54\x09\x1F\x0F\x03\xD8\xB6\xC5\x0C\xE8\xB9\x9E\x0C\x38\x96\x43" + "\xD4\xA6\xC5\x47\xDB\x20\x0E\xE5\xBD\x29\xD4\x7B\x1A\xF8\x41\x57" + "\x49\x69\x9A\x82\xCC\x79\x4A\x43\xEB\x4D\x8B\x2D\xF2\x43\xD5\xA5" + "\xBE\x44\xFD\x36\xAC\x8C\x9B\x02\xF7\x9A\x03\xE8\x19\xA6\x61\xAE" + "\x76\x10\x93\x77\x41\x04\xAB\x4C\xED\x6A\xCC\x14\x1B\x99\x8D\x0C" + "\x6A\x37\x3B\x86\x6C\x51\x37\x5B\x1D\x79\xF2\xA3\x43\x10\xC6\xA7" + "\x21\x79\x6D\xF9\xE9\x04\x6A\xE8\x32\xFF\xAE\xFD\x1C\x7B\x8C\x29" + "\x13\xA3\x0C\xB2\xAD\xEC\x6C\x0F\x8D\x27\x12\x7B\x48\xB2\xDB\x31" + "\x02\x81\x81", /* coefficient - integer of 129 bytes */ + "\x00\x8D\x1B\x05\xCA\x24\x1F\x0C\x53\x19\x52\x74\x63\x21\xFA\x78" + "\x46\x79\xAF\x5C\xDE\x30\xA4\x6C\x20\x38\xE6\x97\x39\xB8\x7A\x70" + "\x0D\x8B\x6C\x6D\x13\x74\xD5\x1C\xDE\xA9\xF4\x60\x37\xFE\x68\x77" + "\x5E\x0B\x4E\x5E\x03\x31\x30\xDF\xD6\xAE\x85\xD0\x81\xBB\x61\xC7" + "\xB1\x04\x5A\xC4\x6D\x56\x1C\xD9\x64\xE7\x85\x7F\x88\x91\xC9\x60" + "\x28\x05\xE2\xC6\x24\x8F\xDD\x61\x64\xD8\x09\xDE\x7E\xD3\x4A\x61" + "\x1A\xD3\x73\x58\x4B\xD8\xA0\x54\x25\x48\x83\x6F\x82\x6C\xAF\x36" + "\x51\x2A\x5D\x14\x2F\x41\x25\x00\xDD\xF8\xF3\x95\xFE\x31\x25\x50" + "\x12", + .key_len = 1191, .m = "\x54\x85\x9b\x34\x2c\x49\xea\x2a", .c = "\xb2\x97\x76\xb4\xae\x3e\x38\x3c\x7e\x64\x1f\xcc\xa2\x7f\xf6\xbe" From ed56f4b5173efba00e3dcd736c18bce43255cf7a Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Fri, 8 Jul 2022 16:20:48 -0400 Subject: [PATCH 199/318] MAINTAINERS: update the LSM maintainer info After many years of shepherding the LSM layer, and Linux kernel access control in general, James has decided to take a small step away from his technical role and has asked me to assume the day-to-day maintenance tasks for the LSM. This patch makes the necessary updates to the MAINTAINERS file. Thanks for all you patience and stewardship over the years James, I promise to do my best not to screw it all up :) Cc: linux-security-module@vger.kernel.org Acked-by: Casey Schaufler Acked-by: Serge Hallyn Acked-by: James Morris Signed-off-by: Paul Moore --- MAINTAINERS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index a6d3bd9d2a8d..5c7c5d04b99d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -17909,12 +17909,13 @@ S: Supported F: Documentation/admin-guide/security-bugs.rst SECURITY SUBSYSTEM +M: Paul Moore M: James Morris M: "Serge E. Hallyn" L: linux-security-module@vger.kernel.org (suggested Cc:) S: Supported W: http://kernsec.org/ -T: git git://git.kernel.org/pub/scm/linux/kernel/git/jmorris/linux-security.git +T: git git://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/lsm.git F: security/ X: security/selinux/ From c808a6ec7166105818e698be9ead396233eb91b8 Mon Sep 17 00:00:00 2001 From: Xiu Jianfeng Date: Mon, 6 Jun 2022 18:10:42 +0800 Subject: [PATCH 200/318] evm: Use IS_ENABLED to initialize .enabled Use IS_ENABLED(CONFIG_XXX) instead of #ifdef/#endif statements to initialize .enabled, minor simplicity improvement. Signed-off-by: Xiu Jianfeng Reviewed-by: Ahmad Fatoum Signed-off-by: Mimi Zohar --- security/integrity/evm/evm_main.c | 52 ++++++++++++++----------------- 1 file changed, 23 insertions(+), 29 deletions(-) diff --git a/security/integrity/evm/evm_main.c b/security/integrity/evm/evm_main.c index cc88f02c7562..397fea5b3fa6 100644 --- a/security/integrity/evm/evm_main.c +++ b/security/integrity/evm/evm_main.c @@ -36,42 +36,36 @@ static const char * const integrity_status_msg[] = { int evm_hmac_attrs; static struct xattr_list evm_config_default_xattrnames[] = { - {.name = XATTR_NAME_SELINUX, -#ifdef CONFIG_SECURITY_SELINUX - .enabled = true -#endif + { + .name = XATTR_NAME_SELINUX, + .enabled = IS_ENABLED(CONFIG_SECURITY_SELINUX) }, - {.name = XATTR_NAME_SMACK, -#ifdef CONFIG_SECURITY_SMACK - .enabled = true -#endif + { + .name = XATTR_NAME_SMACK, + .enabled = IS_ENABLED(CONFIG_SECURITY_SMACK) }, - {.name = XATTR_NAME_SMACKEXEC, -#ifdef CONFIG_EVM_EXTRA_SMACK_XATTRS - .enabled = true -#endif + { + .name = XATTR_NAME_SMACKEXEC, + .enabled = IS_ENABLED(CONFIG_EVM_EXTRA_SMACK_XATTRS) }, - {.name = XATTR_NAME_SMACKTRANSMUTE, -#ifdef CONFIG_EVM_EXTRA_SMACK_XATTRS - .enabled = true -#endif + { + .name = XATTR_NAME_SMACKTRANSMUTE, + .enabled = IS_ENABLED(CONFIG_EVM_EXTRA_SMACK_XATTRS) }, - {.name = XATTR_NAME_SMACKMMAP, -#ifdef CONFIG_EVM_EXTRA_SMACK_XATTRS - .enabled = true -#endif + { + .name = XATTR_NAME_SMACKMMAP, + .enabled = IS_ENABLED(CONFIG_EVM_EXTRA_SMACK_XATTRS) }, - {.name = XATTR_NAME_APPARMOR, -#ifdef CONFIG_SECURITY_APPARMOR - .enabled = true -#endif + { + .name = XATTR_NAME_APPARMOR, + .enabled = IS_ENABLED(CONFIG_SECURITY_APPARMOR) }, - {.name = XATTR_NAME_IMA, -#ifdef CONFIG_IMA_APPRAISE - .enabled = true -#endif + { + .name = XATTR_NAME_IMA, + .enabled = IS_ENABLED(CONFIG_IMA_APPRAISE) }, - {.name = XATTR_NAME_CAPS, + { + .name = XATTR_NAME_CAPS, .enabled = true }, }; From d5a0cdb852b311585b0aff657703da92c874ef8b Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Wed, 13 Jul 2022 06:35:16 +0200 Subject: [PATCH 201/318] MAINTAINERS: mark linux-doc-tw-discuss mailing list moderated After sending a patch to linux-doc-tw-discuss@lists.sourceforge.net, I got the typical response for a moderated list: Your mail to 'linux-doc-tw-discuss' with the subject .... Is being held until the list moderator can review it for approval. The reason it is being held: Post by non-member to a members-only list Mark this mailing list moderated in MAINTAINERS. Signed-off-by: Lukas Bulwahn Link: https://lore.kernel.org/r/20220713043516.19290-1-lukas.bulwahn@gmail.com Signed-off-by: Jonathan Corbet --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 3320dd8bd198..ee8824f2a8a8 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -20244,7 +20244,7 @@ F: tools/tracing/rtla/ TRADITIONAL CHINESE DOCUMENTATION M: Hu Haowen -L: linux-doc-tw-discuss@lists.sourceforge.net +L: linux-doc-tw-discuss@lists.sourceforge.net (moderated for non-subscribers) S: Maintained W: https://github.com/srcres258/linux-doc T: git git://github.com/srcres258/linux-doc.git doc-zh-tw From 12fe434314c8572c8043c6eee6799d2ccac99f26 Mon Sep 17 00:00:00 2001 From: Dov Murik Date: Tue, 12 Jul 2022 10:44:55 +0000 Subject: [PATCH 202/318] Documentation: siphash: Fix typo in the name of offsetofend macro The siphash documentation misspelled "offsetendof" instead of "offsetofend". Fixes: 2c956a60778cbb ("siphash: add cryptographically secure PRF") Signed-off-by: Dov Murik Acked-by: Jason A. Donenfeld Link: https://lore.kernel.org/r/20220712104455.1408150-1-dovmurik@linux.ibm.com Signed-off-by: Jonathan Corbet --- Documentation/security/siphash.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/security/siphash.rst b/Documentation/security/siphash.rst index a10380cb78e5..023bd95c74a5 100644 --- a/Documentation/security/siphash.rst +++ b/Documentation/security/siphash.rst @@ -85,7 +85,7 @@ Often times the XuY functions will not be large enough, and instead you'll want to pass a pre-filled struct to siphash. When doing this, it's important to always ensure the struct has no padding holes. The easiest way to do this is to simply arrange the members of the struct in descending order of size, -and to use offsetendof() instead of sizeof() for getting the size. For +and to use offsetofend() instead of sizeof() for getting the size. For performance reasons, if possible, it's probably a good thing to align the struct to the right boundary. Here's an example:: From c528be5ed203ece2bcb5680f015d18da76174653 Mon Sep 17 00:00:00 2001 From: Binbin Zhou Date: Sun, 10 Jul 2022 21:36:04 +0800 Subject: [PATCH 203/318] docs/zh_CN: core-api: Add watch_queue Chinese translation Translate core-api/watch_queue.rst into Chinese. Last English version used: commit f5461124d59b ("Documentation: move watch_queue to core-api"). Signed-off-by: Binbin Zhou Reviewed-by: Wu XiangCheng Link: https://lore.kernel.org/r/20220710133604.31382-1-zhoubinbin@loongson.cn Signed-off-by: Jonathan Corbet --- .../translations/zh_CN/core-api/index.rst | 1 + .../zh_CN/core-api/watch_queue.rst | 313 ++++++++++++++++++ 2 files changed, 314 insertions(+) create mode 100644 Documentation/translations/zh_CN/core-api/watch_queue.rst diff --git a/Documentation/translations/zh_CN/core-api/index.rst b/Documentation/translations/zh_CN/core-api/index.rst index 26d9913fc8b6..7ca44629860c 100644 --- a/Documentation/translations/zh_CN/core-api/index.rst +++ b/Documentation/translations/zh_CN/core-api/index.rst @@ -28,6 +28,7 @@ printk-basics printk-formats workqueue + watch_queue symbol-namespaces 数据结构和低级实用程序 diff --git a/Documentation/translations/zh_CN/core-api/watch_queue.rst b/Documentation/translations/zh_CN/core-api/watch_queue.rst new file mode 100644 index 000000000000..23b17ae2e4e2 --- /dev/null +++ b/Documentation/translations/zh_CN/core-api/watch_queue.rst @@ -0,0 +1,313 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/core-api/watch_queue.rst + +:翻译: + +周彬彬 Binbin Zhou + +:校译: + +司延腾 Yanteng Si +吴想成 Wu Xiangcheng + + +============ +通用通知机制 +============ + +通用通知机制是建立在标准管道驱动之上的，它可以有效地将来自内核的通知消息拼接到用 +户空间打开的管道中。这可以与以下方面结合使用:: + + * Key/keyring 通知 + +通知缓冲区可以通过以下方式启用： + + “General setup”/“General notification queue” + (CONFIG_WATCH_QUEUE) + +文档包含以下章节： + +.. contents:: :local: + + +概述 +==== + +该设施以一种特殊模式打开的管道形式出现，管道的内部环形缓冲区用于保存内核生成的消 +息。然后通过read()读出这些消息。在此类管道上禁用拼接以及类似的操作，因为它们希望 +在某些情况下将其添加的内容还原到环中-这可能最终会与通知消息重叠。 + +管道的所有者必须告诉内核它想通过该管道观察哪些源。只有连接到该管道上的源才会将消 +息插入其中。请注意，一个源可能绑定到多个管道，并同时将消息插入到所有管道中。 + +还可以将过滤器放置在管道上，以便在不感兴趣时可以忽略某些源类型和子事件。 + +如果环中没有可用的插槽，或者没有预分配的消息缓冲区可用，则将丢弃消息。在这两种情 +况下，read()都会在读取缓冲区中当前的最后一条消息后，将WATCH_META_LOSS_NOTIFICATION +插入到输出缓冲区中。 + +请注意，当生成一个通知时，内核不会等待消费者收集它，而是继续执行。这意味着可以在 +持有自旋锁的同时生成通知，并且还可以保护内核不被用户空间故障无限期地阻碍。 + + +消息结构 +======== + +通知消息由一个简短的头部开始:: + + struct watch_notification { + __u32 type:24; + __u32 subtype:8; + __u32 info; + }; + +“type”表示通知记录的来源，“subtype”表示该来源的记录类型（见下文观测源章节）。该类 +型也可以是“WATCH_TYPE_META”。这是一个由观测队列本身在内部生成的特殊记录类型。有两 +个子类型： + + * WATCH_META_REMOVAL_NOTIFICATION + * WATCH_META_LOSS_NOTIFICATION + +第一个表示安装了观察的对象已被删除或销毁，第二个表示某些消息已丢失。 + +“info”表示一系列东西，包括： + + * 消息的长度，以字节为单位，包括头（带有WATCH_INFO_LENGTH的掩码，并按 + WATCH_INFO_LENGTH__SHIFT移位）。这表示记录的大小，可能在8到127字节之间。 + + * 观测ID（带有WATCH_INFO_ID掩码，并按WATCH_INFO_ID__SHIFT移位）。这表示观测的主 + 叫ID，可能在0到255之间。多个观测组可以共享一个队列，这提供了一种区分它们的方法。 + + * 特定类型的字段（WATCH_INFO_TYPE_INFO）。这是由通知生产者设置的，以指示类型和 + 子类型的某些特定含义。 + +除长度外，信息中的所有内容都可以用于过滤。 + +头部后面可以有补充信息。此格式是由类型和子类型决定的。 + + +观测列表（通知源）API +===================== + +“观测列表“是订阅通知源的观测者的列表。列表可以附加到对象（比如键或超级块），也可 +以是全局的（比如对于设备事件）。从用户空间的角度来看，一个非全局的观测列表通常是 +通过引用它所属的对象来引用的（比如使用KEYCTL_NOTIFY并给它一个密钥序列号来观测特定 +的密钥）。 + +为了管理观测列表，提供了以下函数： + + * :: + + void init_watch_list(struct watch_list *wlist, + void (*release_watch)(struct watch *wlist)); + + 初始化一个观测列表。 如果 ``release_watch`` 不是NULL，那么这表示当watch_list对 + 象被销毁时，应该调用函数来丢弃观测列表对被观测对象的任何引用。 + + * ``void remove_watch_list(struct watch_list *wlist);`` + + 这将删除订阅watch_list的所有观测，并释放它们，然后销毁watch_list对象本身。 + + +观测队列（通知输出）API +======================= + +“观测队列”是由应用程序分配的用以记录通知的缓冲区，其工作原理完全隐藏在管道设备驱 +动中，但必须获得对它的引用才能设置观测。可以通过以下方式进行管理： + + * ``struct watch_queue *get_watch_queue(int fd);`` + + 由于观测队列在内核中通过实现缓冲区的管道的文件描述符表示，用户空间必须通过系 + 统调用传递该文件描述符，这可以用于从系统调用中查找指向观测队列的不透明指针。 + + * ``void put_watch_queue(struct watch_queue *wqueue);`` + + 该函数用以丢弃从 ``get_watch_queue()`` 获得的引用。 + + +观测订阅API +=========== + +“观测”是观测列表上的订阅，表示观测队列，从而表示应写入通知记录的缓冲区。观测队列 +对象还可以携带该对象的过滤规则，由用户空间设置。watch结构体的某些部分可以由驱动程 +序设置:: + + struct watch { + union { + u32 info_id; /* 在info字段中进行OR运算的ID */ + ... + }; + void *private; /* 被观测对象的私有数据 */ + u64 id; /* 内部标识符 */ + ... + }; + +``info_id`` 值是从用户空间获得并按WATCH_INFO_ID__SHIFT移位的8位数字。当通知写入关 +联的观测队列缓冲区时，这将与struct watch_notification::info的WATCH_INFO_ID字段进 +行或运算。 + +``private`` 字段是与watch_list相关联的驱动程序数据，并由 ``watch_list::release_watch()`` +函数清除。 + +``id`` 字段是源的ID。使用不同ID发布的通知将被忽略。 + +提供以下函数来管理观测: + + * ``void init_watch(struct watch *watch, struct watch_queue *wqueue);`` + + 初始化一个观测对象，把它的指针设置到观察队列中，使用适当的限制来避免死锁。 + + * ``int add_watch_to_object(struct watch *watch, struct watch_list *wlist);`` + + 将观测订阅到观测列表（通知源）。watch结构体中的driver-settable字段必须在调用 + 它之前设置。 + + * :: + + int remove_watch_from_object(struct watch_list *wlist, + struct watch_queue *wqueue, + u64 id, false); + + 从观测列表中删除一个观测，该观测必须与指定的观测队列（``wqueue``）和对象标识 + 符（``id``）匹配。通知（``WATCH_META_REMOVAL_NOTIFICATION``）被发送到观测队列 + 表示该观测已被删除。 + + * ``int remove_watch_from_object(struct watch_list *wlist, NULL, 0, true);`` + + 从观测列表中删除所有观测。预计这将被称为销毁前的准备工作，届时新的观测将无法 + 访问观测列表。通知（``WATCH_META_REMOVAL_NOTIFICATION``）被发送到每个订阅观测 + 的观测队列，以表明该观测已被删除。 + + +通知发布API +=========== + +要将通知发布到观测列表以便订阅的观测可以看到，应使用以下函数:: + + void post_watch_notification(struct watch_list *wlist, + struct watch_notification *n, + const struct cred *cred, + u64 id); + +应预先设置通知格式，并应传入一个指向头部（``n``）的指针。通知可能大于此值，并且缓 +冲槽为单位的大小在 ``n->info & WATCH_INFO_LENGTH`` 中注明。 + +``cred`` 结构体表示源（对象）的证书，并传递给LSM，例如SELinux，以允许或禁止根据该队 +列（对象）的证书在每个单独队列中记录注释。 + +``id`` 是源对象ID（如密钥上的序列号）。只有设置相同ID的观测才能看到这个通知。 + + +观测源 +====== + +任何特定的缓冲区都可以从多个源获取信息。 这些源包括: + + * WATCH_TYPE_KEY_NOTIFY + + 这种类型的通知表示密钥和密钥环的变化，包括密钥环内容或密钥属性的变化。 + + 更多信息请参见Documentation/security/keys/core.rst。 + + +事件过滤 +======== + +当创建观测队列后，我们可以应用一组过滤器以限制接收的事件:: + + struct watch_notification_filter filter = { + ... + }; + ioctl(fd, IOC_WATCH_QUEUE_SET_FILTER, &filter) + +过滤器的描述的类型变量是:: + + struct watch_notification_filter { + __u32 nr_filters; + __u32 __reserved; + struct watch_notification_type_filter filters[]; + }; + +其中“nr_filters”表示filters[]数组中过滤器的数量，而“__reserved”应为0。 +“filter”数组有以下类型的元素:: + + struct watch_notification_type_filter { + __u32 type; + __u32 info_filter; + __u32 info_mask; + __u32 subtype_filter[8]; + }; + +其中： + + * ``type`` 是过滤的事件类型，应类似于“WATCH_TYPE_KEY_NOTIFY”。 + + * ``info_filter`` 与 ``info_mask`` 充当通知记录的信息字段的过滤器，只有在以下情 + 况，通知才会写入缓冲区:: + + (watch.info & info_mask) == info_filter + + 例如，这可以用于忽略不在一个挂载树上的观测点的事件。 + + * ``subtype_filter`` 是一个位掩码，表示感兴趣的子类型。subtype_filter[0]的 + bit[0]对应子类型0，bit[1]对应子类型1，以此类推。 + +若ioctl()的参数为NULL，则过滤器将被移除，并且来自观测源的所有事件都将通过。 + + +用户空间代码示例 +================ + +缓冲区的创建如下所示:: + + pipe2(fds, O_TMPFILE); + ioctl(fds[1], IOC_WATCH_QUEUE_SET_SIZE, 256); + +它可以被设置成接收密钥环变化的通知:: + + keyctl(KEYCTL_WATCH_KEY, KEY_SPEC_SESSION_KEYRING, fds[1], 0x01); + +然后，这些通知可以被如下方式所使用:: + + static void consumer(int rfd, struct watch_queue_buffer *buf) + { + unsigned char buffer[128]; + ssize_t buf_len; + + while (buf_len = read(rfd, buffer, sizeof(buffer)), + buf_len > 0 + ) { + void *p = buffer; + void *end = buffer + buf_len; + while (p < end) { + union { + struct watch_notification n; + unsigned char buf1[128]; + } n; + size_t largest, len; + + largest = end - p; + if (largest > 128) + largest = 128; + memcpy(&n, p, largest); + + len = (n->info & WATCH_INFO_LENGTH) >> + WATCH_INFO_LENGTH__SHIFT; + if (len == 0 || len > largest) + return; + + switch (n.n.type) { + case WATCH_TYPE_META: + got_meta(&n.n); + case WATCH_TYPE_KEY_NOTIFY: + saw_key_change(&n.n); + break; + } + + p += len; + } + } + } From e6bd91a7a45e750b471a24496a7c1a612c0ce899 Mon Sep 17 00:00:00 2001 From: Yixuan Cao Date: Sat, 9 Jul 2022 01:23:51 +0800 Subject: [PATCH 204/318] Documentation/translations/zh_CN/mm/page_owner.rst: adjust some words MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I noticed that there are some Chinese words that can be more accurate. So I fix them as follows. 首先，英文原文中的"release" 在这个语境下 是物理页面“释放”的意思，而不是“发布”。 其次，标准表的第一列和第二列， 表达的是“长短键”的意思，第一列是“短键”， 而第二列是“长键”。这样翻译或会更清晰一些。 Signed-off-by: Yixuan Cao Acked-by: Yanteng Si Link: https://lore.kernel.org/r/20220708172351.20928-1-caoyixuan2019@email.szu.edu.cn Signed-off-by: Jonathan Corbet --- Documentation/translations/zh_CN/vm/page_owner.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/translations/zh_CN/vm/page_owner.rst b/Documentation/translations/zh_CN/vm/page_owner.rst index 0034d2ae1cd5..7bd740bc5bf4 100644 --- a/Documentation/translations/zh_CN/vm/page_owner.rst +++ b/Documentation/translations/zh_CN/vm/page_owner.rst @@ -156,22 +156,22 @@ page owner在默认情况下是禁用的。所以，如果你想使用它，你 --sort的选项: - 键 键长 描述 + 短键 长键 描述 p pid 进程ID tg tgid 线程组ID n name 任务命令名称 st stacktrace 页面分配的堆栈跟踪 T txt 块的全文 - ft free_ts 页面发布时的时间戳 + ft free_ts 页面释放时的时间戳 at alloc_ts 页面被分配时的时间戳 ator allocator 页面的内存分配器 --curl的选项: - 键 键长 描述 + 短键 长键 描述 p pid 进程ID tg tgid 线程组ID n name 任务命令名称 - f free 该页是否已经发布 + f free 该页是否已经释放 st stacktrace 页面分配的堆栈跟踪 ator allocator 页面的内存分配器 From ec7c56812fa56b6726150877f0a1025fc78b44a4 Mon Sep 17 00:00:00 2001 From: Michael Kelley Date: Mon, 11 Jul 2022 10:48:22 -0700 Subject: [PATCH 205/318] Documentation: hyperv: Add overview of Hyper-V enlightenments Add an initial documentation topic for Linux enlightenments to run as a guest on Microsoft's Hyper-V hypervisor, linked under the "virt" documentation area. Update the virt doc index.rst and the MAINTAINERS file. Signed-off-by: Michael Kelley Link: https://lore.kernel.org/r/1657561704-12631-2-git-send-email-mikelley@microsoft.com Signed-off-by: Jonathan Corbet --- Documentation/virt/hyperv/index.rst | 10 ++ Documentation/virt/hyperv/overview.rst | 207 +++++++++++++++++++++++++ Documentation/virt/index.rst | 1 + MAINTAINERS | 1 + 4 files changed, 219 insertions(+) create mode 100644 Documentation/virt/hyperv/index.rst create mode 100644 Documentation/virt/hyperv/overview.rst diff --git a/Documentation/virt/hyperv/index.rst b/Documentation/virt/hyperv/index.rst new file mode 100644 index 000000000000..991bee4884b3 --- /dev/null +++ b/Documentation/virt/hyperv/index.rst @@ -0,0 +1,10 @@ +.. SPDX-License-Identifier: GPL-2.0 + +====================== +Hyper-V Enlightenments +====================== + +.. toctree:: + :maxdepth: 1 + + overview diff --git a/Documentation/virt/hyperv/overview.rst b/Documentation/virt/hyperv/overview.rst new file mode 100644 index 000000000000..cd493332c88a --- /dev/null +++ b/Documentation/virt/hyperv/overview.rst @@ -0,0 +1,207 @@ +.. SPDX-License-Identifier: GPL-2.0 + +Overview +======== +The Linux kernel contains a variety of code for running as a fully +enlightened guest on Microsoft's Hyper-V hypervisor. Hyper-V +consists primarily of a bare-metal hypervisor plus a virtual machine +management service running in the parent partition (roughly +equivalent to KVM and QEMU, for example). Guest VMs run in child +partitions. In this documentation, references to Hyper-V usually +encompass both the hypervisor and the VMM service without making a +distinction about which functionality is provided by which +component. + +Hyper-V runs on x86/x64 and arm64 architectures, and Linux guests +are supported on both. The functionality and behavior of Hyper-V is +generally the same on both architectures unless noted otherwise. + +Linux Guest Communication with Hyper-V +-------------------------------------- +Linux guests communicate with Hyper-V in four different ways: + +* Implicit traps: As defined by the x86/x64 or arm64 architecture, + some guest actions trap to Hyper-V. Hyper-V emulates the action and + returns control to the guest. This behavior is generally invisible + to the Linux kernel. + +* Explicit hypercalls: Linux makes an explicit function call to + Hyper-V, passing parameters. Hyper-V performs the requested action + and returns control to the caller. Parameters are passed in + processor registers or in memory shared between the Linux guest and + Hyper-V. On x86/x64, hypercalls use a Hyper-V specific calling + sequence. On arm64, hypercalls use the ARM standard SMCCC calling + sequence. + +* Synthetic register access: Hyper-V implements a variety of + synthetic registers. On x86/x64 these registers appear as MSRs in + the guest, and the Linux kernel can read or write these MSRs using + the normal mechanisms defined by the x86/x64 architecture. On + arm64, these synthetic registers must be accessed using explicit + hypercalls. + +* VMbus: VMbus is a higher-level software construct that is built on + the other 3 mechanisms. It is a message passing interface between + the Hyper-V host and the Linux guest. It uses memory that is shared + between Hyper-V and the guest, along with various signaling + mechanisms. + +The first three communication mechanisms are documented in the +`Hyper-V Top Level Functional Spec (TLFS)`_. The TLFS describes +general Hyper-V functionality and provides details on the hypercalls +and synthetic registers. The TLFS is currently written for the +x86/x64 architecture only. + +.. _Hyper-V Top Level Functional Spec (TLFS): https://docs.microsoft.com/en-us/virtualization/hyper-v-on-windows/tlfs/tlfs + +VMbus is not documented. This documentation provides a high-level +overview of VMbus and how it works, but the details can be discerned +only from the code. + +Sharing Memory +-------------- +Many aspects are communication between Hyper-V and Linux are based +on sharing memory. Such sharing is generally accomplished as +follows: + +* Linux allocates memory from its physical address space using + standard Linux mechanisms. + +* Linux tells Hyper-V the guest physical address (GPA) of the + allocated memory. Many shared areas are kept to 1 page so that a + single GPA is sufficient. Larger shared areas require a list of + GPAs, which usually do not need to be contiguous in the guest + physical address space. How Hyper-V is told about the GPA or list + of GPAs varies. In some cases, a single GPA is written to a + synthetic register. In other cases, a GPA or list of GPAs is sent + in a VMbus message. + +* Hyper-V translates the GPAs into "real" physical memory addresses, + and creates a virtual mapping that it can use to access the memory. + +* Linux can later revoke sharing it has previously established by + telling Hyper-V to set the shared GPA to zero. + +Hyper-V operates with a page size of 4 Kbytes. GPAs communicated to +Hyper-V may be in the form of page numbers, and always describe a +range of 4 Kbytes. Since the Linux guest page size on x86/x64 is +also 4 Kbytes, the mapping from guest page to Hyper-V page is 1-to-1. +On arm64, Hyper-V supports guests with 4/16/64 Kbyte pages as +defined by the arm64 architecture. If Linux is using 16 or 64 +Kbyte pages, Linux code must be careful to communicate with Hyper-V +only in terms of 4 Kbyte pages. HV_HYP_PAGE_SIZE and related macros +are used in code that communicates with Hyper-V so that it works +correctly in all configurations. + +As described in the TLFS, a few memory pages shared between Hyper-V +and the Linux guest are "overlay" pages. With overlay pages, Linux +uses the usual approach of allocating guest memory and telling +Hyper-V the GPA of the allocated memory. But Hyper-V then replaces +that physical memory page with a page it has allocated, and the +original physical memory page is no longer accessible in the guest +VM. Linux may access the memory normally as if it were the memory +that it originally allocated. The "overlay" behavior is visible +only because the contents of the page (as seen by Linux) change at +the time that Linux originally establishes the sharing and the +overlay page is inserted. Similarly, the contents change if Linux +revokes the sharing, in which case Hyper-V removes the overlay page, +and the guest page originally allocated by Linux becomes visible +again. + +Before Linux does a kexec to a kdump kernel or any other kernel, +memory shared with Hyper-V should be revoked. Hyper-V could modify +a shared page or remove an overlay page after the new kernel is +using the page for a different purpose, corrupting the new kernel. +Hyper-V does not provide a single "set everything" operation to +guest VMs, so Linux code must individually revoke all sharing before +doing kexec. See hv_kexec_handler() and hv_crash_handler(). But +the crash/panic path still has holes in cleanup because some shared +pages are set using per-CPU synthetic registers and there's no +mechanism to revoke the shared pages for CPUs other than the CPU +running the panic path. + +CPU Management +-------------- +Hyper-V does not have a ability to hot-add or hot-remove a CPU +from a running VM. However, Windows Server 2019 Hyper-V and +earlier versions may provide guests with ACPI tables that indicate +more CPUs than are actually present in the VM. As is normal, Linux +treats these additional CPUs as potential hot-add CPUs, and reports +them as such even though Hyper-V will never actually hot-add them. +Starting in Windows Server 2022 Hyper-V, the ACPI tables reflect +only the CPUs actually present in the VM, so Linux does not report +any hot-add CPUs. + +A Linux guest CPU may be taken offline using the normal Linux +mechanisms, provided no VMbus channel interrupts are assigned to +the CPU. See the section on VMbus Interrupts for more details +on how VMbus channel interrupts can be re-assigned to permit +taking a CPU offline. + +32-bit and 64-bit +----------------- +On x86/x64, Hyper-V supports 32-bit and 64-bit guests, and Linux +will build and run in either version. While the 32-bit version is +expected to work, it is used rarely and may suffer from undetected +regressions. + +On arm64, Hyper-V supports only 64-bit guests. + +Endian-ness +----------- +All communication between Hyper-V and guest VMs uses Little-Endian +format on both x86/x64 and arm64. Big-endian format on arm64 is not +supported by Hyper-V, and Linux code does not use endian-ness macros +when accessing data shared with Hyper-V. + +Versioning +---------- +Current Linux kernels operate correctly with older versions of +Hyper-V back to Windows Server 2012 Hyper-V. Support for running +on the original Hyper-V release in Windows Server 2008/2008 R2 +has been removed. + +A Linux guest on Hyper-V outputs in dmesg the version of Hyper-V +it is running on. This version is in the form of a Windows build +number and is for display purposes only. Linux code does not +test this version number at runtime to determine available features +and functionality. Hyper-V indicates feature/function availability +via flags in synthetic MSRs that Hyper-V provides to the guest, +and the guest code tests these flags. + +VMbus has its own protocol version that is negotiated during the +initial VMbus connection from the guest to Hyper-V. This version +number is also output to dmesg during boot. This version number +is checked in a few places in the code to determine if specific +functionality is present. + +Furthermore, each synthetic device on VMbus also has a protocol +version that is separate from the VMbus protocol version. Device +drivers for these synthetic devices typically negotiate the device +protocol version, and may test that protocol version to determine +if specific device functionality is present. + +Code Packaging +-------------- +Hyper-V related code appears in the Linux kernel code tree in three +main areas: + +1. drivers/hv + +2. arch/x86/hyperv and arch/arm64/hyperv + +3. individual device driver areas such as drivers/scsi, drivers/net, + drivers/clocksource, etc. + +A few miscellaneous files appear elsewhere. See the full list under +"Hyper-V/Azure CORE AND DRIVERS" and "DRM DRIVER FOR HYPERV +SYNTHETIC VIDEO DEVICE" in the MAINTAINERS file. + +The code in #1 and #2 is built only when CONFIG_HYPERV is set. +Similarly, the code for most Hyper-V related drivers is built only +when CONFIG_HYPERV is set. + +Most Hyper-V related code in #1 and #3 can be built as a module. +The architecture specific code in #2 must be built-in. Also, +drivers/hv/hv_common.c is low-level code that is common across +architectures and must be built-in. diff --git a/Documentation/virt/index.rst b/Documentation/virt/index.rst index 492f0920b988..2f1cffa87b1b 100644 --- a/Documentation/virt/index.rst +++ b/Documentation/virt/index.rst @@ -14,6 +14,7 @@ Linux Virtualization Support ne_overview acrn/index coco/sev-guest + hyperv/index .. only:: html and subproject diff --git a/MAINTAINERS b/MAINTAINERS index ee8824f2a8a8..6eecd51c93a2 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -9180,6 +9180,7 @@ S: Supported T: git git://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git F: Documentation/ABI/stable/sysfs-bus-vmbus F: Documentation/ABI/testing/debugfs-hyperv +F: Documentation/virt/hyperv F: Documentation/networking/device_drivers/ethernet/microsoft/netvsc.rst F: arch/arm64/hyperv F: arch/arm64/include/asm/hyperv-tlfs.h From ac1129e79e90a1dae495fdb2f4659bd1be90abb1 Mon Sep 17 00:00:00 2001 From: Michael Kelley Date: Mon, 11 Jul 2022 10:48:23 -0700 Subject: [PATCH 206/318] Documentation: hyperv: Add overview of VMbus Add documentation topic for using VMbus when running as a guest on Hyper-V. Signed-off-by: Michael Kelley Link: https://lore.kernel.org/r/1657561704-12631-3-git-send-email-mikelley@microsoft.com Signed-off-by: Jonathan Corbet --- Documentation/virt/hyperv/index.rst | 1 + Documentation/virt/hyperv/vmbus.rst | 303 ++++++++++++++++++++++++++++ 2 files changed, 304 insertions(+) create mode 100644 Documentation/virt/hyperv/vmbus.rst diff --git a/Documentation/virt/hyperv/index.rst b/Documentation/virt/hyperv/index.rst index 991bee4884b3..caa43ab3f85c 100644 --- a/Documentation/virt/hyperv/index.rst +++ b/Documentation/virt/hyperv/index.rst @@ -8,3 +8,4 @@ Hyper-V Enlightenments :maxdepth: 1 overview + vmbus diff --git a/Documentation/virt/hyperv/vmbus.rst b/Documentation/virt/hyperv/vmbus.rst new file mode 100644 index 000000000000..d2012d9022c5 --- /dev/null +++ b/Documentation/virt/hyperv/vmbus.rst @@ -0,0 +1,303 @@ +.. SPDX-License-Identifier: GPL-2.0 + +VMbus +===== +VMbus is a software construct provided by Hyper-V to guest VMs. It +consists of a control path and common facilities used by synthetic +devices that Hyper-V presents to guest VMs. The control path is +used to offer synthetic devices to the guest VM and, in some cases, +to rescind those devices. The common facilities include software +channels for communicating between the device driver in the guest VM +and the synthetic device implementation that is part of Hyper-V, and +signaling primitives to allow Hyper-V and the guest to interrupt +each other. + +VMbus is modeled in Linux as a bus, with the expected /sys/bus/vmbus +entry in a running Linux guest. The VMbus driver (drivers/hv/vmbus_drv.c) +establishes the VMbus control path with the Hyper-V host, then +registers itself as a Linux bus driver. It implements the standard +bus functions for adding and removing devices to/from the bus. + +Most synthetic devices offered by Hyper-V have a corresponding Linux +device driver. These devices include: + +* SCSI controller +* NIC +* Graphics frame buffer +* Keyboard +* Mouse +* PCI device pass-thru +* Heartbeat +* Time Sync +* Shutdown +* Memory balloon +* Key/Value Pair (KVP) exchange with Hyper-V +* Hyper-V online backup (a.k.a. VSS) + +Guest VMs may have multiple instances of the synthetic SCSI +controller, synthetic NIC, and PCI pass-thru devices. Other +synthetic devices are limited to a single instance per VM. Not +listed above are a small number of synthetic devices offered by +Hyper-V that are used only by Windows guests and for which Linux +does not have a driver. + +Hyper-V uses the terms "VSP" and "VSC" in describing synthetic +devices. "VSP" refers to the Hyper-V code that implements a +particular synthetic device, while "VSC" refers to the driver for +the device in the guest VM. For example, the Linux driver for the +synthetic NIC is referred to as "netvsc" and the Linux driver for +the synthetic SCSI controller is "storvsc". These drivers contain +functions with names like "storvsc_connect_to_vsp". + +VMbus channels +-------------- +An instance of a synthetic device uses VMbus channels to communicate +between the VSP and the VSC. Channels are bi-directional and used +for passing messages. Most synthetic devices use a single channel, +but the synthetic SCSI controller and synthetic NIC may use multiple +channels to achieve higher performance and greater parallelism. + +Each channel consists of two ring buffers. These are classic ring +buffers from a university data structures textbook. If the read +and writes pointers are equal, the ring buffer is considered to be +empty, so a full ring buffer always has at least one byte unused. +The "in" ring buffer is for messages from the Hyper-V host to the +guest, and the "out" ring buffer is for messages from the guest to +the Hyper-V host. In Linux, the "in" and "out" designations are as +viewed by the guest side. The ring buffers are memory that is +shared between the guest and the host, and they follow the standard +paradigm where the memory is allocated by the guest, with the list +of GPAs that make up the ring buffer communicated to the host. Each +ring buffer consists of a header page (4 Kbytes) with the read and +write indices and some control flags, followed by the memory for the +actual ring. The size of the ring is determined by the VSC in the +guest and is specific to each synthetic device. The list of GPAs +making up the ring is communicated to the Hyper-V host over the +VMbus control path as a GPA Descriptor List (GPADL). See function +vmbus_establish_gpadl(). + +Each ring buffer is mapped into contiguous Linux kernel virtual +space in three parts: 1) the 4 Kbyte header page, 2) the memory +that makes up the ring itself, and 3) a second mapping of the memory +that makes up the ring itself. Because (2) and (3) are contiguous +in kernel virtual space, the code that copies data to and from the +ring buffer need not be concerned with ring buffer wrap-around. +Once a copy operation has completed, the read or write index may +need to be reset to point back into the first mapping, but the +actual data copy does not need to be broken into two parts. This +approach also allows complex data structures to be easily accessed +directly in the ring without handling wrap-around. + +On arm64 with page sizes > 4 Kbytes, the header page must still be +passed to Hyper-V as a 4 Kbyte area. But the memory for the actual +ring must be aligned to PAGE_SIZE and have a size that is a multiple +of PAGE_SIZE so that the duplicate mapping trick can be done. Hence +a portion of the header page is unused and not communicated to +Hyper-V. This case is handled by vmbus_establish_gpadl(). + +Hyper-V enforces a limit on the aggregate amount of guest memory +that can be shared with the host via GPADLs. This limit ensures +that a rogue guest can't force the consumption of excessive host +resources. For Windows Server 2019 and later, this limit is +approximately 1280 Mbytes. For versions prior to Windows Server +2019, the limit is approximately 384 Mbytes. + +VMbus messages +-------------- +All VMbus messages have a standard header that includes the message +length, the offset of the message payload, some flags, and a +transactionID. The portion of the message after the header is +unique to each VSP/VSC pair. + +Messages follow one of two patterns: + +* Unidirectional: Either side sends a message and does not + expect a response message +* Request/response: One side (usually the guest) sends a message + and expects a response + +The transactionID (a.k.a. "requestID") is for matching requests & +responses. Some synthetic devices allow multiple requests to be in- +flight simultaneously, so the guest specifies a transactionID when +sending a request. Hyper-V sends back the same transactionID in the +matching response. + +Messages passed between the VSP and VSC are control messages. For +example, a message sent from the storvsc driver might be "execute +this SCSI command". If a message also implies some data transfer +between the guest and the Hyper-V host, the actual data to be +transferred may be embedded with the control message, or it may be +specified as a separate data buffer that the Hyper-V host will +access as a DMA operation. The former case is used when the size of +the data is small and the cost of copying the data to and from the +ring buffer is minimal. For example, time sync messages from the +Hyper-V host to the guest contain the actual time value. When the +data is larger, a separate data buffer is used. In this case, the +control message contains a list of GPAs that describe the data +buffer. For example, the storvsc driver uses this approach to +specify the data buffers to/from which disk I/O is done. + +Three functions exist to send VMbus messages: + +1. vmbus_sendpacket(): Control-only messages and messages with + embedded data -- no GPAs +2. vmbus_sendpacket_pagebuffer(): Message with list of GPAs + identifying data to transfer. An offset and length is + associated with each GPA so that multiple discontinuous areas + of guest memory can be targeted. +3. vmbus_sendpacket_mpb_desc(): Message with list of GPAs + identifying data to transfer. A single offset and length is + associated with a list of GPAs. The GPAs must describe a + single logical area of guest memory to be targeted. + +Historically, Linux guests have trusted Hyper-V to send well-formed +and valid messages, and Linux drivers for synthetic devices did not +fully validate messages. With the introduction of processor +technologies that fully encrypt guest memory and that allow the +guest to not trust the hypervisor (AMD SNP-SEV, Intel TDX), trusting +the Hyper-V host is no longer a valid assumption. The drivers for +VMbus synthetic devices are being updated to fully validate any +values read from memory that is shared with Hyper-V, which includes +messages from VMbus devices. To facilitate such validation, +messages read by the guest from the "in" ring buffer are copied to a +temporary buffer that is not shared with Hyper-V. Validation is +performed in this temporary buffer without the risk of Hyper-V +maliciously modifying the message after it is validated but before +it is used. + +VMbus interrupts +---------------- +VMbus provides a mechanism for the guest to interrupt the host when +the guest has queued new messages in a ring buffer. The host +expects that the guest will send an interrupt only when an "out" +ring buffer transitions from empty to non-empty. If the guest sends +interrupts at other times, the host deems such interrupts to be +unnecessary. If a guest sends an excessive number of unnecessary +interrupts, the host may throttle that guest by suspending its +execution for a few seconds to prevent a denial-of-service attack. + +Similarly, the host will interrupt the guest when it sends a new +message on the VMbus control path, or when a VMbus channel "in" ring +buffer transitions from empty to non-empty. Each CPU in the guest +may receive VMbus interrupts, so they are best modeled as per-CPU +interrupts in Linux. This model works well on arm64 where a single +per-CPU IRQ is allocated for VMbus. Since x86/x64 lacks support for +per-CPU IRQs, an x86 interrupt vector is statically allocated (see +HYPERVISOR_CALLBACK_VECTOR) across all CPUs and explicitly coded to +call the VMbus interrupt service routine. These interrupts are +visible in /proc/interrupts on the "HYP" line. + +The guest CPU that a VMbus channel will interrupt is selected by the +guest when the channel is created, and the host is informed of that +selection. VMbus devices are broadly grouped into two categories: + +1. "Slow" devices that need only one VMbus channel. The devices + (such as keyboard, mouse, heartbeat, and timesync) generate + relatively few interrupts. Their VMbus channels are all + assigned to interrupt the VMBUS_CONNECT_CPU, which is always + CPU 0. + +2. "High speed" devices that may use multiple VMbus channels for + higher parallelism and performance. These devices include the + synthetic SCSI controller and synthetic NIC. Their VMbus + channels interrupts are assigned to CPUs that are spread out + among the available CPUs in the VM so that interrupts on + multiple channels can be processed in parallel. + +The assignment of VMbus channel interrupts to CPUs is done in the +function init_vp_index(). This assignment is done outside of the +normal Linux interrupt affinity mechanism, so the interrupts are +neither "unmanaged" nor "managed" interrupts. + +The CPU that a VMbus channel will interrupt can be seen in +/sys/bus/vmbus/devices// channels//cpu. +When running on later versions of Hyper-V, the CPU can be changed +by writing a new value to this sysfs entry. Because the interrupt +assignment is done outside of the normal Linux affinity mechanism, +there are no entries in /proc/irq corresponding to individual +VMbus channel interrupts. + +An online CPU in a Linux guest may not be taken offline if it has +VMbus channel interrupts assigned to it. Any such channel +interrupts must first be manually reassigned to another CPU as +described above. When no channel interrupts are assigned to the +CPU, it can be taken offline. + +When a guest CPU receives a VMbus interrupt from the host, the +function vmbus_isr() handles the interrupt. It first checks for +channel interrupts by calling vmbus_chan_sched(), which looks at a +bitmap setup by the host to determine which channels have pending +interrupts on this CPU. If multiple channels have pending +interrupts for this CPU, they are processed sequentially. When all +channel interrupts have been processed, vmbus_isr() checks for and +processes any message received on the VMbus control path. + +The VMbus channel interrupt handling code is designed to work +correctly even if an interrupt is received on a CPU other than the +CPU assigned to the channel. Specifically, the code does not use +CPU-based exclusion for correctness. In normal operation, Hyper-V +will interrupt the assigned CPU. But when the CPU assigned to a +channel is being changed via sysfs, the guest doesn't know exactly +when Hyper-V will make the transition. The code must work correctly +even if there is a time lag before Hyper-V starts interrupting the +new CPU. See comments in target_cpu_store(). + +VMbus device creation/deletion +------------------------------ +Hyper-V and the Linux guest have a separate message-passing path +that is used for synthetic device creation and deletion. This +path does not use a VMbus channel. See vmbus_post_msg() and +vmbus_on_msg_dpc(). + +The first step is for the guest to connect to the generic +Hyper-V VMbus mechanism. As part of establishing this connection, +the guest and Hyper-V agree on a VMbus protocol version they will +use. This negotiation allows newer Linux kernels to run on older +Hyper-V versions, and vice versa. + +The guest then tells Hyper-V to "send offers". Hyper-V sends an +offer message to the guest for each synthetic device that the VM +is configured to have. Each VMbus device type has a fixed GUID +known as the "class ID", and each VMbus device instance is also +identified by a GUID. The offer message from Hyper-V contains +both GUIDs to uniquely (within the VM) identify the device. +There is one offer message for each device instance, so a VM with +two synthetic NICs will get two offers messages with the NIC +class ID. The ordering of offer messages can vary from boot-to-boot +and must not be assumed to be consistent in Linux code. Offer +messages may also arrive long after Linux has initially booted +because Hyper-V supports adding devices, such as synthetic NICs, +to running VMs. A new offer message is processed by +vmbus_process_offer(), which indirectly invokes vmbus_add_channel_work(). + +Upon receipt of an offer message, the guest identifies the device +type based on the class ID, and invokes the correct driver to set up +the device. Driver/device matching is performed using the standard +Linux mechanism. + +The device driver probe function opens the primary VMbus channel to +the corresponding VSP. It allocates guest memory for the channel +ring buffers and shares the ring buffer with the Hyper-V host by +giving the host a list of GPAs for the ring buffer memory. See +vmbus_establish_gpadl(). + +Once the ring buffer is set up, the device driver and VSP exchange +setup messages via the primary channel. These messages may include +negotiating the device protocol version to be used between the Linux +VSC and the VSP on the Hyper-V host. The setup messages may also +include creating additional VMbus channels, which are somewhat +mis-named as "sub-channels" since they are functionally +equivalent to the primary channel once they are created. + +Finally, the device driver may create entries in /dev as with +any device driver. + +The Hyper-V host can send a "rescind" message to the guest to +remove a device that was previously offered. Linux drivers must +handle such a rescind message at any time. Rescinding a device +invokes the device driver "remove" function to cleanly shut +down the device and remove it. Once a synthetic device is +rescinded, neither Hyper-V nor Linux retains any state about +its previous existence. Such a device might be re-added later, +in which case it is treated as an entirely new device. See +vmbus_onoffer_rescind(). From ab3e69fc4d693bb1cc6b3d5afcfabd88a17da9d4 Mon Sep 17 00:00:00 2001 From: Michael Kelley Date: Mon, 11 Jul 2022 10:48:24 -0700 Subject: [PATCH 207/318] Documentation: hyperv: Add overview of clocks and timers Add documentation topic for clocks and timers when running as a guest on Hyper-V. Signed-off-by: Michael Kelley Link: https://lore.kernel.org/r/1657561704-12631-4-git-send-email-mikelley@microsoft.com Signed-off-by: Jonathan Corbet --- Documentation/virt/hyperv/clocks.rst | 73 ++++++++++++++++++++++++++++ Documentation/virt/hyperv/index.rst | 1 + 2 files changed, 74 insertions(+) create mode 100644 Documentation/virt/hyperv/clocks.rst diff --git a/Documentation/virt/hyperv/clocks.rst b/Documentation/virt/hyperv/clocks.rst new file mode 100644 index 000000000000..2da2879fad52 --- /dev/null +++ b/Documentation/virt/hyperv/clocks.rst @@ -0,0 +1,73 @@ +.. SPDX-License-Identifier: GPL-2.0 + +Clocks and Timers +================= + +arm64 +----- +On arm64, Hyper-V virtualizes the ARMv8 architectural system counter +and timer. Guest VMs use this virtualized hardware as the Linux +clocksource and clockevents via the standard arm_arch_timer.c +driver, just as they would on bare metal. Linux vDSO support for the +architectural system counter is functional in guest VMs on Hyper-V. +While Hyper-V also provides a synthetic system clock and four synthetic +per-CPU timers as described in the TLFS, they are not used by the +Linux kernel in a Hyper-V guest on arm64. However, older versions +of Hyper-V for arm64 only partially virtualize the ARMv8 +architectural timer, such that the timer does not generate +interrupts in the VM. Because of this limitation, running current +Linux kernel versions on these older Hyper-V versions requires an +out-of-tree patch to use the Hyper-V synthetic clocks/timers instead. + +x86/x64 +------- +On x86/x64, Hyper-V provides guest VMs with a synthetic system clock +and four synthetic per-CPU timers as described in the TLFS. Hyper-V +also provides access to the virtualized TSC via the RDTSC and +related instructions. These TSC instructions do not trap to +the hypervisor and so provide excellent performance in a VM. +Hyper-V performs TSC calibration, and provides the TSC frequency +to the guest VM via a synthetic MSR. Hyper-V initialization code +in Linux reads this MSR to get the frequency, so it skips TSC +calibration and sets tsc_reliable. Hyper-V provides virtualized +versions of the PIT (in Hyper-V Generation 1 VMs only), local +APIC timer, and RTC. Hyper-V does not provide a virtualized HPET in +guest VMs. + +The Hyper-V synthetic system clock can be read via a synthetic MSR, +but this access traps to the hypervisor. As a faster alternative, +the guest can configure a memory page to be shared between the guest +and the hypervisor. Hyper-V populates this memory page with a +64-bit scale value and offset value. To read the synthetic clock +value, the guest reads the TSC and then applies the scale and offset +as described in the Hyper-V TLFS. The resulting value advances +at a constant 10 MHz frequency. In the case of a live migration +to a host with a different TSC frequency, Hyper-V adjusts the +scale and offset values in the shared page so that the 10 MHz +frequency is maintained. + +Starting with Windows Server 2022 Hyper-V, Hyper-V uses hardware +support for TSC frequency scaling to enable live migration of VMs +across Hyper-V hosts where the TSC frequency may be different. +When a Linux guest detects that this Hyper-V functionality is +available, it prefers to use Linux's standard TSC-based clocksource. +Otherwise, it uses the clocksource for the Hyper-V synthetic system +clock implemented via the shared page (identified as +"hyperv_clocksource_tsc_page"). + +The Hyper-V synthetic system clock is available to user space via +vDSO, and gettimeofday() and related system calls can execute +entirely in user space. The vDSO is implemented by mapping the +shared page with scale and offset values into user space. User +space code performs the same algorithm of reading the TSC and +appying the scale and offset to get the constant 10 MHz clock. + +Linux clockevents are based on Hyper-V synthetic timer 0. While +Hyper-V offers 4 synthetic timers for each CPU, Linux only uses +timer 0. Interrupts from stimer0 are recorded on the "HVS" line in +/proc/interrupts. Clockevents based on the virtualized PIT and +local APIC timer also work, but the Hyper-V synthetic timer is +preferred. + +The driver for the Hyper-V synthetic system clock and timers is +drivers/clocksource/hyperv_timer.c. diff --git a/Documentation/virt/hyperv/index.rst b/Documentation/virt/hyperv/index.rst index caa43ab3f85c..4a7a1b738bbe 100644 --- a/Documentation/virt/hyperv/index.rst +++ b/Documentation/virt/hyperv/index.rst @@ -9,3 +9,4 @@ Hyper-V Enlightenments overview vmbus + clocks From f8a291b7e35dcb3ea65c13f60cad7fb250b6b850 Mon Sep 17 00:00:00 2001 From: Wu XiangCheng Date: Thu, 7 Jul 2022 22:30:36 +0800 Subject: [PATCH 208/318] docs/zh_CN: Add a new translation of reporting-regressions.rst Last English version used: commit d2b40ba2cce2 ("docs: *-regressions.rst: explain how quickly issues should be handled") Signed-off-by: Wu XiangCheng Reviewed-by: Yanteng Si Link: https://lore.kernel.org/r/YsbuDGIpUjOzfAAh@bobwxc.mipc Signed-off-by: Jonathan Corbet --- .../translations/zh_CN/admin-guide/index.rst | 2 +- .../admin-guide/reporting-regressions.rst | 370 ++++++++++++++++++ 2 files changed, 371 insertions(+), 1 deletion(-) create mode 100644 Documentation/translations/zh_CN/admin-guide/reporting-regressions.rst diff --git a/Documentation/translations/zh_CN/admin-guide/index.rst b/Documentation/translations/zh_CN/admin-guide/index.rst index be535ffaf4b0..2f6970d0a032 100644 --- a/Documentation/translations/zh_CN/admin-guide/index.rst +++ b/Documentation/translations/zh_CN/admin-guide/index.rst @@ -36,6 +36,7 @@ Todolist: :maxdepth: 1 reporting-issues + reporting-regressions security-bugs bug-hunting bug-bisect @@ -44,7 +45,6 @@ Todolist: Todolist: -* reporting-bugs * ramoops * dynamic-debug-howto * kdump/index diff --git a/Documentation/translations/zh_CN/admin-guide/reporting-regressions.rst b/Documentation/translations/zh_CN/admin-guide/reporting-regressions.rst new file mode 100644 index 000000000000..c0461ee855bc --- /dev/null +++ b/Documentation/translations/zh_CN/admin-guide/reporting-regressions.rst @@ -0,0 +1,370 @@ +.. SPDX-License-Identifier: (GPL-2.0+ OR CC-BY-4.0) +.. 【重分发信息参见本文件结尾】 + +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/admin-guide/reporting-regressions.rst + +:译者: + + 吴想成 Wu XiangCheng + + +============ +报告回归问题 +============ + +“*我们拒绝出现回归*”是Linux内核开发的首要规则；Linux的发起者和领军开发者Linus +Torvalds立下了此规则并确保它被落实。 + +本文档描述了这条规则对用户的意义，以及Linux内核开发模型如何确保解决所有被报告 +的回归；关于内核开发者如何处理的方面参见 Documentation/process/handling-regressions.rst 。 + + +本文重点（亦即“太长不看”） +========================== + +#. 如果某程序在原先的Linux内核上运行良好，但在较新版本上效果更差、或者根本不 + 能用，那么你就碰见回归问题了。注意，新内核需要使用类似配置编译；更多相关细 + 节参见下方。 + +#. 按照 Documentation/translations/zh_CN/admin-guide/reporting-issues.rst 中 + 所说的报告你的问题，该文档已经包含了所有关于回归的重要方面，为了方便起见也 + 复制到了下面。两个重点：在报告主题中使用“[REGRESSION]”开头并抄送或转发到 + `回归邮件列表 `_ + （regressions@lists.linux.dev）。 + +#. 可选但是建议：在发送或转发报告时，指明该回归发生的起点，以便Linux内核回归 + 追踪机器人“regzbot”可以追踪此问题:: + + #regzbot introduced v5.13..v5.14-rc1 + + +与用户相关的所有Linux内核回归细节 +================================= + + +基本重点 +-------- + + +什么是“回归”以及什么是“无回归规则”？ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +如果某程序/实例在原先的Linux内核上运行良好，但在较新版本上效果更差、或者根本 +不能用，那么你就碰见回归问题了。“无回归规则”不允许出现这种情况。如果偶然发 +生了，导致问题的开发者应当迅速修复问题。 + +也就是说，若Linux 5.13中的WiFi驱动程序运行良好，但是在5.14版本上却不能用、速 +度明显变慢或出现错误，那就出现了回归。如果某正常工作的应用程序突然在新内核上 +出现不稳定，这也是回归；这些问题可能是由于procfs、sysfs或Linux提供给用户空间 +软件的许多其他接口之一的变化。但请记住，前述例子中的5.14需要使用类似于5.13的 +配置构建。这可以用 ``make olddefconfig`` 实现，详细解释见下。 + +注意本节第一句话中的“实例”：即使开发者需要遵循“无回归”规则，但仍可自由地改 +变内核的任何方面，甚至是导出到用户空间的API或ABI，只要别破坏现有的应用程序或 +用例。 + +还需注意，“无回归”规则只限制内核提供给用户空间的接口。它不适用于内核内部接 +口，比如一些外部开发的驱动程序用来插入钩子到内核的模块API。 + +如何报告回归？ +~~~~~~~~~~~~~~ + +只需按照 Documentation/translations/zh_CN/admin-guide/reporting-issues.rst 中 +所说的报告你的问题，该文档已经包含了要点。下面几点概述了一下只在回归中重要的 +方面： + + * 在检查可加入讨论的现有报告时，别忘了搜索 `Linux回归邮件列表 + `_ 和 `regzbot网页界面 + `_ 。 + + * 在报告主题的开头加上“[REGRESSION]”。 + + * 在你的报告中明确最后一个正常工作的内核版本和首个出问题的版本。如若可能， + 用二分法尝试找出导致回归的变更，更多细节见下。 + + * 记得把报告发到Linux回归邮件列表（regressions@lists.linux.dev）。 + + * 如果通过邮件报告回归，请抄送回归列表。 + + * 如果你使用某些缺陷追踪器报告回归，请通过邮件转发已提交的报告到回归列表， + 并抄送维护者以及出问题的相关子系统的邮件列表。 + + 如果是稳定版或长期支持版系列（如v5.15.3…v5.15.5）的回归，请记得抄送 + `Linux稳定版邮件列表 `_ （stable@vger.kernel.org）。 + + 如果你成功地执行了二分，请抄送肇事提交的信息中所有签了“Signed-off-by:”的人。 + +在抄送你的报告到列表时，也请记得通知前述的Linux内核回归追踪机器人。只需在邮件 +中包含如下片段:: + + #regzbot introduced: v5.13..v5.14-rc1 + +Regzbot会就将你的邮件视为在某个特定版本区间的回归报告。上例中即linux v5.13仍 +然正常，而Linux 5.14-rc1是首个您遇到问题的版本。如果你执行了二分以查找导致回 +归的提交，请使用指定肇事提交的id代替:: + + #regzbot introduced: 1f2e3d4c5d + +添加这样的“regzbot命令”对你是有好处的，它会确保报告不会被忽略。如果你省略了 +它，Linux内核的回归跟踪者会把你的回归告诉regzbot，只要你发送了一个副本到回归 +邮件列表。但是回归跟踪者只有一个人，有时不得不休息或甚至偶尔享受可以远离电脑 +的时光（听起来很疯狂）。因此，依赖此人手动将回归添加到 `已追踪且尚未解决的 +Linux内核回归列表 `_ 和 +regzbot发送的每周回归报告，可能会出现延迟。 这样的延误会导致Linus Torvalds +在决定“继续开发还是发布新版本？”时忽略严重的回归。 + +真的修复了所有的回归吗？ +~~~~~~~~~~~~~~~~~~~~~~~~ + +几乎所有都是，只要引起问题的变更（肇事提交）被可靠定位。也有些回归可以不用这 +样，但通常是必须的。 + +谁需要找出回归的根本原因？ +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +受影响代码区域的开发者应该自行尝试定位问题所在。但仅靠他们的努力往往是不可 +能做到的，很多问题只发生在开发者的无法接触的其他特定外部环境中——例如特定的 +硬件平台、固件、Linux发行版、系统的配置或应用程序。这就是为什么最终往往是报 +告者定位肇事提交；有时用户甚至需要再运行额外测试以查明确切的根本原因。开发 +者应该提供建议和可能的帮助，以使普通用户更容易完成该流程。 + +如何找到罪魁祸首？ +~~~~~~~~~~~~~~~~~~ + +如 Documentation/translations/zh_CN/admin-guide/reporting-issues.rst （简要） +和 Documentation/translations/zh_CN/admin-guide/bug-bisect.rst （详细）中所 +述，执行二分。听起来工作量很大，但大部分情况下很快就能找到罪魁祸首。如果这很 +困难或可靠地重现问题很耗时，请考虑与其他受影响的用户合作，一起缩小搜索范围。 + +当出现回归时我可以向谁寻求建议？ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +发送邮件到回归邮件列表（regressions@lists.linux.dev）同时抄送Linux内核的回归 +跟踪者（regressions@leemhuis.info）;如果问题需要保密处理，可以省略列表。 + + +关于回归的更多细节 +------------------ + + +“无回归规则”的目标是什么？ +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +用户应该放心升级内核版本，而不必担心有程序可能崩溃。这符合内核开发者的利益， +可以使更新有吸引力：他们不希望用户停留在停止维护或超过一年半的稳定/长期Linux +版本系列上。这也符合所有人的利益，因为 `那些系列可能含有已知的缺陷、安全问题 +或其他后续版本已经修复的问题 +`_ 。 +此外，内核开发者希望使用户测试最新的预发行版或常规发行版变得简单而有吸引力。 +这同样符合所有人的利益，如果新版本出来后很快就有相关报告，会使追踪和修复问题 +更容易。 + +实际中“无回归”规则真的可行吗？ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +这不是句玩笑话，请见Linux创建者和主要开发人员Linus Torvalds在邮件列表中的许 +多发言，其中一些在 Documentation/process/handling-regressions.rst 中被引用。 + +此规则的例外情况极为罕见；之前当开发者认为某个特定的情况有必要援引例外时， +基本都被证明错了。 + +谁来确保“无回归”被落实？ +~~~~~~~~~~~~~~~~~~~~~~~~ + +照看和支撑树的子系统维护者应该关心这一点——例如，Linus Torvalds之于主线， +Greg Kroah-Hartman等人之于各种稳定/长期系列。 + +他们都得到了别人的帮助，以确保回归报告不会被遗漏。其中之一是Thorsten +Leemhuis，他目前担任Linux内核的“回归跟踪者”；为了做好这项工作，他使用了 +regzbot——Linux内核回归跟踪机器人。所以这就是为什么要抄送或转发你的报告到 +回归邮件列表来通知这些人，已经最好在你的邮件中包含“regzbot命令”来立即追踪它。 + +回归通常多久能修复？ +~~~~~~~~~~~~~~~~~~~~ + +开发者应该尽快修复任何被报告的回归，以提供及时为受影响的用户提供解决方案，并 +防止更多用户遇到问题；然而，开发人员需要花足够的时间和注意力确保回归修复不会 +造成额外的损害。 + +因此，答案取决于各种因素，如回归的影响、存在时长或出现于哪个Linux版本系列。 +但最终，大多数的回归应该在两周内修复。 + +当问题可以通过升级某些软件解决时，是回归吗？ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +基本都是。如果开发人员告诉您其他情况，请咨询上述回归跟踪者。 + +当新内核变慢或能耗增加，是回归吗？ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +是的，但有一些差别。在微型基准测试中变慢5%不太可能被视为回归，除非它也会对 +广泛基准测试的结果产生超过1%的影响。如果有疑问，请寻求建议。 + +当更新Linux时外部内核模块崩溃了，是回归吗？ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +不，因为“无回归”规则仅限于Linux内核提供给用户空间的接口和服务。因此，它不包括 +构建或运行外部开发的内核模块，因为它们在内核空间中运行与挂进内核使用的内部接 +口偶尔会变化。 + +如何处理安全修复引起的回归？ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +在极为罕见的情况下，安全问题无法在不引起回归的情况下修复；这些修复都被放弃了， +因为它们终究会引起问题。幸运的是这种两难境地基本都可以避免，受影响区域的主要 +开发者以及Linus Torvalds本人通常都会努力在不引入回归的情况下解决安全问题。 + +如果你仍然面临此种情况，请查看邮件列表档案是否有人尽力避免过回归。如果没有， +请报告它；如有疑问，请如上所述寻求建议。 + +当修复回归时不可避免会引入另一个，如何处理？ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +很遗憾这种事确实会出现，但幸运的是并不经常出现；如果发生了，受影响代码区的资 +深开发者应当调查该问题以找到避免回归的解决方法，至少避免它们的影响。如果你遇 +到这样的情况，如上所述：检查之前的讨论是否有人已经尽了最大努力，如有疑问请寻 +求建议。 + +小提示：如果人们在每个开发周期中定期给出主线预发布（即v5.15-rc1或-rc3）以供 +测试，则可以避免这种情况。为了更好地解释，可以设想一个在Linux v5.14和v5.15-rc1 +之间集成的更改，该更改导致了回归，但同时是应用于5.15-rc1的其他改进的强依赖。 +如果有人在5.15发布之前就发现并报告了这个问题，那么所有更改都可以直接撤销，从 +而解决回归问题。而就在几天或几周后，此解决方案变成了不可能，因为一些软件可能 +已经开始依赖于后续更改之一：撤销所有更改将导致上述用户软件出现回归，这是不可 +接受的。 + +若我所依赖的功能在数月前被移除了，是回归吗？ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +是的，但如前节所述，通常很难修复此类回归。因此需要逐案处理。这也是定期测试主 +线预发布对所有人有好处的另一个原因。 + +如果我似乎是唯一受影响的人，是否仍适用“无回归”规则？ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +适用，但仅限于实际使用：Linux开发人员希望能够自由地取消那些只能在阁楼和博物 +馆中找到的硬件的支持。 + +请注意，有时为了取得进展，不得不出现回归——后者也是防止Linux停滞不前所必需 +的。因此如果回归所影响的用户很少，那么为了他们和其他人更大的利益，还是让事情 +过去吧。尤其是存在某种规避回归的简单方法，例如更新一些软件或者使用专门为此目 +的创建的内核参数。 + +回归规则是否也适用于staging树中的代码？ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +不，参见 `适用于所有staging代码配置选项的帮助文本 +`_ ， +其早已声明:: + + 请注意：这些驱动正在积极开发中，可能无法正常工作，并可能包含会在不久的 + 将来发生变化的用户接口。 + +虽然staging开发人员通常坚持“无回归”的原则，但有时为了取得进展也会违背它。这就 +是为什么当staging树的WiFi驱动被基本推倒重来时，有些用户不得不处理回归（通常可 +以忽略）。 + +为什么较新版本必须“使用相似配置编译”？ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +因为Linux内核开发人员有时会集成已知的会导致回归的变更，但使它们成为可选的，并 +在内核的默认配置下禁用它们。这一技巧允许进步，否则“无回归”规则将导致停滞。 + +例如，试想一个新的可以阻止恶意软件滥用某个内核的接口的安全特性，同时又需要满足 +另一个很罕见的应用程序。上述的方法可使两方都满意：使用这些应用程序的人可以关闭 +新的安全功能，而其他不会遇到麻烦的人可以启用它。 + +如何创建与旧内核相似的配置？ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +用一个已知良好的内核启动机器，并用 ``make olddefconfig`` 配置新版的Linux。这 +会让内核的构建脚本从正在运行的内核中摘录配置文件（“.config”文件），作为即将编 +译的新版本的基础配置；同时将所有新的配置选项设为默认值，以禁用可能导致回归的 +新功能。 + +如何报告在预编译的普通内核中发现的回归？ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +您需要确保新的内核是用与旧版相似的配置编译（见上文），因为那些构建它们的人可 +能启用了一些已知的与新内核不兼容的特性。如有疑问，请向内核的提供者报告问题并 +寻求建议。 + + +用“regzbot”追踪回归的更多信息 +----------------------------- + +什么是回归追踪？为啥我需要关心它？ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +像“无回归”这样的规则需要有人来确保它们被遵守，否则会被有意/无意打破。历史证 +明了这一点对于Linux内核开发也适用。这就是为什么Linux内核的回归跟踪者Thorsten +Leemhuis，，和另一些人尽力关注所有的回归直到他们解决。他们从未为此获得报酬， +因此这项工作是在尽最大努力的基础上完成的。 + +为什么/如何使用机器人追踪Linux内核回归？ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +由于Linux内核开发过程的分布式和松散结构，完全手动跟踪回归已经被证明是相当困难 +的。因此Linux内核的回归跟踪者开发了regzbot来促进这项工作，其长期目标是尽可能为 +所有相关人员自动化回归跟踪。 + +Regzbot通过监视跟踪的回归报告的回复来工作。此外，它还查找用“Link:”标签引用这 +些报告的补丁；对这些补丁的回复也会被跟踪。结合这些数据，可以很好地了解当前修 +复过程的状态。 + +如何查看regzbot当前追踪的回归？ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +参见 `regzbot在线 `_ 。 + +何种问题可以由regzbot追踪？ +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +该机器人只为了跟踪回归，因此请不要让regzbot涉及常规问题。但是对于Linux内核的 +回归跟踪者来说，让regzbot跟踪严重问题也可以，如有关挂起、损坏数据或内部错误 +（Panic、Oops、BUG（）、warning…）的报告。 + +如何修改被追踪回归的相关信息？ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +在直接或间接回复报告邮件时使用“regzbot命令”即可。最简单的方法是：在“已发送”文 +件夹或邮件列表存档中找到报告，然后使用邮件客户端的“全部回复”功能对其进行回复。 +在该邮件中的独立段落中可使用以下命令之一（即使用空行将这些命令中的一个或多个与 +其余邮件文本分隔开）。 + + * 更新回归引入起点，例如在执行二分之后:: + + #regzbot introduced: 1f2e3d4c5d + + * 设置或更新标题:: + + #regzbot title: foo + + * 监视讨论或bugzilla.kernel.org上有关讨论或修复的工单:: + + #regzbot monitor: https://lore.kernel.org/r/30th.anniversary.repost@klaava.Helsinki.FI/ + #regzbot monitor: https://bugzilla.kernel.org/show_bug.cgi?id=123456789 + + * 标记一个有更多相关细节的地方，例如有关但主题不同的邮件列表帖子或缺陷追踪器中的工单:: + + #regzbot link: https://bugzilla.kernel.org/show_bug.cgi?id=123456789 + + * 标记回归已失效:: + + #regzbot invalid: wasn't a regression, problem has always existed + +Regzbot还支持其他一些主要由开发人员或回归追踪人员使用的命令。命令的更多细节请 +参考 `入门指南 `_ +和 `参考手册 `_ 。 + +.. + 正文结束 +.. + 如本文件开头所述，本文以GPL-2.0+或CC-BY-4.0许可发行。如您想仅在CC-BY-4.0许 + 可下重分发本文，请用“Linux内核开发者”作为作者，并用如下链接作为来源： + https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/plain/Documentation/translations/zh_CN/admin-guide/reporting-regressions.rst +.. + 注意：本RST文件内容只有在来自Linux内核源代码时是使用CC-BY-4.0许可的，因为经 + 过处理的版本（如经内核的构建系统）可能包含来自使用更严格许可证的文件的内容。 From a4c174ca8d8f747b7c5e82f64a2c5ddca67cd1ac Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Mon, 4 Jul 2022 14:25:27 +0200 Subject: [PATCH 209/318] docs: kernel-docs: order reference from newest to oldest The documents on each section of this document are ordered by its published date, from the newest to the oldest. In the kernel-docs.rst, the references on each section of this document are intended to be ordered by its published date, from the newest to the oldest. The Linux Kernel Module Programming Guide was published in 2021; so, it is placed at the top as the most recent publication after the rolling-version "Linux Kernel Mailing List Glossary" reference. Fixes: 630c8fa02f9a ("Documentation: Update details of The Linux Kernel Module Programming Guide") Signed-off-by: Lukas Bulwahn Link: https://lore.kernel.org/r/20220704122537.3407-2-lukas.bulwahn@gmail.com Signed-off-by: Jonathan Corbet --- Documentation/process/kernel-docs.rst | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Documentation/process/kernel-docs.rst b/Documentation/process/kernel-docs.rst index da9527502ef0..b4d98f6f797a 100644 --- a/Documentation/process/kernel-docs.rst +++ b/Documentation/process/kernel-docs.rst @@ -83,6 +83,18 @@ On-line docs Finally this trace-log is used as base for more a exact conceptual exploration and description of the Linux TCP/IP implementation.* + * Title: **The Linux Kernel Module Programming Guide** + + :Author: Peter Jay Salzman, Michael Burian, Ori Pomerantz, Bob Mottram, + Jim Huang. + :URL: https://sysprog21.github.io/lkmpg/ + :Date: 2021 + :Keywords: modules, GPL book, /proc, ioctls, system calls, + interrupt handlers . + :Description: A very nice GPL book on the topic of modules + programming. Lots of examples. Currently the new version is being + actively maintained at https://github.com/sysprog21/lkmpg. + * Title: **On submitting kernel Patches** :Author: Andi Kleen @@ -126,18 +138,6 @@ On-line docs describes how to write user-mode utilities for communicating with Card Services. - * Title: **The Linux Kernel Module Programming Guide** - - :Author: Peter Jay Salzman, Michael Burian, Ori Pomerantz, Bob Mottram, - Jim Huang. - :URL: https://sysprog21.github.io/lkmpg/ - :Date: 2021 - :Keywords: modules, GPL book, /proc, ioctls, system calls, - interrupt handlers . - :Description: A very nice GPL book on the topic of modules - programming. Lots of examples. Currently the new version is being - actively maintained at https://github.com/sysprog21/lkmpg. - * Title: **Global spinlock list and usage** :Author: Rick Lindsley. From 615041d42a1a03a024d0013390961d105bf2e05e Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Mon, 4 Jul 2022 14:25:28 +0200 Subject: [PATCH 210/318] docs: kernel-docs: shorten the lengthy doc title The original title comes from copying the content from a web page that covered various mixed computer-science material. Within the kernel documentation and its current structure, the title can be shortened. Other titles considered, but not selected were: - Index of More Kernel Documentation - Further Kernel Documentation - References to Further Kernel Documentation Shorten the title. Signed-off-by: Lukas Bulwahn Link: https://lore.kernel.org/r/20220704122537.3407-3-lukas.bulwahn@gmail.com Signed-off-by: Jonathan Corbet --- Documentation/process/kernel-docs.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/process/kernel-docs.rst b/Documentation/process/kernel-docs.rst index b4d98f6f797a..5d6fa71895cc 100644 --- a/Documentation/process/kernel-docs.rst +++ b/Documentation/process/kernel-docs.rst @@ -1,7 +1,7 @@ .. _kernel_docs: -Index of Documentation for People Interested in Writing and/or Understanding the Linux Kernel -============================================================================================= +Index of Further Kernel Documentation +===================================== Juan-Mariano de Goyeneche From f46b4b168cdb3b16ae8c7792c28d0b921db1932b Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Mon, 4 Jul 2022 14:25:29 +0200 Subject: [PATCH 211/318] docs: kernel-docs: reflect that it is community-maintained Remove and rephrase statements that only make sense if a single author exclusively would maintain this document, but we would really want to consider this being a page maintained by the kernel community, as it is placed in the kernel repository, and let us hope that more contributors suggest some more documents. Further, do some minor word-smithing. Signed-off-by: Lukas Bulwahn Link: https://lore.kernel.org/r/20220704122537.3407-4-lukas.bulwahn@gmail.com Signed-off-by: Jonathan Corbet --- Documentation/process/kernel-docs.rst | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/Documentation/process/kernel-docs.rst b/Documentation/process/kernel-docs.rst index 5d6fa71895cc..b2523267ffc7 100644 --- a/Documentation/process/kernel-docs.rst +++ b/Documentation/process/kernel-docs.rst @@ -3,7 +3,8 @@ Index of Further Kernel Documentation ===================================== - Juan-Mariano de Goyeneche +Initial Author: Juan-Mariano de Goyeneche (; +email address is defunct now.) The need for a document like this one became apparent in the linux-kernel mailing list as the same questions, asking for pointers @@ -16,21 +17,16 @@ philosophy and design decisions behind this code. Unfortunately, not many documents are available for beginners to start. And, even if they exist, there was no "well-known" place which -kept track of them. These lines try to cover this lack. All documents -available on line known by the author are listed, while some reference -books are also mentioned. +kept track of them. These lines try to cover this lack. PLEASE, if you know any paper not listed here or write a new document, -send me an e-mail, and I'll include a reference to it here. Any -corrections, ideas or comments are also welcomed. +include a reference to it here, following the kernel's patch submission +process. Any corrections, ideas or comments are also welcome. -The papers that follow are listed in no particular order. All are -cataloged with the following fields: the document's "Title", the -"Author"/s, the "URL" where they can be found, some "Keywords" helpful -when searching for specific topics, and a brief "Description" of the -Document. - -Enjoy! +All documents are cataloged with the following fields: the document's +"Title", the "Author"/s, the "URL" where they can be found, some +"Keywords" helpful when searching for specific topics, and a brief +"Description" of the Document. .. note:: From 6c568f6a42ab77e5a299ea152efc89f38d88f334 Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Mon, 4 Jul 2022 14:25:30 +0200 Subject: [PATCH 212/318] docs: kernel-docs: add a reference mentioned in submitting-drivers.rst One section in submitting-drivers.rst was just a collection of references to other external documentation. All except the one added in this commit is already mentioned in kernel-docs or other places in the kernel documentation. Add Arjan van de Ven's article on How to NOT write kernel driver to this index of further kernel documentation. Signed-off-by: Lukas Bulwahn Link: https://lore.kernel.org/r/20220704122537.3407-5-lukas.bulwahn@gmail.com Signed-off-by: Jonathan Corbet --- Documentation/process/kernel-docs.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Documentation/process/kernel-docs.rst b/Documentation/process/kernel-docs.rst index b2523267ffc7..502289d63385 100644 --- a/Documentation/process/kernel-docs.rst +++ b/Documentation/process/kernel-docs.rst @@ -134,6 +134,20 @@ On-line docs describes how to write user-mode utilities for communicating with Card Services. + * Title: **How NOT to write kernel drivers** + + :Author: Arjan van de Ven. + :URL: https://landley.net/kdocs/ols/2002/ols2002-pages-545-555.pdf + :Date: 2002 + :Keywords: driver. + :Description: Programming bugs and Do-nots in kernel driver development + :Abstract: *Quit a few tutorials, articles and books give an introduction + on how to write Linux kernel drivers. Unfortunately the things one + should NOT do in Linux kernel code is either only a minor appendix + or, more commonly, completely absent. This paper tries to briefly touch + the areas in which the most common and serious bugs and do-nots are + encountered.* + * Title: **Global spinlock list and usage** :Author: Rick Lindsley. From 3cb5e516869e2f53b8fc5732f9db140fe539d53d Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Mon, 4 Jul 2022 14:25:31 +0200 Subject: [PATCH 213/318] docs: admin: devices: drop confusing outdated statement on Latex The statement that the Latex version of Linux Device List is unmaintained, must go back to some historic time where there were actually two separate document sources, one in a txt format and another in a tex format (maybe even just on lanana.org). Nowadays, html and tex are generated from the ReST document file and the statement might be confused, believing the actual generated LaTeX document from the maintained ReST document could be an unmaintained one. Remove this statement on the LaTeX version and only keep pointing out that the version on lanana.org is no longer maintained. Signed-off-by: Lukas Bulwahn Link: https://lore.kernel.org/r/20220704122537.3407-6-lukas.bulwahn@gmail.com Signed-off-by: Jonathan Corbet --- Documentation/admin-guide/devices.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Documentation/admin-guide/devices.rst b/Documentation/admin-guide/devices.rst index 035275fedbdd..e3776d77374b 100644 --- a/Documentation/admin-guide/devices.rst +++ b/Documentation/admin-guide/devices.rst @@ -7,10 +7,9 @@ This list is the Linux Device List, the official registry of allocated device numbers and ``/dev`` directory nodes for the Linux operating system. -The LaTeX version of this document is no longer maintained, nor is -the document that used to reside at lanana.org. This version in the -mainline Linux kernel is the master document. Updates shall be sent -as patches to the kernel maintainers (see the +The version of this document at lanana.org is no longer maintained. This +version in the mainline Linux kernel is the master document. Updates +shall be sent as patches to the kernel maintainers (see the :ref:`Documentation/process/submitting-patches.rst ` document). Specifically explore the sections titled "CHAR and MISC DRIVERS", and "BLOCK LAYER" in the MAINTAINERS file to find the right maintainers From 9db370de278087321840ea904d877803a78fba71 Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Mon, 4 Jul 2022 14:25:32 +0200 Subject: [PATCH 214/318] docs: process: remove outdated submitting-drivers.rst Commit 31b24bee3357 ("docs: add a warning to submitting-drivers.rst") in October 2016 already warns "This (...) should maybe just be deleted, but I'm not quite ready to do that yet". Maybe, six years ago, we were not ready but let us remove old content for the better now and structure and maintain less content in the kernel documentation with a better result. Drop this already outdated document and adjust all textual references. Here is an argument why deleting the content will not remove any useful information to the existing kernel documentation, individually broken down for each section. Section "Allocating Device Numbers" refers to https://www.lanana.org/, and then refers to Documentation/admin-guide/devices.rst. However, the devices.rst clearly states: "The version of this document at lanana.org is no longer maintained." Everything needed for submitting drivers is already stated in devices.rst and the reference to https://www.lanana.org/ is outdated, and should be just deleted. Section "Who To Submit Drivers To" is all about Linux 2.0 - 2.6, before the new release version scheme; the mentioned developers are still around, but actually not the first developers to contact anymore. Section "What Criteria Determine Acceptance" has a few bullet points: Licensing and Copyright is well-covered in process/kernel-license.rst. Interfaces, Code, Portability, Clarity state some obvious things about ensuring kernel code quality. Control suggests to add a MAINTAINERS entry, which is already mentioned in 6.Followthrough.rst: "... added yourself to the MAINTAINERS file..." PM support states a bit about implementing and testing power management of a driver, it remains an open question where to place that in the process documents. Driver developers interested in power management will find the corresponding part on power management in the kernel documentation anyway. In section "What Criteria Do Not Determine Acceptance", the points Vendor and Author states something basic consequence of the kernel being an open-source community software development. Probably no need to mention it nowadays. Section "Resources" lists resources that are also mentioned elsewhere more central. - Linux kernel tree and mailing list is mentioned in many places. - https://lwn.net/Kernel/LDD3/ is mentioned in Documentation/process/kernel-docs.rst. - https://lwn.net/ is mentioned in: - Documentation/process/8.Conclusion.rst - Documentation/process/kernel-docs.rst - https://kernelnewbies.org/ is mentioned in: - Documentation/process/8.Conclusion.rst - Documentation/process/kernel-docs.rst - http://www.linux-usb.org/ is mentioned in Documentation/driver-api/usb/usb.rst - https://landley.net/kdocs/ols/2002/ols2002-pages-545-555.pdf is mentioned in Documentation/process/kernel-docs.rst - https://kernelnewbies.org/KernelJanitors is mentioned in Documentation/process/howto.rst - https://git-scm.com/ is mentioned in - Documentation/process/2.Process.rst - Documentation/process/7.AdvancedTopics.rst - Documentation/process/howto.rst Signed-off-by: Lukas Bulwahn Link: https://lore.kernel.org/r/20220704122537.3407-7-lukas.bulwahn@gmail.com Signed-off-by: Jonathan Corbet --- Documentation/hwmon/submitting-patches.rst | 1 - Documentation/kernel-hacking/hacking.rst | 3 +- Documentation/process/5.Posting.rst | 3 +- Documentation/process/8.Conclusion.rst | 16 +- Documentation/process/howto.rst | 4 +- Documentation/process/index.rst | 1 - Documentation/process/submitting-drivers.rst | 194 ------------------- Documentation/process/submitting-patches.rst | 5 +- 8 files changed, 13 insertions(+), 214 deletions(-) delete mode 100644 Documentation/process/submitting-drivers.rst diff --git a/Documentation/hwmon/submitting-patches.rst b/Documentation/hwmon/submitting-patches.rst index 9a218ea996d8..d953ee763c96 100644 --- a/Documentation/hwmon/submitting-patches.rst +++ b/Documentation/hwmon/submitting-patches.rst @@ -12,7 +12,6 @@ increase the chances of your change being accepted. * It should be unnecessary to mention, but please read and follow: - Documentation/process/submit-checklist.rst - - Documentation/process/submitting-drivers.rst - Documentation/process/submitting-patches.rst - Documentation/process/coding-style.rst diff --git a/Documentation/kernel-hacking/hacking.rst b/Documentation/kernel-hacking/hacking.rst index ebd9d90882ea..9a1f020c8449 100644 --- a/Documentation/kernel-hacking/hacking.rst +++ b/Documentation/kernel-hacking/hacking.rst @@ -755,8 +755,7 @@ make a neat patch, there's administrative work to be done: it implies a more-than-passing commitment to some part of the code. - Finally, don't forget to read - ``Documentation/process/submitting-patches.rst`` and possibly - ``Documentation/process/submitting-drivers.rst``. + ``Documentation/process/submitting-patches.rst`` Kernel Cantrips =============== diff --git a/Documentation/process/5.Posting.rst b/Documentation/process/5.Posting.rst index bd36ecb29409..906235c11c24 100644 --- a/Documentation/process/5.Posting.rst +++ b/Documentation/process/5.Posting.rst @@ -10,8 +10,7 @@ of conventions and procedures which are used in the posting of patches; following them will make life much easier for everybody involved. This document will attempt to cover these expectations in reasonable detail; more information can also be found in the files -:ref:`Documentation/process/submitting-patches.rst `, -:ref:`Documentation/process/submitting-drivers.rst ` +:ref:`Documentation/process/submitting-patches.rst ` and :ref:`Documentation/process/submit-checklist.rst `. diff --git a/Documentation/process/8.Conclusion.rst b/Documentation/process/8.Conclusion.rst index b32a40215858..8c847dffe76b 100644 --- a/Documentation/process/8.Conclusion.rst +++ b/Documentation/process/8.Conclusion.rst @@ -5,15 +5,13 @@ For more information There are numerous sources of information on Linux kernel development and related topics. First among those will always be the Documentation -directory found in the kernel source distribution. The top-level :ref:`process/howto.rst ` -file is an important starting point; :ref:`process/submitting-patches.rst ` -and :ref:`process/submitting-drivers.rst ` -are also something which all kernel developers should -read. Many internal kernel APIs are documented using the kerneldoc -mechanism; "make htmldocs" or "make pdfdocs" can be used to generate those -documents in HTML or PDF format (though the version of TeX shipped by some -distributions runs into internal limits and fails to process the documents -properly). +directory found in the kernel source distribution. Start with the +top-level :ref:`process/howto.rst `; also read +:ref:`process/submitting-patches.rst `. Many internal +kernel APIs are documented using the kerneldoc mechanism; "make htmldocs" +or "make pdfdocs" can be used to generate those documents in HTML or PDF +format (though the version of TeX shipped by some distributions runs into +internal limits and fails to process the documents properly). Various web sites discuss kernel development at all levels of detail. Your author would like to humbly suggest https://lwn.net/ as a source; diff --git a/Documentation/process/howto.rst b/Documentation/process/howto.rst index e4beeca57e5f..cd6997a9d203 100644 --- a/Documentation/process/howto.rst +++ b/Documentation/process/howto.rst @@ -105,8 +105,8 @@ required reading: patches if these rules are followed, and many people will only review code if it is in the proper style. - :ref:`Documentation/process/submitting-patches.rst ` and :ref:`Documentation/process/submitting-drivers.rst ` - These files describe in explicit detail how to successfully create + :ref:`Documentation/process/submitting-patches.rst ` + This file describes in explicit detail how to successfully create and send a patch, including (but not limited to): - Email contents diff --git a/Documentation/process/index.rst b/Documentation/process/index.rst index 3587dae4d0ef..2ba2a1582bbe 100644 --- a/Documentation/process/index.rst +++ b/Documentation/process/index.rst @@ -40,7 +40,6 @@ Other guides to the community that are of interest to most developers are: :maxdepth: 1 changes - submitting-drivers stable-api-nonsense management-style stable-kernel-rules diff --git a/Documentation/process/submitting-drivers.rst b/Documentation/process/submitting-drivers.rst deleted file mode 100644 index 8413b693d10d..000000000000 --- a/Documentation/process/submitting-drivers.rst +++ /dev/null @@ -1,194 +0,0 @@ -.. _submittingdrivers: - -Submitting Drivers For The Linux Kernel -======================================= - -This document is intended to explain how to submit device drivers to the -various kernel trees. Note that if you are interested in video card drivers -you should probably talk to XFree86 (https://www.xfree86.org/) and/or X.Org -(https://x.org/) instead. - -.. note:: - - This document is old and has seen little maintenance in recent years; it - should probably be updated or, perhaps better, just deleted. Most of - what is here can be found in the other development documents anyway. - - Oh, and we don't really recommend submitting changes to XFree86 :) - -Also read the :ref:`Documentation/process/submitting-patches.rst ` -document. - - -Allocating Device Numbers -------------------------- - -Major and minor numbers for block and character devices are allocated -by the Linux assigned name and number authority (currently this is -Torben Mathiasen). The site is https://www.lanana.org/. This -also deals with allocating numbers for devices that are not going to -be submitted to the mainstream kernel. -See :ref:`Documentation/admin-guide/devices.rst ` -for more information on this. - -If you don't use assigned numbers then when your device is submitted it will -be given an assigned number even if that is different from values you may -have shipped to customers before. - -Who To Submit Drivers To ------------------------- - -Linux 2.0: - No new drivers are accepted for this kernel tree. - -Linux 2.2: - No new drivers are accepted for this kernel tree. - -Linux 2.4: - If the code area has a general maintainer then please submit it to - the maintainer listed in MAINTAINERS in the kernel file. If the - maintainer does not respond or you cannot find the appropriate - maintainer then please contact Willy Tarreau . - -Linux 2.6 and upper: - The same rules apply as 2.4 except that you should follow linux-kernel - to track changes in API's. The final contact point for Linux 2.6+ - submissions is Andrew Morton. - -What Criteria Determine Acceptance ----------------------------------- - -Licensing: - The code must be released to us under the - GNU General Public License. If you wish the driver to be - useful to other communities such as BSD you may release - under multiple licenses. If you choose to release under - licenses other than the GPL, you should include your - rationale for your license choices in your cover letter. - See accepted licenses at include/linux/module.h - -Copyright: - The copyright owner must agree to use of GPL. - It's best if the submitter and copyright owner - are the same person/entity. If not, the name of - the person/entity authorizing use of GPL should be - listed in case it's necessary to verify the will of - the copyright owner. - -Interfaces: - If your driver uses existing interfaces and behaves like - other drivers in the same class it will be much more likely - to be accepted than if it invents gratuitous new ones. - If you need to implement a common API over Linux and NT - drivers do it in userspace. - -Code: - Please use the Linux style of code formatting as documented - in :ref:`Documentation/process/coding-style.rst `. - If you have sections of code - that need to be in other formats, for example because they - are shared with a windows driver kit and you want to - maintain them just once separate them out nicely and note - this fact. - -Portability: - Pointers are not always 32bits, not all computers are little - endian, people do not all have floating point and you - shouldn't use inline x86 assembler in your driver without - careful thought. Pure x86 drivers generally are not popular. - If you only have x86 hardware it is hard to test portability - but it is easy to make sure the code can easily be made - portable. - -Clarity: - It helps if anyone can see how to fix the driver. It helps - you because you get patches not bug reports. If you submit a - driver that intentionally obfuscates how the hardware works - it will go in the bitbucket. - -PM support: - Since Linux is used on many portable and desktop systems, your - driver is likely to be used on such a system and therefore it - should support basic power management by implementing, if - necessary, the .suspend and .resume methods used during the - system-wide suspend and resume transitions. You should verify - that your driver correctly handles the suspend and resume, but - if you are unable to ensure that, please at least define the - .suspend method returning the -ENOSYS ("Function not - implemented") error. You should also try to make sure that your - driver uses as little power as possible when it's not doing - anything. For the driver testing instructions see - Documentation/power/drivers-testing.rst and for a relatively - complete overview of the power management issues related to - drivers see :ref:`Documentation/driver-api/pm/devices.rst `. - -Control: - In general if there is active maintenance of a driver by - the author then patches will be redirected to them unless - they are totally obvious and without need of checking. - If you want to be the contact and update point for the - driver it is a good idea to state this in the comments, - and include an entry in MAINTAINERS for your driver. - -What Criteria Do Not Determine Acceptance ------------------------------------------ - -Vendor: - Being the hardware vendor and maintaining the driver is - often a good thing. If there is a stable working driver from - other people already in the tree don't expect 'we are the - vendor' to get your driver chosen. Ideally work with the - existing driver author to build a single perfect driver. - -Author: - It doesn't matter if a large Linux company wrote the driver, - or you did. Nobody has any special access to the kernel - tree. Anyone who tells you otherwise isn't telling the - whole story. - - -Resources ---------- - -Linux kernel master tree: - ftp.\ *country_code*\ .kernel.org:/pub/linux/kernel/... - - where *country_code* == your country code, such as - **us**, **uk**, **fr**, etc. - - https://git.kernel.org/?p=linux/kernel/git/torvalds/linux.git - -Linux kernel mailing list: - linux-kernel@vger.kernel.org - [mail majordomo@vger.kernel.org to subscribe] - -Linux Device Drivers, Third Edition (covers 2.6.10): - https://lwn.net/Kernel/LDD3/ (free version) - -LWN.net: - Weekly summary of kernel development activity - https://lwn.net/ - - 2.6 API changes: - - https://lwn.net/Articles/2.6-kernel-api/ - - Porting drivers from prior kernels to 2.6: - - https://lwn.net/Articles/driver-porting/ - -KernelNewbies: - Documentation and assistance for new kernel programmers - - https://kernelnewbies.org/ - -Linux USB project: - http://www.linux-usb.org/ - -How to NOT write kernel driver by Arjan van de Ven: - https://landley.net/kdocs/ols/2002/ols2002-pages-545-555.pdf - -Kernel Janitor: - https://kernelnewbies.org/KernelJanitors - -GIT, Fast Version Control System: - https://git-scm.com/ diff --git a/Documentation/process/submitting-patches.rst b/Documentation/process/submitting-patches.rst index a1cb6280fbcf..be49d8f2601b 100644 --- a/Documentation/process/submitting-patches.rst +++ b/Documentation/process/submitting-patches.rst @@ -12,9 +12,8 @@ This document contains a large number of suggestions in a relatively terse format. For detailed information on how the kernel development process works, see Documentation/process/development-process.rst. Also, read Documentation/process/submit-checklist.rst -for a list of items to check before submitting code. If you are submitting -a driver, also read Documentation/process/submitting-drivers.rst; for device -tree binding patches, read +for a list of items to check before submitting code. +For device tree binding patches, read Documentation/devicetree/bindings/submitting-patches.rst. This documentation assumes that you're using ``git`` to prepare your patches. From 26f200c9e56fefbcd7d302db1019b955ea20bc98 Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Mon, 4 Jul 2022 14:25:33 +0200 Subject: [PATCH 215/318] docs: it_IT: align to submitting-drivers removal Adjust the Italian translation to the removal of submitting-drivers in the English kernel documentation, with some assistance from Frederico Vaga. Link: https://lore.kernel.org/linux-doc/20220628091647.xf5cygybqemwprgb@numero86.vaga.pv.it/ Signed-off-by: Lukas Bulwahn Link: https://lore.kernel.org/r/20220704122537.3407-8-lukas.bulwahn@gmail.com Signed-off-by: Jonathan Corbet --- .../it_IT/kernel-hacking/hacking.rst | 3 +-- .../translations/it_IT/process/5.Posting.rst | 5 ++--- .../translations/it_IT/process/8.Conclusion.rst | 5 ++--- .../translations/it_IT/process/howto.rst | 3 +-- .../translations/it_IT/process/index.rst | 1 - .../it_IT/process/submitting-drivers.rst | 16 ---------------- .../it_IT/process/submitting-patches.rst | 6 ++---- 7 files changed, 8 insertions(+), 31 deletions(-) delete mode 100644 Documentation/translations/it_IT/process/submitting-drivers.rst diff --git a/Documentation/translations/it_IT/kernel-hacking/hacking.rst b/Documentation/translations/it_IT/kernel-hacking/hacking.rst index d5c521327f6a..4bec4669cf48 100644 --- a/Documentation/translations/it_IT/kernel-hacking/hacking.rst +++ b/Documentation/translations/it_IT/kernel-hacking/hacking.rst @@ -795,8 +795,7 @@ anche per avere patch pulite, c'è del lavoro amministrativo da fare: di un semplice impegno su una parte del codice. - Infine, non dimenticatevi di leggere - ``Documentation/process/submitting-patches.rst`` e possibilmente anche - ``Documentation/process/submitting-drivers.rst``. + ``Documentation/process/submitting-patches.rst``. Trucchetti del kernel ===================== diff --git a/Documentation/translations/it_IT/process/5.Posting.rst b/Documentation/translations/it_IT/process/5.Posting.rst index 1476d51eb5e5..a036f38fc82e 100644 --- a/Documentation/translations/it_IT/process/5.Posting.rst +++ b/Documentation/translations/it_IT/process/5.Posting.rst @@ -16,9 +16,8 @@ e di procedure per la pubblicazione delle patch; seguirle renderà la vita più facile a tutti quanti. Questo documento cercherà di coprire questi argomenti con un ragionevole livello di dettaglio; più informazioni possono essere trovare nella cartella 'Documentation', nei file -:ref:`translations/it_IT/process/submitting-patches.rst `, -:ref:`translations/it_IT/process/submitting-drivers.rst `, e -:ref:`translations/it_IT/process/submit-checklist.rst `. +:ref:`translations/it_IT/process/submitting-patches.rst ` +e :ref:`translations/it_IT/process/submit-checklist.rst `. Quando pubblicarle diff --git a/Documentation/translations/it_IT/process/8.Conclusion.rst b/Documentation/translations/it_IT/process/8.Conclusion.rst index 039bfc5a4108..32659ff467c0 100644 --- a/Documentation/translations/it_IT/process/8.Conclusion.rst +++ b/Documentation/translations/it_IT/process/8.Conclusion.rst @@ -13,9 +13,8 @@ e argomenti correlati. Primo tra questi sarà sempre la cartella Documentation che si trova nei sorgenti kernel. Il file :ref:`process/howto.rst ` è un punto di partenza -importante; :ref:`process/submitting-patches.rst ` e -:ref:`process/submitting-drivers.rst ` sono -anch'essi qualcosa che tutti gli sviluppatori del kernel dovrebbero leggere. +importante; :ref:`process/submitting-patches.rst ` è +anch'esso qualcosa che tutti gli sviluppatori del kernel dovrebbero leggere. Molte API interne al kernel sono documentate utilizzando il meccanismo kerneldoc; "make htmldocs" o "make pdfdocs" possono essere usati per generare quei documenti in HTML o PDF (sebbene le versioni di TeX di alcune diff --git a/Documentation/translations/it_IT/process/howto.rst b/Documentation/translations/it_IT/process/howto.rst index 9554368a2ae2..16ad5622d549 100644 --- a/Documentation/translations/it_IT/process/howto.rst +++ b/Documentation/translations/it_IT/process/howto.rst @@ -109,8 +109,7 @@ Di seguito una lista di file che sono presenti nei sorgente del kernel e che accetteranno patch solo se queste osserveranno tali regole, e molte persone revisioneranno il codice solo se scritto nello stile appropriato. - :ref:`Documentation/translations/it_IT/process/submitting-patches.rst ` e - :ref:`Documentation/translations/it_IT/process/submitting-drivers.rst ` + :ref:`Documentation/translations/it_IT/process/submitting-patches.rst ` Questo file descrive dettagliatamente come creare ed inviare una patch con successo, includendo (ma non solo questo): diff --git a/Documentation/translations/it_IT/process/index.rst b/Documentation/translations/it_IT/process/index.rst index c4c867132c88..b223b70a4a95 100644 --- a/Documentation/translations/it_IT/process/index.rst +++ b/Documentation/translations/it_IT/process/index.rst @@ -41,7 +41,6 @@ degli sviluppatori: :maxdepth: 1 changes - submitting-drivers stable-api-nonsense management-style stable-kernel-rules diff --git a/Documentation/translations/it_IT/process/submitting-drivers.rst b/Documentation/translations/it_IT/process/submitting-drivers.rst deleted file mode 100644 index dadd77e47613..000000000000 --- a/Documentation/translations/it_IT/process/submitting-drivers.rst +++ /dev/null @@ -1,16 +0,0 @@ -.. include:: ../disclaimer-ita.rst - -:Original: :ref:`Documentation/process/submitting-drivers.rst ` -:Translator: Federico Vaga - -.. _it_submittingdrivers: - -Sottomettere driver per il kernel Linux -======================================= - -.. note:: - - Questo documento è vecchio e negli ultimi anni non è stato più aggiornato; - dovrebbe essere aggiornato, o forse meglio, rimosso. La maggior parte di - quello che viene detto qui può essere trovato anche negli altri documenti - dedicati allo sviluppo. Per questo motivo il documento non verrà tradotto. diff --git a/Documentation/translations/it_IT/process/submitting-patches.rst b/Documentation/translations/it_IT/process/submitting-patches.rst index 4fb5b3aa306d..f117ee0a36e2 100644 --- a/Documentation/translations/it_IT/process/submitting-patches.rst +++ b/Documentation/translations/it_IT/process/submitting-patches.rst @@ -18,10 +18,8 @@ Questo documento contiene un vasto numero di suggerimenti concisi. Per maggiori dettagli su come funziona il processo di sviluppo del kernel leggete Documentation/translations/it_IT/process/development-process.rst. Leggete anche Documentation/translations/it_IT/process/submit-checklist.rst per una lista di -punti da verificare prima di inviare del codice. Se state inviando un driver, -allora leggete anche -Documentation/translations/it_IT/process/submitting-drivers.rst; per delle patch -relative alle associazioni per Device Tree leggete +punti da verificare prima di inviare del codice. +Per delle patch relative alle associazioni per Device Tree leggete Documentation/translations/it_IT/process/submitting-patches.rst. Questa documentazione assume che sappiate usare ``git`` per preparare le patch. From aa3b2a7ffae17b708e926e2cd148c08a3d189fe1 Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Mon, 4 Jul 2022 14:25:34 +0200 Subject: [PATCH 216/318] docs: ja_JP: howto: remove reference to removed submitting-drivers The document submitting-drivers.rst was deleted. This removes the corresponding reference in the Japanese translation of the howto, with some assistance from Akira Yokasawa. Link: https://lore.kernel.org/linux-doc/a2be48e5-e559-17d7-5ae7-d1205a737ea4@gmail.com/ Signed-off-by: Lukas Bulwahn Reviewed-by: Akira Yokosawa Link: https://lore.kernel.org/r/20220704122537.3407-9-lukas.bulwahn@gmail.com Signed-off-by: Jonathan Corbet --- Documentation/translations/ja_JP/howto.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/translations/ja_JP/howto.rst b/Documentation/translations/ja_JP/howto.rst index 38fed6fe62fe..649e2ff2a407 100644 --- a/Documentation/translations/ja_JP/howto.rst +++ b/Documentation/translations/ja_JP/howto.rst @@ -129,8 +129,8 @@ linux-api@vger.kernel.org に送ることを勧めます。 ルに従っているものだけを受け付け、多くの人は正しいスタイルのコード だけをレビューします。 - :ref:`Documentation/process/submitting-patches.rst ` と :ref:`Documentation/process/submitting-drivers.rst ` - これらのファイルには、どうやってうまくパッチを作って投稿するかにつ + :ref:`Documentation/process/submitting-patches.rst ` + このファイルには、どうやってうまくパッチを作って投稿するかにつ いて非常に詳しく書かれており、以下を含みます (これだけに限らない けれども) From 481bb592d300493a920319e03c0396098119eddf Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Mon, 4 Jul 2022 14:25:35 +0200 Subject: [PATCH 217/318] docs: ko_KR: howto: remove reference to removed submitting-drivers The document submitting-drivers.rst was deleted. This removes the corresponding reference in the Korean translation of the howto. Signed-off-by: Lukas Bulwahn Link: https://lore.kernel.org/r/20220704122537.3407-10-lukas.bulwahn@gmail.com Signed-off-by: Jonathan Corbet --- Documentation/translations/ko_KR/howto.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/translations/ko_KR/howto.rst b/Documentation/translations/ko_KR/howto.rst index e3cdf0c84892..e43970584ca4 100644 --- a/Documentation/translations/ko_KR/howto.rst +++ b/Documentation/translations/ko_KR/howto.rst @@ -124,7 +124,7 @@ mtk.manpages@gmail.com의 메인테이너에게 보낼 것을 권장한다. 메인테이너들은 이 규칙을 따르는 패치들만을 받아들일 것이고 많은 사람들이 그 패치가 올바른 스타일일 경우만 코드를 검토할 것이다. - :ref:`Documentation/process/submitting-patches.rst ` 와 :ref:`Documentation/process/submitting-drivers.rst ` + :ref:`Documentation/process/submitting-patches.rst ` 이 파일들은 성공적으로 패치를 만들고 보내는 법을 다음의 내용들로 굉장히 상세히 설명하고 있다(그러나 다음으로 한정되진 않는다). From 9d4e2eed3f72268ba87af22cf87c9c3bccb8c3c4 Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Mon, 4 Jul 2022 14:25:36 +0200 Subject: [PATCH 218/318] docs: zh_CN: align to submitting-drivers removal Adjust the Chinese translation to the removal of submitting-drivers in the English kernel documentation. Signed-off-by: Lukas Bulwahn Reviewed-by: Alex Shi Link: https://lore.kernel.org/r/20220704122537.3407-11-lukas.bulwahn@gmail.com Signed-off-by: Jonathan Corbet --- .../zh_CN/kernel-hacking/hacking.rst | 3 +- .../translations/zh_CN/process/5.Posting.rst | 3 +- .../translations/zh_CN/process/howto.rst | 1 - .../translations/zh_CN/process/index.rst | 1 - .../zh_CN/process/submitting-drivers.rst | 160 ------------------ 5 files changed, 2 insertions(+), 166 deletions(-) delete mode 100644 Documentation/translations/zh_CN/process/submitting-drivers.rst diff --git a/Documentation/translations/zh_CN/kernel-hacking/hacking.rst b/Documentation/translations/zh_CN/kernel-hacking/hacking.rst index bda79646bb1e..9112949b993c 100644 --- a/Documentation/translations/zh_CN/kernel-hacking/hacking.rst +++ b/Documentation/translations/zh_CN/kernel-hacking/hacking.rst @@ -633,8 +633,7 @@ C++ 意味着您希望在对子系统进行更改时得到询问，并了解缺陷；这意味着对某部分 代码做出更多承诺。 -- 最后，别忘记去阅读 Documentation/process/submitting-patches.rst ， - 也许还有 Documentation/process/submitting-drivers.rst 。 +- 最后，别忘记去阅读 Documentation/process/submitting-patches.rst。 Kernel 仙女棒 =============== diff --git a/Documentation/translations/zh_CN/process/5.Posting.rst b/Documentation/translations/zh_CN/process/5.Posting.rst index 4ee7de13f373..6a469e1c7deb 100644 --- a/Documentation/translations/zh_CN/process/5.Posting.rst +++ b/Documentation/translations/zh_CN/process/5.Posting.rst @@ -19,8 +19,7 @@ 内核开发社区已经发展出一套用于发布补丁的约定和过程；遵循这些约定和过程将使 参与其中的每个人的生活更加轻松。本文档试图描述这些约定的部分细节；更多信息 也可在以下文档中找到 -:ref:`Documentation/translations/zh_CN/process/submitting-patches.rst `， -:ref:`Documentation/translations/zh_CN/process/submitting-drivers.rst ` +:ref:`Documentation/translations/zh_CN/process/submitting-patches.rst ` 和 :ref:`Documentation/translations/zh_CN/process/submit-checklist.rst `。 何时寄送 diff --git a/Documentation/translations/zh_CN/process/howto.rst b/Documentation/translations/zh_CN/process/howto.rst index 1334cdb32a3c..1455190dc087 100644 --- a/Documentation/translations/zh_CN/process/howto.rst +++ b/Documentation/translations/zh_CN/process/howto.rst @@ -96,7 +96,6 @@ Linux内核代码中包含有大量的文档。这些文档对于学习如何与 的代码。 :ref:`Documentation/translations/zh_CN/process/submitting-patches.rst ` - :ref:`Documentation/process/submitting-drivers.rst ` 这两份文档明确描述如何创建和发送补丁，其中包括（但不仅限于)： - 邮件内容 diff --git a/Documentation/translations/zh_CN/process/index.rst b/Documentation/translations/zh_CN/process/index.rst index 39e9c88fbaa6..a683dbea0c83 100644 --- a/Documentation/translations/zh_CN/process/index.rst +++ b/Documentation/translations/zh_CN/process/index.rst @@ -40,7 +40,6 @@ .. toctree:: :maxdepth: 1 - submitting-drivers submit-checklist stable-api-nonsense stable-kernel-rules diff --git a/Documentation/translations/zh_CN/process/submitting-drivers.rst b/Documentation/translations/zh_CN/process/submitting-drivers.rst deleted file mode 100644 index 98341e7cd812..000000000000 --- a/Documentation/translations/zh_CN/process/submitting-drivers.rst +++ /dev/null @@ -1,160 +0,0 @@ -.. _cn_submittingdrivers: - -.. include:: ../disclaimer-zh_CN.rst - -:Original: :ref:`Documentation/process/submitting-drivers.rst - ` - -如果想评论或更新本文的内容，请直接联系原文档的维护者。如果你使用英文 -交流有困难的话，也可以向中文版维护者求助。如果本翻译更新不及时或者翻 -译存在问题，请联系中文版维护者:: - - 中文版维护者： 李阳 Li Yang - 中文版翻译者： 李阳 Li Yang - 中文版校译者： 陈琦 Maggie Chen - 王聪 Wang Cong - 张巍 Zhang Wei - -如何向 Linux 内核提交驱动程序 -============================= - -这篇文档将会解释如何向不同的内核源码树提交设备驱动程序。请注意，如果你感 -兴趣的是显卡驱动程序，你也许应该访问 XFree86 项目(https://www.xfree86.org/) -和／或 X.org 项目 (https://x.org)。 - -另请参阅 Documentation/translations/zh_CN/process/submitting-patches.rst 文档。 - - -分配设备号 ----------- - -块设备和字符设备的主设备号与从设备号是由 Linux 命名编号分配权威 LANANA（ -现在是 Torben Mathiasen）负责分配。申请的网址是 https://www.lanana.org/。 -即使不准备提交到主流内核的设备驱动也需要在这里分配设备号。有关详细信息， -请参阅 Documentation/admin-guide/devices.rst。 - -如果你使用的不是已经分配的设备号，那么当你提交设备驱动的时候，它将会被强 -制分配一个新的设备号，即便这个设备号和你之前发给客户的截然不同。 - -设备驱动的提交对象 ------------------- - -Linux 2.0: - 此内核源码树不接受新的驱动程序。 - -Linux 2.2: - 此内核源码树不接受新的驱动程序。 - -Linux 2.4: - 如果所属的代码领域在内核的 MAINTAINERS 文件中列有一个总维护者， - 那么请将驱动程序提交给他。如果此维护者没有回应或者你找不到恰当的 - 维护者，那么请联系 Willy Tarreau 。 - -Linux 2.6: - 除了遵循和 2.4 版内核同样的规则外，你还需要在 linux-kernel 邮件 - 列表上跟踪最新的 API 变化。向 Linux 2.6 内核提交驱动的顶级联系人 - 是 Andrew Morton 。 - -决定设备驱动能否被接受的条件 ----------------------------- - -许可： 代码必须使用 GNU 通用公开许可证 (GPL) 提交给 Linux，但是 - 我们并不要求 GPL 是唯一的许可。你或许会希望同时使用多种 - 许可证发布，如果希望驱动程序可以被其他开源社区（比如BSD） - 使用。请参考 include/linux/module.h 文件中所列出的可被 - 接受共存的许可。 - -版权： 版权所有者必须同意使用 GPL 许可。最好提交者和版权所有者 - 是相同个人或实体。否则，必需列出授权使用 GPL 的版权所有 - 人或实体，以备验证之需。 - -接口： 如果你的驱动程序使用现成的接口并且和其他同类的驱动程序行 - 为相似，而不是去发明无谓的新接口，那么它将会更容易被接受。 - 如果你需要一个 Linux 和 NT 的通用驱动接口，那么请在用 - 户空间实现它。 - -代码： 请使用 Documentation/process/coding-style.rst 中所描述的 Linux 代码风 - 格。如果你的某些代码段（例如那些与 Windows 驱动程序包共 - 享的代码段）需要使用其他格式，而你却只希望维护一份代码， - 那么请将它们很好地区分出来，并且注明原因。 - -可移植性： 请注意，指针并不永远是 32 位的，不是所有的计算机都使用小 - 尾模式 (little endian) 存储数据，不是所有的人都拥有浮点 - 单元，不要随便在你的驱动程序里嵌入 x86 汇编指令。只能在 - x86 上运行的驱动程序一般是不受欢迎的。虽然你可能只有 x86 - 硬件，很难测试驱动程序在其他平台上是否可用，但是确保代码 - 可以被轻松地移植却是很简单的。 - -清晰度： 做到所有人都能修补这个驱动程序将会很有好处，因为这样你将 - 会直接收到修复的补丁而不是 bug 报告。如果你提交一个试图 - 隐藏硬件工作机理的驱动程序，那么它将会被扔进废纸篓。 - -电源管理： 因为 Linux 正在被很多移动设备和桌面系统使用，所以你的驱 - 动程序也很有可能被使用在这些设备上。它应该支持最基本的电 - 源管理，即在需要的情况下实现系统级休眠和唤醒要用到的 - .suspend 和 .resume 函数。你应该检查你的驱动程序是否能正 - 确地处理休眠与唤醒，如果实在无法确认，请至少把 .suspend - 函数定义成返回 -ENOSYS（功能未实现）错误。你还应该尝试确 - 保你的驱动在什么都不干的情况下将耗电降到最低。要获得驱动 - 程序测试的指导，请参阅 - Documentation/power/drivers-testing.rst。有关驱动程序电 - 源管理问题相对全面的概述，请参阅 - Documentation/driver-api/pm/devices.rst。 - -管理： 如果一个驱动程序的作者还在进行有效的维护，那么通常除了那 - 些明显正确且不需要任何检查的补丁以外，其他所有的补丁都会 - 被转发给作者。如果你希望成为驱动程序的联系人和更新者，最 - 好在代码注释中写明并且在 MAINTAINERS 文件中加入这个驱动 - 程序的条目。 - -不影响设备驱动能否被接受的条件 ------------------------------- - -供应商： 由硬件供应商来维护驱动程序通常是一件好事。不过，如果源码 - 树里已经有其他人提供了可稳定工作的驱动程序，那么请不要期 - 望“我是供应商”会成为内核改用你的驱动程序的理由。理想的情 - 况是：供应商与现有驱动程序的作者合作，构建一个统一完美的 - 驱动程序。 - -作者： 驱动程序是由大的 Linux 公司研发还是由你个人编写，并不影 - 响其是否能被内核接受。没有人对内核源码树享有特权。只要你 - 充分了解内核社区，你就会发现这一点。 - - -资源列表 --------- - -Linux 内核主源码树： - ftp.??.kernel.org:/pub/linux/kernel/... - ?? == 你的国家代码，例如 "cn"、"us"、"uk"、"fr" 等等 - -Linux 内核邮件列表： - linux-kernel@vger.kernel.org - [可通过向majordomo@vger.kernel.org发邮件来订阅] - -Linux 设备驱动程序，第三版（探讨 2.6.10 版内核）： - https://lwn.net/Kernel/LDD3/ （免费版） - -LWN.net: - 每周内核开发活动摘要 - https://lwn.net/ - - 2.6 版中 API 的变更： - - https://lwn.net/Articles/2.6-kernel-api/ - - 将旧版内核的驱动程序移植到 2.6 版： - - https://lwn.net/Articles/driver-porting/ - -内核新手(KernelNewbies): - 为新的内核开发者提供文档和帮助 - https://kernelnewbies.org/ - -Linux USB项目： - http://www.linux-usb.org/ - -写内核驱动的“不要”（Arjan van de Ven著）: - http://www.fenrus.org/how-to-not-write-a-device-driver-paper.pdf - -内核清洁工 (Kernel Janitor): - https://kernelnewbies.org/KernelJanitors From 374e4e6f7c77bf742e5383e787cc302d655e6016 Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Mon, 4 Jul 2022 14:25:37 +0200 Subject: [PATCH 219/318] docs: zh_TW: align to submitting-drivers removal Adjust the Taiwanese translation to the removal of submitting-drivers in the English kernel documentation. Signed-off-by: Lukas Bulwahn Link: https://lore.kernel.org/r/20220704122537.3407-12-lukas.bulwahn@gmail.com Signed-off-by: Jonathan Corbet --- .../translations/zh_TW/process/5.Posting.rst | 3 +- .../translations/zh_TW/process/howto.rst | 1 - .../translations/zh_TW/process/index.rst | 1 - .../zh_TW/process/submitting-drivers.rst | 164 ------------------ 4 files changed, 1 insertion(+), 168 deletions(-) delete mode 100644 Documentation/translations/zh_TW/process/submitting-drivers.rst diff --git a/Documentation/translations/zh_TW/process/5.Posting.rst b/Documentation/translations/zh_TW/process/5.Posting.rst index 5578bca403e6..280a8832ecc0 100644 --- a/Documentation/translations/zh_TW/process/5.Posting.rst +++ b/Documentation/translations/zh_TW/process/5.Posting.rst @@ -22,8 +22,7 @@ 內核開發社區已經發展出一套用於發布補丁的約定和過程；遵循這些約定和過程將使 參與其中的每個人的生活更加輕鬆。本文檔試圖描述這些約定的部分細節；更多信息 也可在以下文檔中找到 -:ref:`Documentation/translations/zh_TW/process/submitting-patches.rst `， -:ref:`Documentation/translations/zh_TW/process/submitting-drivers.rst ` +:ref:`Documentation/translations/zh_TW/process/submitting-patches.rst ` 和 :ref:`Documentation/translations/zh_TW/process/submit-checklist.rst `。 何時郵寄 diff --git a/Documentation/translations/zh_TW/process/howto.rst b/Documentation/translations/zh_TW/process/howto.rst index 2043691b92e3..68ae4411285b 100644 --- a/Documentation/translations/zh_TW/process/howto.rst +++ b/Documentation/translations/zh_TW/process/howto.rst @@ -99,7 +99,6 @@ Linux內核代碼中包含有大量的文檔。這些文檔對於學習如何與 的代碼。 :ref:`Documentation/translations/zh_TW/process/submitting-patches.rst ` - :ref:`Documentation/process/submitting-drivers.rst ` 這兩份文檔明確描述如何創建和發送補丁，其中包括（但不僅限於)： - 郵件內容 diff --git a/Documentation/translations/zh_TW/process/index.rst b/Documentation/translations/zh_TW/process/index.rst index ec7ad14bfd13..c5c59b4fd595 100644 --- a/Documentation/translations/zh_TW/process/index.rst +++ b/Documentation/translations/zh_TW/process/index.rst @@ -43,7 +43,6 @@ .. toctree:: :maxdepth: 1 - submitting-drivers submit-checklist stable-api-nonsense stable-kernel-rules diff --git a/Documentation/translations/zh_TW/process/submitting-drivers.rst b/Documentation/translations/zh_TW/process/submitting-drivers.rst deleted file mode 100644 index 2fdd742318ba..000000000000 --- a/Documentation/translations/zh_TW/process/submitting-drivers.rst +++ /dev/null @@ -1,164 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -.. _tw_submittingdrivers: - -.. include:: ../disclaimer-zh_TW.rst - -:Original: :ref:`Documentation/process/submitting-drivers.rst - ` - -如果想評論或更新本文的內容，請直接聯繫原文檔的維護者。如果你使用英文 -交流有困難的話，也可以向中文版維護者求助。如果本翻譯更新不及時或者翻 -譯存在問題，請聯繫中文版維護者:: - - 中文版維護者： 李陽 Li Yang - 中文版翻譯者： 李陽 Li Yang - 中文版校譯者： 陳琦 Maggie Chen - 王聰 Wang Cong - 張巍 Zhang Wei - 胡皓文 Hu Haowen - -如何向 Linux 內核提交驅動程序 -============================= - -這篇文檔將會解釋如何向不同的內核源碼樹提交設備驅動程序。請注意，如果你感 -興趣的是顯卡驅動程序，你也許應該訪問 XFree86 項目(https://www.xfree86.org/) -和／或 X.org 項目 (https://x.org)。 - -另請參閱 Documentation/translations/zh_TW/process/submitting-patches.rst 文檔。 - - -分配設備號 ----------- - -塊設備和字符設備的主設備號與從設備號是由 Linux 命名編號分配權威 LANANA（ -現在是 Torben Mathiasen）負責分配。申請的網址是 https://www.lanana.org/。 -即使不準備提交到主流內核的設備驅動也需要在這裡分配設備號。有關詳細信息， -請參閱 Documentation/admin-guide/devices.rst。 - -如果你使用的不是已經分配的設備號，那麼當你提交設備驅動的時候，它將會被強 -制分配一個新的設備號，即便這個設備號和你之前發給客戶的截然不同。 - -設備驅動的提交對象 ------------------- - -Linux 2.0: - 此內核源碼樹不接受新的驅動程序。 - -Linux 2.2: - 此內核源碼樹不接受新的驅動程序。 - -Linux 2.4: - 如果所屬的代碼領域在內核的 MAINTAINERS 文件中列有一個總維護者， - 那麼請將驅動程序提交給他。如果此維護者沒有回應或者你找不到恰當的 - 維護者，那麼請聯繫 Willy Tarreau 。 - -Linux 2.6: - 除了遵循和 2.4 版內核同樣的規則外，你還需要在 linux-kernel 郵件 - 列表上跟蹤最新的 API 變化。向 Linux 2.6 內核提交驅動的頂級聯繫人 - 是 Andrew Morton 。 - -決定設備驅動能否被接受的條件 ----------------------------- - -許可： 代碼必須使用 GNU 通用公開許可證 (GPL) 提交給 Linux，但是 - 我們並不要求 GPL 是唯一的許可。你或許會希望同時使用多種 - 許可證發布，如果希望驅動程序可以被其他開源社區（比如BSD） - 使用。請參考 include/linux/module.h 文件中所列出的可被 - 接受共存的許可。 - -版權： 版權所有者必須同意使用 GPL 許可。最好提交者和版權所有者 - 是相同個人或實體。否則，必需列出授權使用 GPL 的版權所有 - 人或實體，以備驗證之需。 - -接口： 如果你的驅動程序使用現成的接口並且和其他同類的驅動程序行 - 爲相似，而不是去發明無謂的新接口，那麼它將會更容易被接受。 - 如果你需要一個 Linux 和 NT 的通用驅動接口，那麼請在用 - 戶空間實現它。 - -代碼： 請使用 Documentation/process/coding-style.rst 中所描述的 Linux 代碼風 - 格。如果你的某些代碼段（例如那些與 Windows 驅動程序包共 - 享的代碼段）需要使用其他格式，而你卻只希望維護一份代碼， - 那麼請將它們很好地區分出來，並且註明原因。 - -可移植性： 請注意，指針並不永遠是 32 位的，不是所有的計算機都使用小 - 尾模式 (little endian) 存儲數據，不是所有的人都擁有浮點 - 單元，不要隨便在你的驅動程序里嵌入 x86 彙編指令。只能在 - x86 上運行的驅動程序一般是不受歡迎的。雖然你可能只有 x86 - 硬體，很難測試驅動程序在其他平台上是否可用，但是確保代碼 - 可以被輕鬆地移植卻是很簡單的。 - -清晰度： 做到所有人都能修補這個驅動程序將會很有好處，因爲這樣你將 - 會直接收到修復的補丁而不是 bug 報告。如果你提交一個試圖 - 隱藏硬體工作機理的驅動程序，那麼它將會被扔進廢紙簍。 - -電源管理： 因爲 Linux 正在被很多行動裝置和桌面系統使用，所以你的驅 - 動程序也很有可能被使用在這些設備上。它應該支持最基本的電 - 源管理，即在需要的情況下實現系統級休眠和喚醒要用到的 - .suspend 和 .resume 函數。你應該檢查你的驅動程序是否能正 - 確地處理休眠與喚醒，如果實在無法確認，請至少把 .suspend - 函數定義成返回 -ENOSYS（功能未實現）錯誤。你還應該嘗試確 - 保你的驅動在什麼都不乾的情況下將耗電降到最低。要獲得驅動 - 程序測試的指導，請參閱 - Documentation/power/drivers-testing.rst。有關驅動程序電 - 源管理問題相對全面的概述，請參閱 - Documentation/driver-api/pm/devices.rst。 - -管理： 如果一個驅動程序的作者還在進行有效的維護，那麼通常除了那 - 些明顯正確且不需要任何檢查的補丁以外，其他所有的補丁都會 - 被轉發給作者。如果你希望成爲驅動程序的聯繫人和更新者，最 - 好在代碼注釋中寫明並且在 MAINTAINERS 文件中加入這個驅動 - 程序的條目。 - -不影響設備驅動能否被接受的條件 ------------------------------- - -供應商： 由硬體供應商來維護驅動程序通常是一件好事。不過，如果源碼 - 樹里已經有其他人提供了可穩定工作的驅動程序，那麼請不要期 - 望「我是供應商」會成爲內核改用你的驅動程序的理由。理想的情 - 況是：供應商與現有驅動程序的作者合作，構建一個統一完美的 - 驅動程序。 - -作者： 驅動程序是由大的 Linux 公司研發還是由你個人編寫，並不影 - 響其是否能被內核接受。沒有人對內核源碼樹享有特權。只要你 - 充分了解內核社區，你就會發現這一點。 - - -資源列表 --------- - -Linux 內核主源碼樹： - ftp.??.kernel.org:/pub/linux/kernel/... - ?? == 你的國家代碼，例如 "cn"、"us"、"uk"、"fr" 等等 - -Linux 內核郵件列表： - linux-kernel@vger.kernel.org - [可通過向majordomo@vger.kernel.org發郵件來訂閱] - -Linux 設備驅動程序，第三版（探討 2.6.10 版內核）： - https://lwn.net/Kernel/LDD3/ （免費版） - -LWN.net: - 每周內核開發活動摘要 - https://lwn.net/ - - 2.6 版中 API 的變更： - - https://lwn.net/Articles/2.6-kernel-api/ - - 將舊版內核的驅動程序移植到 2.6 版： - - https://lwn.net/Articles/driver-porting/ - -內核新手(KernelNewbies): - 爲新的內核開發者提供文檔和幫助 - https://kernelnewbies.org/ - -Linux USB項目： - http://www.linux-usb.org/ - -寫內核驅動的「不要」（Arjan van de Ven著）: - http://www.fenrus.org/how-to-not-write-a-device-driver-paper.pdf - -內核清潔工 (Kernel Janitor): - https://kernelnewbies.org/KernelJanitors - From 0582591cc454e48b027d23c84bc768198d6a82e4 Mon Sep 17 00:00:00 2001 From: Alex Shi Date: Thu, 14 Jul 2022 13:40:13 +0800 Subject: [PATCH 220/318] doc/zh_CN: remove submitting-driver reference from docs The doc is removed, so related reference should be removed too. Signed-off-by: Alex Shi Cc: Yanteng Si Cc: Hu Haowen Link: https://lore.kernel.org/r/20220714054013.214031-1-alexs@kernel.org Signed-off-by: Jonathan Corbet --- Documentation/translations/zh_CN/process/8.Conclusion.rst | 1 - .../translations/zh_CN/process/submitting-patches.rst | 4 +--- Documentation/translations/zh_TW/process/8.Conclusion.rst | 1 - .../translations/zh_TW/process/submitting-patches.rst | 4 +--- 4 files changed, 2 insertions(+), 8 deletions(-) diff --git a/Documentation/translations/zh_CN/process/8.Conclusion.rst b/Documentation/translations/zh_CN/process/8.Conclusion.rst index 4707f0101964..643b88af97bb 100644 --- a/Documentation/translations/zh_CN/process/8.Conclusion.rst +++ b/Documentation/translations/zh_CN/process/8.Conclusion.rst @@ -19,7 +19,6 @@ :ref:`Documentation/translations/zh_CN/process/howto.rst ` 文件是一个重要的起点； :ref:`Documentation/translations/zh_CN/process/submitting-patches.rst ` -和 :ref:`Documentation/translations/zh_CN/process/submitting-drivers.rst ` 也是所有内核开发人员都应该阅读的内容。许多内部内核API都是使用kerneldoc机制 记录的；“make htmldocs”或“make pdfdocs”可用于以HTML或PDF格式生成这些文档 （尽管某些发行版提供的tex版本会遇到内部限制，无法正确处理文档）。 diff --git a/Documentation/translations/zh_CN/process/submitting-patches.rst b/Documentation/translations/zh_CN/process/submitting-patches.rst index a9570165582a..ebb7f37575c1 100644 --- a/Documentation/translations/zh_CN/process/submitting-patches.rst +++ b/Documentation/translations/zh_CN/process/submitting-patches.rst @@ -23,9 +23,7 @@ 以下文档含有大量简洁的建议， 具体请见： :ref:`Documentation/process ` 同样，:ref:`Documentation/translations/zh_CN/process/submit-checklist.rst ` -给出在提交代码前需要检查的项目的列表。如果你在提交一个驱动程序，那么 -同时阅读一下: -:ref:`Documentation/process/submitting-drivers.rst ` +给出在提交代码前需要检查的项目的列表。 其中许多步骤描述了Git版本控制系统的默认行为；如果您使用Git来准备补丁， 您将发现它为您完成的大部分机械工作，尽管您仍然需要准备和记录一组合理的 diff --git a/Documentation/translations/zh_TW/process/8.Conclusion.rst b/Documentation/translations/zh_TW/process/8.Conclusion.rst index 7572b17667d9..044fcc118bef 100644 --- a/Documentation/translations/zh_TW/process/8.Conclusion.rst +++ b/Documentation/translations/zh_TW/process/8.Conclusion.rst @@ -22,7 +22,6 @@ :ref:`Documentation/translations/zh_TW/process/howto.rst ` 文件是一個重要的起點； :ref:`Documentation/translations/zh_TW/process/submitting-patches.rst ` -和 :ref:`Documentation/translations/zh_TW/process/submitting-drivers.rst ` 也是所有內核開發人員都應該閱讀的內容。許多內部內核API都是使用kerneldoc機制 記錄的；「make htmldocs」或「make pdfdocs」可用於以HTML或PDF格式生成這些文檔 （儘管某些發行版提供的tex版本會遇到內部限制，無法正確處理文檔）。 diff --git a/Documentation/translations/zh_TW/process/submitting-patches.rst b/Documentation/translations/zh_TW/process/submitting-patches.rst index c4fd48f5bd8b..3f77ef5d48a0 100644 --- a/Documentation/translations/zh_TW/process/submitting-patches.rst +++ b/Documentation/translations/zh_TW/process/submitting-patches.rst @@ -26,9 +26,7 @@ 以下文檔含有大量簡潔的建議， 具體請見： :ref:`Documentation/process ` 同樣，:ref:`Documentation/translations/zh_TW/process/submit-checklist.rst ` -給出在提交代碼前需要檢查的項目的列表。如果你在提交一個驅動程序，那麼 -同時閱讀一下: -:ref:`Documentation/process/submitting-drivers.rst ` +給出在提交代碼前需要檢查的項目的列表。 其中許多步驟描述了Git版本控制系統的默認行爲；如果您使用Git來準備補丁， 您將發現它爲您完成的大部分機械工作，儘管您仍然需要準備和記錄一組合理的 From e4e712bbbd6d73263d964d6cb390b373738b62ab Mon Sep 17 00:00:00 2001 From: Taehee Yoo Date: Mon, 4 Jul 2022 09:42:48 +0000 Subject: [PATCH 221/318] crypto: aria - Implement ARIA symmetric cipher algorithm ARIA(RFC 5794) is a symmetric block cipher algorithm. This algorithm is being used widely in South Korea as a standard cipher algorithm. This code is written based on the ARIA implementation of OpenSSL. The OpenSSL code is based on the distributed source code[1] by KISA. ARIA has three key sizes and corresponding rounds. ARIA128: 12 rounds. ARIA192: 14 rounds. ARIA245: 16 rounds. [1] https://seed.kisa.or.kr/kisa/Board/19/detailView.do (Korean) Signed-off-by: Taehee Yoo Signed-off-by: Herbert Xu --- crypto/Kconfig | 15 ++ crypto/Makefile | 1 + crypto/aria.c | 288 ++++++++++++++++++++++++++ include/crypto/aria.h | 461 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 765 insertions(+) create mode 100644 crypto/aria.c create mode 100644 include/crypto/aria.h diff --git a/crypto/Kconfig b/crypto/Kconfig index 59489a300cd1..7d98a2b4ac9c 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1508,6 +1508,21 @@ config CRYPTO_SEED See also: +config CRYPTO_ARIA + tristate "ARIA cipher algorithm" + select CRYPTO_ALGAPI + help + ARIA cipher algorithm (RFC5794). + + ARIA is a standard encryption algorithm of the Republic of Korea. + The ARIA specifies three key sizes and rounds. + 128-bit: 12 rounds. + 192-bit: 14 rounds. + 256-bit: 16 rounds. + + See also: + + config CRYPTO_SERPENT tristate "Serpent cipher algorithm" select CRYPTO_ALGAPI diff --git a/crypto/Makefile b/crypto/Makefile index a4a84860fe43..167c004dbf4f 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -148,6 +148,7 @@ obj-$(CONFIG_CRYPTO_TEA) += tea.o obj-$(CONFIG_CRYPTO_KHAZAD) += khazad.o obj-$(CONFIG_CRYPTO_ANUBIS) += anubis.o obj-$(CONFIG_CRYPTO_SEED) += seed.o +obj-$(CONFIG_CRYPTO_ARIA) += aria.o obj-$(CONFIG_CRYPTO_CHACHA20) += chacha_generic.o obj-$(CONFIG_CRYPTO_POLY1305) += poly1305_generic.o obj-$(CONFIG_CRYPTO_DEFLATE) += deflate.o diff --git a/crypto/aria.c b/crypto/aria.c new file mode 100644 index 000000000000..ac3dffac34bb --- /dev/null +++ b/crypto/aria.c @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Cryptographic API. + * + * ARIA Cipher Algorithm. + * + * Documentation of ARIA can be found in RFC 5794. + * Copyright (c) 2022 Taehee Yoo + * + * Information for ARIA + * http://210.104.33.10/ARIA/index-e.html (English) + * http://seed.kisa.or.kr/ (Korean) + * + * Public domain version is distributed above. + */ + +#include + +static void aria_set_encrypt_key(struct aria_ctx *ctx, const u8 *in_key, + unsigned int key_len) +{ + const __be32 *key = (const __be32 *)in_key; + u32 w0[4], w1[4], w2[4], w3[4]; + u32 reg0, reg1, reg2, reg3; + const u32 *ck; + int rkidx = 0; + + ck = &key_rc[(key_len - 16) / 8][0]; + + w0[0] = be32_to_cpu(key[0]); + w0[1] = be32_to_cpu(key[1]); + w0[2] = be32_to_cpu(key[2]); + w0[3] = be32_to_cpu(key[3]); + + reg0 = w0[0] ^ ck[0]; + reg1 = w0[1] ^ ck[1]; + reg2 = w0[2] ^ ck[2]; + reg3 = w0[3] ^ ck[3]; + + aria_subst_diff_odd(®0, ®1, ®2, ®3); + + if (key_len > 16) { + w1[0] = be32_to_cpu(key[4]); + w1[1] = be32_to_cpu(key[5]); + if (key_len > 24) { + w1[2] = be32_to_cpu(key[6]); + w1[3] = be32_to_cpu(key[7]); + } else { + w1[2] = 0; + w1[3] = 0; + } + } else { + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + } + + w1[0] ^= reg0; + w1[1] ^= reg1; + w1[2] ^= reg2; + w1[3] ^= reg3; + + reg0 = w1[0]; + reg1 = w1[1]; + reg2 = w1[2]; + reg3 = w1[3]; + + reg0 ^= ck[4]; + reg1 ^= ck[5]; + reg2 ^= ck[6]; + reg3 ^= ck[7]; + + aria_subst_diff_even(®0, ®1, ®2, ®3); + + reg0 ^= w0[0]; + reg1 ^= w0[1]; + reg2 ^= w0[2]; + reg3 ^= w0[3]; + + w2[0] = reg0; + w2[1] = reg1; + w2[2] = reg2; + w2[3] = reg3; + + reg0 ^= ck[8]; + reg1 ^= ck[9]; + reg2 ^= ck[10]; + reg3 ^= ck[11]; + + aria_subst_diff_odd(®0, ®1, ®2, ®3); + + w3[0] = reg0 ^ w1[0]; + w3[1] = reg1 ^ w1[1]; + w3[2] = reg2 ^ w1[2]; + w3[3] = reg3 ^ w1[3]; + + aria_gsrk(ctx->enc_key[rkidx], w0, w1, 19); + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w1, w2, 19); + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w2, w3, 19); + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w3, w0, 19); + + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w0, w1, 31); + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w1, w2, 31); + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w2, w3, 31); + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w3, w0, 31); + + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w0, w1, 67); + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w1, w2, 67); + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w2, w3, 67); + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w3, w0, 67); + + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w0, w1, 97); + if (key_len > 16) { + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w1, w2, 97); + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w2, w3, 97); + + if (key_len > 24) { + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w3, w0, 97); + + rkidx++; + aria_gsrk(ctx->enc_key[rkidx], w0, w1, 109); + } + } +} + +static void aria_set_decrypt_key(struct aria_ctx *ctx) +{ + int i; + + for (i = 0; i < 4; i++) { + ctx->dec_key[0][i] = ctx->enc_key[ctx->rounds][i]; + ctx->dec_key[ctx->rounds][i] = ctx->enc_key[0][i]; + } + + for (i = 1; i < ctx->rounds; i++) { + ctx->dec_key[i][0] = aria_m(ctx->enc_key[ctx->rounds - i][0]); + ctx->dec_key[i][1] = aria_m(ctx->enc_key[ctx->rounds - i][1]); + ctx->dec_key[i][2] = aria_m(ctx->enc_key[ctx->rounds - i][2]); + ctx->dec_key[i][3] = aria_m(ctx->enc_key[ctx->rounds - i][3]); + + aria_diff_word(&ctx->dec_key[i][0], &ctx->dec_key[i][1], + &ctx->dec_key[i][2], &ctx->dec_key[i][3]); + aria_diff_byte(&ctx->dec_key[i][1], + &ctx->dec_key[i][2], &ctx->dec_key[i][3]); + aria_diff_word(&ctx->dec_key[i][0], &ctx->dec_key[i][1], + &ctx->dec_key[i][2], &ctx->dec_key[i][3]); + } +} + +static int aria_set_key(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct aria_ctx *ctx = crypto_tfm_ctx(tfm); + + if (key_len != 16 && key_len != 24 && key_len != 32) + return -EINVAL; + + ctx->key_length = key_len; + ctx->rounds = (key_len + 32) / 4; + + aria_set_encrypt_key(ctx, in_key, key_len); + aria_set_decrypt_key(ctx); + + return 0; +} + +static void __aria_crypt(struct aria_ctx *ctx, u8 *out, const u8 *in, + u32 key[][ARIA_RD_KEY_WORDS]) +{ + const __be32 *src = (const __be32 *)in; + __be32 *dst = (__be32 *)out; + u32 reg0, reg1, reg2, reg3; + int rounds, rkidx = 0; + + rounds = ctx->rounds; + + reg0 = be32_to_cpu(src[0]); + reg1 = be32_to_cpu(src[1]); + reg2 = be32_to_cpu(src[2]); + reg3 = be32_to_cpu(src[3]); + + aria_add_round_key(key[rkidx], ®0, ®1, ®2, ®3); + rkidx++; + + aria_subst_diff_odd(®0, ®1, ®2, ®3); + aria_add_round_key(key[rkidx], ®0, ®1, ®2, ®3); + rkidx++; + + while ((rounds -= 2) > 0) { + aria_subst_diff_even(®0, ®1, ®2, ®3); + aria_add_round_key(key[rkidx], ®0, ®1, ®2, ®3); + rkidx++; + + aria_subst_diff_odd(®0, ®1, ®2, ®3); + aria_add_round_key(key[rkidx], ®0, ®1, ®2, ®3); + rkidx++; + } + + reg0 = key[rkidx][0] ^ make_u32((u8)(x1[get_u8(reg0, 0)]), + (u8)(x2[get_u8(reg0, 1)] >> 8), + (u8)(s1[get_u8(reg0, 2)]), + (u8)(s2[get_u8(reg0, 3)])); + reg1 = key[rkidx][1] ^ make_u32((u8)(x1[get_u8(reg1, 0)]), + (u8)(x2[get_u8(reg1, 1)] >> 8), + (u8)(s1[get_u8(reg1, 2)]), + (u8)(s2[get_u8(reg1, 3)])); + reg2 = key[rkidx][2] ^ make_u32((u8)(x1[get_u8(reg2, 0)]), + (u8)(x2[get_u8(reg2, 1)] >> 8), + (u8)(s1[get_u8(reg2, 2)]), + (u8)(s2[get_u8(reg2, 3)])); + reg3 = key[rkidx][3] ^ make_u32((u8)(x1[get_u8(reg3, 0)]), + (u8)(x2[get_u8(reg3, 1)] >> 8), + (u8)(s1[get_u8(reg3, 2)]), + (u8)(s2[get_u8(reg3, 3)])); + + dst[0] = cpu_to_be32(reg0); + dst[1] = cpu_to_be32(reg1); + dst[2] = cpu_to_be32(reg2); + dst[3] = cpu_to_be32(reg3); +} + +static void aria_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) +{ + struct aria_ctx *ctx = crypto_tfm_ctx(tfm); + + __aria_crypt(ctx, out, in, ctx->enc_key); +} + +static void aria_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) +{ + struct aria_ctx *ctx = crypto_tfm_ctx(tfm); + + __aria_crypt(ctx, out, in, ctx->dec_key); +} + +static struct crypto_alg aria_alg = { + .cra_name = "aria", + .cra_driver_name = "aria-generic", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_TYPE_CIPHER, + .cra_blocksize = ARIA_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aria_ctx), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + .cra_u = { + .cipher = { + .cia_min_keysize = ARIA_MIN_KEY_SIZE, + .cia_max_keysize = ARIA_MAX_KEY_SIZE, + .cia_setkey = aria_set_key, + .cia_encrypt = aria_encrypt, + .cia_decrypt = aria_decrypt + } + } +}; + +static int __init aria_init(void) +{ + return crypto_register_alg(&aria_alg); +} + +static void __exit aria_fini(void) +{ + crypto_unregister_alg(&aria_alg); +} + +subsys_initcall(aria_init); +module_exit(aria_fini); + +MODULE_DESCRIPTION("ARIA Cipher Algorithm"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Taehee Yoo "); +MODULE_ALIAS_CRYPTO("aria"); diff --git a/include/crypto/aria.h b/include/crypto/aria.h new file mode 100644 index 000000000000..4a86661788e8 --- /dev/null +++ b/include/crypto/aria.h @@ -0,0 +1,461 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Cryptographic API. + * + * ARIA Cipher Algorithm. + * + * Documentation of ARIA can be found in RFC 5794. + * Copyright (c) 2022 Taehee Yoo + * Copyright (c) 2022 Taehee Yoo + * + * Information for ARIA + * http://210.104.33.10/ARIA/index-e.html (English) + * http://seed.kisa.or.kr/ (Korean) + * + * Public domain version is distributed above. + */ + +#ifndef _CRYPTO_ARIA_H +#define _CRYPTO_ARIA_H + +#include +#include +#include +#include +#include +#include + +#define ARIA_MIN_KEY_SIZE 16 +#define ARIA_MAX_KEY_SIZE 32 +#define ARIA_BLOCK_SIZE 16 +#define ARIA_MAX_RD_KEYS 17 +#define ARIA_RD_KEY_WORDS (ARIA_BLOCK_SIZE / sizeof(u32)) + +struct aria_ctx { + int key_length; + int rounds; + u32 enc_key[ARIA_MAX_RD_KEYS][ARIA_RD_KEY_WORDS]; + u32 dec_key[ARIA_MAX_RD_KEYS][ARIA_RD_KEY_WORDS]; +}; + +static const u32 key_rc[5][4] = { + { 0x517cc1b7, 0x27220a94, 0xfe13abe8, 0xfa9a6ee0 }, + { 0x6db14acc, 0x9e21c820, 0xff28b1d5, 0xef5de2b0 }, + { 0xdb92371d, 0x2126e970, 0x03249775, 0x04e8c90e }, + { 0x517cc1b7, 0x27220a94, 0xfe13abe8, 0xfa9a6ee0 }, + { 0x6db14acc, 0x9e21c820, 0xff28b1d5, 0xef5de2b0 } +}; + +static const u32 s1[256] = { + 0x00636363, 0x007c7c7c, 0x00777777, 0x007b7b7b, + 0x00f2f2f2, 0x006b6b6b, 0x006f6f6f, 0x00c5c5c5, + 0x00303030, 0x00010101, 0x00676767, 0x002b2b2b, + 0x00fefefe, 0x00d7d7d7, 0x00ababab, 0x00767676, + 0x00cacaca, 0x00828282, 0x00c9c9c9, 0x007d7d7d, + 0x00fafafa, 0x00595959, 0x00474747, 0x00f0f0f0, + 0x00adadad, 0x00d4d4d4, 0x00a2a2a2, 0x00afafaf, + 0x009c9c9c, 0x00a4a4a4, 0x00727272, 0x00c0c0c0, + 0x00b7b7b7, 0x00fdfdfd, 0x00939393, 0x00262626, + 0x00363636, 0x003f3f3f, 0x00f7f7f7, 0x00cccccc, + 0x00343434, 0x00a5a5a5, 0x00e5e5e5, 0x00f1f1f1, + 0x00717171, 0x00d8d8d8, 0x00313131, 0x00151515, + 0x00040404, 0x00c7c7c7, 0x00232323, 0x00c3c3c3, + 0x00181818, 0x00969696, 0x00050505, 0x009a9a9a, + 0x00070707, 0x00121212, 0x00808080, 0x00e2e2e2, + 0x00ebebeb, 0x00272727, 0x00b2b2b2, 0x00757575, + 0x00090909, 0x00838383, 0x002c2c2c, 0x001a1a1a, + 0x001b1b1b, 0x006e6e6e, 0x005a5a5a, 0x00a0a0a0, + 0x00525252, 0x003b3b3b, 0x00d6d6d6, 0x00b3b3b3, + 0x00292929, 0x00e3e3e3, 0x002f2f2f, 0x00848484, + 0x00535353, 0x00d1d1d1, 0x00000000, 0x00ededed, + 0x00202020, 0x00fcfcfc, 0x00b1b1b1, 0x005b5b5b, + 0x006a6a6a, 0x00cbcbcb, 0x00bebebe, 0x00393939, + 0x004a4a4a, 0x004c4c4c, 0x00585858, 0x00cfcfcf, + 0x00d0d0d0, 0x00efefef, 0x00aaaaaa, 0x00fbfbfb, + 0x00434343, 0x004d4d4d, 0x00333333, 0x00858585, + 0x00454545, 0x00f9f9f9, 0x00020202, 0x007f7f7f, + 0x00505050, 0x003c3c3c, 0x009f9f9f, 0x00a8a8a8, + 0x00515151, 0x00a3a3a3, 0x00404040, 0x008f8f8f, + 0x00929292, 0x009d9d9d, 0x00383838, 0x00f5f5f5, + 0x00bcbcbc, 0x00b6b6b6, 0x00dadada, 0x00212121, + 0x00101010, 0x00ffffff, 0x00f3f3f3, 0x00d2d2d2, + 0x00cdcdcd, 0x000c0c0c, 0x00131313, 0x00ececec, + 0x005f5f5f, 0x00979797, 0x00444444, 0x00171717, + 0x00c4c4c4, 0x00a7a7a7, 0x007e7e7e, 0x003d3d3d, + 0x00646464, 0x005d5d5d, 0x00191919, 0x00737373, + 0x00606060, 0x00818181, 0x004f4f4f, 0x00dcdcdc, + 0x00222222, 0x002a2a2a, 0x00909090, 0x00888888, + 0x00464646, 0x00eeeeee, 0x00b8b8b8, 0x00141414, + 0x00dedede, 0x005e5e5e, 0x000b0b0b, 0x00dbdbdb, + 0x00e0e0e0, 0x00323232, 0x003a3a3a, 0x000a0a0a, + 0x00494949, 0x00060606, 0x00242424, 0x005c5c5c, + 0x00c2c2c2, 0x00d3d3d3, 0x00acacac, 0x00626262, + 0x00919191, 0x00959595, 0x00e4e4e4, 0x00797979, + 0x00e7e7e7, 0x00c8c8c8, 0x00373737, 0x006d6d6d, + 0x008d8d8d, 0x00d5d5d5, 0x004e4e4e, 0x00a9a9a9, + 0x006c6c6c, 0x00565656, 0x00f4f4f4, 0x00eaeaea, + 0x00656565, 0x007a7a7a, 0x00aeaeae, 0x00080808, + 0x00bababa, 0x00787878, 0x00252525, 0x002e2e2e, + 0x001c1c1c, 0x00a6a6a6, 0x00b4b4b4, 0x00c6c6c6, + 0x00e8e8e8, 0x00dddddd, 0x00747474, 0x001f1f1f, + 0x004b4b4b, 0x00bdbdbd, 0x008b8b8b, 0x008a8a8a, + 0x00707070, 0x003e3e3e, 0x00b5b5b5, 0x00666666, + 0x00484848, 0x00030303, 0x00f6f6f6, 0x000e0e0e, + 0x00616161, 0x00353535, 0x00575757, 0x00b9b9b9, + 0x00868686, 0x00c1c1c1, 0x001d1d1d, 0x009e9e9e, + 0x00e1e1e1, 0x00f8f8f8, 0x00989898, 0x00111111, + 0x00696969, 0x00d9d9d9, 0x008e8e8e, 0x00949494, + 0x009b9b9b, 0x001e1e1e, 0x00878787, 0x00e9e9e9, + 0x00cecece, 0x00555555, 0x00282828, 0x00dfdfdf, + 0x008c8c8c, 0x00a1a1a1, 0x00898989, 0x000d0d0d, + 0x00bfbfbf, 0x00e6e6e6, 0x00424242, 0x00686868, + 0x00414141, 0x00999999, 0x002d2d2d, 0x000f0f0f, + 0x00b0b0b0, 0x00545454, 0x00bbbbbb, 0x00161616 +}; + +static const u32 s2[256] = { + 0xe200e2e2, 0x4e004e4e, 0x54005454, 0xfc00fcfc, + 0x94009494, 0xc200c2c2, 0x4a004a4a, 0xcc00cccc, + 0x62006262, 0x0d000d0d, 0x6a006a6a, 0x46004646, + 0x3c003c3c, 0x4d004d4d, 0x8b008b8b, 0xd100d1d1, + 0x5e005e5e, 0xfa00fafa, 0x64006464, 0xcb00cbcb, + 0xb400b4b4, 0x97009797, 0xbe00bebe, 0x2b002b2b, + 0xbc00bcbc, 0x77007777, 0x2e002e2e, 0x03000303, + 0xd300d3d3, 0x19001919, 0x59005959, 0xc100c1c1, + 0x1d001d1d, 0x06000606, 0x41004141, 0x6b006b6b, + 0x55005555, 0xf000f0f0, 0x99009999, 0x69006969, + 0xea00eaea, 0x9c009c9c, 0x18001818, 0xae00aeae, + 0x63006363, 0xdf00dfdf, 0xe700e7e7, 0xbb00bbbb, + 0x00000000, 0x73007373, 0x66006666, 0xfb00fbfb, + 0x96009696, 0x4c004c4c, 0x85008585, 0xe400e4e4, + 0x3a003a3a, 0x09000909, 0x45004545, 0xaa00aaaa, + 0x0f000f0f, 0xee00eeee, 0x10001010, 0xeb00ebeb, + 0x2d002d2d, 0x7f007f7f, 0xf400f4f4, 0x29002929, + 0xac00acac, 0xcf00cfcf, 0xad00adad, 0x91009191, + 0x8d008d8d, 0x78007878, 0xc800c8c8, 0x95009595, + 0xf900f9f9, 0x2f002f2f, 0xce00cece, 0xcd00cdcd, + 0x08000808, 0x7a007a7a, 0x88008888, 0x38003838, + 0x5c005c5c, 0x83008383, 0x2a002a2a, 0x28002828, + 0x47004747, 0xdb00dbdb, 0xb800b8b8, 0xc700c7c7, + 0x93009393, 0xa400a4a4, 0x12001212, 0x53005353, + 0xff00ffff, 0x87008787, 0x0e000e0e, 0x31003131, + 0x36003636, 0x21002121, 0x58005858, 0x48004848, + 0x01000101, 0x8e008e8e, 0x37003737, 0x74007474, + 0x32003232, 0xca00caca, 0xe900e9e9, 0xb100b1b1, + 0xb700b7b7, 0xab00abab, 0x0c000c0c, 0xd700d7d7, + 0xc400c4c4, 0x56005656, 0x42004242, 0x26002626, + 0x07000707, 0x98009898, 0x60006060, 0xd900d9d9, + 0xb600b6b6, 0xb900b9b9, 0x11001111, 0x40004040, + 0xec00ecec, 0x20002020, 0x8c008c8c, 0xbd00bdbd, + 0xa000a0a0, 0xc900c9c9, 0x84008484, 0x04000404, + 0x49004949, 0x23002323, 0xf100f1f1, 0x4f004f4f, + 0x50005050, 0x1f001f1f, 0x13001313, 0xdc00dcdc, + 0xd800d8d8, 0xc000c0c0, 0x9e009e9e, 0x57005757, + 0xe300e3e3, 0xc300c3c3, 0x7b007b7b, 0x65006565, + 0x3b003b3b, 0x02000202, 0x8f008f8f, 0x3e003e3e, + 0xe800e8e8, 0x25002525, 0x92009292, 0xe500e5e5, + 0x15001515, 0xdd00dddd, 0xfd00fdfd, 0x17001717, + 0xa900a9a9, 0xbf00bfbf, 0xd400d4d4, 0x9a009a9a, + 0x7e007e7e, 0xc500c5c5, 0x39003939, 0x67006767, + 0xfe00fefe, 0x76007676, 0x9d009d9d, 0x43004343, + 0xa700a7a7, 0xe100e1e1, 0xd000d0d0, 0xf500f5f5, + 0x68006868, 0xf200f2f2, 0x1b001b1b, 0x34003434, + 0x70007070, 0x05000505, 0xa300a3a3, 0x8a008a8a, + 0xd500d5d5, 0x79007979, 0x86008686, 0xa800a8a8, + 0x30003030, 0xc600c6c6, 0x51005151, 0x4b004b4b, + 0x1e001e1e, 0xa600a6a6, 0x27002727, 0xf600f6f6, + 0x35003535, 0xd200d2d2, 0x6e006e6e, 0x24002424, + 0x16001616, 0x82008282, 0x5f005f5f, 0xda00dada, + 0xe600e6e6, 0x75007575, 0xa200a2a2, 0xef00efef, + 0x2c002c2c, 0xb200b2b2, 0x1c001c1c, 0x9f009f9f, + 0x5d005d5d, 0x6f006f6f, 0x80008080, 0x0a000a0a, + 0x72007272, 0x44004444, 0x9b009b9b, 0x6c006c6c, + 0x90009090, 0x0b000b0b, 0x5b005b5b, 0x33003333, + 0x7d007d7d, 0x5a005a5a, 0x52005252, 0xf300f3f3, + 0x61006161, 0xa100a1a1, 0xf700f7f7, 0xb000b0b0, + 0xd600d6d6, 0x3f003f3f, 0x7c007c7c, 0x6d006d6d, + 0xed00eded, 0x14001414, 0xe000e0e0, 0xa500a5a5, + 0x3d003d3d, 0x22002222, 0xb300b3b3, 0xf800f8f8, + 0x89008989, 0xde00dede, 0x71007171, 0x1a001a1a, + 0xaf00afaf, 0xba00baba, 0xb500b5b5, 0x81008181 +}; + +static const u32 x1[256] = { + 0x52520052, 0x09090009, 0x6a6a006a, 0xd5d500d5, + 0x30300030, 0x36360036, 0xa5a500a5, 0x38380038, + 0xbfbf00bf, 0x40400040, 0xa3a300a3, 0x9e9e009e, + 0x81810081, 0xf3f300f3, 0xd7d700d7, 0xfbfb00fb, + 0x7c7c007c, 0xe3e300e3, 0x39390039, 0x82820082, + 0x9b9b009b, 0x2f2f002f, 0xffff00ff, 0x87870087, + 0x34340034, 0x8e8e008e, 0x43430043, 0x44440044, + 0xc4c400c4, 0xdede00de, 0xe9e900e9, 0xcbcb00cb, + 0x54540054, 0x7b7b007b, 0x94940094, 0x32320032, + 0xa6a600a6, 0xc2c200c2, 0x23230023, 0x3d3d003d, + 0xeeee00ee, 0x4c4c004c, 0x95950095, 0x0b0b000b, + 0x42420042, 0xfafa00fa, 0xc3c300c3, 0x4e4e004e, + 0x08080008, 0x2e2e002e, 0xa1a100a1, 0x66660066, + 0x28280028, 0xd9d900d9, 0x24240024, 0xb2b200b2, + 0x76760076, 0x5b5b005b, 0xa2a200a2, 0x49490049, + 0x6d6d006d, 0x8b8b008b, 0xd1d100d1, 0x25250025, + 0x72720072, 0xf8f800f8, 0xf6f600f6, 0x64640064, + 0x86860086, 0x68680068, 0x98980098, 0x16160016, + 0xd4d400d4, 0xa4a400a4, 0x5c5c005c, 0xcccc00cc, + 0x5d5d005d, 0x65650065, 0xb6b600b6, 0x92920092, + 0x6c6c006c, 0x70700070, 0x48480048, 0x50500050, + 0xfdfd00fd, 0xeded00ed, 0xb9b900b9, 0xdada00da, + 0x5e5e005e, 0x15150015, 0x46460046, 0x57570057, + 0xa7a700a7, 0x8d8d008d, 0x9d9d009d, 0x84840084, + 0x90900090, 0xd8d800d8, 0xabab00ab, 0x00000000, + 0x8c8c008c, 0xbcbc00bc, 0xd3d300d3, 0x0a0a000a, + 0xf7f700f7, 0xe4e400e4, 0x58580058, 0x05050005, + 0xb8b800b8, 0xb3b300b3, 0x45450045, 0x06060006, + 0xd0d000d0, 0x2c2c002c, 0x1e1e001e, 0x8f8f008f, + 0xcaca00ca, 0x3f3f003f, 0x0f0f000f, 0x02020002, + 0xc1c100c1, 0xafaf00af, 0xbdbd00bd, 0x03030003, + 0x01010001, 0x13130013, 0x8a8a008a, 0x6b6b006b, + 0x3a3a003a, 0x91910091, 0x11110011, 0x41410041, + 0x4f4f004f, 0x67670067, 0xdcdc00dc, 0xeaea00ea, + 0x97970097, 0xf2f200f2, 0xcfcf00cf, 0xcece00ce, + 0xf0f000f0, 0xb4b400b4, 0xe6e600e6, 0x73730073, + 0x96960096, 0xacac00ac, 0x74740074, 0x22220022, + 0xe7e700e7, 0xadad00ad, 0x35350035, 0x85850085, + 0xe2e200e2, 0xf9f900f9, 0x37370037, 0xe8e800e8, + 0x1c1c001c, 0x75750075, 0xdfdf00df, 0x6e6e006e, + 0x47470047, 0xf1f100f1, 0x1a1a001a, 0x71710071, + 0x1d1d001d, 0x29290029, 0xc5c500c5, 0x89890089, + 0x6f6f006f, 0xb7b700b7, 0x62620062, 0x0e0e000e, + 0xaaaa00aa, 0x18180018, 0xbebe00be, 0x1b1b001b, + 0xfcfc00fc, 0x56560056, 0x3e3e003e, 0x4b4b004b, + 0xc6c600c6, 0xd2d200d2, 0x79790079, 0x20200020, + 0x9a9a009a, 0xdbdb00db, 0xc0c000c0, 0xfefe00fe, + 0x78780078, 0xcdcd00cd, 0x5a5a005a, 0xf4f400f4, + 0x1f1f001f, 0xdddd00dd, 0xa8a800a8, 0x33330033, + 0x88880088, 0x07070007, 0xc7c700c7, 0x31310031, + 0xb1b100b1, 0x12120012, 0x10100010, 0x59590059, + 0x27270027, 0x80800080, 0xecec00ec, 0x5f5f005f, + 0x60600060, 0x51510051, 0x7f7f007f, 0xa9a900a9, + 0x19190019, 0xb5b500b5, 0x4a4a004a, 0x0d0d000d, + 0x2d2d002d, 0xe5e500e5, 0x7a7a007a, 0x9f9f009f, + 0x93930093, 0xc9c900c9, 0x9c9c009c, 0xefef00ef, + 0xa0a000a0, 0xe0e000e0, 0x3b3b003b, 0x4d4d004d, + 0xaeae00ae, 0x2a2a002a, 0xf5f500f5, 0xb0b000b0, + 0xc8c800c8, 0xebeb00eb, 0xbbbb00bb, 0x3c3c003c, + 0x83830083, 0x53530053, 0x99990099, 0x61610061, + 0x17170017, 0x2b2b002b, 0x04040004, 0x7e7e007e, + 0xbaba00ba, 0x77770077, 0xd6d600d6, 0x26260026, + 0xe1e100e1, 0x69690069, 0x14140014, 0x63630063, + 0x55550055, 0x21210021, 0x0c0c000c, 0x7d7d007d +}; + +static const u32 x2[256] = { + 0x30303000, 0x68686800, 0x99999900, 0x1b1b1b00, + 0x87878700, 0xb9b9b900, 0x21212100, 0x78787800, + 0x50505000, 0x39393900, 0xdbdbdb00, 0xe1e1e100, + 0x72727200, 0x09090900, 0x62626200, 0x3c3c3c00, + 0x3e3e3e00, 0x7e7e7e00, 0x5e5e5e00, 0x8e8e8e00, + 0xf1f1f100, 0xa0a0a000, 0xcccccc00, 0xa3a3a300, + 0x2a2a2a00, 0x1d1d1d00, 0xfbfbfb00, 0xb6b6b600, + 0xd6d6d600, 0x20202000, 0xc4c4c400, 0x8d8d8d00, + 0x81818100, 0x65656500, 0xf5f5f500, 0x89898900, + 0xcbcbcb00, 0x9d9d9d00, 0x77777700, 0xc6c6c600, + 0x57575700, 0x43434300, 0x56565600, 0x17171700, + 0xd4d4d400, 0x40404000, 0x1a1a1a00, 0x4d4d4d00, + 0xc0c0c000, 0x63636300, 0x6c6c6c00, 0xe3e3e300, + 0xb7b7b700, 0xc8c8c800, 0x64646400, 0x6a6a6a00, + 0x53535300, 0xaaaaaa00, 0x38383800, 0x98989800, + 0x0c0c0c00, 0xf4f4f400, 0x9b9b9b00, 0xededed00, + 0x7f7f7f00, 0x22222200, 0x76767600, 0xafafaf00, + 0xdddddd00, 0x3a3a3a00, 0x0b0b0b00, 0x58585800, + 0x67676700, 0x88888800, 0x06060600, 0xc3c3c300, + 0x35353500, 0x0d0d0d00, 0x01010100, 0x8b8b8b00, + 0x8c8c8c00, 0xc2c2c200, 0xe6e6e600, 0x5f5f5f00, + 0x02020200, 0x24242400, 0x75757500, 0x93939300, + 0x66666600, 0x1e1e1e00, 0xe5e5e500, 0xe2e2e200, + 0x54545400, 0xd8d8d800, 0x10101000, 0xcecece00, + 0x7a7a7a00, 0xe8e8e800, 0x08080800, 0x2c2c2c00, + 0x12121200, 0x97979700, 0x32323200, 0xababab00, + 0xb4b4b400, 0x27272700, 0x0a0a0a00, 0x23232300, + 0xdfdfdf00, 0xefefef00, 0xcacaca00, 0xd9d9d900, + 0xb8b8b800, 0xfafafa00, 0xdcdcdc00, 0x31313100, + 0x6b6b6b00, 0xd1d1d100, 0xadadad00, 0x19191900, + 0x49494900, 0xbdbdbd00, 0x51515100, 0x96969600, + 0xeeeeee00, 0xe4e4e400, 0xa8a8a800, 0x41414100, + 0xdadada00, 0xffffff00, 0xcdcdcd00, 0x55555500, + 0x86868600, 0x36363600, 0xbebebe00, 0x61616100, + 0x52525200, 0xf8f8f800, 0xbbbbbb00, 0x0e0e0e00, + 0x82828200, 0x48484800, 0x69696900, 0x9a9a9a00, + 0xe0e0e000, 0x47474700, 0x9e9e9e00, 0x5c5c5c00, + 0x04040400, 0x4b4b4b00, 0x34343400, 0x15151500, + 0x79797900, 0x26262600, 0xa7a7a700, 0xdedede00, + 0x29292900, 0xaeaeae00, 0x92929200, 0xd7d7d700, + 0x84848400, 0xe9e9e900, 0xd2d2d200, 0xbababa00, + 0x5d5d5d00, 0xf3f3f300, 0xc5c5c500, 0xb0b0b000, + 0xbfbfbf00, 0xa4a4a400, 0x3b3b3b00, 0x71717100, + 0x44444400, 0x46464600, 0x2b2b2b00, 0xfcfcfc00, + 0xebebeb00, 0x6f6f6f00, 0xd5d5d500, 0xf6f6f600, + 0x14141400, 0xfefefe00, 0x7c7c7c00, 0x70707000, + 0x5a5a5a00, 0x7d7d7d00, 0xfdfdfd00, 0x2f2f2f00, + 0x18181800, 0x83838300, 0x16161600, 0xa5a5a500, + 0x91919100, 0x1f1f1f00, 0x05050500, 0x95959500, + 0x74747400, 0xa9a9a900, 0xc1c1c100, 0x5b5b5b00, + 0x4a4a4a00, 0x85858500, 0x6d6d6d00, 0x13131300, + 0x07070700, 0x4f4f4f00, 0x4e4e4e00, 0x45454500, + 0xb2b2b200, 0x0f0f0f00, 0xc9c9c900, 0x1c1c1c00, + 0xa6a6a600, 0xbcbcbc00, 0xececec00, 0x73737300, + 0x90909000, 0x7b7b7b00, 0xcfcfcf00, 0x59595900, + 0x8f8f8f00, 0xa1a1a100, 0xf9f9f900, 0x2d2d2d00, + 0xf2f2f200, 0xb1b1b100, 0x00000000, 0x94949400, + 0x37373700, 0x9f9f9f00, 0xd0d0d000, 0x2e2e2e00, + 0x9c9c9c00, 0x6e6e6e00, 0x28282800, 0x3f3f3f00, + 0x80808000, 0xf0f0f000, 0x3d3d3d00, 0xd3d3d300, + 0x25252500, 0x8a8a8a00, 0xb5b5b500, 0xe7e7e700, + 0x42424200, 0xb3b3b300, 0xc7c7c700, 0xeaeaea00, + 0xf7f7f700, 0x4c4c4c00, 0x11111100, 0x33333300, + 0x03030300, 0xa2a2a200, 0xacacac00, 0x60606000 +}; + +static inline u32 rotl32(u32 v, u32 r) +{ + return ((v << r) | (v >> (32 - r))); +} + +static inline u32 rotr32(u32 v, u32 r) +{ + return ((v >> r) | (v << (32 - r))); +} + +static inline u32 bswap32(u32 v) +{ + return ((v << 24) ^ + (v >> 24) ^ + ((v & 0x0000ff00) << 8) ^ + ((v & 0x00ff0000) >> 8)); +} + +static inline u8 get_u8(u32 x, u32 y) +{ + return (x >> ((3 - y) * 8)); +} + +static inline u32 make_u32(u8 v0, u8 v1, u8 v2, u8 v3) +{ + return ((u32)v0 << 24) | ((u32)v1 << 16) | ((u32)v2 << 8) | ((u32)v3); +} + +static inline u32 aria_m(u32 t0) +{ + return rotr32(t0, 8) ^ rotr32(t0 ^ rotr32(t0, 8), 16); +} + +/* S-Box Layer 1 + M */ +static inline void aria_sbox_layer1_with_pre_diff(u32 *t0, u32 *t1, u32 *t2, + u32 *t3) +{ + *t0 = s1[get_u8(*t0, 0)] ^ + s2[get_u8(*t0, 1)] ^ + x1[get_u8(*t0, 2)] ^ + x2[get_u8(*t0, 3)]; + *t1 = s1[get_u8(*t1, 0)] ^ + s2[get_u8(*t1, 1)] ^ + x1[get_u8(*t1, 2)] ^ + x2[get_u8(*t1, 3)]; + *t2 = s1[get_u8(*t2, 0)] ^ + s2[get_u8(*t2, 1)] ^ + x1[get_u8(*t2, 2)] ^ + x2[get_u8(*t2, 3)]; + *t3 = s1[get_u8(*t3, 0)] ^ + s2[get_u8(*t3, 1)] ^ + x1[get_u8(*t3, 2)] ^ + x2[get_u8(*t3, 3)]; +} + +/* S-Box Layer 2 + M */ +static inline void aria_sbox_layer2_with_pre_diff(u32 *t0, u32 *t1, u32 *t2, + u32 *t3) +{ + *t0 = x1[get_u8(*t0, 0)] ^ + x2[get_u8(*t0, 1)] ^ + s1[get_u8(*t0, 2)] ^ + s2[get_u8(*t0, 3)]; + *t1 = x1[get_u8(*t1, 0)] ^ + x2[get_u8(*t1, 1)] ^ + s1[get_u8(*t1, 2)] ^ + s2[get_u8(*t1, 3)]; + *t2 = x1[get_u8(*t2, 0)] ^ + x2[get_u8(*t2, 1)] ^ + s1[get_u8(*t2, 2)] ^ + s2[get_u8(*t2, 3)]; + *t3 = x1[get_u8(*t3, 0)] ^ + x2[get_u8(*t3, 1)] ^ + s1[get_u8(*t3, 2)] ^ + s2[get_u8(*t3, 3)]; +} + +/* Word-level diffusion */ +static inline void aria_diff_word(u32 *t0, u32 *t1, u32 *t2, u32 *t3) +{ + *t1 ^= *t2; + *t2 ^= *t3; + *t0 ^= *t1; + + *t3 ^= *t1; + *t2 ^= *t0; + *t1 ^= *t2; +} + +/* Byte-level diffusion */ +static inline void aria_diff_byte(u32 *t1, u32 *t2, u32 *t3) +{ + *t1 = ((*t1 << 8) & 0xff00ff00) ^ ((*t1 >> 8) & 0x00ff00ff); + *t2 = rotr32(*t2, 16); + *t3 = bswap32(*t3); +} + +/* Key XOR Layer */ +static inline void aria_add_round_key(u32 *rk, u32 *t0, u32 *t1, u32 *t2, + u32 *t3) +{ + *t0 ^= rk[0]; + *t1 ^= rk[1]; + *t2 ^= rk[2]; + *t3 ^= rk[3]; +} +/* Odd round Substitution & Diffusion */ +static inline void aria_subst_diff_odd(u32 *t0, u32 *t1, u32 *t2, u32 *t3) +{ + aria_sbox_layer1_with_pre_diff(t0, t1, t2, t3); + aria_diff_word(t0, t1, t2, t3); + aria_diff_byte(t1, t2, t3); + aria_diff_word(t0, t1, t2, t3); +} + +/* Even round Substitution & Diffusion */ +static inline void aria_subst_diff_even(u32 *t0, u32 *t1, u32 *t2, u32 *t3) +{ + aria_sbox_layer2_with_pre_diff(t0, t1, t2, t3); + aria_diff_word(t0, t1, t2, t3); + aria_diff_byte(t3, t0, t1); + aria_diff_word(t0, t1, t2, t3); +} + +/* Q, R Macro expanded ARIA GSRK */ +static inline void aria_gsrk(u32 *rk, u32 *x, u32 *y, u32 n) +{ + int q = 4 - (n / 32); + int r = n % 32; + + rk[0] = (x[0]) ^ + ((y[q % 4]) >> r) ^ + ((y[(q + 3) % 4]) << (32 - r)); + rk[1] = (x[1]) ^ + ((y[(q + 1) % 4]) >> r) ^ + ((y[q % 4]) << (32 - r)); + rk[2] = (x[2]) ^ + ((y[(q + 2) % 4]) >> r) ^ + ((y[(q + 1) % 4]) << (32 - r)); + rk[3] = (x[3]) ^ + ((y[(q + 3) % 4]) >> r) ^ + ((y[(q + 2) % 4]) << (32 - r)); +} + +#endif From 01ce31de7043e17b0d7d47f5e038f067db618113 Mon Sep 17 00:00:00 2001 From: Taehee Yoo Date: Mon, 4 Jul 2022 09:42:49 +0000 Subject: [PATCH 222/318] crypto: testmgr - add ARIA testmgr tests It contains ARIA ecb(aria), cbc(aria), cfb(aria), ctr(aria), and gcm(aria). ecb testvector is from RFC standard. cbc, cfb, and ctr testvectors are from KISA[1], who developed ARIA algorithm. gcm(aria) is from openssl test vector. [1] https://seed.kisa.or.kr/kisa/kcmvp/EgovVerification.do (Korean) Signed-off-by: Taehee Yoo Signed-off-by: Herbert Xu --- crypto/tcrypt.c | 38 +- crypto/testmgr.c | 31 + crypto/testmgr.h | 2860 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 2928 insertions(+), 1 deletion(-) diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index a8831060c4ce..f56d1a9cf0a7 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -71,7 +71,7 @@ static const char *check[] = { "blowfish", "twofish", "serpent", "sha384", "sha512", "md4", "aes", "cast6", "arc4", "michael_mic", "deflate", "crc32c", "tea", "xtea", "khazad", "wp512", "wp384", "wp256", "xeta", "fcrypt", - "camellia", "seed", "rmd160", + "camellia", "seed", "rmd160", "aria", "lzo", "lzo-rle", "cts", "sha3-224", "sha3-256", "sha3-384", "sha3-512", "streebog256", "streebog512", NULL @@ -1730,6 +1730,10 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) ret += tcrypt_test("polyval"); break; + case 58: + ret += tcrypt_test("gcm(aria)"); + break; + case 100: ret += tcrypt_test("hmac(md5)"); break; @@ -1866,6 +1870,12 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) ret += tcrypt_test("cfb(sm4)"); ret += tcrypt_test("ctr(sm4)"); break; + case 192: + ret += tcrypt_test("ecb(aria)"); + ret += tcrypt_test("cbc(aria)"); + ret += tcrypt_test("cfb(aria)"); + ret += tcrypt_test("ctr(aria)"); + break; case 200: test_cipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0, speed_template_16_24_32); @@ -2192,6 +2202,32 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) 0, speed_template_32); break; + case 227: + test_cipher_speed("ecb(aria)", ENCRYPT, sec, NULL, 0, + speed_template_16_24_32); + test_cipher_speed("ecb(aria)", DECRYPT, sec, NULL, 0, + speed_template_16_24_32); + test_cipher_speed("cbc(aria)", ENCRYPT, sec, NULL, 0, + speed_template_16_24_32); + test_cipher_speed("cbc(aria)", DECRYPT, sec, NULL, 0, + speed_template_16_24_32); + test_cipher_speed("cfb(aria)", ENCRYPT, sec, NULL, 0, + speed_template_16_24_32); + test_cipher_speed("cfb(aria)", DECRYPT, sec, NULL, 0, + speed_template_16_24_32); + test_cipher_speed("ctr(aria)", ENCRYPT, sec, NULL, 0, + speed_template_16_24_32); + test_cipher_speed("ctr(aria)", DECRYPT, sec, NULL, 0, + speed_template_16_24_32); + break; + + case 228: + test_aead_speed("gcm(aria)", ENCRYPT, sec, + NULL, 0, 16, 8, speed_template_16_24_32); + test_aead_speed("gcm(aria)", DECRYPT, sec, + NULL, 0, 16, 8, speed_template_16_24_32); + break; + case 300: if (alg) { test_hash_speed(alg, sec, generic_hash_speed_template); diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 7a8a56749960..5349ffee6bbd 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -4388,6 +4388,12 @@ static const struct alg_test_desc alg_test_descs[] = { .suite = { .cipher = __VECS(anubis_cbc_tv_template) }, + }, { + .alg = "cbc(aria)", + .test = alg_test_skcipher, + .suite = { + .cipher = __VECS(aria_cbc_tv_template) + }, }, { .alg = "cbc(blowfish)", .test = alg_test_skcipher, @@ -4505,6 +4511,12 @@ static const struct alg_test_desc alg_test_descs[] = { .suite = { .cipher = __VECS(aes_cfb_tv_template) }, + }, { + .alg = "cfb(aria)", + .test = alg_test_skcipher, + .suite = { + .cipher = __VECS(aria_cfb_tv_template) + }, }, { .alg = "cfb(sm4)", .test = alg_test_skcipher, @@ -4574,6 +4586,12 @@ static const struct alg_test_desc alg_test_descs[] = { .suite = { .cipher = __VECS(aes_ctr_tv_template) } + }, { + .alg = "ctr(aria)", + .test = alg_test_skcipher, + .suite = { + .cipher = __VECS(aria_ctr_tv_template) + } }, { .alg = "ctr(blowfish)", .test = alg_test_skcipher, @@ -4834,6 +4852,12 @@ static const struct alg_test_desc alg_test_descs[] = { .suite = { .cipher = __VECS(arc4_tv_template) } + }, { + .alg = "ecb(aria)", + .test = alg_test_skcipher, + .suite = { + .cipher = __VECS(aria_tv_template) + } }, { .alg = "ecb(blowfish)", .test = alg_test_skcipher, @@ -5050,6 +5074,13 @@ static const struct alg_test_desc alg_test_descs[] = { .suite = { .aead = __VECS(aes_gcm_tv_template) } + }, { + .alg = "gcm(aria)", + .generic_driver = "gcm_base(ctr(aria-generic),ghash-generic)", + .test = alg_test_aead, + .suite = { + .aead = __VECS(aria_gcm_tv_template) + } }, { .alg = "gcm(sm4)", .generic_driver = "gcm_base(ctr(sm4-generic),ghash-generic)", diff --git a/crypto/testmgr.h b/crypto/testmgr.h index f1dffdace219..dee88510f58d 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -26536,6 +26536,2866 @@ static const struct cipher_testvec seed_tv_template[] = { } }; +/* + * ARIA test vectors + */ +static const struct cipher_testvec aria_tv_template[] = { + { + .key = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + .klen = 16, + .ptext = "\x00\x11\x22\x33\x44\x55\x66\x77" + "\x88\x99\xaa\xbb\xcc\xdd\xee\xff", + .ctext = "\xd7\x18\xfb\xd6\xab\x64\x4c\x73" + "\x9d\xa9\x5f\x3b\xe6\x45\x17\x78", + .len = 16, + }, { + .key = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + "\x10\x11\x12\x13\x14\x15\x16\x17", + .klen = 24, + .ptext = "\x00\x11\x22\x33\x44\x55\x66\x77" + "\x88\x99\xaa\xbb\xcc\xdd\xee\xff", + .ctext = "\x26\x44\x9c\x18\x05\xdb\xe7\xaa" + "\x25\xa4\x68\xce\x26\x3a\x9e\x79", + .len = 16, + }, { + .key = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + "\x10\x11\x12\x13\x14\x15\x16\x17" + "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + .klen = 32, + .ptext = "\x00\x11\x22\x33\x44\x55\x66\x77" + "\x88\x99\xaa\xbb\xcc\xdd\xee\xff", + .ctext = "\xf9\x2b\xd7\xc7\x9f\xb7\x2e\x2f" + "\x2b\x8f\x80\xc1\x97\x2d\x24\xfc", + .len = 16, + } +}; + +static const struct cipher_testvec aria_cbc_tv_template[] = { + { + .key = "\x7c\x95\x0d\x07\xe6\x14\x98\x92" + "\x07\xac\x22\x41\x4d\x23\x27\x37", + .klen = 16, + .iv = "\x9d\xd5\x62\xce\x3d\x07\xd9\x89" + "\xf2\x78\x19\x4b\x65\x39\xc3\xc6", + .ptext = "\xcb\xbf\x47\x35\xc5\x37\xf0\x4e" + "\x85\x19\x21\x72\x33\x00\xde\x28", + .ctext = "\xf4\x80\x89\x89\x4a\x37\xda\x98" + "\x80\x52\x74\x75\xd9\xef\x58\xff", + .len = 16, + }, { + .key = "\x8f\xb9\x8d\xc9\xd7\x99\xfe\x7d" + "\xeb\x14\xaa\x65\xaf\x8c\x38\x1a", + .klen = 16, + .iv = "\xb1\x67\x46\x57\x0c\x64\x65\xf2" + "\x8c\x2f\x65\x11\x12\x33\xd4\x9a", + .ptext = "\x3a\xaf\xc1\xeb\x3c\x0c\xc5\xcc" + "\x10\x6e\x45\xa1\xd6\x89\xf1\xe5" + "\x74\xb6\x90\xd3\x81\x45\x00\x66" + "\x62\x15\x78\x84\xb2\x63\x11\x76", + .ctext = "\x3d\x7d\x3a\xeb\x23\x85\x3e\x72" + "\x12\x45\xbb\x5b\x42\x99\xec\xa0" + "\xa2\xbe\x75\xd6\xb1\xd8\xea\x6f" + "\x97\xfe\xfd\xcc\xfc\x08\x38\x00", + .len = 32, + }, { + .key = "\xe8\xe0\x85\x9c\x33\x06\x36\x5f" + "\xa9\xab\x72\x66\xa1\xd7\xf5\x0d", + .klen = 16, + .iv = "\x5d\xd3\xaf\x13\xed\x82\xc8\x92" + "\x4f\xf4\xe2\x35\xdb\x39\x9e\xa5", + .ptext = "\xdf\x73\x61\x44\x86\x2f\x58\x1e" + "\xfe\xf6\xb9\x1d\xd9\x1e\x4c\x7c" + "\xb4\xe6\x2b\x7d\x17\xc3\xc6\x5f" + "\x9d\xf4\x29\x8a\x55\x5c\x82\x0e" + "\x67\x91\xdd\x4b\xfb\x31\x33\xf1" + "\x56\x75\xa3\x2c\x46\x08\xff\x18", + .ctext = "\x85\x07\x8c\x88\x70\x7b\x39\xb8" + "\xfd\x1d\xa1\xd0\x89\x5f\x3f\x85" + "\x18\x5a\xde\x64\xbd\x54\xd5\x67" + "\xd1\x27\x4c\x98\x82\x76\xea\x22" + "\x52\x98\x79\xb4\x1d\xe8\x16\xd0" + "\xc6\xea\xf7\xbb\x38\x89\xf2\x5d", + .len = 48, + }, { + .key = "\xc1\x19\x8a\x7b\xc9\xaf\x00\xb3" + "\x92\x3c\xd7\xed\xe7\x76\xc5\x98", + .klen = 16, + .iv = "\xca\x62\x82\x1a\x5b\xb1\xcf\xc1" + "\xfb\x50\xb7\xfc\xb0\x3b\x15\xcb", + .ptext = "\xcb\x92\x56\x74\xc9\xee\x80\x78" + "\x78\xf5\x73\xc5\x5b\x2c\x70\x2d" + "\x4e\x0d\xd7\x17\x6d\x5a\x35\x74" + "\x33\xb0\x7d\xf5\xdf\x5f\x96\x7b" + "\x1c\x79\x16\xd0\xe0\x29\x4e\x94" + "\x95\x46\x86\x7a\x77\x28\x89\xb4" + "\x3d\xbb\x65\xab\xfb\xd1\x6c\xf4" + "\x47\xbd\x7e\x7f\x9b\x1d\x8b\x12", + .ctext = "\x69\xd2\x56\xdf\xa8\x1a\x97\xbd" + "\x69\xb5\xbb\x6b\x29\x1d\x5f\x0f" + "\xdf\x5f\x63\xc0\x83\x0b\xd7\xb1" + "\x31\x2d\xbf\x73\xe1\xe5\x5d\x0e" + "\x0c\x8d\xc4\x8a\xa9\xbd\x5f\xc7" + "\xb5\x61\xa0\x2b\x90\x64\x1a\xde" + "\xd2\xe1\x61\xb9\xce\xf4\x0b\x1c" + "\x9c\x43\x69\x6d\xb2\x32\x98\x44", + .len = 64, + }, { + .key = "\xfa\xf7\x53\xf6\xd6\x08\x70\xf1" + "\x32\x58\x97\x74\x04\x12\x1b\x14", + .klen = 16, + .iv = "\xdd\x93\xb2\x3e\xcb\xc1\x7c\x27" + "\x7f\x9e\x41\x03\xab\x1d\xfb\x77", + .ptext = "\xae\x34\x94\x50\x73\x32\xf0\x75" + "\x96\x53\x2e\x1a\xc9\x91\x2b\x37" + "\x77\xbe\x48\x39\xa7\xd0\x6e\xf7" + "\x22\x7c\x4f\xe7\xd8\x06\xee\x92" + "\x80\x57\x61\x45\x7f\x50\xd5\x0a" + "\x0b\x5e\xd4\xd6\x90\x4e\xc3\x04" + "\x52\x63\xaf\x02\x55\xa6\x49\x4b" + "\x7a\x7e\x2e\x95\xea\x80\x6c\x4b" + "\xb7\x88\x42\x3d\xc1\x09\x28\x97" + "\xd7\xa1\x0f\x0f\x1f\xf1\xea\x63", + .ctext = "\x6b\x83\x00\xf1\x79\xb2\x23\xbf" + "\x17\x26\x8a\xef\xd3\xe1\x0e\x82" + "\x5b\xc7\xde\x3e\x39\x72\x2d\xb0" + "\xad\x25\x3b\xe6\x3b\x9f\xe9\x4b" + "\x6e\xe8\x77\xf5\x9d\x7d\x00\xae" + "\x73\x7b\x81\xff\xe3\x55\x8e\x90" + "\xdf\xe4\xcd\xd5\xdc\x16\x8b\x7a" + "\xe5\x04\x92\x18\xff\xcc\x63\x1b" + "\x53\xf3\x26\x44\x5c\x48\x1d\xa2" + "\x1f\x3f\xe0\x8b\x8f\x6f\xc2\x38", + .len = 80, + }, { + .key = "\xb8\xab\x6d\x03\x9d\xec\x15\x0a" + "\xcd\xcd\x68\x73\xa9\x35\x7e\x8a", + .klen = 16, + .iv = "\x9d\xf1\xc0\xa0\x02\x06\xf0\x03" + "\x43\x45\x6a\x2e\x3f\x21\xa9\x3c", + .ptext = "\xef\xbe\x0c\xa3\x49\x4a\xda\x1e" + "\x64\x90\x85\xeb\xdc\xca\x2b\x37" + "\x78\xb7\x62\xd7\x0a\xee\x35\x38" + "\x97\x72\x6a\x99\xb8\x86\x07\x77" + "\x40\xc3\x14\x49\x1f\x67\xa1\x6e" + "\x87\xf0\x0b\x64\x4d\xea\x7c\x3a" + "\x91\x05\xb1\x48\xa1\x6a\x00\x1d" + "\x1b\x4f\x99\xb9\x52\xc9\x0c\xfd" + "\xf3\xe2\x0b\x5f\xe9\xec\x71\xe2" + "\x7d\x15\x84\x46\xc2\x3b\x77\x7b" + "\x30\x01\x34\x5c\x8f\x22\x58\x9a" + "\x17\x05\x7e\xf6\xd5\x92\xc0\xb4", + .ctext = "\x79\x50\x9b\x34\xd7\x22\x9a\x72" + "\x61\xd7\xd8\xa9\xdb\xcf\x2f\xb0" + "\x81\x11\xe3\xed\xa0\xe4\xbd\x8d" + "\xe6\xf2\x52\x52\x40\xec\x9f\x3b" + "\xd4\x48\xc6\xdf\xfd\x36\x90\x8a" + "\x2f\x3b\xb0\xfb\xf4\x2b\x99\xa5" + "\xb2\x39\xc7\x52\x57\x2b\xbc\xd7" + "\x3f\x06\x10\x15\x2e\xf7\xaa\x79" + "\xd6\x6a\xe5\x4e\x2d\x0f\x5f\xaf" + "\xf9\x5a\x63\x28\x33\xf0\x85\x8a" + "\x06\x45\xce\x73\xaa\x96\x1d\xcc" + "\x6e\xb9\x25\xb8\x4c\xfe\xeb\x64", + .len = 96, + }, { + .key = "\x50\x45\x7b\x4c\x6d\x80\x53\x62" + "\x90\x26\x77\xf8\x04\x65\x26\xe3", + .klen = 16, + .iv = "\x9d\xd3\x73\x7b\x9b\xbd\x45\x97" + "\xd2\xbb\xa1\xb9\x08\x88\x2c\x85", + .ptext = "\x9f\x11\xeb\x78\x74\xcc\x4e\xd6" + "\x06\x4b\x6d\xe4\xdb\x11\x91\x58" + "\x1f\xa4\xf6\x0e\x8f\xe4\xcf\xfc" + "\x95\x9a\x8b\x68\xb4\x54\x57\x58" + "\x27\x71\xe4\x4b\xc5\x78\x6a\x26" + "\x28\xae\xed\x71\x0e\xe7\xbf\xc3" + "\xff\x9c\x46\x7b\x31\x3e\xff\xb1" + "\xa8\xca\xc3\x6d\xa1\x9e\x49\x16" + "\x31\x8b\xed\x2d\x2a\x2b\xaf\x3b" + "\x3e\x74\x7f\x07\x67\x8e\xb8\x0d" + "\x86\xe2\xea\x2c\x4a\x74\xdc\x9f" + "\x53\x72\xd1\x2e\x97\x0d\x0b\xa5" + "\x05\x87\x8e\x86\x69\x8d\x26\xfb" + "\x90\xc8\xab\x0e\xac\xaf\x84\x1c", + .ctext = "\x3c\x91\xab\x71\xe4\x77\x3e\xb0" + "\x7f\x20\x2e\xd0\xe1\xbe\xfd\x3c" + "\x06\x6c\x36\x75\x46\x27\xfd\x2d" + "\xba\x0f\xf0\x3c\x6d\x1e\x4b\x20" + "\xe9\x5e\x30\xd8\x03\xc6\xa0\x86" + "\xa8\xc7\xa4\x7f\x0e\x1f\x35\x55" + "\x24\x53\x02\xd5\x77\x30\x73\xdc" + "\xa5\xaf\x19\x92\x5b\x36\x86\x0e" + "\xcf\xf2\x5c\x00\xde\x92\xbf\x89" + "\x76\x46\xd5\x26\xb1\x8d\xa4\xef" + "\x61\x7e\x78\xb4\x68\xf5\x5b\x1d" + "\x39\x65\x32\x3a\xad\xff\x8b\x37" + "\x60\xc2\x8a\xaf\x48\x96\x8b\x9f" + "\x12\x6c\x70\x77\x95\xf3\x58\xb0", + .len = 112, + }, { + .key = "\xf9\x9f\x6a\x87\xa1\x2d\x6e\xac" + "\xde\xbb\x3e\x15\x5e\x49\xa4\xef", + .klen = 16, + .iv = "\xeb\x8e\x4f\xbe\x4b\x47\xd6\x4f" + "\x65\xd0\xfa\xee\xa6\xf1\x2c\xda", + .ptext = "\xa3\xfa\x4f\xf6\x00\x12\xbe\xc1" + "\x90\xcc\x91\x88\xbd\xfb\x1c\xdb" + "\x2b\xc8\xb9\x3d\x98\x01\xc8\x1f" + "\x07\xb4\xf3\x10\x1d\xfd\xb7\x2e" + "\xcb\x1c\x1f\xe0\x2d\xca\xd3\xc7" + "\xb2\xce\x52\xf1\x7e\xcb\x7c\x50" + "\x0c\x5c\x53\x6b\x18\x62\x02\x54" + "\xbc\x9d\x1f\xda\xd9\x7a\x2d\xff" + "\xb8\x2c\x65\xad\xf1\xfe\xb6\xa4" + "\x8c\xe8\x0a\xb7\x67\x60\xcb\x38" + "\xd7\x72\xa5\xb1\x92\x13\x8e\xd4" + "\xcd\xb3\x04\xb5\xa1\x11\x96\x37" + "\xb3\x53\xa6\xc4\x14\x56\x6d\x42" + "\x66\x43\x40\x42\x41\x63\x11\x7a" + "\xd5\x34\x38\x75\xd0\xbc\x74\x89" + "\x82\x1d\x2c\x0a\x3e\x6a\xfb\xbd", + .ctext = "\x09\x58\xf3\x22\xe5\x10\xf6\x3d" + "\xba\xb1\xfa\x5a\x16\xfe\xc5\x32" + "\x3d\x34\x59\x2e\x81\xde\x99\x2f" + "\xeb\x6a\x97\x86\x1f\x47\x8d\xe6" + "\x87\x79\x0e\xfe\xa4\xca\x09\xdc" + "\x24\x9b\xbb\xb1\x90\x33\xce\xd7" + "\x62\xfd\xfd\xa3\x65\x50\x07\x7c" + "\x4c\xa2\x10\xc7\x32\x0a\x0d\x5e" + "\x22\x29\x40\x71\xe5\xcc\x3a\x5b" + "\x5b\x53\x51\xa5\x5b\xc1\x76\x05" + "\x84\x6e\xe3\x58\x2b\xf2\x28\x76" + "\x5c\x66\x90\xfe\x63\x30\x1c\x45" + "\x26\x34\x80\xfe\x76\x87\x5b\xb1" + "\x63\x10\x09\xf6\x9d\x35\xcb\xee" + "\x3c\x60\x9d\x77\x5b\x36\x70\x09" + "\x4b\x63\x63\x90\x97\x3a\x6c\x8a", + .len = 128, + }, { + .key = "\x04\xb9\x6c\x8f\x5e\x79\x02\x87" + "\x88\x06\x7c\xfa\xd3\x7b\x56\xfe", + .klen = 16, + .iv = "\x4b\xc8\x93\x20\x98\x04\xba\x5a" + "\x22\x04\x1f\x3f\x79\x2c\x63\x79", + .ptext = "\xf3\x85\x3e\x75\x97\x10\x7c\x5d" + "\x39\x5a\x46\x47\xe7\x51\xa3\xac" + "\x84\x56\x3f\x1b\xb3\x93\x6a\x2e" + "\xf7\x8f\x63\xbe\x18\xff\xd7\x53" + "\xc8\xe0\xa5\xde\x86\xc2\xe4\xab" + "\xc3\x67\x27\x91\x43\x8c\xff\x6c" + "\xc7\x07\xc2\xcd\xe9\x12\x8b\xef" + "\x47\xe7\x82\xed\xe3\x8d\x5e\x33" + "\xca\xf1\x28\x32\xf4\x38\x41\x59" + "\x6c\x54\xa6\x40\xb0\xd5\x73\x26" + "\x5b\x02\xa6\x9d\x01\x29\x26\x84" + "\x5b\x33\x04\x36\xa4\x7b\x00\x01" + "\x42\xe1\x4f\xda\xa9\x1a\x9b\x4e" + "\x7d\x4a\x4c\xbc\xf6\xd4\x06\xc2" + "\x89\x70\x72\xf5\xc5\x7f\x42\xd5" + "\x7b\x9c\x6f\x00\x21\x74\xc5\xa5" + "\x78\xd7\xa2\x3c\x6d\x0f\xfb\x74" + "\x3d\x70\x9f\x6d\xdd\x30\xc0\x28", + .ctext = "\xc0\x49\x98\xb9\xf6\x58\xeb\x56" + "\x36\x76\x7a\x40\x7c\x27\x80\x62" + "\xe3\xcb\x9c\x87\x2c\x03\xc2\x0c" + "\x82\x00\x50\xd2\xe4\x61\x4d\x54" + "\x88\x10\x6f\x0a\xb4\x25\x57\xba" + "\xf0\x07\xe3\x55\x06\xb3\x72\xe9" + "\x2f\x9f\x1e\x50\xa8\x15\x69\x71" + "\xe3\xe5\x50\x32\xe5\xe0\x47\x0f" + "\x3a\xaa\x7d\xc0\x09\x0e\xdb\x1a" + "\xae\xb6\xa5\x87\x63\xd6\xbe\x8b" + "\xb2\x3d\x10\x1e\xb3\x68\xcf\x8a" + "\xe5\xa8\x89\xa9\xfe\x79\x13\x77" + "\xc4\x3f\x6f\x9f\xdd\x76\x5b\xf2" + "\x05\x67\x8a\x58\xb4\x31\xac\x64" + "\x6f\xc4\xc1\x6b\x08\x79\x3f\xe5" + "\x1c\x9a\x66\x3f\x7d\x1f\x18\xb1" + "\x07\xa5\x7b\x4f\x2c\x43\x33\x84" + "\xab\x1b\xc0\x7d\x49\x2f\x27\x9b", + .len = 144, + }, { + .key = "\x99\x79\xaf\x3c\xfb\xbd\xe7\xca" + "\xee\x4a\x4d\xb2\x23\x1e\xb6\x07", + .klen = 16, + .iv = "\xb4\xfc\xaa\xc1\x08\xbf\x68\xb2" + "\xf6\xef\x29\xbc\x2d\x92\xa9\x40", + .ptext = "\xd3\x44\xe4\xd9\x6c\x8a\x1d\x4b" + "\xfe\x64\x25\xb6\x72\x21\xda\x10" + "\x3e\x77\xee\xd1\x41\xd3\xea\xf0" + "\xee\xee\x72\x0f\xad\xa1\xca\xf3" + "\x7e\xfa\x99\x36\xe0\x8f\xed\x40" + "\xf1\x12\x80\x73\xd6\x26\x3a\xa6" + "\x5d\x71\xf6\xd5\xe1\xf3\x89\x16" + "\x6f\x96\x00\xcf\x26\x06\x2a\x27" + "\xe4\xc2\x57\xba\x1f\x74\x5e\x91" + "\x10\x7e\xe5\x51\x17\xd5\xdc\xb2" + "\x5b\x12\x4b\x33\xb1\xc6\x4e\x0d" + "\xbf\x0e\x5d\x65\x61\x68\xd1\xc5" + "\x4b\xc5\xa4\xcd\xf0\xe0\x79\x26" + "\xa3\xcd\xdc\xb8\xfc\xd5\xca\x1d" + "\x7e\x81\x74\x55\x76\xf5\x40\xbb" + "\x26\x7f\x11\x37\x23\x70\xc8\xb6" + "\xfc\x2b\x0b\xd7\x1c\x7b\x45\xe7" + "\xf2\x2a\xed\x10\x4f\xcf\x0c\xcd" + "\x0f\xe7\xf9\xa1\xfb\x27\x67\x09" + "\xee\x11\xa2\xaf\x37\xc6\x16\xe0", + .ctext = "\x60\xce\x9a\xdb\xb2\xe8\xa2\x64" + "\x35\x9c\x5b\x97\x21\x9b\x95\x89" + "\x7b\x89\x15\x01\x97\x8b\xec\x9b" + "\xb9\xce\x7d\xb9\x9d\xcc\xd0\xa0" + "\xda\x39\x5d\xfd\xb9\x51\xe7\x2f" + "\xe7\x9b\x73\x1b\x07\xfb\xfd\xbb" + "\xce\x84\x68\x76\x12\xc9\x6c\x38" + "\xc0\xdc\x67\x96\x5e\x63\xcf\xe5" + "\x57\x84\x7a\x14\x8c\xab\x38\x94" + "\x1c\x27\xc3\xe0\x03\x58\xfe\x98" + "\x97\xfc\x96\xba\x65\x87\x1e\x44" + "\xf8\x00\x91\x6a\x14\x05\xf3\xf9" + "\x8e\x3e\x7a\x3c\x41\x96\x15\x4f" + "\xa8\xc0\x73\x1f\x1b\xeb\xaf\xec" + "\xc4\x5a\x35\xed\x42\x2f\x47\xea" + "\xfd\x2f\x29\xf6\x0f\x58\x8b\x3d" + "\x15\x81\xe3\xa4\xa6\x5f\x33\x33" + "\xe9\x0d\x06\x4f\x7f\x89\x2c\x3d" + "\x18\x45\x1f\xd1\xc5\x74\xf7\x52" + "\x2f\x9b\x72\x3d\x1f\xad\x12\x1b", + .len = 160, + }, { + .key = "\x7f\x92\xd5\x06\x30\x6b\xc0\x23" + "\x87\xa8\x8e\x6d\xc7\xc5\xd7\xf1" + "\x5f\xce\x89\xb3\xd5\x7f\x7f\xf0", + .klen = 24, + .iv = "\xfd\xab\x56\xa6\x6e\xda\x7c\x57" + "\x36\x36\x89\x09\xcd\xa8\xd3\x91", + .ptext = "\x48\x3e\x3c\x11\xcf\xd0\x4f\xc0" + "\x51\xe3\x8c\xe9\x76\xcd\xff\x37", + .ctext = "\x2d\x8f\x39\x71\x0a\x2c\xc9\x93" + "\xb6\x1a\x5c\x53\x06\x4d\xaa\xcf", + .len = 16, + }, { + .key = "\xd6\x1a\x18\x2f\x68\x2f\xb6\xfe" + "\x3d\x2d\x85\x75\x6e\x18\x8a\x52" + "\x53\x39\xfc\xc1\xf5\xc0\x56\x22", + .klen = 24, + .iv = "\xc6\xae\xaa\x0d\x90\xf2\x38\x93" + "\xac\xd2\x3f\xc7\x74\x8d\x13\x7e", + .ptext = "\xfa\x3f\x70\x52\xfb\x04\x0e\xed" + "\x0e\x60\x75\x84\x21\xdf\x13\xa1" + "\x26\xf8\x8c\x26\x0a\x37\x51\x8f" + "\xe7\x9c\x74\x77\x7a\x3e\xbb\x5d", + .ctext = "\xc1\x53\x86\xf8\x60\x5d\x72\x59" + "\x7e\xdf\xc8\xdb\x85\xd6\x9f\x2a" + "\xa1\xda\xe5\x85\x78\x4f\x1b\x6f" + "\x58\xf3\x2b\xff\x34\xe4\x97\x4e", + .len = 32, + }, { + .key = "\xd7\x33\xf3\xa9\x5b\xb4\x86\xea" + "\xe3\x7d\x50\x62\x3b\x73\xaf\xc4" + "\xda\x89\xd9\x3c\xcc\xe4\x73\xb0", + .klen = 24, + .iv = "\xef\x3e\x5f\x46\x62\x88\xd5\x26" + "\x3b\xd3\xb5\x81\x78\x70\x1b\xd2", + .ptext = "\x39\x56\x34\x63\x2c\xc5\x51\x13" + "\x48\x29\x3a\x58\xbe\x41\xc5\x80" + "\x2c\x80\xa7\x3c\x14\xb4\x89\x5e" + "\x8e\xe5\x5f\xe2\x39\x80\xf5\x2b" + "\x77\xb5\xca\x90\xda\x1d\x22\x17" + "\xd9\xa0\x57\x80\xc8\x96\x70\x86", + .ctext = "\x25\x5f\x66\x15\xb5\x62\xfb\x55" + "\xb3\x77\xa1\x7d\x03\xba\x86\x0a" + "\x0d\x5b\xbb\x06\xe9\xe2\xa8\x41" + "\xa3\x58\xd6\x4b\xcb\x7f\xd0\x15" + "\x3b\x02\x74\x5d\x4c\x4c\xb0\xa5" + "\x06\xc9\x59\x53\x2a\x36\xeb\x59", + .len = 48, + }, { + .key = "\x07\x2c\xf4\x61\x79\x09\x01\x8f" + "\x37\x32\x98\xd4\x86\x2b\x3b\x80" + "\x07\x60\xba\xf0\x2e\xc3\x4a\x57", + .klen = 24, + .iv = "\xf5\xb5\xd7\xbf\xd2\x2a\x9b\x4a" + "\xe6\x08\xf0\xbe\x77\xd1\x62\x40", + .ptext = "\xa0\x82\x09\x60\x47\xbb\x16\x56" + "\x50\x1f\xab\x8b\x10\xfe\xf0\x5c" + "\x05\x32\x63\x1a\xc4\x46\x6f\x55" + "\x32\xde\x41\x5a\xf7\x52\xd7\xfa" + "\x30\x9d\x59\x8d\x64\x76\xad\x37" + "\xba\xbc\x46\x6a\x69\x17\x3c\xac" + "\x6f\xdd\xa2\x9b\x86\x32\x14\x2e" + "\x54\x74\x8f\x3d\xe2\xd6\x85\x44", + .ctext = "\x91\x02\xa9\xd3\x4b\x9a\x8f\xe6" + "\x9f\xe4\x51\x57\xc9\x42\xda\x68" + "\xca\xf6\x54\x51\x90\xec\x20\x2e" + "\xab\x25\x6c\xd9\x8b\x99\xa6\x1c" + "\x72\xc9\x01\xd6\xbc\x2b\x26\x78" + "\x42\x00\x84\x0a\xdd\xa8\xd9\xb5" + "\xc6\xc8\x30\xb6\xab\xea\x71\x84" + "\xb2\x57\x97\x32\xdb\x35\x23\xd8", + .len = 64, + }, { + .key = "\x4f\x4a\x31\x64\xc6\xa5\x29\xaa" + "\xad\xfd\x32\x94\x1f\x56\x57\xd1" + "\x9d\x7e\x3d\x49\x00\x36\xb1\x5d", + .klen = 24, + .iv = "\xb2\x92\x83\x70\x1e\xa3\x97\xa6" + "\x65\x53\x39\xeb\x53\x8f\xb1\x38", + .ptext = "\x91\xac\x17\x11\x1c\x03\x69\x53" + "\xf5\xdf\xdb\x2c\x1b\x9a\x6e\x6b" + "\xb6\x02\xc4\xfa\x95\x01\x33\xa8" + "\xda\x7e\x18\x2c\xf4\x7e\x6e\x67" + "\xce\x8f\x9f\xea\x46\x66\x99\xb8" + "\xe1\xc7\x25\x4d\xbd\xa5\x74\xdf" + "\xc7\x8b\xfb\xe3\x2d\x3a\x82\xd3" + "\x17\x94\x77\x2f\x92\xb8\x87\xc2" + "\xcc\x6f\x70\x26\x87\xc7\x10\x8a" + "\xc8\xfd\xc2\xb3\xcf\xa0\xeb\x41", + .ctext = "\x28\x23\x3a\x4a\x18\xb7\xb6\x05" + "\xd4\x1b\x6a\x9e\xa7\xf2\x38\x01" + "\x78\xd3\xb0\x1b\x95\x68\x59\xf1" + "\xc0\xed\x30\x46\x2e\xb9\xa6\xdc" + "\xde\xef\xa6\x85\x19\xfc\x4d\x36" + "\x5d\x24\x92\x62\x75\x32\x76\x6d" + "\x6d\xa9\x07\xe1\x4f\x59\x84\x1a" + "\x68\x9a\x07\x48\xd3\x86\xf6\xf1" + "\x5b\xf9\x35\xec\x7c\xaf\x47\x13" + "\x9c\xc9\x33\x12\x10\x2f\x94\x8a", + .len = 80, + }, { + .key = "\x4c\xf4\xd0\x34\xd0\x95\xab\xae" + "\x82\x5c\xfd\xfa\x13\x86\x25\xce" + "\xf4\x13\x32\xcd\xc6\x6d\xf6\x50", + .klen = 24, + .iv = "\x12\x4a\x5b\x66\x3a\xd3\xfb\x1a" + "\xaf\x06\xea\xf4\x65\x59\xd6\xc2", + .ptext = "\x84\xa0\x53\x97\x61\x30\x70\x15" + "\xac\x45\x8e\xe8\xeb\xa1\x72\x93" + "\x26\x76\x98\x6f\xe4\x86\xca\xf0" + "\x57\x89\xf2\x2b\xd4\xcf\x2d\x95" + "\x86\x26\x20\x0e\x62\xfe\x8f\x1e" + "\x5d\xcb\x2b\x7e\xdd\xab\xac\xda" + "\x6e\x49\x20\xd5\xb7\x01\x83\x4e" + "\xac\x45\x8f\xe1\x05\x3f\xd5\xb1" + "\xee\xb7\x0d\x65\x00\x38\xab\x71" + "\x70\x6e\xb3\x97\x86\xd3\xcd\xad" + "\x51\x8b\x9c\xa0\x9a\x8b\x4c\xb9" + "\x16\x01\x6a\x1f\xdf\xf0\xf9\x9e", + .ctext = "\x38\x5b\x16\xef\xb8\x8c\x74\x7a" + "\x55\x17\x71\xa7\x7d\x34\xd7\x6a" + "\xc6\x31\x55\x6f\xbb\x61\xf4\x12" + "\x81\x8c\x91\x0d\x10\xdb\xd5\x22" + "\x77\x36\x32\xb6\x77\xb1\x5e\x21" + "\xb5\xec\xf9\x64\x04\x90\x6f\xc6" + "\x8a\x86\x23\xb5\xfe\xa4\xb6\x84" + "\x91\xa1\x60\xe3\xd7\xf3\xb9\xda" + "\x96\x23\x4a\xb3\xab\x75\x84\x04" + "\x15\x1a\xbb\xe8\x02\x1e\x80\x7c" + "\xc1\x93\x01\x0f\x5c\x4a\xde\x85" + "\xbb\x93\x05\x66\x53\x74\x40\x56", + .len = 96, + }, { + .key = "\x25\x1b\xc2\xa6\x21\x25\xeb\x97" + "\x4b\xf6\xcb\x3b\xcd\x61\xfd\x94" + "\x37\x03\xb3\xd9\x74\x6e\x4d\xbb", + .klen = 24, + .iv = "\xfd\x87\x2b\xec\x4c\x2c\xbf\xe2" + "\x94\x1a\xe6\xd9\xaf\x0e\x78\x17", + .ptext = "\x58\x2b\x1d\x73\x9a\x9c\x63\x18" + "\x88\x7a\x0e\x87\x2f\xf0\xb0\xdb" + "\xc9\x9d\x79\x51\x34\x39\x4f\x07" + "\xa2\x7c\x21\x04\x91\x3b\x79\x79" + "\xfe\xd5\x51\x46\xd5\xcd\x28\xc0" + "\xad\xb8\x55\xb2\xb2\x5a\x9a\xa2" + "\xe2\x0c\xfc\x55\x7d\x60\xd2\x95" + "\xb6\x08\x1d\x31\xaf\xf4\x17\x46" + "\xa4\xbb\x0f\xbd\x67\x3c\x73\x15" + "\x0c\x85\x2f\x62\xe5\xf4\x35\x96" + "\xb1\x9b\x5d\x00\x10\xe9\x70\x12" + "\x3a\x87\x7f\x67\xf1\x81\x7a\x05" + "\xb4\xa6\xfe\xdf\x36\x31\x6d\x9e" + "\x0e\xa9\x44\xa0\xb0\x05\xa9\x41", + .ctext = "\x4b\x56\xe0\xc2\x65\x2f\x7c\x6f" + "\xee\x22\xeb\x34\x1c\xa5\xb7\xc8" + "\x35\xd7\x51\xfd\x6a\xf4\xdd\xc3" + "\x38\xf4\xfc\x9d\x2e\xc2\x77\xb7" + "\x93\x8e\x8c\xb3\x44\x9b\xaf\xbb" + "\x99\xb9\xa8\x38\x1c\xfe\x63\xfb" + "\x1f\xa0\xaa\x35\x29\x7b\x87\x49" + "\x8e\x93\xa5\xb8\x5a\x85\x37\xa7" + "\x67\x69\x49\xbd\xc3\xfa\x89\x1c" + "\xf5\x60\x9b\xe7\x71\x96\x95\xd9" + "\x0b\x98\xe6\x74\x1d\xa3\xd9\x89" + "\x03\xe4\xf6\x66\xb3\x73\xb1\xac" + "\x9f\xee\x8f\xc2\x96\xcc\x97\x78" + "\x1b\x96\x63\x64\x00\x9c\x2d\x29", + .len = 112, + }, { + .key = "\x9c\x14\x44\x5a\xd5\x1c\x50\x08" + "\x95\xc2\xf2\xaf\x3f\x29\xc9\x3e" + "\x95\x5e\xc6\xb4\x2b\xf4\x3e\xe3", + .klen = 24, + .iv = "\x1b\xeb\x3d\x73\xfb\xd7\x1e\x2b" + "\x0c\x3d\x58\x6c\xb4\x41\x9b\xfe", + .ptext = "\x2f\x7e\x1c\x10\x81\x36\x2d\x79" + "\xaf\xab\x10\x44\x2e\xcc\x0d\x6c" + "\x9c\x14\xc2\xe4\xae\xb0\xbb\xda" + "\x6a\xe0\x42\x3d\x96\x9f\x78\x7d" + "\x70\x86\xa5\x92\x9f\xee\xcd\x3f" + "\x6a\x55\x84\x98\x28\x03\x02\xc2" + "\xf7\xec\x7a\xfa\xb1\xd9\xa8\xd8" + "\x1c\xc3\xaa\xd5\x61\x7f\x10\x0c" + "\xc0\xa1\x36\x3d\x81\x9a\xd2\x17" + "\x2e\x23\xc9\xb7\xff\xdf\x47\x6c" + "\x96\x3b\x0e\xbd\xec\x9a\x0e\xad" + "\x8c\xaf\x36\x3d\xff\x29\x8b\x33" + "\x87\x96\x77\x1a\x10\x81\x63\x8a" + "\x63\xde\x88\xa9\x9d\xa9\x01\xf2" + "\xdf\xc9\x25\x35\x48\x3a\x15\xdf" + "\x20\x6b\x91\x7c\x56\xe5\x10\x7a", + .ctext = "\x4d\x35\x70\xf1\x25\x02\x1d\x7f" + "\x9e\x0f\x5b\x4b\x65\xab\xcc\x6b" + "\x62\xab\x2b\xfa\xc0\x66\xee\x56" + "\xb4\x66\x95\x22\x84\x39\xd8\x3f" + "\x74\xba\x4f\x3f\xcd\xef\xcf\xf6" + "\x76\xeb\x9e\x8a\xec\x9c\x31\xa0" + "\x3e\x0c\xf9\xfa\x57\x90\xb4\x02" + "\xac\xc8\x28\xda\xa0\x05\xb7\x7e" + "\x75\x9c\x79\x36\xa9\x2f\x1a\x36" + "\x56\x77\xda\x74\xc7\xb3\xdf\xf3" + "\xb9\x83\x10\xf3\x6b\xe1\xdf\xcb" + "\x11\x70\xb1\xa0\x68\x48\x26\x95" + "\x10\x91\x94\xf3\xe9\x82\xb4\x8a" + "\xaa\xde\xf8\x9f\xce\x82\x47\x18" + "\x37\x5d\xda\x34\x74\x4d\x36\xbd" + "\xa5\x6c\xa4\xb3\x70\xad\x00\xbd", + .len = 128, + }, { + .key = "\x2d\x2e\x0f\x30\x32\xed\xa9\x1f" + "\x71\x4e\x68\x77\xe8\xa8\x5b\xdd" + "\x3c\x5e\x68\x6b\xab\x03\xe4\xf8", + .klen = 24, + .iv = "\x42\xc1\x61\x9a\x50\xfb\xc7\x6a" + "\x1a\x31\xa7\x87\xd0\x24\xcb\x5e", + .ptext = "\xc0\x3b\x12\x28\xca\x26\x7b\xb3" + "\x14\xc1\x7f\x66\xff\x3b\xa4\x80" + "\x59\x77\x4f\xa0\xd4\xb2\xd9\x8a" + "\xb6\x67\xe6\x28\xd3\x6f\xf2\xcf" + "\xb8\x6d\x2d\xc4\x2a\x69\x89\xff" + "\xcf\xbb\x11\x2e\x2a\x2b\x7c\xfd" + "\xcd\x56\x02\x95\xc9\x54\x6e\x62" + "\x6a\x97\x75\x1a\x21\x16\x46\xfb" + "\xc2\xab\x62\x54\xef\xba\xae\x46" + "\xd4\x14\xc6\xcc\x16\x1b\x95\xf9" + "\x05\x26\x23\x81\x19\x27\xad\x7b" + "\x9c\x8b\xfb\x65\xa4\x61\xee\x69" + "\x44\xbf\x59\xde\x03\x61\x11\x12" + "\x8d\x94\x48\x47\xa9\x52\x16\xfb" + "\x6b\xaf\x59\x6d\xab\x74\xbf\x5c" + "\xb6\x09\x21\x12\x42\x98\x13\xa1" + "\xa8\x6f\xb9\x6d\x4d\xa6\xdc\xea" + "\x61\x02\x3c\xa7\xcd\x1a\x28\x8c", + .ctext = "\xa1\x4a\x83\xb2\xe0\xef\x3d\x94" + "\xa4\x34\x66\x93\xb4\x89\x4e\x12" + "\xe5\x61\xc9\xea\xe0\x16\x96\x1a" + "\x3e\x94\x20\x81\xd4\x12\x7f\xf4" + "\xb8\x3f\xc9\xe2\x99\xb5\x0f\x9e" + "\x71\x86\x4f\x13\x78\x4e\xf1\x51" + "\xd4\x7d\x6e\x47\x31\x9a\xd8\xf7" + "\xb9\xb1\x17\xd0\xbd\xbf\x72\x86" + "\xb4\x58\x85\xf0\x05\x67\xc4\x00" + "\xca\xcb\xa7\x1a\x1d\x88\x29\xf4" + "\xe2\xf6\xdd\x5a\x3e\x5a\xbb\x29" + "\x48\x5a\x4a\x18\xcd\x5c\xf1\x09" + "\x5b\xbe\x1a\x43\x12\xc5\x6e\x6e" + "\x5e\x6d\x3b\x22\xf7\x58\xbd\xc8" + "\xb1\x04\xaf\x44\x9c\x2b\x98\x5a" + "\x14\xb7\x35\xb8\x9a\xce\x32\x28" + "\x1f\x8d\x08\x8a\xb9\x82\xf0\xa5" + "\x6a\x37\x29\xb6\x29\x3a\x53\x5e", + .len = 144, + }, { + .key = "\x66\xb8\x4d\x60\x67\x82\xcc\x8d" + "\x1e\xda\x8f\x28\xe5\x02\xdc\x2c" + "\x54\x84\x2a\x06\xb5\xd1\x34\x57", + .klen = 24, + .iv = "\xb8\x28\x4d\xf5\x69\xb9\xf3\x33" + "\x5e\x0b\xa6\x62\x35\x9b\xfb\x97", + .ptext = "\x3e\xc6\xec\xaf\x74\xe8\x72\x91" + "\xb2\xc6\x56\xb3\x23\x29\x43\xe0" + "\xfb\xcc\x21\x38\x64\x78\x9e\x78" + "\xbb\x6e\x0d\x7b\xfd\x05\x74\x01" + "\x7c\x94\xe0\xb0\xd7\x92\xfc\x58" + "\x28\xfc\xe2\x7b\x7f\xf7\x31\x0d" + "\x90\xb7\x60\x78\xa8\x9f\x52\xe3" + "\xe6\xaa\x2a\xb4\xa7\x09\x60\x53" + "\x42\x0e\x15\x31\xf6\x48\xa3\x0a" + "\x20\xf0\x79\x67\xb1\x83\x26\x66" + "\xe0\xb1\xb3\xbd\x1c\x76\x36\xfd" + "\x45\x87\xa4\x14\x1b\xef\xe7\x16" + "\xf7\xfa\x30\x3d\xb9\x52\x8f\x2e" + "\x01\x68\xc1\x7d\xa2\x15\x49\x74" + "\x53\x82\xc2\x10\xa8\x45\x73\x4d" + "\x41\xcc\x24\xa3\x42\xff\x30\xd1" + "\x02\x21\xdc\xd9\x08\xf7\xe7\x4c" + "\x33\x2d\x62\xc7\x38\xf5\xc2\xbe" + "\x52\xf1\x34\x78\x34\x53\x30\x5b" + "\x43\x43\x51\x6a\x02\x81\x64\x0c", + .ctext = "\xd9\xed\xc8\xc7\x66\xcd\x06\xc5" + "\xc1\x25\x9b\xf5\x14\x71\x1d\x69" + "\xc9\x7c\x04\x40\xab\xc0\x44\xf4" + "\xa1\xe6\x57\x8b\x35\x62\x4e\x3f" + "\xce\x4a\x99\xcd\x95\xc4\xd1\xf3" + "\xbc\x25\xa2\x18\xe6\xd1\xf7\xc0" + "\x13\x98\x60\x4c\x5c\xb1\x4f\x7a" + "\xbc\x45\x12\x52\xe8\x71\xb0\xf1" + "\x18\xef\x6f\x8a\x63\x35\x17\xae" + "\x90\x31\x41\x9d\xf4\xdc\x35\xcc" + "\x49\x72\x10\x11\x3b\xe3\x40\x7a" + "\x8e\x21\x39\xd0\x5b\x82\xb1\xe9" + "\x0c\x37\x5a\x7c\x11\xcb\x96\xd9" + "\xd4\x1c\x47\x4b\x70\xcb\xca\x08" + "\x5f\x71\xe9\x48\xf6\x29\xd8\xbb" + "\x5c\xad\x9b\x23\x9f\x62\xaf\xef" + "\x8e\xd8\x99\x1d\x60\xad\xc3\x6f" + "\xed\x06\x1a\xec\xfa\xc0\x0f\x0d" + "\xb7\x00\x02\x45\x7c\x94\x23\xb6" + "\xd7\x26\x6a\x16\x62\xc4\xd9\xee", + .len = 160, + }, { + .key = "\x7f\x92\xd5\x06\x30\x6b\xc0\x23" + "\x87\xa8\x8e\x6d\xc7\xc5\xd7\xf1" + "\x5f\xce\x89\xb3\xd5\x7f\x7f\xf0" + "\xfd\xab\x56\xa6\x6e\xda\x7c\x57", + .klen = 32, + .iv = "\x36\x36\x89\x09\xcd\xa8\xd3\x91" + "\x48\x3e\x3c\x11\xcf\xd0\x4f\xc0", + .ptext = "\x51\xe3\x8c\xe9\x76\xcd\xff\x37" + "\xd6\x1a\x18\x2f\x68\x2f\xb6\xfe", + .ctext = "\x05\x31\x46\x6d\xb8\xf4\x92\x64" + "\x46\xfd\x0d\x96\x60\x01\xd7\x94", + .len = 16, + }, { + .key = "\x3d\x2d\x85\x75\x6e\x18\x8a\x52" + "\x53\x39\xfc\xc1\xf5\xc0\x56\x22" + "\xc6\xae\xaa\x0d\x90\xf2\x38\x93" + "\xac\xd2\x3f\xc7\x74\x8d\x13\x7e", + .klen = 32, + .iv = "\xfa\x3f\x70\x52\xfb\x04\x0e\xed" + "\x0e\x60\x75\x84\x21\xdf\x13\xa1", + .ptext = "\x26\xf8\x8c\x26\x0a\x37\x51\x8f" + "\xe7\x9c\x74\x77\x7a\x3e\xbb\x5d" + "\xd7\x33\xf3\xa9\x5b\xb4\x86\xea" + "\xe3\x7d\x50\x62\x3b\x73\xaf\xc4", + .ctext = "\x24\x36\xe4\x14\xb7\xe1\x56\x8a" + "\xf3\xc5\xaf\x0e\xa7\xeb\xbd\xcd" + "\x2d\xe9\xd7\x19\xae\x24\x5d\x3b" + "\x1d\xfb\xdc\x21\xb3\x1a\x37\x0b", + .len = 32, + }, { + .key = "\xda\x89\xd9\x3c\xcc\xe4\x73\xb0" + "\xef\x3e\x5f\x46\x62\x88\xd5\x26" + "\x3b\xd3\xb5\x81\x78\x70\x1b\xd2" + "\x39\x56\x34\x63\x2c\xc5\x51\x13", + .klen = 32, + .iv = "\x48\x29\x3a\x58\xbe\x41\xc5\x80" + "\x2c\x80\xa7\x3c\x14\xb4\x89\x5e", + .ptext = "\x8e\xe5\x5f\xe2\x39\x80\xf5\x2b" + "\x77\xb5\xca\x90\xda\x1d\x22\x17" + "\xd9\xa0\x57\x80\xc8\x96\x70\x86" + "\x07\x2c\xf4\x61\x79\x09\x01\x8f" + "\x37\x32\x98\xd4\x86\x2b\x3b\x80" + "\x07\x60\xba\xf0\x2e\xc3\x4a\x57", + .ctext = "\x2e\x73\x60\xec\xd3\x95\x78\xe8" + "\x0f\x98\x1a\xc2\x92\x49\x0b\x49" + "\x71\x42\xf4\xb0\xaa\x8b\xf8\x53" + "\x16\xab\x6d\x74\xc0\xda\xab\xcd" + "\x85\x52\x11\x20\x2c\x59\x16\x00" + "\x26\x47\x4a\xea\x08\x5f\x38\x68", + .len = 48, + }, { + .key = "\xf5\xb5\xd7\xbf\xd2\x2a\x9b\x4a" + "\xe6\x08\xf0\xbe\x77\xd1\x62\x40" + "\xa0\x82\x09\x60\x47\xbb\x16\x56" + "\x50\x1f\xab\x8b\x10\xfe\xf0\x5c", + .klen = 32, + .iv = "\x05\x32\x63\x1a\xc4\x46\x6f\x55" + "\x32\xde\x41\x5a\xf7\x52\xd7\xfa", + .ptext = "\x30\x9d\x59\x8d\x64\x76\xad\x37" + "\xba\xbc\x46\x6a\x69\x17\x3c\xac" + "\x6f\xdd\xa2\x9b\x86\x32\x14\x2e" + "\x54\x74\x8f\x3d\xe2\xd6\x85\x44" + "\x4f\x4a\x31\x64\xc6\xa5\x29\xaa" + "\xad\xfd\x32\x94\x1f\x56\x57\xd1" + "\x9d\x7e\x3d\x49\x00\x36\xb1\x5d" + "\xb2\x92\x83\x70\x1e\xa3\x97\xa6", + .ctext = "\xfb\xd3\xc3\x8b\xf7\x89\xcc\x31" + "\xb1\x7f\xc3\x91\xdc\x04\xc6\xd7" + "\x33\xbd\xe0\xee\x0c\xd5\x70\xed" + "\x1b\x1d\xad\x49\x6f\x5c\xa1\x68" + "\xd7\x03\xc9\x65\xa7\x90\x30\x2b" + "\x26\xeb\xf4\x7a\xac\xcc\x03\xe1" + "\x6a\xe5\xdb\x23\x10\x8a\xcd\x70" + "\x39\x4d\x7a\xc9\xcd\x62\xd1\x65", + .len = 64, + }, { + .key = "\x65\x53\x39\xeb\x53\x8f\xb1\x38" + "\x91\xac\x17\x11\x1c\x03\x69\x53" + "\xf5\xdf\xdb\x2c\x1b\x9a\x6e\x6b" + "\xb6\x02\xc4\xfa\x95\x01\x33\xa8", + .klen = 32, + .iv = "\xda\x7e\x18\x2c\xf4\x7e\x6e\x67" + "\xce\x8f\x9f\xea\x46\x66\x99\xb8", + .ptext = "\xe1\xc7\x25\x4d\xbd\xa5\x74\xdf" + "\xc7\x8b\xfb\xe3\x2d\x3a\x82\xd3" + "\x17\x94\x77\x2f\x92\xb8\x87\xc2" + "\xcc\x6f\x70\x26\x87\xc7\x10\x8a" + "\xc8\xfd\xc2\xb3\xcf\xa0\xeb\x41" + "\x4c\xf4\xd0\x34\xd0\x95\xab\xae" + "\x82\x5c\xfd\xfa\x13\x86\x25\xce" + "\xf4\x13\x32\xcd\xc6\x6d\xf6\x50" + "\x12\x4a\x5b\x66\x3a\xd3\xfb\x1a" + "\xaf\x06\xea\xf4\x65\x59\xd6\xc2", + .ctext = "\xa2\x51\x28\xc2\x5e\x58\x1c\xaf" + "\x84\x92\x1c\xe1\x92\xf0\xf9\x9e" + "\xf2\xb3\xc6\x2b\x34\xd2\x8d\xa0" + "\xb3\xd7\x87\x56\xeb\xd9\x32\x6a" + "\xca\x90\x28\x26\x49\x34\xca\x41" + "\xce\xc5\x9e\xd6\xfe\x57\x71\x3c" + "\x98\xaf\xdd\xfc\x7d\xdf\x26\x7e" + "\xb7\x9c\xd5\x15\xe5\x81\x7a\x4f" + "\x4f\x4f\xe5\x77\xf2\x2e\x67\x68" + "\x52\xc1\xac\x28\x2c\x88\xf4\x38", + .len = 80, + }, { + .key = "\x84\xa0\x53\x97\x61\x30\x70\x15" + "\xac\x45\x8e\xe8\xeb\xa1\x72\x93" + "\x26\x76\x98\x6f\xe4\x86\xca\xf0" + "\x57\x89\xf2\x2b\xd4\xcf\x2d\x95", + .klen = 32, + .iv = "\x86\x26\x20\x0e\x62\xfe\x8f\x1e" + "\x5d\xcb\x2b\x7e\xdd\xab\xac\xda", + .ptext = "\x6e\x49\x20\xd5\xb7\x01\x83\x4e" + "\xac\x45\x8f\xe1\x05\x3f\xd5\xb1" + "\xee\xb7\x0d\x65\x00\x38\xab\x71" + "\x70\x6e\xb3\x97\x86\xd3\xcd\xad" + "\x51\x8b\x9c\xa0\x9a\x8b\x4c\xb9" + "\x16\x01\x6a\x1f\xdf\xf0\xf9\x9e" + "\x25\x1b\xc2\xa6\x21\x25\xeb\x97" + "\x4b\xf6\xcb\x3b\xcd\x61\xfd\x94" + "\x37\x03\xb3\xd9\x74\x6e\x4d\xbb" + "\xfd\x87\x2b\xec\x4c\x2c\xbf\xe2" + "\x94\x1a\xe6\xd9\xaf\x0e\x78\x17" + "\x58\x2b\x1d\x73\x9a\x9c\x63\x18", + .ctext = "\xd1\xce\xbe\xe0\x4a\x6e\x6d\x7f" + "\x89\x19\x28\xb1\xca\xe8\xc1\x9c" + "\x8c\x0b\x7d\x63\xfe\xff\x3d\xf4" + "\x65\x9e\xd6\xe7\x2f\x5a\xc1\x31" + "\x1e\xe7\x59\x27\x54\x92\xcc\xaa" + "\x5b\x3d\xeb\xe7\x96\xc1\x49\x54" + "\x18\xf3\x14\xaa\x56\x03\x28\x53" + "\xaa\x0a\x91\xdf\x92\x96\x9b\x06" + "\x1a\x24\x02\x09\xe7\xa6\xdc\x75" + "\xeb\x00\x1d\xf5\xf2\xa7\x4a\x9d" + "\x75\x80\xb7\x47\x63\xfc\xad\x18" + "\x85\x5f\xfc\x64\x03\x72\x38\xe7", + .len = 96, + }, { + .key = "\x88\x7a\x0e\x87\x2f\xf0\xb0\xdb" + "\xc9\x9d\x79\x51\x34\x39\x4f\x07" + "\xa2\x7c\x21\x04\x91\x3b\x79\x79" + "\xfe\xd5\x51\x46\xd5\xcd\x28\xc0", + .klen = 32, + .iv = "\xad\xb8\x55\xb2\xb2\x5a\x9a\xa2" + "\xe2\x0c\xfc\x55\x7d\x60\xd2\x95", + .ptext = "\xb6\x08\x1d\x31\xaf\xf4\x17\x46" + "\xa4\xbb\x0f\xbd\x67\x3c\x73\x15" + "\x0c\x85\x2f\x62\xe5\xf4\x35\x96" + "\xb1\x9b\x5d\x00\x10\xe9\x70\x12" + "\x3a\x87\x7f\x67\xf1\x81\x7a\x05" + "\xb4\xa6\xfe\xdf\x36\x31\x6d\x9e" + "\x0e\xa9\x44\xa0\xb0\x05\xa9\x41" + "\x9c\x14\x44\x5a\xd5\x1c\x50\x08" + "\x95\xc2\xf2\xaf\x3f\x29\xc9\x3e" + "\x95\x5e\xc6\xb4\x2b\xf4\x3e\xe3" + "\x1b\xeb\x3d\x73\xfb\xd7\x1e\x2b" + "\x0c\x3d\x58\x6c\xb4\x41\x9b\xfe" + "\x2f\x7e\x1c\x10\x81\x36\x2d\x79" + "\xaf\xab\x10\x44\x2e\xcc\x0d\x6c", + .ctext = "\x0b\x07\xdc\x6a\x47\x45\xd2\xb0" + "\xa3\xf2\x42\x2f\xa4\x79\x6b\x4c" + "\x53\x9c\x8a\x2f\x48\x9c\xf2\x89" + "\x73\x8b\xdd\x97\xde\x41\x06\xc8" + "\x8a\x30\x7a\xa9\x90\x4a\x43\xd0" + "\xd5\xee\x16\x51\x44\xda\xe4\xb8" + "\xe8\x5f\x6f\xef\x84\xf3\x44\x43" + "\xbd\xdc\xc3\xdf\x65\x2b\xaf\xf6" + "\xfe\xd0\x4a\x5b\x30\x47\x8c\xaf" + "\x8d\xed\x2d\x91\xa1\x03\x9a\x80" + "\x58\xdd\xaa\x8f\x3b\x6b\x39\x10" + "\xe5\x92\xbc\xac\xaa\x25\xa1\x13" + "\x7e\xaa\x03\x83\x05\x83\x11\xfe" + "\x19\x5f\x04\x01\x48\x00\x3b\x58", + .len = 112, + }, { + .key = "\x9c\x14\xc2\xe4\xae\xb0\xbb\xda" + "\x6a\xe0\x42\x3d\x96\x9f\x78\x7d" + "\x70\x86\xa5\x92\x9f\xee\xcd\x3f" + "\x6a\x55\x84\x98\x28\x03\x02\xc2", + .klen = 32, + .iv = "\xf7\xec\x7a\xfa\xb1\xd9\xa8\xd8" + "\x1c\xc3\xaa\xd5\x61\x7f\x10\x0c", + .ptext = "\xc0\xa1\x36\x3d\x81\x9a\xd2\x17" + "\x2e\x23\xc9\xb7\xff\xdf\x47\x6c" + "\x96\x3b\x0e\xbd\xec\x9a\x0e\xad" + "\x8c\xaf\x36\x3d\xff\x29\x8b\x33" + "\x87\x96\x77\x1a\x10\x81\x63\x8a" + "\x63\xde\x88\xa9\x9d\xa9\x01\xf2" + "\xdf\xc9\x25\x35\x48\x3a\x15\xdf" + "\x20\x6b\x91\x7c\x56\xe5\x10\x7a" + "\x2d\x2e\x0f\x30\x32\xed\xa9\x1f" + "\x71\x4e\x68\x77\xe8\xa8\x5b\xdd" + "\x3c\x5e\x68\x6b\xab\x03\xe4\xf8" + "\x42\xc1\x61\x9a\x50\xfb\xc7\x6a" + "\x1a\x31\xa7\x87\xd0\x24\xcb\x5e" + "\xc0\x3b\x12\x28\xca\x26\x7b\xb3" + "\x14\xc1\x7f\x66\xff\x3b\xa4\x80" + "\x59\x77\x4f\xa0\xd4\xb2\xd9\x8a", + .ctext = "\xfe\xba\x8f\x68\x47\x55\xaa\x61" + "\x48\xdd\xf3\x7c\xc4\xdc\xa6\x93" + "\x4e\x72\x3f\xc7\xd0\x2b\x9b\xac" + "\xc1\xb5\x95\xf8\x8e\x75\x62\x0c" + "\x05\x6a\x90\x76\x35\xed\x73\xf2" + "\x0f\x44\x3d\xaf\xd4\x00\xeb\x1d" + "\xad\x27\xf2\x2f\x55\x65\x91\x0f" + "\xe4\x04\x9c\xfb\x8a\x18\x22\x8e" + "\x21\xbe\x93\x09\xdd\x3e\x93\x34" + "\x60\x82\xcd\xff\x42\x10\xed\x43" + "\x3a\x4b\xb8\x5c\x6c\xa8\x9e\x1c" + "\x95\x6a\x17\xa7\xa3\xe0\x7d\xdb" + "\x6e\xca\xaf\xc1\x1f\xb2\x86\x15" + "\xf0\xc1\x55\x72\xf2\x74\x44\xeb" + "\x09\x09\x83\x8b\x2c\xc9\x63\x13" + "\x99\xe3\xe1\x4b\x5c\xf7\xb1\x04", + .len = 128, + }, { + .key = "\xb6\x67\xe6\x28\xd3\x6f\xf2\xcf" + "\xb8\x6d\x2d\xc4\x2a\x69\x89\xff" + "\xcf\xbb\x11\x2e\x2a\x2b\x7c\xfd" + "\xcd\x56\x02\x95\xc9\x54\x6e\x62", + .klen = 32, + .iv = "\x6a\x97\x75\x1a\x21\x16\x46\xfb" + "\xc2\xab\x62\x54\xef\xba\xae\x46", + .ptext = "\xd4\x14\xc6\xcc\x16\x1b\x95\xf9" + "\x05\x26\x23\x81\x19\x27\xad\x7b" + "\x9c\x8b\xfb\x65\xa4\x61\xee\x69" + "\x44\xbf\x59\xde\x03\x61\x11\x12" + "\x8d\x94\x48\x47\xa9\x52\x16\xfb" + "\x6b\xaf\x59\x6d\xab\x74\xbf\x5c" + "\xb6\x09\x21\x12\x42\x98\x13\xa1" + "\xa8\x6f\xb9\x6d\x4d\xa6\xdc\xea" + "\x61\x02\x3c\xa7\xcd\x1a\x28\x8c" + "\x66\xb8\x4d\x60\x67\x82\xcc\x8d" + "\x1e\xda\x8f\x28\xe5\x02\xdc\x2c" + "\x54\x84\x2a\x06\xb5\xd1\x34\x57" + "\xb8\x28\x4d\xf5\x69\xb9\xf3\x33" + "\x5e\x0b\xa6\x62\x35\x9b\xfb\x97" + "\x3e\xc6\xec\xaf\x74\xe8\x72\x91" + "\xb2\xc6\x56\xb3\x23\x29\x43\xe0" + "\xfb\xcc\x21\x38\x64\x78\x9e\x78" + "\xbb\x6e\x0d\x7b\xfd\x05\x74\x01", + .ctext = "\xa5\x19\x33\xad\x2d\x1a\x7b\x34" + "\xb0\x21\x68\x0e\x20\x11\x7a\x37" + "\xef\x35\x33\x64\x31\x0a\x42\x77" + "\x2c\x7f\x1a\x34\xd6\x93\x2d\xe9" + "\x26\xb9\x15\xec\x4f\x83\xbd\x48" + "\x5b\xe9\x63\xea\x10\x3b\xec\xfb" + "\xb0\x5e\x81\x90\xf0\x07\x43\xc4" + "\xda\x54\x69\x98\x13\x5d\x93\x16" + "\xca\x06\x81\x64\x36\xbe\x36\xa2" + "\xd4\xd8\x48\x63\xc7\x53\x39\x93" + "\x6d\x6b\xd6\x49\x00\x72\x5e\x02" + "\xc7\x88\x61\x0f\x10\x88\xd4\x9e" + "\x17\x81\xa4\xdc\x43\x4e\x83\x43" + "\xd4\xc3\xd7\x25\x9a\xd4\x76\xde" + "\x88\xe3\x98\x5a\x0e\x80\x23\xfb" + "\x49\xb3\x83\xf6\xb9\x16\x00\x06" + "\xa5\x06\x24\x17\x65\xbb\x68\xa9" + "\x56\x6d\xeb\xcd\x3c\x14\xd2\x64", + .len = 144, + }, { + .key = "\x7c\x94\xe0\xb0\xd7\x92\xfc\x58" + "\x28\xfc\xe2\x7b\x7f\xf7\x31\x0d" + "\x90\xb7\x60\x78\xa8\x9f\x52\xe3" + "\xe6\xaa\x2a\xb4\xa7\x09\x60\x53", + .klen = 32, + .iv = "\x42\x0e\x15\x31\xf6\x48\xa3\x0a" + "\x20\xf0\x79\x67\xb1\x83\x26\x66", + .ptext = "\xe0\xb1\xb3\xbd\x1c\x76\x36\xfd" + "\x45\x87\xa4\x14\x1b\xef\xe7\x16" + "\xf7\xfa\x30\x3d\xb9\x52\x8f\x2e" + "\x01\x68\xc1\x7d\xa2\x15\x49\x74" + "\x53\x82\xc2\x10\xa8\x45\x73\x4d" + "\x41\xcc\x24\xa3\x42\xff\x30\xd1" + "\x02\x21\xdc\xd9\x08\xf7\xe7\x4c" + "\x33\x2d\x62\xc7\x38\xf5\xc2\xbe" + "\x52\xf1\x34\x78\x34\x53\x30\x5b" + "\x43\x43\x51\x6a\x02\x81\x64\x0c" + "\xcd\x4b\xbf\x0f\xcb\x81\xd4\xec" + "\x1e\x07\x05\x4d\x5c\x6b\xba\xcc" + "\x43\xc7\xb1\xfe\xa8\xe9\x96\xb0" + "\xb1\xb2\xd4\x70\x44\xbc\xaa\x50" + "\xbf\x3f\x81\xe6\xea\x36\x7d\x97" + "\x2a\xbd\x52\x16\xf7\xbe\x59\x27" + "\x8f\xcc\xe3\xa9\xec\x4f\xcd\xd3" + "\xf4\xe2\x54\xbe\xf1\xf9\x2b\x23" + "\x40\xc7\xcb\x67\x4d\x5f\x0b\xd4" + "\xbf\x19\xf0\x2a\xef\x37\xc6\x56", + .ctext = "\x0a\x69\xd8\x67\x33\x2a\x2f\xa9" + "\x26\x79\x65\xd6\x75\x1e\x98\xe8" + "\x52\x56\x32\xbf\x67\x71\xf4\x01" + "\xb1\x6f\xef\xf9\xc9\xad\xb3\x49" + "\x7a\x4f\x24\x9a\xae\x06\x62\x26" + "\x3e\xe4\xa7\x6f\x5a\xbf\xe9\x52" + "\x13\x01\x74\x8b\x6e\xb1\x65\x24" + "\xaa\x8d\xbb\x54\x21\x20\x60\xa4" + "\xb7\xa5\xf9\x4e\x7b\xf5\x0b\x70" + "\xd2\xb9\xdc\x9b\xdb\x2c\xb2\x43" + "\xf7\x71\x30\xa5\x13\x6f\x16\x75" + "\xd0\xdf\x72\xae\xe4\xed\xc1\xa3" + "\x81\xe0\xd5\xc0\x0e\x62\xe8\xe5" + "\x86\x2c\x37\xde\xf8\xb0\x21\xe4" + "\xcd\xa6\x76\x9b\xa1\x56\xd3\x67" + "\x70\x69\xd6\x5d\xc7\x65\x19\x59" + "\x43\x9c\xca\x32\xe9\xd1\x48\x92" + "\x71\x79\x87\x73\x24\xcb\xc0\x0f" + "\x23\x3b\x8f\x51\x8a\xb3\x3a\x9c" + "\x74\xa4\x19\xa7\xe4\x4f\x6b\x32", + .len = 160, + } +}; + +static const struct cipher_testvec aria_ctr_tv_template[] = { + { + .key = "\x7f\x92\xd5\x06\x30\x6b\xc0\x23" + "\x87\xa8\x8e\x6d\xc7\xc5\xd7\xf1", + .klen = 16, + .iv = "\x5f\xce\x89\xb3\xd5\x7f\x7f\xf0" + "\xfd\xab\x56\xa6\x6e\xda\x7c\x57", + .ptext = "\x36\x36\x89\x09\xcd\xa8\xd3\x91" + "\x48\x3e\x3c\x11\xcf\xd0\x4f\xc0", + .ctext = "\x19\x28\xb5\xf2\x1c\xbc\xf8\xaf" + "\xb9\xae\x1b\x23\x4f\xe1\x6e\x40", + .len = 16, + }, { + .key = "\x51\xe3\x8c\xe9\x76\xcd\xff\x37" + "\xd6\x1a\x18\x2f\x68\x2f\xb6\xfe", + .klen = 16, + .iv = "\x3d\x2d\x85\x75\x6e\x18\x8a\x52" + "\x53\x39\xfc\xc1\xf5\xc0\x56\x22", + .ptext = "\xc6\xae\xaa\x0d\x90\xf2\x38\x93" + "\xac\xd2\x3f\xc7\x74\x8d\x13\x7e" + "\xfa\x3f\x70\x52\xfb\x04\x0e\xed" + "\x0e\x60\x75\x84\x21\xdf\x13\xa1", + .ctext = "\x3f\x8c\xa9\x19\xd6\xb4\xfb\xed" + "\x9c\x6d\xaa\x1b\xe1\xc1\xe6\xa8" + "\xa9\x0a\x63\xd3\xa2\x1e\x6b\xa8" + "\x52\x97\x1e\x81\x34\x6f\x98\x0e", + .len = 32, + }, { + .key = "\x26\xf8\x8c\x26\x0a\x37\x51\x8f" + "\xe7\x9c\x74\x77\x7a\x3e\xbb\x5d", + .klen = 16, + .iv = "\xd7\x33\xf3\xa9\x5b\xb4\x86\xea" + "\xe3\x7d\x50\x62\x3b\x73\xaf\xc4", + .ptext = "\xda\x89\xd9\x3c\xcc\xe4\x73\xb0" + "\xef\x3e\x5f\x46\x62\x88\xd5\x26" + "\x3b\xd3\xb5\x81\x78\x70\x1b\xd2" + "\x39\x56\x34\x63\x2c\xc5\x51\x13" + "\x48\x29\x3a\x58\xbe\x41\xc5\x80" + "\x2c\x80\xa7\x3c\x14\xb4\x89\x5e", + .ctext = "\x28\xd8\xa7\xf8\x74\x98\x00\xfc" + "\xd6\x48\xad\xbd\xbe\x3f\x0e\x7b" + "\x3d\x46\xfd\xde\x3e\x4f\x12\x43" + "\xac\x85\xda\xff\x70\x24\x44\x9d" + "\x1e\xf8\x9f\x30\xba\xca\xe0\x97" + "\x03\x6d\xe1\x1d\xc7\x21\x79\x37", + .len = 48, + }, { + .key = "\x8e\xe5\x5f\xe2\x39\x80\xf5\x2b" + "\x77\xb5\xca\x90\xda\x1d\x22\x17", + .klen = 16, + .iv = "\xd9\xa0\x57\x80\xc8\x96\x70\x86" + "\x07\x2c\xf4\x61\x79\x09\x01\x8f", + .ptext = "\x37\x32\x98\xd4\x86\x2b\x3b\x80" + "\x07\x60\xba\xf0\x2e\xc3\x4a\x57" + "\xf5\xb5\xd7\xbf\xd2\x2a\x9b\x4a" + "\xe6\x08\xf0\xbe\x77\xd1\x62\x40" + "\xa0\x82\x09\x60\x47\xbb\x16\x56" + "\x50\x1f\xab\x8b\x10\xfe\xf0\x5c" + "\x05\x32\x63\x1a\xc4\x46\x6f\x55" + "\x32\xde\x41\x5a\xf7\x52\xd7\xfa", + .ctext = "\x29\x31\x55\xd2\xe5\x0b\x81\x39" + "\xf9\xbc\x63\xe2\xfa\x26\x99\xde" + "\xde\x18\x93\x68\x81\x7b\x0a\x4d" + "\xf6\x03\xe1\xee\xf9\x0e\x1f\xe8" + "\xa8\x80\x81\x46\xdc\x24\x43\x3f" + "\xff\xfe\x8c\x3e\x17\x0a\x6d\xa2" + "\x47\x55\x62\xa0\x03\x4e\x48\x67" + "\xa2\x64\xc0\x9b\x6c\xa4\xfd\x6a", + .len = 64, + }, { + .key = "\x30\x9d\x59\x8d\x64\x76\xad\x37" + "\xba\xbc\x46\x6a\x69\x17\x3c\xac", + .klen = 16, + .iv = "\x6f\xdd\xa2\x9b\x86\x32\x14\x2e" + "\x54\x74\x8f\x3d\xe2\xd6\x85\x44", + .ptext = "\x4f\x4a\x31\x64\xc6\xa5\x29\xaa" + "\xad\xfd\x32\x94\x1f\x56\x57\xd1" + "\x9d\x7e\x3d\x49\x00\x36\xb1\x5d" + "\xb2\x92\x83\x70\x1e\xa3\x97\xa6" + "\x65\x53\x39\xeb\x53\x8f\xb1\x38" + "\x91\xac\x17\x11\x1c\x03\x69\x53" + "\xf5\xdf\xdb\x2c\x1b\x9a\x6e\x6b" + "\xb6\x02\xc4\xfa\x95\x01\x33\xa8" + "\xda\x7e\x18\x2c\xf4\x7e\x6e\x67" + "\xce\x8f\x9f\xea\x46\x66\x99\xb8", + .ctext = "\x38\xbc\xf5\x9d\x0e\x26\xa6\x18" + "\x95\x0b\x23\x54\x09\xa1\xf9\x46" + "\x12\xf1\x42\x57\xa1\xaa\x52\xfa" + "\x8a\xbd\xf2\x03\x63\x4e\xbc\xf7" + "\x21\xea\xed\xca\xdd\x42\x41\x94" + "\xe4\x6c\x07\x06\x19\x59\x30\xff" + "\x8c\x9d\x51\xbf\x2c\x2e\x5b\xa5" + "\x7d\x11\xec\x6b\x21\x08\x12\x18" + "\xe4\xdf\x5a\xfd\xa6\x5f\xee\x2f" + "\x5c\x24\xb7\xea\xc1\xcd\x6d\x68", + .len = 80, + }, { + .key = "\xe1\xc7\x25\x4d\xbd\xa5\x74\xdf" + "\xc7\x8b\xfb\xe3\x2d\x3a\x82\xd3", + .klen = 16, + .iv = "\x17\x94\x77\x2f\x92\xb8\x87\xc2" + "\xcc\x6f\x70\x26\x87\xc7\x10\x8a", + .ptext = "\xc8\xfd\xc2\xb3\xcf\xa0\xeb\x41" + "\x4c\xf4\xd0\x34\xd0\x95\xab\xae" + "\x82\x5c\xfd\xfa\x13\x86\x25\xce" + "\xf4\x13\x32\xcd\xc6\x6d\xf6\x50" + "\x12\x4a\x5b\x66\x3a\xd3\xfb\x1a" + "\xaf\x06\xea\xf4\x65\x59\xd6\xc2" + "\x84\xa0\x53\x97\x61\x30\x70\x15" + "\xac\x45\x8e\xe8\xeb\xa1\x72\x93" + "\x26\x76\x98\x6f\xe4\x86\xca\xf0" + "\x57\x89\xf2\x2b\xd4\xcf\x2d\x95" + "\x86\x26\x20\x0e\x62\xfe\x8f\x1e" + "\x5d\xcb\x2b\x7e\xdd\xab\xac\xda", + .ctext = "\xdf\x79\x58\x30\x6f\x47\x12\x78" + "\x04\xb2\x0b\x1a\x62\x22\xe2\x9f" + "\xfe\xc2\xf5\x6d\x9e\x0e\x2e\x56" + "\x76\x01\x7f\x25\x8f\x6e\xc5\xf3" + "\x91\xff\xcd\x67\xc6\xae\x0b\x01" + "\x4d\x5f\x40\x25\x88\xc5\xe0\x3d" + "\x37\x62\x12\x58\xfe\xc5\x4a\x21" + "\x4a\x86\x8d\x94\xdd\xfd\xe6\xf6" + "\x1e\xa6\x78\x4f\x90\x66\xda\xe4" + "\x4e\x64\xa8\x05\xc6\xd8\x7d\xfb" + "\xac\xc9\x1d\x14\xb5\xb0\xfa\x9c" + "\xe8\x84\xef\x87\xbe\xb4\x2a\x87", + .len = 96, + }, { + .key = "\x6e\x49\x20\xd5\xb7\x01\x83\x4e" + "\xac\x45\x8f\xe1\x05\x3f\xd5\xb1", + .klen = 16, + .iv = "\xee\xb7\x0d\x65\x00\x38\xab\x71" + "\x70\x6e\xb3\x97\x86\xd3\xcd\xad", + .ptext = "\x51\x8b\x9c\xa0\x9a\x8b\x4c\xb9" + "\x16\x01\x6a\x1f\xdf\xf0\xf9\x9e" + "\x25\x1b\xc2\xa6\x21\x25\xeb\x97" + "\x4b\xf6\xcb\x3b\xcd\x61\xfd\x94" + "\x37\x03\xb3\xd9\x74\x6e\x4d\xbb" + "\xfd\x87\x2b\xec\x4c\x2c\xbf\xe2" + "\x94\x1a\xe6\xd9\xaf\x0e\x78\x17" + "\x58\x2b\x1d\x73\x9a\x9c\x63\x18" + "\x88\x7a\x0e\x87\x2f\xf0\xb0\xdb" + "\xc9\x9d\x79\x51\x34\x39\x4f\x07" + "\xa2\x7c\x21\x04\x91\x3b\x79\x79" + "\xfe\xd5\x51\x46\xd5\xcd\x28\xc0" + "\xad\xb8\x55\xb2\xb2\x5a\x9a\xa2" + "\xe2\x0c\xfc\x55\x7d\x60\xd2\x95", + .ctext = "\xe4\x25\x0d\x22\xeb\xbe\x5e\x90" + "\x01\xe5\xae\xc9\x94\xbd\x93\x89" + "\x5f\x98\xf1\x46\x6a\x50\x3b\xa2" + "\x79\xd9\xe4\x9c\x9a\xde\xf2\x8c" + "\x25\x49\x4c\xda\xb4\x2c\x76\xab" + "\x0a\xa8\x51\xaf\xc0\x62\x1b\xe9" + "\xe9\x7a\x35\x6a\x4b\x1f\x48\x00" + "\xeb\x24\x1d\x5e\xdd\x06\x09\x23" + "\x2a\xfa\x8f\x3b\x3e\x9e\x14\x6f" + "\x2a\x3c\xef\x6d\x73\x67\xdd\x6c" + "\xc8\xa5\x57\xc8\x02\xb6\x9a\xe8" + "\x8d\xcf\x10\xfa\x3e\x9c\x4d\xeb" + "\x44\xd2\x05\x31\x40\x94\x77\x87" + "\xf0\x83\xb5\xd2\x2a\x9c\xbc\xe4", + .len = 112, + }, { + .key = "\xb6\x08\x1d\x31\xaf\xf4\x17\x46" + "\xa4\xbb\x0f\xbd\x67\x3c\x73\x15", + .klen = 16, + .iv = "\x0c\x85\x2f\x62\xe5\xf4\x35\x96" + "\xb1\x9b\x5d\x00\x10\xe9\x70\x12", + .ptext = "\x3a\x87\x7f\x67\xf1\x81\x7a\x05" + "\xb4\xa6\xfe\xdf\x36\x31\x6d\x9e" + "\x0e\xa9\x44\xa0\xb0\x05\xa9\x41" + "\x9c\x14\x44\x5a\xd5\x1c\x50\x08" + "\x95\xc2\xf2\xaf\x3f\x29\xc9\x3e" + "\x95\x5e\xc6\xb4\x2b\xf4\x3e\xe3" + "\x1b\xeb\x3d\x73\xfb\xd7\x1e\x2b" + "\x0c\x3d\x58\x6c\xb4\x41\x9b\xfe" + "\x2f\x7e\x1c\x10\x81\x36\x2d\x79" + "\xaf\xab\x10\x44\x2e\xcc\x0d\x6c" + "\x9c\x14\xc2\xe4\xae\xb0\xbb\xda" + "\x6a\xe0\x42\x3d\x96\x9f\x78\x7d" + "\x70\x86\xa5\x92\x9f\xee\xcd\x3f" + "\x6a\x55\x84\x98\x28\x03\x02\xc2" + "\xf7\xec\x7a\xfa\xb1\xd9\xa8\xd8" + "\x1c\xc3\xaa\xd5\x61\x7f\x10\x0c", + .ctext = "\xa7\x4c\x96\x55\x7c\x07\xce\xb2" + "\x6f\x63\x9f\xc6\x8b\x6f\xc6\x4a" + "\x2c\x47\x8d\x99\xdf\x65\x75\x96" + "\xb7\x1d\x50\x5b\x57\x4a\x69\xcc" + "\xc9\x3a\x18\x8a\xd1\xab\x70\x4a" + "\xa3\x13\x80\xdd\x48\xc0\x6a\x7d" + "\x21\xa8\x22\x06\x32\x47\xc0\x16" + "\x1f\x9a\xc0\x21\x33\x66\xf2\xd8" + "\x69\x79\xae\x02\x82\x3f\xaf\xa6" + "\x98\xdb\xcd\x2a\xe5\x12\x39\x80" + "\x8a\xc1\x73\x99\xe5\xe4\x17\xe3" + "\x56\xc2\x43\xa6\x41\x6b\xb2\xa4" + "\x9f\x81\xc4\xe9\xf4\x29\x65\x50" + "\x69\x81\x80\x4b\x86\xab\x5e\x30" + "\xd0\x81\x9d\x6f\x24\x59\x42\xc7" + "\x6d\x5e\x41\xb8\xf5\x99\xc2\xae", + .len = 128, + }, { + .key = "\xc0\xa1\x36\x3d\x81\x9a\xd2\x17" + "\x2e\x23\xc9\xb7\xff\xdf\x47\x6c", + .klen = 16, + .iv = "\x96\x3b\x0e\xbd\xec\x9a\x0e\xad" + "\x8c\xaf\x36\x3d\xff\x29\x8b\x33", + .ptext = "\x87\x96\x77\x1a\x10\x81\x63\x8a" + "\x63\xde\x88\xa9\x9d\xa9\x01\xf2" + "\xdf\xc9\x25\x35\x48\x3a\x15\xdf" + "\x20\x6b\x91\x7c\x56\xe5\x10\x7a" + "\x2d\x2e\x0f\x30\x32\xed\xa9\x1f" + "\x71\x4e\x68\x77\xe8\xa8\x5b\xdd" + "\x3c\x5e\x68\x6b\xab\x03\xe4\xf8" + "\x42\xc1\x61\x9a\x50\xfb\xc7\x6a" + "\x1a\x31\xa7\x87\xd0\x24\xcb\x5e" + "\xc0\x3b\x12\x28\xca\x26\x7b\xb3" + "\x14\xc1\x7f\x66\xff\x3b\xa4\x80" + "\x59\x77\x4f\xa0\xd4\xb2\xd9\x8a" + "\xb6\x67\xe6\x28\xd3\x6f\xf2\xcf" + "\xb8\x6d\x2d\xc4\x2a\x69\x89\xff" + "\xcf\xbb\x11\x2e\x2a\x2b\x7c\xfd" + "\xcd\x56\x02\x95\xc9\x54\x6e\x62" + "\x6a\x97\x75\x1a\x21\x16\x46\xfb" + "\xc2\xab\x62\x54\xef\xba\xae\x46", + .ctext = "\x11\x7f\xea\x49\xaf\x24\x52\xa2" + "\xde\x60\x99\x58\x23\xf9\x9e\x91" + "\x73\xd5\x9a\xcb\xdd\x10\xcd\x68" + "\xb8\x9e\xef\xa4\xe9\x2d\xf0\x27" + "\x44\xd4\x9a\xd6\xb6\x9c\x7a\xec" + "\x17\x17\xea\xa7\x8e\xa8\x40\x6b" + "\x43\x3d\x50\x59\x0f\x74\x1b\x9e" + "\x03\xed\x4f\x2f\xb8\xda\xef\xc3" + "\x3f\x29\xb3\xf4\x5c\xcd\xce\x3c" + "\xba\xfb\xc6\xd1\x1d\x6f\x61\x3a" + "\x2b\xbd\xde\x30\xc5\x53\xe0\x6e" + "\xbe\xae\x2f\x81\x13\x0f\xd2\xd5" + "\x14\xda\xd3\x60\x9c\xf8\x00\x86" + "\xe9\x97\x3e\x05\xb3\x95\xb3\x21" + "\x1f\x3c\x56\xef\xcb\x32\x49\x5c" + "\x89\xf1\x34\xe4\x8d\x7f\xde\x01" + "\x1f\xd9\x25\x6d\x34\x1d\x6b\x71" + "\xc9\xa9\xd6\x14\x1a\xf1\x44\x59", + .len = 144, + }, { + .key = "\xd4\x14\xc6\xcc\x16\x1b\x95\xf9" + "\x05\x26\x23\x81\x19\x27\xad\x7b", + .klen = 16, + .iv = "\x9c\x8b\xfb\x65\xa4\x61\xee\x69" + "\x44\xbf\x59\xde\x03\x61\x11\x12", + .ptext = "\x8d\x94\x48\x47\xa9\x52\x16\xfb" + "\x6b\xaf\x59\x6d\xab\x74\xbf\x5c" + "\xb6\x09\x21\x12\x42\x98\x13\xa1" + "\xa8\x6f\xb9\x6d\x4d\xa6\xdc\xea" + "\x61\x02\x3c\xa7\xcd\x1a\x28\x8c" + "\x66\xb8\x4d\x60\x67\x82\xcc\x8d" + "\x1e\xda\x8f\x28\xe5\x02\xdc\x2c" + "\x54\x84\x2a\x06\xb5\xd1\x34\x57" + "\xb8\x28\x4d\xf5\x69\xb9\xf3\x33" + "\x5e\x0b\xa6\x62\x35\x9b\xfb\x97" + "\x3e\xc6\xec\xaf\x74\xe8\x72\x91" + "\xb2\xc6\x56\xb3\x23\x29\x43\xe0" + "\xfb\xcc\x21\x38\x64\x78\x9e\x78" + "\xbb\x6e\x0d\x7b\xfd\x05\x74\x01" + "\x7c\x94\xe0\xb0\xd7\x92\xfc\x58" + "\x28\xfc\xe2\x7b\x7f\xf7\x31\x0d" + "\x90\xb7\x60\x78\xa8\x9f\x52\xe3" + "\xe6\xaa\x2a\xb4\xa7\x09\x60\x53" + "\x42\x0e\x15\x31\xf6\x48\xa3\x0a" + "\x20\xf0\x79\x67\xb1\x83\x26\x66", + .ctext = "\x5b\xc0\xe8\x17\xa4\xf9\xea\xce" + "\x9e\xf9\xe0\xb1\xac\x37\xe9\x41" + "\x0b\x57\xc6\x55\x54\x50\xfa\xa9" + "\x60\xaf\x7a\x4e\x98\x56\xde\x81" + "\x14\xfc\xac\x21\x81\x3e\xf4\x0f" + "\x40\x92\x30\xa8\x16\x88\x1a\xc3" + "\xf1\x39\xbd\x0a\xb9\x44\xc8\x67" + "\x8c\xaa\x2b\x45\x8b\x5b\x7b\x24" + "\xd5\xd8\x9e\xd3\x59\xa5\xd7\x69" + "\xdf\xf4\x50\xf9\x5f\x4f\x44\x1f" + "\x2c\x75\x68\x6e\x3a\xa8\xae\x4b" + "\x84\xf0\x42\x6c\xc0\x3c\x42\xaf" + "\x87\x2b\x89\xe9\x51\x69\x16\x63" + "\xc5\x62\x13\x05\x4c\xb2\xa9\x69" + "\x01\x14\x73\x88\x8e\x41\x47\xb6" + "\x68\x74\xbc\xe9\xad\xda\x94\xa1" + "\x0c\x12\x8e\xd4\x38\x15\x02\x97" + "\x27\x72\x4d\xdf\x61\xcc\x86\x3d" + "\xd6\x32\x4a\xc3\xa9\x4c\x35\x4f" + "\x5b\x91\x7d\x5c\x79\x59\xb3\xd5", + .len = 160, + }, { + .key = "\x7f\x92\xd5\x06\x30\x6b\xc0\x23" + "\x87\xa8\x8e\x6d\xc7\xc5\xd7\xf1" + "\x5f\xce\x89\xb3\xd5\x7f\x7f\xf0", + .klen = 24, + .iv = "\xfd\xab\x56\xa6\x6e\xda\x7c\x57" + "\x36\x36\x89\x09\xcd\xa8\xd3\x91", + .ptext = "\x48\x3e\x3c\x11\xcf\xd0\x4f\xc0" + "\x51\xe3\x8c\xe9\x76\xcd\xff\x37", + .ctext = "\xa4\x12\x2f\xc4\xf0\x6d\xd9\x46" + "\xe4\xe6\xd1\x0b\x6d\x14\xf0\x8f", + .len = 16, + }, { + .key = "\xd6\x1a\x18\x2f\x68\x2f\xb6\xfe" + "\x3d\x2d\x85\x75\x6e\x18\x8a\x52" + "\x53\x39\xfc\xc1\xf5\xc0\x56\x22", + .klen = 24, + .iv = "\xc6\xae\xaa\x0d\x90\xf2\x38\x93" + "\xac\xd2\x3f\xc7\x74\x8d\x13\x7e", + .ptext = "\xfa\x3f\x70\x52\xfb\x04\x0e\xed" + "\x0e\x60\x75\x84\x21\xdf\x13\xa1" + "\x26\xf8\x8c\x26\x0a\x37\x51\x8f" + "\xe7\x9c\x74\x77\x7a\x3e\xbb\x5d", + .ctext = "\x80\x2b\xf0\x88\xb9\x4b\x8d\xf5" + "\xc3\x0e\x15\x5b\xea\x5d\x5b\xa8" + "\x07\x95\x78\x72\xc0\xb9\xbf\x25" + "\x33\x22\xd1\x05\x56\x46\x62\x25", + .len = 32, + }, { + .key = "\xd7\x33\xf3\xa9\x5b\xb4\x86\xea" + "\xe3\x7d\x50\x62\x3b\x73\xaf\xc4" + "\xda\x89\xd9\x3c\xcc\xe4\x73\xb0", + .klen = 24, + .iv = "\xef\x3e\x5f\x46\x62\x88\xd5\x26" + "\x3b\xd3\xb5\x81\x78\x70\x1b\xd2", + .ptext = "\x39\x56\x34\x63\x2c\xc5\x51\x13" + "\x48\x29\x3a\x58\xbe\x41\xc5\x80" + "\x2c\x80\xa7\x3c\x14\xb4\x89\x5e" + "\x8e\xe5\x5f\xe2\x39\x80\xf5\x2b" + "\x77\xb5\xca\x90\xda\x1d\x22\x17" + "\xd9\xa0\x57\x80\xc8\x96\x70\x86", + .ctext = "\x65\x01\x3c\xb0\xac\x4c\x63\xb6" + "\xe7\xf1\xf4\x61\x35\xf4\x36\xde" + "\x7f\x85\xba\x41\xa8\xb0\x27\x11" + "\x86\x2c\x71\x16\x05\x1d\xcf\x70" + "\x35\xef\x23\x17\xfc\xed\x3f\x1a" + "\x8e\xb3\xe5\xdb\x90\xb4\xb8\x35", + .len = 48, + }, { + .key = "\x07\x2c\xf4\x61\x79\x09\x01\x8f" + "\x37\x32\x98\xd4\x86\x2b\x3b\x80" + "\x07\x60\xba\xf0\x2e\xc3\x4a\x57", + .klen = 24, + .iv = "\xf5\xb5\xd7\xbf\xd2\x2a\x9b\x4a" + "\xe6\x08\xf0\xbe\x77\xd1\x62\x40", + .ptext = "\xa0\x82\x09\x60\x47\xbb\x16\x56" + "\x50\x1f\xab\x8b\x10\xfe\xf0\x5c" + "\x05\x32\x63\x1a\xc4\x46\x6f\x55" + "\x32\xde\x41\x5a\xf7\x52\xd7\xfa" + "\x30\x9d\x59\x8d\x64\x76\xad\x37" + "\xba\xbc\x46\x6a\x69\x17\x3c\xac" + "\x6f\xdd\xa2\x9b\x86\x32\x14\x2e" + "\x54\x74\x8f\x3d\xe2\xd6\x85\x44", + .ctext = "\x5a\xfb\xb1\x2c\x6e\xe5\xb8\xe0" + "\x80\xb6\x77\xa8\xfe\x10\x3a\x99" + "\x00\x8e\x30\x23\x7d\x50\x87\xda" + "\xc6\x46\x73\x37\x8b\xf1\xab\x26" + "\x2d\xa8\x0c\xa8\x9e\x77\xee\xfc" + "\x78\x4f\x03\x0f\xeb\xc6\x03\x34" + "\xb9\x9c\x4f\x59\x55\xc5\x99\x47" + "\xd4\x7e\xe8\x06\x43\x5f\xa1\x6b", + .len = 64, + }, { + .key = "\x4f\x4a\x31\x64\xc6\xa5\x29\xaa" + "\xad\xfd\x32\x94\x1f\x56\x57\xd1" + "\x9d\x7e\x3d\x49\x00\x36\xb1\x5d", + .klen = 24, + .iv = "\xb2\x92\x83\x70\x1e\xa3\x97\xa6" + "\x65\x53\x39\xeb\x53\x8f\xb1\x38", + .ptext = "\x91\xac\x17\x11\x1c\x03\x69\x53" + "\xf5\xdf\xdb\x2c\x1b\x9a\x6e\x6b" + "\xb6\x02\xc4\xfa\x95\x01\x33\xa8" + "\xda\x7e\x18\x2c\xf4\x7e\x6e\x67" + "\xce\x8f\x9f\xea\x46\x66\x99\xb8" + "\xe1\xc7\x25\x4d\xbd\xa5\x74\xdf" + "\xc7\x8b\xfb\xe3\x2d\x3a\x82\xd3" + "\x17\x94\x77\x2f\x92\xb8\x87\xc2" + "\xcc\x6f\x70\x26\x87\xc7\x10\x8a" + "\xc8\xfd\xc2\xb3\xcf\xa0\xeb\x41", + .ctext = "\xc9\x5f\xe0\x60\x61\x38\x7e\x79" + "\x52\x68\x64\x8f\x55\x9b\x6b\x72" + "\xbf\x09\xef\x2f\xb2\x92\xbb\xa3" + "\xe1\x6a\xeb\xe6\x4e\x7c\x5d\xe0" + "\x6a\x4b\xd0\x57\x3b\x28\x8a\x83" + "\x75\xd4\x5a\x2e\xd1\x9a\x57\xe3" + "\xc5\x43\x36\xde\x02\xac\x2c\x75" + "\xea\x33\x3a\x7e\x5d\xb8\xf6\x12" + "\x42\xbd\x06\x8a\x09\x6b\xd6\xb6" + "\x25\x59\xcd\xbd\x17\xeb\x69\xb3", + .len = 80, + }, { + .key = "\x4c\xf4\xd0\x34\xd0\x95\xab\xae" + "\x82\x5c\xfd\xfa\x13\x86\x25\xce" + "\xf4\x13\x32\xcd\xc6\x6d\xf6\x50", + .klen = 24, + .iv = "\x12\x4a\x5b\x66\x3a\xd3\xfb\x1a" + "\xaf\x06\xea\xf4\x65\x59\xd6\xc2", + .ptext = "\x84\xa0\x53\x97\x61\x30\x70\x15" + "\xac\x45\x8e\xe8\xeb\xa1\x72\x93" + "\x26\x76\x98\x6f\xe4\x86\xca\xf0" + "\x57\x89\xf2\x2b\xd4\xcf\x2d\x95" + "\x86\x26\x20\x0e\x62\xfe\x8f\x1e" + "\x5d\xcb\x2b\x7e\xdd\xab\xac\xda" + "\x6e\x49\x20\xd5\xb7\x01\x83\x4e" + "\xac\x45\x8f\xe1\x05\x3f\xd5\xb1" + "\xee\xb7\x0d\x65\x00\x38\xab\x71" + "\x70\x6e\xb3\x97\x86\xd3\xcd\xad" + "\x51\x8b\x9c\xa0\x9a\x8b\x4c\xb9" + "\x16\x01\x6a\x1f\xdf\xf0\xf9\x9e", + .ctext = "\x03\x2c\x39\x24\x99\xb5\xf6\x79" + "\x91\x89\xb7\xf8\x89\x68\x37\x9d" + "\xe7\x4d\x7d\x1c\x36\xae\x98\xd2" + "\xbf\x2a\xa4\x30\x38\x30\xe7\x5d" + "\xbb\x00\x09\x40\x34\xa4\xef\x82" + "\x23\xca\x0e\xb3\x71\x80\x29\x0a" + "\xa9\x0b\x26\x65\x9a\x12\xbf\x18" + "\xfb\xf8\xe4\xc2\x62\x57\x18\xfb" + "\x1e\x98\xea\x5b\xf6\xd6\x7c\x52" + "\x7a\xba\x0e\x6a\x54\x19\xb6\xfa" + "\xe5\xd7\x60\x40\xb0\x1a\xf1\x09" + "\x70\x96\x23\x49\x98\xfc\x79\xd2", + .len = 96, + }, { + .key = "\x25\x1b\xc2\xa6\x21\x25\xeb\x97" + "\x4b\xf6\xcb\x3b\xcd\x61\xfd\x94" + "\x37\x03\xb3\xd9\x74\x6e\x4d\xbb", + .klen = 24, + .iv = "\xfd\x87\x2b\xec\x4c\x2c\xbf\xe2" + "\x94\x1a\xe6\xd9\xaf\x0e\x78\x17", + .ptext = "\x58\x2b\x1d\x73\x9a\x9c\x63\x18" + "\x88\x7a\x0e\x87\x2f\xf0\xb0\xdb" + "\xc9\x9d\x79\x51\x34\x39\x4f\x07" + "\xa2\x7c\x21\x04\x91\x3b\x79\x79" + "\xfe\xd5\x51\x46\xd5\xcd\x28\xc0" + "\xad\xb8\x55\xb2\xb2\x5a\x9a\xa2" + "\xe2\x0c\xfc\x55\x7d\x60\xd2\x95" + "\xb6\x08\x1d\x31\xaf\xf4\x17\x46" + "\xa4\xbb\x0f\xbd\x67\x3c\x73\x15" + "\x0c\x85\x2f\x62\xe5\xf4\x35\x96" + "\xb1\x9b\x5d\x00\x10\xe9\x70\x12" + "\x3a\x87\x7f\x67\xf1\x81\x7a\x05" + "\xb4\xa6\xfe\xdf\x36\x31\x6d\x9e" + "\x0e\xa9\x44\xa0\xb0\x05\xa9\x41", + .ctext = "\xd4\x9a\x04\x54\x05\xd2\xe6\x3f" + "\xb0\xa4\x36\x5e\x1e\x9c\x35\xb0" + "\xa6\x62\x35\x47\xf4\x4d\x08\x9e" + "\x1c\x22\x91\x8e\x7f\x00\xa6\x3e" + "\x0a\x04\x42\x0f\xc4\xa6\x5d\xe2" + "\x49\x4c\x61\x12\xea\x9d\x7d\x7c" + "\xfa\x93\x74\x6b\x79\x8c\xdb\xc6" + "\x47\xf6\xea\x84\x3e\x97\x7d\x87" + "\x40\x38\x92\xc7\x44\xef\xdf\x63" + "\x29\xe4\x5b\x3a\x87\x22\xa1\x3f" + "\x2b\x31\xb1\xa4\x0d\xea\xf3\x0b" + "\xd7\x4f\xb6\x9c\xba\x40\xa3\x2f" + "\x21\x2b\x05\xe4\xca\xef\x87\x04" + "\xe6\xd0\x29\x2c\x29\x26\x57\xcd", + .len = 112, + }, { + .key = "\x9c\x14\x44\x5a\xd5\x1c\x50\x08" + "\x95\xc2\xf2\xaf\x3f\x29\xc9\x3e" + "\x95\x5e\xc6\xb4\x2b\xf4\x3e\xe3", + .klen = 24, + .iv = "\x1b\xeb\x3d\x73\xfb\xd7\x1e\x2b" + "\x0c\x3d\x58\x6c\xb4\x41\x9b\xfe", + .ptext = "\x2f\x7e\x1c\x10\x81\x36\x2d\x79" + "\xaf\xab\x10\x44\x2e\xcc\x0d\x6c" + "\x9c\x14\xc2\xe4\xae\xb0\xbb\xda" + "\x6a\xe0\x42\x3d\x96\x9f\x78\x7d" + "\x70\x86\xa5\x92\x9f\xee\xcd\x3f" + "\x6a\x55\x84\x98\x28\x03\x02\xc2" + "\xf7\xec\x7a\xfa\xb1\xd9\xa8\xd8" + "\x1c\xc3\xaa\xd5\x61\x7f\x10\x0c" + "\xc0\xa1\x36\x3d\x81\x9a\xd2\x17" + "\x2e\x23\xc9\xb7\xff\xdf\x47\x6c" + "\x96\x3b\x0e\xbd\xec\x9a\x0e\xad" + "\x8c\xaf\x36\x3d\xff\x29\x8b\x33" + "\x87\x96\x77\x1a\x10\x81\x63\x8a" + "\x63\xde\x88\xa9\x9d\xa9\x01\xf2" + "\xdf\xc9\x25\x35\x48\x3a\x15\xdf" + "\x20\x6b\x91\x7c\x56\xe5\x10\x7a", + .ctext = "\xbc\x57\x2a\x88\x0a\xd0\x06\x4f" + "\xdb\x7b\x03\x9f\x97\x1a\x20\xfe" + "\xdb\xdc\x8e\x7b\x68\x13\xc8\xf5" + "\x06\xe3\xe0\x7e\xd3\x51\x21\x86" + "\x4f\x32\xdb\x78\xe3\x26\xbe\x34" + "\x52\x4c\x4e\x6b\x85\x52\x63\x8b" + "\x8c\x5c\x0e\x33\xf5\xa3\x88\x2d" + "\x04\xdc\x01\x2d\xbe\xa1\x48\x6d" + "\x50\xf4\x16\xb1\xd7\x4d\x1e\x99" + "\xa8\x1d\x54\xcb\x13\xf9\x85\x51" + "\x18\x9f\xef\x45\x62\x5d\x48\xe5" + "\x0c\x54\xf7\x7b\x33\x18\xce\xb0" + "\xd5\x82\x1b\xe2\x91\xae\xdc\x09" + "\xe2\x97\xa8\x27\x13\x78\xc6\xb8" + "\x20\x06\x1a\x71\x5a\xb3\xbc\x1b" + "\x69\x1f\xcd\x57\x70\xa7\x1e\x35", + .len = 128, + }, { + .key = "\x2d\x2e\x0f\x30\x32\xed\xa9\x1f" + "\x71\x4e\x68\x77\xe8\xa8\x5b\xdd" + "\x3c\x5e\x68\x6b\xab\x03\xe4\xf8", + .klen = 24, + .iv = "\x42\xc1\x61\x9a\x50\xfb\xc7\x6a" + "\x1a\x31\xa7\x87\xd0\x24\xcb\x5e", + .ptext = "\xc0\x3b\x12\x28\xca\x26\x7b\xb3" + "\x14\xc1\x7f\x66\xff\x3b\xa4\x80" + "\x59\x77\x4f\xa0\xd4\xb2\xd9\x8a" + "\xb6\x67\xe6\x28\xd3\x6f\xf2\xcf" + "\xb8\x6d\x2d\xc4\x2a\x69\x89\xff" + "\xcf\xbb\x11\x2e\x2a\x2b\x7c\xfd" + "\xcd\x56\x02\x95\xc9\x54\x6e\x62" + "\x6a\x97\x75\x1a\x21\x16\x46\xfb" + "\xc2\xab\x62\x54\xef\xba\xae\x46" + "\xd4\x14\xc6\xcc\x16\x1b\x95\xf9" + "\x05\x26\x23\x81\x19\x27\xad\x7b" + "\x9c\x8b\xfb\x65\xa4\x61\xee\x69" + "\x44\xbf\x59\xde\x03\x61\x11\x12" + "\x8d\x94\x48\x47\xa9\x52\x16\xfb" + "\x6b\xaf\x59\x6d\xab\x74\xbf\x5c" + "\xb6\x09\x21\x12\x42\x98\x13\xa1" + "\xa8\x6f\xb9\x6d\x4d\xa6\xdc\xea" + "\x61\x02\x3c\xa7\xcd\x1a\x28\x8c", + .ctext = "\xd7\xb4\xfc\xcc\x1f\xf7\xfc\x7d" + "\x69\xfa\xcb\x01\x60\xf3\x5a\x14" + "\x88\xf7\xea\x43\xaa\x47\xf1\x8a" + "\x4e\xd0\x3c\x50\x58\x35\x95\x21" + "\x5f\xcc\x73\x0b\x97\xa0\x2c\x6b" + "\x70\x4d\x3d\xa8\x21\xbe\xfc\xec" + "\xb6\x55\xf0\x48\x2b\x11\xcc\x4b" + "\xda\xf7\x09\xd9\x18\x7b\x4f\x00" + "\x76\x40\xe0\x7d\x33\xcf\x4f\x77" + "\x91\x97\x63\xfa\x72\xba\x5c\x3d" + "\xcf\x2e\xb8\x19\x56\x4a\xa5\x02" + "\xc3\xb1\x80\xa8\x57\x03\x32\x57" + "\xa8\xe1\x65\xf7\xd3\x52\xc5\xcf" + "\x55\x1e\x34\xe3\x77\xab\x83\xdb" + "\xaf\xd3\x8a\xcc\x96\x1c\xc9\x73" + "\xd9\x0b\xb6\x4c\x31\xac\x2c\x82" + "\xb8\xb4\xc8\xe1\xa5\x71\xcc\xb3" + "\x7e\x85\xb8\xfa\x6b\xef\x41\x24", + .len = 144, + }, { + .key = "\x66\xb8\x4d\x60\x67\x82\xcc\x8d" + "\x1e\xda\x8f\x28\xe5\x02\xdc\x2c" + "\x54\x84\x2a\x06\xb5\xd1\x34\x57", + .klen = 24, + .iv = "\xb8\x28\x4d\xf5\x69\xb9\xf3\x33" + "\x5e\x0b\xa6\x62\x35\x9b\xfb\x97", + .ptext = "\x3e\xc6\xec\xaf\x74\xe8\x72\x91" + "\xb2\xc6\x56\xb3\x23\x29\x43\xe0" + "\xfb\xcc\x21\x38\x64\x78\x9e\x78" + "\xbb\x6e\x0d\x7b\xfd\x05\x74\x01" + "\x7c\x94\xe0\xb0\xd7\x92\xfc\x58" + "\x28\xfc\xe2\x7b\x7f\xf7\x31\x0d" + "\x90\xb7\x60\x78\xa8\x9f\x52\xe3" + "\xe6\xaa\x2a\xb4\xa7\x09\x60\x53" + "\x42\x0e\x15\x31\xf6\x48\xa3\x0a" + "\x20\xf0\x79\x67\xb1\x83\x26\x66" + "\xe0\xb1\xb3\xbd\x1c\x76\x36\xfd" + "\x45\x87\xa4\x14\x1b\xef\xe7\x16" + "\xf7\xfa\x30\x3d\xb9\x52\x8f\x2e" + "\x01\x68\xc1\x7d\xa2\x15\x49\x74" + "\x53\x82\xc2\x10\xa8\x45\x73\x4d" + "\x41\xcc\x24\xa3\x42\xff\x30\xd1" + "\x02\x21\xdc\xd9\x08\xf7\xe7\x4c" + "\x33\x2d\x62\xc7\x38\xf5\xc2\xbe" + "\x52\xf1\x34\x78\x34\x53\x30\x5b" + "\x43\x43\x51\x6a\x02\x81\x64\x0c", + .ctext = "\x71\xf6\x96\x02\x07\x71\x1a\x08" + "\x7c\xfe\x33\xc4\xc9\xbe\xe2\xed" + "\xf8\x46\x69\xce\x1b\xdc\xd3\x05" + "\x7a\xec\x26\x4d\x27\x2a\x49\x36" + "\x85\xe1\x5d\xd3\x91\xd7\x68\xb8" + "\x55\xa5\x27\x55\x2d\xc1\x78\x27" + "\x0c\x49\x0a\x24\x3b\x76\x3f\x5f" + "\x29\x1c\x37\x2f\x30\xfc\x50\xcb" + "\xe2\x54\x26\x7d\x97\xa7\xf3\x58" + "\x15\xe1\x4c\xeb\x35\xc9\xd1\x1e" + "\x7e\x7d\xa0\xe5\x62\xa5\x2d\xf6" + "\x77\xb0\xef\x13\x55\xb4\x66\x2c" + "\x3b\x50\x1b\x4d\xc2\x64\xce\xc6" + "\xfe\xf2\xad\xfe\x26\x73\x36\x66" + "\x0c\x2f\x10\x35\x97\x3c\x9c\x98" + "\xc1\x90\xa8\x82\xd7\xc6\x31\x68" + "\xcf\x77\xa8\x5b\xdf\xf9\x5a\x8e" + "\x84\xb5\x0b\x6e\x5b\xec\x36\x89" + "\x0b\xb1\xbf\xb9\x70\x02\x5c\x22" + "\xc3\xd5\xc1\xc6\xfd\x07\xdb\x70", + .len = 160, + }, { + .key = "\x82\x8e\x9e\x06\x7b\xc2\xe9\xb3" + "\x06\xa3\xfa\x99\x42\x67\x87\xac" + "\x21\xc7\xb0\x98\x6c\xf8\x26\x57" + "\x08\xdd\x92\x02\x77\x7b\x35\xe7", + .klen = 32, + .iv = "\xa1\xad\xcb\xdd\xd5\x19\xb6\xd4" + "\x0b\x62\x58\xb0\x6c\xa0\xc1\x58", + .ptext = "\x14\x0d\x8a\x09\x16\x00\x00\xf1" + "\xc0\x20\x86\xf9\x21\xd1\x34\xe2", + .ctext = "\x05\xe3\x34\xaf\x6c\x83\x14\x8b" + "\x9d\x1c\xd6\x87\x74\x91\xdf\x17", + .len = 16, + }, { + .key = "\xc9\xf3\xc4\x93\xd0\xcc\xaf\xb1" + "\x1a\x42\x93\x71\xd8\x4e\xd8\xaa" + "\x52\xad\x93\x2f\xe5\xd9\xaa\x5b" + "\x47\x37\x3a\xed\x13\x92\x35\x16", + .klen = 32, + .iv = "\x81\xc8\x50\xd1\x74\xc3\x1c\x73" + "\xbb\xab\x72\x83\x90\x5a\x15\xcb", + .ptext = "\x65\x11\x93\xaf\xe1\x69\x6c\xbe" + "\x25\x8c\x76\x87\x53\xa4\x80\xae" + "\x51\x94\x36\x3f\xca\xe7\x45\x41" + "\x76\x05\xbf\x8f\x9c\xad\xc0\xe3", + .ctext = "\x6b\x00\x6e\x49\x7a\x6d\xe3\x04" + "\x4e\xf7\x9f\x8a\x1f\x14\xbd\xb1" + "\x51\xbf\x13\x9f\x29\x95\x51\x16" + "\xd0\x23\x9a\x1a\x45\xc2\xc3\xd1", + .len = 32, + }, { + .key = "\xd5\x9f\x52\x34\x12\x99\x8e\x42" + "\xe0\x85\x04\x6f\xeb\xf1\x5d\xd0" + "\xc1\xbf\x3f\x84\xd9\x1e\x71\x44" + "\xd4\xb9\x40\x3c\x02\x2e\x21\x19", + .klen = 32, + .iv = "\x28\xc1\x97\x64\x81\x52\x57\x0e" + "\x02\x8c\xab\x4c\xe2\x60\x14\xa5", + .ptext = "\x5a\xb1\x33\x48\xaa\x51\xe9\xa4" + "\x5c\x2d\xbe\x33\xcc\xc4\x7f\x96" + "\xe8\xde\x2b\xe7\x35\x7a\x11\x4b" + "\x13\x08\x32\xc6\x41\xd8\xec\x54" + "\xa3\xd3\xda\x35\x43\x69\xf6\x88" + "\x97\xca\x00\x1b\x02\x59\x24\x82", + .ctext = "\x03\xaf\x76\xbd\x5e\x5b\xca\xc0" + "\xae\x44\xa2\x2f\xc2\x76\x2f\x50" + "\xfa\x94\x94\x5a\x48\x9d\x9c\x38" + "\xc9\x75\xc9\xb2\x56\x0a\x2d\x91" + "\xb8\xe8\x4e\xaa\xcb\x51\x9b\x6a" + "\x20\x9b\x2b\xc5\xb0\x18\x9d\x01", + .len = 48, + }, { + .key = "\x9c\x5d\xd7\x66\x36\xfa\x02\x20" + "\x99\x61\x62\x86\x0f\x43\x2e\x05" + "\x25\x8b\xfb\xf1\xae\x4c\xde\x18" + "\x0b\xf8\xd0\x9d\xaa\xd4\x56\x04", + .klen = 32, + .iv = "\xcd\xa8\x61\x89\x8d\xbb\x72\xb6" + "\x1e\xfe\x03\x34\x54\x88\x23\xe2", + .ptext = "\x66\x42\x60\x24\xf3\xe4\xe9\x7e" + "\x42\x20\xf4\x61\xce\x1c\x5e\x44" + "\x02\x26\x91\xf7\x41\xa4\xab\x34" + "\x29\x49\xdd\x78\x19\x8f\x10\x10" + "\xf0\x61\xcf\x77\x18\x17\x61\xdf" + "\xc4\xa8\x35\x0e\x75\x1b\x84\x6b" + "\xc3\x3f\x31\x59\x5a\x9c\xf4\xc3" + "\x43\xa9\xb7\xf8\x65\x40\x40\xba", + .ctext = "\xb6\x41\x55\x8f\xeb\x16\x1e\x4c" + "\x81\xa0\x85\x6c\xf0\x07\xa5\x2a" + "\x19\x91\xed\x3e\xd6\x30\x8c\xca" + "\x5d\x0f\x58\xca\xd2\x8a\xac\xa2" + "\x2b\x86\x4f\xb5\x85\x4d\xac\x6d" + "\xe5\x39\x1b\x02\x23\x89\x4e\x4f" + "\x02\x00\xe8\x1b\x40\x85\x21\x2b" + "\xc6\xb1\x98\xed\x70\xb3\xf8\xc3", + .len = 64, + }, { + .key = "\x4b\x4e\x11\x91\x27\xcf\x8c\x66" + "\x17\xfa\x5b\x4c\xa8\xb8\x0f\xa1" + "\x99\x5b\x07\x56\xe1\x8d\x94\x8b" + "\xf2\x86\x5a\x5f\x40\x83\xfa\x06", + .klen = 32, + .iv = "\xfd\x73\xee\x1c\x27\xf3\xb4\x38" + "\xc5\x7c\x2e\xc5\x6e\xdb\x49\x0d", + .ptext = "\x0a\xe2\xdd\x97\xdd\x5e\xd4\xb3" + "\xc1\x49\x8f\x53\xb2\x40\x85\x1c" + "\x90\x37\x2d\xbd\x21\x6b\x1f\x80" + "\x56\x98\x76\x1e\xcf\x6c\x78\xd8" + "\xa0\x3c\x79\xc3\x56\xf7\xfc\x64" + "\x35\x58\x1c\x7c\xc4\x5f\x2a\x25" + "\x8c\x01\x98\x1e\x1c\x1f\x15\x64" + "\x50\xb5\xfa\x02\xd3\x54\xe5\x29" + "\xe3\xd2\xa3\x83\x54\x40\x54\xc5" + "\xd8\x1c\xc9\x84\x7d\xc8\x31\x49", + .ctext = "\x53\x2a\xa8\xa0\x15\xaf\x2f\xc4" + "\x7d\x31\xb4\x61\x80\x5f\xd1\xb6" + "\x7c\xca\x86\xb9\x28\x6e\xb6\x2b" + "\xe3\x4b\x7e\xea\xb3\x4f\xa2\xa2" + "\x4e\x8f\xbe\x22\x66\xb3\x92\xbc" + "\x70\x91\xaf\xa6\x09\x5d\xe2\x05" + "\x38\x62\xd3\x6e\x07\x63\x91\xad" + "\x48\x5a\x42\xe7\xdc\x0d\xb1\xe3" + "\x92\x88\x64\xee\x93\xaa\xaf\x31" + "\x68\x57\x35\x8d\x54\x2c\xfa\xb1", + .len = 80, + }, { + .key = "\x77\x3b\xf5\xe7\x20\xf7\xe0\x0c" + "\x3d\x3a\x83\x17\x83\x79\xd8\x29" + "\x5a\x0a\x25\x7f\xe0\x21\x23\xff" + "\x31\xfd\x60\x10\xe6\x63\xe2\xaf", + .klen = 32, + .iv = "\xdb\x4c\x0d\xc0\x36\xdb\xc7\xa1" + "\xa4\x91\xd9\x05\xe6\xc4\x98\x00", + .ptext = "\x8d\x4d\xc6\x5e\x01\x82\xb3\x39" + "\xc8\x64\xa7\xcb\x05\x19\x84\x80" + "\x3f\x9c\xa8\x4f\x64\xb3\x11\x4b" + "\x0e\x21\xc4\x75\x04\x1d\x6f\xd5" + "\x04\x04\x4d\xc9\xc0\x4b\x4a\x9c" + "\x26\xb7\x68\x5a\xe4\xd0\x61\xe3" + "\x2c\x93\x8e\x3f\xb4\x67\x07\x31" + "\x02\x52\x0c\x0f\xe6\x6d\xa3\xd0" + "\x48\x95\x83\x67\x23\x64\x31\x50" + "\xd2\x5f\x69\x68\x8b\x71\xbf\x01" + "\x29\x99\x86\x36\x2e\xdf\xf1\x7c" + "\x08\x8c\x78\x7a\x93\x9a\x7d\x1b", + .ctext = "\x92\x90\x48\x2f\x3a\x6b\x68\x43" + "\x28\x9b\x7d\x1e\x46\x28\xd8\x58" + "\xd9\x1e\x44\xd7\x24\x91\x65\xb1" + "\x15\xde\xc4\x63\xf1\xb1\x34\x9e" + "\xae\x8c\x51\x94\xc5\x22\x65\x8d" + "\x3d\x85\xf5\x34\x5f\x04\x68\x95" + "\xf2\x66\x62\xbb\xc8\x3f\xe4\x0a" + "\x8a\xb2\x70\xc0\x77\xd5\x96\xef" + "\x9e\x39\x3a\x3e\x0d\x2b\xf9\xfe" + "\xa9\xbc\x00\xba\xc5\x43\xd7\x70" + "\x2f\xef\x1e\x1e\x93\xc2\x5d\xf1" + "\xb5\x50\xb8\xf5\xee\xf4\x26\x6f", + .len = 96, + }, { + .key = "\xe0\x6a\x30\xe1\x35\xb5\xb0\x7c" + "\x54\xc5\x73\x9b\x00\xe5\xe7\x02" + "\xbe\x16\x59\xdc\xd9\x03\x17\x53" + "\xa8\x37\xd1\x5f\x13\x8e\x45\xdb", + .klen = 32, + .iv = "\x54\xe9\x1c\xde\xfb\x26\x0e\x48" + "\x35\x50\x4d\x9b\x4d\x12\x21\x0d", + .ptext = "\x73\x72\xcf\xdb\xbd\xbc\xc0\xdf" + "\x6b\xbb\xdf\x65\x6f\x2f\x43\x3b" + "\x2d\x7c\x0e\x07\x7f\xa0\x95\xdd" + "\xfc\x67\xc1\x11\x7a\xe2\xb5\x4a" + "\xd1\x15\xb0\xd8\xe2\xf0\x35\x48" + "\xd8\x81\x6a\x35\xae\x67\xbf\x61" + "\xf2\x8a\xcf\x04\xc8\x09\x8b\x63" + "\x31\x74\x95\xa5\x8d\x3c\xea\xe2" + "\x5f\x67\xc4\x7e\x51\x88\xbf\xb5" + "\x78\xef\x3a\x76\xd8\x1d\x00\x75" + "\x2b\x7b\x28\x7c\xde\x4b\x39\x01" + "\x5d\xde\x92\xfe\x90\x07\x09\xfd" + "\xa5\xd1\xd3\x72\x11\x6d\xa4\x4e" + "\xd1\x6e\x16\xd1\xf6\x39\x4f\xa0", + .ctext = "\x3b\xc5\xee\xfc\x05\xaf\xa6\xb7" + "\xfe\x12\x24\x79\x31\xad\x32\xb5" + "\xfb\x71\x9b\x02\xad\xf4\x94\x20" + "\x25\x7b\xdb\xdf\x97\x99\xca\xea" + "\xc4\xed\x32\x26\x6b\xc8\xd4\x7b" + "\x5b\x55\xfa\xf9\x5b\xab\x88\xdb" + "\x48\xfe\x67\xd5\x5a\x47\x81\x4e" + "\x3e\x1e\x83\xca\x1d\x04\xe1\xb5" + "\x6c\x1b\xbd\xf2\x2d\xf1\xae\x75" + "\x09\x6a\xf8\xb2\xc3\x27\xee\x08" + "\x66\x94\x72\xc0\x2b\x12\x47\x23" + "\x4d\xde\xb4\xca\xf7\x66\xca\x14" + "\xe7\x68\x1b\xfb\x48\x70\x3e\x4c" + "\x43\xbb\x88\x32\x25\xff\x77\x6a", + .len = 112, + }, { + .key = "\x60\xb6\xde\x17\xca\x4c\xe7\xe0" + "\x07\x0d\x80\xc5\x8a\x2d\x5a\xc2" + "\x2c\xb9\xa4\x5f\x2a\x85\x2c\x3d" + "\x6d\x67\xc8\xee\x0f\xa2\xf4\x09", + .klen = 32, + .iv = "\x1a\xa5\xbc\x7e\x93\xf6\xdd\x28" + "\xb7\x69\x27\xa1\x84\x95\x25\x5a", + .ptext = "\x7b\x88\x00\xeb\xa5\xba\xa1\xa7" + "\xd4\x40\x16\x74\x2b\x42\x37\xda" + "\xe0\xaf\x89\x59\x41\x2f\x62\x00" + "\xf5\x5a\x4e\x3b\x85\x27\xb2\xed" + "\x1b\xa7\xaf\xbe\x89\xf3\x49\xb7" + "\x8c\x63\xc9\x0c\x52\x00\x5f\x38" + "\x3b\x3c\x0c\x4f\xdd\xe1\xbf\x90" + "\x4a\x48\xbf\x3a\x95\xcb\x48\xa2" + "\x92\x7c\x79\x81\xde\x18\x6e\x92" + "\x1f\x36\xa9\x5d\x8d\xc4\xb6\x4d" + "\xb2\xb4\x0e\x09\x6d\xf3\x3d\x01" + "\x3d\x9b\x40\x47\xbc\x69\x31\xa1" + "\x6a\x71\x26\xdc\xac\x10\x56\x63" + "\x15\x23\x7d\x10\xe3\x76\x82\x41" + "\xcd\x80\x57\x2f\xfc\x4d\x22\x7b" + "\x57\xbb\x9a\x0a\x03\xe9\xb3\x13", + .ctext = "\x37\x0d\x47\x21\xbc\x28\x0b\xf7" + "\x85\x5f\x60\x57\xf2\x7f\x92\x20" + "\x5f\xa7\xf6\xf4\xa6\xf5\xdf\x1e" + "\xae\x8e\xeb\x97\xfc\xce\x6a\x25" + "\x6d\x6a\x5b\xd1\x99\xf6\x27\x77" + "\x52\x0c\xf1\xd7\x94\xa0\x67\x5d" + "\x60\x35\xb0\x6d\x01\x45\x52\xc8" + "\x05\xd8\x7f\x69\xaf\x8e\x68\x05" + "\xa8\xa5\x24\x2f\x95\xef\xf1\xd2" + "\x8c\x45\x12\xc5\x7a\xcf\xbb\x99" + "\x25\xaa\xa3\x9b\x3f\xf1\xfc\x9d" + "\xfa\x2c\x26\x9b\x92\x47\x61\x6b" + "\x63\x1e\x41\x67\xcb\xb7\x0f\x52" + "\x70\xd4\x0d\x7e\xef\x34\xa2\x75" + "\x4f\x6a\x55\x9c\x2b\x4a\x02\xdd" + "\x96\x5d\xcb\xca\x45\xa1\xec\xaa", + .len = 128, + }, { + .key = "\x2a\xed\x7d\x76\xfc\xc5\x49\x50" + "\xf4\x90\x0f\xcc\x5d\xff\x0c\x3c" + "\x14\x06\xaf\x68\x8f\xd7\xb6\x25" + "\x1e\x10\x95\x2a\x71\x33\x17\x20", + .klen = 32, + .iv = "\x5b\x58\x47\xf8\xd5\x1e\x91\x81" + "\x46\xe7\x25\x3a\x02\x45\x9c\x65", + .ptext = "\x10\xaf\xde\x5c\x30\x79\x43\x28" + "\x1c\x03\xf8\x50\x0f\x30\xa5\xef" + "\x84\x19\x4c\x09\x40\x03\x75\x1f" + "\x92\x8f\x88\x01\xda\x31\x7a\xe4" + "\x48\xe3\xab\xb4\xe6\x1b\x0f\xac" + "\xd9\xfa\x8d\x23\xe4\xc6\xa4\xa9" + "\x2d\x9a\x54\x52\x44\x5c\x3c\x52" + "\x61\xf0\x00\xca\xed\xab\xed\xe2" + "\x44\x0b\xe0\x18\xba\xa5\x63\xd8" + "\xdc\x5e\x1a\x4c\xf8\xde\x5e\x75" + "\xdf\x42\x27\x7b\xe9\x11\x2f\x41" + "\x3a\x72\x54\x3d\x44\x9c\x3e\x87" + "\x8d\x8d\x43\x2f\xb2\xff\x87\xd4" + "\xad\x98\x68\x72\x53\x61\x19\x7c" + "\x20\x79\x8c\x2b\x37\x0b\x96\x15" + "\xa5\x7d\x4e\x01\xe6\xea\xb6\xfa" + "\xaa\xd3\x9d\xa2\xd9\x11\xc3\xc9" + "\xd4\x0e\x3f\x3e\xfe\x35\x1e\xe5", + .ctext = "\xb0\x2b\x75\x5f\x33\x1b\x05\x49" + "\x06\xf1\x43\x91\xc2\x85\xfa\xac" + "\x3f\x47\xf3\x89\x73\xb2\x0e\xa4" + "\x30\xcb\x87\x39\x53\x5d\x36\x89" + "\x77\xd9\x17\x01\x95\xa6\xe9\x71" + "\x51\x53\xd9\x4f\xa6\xc2\x79\x3d" + "\x2e\x50\x90\x52\x0d\x27\x1a\x46" + "\xf1\xe8\x6e\x7e\x7b\x32\xe5\x22" + "\x22\x1f\xba\x5e\xcf\x25\x6b\x26" + "\x76\xf0\xca\x8e\xdd\x5b\xd3\x09" + "\x6f\x82\x08\x56\x1f\x51\x72\x57" + "\xca\xd1\x60\x07\xfb\x9f\x71\x54" + "\x0f\xf6\x48\x71\xfa\x8f\xcb\xdd" + "\xce\xd3\x16\xcd\xae\x0e\x67\x5e" + "\xea\x8d\xa2\x4a\x4f\x11\xc8\xc8" + "\x2f\x04\xfe\xa8\x2a\x07\x1c\xb1" + "\x77\x39\xda\x8b\xd9\x5c\x94\x6c" + "\x4d\x4d\x13\x51\x6f\x07\x06\x5b", + .len = 144, + }, { + .key = "\x7b\xa7\x4d\x0a\x37\x30\xb9\xf5" + "\x2a\x79\xb4\xbf\xdb\x7f\x9b\x64" + "\x23\x43\xb5\x18\x34\xc4\x5f\xdf" + "\xd9\x2a\x66\x58\x00\x44\xb5\xd9", + .klen = 32, + .iv = "\x75\x34\x30\xc1\xf0\x69\xdf\x0a" + "\x52\xce\x4f\x1e\x2c\x41\x35\xec", + .ptext = "\x81\x47\x55\x3a\xcd\xfe\xa2\x3d" + "\x45\x53\xa7\x67\x61\x74\x25\x80" + "\x98\x89\xfe\xf8\x6a\x9f\x51\x7c" + "\xa4\xe4\xe7\xc7\xe0\x1a\xce\xbb" + "\x4b\x46\x43\xb0\xab\xa8\xd6\x0c" + "\xa0\xf0\xc8\x13\x29\xaf\xb8\x01" + "\x6b\x0c\x7e\x56\xae\xb8\x58\x72" + "\xa9\x24\x44\x61\xff\xf1\xac\xf8" + "\x09\xa8\x48\x21\xd6\xab\x41\x73" + "\x70\x6b\x92\x06\x61\xdc\xb4\x85" + "\x76\x26\x7a\x84\xc3\x9e\x3a\x14" + "\xe7\xf4\x2d\x95\x92\xad\x18\xcc" + "\x44\xd4\x2c\x36\x57\xed\x2b\x9b" + "\x3f\x2b\xcd\xe5\x11\xe3\x62\x33" + "\x42\x3f\xb8\x2a\xb1\x37\x3f\x8b" + "\xe8\xbd\x6b\x0b\x9f\x38\x5a\x5f" + "\x82\x34\xb7\x96\x35\x58\xde\xab" + "\x94\x98\x41\x5b\x3f\xac\x0a\x34" + "\x56\xc0\x02\xef\x81\x6d\xb1\xff" + "\x34\xe8\xc7\x6a\x31\x79\xba\xd8", + .ctext = "\x4e\x00\x7c\x52\x45\x76\xf9\x3d" + "\x1a\xd1\x72\xbc\xb9\x0f\xa9\xfb" + "\x0e\x5b\xe2\x3c\xc7\xae\x92\xf6" + "\xb8\x0b\x0a\x95\x40\xe9\x7f\xe0" + "\x54\x10\xf9\xf6\x23\x1f\x51\xc8" + "\x16\x8b\x2e\x79\xe1\x8c\x0b\x43" + "\xe5\xeb\xb5\x9d\x1e\xc3\x28\x07" + "\x5c\x8d\xb1\xe7\x80\xd3\xce\x62" + "\x8d\xf8\x31\x1f\x29\x8b\x90\xee" + "\xe5\xc3\xfa\x16\xc4\xf0\xc3\x99" + "\xe9\x5e\x19\xba\x37\xb8\xc0\x87" + "\xb5\xc6\xc9\x31\xcb\x6e\x30\xce" + "\x03\x1d\xfe\xce\x08\x32\x00\xeb" + "\x86\xc4\xfb\x48\x01\xda\x93\x73" + "\xcc\xb7\xae\x4e\x94\x20\xeb\xc7" + "\xe3\x33\x4c\xeb\xed\xe2\xfc\x86" + "\x0e\x73\x32\xf9\x1b\xf3\x25\xf3" + "\x74\xad\xd1\xf4\x2c\x45\xa4\xfd" + "\x52\x40\xa2\x4e\xa5\x62\xf6\x02" + "\xbb\xb0\xe3\x23\x86\x67\xb8\xf6", + .len = 160, + } +}; + +static const struct cipher_testvec aria_cfb_tv_template[] = { + { + .key = "\x7f\x92\xd5\x06\x30\x6b\xc0\x23" + "\x87\xa8\x8e\x6d\xc7\xc5\xd7\xf1", + .klen = 16, + .iv = "\x5f\xce\x89\xb3\xd5\x7f\x7f\xf0" + "\xfd\xab\x56\xa6\x6e\xda\x7c\x57", + .ptext = "\x36\x36\x89\x09\xcd\xa8\xd3\x91" + "\x48\x3e\x3c\x11\xcf\xd0\x4f\xc0", + .ctext = "\x19\x28\xb5\xf2\x1c\xbc\xf8\xaf" + "\xb9\xae\x1b\x23\x4f\xe1\x6e\x40", + }, { + .key = "\x51\xe3\x8c\xe9\x76\xcd\xff\x37" + "\xd6\x1a\x18\x2f\x68\x2f\xb6\xfe", + .klen = 16, + .iv = "\x3d\x2d\x85\x75\x6e\x18\x8a\x52" + "\x53\x39\xfc\xc1\xf5\xc0\x56\x22", + .ptext = "\xc6\xae\xaa\x0d\x90\xf2\x38\x93" + "\xac\xd2\x3f\xc7\x74\x8d\x13\x7e" + "\xfa\x3f\x70\x52\xfb\x04\x0e\xed" + "\x0e\x60\x75\x84\x21\xdf\x13\xa1", + .ctext = "\x3f\x8c\xa9\x19\xd6\xb4\xfb\xed" + "\x9c\x6d\xaa\x1b\xe1\xc1\xe6\xa8" + "\x47\x35\x7d\xa3\x96\x7d\x53\x60" + "\xa9\x33\x9c\x34\xae\x7d\x7c\x74", + .len = 32, + }, { + .key = "\x26\xf8\x8c\x26\x0a\x37\x51\x8f" + "\xe7\x9c\x74\x77\x7a\x3e\xbb\x5d", + .klen = 16, + .iv = "\xd7\x33\xf3\xa9\x5b\xb4\x86\xea" + "\xe3\x7d\x50\x62\x3b\x73\xaf\xc4", + .ptext = "\xda\x89\xd9\x3c\xcc\xe4\x73\xb0" + "\xef\x3e\x5f\x46\x62\x88\xd5\x26" + "\x3b\xd3\xb5\x81\x78\x70\x1b\xd2" + "\x39\x56\x34\x63\x2c\xc5\x51\x13" + "\x48\x29\x3a\x58\xbe\x41\xc5\x80" + "\x2c\x80\xa7\x3c\x14\xb4\x89\x5e", + .ctext = "\x28\xd8\xa7\xf8\x74\x98\x00\xfc" + "\xd6\x48\xad\xbd\xbe\x3f\x0e\x7b" + "\xa3\xec\x03\x6a\xfb\xc9\x01\x83" + "\xb3\x2f\xda\x5e\x66\xa0\xc3\xec" + "\xe9\xd4\x72\x2a\xa2\x90\x41\xcf" + "\xde\x30\x79\xc3\x82\x10\x51\xe1", + .len = 48, + }, { + .key = "\x8e\xe5\x5f\xe2\x39\x80\xf5\x2b" + "\x77\xb5\xca\x90\xda\x1d\x22\x17", + .klen = 16, + .iv = "\xd9\xa0\x57\x80\xc8\x96\x70\x86" + "\x07\x2c\xf4\x61\x79\x09\x01\x8f", + .ptext = "\x37\x32\x98\xd4\x86\x2b\x3b\x80" + "\x07\x60\xba\xf0\x2e\xc3\x4a\x57" + "\xf5\xb5\xd7\xbf\xd2\x2a\x9b\x4a" + "\xe6\x08\xf0\xbe\x77\xd1\x62\x40" + "\xa0\x82\x09\x60\x47\xbb\x16\x56" + "\x50\x1f\xab\x8b\x10\xfe\xf0\x5c" + "\x05\x32\x63\x1a\xc4\x46\x6f\x55" + "\x32\xde\x41\x5a\xf7\x52\xd7\xfa", + .ctext = "\x29\x31\x55\xd2\xe5\x0b\x81\x39" + "\xf9\xbc\x63\xe2\xfa\x26\x99\xde" + "\x5c\xd3\x0a\x56\xe5\xfc\x83\xdd" + "\xab\x26\x90\x7d\xa8\x0f\x01\xa6" + "\x0e\x01\xdc\x1f\xfa\xa7\xdd\x09" + "\xf9\xbf\x12\xf4\xc6\x9f\xbd\x57" + "\x23\x68\x54\x0f\xe0\xcf\x1c\x6d" + "\xe1\x5e\x0b\x4a\x1e\x71\x1d\xaa", + .len = 64, + }, { + .key = "\x30\x9d\x59\x8d\x64\x76\xad\x37" + "\xba\xbc\x46\x6a\x69\x17\x3c\xac", + .klen = 16, + .iv = "\x6f\xdd\xa2\x9b\x86\x32\x14\x2e" + "\x54\x74\x8f\x3d\xe2\xd6\x85\x44", + .ptext = "\x4f\x4a\x31\x64\xc6\xa5\x29\xaa" + "\xad\xfd\x32\x94\x1f\x56\x57\xd1" + "\x9d\x7e\x3d\x49\x00\x36\xb1\x5d" + "\xb2\x92\x83\x70\x1e\xa3\x97\xa6" + "\x65\x53\x39\xeb\x53\x8f\xb1\x38" + "\x91\xac\x17\x11\x1c\x03\x69\x53" + "\xf5\xdf\xdb\x2c\x1b\x9a\x6e\x6b" + "\xb6\x02\xc4\xfa\x95\x01\x33\xa8" + "\xda\x7e\x18\x2c\xf4\x7e\x6e\x67" + "\xce\x8f\x9f\xea\x46\x66\x99\xb8", + .ctext = "\x38\xbc\xf5\x9d\x0e\x26\xa6\x18" + "\x95\x0b\x23\x54\x09\xa1\xf9\x46" + "\x7a\x31\xa0\xd7\x4a\xec\xb3\x10" + "\x8a\x8e\x99\x78\x6c\x6e\x76\xf2" + "\x63\x8a\x3b\x90\xaa\xd5\x64\x65" + "\x5a\x52\xb0\x36\x4c\xce\xed\xc7" + "\x51\x3c\x06\xb0\xee\x54\xec\x10" + "\xc0\x5f\xfd\xa9\x44\x9a\x29\x32" + "\x19\x79\x7d\x2b\x14\x26\x96\x13" + "\x9d\xa5\x61\xbd\xb6\x72\x37\x26", + .len = 80, + }, { + .key = "\xe1\xc7\x25\x4d\xbd\xa5\x74\xdf" + "\xc7\x8b\xfb\xe3\x2d\x3a\x82\xd3", + .klen = 16, + .iv = "\x17\x94\x77\x2f\x92\xb8\x87\xc2" + "\xcc\x6f\x70\x26\x87\xc7\x10\x8a", + .ptext = "\xc8\xfd\xc2\xb3\xcf\xa0\xeb\x41" + "\x4c\xf4\xd0\x34\xd0\x95\xab\xae" + "\x82\x5c\xfd\xfa\x13\x86\x25\xce" + "\xf4\x13\x32\xcd\xc6\x6d\xf6\x50" + "\x12\x4a\x5b\x66\x3a\xd3\xfb\x1a" + "\xaf\x06\xea\xf4\x65\x59\xd6\xc2" + "\x84\xa0\x53\x97\x61\x30\x70\x15" + "\xac\x45\x8e\xe8\xeb\xa1\x72\x93" + "\x26\x76\x98\x6f\xe4\x86\xca\xf0" + "\x57\x89\xf2\x2b\xd4\xcf\x2d\x95" + "\x86\x26\x20\x0e\x62\xfe\x8f\x1e" + "\x5d\xcb\x2b\x7e\xdd\xab\xac\xda", + .ctext = "\xdf\x79\x58\x30\x6f\x47\x12\x78" + "\x04\xb2\x0b\x1a\x62\x22\xe2\x9f" + "\xfe\x90\x50\x41\x1b\x6a\x6a\x9c" + "\x4e\x77\x8f\xca\xd1\x68\x31\xcd" + "\x41\x82\xa5\x5b\xc0\x08\x2b\x37" + "\x62\xec\x95\xf1\x56\x12\x38\x66" + "\x84\x82\x72\xda\x00\x21\x96\x82" + "\x33\xd4\x99\xaa\xb9\xeb\xd5\xc3" + "\x2b\xa8\xf7\xdc\x13\x0e\x21\x9f" + "\x4b\xf9\x42\x58\xa8\x39\x10\xd5" + "\x86\xa5\xc6\x78\x3b\x34\x05\x03" + "\x54\x43\x2b\x80\xa9\x53\x4d\x0e", + .len = 96, + }, { + .key = "\x6e\x49\x20\xd5\xb7\x01\x83\x4e" + "\xac\x45\x8f\xe1\x05\x3f\xd5\xb1", + .klen = 16, + .iv = "\xee\xb7\x0d\x65\x00\x38\xab\x71" + "\x70\x6e\xb3\x97\x86\xd3\xcd\xad", + .ptext = "\x51\x8b\x9c\xa0\x9a\x8b\x4c\xb9" + "\x16\x01\x6a\x1f\xdf\xf0\xf9\x9e" + "\x25\x1b\xc2\xa6\x21\x25\xeb\x97" + "\x4b\xf6\xcb\x3b\xcd\x61\xfd\x94" + "\x37\x03\xb3\xd9\x74\x6e\x4d\xbb" + "\xfd\x87\x2b\xec\x4c\x2c\xbf\xe2" + "\x94\x1a\xe6\xd9\xaf\x0e\x78\x17" + "\x58\x2b\x1d\x73\x9a\x9c\x63\x18" + "\x88\x7a\x0e\x87\x2f\xf0\xb0\xdb" + "\xc9\x9d\x79\x51\x34\x39\x4f\x07" + "\xa2\x7c\x21\x04\x91\x3b\x79\x79" + "\xfe\xd5\x51\x46\xd5\xcd\x28\xc0" + "\xad\xb8\x55\xb2\xb2\x5a\x9a\xa2" + "\xe2\x0c\xfc\x55\x7d\x60\xd2\x95", + .ctext = "\xe4\x25\x0d\x22\xeb\xbe\x5e\x90" + "\x01\xe5\xae\xc9\x94\xbd\x93\x89" + "\x5e\x5a\x5a\x2f\xf6\xdf\xf8\x16" + "\xd3\xb2\xed\x29\x51\xe2\x75\xb0" + "\x1a\x48\xb5\xe6\xd3\x58\x40\xc7" + "\x6f\x6f\xcf\x57\x82\x43\x5a\x36" + "\xef\x27\xe1\x34\x85\x01\xec\x98" + "\x00\xbd\x94\x6f\x12\x39\xa8\x13" + "\xfe\x3c\x39\xc0\xc6\xe1\xcc\x05" + "\x0e\xd5\xc9\xda\xbd\xdd\xdb\xaa" + "\x5a\xaa\x8e\xe8\xa8\x0a\xc5\x18" + "\xb4\x1d\x13\x81\xc9\xc4\xaa\x61" + "\xa9\xbd\xaa\x03\x12\x93\xbb\xed" + "\x0c\x6e\xbd\x1c\x05\x16\x8a\x59", + .len = 112, + }, { + .key = "\xb6\x08\x1d\x31\xaf\xf4\x17\x46" + "\xa4\xbb\x0f\xbd\x67\x3c\x73\x15", + .klen = 16, + .iv = "\x0c\x85\x2f\x62\xe5\xf4\x35\x96" + "\xb1\x9b\x5d\x00\x10\xe9\x70\x12", + .ptext = "\x3a\x87\x7f\x67\xf1\x81\x7a\x05" + "\xb4\xa6\xfe\xdf\x36\x31\x6d\x9e" + "\x0e\xa9\x44\xa0\xb0\x05\xa9\x41" + "\x9c\x14\x44\x5a\xd5\x1c\x50\x08" + "\x95\xc2\xf2\xaf\x3f\x29\xc9\x3e" + "\x95\x5e\xc6\xb4\x2b\xf4\x3e\xe3" + "\x1b\xeb\x3d\x73\xfb\xd7\x1e\x2b" + "\x0c\x3d\x58\x6c\xb4\x41\x9b\xfe" + "\x2f\x7e\x1c\x10\x81\x36\x2d\x79" + "\xaf\xab\x10\x44\x2e\xcc\x0d\x6c" + "\x9c\x14\xc2\xe4\xae\xb0\xbb\xda" + "\x6a\xe0\x42\x3d\x96\x9f\x78\x7d" + "\x70\x86\xa5\x92\x9f\xee\xcd\x3f" + "\x6a\x55\x84\x98\x28\x03\x02\xc2" + "\xf7\xec\x7a\xfa\xb1\xd9\xa8\xd8" + "\x1c\xc3\xaa\xd5\x61\x7f\x10\x0c", + .ctext = "\xa7\x4c\x96\x55\x7c\x07\xce\xb2" + "\x6f\x63\x9f\xc6\x8b\x6f\xc6\x4a" + "\x85\xf2\x4b\xdf\x62\x0c\x6c\x8d" + "\x13\x5d\xd3\x40\x58\xa6\xf9\x03" + "\xd9\xf2\x48\x4e\x12\x64\x9a\x55" + "\xa2\xa3\xd0\x19\xe5\x5b\xaa\x62" + "\x7b\xe9\x2a\x23\xab\xb5\xa6\xcf" + "\x53\x59\x70\xc6\xb8\x92\x12\x3b" + "\x93\x68\x24\xba\x7d\xd6\xc0\x5b" + "\x06\x2e\x7f\x2e\x32\x5d\x42\x9c" + "\x13\x8e\x92\x3c\x99\x20\x32\x2b" + "\x4a\x41\xb2\x4a\x81\xe8\x6e\x7f" + "\x5b\x8e\xca\x4d\xd7\x29\x96\xde" + "\x30\x9c\xa6\x84\x90\xe7\xc2\xae" + "\xf4\x7e\x73\x32\x4c\x25\xec\xef" + "\x58\x69\x63\x3f\x4e\x71\x4b\x1c", + .len = 128, + }, { + .key = "\xc0\xa1\x36\x3d\x81\x9a\xd2\x17" + "\x2e\x23\xc9\xb7\xff\xdf\x47\x6c", + .klen = 16, + .iv = "\x96\x3b\x0e\xbd\xec\x9a\x0e\xad" + "\x8c\xaf\x36\x3d\xff\x29\x8b\x33", + .ptext = "\x87\x96\x77\x1a\x10\x81\x63\x8a" + "\x63\xde\x88\xa9\x9d\xa9\x01\xf2" + "\xdf\xc9\x25\x35\x48\x3a\x15\xdf" + "\x20\x6b\x91\x7c\x56\xe5\x10\x7a" + "\x2d\x2e\x0f\x30\x32\xed\xa9\x1f" + "\x71\x4e\x68\x77\xe8\xa8\x5b\xdd" + "\x3c\x5e\x68\x6b\xab\x03\xe4\xf8" + "\x42\xc1\x61\x9a\x50\xfb\xc7\x6a" + "\x1a\x31\xa7\x87\xd0\x24\xcb\x5e" + "\xc0\x3b\x12\x28\xca\x26\x7b\xb3" + "\x14\xc1\x7f\x66\xff\x3b\xa4\x80" + "\x59\x77\x4f\xa0\xd4\xb2\xd9\x8a" + "\xb6\x67\xe6\x28\xd3\x6f\xf2\xcf" + "\xb8\x6d\x2d\xc4\x2a\x69\x89\xff" + "\xcf\xbb\x11\x2e\x2a\x2b\x7c\xfd" + "\xcd\x56\x02\x95\xc9\x54\x6e\x62" + "\x6a\x97\x75\x1a\x21\x16\x46\xfb" + "\xc2\xab\x62\x54\xef\xba\xae\x46", + .ctext = "\x11\x7f\xea\x49\xaf\x24\x52\xa2" + "\xde\x60\x99\x58\x23\xf9\x9e\x91" + "\x94\x52\x31\xa3\x28\x07\x14\xad" + "\x00\x24\x4a\x4a\xe7\x18\xd7\x24" + "\xcc\x8b\x66\x53\x82\x65\x31\xa5" + "\x54\x76\x59\x0b\x69\x6f\x90\x2c" + "\x8d\xa5\x2b\x61\x05\x80\xfb\xe0" + "\xf9\x6e\xaf\xb9\xc4\x15\x67\xcc" + "\x15\xce\xa0\xc0\xf2\xae\xa6\x15" + "\x24\x9a\xe5\xcb\x09\x42\xcf\x41" + "\x95\xa4\x8d\xbf\xe8\xb8\x40\xcd" + "\xb0\x33\x2c\xb3\xc4\xdd\xf9\x45" + "\xda\xb2\xeb\xb3\xf8\xfa\x7f\xe3" + "\xc0\x3a\x98\xe7\x17\x4a\x0c\x60" + "\xb2\x22\xba\x3b\x21\x85\x27\x56" + "\xe0\xb2\xf7\x2a\x59\xb1\x56\x20" + "\x0b\xa9\x13\x73\xe0\x6f\x61\x32" + "\xa5\x38\x14\xb3\xe3\xaa\x70\x44", + .len = 144, + }, { + .key = "\xd4\x14\xc6\xcc\x16\x1b\x95\xf9" + "\x05\x26\x23\x81\x19\x27\xad\x7b", + .klen = 16, + .iv = "\x9c\x8b\xfb\x65\xa4\x61\xee\x69" + "\x44\xbf\x59\xde\x03\x61\x11\x12", + .ptext = "\x8d\x94\x48\x47\xa9\x52\x16\xfb" + "\x6b\xaf\x59\x6d\xab\x74\xbf\x5c" + "\xb6\x09\x21\x12\x42\x98\x13\xa1" + "\xa8\x6f\xb9\x6d\x4d\xa6\xdc\xea" + "\x61\x02\x3c\xa7\xcd\x1a\x28\x8c" + "\x66\xb8\x4d\x60\x67\x82\xcc\x8d" + "\x1e\xda\x8f\x28\xe5\x02\xdc\x2c" + "\x54\x84\x2a\x06\xb5\xd1\x34\x57" + "\xb8\x28\x4d\xf5\x69\xb9\xf3\x33" + "\x5e\x0b\xa6\x62\x35\x9b\xfb\x97" + "\x3e\xc6\xec\xaf\x74\xe8\x72\x91" + "\xb2\xc6\x56\xb3\x23\x29\x43\xe0" + "\xfb\xcc\x21\x38\x64\x78\x9e\x78" + "\xbb\x6e\x0d\x7b\xfd\x05\x74\x01" + "\x7c\x94\xe0\xb0\xd7\x92\xfc\x58" + "\x28\xfc\xe2\x7b\x7f\xf7\x31\x0d" + "\x90\xb7\x60\x78\xa8\x9f\x52\xe3" + "\xe6\xaa\x2a\xb4\xa7\x09\x60\x53" + "\x42\x0e\x15\x31\xf6\x48\xa3\x0a" + "\x20\xf0\x79\x67\xb1\x83\x26\x66", + .ctext = "\x5b\xc0\xe8\x17\xa4\xf9\xea\xce" + "\x9e\xf9\xe0\xb1\xac\x37\xe9\x41" + "\xc8\x06\xf9\x1c\x1a\xfc\xe8\x7a" + "\x38\xf2\x80\x66\xc2\x70\x59\x4e" + "\xe0\x32\x5b\x27\x39\xf5\xfb\x03" + "\xc8\xaf\xd6\x7e\x57\xc7\xc6\x71" + "\xd9\xd0\x48\x39\xb1\x0d\xa8\x1a" + "\x23\x8a\x3d\x05\xe2\x90\x7e\x18" + "\xd7\x20\x04\x3b\x82\x76\x3f\xaa" + "\xc2\x89\xb6\x9e\x14\x2f\x46\xcd" + "\x51\x9b\xa8\x7b\x62\x7b\x9c\x17" + "\xc4\xe1\x8b\x3f\xb5\x4d\xac\x66" + "\x49\xf6\xb6\x4c\x3e\x16\x46\xb0" + "\xca\x04\xef\x72\x5c\x03\x0a\xe5" + "\x2f\x4e\x36\x38\x36\x9f\xf4\xe2" + "\x81\x7a\x4c\xdf\x36\x27\xd5\x9d" + "\x03\xad\x1d\x3a\xe9\x2a\x99\xb0" + "\x2c\xba\x13\x75\xc8\x37\x97\x11" + "\xf4\x15\x0f\xb7\x75\x26\xa1\x14" + "\x79\xec\x1f\xab\xd2\x10\x8c\x5f", + .len = 160, + }, { + .key = "\x7f\x92\xd5\x06\x30\x6b\xc0\x23" + "\x87\xa8\x8e\x6d\xc7\xc5\xd7\xf1" + "\x5f\xce\x89\xb3\xd5\x7f\x7f\xf0", + .klen = 24, + .iv = "\xfd\xab\x56\xa6\x6e\xda\x7c\x57" + "\x36\x36\x89\x09\xcd\xa8\xd3\x91", + .ptext = "\x48\x3e\x3c\x11\xcf\xd0\x4f\xc0" + "\x51\xe3\x8c\xe9\x76\xcd\xff\x37", + .ctext = "\xa4\x12\x2f\xc4\xf0\x6d\xd9\x46" + "\xe4\xe6\xd1\x0b\x6d\x14\xf0\x8f", + .len = 16, + }, { + .key = "\xd6\x1a\x18\x2f\x68\x2f\xb6\xfe" + "\x3d\x2d\x85\x75\x6e\x18\x8a\x52" + "\x53\x39\xfc\xc1\xf5\xc0\x56\x22", + .klen = 24, + .iv = "\xc6\xae\xaa\x0d\x90\xf2\x38\x93" + "\xac\xd2\x3f\xc7\x74\x8d\x13\x7e", + .ptext = "\xfa\x3f\x70\x52\xfb\x04\x0e\xed" + "\x0e\x60\x75\x84\x21\xdf\x13\xa1" + "\x26\xf8\x8c\x26\x0a\x37\x51\x8f" + "\xe7\x9c\x74\x77\x7a\x3e\xbb\x5d", + .ctext = "\x80\x2b\xf0\x88\xb9\x4b\x8d\xf5" + "\xc3\x0e\x15\x5b\xea\x5d\x5b\xa8" + "\x52\xe7\x83\x3c\xa1\x51\x1c\x1f" + "\x38\xd9\x7c\x88\x3c\x3a\xcd\x3e", + .len = 32, + }, { + .key = "\xd7\x33\xf3\xa9\x5b\xb4\x86\xea" + "\xe3\x7d\x50\x62\x3b\x73\xaf\xc4" + "\xda\x89\xd9\x3c\xcc\xe4\x73\xb0", + .klen = 24, + .iv = "\xef\x3e\x5f\x46\x62\x88\xd5\x26" + "\x3b\xd3\xb5\x81\x78\x70\x1b\xd2", + .ptext = "\x39\x56\x34\x63\x2c\xc5\x51\x13" + "\x48\x29\x3a\x58\xbe\x41\xc5\x80" + "\x2c\x80\xa7\x3c\x14\xb4\x89\x5e" + "\x8e\xe5\x5f\xe2\x39\x80\xf5\x2b" + "\x77\xb5\xca\x90\xda\x1d\x22\x17" + "\xd9\xa0\x57\x80\xc8\x96\x70\x86", + .ctext = "\x65\x01\x3c\xb0\xac\x4c\x63\xb6" + "\xe7\xf1\xf4\x61\x35\xf4\x36\xde" + "\xeb\x0f\x8c\x34\xd1\x78\xb4\x00" + "\xb2\xc1\x7c\x28\xb2\xb7\xbb\xa3" + "\xc6\xb7\x27\xf7\x6d\x56\x79\xfa" + "\x61\x57\xba\x30\x6f\x56\xe9\x8c", + .len = 48, + }, { + .key = "\x07\x2c\xf4\x61\x79\x09\x01\x8f" + "\x37\x32\x98\xd4\x86\x2b\x3b\x80" + "\x07\x60\xba\xf0\x2e\xc3\x4a\x57", + .klen = 24, + .iv = "\xf5\xb5\xd7\xbf\xd2\x2a\x9b\x4a" + "\xe6\x08\xf0\xbe\x77\xd1\x62\x40", + .ptext = "\xa0\x82\x09\x60\x47\xbb\x16\x56" + "\x50\x1f\xab\x8b\x10\xfe\xf0\x5c" + "\x05\x32\x63\x1a\xc4\x46\x6f\x55" + "\x32\xde\x41\x5a\xf7\x52\xd7\xfa" + "\x30\x9d\x59\x8d\x64\x76\xad\x37" + "\xba\xbc\x46\x6a\x69\x17\x3c\xac" + "\x6f\xdd\xa2\x9b\x86\x32\x14\x2e" + "\x54\x74\x8f\x3d\xe2\xd6\x85\x44", + .ctext = "\x5a\xfb\xb1\x2c\x6e\xe5\xb8\xe0" + "\x80\xb6\x77\xa8\xfe\x10\x3a\x99" + "\xbf\xc0\x2a\xfe\x6f\x38\xf2\x1d" + "\x53\x6c\x05\x83\xb1\x13\x00\x87" + "\x92\x92\x42\x70\xcf\x9f\xf7\x8f" + "\x53\x55\x18\x6f\x35\x68\x35\x50" + "\x3a\xc8\x45\x3e\xa3\xf1\x33\x2e" + "\xa1\x65\x42\xe2\x6d\x31\x8c\x4b", + .len = 64, + }, { + .key = "\x4f\x4a\x31\x64\xc6\xa5\x29\xaa" + "\xad\xfd\x32\x94\x1f\x56\x57\xd1" + "\x9d\x7e\x3d\x49\x00\x36\xb1\x5d", + .klen = 24, + .iv = "\xb2\x92\x83\x70\x1e\xa3\x97\xa6" + "\x65\x53\x39\xeb\x53\x8f\xb1\x38", + .ptext = "\x91\xac\x17\x11\x1c\x03\x69\x53" + "\xf5\xdf\xdb\x2c\x1b\x9a\x6e\x6b" + "\xb6\x02\xc4\xfa\x95\x01\x33\xa8" + "\xda\x7e\x18\x2c\xf4\x7e\x6e\x67" + "\xce\x8f\x9f\xea\x46\x66\x99\xb8" + "\xe1\xc7\x25\x4d\xbd\xa5\x74\xdf" + "\xc7\x8b\xfb\xe3\x2d\x3a\x82\xd3" + "\x17\x94\x77\x2f\x92\xb8\x87\xc2" + "\xcc\x6f\x70\x26\x87\xc7\x10\x8a" + "\xc8\xfd\xc2\xb3\xcf\xa0\xeb\x41", + .ctext = "\xc9\x5f\xe0\x60\x61\x38\x7e\x79" + "\x52\x68\x64\x8f\x55\x9b\x6b\x72" + "\xa5\x17\x61\xb7\xce\x02\xa9\xa4" + "\x5c\x73\x45\x33\xd1\x07\x5e\xdc" + "\xe5\xbe\xa7\xde\x69\xa0\x97\x98" + "\x02\xef\xa4\x67\x51\x60\x69\x4f" + "\x03\xf5\xa8\x5f\x03\x69\xbc\xc2" + "\x34\x59\x7e\xd4\xd2\xb3\x32\x2f" + "\x0c\xb4\x37\xca\xc4\xc7\x93\xf4" + "\xa4\xab\x01\x3f\x91\x29\x55\x98", + .len = 80, + }, { + .key = "\x4c\xf4\xd0\x34\xd0\x95\xab\xae" + "\x82\x5c\xfd\xfa\x13\x86\x25\xce" + "\xf4\x13\x32\xcd\xc6\x6d\xf6\x50", + .klen = 24, + .iv = "\x12\x4a\x5b\x66\x3a\xd3\xfb\x1a" + "\xaf\x06\xea\xf4\x65\x59\xd6\xc2", + .ptext = "\x84\xa0\x53\x97\x61\x30\x70\x15" + "\xac\x45\x8e\xe8\xeb\xa1\x72\x93" + "\x26\x76\x98\x6f\xe4\x86\xca\xf0" + "\x57\x89\xf2\x2b\xd4\xcf\x2d\x95" + "\x86\x26\x20\x0e\x62\xfe\x8f\x1e" + "\x5d\xcb\x2b\x7e\xdd\xab\xac\xda" + "\x6e\x49\x20\xd5\xb7\x01\x83\x4e" + "\xac\x45\x8f\xe1\x05\x3f\xd5\xb1" + "\xee\xb7\x0d\x65\x00\x38\xab\x71" + "\x70\x6e\xb3\x97\x86\xd3\xcd\xad" + "\x51\x8b\x9c\xa0\x9a\x8b\x4c\xb9" + "\x16\x01\x6a\x1f\xdf\xf0\xf9\x9e", + .ctext = "\x03\x2c\x39\x24\x99\xb5\xf6\x79" + "\x91\x89\xb7\xf8\x89\x68\x37\x9d" + "\xa2\x80\x95\x74\x87\x64\xb9\xeb" + "\x85\x28\x92\x9a\x6e\xd3\x3b\x50" + "\x4c\x80\x5b\xe4\xf2\x7e\xda\x2a" + "\xd4\xf8\xcb\xe3\x6f\xdf\xae\x0e" + "\xc5\x6c\x0b\x49\x2e\x29\x1c\xf2" + "\x3f\x44\x44\x12\x67\xa6\xff\x44" + "\xe0\xec\xd8\xf7\x32\xde\x21\x15" + "\xab\x8f\x98\x4d\xed\xb0\x42\xfd" + "\x83\x94\xe2\xcc\x69\x6d\xe8\xdb" + "\x62\x93\x1f\xd0\xf4\x8c\x62\xc0", + .len = 96, + }, { + .key = "\x25\x1b\xc2\xa6\x21\x25\xeb\x97" + "\x4b\xf6\xcb\x3b\xcd\x61\xfd\x94" + "\x37\x03\xb3\xd9\x74\x6e\x4d\xbb", + .klen = 24, + .iv = "\xfd\x87\x2b\xec\x4c\x2c\xbf\xe2" + "\x94\x1a\xe6\xd9\xaf\x0e\x78\x17", + .ptext = "\x58\x2b\x1d\x73\x9a\x9c\x63\x18" + "\x88\x7a\x0e\x87\x2f\xf0\xb0\xdb" + "\xc9\x9d\x79\x51\x34\x39\x4f\x07" + "\xa2\x7c\x21\x04\x91\x3b\x79\x79" + "\xfe\xd5\x51\x46\xd5\xcd\x28\xc0" + "\xad\xb8\x55\xb2\xb2\x5a\x9a\xa2" + "\xe2\x0c\xfc\x55\x7d\x60\xd2\x95" + "\xb6\x08\x1d\x31\xaf\xf4\x17\x46" + "\xa4\xbb\x0f\xbd\x67\x3c\x73\x15" + "\x0c\x85\x2f\x62\xe5\xf4\x35\x96" + "\xb1\x9b\x5d\x00\x10\xe9\x70\x12" + "\x3a\x87\x7f\x67\xf1\x81\x7a\x05" + "\xb4\xa6\xfe\xdf\x36\x31\x6d\x9e" + "\x0e\xa9\x44\xa0\xb0\x05\xa9\x41", + .ctext = "\xd4\x9a\x04\x54\x05\xd2\xe6\x3f" + "\xb0\xa4\x36\x5e\x1e\x9c\x35\xb0" + "\xc0\x89\xbd\x1c\xaa\x45\xa6\xc8" + "\x16\x68\x4a\x06\x93\x67\x88\xd7" + "\x72\x6e\x48\x0a\x17\xa3\x52\x8b" + "\x96\x5f\x41\xf6\x17\x64\x55\x8b" + "\xac\xce\xf6\x8c\xce\xd2\xd4\xd4" + "\x8d\x92\x32\xe0\x0d\xb4\xf7\x4a" + "\x90\xaf\x7b\x85\x21\x46\x2e\xa6" + "\x9e\xac\x0d\x22\xf2\x26\xf6\xd3" + "\x27\xcd\x59\xa0\xe2\xbb\x22\xcd" + "\x35\xb6\x28\x45\x0a\x46\xb0\x3a" + "\xac\x3e\xd3\x5b\xc6\x54\xa2\xa3" + "\x6d\xbb\xb3\xcd\xc5\x64\x62\x92", + .len = 112, + }, { + .key = "\x9c\x14\x44\x5a\xd5\x1c\x50\x08" + "\x95\xc2\xf2\xaf\x3f\x29\xc9\x3e" + "\x95\x5e\xc6\xb4\x2b\xf4\x3e\xe3", + .klen = 24, + .iv = "\x1b\xeb\x3d\x73\xfb\xd7\x1e\x2b" + "\x0c\x3d\x58\x6c\xb4\x41\x9b\xfe", + .ptext = "\x2f\x7e\x1c\x10\x81\x36\x2d\x79" + "\xaf\xab\x10\x44\x2e\xcc\x0d\x6c" + "\x9c\x14\xc2\xe4\xae\xb0\xbb\xda" + "\x6a\xe0\x42\x3d\x96\x9f\x78\x7d" + "\x70\x86\xa5\x92\x9f\xee\xcd\x3f" + "\x6a\x55\x84\x98\x28\x03\x02\xc2" + "\xf7\xec\x7a\xfa\xb1\xd9\xa8\xd8" + "\x1c\xc3\xaa\xd5\x61\x7f\x10\x0c" + "\xc0\xa1\x36\x3d\x81\x9a\xd2\x17" + "\x2e\x23\xc9\xb7\xff\xdf\x47\x6c" + "\x96\x3b\x0e\xbd\xec\x9a\x0e\xad" + "\x8c\xaf\x36\x3d\xff\x29\x8b\x33" + "\x87\x96\x77\x1a\x10\x81\x63\x8a" + "\x63\xde\x88\xa9\x9d\xa9\x01\xf2" + "\xdf\xc9\x25\x35\x48\x3a\x15\xdf" + "\x20\x6b\x91\x7c\x56\xe5\x10\x7a", + .ctext = "\xbc\x57\x2a\x88\x0a\xd0\x06\x4f" + "\xdb\x7b\x03\x9f\x97\x1a\x20\xfe" + "\x15\x91\xb4\xed\x5d\x78\x89\x2a" + "\x67\x6b\x9c\x47\x36\xc2\x80\x0e" + "\x03\x8d\x6f\xfc\x94\xc7\xc5\xc2" + "\xeb\x43\x74\x5d\xfe\xc4\x5a\xa1" + "\x80\x51\x8a\x63\xd1\x27\x1b\x0a" + "\x88\x2c\xc4\x7f\x1a\xa3\x28\xe5" + "\xfd\xd0\x8a\xd4\x36\xa6\x19\xd5" + "\xff\x41\x7a\x8b\x6e\x9a\x97\x14" + "\x2a\xc8\xd0\xb8\xa3\x8e\x64\x32" + "\xb7\x2d\x76\x9b\x3b\xe2\x3f\x91" + "\xb4\x64\xbf\x59\x67\x14\xc3\xf5" + "\xa8\x92\x4b\x85\xdf\x80\xcb\xb5" + "\xc7\x80\xf9\x4a\xbc\xed\x67\x5a" + "\x0b\x58\x65\x1f\xc9\x6e\x9b\x0a", + .len = 128, + }, { + .key = "\x2d\x2e\x0f\x30\x32\xed\xa9\x1f" + "\x71\x4e\x68\x77\xe8\xa8\x5b\xdd" + "\x3c\x5e\x68\x6b\xab\x03\xe4\xf8", + .klen = 24, + .iv = "\x42\xc1\x61\x9a\x50\xfb\xc7\x6a" + "\x1a\x31\xa7\x87\xd0\x24\xcb\x5e", + .ptext = "\xc0\x3b\x12\x28\xca\x26\x7b\xb3" + "\x14\xc1\x7f\x66\xff\x3b\xa4\x80" + "\x59\x77\x4f\xa0\xd4\xb2\xd9\x8a" + "\xb6\x67\xe6\x28\xd3\x6f\xf2\xcf" + "\xb8\x6d\x2d\xc4\x2a\x69\x89\xff" + "\xcf\xbb\x11\x2e\x2a\x2b\x7c\xfd" + "\xcd\x56\x02\x95\xc9\x54\x6e\x62" + "\x6a\x97\x75\x1a\x21\x16\x46\xfb" + "\xc2\xab\x62\x54\xef\xba\xae\x46" + "\xd4\x14\xc6\xcc\x16\x1b\x95\xf9" + "\x05\x26\x23\x81\x19\x27\xad\x7b" + "\x9c\x8b\xfb\x65\xa4\x61\xee\x69" + "\x44\xbf\x59\xde\x03\x61\x11\x12" + "\x8d\x94\x48\x47\xa9\x52\x16\xfb" + "\x6b\xaf\x59\x6d\xab\x74\xbf\x5c" + "\xb6\x09\x21\x12\x42\x98\x13\xa1" + "\xa8\x6f\xb9\x6d\x4d\xa6\xdc\xea" + "\x61\x02\x3c\xa7\xcd\x1a\x28\x8c", + .ctext = "\xd7\xb4\xfc\xcc\x1f\xf7\xfc\x7d" + "\x69\xfa\xcb\x01\x60\xf3\x5a\x14" + "\xfe\x8c\x4e\xfa\x09\xb5\x0d\xda" + "\xff\xdd\xba\xdf\xa3\x6b\x3a\x87" + "\x21\xbb\xf8\x62\x14\x22\xdd\x9b" + "\x92\x23\xaa\xd7\xcc\xb2\x15\xd0" + "\xbd\x81\x95\x24\xc2\xc6\x53\x5b" + "\xf7\x3c\xa0\xf7\x36\xbc\xbf\xf3" + "\xfc\x1c\x6e\xe0\x71\x8d\xa1\x3d" + "\x8e\x1a\xc5\xba\xd5\x68\xd4\x7a" + "\xe0\x4f\x0a\x14\x89\x0b\xa6\x2f" + "\x18\xc5\x38\x76\xf1\xe7\x5c\xae" + "\x7a\xbb\x27\x1c\xf0\x7c\x6c\x14" + "\x07\xb7\x49\x6e\x29\x04\x38\x31" + "\x91\xe8\x1d\x0f\xfc\x3b\xb8\x20" + "\x58\x64\x11\xa1\xf5\xba\xa3\x62" + "\x92\xcf\x44\x63\x2c\xe8\x10\xb5" + "\xf0\x97\x86\xcb\x5f\xc1\x80\x7a", + .len = 144, + }, { + .key = "\x66\xb8\x4d\x60\x67\x82\xcc\x8d" + "\x1e\xda\x8f\x28\xe5\x02\xdc\x2c" + "\x54\x84\x2a\x06\xb5\xd1\x34\x57", + .klen = 24, + .iv = "\xb8\x28\x4d\xf5\x69\xb9\xf3\x33" + "\x5e\x0b\xa6\x62\x35\x9b\xfb\x97", + .ptext = "\x3e\xc6\xec\xaf\x74\xe8\x72\x91" + "\xb2\xc6\x56\xb3\x23\x29\x43\xe0" + "\xfb\xcc\x21\x38\x64\x78\x9e\x78" + "\xbb\x6e\x0d\x7b\xfd\x05\x74\x01" + "\x7c\x94\xe0\xb0\xd7\x92\xfc\x58" + "\x28\xfc\xe2\x7b\x7f\xf7\x31\x0d" + "\x90\xb7\x60\x78\xa8\x9f\x52\xe3" + "\xe6\xaa\x2a\xb4\xa7\x09\x60\x53" + "\x42\x0e\x15\x31\xf6\x48\xa3\x0a" + "\x20\xf0\x79\x67\xb1\x83\x26\x66" + "\xe0\xb1\xb3\xbd\x1c\x76\x36\xfd" + "\x45\x87\xa4\x14\x1b\xef\xe7\x16" + "\xf7\xfa\x30\x3d\xb9\x52\x8f\x2e" + "\x01\x68\xc1\x7d\xa2\x15\x49\x74" + "\x53\x82\xc2\x10\xa8\x45\x73\x4d" + "\x41\xcc\x24\xa3\x42\xff\x30\xd1" + "\x02\x21\xdc\xd9\x08\xf7\xe7\x4c" + "\x33\x2d\x62\xc7\x38\xf5\xc2\xbe" + "\x52\xf1\x34\x78\x34\x53\x30\x5b" + "\x43\x43\x51\x6a\x02\x81\x64\x0c", + .ctext = "\x71\xf6\x96\x02\x07\x71\x1a\x08" + "\x7c\xfe\x33\xc4\xc9\xbe\xe2\xed" + "\xd0\xcc\x5d\x27\x75\xb4\x5d\x8d" + "\x24\x03\xe4\x96\x31\x94\x0e\x38" + "\x14\x4f\xad\x16\x58\x0d\x73\xdc" + "\xbe\x5b\xcb\x38\xeb\x4d\xbc\x9a" + "\x44\x69\x7a\x12\x91\x14\x52\xfa" + "\xd2\xa2\xc5\x66\xd7\xaf\x4d\xb9" + "\xb1\x58\x24\x10\xde\x6a\xee\x7e" + "\x45\xf3\x76\xea\x47\x8a\xe6\x96" + "\x41\xf2\x96\x2d\x3c\xec\xcf\xc6" + "\x1d\xf4\x26\xc0\xea\x90\x27\x6e" + "\x87\xef\xb5\x39\x38\xdb\xad\xbf" + "\x57\x9a\x1d\xbc\x1d\xe5\x16\x91" + "\x41\x45\xbe\x67\x6c\x42\x0f\xad" + "\xcf\xfb\xcd\xf1\x4c\xd8\x73\xe7" + "\x24\x3b\xd7\x03\xeb\xd1\xb1\x1b" + "\x7d\xc9\x3d\x34\xd7\xb8\x69\x03" + "\x76\x95\x32\x26\xed\x88\x76\x89" + "\x13\xc6\xc8\xa6\x60\xf9\x73\x4d", + .len = 160, + }, { + .key = "\x82\x8e\x9e\x06\x7b\xc2\xe9\xb3" + "\x06\xa3\xfa\x99\x42\x67\x87\xac" + "\x21\xc7\xb0\x98\x6c\xf8\x26\x57" + "\x08\xdd\x92\x02\x77\x7b\x35\xe7", + .klen = 32, + .iv = "\xa1\xad\xcb\xdd\xd5\x19\xb6\xd4" + "\x0b\x62\x58\xb0\x6c\xa0\xc1\x58", + .ptext = "\x14\x0d\x8a\x09\x16\x00\x00\xf1" + "\xc0\x20\x86\xf9\x21\xd1\x34\xe2", + .ctext = "\x05\xe3\x34\xaf\x6c\x83\x14\x8b" + "\x9d\x1c\xd6\x87\x74\x91\xdf\x17", + .len = 16, + }, { + .key = "\xc9\xf3\xc4\x93\xd0\xcc\xaf\xb1" + "\x1a\x42\x93\x71\xd8\x4e\xd8\xaa" + "\x52\xad\x93\x2f\xe5\xd9\xaa\x5b" + "\x47\x37\x3a\xed\x13\x92\x35\x16", + .klen = 32, + .iv = "\x81\xc8\x50\xd1\x74\xc3\x1c\x73" + "\xbb\xab\x72\x83\x90\x5a\x15\xcb", + .ptext = "\x65\x11\x93\xaf\xe1\x69\x6c\xbe" + "\x25\x8c\x76\x87\x53\xa4\x80\xae" + "\x51\x94\x36\x3f\xca\xe7\x45\x41" + "\x76\x05\xbf\x8f\x9c\xad\xc0\xe3", + .ctext = "\x6B\x00\x6E\x49\x7A\x6D\xE3\x04" + "\x4E\xF7\x9F\x8A\x1F\x14\xBD\xB1" + "\xD3\x5D\xA4\x30\x26\x85\x85\xEF" + "\x12\xBC\xC7\xA1\x65\x82\xA7\x74", + .len = 32, + }, { + .key = "\xd5\x9f\x52\x34\x12\x99\x8e\x42" + "\xe0\x85\x04\x6f\xeb\xf1\x5d\xd0" + "\xc1\xbf\x3f\x84\xd9\x1e\x71\x44" + "\xd4\xb9\x40\x3c\x02\x2e\x21\x19", + .klen = 32, + .iv = "\x28\xc1\x97\x64\x81\x52\x57\x0e" + "\x02\x8c\xab\x4c\xe2\x60\x14\xa5", + .ptext = "\x5a\xb1\x33\x48\xaa\x51\xe9\xa4" + "\x5c\x2d\xbe\x33\xcc\xc4\x7f\x96" + "\xe8\xde\x2b\xe7\x35\x7a\x11\x4b" + "\x13\x08\x32\xc6\x41\xd8\xec\x54" + "\xa3\xd3\xda\x35\x43\x69\xf6\x88" + "\x97\xca\x00\x1b\x02\x59\x24\x82", + .ctext = "\x03\xaf\x76\xbd\x5e\x5b\xca\xc0" + "\xae\x44\xa2\x2f\xc2\x76\x2f\x50" + "\x6a\x73\x28\xf2\xba\xe8\xb2\xb8" + "\x43\x61\x41\x92\xff\xac\xcb\xa6" + "\x84\x31\xe3\x34\xd0\x37\x81\xab" + "\x2b\x0e\x97\x3c\x4a\x2d\xa4\x83", + .len = 48, + }, { + .key = "\x9c\x5d\xd7\x66\x36\xfa\x02\x20" + "\x99\x61\x62\x86\x0f\x43\x2e\x05" + "\x25\x8b\xfb\xf1\xae\x4c\xde\x18" + "\x0b\xf8\xd0\x9d\xaa\xd4\x56\x04", + .klen = 32, + .iv = "\xcd\xa8\x61\x89\x8d\xbb\x72\xb6" + "\x1e\xfe\x03\x34\x54\x88\x23\xe2", + .ptext = "\x66\x42\x60\x24\xf3\xe4\xe9\x7e" + "\x42\x20\xf4\x61\xce\x1c\x5e\x44" + "\x02\x26\x91\xf7\x41\xa4\xab\x34" + "\x29\x49\xdd\x78\x19\x8f\x10\x10" + "\xf0\x61\xcf\x77\x18\x17\x61\xdf" + "\xc4\xa8\x35\x0e\x75\x1b\x84\x6b" + "\xc3\x3f\x31\x59\x5a\x9c\xf4\xc3" + "\x43\xa9\xb7\xf8\x65\x40\x40\xba", + .ctext = "\xb6\x41\x55\x8f\xeb\x16\x1e\x4c" + "\x81\xa0\x85\x6c\xf0\x07\xa5\x2a" + "\x12\x0f\x1d\xb2\xaa\xba\x85\x0f" + "\xa6\x27\x1a\x91\xa6\xc5\x8c\x2a" + "\xde\x8d\x3a\xa9\x8b\xcf\x24\xf1" + "\x82\x51\x6b\xc8\x01\xd7\x7b\x89" + "\x6c\xfc\xb1\x96\x6c\xa2\xd7\x1f" + "\x4b\x7a\xd9\x8d\x34\xaa\xa0\x8a", + .len = 64, + }, { + .key = "\x4b\x4e\x11\x91\x27\xcf\x8c\x66" + "\x17\xfa\x5b\x4c\xa8\xb8\x0f\xa1" + "\x99\x5b\x07\x56\xe1\x8d\x94\x8b" + "\xf2\x86\x5a\x5f\x40\x83\xfa\x06", + .klen = 32, + .iv = "\xfd\x73\xee\x1c\x27\xf3\xb4\x38" + "\xc5\x7c\x2e\xc5\x6e\xdb\x49\x0d", + .ptext = "\x0a\xe2\xdd\x97\xdd\x5e\xd4\xb3" + "\xc1\x49\x8f\x53\xb2\x40\x85\x1c" + "\x90\x37\x2d\xbd\x21\x6b\x1f\x80" + "\x56\x98\x76\x1e\xcf\x6c\x78\xd8" + "\xa0\x3c\x79\xc3\x56\xf7\xfc\x64" + "\x35\x58\x1c\x7c\xc4\x5f\x2a\x25" + "\x8c\x01\x98\x1e\x1c\x1f\x15\x64" + "\x50\xb5\xfa\x02\xd3\x54\xe5\x29" + "\xe3\xd2\xa3\x83\x54\x40\x54\xc5" + "\xd8\x1c\xc9\x84\x7d\xc8\x31\x49", + .ctext = "\x53\x2a\xa8\xa0\x15\xaf\x2f\xc4" + "\x7d\x31\xb4\x61\x80\x5f\xd1\xb6" + "\xa4\x29\x40\x72\x1b\xb2\x96\xb7" + "\x4d\x5e\x5b\x53\x44\xa4\xf1\xe9" + "\xf0\x27\x2f\x26\x84\x66\x13\xa4" + "\xb2\x19\x55\xb1\x18\xf3\x69\xfd" + "\xb0\x2f\x08\x3f\xa5\x41\xe2\x34" + "\x5e\x63\x57\x0e\xef\x17\x78\xbc" + "\xc3\x65\x7c\xbe\x6b\xa3\xa3\xef" + "\x58\x05\x30\x5a\x08\xbd\xf7\x0e", + .len = 80, + }, { + .key = "\x77\x3b\xf5\xe7\x20\xf7\xe0\x0c" + "\x3d\x3a\x83\x17\x83\x79\xd8\x29" + "\x5a\x0a\x25\x7f\xe0\x21\x23\xff" + "\x31\xfd\x60\x10\xe6\x63\xe2\xaf", + .klen = 32, + .iv = "\xdb\x4c\x0d\xc0\x36\xdb\xc7\xa1" + "\xa4\x91\xd9\x05\xe6\xc4\x98\x00", + .ptext = "\x8d\x4d\xc6\x5e\x01\x82\xb3\x39" + "\xc8\x64\xa7\xcb\x05\x19\x84\x80" + "\x3f\x9c\xa8\x4f\x64\xb3\x11\x4b" + "\x0e\x21\xc4\x75\x04\x1d\x6f\xd5" + "\x04\x04\x4d\xc9\xc0\x4b\x4a\x9c" + "\x26\xb7\x68\x5a\xe4\xd0\x61\xe3" + "\x2c\x93\x8e\x3f\xb4\x67\x07\x31" + "\x02\x52\x0c\x0f\xe6\x6d\xa3\xd0" + "\x48\x95\x83\x67\x23\x64\x31\x50" + "\xd2\x5f\x69\x68\x8b\x71\xbf\x01" + "\x29\x99\x86\x36\x2e\xdf\xf1\x7c" + "\x08\x8c\x78\x7a\x93\x9a\x7d\x1b", + .ctext = "\x92\x90\x48\x2f\x3a\x6b\x68\x43" + "\x28\x9b\x7d\x1e\x46\x28\xd8\x58" + "\x0f\x47\x8b\xb5\x83\x35\x35\x3e" + "\xdf\x59\x3d\xb3\x47\xfc\xfc\x52" + "\x86\xeb\xb3\x58\x54\xd5\x0a\xb4" + "\xad\xbd\x5c\x09\xfc\x08\xc2\x01" + "\x5e\x9b\x30\x11\xc4\x40\x2e\x32" + "\x9c\xa0\xf1\xfd\xae\xd4\x75\x5e" + "\x52\xd9\x19\x4d\xc1\xd4\xb6\x19" + "\x88\xfb\x29\x17\x15\xbb\x60\xd6" + "\x5a\xe9\x82\x89\xaf\x30\x4e\xd4" + "\x47\xde\x86\x88\x95\x4c\x13\x59", + .len = 96, + }, { + .key = "\xe0\x6a\x30\xe1\x35\xb5\xb0\x7c" + "\x54\xc5\x73\x9b\x00\xe5\xe7\x02" + "\xbe\x16\x59\xdc\xd9\x03\x17\x53" + "\xa8\x37\xd1\x5f\x13\x8e\x45\xdb", + .klen = 32, + .iv = "\x54\xe9\x1c\xde\xfb\x26\x0e\x48" + "\x35\x50\x4d\x9b\x4d\x12\x21\x0d", + .ptext = "\x73\x72\xcf\xdb\xbd\xbc\xc0\xdf" + "\x6b\xbb\xdf\x65\x6f\x2f\x43\x3b" + "\x2d\x7c\x0e\x07\x7f\xa0\x95\xdd" + "\xfc\x67\xc1\x11\x7a\xe2\xb5\x4a" + "\xd1\x15\xb0\xd8\xe2\xf0\x35\x48" + "\xd8\x81\x6a\x35\xae\x67\xbf\x61" + "\xf2\x8a\xcf\x04\xc8\x09\x8b\x63" + "\x31\x74\x95\xa5\x8d\x3c\xea\xe2" + "\x5f\x67\xc4\x7e\x51\x88\xbf\xb5" + "\x78\xef\x3a\x76\xd8\x1d\x00\x75" + "\x2b\x7b\x28\x7c\xde\x4b\x39\x01" + "\x5d\xde\x92\xfe\x90\x07\x09\xfd" + "\xa5\xd1\xd3\x72\x11\x6d\xa4\x4e" + "\xd1\x6e\x16\xd1\xf6\x39\x4f\xa0", + .ctext = "\x3b\xc5\xee\xfc\x05\xaf\xa6\xb7" + "\xfe\x12\x24\x79\x31\xad\x32\xb5" + "\x64\x5a\x17\xc9\xbf\x1f\xdc\xce" + "\x8d\x73\x00\x71\xd9\xfb\xd2\xe6" + "\xc3\x54\xb4\xf3\x36\xe8\x89\x12" + "\x5a\x32\x0b\xa6\xec\x5f\x89\xe7" + "\xe8\x34\x92\xa6\xce\xde\x8f\xf9" + "\x4f\xda\xed\x61\x8e\xb2\x81\xbe" + "\xf2\x15\x85\xbe\xa1\x5f\x19\x85" + "\x71\x7e\xda\x46\x59\xed\x5d\xb0" + "\xd9\x68\x97\xe0\xcd\x1d\x1b\x65" + "\xf5\xc9\x44\xe2\xb4\x42\x17\x7c" + "\xe7\x58\xf3\x2f\xcf\xbe\x5c\x66" + "\xaa\xd3\x61\xa5\x9a\x79\xbb\xa0", + .len = 112, + }, { + .key = "\x60\xb6\xde\x17\xca\x4c\xe7\xe0" + "\x07\x0d\x80\xc5\x8a\x2d\x5a\xc2" + "\x2c\xb9\xa4\x5f\x2a\x85\x2c\x3d" + "\x6d\x67\xc8\xee\x0f\xa2\xf4\x09", + .klen = 32, + .iv = "\x1a\xa5\xbc\x7e\x93\xf6\xdd\x28" + "\xb7\x69\x27\xa1\x84\x95\x25\x5a", + .ptext = "\x7b\x88\x00\xeb\xa5\xba\xa1\xa7" + "\xd4\x40\x16\x74\x2b\x42\x37\xda" + "\xe0\xaf\x89\x59\x41\x2f\x62\x00" + "\xf5\x5a\x4e\x3b\x85\x27\xb2\xed" + "\x1b\xa7\xaf\xbe\x89\xf3\x49\xb7" + "\x8c\x63\xc9\x0c\x52\x00\x5f\x38" + "\x3b\x3c\x0c\x4f\xdd\xe1\xbf\x90" + "\x4a\x48\xbf\x3a\x95\xcb\x48\xa2" + "\x92\x7c\x79\x81\xde\x18\x6e\x92" + "\x1f\x36\xa9\x5d\x8d\xc4\xb6\x4d" + "\xb2\xb4\x0e\x09\x6d\xf3\x3d\x01" + "\x3d\x9b\x40\x47\xbc\x69\x31\xa1" + "\x6a\x71\x26\xdc\xac\x10\x56\x63" + "\x15\x23\x7d\x10\xe3\x76\x82\x41" + "\xcd\x80\x57\x2f\xfc\x4d\x22\x7b" + "\x57\xbb\x9a\x0a\x03\xe9\xb3\x13", + .ctext = "\x37\x0d\x47\x21\xbc\x28\x0b\xf7" + "\x85\x5f\x60\x57\xf2\x7f\x92\x20" + "\x53\x1a\xbf\xd1\x7f\x8c\x39\x29" + "\x0e\x18\xab\x0c\x00\x92\xd3\x68" + "\x60\x56\x3b\x00\xef\xf8\x02\xfa" + "\xcb\x92\x1a\x91\xe1\xf0\x4f\x8a" + "\xc6\x4f\x65\x16\x71\x8b\x5d\xd5" + "\x79\xa9\x6d\x68\x1b\x59\xe7\x2a" + "\x1c\xd0\x5d\xfb\x06\x3b\x15\x72" + "\xa8\xd1\x59\x9a\xb2\x6c\xf2\xd5" + "\x19\xef\xde\x03\x4c\x75\x65\x38" + "\x5b\xda\xc9\xf0\x44\x99\xb2\x6e" + "\x78\xfb\x85\x5a\x92\x91\x1a\x0a" + "\x13\x0c\x1b\x1c\xbe\xbe\x46\x6e" + "\x73\xff\xc2\x6e\xb9\x06\x16\x7e" + "\xf6\xc0\x01\x30\x34\x56\x46\x55", + .len = 128, + }, { + .key = "\x2a\xed\x7d\x76\xfc\xc5\x49\x50" + "\xf4\x90\x0f\xcc\x5d\xff\x0c\x3c" + "\x14\x06\xaf\x68\x8f\xd7\xb6\x25" + "\x1e\x10\x95\x2a\x71\x33\x17\x20", + .klen = 32, + .iv = "\x5b\x58\x47\xf8\xd5\x1e\x91\x81" + "\x46\xe7\x25\x3a\x02\x45\x9c\x65", + .ptext = "\x10\xaf\xde\x5c\x30\x79\x43\x28" + "\x1c\x03\xf8\x50\x0f\x30\xa5\xef" + "\x84\x19\x4c\x09\x40\x03\x75\x1f" + "\x92\x8f\x88\x01\xda\x31\x7a\xe4" + "\x48\xe3\xab\xb4\xe6\x1b\x0f\xac" + "\xd9\xfa\x8d\x23\xe4\xc6\xa4\xa9" + "\x2d\x9a\x54\x52\x44\x5c\x3c\x52" + "\x61\xf0\x00\xca\xed\xab\xed\xe2" + "\x44\x0b\xe0\x18\xba\xa5\x63\xd8" + "\xdc\x5e\x1a\x4c\xf8\xde\x5e\x75" + "\xdf\x42\x27\x7b\xe9\x11\x2f\x41" + "\x3a\x72\x54\x3d\x44\x9c\x3e\x87" + "\x8d\x8d\x43\x2f\xb2\xff\x87\xd4" + "\xad\x98\x68\x72\x53\x61\x19\x7c" + "\x20\x79\x8c\x2b\x37\x0b\x96\x15" + "\xa5\x7d\x4e\x01\xe6\xea\xb6\xfa" + "\xaa\xd3\x9d\xa2\xd9\x11\xc3\xc9" + "\xd4\x0e\x3f\x3e\xfe\x35\x1e\xe5", + .ctext = "\xb0\x2b\x75\x5f\x33\x1b\x05\x49" + "\x06\xf1\x43\x91\xc2\x85\xfa\xac" + "\x74\xd5\x8c\xc9\x47\x6e\x5a\xf6" + "\x69\x33\x4c\xcb\x2f\x36\x4b\x41" + "\xec\x05\x69\xab\x7f\x42\xc9\xd2" + "\x26\x64\x51\x9e\x3d\x65\x35\xf0" + "\x8d\x5e\x8a\xb1\xee\xdf\x1a\x98" + "\x36\xd2\x37\x49\x5b\xe2\x57\x00" + "\x1d\x72\x7e\xe8\x38\x11\x83\x15" + "\xc7\x4e\x65\xa4\x2c\x9e\x6a\x3e" + "\xb4\x78\x3f\xe9\x91\x5d\x06\xa9" + "\xf1\xfc\x6b\x08\xe5\x2b\x2a\x99" + "\x65\xa7\x2e\x47\xf9\xc2\xb1\x8b" + "\x88\x2f\xb7\x62\x84\x63\x94\x00" + "\x49\xa7\xd0\x2b\x54\x7a\x69\xb3" + "\x04\x66\xfc\x97\x40\x92\xd1\xb8" + "\xb4\x2a\x9e\xdb\x31\xcd\x48\x84" + "\x29\x3b\x02\xac\xb8\x54\x95\xb4", + .len = 144, + }, { + .key = "\x7b\xa7\x4d\x0a\x37\x30\xb9\xf5" + "\x2a\x79\xb4\xbf\xdb\x7f\x9b\x64" + "\x23\x43\xb5\x18\x34\xc4\x5f\xdf" + "\xd9\x2a\x66\x58\x00\x44\xb5\xd9", + .klen = 32, + .iv = "\x75\x34\x30\xc1\xf0\x69\xdf\x0a" + "\x52\xce\x4f\x1e\x2c\x41\x35\xec", + .ptext = "\x81\x47\x55\x3a\xcd\xfe\xa2\x3d" + "\x45\x53\xa7\x67\x61\x74\x25\x80" + "\x98\x89\xfe\xf8\x6a\x9f\x51\x7c" + "\xa4\xe4\xe7\xc7\xe0\x1a\xce\xbb" + "\x4b\x46\x43\xb0\xab\xa8\xd6\x0c" + "\xa0\xf0\xc8\x13\x29\xaf\xb8\x01" + "\x6b\x0c\x7e\x56\xae\xb8\x58\x72" + "\xa9\x24\x44\x61\xff\xf1\xac\xf8" + "\x09\xa8\x48\x21\xd6\xab\x41\x73" + "\x70\x6b\x92\x06\x61\xdc\xb4\x85" + "\x76\x26\x7a\x84\xc3\x9e\x3a\x14" + "\xe7\xf4\x2d\x95\x92\xad\x18\xcc" + "\x44\xd4\x2c\x36\x57\xed\x2b\x9b" + "\x3f\x2b\xcd\xe5\x11\xe3\x62\x33" + "\x42\x3f\xb8\x2a\xb1\x37\x3f\x8b" + "\xe8\xbd\x6b\x0b\x9f\x38\x5a\x5f" + "\x82\x34\xb7\x96\x35\x58\xde\xab" + "\x94\x98\x41\x5b\x3f\xac\x0a\x34" + "\x56\xc0\x02\xef\x81\x6d\xb1\xff" + "\x34\xe8\xc7\x6a\x31\x79\xba\xd8", + .ctext = "\x4e\x00\x7c\x52\x45\x76\xf9\x3d" + "\x1a\xd1\x72\xbc\xb9\x0f\xa9\xfb" + "\x0a\xf5\xe8\x11\x66\x8b\xad\x68" + "\x5a\x2e\xbf\x09\x33\x9d\xb6\x67" + "\xe5\xcb\x0a\xe0\xac\xed\x73\x4b" + "\xbb\x15\xde\xd8\xab\x33\x28\x5f" + "\x96\x07\x3c\x28\x79\x88\x84\xc7" + "\x13\xf7\x0d\xa5\x97\x3b\xd9\xb1" + "\xf2\x65\xb0\xac\xbb\x8a\x97\xd1" + "\x70\x3a\x91\x65\xc8\x39\x04\xe7" + "\x1a\x9c\x80\x65\x2b\x69\x4b\xdc" + "\xdc\xc7\xf1\x31\xda\xab\xb4\xd7" + "\x46\x2e\x1d\xc9\x2e\xe9\x46\xec" + "\xa4\xa1\x91\x6b\x4a\x09\xf9\x39" + "\x7b\x7d\x6d\xf5\x43\x7f\xcc\x74" + "\x96\xfa\x48\xd0\xe1\x74\x24\xd0" + "\x19\x22\x24\x84\x2b\x12\x10\x46" + "\x90\xbd\xa9\x93\xb7\xf7\x36\xd4" + "\x48\xc7\x32\x83\x8c\xa9\xcd\x5a" + "\x2f\x05\x33\xc1\x5b\x50\x70\xc4", + .len = 160, + } +}; + +static const struct aead_testvec aria_gcm_tv_template[] = { + { + .key = "\xe9\x1e\x5e\x75\xda\x65\x55\x4a" + "\x48\x18\x1f\x38\x46\x34\x95\x62", + .klen = 16, + .iv = "\x00\x00\x20\xe8\xf5\xeb\x00\x00" + "\x00\x00\x31\x5e", + .assoc = "\x80\x08\x31\x5e\xbf\x2e\x6f\xe0" + "\x20\xe8\xf5\xeb", + .alen = 12, + .ptext = "\xf5\x7a\xf5\xfd\x4a\xe1\x95\x62" + "\x97\x6e\xc5\x7a\x5a\x7a\xd5\x5a" + "\x5a\xf5\xc5\xe5\xc5\xfd\xf5\xc5" + "\x5a\xd5\x7a\x4a\x72\x72\xd5\x72" + "\x62\xe9\x72\x95\x66\xed\x66\xe9" + "\x7a\xc5\x4a\x4a\x5a\x7a\xd5\xe1" + "\x5a\xe5\xfd\xd5\xfd\x5a\xc5\xd5" + "\x6a\xe5\x6a\xd5\xc5\x72\xd5\x4a" + "\xe5\x4a\xc5\x5a\x95\x6a\xfd\x6a" + "\xed\x5a\x4a\xc5\x62\x95\x7a\x95" + "\x16\x99\x16\x91\xd5\x72\xfd\x14" + "\xe9\x7a\xe9\x62\xed\x7a\x9f\x4a" + "\x95\x5a\xf5\x72\xe1\x62\xf5\x7a" + "\x95\x66\x66\xe1\x7a\xe1\xf5\x4a" + "\x95\xf5\x66\xd5\x4a\x66\xe1\x6e" + "\x4a\xfd\x6a\x9f\x7a\xe1\xc5\xc5" + "\x5a\xe5\xd5\x6a\xfd\xe9\x16\xc5" + "\xe9\x4a\x6e\xc5\x66\x95\xe1\x4a" + "\xfd\xe1\x14\x84\x16\xe9\x4a\xd5" + "\x7a\xc5\x14\x6e\xd5\x9d\x1c\xc5", + .plen = 160, + .ctext = "\x4d\x8a\x9a\x06\x75\x55\x0c\x70" + "\x4b\x17\xd8\xc9\xdd\xc8\x1a\x5c" + "\xd6\xf7\xda\x34\xf2\xfe\x1b\x3d" + "\xb7\xcb\x3d\xfb\x96\x97\x10\x2e" + "\xa0\xf3\xc1\xfc\x2d\xbc\x87\x3d" + "\x44\xbc\xee\xae\x8e\x44\x42\x97" + "\x4b\xa2\x1f\xf6\x78\x9d\x32\x72" + "\x61\x3f\xb9\x63\x1a\x7c\xf3\xf1" + "\x4b\xac\xbe\xb4\x21\x63\x3a\x90" + "\xff\xbe\x58\xc2\xfa\x6b\xdc\xa5" + "\x34\xf1\x0d\x0d\xe0\x50\x2c\xe1" + "\xd5\x31\xb6\x33\x6e\x58\x87\x82" + "\x78\x53\x1e\x5c\x22\xbc\x6c\x85" + "\xbb\xd7\x84\xd7\x8d\x9e\x68\x0a" + "\xa1\x90\x31\xaa\xf8\x91\x01\xd6" + "\x69\xd7\xa3\x96\x5c\x1f\x7e\x16" + "\x22\x9d\x74\x63\xe0\x53\x5f\x4e" + "\x25\x3f\x5d\x18\x18\x7d\x40\xb8" + "\xae\x0f\x56\x4b\xd9\x70\xb5\xe7" + "\xe2\xad\xfb\x21\x1e\x89\xa9\x53" + "\x5a\xba\xce\x3f\x37\xf5\xa7\x36" + "\xf4\xbe\x98\x4b\xbf\xfb\xed\xc1", + .clen = 176, + }, { + .key = "\x0c\x5f\xfd\x37\xa1\x1e\xdc\x42" + "\xc3\x25\x28\x7f\xc0\x60\x4f\x2e" + "\x3e\x8c\xd5\x67\x1a\x00\xfe\x32" + "\x16\xaa\x5e\xb1\x05\x78\x3b\x54", + .klen = 32, + .iv = "\x00\x00\x20\xe8\xf5\xeb\x00\x00" + "\x00\x00\x31\x5e", + .assoc = "\x80\x08\x31\x5e\xbf\x2e\x6f\xe0" + "\x20\xe8\xf5\xeb", + .alen = 12, + .ptext = "\xf5\x7a\xf5\xfd\x4a\xe1\x95\x62" + "\x97\x6e\xc5\x7a\x5a\x7a\xd5\x5a" + "\x5a\xf5\xc5\xe5\xc5\xfd\xf5\xc5" + "\x5a\xd5\x7a\x4a\x72\x72\xd5\x72" + "\x62\xe9\x72\x95\x66\xed\x66\xe9" + "\x7a\xc5\x4a\x4a\x5a\x7a\xd5\xe1" + "\x5a\xe5\xfd\xd5\xfd\x5a\xc5\xd5" + "\x6a\xe5\x6a\xd5\xc5\x72\xd5\x4a" + "\xe5\x4a\xc5\x5a\x95\x6a\xfd\x6a" + "\xed\x5a\x4a\xc5\x62\x95\x7a\x95" + "\x16\x99\x16\x91\xd5\x72\xfd\x14" + "\xe9\x7a\xe9\x62\xed\x7a\x9f\x4a" + "\x95\x5a\xf5\x72\xe1\x62\xf5\x7a" + "\x95\x66\x66\xe1\x7a\xe1\xf5\x4a" + "\x95\xf5\x66\xd5\x4a\x66\xe1\x6e" + "\x4a\xfd\x6a\x9f\x7a\xe1\xc5\xc5" + "\x5a\xe5\xd5\x6a\xfd\xe9\x16\xc5" + "\xe9\x4a\x6e\xc5\x66\x95\xe1\x4a" + "\xfd\xe1\x14\x84\x16\xe9\x4a\xd5" + "\x7a\xc5\x14\x6e\xd5\x9d\x1c\xc5", + .plen = 160, + .ctext = "\x6f\x9e\x4b\xcb\xc8\xc8\x5f\xc0" + "\x12\x8f\xb1\xe4\xa0\xa2\x0c\xb9" + "\x93\x2f\xf7\x45\x81\xf5\x4f\xc0" + "\x13\xdd\x05\x4b\x19\xf9\x93\x71" + "\x42\x5b\x35\x2d\x97\xd3\xf3\x37" + "\xb9\x0b\x63\xd1\xb0\x82\xad\xee" + "\xea\x9d\x2d\x73\x91\x89\x7d\x59" + "\x1b\x98\x5e\x55\xfb\x50\xcb\x53" + "\x50\xcf\x7d\x38\xdc\x27\xdd\xa1" + "\x27\xc0\x78\xa1\x49\xc8\xeb\x98" + "\x08\x3d\x66\x36\x3a\x46\xe3\x72" + "\x6a\xf2\x17\xd3\xa0\x02\x75\xad" + "\x5b\xf7\x72\xc7\x61\x0e\xa4\xc2" + "\x30\x06\x87\x8f\x0e\xe6\x9a\x83" + "\x97\x70\x31\x69\xa4\x19\x30\x3f" + "\x40\xb7\x2e\x45\x73\x71\x4d\x19" + "\xe2\x69\x7d\xf6\x1e\x7c\x72\x52" + "\xe5\xab\xc6\xba\xde\x87\x6a\xc4" + "\x96\x1b\xfa\xc4\xd5\xe8\x67\xaf" + "\xca\x35\x1a\x48\xae\xd5\x28\x22" + "\xe2\x10\xd6\xce\xd2\xcf\x43\x0f" + "\xf8\x41\x47\x29\x15\xe7\xef\x48", + .clen = 176, + } +}; + static const struct cipher_testvec chacha20_tv_template[] = { { /* RFC7539 A.2. Test Vector #1 */ .key = "\x00\x00\x00\x00\x00\x00\x00\x00" From 30fb034361ff1b9bfc569b2d8d66b544ea3eb18f Mon Sep 17 00:00:00 2001 From: Yuan Can Date: Mon, 4 Jul 2022 13:01:45 +0000 Subject: [PATCH 223/318] crypto: ccree - Add missing clk_disable_unprepare() in cc_pm_resume() Add clk_disable_unprepare() on error path in cc_pm_resume(). Reported-by: Hulk Robot Signed-off-by: Yuan Can Signed-off-by: Herbert Xu --- drivers/crypto/ccree/cc_pm.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/crypto/ccree/cc_pm.c b/drivers/crypto/ccree/cc_pm.c index d5421b0c6831..6124fbbbed94 100644 --- a/drivers/crypto/ccree/cc_pm.c +++ b/drivers/crypto/ccree/cc_pm.c @@ -41,6 +41,7 @@ static int cc_pm_resume(struct device *dev) /* wait for Cryptocell reset completion */ if (!cc_wait_for_reset_completion(drvdata)) { dev_err(dev, "Cryptocell reset not completed"); + clk_disable_unprepare(drvdata->clk); return -EBUSY; } @@ -48,6 +49,7 @@ static int cc_pm_resume(struct device *dev) rc = init_cc_regs(drvdata); if (rc) { dev_err(dev, "init_cc_regs (%x)\n", rc); + clk_disable_unprepare(drvdata->clk); return rc; } /* check if tee fips error occurred during power down */ From 4fdcabb86ab17621b54695e5bb52ceddef434e82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Tue, 5 Jul 2022 22:51:38 +0200 Subject: [PATCH 224/318] crypto: atmel-aes - Drop if with an always false condition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The remove callback is only called after probe completed successfully. In this case platform_set_drvdata() was called with a non-NULL argument and so aes_dd is never NULL. This is a preparation for making platform remove callbacks return void. Signed-off-by: Uwe Kleine-König Signed-off-by: Herbert Xu --- drivers/crypto/atmel-aes.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c index f72c6b3e4ad8..886bf258544c 100644 --- a/drivers/crypto/atmel-aes.c +++ b/drivers/crypto/atmel-aes.c @@ -2669,8 +2669,7 @@ static int atmel_aes_remove(struct platform_device *pdev) struct atmel_aes_dev *aes_dd; aes_dd = platform_get_drvdata(pdev); - if (!aes_dd) - return -ENODEV; + spin_lock(&atmel_aes.lock); list_del(&aes_dd->list); spin_unlock(&atmel_aes.lock); From 25edb4cddb0f3deb260b52106b73c3d0f4968489 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Tue, 5 Jul 2022 22:51:39 +0200 Subject: [PATCH 225/318] crypto: atmel-sha - Drop if with an always false condition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The remove callback is only called after probe completed successfully. In this case platform_set_drvdata() was called with a non-NULL argument and so sha_dd is never NULL. This is a preparation for making platform remove callbacks return void. Signed-off-by: Uwe Kleine-König Reviewed-by: Claudiu Beznea Signed-off-by: Herbert Xu --- drivers/crypto/atmel-sha.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c index d1628112dacc..e054e0ac6fc2 100644 --- a/drivers/crypto/atmel-sha.c +++ b/drivers/crypto/atmel-sha.c @@ -2669,8 +2669,7 @@ static int atmel_sha_remove(struct platform_device *pdev) struct atmel_sha_dev *sha_dd; sha_dd = platform_get_drvdata(pdev); - if (!sha_dd) - return -ENODEV; + spin_lock(&atmel_sha.lock); list_del(&sha_dd->list); spin_unlock(&atmel_sha.lock); From 515f4fc66bf176b91d24d89deddc18e0be12543f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Tue, 5 Jul 2022 22:51:40 +0200 Subject: [PATCH 226/318] crypto: atmel-tdes - Drop if with an always false condition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The remove callback is only called after probe completed successfully. In this case platform_set_drvdata() was called with a non-NULL argument and so tdes_dd is never NULL. This is a preparation for making platform remove callbacks return void. Signed-off-by: Uwe Kleine-König Reviewed-by: Claudiu Beznea Signed-off-by: Herbert Xu --- drivers/crypto/atmel-tdes.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c index 9fd7b8e439d2..a5e78aa08bf0 100644 --- a/drivers/crypto/atmel-tdes.c +++ b/drivers/crypto/atmel-tdes.c @@ -1266,8 +1266,7 @@ static int atmel_tdes_remove(struct platform_device *pdev) struct atmel_tdes_dev *tdes_dd; tdes_dd = platform_get_drvdata(pdev); - if (!tdes_dd) - return -ENODEV; + spin_lock(&atmel_tdes.lock); list_del(&tdes_dd->list); spin_unlock(&atmel_tdes.lock); From da1e716864aeeaeef0a32a7cdddfb8c6ebd8c1f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Tue, 5 Jul 2022 22:51:41 +0200 Subject: [PATCH 227/318] crypto: omap-aes - Drop if with an always false condition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The remove callback is only called after probe completed successfully. In this case platform_set_drvdata() was called with a non-NULL argument and so dd is never NULL. This is a preparation for making platform remove callbacks return void. While touching this driver remove a stray empty line. Signed-off-by: Uwe Kleine-König Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 581211a92628..67a99c760bc4 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -1261,9 +1261,6 @@ static int omap_aes_remove(struct platform_device *pdev) struct aead_alg *aalg; int i, j; - if (!dd) - return -ENODEV; - spin_lock_bh(&list_lock); list_del(&dd->list); spin_unlock_bh(&list_lock); @@ -1279,7 +1276,6 @@ static int omap_aes_remove(struct platform_device *pdev) aalg = &dd->pdata->aead_algs_info->algs_list[i]; crypto_unregister_aead(aalg); dd->pdata->aead_algs_info->registered--; - } crypto_engine_exit(dd->engine); From 8ce715e711659322a6e05cad154aa1355bb2e14f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Tue, 5 Jul 2022 22:51:42 +0200 Subject: [PATCH 228/318] crypto: omap-des - Drop if with an always false condition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The remove callback is only called after probe completed successfully. In this case platform_set_drvdata() was called with a non-NULL argument and so dd is never NULL. This is a preparation for making platform remove callbacks return void. While touching this driver remove an assignment without effect. Signed-off-by: Uwe Kleine-König Signed-off-by: Herbert Xu --- drivers/crypto/omap-des.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c index 538aff80869f..f783769ea110 100644 --- a/drivers/crypto/omap-des.c +++ b/drivers/crypto/omap-des.c @@ -1091,9 +1091,6 @@ static int omap_des_remove(struct platform_device *pdev) struct omap_des_dev *dd = platform_get_drvdata(pdev); int i, j; - if (!dd) - return -ENODEV; - spin_lock_bh(&list_lock); list_del(&dd->list); spin_unlock_bh(&list_lock); @@ -1106,7 +1103,6 @@ static int omap_des_remove(struct platform_device *pdev) tasklet_kill(&dd->done_task); omap_des_dma_cleanup(dd); pm_runtime_disable(dd->dev); - dd = NULL; return 0; } From 35b22c19daa1bbe77dce2f7f776c0cf2acff1f00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Tue, 5 Jul 2022 22:51:43 +0200 Subject: [PATCH 229/318] crypto: omap-sham - Drop if with an always false condition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The remove callback is only called after probe completed successfully. In this case platform_set_drvdata() was called with a non-NULL argument and so dd is never NULL. This is a preparation for making platform remove callbacks return void. Signed-off-by: Uwe Kleine-König Signed-off-by: Herbert Xu --- drivers/crypto/omap-sham.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 4b37dc69a50c..655a7f5a406a 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -2197,8 +2197,7 @@ static int omap_sham_remove(struct platform_device *pdev) int i, j; dd = platform_get_drvdata(pdev); - if (!dd) - return -ENODEV; + spin_lock_bh(&sham.lock); list_del(&dd->list); spin_unlock_bh(&sham.lock); From 1d5390a33a4b54b2129316656792d55755a03d06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Tue, 5 Jul 2022 22:51:44 +0200 Subject: [PATCH 230/318] crypto: s5p-sss - Drop if with an always false condition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The remove callback is only called after probe completed successfully. In this case platform_set_drvdata() was called with a non-NULL argument and so pdata is never NULL. This is a preparation for making platform remove callbacks return void. Signed-off-by: Uwe Kleine-König Acked-by: Vladimir Zapolskiy Reviewed-by: Krzysztof Kozlowski Signed-off-by: Herbert Xu --- drivers/crypto/s5p-sss.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c index 7717e9e5977b..b79e49aa724f 100644 --- a/drivers/crypto/s5p-sss.c +++ b/drivers/crypto/s5p-sss.c @@ -2321,9 +2321,6 @@ static int s5p_aes_remove(struct platform_device *pdev) struct s5p_aes_dev *pdata = platform_get_drvdata(pdev); int i; - if (!pdata) - return -ENODEV; - for (i = 0; i < ARRAY_SIZE(algs); i++) crypto_unregister_skcipher(&algs[i]); From 2e26efb384d8a38ac62dc7022cff448cf54b80ee Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Tue, 5 Jul 2022 15:58:57 -0700 Subject: [PATCH 231/318] crypto: caam/qi2 - switch to netif_napi_add_tx_weight() caam has its own special NAPI weights. It's also a crypto device so presumably it can't be used for packet Rx. Switch to the (new) correct API. Signed-off-by: Jakub Kicinski Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg_qi2.c | 5 +++-- drivers/crypto/caam/qi.c | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c index 4b81fb33f199..4482cb145d05 100644 --- a/drivers/crypto/caam/caamalg_qi2.c +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -5083,8 +5083,9 @@ static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev) ppriv->net_dev.dev = *dev; INIT_LIST_HEAD(&ppriv->net_dev.napi_list); - netif_napi_add(&ppriv->net_dev, &ppriv->napi, dpaa2_dpseci_poll, - DPAA2_CAAM_NAPI_WEIGHT); + netif_napi_add_tx_weight(&ppriv->net_dev, &ppriv->napi, + dpaa2_dpseci_poll, + DPAA2_CAAM_NAPI_WEIGHT); } return 0; diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c index 49439d0d1b3c..c36f27376d7e 100644 --- a/drivers/crypto/caam/qi.c +++ b/drivers/crypto/caam/qi.c @@ -749,8 +749,8 @@ int caam_qi_init(struct platform_device *caam_pdev) net_dev->dev = *qidev; INIT_LIST_HEAD(&net_dev->napi_list); - netif_napi_add(net_dev, irqtask, caam_qi_poll, - CAAM_NAPI_WEIGHT); + netif_napi_add_tx_weight(net_dev, irqtask, caam_qi_poll, + CAAM_NAPI_WEIGHT); napi_enable(irqtask); } From c6a16f4bbf5518603b51df23f23e46223244dcb0 Mon Sep 17 00:00:00 2001 From: Claudiu Beznea Date: Wed, 6 Jul 2022 11:58:44 +0300 Subject: [PATCH 232/318] crypto: atmel-sha - initialize sha_dd while declaring Initialize sha_dd with platform_get_drvdata() when declaring it. Signed-off-by: Claudiu Beznea Signed-off-by: Herbert Xu --- drivers/crypto/atmel-sha.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c index e054e0ac6fc2..ca4b01926d1b 100644 --- a/drivers/crypto/atmel-sha.c +++ b/drivers/crypto/atmel-sha.c @@ -2666,9 +2666,7 @@ err_tasklet_kill: static int atmel_sha_remove(struct platform_device *pdev) { - struct atmel_sha_dev *sha_dd; - - sha_dd = platform_get_drvdata(pdev); + struct atmel_sha_dev *sha_dd = platform_get_drvdata(pdev); spin_lock(&atmel_sha.lock); list_del(&sha_dd->list); From 6c14a9650b8729882995f5fcd6c62336b40aeb26 Mon Sep 17 00:00:00 2001 From: Claudiu Beznea Date: Wed, 6 Jul 2022 12:17:27 +0300 Subject: [PATCH 233/318] crypto: atmel-tdes - initialize tdes_dd while declaring Initialize sha_dd with platform_get_drvdata() when declaring it. Signed-off-by: Claudiu Beznea Signed-off-by: Herbert Xu --- drivers/crypto/atmel-tdes.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c index a5e78aa08bf0..8b7bc1076e0d 100644 --- a/drivers/crypto/atmel-tdes.c +++ b/drivers/crypto/atmel-tdes.c @@ -1263,9 +1263,7 @@ err_tasklet_kill: static int atmel_tdes_remove(struct platform_device *pdev) { - struct atmel_tdes_dev *tdes_dd; - - tdes_dd = platform_get_drvdata(pdev); + struct atmel_tdes_dev *tdes_dd = platform_get_drvdata(pdev); spin_lock(&atmel_tdes.lock); list_del(&tdes_dd->list); From a65c9a2a0b43118ee6f00eeeb73aefdcbd89728f Mon Sep 17 00:00:00 2001 From: Andrew Davis Date: Wed, 6 Jul 2022 14:11:43 -0500 Subject: [PATCH 234/318] crypto: sa2ul - Set the supported_algos bits individually Setting these individually gives a better picture of supported functions at a glance. Plus if the list changes an unwanted one will not accidentally get set with GENMASK. Signed-off-by: Andrew Davis Signed-off-by: Herbert Xu --- drivers/crypto/sa2ul.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c index 6957a125b447..1d732113b81e 100644 --- a/drivers/crypto/sa2ul.c +++ b/drivers/crypto/sa2ul.c @@ -2361,7 +2361,15 @@ static int sa_link_child(struct device *dev, void *data) static struct sa_match_data am654_match_data = { .priv = 1, .priv_id = 1, - .supported_algos = GENMASK(SA_ALG_AUTHENC_SHA256_AES, 0), + .supported_algos = BIT(SA_ALG_CBC_AES) | + BIT(SA_ALG_EBC_AES) | + BIT(SA_ALG_CBC_DES3) | + BIT(SA_ALG_ECB_DES3) | + BIT(SA_ALG_SHA1) | + BIT(SA_ALG_SHA256) | + BIT(SA_ALG_SHA512) | + BIT(SA_ALG_AUTHENC_SHA1_AES) | + BIT(SA_ALG_AUTHENC_SHA256_AES), }; static struct sa_match_data am64_match_data = { From b77e34f5b10de529255c9468203d0644a7af3b81 Mon Sep 17 00:00:00 2001 From: Andrew Davis Date: Wed, 6 Jul 2022 14:11:44 -0500 Subject: [PATCH 235/318] crypto: sa2ul - Check engine status before enabling There is a engine status register that can be used to check if the different HW crypto engines are enabled. Check that first and then only try to enable the engines if they are not already on. This has a couple benefits. First we don't need to use match_data for this. Second, this driver can now work on HS devices where the engine control registers are read-only and writing causes a firewall exception. Signed-off-by: Andrew Davis Signed-off-by: Herbert Xu --- drivers/crypto/sa2ul.c | 15 +++++++-------- drivers/crypto/sa2ul.h | 1 + 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c index 1d732113b81e..f4bc06c24ad8 100644 --- a/drivers/crypto/sa2ul.c +++ b/drivers/crypto/sa2ul.c @@ -86,7 +86,6 @@ struct sa_match_data { u8 priv; u8 priv_id; u32 supported_algos; - bool skip_engine_control; }; static struct device *sa_k3_dev; @@ -2380,7 +2379,6 @@ static struct sa_match_data am64_match_data = { BIT(SA_ALG_SHA256) | BIT(SA_ALG_SHA512) | BIT(SA_ALG_AUTHENC_SHA256_AES), - .skip_engine_control = true, }; static const struct of_device_id of_match[] = { @@ -2398,6 +2396,7 @@ static int sa_ul_probe(struct platform_device *pdev) struct device_node *node = dev->of_node; static void __iomem *saul_base; struct sa_crypto_data *dev_data; + u32 status, val; int ret; dev_data = devm_kzalloc(dev, sizeof(*dev_data), GFP_KERNEL); @@ -2434,13 +2433,13 @@ static int sa_ul_probe(struct platform_device *pdev) spin_lock_init(&dev_data->scid_lock); - if (!dev_data->match_data->skip_engine_control) { - u32 val = SA_EEC_ENCSS_EN | SA_EEC_AUTHSS_EN | SA_EEC_CTXCACH_EN | - SA_EEC_CPPI_PORT_IN_EN | SA_EEC_CPPI_PORT_OUT_EN | - SA_EEC_TRNG_EN; - + val = SA_EEC_ENCSS_EN | SA_EEC_AUTHSS_EN | SA_EEC_CTXCACH_EN | + SA_EEC_CPPI_PORT_IN_EN | SA_EEC_CPPI_PORT_OUT_EN | + SA_EEC_TRNG_EN; + status = readl_relaxed(saul_base + SA_ENGINE_STATUS); + /* Only enable engines if all are not already enabled */ + if (val & ~status) writel_relaxed(val, saul_base + SA_ENGINE_ENABLE_CONTROL); - } sa_register_algos(dev_data); diff --git a/drivers/crypto/sa2ul.h b/drivers/crypto/sa2ul.h index ed66d1f111db..92bf97232a29 100644 --- a/drivers/crypto/sa2ul.h +++ b/drivers/crypto/sa2ul.h @@ -16,6 +16,7 @@ #include #include +#define SA_ENGINE_STATUS 0x0008 #define SA_ENGINE_ENABLE_CONTROL 0x1000 struct sa_tfm_ctx; From 1353e576ae3b7b9703b74f6f2276b6dd450f4a2e Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Thu, 7 Jul 2022 09:05:46 +0100 Subject: [PATCH 236/318] crypto: x86/blowfish - remove redundant assignment to variable nytes Variable nbytes is being assigned a value that is never read, it is being re-assigned in the next statement in the while-loop. The assignment is redundant and can be removed. Cleans up clang scan-build warnings, e.g.: arch/x86/crypto/blowfish_glue.c:147:10: warning: Although the value stored to 'nbytes' is used in the enclosing expression, the value is never actually read from 'nbytes' Signed-off-by: Colin Ian King Signed-off-by: Herbert Xu --- arch/x86/crypto/blowfish_glue.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/crypto/blowfish_glue.c b/arch/x86/crypto/blowfish_glue.c index ba06322c1e39..019c64c1340a 100644 --- a/arch/x86/crypto/blowfish_glue.c +++ b/arch/x86/crypto/blowfish_glue.c @@ -144,7 +144,7 @@ static int cbc_encrypt(struct skcipher_request *req) err = skcipher_walk_virt(&walk, req, false); - while ((nbytes = walk.nbytes)) { + while (walk.nbytes) { nbytes = __cbc_encrypt(ctx, &walk); err = skcipher_walk_done(&walk, nbytes); } @@ -225,7 +225,7 @@ static int cbc_decrypt(struct skcipher_request *req) err = skcipher_walk_virt(&walk, req, false); - while ((nbytes = walk.nbytes)) { + while (walk.nbytes) { nbytes = __cbc_decrypt(ctx, &walk); err = skcipher_walk_done(&walk, nbytes); } From 5a44749f65b2342d43dea82024e4febdac33c78d Mon Sep 17 00:00:00 2001 From: Vladis Dronov Date: Fri, 8 Jul 2022 14:33:13 +0200 Subject: [PATCH 237/318] crypto: fips - make proc files report fips module name and version FIPS 140-3 introduced a requirement for the FIPS module to return information about itself, specifically a name and a version. These values must match the values reported on FIPS certificates. This patch adds two files to read a name and a version from: /proc/sys/crypto/fips_name /proc/sys/crypto/fips_version v2: removed redundant parentheses in config entries. v3: move FIPS_MODULE_* defines to fips.c where they are used. v4: return utsrelease.h inclusion Signed-off-by: Simo Sorce Signed-off-by: Vladis Dronov Signed-off-by: Herbert Xu --- crypto/Kconfig | 21 +++++++++++++++++++++ crypto/fips.c | 35 ++++++++++++++++++++++++++++++----- 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index 7d98a2b4ac9c..54bdcf2ce331 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -33,6 +33,27 @@ config CRYPTO_FIPS certification. You should say no unless you know what this is. +config CRYPTO_FIPS_NAME + string "FIPS Module Name" + default "Linux Kernel Cryptographic API" + depends on CRYPTO_FIPS + help + This option sets the FIPS Module name reported by the Crypto API via + the /proc/sys/crypto/fips_name file. + +config CRYPTO_FIPS_CUSTOM_VERSION + bool "Use Custom FIPS Module Version" + depends on CRYPTO_FIPS + default n + +config CRYPTO_FIPS_VERSION + string "FIPS Module Version" + default "(none)" + depends on CRYPTO_FIPS_CUSTOM_VERSION + help + This option provides the ability to override the FIPS Module Version. + By default the KERNELRELEASE value is used. + config CRYPTO_ALGAPI tristate select CRYPTO_ALGAPI2 diff --git a/crypto/fips.c b/crypto/fips.c index 7b1d8caee669..b05d3c7b3ca5 100644 --- a/crypto/fips.c +++ b/crypto/fips.c @@ -12,6 +12,7 @@ #include #include #include +#include int fips_enabled; EXPORT_SYMBOL_GPL(fips_enabled); @@ -30,13 +31,37 @@ static int fips_enable(char *str) __setup("fips=", fips_enable); +#define FIPS_MODULE_NAME CONFIG_CRYPTO_FIPS_NAME +#ifdef CONFIG_CRYPTO_FIPS_CUSTOM_VERSION +#define FIPS_MODULE_VERSION CONFIG_CRYPTO_FIPS_VERSION +#else +#define FIPS_MODULE_VERSION UTS_RELEASE +#endif + +static char fips_name[] = FIPS_MODULE_NAME; +static char fips_version[] = FIPS_MODULE_VERSION; + static struct ctl_table crypto_sysctl_table[] = { { - .procname = "fips_enabled", - .data = &fips_enabled, - .maxlen = sizeof(int), - .mode = 0444, - .proc_handler = proc_dointvec + .procname = "fips_enabled", + .data = &fips_enabled, + .maxlen = sizeof(int), + .mode = 0444, + .proc_handler = proc_dointvec + }, + { + .procname = "fips_name", + .data = &fips_name, + .maxlen = 64, + .mode = 0444, + .proc_handler = proc_dostring + }, + { + .procname = "fips_version", + .data = &fips_version, + .maxlen = 64, + .mode = 0444, + .proc_handler = proc_dostring }, {} }; From 463f74089ff9148e3e46af454a6977d40b98cd10 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 9 Jul 2022 14:18:48 -0700 Subject: [PATCH 238/318] crypto: lib - move lib/sha1.c into lib/crypto/ SHA-1 is a crypto algorithm (or at least was intended to be -- it's not considered secure anymore), so move it out of the top-level library directory and into lib/crypto/. Signed-off-by: Eric Biggers Reviewed-by: Jason A. Donenfeld Signed-off-by: Herbert Xu --- lib/Makefile | 2 +- lib/crypto/Makefile | 2 ++ lib/{ => crypto}/sha1.c | 0 3 files changed, 3 insertions(+), 1 deletion(-) rename lib/{ => crypto}/sha1.c (100%) diff --git a/lib/Makefile b/lib/Makefile index f99bf61f8bbc..67482f5ec0e8 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -29,7 +29,7 @@ endif lib-y := ctype.o string.o vsprintf.o cmdline.o \ rbtree.o radix-tree.o timerqueue.o xarray.o \ - idr.o extable.o sha1.o irq_regs.o argv_split.o \ + idr.o extable.o irq_regs.o argv_split.o \ flex_proportions.o ratelimit.o show_mem.o \ is_single_threaded.o plist.o decompress.o kobject_uevent.o \ earlycpio.o seq_buf.o siphash.o dec_and_lock.o \ diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 26be2bbe09c5..d28111ba54fc 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -34,6 +34,8 @@ libpoly1305-y := poly1305-donna32.o libpoly1305-$(CONFIG_ARCH_SUPPORTS_INT128) := poly1305-donna64.o libpoly1305-y += poly1305.o +obj-y += sha1.o + obj-$(CONFIG_CRYPTO_LIB_SHA256) += libsha256.o libsha256-y := sha256.o diff --git a/lib/sha1.c b/lib/crypto/sha1.c similarity index 100% rename from lib/sha1.c rename to lib/crypto/sha1.c From ec8f7f4821d5e70d71601519bc2325b311324a96 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 9 Jul 2022 14:18:49 -0700 Subject: [PATCH 239/318] crypto: lib - make the sha1 library optional Since the Linux RNG no longer uses sha1_transform(), the SHA-1 library is no longer needed unconditionally. Make it possible to build the Linux kernel without the SHA-1 library by putting it behind a kconfig option, and selecting this new option from the kconfig options that gate the remaining users: CRYPTO_SHA1 for crypto/sha1_generic.c, BPF for kernel/bpf/core.c, and IPV6 for net/ipv6/addrconf.c. Unfortunately, since BPF is selected by NET, for now this can only make a difference for kernels built without networking support. Signed-off-by: Eric Biggers Reviewed-by: Jason A. Donenfeld Acked-by: Jakub Kicinski Acked-by: Alexei Starovoitov Signed-off-by: Herbert Xu --- crypto/Kconfig | 1 + init/Kconfig | 1 + lib/crypto/Kconfig | 3 +++ lib/crypto/Makefile | 3 ++- net/ipv6/Kconfig | 1 + 5 files changed, 8 insertions(+), 1 deletion(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index 54bdcf2ce331..75c71d9a5ffb 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -901,6 +901,7 @@ config CRYPTO_RMD160 config CRYPTO_SHA1 tristate "SHA1 digest algorithm" select CRYPTO_HASH + select CRYPTO_LIB_SHA1 help SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2). diff --git a/init/Kconfig b/init/Kconfig index c984afc489de..d8d0b4bdfe41 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1472,6 +1472,7 @@ config HAVE_PCSPKR_PLATFORM # interpreter that classic socket filters depend on config BPF bool + select CRYPTO_LIB_SHA1 menuconfig EXPERT bool "Configure standard kernel features (expert users)" diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 2082af43d51f..9ff549f63540 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -121,6 +121,9 @@ config CRYPTO_LIB_CHACHA20POLY1305 select CRYPTO_LIB_POLY1305 select CRYPTO_ALGAPI +config CRYPTO_LIB_SHA1 + tristate + config CRYPTO_LIB_SHA256 tristate diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index d28111ba54fc..919cbb2c220d 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -34,7 +34,8 @@ libpoly1305-y := poly1305-donna32.o libpoly1305-$(CONFIG_ARCH_SUPPORTS_INT128) := poly1305-donna64.o libpoly1305-y += poly1305.o -obj-y += sha1.o +obj-$(CONFIG_CRYPTO_LIB_SHA1) += libsha1.o +libsha1-y := sha1.o obj-$(CONFIG_CRYPTO_LIB_SHA256) += libsha256.o libsha256-y := sha256.o diff --git a/net/ipv6/Kconfig b/net/ipv6/Kconfig index bf2e5e5fe142..658bfed1df8b 100644 --- a/net/ipv6/Kconfig +++ b/net/ipv6/Kconfig @@ -7,6 +7,7 @@ menuconfig IPV6 tristate "The IPv6 protocol" default y + select CRYPTO_LIB_SHA1 help Support for IP version 6 (IPv6). From 65d9a9a60fd71be964effb2e94747a6acb6e7015 Mon Sep 17 00:00:00 2001 From: "Naveen N. Rao" Date: Fri, 1 Jul 2022 13:04:04 +0530 Subject: [PATCH 240/318] kexec_file: drop weak attribute from functions As requested (http://lkml.kernel.org/r/87ee0q7b92.fsf@email.froward.int.ebiederm.org), this series converts weak functions in kexec to use the #ifdef approach. Quoting the 3e35142ef99fe ("kexec_file: drop weak attribute from arch_kexec_apply_relocations[_add]") changelog: : Since commit d1bcae833b32f1 ("ELF: Don't generate unused section symbols") : [1], binutils (v2.36+) started dropping section symbols that it thought : were unused. This isn't an issue in general, but with kexec_file.c, gcc : is placing kexec_arch_apply_relocations[_add] into a separate : .text.unlikely section and the section symbol ".text.unlikely" is being : dropped. Due to this, recordmcount is unable to find a non-weak symbol in : .text.unlikely to generate a relocation record against. This patch (of 2); Drop __weak attribute from functions in kexec_file.c: - arch_kexec_kernel_image_probe() - arch_kimage_file_post_load_cleanup() - arch_kexec_kernel_image_load() - arch_kexec_locate_mem_hole() - arch_kexec_kernel_verify_sig() arch_kexec_kernel_image_load() calls into kexec_image_load_default(), so drop the static attribute for the latter. arch_kexec_kernel_verify_sig() is not overridden by any architecture, so drop the __weak attribute. Link: https://lkml.kernel.org/r/cover.1656659357.git.naveen.n.rao@linux.vnet.ibm.com Link: https://lkml.kernel.org/r/2cd7ca1fe4d6bb6ca38e3283c717878388ed6788.1656659357.git.naveen.n.rao@linux.vnet.ibm.com Signed-off-by: Naveen N. Rao Suggested-by: Eric Biederman Signed-off-by: Andrew Morton Signed-off-by: Mimi Zohar --- arch/arm64/include/asm/kexec.h | 4 ++- arch/powerpc/include/asm/kexec.h | 9 +++++++ arch/s390/include/asm/kexec.h | 3 +++ arch/x86/include/asm/kexec.h | 6 +++++ include/linux/kexec.h | 44 +++++++++++++++++++++++++++----- kernel/kexec_file.c | 35 ++----------------------- 6 files changed, 61 insertions(+), 40 deletions(-) diff --git a/arch/arm64/include/asm/kexec.h b/arch/arm64/include/asm/kexec.h index 9839bfc163d7..78d272b26ebd 100644 --- a/arch/arm64/include/asm/kexec.h +++ b/arch/arm64/include/asm/kexec.h @@ -115,7 +115,9 @@ extern const struct kexec_file_ops kexec_image_ops; struct kimage; -extern int arch_kimage_file_post_load_cleanup(struct kimage *image); +int arch_kimage_file_post_load_cleanup(struct kimage *image); +#define arch_kimage_file_post_load_cleanup arch_kimage_file_post_load_cleanup + extern int load_other_segments(struct kimage *image, unsigned long kernel_load_addr, unsigned long kernel_size, char *initrd, unsigned long initrd_len, diff --git a/arch/powerpc/include/asm/kexec.h b/arch/powerpc/include/asm/kexec.h index 2aefe14e1442..1e5e9b6ec78d 100644 --- a/arch/powerpc/include/asm/kexec.h +++ b/arch/powerpc/include/asm/kexec.h @@ -120,6 +120,15 @@ int setup_purgatory(struct kimage *image, const void *slave_code, #ifdef CONFIG_PPC64 struct kexec_buf; +int arch_kexec_kernel_image_probe(struct kimage *image, void *buf, unsigned long buf_len); +#define arch_kexec_kernel_image_probe arch_kexec_kernel_image_probe + +int arch_kimage_file_post_load_cleanup(struct kimage *image); +#define arch_kimage_file_post_load_cleanup arch_kimage_file_post_load_cleanup + +int arch_kexec_locate_mem_hole(struct kexec_buf *kbuf); +#define arch_kexec_locate_mem_hole arch_kexec_locate_mem_hole + int load_crashdump_segments_ppc64(struct kimage *image, struct kexec_buf *kbuf); int setup_purgatory_ppc64(struct kimage *image, const void *slave_code, diff --git a/arch/s390/include/asm/kexec.h b/arch/s390/include/asm/kexec.h index 649ecdcc8734..8886aadc11a3 100644 --- a/arch/s390/include/asm/kexec.h +++ b/arch/s390/include/asm/kexec.h @@ -92,5 +92,8 @@ int arch_kexec_apply_relocations_add(struct purgatory_info *pi, const Elf_Shdr *relsec, const Elf_Shdr *symtab); #define arch_kexec_apply_relocations_add arch_kexec_apply_relocations_add + +int arch_kimage_file_post_load_cleanup(struct kimage *image); +#define arch_kimage_file_post_load_cleanup arch_kimage_file_post_load_cleanup #endif #endif /*_S390_KEXEC_H */ diff --git a/arch/x86/include/asm/kexec.h b/arch/x86/include/asm/kexec.h index 6ad8d946cd3e..5ec359c1b50c 100644 --- a/arch/x86/include/asm/kexec.h +++ b/arch/x86/include/asm/kexec.h @@ -193,6 +193,12 @@ int arch_kexec_apply_relocations_add(struct purgatory_info *pi, const Elf_Shdr *relsec, const Elf_Shdr *symtab); #define arch_kexec_apply_relocations_add arch_kexec_apply_relocations_add + +void *arch_kexec_kernel_image_load(struct kimage *image); +#define arch_kexec_kernel_image_load arch_kexec_kernel_image_load + +int arch_kimage_file_post_load_cleanup(struct kimage *image); +#define arch_kimage_file_post_load_cleanup arch_kimage_file_post_load_cleanup #endif #endif diff --git a/include/linux/kexec.h b/include/linux/kexec.h index 475683cd67f1..6958c6b471f4 100644 --- a/include/linux/kexec.h +++ b/include/linux/kexec.h @@ -188,21 +188,53 @@ int kexec_purgatory_get_set_symbol(struct kimage *image, const char *name, void *buf, unsigned int size, bool get_value); void *kexec_purgatory_get_symbol_addr(struct kimage *image, const char *name); +void *kexec_image_load_default(struct kimage *image); + +#ifndef arch_kexec_kernel_image_probe +static inline int +arch_kexec_kernel_image_probe(struct kimage *image, void *buf, unsigned long buf_len) +{ + return kexec_image_probe_default(image, buf, buf_len); +} +#endif + +#ifndef arch_kimage_file_post_load_cleanup +static inline int arch_kimage_file_post_load_cleanup(struct kimage *image) +{ + return kexec_image_post_load_cleanup_default(image); +} +#endif + +#ifndef arch_kexec_kernel_image_load +static inline void *arch_kexec_kernel_image_load(struct kimage *image) +{ + return kexec_image_load_default(image); +} +#endif -/* Architectures may override the below functions */ -int arch_kexec_kernel_image_probe(struct kimage *image, void *buf, - unsigned long buf_len); -void *arch_kexec_kernel_image_load(struct kimage *image); -int arch_kimage_file_post_load_cleanup(struct kimage *image); #ifdef CONFIG_KEXEC_SIG int arch_kexec_kernel_verify_sig(struct kimage *image, void *buf, unsigned long buf_len); #endif -int arch_kexec_locate_mem_hole(struct kexec_buf *kbuf); extern int kexec_add_buffer(struct kexec_buf *kbuf); int kexec_locate_mem_hole(struct kexec_buf *kbuf); +#ifndef arch_kexec_locate_mem_hole +/** + * arch_kexec_locate_mem_hole - Find free memory to place the segments. + * @kbuf: Parameters for the memory search. + * + * On success, kbuf->mem will have the start address of the memory region found. + * + * Return: 0 on success, negative errno on error. + */ +static inline int arch_kexec_locate_mem_hole(struct kexec_buf *kbuf) +{ + return kexec_locate_mem_hole(kbuf); +} +#endif + /* Alignment required for elf header segment */ #define ELF_CORE_HEADER_ALIGN 4096 diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index f9261c07b048..0c27c81351ee 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -62,14 +62,7 @@ int kexec_image_probe_default(struct kimage *image, void *buf, return ret; } -/* Architectures can provide this probe function */ -int __weak arch_kexec_kernel_image_probe(struct kimage *image, void *buf, - unsigned long buf_len) -{ - return kexec_image_probe_default(image, buf, buf_len); -} - -static void *kexec_image_load_default(struct kimage *image) +void *kexec_image_load_default(struct kimage *image) { if (!image->fops || !image->fops->load) return ERR_PTR(-ENOEXEC); @@ -80,11 +73,6 @@ static void *kexec_image_load_default(struct kimage *image) image->cmdline_buf_len); } -void * __weak arch_kexec_kernel_image_load(struct kimage *image) -{ - return kexec_image_load_default(image); -} - int kexec_image_post_load_cleanup_default(struct kimage *image) { if (!image->fops || !image->fops->cleanup) @@ -93,11 +81,6 @@ int kexec_image_post_load_cleanup_default(struct kimage *image) return image->fops->cleanup(image->image_loader_data); } -int __weak arch_kimage_file_post_load_cleanup(struct kimage *image) -{ - return kexec_image_post_load_cleanup_default(image); -} - #ifdef CONFIG_KEXEC_SIG static int kexec_image_verify_sig_default(struct kimage *image, void *buf, unsigned long buf_len) @@ -110,8 +93,7 @@ static int kexec_image_verify_sig_default(struct kimage *image, void *buf, return image->fops->verify_sig(buf, buf_len); } -int __weak arch_kexec_kernel_verify_sig(struct kimage *image, void *buf, - unsigned long buf_len) +int arch_kexec_kernel_verify_sig(struct kimage *image, void *buf, unsigned long buf_len) { return kexec_image_verify_sig_default(image, buf, buf_len); } @@ -621,19 +603,6 @@ int kexec_locate_mem_hole(struct kexec_buf *kbuf) return ret == 1 ? 0 : -EADDRNOTAVAIL; } -/** - * arch_kexec_locate_mem_hole - Find free memory to place the segments. - * @kbuf: Parameters for the memory search. - * - * On success, kbuf->mem will have the start address of the memory region found. - * - * Return: 0 on success, negative errno on error. - */ -int __weak arch_kexec_locate_mem_hole(struct kexec_buf *kbuf) -{ - return kexec_locate_mem_hole(kbuf); -} - /** * kexec_add_buffer - place a buffer in a kexec segment * @kbuf: Buffer contents and memory parameters. From 0738eceb6201691534df07e0928d0a6168a35787 Mon Sep 17 00:00:00 2001 From: "Naveen N. Rao" Date: Fri, 1 Jul 2022 13:04:05 +0530 Subject: [PATCH 241/318] kexec: drop weak attribute from functions Drop __weak attribute from functions in kexec_core.c: - machine_kexec_post_load() - arch_kexec_protect_crashkres() - arch_kexec_unprotect_crashkres() - crash_free_reserved_phys_range() Link: https://lkml.kernel.org/r/c0f6219e03cb399d166d518ab505095218a902dd.1656659357.git.naveen.n.rao@linux.vnet.ibm.com Signed-off-by: Naveen N. Rao Suggested-by: Eric Biederman Signed-off-by: Andrew Morton Signed-off-by: Mimi Zohar --- arch/arm64/include/asm/kexec.h | 16 ++++++++++++++-- arch/powerpc/include/asm/kexec.h | 5 +++++ arch/s390/include/asm/kexec.h | 11 +++++++++++ arch/x86/include/asm/kexec.h | 6 ++++++ include/linux/kexec.h | 32 ++++++++++++++++++++++++++++---- kernel/kexec_core.c | 27 --------------------------- 6 files changed, 64 insertions(+), 33 deletions(-) diff --git a/arch/arm64/include/asm/kexec.h b/arch/arm64/include/asm/kexec.h index 78d272b26ebd..559bfae26715 100644 --- a/arch/arm64/include/asm/kexec.h +++ b/arch/arm64/include/asm/kexec.h @@ -84,16 +84,30 @@ static inline void crash_setup_regs(struct pt_regs *newregs, extern bool crash_is_nosave(unsigned long pfn); extern void crash_prepare_suspend(void); extern void crash_post_resume(void); + +void crash_free_reserved_phys_range(unsigned long begin, unsigned long end); +#define crash_free_reserved_phys_range crash_free_reserved_phys_range #else static inline bool crash_is_nosave(unsigned long pfn) {return false; } static inline void crash_prepare_suspend(void) {} static inline void crash_post_resume(void) {} #endif +struct kimage; + #if defined(CONFIG_KEXEC_CORE) void cpu_soft_restart(unsigned long el2_switch, unsigned long entry, unsigned long arg0, unsigned long arg1, unsigned long arg2); + +int machine_kexec_post_load(struct kimage *image); +#define machine_kexec_post_load machine_kexec_post_load + +void arch_kexec_protect_crashkres(void); +#define arch_kexec_protect_crashkres arch_kexec_protect_crashkres + +void arch_kexec_unprotect_crashkres(void); +#define arch_kexec_unprotect_crashkres arch_kexec_unprotect_crashkres #endif #define ARCH_HAS_KIMAGE_ARCH @@ -113,8 +127,6 @@ struct kimage_arch { #ifdef CONFIG_KEXEC_FILE extern const struct kexec_file_ops kexec_image_ops; -struct kimage; - int arch_kimage_file_post_load_cleanup(struct kimage *image); #define arch_kimage_file_post_load_cleanup arch_kimage_file_post_load_cleanup diff --git a/arch/powerpc/include/asm/kexec.h b/arch/powerpc/include/asm/kexec.h index 1e5e9b6ec78d..d6f4edfe4737 100644 --- a/arch/powerpc/include/asm/kexec.h +++ b/arch/powerpc/include/asm/kexec.h @@ -98,6 +98,11 @@ void relocate_new_kernel(unsigned long indirection_page, unsigned long reboot_co void kexec_copy_flush(struct kimage *image); +#if defined(CONFIG_CRASH_DUMP) && defined(CONFIG_PPC_RTAS) +void crash_free_reserved_phys_range(unsigned long begin, unsigned long end); +#define crash_free_reserved_phys_range crash_free_reserved_phys_range +#endif + #ifdef CONFIG_KEXEC_FILE extern const struct kexec_file_ops kexec_elf64_ops; diff --git a/arch/s390/include/asm/kexec.h b/arch/s390/include/asm/kexec.h index 8886aadc11a3..1bd08eb56d5f 100644 --- a/arch/s390/include/asm/kexec.h +++ b/arch/s390/include/asm/kexec.h @@ -85,6 +85,17 @@ struct kimage_arch { extern const struct kexec_file_ops s390_kexec_image_ops; extern const struct kexec_file_ops s390_kexec_elf_ops; +#ifdef CONFIG_CRASH_DUMP +void crash_free_reserved_phys_range(unsigned long begin, unsigned long end); +#define crash_free_reserved_phys_range crash_free_reserved_phys_range + +void arch_kexec_protect_crashkres(void); +#define arch_kexec_protect_crashkres arch_kexec_protect_crashkres + +void arch_kexec_unprotect_crashkres(void); +#define arch_kexec_unprotect_crashkres arch_kexec_unprotect_crashkres +#endif + #ifdef CONFIG_KEXEC_FILE struct purgatory_info; int arch_kexec_apply_relocations_add(struct purgatory_info *pi, diff --git a/arch/x86/include/asm/kexec.h b/arch/x86/include/asm/kexec.h index 5ec359c1b50c..a3760ca796aa 100644 --- a/arch/x86/include/asm/kexec.h +++ b/arch/x86/include/asm/kexec.h @@ -186,6 +186,12 @@ extern int arch_kexec_post_alloc_pages(void *vaddr, unsigned int pages, extern void arch_kexec_pre_free_pages(void *vaddr, unsigned int pages); #define arch_kexec_pre_free_pages arch_kexec_pre_free_pages +void arch_kexec_protect_crashkres(void); +#define arch_kexec_protect_crashkres arch_kexec_protect_crashkres + +void arch_kexec_unprotect_crashkres(void); +#define arch_kexec_unprotect_crashkres arch_kexec_unprotect_crashkres + #ifdef CONFIG_KEXEC_FILE struct purgatory_info; int arch_kexec_apply_relocations_add(struct purgatory_info *pi, diff --git a/include/linux/kexec.h b/include/linux/kexec.h index 6958c6b471f4..8107606ad1e8 100644 --- a/include/linux/kexec.h +++ b/include/linux/kexec.h @@ -390,7 +390,10 @@ extern void machine_kexec_cleanup(struct kimage *image); extern int kernel_kexec(void); extern struct page *kimage_alloc_control_pages(struct kimage *image, unsigned int order); -int machine_kexec_post_load(struct kimage *image); + +#ifndef machine_kexec_post_load +static inline int machine_kexec_post_load(struct kimage *image) { return 0; } +#endif extern void __crash_kexec(struct pt_regs *); extern void crash_kexec(struct pt_regs *); @@ -423,10 +426,21 @@ extern bool kexec_in_progress; int crash_shrink_memory(unsigned long new_size); size_t crash_get_memory_size(void); -void crash_free_reserved_phys_range(unsigned long begin, unsigned long end); -void arch_kexec_protect_crashkres(void); -void arch_kexec_unprotect_crashkres(void); +#ifndef arch_kexec_protect_crashkres +/* + * Protection mechanism for crashkernel reserved memory after + * the kdump kernel is loaded. + * + * Provide an empty default implementation here -- architecture + * code may override this + */ +static inline void arch_kexec_protect_crashkres(void) { } +#endif + +#ifndef arch_kexec_unprotect_crashkres +static inline void arch_kexec_unprotect_crashkres(void) { } +#endif #ifndef page_to_boot_pfn static inline unsigned long page_to_boot_pfn(struct page *page) @@ -456,6 +470,16 @@ static inline phys_addr_t boot_phys_to_phys(unsigned long boot_phys) } #endif +#ifndef crash_free_reserved_phys_range +static inline void crash_free_reserved_phys_range(unsigned long begin, unsigned long end) +{ + unsigned long addr; + + for (addr = begin; addr < end; addr += PAGE_SIZE) + free_reserved_page(boot_pfn_to_page(addr >> PAGE_SHIFT)); +} +#endif + static inline unsigned long virt_to_boot_phys(void *addr) { return phys_to_boot_phys(__pa((unsigned long)addr)); diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 4d34c78334ce..acd029b307e4 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -591,11 +591,6 @@ static void kimage_free_extra_pages(struct kimage *image) } -int __weak machine_kexec_post_load(struct kimage *image) -{ - return 0; -} - void kimage_terminate(struct kimage *image) { if (*image->entry != 0) @@ -1020,15 +1015,6 @@ size_t crash_get_memory_size(void) return size; } -void __weak crash_free_reserved_phys_range(unsigned long begin, - unsigned long end) -{ - unsigned long addr; - - for (addr = begin; addr < end; addr += PAGE_SIZE) - free_reserved_page(boot_pfn_to_page(addr >> PAGE_SHIFT)); -} - int crash_shrink_memory(unsigned long new_size) { int ret = 0; @@ -1225,16 +1211,3 @@ int kernel_kexec(void) mutex_unlock(&kexec_mutex); return error; } - -/* - * Protection mechanism for crashkernel reserved memory after - * the kdump kernel is loaded. - * - * Provide an empty default implementation here -- architecture - * code may override this - */ -void __weak arch_kexec_protect_crashkres(void) -{} - -void __weak arch_kexec_unprotect_crashkres(void) -{} From 689a71493bd2f31c024f8c0395f85a1fd4b2138e Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Thu, 14 Jul 2022 21:40:24 +0800 Subject: [PATCH 242/318] kexec: clean up arch_kexec_kernel_verify_sig Before commit 105e10e2cf1c ("kexec_file: drop weak attribute from functions"), there was already no arch-specific implementation of arch_kexec_kernel_verify_sig. With weak attribute dropped by that commit, arch_kexec_kernel_verify_sig is completely useless. So clean it up. Note later patches are dependent on this patch so it should be backported to the stable tree as well. Cc: stable@vger.kernel.org Suggested-by: Eric W. Biederman Reviewed-by: Michal Suchanek Acked-by: Baoquan He Signed-off-by: Coiby Xu [zohar@linux.ibm.com: reworded patch description "Note"] Link: https://lore.kernel.org/linux-integrity/20220714134027.394370-1-coxu@redhat.com/ Signed-off-by: Mimi Zohar --- include/linux/kexec.h | 5 ----- kernel/kexec_file.c | 33 +++++++++++++-------------------- 2 files changed, 13 insertions(+), 25 deletions(-) diff --git a/include/linux/kexec.h b/include/linux/kexec.h index 8107606ad1e8..7f710fb3712b 100644 --- a/include/linux/kexec.h +++ b/include/linux/kexec.h @@ -212,11 +212,6 @@ static inline void *arch_kexec_kernel_image_load(struct kimage *image) } #endif -#ifdef CONFIG_KEXEC_SIG -int arch_kexec_kernel_verify_sig(struct kimage *image, void *buf, - unsigned long buf_len); -#endif - extern int kexec_add_buffer(struct kexec_buf *kbuf); int kexec_locate_mem_hole(struct kexec_buf *kbuf); diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index 0c27c81351ee..6dc1294c90fc 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -81,24 +81,6 @@ int kexec_image_post_load_cleanup_default(struct kimage *image) return image->fops->cleanup(image->image_loader_data); } -#ifdef CONFIG_KEXEC_SIG -static int kexec_image_verify_sig_default(struct kimage *image, void *buf, - unsigned long buf_len) -{ - if (!image->fops || !image->fops->verify_sig) { - pr_debug("kernel loader does not support signature verification.\n"); - return -EKEYREJECTED; - } - - return image->fops->verify_sig(buf, buf_len); -} - -int arch_kexec_kernel_verify_sig(struct kimage *image, void *buf, unsigned long buf_len) -{ - return kexec_image_verify_sig_default(image, buf, buf_len); -} -#endif - /* * Free up memory used by kernel, initrd, and command line. This is temporary * memory allocation which is not needed any more after these buffers have @@ -141,13 +123,24 @@ void kimage_file_post_load_cleanup(struct kimage *image) } #ifdef CONFIG_KEXEC_SIG +static int kexec_image_verify_sig(struct kimage *image, void *buf, + unsigned long buf_len) +{ + if (!image->fops || !image->fops->verify_sig) { + pr_debug("kernel loader does not support signature verification.\n"); + return -EKEYREJECTED; + } + + return image->fops->verify_sig(buf, buf_len); +} + static int kimage_validate_signature(struct kimage *image) { int ret; - ret = arch_kexec_kernel_verify_sig(image, image->kernel_buf, - image->kernel_buf_len); + ret = kexec_image_verify_sig(image, image->kernel_buf, + image->kernel_buf_len); if (ret) { if (sig_enforce) { From c903dae8941deb55043ee46ded29e84e97cd84bb Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Thu, 14 Jul 2022 21:40:25 +0800 Subject: [PATCH 243/318] kexec, KEYS: make the code in bzImage64_verify_sig generic commit 278311e417be ("kexec, KEYS: Make use of platform keyring for signature verify") adds platform keyring support on x86 kexec but not arm64. The code in bzImage64_verify_sig uses the keys on the .builtin_trusted_keys, .machine, if configured and enabled, .secondary_trusted_keys, also if configured, and .platform keyrings to verify the signed kernel image as PE file. Cc: kexec@lists.infradead.org Cc: keyrings@vger.kernel.org Cc: linux-security-module@vger.kernel.org Reviewed-by: Michal Suchanek Signed-off-by: Coiby Xu Signed-off-by: Mimi Zohar --- arch/x86/kernel/kexec-bzimage64.c | 20 +------------------- include/linux/kexec.h | 7 +++++++ kernel/kexec_file.c | 17 +++++++++++++++++ 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c index 170d0fd68b1f..f299b48f9c9f 100644 --- a/arch/x86/kernel/kexec-bzimage64.c +++ b/arch/x86/kernel/kexec-bzimage64.c @@ -17,7 +17,6 @@ #include #include #include -#include #include #include @@ -528,28 +527,11 @@ static int bzImage64_cleanup(void *loader_data) return 0; } -#ifdef CONFIG_KEXEC_BZIMAGE_VERIFY_SIG -static int bzImage64_verify_sig(const char *kernel, unsigned long kernel_len) -{ - int ret; - - ret = verify_pefile_signature(kernel, kernel_len, - VERIFY_USE_SECONDARY_KEYRING, - VERIFYING_KEXEC_PE_SIGNATURE); - if (ret == -ENOKEY && IS_ENABLED(CONFIG_INTEGRITY_PLATFORM_KEYRING)) { - ret = verify_pefile_signature(kernel, kernel_len, - VERIFY_USE_PLATFORM_KEYRING, - VERIFYING_KEXEC_PE_SIGNATURE); - } - return ret; -} -#endif - const struct kexec_file_ops kexec_bzImage64_ops = { .probe = bzImage64_probe, .load = bzImage64_load, .cleanup = bzImage64_cleanup, #ifdef CONFIG_KEXEC_BZIMAGE_VERIFY_SIG - .verify_sig = bzImage64_verify_sig, + .verify_sig = kexec_kernel_verify_pe_sig, #endif }; diff --git a/include/linux/kexec.h b/include/linux/kexec.h index 7f710fb3712b..13e6c4b58f07 100644 --- a/include/linux/kexec.h +++ b/include/linux/kexec.h @@ -19,6 +19,7 @@ #include #include +#include /* Location of a reserved region to hold the crash kernel. */ @@ -212,6 +213,12 @@ static inline void *arch_kexec_kernel_image_load(struct kimage *image) } #endif +#ifdef CONFIG_KEXEC_SIG +#ifdef CONFIG_SIGNED_PE_FILE_VERIFICATION +int kexec_kernel_verify_pe_sig(const char *kernel, unsigned long kernel_len); +#endif +#endif + extern int kexec_add_buffer(struct kexec_buf *kbuf); int kexec_locate_mem_hole(struct kexec_buf *kbuf); diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index 6dc1294c90fc..a7b411c22f19 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -123,6 +123,23 @@ void kimage_file_post_load_cleanup(struct kimage *image) } #ifdef CONFIG_KEXEC_SIG +#ifdef CONFIG_SIGNED_PE_FILE_VERIFICATION +int kexec_kernel_verify_pe_sig(const char *kernel, unsigned long kernel_len) +{ + int ret; + + ret = verify_pefile_signature(kernel, kernel_len, + VERIFY_USE_SECONDARY_KEYRING, + VERIFYING_KEXEC_PE_SIGNATURE); + if (ret == -ENOKEY && IS_ENABLED(CONFIG_INTEGRITY_PLATFORM_KEYRING)) { + ret = verify_pefile_signature(kernel, kernel_len, + VERIFY_USE_PLATFORM_KEYRING, + VERIFYING_KEXEC_PE_SIGNATURE); + } + return ret; +} +#endif + static int kexec_image_verify_sig(struct kimage *image, void *buf, unsigned long buf_len) { From 0d519cadf75184a24313568e7f489a7fc9b1be3b Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Thu, 14 Jul 2022 21:40:26 +0800 Subject: [PATCH 244/318] arm64: kexec_file: use more system keyrings to verify kernel image signature Currently, when loading a kernel image via the kexec_file_load() system call, arm64 can only use the .builtin_trusted_keys keyring to verify a signature whereas x86 can use three more keyrings i.e. .secondary_trusted_keys, .machine and .platform keyrings. For example, one resulting problem is kexec'ing a kernel image would be rejected with the error "Lockdown: kexec: kexec of unsigned images is restricted; see man kernel_lockdown.7". This patch set enables arm64 to make use of the same keyrings as x86 to verify the signature kexec'ed kernel image. Fixes: 732b7b93d849 ("arm64: kexec_file: add kernel signature verification support") Cc: stable@vger.kernel.org # 105e10e2cf1c: kexec_file: drop weak attribute from functions Cc: stable@vger.kernel.org # 34d5960af253: kexec: clean up arch_kexec_kernel_verify_sig Cc: stable@vger.kernel.org # 83b7bb2d49ae: kexec, KEYS: make the code in bzImage64_verify_sig generic Acked-by: Baoquan He Cc: kexec@lists.infradead.org Cc: keyrings@vger.kernel.org Cc: linux-security-module@vger.kernel.org Co-developed-by: Michal Suchanek Signed-off-by: Michal Suchanek Acked-by: Will Deacon Signed-off-by: Coiby Xu Signed-off-by: Mimi Zohar --- arch/arm64/kernel/kexec_image.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/arch/arm64/kernel/kexec_image.c b/arch/arm64/kernel/kexec_image.c index 9ec34690e255..5ed6a585f21f 100644 --- a/arch/arm64/kernel/kexec_image.c +++ b/arch/arm64/kernel/kexec_image.c @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -130,18 +129,10 @@ static void *image_load(struct kimage *image, return NULL; } -#ifdef CONFIG_KEXEC_IMAGE_VERIFY_SIG -static int image_verify_sig(const char *kernel, unsigned long kernel_len) -{ - return verify_pefile_signature(kernel, kernel_len, NULL, - VERIFYING_KEXEC_PE_SIGNATURE); -} -#endif - const struct kexec_file_ops kexec_image_ops = { .probe = image_probe, .load = image_load, #ifdef CONFIG_KEXEC_IMAGE_VERIFY_SIG - .verify_sig = image_verify_sig, + .verify_sig = kexec_kernel_verify_pe_sig, #endif }; From 0828c4a39be57768b8788e8cbd0d84683ea757e5 Mon Sep 17 00:00:00 2001 From: Michal Suchanek Date: Thu, 14 Jul 2022 21:40:27 +0800 Subject: [PATCH 245/318] kexec, KEYS, s390: Make use of built-in and secondary keyring for signature verification commit e23a8020ce4e ("s390/kexec_file: Signature verification prototype") adds support for KEXEC_SIG verification with keys from platform keyring but the built-in keys and secondary keyring are not used. Add support for the built-in keys and secondary keyring as x86 does. Fixes: e23a8020ce4e ("s390/kexec_file: Signature verification prototype") Cc: stable@vger.kernel.org Cc: Philipp Rudo Cc: kexec@lists.infradead.org Cc: keyrings@vger.kernel.org Cc: linux-security-module@vger.kernel.org Signed-off-by: Michal Suchanek Reviewed-by: "Lee, Chun-Yi" Acked-by: Baoquan He Signed-off-by: Coiby Xu Acked-by: Heiko Carstens Signed-off-by: Mimi Zohar --- arch/s390/kernel/machine_kexec_file.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/arch/s390/kernel/machine_kexec_file.c b/arch/s390/kernel/machine_kexec_file.c index 8f43575a4dd3..fc6d5f58debe 100644 --- a/arch/s390/kernel/machine_kexec_file.c +++ b/arch/s390/kernel/machine_kexec_file.c @@ -31,6 +31,7 @@ int s390_verify_sig(const char *kernel, unsigned long kernel_len) const unsigned long marker_len = sizeof(MODULE_SIG_STRING) - 1; struct module_signature *ms; unsigned long sig_len; + int ret; /* Skip signature verification when not secure IPLed. */ if (!ipl_secure_flag) @@ -65,11 +66,18 @@ int s390_verify_sig(const char *kernel, unsigned long kernel_len) return -EBADMSG; } - return verify_pkcs7_signature(kernel, kernel_len, - kernel + kernel_len, sig_len, - VERIFY_USE_PLATFORM_KEYRING, - VERIFYING_MODULE_SIGNATURE, - NULL, NULL); + ret = verify_pkcs7_signature(kernel, kernel_len, + kernel + kernel_len, sig_len, + VERIFY_USE_SECONDARY_KEYRING, + VERIFYING_MODULE_SIGNATURE, + NULL, NULL); + if (ret == -ENOKEY && IS_ENABLED(CONFIG_INTEGRITY_PLATFORM_KEYRING)) + ret = verify_pkcs7_signature(kernel, kernel_len, + kernel + kernel_len, sig_len, + VERIFY_USE_PLATFORM_KEYRING, + VERIFYING_MODULE_SIGNATURE, + NULL, NULL); + return ret; } #endif /* CONFIG_KEXEC_SIG */ From 92c005a1176288c98a0dc49f37376da35bbea071 Mon Sep 17 00:00:00 2001 From: Micah Morton Date: Wed, 15 Jun 2022 21:19:06 +0000 Subject: [PATCH 246/318] LSM: SafeSetID: fix userns bug in selftest Not sure how this bug got in here but its been there since the original merge. I think I tested the code on a system that wouldn't let me clone() with CLONE_NEWUSER flag set so had to comment out these test_userns invocations. Trying to map UID 0 inside the userns to UID 0 outside will never work, even with CAP_SETUID. The code is supposed to test whether we can map UID 0 in the userns to the UID of the parent process (the one with CAP_SETUID that is writing the /proc/[pid]/uid_map file). Signed-off-by: Micah Morton --- tools/testing/selftests/safesetid/safesetid-test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/safesetid/safesetid-test.c b/tools/testing/selftests/safesetid/safesetid-test.c index 4b809c93ba36..111dcbcc0491 100644 --- a/tools/testing/selftests/safesetid/safesetid-test.c +++ b/tools/testing/selftests/safesetid/safesetid-test.c @@ -194,7 +194,7 @@ static bool test_userns(bool expect_success) printf("preparing file name string failed"); return false; } - success = write_file(map_file_name, "0 0 1", uid); + success = write_file(map_file_name, "0 %d 1", uid); return success == expect_success; } From b2927170d4fbdf30545eb482133425477625a665 Mon Sep 17 00:00:00 2001 From: Micah Morton Date: Wed, 15 Jun 2022 22:14:14 +0000 Subject: [PATCH 247/318] LSM: SafeSetID: selftest cleanup and prepare for GIDs Add some notes on how to run the test, update the policy file paths to reflect recent upstream changes, prepare test for adding GID testing. Signed-off-by: Micah Morton --- tools/testing/selftests/safesetid/Makefile | 2 +- .../selftests/safesetid/safesetid-test.c | 93 ++++++++++--------- 2 files changed, 51 insertions(+), 44 deletions(-) diff --git a/tools/testing/selftests/safesetid/Makefile b/tools/testing/selftests/safesetid/Makefile index fa02c4d5ec13..e815bbf2d0f4 100644 --- a/tools/testing/selftests/safesetid/Makefile +++ b/tools/testing/selftests/safesetid/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -# Makefile for mount selftests. +# Makefile for SafeSetID selftest. CFLAGS = -Wall -O2 LDLIBS = -lcap diff --git a/tools/testing/selftests/safesetid/safesetid-test.c b/tools/testing/selftests/safesetid/safesetid-test.c index 111dcbcc0491..c16977e4b913 100644 --- a/tools/testing/selftests/safesetid/safesetid-test.c +++ b/tools/testing/selftests/safesetid/safesetid-test.c @@ -16,17 +16,28 @@ #include #include +/* + * NOTES about this test: + * - requries libcap-dev to be installed on test system + * - requires securityfs to me mounted at /sys/kernel/security, e.g.: + * mount -n -t securityfs -o nodev,noexec,nosuid securityfs /sys/kernel/security + * - needs CONFIG_SECURITYFS and CONFIG_SAFESETID to be enabled + */ + #ifndef CLONE_NEWUSER # define CLONE_NEWUSER 0x10000000 #endif -#define ROOT_USER 0 -#define RESTRICTED_PARENT 1 -#define ALLOWED_CHILD1 2 -#define ALLOWED_CHILD2 3 -#define NO_POLICY_USER 4 +#define ROOT_UGID 0 +#define RESTRICTED_PARENT_UGID 1 +#define ALLOWED_CHILD1_UGID 2 +#define ALLOWED_CHILD2_UGID 3 +#define NO_POLICY_UGID 4 -char* add_whitelist_policy_file = "/sys/kernel/security/safesetid/add_whitelist_policy"; +#define UGID_POLICY_STRING "1:2\n1:3\n2:2\n3:3\n" + +char* add_uid_whitelist_policy_file = "/sys/kernel/security/safesetid/uid_allowlist_policy"; +char* add_gid_whitelist_policy_file = "/sys/kernel/security/safesetid/gid_allowlist_policy"; static void die(char *fmt, ...) { @@ -106,7 +117,7 @@ static void ensure_user_exists(uid_t uid) die("couldn't open file\n"); if (fseek(fd, 0, SEEK_END)) die("couldn't fseek\n"); - snprintf(name_str, 10, "%d", uid); + snprintf(name_str, 10, "user %d", uid); p.pw_name=name_str; p.pw_uid=uid; p.pw_gecos="Test account"; @@ -122,7 +133,7 @@ static void ensure_user_exists(uid_t uid) static void ensure_securityfs_mounted(void) { - int fd = open(add_whitelist_policy_file, O_WRONLY); + int fd = open(add_uid_whitelist_policy_file, O_WRONLY); if (fd < 0) { if (errno == ENOENT) { // Need to mount securityfs @@ -135,36 +146,32 @@ static void ensure_securityfs_mounted(void) } else { if (close(fd) != 0) { die("close of %s failed: %s\n", - add_whitelist_policy_file, strerror(errno)); + add_uid_whitelist_policy_file, strerror(errno)); } } } -static void write_policies(void) +static void write_uid_policies() { - static char *policy_str = - "1:2\n" - "1:3\n" - "2:2\n" - "3:3\n"; + static char *policy_str = UGID_POLICY_STRING; ssize_t written; int fd; - fd = open(add_whitelist_policy_file, O_WRONLY); + fd = open(add_uid_whitelist_policy_file, O_WRONLY); if (fd < 0) - die("can't open add_whitelist_policy file\n"); + die("can't open add_uid_whitelist_policy file\n"); written = write(fd, policy_str, strlen(policy_str)); if (written != strlen(policy_str)) { if (written >= 0) { - die("short write to %s\n", add_whitelist_policy_file); + die("short write to %s\n", add_uid_whitelist_policy_file); } else { die("write to %s failed: %s\n", - add_whitelist_policy_file, strerror(errno)); + add_uid_whitelist_policy_file, strerror(errno)); } } if (close(fd) != 0) { die("close of %s failed: %s\n", - add_whitelist_policy_file, strerror(errno)); + add_uid_whitelist_policy_file, strerror(errno)); } } @@ -260,11 +267,11 @@ static void test_setuid(uid_t child_uid, bool expect_success) static void ensure_users_exist(void) { - ensure_user_exists(ROOT_USER); - ensure_user_exists(RESTRICTED_PARENT); - ensure_user_exists(ALLOWED_CHILD1); - ensure_user_exists(ALLOWED_CHILD2); - ensure_user_exists(NO_POLICY_USER); + ensure_user_exists(ROOT_UGID); + ensure_user_exists(RESTRICTED_PARENT_UGID); + ensure_user_exists(ALLOWED_CHILD1_UGID); + ensure_user_exists(ALLOWED_CHILD2_UGID); + ensure_user_exists(NO_POLICY_UGID); } static void drop_caps(bool setid_retained) @@ -285,39 +292,38 @@ int main(int argc, char **argv) { ensure_users_exist(); ensure_securityfs_mounted(); - write_policies(); + write_uid_policies(); if (prctl(PR_SET_KEEPCAPS, 1L)) die("Error with set keepcaps\n"); - // First test to make sure we can write userns mappings from a user - // that doesn't have any restrictions (as long as it has CAP_SETUID); - if (setuid(NO_POLICY_USER) < 0) - die("Error with set uid(%d)\n", NO_POLICY_USER); - if (setgid(NO_POLICY_USER) < 0) - die("Error with set gid(%d)\n", NO_POLICY_USER); - + // First test to make sure we can write userns mappings from a non-root + // user that doesn't have any restrictions (as long as it has + // CAP_SETUID); + if (setgid(NO_POLICY_UGID) < 0) + die("Error with set gid(%d)\n", NO_POLICY_UGID); + if (setuid(NO_POLICY_UGID) < 0) + die("Error with set uid(%d)\n", NO_POLICY_UGID); // Take away all but setid caps drop_caps(true); - // Need PR_SET_DUMPABLE flag set so we can write /proc/[pid]/uid_map // from non-root parent process. if (prctl(PR_SET_DUMPABLE, 1, 0, 0, 0)) die("Error with set dumpable\n"); - if (!test_userns(true)) { die("test_userns failed when it should work\n"); } - if (setuid(RESTRICTED_PARENT) < 0) - die("Error with set uid(%d)\n", RESTRICTED_PARENT); - if (setgid(RESTRICTED_PARENT) < 0) - die("Error with set gid(%d)\n", RESTRICTED_PARENT); + // Now switch to a user/group with restrictions + if (setgid(RESTRICTED_PARENT_UGID) < 0) + die("Error with set gid(%d)\n", RESTRICTED_PARENT_UGID); + if (setuid(RESTRICTED_PARENT_UGID) < 0) + die("Error with set uid(%d)\n", RESTRICTED_PARENT_UGID); - test_setuid(ROOT_USER, false); - test_setuid(ALLOWED_CHILD1, true); - test_setuid(ALLOWED_CHILD2, true); - test_setuid(NO_POLICY_USER, false); + test_setuid(ROOT_UGID, false); + test_setuid(ALLOWED_CHILD1_UGID, true); + test_setuid(ALLOWED_CHILD2_UGID, true); + test_setuid(NO_POLICY_UGID, false); if (!test_userns(false)) { die("test_userns worked when it should fail\n"); @@ -331,5 +337,6 @@ int main(int argc, char **argv) // NOTE: this test doesn't clean up users that were created in // /etc/passwd or flush policies that were added to the LSM. + printf("test successful!\n"); return EXIT_SUCCESS; } From a1732d6898ced0523cb4073c7f02d236edf312b1 Mon Sep 17 00:00:00 2001 From: Micah Morton Date: Wed, 15 Jun 2022 22:17:40 +0000 Subject: [PATCH 248/318] LSM: SafeSetID: add GID testing to selftest GID security policies were added back in v5.10, update the selftest to reflect this. Signed-off-by: Micah Morton --- .../selftests/safesetid/safesetid-test.c | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) diff --git a/tools/testing/selftests/safesetid/safesetid-test.c b/tools/testing/selftests/safesetid/safesetid-test.c index c16977e4b913..a653c47a4ab5 100644 --- a/tools/testing/selftests/safesetid/safesetid-test.c +++ b/tools/testing/selftests/safesetid/safesetid-test.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -120,6 +121,7 @@ static void ensure_user_exists(uid_t uid) snprintf(name_str, 10, "user %d", uid); p.pw_name=name_str; p.pw_uid=uid; + p.pw_gid=uid; p.pw_gecos="Test account"; p.pw_dir="/dev/null"; p.pw_shell="/bin/false"; @@ -131,6 +133,33 @@ static void ensure_user_exists(uid_t uid) } } +static void ensure_group_exists(gid_t gid) +{ + struct group g; + + FILE *fd; + char name_str[10]; + + if (getgrgid(gid) == NULL) { + memset(&g,0x00,sizeof(g)); + fd=fopen("/etc/group","a"); + if (fd == NULL) + die("couldn't open group file\n"); + if (fseek(fd, 0, SEEK_END)) + die("couldn't fseek group file\n"); + snprintf(name_str, 10, "group %d", gid); + g.gr_name=name_str; + g.gr_gid=gid; + g.gr_passwd=NULL; + g.gr_mem=NULL; + int value = putgrent(&g,fd); + if (value != 0) + die("putgrent failed\n"); + if (fclose(fd)) + die("fclose failed\n"); + } +} + static void ensure_securityfs_mounted(void) { int fd = open(add_uid_whitelist_policy_file, O_WRONLY); @@ -175,6 +204,31 @@ static void write_uid_policies() } } +static void write_gid_policies() +{ + static char *policy_str = UGID_POLICY_STRING; + ssize_t written; + int fd; + + fd = open(add_gid_whitelist_policy_file, O_WRONLY); + if (fd < 0) + die("can't open add_gid_whitelist_policy file\n"); + written = write(fd, policy_str, strlen(policy_str)); + if (written != strlen(policy_str)) { + if (written >= 0) { + die("short write to %s\n", add_gid_whitelist_policy_file); + } else { + die("write to %s failed: %s\n", + add_gid_whitelist_policy_file, strerror(errno)); + } + } + if (close(fd) != 0) { + die("close of %s failed: %s\n", + add_gid_whitelist_policy_file, strerror(errno)); + } +} + + static bool test_userns(bool expect_success) { uid_t uid; @@ -265,6 +319,63 @@ static void test_setuid(uid_t child_uid, bool expect_success) die("should not reach here\n"); } +static void test_setgid(gid_t child_gid, bool expect_success) +{ + pid_t cpid, w; + int wstatus; + + cpid = fork(); + if (cpid == -1) { + die("fork\n"); + } + + if (cpid == 0) { /* Code executed by child */ + if (setgid(child_gid) < 0) + exit(EXIT_FAILURE); + if (getgid() == child_gid) + exit(EXIT_SUCCESS); + else + exit(EXIT_FAILURE); + } else { /* Code executed by parent */ + do { + w = waitpid(cpid, &wstatus, WUNTRACED | WCONTINUED); + if (w == -1) { + die("waitpid\n"); + } + + if (WIFEXITED(wstatus)) { + if (WEXITSTATUS(wstatus) == EXIT_SUCCESS) { + if (expect_success) { + return; + } else { + die("unexpected success\n"); + } + } else { + if (expect_success) { + die("unexpected failure\n"); + } else { + return; + } + } + } else if (WIFSIGNALED(wstatus)) { + if (WTERMSIG(wstatus) == 9) { + if (expect_success) + die("killed unexpectedly\n"); + else + return; + } else { + die("unexpected signal: %d\n", wstatus); + } + } else { + die("unexpected status: %d\n", wstatus); + } + } while (!WIFEXITED(wstatus) && !WIFSIGNALED(wstatus)); + } + + die("should not reach here\n"); +} + + static void ensure_users_exist(void) { ensure_user_exists(ROOT_UGID); @@ -274,6 +385,15 @@ static void ensure_users_exist(void) ensure_user_exists(NO_POLICY_UGID); } +static void ensure_groups_exist(void) +{ + ensure_group_exists(ROOT_UGID); + ensure_group_exists(RESTRICTED_PARENT_UGID); + ensure_group_exists(ALLOWED_CHILD1_UGID); + ensure_group_exists(ALLOWED_CHILD2_UGID); + ensure_group_exists(NO_POLICY_UGID); +} + static void drop_caps(bool setid_retained) { cap_value_t cap_values[] = {CAP_SETUID, CAP_SETGID}; @@ -290,9 +410,11 @@ static void drop_caps(bool setid_retained) int main(int argc, char **argv) { + ensure_groups_exist(); ensure_users_exist(); ensure_securityfs_mounted(); write_uid_policies(); + write_gid_policies(); if (prctl(PR_SET_KEEPCAPS, 1L)) die("Error with set keepcaps\n"); @@ -325,6 +447,12 @@ int main(int argc, char **argv) test_setuid(ALLOWED_CHILD2_UGID, true); test_setuid(NO_POLICY_UGID, false); + test_setgid(ROOT_UGID, false); + test_setgid(ALLOWED_CHILD1_UGID, true); + test_setgid(ALLOWED_CHILD2_UGID, true); + test_setgid(NO_POLICY_UGID, false); + + if (!test_userns(false)) { die("test_userns worked when it should fail\n"); } @@ -334,6 +462,9 @@ int main(int argc, char **argv) test_setuid(2, false); test_setuid(3, false); test_setuid(4, false); + test_setgid(2, false); + test_setgid(3, false); + test_setgid(4, false); // NOTE: this test doesn't clean up users that were created in // /etc/passwd or flush policies that were added to the LSM. From fcfe0ac2fcfae7d5fcad3d0375cb8ff38caf8aba Mon Sep 17 00:00:00 2001 From: Micah Morton Date: Wed, 8 Jun 2022 20:57:11 +0000 Subject: [PATCH 249/318] security: Add LSM hook to setgroups() syscall Give the LSM framework the ability to filter setgroups() syscalls. There are already analagous hooks for the set*uid() and set*gid() syscalls. The SafeSetID LSM will use this new hook to ensure setgroups() calls are allowed by the installed security policy. Tested by putting print statement in security_task_fix_setgroups() hook and confirming that it gets hit when userspace does a setgroups() syscall. Acked-by: Casey Schaufler Reviewed-by: Serge Hallyn Signed-off-by: Micah Morton --- include/linux/lsm_hook_defs.h | 1 + include/linux/lsm_hooks.h | 7 +++++++ include/linux/security.h | 7 +++++++ kernel/groups.c | 13 +++++++++++++ security/security.c | 5 +++++ 5 files changed, 33 insertions(+) diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index eafa1d2489fd..806448173033 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -201,6 +201,7 @@ LSM_HOOK(int, 0, task_fix_setuid, struct cred *new, const struct cred *old, int flags) LSM_HOOK(int, 0, task_fix_setgid, struct cred *new, const struct cred * old, int flags) +LSM_HOOK(int, 0, task_fix_setgroups, struct cred *new, const struct cred * old) LSM_HOOK(int, 0, task_setpgid, struct task_struct *p, pid_t pgid) LSM_HOOK(int, 0, task_getpgid, struct task_struct *p) LSM_HOOK(int, 0, task_getsid, struct task_struct *p) diff --git a/include/linux/lsm_hooks.h b/include/linux/lsm_hooks.h index 91c8146649f5..84a0d7e02176 100644 --- a/include/linux/lsm_hooks.h +++ b/include/linux/lsm_hooks.h @@ -702,6 +702,13 @@ * @old is the set of credentials that are being replaced. * @flags contains one of the LSM_SETID_* values. * Return 0 on success. + * @task_fix_setgroups: + * Update the module's state after setting the supplementary group + * identity attributes of the current process. + * @new is the set of credentials that will be installed. Modifications + * should be made to this rather than to @current->cred. + * @old is the set of credentials that are being replaced. + * Return 0 on success. * @task_setpgid: * Check permission before setting the process group identifier of the * process @p to @pgid. diff --git a/include/linux/security.h b/include/linux/security.h index 7fc4e9f49f54..1dfd32c49fa3 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -415,6 +415,7 @@ int security_task_fix_setuid(struct cred *new, const struct cred *old, int flags); int security_task_fix_setgid(struct cred *new, const struct cred *old, int flags); +int security_task_fix_setgroups(struct cred *new, const struct cred *old); int security_task_setpgid(struct task_struct *p, pid_t pgid); int security_task_getpgid(struct task_struct *p); int security_task_getsid(struct task_struct *p); @@ -1098,6 +1099,12 @@ static inline int security_task_fix_setgid(struct cred *new, return 0; } +static inline int security_task_fix_setgroups(struct cred *new, + const struct cred *old) +{ + return 0; +} + static inline int security_task_setpgid(struct task_struct *p, pid_t pgid) { return 0; diff --git a/kernel/groups.c b/kernel/groups.c index 787b381c7c00..9aaed2a31073 100644 --- a/kernel/groups.c +++ b/kernel/groups.c @@ -134,13 +134,26 @@ EXPORT_SYMBOL(set_groups); int set_current_groups(struct group_info *group_info) { struct cred *new; + const struct cred *old; + int retval; new = prepare_creds(); if (!new) return -ENOMEM; + old = current_cred(); + set_groups(new, group_info); + + retval = security_task_fix_setgroups(new, old); + if (retval < 0) + goto error; + return commit_creds(new); + +error: + abort_creds(new); + return retval; } EXPORT_SYMBOL(set_current_groups); diff --git a/security/security.c b/security/security.c index 188b8f782220..15c686145ad6 100644 --- a/security/security.c +++ b/security/security.c @@ -1803,6 +1803,11 @@ int security_task_fix_setgid(struct cred *new, const struct cred *old, return call_int_hook(task_fix_setgid, 0, new, old, flags); } +int security_task_fix_setgroups(struct cred *new, const struct cred *old) +{ + return call_int_hook(task_fix_setgroups, 0, new, old); +} + int security_task_setpgid(struct task_struct *p, pid_t pgid) { return call_int_hook(task_setpgid, 0, p, pgid); From 3e3374d382ff250502fbc4407001ac793d5c4e7f Mon Sep 17 00:00:00 2001 From: Micah Morton Date: Wed, 8 Jun 2022 22:27:27 +0000 Subject: [PATCH 250/318] LSM: SafeSetID: Add setgroups() security policy handling The SafeSetID LSM has functionality for restricting setuid()/setgid() syscalls based on its configured security policies. This patch adds the analogous functionality for the setgroups() syscall. Security policy for the setgroups() syscall follows the same policies that are installed on the system for setgid() syscalls. Signed-off-by: Micah Morton --- security/safesetid/lsm.c | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/security/safesetid/lsm.c b/security/safesetid/lsm.c index 963f4ad9cb66..e806739f7868 100644 --- a/security/safesetid/lsm.c +++ b/security/safesetid/lsm.c @@ -97,15 +97,9 @@ static int safesetid_security_capable(const struct cred *cred, return 0; /* - * If CAP_SET{U/G}ID is currently used for a setid() syscall, we want to - * let it go through here; the real security check happens later, in the - * task_fix_set{u/g}id hook. - * - * NOTE: - * Until we add support for restricting setgroups() calls, GID security - * policies offer no meaningful security since we always return 0 here - * when called from within the setgroups() syscall and there is no - * additional hook later on to enforce security policies for setgroups(). + * If CAP_SET{U/G}ID is currently used for a setid or setgroups syscall, we + * want to let it go through here; the real security check happens later, in + * the task_fix_set{u/g}id or task_fix_setgroups hooks. */ if ((opts & CAP_OPT_INSETID) != 0) return 0; @@ -241,9 +235,36 @@ static int safesetid_task_fix_setgid(struct cred *new, return -EACCES; } +static int safesetid_task_fix_setgroups(struct cred *new, const struct cred *old) +{ + int i; + + /* Do nothing if there are no setgid restrictions for our old RGID. */ + if (setid_policy_lookup((kid_t){.gid = old->gid}, INVALID_ID, GID) == SIDPOL_DEFAULT) + return 0; + + get_group_info(new->group_info); + for (i = 0; i < new->group_info->ngroups; i++) { + if (!id_permitted_for_cred(old, (kid_t){.gid = new->group_info->gid[i]}, GID)) { + put_group_info(new->group_info); + /* + * Kill this process to avoid potential security vulnerabilities + * that could arise from a missing allowlist entry preventing a + * privileged process from dropping to a lesser-privileged one. + */ + force_sig(SIGKILL); + return -EACCES; + } + } + + put_group_info(new->group_info); + return 0; +} + static struct security_hook_list safesetid_security_hooks[] = { LSM_HOOK_INIT(task_fix_setuid, safesetid_task_fix_setuid), LSM_HOOK_INIT(task_fix_setgid, safesetid_task_fix_setgid), + LSM_HOOK_INIT(task_fix_setgroups, safesetid_task_fix_setgroups), LSM_HOOK_INIT(capable, safesetid_security_capable) }; From 64b634830c919979de4b18163e15d30df66e64a8 Mon Sep 17 00:00:00 2001 From: Micah Morton Date: Thu, 16 Jun 2022 17:09:55 +0000 Subject: [PATCH 251/318] LSM: SafeSetID: add setgroups() testing to selftest Selftest already has support for testing UID and GID transitions. Signed-off-by: Micah Morton --- .../selftests/safesetid/safesetid-test.c | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/tools/testing/selftests/safesetid/safesetid-test.c b/tools/testing/selftests/safesetid/safesetid-test.c index a653c47a4ab5..eb9bf0aee951 100644 --- a/tools/testing/selftests/safesetid/safesetid-test.c +++ b/tools/testing/selftests/safesetid/safesetid-test.c @@ -375,6 +375,71 @@ static void test_setgid(gid_t child_gid, bool expect_success) die("should not reach here\n"); } +static void test_setgroups(gid_t* child_groups, size_t len, bool expect_success) +{ + pid_t cpid, w; + int wstatus; + gid_t groupset[len]; + int i, j; + + cpid = fork(); + if (cpid == -1) { + die("fork\n"); + } + + if (cpid == 0) { /* Code executed by child */ + if (setgroups(len, child_groups) != 0) + exit(EXIT_FAILURE); + if (getgroups(len, groupset) != len) + exit(EXIT_FAILURE); + for (i = 0; i < len; i++) { + for (j = 0; j < len; j++) { + if (child_groups[i] == groupset[j]) + break; + if (j == len - 1) + exit(EXIT_FAILURE); + } + } + exit(EXIT_SUCCESS); + } else { /* Code executed by parent */ + do { + w = waitpid(cpid, &wstatus, WUNTRACED | WCONTINUED); + if (w == -1) { + die("waitpid\n"); + } + + if (WIFEXITED(wstatus)) { + if (WEXITSTATUS(wstatus) == EXIT_SUCCESS) { + if (expect_success) { + return; + } else { + die("unexpected success\n"); + } + } else { + if (expect_success) { + die("unexpected failure\n"); + } else { + return; + } + } + } else if (WIFSIGNALED(wstatus)) { + if (WTERMSIG(wstatus) == 9) { + if (expect_success) + die("killed unexpectedly\n"); + else + return; + } else { + die("unexpected signal: %d\n", wstatus); + } + } else { + die("unexpected status: %d\n", wstatus); + } + } while (!WIFEXITED(wstatus) && !WIFSIGNALED(wstatus)); + } + + die("should not reach here\n"); +} + static void ensure_users_exist(void) { @@ -452,6 +517,10 @@ int main(int argc, char **argv) test_setgid(ALLOWED_CHILD2_UGID, true); test_setgid(NO_POLICY_UGID, false); + gid_t allowed_supp_groups[2] = {ALLOWED_CHILD1_UGID, ALLOWED_CHILD2_UGID}; + gid_t disallowed_supp_groups[2] = {ROOT_UGID, NO_POLICY_UGID}; + test_setgroups(allowed_supp_groups, 2, true); + test_setgroups(disallowed_supp_groups, 2, false); if (!test_userns(false)) { die("test_userns worked when it should fail\n"); From 8da572c52a9be6d006bae290339c629fc6501910 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 9 Jun 2022 17:06:16 -0700 Subject: [PATCH 252/318] fs-verity: mention btrfs support btrfs supports fs-verity since Linux v5.15. Document this. Signed-off-by: Eric Biggers Acked-by: David Sterba Link: https://lore.kernel.org/r/20220610000616.18225-1-ebiggers@kernel.org --- Documentation/filesystems/fsverity.rst | 53 +++++++++++++++----------- fs/verity/Kconfig | 10 ++--- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/Documentation/filesystems/fsverity.rst b/Documentation/filesystems/fsverity.rst index 756f2c215ba1..cb8e7573882a 100644 --- a/Documentation/filesystems/fsverity.rst +++ b/Documentation/filesystems/fsverity.rst @@ -11,9 +11,9 @@ Introduction fs-verity (``fs/verity/``) is a support layer that filesystems can hook into to support transparent integrity and authenticity protection -of read-only files. Currently, it is supported by the ext4 and f2fs -filesystems. Like fscrypt, not too much filesystem-specific code is -needed to support fs-verity. +of read-only files. Currently, it is supported by the ext4, f2fs, and +btrfs filesystems. Like fscrypt, not too much filesystem-specific +code is needed to support fs-verity. fs-verity is similar to `dm-verity `_ @@ -473,9 +473,9 @@ files being swapped around. Filesystem support ================== -fs-verity is currently supported by the ext4 and f2fs filesystems. -The CONFIG_FS_VERITY kconfig option must be enabled to use fs-verity -on either filesystem. +fs-verity is supported by several filesystems, described below. The +CONFIG_FS_VERITY kconfig option must be enabled to use fs-verity on +any of these filesystems. ``include/linux/fsverity.h`` declares the interface between the ``fs/verity/`` support layer and filesystems. Briefly, filesystems @@ -544,6 +544,13 @@ Currently, f2fs verity only supports a Merkle tree block size of 4096. Also, f2fs doesn't support enabling verity on files that currently have atomic or volatile writes pending. +btrfs +----- + +btrfs supports fs-verity since Linux v5.15. Verity-enabled inodes are +marked with a RO_COMPAT inode flag, and the verity metadata is stored +in separate btree items. + Implementation details ====================== @@ -622,14 +629,14 @@ workqueue, and then the workqueue work does the decryption or verification. Finally, pages where no decryption or verity error occurred are marked Uptodate, and the pages are unlocked. -Files on ext4 and f2fs may contain holes. Normally, ``->readahead()`` -simply zeroes holes and sets the corresponding pages Uptodate; no bios -are issued. To prevent this case from bypassing fs-verity, these -filesystems use fsverity_verify_page() to verify hole pages. +On many filesystems, files can contain holes. Normally, +``->readahead()`` simply zeroes holes and sets the corresponding pages +Uptodate; no bios are issued. To prevent this case from bypassing +fs-verity, these filesystems use fsverity_verify_page() to verify hole +pages. -ext4 and f2fs disable direct I/O on verity files, since otherwise -direct I/O would bypass fs-verity. (They also do the same for -encrypted files.) +Filesystems also disable direct I/O on verity files, since otherwise +direct I/O would bypass fs-verity. Userspace utility ================= @@ -648,7 +655,7 @@ Tests To test fs-verity, use xfstests. For example, using `kvm-xfstests `_:: - kvm-xfstests -c ext4,f2fs -g verity + kvm-xfstests -c ext4,f2fs,btrfs -g verity FAQ === @@ -771,15 +778,15 @@ weren't already directly answered in other parts of this document. e.g. magically trigger construction of a Merkle tree. :Q: Does fs-verity support remote filesystems? -:A: Only ext4 and f2fs support is implemented currently, but in - principle any filesystem that can store per-file verity metadata - can support fs-verity, regardless of whether it's local or remote. - Some filesystems may have fewer options of where to store the - verity metadata; one possibility is to store it past the end of - the file and "hide" it from userspace by manipulating i_size. The - data verification functions provided by ``fs/verity/`` also assume - that the filesystem uses the Linux pagecache, but both local and - remote filesystems normally do so. +:A: So far all filesystems that have implemented fs-verity support are + local filesystems, but in principle any filesystem that can store + per-file verity metadata can support fs-verity, regardless of + whether it's local or remote. Some filesystems may have fewer + options of where to store the verity metadata; one possibility is + to store it past the end of the file and "hide" it from userspace + by manipulating i_size. The data verification functions provided + by ``fs/verity/`` also assume that the filesystem uses the Linux + pagecache, but both local and remote filesystems normally do so. :Q: Why is anything filesystem-specific at all? Shouldn't fs-verity be implemented entirely at the VFS level? diff --git a/fs/verity/Kconfig b/fs/verity/Kconfig index 54598cd80145..aad1f1d998b9 100644 --- a/fs/verity/Kconfig +++ b/fs/verity/Kconfig @@ -14,11 +14,11 @@ config FS_VERITY help This option enables fs-verity. fs-verity is the dm-verity mechanism implemented at the file level. On supported - filesystems (currently EXT4 and F2FS), userspace can use an - ioctl to enable verity for a file, which causes the filesystem - to build a Merkle tree for the file. The filesystem will then - transparently verify any data read from the file against the - Merkle tree. The file is also made read-only. + filesystems (currently ext4, f2fs, and btrfs), userspace can + use an ioctl to enable verity for a file, which causes the + filesystem to build a Merkle tree for the file. The filesystem + will then transparently verify any data read from the file + against the Merkle tree. The file is also made read-only. This serves as an integrity check, but the availability of the Merkle tree root hash also allows efficiently supporting From 9592eef7c16ec5fb9f36c4d9abe8eeffc2e1d2f3 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Tue, 5 Jul 2022 20:48:41 +0200 Subject: [PATCH 253/318] random: remove CONFIG_ARCH_RANDOM When RDRAND was introduced, there was much discussion on whether it should be trusted and how the kernel should handle that. Initially, two mechanisms cropped up, CONFIG_ARCH_RANDOM, a compile time switch, and "nordrand", a boot-time switch. Later the thinking evolved. With a properly designed RNG, using RDRAND values alone won't harm anything, even if the outputs are malicious. Rather, the issue is whether those values are being *trusted* to be good or not. And so a new set of options were introduced as the real ones that people use -- CONFIG_RANDOM_TRUST_CPU and "random.trust_cpu". With these options, RDRAND is used, but it's not always credited. So in the worst case, it does nothing, and in the best case, maybe it helps. Along the way, CONFIG_ARCH_RANDOM's meaning got sort of pulled into the center and became something certain platforms force-select. The old options don't really help with much, and it's a bit odd to have special handling for these instructions when the kernel can deal fine with the existence or untrusted existence or broken existence or non-existence of that CPU capability. Simplify the situation by removing CONFIG_ARCH_RANDOM and using the ordinary asm-generic fallback pattern instead, keeping the two options that are actually used. For now it leaves "nordrand" for now, as the removal of that will take a different route. Acked-by: Michael Ellerman Acked-by: Catalin Marinas Acked-by: Borislav Petkov Acked-by: Heiko Carstens Acked-by: Greg Kroah-Hartman Signed-off-by: Jason A. Donenfeld --- arch/arm/include/asm/archrandom.h | 2 ++ arch/arm64/Kconfig | 8 ------ arch/arm64/include/asm/archrandom.h | 10 -------- arch/arm64/kernel/cpufeature.c | 2 -- arch/powerpc/Kconfig | 3 --- arch/powerpc/include/asm/archrandom.h | 3 --- arch/powerpc/include/asm/machdep.h | 2 -- arch/powerpc/platforms/microwatt/Kconfig | 1 - arch/powerpc/platforms/powernv/Kconfig | 1 - arch/powerpc/platforms/pseries/Kconfig | 1 - arch/s390/Kconfig | 15 ----------- arch/s390/configs/zfcpdump_defconfig | 1 - arch/s390/crypto/Makefile | 2 +- arch/s390/include/asm/archrandom.h | 3 --- arch/s390/kernel/setup.c | 2 -- arch/x86/Kconfig | 9 ------- arch/x86/include/asm/archrandom.h | 14 +++-------- arch/x86/kernel/cpu/rdrand.c | 2 -- drivers/char/Kconfig | 1 - drivers/char/hw_random/s390-trng.c | 9 ------- include/asm-generic/Kbuild | 1 + include/asm-generic/archrandom.h | 25 +++++++++++++++++++ include/linux/random.h | 9 +------ .../selftests/wireguard/qemu/kernel.config | 1 - 24 files changed, 34 insertions(+), 93 deletions(-) create mode 100644 include/asm-generic/archrandom.h diff --git a/arch/arm/include/asm/archrandom.h b/arch/arm/include/asm/archrandom.h index a8e84ca5c2ee..cc4714eb1a75 100644 --- a/arch/arm/include/asm/archrandom.h +++ b/arch/arm/include/asm/archrandom.h @@ -7,4 +7,6 @@ static inline bool __init smccc_probe_trng(void) return false; } +#include + #endif /* _ASM_ARCHRANDOM_H */ diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 1652a9800ebe..1880f71c2547 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1858,14 +1858,6 @@ config ARM64_E0PD This option enables E0PD for TTBR1 where available. -config ARCH_RANDOM - bool "Enable support for random number generation" - default y - help - Random number generation (part of the ARMv8.5 Extensions) - provides a high bandwidth, cryptographically secure - hardware random number generator. - config ARM64_AS_HAS_MTE # Initial support for MTE went in binutils 2.32.0, checked with # ".arch armv8.5-a+memtag" below. However, this was incomplete diff --git a/arch/arm64/include/asm/archrandom.h b/arch/arm64/include/asm/archrandom.h index 3a6b6d38c5b8..c3b9fa56af67 100644 --- a/arch/arm64/include/asm/archrandom.h +++ b/arch/arm64/include/asm/archrandom.h @@ -2,8 +2,6 @@ #ifndef _ASM_ARCHRANDOM_H #define _ASM_ARCHRANDOM_H -#ifdef CONFIG_ARCH_RANDOM - #include #include #include @@ -167,12 +165,4 @@ arch_get_random_seed_long_early(unsigned long *v) } #define arch_get_random_seed_long_early arch_get_random_seed_long_early -#else /* !CONFIG_ARCH_RANDOM */ - -static inline bool __init smccc_probe_trng(void) -{ - return false; -} - -#endif /* CONFIG_ARCH_RANDOM */ #endif /* _ASM_ARCHRANDOM_H */ diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 8d88433de81d..0e9462abeb77 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2416,7 +2416,6 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .cpu_enable = cpu_enable_e0pd, }, #endif -#ifdef CONFIG_ARCH_RANDOM { .desc = "Random Number Generator", .capability = ARM64_HAS_RNG, @@ -2428,7 +2427,6 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .sign = FTR_UNSIGNED, .min_field_value = 1, }, -#endif #ifdef CONFIG_ARM64_BTI { .desc = "Branch Target Identification", diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 7aa12e88c580..623deb5bedf6 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -1252,9 +1252,6 @@ config PHYSICAL_START default "0x00000000" endif -config ARCH_RANDOM - def_bool n - config PPC_LIB_RHEAP bool diff --git a/arch/powerpc/include/asm/archrandom.h b/arch/powerpc/include/asm/archrandom.h index 9a53e29680f4..25ba65df6b1a 100644 --- a/arch/powerpc/include/asm/archrandom.h +++ b/arch/powerpc/include/asm/archrandom.h @@ -2,8 +2,6 @@ #ifndef _ASM_POWERPC_ARCHRANDOM_H #define _ASM_POWERPC_ARCHRANDOM_H -#ifdef CONFIG_ARCH_RANDOM - #include static inline bool __must_check arch_get_random_long(unsigned long *v) @@ -35,7 +33,6 @@ static inline bool __must_check arch_get_random_seed_int(unsigned int *v) return rc; } -#endif /* CONFIG_ARCH_RANDOM */ #ifdef CONFIG_PPC_POWERNV int powernv_hwrng_present(void); diff --git a/arch/powerpc/include/asm/machdep.h b/arch/powerpc/include/asm/machdep.h index 358d171ae8e0..6c1002043367 100644 --- a/arch/powerpc/include/asm/machdep.h +++ b/arch/powerpc/include/asm/machdep.h @@ -200,9 +200,7 @@ struct machdep_calls { ssize_t (*cpu_release)(const char *, size_t); #endif -#ifdef CONFIG_ARCH_RANDOM int (*get_random_seed)(unsigned long *v); -#endif }; extern void e500_idle(void); diff --git a/arch/powerpc/platforms/microwatt/Kconfig b/arch/powerpc/platforms/microwatt/Kconfig index 5e320f49583a..6af443a1db99 100644 --- a/arch/powerpc/platforms/microwatt/Kconfig +++ b/arch/powerpc/platforms/microwatt/Kconfig @@ -6,7 +6,6 @@ config PPC_MICROWATT select PPC_ICS_NATIVE select PPC_ICP_NATIVE select PPC_UDBG_16550 - select ARCH_RANDOM help This option enables support for FPGA-based Microwatt implementations. diff --git a/arch/powerpc/platforms/powernv/Kconfig b/arch/powerpc/platforms/powernv/Kconfig index 161dfe024085..e1a05c5a9004 100644 --- a/arch/powerpc/platforms/powernv/Kconfig +++ b/arch/powerpc/platforms/powernv/Kconfig @@ -12,7 +12,6 @@ config PPC_POWERNV select EPAPR_BOOT select PPC_INDIRECT_PIO select PPC_UDBG_16550 - select ARCH_RANDOM select CPU_FREQ select PPC_DOORBELL select MMU_NOTIFIER diff --git a/arch/powerpc/platforms/pseries/Kconfig b/arch/powerpc/platforms/pseries/Kconfig index f7fd91d153a4..f4a647c1f0b2 100644 --- a/arch/powerpc/platforms/pseries/Kconfig +++ b/arch/powerpc/platforms/pseries/Kconfig @@ -19,7 +19,6 @@ config PPC_PSERIES select PPC_UDBG_16550 select PPC_DOORBELL select HOTPLUG_CPU - select ARCH_RANDOM select FORCE_SMP select SWIOTLB default y diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 8cd9e56c629b..9b6e4e7cb17b 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -507,21 +507,6 @@ config KEXEC_SIG verification for the corresponding kernel image type being loaded in order for this to work. -config ARCH_RANDOM - def_bool y - prompt "s390 architectural random number generation API" - help - Enable the s390 architectural random number generation API - to provide random data for all consumers within the Linux - kernel. - - When enabled the arch_random_* functions declared in linux/random.h - are implemented. The implementation is based on the s390 CPACF - instruction subfunction TRNG which provides a real true random - number generator. - - If unsure, say Y. - config KERNEL_NOBP def_bool n prompt "Enable modified branch prediction for the kernel by default" diff --git a/arch/s390/configs/zfcpdump_defconfig b/arch/s390/configs/zfcpdump_defconfig index a87fcc45e307..f4976f611b94 100644 --- a/arch/s390/configs/zfcpdump_defconfig +++ b/arch/s390/configs/zfcpdump_defconfig @@ -15,7 +15,6 @@ CONFIG_TUNE_ZEC12=y # CONFIG_COMPAT is not set CONFIG_NR_CPUS=2 CONFIG_HZ_100=y -# CONFIG_ARCH_RANDOM is not set # CONFIG_RELOCATABLE is not set # CONFIG_CHSC_SCH is not set # CONFIG_SCM_BUS is not set diff --git a/arch/s390/crypto/Makefile b/arch/s390/crypto/Makefile index c63abfeb6d17..1b1cc478fa94 100644 --- a/arch/s390/crypto/Makefile +++ b/arch/s390/crypto/Makefile @@ -15,7 +15,7 @@ obj-$(CONFIG_CRYPTO_CHACHA_S390) += chacha_s390.o obj-$(CONFIG_S390_PRNG) += prng.o obj-$(CONFIG_CRYPTO_GHASH_S390) += ghash_s390.o obj-$(CONFIG_CRYPTO_CRC32_S390) += crc32-vx_s390.o -obj-$(CONFIG_ARCH_RANDOM) += arch_random.o +obj-y += arch_random.o crc32-vx_s390-y := crc32-vx.o crc32le-vx.o crc32be-vx.o chacha_s390-y := chacha-glue.o chacha-s390.o diff --git a/arch/s390/include/asm/archrandom.h b/arch/s390/include/asm/archrandom.h index 2c6e1c6ecbe7..0a1c2e66c709 100644 --- a/arch/s390/include/asm/archrandom.h +++ b/arch/s390/include/asm/archrandom.h @@ -11,8 +11,6 @@ #ifndef _ASM_S390_ARCHRANDOM_H #define _ASM_S390_ARCHRANDOM_H -#ifdef CONFIG_ARCH_RANDOM - #include #include #include @@ -50,5 +48,4 @@ static inline bool __must_check arch_get_random_seed_int(unsigned int *v) return false; } -#endif /* CONFIG_ARCH_RANDOM */ #endif /* _ASM_S390_ARCHRANDOM_H */ diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 0a37f5de2863..ebad41afe355 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -876,10 +876,8 @@ static void __init setup_randomness(void) add_device_randomness(&vmms->vm, sizeof(vmms->vm[0]) * vmms->count); memblock_free(vmms, PAGE_SIZE); -#ifdef CONFIG_ARCH_RANDOM if (cpacf_query_func(CPACF_PRNO, CPACF_PRNO_TRNG)) static_branch_enable(&s390_arch_random_available); -#endif } /* diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index e58798f636d4..ba13749c09c8 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1810,15 +1810,6 @@ config ARCH_USES_PG_UNCACHED def_bool y depends on X86_PAT -config ARCH_RANDOM - def_bool y - prompt "x86 architectural random number generator" if EXPERT - help - Enable the x86 architectural RDRAND instruction - (Intel Bull Mountain technology) to generate random numbers. - If supported, this is a high bandwidth, cryptographically - secure hardware random number generator. - config X86_UMIP def_bool y prompt "User Mode Instruction Prevention" if EXPERT diff --git a/arch/x86/include/asm/archrandom.h b/arch/x86/include/asm/archrandom.h index ebc248e49549..fb235b696175 100644 --- a/arch/x86/include/asm/archrandom.h +++ b/arch/x86/include/asm/archrandom.h @@ -65,10 +65,8 @@ static inline bool __must_check rdseed_int(unsigned int *v) /* * These are the generic interfaces; they must not be declared if the - * stubs in are to be invoked, - * i.e. CONFIG_ARCH_RANDOM is not defined. + * stubs in are to be invoked. */ -#ifdef CONFIG_ARCH_RANDOM static inline bool __must_check arch_get_random_long(unsigned long *v) { @@ -90,12 +88,8 @@ static inline bool __must_check arch_get_random_seed_int(unsigned int *v) return static_cpu_has(X86_FEATURE_RDSEED) ? rdseed_int(v) : false; } -extern void x86_init_rdrand(struct cpuinfo_x86 *c); - -#else /* !CONFIG_ARCH_RANDOM */ - -static inline void x86_init_rdrand(struct cpuinfo_x86 *c) { } - -#endif /* !CONFIG_ARCH_RANDOM */ +#ifndef CONFIG_UML +void x86_init_rdrand(struct cpuinfo_x86 *c); +#endif #endif /* ASM_X86_ARCHRANDOM_H */ diff --git a/arch/x86/kernel/cpu/rdrand.c b/arch/x86/kernel/cpu/rdrand.c index c4be62058dd9..8f216669ecb8 100644 --- a/arch/x86/kernel/cpu/rdrand.c +++ b/arch/x86/kernel/cpu/rdrand.c @@ -26,7 +26,6 @@ __setup("nordrand", x86_rdrand_setup); */ #define SANITY_CHECK_LOOPS 8 -#ifdef CONFIG_ARCH_RANDOM void x86_init_rdrand(struct cpuinfo_x86 *c) { unsigned int changed = 0; @@ -63,4 +62,3 @@ void x86_init_rdrand(struct cpuinfo_x86 *c) "RDRAND gives funky smelling output, might consider not using it by booting with \"nordrand\""); } -#endif diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig index 0b6c03643ddc..30192e123e5f 100644 --- a/drivers/char/Kconfig +++ b/drivers/char/Kconfig @@ -431,7 +431,6 @@ config ADI config RANDOM_TRUST_CPU bool "Initialize RNG using CPU RNG instructions" default y - depends on ARCH_RANDOM help Initialize the RNG using random numbers supplied by the CPU's RNG instructions (e.g. RDRAND), if supported and available. These diff --git a/drivers/char/hw_random/s390-trng.c b/drivers/char/hw_random/s390-trng.c index 2beaa35c0d74..488808dc17a2 100644 --- a/drivers/char/hw_random/s390-trng.c +++ b/drivers/char/hw_random/s390-trng.c @@ -108,7 +108,6 @@ static ssize_t trng_counter_show(struct device *dev, { u64 dev_counter = atomic64_read(&trng_dev_counter); u64 hwrng_counter = atomic64_read(&trng_hwrng_counter); -#if IS_ENABLED(CONFIG_ARCH_RANDOM) u64 arch_counter = atomic64_read(&s390_arch_random_counter); return sysfs_emit(buf, @@ -118,14 +117,6 @@ static ssize_t trng_counter_show(struct device *dev, "total: %llu\n", dev_counter, hwrng_counter, arch_counter, dev_counter + hwrng_counter + arch_counter); -#else - return sysfs_emit(buf, - "trng: %llu\n" - "hwrng: %llu\n" - "total: %llu\n", - dev_counter, hwrng_counter, - dev_counter + hwrng_counter); -#endif } static DEVICE_ATTR(byte_counter, 0444, trng_counter_show, NULL); diff --git a/include/asm-generic/Kbuild b/include/asm-generic/Kbuild index 8e47d483b524..36db8b9eb68a 100644 --- a/include/asm-generic/Kbuild +++ b/include/asm-generic/Kbuild @@ -5,6 +5,7 @@ # asm headers from the host architecutre.) mandatory-y += atomic.h +mandatory-y += archrandom.h mandatory-y += barrier.h mandatory-y += bitops.h mandatory-y += bug.h diff --git a/include/asm-generic/archrandom.h b/include/asm-generic/archrandom.h new file mode 100644 index 000000000000..3a5ee202dd86 --- /dev/null +++ b/include/asm-generic/archrandom.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_GENERIC_ARCHRANDOM_H__ +#define __ASM_GENERIC_ARCHRANDOM_H__ + +static inline bool __must_check arch_get_random_long(unsigned long *v) +{ + return false; +} + +static inline bool __must_check arch_get_random_int(unsigned int *v) +{ + return false; +} + +static inline bool __must_check arch_get_random_seed_long(unsigned long *v) +{ + return false; +} + +static inline bool __must_check arch_get_random_seed_int(unsigned int *v) +{ + return false; +} + +#endif diff --git a/include/linux/random.h b/include/linux/random.h index 20e389a14e5c..865770e29f3e 100644 --- a/include/linux/random.h +++ b/include/linux/random.h @@ -106,14 +106,7 @@ declare_get_random_var_wait(long, unsigned long) */ #include -#ifdef CONFIG_ARCH_RANDOM -# include -#else -static inline bool __must_check arch_get_random_long(unsigned long *v) { return false; } -static inline bool __must_check arch_get_random_int(unsigned int *v) { return false; } -static inline bool __must_check arch_get_random_seed_long(unsigned long *v) { return false; } -static inline bool __must_check arch_get_random_seed_int(unsigned int *v) { return false; } -#endif +#include /* * Called from the boot CPU during startup; not valid to call once diff --git a/tools/testing/selftests/wireguard/qemu/kernel.config b/tools/testing/selftests/wireguard/qemu/kernel.config index bad88f4b0a03..e1858ce7003f 100644 --- a/tools/testing/selftests/wireguard/qemu/kernel.config +++ b/tools/testing/selftests/wireguard/qemu/kernel.config @@ -58,7 +58,6 @@ CONFIG_NO_HZ_IDLE=y CONFIG_NO_HZ_FULL=n CONFIG_HZ_PERIODIC=n CONFIG_HIGH_RES_TIMERS=y -CONFIG_ARCH_RANDOM=y CONFIG_FILE_LOCKING=y CONFIG_POSIX_TIMERS=y CONFIG_DEVTMPFS=y From 049f9ae93d033be6758ad865a9b89650f9f075ec Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Sat, 9 Jul 2022 12:43:06 +0200 Subject: [PATCH 254/318] x86/rdrand: Remove "nordrand" flag in favor of "random.trust_cpu" The decision of whether or not to trust RDRAND is controlled by the "random.trust_cpu" boot time parameter or the CONFIG_RANDOM_TRUST_CPU compile time default. The "nordrand" flag was added during the early days of RDRAND, when there were worries that merely using its values could compromise the RNG. However, these days, RDRAND values are not used directly but always go through the RNG's hash function, making "nordrand" no longer useful. Rather, the correct switch is "random.trust_cpu", which not only handles the relevant trust issue directly, but also is general to multiple CPU types, not just x86. However, x86 RDRAND does have a history of being occasionally problematic. Prior, when the kernel would notice something strange, it'd warn in dmesg and suggest enabling "nordrand". We can improve on that by making the test a little bit better and then taking the step of automatically disabling RDRAND if we detect it's problematic. Also disable RDSEED if the RDRAND test fails. Cc: x86@kernel.org Cc: Theodore Ts'o Suggested-by: H. Peter Anvin Suggested-by: Borislav Petkov Acked-by: Borislav Petkov Signed-off-by: Jason A. Donenfeld --- .../admin-guide/kernel-parameters.txt | 5 -- arch/x86/kernel/cpu/amd.c | 2 +- arch/x86/kernel/cpu/rdrand.c | 57 +++++++------------ 3 files changed, 22 insertions(+), 42 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index f2d26cb7e853..1e2307f11105 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3733,11 +3733,6 @@ noreplace-smp [X86-32,SMP] Don't replace SMP instructions with UP alternatives - nordrand [X86] Disable kernel use of the RDRAND and - RDSEED instructions even if they are supported - by the processor. RDRAND and RDSEED are still - available to user space applications. - noresume [SWSUSP] Disables resume and restores original swap space. diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 35d5288394cb..48276c0e479d 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -808,7 +808,7 @@ static void clear_rdrand_cpuid_bit(struct cpuinfo_x86 *c) return; /* - * The nordrand option can clear X86_FEATURE_RDRAND, so check for + * The self-test can clear X86_FEATURE_RDRAND, so check for * RDRAND support using the CPUID function directly. */ if (!(cpuid_ecx(1) & BIT(30)) || rdrand_force) diff --git a/arch/x86/kernel/cpu/rdrand.c b/arch/x86/kernel/cpu/rdrand.c index 8f216669ecb8..26a427fa84ea 100644 --- a/arch/x86/kernel/cpu/rdrand.c +++ b/arch/x86/kernel/cpu/rdrand.c @@ -11,54 +11,39 @@ #include #include -static int __init x86_rdrand_setup(char *s) -{ - setup_clear_cpu_cap(X86_FEATURE_RDRAND); - setup_clear_cpu_cap(X86_FEATURE_RDSEED); - return 1; -} -__setup("nordrand", x86_rdrand_setup); - /* * RDRAND has Built-In-Self-Test (BIST) that runs on every invocation. - * Run the instruction a few times as a sanity check. - * If it fails, it is simple to disable RDRAND here. + * Run the instruction a few times as a sanity check. Also make sure + * it's not outputting the same value over and over, which has happened + * as a result of past CPU bugs. + * + * If it fails, it is simple to disable RDRAND and RDSEED here. */ -#define SANITY_CHECK_LOOPS 8 void x86_init_rdrand(struct cpuinfo_x86 *c) { - unsigned int changed = 0; - unsigned long tmp, prev; - int i; + enum { SAMPLES = 8, MIN_CHANGE = 5 }; + unsigned long sample, prev; + bool failure = false; + size_t i, changed; if (!cpu_has(c, X86_FEATURE_RDRAND)) return; - for (i = 0; i < SANITY_CHECK_LOOPS; i++) { - if (!rdrand_long(&tmp)) { - clear_cpu_cap(c, X86_FEATURE_RDRAND); - pr_warn_once("rdrand: disabled\n"); - return; + for (changed = 0, i = 0; i < SAMPLES; ++i) { + if (!rdrand_long(&sample)) { + failure = true; + break; } + changed += i && sample != prev; + prev = sample; } + if (changed < MIN_CHANGE) + failure = true; - /* - * Stupid sanity-check whether RDRAND does *actually* generate - * some at least random-looking data. - */ - prev = tmp; - for (i = 0; i < SANITY_CHECK_LOOPS; i++) { - if (rdrand_long(&tmp)) { - if (prev != tmp) - changed++; - - prev = tmp; - } + if (failure) { + clear_cpu_cap(c, X86_FEATURE_RDRAND); + clear_cpu_cap(c, X86_FEATURE_RDSEED); + pr_emerg("RDRAND is not reliable on this platform; disabling.\n"); } - - if (WARN_ON_ONCE(!changed)) - pr_emerg( -"RDRAND gives funky smelling output, might consider not using it by booting with \"nordrand\""); - } From b8ac29b40183a6038919768b5d189c9bd91ce9b4 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Sun, 17 Jul 2022 23:53:34 +0200 Subject: [PATCH 255/318] timekeeping: contribute wall clock to rng on time change The rng's random_init() function contributes the real time to the rng at boot time, so that events can at least start in relation to something particular in the real world. But this clock might not yet be set that point in boot, so nothing is contributed. In addition, the relation between minor clock changes from, say, NTP, and the cycle counter is potentially useful entropic data. This commit addresses this by mixing in a time stamp on calls to settimeofday and adjtimex. No entropy is credited in doing so, so it doesn't make initialization faster, but it is still useful input to have. Fixes: 1da177e4c3f4 ("Linux-2.6.12-rc2") Cc: stable@vger.kernel.org Reviewed-by: Thomas Gleixner Reviewed-by: Eric Biggers Signed-off-by: Jason A. Donenfeld --- kernel/time/timekeeping.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 8e4b3c32fcf9..f72b9f1de178 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -23,6 +23,7 @@ #include #include #include +#include #include "tick-internal.h" #include "ntp_internal.h" @@ -1343,8 +1344,10 @@ out: /* Signal hrtimers about time change */ clock_was_set(CLOCK_SET_WALL); - if (!ret) + if (!ret) { audit_tk_injoffset(ts_delta); + add_device_randomness(ts, sizeof(*ts)); + } return ret; } @@ -2430,6 +2433,7 @@ int do_adjtimex(struct __kernel_timex *txc) ret = timekeeping_validate_timex(txc); if (ret) return ret; + add_device_randomness(txc, sizeof(*txc)); if (txc->modes & ADJ_SETOFFSET) { struct timespec64 delta; @@ -2447,6 +2451,7 @@ int do_adjtimex(struct __kernel_timex *txc) audit_ntp_init(&ad); ktime_get_real_ts64(&ts); + add_device_randomness(&ts, sizeof(ts)); raw_spin_lock_irqsave(&timekeeper_lock, flags); write_seqcount_begin(&tk_core.seq); From b7a68f67ff4911e8a842d03f6f97fa91a8d483f5 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Thu, 14 Jul 2022 20:28:22 +0200 Subject: [PATCH 256/318] random: use try_cmpxchg in _credit_init_bits Use `!try_cmpxchg(ptr, &orig, new)` instead of `cmpxchg(ptr, orig, new) != orig` in _credit_init_bits. This has two benefits: - The x86 cmpxchg instruction returns success in the ZF flag, so this change saves a compare after cmpxchg, as well as a related move instruction in front of cmpxchg. - try_cmpxchg implicitly assigns the *ptr value to &orig when cmpxchg fails, enabling further code simplifications. This patch has no functional change. Signed-off-by: Uros Bizjak Signed-off-by: Jason A. Donenfeld --- drivers/char/random.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/char/random.c b/drivers/char/random.c index a1af90bacc9f..0c6568ae5f68 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -643,10 +643,10 @@ static void __cold _credit_init_bits(size_t bits) add = min_t(size_t, bits, POOL_BITS); + orig = READ_ONCE(input_pool.init_bits); do { - orig = READ_ONCE(input_pool.init_bits); new = min_t(unsigned int, POOL_BITS, orig + add); - } while (cmpxchg(&input_pool.init_bits, orig, new) != orig); + } while (!try_cmpxchg(&input_pool.init_bits, &orig, new)); if (orig < POOL_READY_BITS && new >= POOL_READY_BITS) { crng_reseed(); /* Sets crng_init to CRNG_READY under base_crng.lock. */ From 0b9ba6135d7f18b82f3d8bebb55ded725ba88e0e Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Wed, 13 Jul 2022 01:12:21 +0200 Subject: [PATCH 257/318] um: seed rng using host OS rng UML generally does not provide access to special CPU instructions like RDRAND, and execution tends to be rather deterministic, with no real hardware interrupts, making good randomness really very hard, if not all together impossible. Not only is this a security eyebrow raiser, but it's also quite annoying when trying to do various pieces of UML-based automation that takes a long time to boot, if ever. Fix this by trivially calling getrandom() in the host and using that seed as "bootloader randomness", which initializes the rng immediately at UML boot. The old behavior can be restored the same way as on any other arch, by way of CONFIG_TRUST_BOOTLOADER_RANDOMNESS=n or random.trust_bootloader=0. So seen from that perspective, this just makes UML act like other archs, which is positive in its own right. Additionally, wire up arch_get_random_{int,long}() in the same way, so that reseeds can also make use of the host RNG, controllable by CONFIG_TRUST_CPU_RANDOMNESS and random.trust_cpu, per usual. Cc: stable@vger.kernel.org Acked-by: Johannes Berg Acked-By: Anton Ivanov Signed-off-by: Jason A. Donenfeld --- arch/um/include/asm/archrandom.h | 30 ++++++++++++++++++++++++++++++ arch/um/include/shared/os.h | 7 +++++++ arch/um/kernel/um_arch.c | 8 ++++++++ arch/um/os-Linux/util.c | 6 ++++++ 4 files changed, 51 insertions(+) create mode 100644 arch/um/include/asm/archrandom.h diff --git a/arch/um/include/asm/archrandom.h b/arch/um/include/asm/archrandom.h new file mode 100644 index 000000000000..2f24cb96391d --- /dev/null +++ b/arch/um/include/asm/archrandom.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_UM_ARCHRANDOM_H__ +#define __ASM_UM_ARCHRANDOM_H__ + +#include + +/* This is from , but better not to #include that in a global header here. */ +ssize_t os_getrandom(void *buf, size_t len, unsigned int flags); + +static inline bool __must_check arch_get_random_long(unsigned long *v) +{ + return os_getrandom(v, sizeof(*v), 0) == sizeof(*v); +} + +static inline bool __must_check arch_get_random_int(unsigned int *v) +{ + return os_getrandom(v, sizeof(*v), 0) == sizeof(*v); +} + +static inline bool __must_check arch_get_random_seed_long(unsigned long *v) +{ + return false; +} + +static inline bool __must_check arch_get_random_seed_int(unsigned int *v) +{ + return false; +} + +#endif diff --git a/arch/um/include/shared/os.h b/arch/um/include/shared/os.h index fafde1d5416e..0df646c6651e 100644 --- a/arch/um/include/shared/os.h +++ b/arch/um/include/shared/os.h @@ -11,6 +11,12 @@ #include #include #include +/* This is to get size_t */ +#ifndef __UM_HOST__ +#include +#else +#include +#endif #define CATCH_EINTR(expr) while ((errno = 0, ((expr) < 0)) && (errno == EINTR)) @@ -243,6 +249,7 @@ extern void stack_protections(unsigned long address); extern int raw(int fd); extern void setup_machinename(char *machine_out); extern void setup_hostinfo(char *buf, int len); +extern ssize_t os_getrandom(void *buf, size_t len, unsigned int flags); extern void os_dump_core(void) __attribute__ ((noreturn)); extern void um_early_printk(const char *s, unsigned int n); extern void os_fix_helper_signals(void); diff --git a/arch/um/kernel/um_arch.c b/arch/um/kernel/um_arch.c index 0760e24f2eba..74f3efd96bd4 100644 --- a/arch/um/kernel/um_arch.c +++ b/arch/um/kernel/um_arch.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -406,6 +407,8 @@ int __init __weak read_initrd(void) void __init setup_arch(char **cmdline_p) { + u8 rng_seed[32]; + stack_protections((unsigned long) &init_thread_info); setup_physmem(uml_physmem, uml_reserved, physmem_size, highmem); mem_total_pages(physmem_size, iomem_size, highmem); @@ -416,6 +419,11 @@ void __init setup_arch(char **cmdline_p) strlcpy(boot_command_line, command_line, COMMAND_LINE_SIZE); *cmdline_p = command_line; setup_hostinfo(host_info, sizeof host_info); + + if (os_getrandom(rng_seed, sizeof(rng_seed), 0) == sizeof(rng_seed)) { + add_bootloader_randomness(rng_seed, sizeof(rng_seed)); + memzero_explicit(rng_seed, sizeof(rng_seed)); + } } void __init check_bugs(void) diff --git a/arch/um/os-Linux/util.c b/arch/um/os-Linux/util.c index 41297ec404bf..fc0f2a9dee5a 100644 --- a/arch/um/os-Linux/util.c +++ b/arch/um/os-Linux/util.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -96,6 +97,11 @@ static inline void __attribute__ ((noreturn)) uml_abort(void) exit(127); } +ssize_t os_getrandom(void *buf, size_t len, unsigned int flags) +{ + return getrandom(buf, len, flags); +} + /* * UML helper threads must not handle SIGWINCH/INT/TERM */ From fb77dccfc701b6ebcc232574c828bc69146cf90a Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 12 Apr 2022 15:08:14 -0700 Subject: [PATCH 258/318] rcu: Decrease FQS scan wait time in case of callback overloading The force-quiesce-state loop function rcu_gp_fqs_loop() checks for callback overloading and does an immediate initial scan for idle CPUs if so. However, subsequent rescans will be carried out at as leisurely a rate as they always are, as specified by the rcutree.jiffies_till_next_fqs module parameter. It might be tempting to just continue immediately rescanning, but this turns the RCU grace-period kthread into a CPU hog. It might also be tempting to reduce the time between rescans to a single jiffy, but this can be problematic on larger systems. This commit therefore divides the normal time between rescans by three, rounding up. Thus a small system running at HZ=1000 that is suffering from callback overload will wait only one jiffy instead of the normal three between rescans. [ paulmck: Apply Neeraj Upadhyay feedback. ] Signed-off-by: Paul E. McKenney Reviewed-by: Neeraj Upadhyay --- kernel/rcu/tree.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index c25ba442044a..52094e72866e 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -1983,7 +1983,12 @@ static noinline_for_stack void rcu_gp_fqs_loop(void) gf = RCU_GP_FLAG_OVLD; ret = 0; for (;;) { - if (!ret) { + if (rcu_state.cbovld) { + j = (j + 2) / 3; + if (j <= 0) + j = 1; + } + if (!ret || time_before(jiffies + j, rcu_state.jiffies_force_qs)) { WRITE_ONCE(rcu_state.jiffies_force_qs, jiffies + j); /* * jiffies_force_qs before RCU_GP_WAIT_FQS state From 48f8070f5dd8e13148ae4647780a452d53c457a2 Mon Sep 17 00:00:00 2001 From: Patrick Wang Date: Tue, 26 Apr 2022 18:45:02 +0800 Subject: [PATCH 259/318] rcu: Avoid tracing a few functions executed in stop machine Stop-machine recently started calling additional functions while waiting: ---------------------------------------------------------------- Former stop machine wait loop: do { cpu_relax(); => macro ... } while (curstate != STOPMACHINE_EXIT); ----------------------------------------------------------------- Current stop machine wait loop: do { stop_machine_yield(cpumask); => function (notraced) ... touch_nmi_watchdog(); => function (notraced, inside calls also notraced) ... rcu_momentary_dyntick_idle(); => function (notraced, inside calls traced) } while (curstate != MULTI_STOP_EXIT); ------------------------------------------------------------------ These functions (and the functions that they call) must be marked notrace to prevent them from being updated while they are executing. The consequences of failing to mark these functions can be severe: rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: rcu: 1-...!: (0 ticks this GP) idle=14f/1/0x4000000000000000 softirq=3397/3397 fqs=0 rcu: 3-...!: (0 ticks this GP) idle=ee9/1/0x4000000000000000 softirq=5168/5168 fqs=0 (detected by 0, t=8137 jiffies, g=5889, q=2 ncpus=4) Task dump for CPU 1: task:migration/1 state:R running task stack: 0 pid: 19 ppid: 2 flags:0x00000000 Stopper: multi_cpu_stop+0x0/0x18c <- stop_machine_cpuslocked+0x128/0x174 Call Trace: Task dump for CPU 3: task:migration/3 state:R running task stack: 0 pid: 29 ppid: 2 flags:0x00000000 Stopper: multi_cpu_stop+0x0/0x18c <- stop_machine_cpuslocked+0x128/0x174 Call Trace: rcu: rcu_preempt kthread timer wakeup didn't happen for 8136 jiffies! g5889 f0x0 RCU_GP_WAIT_FQS(5) ->state=0x402 rcu: Possible timer handling issue on cpu=2 timer-softirq=594 rcu: rcu_preempt kthread starved for 8137 jiffies! g5889 f0x0 RCU_GP_WAIT_FQS(5) ->state=0x402 ->cpu=2 rcu: Unless rcu_preempt kthread gets sufficient CPU time, OOM is now expected behavior. rcu: RCU grace-period kthread stack dump: task:rcu_preempt state:I stack: 0 pid: 14 ppid: 2 flags:0x00000000 Call Trace: schedule+0x56/0xc2 schedule_timeout+0x82/0x184 rcu_gp_fqs_loop+0x19a/0x318 rcu_gp_kthread+0x11a/0x140 kthread+0xee/0x118 ret_from_exception+0x0/0x14 rcu: Stack dump where RCU GP kthread last ran: Task dump for CPU 2: task:migration/2 state:R running task stack: 0 pid: 24 ppid: 2 flags:0x00000000 Stopper: multi_cpu_stop+0x0/0x18c <- stop_machine_cpuslocked+0x128/0x174 Call Trace: This commit therefore marks these functions notrace: rcu_preempt_deferred_qs() rcu_preempt_need_deferred_qs() rcu_preempt_deferred_qs_irqrestore() [ paulmck: Apply feedback from Neeraj Upadhyay. ] Signed-off-by: Patrick Wang Acked-by: Steven Rostedt (Google) Signed-off-by: Paul E. McKenney Reviewed-by: Neeraj Upadhyay --- kernel/rcu/tree_plugin.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index c8ba0fe17267..7a07f2ca153e 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -460,7 +460,7 @@ static bool rcu_preempt_has_tasks(struct rcu_node *rnp) * be quite short, for example, in the case of the call from * rcu_read_unlock_special(). */ -static void +static notrace void rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags) { bool empty_exp; @@ -581,7 +581,7 @@ rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags) * is disabled. This function cannot be expected to understand these * nuances, so the caller must handle them. */ -static bool rcu_preempt_need_deferred_qs(struct task_struct *t) +static notrace bool rcu_preempt_need_deferred_qs(struct task_struct *t) { return (__this_cpu_read(rcu_data.cpu_no_qs.b.exp) || READ_ONCE(t->rcu_read_unlock_special.s)) && @@ -595,7 +595,7 @@ static bool rcu_preempt_need_deferred_qs(struct task_struct *t) * evaluate safety in terms of interrupt, softirq, and preemption * disabling. */ -static void rcu_preempt_deferred_qs(struct task_struct *t) +static notrace void rcu_preempt_deferred_qs(struct task_struct *t) { unsigned long flags; @@ -926,7 +926,7 @@ static bool rcu_preempt_has_tasks(struct rcu_node *rnp) * Because there is no preemptible RCU, there can be no deferred quiescent * states. */ -static bool rcu_preempt_need_deferred_qs(struct task_struct *t) +static notrace bool rcu_preempt_need_deferred_qs(struct task_struct *t) { return false; } @@ -935,7 +935,7 @@ static bool rcu_preempt_need_deferred_qs(struct task_struct *t) // period for a quiescent state from this CPU. Note that requests from // tasks are handled when removing the task from the blocked-tasks list // below. -static void rcu_preempt_deferred_qs(struct task_struct *t) +static notrace void rcu_preempt_deferred_qs(struct task_struct *t) { struct rcu_data *rdp = this_cpu_ptr(&rcu_data); From 52c1d81ee2911ef592048582c6d07975b7399726 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Thu, 5 May 2022 23:52:36 +0800 Subject: [PATCH 260/318] rcu: Add rnp->cbovldmask check in rcutree_migrate_callbacks() Currently, the rcu_node structure's ->cbovlmask field is set in call_rcu() when a given CPU is suffering from callback overload. But if that CPU goes offline, the outgoing CPU's callbacks is migrated to the running CPU, which is likely to overload the running CPU. However, that CPU's bit in its leaf rcu_node structure's ->cbovlmask field remains zero. Initially, this is OK because the outgoing CPU's bit remains set. However, that bit will be cleared at the next end of a grace period, at which time it is quite possible that the running CPU will still be overloaded. If the running CPU invokes call_rcu(), then overload will be checked for and the bit will be set. Except that there is no guarantee that the running CPU will invoke call_rcu(), in which case the next grace period will fail to take the running CPU's overload condition into account. Plus, because the bit is not set, the end of the grace period won't check for overload on this CPU. This commit therefore adds a call to check_cb_ovld_locked() in rcutree_migrate_callbacks() to set the running CPU's ->cbovlmask bit appropriately. Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney Reviewed-by: Neeraj Upadhyay --- kernel/rcu/tree.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 52094e72866e..7c62af481981 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -4491,6 +4491,7 @@ void rcutree_migrate_callbacks(int cpu) needwake = needwake || rcu_advance_cbs(my_rnp, my_rdp); rcu_segcblist_disable(&rdp->cblist); WARN_ON_ONCE(rcu_segcblist_empty(&my_rdp->cblist) != !rcu_segcblist_n_cbs(&my_rdp->cblist)); + check_cb_ovld_locked(my_rdp, my_rnp); if (rcu_rdp_is_offloaded(my_rdp)) { raw_spin_unlock_rcu_node(my_rnp); /* irqs remain disabled. */ __call_rcu_nocb_wake(my_rdp, true, flags); From 70a82c3c55c8665d3996dcb9968adcf24d52bbc4 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Fri, 13 May 2022 08:42:55 +0800 Subject: [PATCH 261/318] rcu: Immediately boost preempted readers for strict grace periods The intent of the CONFIG_RCU_STRICT_GRACE_PERIOD Konfig option is to cause normal grace periods to complete quickly in order to better catch errors resulting from improperly leaking pointers from RCU read-side critical sections. However, kernels built with this option enabled still wait for some hundreds of milliseconds before boosting RCU readers that have been preempted within their current critical section. The value of this delay is set by the CONFIG_RCU_BOOST_DELAY Kconfig option, which defaults to 500 milliseconds. This commit therefore causes kernels build with strict grace periods to ignore CONFIG_RCU_BOOST_DELAY. This causes rcu_initiate_boost() to start boosting immediately after all CPUs on a given leaf rcu_node structure have passed through their quiescent states. Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney Reviewed-by: Neeraj Upadhyay --- kernel/rcu/tree_plugin.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 7a07f2ca153e..4c92034e2670 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -1140,7 +1140,8 @@ static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags) (rnp->gp_tasks != NULL && rnp->boost_tasks == NULL && rnp->qsmask == 0 && - (!time_after(rnp->boost_time, jiffies) || rcu_state.cbovld))) { + (!time_after(rnp->boost_time, jiffies) || rcu_state.cbovld || + IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD)))) { if (rnp->exp_tasks == NULL) WRITE_ONCE(rnp->boost_tasks, rnp->gp_tasks); raw_spin_unlock_irqrestore_rcu_node(rnp, flags); From b3ade95b8ee507d650d9e163abfdf645a9f3886d Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 17 May 2022 18:03:36 -0700 Subject: [PATCH 262/318] rcu: Forbid RCU_STRICT_GRACE_PERIOD in TINY_RCU kernels The RCU_STRICT_GRACE_PERIOD Kconfig option does nothing in kernels built with CONFIG_TINY_RCU=y, so this commit adjusts the dependencies to disallow this combination. Signed-off-by: Paul E. McKenney Reviewed-by: Neeraj Upadhyay --- kernel/rcu/Kconfig.debug | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/rcu/Kconfig.debug b/kernel/rcu/Kconfig.debug index 9b64e55d4f61..4da05beb13d7 100644 --- a/kernel/rcu/Kconfig.debug +++ b/kernel/rcu/Kconfig.debug @@ -121,7 +121,7 @@ config RCU_EQS_DEBUG config RCU_STRICT_GRACE_PERIOD bool "Provide debug RCU implementation with short grace periods" - depends on DEBUG_KERNEL && RCU_EXPERT && NR_CPUS <= 4 + depends on DEBUG_KERNEL && RCU_EXPERT && NR_CPUS <= 4 && !TINY_RCU default n select PREEMPT_COUNT if PREEMPT=n help From 9c9b26b0df270d4f9246e483a44686fca951a29c Mon Sep 17 00:00:00 2001 From: Chen Zhongjin Date: Tue, 10 May 2022 17:46:39 +0800 Subject: [PATCH 263/318] locking/csd_lock: Change csdlock_debug from early_param to __setup The csdlock_debug kernel-boot parameter is parsed by the early_param() function csdlock_debug(). If set, csdlock_debug() invokes static_branch_enable() to enable csd_lock_wait feature, which triggers a panic on arm64 for kernels built with CONFIG_SPARSEMEM=y and CONFIG_SPARSEMEM_VMEMMAP=n. With CONFIG_SPARSEMEM_VMEMMAP=n, __nr_to_section is called in static_key_enable() and returns NULL, resulting in a NULL dereference because mem_section is initialized only later in sparse_init(). This is also a problem for powerpc because early_param() functions are invoked earlier than jump_label_init(), also resulting in static_key_enable() failures. These failures cause the warning "static key 'xxx' used before call to jump_label_init()". Thus, early_param is too early for csd_lock_wait to run static_branch_enable(), so changes it to __setup to fix these. Fixes: 8d0968cc6b8f ("locking/csd_lock: Add boot parameter for controlling CSD lock debugging") Cc: stable@vger.kernel.org Reported-by: Chen jingwen Signed-off-by: Chen Zhongjin Signed-off-by: Paul E. McKenney --- kernel/smp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/smp.c b/kernel/smp.c index dd215f439426..650810a6f29b 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -174,9 +174,9 @@ static int __init csdlock_debug(char *str) if (val) static_branch_enable(&csdlock_debug_enabled); - return 0; + return 1; } -early_param("csdlock_debug", csdlock_debug); +__setup("csdlock_debug=", csdlock_debug); static DEFINE_PER_CPU(call_single_data_t *, cur_csd); static DEFINE_PER_CPU(smp_call_func_t, cur_csd_func); From 800d6acf40e5ce676b53a1259fb4e93e56279367 Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Fri, 27 May 2022 17:07:45 +0200 Subject: [PATCH 264/318] rcu: tiny: Record kvfree_call_rcu() call stack for KASAN When running KASAN with Tiny RCU (e.g. under ARCH=um, where a working KASAN patch is now available), we don't get any information on the original kfree_rcu() (or similar) caller when a problem is reported, as Tiny RCU doesn't record this. Add the recording, which required pulling kvfree_call_rcu() out of line for the KASAN case since the recording function (kasan_record_aux_stack_noalloc) is neither exported, nor can we include kasan.h into rcutiny.h. without KASAN, the patch has no size impact (ARCH=um kernel): text data bss dec hex filename 6151515 4423154 33148520 43723189 29b29b5 linux 6151515 4423154 33148520 43723189 29b29b5 linux + patch with KASAN, the impact on my build was minimal: text data bss dec hex filename 13915539 7388050 33282304 54585893 340ea25 linux 13911266 7392114 33282304 54585684 340e954 linux + patch -4273 +4064 +-0 -209 Acked-by: Dmitry Vyukov Signed-off-by: Johannes Berg Signed-off-by: Paul E. McKenney --- include/linux/rcutiny.h | 11 ++++++++++- kernel/rcu/tiny.c | 14 ++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h index 5fed476f977f..d84e13f2c384 100644 --- a/include/linux/rcutiny.h +++ b/include/linux/rcutiny.h @@ -38,7 +38,7 @@ static inline void synchronize_rcu_expedited(void) */ extern void kvfree(const void *addr); -static inline void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) +static inline void __kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) { if (head) { call_rcu(head, func); @@ -51,6 +51,15 @@ static inline void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) kvfree((void *) func); } +#ifdef CONFIG_KASAN_GENERIC +void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func); +#else +static inline void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) +{ + __kvfree_call_rcu(head, func); +} +#endif + void rcu_qs(void); static inline void rcu_softirq_qs(void) diff --git a/kernel/rcu/tiny.c b/kernel/rcu/tiny.c index 340b3f8b090d..58ff3721d975 100644 --- a/kernel/rcu/tiny.c +++ b/kernel/rcu/tiny.c @@ -217,6 +217,20 @@ bool poll_state_synchronize_rcu(unsigned long oldstate) } EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu); +#ifdef CONFIG_KASAN_GENERIC +void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) +{ + if (head) { + void *ptr = (void *) head - (unsigned long) func; + + kasan_record_aux_stack_noalloc(ptr); + } + + __kvfree_call_rcu(head, func); +} +EXPORT_SYMBOL_GPL(kvfree_call_rcu); +#endif + void __init rcu_init(void) { open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); From e2bb1288a381e9239aaf606ae8c1e20ea71c20bd Mon Sep 17 00:00:00 2001 From: Zqiang Date: Thu, 26 May 2022 09:55:12 +0800 Subject: [PATCH 265/318] rcu: Cleanup RCU urgency state for offline CPU When a CPU is slow to provide a quiescent state for a given grace period, RCU takes steps to encourage that CPU to get with the quiescent-state program in a more timely fashion. These steps include these flags in the rcu_data structure: 1. ->rcu_urgent_qs, which causes the scheduling-clock interrupt to request an otherwise pointless context switch from the scheduler. 2. ->rcu_need_heavy_qs, which causes both cond_resched() and RCU's context-switch hook to do an immediate momentary quiscent state. 3. ->rcu_need_heavy_qs, which causes the scheduler-clock tick to be enabled even on nohz_full CPUs with only one runnable task. These flags are of course cleared once the corresponding CPU has passed through a quiescent state. Unless that quiescent state is the CPU going offline, which means that when the CPU comes back online, it will needlessly consume additional CPU time and incur additional latency, which constitutes a minor but very real performance bug. This commit therefore adds the call to rcu_disable_urgency_upon_qs() that clears these flags to the CPU-hotplug offlining code path. Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney Reviewed-by: Neeraj Upadhyay --- kernel/rcu/tree.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 7c62af481981..9d9a2a657823 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -4446,6 +4446,7 @@ void rcu_report_dead(unsigned int cpu) rdp->rcu_ofl_gp_flags = READ_ONCE(rcu_state.gp_flags); if (rnp->qsmask & mask) { /* RCU waiting on outgoing CPU? */ /* Report quiescent state -before- changing ->qsmaskinitnext! */ + rcu_disable_urgency_upon_qs(rdp); rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags); raw_spin_lock_irqsave_rcu_node(rnp, flags); } From 82d26c36cc68e781400eb4e541f943008208f2d6 Mon Sep 17 00:00:00 2001 From: "Joel Fernandes (Google)" Date: Thu, 2 Jun 2022 10:06:43 +0200 Subject: [PATCH 266/318] rcu/kvfree: Remove useless monitor_todo flag monitor_todo is not needed as the work struct already tracks if work is pending. Just use that to know if work is pending using schedule_delayed_work() helper. Signed-off-by: Joel Fernandes (Google) Signed-off-by: Uladzislau Rezki (Sony) Signed-off-by: Paul E. McKenney Reviewed-by: Neeraj Upadhyay --- kernel/rcu/tree.c | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 9d9a2a657823..6f4656aed962 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -3216,7 +3216,6 @@ struct kfree_rcu_cpu_work { * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period * @lock: Synchronize access to this structure * @monitor_work: Promote @head to @head_free after KFREE_DRAIN_JIFFIES - * @monitor_todo: Tracks whether a @monitor_work delayed work is pending * @initialized: The @rcu_work fields have been initialized * @count: Number of objects for which GP not started * @bkvcache: @@ -3241,7 +3240,6 @@ struct kfree_rcu_cpu { struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES]; raw_spinlock_t lock; struct delayed_work monitor_work; - bool monitor_todo; bool initialized; int count; @@ -3421,6 +3419,18 @@ static void kfree_rcu_work(struct work_struct *work) } } +static bool +need_offload_krc(struct kfree_rcu_cpu *krcp) +{ + int i; + + for (i = 0; i < FREE_N_CHANNELS; i++) + if (krcp->bkvhead[i]) + return true; + + return !!krcp->head; +} + /* * This function is invoked after the KFREE_DRAIN_JIFFIES timeout. */ @@ -3477,9 +3487,7 @@ static void kfree_rcu_monitor(struct work_struct *work) // of the channels that is still busy we should rearm the // work to repeat an attempt. Because previous batches are // still in progress. - if (!krcp->bkvhead[0] && !krcp->bkvhead[1] && !krcp->head) - krcp->monitor_todo = false; - else + if (need_offload_krc(krcp)) schedule_delayed_work(&krcp->monitor_work, KFREE_DRAIN_JIFFIES); raw_spin_unlock_irqrestore(&krcp->lock, flags); @@ -3667,11 +3675,8 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) WRITE_ONCE(krcp->count, krcp->count + 1); // Set timer to drain after KFREE_DRAIN_JIFFIES. - if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING && - !krcp->monitor_todo) { - krcp->monitor_todo = true; + if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING) schedule_delayed_work(&krcp->monitor_work, KFREE_DRAIN_JIFFIES); - } unlock_return: krc_this_cpu_unlock(krcp, flags); @@ -3746,14 +3751,8 @@ void __init kfree_rcu_scheduler_running(void) struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu); raw_spin_lock_irqsave(&krcp->lock, flags); - if ((!krcp->bkvhead[0] && !krcp->bkvhead[1] && !krcp->head) || - krcp->monitor_todo) { - raw_spin_unlock_irqrestore(&krcp->lock, flags); - continue; - } - krcp->monitor_todo = true; - schedule_delayed_work_on(cpu, &krcp->monitor_work, - KFREE_DRAIN_JIFFIES); + if (need_offload_krc(krcp)) + schedule_delayed_work_on(cpu, &krcp->monitor_work, KFREE_DRAIN_JIFFIES); raw_spin_unlock_irqrestore(&krcp->lock, flags); } } From 9bdb5b3a8d8ad1c92db309219859fe1c87c95351 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 8 Jun 2022 09:34:10 -0700 Subject: [PATCH 267/318] rcu: Initialize first_gp_fqs at declaration in rcu_gp_fqs() This commit saves a line of code by initializing the rcu_gp_fqs() function's first_gp_fqs local variable in its declaration. Reported-by: Frederic Weisbecker Reported-by: Neeraj Upadhyay Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 6f4656aed962..a23877a773be 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -1971,13 +1971,12 @@ static void rcu_gp_fqs(bool first_time) */ static noinline_for_stack void rcu_gp_fqs_loop(void) { - bool first_gp_fqs; + bool first_gp_fqs = true; int gf = 0; unsigned long j; int ret; struct rcu_node *rnp = rcu_get_root(); - first_gp_fqs = true; j = READ_ONCE(jiffies_till_first_fqs); if (rcu_state.cbovld) gf = RCU_GP_FLAG_OVLD; From a03ae49c4785c1bc7b940e38bbdf2e63d79d1470 Mon Sep 17 00:00:00 2001 From: Neeraj Upadhyay Date: Thu, 9 Jun 2022 12:43:40 +0530 Subject: [PATCH 268/318] rcu/tree: Add comment to describe GP-done condition in fqs loop Add a comment to explain why !rcu_preempt_blocked_readers_cgp() condition is required on root rnp node, for GP completion check in rcu_gp_fqs_loop(). Reviewed-by: Joel Fernandes (Google) Signed-off-by: Neeraj Upadhyay Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index a23877a773be..7c2231823e84 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2005,7 +2005,15 @@ static noinline_for_stack void rcu_gp_fqs_loop(void) rcu_gp_torture_wait(); WRITE_ONCE(rcu_state.gp_state, RCU_GP_DOING_FQS); /* Locking provides needed memory barriers. */ - /* If grace period done, leave loop. */ + /* + * Exit the loop if the root rcu_node structure indicates that the grace period + * has ended, leave the loop. The rcu_preempt_blocked_readers_cgp(rnp) check + * is required only for single-node rcu_node trees because readers blocking + * the current grace period are queued only on leaf rcu_node structures. + * For multi-node trees, checking the root node's ->qsmask suffices, because a + * given root node's ->qsmask bit is cleared only when all CPUs and tasks from + * the corresponding leaf nodes have passed through their quiescent state. + */ if (!READ_ONCE(rnp->qsmask) && !rcu_preempt_blocked_readers_cgp(rnp)) break; From 1598f4a4762be0ea6a1bcd229c2c9ff1ebb212bb Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Tue, 19 Apr 2022 14:23:18 +0200 Subject: [PATCH 269/318] rcu/nocb: Add/del rdp to iterate from rcuog itself NOCB rdp's are part of a group whose list is iterated by the corresponding rdp leader. This list is RCU traversed because an rdp can be either added or deleted concurrently. Upon addition, a new iteration to the list after a synchronization point (a pair of LOCK/UNLOCK ->nocb_gp_lock) is forced to make sure: 1) we didn't miss a new element added in the middle of an iteration 2) we didn't ignore a whole subset of the list due to an element being quickly deleted and then re-added. 3) we prevent from probably other surprises... Although this layout is expected to be safe, it doesn't help anybody to sleep well. Simplify instead the nocb state toggling with moving the list modification from the nocb (de-)offloading workqueue to the rcuog kthreads instead. Whenever the rdp leader is expected to (re-)set the SEGCBLIST_KTHREAD_GP flag of a target rdp, the latter is queued so that the leader handles the flag flip along with adding or deleting the target rdp to the list to iterate. This way the list modification and iteration happen from the same kthread and those operations can't race altogether. As a bonus, the flags for each rdp don't need to be checked locklessly before each iteration, which is one less opportunity to produce nightmares. Signed-off-by: Frederic Weisbecker Cc: Neeraj Upadhyay Cc: Boqun Feng Cc: Uladzislau Rezki Cc: Joel Fernandes Cc: Zqiang Signed-off-by: Paul E. McKenney Reviewed-by: Neeraj Upadhyay --- kernel/rcu/tree.h | 1 + kernel/rcu/tree_nocb.h | 138 +++++++++++++++++++++-------------------- 2 files changed, 71 insertions(+), 68 deletions(-) diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 2ccf5845957d..4f8532c33558 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -235,6 +235,7 @@ struct rcu_data { * if rdp_gp. */ struct list_head nocb_entry_rdp; /* rcu_data node in wakeup chain. */ + struct rcu_data *nocb_toggling_rdp; /* rdp queued for (de-)offloading */ /* The following fields are used by CB kthread, hence new cacheline. */ struct rcu_data *nocb_gp_rdp ____cacheline_internodealigned_in_smp; diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index 46694e13398a..dac74952e1d1 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -546,52 +546,43 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, } } -/* - * Check if we ignore this rdp. - * - * We check that without holding the nocb lock but - * we make sure not to miss a freshly offloaded rdp - * with the current ordering: - * - * rdp_offload_toggle() nocb_gp_enabled_cb() - * ------------------------- ---------------------------- - * WRITE flags LOCK nocb_gp_lock - * LOCK nocb_gp_lock READ/WRITE nocb_gp_sleep - * READ/WRITE nocb_gp_sleep UNLOCK nocb_gp_lock - * UNLOCK nocb_gp_lock READ flags - */ -static inline bool nocb_gp_enabled_cb(struct rcu_data *rdp) -{ - u8 flags = SEGCBLIST_OFFLOADED | SEGCBLIST_KTHREAD_GP; - - return rcu_segcblist_test_flags(&rdp->cblist, flags); -} - -static inline bool nocb_gp_update_state_deoffloading(struct rcu_data *rdp, - bool *needwake_state) +static int nocb_gp_toggle_rdp(struct rcu_data *rdp, + bool *wake_state) { struct rcu_segcblist *cblist = &rdp->cblist; + unsigned long flags; + int ret; - if (rcu_segcblist_test_flags(cblist, SEGCBLIST_OFFLOADED)) { - if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)) { - rcu_segcblist_set_flags(cblist, SEGCBLIST_KTHREAD_GP); - if (rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB)) - *needwake_state = true; - } - return false; + rcu_nocb_lock_irqsave(rdp, flags); + if (rcu_segcblist_test_flags(cblist, SEGCBLIST_OFFLOADED) && + !rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)) { + /* + * Offloading. Set our flag and notify the offload worker. + * We will handle this rdp until it ever gets de-offloaded. + */ + rcu_segcblist_set_flags(cblist, SEGCBLIST_KTHREAD_GP); + if (rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB)) + *wake_state = true; + ret = 1; + } else if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_OFFLOADED) && + rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)) { + /* + * De-offloading. Clear our flag and notify the de-offload worker. + * We will ignore this rdp until it ever gets re-offloaded. + */ + rcu_segcblist_clear_flags(cblist, SEGCBLIST_KTHREAD_GP); + if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB)) + *wake_state = true; + ret = 0; + } else { + WARN_ON_ONCE(1); + ret = -1; } - /* - * De-offloading. Clear our flag and notify the de-offload worker. - * We will ignore this rdp until it ever gets re-offloaded. - */ - WARN_ON_ONCE(!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)); - rcu_segcblist_clear_flags(cblist, SEGCBLIST_KTHREAD_GP); - if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB)) - *needwake_state = true; - return true; -} + rcu_nocb_unlock_irqrestore(rdp, flags); + return ret; +} /* * No-CBs GP kthreads come here to wait for additional callbacks to show up @@ -609,7 +600,7 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) bool needwait_gp = false; // This prevents actual uninitialized use. bool needwake; bool needwake_gp; - struct rcu_data *rdp; + struct rcu_data *rdp, *rdp_toggling = NULL; struct rcu_node *rnp; unsigned long wait_gp_seq = 0; // Suppress "use uninitialized" warning. bool wasempty = false; @@ -634,19 +625,10 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) * is added to the list, so the skipped-over rcu_data structures * won't be ignored for long. */ - list_for_each_entry_rcu(rdp, &my_rdp->nocb_head_rdp, nocb_entry_rdp, 1) { - bool needwake_state = false; - - if (!nocb_gp_enabled_cb(rdp)) - continue; + list_for_each_entry(rdp, &my_rdp->nocb_head_rdp, nocb_entry_rdp) { trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Check")); rcu_nocb_lock_irqsave(rdp, flags); - if (nocb_gp_update_state_deoffloading(rdp, &needwake_state)) { - rcu_nocb_unlock_irqrestore(rdp, flags); - if (needwake_state) - swake_up_one(&rdp->nocb_state_wq); - continue; - } + lockdep_assert_held(&rdp->nocb_lock); bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); if (bypass_ncbs && (time_after(j, READ_ONCE(rdp->nocb_bypass_first) + 1) || @@ -656,8 +638,6 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); } else if (!bypass_ncbs && rcu_segcblist_empty(&rdp->cblist)) { rcu_nocb_unlock_irqrestore(rdp, flags); - if (needwake_state) - swake_up_one(&rdp->nocb_state_wq); continue; /* No callbacks here, try next. */ } if (bypass_ncbs) { @@ -705,8 +685,6 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) } if (needwake_gp) rcu_gp_kthread_wake(); - if (needwake_state) - swake_up_one(&rdp->nocb_state_wq); } my_rdp->nocb_gp_bypass = bypass; @@ -739,15 +717,49 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) !READ_ONCE(my_rdp->nocb_gp_sleep)); trace_rcu_this_gp(rnp, my_rdp, wait_gp_seq, TPS("EndWait")); } + if (!rcu_nocb_poll) { raw_spin_lock_irqsave(&my_rdp->nocb_gp_lock, flags); + // (De-)queue an rdp to/from the group if its nocb state is changing + rdp_toggling = my_rdp->nocb_toggling_rdp; + if (rdp_toggling) + my_rdp->nocb_toggling_rdp = NULL; + if (my_rdp->nocb_defer_wakeup > RCU_NOCB_WAKE_NOT) { WRITE_ONCE(my_rdp->nocb_defer_wakeup, RCU_NOCB_WAKE_NOT); del_timer(&my_rdp->nocb_timer); } WRITE_ONCE(my_rdp->nocb_gp_sleep, true); raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags); + } else { + rdp_toggling = READ_ONCE(my_rdp->nocb_toggling_rdp); + if (rdp_toggling) { + /* + * Paranoid locking to make sure nocb_toggling_rdp is well + * reset *before* we (re)set SEGCBLIST_KTHREAD_GP or we could + * race with another round of nocb toggling for this rdp. + * Nocb locking should prevent from that already but we stick + * to paranoia, especially in rare path. + */ + raw_spin_lock_irqsave(&my_rdp->nocb_gp_lock, flags); + my_rdp->nocb_toggling_rdp = NULL; + raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags); + } } + + if (rdp_toggling) { + bool wake_state = false; + int ret; + + ret = nocb_gp_toggle_rdp(rdp_toggling, &wake_state); + if (ret == 1) + list_add_tail(&rdp_toggling->nocb_entry_rdp, &my_rdp->nocb_head_rdp); + else if (ret == 0) + list_del(&rdp_toggling->nocb_entry_rdp); + if (wake_state) + swake_up_one(&rdp_toggling->nocb_state_wq); + } + my_rdp->nocb_gp_seq = -1; WARN_ON(signal_pending(current)); } @@ -966,6 +978,8 @@ static int rdp_offload_toggle(struct rcu_data *rdp, swake_up_one(&rdp->nocb_cb_wq); raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags); + // Queue this rdp for add/del to/from the list to iterate on rcuog + WRITE_ONCE(rdp_gp->nocb_toggling_rdp, rdp); if (rdp_gp->nocb_gp_sleep) { rdp_gp->nocb_gp_sleep = false; wake_gp = true; @@ -1013,8 +1027,6 @@ static long rcu_nocb_rdp_deoffload(void *arg) swait_event_exclusive(rdp->nocb_state_wq, !rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP)); - /* Stop nocb_gp_wait() from iterating over this structure. */ - list_del_rcu(&rdp->nocb_entry_rdp); /* * Lock one last time to acquire latest callback updates from kthreads * so we can later handle callbacks locally without locking. @@ -1079,16 +1091,6 @@ static long rcu_nocb_rdp_offload(void *arg) pr_info("Offloading %d\n", rdp->cpu); - /* - * Cause future nocb_gp_wait() invocations to iterate over - * structure, resetting ->nocb_gp_sleep and waking up the related - * "rcuog". Since nocb_gp_wait() in turn locks ->nocb_gp_lock - * before setting ->nocb_gp_sleep again, we are guaranteed to - * iterate this newly added structure before "rcuog" goes to - * sleep again. - */ - list_add_tail_rcu(&rdp->nocb_entry_rdp, &rdp->nocb_gp_rdp->nocb_head_rdp); - /* * Can't use rcu_nocb_lock_irqsave() before SEGCBLIST_LOCKING * is set. From 24a57affd242566fef2935f04f062350b8275187 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Tue, 19 Apr 2022 14:23:19 +0200 Subject: [PATCH 270/318] rcu/nocb: Invert rcu_state.barrier_mutex VS hotplug lock locking order In case of failure to spawn either rcuog or rcuo[p] kthreads for a given rdp, rcu_nocb_rdp_deoffload() needs to be called with the hotplug lock and the barrier_mutex held. However cpus write lock is already held while calling rcutree_prepare_cpu(). It's not possible to call rcu_nocb_rdp_deoffload() from there with just locking the barrier_mutex or this would result in a locking inversion against rcu_nocb_cpu_deoffload() which holds both locks in the reverse order. Simply solve this with inverting the locking order inside rcu_nocb_cpu_[de]offload(). This will be a pre-requisite to toggle NOCB states toward cpusets anyway. Signed-off-by: Zqiang Cc: Neeraj Upadhyay Cc: Boqun Feng Cc: Uladzislau Rezki Cc: Joel Fernandes Signed-off-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney Reviewed-by: Neeraj Upadhyay --- kernel/rcu/tree_nocb.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index dac74952e1d1..f2f2cab6285a 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -1055,8 +1055,8 @@ int rcu_nocb_cpu_deoffload(int cpu) struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); int ret = 0; - mutex_lock(&rcu_state.barrier_mutex); cpus_read_lock(); + mutex_lock(&rcu_state.barrier_mutex); if (rcu_rdp_is_offloaded(rdp)) { if (cpu_online(cpu)) { ret = work_on_cpu(cpu, rcu_nocb_rdp_deoffload, rdp); @@ -1067,8 +1067,8 @@ int rcu_nocb_cpu_deoffload(int cpu) ret = -EINVAL; } } - cpus_read_unlock(); mutex_unlock(&rcu_state.barrier_mutex); + cpus_read_unlock(); return ret; } @@ -1134,8 +1134,8 @@ int rcu_nocb_cpu_offload(int cpu) struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); int ret = 0; - mutex_lock(&rcu_state.barrier_mutex); cpus_read_lock(); + mutex_lock(&rcu_state.barrier_mutex); if (!rcu_rdp_is_offloaded(rdp)) { if (cpu_online(cpu)) { ret = work_on_cpu(cpu, rcu_nocb_rdp_offload, rdp); @@ -1146,8 +1146,8 @@ int rcu_nocb_cpu_offload(int cpu) ret = -EINVAL; } } - cpus_read_unlock(); mutex_unlock(&rcu_state.barrier_mutex); + cpus_read_unlock(); return ret; } From 3a5761dc025da47960755ac64d9fbf1c32e8cd80 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Tue, 19 Apr 2022 14:23:20 +0200 Subject: [PATCH 271/318] rcu/nocb: Fix NOCB kthreads spawn failure with rcu_nocb_rdp_deoffload() direct call If the rcuog/o[p] kthreads spawn failed, the offloaded rdp needs to be explicitly deoffloaded, otherwise the target rdp is still considered offloaded even though nothing actually handles the callbacks. Signed-off-by: Zqiang Cc: Neeraj Upadhyay Cc: Boqun Feng Cc: Uladzislau Rezki Cc: Joel Fernandes Signed-off-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney Reviewed-by: Neeraj Upadhyay --- kernel/rcu/tree_nocb.h | 80 +++++++++++++++++++++++++++++++++--------- 1 file changed, 64 insertions(+), 16 deletions(-) diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index f2f2cab6285a..4cf9a29bba79 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -986,10 +986,7 @@ static int rdp_offload_toggle(struct rcu_data *rdp, } raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags); - if (wake_gp) - wake_up_process(rdp_gp->nocb_gp_kthread); - - return 0; + return wake_gp; } static long rcu_nocb_rdp_deoffload(void *arg) @@ -997,9 +994,15 @@ static long rcu_nocb_rdp_deoffload(void *arg) struct rcu_data *rdp = arg; struct rcu_segcblist *cblist = &rdp->cblist; unsigned long flags; - int ret; + int wake_gp; + struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; - WARN_ON_ONCE(rdp->cpu != raw_smp_processor_id()); + /* + * rcu_nocb_rdp_deoffload() may be called directly if + * rcuog/o[p] spawn failed, because at this time the rdp->cpu + * is not online yet. + */ + WARN_ON_ONCE((rdp->cpu != raw_smp_processor_id()) && cpu_online(rdp->cpu)); pr_info("De-offloading %d\n", rdp->cpu); @@ -1023,10 +1026,41 @@ static long rcu_nocb_rdp_deoffload(void *arg) */ rcu_segcblist_set_flags(cblist, SEGCBLIST_RCU_CORE); invoke_rcu_core(); - ret = rdp_offload_toggle(rdp, false, flags); - swait_event_exclusive(rdp->nocb_state_wq, - !rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB | - SEGCBLIST_KTHREAD_GP)); + wake_gp = rdp_offload_toggle(rdp, false, flags); + + mutex_lock(&rdp_gp->nocb_gp_kthread_mutex); + if (rdp_gp->nocb_gp_kthread) { + if (wake_gp) + wake_up_process(rdp_gp->nocb_gp_kthread); + + /* + * If rcuo[p] kthread spawn failed, directly remove SEGCBLIST_KTHREAD_CB. + * Just wait SEGCBLIST_KTHREAD_GP to be cleared by rcuog. + */ + if (!rdp->nocb_cb_kthread) { + rcu_nocb_lock_irqsave(rdp, flags); + rcu_segcblist_clear_flags(&rdp->cblist, SEGCBLIST_KTHREAD_CB); + rcu_nocb_unlock_irqrestore(rdp, flags); + } + + swait_event_exclusive(rdp->nocb_state_wq, + !rcu_segcblist_test_flags(cblist, + SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP)); + } else { + /* + * No kthread to clear the flags for us or remove the rdp from the nocb list + * to iterate. Do it here instead. Locking doesn't look stricly necessary + * but we stick to paranoia in this rare path. + */ + rcu_nocb_lock_irqsave(rdp, flags); + rcu_segcblist_clear_flags(&rdp->cblist, + SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP); + rcu_nocb_unlock_irqrestore(rdp, flags); + + list_del(&rdp->nocb_entry_rdp); + } + mutex_unlock(&rdp_gp->nocb_gp_kthread_mutex); + /* * Lock one last time to acquire latest callback updates from kthreads * so we can later handle callbacks locally without locking. @@ -1047,7 +1081,7 @@ static long rcu_nocb_rdp_deoffload(void *arg) WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); - return ret; + return 0; } int rcu_nocb_cpu_deoffload(int cpu) @@ -1079,7 +1113,8 @@ static long rcu_nocb_rdp_offload(void *arg) struct rcu_data *rdp = arg; struct rcu_segcblist *cblist = &rdp->cblist; unsigned long flags; - int ret; + int wake_gp; + struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; WARN_ON_ONCE(rdp->cpu != raw_smp_processor_id()); /* @@ -1089,6 +1124,9 @@ static long rcu_nocb_rdp_offload(void *arg) if (!rdp->nocb_gp_rdp) return -EINVAL; + if (WARN_ON_ONCE(!rdp_gp->nocb_gp_kthread)) + return -EINVAL; + pr_info("Offloading %d\n", rdp->cpu); /* @@ -1113,7 +1151,9 @@ static long rcu_nocb_rdp_offload(void *arg) * WRITE flags READ callbacks * rcu_nocb_unlock() rcu_nocb_unlock() */ - ret = rdp_offload_toggle(rdp, true, flags); + wake_gp = rdp_offload_toggle(rdp, true, flags); + if (wake_gp) + wake_up_process(rdp_gp->nocb_gp_kthread); swait_event_exclusive(rdp->nocb_state_wq, rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB) && rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)); @@ -1126,7 +1166,7 @@ static long rcu_nocb_rdp_offload(void *arg) rcu_segcblist_clear_flags(cblist, SEGCBLIST_RCU_CORE); rcu_nocb_unlock_irqrestore(rdp, flags); - return ret; + return 0; } int rcu_nocb_cpu_offload(int cpu) @@ -1248,7 +1288,7 @@ static void rcu_spawn_cpu_nocb_kthread(int cpu) "rcuog/%d", rdp_gp->cpu); if (WARN_ONCE(IS_ERR(t), "%s: Could not start rcuo GP kthread, OOM is now expected behavior\n", __func__)) { mutex_unlock(&rdp_gp->nocb_gp_kthread_mutex); - return; + goto end; } WRITE_ONCE(rdp_gp->nocb_gp_kthread, t); if (kthread_prio) @@ -1260,12 +1300,20 @@ static void rcu_spawn_cpu_nocb_kthread(int cpu) t = kthread_run(rcu_nocb_cb_kthread, rdp, "rcuo%c/%d", rcu_state.abbr, cpu); if (WARN_ONCE(IS_ERR(t), "%s: Could not start rcuo CB kthread, OOM is now expected behavior\n", __func__)) - return; + goto end; if (kthread_prio) sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); WRITE_ONCE(rdp->nocb_cb_kthread, t); WRITE_ONCE(rdp->nocb_gp_kthread, rdp_gp->nocb_gp_kthread); + return; +end: + mutex_lock(&rcu_state.barrier_mutex); + if (rcu_rdp_is_offloaded(rdp)) { + rcu_nocb_rdp_deoffload(rdp); + cpumask_clear_cpu(cpu, rcu_nocb_mask); + } + mutex_unlock(&rcu_state.barrier_mutex); } /* How many CB CPU IDs per GP kthread? Default of -1 for sqrt(nr_cpu_ids). */ From b37a667c62421b34e96b05613457b9fb0ed66ea1 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Fri, 22 Apr 2022 17:52:47 +0000 Subject: [PATCH 272/318] rcu/nocb: Add an option to offload all CPUs on boot Systems built with CONFIG_RCU_NOCB_CPU=y but booted without either the rcu_nocbs= or rcu_nohz_full= kernel-boot parameters will not have callback offloading on any of the CPUs, nor can any of the CPUs be switched to enable callback offloading at runtime. Although this is intentional, it would be nice to have a way to offload all the CPUs without having to make random bootloaders specify either the rcu_nocbs= or the rcu_nohz_full= kernel-boot parameters. This commit therefore provides a new CONFIG_RCU_NOCB_CPU_DEFAULT_ALL Kconfig option that switches the default so as to offload callback processing on all of the CPUs. This default can still be overridden using the rcu_nocbs= and rcu_nohz_full= kernel-boot parameters. Reviewed-by: Kalesh Singh Reviewed-by: Uladzislau Rezki (In v4.1, fixed issues with CONFIG maze reported by kernel test robot). Reported-by: kernel test robot Signed-off-by: Joel Fernandes Signed-off-by: Paul E. McKenney Reviewed-by: Neeraj Upadhyay --- Documentation/admin-guide/kernel-parameters.txt | 6 ++++++ kernel/rcu/Kconfig | 13 +++++++++++++ kernel/rcu/tree_nocb.h | 17 +++++++++++++++-- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 2522b11e593f..34605c275294 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3659,6 +3659,9 @@ just as if they had also been called out in the rcu_nocbs= boot parameter. + Note that this argument takes precedence over + the CONFIG_RCU_NOCB_CPU_DEFAULT_ALL option. + noiotrap [SH] Disables trapped I/O port accesses. noirqdebug [X86-32] Disables the code which attempts to detect and @@ -4557,6 +4560,9 @@ no-callback mode from boot but the mode may be toggled at runtime via cpusets. + Note that this argument takes precedence over + the CONFIG_RCU_NOCB_CPU_DEFAULT_ALL option. + rcu_nocb_poll [KNL] Rather than requiring that offloaded CPUs (specified by rcu_nocbs= above) explicitly diff --git a/kernel/rcu/Kconfig b/kernel/rcu/Kconfig index 1c630e573548..27aab870ae4c 100644 --- a/kernel/rcu/Kconfig +++ b/kernel/rcu/Kconfig @@ -262,6 +262,19 @@ config RCU_NOCB_CPU Say Y here if you need reduced OS jitter, despite added overhead. Say N here if you are unsure. +config RCU_NOCB_CPU_DEFAULT_ALL + bool "Offload RCU callback processing from all CPUs by default" + depends on RCU_NOCB_CPU + default n + help + Use this option to offload callback processing from all CPUs + by default, in the absence of the rcu_nocbs or nohz_full boot + parameter. This also avoids the need to use any boot parameters + to achieve the effect of offloading all CPUs on boot. + + Say Y here if you want offload all CPUs by default on boot. + Say N here if you are unsure. + config TASKS_TRACE_RCU_READ_MB bool "Tasks Trace RCU readers use memory barriers in user and idle" depends on RCU_EXPERT && TASKS_TRACE_RCU diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index 4cf9a29bba79..60cc92cc6655 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -1197,11 +1197,21 @@ void __init rcu_init_nohz(void) { int cpu; bool need_rcu_nocb_mask = false; + bool offload_all = false; struct rcu_data *rdp; -#if defined(CONFIG_NO_HZ_FULL) - if (tick_nohz_full_running && !cpumask_empty(tick_nohz_full_mask)) +#if defined(CONFIG_RCU_NOCB_CPU_DEFAULT_ALL) + if (!rcu_state.nocb_is_setup) { need_rcu_nocb_mask = true; + offload_all = true; + } +#endif /* #if defined(CONFIG_RCU_NOCB_CPU_DEFAULT_ALL) */ + +#if defined(CONFIG_NO_HZ_FULL) + if (tick_nohz_full_running && !cpumask_empty(tick_nohz_full_mask)) { + need_rcu_nocb_mask = true; + offload_all = false; /* NO_HZ_FULL has its own mask. */ + } #endif /* #if defined(CONFIG_NO_HZ_FULL) */ if (need_rcu_nocb_mask) { @@ -1222,6 +1232,9 @@ void __init rcu_init_nohz(void) cpumask_or(rcu_nocb_mask, rcu_nocb_mask, tick_nohz_full_mask); #endif /* #if defined(CONFIG_NO_HZ_FULL) */ + if (offload_all) + cpumask_setall(rcu_nocb_mask); + if (!cpumask_subset(rcu_nocb_mask, cpu_possible_mask)) { pr_info("\tNote: kernel parameter 'rcu_nocbs=', 'nohz_full', or 'isolcpus=' contains nonexistent CPUs.\n"); cpumask_and(rcu_nocb_mask, cpu_possible_mask, From 5103850654fdc651f0a7076ac753b958f018bb85 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Fri, 29 Apr 2022 20:42:22 +0800 Subject: [PATCH 273/318] rcu: Add nocb_cb_kthread check to rcu_is_callbacks_kthread() Callbacks are invoked in RCU kthreads when calbacks are offloaded (rcu_nocbs boot parameter) or when RCU's softirq handler has been offloaded to rcuc kthreads (use_softirq==0). The current code allows for the rcu_nocbs case but not the use_softirq case. This commit adds support for the use_softirq case. Reported-by: kernel test robot Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney Reviewed-by: Neeraj Upadhyay --- kernel/rcu/tree.c | 4 ++-- kernel/rcu/tree.h | 2 +- kernel/rcu/tree_plugin.h | 33 +++++++++++++++++++-------------- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index c25ba442044a..74455671e6cf 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2530,7 +2530,7 @@ static void rcu_do_batch(struct rcu_data *rdp) trace_rcu_batch_end(rcu_state.name, 0, !rcu_segcblist_empty(&rdp->cblist), need_resched(), is_idle_task(current), - rcu_is_callbacks_kthread()); + rcu_is_callbacks_kthread(rdp)); return; } @@ -2608,7 +2608,7 @@ static void rcu_do_batch(struct rcu_data *rdp) rcu_nocb_lock_irqsave(rdp, flags); rdp->n_cbs_invoked += count; trace_rcu_batch_end(rcu_state.name, count, !!rcl.head, need_resched(), - is_idle_task(current), rcu_is_callbacks_kthread()); + is_idle_task(current), rcu_is_callbacks_kthread(rdp)); /* Update counts and requeue any remaining callbacks. */ rcu_segcblist_insert_done_cbs(&rdp->cblist, &rcl); diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 4f8532c33558..649ad4f0129b 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -426,7 +426,7 @@ static void rcu_flavor_sched_clock_irq(int user); static void dump_blkd_tasks(struct rcu_node *rnp, int ncheck); static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags); static void rcu_preempt_boost_start_gp(struct rcu_node *rnp); -static bool rcu_is_callbacks_kthread(void); +static bool rcu_is_callbacks_kthread(struct rcu_data *rdp); static void rcu_cpu_kthread_setup(unsigned int cpu); static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp); static bool rcu_preempt_has_tasks(struct rcu_node *rnp); diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index c8ba0fe17267..0483e1338c41 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -1012,6 +1012,25 @@ static void rcu_cpu_kthread_setup(unsigned int cpu) WRITE_ONCE(rdp->rcuc_activity, jiffies); } +static bool rcu_is_callbacks_nocb_kthread(struct rcu_data *rdp) +{ +#ifdef CONFIG_RCU_NOCB_CPU + return rdp->nocb_cb_kthread == current; +#else + return false; +#endif +} + +/* + * Is the current CPU running the RCU-callbacks kthread? + * Caller must have preemption disabled. + */ +static bool rcu_is_callbacks_kthread(struct rcu_data *rdp) +{ + return rdp->rcu_cpu_kthread_task == current || + rcu_is_callbacks_nocb_kthread(rdp); +} + #ifdef CONFIG_RCU_BOOST /* @@ -1151,15 +1170,6 @@ static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags) } } -/* - * Is the current CPU running the RCU-callbacks kthread? - * Caller must have preemption disabled. - */ -static bool rcu_is_callbacks_kthread(void) -{ - return __this_cpu_read(rcu_data.rcu_cpu_kthread_task) == current; -} - #define RCU_BOOST_DELAY_JIFFIES DIV_ROUND_UP(CONFIG_RCU_BOOST_DELAY * HZ, 1000) /* @@ -1242,11 +1252,6 @@ static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags) raw_spin_unlock_irqrestore_rcu_node(rnp, flags); } -static bool rcu_is_callbacks_kthread(void) -{ - return false; -} - static void rcu_preempt_boost_start_gp(struct rcu_node *rnp) { } From 8f489b4da5278fc6e5fc8f0029ae7fb51c060215 Mon Sep 17 00:00:00 2001 From: "Uladzislau Rezki (Sony)" Date: Wed, 11 May 2022 10:57:03 +0200 Subject: [PATCH 274/318] rcu/nocb: Add option to opt rcuo kthreads out of RT priority This commit introduces a RCU_NOCB_CPU_CB_BOOST Kconfig option that prevents rcuo kthreads from running at real-time priority, even in kernels built with RCU_BOOST. This capability is important to devices needing low-latency (as in a few milliseconds) response from expedited RCU grace periods, but which are not running a classic real-time workload. On such devices, permitting the rcuo kthreads to run at real-time priority results in unacceptable latencies imposed on the application tasks, which run as SCHED_OTHER. See for example the following trace output: <...>-60 [006] d..1 2979.028717: rcu_batch_start: rcu_preempt CBs=34619 bl=270 If that rcuop kthread were permitted to run at real-time SCHED_FIFO priority, it would monopolize its CPU for hundreds of milliseconds while invoking those 34619 RCU callback functions, which would cause an unacceptably long latency spike for many application stacks on Android platforms. However, some existing real-time workloads require that callback invocation run at SCHED_FIFO priority, for example, those running on systems with heavy SCHED_OTHER background loads. (It is the real-time system's administrator's responsibility to make sure that important real-time tasks run at a higher priority than do RCU's kthreads.) Therefore, this new RCU_NOCB_CPU_CB_BOOST Kconfig option defaults to "y" on kernels built with PREEMPT_RT and defaults to "n" otherwise. The effect is to preserve current behavior for real-time systems, but for other systems to allow expedited RCU grace periods to run with real-time priority while continuing to invoke RCU callbacks as SCHED_OTHER. As you would expect, this RCU_NOCB_CPU_CB_BOOST Kconfig option has no effect except on CPUs with offloaded RCU callbacks. Signed-off-by: Uladzislau Rezki (Sony) Signed-off-by: Paul E. McKenney Acked-by: Joel Fernandes (Google) Reviewed-by: Neeraj Upadhyay --- kernel/rcu/Kconfig | 16 ++++++++++++++++ kernel/rcu/tree.c | 6 +++++- kernel/rcu/tree_nocb.h | 3 ++- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/Kconfig b/kernel/rcu/Kconfig index 27aab870ae4c..c05ca52cdf64 100644 --- a/kernel/rcu/Kconfig +++ b/kernel/rcu/Kconfig @@ -275,6 +275,22 @@ config RCU_NOCB_CPU_DEFAULT_ALL Say Y here if you want offload all CPUs by default on boot. Say N here if you are unsure. +config RCU_NOCB_CPU_CB_BOOST + bool "Offload RCU callback from real-time kthread" + depends on RCU_NOCB_CPU && RCU_BOOST + default y if PREEMPT_RT + help + Use this option to invoke offloaded callbacks as SCHED_FIFO + to avoid starvation by heavy SCHED_OTHER background load. + Of course, running as SCHED_FIFO during callback floods will + cause the rcuo[ps] kthreads to monopolize the CPU for hundreds + of milliseconds or more. Therefore, when enabling this option, + it is your responsibility to ensure that latency-sensitive + tasks either run with higher priority or run on some other CPU. + + Say Y here if you want to set RT priority for offloading kthreads. + Say N here if you are building a !PREEMPT_RT kernel and are unsure. + config TASKS_TRACE_RCU_READ_MB bool "Tasks Trace RCU readers use memory barriers in user and idle" depends on RCU_EXPERT && TASKS_TRACE_RCU diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 74455671e6cf..3b9f45ebb499 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -154,7 +154,11 @@ static void sync_sched_exp_online_cleanup(int cpu); static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp); static bool rcu_rdp_is_offloaded(struct rcu_data *rdp); -/* rcuc/rcub/rcuop kthread realtime priority */ +/* + * rcuc/rcub/rcuop kthread realtime priority. The "rcuop" + * real-time priority(enabling/disabling) is controlled by + * the extra CONFIG_RCU_NOCB_CPU_CB_BOOST configuration. + */ static int kthread_prio = IS_ENABLED(CONFIG_RCU_BOOST) ? 1 : 0; module_param(kthread_prio, int, 0444); diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index 60cc92cc6655..fa8e4f82e60c 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -1315,8 +1315,9 @@ static void rcu_spawn_cpu_nocb_kthread(int cpu) if (WARN_ONCE(IS_ERR(t), "%s: Could not start rcuo CB kthread, OOM is now expected behavior\n", __func__)) goto end; - if (kthread_prio) + if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_CB_BOOST) && kthread_prio) sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); + WRITE_ONCE(rdp->nocb_cb_kthread, t); WRITE_ONCE(rdp->nocb_gp_kthread, rdp_gp->nocb_gp_kthread); return; From 0578e14c945b1739e15c0e993280151fa5b99ca2 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Sat, 11 Jun 2022 19:00:44 +0800 Subject: [PATCH 275/318] rcu/nocb: Avoid polling when my_rdp->nocb_head_rdp list is empty Currently, if the 'rcu_nocb_poll' kernel boot parameter is enabled, all rcuog kthreads enter polling mode. However, if all of a given group of rcuo kthreads correspond to CPUs that have been de-offloaded, the corresponding rcuog kthread will nonetheless still wake up periodically, unnecessarily consuming power and perturbing workloads. Fortunately, this situation is easily detected by the fact that the rcuog kthread's CPU's rcu_data structure's ->nocb_head_rdp list is empty. This commit saves power and avoids unnecessarily perturbing workloads by putting an rcuog kthread to sleep during any time period when all of its rcuo kthreads' CPUs are de-offloaded. Co-developed-by: Frederic Weisbecker Signed-off-by: Frederic Weisbecker Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney Reviewed-by: Neeraj Upadhyay --- kernel/rcu/tree_nocb.h | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index fa8e4f82e60c..a8f574d8850d 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -584,6 +584,14 @@ static int nocb_gp_toggle_rdp(struct rcu_data *rdp, return ret; } +static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu) +{ + trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Sleep")); + swait_event_interruptible_exclusive(my_rdp->nocb_gp_wq, + !READ_ONCE(my_rdp->nocb_gp_sleep)); + trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("EndSleep")); +} + /* * No-CBs GP kthreads come here to wait for additional callbacks to show up * or for grace periods to end. @@ -701,13 +709,19 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) /* Polling, so trace if first poll in the series. */ if (gotcbs) trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Poll")); - schedule_timeout_idle(1); + if (list_empty(&my_rdp->nocb_head_rdp)) { + raw_spin_lock_irqsave(&my_rdp->nocb_gp_lock, flags); + if (!my_rdp->nocb_toggling_rdp) + WRITE_ONCE(my_rdp->nocb_gp_sleep, true); + raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags); + /* Wait for any offloading rdp */ + nocb_gp_sleep(my_rdp, cpu); + } else { + schedule_timeout_idle(1); + } } else if (!needwait_gp) { /* Wait for callbacks to appear. */ - trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Sleep")); - swait_event_interruptible_exclusive(my_rdp->nocb_gp_wq, - !READ_ONCE(my_rdp->nocb_gp_sleep)); - trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("EndSleep")); + nocb_gp_sleep(my_rdp, cpu); } else { rnp = my_rdp->mynode; trace_rcu_this_gp(rnp, my_rdp, wait_gp_seq, TPS("StartWait")); From ed221835a7ae8d0c5d9967d70518d0f230b8e9c5 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 19 Jul 2022 03:04:15 +0000 Subject: [PATCH 276/318] crypto: lib - add module license to libsha1 libsha1 can be a module, so it needs a MODULE_LICENSE. Fixes: ec8f7f4821d5 ("crypto: lib - make the sha1 library optional") Reported-by: Randy Dunlap Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- lib/crypto/sha1.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/crypto/sha1.c b/lib/crypto/sha1.c index 0494766fc574..1aebe7be9401 100644 --- a/lib/crypto/sha1.c +++ b/lib/crypto/sha1.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -135,3 +136,5 @@ void sha1_init(__u32 *buf) buf[4] = 0xc3d2e1f0; } EXPORT_SYMBOL(sha1_init); + +MODULE_LICENSE("GPL"); From 40b1aa5973e48e1055161990dc9079fd3bea5dad Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Wed, 20 Jul 2022 14:31:31 +0200 Subject: [PATCH 277/318] docs: ABI: correct QEMU fw_cfg spec path A few weeks ago, QEMU switched docs/specs/fw_cfg.txt to be docs/specs/fw_cfg.rst, so update the reference in the kernel docs to reflect this. Also add a link to the online documentation to make it easier to find. Cc: Simon Sapin Cc: Gabriel Somlo Signed-off-by: Jason A. Donenfeld Link: https://lore.kernel.org/r/20220720123131.341755-1-Jason@zx2c4.com Signed-off-by: Jonathan Corbet --- Documentation/ABI/testing/sysfs-firmware-qemu_fw_cfg | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-firmware-qemu_fw_cfg b/Documentation/ABI/testing/sysfs-firmware-qemu_fw_cfg index ee0d6dbc810e..54d1bfd0db12 100644 --- a/Documentation/ABI/testing/sysfs-firmware-qemu_fw_cfg +++ b/Documentation/ABI/testing/sysfs-firmware-qemu_fw_cfg @@ -12,8 +12,9 @@ Description: configuration data to the guest userspace. The authoritative guest-side hardware interface documentation - to the fw_cfg device can be found in "docs/specs/fw_cfg.txt" - in the QEMU source tree. + to the fw_cfg device can be found in "docs/specs/fw_cfg.rst" + in the QEMU source tree, or online at: + https://qemu-project.gitlab.io/qemu/specs/fw_cfg.html **SysFS fw_cfg Interface** From cbf4adfd4d19c39bc929359a04a466166f7b7d1e Mon Sep 17 00:00:00 2001 From: Sotir Danailov Date: Fri, 15 Jul 2022 23:13:07 +0200 Subject: [PATCH 278/318] Documentation: process: Update email client instructions for Thunderbird The instructions don't match with the current Thunderbird interface. Clarification on using external extensions. New information on how to avoid writing HTML emails. Tell user to restart Thunderbird after modifications. Tested-by: Randy Dunlap Acked-by: Randy Dunlap Signed-off-by: Sotir Danailov Link: https://lore.kernel.org/r/20220715211307.9358-1-sndanailov@gmail.com Signed-off-by: Jonathan Corbet --- Documentation/process/email-clients.rst | 69 +++++++++++++++++-------- 1 file changed, 47 insertions(+), 22 deletions(-) diff --git a/Documentation/process/email-clients.rst b/Documentation/process/email-clients.rst index 16586f6cc888..fc2c46f3f82d 100644 --- a/Documentation/process/email-clients.rst +++ b/Documentation/process/email-clients.rst @@ -277,36 +277,61 @@ Thunderbird (GUI) Thunderbird is an Outlook clone that likes to mangle text, but there are ways to coerce it into behaving. -- Allow use of an external editor: - The easiest thing to do with Thunderbird and patches is to use an - "external editor" extension and then just use your favorite ``$EDITOR`` - for reading/merging patches into the body text. To do this, download - and install the extension, then add a button for it using - :menuselection:`View-->Toolbars-->Customize...` and finally just click on it - when in the :menuselection:`Compose` dialog. +After doing the modifications, this includes installing the extensions, +you need to restart Thunderbird. - Please note that "external editor" requires that your editor must not - fork, or in other words, the editor must not return before closing. - You may have to pass additional flags or change the settings of your - editor. Most notably if you are using gvim then you must pass the -f - option to gvim by putting ``/usr/bin/gvim -f`` (if the binary is in - ``/usr/bin``) to the text editor field in :menuselection:`external editor` - settings. If you are using some other editor then please read its manual - to find out how to do this. +- Allow use of an external editor: + + The easiest thing to do with Thunderbird and patches is to use extensions + which open your favorite external editor. + + Here are some example extensions which are capable of doing this. + + - "External Editor Revived" + + https://github.com/Frederick888/external-editor-revived + + https://addons.thunderbird.net/en-GB/thunderbird/addon/external-editor-revived/ + + It requires installing a "native messaging host". + Please read the wiki which can be found here: + https://github.com/Frederick888/external-editor-revived/wiki + + - "External Editor" + + https://github.com/exteditor/exteditor + + To do this, download and install the extension, then open the + :menuselection:`compose` window, add a button for it using + :menuselection:`View-->Toolbars-->Customize...` + then just click on the new button when you wish to use the external editor. + + Please note that "External Editor" requires that your editor must not + fork, or in other words, the editor must not return before closing. + You may have to pass additional flags or change the settings of your + editor. Most notably if you are using gvim then you must pass the -f + option to gvim by putting ``/usr/bin/gvim --nofork"`` (if the binary is in + ``/usr/bin``) to the text editor field in :menuselection:`external editor` + settings. If you are using some other editor then please read its manual + to find out how to do this. To beat some sense out of the internal editor, do this: -- Edit your Thunderbird config settings so that it won't use ``format=flowed``. - Go to :menuselection:`edit-->preferences-->advanced-->config editor` to bring up - the thunderbird's registry editor. +- Edit your Thunderbird config settings so that it won't use ``format=flowed``! + Go to your main window and find the button for your main dropdown menu. + :menuselection:`Main Menu-->Preferences-->General-->Config Editor...` + to bring up the thunderbird's registry editor. -- Set ``mailnews.send_plaintext_flowed`` to ``false`` + - Set ``mailnews.send_plaintext_flowed`` to ``false`` -- Set ``mailnews.wraplength`` from ``72`` to ``0`` + - Set ``mailnews.wraplength`` from ``72`` to ``0`` -- :menuselection:`View-->Message Body As-->Plain Text` +- Don't write HTML messages! Go to the main window + :menuselection:`Main Menu-->Account Settings-->youracc@server.something-->Composition & Addressing`! + There you can disable the option "Compose messages in HTML format". -- :menuselection:`View-->Character Encoding-->Unicode (UTF-8)` +- Open messages only as plain text! Go to the main window + :menuselection:`Main Menu-->View-->Message Body As-->Plain Text`! TkRat (GUI) *********** From 5a491c9714d00ce81df18c3832c8602e1b137f5b Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Wed, 20 Jul 2022 15:23:13 -0600 Subject: [PATCH 279/318] docs: Remove spurious tag from admin-guide/mm/overcommit-accounting.rst This tag is not used anywhere and clutters the file; just remove it. Signed-off-by: Jonathan Corbet --- Documentation/vm/overcommit-accounting.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/Documentation/vm/overcommit-accounting.rst b/Documentation/vm/overcommit-accounting.rst index 1addb0c374a4..a4895d6fc1c2 100644 --- a/Documentation/vm/overcommit-accounting.rst +++ b/Documentation/vm/overcommit-accounting.rst @@ -1,5 +1,3 @@ -.. _overcommit_accounting: - ===================== Overcommit Accounting ===================== From bf95b2bc3e42f11f4d7a5e8a98376c2b4a2aa82f Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 13 Apr 2022 17:46:15 -0700 Subject: [PATCH 280/318] rcu: Switch polled grace-period APIs to ->gp_seq_polled This commit switches the existing polled grace-period APIs to use a new ->gp_seq_polled counter in the rcu_state structure. An additional ->gp_seq_polled_snap counter in that same structure allows the normal grace period kthread to interact properly with the !SMP !PREEMPT fastpath through synchronize_rcu(). The first of the two to note the end of a given grace period will make knowledge of this transition available to the polled API. This commit is in preparation for polled expedited grace periods. [ paulmck: Fix use of rcu_state.gp_seq_polled to start normal grace period. ] Link: https://lore.kernel.org/all/20220121142454.1994916-1-bfoster@redhat.com/ Link: https://docs.google.com/document/d/1RNKWW9jQyfjxw2E8dsXVTdvZYh0HnYeSHDKog9jhdN8/edit?usp=sharing Cc: Brian Foster Cc: Dave Chinner Cc: Al Viro Cc: Ian Kent Co-developed-by: Boqun Feng Signed-off-by: Boqun Feng Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 98 +++++++++++++++++++++++++++++++++++++++++++++-- kernel/rcu/tree.h | 2 + 2 files changed, 96 insertions(+), 4 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 46cfceea8784..b40a5a19ddd2 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -1775,6 +1775,78 @@ static void rcu_strict_gp_boundary(void *unused) invoke_rcu_core(); } +// Has rcu_init() been invoked? This is used (for example) to determine +// whether spinlocks may be acquired safely. +static bool rcu_init_invoked(void) +{ + return !!rcu_state.n_online_cpus; +} + +// Make the polled API aware of the beginning of a grace period. +static void rcu_poll_gp_seq_start(unsigned long *snap) +{ + struct rcu_node *rnp = rcu_get_root(); + + if (rcu_init_invoked()) + raw_lockdep_assert_held_rcu_node(rnp); + + // If RCU was idle, note beginning of GP. + if (!rcu_seq_state(rcu_state.gp_seq_polled)) + rcu_seq_start(&rcu_state.gp_seq_polled); + + // Either way, record current state. + *snap = rcu_state.gp_seq_polled; +} + +// Make the polled API aware of the end of a grace period. +static void rcu_poll_gp_seq_end(unsigned long *snap) +{ + struct rcu_node *rnp = rcu_get_root(); + + if (rcu_init_invoked()) + raw_lockdep_assert_held_rcu_node(rnp); + + // If the previously noted GP is still in effect, record the + // end of that GP. Either way, zero counter to avoid counter-wrap + // problems. + if (*snap && *snap == rcu_state.gp_seq_polled) { + rcu_seq_end(&rcu_state.gp_seq_polled); + rcu_state.gp_seq_polled_snap = 0; + } else { + *snap = 0; + } +} + +// Make the polled API aware of the beginning of a grace period, but +// where caller does not hold the root rcu_node structure's lock. +static void rcu_poll_gp_seq_start_unlocked(unsigned long *snap) +{ + struct rcu_node *rnp = rcu_get_root(); + + if (rcu_init_invoked()) { + lockdep_assert_irqs_enabled(); + raw_spin_lock_irq_rcu_node(rnp); + } + rcu_poll_gp_seq_start(snap); + if (rcu_init_invoked()) + raw_spin_unlock_irq_rcu_node(rnp); +} + +// Make the polled API aware of the end of a grace period, but where +// caller does not hold the root rcu_node structure's lock. +static void rcu_poll_gp_seq_end_unlocked(unsigned long *snap) +{ + struct rcu_node *rnp = rcu_get_root(); + + if (rcu_init_invoked()) { + lockdep_assert_irqs_enabled(); + raw_spin_lock_irq_rcu_node(rnp); + } + rcu_poll_gp_seq_end(snap); + if (rcu_init_invoked()) + raw_spin_unlock_irq_rcu_node(rnp); +} + /* * Initialize a new grace period. Return false if no grace period required. */ @@ -1810,6 +1882,7 @@ static noinline_for_stack bool rcu_gp_init(void) rcu_seq_start(&rcu_state.gp_seq); ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq); trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("start")); + rcu_poll_gp_seq_start(&rcu_state.gp_seq_polled_snap); raw_spin_unlock_irq_rcu_node(rnp); /* @@ -2069,6 +2142,7 @@ static noinline void rcu_gp_cleanup(void) * safe for us to drop the lock in order to mark the grace * period as completed in all of the rcu_node structures. */ + rcu_poll_gp_seq_end(&rcu_state.gp_seq_polled_snap); raw_spin_unlock_irq_rcu_node(rnp); /* @@ -3837,8 +3911,18 @@ void synchronize_rcu(void) lock_is_held(&rcu_lock_map) || lock_is_held(&rcu_sched_lock_map), "Illegal synchronize_rcu() in RCU read-side critical section"); - if (rcu_blocking_is_gp()) + if (rcu_blocking_is_gp()) { + // Note well that this code runs with !PREEMPT && !SMP. + // In addition, all code that advances grace periods runs + // at process level. Therefore, this GP overlaps with other + // GPs only by being fully nested within them, which allows + // reuse of ->gp_seq_polled_snap. + rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_snap); + rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_snap); + if (rcu_init_invoked()) + cond_resched_tasks_rcu_qs(); return; // Context allows vacuous grace periods. + } if (rcu_gp_is_expedited()) synchronize_rcu_expedited(); else @@ -3860,7 +3944,7 @@ unsigned long get_state_synchronize_rcu(void) * before the load from ->gp_seq. */ smp_mb(); /* ^^^ */ - return rcu_seq_snap(&rcu_state.gp_seq); + return rcu_seq_snap(&rcu_state.gp_seq_polled); } EXPORT_SYMBOL_GPL(get_state_synchronize_rcu); @@ -3889,7 +3973,13 @@ unsigned long start_poll_synchronize_rcu(void) rdp = this_cpu_ptr(&rcu_data); rnp = rdp->mynode; raw_spin_lock_rcu_node(rnp); // irqs already disabled. - needwake = rcu_start_this_gp(rnp, rdp, gp_seq); + // Note it is possible for a grace period to have elapsed between + // the above call to get_state_synchronize_rcu() and the below call + // to rcu_seq_snap. This is OK, the worst that happens is that we + // get a grace period that no one needed. These accesses are ordered + // by smp_mb(), and we are accessing them in the opposite order + // from which they are updated at grace-period start, as required. + needwake = rcu_start_this_gp(rnp, rdp, rcu_seq_snap(&rcu_state.gp_seq)); raw_spin_unlock_irqrestore_rcu_node(rnp, flags); if (needwake) rcu_gp_kthread_wake(); @@ -3925,7 +4015,7 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu); bool poll_state_synchronize_rcu(unsigned long oldstate) { if (oldstate == RCU_GET_STATE_COMPLETED || - rcu_seq_done_exact(&rcu_state.gp_seq, oldstate)) { + rcu_seq_done_exact(&rcu_state.gp_seq_polled, oldstate)) { smp_mb(); /* Ensure GP ends before subsequent accesses. */ return true; } diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 2ccf5845957d..9c853033f159 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -323,6 +323,8 @@ struct rcu_state { short gp_state; /* GP kthread sleep state. */ unsigned long gp_wake_time; /* Last GP kthread wake. */ unsigned long gp_wake_seq; /* ->gp_seq at ^^^. */ + unsigned long gp_seq_polled; /* GP seq for polled API. */ + unsigned long gp_seq_polled_snap; /* ->gp_seq_polled at normal GP start. */ /* End of fields guarded by root rcu_node's lock. */ From dd04140531b5d38b77ad9ff7b18117654be5bf5c Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 14 Apr 2022 06:56:35 -0700 Subject: [PATCH 281/318] rcu: Make polled grace-period API account for expedited grace periods Currently, this code could splat: oldstate = get_state_synchronize_rcu(); synchronize_rcu_expedited(); WARN_ON_ONCE(!poll_state_synchronize_rcu(oldstate)); This situation is counter-intuitive and user-unfriendly. After all, there really was a perfectly valid full grace period right after the call to get_state_synchronize_rcu(), so why shouldn't poll_state_synchronize_rcu() know about it? This commit therefore makes the polled grace-period API aware of expedited grace periods in addition to the normal grace periods that it is already aware of. With this change, the above code is guaranteed not to splat. Please note that the above code can still splat due to counter wrap on the one hand and situations involving partially overlapping normal/expedited grace periods on the other. On 64-bit systems, the second is of course much more likely than the first. It is possible to modify this approach to prevent overlapping grace periods from causing splats, but only at the expense of greatly increasing the probability of counter wrap, as in within milliseconds on 32-bit systems and within minutes on 64-bit systems. This commit is in preparation for polled expedited grace periods. Link: https://lore.kernel.org/all/20220121142454.1994916-1-bfoster@redhat.com/ Link: https://docs.google.com/document/d/1RNKWW9jQyfjxw2E8dsXVTdvZYh0HnYeSHDKog9jhdN8/edit?usp=sharing Cc: Brian Foster Cc: Dave Chinner Cc: Al Viro Cc: Ian Kent Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 9 +++++---- kernel/rcu/tree.h | 1 + kernel/rcu/tree_exp.h | 16 ++++++++++++++-- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index b40a5a19ddd2..1505b02b4e53 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -1812,6 +1812,7 @@ static void rcu_poll_gp_seq_end(unsigned long *snap) if (*snap && *snap == rcu_state.gp_seq_polled) { rcu_seq_end(&rcu_state.gp_seq_polled); rcu_state.gp_seq_polled_snap = 0; + rcu_state.gp_seq_polled_exp_snap = 0; } else { *snap = 0; } @@ -3913,10 +3914,10 @@ void synchronize_rcu(void) "Illegal synchronize_rcu() in RCU read-side critical section"); if (rcu_blocking_is_gp()) { // Note well that this code runs with !PREEMPT && !SMP. - // In addition, all code that advances grace periods runs - // at process level. Therefore, this GP overlaps with other - // GPs only by being fully nested within them, which allows - // reuse of ->gp_seq_polled_snap. + // In addition, all code that advances grace periods runs at + // process level. Therefore, this normal GP overlaps with + // other normal GPs only by being fully nested within them, + // which allows reuse of ->gp_seq_polled_snap. rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_snap); rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_snap); if (rcu_init_invoked()) diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 9c853033f159..5634e76106c4 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -325,6 +325,7 @@ struct rcu_state { unsigned long gp_wake_seq; /* ->gp_seq at ^^^. */ unsigned long gp_seq_polled; /* GP seq for polled API. */ unsigned long gp_seq_polled_snap; /* ->gp_seq_polled at normal GP start. */ + unsigned long gp_seq_polled_exp_snap; /* ->gp_seq_polled at expedited GP start. */ /* End of fields guarded by root rcu_node's lock. */ diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index 0f70f62039a9..e0258066b881 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -18,6 +18,7 @@ static int rcu_print_task_exp_stall(struct rcu_node *rnp); static void rcu_exp_gp_seq_start(void) { rcu_seq_start(&rcu_state.expedited_sequence); + rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_exp_snap); } /* @@ -34,6 +35,7 @@ static __maybe_unused unsigned long rcu_exp_gp_seq_endval(void) */ static void rcu_exp_gp_seq_end(void) { + rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_exp_snap); rcu_seq_end(&rcu_state.expedited_sequence); smp_mb(); /* Ensure that consecutive grace periods serialize. */ } @@ -913,8 +915,18 @@ void synchronize_rcu_expedited(void) "Illegal synchronize_rcu_expedited() in RCU read-side critical section"); /* Is the state is such that the call is a grace period? */ - if (rcu_blocking_is_gp()) - return; + if (rcu_blocking_is_gp()) { + // Note well that this code runs with !PREEMPT && !SMP. + // In addition, all code that advances grace periods runs + // at process level. Therefore, this expedited GP overlaps + // with other expedited GPs only by being fully nested within + // them, which allows reuse of ->gp_seq_polled_exp_snap. + rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_exp_snap); + rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_exp_snap); + if (rcu_init_invoked()) + cond_resched(); + return; // Context allows vacuous grace periods. + } /* If expedited grace periods are prohibited, fall back to normal. */ if (rcu_gp_is_normal()) { From 7f4535366f8f77b3ddbc79d4ba82df966c5c2aab Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 14 Apr 2022 11:49:58 -0700 Subject: [PATCH 282/318] rcu: Make Tiny RCU grace periods visible to polled APIs This commit makes the Tiny RCU implementation of synchronize_rcu() increment the rcu_ctrlblk.gp_seq counter, thus making both synchronize_rcu() and synchronize_rcu_expedited() visible to get_state_synchronize_rcu() and friends. Link: https://lore.kernel.org/all/20220121142454.1994916-1-bfoster@redhat.com/ Link: https://docs.google.com/document/d/1RNKWW9jQyfjxw2E8dsXVTdvZYh0HnYeSHDKog9jhdN8/edit?usp=sharing Cc: Brian Foster Cc: Dave Chinner Cc: Al Viro Cc: Ian Kent Signed-off-by: Paul E. McKenney --- kernel/rcu/tiny.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/tiny.c b/kernel/rcu/tiny.c index dbee6bea6726..60071817d939 100644 --- a/kernel/rcu/tiny.c +++ b/kernel/rcu/tiny.c @@ -139,8 +139,10 @@ static __latent_entropy void rcu_process_callbacks(struct softirq_action *unused /* * Wait for a grace period to elapse. But it is illegal to invoke * synchronize_rcu() from within an RCU read-side critical section. - * Therefore, any legal call to synchronize_rcu() is a quiescent - * state, and so on a UP system, synchronize_rcu() need do nothing. + * Therefore, any legal call to synchronize_rcu() is a quiescent state, + * and so on a UP system, synchronize_rcu() need do nothing, other than + * let the polled APIs know that another grace period elapsed. + * * (But Lai Jiangshan points out the benefits of doing might_sleep() * to reduce latency.) * @@ -152,6 +154,7 @@ void synchronize_rcu(void) lock_is_held(&rcu_lock_map) || lock_is_held(&rcu_sched_lock_map), "Illegal synchronize_rcu() in RCU read-side critical section"); + WRITE_ONCE(rcu_ctrlblk.gp_seq, rcu_ctrlblk.gp_seq + 2); } EXPORT_SYMBOL_GPL(synchronize_rcu); From e4333cb20f047d96485a9416a93ae4b2ec3b27dd Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 14 Apr 2022 09:09:11 -0700 Subject: [PATCH 283/318] rcutorture: Verify that polled GP API sees synchronous grace periods This commit causes rcu_torture_writer() to use WARN_ON_ONCE() to check that the cookie returned by the current RCU flavor's ->get_gp_state() function (get_state_synchronize_rcu() for vanilla RCU) causes that flavor's ->poll_gp_state function (poll_state_synchronize_rcu() for vanilla RCU) to unconditionally return true. Note that a pair calls to synchronous grace-period-wait functions are used. This is necessary to account for partially overlapping normal and expedited grace periods aligning in just the wrong way with polled API invocations, which can cause those polled API invocations to ignore one or the other of those partially overlapping grace periods. It is unlikely that this sort of ignored grace period will be a problem in production, but rcutorture can make it happen quite within a few tens of seconds. This commit is in preparation for polled expedited grace periods. [ paulmck: Apply feedback from Frederic Weisbecker. ] Link: https://lore.kernel.org/all/20220121142454.1994916-1-bfoster@redhat.com/ Link: https://docs.google.com/document/d/1RNKWW9jQyfjxw2E8dsXVTdvZYh0HnYeSHDKog9jhdN8/edit?usp=sharing Cc: Brian Foster Cc: Dave Chinner Cc: Al Viro Cc: Ian Kent Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 4ceec9f4169c..d2edc763bb92 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -1269,7 +1269,12 @@ rcu_torture_writer(void *arg) break; case RTWS_EXP_SYNC: rcu_torture_writer_state = RTWS_EXP_SYNC; + if (cur_ops->get_gp_state && cur_ops->poll_gp_state) + cookie = cur_ops->get_gp_state(); cur_ops->exp_sync(); + cur_ops->exp_sync(); + if (cur_ops->get_gp_state && cur_ops->poll_gp_state) + WARN_ON_ONCE(!cur_ops->poll_gp_state(cookie)); rcu_torture_pipe_update(old_rp); break; case RTWS_COND_GET: @@ -1291,7 +1296,12 @@ rcu_torture_writer(void *arg) break; case RTWS_SYNC: rcu_torture_writer_state = RTWS_SYNC; + if (cur_ops->get_gp_state && cur_ops->poll_gp_state) + cookie = cur_ops->get_gp_state(); cur_ops->sync(); + cur_ops->sync(); + if (cur_ops->get_gp_state && cur_ops->poll_gp_state) + WARN_ON_ONCE(!cur_ops->poll_gp_state(cookie)); rcu_torture_pipe_update(old_rp); break; default: From d96c52fe4907c68adc5e61a0bef7aec0933223d5 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 15 Apr 2022 10:55:42 -0700 Subject: [PATCH 284/318] rcu: Add polled expedited grace-period primitives This commit adds expedited grace-period functionality to RCU's polled grace-period API, adding start_poll_synchronize_rcu_expedited() and cond_synchronize_rcu_expedited(), which are similar to the existing start_poll_synchronize_rcu() and cond_synchronize_rcu() functions, respectively. Note that although start_poll_synchronize_rcu_expedited() can be invoked very early, the resulting expedited grace periods are not guaranteed to start until after workqueues are fully initialized. On the other hand, both synchronize_rcu() and synchronize_rcu_expedited() can also be invoked very early, and the resulting grace periods will be taken into account as they occur. [ paulmck: Apply feedback from Neeraj Upadhyay. ] Link: https://lore.kernel.org/all/20220121142454.1994916-1-bfoster@redhat.com/ Link: https://docs.google.com/document/d/1RNKWW9jQyfjxw2E8dsXVTdvZYh0HnYeSHDKog9jhdN8/edit?usp=sharing Cc: Brian Foster Cc: Dave Chinner Cc: Al Viro Cc: Ian Kent Signed-off-by: Paul E. McKenney --- include/linux/rcutiny.h | 10 +++++ include/linux/rcutree.h | 2 + kernel/rcu/tree.c | 17 ++++++--- kernel/rcu/tree.h | 7 ++++ kernel/rcu/tree_exp.h | 85 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 116 insertions(+), 5 deletions(-) diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h index 5fed476f977f..ab7e20dfb07b 100644 --- a/include/linux/rcutiny.h +++ b/include/linux/rcutiny.h @@ -23,6 +23,16 @@ static inline void cond_synchronize_rcu(unsigned long oldstate) might_sleep(); } +static inline unsigned long start_poll_synchronize_rcu_expedited(void) +{ + return start_poll_synchronize_rcu(); +} + +static inline void cond_synchronize_rcu_expedited(unsigned long oldstate) +{ + cond_synchronize_rcu(oldstate); +} + extern void rcu_barrier(void); static inline void synchronize_rcu_expedited(void) diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index 9c6cfb742504..20dbaa9a3882 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h @@ -40,6 +40,8 @@ bool rcu_eqs_special_set(int cpu); void rcu_momentary_dyntick_idle(void); void kfree_rcu_scheduler_running(void); bool rcu_gp_might_be_stalled(void); +unsigned long start_poll_synchronize_rcu_expedited(void); +void cond_synchronize_rcu_expedited(unsigned long oldstate); unsigned long get_state_synchronize_rcu(void); unsigned long start_poll_synchronize_rcu(void); bool poll_state_synchronize_rcu(unsigned long oldstate); diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 1505b02b4e53..6cf5b51622cd 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -4027,20 +4027,20 @@ EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu); /** * cond_synchronize_rcu - Conditionally wait for an RCU grace period * - * @oldstate: value from get_state_synchronize_rcu() or start_poll_synchronize_rcu() + * @oldstate: value from get_state_synchronize_rcu(), start_poll_synchronize_rcu(), or start_poll_synchronize_rcu_expedited() * * If a full RCU grace period has elapsed since the earlier call to * get_state_synchronize_rcu() or start_poll_synchronize_rcu(), just return. * Otherwise, invoke synchronize_rcu() to wait for a full grace period. * - * Yes, this function does not take counter wrap into account. But - * counter wrap is harmless. If the counter wraps, we have waited for + * Yes, this function does not take counter wrap into account. + * But counter wrap is harmless. If the counter wraps, we have waited for * more than 2 billion grace periods (and way more on a 64-bit system!), - * so waiting for one additional grace period should be just fine. + * so waiting for a couple of additional grace periods should be just fine. * * This function provides the same memory-ordering guarantees that * would be provided by a synchronize_rcu() that was invoked at the call - * to the function that provided @oldstate, and that returned at the end + * to the function that provided @oldstate and that returned at the end * of this function. */ void cond_synchronize_rcu(unsigned long oldstate) @@ -4793,6 +4793,9 @@ static void __init rcu_init_one(void) init_waitqueue_head(&rnp->exp_wq[3]); spin_lock_init(&rnp->exp_lock); mutex_init(&rnp->boost_kthread_mutex); + raw_spin_lock_init(&rnp->exp_poll_lock); + rnp->exp_seq_poll_rq = RCU_GET_STATE_COMPLETED; + INIT_WORK(&rnp->exp_poll_wq, sync_rcu_do_polled_gp); } } @@ -5018,6 +5021,10 @@ void __init rcu_init(void) qovld_calc = DEFAULT_RCU_QOVLD_MULT * qhimark; else qovld_calc = qovld; + + // Kick-start any polled grace periods that started early. + if (!(per_cpu_ptr(&rcu_data, cpu)->mynode->exp_seq_poll_rq & 0x1)) + (void)start_poll_synchronize_rcu_expedited(); } #include "tree_stall.h" diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 5634e76106c4..fb77deca5f5c 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -133,6 +133,10 @@ struct rcu_node { wait_queue_head_t exp_wq[4]; struct rcu_exp_work rew; bool exp_need_flush; /* Need to flush workitem? */ + raw_spinlock_t exp_poll_lock; + /* Lock and data for polled expedited grace periods. */ + unsigned long exp_seq_poll_rq; + struct work_struct exp_poll_wq; } ____cacheline_internodealigned_in_smp; /* @@ -484,3 +488,6 @@ static void rcu_iw_handler(struct irq_work *iwp); static void check_cpu_stall(struct rcu_data *rdp); static void rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp, const unsigned long gpssdelay); + +/* Forward declarations for tree_exp.h. */ +static void sync_rcu_do_polled_gp(struct work_struct *wp); diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index e0258066b881..571b0a700cce 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -962,3 +962,88 @@ void synchronize_rcu_expedited(void) synchronize_rcu_expedited_destroy_work(&rew); } EXPORT_SYMBOL_GPL(synchronize_rcu_expedited); + +/* + * Ensure that start_poll_synchronize_rcu_expedited() has the expedited + * RCU grace periods that it needs. + */ +static void sync_rcu_do_polled_gp(struct work_struct *wp) +{ + unsigned long flags; + struct rcu_node *rnp = container_of(wp, struct rcu_node, exp_poll_wq); + unsigned long s; + + raw_spin_lock_irqsave(&rnp->exp_poll_lock, flags); + s = rnp->exp_seq_poll_rq; + rnp->exp_seq_poll_rq = RCU_GET_STATE_COMPLETED; + raw_spin_unlock_irqrestore(&rnp->exp_poll_lock, flags); + if (s == RCU_GET_STATE_COMPLETED) + return; + while (!poll_state_synchronize_rcu(s)) + synchronize_rcu_expedited(); + raw_spin_lock_irqsave(&rnp->exp_poll_lock, flags); + s = rnp->exp_seq_poll_rq; + if (poll_state_synchronize_rcu(s)) + rnp->exp_seq_poll_rq = RCU_GET_STATE_COMPLETED; + raw_spin_unlock_irqrestore(&rnp->exp_poll_lock, flags); +} + +/** + * start_poll_synchronize_rcu_expedited - Snapshot current RCU state and start expedited grace period + * + * Returns a cookie to pass to a call to cond_synchronize_rcu(), + * cond_synchronize_rcu_expedited(), or poll_state_synchronize_rcu(), + * allowing them to determine whether or not any sort of grace period has + * elapsed in the meantime. If the needed expedited grace period is not + * already slated to start, initiates that grace period. + */ +unsigned long start_poll_synchronize_rcu_expedited(void) +{ + unsigned long flags; + struct rcu_data *rdp; + struct rcu_node *rnp; + unsigned long s; + + s = get_state_synchronize_rcu(); + rdp = per_cpu_ptr(&rcu_data, raw_smp_processor_id()); + rnp = rdp->mynode; + if (rcu_init_invoked()) + raw_spin_lock_irqsave(&rnp->exp_poll_lock, flags); + if (!poll_state_synchronize_rcu(s)) { + rnp->exp_seq_poll_rq = s; + if (rcu_init_invoked()) + queue_work(rcu_gp_wq, &rnp->exp_poll_wq); + } + if (rcu_init_invoked()) + raw_spin_unlock_irqrestore(&rnp->exp_poll_lock, flags); + + return s; +} +EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu_expedited); + +/** + * cond_synchronize_rcu_expedited - Conditionally wait for an expedited RCU grace period + * + * @oldstate: value from get_state_synchronize_rcu(), start_poll_synchronize_rcu(), or start_poll_synchronize_rcu_expedited() + * + * If any type of full RCU grace period has elapsed since the earlier + * call to get_state_synchronize_rcu(), start_poll_synchronize_rcu(), + * or start_poll_synchronize_rcu_expedited(), just return. Otherwise, + * invoke synchronize_rcu_expedited() to wait for a full grace period. + * + * Yes, this function does not take counter wrap into account. + * But counter wrap is harmless. If the counter wraps, we have waited for + * more than 2 billion grace periods (and way more on a 64-bit system!), + * so waiting for a couple of additional grace periods should be just fine. + * + * This function provides the same memory-ordering guarantees that + * would be provided by a synchronize_rcu() that was invoked at the call + * to the function that provided @oldstate and that returned at the end + * of this function. + */ +void cond_synchronize_rcu_expedited(unsigned long oldstate) +{ + if (!poll_state_synchronize_rcu(oldstate)) + synchronize_rcu_expedited(); +} +EXPORT_SYMBOL_GPL(cond_synchronize_rcu_expedited); From 11d62f0f43a35d7c62aabc06a99cd4691a47ccb4 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 1 Feb 2022 07:01:20 -0800 Subject: [PATCH 285/318] rcutorture: Test polled expedited grace-period primitives This commit adds tests of start_poll_synchronize_rcu_expedited() and poll_state_synchronize_rcu_expedited(). Link: https://lore.kernel.org/all/20220121142454.1994916-1-bfoster@redhat.com/ Link: https://docs.google.com/document/d/1RNKWW9jQyfjxw2E8dsXVTdvZYh0HnYeSHDKog9jhdN8/edit?usp=sharing Cc: Brian Foster Cc: Dave Chinner Cc: Al Viro Cc: Ian Kent Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 87 +++++++++++++++++++++++++++++++++++------ 1 file changed, 74 insertions(+), 13 deletions(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index d2edc763bb92..0788ef2a4491 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -86,10 +86,12 @@ torture_param(int, fwd_progress_holdoff, 60, torture_param(bool, fwd_progress_need_resched, 1, "Hide cond_resched() behind need_resched()"); torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives"); +torture_param(bool, gp_cond_exp, false, "Use conditional/async expedited GP wait primitives"); torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); torture_param(bool, gp_normal, false, "Use normal (non-expedited) GP wait primitives"); torture_param(bool, gp_poll, false, "Use polling GP wait primitives"); +torture_param(bool, gp_poll_exp, false, "Use polling expedited GP wait primitives"); torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives"); torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers"); torture_param(int, leakpointer, 0, "Leak pointer dereferences from readers"); @@ -209,12 +211,16 @@ static int rcu_torture_writer_state; #define RTWS_DEF_FREE 3 #define RTWS_EXP_SYNC 4 #define RTWS_COND_GET 5 -#define RTWS_COND_SYNC 6 -#define RTWS_POLL_GET 7 -#define RTWS_POLL_WAIT 8 -#define RTWS_SYNC 9 -#define RTWS_STUTTER 10 -#define RTWS_STOPPING 11 +#define RTWS_COND_GET_EXP 6 +#define RTWS_COND_SYNC 7 +#define RTWS_COND_SYNC_EXP 8 +#define RTWS_POLL_GET 9 +#define RTWS_POLL_GET_EXP 10 +#define RTWS_POLL_WAIT 11 +#define RTWS_POLL_WAIT_EXP 12 +#define RTWS_SYNC 13 +#define RTWS_STUTTER 14 +#define RTWS_STOPPING 15 static const char * const rcu_torture_writer_state_names[] = { "RTWS_FIXED_DELAY", "RTWS_DELAY", @@ -222,9 +228,13 @@ static const char * const rcu_torture_writer_state_names[] = { "RTWS_DEF_FREE", "RTWS_EXP_SYNC", "RTWS_COND_GET", + "RTWS_COND_GET_EXP", "RTWS_COND_SYNC", + "RTWS_COND_SYNC_EXP", "RTWS_POLL_GET", + "RTWS_POLL_GET_EXP", "RTWS_POLL_WAIT", + "RTWS_POLL_WAIT_EXP", "RTWS_SYNC", "RTWS_STUTTER", "RTWS_STOPPING", @@ -337,6 +347,10 @@ struct rcu_torture_ops { void (*deferred_free)(struct rcu_torture *p); void (*sync)(void); void (*exp_sync)(void); + unsigned long (*get_gp_state_exp)(void); + unsigned long (*start_gp_poll_exp)(void); + bool (*poll_gp_state_exp)(unsigned long oldstate); + void (*cond_sync_exp)(unsigned long oldstate); unsigned long (*get_gp_state)(void); unsigned long (*get_gp_completed)(void); unsigned long (*start_gp_poll)(void); @@ -509,6 +523,10 @@ static struct rcu_torture_ops rcu_ops = { .start_gp_poll = start_poll_synchronize_rcu, .poll_gp_state = poll_state_synchronize_rcu, .cond_sync = cond_synchronize_rcu, + .get_gp_state_exp = get_state_synchronize_rcu, + .start_gp_poll_exp = start_poll_synchronize_rcu_expedited, + .poll_gp_state_exp = poll_state_synchronize_rcu, + .cond_sync_exp = cond_synchronize_rcu_expedited, .call = call_rcu, .cb_barrier = rcu_barrier, .fqs = rcu_force_quiescent_state, @@ -1138,9 +1156,8 @@ rcu_torture_fqs(void *arg) return 0; } -// Used by writers to randomly choose from the available grace-period -// primitives. The only purpose of the initialization is to size the array. -static int synctype[] = { RTWS_DEF_FREE, RTWS_EXP_SYNC, RTWS_COND_GET, RTWS_POLL_GET, RTWS_SYNC }; +// Used by writers to randomly choose from the available grace-period primitives. +static int synctype[ARRAY_SIZE(rcu_torture_writer_state_names)] = { }; static int nsynctypes; /* @@ -1148,18 +1165,27 @@ static int nsynctypes; */ static void rcu_torture_write_types(void) { - bool gp_cond1 = gp_cond, gp_exp1 = gp_exp, gp_normal1 = gp_normal; - bool gp_poll1 = gp_poll, gp_sync1 = gp_sync; + bool gp_cond1 = gp_cond, gp_cond_exp1 = gp_cond_exp, gp_exp1 = gp_exp; + bool gp_poll_exp1 = gp_poll_exp, gp_normal1 = gp_normal, gp_poll1 = gp_poll; + bool gp_sync1 = gp_sync; /* Initialize synctype[] array. If none set, take default. */ - if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_poll1 && !gp_sync1) - gp_cond1 = gp_exp1 = gp_normal1 = gp_poll1 = gp_sync1 = true; + if (!gp_cond1 && !gp_cond_exp1 && !gp_exp1 && !gp_poll_exp && + !gp_normal1 && !gp_poll1 && !gp_sync1) + gp_cond1 = gp_cond_exp1 = gp_exp1 = gp_poll_exp1 = + gp_normal1 = gp_poll1 = gp_sync1 = true; if (gp_cond1 && cur_ops->get_gp_state && cur_ops->cond_sync) { synctype[nsynctypes++] = RTWS_COND_GET; pr_info("%s: Testing conditional GPs.\n", __func__); } else if (gp_cond && (!cur_ops->get_gp_state || !cur_ops->cond_sync)) { pr_alert("%s: gp_cond without primitives.\n", __func__); } + if (gp_cond_exp1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp) { + synctype[nsynctypes++] = RTWS_COND_GET_EXP; + pr_info("%s: Testing conditional expedited GPs.\n", __func__); + } else if (gp_cond_exp && (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp)) { + pr_alert("%s: gp_cond_exp without primitives.\n", __func__); + } if (gp_exp1 && cur_ops->exp_sync) { synctype[nsynctypes++] = RTWS_EXP_SYNC; pr_info("%s: Testing expedited GPs.\n", __func__); @@ -1178,6 +1204,12 @@ static void rcu_torture_write_types(void) } else if (gp_poll && (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)) { pr_alert("%s: gp_poll without primitives.\n", __func__); } + if (gp_poll_exp1 && cur_ops->start_gp_poll_exp && cur_ops->poll_gp_state_exp) { + synctype[nsynctypes++] = RTWS_POLL_GET_EXP; + pr_info("%s: Testing polling expedited GPs.\n", __func__); + } else if (gp_poll_exp && (!cur_ops->start_gp_poll_exp || !cur_ops->poll_gp_state_exp)) { + pr_alert("%s: gp_poll_exp without primitives.\n", __func__); + } if (gp_sync1 && cur_ops->sync) { synctype[nsynctypes++] = RTWS_SYNC; pr_info("%s: Testing normal GPs.\n", __func__); @@ -1285,6 +1317,14 @@ rcu_torture_writer(void *arg) cur_ops->cond_sync(gp_snap); rcu_torture_pipe_update(old_rp); break; + case RTWS_COND_GET_EXP: + rcu_torture_writer_state = RTWS_COND_GET_EXP; + gp_snap = cur_ops->get_gp_state_exp(); + torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); + rcu_torture_writer_state = RTWS_COND_SYNC_EXP; + cur_ops->cond_sync_exp(gp_snap); + rcu_torture_pipe_update(old_rp); + break; case RTWS_POLL_GET: rcu_torture_writer_state = RTWS_POLL_GET; gp_snap = cur_ops->start_gp_poll(); @@ -1294,6 +1334,15 @@ rcu_torture_writer(void *arg) &rand); rcu_torture_pipe_update(old_rp); break; + case RTWS_POLL_GET_EXP: + rcu_torture_writer_state = RTWS_POLL_GET_EXP; + gp_snap = cur_ops->start_gp_poll_exp(); + rcu_torture_writer_state = RTWS_POLL_WAIT_EXP; + while (!cur_ops->poll_gp_state_exp(gp_snap)) + torture_hrtimeout_jiffies(torture_random(&rand) % 16, + &rand); + rcu_torture_pipe_update(old_rp); + break; case RTWS_SYNC: rcu_torture_writer_state = RTWS_SYNC; if (cur_ops->get_gp_state && cur_ops->poll_gp_state) @@ -1400,6 +1449,11 @@ rcu_torture_fakewriter(void *arg) torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); cur_ops->cond_sync(gp_snap); break; + case RTWS_COND_GET_EXP: + gp_snap = cur_ops->get_gp_state_exp(); + torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); + cur_ops->cond_sync_exp(gp_snap); + break; case RTWS_POLL_GET: gp_snap = cur_ops->start_gp_poll(); while (!cur_ops->poll_gp_state(gp_snap)) { @@ -1407,6 +1461,13 @@ rcu_torture_fakewriter(void *arg) &rand); } break; + case RTWS_POLL_GET_EXP: + gp_snap = cur_ops->start_gp_poll_exp(); + while (!cur_ops->poll_gp_state_exp(gp_snap)) { + torture_hrtimeout_jiffies(torture_random(&rand) % 16, + &rand); + } + break; case RTWS_SYNC: cur_ops->sync(); break; From ef4f9d9b9230fcf4fca9801f03c31d99ed06a716 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Fri, 22 Apr 2022 21:15:18 +0800 Subject: [PATCH 286/318] rcu: Put panic_on_rcu_stall() after expedited RCU CPU stall warnings When a normal RCU CPU stall warning is encountered with the panic_on_rcu_stall sysfs variable is set, the system panics only after the stall warning is printed. But when an expedited RCU CPU stall warning is encountered with the panic_on_rcu_stall sysfs variable is set, the system panics first, thus never printing the stall warning. This commit therefore brings the expedited stall warning into line with the normal stall warning by printing first and panicking afterwards. Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/tree_exp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index 571b0a700cce..f05a15b11fa0 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -623,7 +623,6 @@ static void synchronize_rcu_expedited_wait(void) return; if (rcu_stall_is_suppressed()) continue; - panic_on_rcu_stall(); trace_rcu_stall_warning(rcu_state.name, TPS("ExpeditedStall")); pr_err("INFO: %s detected expedited stalls on CPUs/tasks: {", rcu_state.name); @@ -671,6 +670,7 @@ static void synchronize_rcu_expedited_wait(void) } } jiffies_stall = 3 * rcu_exp_jiffies_till_stall_check() + 3; + panic_on_rcu_stall(); } } From 82e445697d6a14d6b7462c13c613ebdd96468818 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 9 May 2022 09:49:05 -0700 Subject: [PATCH 287/318] rcu: Diagnose extended sync_rcu_do_polled_gp() loops This commit dumps out state when the sync_rcu_do_polled_gp() function loops more than expected. This is a debugging aid. Signed-off-by: Paul E. McKenney --- kernel/rcu/tree_exp.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index f05a15b11fa0..4c7037b50703 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -970,6 +970,7 @@ EXPORT_SYMBOL_GPL(synchronize_rcu_expedited); static void sync_rcu_do_polled_gp(struct work_struct *wp) { unsigned long flags; + int i = 0; struct rcu_node *rnp = container_of(wp, struct rcu_node, exp_poll_wq); unsigned long s; @@ -979,8 +980,12 @@ static void sync_rcu_do_polled_gp(struct work_struct *wp) raw_spin_unlock_irqrestore(&rnp->exp_poll_lock, flags); if (s == RCU_GET_STATE_COMPLETED) return; - while (!poll_state_synchronize_rcu(s)) + while (!poll_state_synchronize_rcu(s)) { synchronize_rcu_expedited(); + if (i == 10 || i == 20) + pr_info("%s: i = %d s = %lx gp_seq_polled = %lx\n", __func__, i, s, READ_ONCE(rcu_state.gp_seq_polled)); + i++; + } raw_spin_lock_irqsave(&rnp->exp_poll_lock, flags); s = rnp->exp_seq_poll_rq; if (poll_state_synchronize_rcu(s)) From 28787e04fb67963673cbe6f77fb27137eba42718 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Wed, 18 May 2022 19:43:10 +0800 Subject: [PATCH 288/318] rcu: Add irqs-disabled indicator to expedited RCU CPU stall warnings If a CPU has interrupts disabled continuously starting before the beginning of a given expedited RCU grace period, that CPU will not execute that grace period's IPI handler. This will in turn mean that the ->cpu_no_qs.b.exp field in that CPU's rcu_data structure will continue to contain the boolean value false. Knowing whether or not a CPU has had interrupts disabled can be helpful when debugging an expedited RCU CPU stall warning, so this commit adds a "D" indicator expedited RCU CPU stall warnings that signifies that the corresponding CPU has had interrupts disabled throughout. This capability was tested as follows: runqemu kvm slirp nographic qemuparams="-m 4096 -smp 4" bootparams= "isolcpus=2,3 nohz_full=2,3 rcu_nocbs=2,3 rcutree.dump_tree=1 rcutorture.stall_cpu_holdoff=30 rcutorture.stall_cpu=40 rcutorture.stall_cpu_irqsoff=1 rcutorture.stall_cpu_block=0 rcutorture.stall_no_softlockup=1" -d The rcu_torture_stall() function ran on CPU 1, which displays the "D" as expected given the rcutorture.stall_cpu_irqsoff=1 module parameter: ............ rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 1-...D } 26467 jiffies s: 13317 root: 0x1/. rcu: blocking rcu_node structures (internal RCU debug): l=1:0-1:0x2/. Task dump for CPU 1: task:rcu_torture_sta state:R running task stack: 0 pid: 76 ppid: 2 flags:0x00004008 Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/tree_exp.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index 4c7037b50703..f092c7f18a5f 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -637,10 +637,11 @@ static void synchronize_rcu_expedited_wait(void) continue; ndetected++; rdp = per_cpu_ptr(&rcu_data, cpu); - pr_cont(" %d-%c%c%c", cpu, + pr_cont(" %d-%c%c%c%c", cpu, "O."[!!cpu_online(cpu)], "o."[!!(rdp->grpmask & rnp->expmaskinit)], - "N."[!!(rdp->grpmask & rnp->expmaskinitnext)]); + "N."[!!(rdp->grpmask & rnp->expmaskinitnext)], + "D."[!!(rdp->cpu_no_qs.b.exp)]); } } pr_cont(" } %lu jiffies s: %lu root: %#lx/%c\n", From 19cdbdb7cda0cb4948dfaab613d8b4f63c88a53e Mon Sep 17 00:00:00 2001 From: Bagas Sanjaya Date: Wed, 13 Jul 2022 15:56:27 +0700 Subject: [PATCH 289/318] Documentation: qat: Use code block for qat sysfs example kernel test robot and Stephen Rothwell reported htmldocs warning: Documentation/ABI/testing/sysfs-driver-qat:24: WARNING: Unexpected indentation. The warning isn't caused by Date: field pointed by the warning, but rather by sysfs example that isn't in literal code block. Add the code block marker. Link: https://lore.kernel.org/linux-next/20220711204932.333379b4@canb.auug.org.au/ Link: https://lore.kernel.org/linux-doc/202207090803.TEGI95qw-lkp@intel.com/ Reported-by: kernel test robot Reported-by: Stephen Rothwell Fixes: d4cfb144f60551 ("crypto: qat - expose device config through sysfs for 4xxx") Acked-by: Giovanni Cabiddu Cc: Wojciech Ziemba Cc: Adam Guerin Cc: Fiona Trahe Cc: Herbert Xu Cc: Vladis Dronov Cc: Tomasz Kowallik Cc: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Bagas Sanjaya Acked-by: Vladis Dronov Signed-off-by: Herbert Xu --- Documentation/ABI/testing/sysfs-driver-qat | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation/ABI/testing/sysfs-driver-qat b/Documentation/ABI/testing/sysfs-driver-qat index a600531e9562..43e081ec22cc 100644 --- a/Documentation/ABI/testing/sysfs-driver-qat +++ b/Documentation/ABI/testing/sysfs-driver-qat @@ -46,7 +46,8 @@ Description: Reports the current configuration of the QAT device and allows The following example shows how to change the configuration of a device configured for running crypto services in order to - run data compression: + run data compression:: + # cat /sys/bus/pci/devices//qat/state up # cat /sys/bus/pci/devices//qat/cfg_services From 1b466b8cbfd8321408bb0792b241a2b9222aed62 Mon Sep 17 00:00:00 2001 From: Bagas Sanjaya Date: Wed, 13 Jul 2022 15:56:28 +0700 Subject: [PATCH 290/318] Documentation: qat: rewrite description The sysfs description contains redundancy on returned and allowed values list, due to the described sysfs is read-write. Rewrite. Acked-by: Giovanni Cabiddu Cc: Adam Guerin Cc: Tomasz Kowallik Cc: Giovanni Cabiddu Cc: Wojciech Ziemba Cc: Fiona Trahe Cc: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Bagas Sanjaya Signed-off-by: Herbert Xu --- Documentation/ABI/testing/sysfs-driver-qat | 34 +++++++--------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-driver-qat b/Documentation/ABI/testing/sysfs-driver-qat index 43e081ec22cc..185f81a2aab3 100644 --- a/Documentation/ABI/testing/sysfs-driver-qat +++ b/Documentation/ABI/testing/sysfs-driver-qat @@ -2,18 +2,14 @@ What: /sys/bus/pci/devices//qat/state Date: June 2022 KernelVersion: 5.20 Contact: qat-linux@intel.com -Description: Reports the current state of the QAT device and allows to - change it. +Description: (RW) Reports the current state of the QAT device. Write to + the file to start or stop the device. - This attribute is RW. + The values are: - Returned values: - up: the device is up and running - down: the device is down + * up: the device is up and running + * down: the device is down - Allowed values: - up: initialize and start the device - down: stop the device and bring it down It is possible to transition the device from up to down only if the device is up and vice versa. @@ -24,22 +20,14 @@ What: /sys/bus/pci/devices//qat/cfg_services Date: June 2022 KernelVersion: 5.20 Contact: qat-linux@intel.com -Description: Reports the current configuration of the QAT device and allows - to change it. +Description: (RW) Reports the current configuration of the QAT device. + Write to the file to change the configured services. - This attribute is RW. + The values are: - Returned values: - sym;asym: the device is configured for running - crypto services - dc: the device is configured for running - compression services - - Allowed values: - sym;asym: configure the device for running - crypto services - dc: configure the device for running - compression services + * sym;asym: the device is configured for running crypto + services + * dc: the device is configured for running compression services It is possible to set the configuration only if the device is in the `down` state (see /sys/bus/pci/devices//qat/state) From 693b8755e1b1dd3c0bc22920a8bf2bd495688909 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Thu, 14 Jul 2022 23:28:20 +0200 Subject: [PATCH 291/318] crypto: keembay-ocs-ecc - Drop if with an always false condition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The remove callback is only called after probe completed successfully. In this case platform_set_drvdata() was called with a non-NULL argument and so ecc_dev is never NULL. This is a preparation for making platform remove callbacks return void. Signed-off-by: Uwe Kleine-König Signed-off-by: Herbert Xu --- drivers/crypto/keembay/keembay-ocs-ecc.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/crypto/keembay/keembay-ocs-ecc.c b/drivers/crypto/keembay/keembay-ocs-ecc.c index 5d0785d3f1b5..2269df17514c 100644 --- a/drivers/crypto/keembay/keembay-ocs-ecc.c +++ b/drivers/crypto/keembay/keembay-ocs-ecc.c @@ -976,8 +976,6 @@ static int kmb_ocs_ecc_remove(struct platform_device *pdev) struct ocs_ecc_dev *ecc_dev; ecc_dev = platform_get_drvdata(pdev); - if (!ecc_dev) - return -ENODEV; crypto_unregister_kpp(&ocs_ecdh_p384); crypto_unregister_kpp(&ocs_ecdh_p256); From 4cbdecd02fd29eb69a376ffdac47aff441c4d19f Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Thu, 14 Jul 2022 18:59:14 -0700 Subject: [PATCH 292/318] crypto: rmd160 - fix Kconfig "its" grammar Use the possessive "its" instead of the contraction "it's" where appropriate. Signed-off-by: Randy Dunlap Cc: Herbert Xu Cc: "David S. Miller" Cc: linux-crypto@vger.kernel.org Signed-off-by: Herbert Xu --- crypto/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index 75c71d9a5ffb..e72bf3fc298c 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -889,7 +889,7 @@ config CRYPTO_RMD160 RIPEMD-160 is a 160-bit cryptographic hash function. It is intended to be used as a secure replacement for the 128-bit hash functions - MD4, MD5 and it's predecessor RIPEMD + MD4, MD5 and its predecessor RIPEMD (not to be confused with RIPEMD-128). It's speed is comparable to SHA1 and there are no known attacks From 824b94a88320eaa5e3e059b494e457ed25987a63 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Fri, 15 Jul 2022 12:50:07 +0800 Subject: [PATCH 293/318] crypto: twofish - Fix comment typo The double `that' is duplicated in line 301, remove one. Signed-off-by: Jason Wang Signed-off-by: Herbert Xu --- crypto/twofish_common.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/twofish_common.c b/crypto/twofish_common.c index d23fa531b91f..f921f30334f4 100644 --- a/crypto/twofish_common.c +++ b/crypto/twofish_common.c @@ -298,7 +298,7 @@ static const u32 mds[4][256] = { * multiplication is inefficient without hardware support. To multiply * faster, I make use of the fact x is a generator for the nonzero elements, * so that every element p of GF(2)[x]/w(x) is either 0 or equal to (x)^n for - * some n in 0..254. Note that that caret is exponentiation in GF(2^8), + * some n in 0..254. Note that caret is exponentiation in GF(2^8), * *not* polynomial notation. So if I want to compute pq where p and q are * in GF(2^8), I can just say: * 1. if p=0 or q=0 then pq=0 From 85796a9b7583a0b00ee9e69b932daafb41515a76 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Fri, 15 Jul 2022 13:40:50 +0800 Subject: [PATCH 294/318] hwrng: via - Fix comment typo The double `close' is duplicated in line 148, remove one. Signed-off-by: Jason Wang Signed-off-by: Herbert Xu --- drivers/char/hw_random/via-rng.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/char/hw_random/via-rng.c b/drivers/char/hw_random/via-rng.c index 7444cc146e86..a9a0a3b09c8b 100644 --- a/drivers/char/hw_random/via-rng.c +++ b/drivers/char/hw_random/via-rng.c @@ -145,7 +145,7 @@ static int via_rng_init(struct hwrng *rng) } /* Control the RNG via MSR. Tread lightly and pay very close - * close attention to values written, as the reserved fields + * attention to values written, as the reserved fields * are documented to be "undefined and unpredictable"; but it * does not say to write them as zero, so I make a guess that * we restore the values we find in the register. From 647c952eac5d27569365e9407c96db3c6915e08f Mon Sep 17 00:00:00 2001 From: shaom Deng Date: Sun, 17 Jul 2022 00:56:33 +0800 Subject: [PATCH 295/318] cyrpto: powerpc/aes - delete the rebundant word "block" in comments there is rebundant word "block" in comments, so remove it Signed-off-by: shaom Deng Signed-off-by: Herbert Xu --- arch/powerpc/crypto/aes-spe-glue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/powerpc/crypto/aes-spe-glue.c b/arch/powerpc/crypto/aes-spe-glue.c index e8dfe9fb0266..efab78a3a8f6 100644 --- a/arch/powerpc/crypto/aes-spe-glue.c +++ b/arch/powerpc/crypto/aes-spe-glue.c @@ -28,7 +28,7 @@ * instructions per clock cycle using one 32/64 bit unit (SU1) and one 32 * bit unit (SU2). One of these can be a memory access that is executed via * a single load and store unit (LSU). XTS-AES-256 takes ~780 operations per - * 16 byte block block or 25 cycles per byte. Thus 768 bytes of input data + * 16 byte block or 25 cycles per byte. Thus 768 bytes of input data * will need an estimated maximum of 20,000 cycles. Headroom for cache misses * included. Even with the low end model clocked at 667 MHz this equals to a * critical time window of less than 30us. The value has been chosen to From 9d2bb9a74b2877f100637d6ab5685bcd33c69d44 Mon Sep 17 00:00:00 2001 From: Ignat Korchagin Date: Sun, 17 Jul 2022 14:37:46 +0100 Subject: [PATCH 296/318] crypto: testmgr - some more fixes to RSA test vectors Two more fixes: * some test vectors in commit 79e6e2f3f3ff ("crypto: testmgr - populate RSA CRT parameters in RSA test vectors") had misplaced commas, which break the test and trigger KASAN warnings at least on x86-64 * pkcs1pad test vector did not have its CRT parameters Fixes: 79e6e2f3f3ff ("crypto: testmgr - populate RSA CRT parameters in RSA test vectors") Reported-by: Eric Biggers Signed-off-by: Ignat Korchagin Signed-off-by: Herbert Xu --- crypto/testmgr.h | 102 +++++++++++++++++++++++++++++------------------ 1 file changed, 63 insertions(+), 39 deletions(-) diff --git a/crypto/testmgr.h b/crypto/testmgr.h index dee88510f58d..d6088e26f326 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -273,7 +273,7 @@ static const struct akcipher_testvec rsa_tv_template[] = { "\x61\xAD\xBD\x3A\x8A\x7E\x99\x1C\x5C\x05\x56\xA9\x4C\x31\x46\xA7" "\xF9\x80\x3F\x8F\x6F\x8A\xE3\x42\xE9\x31\xFD\x8A\xE4\x7A\x22\x0D" "\x1B\x99\xA4\x95\x84\x98\x07\xFE\x39\xF9\x24\x5A\x98\x36\xDA\x3D" - "\x02\x41", /* coefficient - integer of 65 bytes */ + "\x02\x41" /* coefficient - integer of 65 bytes */ "\x00\xB0\x6C\x4F\xDA\xBB\x63\x01\x19\x8D\x26\x5B\xDB\xAE\x94\x23" "\xB3\x80\xF2\x71\xF7\x34\x53\x88\x50\x93\x07\x7F\xCD\x39\xE2\x11" "\x9F\xC9\x86\x32\x15\x4F\x58\x83\xB1\x67\xA9\x67\xBF\x40\x2B\x4E" @@ -370,7 +370,7 @@ static const struct akcipher_testvec rsa_tv_template[] = { "\x6A\x37\x3B\x86\x6C\x51\x37\x5B\x1D\x79\xF2\xA3\x43\x10\xC6\xA7" "\x21\x79\x6D\xF9\xE9\x04\x6A\xE8\x32\xFF\xAE\xFD\x1C\x7B\x8C\x29" "\x13\xA3\x0C\xB2\xAD\xEC\x6C\x0F\x8D\x27\x12\x7B\x48\xB2\xDB\x31" - "\x02\x81\x81", /* coefficient - integer of 129 bytes */ + "\x02\x81\x81" /* coefficient - integer of 129 bytes */ "\x00\x8D\x1B\x05\xCA\x24\x1F\x0C\x53\x19\x52\x74\x63\x21\xFA\x78" "\x46\x79\xAF\x5C\xDE\x30\xA4\x6C\x20\x38\xE6\x97\x39\xB8\x7A\x70" "\x0D\x8B\x6C\x6D\x13\x74\xD5\x1C\xDE\xA9\xF4\x60\x37\xFE\x68\x77" @@ -1231,7 +1231,7 @@ static const struct akcipher_testvec ecrdsa_tv_template[] = { static const struct akcipher_testvec pkcs1pad_rsa_tv_template[] = { { .key = - "\x30\x82\x03\x1f\x02\x01\x00\x02\x82\x01\x01\x00\xd7\x1e\x77\x82" + "\x30\x82\x04\xa5\x02\x01\x00\x02\x82\x01\x01\x00\xd7\x1e\x77\x82" "\x8c\x92\x31\xe7\x69\x02\xa2\xd5\x5c\x78\xde\xa2\x0c\x8f\xfe\x28" "\x59\x31\xdf\x40\x9c\x60\x61\x06\xb9\x2f\x62\x40\x80\x76\xcb\x67" "\x4a\xb5\x59\x56\x69\x17\x07\xfa\xf9\x4c\xbd\x6c\x37\x7a\x46\x7d" @@ -1247,42 +1247,66 @@ static const struct akcipher_testvec pkcs1pad_rsa_tv_template[] = { "\x9e\x49\x63\x6e\x02\xc1\xc9\x3a\x9b\xa5\x22\x1b\x07\x95\xd6\x10" "\x02\x50\xfd\xfd\xd1\x9b\xbe\xab\xc2\xc0\x74\xd7\xec\x00\xfb\x11" "\x71\xcb\x7a\xdc\x81\x79\x9f\x86\x68\x46\x63\x82\x4d\xb7\xf1\xe6" - "\x16\x6f\x42\x63\xf4\x94\xa0\xca\x33\xcc\x75\x13\x02\x82\x01\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x01" - "\x02\x82\x01\x00\x62\xb5\x60\x31\x4f\x3f\x66\x16\xc1\x60\xac\x47" - "\x2a\xff\x6b\x69\x00\x4a\xb2\x5c\xe1\x50\xb9\x18\x74\xa8\xe4\xdc" - "\xa8\xec\xcd\x30\xbb\xc1\xc6\xe3\xc6\xac\x20\x2a\x3e\x5e\x8b\x12" - "\xe6\x82\x08\x09\x38\x0b\xab\x7c\xb3\xcc\x9c\xce\x97\x67\xdd\xef" - "\x95\x40\x4e\x92\xe2\x44\xe9\x1d\xc1\x14\xfd\xa9\xb1\xdc\x71\x9c" - "\x46\x21\xbd\x58\x88\x6e\x22\x15\x56\xc1\xef\xe0\xc9\x8d\xe5\x80" - "\x3e\xda\x7e\x93\x0f\x52\xf6\xf5\xc1\x91\x90\x9e\x42\x49\x4f\x8d" - "\x9c\xba\x38\x83\xe9\x33\xc2\x50\x4f\xec\xc2\xf0\xa8\xb7\x6e\x28" - "\x25\x56\x6b\x62\x67\xfe\x08\xf1\x56\xe5\x6f\x0e\x99\xf1\xe5\x95" - "\x7b\xef\xeb\x0a\x2c\x92\x97\x57\x23\x33\x36\x07\xdd\xfb\xae\xf1" - "\xb1\xd8\x33\xb7\x96\x71\x42\x36\xc5\xa4\xa9\x19\x4b\x1b\x52\x4c" - "\x50\x69\x91\xf0\x0e\xfa\x80\x37\x4b\xb5\xd0\x2f\xb7\x44\x0d\xd4" - "\xf8\x39\x8d\xab\x71\x67\x59\x05\x88\x3d\xeb\x48\x48\x33\x88\x4e" - "\xfe\xf8\x27\x1b\xd6\x55\x60\x5e\x48\xb7\x6d\x9a\xa8\x37\xf9\x7a" - "\xde\x1b\xcd\x5d\x1a\x30\xd4\xe9\x9e\x5b\x3c\x15\xf8\x9c\x1f\xda" - "\xd1\x86\x48\x55\xce\x83\xee\x8e\x51\xc7\xde\x32\x12\x47\x7d\x46" - "\xb8\x35\xdf\x41\x02\x01\x00\x02\x01\x00\x02\x01\x00\x02\x01\x00" - "\x02\x01\x00", - .key_len = 803, + "\x16\x6f\x42\x63\xf4\x94\xa0\xca\x33\xcc\x75\x13\x02\x03\x01\x00" + "\x01\x02\x82\x01\x00\x62\xb5\x60\x31\x4f\x3f\x66\x16\xc1\x60\xac" + "\x47\x2a\xff\x6b\x69\x00\x4a\xb2\x5c\xe1\x50\xb9\x18\x74\xa8\xe4" + "\xdc\xa8\xec\xcd\x30\xbb\xc1\xc6\xe3\xc6\xac\x20\x2a\x3e\x5e\x8b" + "\x12\xe6\x82\x08\x09\x38\x0b\xab\x7c\xb3\xcc\x9c\xce\x97\x67\xdd" + "\xef\x95\x40\x4e\x92\xe2\x44\xe9\x1d\xc1\x14\xfd\xa9\xb1\xdc\x71" + "\x9c\x46\x21\xbd\x58\x88\x6e\x22\x15\x56\xc1\xef\xe0\xc9\x8d\xe5" + "\x80\x3e\xda\x7e\x93\x0f\x52\xf6\xf5\xc1\x91\x90\x9e\x42\x49\x4f" + "\x8d\x9c\xba\x38\x83\xe9\x33\xc2\x50\x4f\xec\xc2\xf0\xa8\xb7\x6e" + "\x28\x25\x56\x6b\x62\x67\xfe\x08\xf1\x56\xe5\x6f\x0e\x99\xf1\xe5" + "\x95\x7b\xef\xeb\x0a\x2c\x92\x97\x57\x23\x33\x36\x07\xdd\xfb\xae" + "\xf1\xb1\xd8\x33\xb7\x96\x71\x42\x36\xc5\xa4\xa9\x19\x4b\x1b\x52" + "\x4c\x50\x69\x91\xf0\x0e\xfa\x80\x37\x4b\xb5\xd0\x2f\xb7\x44\x0d" + "\xd4\xf8\x39\x8d\xab\x71\x67\x59\x05\x88\x3d\xeb\x48\x48\x33\x88" + "\x4e\xfe\xf8\x27\x1b\xd6\x55\x60\x5e\x48\xb7\x6d\x9a\xa8\x37\xf9" + "\x7a\xde\x1b\xcd\x5d\x1a\x30\xd4\xe9\x9e\x5b\x3c\x15\xf8\x9c\x1f" + "\xda\xd1\x86\x48\x55\xce\x83\xee\x8e\x51\xc7\xde\x32\x12\x47\x7d" + "\x46\xb8\x35\xdf\x41\x02\x81\x81\x00\xe4\x4c\xae\xde\x16\xfd\x9f" + "\x83\x55\x5b\x84\x4a\xcf\x1c\xf1\x37\x95\xad\xca\x29\x7f\x2d\x6e" + "\x32\x81\xa4\x2b\x26\x14\x96\x1d\x40\x05\xec\x0c\xaf\x3f\x2c\x6f" + "\x2c\xe8\xbf\x1d\xee\xd0\xb3\xef\x7c\x5b\x9e\x88\x4f\x2a\x8b\x0e" + "\x4a\xbd\xb7\x8c\xfa\x10\x0e\x3b\xda\x68\xad\x41\x2b\xe4\x96\xfa" + "\x7f\x80\x52\x5f\x07\x9f\x0e\x3b\x5e\x96\x45\x1a\x13\x2b\x94\xce" + "\x1f\x07\x69\x85\x35\xfc\x69\x63\x5b\xf8\xf8\x3f\xce\x9d\x40\x1e" + "\x7c\xad\xfb\x9e\xce\xe0\x01\xf8\xef\x59\x5d\xdc\x00\x79\xab\x8a" + "\x3f\x80\xa2\x76\x32\x94\xa9\xea\x65\x02\x81\x81\x00\xf1\x38\x60" + "\x90\x0d\x0c\x2e\x3d\x34\xe5\x90\xea\x21\x43\x1f\x68\x63\x16\x7b" + "\x25\x8d\xde\x82\x2b\x52\xf8\xa3\xfd\x0f\x39\xe7\xe9\x5e\x32\x75" + "\x15\x7d\xd0\xc9\xce\x06\xe5\xfb\xa9\xcb\x22\xe5\xdb\x49\x09\xf2" + "\xe6\xb7\xa5\xa7\x75\x2e\x91\x2d\x2b\x5d\xf1\x48\x61\x45\x43\xd7" + "\xbd\xfc\x11\x73\xb5\x11\x9f\xb2\x18\x3a\x6f\x36\xa7\xc2\xd3\x18" + "\x4d\xf0\xc5\x1f\x70\x8c\x9b\xc5\x1d\x95\xa8\x5a\x9e\x8c\xb1\x4b" + "\x6a\x2a\x84\x76\x2c\xd8\x4f\x47\xb0\x81\x84\x02\x45\xf0\x85\xf8" + "\x0c\x6d\xa7\x0c\x4d\x2c\xb2\x5b\x81\x70\xfd\x6e\x17\x02\x81\x81" + "\x00\x8d\x07\xc5\xfa\x92\x4f\x48\xcb\xd3\xdd\xfe\x02\x4c\xa1\x7f" + "\x6d\xab\xfc\x38\xe7\x9b\x95\xcf\xfe\x49\x51\xc6\x09\xf7\x2b\xa8" + "\x94\x15\x54\x75\x9d\x88\xb4\x05\x55\xc3\xcd\xd4\x4a\xe4\x08\x53" + "\xc8\x09\xbd\x0c\x4d\x83\x65\x75\x85\xbc\x5e\xf8\x2a\xbd\xe2\x5d" + "\x1d\x16\x0e\xf9\x34\x89\x38\xaf\x34\x36\x6c\x2c\x22\x44\x22\x81" + "\x90\x73\xd9\xea\x3a\xaf\x70\x74\x48\x7c\xc6\xb5\xb0\xdc\xe5\xa9" + "\xa8\x76\x4b\xbc\xf7\x00\xf3\x4c\x22\x0f\x44\x62\x1d\x40\x0a\x57" + "\xe2\x5b\xdd\x7c\x7b\x9a\xad\xda\x70\x52\x21\x8a\x4c\xc2\xc3\x98" + "\x75\x02\x81\x81\x00\xed\x24\x5c\xa2\x21\x81\xa1\x0f\xa1\x2a\x33" + "\x0e\x49\xc7\x00\x60\x92\x51\x6e\x9d\x9b\xdc\x6d\x22\x04\x7e\xd6" + "\x51\x19\x9f\xf6\xe3\x91\x2c\x8f\xb8\xa2\x29\x19\xcc\x47\x31\xdf" + "\xf8\xab\xf0\xd2\x02\x83\xca\x99\x16\xc2\xe2\xc3\x3f\x4b\x99\x83" + "\xcb\x87\x9e\x86\x66\xc2\x3e\x91\x21\x80\x66\xf3\xd6\xc5\xcd\xb6" + "\xbb\x64\xef\x22\xcf\x48\x94\x58\xe7\x7e\xd5\x7c\x34\x1c\xb7\xa2" + "\xd0\x93\xe9\x9f\xb5\x11\x61\xd7\x5f\x37\x0f\x64\x52\x70\x11\x78" + "\xcc\x08\x77\xeb\xf8\x30\x1e\xb4\x9e\x1b\x4a\xc7\xa8\x33\x51\xe0" + "\xed\xdf\x53\xf6\xdf\x02\x81\x81\x00\x86\xd9\x4c\xee\x65\x61\xc1" + "\x19\xa9\xd5\x74\x9b\xd5\xca\xf6\x83\x2b\x06\xb4\x20\xfe\x45\x29" + "\xe8\xe3\xfa\xe1\x4f\x28\x8e\x63\x2f\x74\xc3\x3a\x5c\x9a\xf5\x9e" + "\x0e\x0d\xc5\xfe\xa0\x4c\x00\xce\x7b\xa4\x19\x17\x59\xaf\x13\x3a" + "\x03\x8f\x54\xf5\x60\x39\x2e\xd9\x06\xb3\x7c\xd6\x90\x06\x41\x77" + "\xf3\x93\xe1\x7a\x01\x41\xc1\x8f\xfe\x4c\x88\x39\xdb\xde\x71\x9e" + "\x58\xd1\x49\x50\x80\xb2\x5a\x4f\x69\x8b\xb8\xfe\x63\xd4\x42\x3d" + "\x37\x61\xa8\x4c\xff\xb6\x99\x4c\xf4\x51\xe0\x44\xaa\x69\x79\x3f" + "\x81\xa4\x61\x3d\x26\xe9\x04\x52\x64", + .key_len = 1193, /* * m is SHA256 hash of following message: * "\x49\x41\xbe\x0a\x0c\xc9\xf6\x35\x51\xe4\x27\x56\x13\x71\x4b\xd0" From d349ab99eec7ab0f977fc4aac27aa476907acf90 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Sun, 17 Jul 2022 12:35:24 +0200 Subject: [PATCH 297/318] random: handle archrandom with multiple longs The archrandom interface was originally designed for x86, which supplies RDRAND/RDSEED for receiving random words into registers, resulting in one function to generate an int and another to generate a long. However, other architectures don't follow this. On arm64, the SMCCC TRNG interface can return between one and three longs. On s390, the CPACF TRNG interface can return arbitrary amounts, with four longs having the same cost as one. On UML, the os_getrandom() interface can return arbitrary amounts. So change the api signature to take a "max_longs" parameter designating the maximum number of longs requested, and then return the number of longs generated. Since callers need to check this return value and loop anyway, each arch implementation does not bother implementing its own loop to try again to fill the maximum number of longs. Additionally, all existing callers pass in a constant max_longs parameter. Taken together, these two things mean that the codegen doesn't really change much for one-word-at-a-time platforms, while performance is greatly improved on platforms such as s390. Acked-by: Heiko Carstens Acked-by: Catalin Marinas Acked-by: Mark Rutland Acked-by: Michael Ellerman Acked-by: Borislav Petkov Signed-off-by: Jason A. Donenfeld --- arch/arm64/include/asm/archrandom.h | 102 ++++++++++++-------------- arch/arm64/kernel/kaslr.c | 2 +- arch/powerpc/include/asm/archrandom.h | 30 ++------ arch/powerpc/kvm/book3s_hv.c | 2 +- arch/s390/include/asm/archrandom.h | 29 ++------ arch/um/include/asm/archrandom.h | 23 +++--- arch/x86/include/asm/archrandom.h | 41 +---------- arch/x86/kernel/espfix_64.c | 2 +- drivers/char/random.c | 45 ++++++++---- include/asm-generic/archrandom.h | 18 +---- include/linux/random.h | 12 +-- 11 files changed, 117 insertions(+), 189 deletions(-) diff --git a/arch/arm64/include/asm/archrandom.h b/arch/arm64/include/asm/archrandom.h index c3b9fa56af67..109e2a4454be 100644 --- a/arch/arm64/include/asm/archrandom.h +++ b/arch/arm64/include/asm/archrandom.h @@ -58,7 +58,7 @@ static inline bool __arm64_rndrrs(unsigned long *v) return ok; } -static inline bool __must_check arch_get_random_long(unsigned long *v) +static inline size_t __must_check arch_get_random_longs(unsigned long *v, size_t max_longs) { /* * Only support the generic interface after we have detected @@ -66,27 +66,15 @@ static inline bool __must_check arch_get_random_long(unsigned long *v) * cpufeature code and with potential scheduling between CPUs * with and without the feature. */ - if (cpus_have_const_cap(ARM64_HAS_RNG) && __arm64_rndr(v)) - return true; - return false; + if (max_longs && cpus_have_const_cap(ARM64_HAS_RNG) && __arm64_rndr(v)) + return 1; + return 0; } -static inline bool __must_check arch_get_random_int(unsigned int *v) +static inline size_t __must_check arch_get_random_seed_longs(unsigned long *v, size_t max_longs) { - if (cpus_have_const_cap(ARM64_HAS_RNG)) { - unsigned long val; - - if (__arm64_rndr(&val)) { - *v = val; - return true; - } - } - return false; -} - -static inline bool __must_check arch_get_random_seed_long(unsigned long *v) -{ - struct arm_smccc_res res; + if (!max_longs) + return 0; /* * We prefer the SMCCC call, since its semantics (return actual @@ -95,10 +83,23 @@ static inline bool __must_check arch_get_random_seed_long(unsigned long *v) * (the output of a pseudo RNG freshly seeded by a TRNG). */ if (smccc_trng_available) { - arm_smccc_1_1_invoke(ARM_SMCCC_TRNG_RND64, 64, &res); + struct arm_smccc_res res; + + max_longs = min_t(size_t, 3, max_longs); + arm_smccc_1_1_invoke(ARM_SMCCC_TRNG_RND64, max_longs * 64, &res); if ((int)res.a0 >= 0) { - *v = res.a3; - return true; + switch (max_longs) { + case 3: + *v++ = res.a1; + fallthrough; + case 2: + *v++ = res.a2; + fallthrough; + case 1: + *v++ = res.a3; + break; + } + return max_longs; } } @@ -108,32 +109,9 @@ static inline bool __must_check arch_get_random_seed_long(unsigned long *v) * enough to implement this API if no other entropy source exists. */ if (cpus_have_const_cap(ARM64_HAS_RNG) && __arm64_rndrrs(v)) - return true; + return 1; - return false; -} - -static inline bool __must_check arch_get_random_seed_int(unsigned int *v) -{ - struct arm_smccc_res res; - unsigned long val; - - if (smccc_trng_available) { - arm_smccc_1_1_invoke(ARM_SMCCC_TRNG_RND64, 32, &res); - if ((int)res.a0 >= 0) { - *v = res.a3 & GENMASK(31, 0); - return true; - } - } - - if (cpus_have_const_cap(ARM64_HAS_RNG)) { - if (__arm64_rndrrs(&val)) { - *v = val; - return true; - } - } - - return false; + return 0; } static inline bool __init __early_cpu_has_rndr(void) @@ -143,26 +121,40 @@ static inline bool __init __early_cpu_has_rndr(void) return (ftr >> ID_AA64ISAR0_EL1_RNDR_SHIFT) & 0xf; } -static inline bool __init __must_check -arch_get_random_seed_long_early(unsigned long *v) +static inline size_t __init __must_check +arch_get_random_seed_longs_early(unsigned long *v, size_t max_longs) { WARN_ON(system_state != SYSTEM_BOOTING); + if (!max_longs) + return 0; + if (smccc_trng_available) { struct arm_smccc_res res; - arm_smccc_1_1_invoke(ARM_SMCCC_TRNG_RND64, 64, &res); + max_longs = min_t(size_t, 3, max_longs); + arm_smccc_1_1_invoke(ARM_SMCCC_TRNG_RND64, max_longs * 64, &res); if ((int)res.a0 >= 0) { - *v = res.a3; - return true; + switch (max_longs) { + case 3: + *v++ = res.a1; + fallthrough; + case 2: + *v++ = res.a2; + fallthrough; + case 1: + *v++ = res.a3; + break; + } + return max_longs; } } if (__early_cpu_has_rndr() && __arm64_rndr(v)) - return true; + return 1; - return false; + return 0; } -#define arch_get_random_seed_long_early arch_get_random_seed_long_early +#define arch_get_random_seed_longs_early arch_get_random_seed_longs_early #endif /* _ASM_ARCHRANDOM_H */ diff --git a/arch/arm64/kernel/kaslr.c b/arch/arm64/kernel/kaslr.c index 418b2bba1521..c5d541f358d3 100644 --- a/arch/arm64/kernel/kaslr.c +++ b/arch/arm64/kernel/kaslr.c @@ -106,7 +106,7 @@ u64 __init kaslr_early_init(void) * and supported. */ - if (arch_get_random_seed_long_early(&raw)) + if (arch_get_random_seed_longs_early(&raw, 1)) seed ^= raw; if (!seed) { diff --git a/arch/powerpc/include/asm/archrandom.h b/arch/powerpc/include/asm/archrandom.h index 25ba65df6b1a..0e365c5b2396 100644 --- a/arch/powerpc/include/asm/archrandom.h +++ b/arch/powerpc/include/asm/archrandom.h @@ -4,34 +4,16 @@ #include -static inline bool __must_check arch_get_random_long(unsigned long *v) +static inline size_t __must_check arch_get_random_longs(unsigned long *v, size_t max_longs) { - return false; + return 0; } -static inline bool __must_check arch_get_random_int(unsigned int *v) +static inline size_t __must_check arch_get_random_seed_longs(unsigned long *v, size_t max_longs) { - return false; -} - -static inline bool __must_check arch_get_random_seed_long(unsigned long *v) -{ - if (ppc_md.get_random_seed) - return ppc_md.get_random_seed(v); - - return false; -} - -static inline bool __must_check arch_get_random_seed_int(unsigned int *v) -{ - unsigned long val; - bool rc; - - rc = arch_get_random_seed_long(&val); - if (rc) - *v = val; - - return rc; + if (max_longs && ppc_md.get_random_seed && ppc_md.get_random_seed(v)) + return 1; + return 0; } #ifdef CONFIG_PPC_POWERNV diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index e08fb3124dca..631062cde6b4 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -1207,7 +1207,7 @@ int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu) break; #endif case H_RANDOM: - if (!arch_get_random_seed_long(&vcpu->arch.regs.gpr[4])) + if (!arch_get_random_seed_longs(&vcpu->arch.regs.gpr[4], 1)) ret = H_HARDWARE; break; case H_RPT_INVALIDATE: diff --git a/arch/s390/include/asm/archrandom.h b/arch/s390/include/asm/archrandom.h index 0a1c2e66c709..cf5e000df0a1 100644 --- a/arch/s390/include/asm/archrandom.h +++ b/arch/s390/include/asm/archrandom.h @@ -18,34 +18,19 @@ DECLARE_STATIC_KEY_FALSE(s390_arch_random_available); extern atomic64_t s390_arch_random_counter; -static inline bool __must_check arch_get_random_long(unsigned long *v) +static inline size_t __must_check arch_get_random_longs(unsigned long *v, size_t max_longs) { - return false; + return 0; } -static inline bool __must_check arch_get_random_int(unsigned int *v) -{ - return false; -} - -static inline bool __must_check arch_get_random_seed_long(unsigned long *v) +static inline size_t __must_check arch_get_random_seed_longs(unsigned long *v, size_t max_longs) { if (static_branch_likely(&s390_arch_random_available)) { - cpacf_trng(NULL, 0, (u8 *)v, sizeof(*v)); - atomic64_add(sizeof(*v), &s390_arch_random_counter); - return true; + cpacf_trng(NULL, 0, (u8 *)v, max_longs * sizeof(*v)); + atomic64_add(max_longs * sizeof(*v), &s390_arch_random_counter); + return max_longs; } - return false; -} - -static inline bool __must_check arch_get_random_seed_int(unsigned int *v) -{ - if (static_branch_likely(&s390_arch_random_available)) { - cpacf_trng(NULL, 0, (u8 *)v, sizeof(*v)); - atomic64_add(sizeof(*v), &s390_arch_random_counter); - return true; - } - return false; + return 0; } #endif /* _ASM_S390_ARCHRANDOM_H */ diff --git a/arch/um/include/asm/archrandom.h b/arch/um/include/asm/archrandom.h index 2f24cb96391d..24e16c979c51 100644 --- a/arch/um/include/asm/archrandom.h +++ b/arch/um/include/asm/archrandom.h @@ -7,24 +7,19 @@ /* This is from , but better not to #include that in a global header here. */ ssize_t os_getrandom(void *buf, size_t len, unsigned int flags); -static inline bool __must_check arch_get_random_long(unsigned long *v) +static inline size_t __must_check arch_get_random_longs(unsigned long *v, size_t max_longs) { - return os_getrandom(v, sizeof(*v), 0) == sizeof(*v); + ssize_t ret; + + ret = os_getrandom(v, max_longs * sizeof(*v), 0); + if (ret < 0) + return 0; + return ret / sizeof(*v); } -static inline bool __must_check arch_get_random_int(unsigned int *v) +static inline size_t __must_check arch_get_random_seed_longs(unsigned long *v, size_t max_longs) { - return os_getrandom(v, sizeof(*v), 0) == sizeof(*v); -} - -static inline bool __must_check arch_get_random_seed_long(unsigned long *v) -{ - return false; -} - -static inline bool __must_check arch_get_random_seed_int(unsigned int *v) -{ - return false; + return 0; } #endif diff --git a/arch/x86/include/asm/archrandom.h b/arch/x86/include/asm/archrandom.h index fb235b696175..02bae8e0758b 100644 --- a/arch/x86/include/asm/archrandom.h +++ b/arch/x86/include/asm/archrandom.h @@ -31,20 +31,6 @@ static inline bool __must_check rdrand_long(unsigned long *v) return false; } -static inline bool __must_check rdrand_int(unsigned int *v) -{ - bool ok; - unsigned int retry = RDRAND_RETRY_LOOPS; - do { - asm volatile("rdrand %[out]" - CC_SET(c) - : CC_OUT(c) (ok), [out] "=r" (*v)); - if (ok) - return true; - } while (--retry); - return false; -} - static inline bool __must_check rdseed_long(unsigned long *v) { bool ok; @@ -54,38 +40,19 @@ static inline bool __must_check rdseed_long(unsigned long *v) return ok; } -static inline bool __must_check rdseed_int(unsigned int *v) -{ - bool ok; - asm volatile("rdseed %[out]" - CC_SET(c) - : CC_OUT(c) (ok), [out] "=r" (*v)); - return ok; -} - /* * These are the generic interfaces; they must not be declared if the * stubs in are to be invoked. */ -static inline bool __must_check arch_get_random_long(unsigned long *v) +static inline size_t __must_check arch_get_random_longs(unsigned long *v, size_t max_longs) { - return static_cpu_has(X86_FEATURE_RDRAND) ? rdrand_long(v) : false; + return max_longs && static_cpu_has(X86_FEATURE_RDRAND) && rdrand_long(v) ? 1 : 0; } -static inline bool __must_check arch_get_random_int(unsigned int *v) +static inline size_t __must_check arch_get_random_seed_longs(unsigned long *v, size_t max_longs) { - return static_cpu_has(X86_FEATURE_RDRAND) ? rdrand_int(v) : false; -} - -static inline bool __must_check arch_get_random_seed_long(unsigned long *v) -{ - return static_cpu_has(X86_FEATURE_RDSEED) ? rdseed_long(v) : false; -} - -static inline bool __must_check arch_get_random_seed_int(unsigned int *v) -{ - return static_cpu_has(X86_FEATURE_RDSEED) ? rdseed_int(v) : false; + return max_longs && static_cpu_has(X86_FEATURE_RDSEED) && rdseed_long(v) ? 1 : 0; } #ifndef CONFIG_UML diff --git a/arch/x86/kernel/espfix_64.c b/arch/x86/kernel/espfix_64.c index 4fe7af58cfe1..9417d5aa7305 100644 --- a/arch/x86/kernel/espfix_64.c +++ b/arch/x86/kernel/espfix_64.c @@ -100,7 +100,7 @@ static void init_espfix_random(void) * This is run before the entropy pools are initialized, * but this is hopefully better than nothing. */ - if (!arch_get_random_long(&rand)) { + if (!arch_get_random_longs(&rand, 1)) { /* The constant is an arbitrary large prime */ rand = rdtsc(); rand *= 0xc345c6b72fd16123UL; diff --git a/drivers/char/random.c b/drivers/char/random.c index 0c6568ae5f68..7bf11fa66265 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -596,12 +596,20 @@ static void extract_entropy(void *buf, size_t len) unsigned long rdseed[32 / sizeof(long)]; size_t counter; } block; - size_t i; + size_t i, longs; - for (i = 0; i < ARRAY_SIZE(block.rdseed); ++i) { - if (!arch_get_random_seed_long(&block.rdseed[i]) && - !arch_get_random_long(&block.rdseed[i])) - block.rdseed[i] = random_get_entropy(); + for (i = 0; i < ARRAY_SIZE(block.rdseed);) { + longs = arch_get_random_seed_longs(&block.rdseed[i], ARRAY_SIZE(block.rdseed) - i); + if (longs) { + i += longs; + continue; + } + longs = arch_get_random_longs(&block.rdseed[i], ARRAY_SIZE(block.rdseed) - i); + if (longs) { + i += longs; + continue; + } + block.rdseed[i++] = random_get_entropy(); } spin_lock_irqsave(&input_pool.lock, flags); @@ -776,22 +784,31 @@ static struct notifier_block pm_notifier = { .notifier_call = random_pm_notifica int __init random_init(const char *command_line) { ktime_t now = ktime_get_real(); - unsigned int i, arch_bits; - unsigned long entropy; + size_t i, longs, arch_bits; + unsigned long entropy[BLAKE2S_BLOCK_SIZE / sizeof(long)]; #if defined(LATENT_ENTROPY_PLUGIN) static const u8 compiletime_seed[BLAKE2S_BLOCK_SIZE] __initconst __latent_entropy; _mix_pool_bytes(compiletime_seed, sizeof(compiletime_seed)); #endif - for (i = 0, arch_bits = BLAKE2S_BLOCK_SIZE * 8; - i < BLAKE2S_BLOCK_SIZE; i += sizeof(entropy)) { - if (!arch_get_random_seed_long_early(&entropy) && - !arch_get_random_long_early(&entropy)) { - entropy = random_get_entropy(); - arch_bits -= sizeof(entropy) * 8; + for (i = 0, arch_bits = sizeof(entropy) * 8; i < ARRAY_SIZE(entropy);) { + longs = arch_get_random_seed_longs(entropy, ARRAY_SIZE(entropy) - i); + if (longs) { + _mix_pool_bytes(entropy, sizeof(*entropy) * longs); + i += longs; + continue; } - _mix_pool_bytes(&entropy, sizeof(entropy)); + longs = arch_get_random_longs(entropy, ARRAY_SIZE(entropy) - i); + if (longs) { + _mix_pool_bytes(entropy, sizeof(*entropy) * longs); + i += longs; + continue; + } + entropy[0] = random_get_entropy(); + _mix_pool_bytes(entropy, sizeof(*entropy)); + arch_bits -= sizeof(*entropy) * 8; + ++i; } _mix_pool_bytes(&now, sizeof(now)); _mix_pool_bytes(utsname(), sizeof(*(utsname()))); diff --git a/include/asm-generic/archrandom.h b/include/asm-generic/archrandom.h index 3a5ee202dd86..3cd7f980cfdc 100644 --- a/include/asm-generic/archrandom.h +++ b/include/asm-generic/archrandom.h @@ -2,24 +2,14 @@ #ifndef __ASM_GENERIC_ARCHRANDOM_H__ #define __ASM_GENERIC_ARCHRANDOM_H__ -static inline bool __must_check arch_get_random_long(unsigned long *v) +static inline size_t __must_check arch_get_random_longs(unsigned long *v, size_t max_longs) { - return false; + return 0; } -static inline bool __must_check arch_get_random_int(unsigned int *v) +static inline size_t __must_check arch_get_random_seed_longs(unsigned long *v, size_t max_longs) { - return false; -} - -static inline bool __must_check arch_get_random_seed_long(unsigned long *v) -{ - return false; -} - -static inline bool __must_check arch_get_random_seed_int(unsigned int *v) -{ - return false; + return 0; } #endif diff --git a/include/linux/random.h b/include/linux/random.h index 865770e29f3e..3fec206487f6 100644 --- a/include/linux/random.h +++ b/include/linux/random.h @@ -112,19 +112,19 @@ declare_get_random_var_wait(long, unsigned long) * Called from the boot CPU during startup; not valid to call once * secondary CPUs are up and preemption is possible. */ -#ifndef arch_get_random_seed_long_early -static inline bool __init arch_get_random_seed_long_early(unsigned long *v) +#ifndef arch_get_random_seed_longs_early +static inline size_t __init arch_get_random_seed_longs_early(unsigned long *v, size_t max_longs) { WARN_ON(system_state != SYSTEM_BOOTING); - return arch_get_random_seed_long(v); + return arch_get_random_seed_longs(v, max_longs); } #endif -#ifndef arch_get_random_long_early -static inline bool __init arch_get_random_long_early(unsigned long *v) +#ifndef arch_get_random_longs_early +static inline bool __init arch_get_random_longs_early(unsigned long *v, size_t max_longs) { WARN_ON(system_state != SYSTEM_BOOTING); - return arch_get_random_long(v); + return arch_get_random_longs(v, max_longs); } #endif From da1d9caf95def6f0320819cf941c9fd1069ba9e1 Mon Sep 17 00:00:00 2001 From: Federico Vaga Date: Sat, 2 Jul 2022 23:08:20 +0200 Subject: [PATCH 298/318] doc:it_IT: align Italian documentation Translation for the following patches commit df05c0e9496c ("Documentation: Raise the minimum supported version of LLVM to 11.0.0") commit 333b11e541fe ("Documentation: Add minimum pahole version") commit 6d6a8d6a4ed0 ("docs: Update Sphinx requirements") commit 76ae847497bc ("Documentation: raise minimum supported version of GCC to 5.1") commit 59c6a716b14b ("Documentation/process/maintainer-pgp-guide: Replace broken link to PGP path finder") commit 85eafc63d032 ("docs: update file link location") commit 869f496e1aa6 ("docs: process: submitting-patches: Clarify the Reported-by usage") commit 6c5ccd24ff17 ("Remove mentions of the Trivial Patch Monkey") commit aa9b5e0df226 ("Documentation/process: fix self reference") commit b96ff02ab2be ("Documentation/process: fix a cross reference") commit 1f57bd42b77c ("docs: submitting-patches: make section about the Link: tag more explicit") commit a9d85efb25fb ("docs: use the lore redirector everywhere") commit 31c9d7c82975 ("Documentation/process: Add tip tree handbook") commit 604370e106cc ("Documentation/process: Add maintainer handbooks section") commit bf33a9d42d0c ("docs: 5.Posting.rst: describe Fixes: and Link: tags") commit c04639a7d2fb ("coding-style.rst: trivial: fix location of driver model macros") commit d5b421fe0282 ("docs: Explain the desired position of function attributes") commit 3577cdb23b8f ("docs: deprecated.rst: Clarify open-coded arithmetic with literals") commit db67eb748e7a ("docs: discourage use of list tables") commit 0e805b118662 ("docs: address some text issues with css/theme support") commit 135707d3765e ("docs: allow to pass extra DOCS_CSS themes via make") commit fe450eeb4e6f ("Documentation: in_irq() cleanup") commit 10855b45a428 ("docs: fix typo in Documentation/kernel-hacking/locking.rst") commit bc67f1c454fb ("docs: futex: Fix kernel-doc references") commit abf36fe0be7d ("docs: kernel-hacking: Remove inappropriate text") commit f35cf1a59e9a ("Documentation: kernel-hacking: minor edits for style") commit f35cf1a59e9a ("Documentation: kernel-hacking: minor edits for style") commit 980c3799c500 ("Documentation: kernel-doc: Promote two chapter headings to page title") commit e1be43d9b5d0 ("overflow: Implement size_t saturating arithmetic helpers") commit 615f3eea0d5f ("Documentation: add note block surrounding security patch note") commit 587d39b260c4 ("Documentation: add link to stable release candidate tree") commit 555d44932c67 ("Documentation: update stable tree link") commit 88d99e870143 ("Documentation: update stable review cycle documentation") commit 0c603a5c704f ("Documentation/process: mention patch changelog in review process") commit 6d5aa418b3bd ("docs: submitting-patches: Fix crossref to 'The canonical patch format'") commit f1a693994b1c ("Documentation/process: use scripts/get_maintainer.pl on patches") commit 69ef0920bdd3 ("Docs: Add cpio requirement to changes.rst") commit 5a5866c28b43 ("Docs: Replace version by 'current' in changes.rst") commit 9b5a7f4a2a8d ("x86/configs: Add x86 debugging Kconfig fragment plus docs") commit f1a693994b1c ("Documentation/process: use scripts/get_maintainer.pl on patches") commit e8c07082a810 ("Kbuild: move to -std=gnu11") Signed-off-by: Federico Vaga Link: https://lore.kernel.org/r/20220702210820.13118-1-federico.vaga@vaga.pv.it Signed-off-by: Jonathan Corbet --- .../bindings/submitting-patches.rst | 11 ++++ .../it_IT/doc-guide/kernel-doc.rst | 2 + .../translations/it_IT/doc-guide/sphinx.rst | 18 +++-- .../it_IT/kernel-hacking/hacking.rst | 24 ++++--- .../it_IT/kernel-hacking/locking.rst | 14 +--- .../it_IT/maintainer/configure-git.rst | 10 +++ .../it_IT/process/3.Early-stage.rst | 17 ++--- .../translations/it_IT/process/5.Posting.rst | 27 ++++++-- .../translations/it_IT/process/changes.rst | 25 +++++-- .../it_IT/process/coding-style.rst | 42 +++++++++++- .../translations/it_IT/process/deprecated.rst | 24 +++++-- .../translations/it_IT/process/index.rst | 1 + .../it_IT/process/maintainer-handbooks.rst | 24 +++++++ .../it_IT/process/maintainer-pgp-guide.rst | 14 ++-- .../it_IT/process/maintainer-tip.rst | 10 +++ .../it_IT/process/maintainers.rst | 13 ++++ .../it_IT/process/stable-kernel-rules.rst | 42 ++++++++---- .../it_IT/process/submitting-patches.rst | 66 +++++++++++++------ 18 files changed, 288 insertions(+), 96 deletions(-) create mode 100644 Documentation/translations/it_IT/devicetree/bindings/submitting-patches.rst create mode 100644 Documentation/translations/it_IT/maintainer/configure-git.rst create mode 100644 Documentation/translations/it_IT/process/maintainer-handbooks.rst create mode 100644 Documentation/translations/it_IT/process/maintainer-tip.rst create mode 100644 Documentation/translations/it_IT/process/maintainers.rst diff --git a/Documentation/translations/it_IT/devicetree/bindings/submitting-patches.rst b/Documentation/translations/it_IT/devicetree/bindings/submitting-patches.rst new file mode 100644 index 000000000000..b473cd2190be --- /dev/null +++ b/Documentation/translations/it_IT/devicetree/bindings/submitting-patches.rst @@ -0,0 +1,11 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. include:: ../../disclaimer-ita.rst + +:Original: Documentation/devicetree/bindings/submitting-patches.rst + +================================================ +Sottomettere patch per devicetree (DT) *binding* +================================================ + +.. note:: to be translated diff --git a/Documentation/translations/it_IT/doc-guide/kernel-doc.rst b/Documentation/translations/it_IT/doc-guide/kernel-doc.rst index 009cdac014b6..78082281acf9 100644 --- a/Documentation/translations/it_IT/doc-guide/kernel-doc.rst +++ b/Documentation/translations/it_IT/doc-guide/kernel-doc.rst @@ -5,6 +5,7 @@ .. _it_kernel_doc: +================================= Scrivere i commenti in kernel-doc ================================= @@ -469,6 +470,7 @@ Il titolo che segue ``DOC:`` funziona da intestazione all'interno del file sorgente, ma anche come identificatore per l'estrazione di questi commenti di documentazione. Quindi, il titolo dev'essere unico all'interno del file. +======================================= Includere i commenti di tipo kernel-doc ======================================= diff --git a/Documentation/translations/it_IT/doc-guide/sphinx.rst b/Documentation/translations/it_IT/doc-guide/sphinx.rst index 9762452c584c..64528790dc34 100644 --- a/Documentation/translations/it_IT/doc-guide/sphinx.rst +++ b/Documentation/translations/it_IT/doc-guide/sphinx.rst @@ -5,8 +5,9 @@ .. _it_sphinxdoc: -Introduzione -============ +============================================= +Usare Sphinx per la documentazione del kernel +============================================= Il kernel Linux usa `Sphinx`_ per la generazione della documentazione a partire dai file `reStructuredText`_ che si trovano nella cartella ``Documentation``. @@ -158,6 +159,9 @@ Per poter passare ulteriori opzioni a Sphinx potete utilizzare la variabile make ``SPHINXOPTS``. Per esempio, se volete che Sphinx sia più verboso durante la generazione potete usare il seguente comando ``make SPHINXOPTS=-v htmldocs``. +Potete anche personalizzare l'ouptut html passando un livello aggiuntivo +DOCS_CSS usando la rispettiva variabile d'ambiente ``DOCS_CSS``. + Potete eliminare la documentazione generata tramite il comando ``make cleandocs``. @@ -276,11 +280,11 @@ incrociato quando questa ha una voce nell'indice. Se trovate degli usi di Tabelle a liste --------------- -Raccomandiamo l'uso delle tabelle in formato lista (*list table*). Le tabelle -in formato lista sono liste di liste. In confronto all'ASCII-art potrebbero -non apparire di facile lettura nei file in formato testo. Il loro vantaggio è -che sono facili da creare o modificare e che la differenza di una modifica è -molto più significativa perché limitata alle modifiche del contenuto. +Il formato ``list-table`` può essere utile per tutte quelle tabelle che non +possono essere facilmente scritte usando il formato ASCII-art di Sphinx. Però, +questo genere di tabelle sono illeggibili per chi legge direttamente i file di +testo. Dunque, questo formato dovrebbe essere evitato senza forti argomenti che +ne giustifichino l'uso. La ``flat-table`` è anch'essa una lista di liste simile alle ``list-table`` ma con delle funzionalità aggiuntive: diff --git a/Documentation/translations/it_IT/kernel-hacking/hacking.rst b/Documentation/translations/it_IT/kernel-hacking/hacking.rst index 4bec4669cf48..560f1d0482d2 100644 --- a/Documentation/translations/it_IT/kernel-hacking/hacking.rst +++ b/Documentation/translations/it_IT/kernel-hacking/hacking.rst @@ -129,8 +129,7 @@ eseguiti simultaneamente. .. warning:: Il nome 'tasklet' è ingannevole: non hanno niente a che fare - con i 'processi' ('tasks'), e probabilmente hanno più a che vedere - con qualche pessima vodka che Alexey Kuznetsov si fece a quel tempo. + con i 'processi' ('tasks'). Potete determinate se siete in un softirq (o tasklet) utilizzando la macro :c:func:`in_softirq()` (``include/linux/preempt.h``). @@ -308,7 +307,7 @@ esse copiano una quantità arbitraria di dati da e verso lo spazio utente. Al contrario di:c:func:`put_user()` e :c:func:`get_user()`, queste funzioni ritornano la quantità di dati copiati (0 è comunque un successo). -[Sì, questa stupida interfaccia mi imbarazza. La battaglia torna in auge anno +[Sì, questa interfaccia mi imbarazza. La battaglia torna in auge anno dopo anno. --RR] Le funzioni potrebbero dormire implicitamente. Queste non dovrebbero mai essere @@ -679,9 +678,8 @@ tutti sulle spine: questo riflette cambiamenti fondamentati (eg. la funzione non può più essere chiamata con le funzioni attive, o fa controlli aggiuntivi, o non fa più controlli che venivano fatti in precedenza). Solitamente a questo s'accompagna un'adeguata e completa nota sulla lista di discussone -linux-kernel; cercate negli archivi. -Solitamente eseguire una semplice sostituzione su tutto un file rendere -le cose **peggiori**. +più adatta; cercate negli archivi. Solitamente eseguire una semplice +sostituzione su tutto un file rendere le cose **peggiori**. Inizializzazione dei campi d'una struttura ------------------------------------------ @@ -759,14 +757,14 @@ Mettere le vostre cose nel kernel Al fine d'avere le vostre cose in ordine per l'inclusione ufficiale, o anche per avere patch pulite, c'è del lavoro amministrativo da fare: -- Trovare di chi è lo stagno in cui state pisciando. Guardare in cima +- Trovare chi è responsabile del codice che state modificando. Guardare in cima ai file sorgenti, all'interno del file ``MAINTAINERS``, ed alla fine di tutti nel file ``CREDITS``. Dovreste coordinarvi con queste persone per evitare di duplicare gli sforzi, o provare qualcosa che è già stato rigettato. Assicuratevi di mettere il vostro nome ed indirizzo email in cima a - tutti i file che create o che mangeggiate significativamente. Questo è + tutti i file che create o che maneggiate significativamente. Questo è il primo posto dove le persone guarderanno quando troveranno un baco, o quando **loro** vorranno fare una modifica. @@ -787,12 +785,12 @@ anche per avere patch pulite, c'è del lavoro amministrativo da fare: "obj-$(CONFIG_xxx) += xxx.o". La sintassi è documentata nel file ``Documentation/kbuild/makefiles.rst``. -- Aggiungete voi stessi in ``CREDITS`` se avete fatto qualcosa di notevole, - solitamente qualcosa che supera il singolo file (comunque il vostro nome - dovrebbe essere all'inizio dei file sorgenti). ``MAINTAINERS`` significa +- Aggiungete voi stessi in ``CREDITS`` se credete di aver fatto qualcosa di + notevole, solitamente qualcosa che supera il singolo file (comunque il vostro + nome dovrebbe essere all'inizio dei file sorgenti). ``MAINTAINERS`` significa che volete essere consultati quando vengono fatte delle modifiche ad un - sottosistema, e quando ci sono dei bachi; questo implica molto di più - di un semplice impegno su una parte del codice. + sottosistema, e quando ci sono dei bachi; questo implica molto di più di un + semplice impegno su una parte del codice. - Infine, non dimenticatevi di leggere ``Documentation/process/submitting-patches.rst``. diff --git a/Documentation/translations/it_IT/kernel-hacking/locking.rst b/Documentation/translations/it_IT/kernel-hacking/locking.rst index 163f1bd4e857..51af37f2d621 100644 --- a/Documentation/translations/it_IT/kernel-hacking/locking.rst +++ b/Documentation/translations/it_IT/kernel-hacking/locking.rst @@ -102,16 +102,11 @@ che non esistano. Sincronizzazione nel kernel Linux ================================= -Se posso darvi un suggerimento: non dormite mai con qualcuno più pazzo di -voi. Ma se dovessi darvi un suggerimento sulla sincronizzazione: -**mantenetela semplice**. +Se dovessi darvi un suggerimento sulla sincronizzazione: **mantenetela +semplice**. Siate riluttanti nell'introduzione di nuovi *lock*. -Abbastanza strano, quest'ultimo è l'esatto opposto del mio suggerimento -su quando **avete** dormito con qualcuno più pazzo di voi. E dovreste -pensare a prendervi un cane bello grande. - I due principali tipi di *lock* nel kernel: spinlock e mutex ------------------------------------------------------------ @@ -316,7 +311,7 @@ Pete Zaitcev ci offre il seguente riassunto: - Se siete in un contesto utente (una qualsiasi chiamata di sistema) e volete sincronizzarvi con altri processi, usate i mutex. Potete trattenere - il mutex e dormire (``copy_from_user*(`` o ``kmalloc(x,GFP_KERNEL)``). + il mutex e dormire (``copy_from_user(`` o ``kmalloc(x,GFP_KERNEL)``). - Altrimenti (== i dati possono essere manipolati da un'interruzione) usate spin_lock_irqsave() e spin_unlock_irqrestore(). @@ -979,9 +974,6 @@ fallisce nel trovare quello che vuole, quindi rilascia il *lock* di lettura, trattiene un *lock* di scrittura ed inserisce un oggetto; questo genere di codice presenta una corsa critica. -Se non riuscite a capire il perché, per favore state alla larga dal mio -codice. - corsa fra temporizzatori: un passatempo del kernel -------------------------------------------------- diff --git a/Documentation/translations/it_IT/maintainer/configure-git.rst b/Documentation/translations/it_IT/maintainer/configure-git.rst new file mode 100644 index 000000000000..8316fa53946f --- /dev/null +++ b/Documentation/translations/it_IT/maintainer/configure-git.rst @@ -0,0 +1,10 @@ +.. include:: ../disclaimer-ita.rst + +:Original: Documentation/process/botching-up-ioctls.rst + +.. _it_configuregit: + +Configurare Git +=============== + +.. note:: To be translated diff --git a/Documentation/translations/it_IT/process/3.Early-stage.rst b/Documentation/translations/it_IT/process/3.Early-stage.rst index 443ac1e5558f..0809de39108a 100644 --- a/Documentation/translations/it_IT/process/3.Early-stage.rst +++ b/Documentation/translations/it_IT/process/3.Early-stage.rst @@ -168,14 +168,15 @@ in questa ricerca: .../scripts/get_maintainer.pl -Se questo script viene eseguito con l'opzione "-f" ritornerà il -manutentore(i) attuale per un dato file o cartella. Se viene passata una -patch sulla linea di comando, lo script elencherà i manutentori che -dovrebbero riceverne una copia. Ci sono svariate opzioni che regolano -quanto a fondo get_maintainer.pl debba cercare i manutentori; -siate quindi prudenti nell'utilizzare le opzioni più aggressive poiché -potreste finire per includere sviluppatori che non hanno un vero interesse -per il codice che state modificando. +Se questo script viene eseguito con l'opzione "-f" ritornerà il manutentore(i) +attuale per un dato file o cartella. Se viene passata una patch sulla linea di +comando, lo script elencherà i manutentori che dovrebbero riceverne una copia. +Questo è la maniera raccomandata (non quella con "-f") per ottenere la lista di +persone da aggiungere a Cc per le vostre patch. Ci sono svariate opzioni che +regolano quanto a fondo get_maintainer.pl debba cercare i manutentori; siate +quindi prudenti nell'utilizzare le opzioni più aggressive poiché potreste finire +per includere sviluppatori che non hanno un vero interesse per il codice che +state modificando. Se tutto ciò dovesse fallire, parlare con Andrew Morton potrebbe essere un modo efficace per capire chi è il manutentore di un dato pezzo di codice. diff --git a/Documentation/translations/it_IT/process/5.Posting.rst b/Documentation/translations/it_IT/process/5.Posting.rst index a036f38fc82e..cf92a16ed7e5 100644 --- a/Documentation/translations/it_IT/process/5.Posting.rst +++ b/Documentation/translations/it_IT/process/5.Posting.rst @@ -213,13 +213,28 @@ irrilevanti (quelli generati dal processo di generazione, per esempio, o i file di backup del vostro editor). Il file "dontdiff" nella cartella Documentation potrà esservi d'aiuto su questo punto; passatelo a diff con l'opzione "-X". -Le etichette sopra menzionante sono usate per descrivere come i vari -sviluppatori sono stati associati allo sviluppo di una patch. Sono descritte -in dettaglio nel documento :ref:`translations/it_IT/process/submitting-patches.rst `; -quello che segue è un breve riassunto. Ognuna di queste righe ha il seguente -formato: +Le etichette sopracitate danno un'idea di come una patch prende vita e sono +descritte nel dettaglio nel documento +:ref:`Documentation/translations/it_IT/process/submitting-patches.rst `. +Qui di seguito un breve riassunto. -:: +Un'etichetta ci può dire quale commit ha introdotto il problema che viene corretto nella patch:: + + Fixes: 1f2e3d4c5b6a ("The first line of the commit specified by the first 12 characters of its SHA-1 ID") + +Un'altra etichetta viene usata per fornire collegamenti a pagine web contenenti +maggiori informazioni, per esempio un rapporto circa il baco risolto dalla +patch, oppure un documento con le specifiche implementate dalla patch:: + + Link: https://example.com/somewhere.html optional-other-stuff + +Alcuni manutentori aggiungono quest'etichetta alla patch per fare riferimento +alla più recente discussione pubblica. A volte questo è fatto automaticamente da +alcuni strumenti come b4 or un *hook* git come quello descritto qui +'Documentation/translations/it_IT/maintainer/configure-git.rst' + +Un terzo tipo di etichetta viene usato per indicare chi ha contribuito allo +sviluppo della patch. Tutte queste etichette seguono il formato:: tag: Full Name optional-other-stuff diff --git a/Documentation/translations/it_IT/process/changes.rst b/Documentation/translations/it_IT/process/changes.rst index dc7193377b7f..10e0ef9c34b7 100644 --- a/Documentation/translations/it_IT/process/changes.rst +++ b/Documentation/translations/it_IT/process/changes.rst @@ -11,8 +11,8 @@ Requisiti minimi per compilare il kernel Introduzione ============ -Questo documento fornisce una lista dei software necessari per eseguire i -kernel 4.x. +Questo documento fornisce una lista dei software necessari per eseguire questa +versione del kernel. Questo documento è basato sul file "Changes" del kernel 2.0.x e quindi le persone che lo scrissero meritano credito (Jared Mauch, Axel Boldt, @@ -32,12 +32,13 @@ PC Card, per esempio, probabilmente non dovreste preoccuparvi di pcmciautils. ====================== ================= ======================================== Programma Versione minima Comando per verificare la versione ====================== ================= ======================================== -GNU C 4.9 gcc --version -Clang/LLVM (optional) 10.0.1 clang --version +GNU C 5.1 gcc --version +Clang/LLVM (optional) 11.0.0 clang --version GNU make 3.81 make --version binutils 2.23 ld -v flex 2.5.35 flex --version bison 2.0 bison --version +pahole 1.16 pahole --version util-linux 2.10o fdformat --version kmod 13 depmod -V e2fsprogs 1.41.4 e2fsck -V @@ -58,6 +59,7 @@ iptables 1.4.2 iptables -V openssl & libcrypto 1.0.0 openssl version bc 1.06.95 bc --version Sphinx\ [#f1]_ 1.7 sphinx-build --version +cpio any cpio --version ====================== ================= ======================================== .. [#f1] Sphinx è necessario solo per produrre la documentazione del Kernel @@ -111,6 +113,16 @@ Bison Dalla versione 4.16, il sistema di compilazione, durante l'esecuzione, genera un parsificatore. Questo richiede bison 2.0 o successivo. +pahole +------ + +Dalla versione 5.2, quando viene impostato CONFIG_DEBUG_INFO_BTF, il sistema di +compilazione genera BTF (BPF Type Format) a partire da DWARF per vmlinux. Più +tardi anche per i moduli. Questo richiede pahole v1.16 o successivo. + +A seconda della distribuzione, lo si può trovare nei pacchetti 'dwarves' o +'pahole'. Oppure lo si può trovare qui: https://fedorapeople.org/~acme/dwarves/. + Perl ---- @@ -455,6 +467,11 @@ mcelog - +cpio +---- + +- + Rete **** diff --git a/Documentation/translations/it_IT/process/coding-style.rst b/Documentation/translations/it_IT/process/coding-style.rst index ecc74ba50d3e..a393ee4182af 100644 --- a/Documentation/translations/it_IT/process/coding-style.rst +++ b/Documentation/translations/it_IT/process/coding-style.rst @@ -466,14 +466,52 @@ la riga della parentesi graffa di chiusura. Ad esempio: } EXPORT_SYMBOL(system_is_up); +6.1) Prototipi di funzione +************************** + Nei prototipi di funzione, includete i nomi dei parametri e i loro tipi. Nonostante questo non sia richiesto dal linguaggio C, in Linux viene preferito perché è un modo semplice per aggiungere informazioni importanti per il lettore. -Non usate la parola chiave ``extern`` coi prototipi di funzione perché +Non usate la parola chiave ``extern`` con le dichiarazioni di funzione perché rende le righe più lunghe e non è strettamente necessario. +Quando scrivete i prototipi di funzione mantenete `l'ordine degli elementi `_. + +Prendiamo questa dichiarazione di funzione come esempio:: + + __init void * __must_check action(enum magic value, size_t size, u8 count, + char *fmt, ...) __printf(4, 5) __malloc; + +L'ordine suggerito per gli elementi di un prototipo di funzione è il seguente: + +- classe d'archiviazione (in questo caso ``static __always_inline``. Da notare + che ``__always_inline`` è tecnicamente un attributo ma che viene trattato come + ``inline``) +- attributi della classe di archiviazione (in questo caso ``__init``, in altre + parole la sezione, ma anche cose tipo ``__cold``) +- il tipo di ritorno (in questo caso, ``void *``) +- attributi per il valore di ritorno (in questo caso, ``__must_check``) +- il nome della funzione (in questo caso, ``action``) +- i parametri della funzione(in questo caso, + ``(enum magic value, size_t size, u8 count, char *fmt, ...)``, + da notare che va messo anche il nome del parametro) +- attributi dei parametri (in questo caso, ``__printf(4, 5)``) +- attributi per il comportamento della funzione (in questo caso, ``__malloc_``) + +Notate che per la **definizione** di una funzione (il altre parole il corpo +della funzione), il compilatore non permette di usare gli attributi per i +parametri dopo i parametri. In questi casi, devono essere messi dopo gli +attributi della classe d'archiviazione (notate che la posizione di +``__printf(4,5)`` cambia rispetto alla **dichiarazione**):: + + static __always_inline __init __printf(4, 5) void * __must_check action(enum magic value, + size_t size, u8 count, char *fmt, ...) __malloc + { + ... + }*)**``)**``)``)``*)``)``)``)``*``)``)``)*) + 7) Centralizzare il ritorno delle funzioni ------------------------------------------ @@ -855,7 +893,7 @@ I messaggi del kernel non devono terminare con un punto fermo. Scrivere i numeri fra parentesi (%d) non migliora alcunché e per questo dovrebbero essere evitati. -Ci sono alcune macro per la diagnostica in che dovreste +Ci sono alcune macro per la diagnostica in che dovreste usare per assicurarvi che i messaggi vengano associati correttamente ai dispositivi e ai driver, e che siano etichettati correttamente: dev_err(), dev_warn(), dev_info(), e così via. Per messaggi che non sono associati ad diff --git a/Documentation/translations/it_IT/process/deprecated.rst b/Documentation/translations/it_IT/process/deprecated.rst index 987f45ee1804..febf83897783 100644 --- a/Documentation/translations/it_IT/process/deprecated.rst +++ b/Documentation/translations/it_IT/process/deprecated.rst @@ -69,8 +69,8 @@ dovrebbero essere fatto negli argomenti di funzioni di allocazione di memoria piccoli di quelli che il chiamante si aspettava. L'uso di questo modo di allocare può portare ad un overflow della memoria di heap e altri malfunzionamenti. (Si fa eccezione per valori numerici per i quali il -compilatore può generare avvisi circa un potenziale overflow. Tuttavia usare -i valori numerici come suggerito di seguito è innocuo). +compilatore può generare avvisi circa un potenziale overflow. Tuttavia, anche in +questi casi è preferibile riscrivere il codice come suggerito di seguito). Per esempio, non usate ``count * size`` come argomento:: @@ -80,6 +80,9 @@ Al suo posto, si dovrebbe usare l'allocatore a due argomenti:: foo = kmalloc_array(count, size, GFP_KERNEL); +Nello specifico, kmalloc() può essere sostituta da kmalloc_array(), e kzalloc() +da kcalloc(). + Se questo tipo di allocatore non è disponibile, allora dovrebbero essere usate le funzioni del tipo *saturate-on-overflow*:: @@ -100,9 +103,20 @@ Invece, usate la seguente funzione:: invitati a riorganizzare il vostro codice usando il `flexible array member <#zero-length-and-one-element-arrays>`_. -Per maggiori dettagli fate riferimento a array_size(), -array3_size(), e struct_size(), così come la famiglia di -funzioni check_add_overflow() e check_mul_overflow(). +Per altri calcoli, usate le funzioni size_mul(), size_add(), e size_sub(). Per +esempio, al posto di:: + + foo = krealloc(current_size + chunk_size * (count - 3), GFP_KERNEL); + +dovreste scrivere: + + foo = krealloc(size_add(current_size, + size_mul(chunk_size, + size_sub(count, 3))), GFP_KERNEL); + +Per maggiori dettagli fate riferimento a array3_size() e flex_array_size(), ma +anche le funzioni della famiglia check_mul_overflow(), check_add_overflow(), +check_sub_overflow(), e check_shl_overflow(). simple_strtol(), simple_strtoll(), simple_strtoul(), simple_strtoull() ---------------------------------------------------------------------- diff --git a/Documentation/translations/it_IT/process/index.rst b/Documentation/translations/it_IT/process/index.rst index b223b70a4a95..8d4e36a07ff4 100644 --- a/Documentation/translations/it_IT/process/index.rst +++ b/Documentation/translations/it_IT/process/index.rst @@ -46,6 +46,7 @@ degli sviluppatori: stable-kernel-rules submit-checklist kernel-docs + maintainers Ed infine, qui ci sono alcune guide più tecniche che son state messe qua solo perché non si è trovato un posto migliore. diff --git a/Documentation/translations/it_IT/process/maintainer-handbooks.rst b/Documentation/translations/it_IT/process/maintainer-handbooks.rst new file mode 100644 index 000000000000..d840145bcceb --- /dev/null +++ b/Documentation/translations/it_IT/process/maintainer-handbooks.rst @@ -0,0 +1,24 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. include:: ../disclaimer-ita.rst + +:Original: Documentation/process/maintainer-handbooks.rst +:Translator: Federico Vaga + +.. _it_maintainer_handbooks_main: + +Note sul processo di sviluppo dei sottosistemi e dei sorgenti dei manutentori +============================================================================= + +Lo scopo di questo documento è quello di fornire informazioni sul processo di +sviluppo dedicate ai sottosistemi che vanno ad integrare quelle più generali +descritte in :ref:`Documentation/translations/it_IT/process +`. + +Indice: + +.. toctree:: + :numbered: + :maxdepth: 2 + + maintainer-tip diff --git a/Documentation/translations/it_IT/process/maintainer-pgp-guide.rst b/Documentation/translations/it_IT/process/maintainer-pgp-guide.rst index f3c8e8d377ee..a1e98ec9532e 100644 --- a/Documentation/translations/it_IT/process/maintainer-pgp-guide.rst +++ b/Documentation/translations/it_IT/process/maintainer-pgp-guide.rst @@ -931,12 +931,11 @@ che avete nel vostro portachiavi:: uid [ unknown] Linus Torvalds sub rsa2048 2011-09-20 [E] -Poi, aprite il `PGP pathfinder`_. Nel campo "From", incollate l'impronta -digitale della chiave di Linus Torvalds che si vede nell'output qui sopra. -Nel campo "to", incollate il key-id della chiave sconosciuta che avete -trovato con ``gpg --search``, e poi verificare il risultato: - -- `Finding paths to Linus`_ +Poi, cercate un percorso affidabile da Linux Torvalds alla chiave che avete +trovato con ``gpg --search`` usando la chiave sconosciuta.Per farlo potete usare +diversi strumenti come https://github.com/mricon/wotmate, +https://git.kernel.org/pub/scm/docs/kernel/pgpkeys.git/tree/graphs, e +https://the.earth.li/~noodles/pathfind.html. Se trovate un paio di percorsi affidabili è un buon segno circa la validità della chiave. Ora, potete aggiungerla al vostro portachiavi dal keyserver:: @@ -948,6 +947,3 @@ fiducia nell'amministratore del servizio *PGP Pathfinder* sperando che non sia malintenzionato (infatti, questo va contro :ref:`it_devs_not_infra`). Tuttavia, se mantenete con cura la vostra rete di fiducia sarà un deciso miglioramento rispetto alla cieca fiducia nei keyserver. - -.. _`PGP pathfinder`: https://pgp.cs.uu.nl/ -.. _`Finding paths to Linus`: https://pgp.cs.uu.nl/paths/79BE3E4300411886/to/C94035C21B4F2AEB.html diff --git a/Documentation/translations/it_IT/process/maintainer-tip.rst b/Documentation/translations/it_IT/process/maintainer-tip.rst new file mode 100644 index 000000000000..126f17ee541e --- /dev/null +++ b/Documentation/translations/it_IT/process/maintainer-tip.rst @@ -0,0 +1,10 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. include:: ../disclaimer-ita.rst + +:Original: Documentation/process/maintainer-tip.rst + +Il tascabile dei sorgenti tip +============================= + +.. note:: To be translated diff --git a/Documentation/translations/it_IT/process/maintainers.rst b/Documentation/translations/it_IT/process/maintainers.rst new file mode 100644 index 000000000000..3225f7c89fda --- /dev/null +++ b/Documentation/translations/it_IT/process/maintainers.rst @@ -0,0 +1,13 @@ +:Original: Documentation/process/maintainers.rst + +Lista dei manutentori e come inviare modifiche al kernel +======================================================== + +Questa pagina non verrà tradotta. Fate riferimento alla versione originale in +inglese. + +.. note:: La pagina originale usa una direttiva speciale per integrare il file + `MAINTAINERS` in sphinx. La parte di quel documento che si potrebbe + tradurre contiene comunque informazioni già presenti in + :ref:`Documentation/translations/it_IT/process/submitting-patches.rst + `. diff --git a/Documentation/translations/it_IT/process/stable-kernel-rules.rst b/Documentation/translations/it_IT/process/stable-kernel-rules.rst index 83f48afe48b9..0be675b03199 100644 --- a/Documentation/translations/it_IT/process/stable-kernel-rules.rst +++ b/Documentation/translations/it_IT/process/stable-kernel-rules.rst @@ -41,10 +41,10 @@ Regole sul tipo di patch che vengono o non vengono accettate nei sorgenti Procedura per sottomettere patch per i sorgenti -stable ------------------------------------------------------- - - Una patch di sicurezza non dovrebbero essere gestite (solamente) dal processo - di revisione -stable, ma dovrebbe seguire le procedure descritte in - :ref:`Documentation/translations/it_IT/admin-guide/security-bugs.rst `. - +.. note:: + Una patch di sicurezza non dovrebbe essere gestita (solamente) dal processo + di revisione -stable, ma dovrebbe seguire le procedure descritte in + :ref:`Documentation/translations/it_IT/admin-guide/security-bugs.rst `. Per tutte le altre sottomissioni, scegliere una delle seguenti procedure ------------------------------------------------------------------------ @@ -90,9 +90,9 @@ L':ref:`it_option_2` e l':ref:`it_option_3` sono più utili quando, al momento dell'inclusione dei sorgenti principali, si ritiene che non debbano essere incluse anche in quelli stabili (per esempio, perché si crede che si dovrebbero fare più verifiche per eventuali regressioni). L':ref:`it_option_3` è -particolarmente utile se la patch ha bisogno di qualche modifica per essere -applicata ad un kernel più vecchio (per esempio, perché nel frattempo l'API è -cambiata). +particolarmente utile se una patch dev'essere riportata su una versione +precedente (per esempio la patch richiede modifiche a causa di cambiamenti di +API). Notate che per l':ref:`it_option_3`, se la patch è diversa da quella nei sorgenti principali (per esempio perché è stato necessario un lavoro di @@ -167,9 +167,18 @@ Ciclo di una revisione della lista linux-kernel obietta la bontà della patch, sollevando problemi che i manutentori ed i membri non avevano compreso, allora la patch verrà rimossa dalla coda. - - Alla fine del ciclo di revisione tutte le patch che hanno ricevuto l'ACK - verranno aggiunte per il prossimo rilascio -stable, e successivamente - questo nuovo rilascio verrà fatto. + - Le patch che hanno ricevuto un ACK verranno inviate nuovamente come parte di + un rilascio candidato (-rc) al fine di essere verificate dagli sviluppatori e + dai testatori. + - Solitamente si pubblica solo una -rc, tuttavia se si riscontrano problemi + importanti, alcune patch potrebbero essere modificate o essere scartate, + oppure nuove patch potrebbero essere messe in coda. Dunque, verranno pubblicate + nuove -rc e così via finché non si ritiene che non vi siano più problemi. + - Si può rispondere ad una -rc scrivendo sulla lista di discussione un'email + con l'etichetta "Tested-by:". Questa etichetta verrà raccolta ed aggiunta al + commit rilascio. + - Alla fine del ciclo di revisione il nuovo rilascio -stable conterrà tutte le + patch che erano in coda e sono state verificate. - Le patch di sicurezza verranno accettate nei sorgenti -stable direttamente dalla squadra per la sicurezza del kernel, e non passerà per il normale ciclo di revisione. Contattate la suddetta squadra per maggiori dettagli @@ -186,8 +195,19 @@ Sorgenti - Il rilascio definitivo, e marchiato, di tutti i kernel stabili può essere trovato in rami distinti per versione al seguente indirizzo: - https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git + https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git + - I rilasci candidati di tutti i kernel stabili possono essere trovati al + seguente indirizzo: + + https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable-rc.git/ + + + .. warning:: + I sorgenti -stable-rc sono un'istantanea dei sorgenti stable-queue e + subirà frequenti modifiche, dunque verrà anche trapiantato spesso. + Dovrebbe essere usato solo allo scopo di verifica (per esempio in un + sistema di CI) Comitato per la revisione ------------------------- diff --git a/Documentation/translations/it_IT/process/submitting-patches.rst b/Documentation/translations/it_IT/process/submitting-patches.rst index f117ee0a36e2..a3bb0008837a 100644 --- a/Documentation/translations/it_IT/process/submitting-patches.rst +++ b/Documentation/translations/it_IT/process/submitting-patches.rst @@ -26,6 +26,10 @@ Questa documentazione assume che sappiate usare ``git`` per preparare le patch. Se non siete pratici di ``git``, allora è bene che lo impariate; renderà la vostra vita di sviluppatore del kernel molto più semplice. +I sorgenti di alcuni sottosistemi e manutentori contengono più informazioni +riguardo al loro modo di lavorare ed aspettative. Consultate +:ref:`Documentation/translations/it_IT/process/maintainer-handbooks.rst ` + Ottenere i sorgenti attuali --------------------------- @@ -82,11 +86,11 @@ comporti come descritto. I manutentori vi saranno grati se scrivete la descrizione della patch in un formato che sia compatibile con il gestore dei sorgenti usato dal kernel, -``git``, come un "commit log". Leggete :ref:`it_explicit_in_reply_to`. +``git``, come un "commit log". Leggete :ref:`it_the_canonical_patch_format`. Risolvete solo un problema per patch. Se la vostra descrizione inizia ad essere lunga, potrebbe essere un segno che la vostra patch necessita d'essere -divisa. Leggete :ref:`split_changes`. +divisa. Leggete :ref:`it_split_changes`. Quando inviate o rinviate una patch o una serie, includete la descrizione completa delle modifiche e la loro giustificazione. Non limitatevi a dire che @@ -102,17 +106,28 @@ do frotz" piuttosto che "[This patch] makes xyzzy do frotz" or "[I] changed xyzzy to do frotz", come se steste dando ordini al codice di cambiare il suo comportamento. -Se la patch corregge un baco conosciuto, fare riferimento a quel baco inserendo -il suo numero o il suo URL. Se la patch è la conseguenza di una discussione -su una lista di discussione, allora fornite l'URL all'archivio di quella -discussione; usate i collegamenti a https://lore.kernel.org/ con il -``Message-Id``, in questo modo vi assicurerete che il collegamento non diventi -invalido nel tempo. +Se ci sono delle discussioni, o altre informazioni d'interesse, che fanno +riferimento alla patch, allora aggiungete l'etichetta 'Link:' per farvi +riferimento. Per esempio, se la vostra patch corregge un baco potete aggiungere +quest'etichetta per fare riferimento ad un rapporto su una lista di discussione +o un *bug tracker*. Un altro esempio; potete usare quest'etichetta per far +riferimento ad una discussione precedentemente avvenuta su una lista di +discussione, o qualcosa di documentato sul web, da cui poi è nata la patch in +questione. -Tuttavia, cercate di rendere la vostra spiegazione comprensibile anche senza -far riferimento a fonti esterne. In aggiunta ai collegamenti a bachi e liste -di discussione, riassumente i punti più importanti della discussione che hanno -portato alla creazione della patch. +Quando volete fare riferimento ad una lista di discussione, preferite il +servizio d'archiviazione lore.kernel.org. Per create un collegamento URL è +sufficiente usare il campo ``Message-Id``, presente nell'intestazione del +messaggio, senza parentesi angolari. Per esempio:: + + Link: https://lore.kernel.org/r/30th.anniversary.repost@klaava.Helsinki.FI/ + +Prima d'inviare il messaggio ricordatevi di verificare che il collegamento così +creato funzioni e che indirizzi verso il messaggio desiderato. + +Tuttavia, provate comunque a dare una spiegazione comprensibile anche senza +accedere alle fonti esterne. Inoltre, riassumente i punti più salienti che hanno +condotto all'invio della patch. Se volete far riferimento a uno specifico commit, non usate solo l'identificativo SHA-1. Per cortesia, aggiungete anche la breve riga @@ -224,10 +239,11 @@ nella vostra patch. Dovreste sempre inviare una copia della patch ai manutentori dei sottosistemi interessati dalle modifiche; date un'occhiata al file MAINTAINERS e alla storia -delle revisioni per scoprire chi si occupa del codice. Lo script -scripts/get_maintainer.pl può esservi d'aiuto. Se non riuscite a trovare un -manutentore per il sottosistema su cui state lavorando, allora Andrew Morton -(akpm@linux-foundation.org) sarà la vostra ultima possibilità. +delle revisioni per scoprire chi si occupa del codice. Lo script +scripts/get_maintainer.pl può esservi d'aiuto (passategli il percorso alle +vostre patch). Se non riuscite a trovare un manutentore per il sottosistema su +cui state lavorando, allora Andrew Morton (akpm@linux-foundation.org) sarà la +vostra ultima possibilità. Normalmente, dovreste anche scegliere una lista di discussione a cui inviare la vostra serie di patch. La lista di discussione linux-kernel@vger.kernel.org @@ -322,14 +338,19 @@ cosa stia accadendo. Assicuratevi di dire ai revisori quali cambiamenti state facendo e di ringraziarli per il loro tempo. Revisionare codice è un lavoro faticoso e che -richiede molto tempo, e a volte i revisori diventano burberi. Tuttavia, anche -in questo caso, rispondete con educazione e concentratevi sul problema che -hanno evidenziato. +richiede molto tempo, e a volte i revisori diventano burberi. Tuttavia, anche in +questo caso, rispondete con educazione e concentratevi sul problema che hanno +evidenziato. Quando inviate una version successiva ricordatevi di aggiungere un +``patch changelog`` alla email di intestazione o ad ogni singola patch spiegando +le differenze rispetto a sottomissioni precedenti (vedere +:ref:`it_the_canonical_patch_format`). Leggete Documentation/translations/it_IT/process/email-clients.rst per le raccomandazioni sui programmi di posta elettronica e l'etichetta da usare sulle liste di discussione. +.. _it_resend_reminders: + Non scoraggiatevi - o impazientitevi ------------------------------------ @@ -502,7 +523,8 @@ Utilizzare Reported-by:, Tested-by:, Reviewed-by:, Suggested-by: e Fixes: L'etichetta Reported-by da credito alle persone che trovano e riportano i bachi e si spera che questo possa ispirarli ad aiutarci nuovamente in futuro. Rammentate che se il baco è stato riportato in privato, dovrete chiedere il -permesso prima di poter utilizzare l'etichetta Reported-by. +permesso prima di poter utilizzare l'etichetta Reported-by. Questa etichetta va +usata per i bachi, dunque non usatela per richieste di nuove funzionalità. L'etichetta Tested-by: indica che la patch è stata verificata con successo (su un qualche sistema) dalla persona citata. Questa etichetta informa i @@ -572,6 +594,8 @@ previste per i kernel stabili, e nemmeno dalla necessità di aggiungere in copia conoscenza stable@vger.kernel.org su tutte le patch per suddetti kernel. +.. _it_the_canonical_patch_format: + Il formato canonico delle patch ------------------------------- @@ -717,6 +741,8 @@ messe correttamente oltre la riga.:: Maggiori dettagli sul formato delle patch nei riferimenti qui di seguito. +.. _it_backtraces: + Aggiungere i *backtrace* nei messaggi di commit ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 659797dc4d64ee4814e5e9a5387933e38a11fb7a Mon Sep 17 00:00:00 2001 From: Yanteng Si Date: Thu, 28 Jul 2022 13:05:56 +0800 Subject: [PATCH 299/318] Docs/zh_CN: Update the translation of iio_configfs to 5.19-rc8 update to commit dafcf4ed8392 ("iio: hrtimer: Allow sub Hz granularity"). c1d82dbcb0a6 ("docs: iio: fix example formatting"). Remove some useless spaces. Signed-off-by: Yanteng Si Reviewed-by: Alex Shi Link: https://lore.kernel.org/r/1c0ae3e0932d813d41de666c1c65379786e79ba6.1658983157.git.siyanteng@loongson.cn Signed-off-by: Jonathan Corbet --- .../translations/zh_CN/iio/iio_configfs.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Documentation/translations/zh_CN/iio/iio_configfs.rst b/Documentation/translations/zh_CN/iio/iio_configfs.rst index d5460e951804..eccaf1c644b4 100644 --- a/Documentation/translations/zh_CN/iio/iio_configfs.rst +++ b/Documentation/translations/zh_CN/iio/iio_configfs.rst @@ -37,10 +37,10 @@ configfs轻松配置的对象（例如：设备，触发器）。 3. 软件触发器 ============= -IIO默认configfs组之一是“触发器”组。 挂载configfs后可以自动访问它，并且可 +IIO默认configfs组之一是“触发器”组。挂载configfs后可以自动访问它，并且可 以在/config/iio/triggers下找到。 -IIO软件触发器为创建多种触发器类型提供了支持。 通常在include/linux/iio +IIO软件触发器为创建多种触发器类型提供了支持。通常在include/linux/iio /sw_trigger.h：中的接口下将新的触发器类型实现为单独的内核模块： :: @@ -76,10 +76,10 @@ IIO软件触发器为创建多种触发器类型提供了支持。 通常在incl .ops = &iio_trig_sample_ops, }; -module_iio_sw_trigger_driver(iio_trig_sample); + module_iio_sw_trigger_driver(iio_trig_sample); -每种触发器类型在/config/iio/triggers下都有其自己的目录。 加载iio-trig-sample -模块将创建“ trig-sample”触发器类型目录/config/iio/triggers/trig-sample. +每种触发器类型在/config/iio/triggers下都有其自己的目录。加载iio-trig-sample +模块将创建“trig-sample”触发器类型目录/config/iio/triggers/trig-sample. 我们支持以下中断源（触发器类型） @@ -102,3 +102,5 @@ module_iio_sw_trigger_driver(iio_trig_sample); ---------------------------- "hrtimer”触发器类型没有来自/config dir的任何可配置属性。 +它确实引入了触发目录的sampling_frequency属性。 +该属性以Hz为单位设置轮询频率，精度为mHz。 \ No newline at end of file From 507f48799ab0915788df79edf794d659ad8104c7 Mon Sep 17 00:00:00 2001 From: Yanteng Si Date: Thu, 28 Jul 2022 13:05:57 +0800 Subject: [PATCH 300/318] Docs/zh_CN: Update the translation of kasan to 5.19-rc8 update to commit 3ff16d30f593 ("kasan: test: improve failure message in KUNIT_EXPECT_KASAN_FAIL()") c9d1af2b780a ("mm/kasan: move kasan.fault to mm/kasan/report.c") 2d27e5851473 ("kasan: Extend KASAN mode kernel parameter") 8479d7b5be2f ("kasan: documentation updates") c2ec0c8f6877 ("kasan: update documentation") Signed-off-by: Yanteng Si Reviewed-by: Alex Shi Link: https://lore.kernel.org/r/abbb6c5cc5a7daf0720a9cb0a8255472d4ac69b4.1658983157.git.siyanteng@loongson.cn Signed-off-by: Jonathan Corbet --- .../translations/zh_CN/dev-tools/kasan.rst | 117 ++++++++++++------ 1 file changed, 77 insertions(+), 40 deletions(-) diff --git a/Documentation/translations/zh_CN/dev-tools/kasan.rst b/Documentation/translations/zh_CN/dev-tools/kasan.rst index 23db9d419047..fe76cbe77ad6 100644 --- a/Documentation/translations/zh_CN/dev-tools/kasan.rst +++ b/Documentation/translations/zh_CN/dev-tools/kasan.rst @@ -11,34 +11,65 @@ 概述 ---- -KernelAddressSANitizer(KASAN)是一种动态内存安全错误检测工具，主要功能是 -检查内存越界访问和使用已释放内存的问题。KASAN有三种模式: +Kernel Address SANitizer(KASAN)是一种动态内存安全错误检测工具，主要功能是 +检查内存越界访问和使用已释放内存的问题。 -1. 通用KASAN（与用户空间的ASan类似） -2. 基于软件标签的KASAN（与用户空间的HWASan类似） -3. 基于硬件标签的KASAN（基于硬件内存标签） +KASAN有三种模式: -由于通用KASAN的内存开销较大，通用KASAN主要用于调试。基于软件标签的KASAN -可用于dogfood测试，因为它具有较低的内存开销，并允许将其用于实际工作量。 -基于硬件标签的KASAN具有较低的内存和性能开销，因此可用于生产。同时可用于 -检测现场内存问题或作为安全缓解措施。 +1. 通用KASAN +2. 基于软件标签的KASAN +3. 基于硬件标签的KASAN -软件KASAN模式（#1和#2）使用编译时工具在每次内存访问之前插入有效性检查， -因此需要一个支持它的编译器版本。 +用CONFIG_KASAN_GENERIC启用的通用KASAN，是用于调试的模式，类似于用户空 +间的ASan。这种模式在许多CPU架构上都被支持，但它有明显的性能和内存开销。 -通用KASAN在GCC和Clang受支持。GCC需要8.3.0或更高版本。任何受支持的Clang -版本都是兼容的，但从Clang 11才开始支持检测全局变量的越界访问。 +基于软件标签的KASAN或SW_TAGS KASAN，通过CONFIG_KASAN_SW_TAGS启用， +可以用于调试和自我测试，类似于用户空间HWASan。这种模式只支持arm64，但其 +适度的内存开销允许在内存受限的设备上用真实的工作负载进行测试。 -基于软件标签的KASAN模式仅在Clang中受支持。 +基于硬件标签的KASAN或HW_TAGS KASAN，用CONFIG_KASAN_HW_TAGS启用，被 +用作现场内存错误检测器或作为安全缓解的模式。这种模式只在支持MTE（内存标签 +扩展）的arm64 CPU上工作，但它的内存和性能开销很低，因此可以在生产中使用。 -硬件KASAN模式（#3）依赖硬件来执行检查，但仍需要支持内存标签指令的编译器 -版本。GCC 10+和Clang 11+支持此模式。 +关于每种KASAN模式的内存和性能影响的细节，请参见相应的Kconfig选项的描述。 -两种软件KASAN模式都适用于SLUB和SLAB内存分配器，而基于硬件标签的KASAN目前 -仅支持SLUB。 +通用模式和基于软件标签的模式通常被称为软件模式。基于软件标签的模式和基于 +硬件标签的模式被称为基于标签的模式。 -目前x86_64、arm、arm64、xtensa、s390、riscv架构支持通用KASAN模式，仅 -arm64架构支持基于标签的KASAN模式。 +支持 +---- + +体系架构 +~~~~~~~~ + +在x86_64、arm、arm64、powerpc、riscv、s390和xtensa上支持通用KASAN， +而基于标签的KASAN模式只在arm64上支持。 + +编译器 +~~~~~~ + +软件KASAN模式使用编译时工具在每个内存访问之前插入有效性检查，因此需要一个 +提供支持的编译器版本。基于硬件标签的模式依靠硬件来执行这些检查，但仍然需要 +一个支持内存标签指令的编译器版本。 + +通用KASAN需要GCC 8.3.0版本或更高版本，或者内核支持的任何Clang版本。 + +基于软件标签的KASAN需要GCC 11+或者内核支持的任何Clang版本。 + +基于硬件标签的KASAN需要GCC 10+或Clang 12+。 + +内存类型 +~~~~~~~~ + +通用KASAN支持在所有的slab、page_alloc、vmap、vmalloc、堆栈和全局内存 +中查找错误。 + +基于软件标签的KASAN支持slab、page_alloc、vmalloc和堆栈内存。 + +基于硬件标签的KASAN支持slab、page_alloc和不可执行的vmalloc内存。 + +对于slab，两种软件KASAN模式都支持SLUB和SLAB分配器，而基于硬件标签的 +KASAN只支持SLUB。 用法 ---- @@ -53,7 +84,7 @@ arm64架构支持基于标签的KASAN模式。 对于软件模式，还可以在 ``CONFIG_KASAN_OUTLINE`` 和 ``CONFIG_KASAN_INLINE`` 之间进行选择。outline和inline是编译器插桩类型。前者产生较小的二进制文件， -而后者快1.1-2倍。 +而后者快2倍。 要将受影响的slab对象的alloc和free堆栈跟踪包含到报告中，请启用 ``CONFIG_STACKTRACE`` 。要包括受影响物理页面的分配和释放堆栈跟踪的话， @@ -172,21 +203,29 @@ KASAN受通用 ``panic_on_warn`` 命令行参数的影响。启用该功能后 默认情况下，KASAN只为第一次无效内存访问打印错误报告。使用 ``kasan_multi_shot`` ， KASAN会针对每个无效访问打印报告。这有效地禁用了KASAN报告的 ``panic_on_warn`` 。 +另外，独立于 ``panic_on_warn`` , ``kasan.fault=`` 引导参数可以用来控制恐慌和报 +告行为: + +- ``kasan.fault=report`` 或 ``=panic`` 控制是只打印KASAN报告还是同时使内核恐慌 + (默认: ``report`` )。即使启用了 ``kasan_multi_shot`` ，也会发生内核恐慌。 + 基于硬件标签的KASAN模式（请参阅下面有关各种模式的部分）旨在在生产中用作安全缓解 -措施。因此，它支持允许禁用KASAN或控制其功能的引导参数。 +措施。因此，它支持允许禁用KASAN或控制其功能的附加引导参数。 - ``kasan=off`` 或 ``=on`` 控制KASAN是否启用 (默认: ``on`` )。 -- ``kasan.mode=sync`` 或 ``=async`` 控制KASAN是否配置为同步或异步执行模式(默认: - ``sync`` )。同步模式：当标签检查错误发生时，立即检测到错误访问。异步模式： - 延迟错误访问检测。当标签检查错误发生时，信息存储在硬件中（在arm64的 +- ``kasan.mode=sync`` 、 ``=async`` 或 ``=asymm`` 控制KASAN是否配置 + 为同步或异步执行模式(默认:``sync`` )。 + 同步模式：当标签检查错误发生时，立即检测到错误访问。 + 异步模式：延迟错误访问检测。当标签检查错误发生时，信息存储在硬件中（在arm64的 TFSR_EL1寄存器中）。内核会定期检查硬件，并且仅在这些检查期间报告标签错误。 + 非对称模式：读取时同步检测不良访问，写入时异步检测。 + +- ``kasan.vmalloc=off`` 或 ``=on`` 禁用或启用vmalloc分配的标记（默认：``on`` ）。 - ``kasan.stacktrace=off`` 或 ``=on`` 禁用或启用alloc和free堆栈跟踪收集 (默认: ``on`` )。 -- ``kasan.fault=report`` 或 ``=panic`` 控制是只打印KASAN报告还是同时使内核恐慌 - (默认: ``report`` )。即使启用了 ``kasan_multi_shot`` ，也会发生内核恐慌。 实施细则 -------- @@ -244,7 +283,6 @@ KASAN会针对每个无效访问打印报告。这有效地禁用了KASAN报告 基于软件标签的KASAN使用0xFF作为匹配所有指针标签（不检查通过带有0xFF指针标签 的指针进行的访问）。值0xFE当前保留用于标记已释放的内存区域。 -基于软件标签的KASAN目前仅支持对Slab和page_alloc内存进行标记。 基于硬件标签的KASAN模式 ~~~~~~~~~~~~~~~~~~~~~~~ @@ -262,8 +300,6 @@ KASAN会针对每个无效访问打印报告。这有效地禁用了KASAN报告 基于硬件标签的KASAN使用0xFF作为匹配所有指针标签（不检查通过带有0xFF指针标签的 指针进行的访问）。值0xFE当前保留用于标记已释放的内存区域。 -基于硬件标签的KASAN目前仅支持对Slab和page_alloc内存进行标记。 - 如果硬件不支持MTE（ARMv8.5之前），则不会启用基于硬件标签的KASAN。在这种情况下， 所有KASAN引导参数都将被忽略。 @@ -275,6 +311,8 @@ KASAN会针对每个无效访问打印报告。这有效地禁用了KASAN报告 影子内存 -------- +本节的内容只适用于软件KASAN模式。 + 内核将内存映射到地址空间的几个不同部分。内核虚拟地址的范围很大：没有足够的真实 内存来支持内核可以访问的每个地址的真实影子区域。因此，KASAN只为地址空间的某些 部分映射真实的影子。 @@ -297,7 +335,7 @@ CONFIG_KASAN_VMALLOC ~~~~~~~~~~~~~~~~~~~~ 使用 ``CONFIG_KASAN_VMALLOC`` ，KASAN可以以更大的内存使用为代价覆盖vmalloc -空间。目前，这在x86、riscv、s390和powerpc上受支持。 +空间。目前，这在arm64、x86、riscv、s390和powerpc上受支持。 这通过连接到vmalloc和vmap并动态分配真实的影子内存来支持映射。 @@ -349,10 +387,10 @@ KASAN连接到vmap基础架构以懒清理未使用的影子内存。 ``kasan_disable_current()``/``kasan_enable_current()`` 部分注释这部分代码。 这也会禁用通过函数调用发生的间接访问的报告。 -对于基于标签的KASAN模式（包括硬件模式），要禁用访问检查，请使用 -``kasan_reset_tag()`` 或 ``page_kasan_tag_reset()`` 。请注意，通过 -``page_kasan_tag_reset()`` 临时禁用访问检查需要通过 ``page_kasan_tag`` -/ ``page_kasan_tag_set`` 保存和恢复每页KASAN标签。 +对于基于标签的KASAN模式，要禁用访问检查，请使用 ``kasan_reset_tag()`` 或 +``page_kasan_tag_reset()`` 。请注意，通过 ``page_kasan_tag_reset()`` +临时禁用访问检查需要通过 ``page_kasan_tag`` / ``page_kasan_tag_set`` 保 +存和恢复每页KASAN标签。 测试 ~~~~ @@ -381,11 +419,10 @@ KASAN连接到vmap基础架构以懒清理未使用的影子内存。 当由于缺少KASAN报告而导致测试失败时:: - # kmalloc_double_kzfree: EXPECTATION FAILED at lib/test_kasan.c:629 - Expected kasan_data->report_expected == kasan_data->report_found, but - kasan_data->report_expected == 1 - kasan_data->report_found == 0 - not ok 28 - kmalloc_double_kzfree + # kmalloc_double_kzfree: EXPECTATION FAILED at lib/test_kasan.c:974 + KASAN failure expected in "kfree_sensitive(ptr)", but none occurred + not ok 44 - kmalloc_double_kzfree + 最后打印所有KASAN测试的累积状态。成功:: From 6a5057e9dc13dd31d1577652162c4ca2869d8950 Mon Sep 17 00:00:00 2001 From: Yanteng Si Date: Thu, 28 Jul 2022 13:05:58 +0800 Subject: [PATCH 301/318] Docs/zh_CN: Update the translation of sparse to 5.19-rc8 Update to commit179fd6ba3bac ("Documentation/sparse: add hints about __CHECKER__") Signed-off-by: Yanteng Si Reviewed-by: Alex Shi Link: https://lore.kernel.org/r/2ecdf7ddc8644e9031e4e2af947b97d63ab046f0.1658983157.git.siyanteng@loongson.cn Signed-off-by: Jonathan Corbet --- Documentation/translations/zh_CN/dev-tools/sparse.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/translations/zh_CN/dev-tools/sparse.rst b/Documentation/translations/zh_CN/dev-tools/sparse.rst index 556282437aad..0664c634bc4f 100644 --- a/Documentation/translations/zh_CN/dev-tools/sparse.rst +++ b/Documentation/translations/zh_CN/dev-tools/sparse.rst @@ -106,3 +106,5 @@ __releases - 指定的锁在函数进入时被持有，但在退出时不被持 make 的可选变量 CHECKFLAGS 可以用来向 sparse 工具传递参数。编译系统会自 动向 sparse 工具传递 -Wbitwise 参数。 + +注意sparse定义了__CHECKER__预处理器符号。 \ No newline at end of file From 63c1d2516b057e2d49f58812d8c149acfd5348a5 Mon Sep 17 00:00:00 2001 From: Yanteng Si Date: Thu, 28 Jul 2022 13:07:48 +0800 Subject: [PATCH 302/318] Docs/zh_CN: Update the translation of testing-overview to 5.19-rc8 Update to commit 12379401c000 ("Documentation: dev-tools: Add a section for static analysis tools") Signed-off-by: Yanteng Si Reviewed-by: Alex Shi Link: https://lore.kernel.org/r/b9911916446f858fe95e4d4bfeaecdab14bc0b6a.1658983157.git.siyanteng@loongson.cn Signed-off-by: Jonathan Corbet --- .../zh_CN/dev-tools/testing-overview.rst | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/Documentation/translations/zh_CN/dev-tools/testing-overview.rst b/Documentation/translations/zh_CN/dev-tools/testing-overview.rst index b7a1d13da6c6..d6f2c65ed511 100644 --- a/Documentation/translations/zh_CN/dev-tools/testing-overview.rst +++ b/Documentation/translations/zh_CN/dev-tools/testing-overview.rst @@ -107,3 +107,28 @@ Documentation/dev-tools/kcov.rst 是能够构建在内核之中，用于在每 之后你就能确保这些错误在测试过程中都不会发生了。 一些工具与KUnit和kselftest集成，并且在检测到问题时会自动打断测试。 + +静态分析工具 +============ + +除了测试运行中的内核，我们还可以使用**静态分析**工具直接分析内核的源代 +码（**在编译时**）。内核中常用的工具允许人们检查整个源代码树或其中的特 +定文件。它们使得在开发过程中更容易发现和修复问题。 + + Sparse可以通过执行类型检查、锁检查、值范围检查来帮助测试内核，此外还 + 可以在检查代码时报告各种错误和警告。关于如何使用它的细节，请参阅 + Documentation/translations/zh_CN/dev-tools/sparse.rst。 + + Smatch扩展了Sparse，并提供了对编程逻辑错误的额外检查，如开关语句中 + 缺少断点，错误检查中未使用的返回值，忘记在错误路径的返回中设置错误代 + 码等。Smatch也有针对更严重问题的测试，如整数溢出、空指针解除引用和内 + 存泄漏。见项目页面http://smatch.sourceforge.net/。 + + Coccinelle是我们可以使用的另一个静态分析器。Coccinelle经常被用来 + 帮助源代码的重构和并行演化，但它也可以帮助避免常见代码模式中出现的某 + 些错误。可用的测试类型包括API测试、内核迭代器的正确使用测试、自由操 + 作的合理性检查、锁定行为的分析，以及已知的有助于保持内核使用一致性的 + 进一步测试。详情请见Documentation/dev-tools/coccinelle.rst。 + + 不过要注意的是，静态分析工具存在**假阳性**的问题。在试图修复错误和警 + 告之前，需要仔细评估它们。 From 83b41bb27b25b4b972d03b593e8da7e3dd5735aa Mon Sep 17 00:00:00 2001 From: Yanteng Si Date: Thu, 28 Jul 2022 13:07:59 +0800 Subject: [PATCH 303/318] Docs/zh_CN: Update the translation of usage to 5.19-rc8 Update to commit b57e39a743e4 ("Docs/admin-guide /damon/sysfs: document 'LRU_DEPRIO' scheme action") 0bcba960b1fa ("Docs/admin-guide/damon/sysfs: document 'LRU_PRIO' scheme action") Signed-off-by: Yanteng Si Reviewed-by: Alex Shi Link: https://lore.kernel.org/r/130795ae1a4097e25e8e354870e65c3018241a8e.1658983157.git.siyanteng@loongson.cn Signed-off-by: Jonathan Corbet --- Documentation/translations/zh_CN/admin-guide/mm/damon/usage.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/translations/zh_CN/admin-guide/mm/damon/usage.rst b/Documentation/translations/zh_CN/admin-guide/mm/damon/usage.rst index eee0e8c5c368..2c7d9106e399 100644 --- a/Documentation/translations/zh_CN/admin-guide/mm/damon/usage.rst +++ b/Documentation/translations/zh_CN/admin-guide/mm/damon/usage.rst @@ -210,6 +210,8 @@ schemes// - ``pageout``: 为具有 ``MADV_PAGEOUT`` 的区域调用 ``madvise()`` 。 - ``hugepage``: 为带有 ``MADV_HUGEPAGE`` 的区域调用 ``madvise()`` 。 - ``nohugepage``: 为带有 ``MADV_NOHUGEPAGE`` 的区域调用 ``madvise()``。 + - ``lru_prio``: 在其LRU列表上对区域进行优先排序。 + - ``lru_deprio``: 对区域的LRU列表进行降低优先处理。 - ``stat``: 什么都不做，只计算统计数据 schemes//access_pattern/ From c78478e164d4163419dc027fba7e3a2261d0c1d9 Mon Sep 17 00:00:00 2001 From: Yanteng Si Date: Thu, 28 Jul 2022 13:08:10 +0800 Subject: [PATCH 304/318] Docs/zh_CN: Update the translation of pci-iov-howto to 5.19-rc8 Update to commit 4f23bd5d09af ("PCI/doc: Convert examples to generic power management") Signed-off-by: Yanteng Si Reviewed-by: Alex Shi Link: https://lore.kernel.org/r/24c53aabc9d942f6fe38b8d3a843329edca3d18c.1658983157.git.siyanteng@loongson.cn Signed-off-by: Jonathan Corbet --- Documentation/translations/zh_CN/PCI/pci-iov-howto.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Documentation/translations/zh_CN/PCI/pci-iov-howto.rst b/Documentation/translations/zh_CN/PCI/pci-iov-howto.rst index fbc83dfdcead..fb023ea1374d 100644 --- a/Documentation/translations/zh_CN/PCI/pci-iov-howto.rst +++ b/Documentation/translations/zh_CN/PCI/pci-iov-howto.rst @@ -123,14 +123,14 @@ nr_virtfn'是要启用的VF的编号。 ... } - static int dev_suspend(struct pci_dev *dev, pm_message_t state) + static int dev_suspend(struct device *dev) { ... return 0; } - static int dev_resume(struct pci_dev *dev) + static int dev_resume(struct device *dev) { ... @@ -163,8 +163,7 @@ nr_virtfn'是要启用的VF的编号。 .id_table = dev_id_table, .probe = dev_probe, .remove = dev_remove, - .suspend = dev_suspend, - .resume = dev_resume, + .driver.pm = &dev_pm_ops .shutdown = dev_shutdown, .sriov_configure = dev_sriov_configure, }; From ce1120076c53681fc084240627bc9c974039da97 Mon Sep 17 00:00:00 2001 From: Yanteng Si Date: Thu, 28 Jul 2022 13:08:26 +0800 Subject: [PATCH 305/318] Docs/zh_CN: Update the translation of pci to 5.19-rc8 Update to commit f21949c14968 ("PCI/doc:Update obsolete pci_set_dma_mask() references") 05b0ebd06ae6 ("PCI/doc: cleanup references to the legacy PCI DMA API") Signed-off-by: Yanteng Si Reviewed-by: Wu XiangCheng Reviewed-by: Alex Shi Link: https://lore.kernel.org/r/bccb98a1c6706ecfd4aaba8c27f1c64024e1c139.1658983157.git.siyanteng@loongson.cn Signed-off-by: Jonathan Corbet --- Documentation/translations/zh_CN/PCI/pci.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/translations/zh_CN/PCI/pci.rst b/Documentation/translations/zh_CN/PCI/pci.rst index 520707787256..83c2a41d38d3 100644 --- a/Documentation/translations/zh_CN/PCI/pci.rst +++ b/Documentation/translations/zh_CN/PCI/pci.rst @@ -255,13 +255,13 @@ pci_set_master()将通过设置PCI_COMMAND寄存器中的总线主控位来启 虽然所有的驱动程序都应该明确指出PCI总线主控的DMA功能（如32位或64位），但对于流式 数据来说，具有超过32位总线主站功能的设备需要驱动程序通过调用带有适当参数的 -``pci_set_dma_mask()`` 来“注册”这种功能。一般来说，在系统RAM高于4G物理地址的情 +``dma_set_mask()`` 来“注册”这种功能。一般来说，在系统RAM高于4G物理地址的情 况下，这允许更有效的DMA。 -所有PCI-X和PCIe兼容设备的驱动程序必须调用 ``pci_set_dma_mask()`` ，因为它们 +所有PCI-X和PCIe兼容设备的驱动程序必须调用 ``dma_set_mask()`` ，因为它们 是64位DMA设备。 -同样，如果设备可以通过调用 ``pci_set_consistent_dma_mask()`` 直接寻址到 +同样，如果设备可以通过调用 ``dma_set_coherent_mask()`` 直接寻址到 4G物理地址以上的系统RAM中的“一致性内存”，那么驱动程序也必须“注册”这种功能。同 样，这包括所有PCI-X和PCIe兼容设备的驱动程序。许多64位“PCI”设备（在PCI-X之前） 和一些PCI-X设备对有效载荷（“流式”）数据具有64位DMA功能，但对控制（“一致性”）数 From 4116ff79749d8842f6a6dfce5a17a5ea15fc290b Mon Sep 17 00:00:00 2001 From: Yanteng Si Date: Thu, 28 Jul 2022 13:08:43 +0800 Subject: [PATCH 306/318] Docs/zh_CN: Update the translation of sched-stats to 5.19-rc8 Update to commit 6c757e9f55f0 ("docs/scheduler: fix unit error") ddb21d27a6a5 ("docs/scheduler: Change unit of cpu_time and rq_time to nanoseconds") Signed-off-by: Yanteng Si Reviewed-by: Wu XiangCheng Reviewed-by: Alex Shi Link: https://lore.kernel.org/r/3cb1c4c466dfa38d72a867dc6e2c833ceb69ecb7.1658983157.git.siyanteng@loongson.cn Signed-off-by: Jonathan Corbet --- .../translations/zh_CN/scheduler/sched-stats.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/translations/zh_CN/scheduler/sched-stats.rst b/Documentation/translations/zh_CN/scheduler/sched-stats.rst index 1c68c3d1c283..c5e0be663837 100644 --- a/Documentation/translations/zh_CN/scheduler/sched-stats.rst +++ b/Documentation/translations/zh_CN/scheduler/sched-stats.rst @@ -57,8 +57,8 @@ cpu 1 2 3 4 5 6 7 8 9 接下来的三个统计数据描述了调度延迟： - 7) 本处理器运行任务的总时间，单位是jiffies - 8) 本处理器任务等待运行的时间，单位是jiffies + 7) 本处理器运行任务的总时间，单位是纳秒 + 8) 本处理器任务等待运行的时间，单位是纳秒 9) 本CPU运行了#个时间片 域统计数据 @@ -146,8 +146,8 @@ domain 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 schedstats还添加了一个新的/proc//schedstat文件，来提供一些进程级的 相同信息。这个文件中，有三个字段与该进程相关： - 1) 在CPU上运行花费的时间 - 2) 在运行队列上等待的时间 + 1) 在CPU上运行花费的时间(单位是纳秒) + 2) 在运行队列上等待的时间(单位是纳秒) 3) 在CPU上运行了#个时间片 可以很容易地编写一个程序，利用这些额外的字段来报告一个特定的进程或一组进程在 From 339170d8d3da5685762619080263abb78700ab4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Paulo=20Rechi=20Vita?= Date: Wed, 27 Jul 2022 10:05:39 -0400 Subject: [PATCH 307/318] docs: efi-stub: Fix paths for x86 / arm stubs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes the paths of x86 / arm efi-stub source files. Signed-off-by: João Paulo Rechi Vita Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20220727140539.10021-1-jprvita@endlessos.org Signed-off-by: Jonathan Corbet --- Documentation/admin-guide/efi-stub.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/efi-stub.rst b/Documentation/admin-guide/efi-stub.rst index 833edb0d0bc4..b24e7c40d832 100644 --- a/Documentation/admin-guide/efi-stub.rst +++ b/Documentation/admin-guide/efi-stub.rst @@ -7,10 +7,10 @@ as a PE/COFF image, thereby convincing EFI firmware loaders to load it as an EFI executable. The code that modifies the bzImage header, along with the EFI-specific entry point that the firmware loader jumps to are collectively known as the "EFI boot stub", and live in -arch/x86/boot/header.S and arch/x86/boot/compressed/eboot.c, +arch/x86/boot/header.S and drivers/firmware/efi/libstub/x86-stub.c, respectively. For ARM the EFI stub is implemented in arch/arm/boot/compressed/efi-header.S and -arch/arm/boot/compressed/efi-stub.c. EFI stub code that is shared +drivers/firmware/efi/libstub/arm32-stub.c. EFI stub code that is shared between architectures is in drivers/firmware/efi/libstub. For arm64, there is no compressed kernel support, so the Image itself From 98dfa9343f37bdd4112966292751e3a93aaf2e56 Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Tue, 19 Jul 2022 12:23:23 +0800 Subject: [PATCH 308/318] crypto: hisilicon/hpre - don't use GFP_KERNEL to alloc mem during softirq The hpre encryption driver may be used to encrypt and decrypt packets during the rx softirq, it is not allowed to use GFP_KERNEL. Fixes: c8b4b477079d ("crypto: hisilicon - add HiSilicon HPRE accelerator") Signed-off-by: Zhengchao Shao Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_crypto.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index 97d54c1465c2..3ba6f15deafc 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -252,7 +252,7 @@ static int hpre_prepare_dma_buf(struct hpre_asym_request *hpre_req, if (unlikely(shift < 0)) return -EINVAL; - ptr = dma_alloc_coherent(dev, ctx->key_sz, tmp, GFP_KERNEL); + ptr = dma_alloc_coherent(dev, ctx->key_sz, tmp, GFP_ATOMIC); if (unlikely(!ptr)) return -ENOMEM; From fa4d57b85786ec0e16565c75a51c208834b0c24d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pali=20Roh=C3=A1r?= Date: Tue, 19 Jul 2022 09:54:03 +0200 Subject: [PATCH 309/318] crypto: inside-secure - Add missing MODULE_DEVICE_TABLE for of MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without MODULE_DEVICE_TABLE, crypto_safexcel.ko module is not automatically loaded on platforms where inside-secure crypto HW is specified in device tree (e.g. Armada 3720). So add missing MODULE_DEVICE_TABLE for of. Fixes: 1b44c5a60c13 ("crypto: inside-secure - add SafeXcel EIP197 crypto engine driver") Signed-off-by: Pali Rohár Acked-by: Marek Behún Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 9b1a158aec29..ad0d8c4a71ac 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1831,6 +1831,8 @@ static const struct of_device_id safexcel_of_match_table[] = { {}, }; +MODULE_DEVICE_TABLE(of, safexcel_of_match_table); + static struct platform_driver crypto_safexcel = { .probe = safexcel_probe, .remove = safexcel_remove, From 96ec8dfdd094b7b2b015e092d581835a732949ff Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Tue, 19 Jul 2022 11:13:28 -0500 Subject: [PATCH 310/318] crypto: ccp - Add support for new CCP/PSP device ID Add a new CCP/PSP PCI device ID. This uses same register offsets as the previously supported structure. Signed-off-by: Mario Limonciello Acked-by: Tom Lendacky Acked-by: Rijo Thomas Signed-off-by: Herbert Xu --- drivers/crypto/ccp/sp-pci.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/crypto/ccp/sp-pci.c b/drivers/crypto/ccp/sp-pci.c index b5970ae54d0e..792d6da7f0c0 100644 --- a/drivers/crypto/ccp/sp-pci.c +++ b/drivers/crypto/ccp/sp-pci.c @@ -427,6 +427,12 @@ static const struct sp_dev_vdata dev_vdata[] = { .bar = 2, #ifdef CONFIG_CRYPTO_DEV_SP_PSP .psp_vdata = &pspv2, +#endif + }, + { /* 6 */ + .bar = 2, +#ifdef CONFIG_CRYPTO_DEV_SP_PSP + .psp_vdata = &pspv3, #endif }, }; @@ -438,6 +444,7 @@ static const struct pci_device_id sp_pci_table[] = { { PCI_VDEVICE(AMD, 0x15DF), (kernel_ulong_t)&dev_vdata[4] }, { PCI_VDEVICE(AMD, 0x1649), (kernel_ulong_t)&dev_vdata[4] }, { PCI_VDEVICE(AMD, 0x14CA), (kernel_ulong_t)&dev_vdata[5] }, + { PCI_VDEVICE(AMD, 0x15C7), (kernel_ulong_t)&dev_vdata[6] }, /* Last entry must be zero */ { 0, } }; From 383ce25dd2b117da0c092dc0b45e7ad0e9ffcbb2 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Wed, 20 Jul 2022 15:28:44 +0200 Subject: [PATCH 311/318] crypto: ccree - Remove a useless dma_supported() call There is no point in calling dma_supported() before calling dma_set_coherent_mask(). This function already calls dma_supported() and returns an error (-EIO) if it fails. So remove the superfluous dma_supported() call. Moreover, setting a larger DMA mask will never fail when setting a smaller one will succeed, so the whole "while" loop can be removed as well. (see [1]) While at it, fix the name of the function reported in a dev_err(). [1]: https://lore.kernel.org/all/YteQ6Vx2C03UtCkG@infradead.org/ Suggested-by: Christoph Hellwig Signed-off-by: Christophe JAILLET Acked-by: Gilad Ben-Yossef Signed-off-by: Herbert Xu --- drivers/crypto/ccree/cc_driver.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/drivers/crypto/ccree/cc_driver.c b/drivers/crypto/ccree/cc_driver.c index 7d1bee86d581..cadead18b59e 100644 --- a/drivers/crypto/ccree/cc_driver.c +++ b/drivers/crypto/ccree/cc_driver.c @@ -372,17 +372,10 @@ static int init_cc_resources(struct platform_device *plat_dev) dev->dma_mask = &dev->coherent_dma_mask; dma_mask = DMA_BIT_MASK(DMA_BIT_MASK_LEN); - while (dma_mask > 0x7fffffffUL) { - if (dma_supported(dev, dma_mask)) { - rc = dma_set_coherent_mask(dev, dma_mask); - if (!rc) - break; - } - dma_mask >>= 1; - } - + rc = dma_set_coherent_mask(dev, dma_mask); if (rc) { - dev_err(dev, "Failed in dma_set_mask, mask=%llx\n", dma_mask); + dev_err(dev, "Failed in dma_set_coherent_mask, mask=%llx\n", + dma_mask); return rc; } From 45f5d0176d8426cc1ab0bab84fbd8ef5c57526c6 Mon Sep 17 00:00:00 2001 From: Kai Ye Date: Thu, 21 Jul 2022 10:18:31 +0800 Subject: [PATCH 312/318] crypto: hisilicon/sec - fix auth key size error The authentication algorithm supports a maximum of 128-byte keys. The allocated key memory is insufficient. Fixes: 2f072d75d1ab ("crypto: hisilicon - Add aead support on SEC2") Signed-off-by: Kai Ye Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sec2/sec_crypto.c | 6 +++--- drivers/crypto/hisilicon/sec2/sec_crypto.h | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 71dfa7db6394..77c9f13cf69a 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -620,7 +620,7 @@ static int sec_auth_init(struct sec_ctx *ctx) { struct sec_auth_ctx *a_ctx = &ctx->a_ctx; - a_ctx->a_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE, + a_ctx->a_key = dma_alloc_coherent(ctx->dev, SEC_MAX_AKEY_SIZE, &a_ctx->a_key_dma, GFP_KERNEL); if (!a_ctx->a_key) return -ENOMEM; @@ -632,8 +632,8 @@ static void sec_auth_uninit(struct sec_ctx *ctx) { struct sec_auth_ctx *a_ctx = &ctx->a_ctx; - memzero_explicit(a_ctx->a_key, SEC_MAX_KEY_SIZE); - dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE, + memzero_explicit(a_ctx->a_key, SEC_MAX_AKEY_SIZE); + dma_free_coherent(ctx->dev, SEC_MAX_AKEY_SIZE, a_ctx->a_key, a_ctx->a_key_dma); } diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.h b/drivers/crypto/hisilicon/sec2/sec_crypto.h index 5e039b50e9d4..d033f63b583f 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.h +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.h @@ -7,6 +7,7 @@ #define SEC_AIV_SIZE 12 #define SEC_IV_SIZE 24 #define SEC_MAX_KEY_SIZE 64 +#define SEC_MAX_AKEY_SIZE 128 #define SEC_COMM_SCENE 0 #define SEC_MIN_BLOCK_SZ 1 From 11364d61314eb97b12d6b6facb1ededada52fcc1 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Thu, 21 Jul 2022 22:58:53 +0200 Subject: [PATCH 313/318] crypto: hisilicon/zip - Use the bitmap API to allocate bitmaps Use bitmap_zalloc()/bitmap_free() instead of hand-writing them. It is less verbose and it improves the semantic. While at it, add an explicit include . Signed-off-by: Christophe JAILLET Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/zip/zip_crypto.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c index 67869513e48c..ad35434a3fdb 100644 --- a/drivers/crypto/hisilicon/zip/zip_crypto.c +++ b/drivers/crypto/hisilicon/zip/zip_crypto.c @@ -2,6 +2,7 @@ /* Copyright (c) 2019 HiSilicon Limited. */ #include #include +#include #include #include #include "zip.h" @@ -606,8 +607,7 @@ static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx) req_q = &ctx->qp_ctx[i].req_q; req_q->size = QM_Q_DEPTH; - req_q->req_bitmap = kcalloc(BITS_TO_LONGS(req_q->size), - sizeof(long), GFP_KERNEL); + req_q->req_bitmap = bitmap_zalloc(req_q->size, GFP_KERNEL); if (!req_q->req_bitmap) { ret = -ENOMEM; if (i == 0) @@ -631,11 +631,11 @@ static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx) return 0; err_free_loop1: - kfree(ctx->qp_ctx[HZIP_QPC_DECOMP].req_q.req_bitmap); + bitmap_free(ctx->qp_ctx[HZIP_QPC_DECOMP].req_q.req_bitmap); err_free_loop0: kfree(ctx->qp_ctx[HZIP_QPC_COMP].req_q.q); err_free_bitmap: - kfree(ctx->qp_ctx[HZIP_QPC_COMP].req_q.req_bitmap); + bitmap_free(ctx->qp_ctx[HZIP_QPC_COMP].req_q.req_bitmap); return ret; } @@ -645,7 +645,7 @@ static void hisi_zip_release_req_q(struct hisi_zip_ctx *ctx) for (i = 0; i < HZIP_CTX_Q_NUM; i++) { kfree(ctx->qp_ctx[i].req_q.q); - kfree(ctx->qp_ctx[i].req_q.req_bitmap); + bitmap_free(ctx->qp_ctx[i].req_q.req_bitmap); } } From 7ae19d422c7da84b5f13bc08b98bd737a08d3a53 Mon Sep 17 00:00:00 2001 From: GUO Zihua Date: Fri, 22 Jul 2022 14:31:57 +0800 Subject: [PATCH 314/318] crypto: arm64/poly1305 - fix a read out-of-bound A kasan error was reported during fuzzing: BUG: KASAN: slab-out-of-bounds in neon_poly1305_blocks.constprop.0+0x1b4/0x250 [poly1305_neon] Read of size 4 at addr ffff0010e293f010 by task syz-executor.5/1646715 CPU: 4 PID: 1646715 Comm: syz-executor.5 Kdump: loaded Not tainted 5.10.0.aarch64 #1 Hardware name: Huawei TaiShan 2280 /BC11SPCD, BIOS 1.59 01/31/2019 Call trace: dump_backtrace+0x0/0x394 show_stack+0x34/0x4c arch/arm64/kernel/stacktrace.c:196 __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x158/0x1e4 lib/dump_stack.c:118 print_address_description.constprop.0+0x68/0x204 mm/kasan/report.c:387 __kasan_report+0xe0/0x140 mm/kasan/report.c:547 kasan_report+0x44/0xe0 mm/kasan/report.c:564 check_memory_region_inline mm/kasan/generic.c:187 [inline] __asan_load4+0x94/0xd0 mm/kasan/generic.c:252 neon_poly1305_blocks.constprop.0+0x1b4/0x250 [poly1305_neon] neon_poly1305_do_update+0x6c/0x15c [poly1305_neon] neon_poly1305_update+0x9c/0x1c4 [poly1305_neon] crypto_shash_update crypto/shash.c:131 [inline] shash_finup_unaligned+0x84/0x15c crypto/shash.c:179 crypto_shash_finup+0x8c/0x140 crypto/shash.c:193 shash_digest_unaligned+0xb8/0xe4 crypto/shash.c:201 crypto_shash_digest+0xa4/0xfc crypto/shash.c:217 crypto_shash_tfm_digest+0xb4/0x150 crypto/shash.c:229 essiv_skcipher_setkey+0x164/0x200 [essiv] crypto_skcipher_setkey+0xb0/0x160 crypto/skcipher.c:612 skcipher_setkey+0x3c/0x50 crypto/algif_skcipher.c:305 alg_setkey+0x114/0x2a0 crypto/af_alg.c:220 alg_setsockopt+0x19c/0x210 crypto/af_alg.c:253 __sys_setsockopt+0x190/0x2e0 net/socket.c:2123 __do_sys_setsockopt net/socket.c:2134 [inline] __se_sys_setsockopt net/socket.c:2131 [inline] __arm64_sys_setsockopt+0x78/0x94 net/socket.c:2131 __invoke_syscall arch/arm64/kernel/syscall.c:36 [inline] invoke_syscall+0x64/0x100 arch/arm64/kernel/syscall.c:48 el0_svc_common.constprop.0+0x220/0x230 arch/arm64/kernel/syscall.c:155 do_el0_svc+0xb4/0xd4 arch/arm64/kernel/syscall.c:217 el0_svc+0x24/0x3c arch/arm64/kernel/entry-common.c:353 el0_sync_handler+0x160/0x164 arch/arm64/kernel/entry-common.c:369 el0_sync+0x160/0x180 arch/arm64/kernel/entry.S:683 This error can be reproduced by the following code compiled as ko on a system with kasan enabled: #include #include #include #include char test_data[] = "\x00\x01\x02\x03\x04\x05\x06\x07" "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" "\x10\x11\x12\x13\x14\x15\x16\x17" "\x18\x19\x1a\x1b\x1c\x1d\x1e"; int init(void) { struct crypto_shash *tfm = NULL; char *data = NULL, *out = NULL; tfm = crypto_alloc_shash("poly1305", 0, 0); data = kmalloc(POLY1305_KEY_SIZE - 1, GFP_KERNEL); out = kmalloc(POLY1305_DIGEST_SIZE, GFP_KERNEL); memcpy(data, test_data, POLY1305_KEY_SIZE - 1); crypto_shash_tfm_digest(tfm, data, POLY1305_KEY_SIZE - 1, out); kfree(data); kfree(out); return 0; } void deinit(void) { } module_init(init) module_exit(deinit) MODULE_LICENSE("GPL"); The root cause of the bug sits in neon_poly1305_blocks. The logic neon_poly1305_blocks() performed is that if it was called with both s[] and r[] uninitialized, it will first try to initialize them with the data from the first "block" that it believed to be 32 bytes in length. First 16 bytes are used as the key and the next 16 bytes for s[]. This would lead to the aforementioned read out-of-bound. However, after calling poly1305_init_arch(), only 16 bytes were deducted from the input and s[] is initialized yet again with the following 16 bytes. The second initialization of s[] is certainly redundent which indicates that the first initialization should be for r[] only. This patch fixes the issue by calling poly1305_init_arm64() instead of poly1305_init_arch(). This is also the implementation for the same algorithm on arm platform. Fixes: f569ca164751 ("crypto: arm64/poly1305 - incorporate OpenSSL/CRYPTOGAMS NEON implementation") Cc: stable@vger.kernel.org Signed-off-by: GUO Zihua Reviewed-by: Eric Biggers Acked-by: Will Deacon Signed-off-by: Herbert Xu --- arch/arm64/crypto/poly1305-glue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/crypto/poly1305-glue.c b/arch/arm64/crypto/poly1305-glue.c index 9c3d86e397bf..1fae18ba11ed 100644 --- a/arch/arm64/crypto/poly1305-glue.c +++ b/arch/arm64/crypto/poly1305-glue.c @@ -52,7 +52,7 @@ static void neon_poly1305_blocks(struct poly1305_desc_ctx *dctx, const u8 *src, { if (unlikely(!dctx->sset)) { if (!dctx->rset) { - poly1305_init_arch(dctx, src); + poly1305_init_arm64(&dctx->h, src); src += POLY1305_BLOCK_SIZE; len -= POLY1305_BLOCK_SIZE; dctx->rset = 1; From af5d35b83f642399c719ea9a8599a13b8a0c4167 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Sat, 23 Jul 2022 17:26:40 +0800 Subject: [PATCH 315/318] crypto: tcrypt - Remove the static variable initialisations to NULL Initialise global and static variable to NULL is always unnecessary. Remove the unnecessary initialisations. Signed-off-by: Jason Wang Signed-off-by: Herbert Xu --- crypto/tcrypt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index f56d1a9cf0a7..59eb8ec36664 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -58,7 +58,7 @@ */ static unsigned int sec; -static char *alg = NULL; +static char *alg; static u32 type; static u32 mask; static int mode; From 7f637be4d46029bd7700c9f244945a42dbd976fa Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Sat, 30 Jul 2022 01:12:25 +0200 Subject: [PATCH 316/318] random: correct spelling of "overwrites" It was missing an 'r'. Fixes: 186873c549df ("random: use simpler fast key erasure flow on per-cpu keys") Signed-off-by: Jason A. Donenfeld --- drivers/char/random.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/char/random.c b/drivers/char/random.c index 7bf11fa66265..d44832e9e709 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -229,7 +229,7 @@ static void crng_reseed(void) /* * This generates a ChaCha block using the provided key, and then - * immediately overwites that key with half the block. It returns + * immediately overwrites that key with half the block. It returns * the resultant ChaCha state to the user, along with the second * half of the block containing 32 bytes of random data that may * be used; random_data_len may not be greater than 32. From 63c3b5d2ca96b4a2a88ae01bea94021e874ce8fe Mon Sep 17 00:00:00 2001 From: "GONG, Ruiqi" Date: Mon, 6 Jun 2022 16:17:14 +0800 Subject: [PATCH 317/318] smack: Replace kzalloc + strncpy with kstrndup Simplify the code by using kstrndup instead of kzalloc and strncpy in smk_parse_smack(), which meanwhile remove strncpy as [1] suggests. [1]: https://github.com/KSPP/linux/issues/90 Signed-off-by: GONG, Ruiqi Signed-off-by: Casey Schaufler --- security/smack/smack_access.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/security/smack/smack_access.c b/security/smack/smack_access.c index d2186e2757be..585e5e35710b 100644 --- a/security/smack/smack_access.c +++ b/security/smack/smack_access.c @@ -465,12 +465,9 @@ char *smk_parse_smack(const char *string, int len) if (i == 0 || i >= SMK_LONGLABEL) return ERR_PTR(-EINVAL); - smack = kzalloc(i + 1, GFP_NOFS); - if (smack == NULL) + smack = kstrndup(string, i, GFP_NOFS); + if (!smack) return ERR_PTR(-ENOMEM); - - strncpy(smack, string, i); - return smack; } From aa16fb4b9e7e1057008d999138e7ae68a40bf167 Mon Sep 17 00:00:00 2001 From: Xiu Jianfeng Date: Fri, 10 Jun 2022 17:23:07 +0800 Subject: [PATCH 318/318] smack: Remove the redundant lsm_inode_alloc It's not possible for inode->i_security to be NULL here because every inode will call inode_init_always and then lsm_inode_alloc to alloc memory for inode->security, this is what LSM infrastructure management do, so remove this redundant code. Signed-off-by: Xiu Jianfeng Signed-off-by: Casey Schaufler --- security/smack/smack_lsm.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c index 6207762dbdb1..001831458fa2 100644 --- a/security/smack/smack_lsm.c +++ b/security/smack/smack_lsm.c @@ -766,13 +766,6 @@ static int smack_set_mnt_opts(struct super_block *sb, if (sp->smk_flags & SMK_SB_INITIALIZED) return 0; - if (inode->i_security == NULL) { - int rc = lsm_inode_alloc(inode); - - if (rc) - return rc; - } - if (!smack_privileged(CAP_MAC_ADMIN)) { /* * Unprivileged mounts don't get to specify Smack values.