diff options
Diffstat (limited to 'crypto')
108 files changed, 3118 insertions, 968 deletions
diff --git a/crypto/842.c b/crypto/842.c index bc26dc942821..5f98393b65d1 100644 --- a/crypto/842.c +++ b/crypto/842.c @@ -144,7 +144,7 @@ static int __init crypto842_mod_init(void) return ret; } -module_init(crypto842_mod_init); +subsys_initcall(crypto842_mod_init); static void __exit crypto842_mod_exit(void) { diff --git a/crypto/Kconfig b/crypto/Kconfig index bbab6bf33519..3d056e7da65f 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -27,8 +27,8 @@ config CRYPTO_FIPS depends on (CRYPTO_ANSI_CPRNG || CRYPTO_DRBG) && !CRYPTO_MANAGER_DISABLE_TESTS depends on (MODULE_SIG || !MODULES) help - This options enables the fips boot option which is - required if you want to system to operate in a FIPS 200 + This option enables the fips boot option which is + required if you want the system to operate in a FIPS 200 certification. You should say no unless you know what this is. @@ -113,29 +113,6 @@ config CRYPTO_ACOMP select CRYPTO_ALGAPI select CRYPTO_ACOMP2 -config CRYPTO_RSA - tristate "RSA algorithm" - select CRYPTO_AKCIPHER - select CRYPTO_MANAGER - select MPILIB - select ASN1 - help - Generic implementation of the RSA public key algorithm. - -config CRYPTO_DH - tristate "Diffie-Hellman algorithm" - select CRYPTO_KPP - select MPILIB - help - Generic implementation of the Diffie-Hellman algorithm. - -config CRYPTO_ECDH - tristate "ECDH algorithm" - select CRYPTO_KPP - select CRYPTO_RNG_DEFAULT - help - Generic implementation of the ECDH algorithm - config CRYPTO_MANAGER tristate "Cryptographic algorithm manager" select CRYPTO_MANAGER2 @@ -253,6 +230,48 @@ config CRYPTO_GLUE_HELPER_X86 config CRYPTO_ENGINE tristate +comment "Public-key cryptography" + +config CRYPTO_RSA + tristate "RSA algorithm" + select CRYPTO_AKCIPHER + select CRYPTO_MANAGER + select MPILIB + select ASN1 + help + Generic implementation of the RSA public key algorithm. + +config CRYPTO_DH + tristate "Diffie-Hellman algorithm" + select CRYPTO_KPP + select MPILIB + help + Generic implementation of the Diffie-Hellman algorithm. + +config CRYPTO_ECC + tristate + +config CRYPTO_ECDH + tristate "ECDH algorithm" + select CRYPTO_ECC + select CRYPTO_KPP + select CRYPTO_RNG_DEFAULT + help + Generic implementation of the ECDH algorithm + +config CRYPTO_ECRDSA + tristate "EC-RDSA (GOST 34.10) algorithm" + select CRYPTO_ECC + select CRYPTO_AKCIPHER + select CRYPTO_STREEBOG + select OID_REGISTRY + select ASN1 + help + Elliptic Curve Russian Digital Signature Algorithm (GOST R 34.10-2012, + RFC 7091, ISO/IEC 14888-3:2018) is one of the Russian cryptographic + standard algorithms (called GOST algorithms). Only signature verification + is implemented. + comment "Authenticated Encryption with Associated Data" config CRYPTO_CCM @@ -310,25 +329,25 @@ config CRYPTO_AEGIS128_AESNI_SSE2 tristate "AEGIS-128 AEAD algorithm (x86_64 AESNI+SSE2 implementation)" depends on X86 && 64BIT select CRYPTO_AEAD - select CRYPTO_CRYPTD + select CRYPTO_SIMD help - AESNI+SSE2 implementation of the AEGSI-128 dedicated AEAD algorithm. + AESNI+SSE2 implementation of the AEGIS-128 dedicated AEAD algorithm. config CRYPTO_AEGIS128L_AESNI_SSE2 tristate "AEGIS-128L AEAD algorithm (x86_64 AESNI+SSE2 implementation)" depends on X86 && 64BIT select CRYPTO_AEAD - select CRYPTO_CRYPTD + select CRYPTO_SIMD help - AESNI+SSE2 implementation of the AEGSI-128L dedicated AEAD algorithm. + AESNI+SSE2 implementation of the AEGIS-128L dedicated AEAD algorithm. config CRYPTO_AEGIS256_AESNI_SSE2 tristate "AEGIS-256 AEAD algorithm (x86_64 AESNI+SSE2 implementation)" depends on X86 && 64BIT select CRYPTO_AEAD - select CRYPTO_CRYPTD + select CRYPTO_SIMD help - AESNI+SSE2 implementation of the AEGSI-256 dedicated AEAD algorithm. + AESNI+SSE2 implementation of the AEGIS-256 dedicated AEAD algorithm. config CRYPTO_MORUS640 tristate "MORUS-640 AEAD algorithm" @@ -340,7 +359,7 @@ config CRYPTO_MORUS640_GLUE tristate depends on X86 select CRYPTO_AEAD - select CRYPTO_CRYPTD + select CRYPTO_SIMD help Common glue for SIMD optimizations of the MORUS-640 dedicated AEAD algorithm. @@ -363,7 +382,7 @@ config CRYPTO_MORUS1280_GLUE tristate depends on X86 select CRYPTO_AEAD - select CRYPTO_CRYPTD + select CRYPTO_SIMD help Common glue for SIMD optimizations of the MORUS-1280 dedicated AEAD algorithm. diff --git a/crypto/Makefile b/crypto/Makefile index fb5bf2a3a666..266a4cdbb9e2 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -147,12 +147,20 @@ obj-$(CONFIG_CRYPTO_USER_API_RNG) += algif_rng.o obj-$(CONFIG_CRYPTO_USER_API_AEAD) += algif_aead.o obj-$(CONFIG_CRYPTO_ZSTD) += zstd.o obj-$(CONFIG_CRYPTO_OFB) += ofb.o +obj-$(CONFIG_CRYPTO_ECC) += ecc.o -ecdh_generic-y := ecc.o ecdh_generic-y += ecdh.o ecdh_generic-y += ecdh_helper.o obj-$(CONFIG_CRYPTO_ECDH) += ecdh_generic.o +$(obj)/ecrdsa_params.asn1.o: $(obj)/ecrdsa_params.asn1.c $(obj)/ecrdsa_params.asn1.h +$(obj)/ecrdsa_pub_key.asn1.o: $(obj)/ecrdsa_pub_key.asn1.c $(obj)/ecrdsa_pub_key.asn1.h +$(obj)/ecrdsa.o: $(obj)/ecrdsa_params.asn1.h $(obj)/ecrdsa_pub_key.asn1.h +ecrdsa_generic-y += ecrdsa.o +ecrdsa_generic-y += ecrdsa_params.asn1.o +ecrdsa_generic-y += ecrdsa_pub_key.asn1.o +obj-$(CONFIG_CRYPTO_ECRDSA) += ecrdsa_generic.o + # # generic algorithms and the async_tx api # diff --git a/crypto/adiantum.c b/crypto/adiantum.c index 5564e73266a6..395a3ddd3707 100644 --- a/crypto/adiantum.c +++ b/crypto/adiantum.c @@ -265,7 +265,6 @@ static int adiantum_hash_message(struct skcipher_request *req, int err; hash_desc->tfm = tctx->hash; - hash_desc->flags = 0; err = crypto_shash_init(hash_desc); if (err) @@ -659,7 +658,7 @@ static void __exit adiantum_module_exit(void) crypto_unregister_template(&adiantum_tmpl); } -module_init(adiantum_module_init); +subsys_initcall(adiantum_module_init); module_exit(adiantum_module_exit); MODULE_DESCRIPTION("Adiantum length-preserving encryption mode"); diff --git a/crypto/aegis128.c b/crypto/aegis128.c index 3718a8341303..d78f77fc5dd1 100644 --- a/crypto/aegis128.c +++ b/crypto/aegis128.c @@ -448,7 +448,7 @@ static void __exit crypto_aegis128_module_exit(void) crypto_unregister_aead(&crypto_aegis128_alg); } -module_init(crypto_aegis128_module_init); +subsys_initcall(crypto_aegis128_module_init); module_exit(crypto_aegis128_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/aegis128l.c b/crypto/aegis128l.c index 275a8616d71b..9bca3d619a22 100644 --- a/crypto/aegis128l.c +++ b/crypto/aegis128l.c @@ -512,7 +512,7 @@ static void __exit crypto_aegis128l_module_exit(void) crypto_unregister_aead(&crypto_aegis128l_alg); } -module_init(crypto_aegis128l_module_init); +subsys_initcall(crypto_aegis128l_module_init); module_exit(crypto_aegis128l_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/aegis256.c b/crypto/aegis256.c index ecd6b7f34a2d..b47fd39595ad 100644 --- a/crypto/aegis256.c +++ b/crypto/aegis256.c @@ -463,7 +463,7 @@ static void __exit crypto_aegis256_module_exit(void) crypto_unregister_aead(&crypto_aegis256_alg); } -module_init(crypto_aegis256_module_init); +subsys_initcall(crypto_aegis256_module_init); module_exit(crypto_aegis256_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/aes_generic.c b/crypto/aes_generic.c index 13df33aca463..f217568917e4 100644 --- a/crypto/aes_generic.c +++ b/crypto/aes_generic.c @@ -64,7 +64,7 @@ static inline u8 byte(const u32 x, const unsigned n) static const u32 rco_tab[10] = { 1, 2, 4, 8, 16, 32, 64, 128, 27, 54 }; /* cacheline-aligned to facilitate prefetching into cache */ -__visible const u32 crypto_ft_tab[4][256] __cacheline_aligned = { +__visible const u32 crypto_ft_tab[4][256] ____cacheline_aligned = { { 0xa56363c6, 0x847c7cf8, 0x997777ee, 0x8d7b7bf6, 0x0df2f2ff, 0xbd6b6bd6, 0xb16f6fde, 0x54c5c591, @@ -328,7 +328,7 @@ __visible const u32 crypto_ft_tab[4][256] __cacheline_aligned = { } }; -__visible const u32 crypto_fl_tab[4][256] __cacheline_aligned = { +__visible const u32 crypto_fl_tab[4][256] ____cacheline_aligned = { { 0x00000063, 0x0000007c, 0x00000077, 0x0000007b, 0x000000f2, 0x0000006b, 0x0000006f, 0x000000c5, @@ -592,7 +592,7 @@ __visible const u32 crypto_fl_tab[4][256] __cacheline_aligned = { } }; -__visible const u32 crypto_it_tab[4][256] __cacheline_aligned = { +__visible const u32 crypto_it_tab[4][256] ____cacheline_aligned = { { 0x50a7f451, 0x5365417e, 0xc3a4171a, 0x965e273a, 0xcb6bab3b, 0xf1459d1f, 0xab58faac, 0x9303e34b, @@ -856,7 +856,7 @@ __visible const u32 crypto_it_tab[4][256] __cacheline_aligned = { } }; -__visible const u32 crypto_il_tab[4][256] __cacheline_aligned = { +__visible const u32 crypto_il_tab[4][256] ____cacheline_aligned = { { 0x00000052, 0x00000009, 0x0000006a, 0x000000d5, 0x00000030, 0x00000036, 0x000000a5, 0x00000038, @@ -1470,7 +1470,7 @@ static void __exit aes_fini(void) crypto_unregister_alg(&aes_alg); } -module_init(aes_init); +subsys_initcall(aes_init); module_exit(aes_fini); MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm"); diff --git a/crypto/akcipher.c b/crypto/akcipher.c index 0cbeae137e0a..780daa436dac 100644 --- a/crypto/akcipher.c +++ b/crypto/akcipher.c @@ -119,10 +119,24 @@ static void akcipher_prepare_alg(struct akcipher_alg *alg) base->cra_flags |= CRYPTO_ALG_TYPE_AKCIPHER; } +static int akcipher_default_op(struct akcipher_request *req) +{ + return -ENOSYS; +} + int crypto_register_akcipher(struct akcipher_alg *alg) { struct crypto_alg *base = &alg->base; + if (!alg->sign) + alg->sign = akcipher_default_op; + if (!alg->verify) + alg->verify = akcipher_default_op; + if (!alg->encrypt) + alg->encrypt = akcipher_default_op; + if (!alg->decrypt) + alg->decrypt = akcipher_default_op; + akcipher_prepare_alg(alg); return crypto_register_alg(base); } diff --git a/crypto/algboss.c b/crypto/algboss.c index 527b44d0af21..bb97cfb38836 100644 --- a/crypto/algboss.c +++ b/crypto/algboss.c @@ -296,7 +296,13 @@ static void __exit cryptomgr_exit(void) BUG_ON(err); } -subsys_initcall(cryptomgr_init); +/* + * This is arch_initcall() so that the crypto self-tests are run on algorithms + * registered early by subsys_initcall(). subsys_initcall() is needed for + * generic implementations so that they're available for comparison tests when + * other implementations are registered later by module_init(). + */ +arch_initcall(cryptomgr_init); module_exit(cryptomgr_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/ansi_cprng.c b/crypto/ansi_cprng.c index eff337ce9003..e7c43ea4ce9d 100644 --- a/crypto/ansi_cprng.c +++ b/crypto/ansi_cprng.c @@ -472,7 +472,7 @@ MODULE_DESCRIPTION("Software Pseudo Random Number Generator"); MODULE_AUTHOR("Neil Horman <nhorman@tuxdriver.com>"); module_param(dbg, int, 0); MODULE_PARM_DESC(dbg, "Boolean to enable debugging (0/1 == off/on)"); -module_init(prng_mod_init); +subsys_initcall(prng_mod_init); module_exit(prng_mod_fini); MODULE_ALIAS_CRYPTO("stdrng"); MODULE_ALIAS_CRYPTO("ansi_cprng"); diff --git a/crypto/anubis.c b/crypto/anubis.c index 4bb187c2a902..673927de0eb9 100644 --- a/crypto/anubis.c +++ b/crypto/anubis.c @@ -699,7 +699,7 @@ static void __exit anubis_mod_fini(void) crypto_unregister_alg(&anubis_alg); } -module_init(anubis_mod_init); +subsys_initcall(anubis_mod_init); module_exit(anubis_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/arc4.c b/crypto/arc4.c index 6c93342e3405..2233d36456e2 100644 --- a/crypto/arc4.c +++ b/crypto/arc4.c @@ -163,7 +163,7 @@ static void __exit arc4_exit(void) crypto_unregister_skcipher(&arc4_skcipher); } -module_init(arc4_init); +subsys_initcall(arc4_init); module_exit(arc4_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/asymmetric_keys/asym_tpm.c b/crypto/asymmetric_keys/asym_tpm.c index 5d4c270463f6..76d2ce3a1b5b 100644 --- a/crypto/asymmetric_keys/asym_tpm.c +++ b/crypto/asymmetric_keys/asym_tpm.c @@ -276,6 +276,10 @@ static int tpm_sign(struct tpm_buf *tb, return datalen; } + +/* Room to fit two u32 zeros for algo id and parameters length. */ +#define SETKEY_PARAMS_SIZE (sizeof(u32) * 2) + /* * Maximum buffer size for the BER/DER encoded public key. The public key * is of the form SEQUENCE { INTEGER n, INTEGER e } where n is a maximum 2048 @@ -286,8 +290,9 @@ static int tpm_sign(struct tpm_buf *tb, * - 257 bytes of n * - max 2 bytes for INTEGER e type/length * - 3 bytes of e + * - 4+4 of zeros for set_pub_key parameters (SETKEY_PARAMS_SIZE) */ -#define PUB_KEY_BUF_SIZE (4 + 4 + 257 + 2 + 3) +#define PUB_KEY_BUF_SIZE (4 + 4 + 257 + 2 + 3 + SETKEY_PARAMS_SIZE) /* * Provide a part of a description of the key for /proc/keys. @@ -364,6 +369,8 @@ static uint32_t derive_pub_key(const void *pub_key, uint32_t len, uint8_t *buf) cur = encode_tag_length(cur, 0x02, sizeof(e)); memcpy(cur, e, sizeof(e)); cur += sizeof(e); + /* Zero parameters to satisfy set_pub_key ABI. */ + memset(cur, 0, SETKEY_PARAMS_SIZE); return cur - buf; } @@ -744,12 +751,10 @@ static int tpm_key_verify_signature(const struct key *key, struct crypto_wait cwait; struct crypto_akcipher *tfm; struct akcipher_request *req; - struct scatterlist sig_sg, digest_sg; + struct scatterlist src_sg[2]; char alg_name[CRYPTO_MAX_ALG_NAME]; uint8_t der_pub_key[PUB_KEY_BUF_SIZE]; uint32_t der_pub_key_len; - void *output; - unsigned int outlen; int ret; pr_devel("==>%s()\n", __func__); @@ -781,37 +786,17 @@ static int tpm_key_verify_signature(const struct key *key, if (!req) goto error_free_tfm; - ret = -ENOMEM; - outlen = crypto_akcipher_maxsize(tfm); - output = kmalloc(outlen, GFP_KERNEL); - if (!output) - goto error_free_req; - - sg_init_one(&sig_sg, sig->s, sig->s_size); - sg_init_one(&digest_sg, output, outlen); - akcipher_request_set_crypt(req, &sig_sg, &digest_sg, sig->s_size, - outlen); + sg_init_table(src_sg, 2); + sg_set_buf(&src_sg[0], sig->s, sig->s_size); + sg_set_buf(&src_sg[1], sig->digest, sig->digest_size); + akcipher_request_set_crypt(req, src_sg, NULL, sig->s_size, + sig->digest_size); crypto_init_wait(&cwait); akcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, crypto_req_done, &cwait); - - /* Perform the verification calculation. This doesn't actually do the - * verification, but rather calculates the hash expected by the - * signature and returns that to us. - */ ret = crypto_wait_req(crypto_akcipher_verify(req), &cwait); - if (ret) - goto out_free_output; - - /* Do the actual verification step. */ - if (req->dst_len != sig->digest_size || - memcmp(sig->digest, output, sig->digest_size) != 0) - ret = -EKEYREJECTED; -out_free_output: - kfree(output); -error_free_req: akcipher_request_free(req); error_free_tfm: crypto_free_akcipher(tfm); diff --git a/crypto/asymmetric_keys/pkcs7_verify.c b/crypto/asymmetric_keys/pkcs7_verify.c index 97c77f66b20d..f7b0980bf02d 100644 --- a/crypto/asymmetric_keys/pkcs7_verify.c +++ b/crypto/asymmetric_keys/pkcs7_verify.c @@ -56,7 +56,6 @@ static int pkcs7_digest(struct pkcs7_message *pkcs7, goto error_no_desc; desc->tfm = tfm; - desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; /* Digest the message [RFC2315 9.3] */ ret = crypto_shash_digest(desc, pkcs7->data, pkcs7->data_len, diff --git a/crypto/asymmetric_keys/public_key.c b/crypto/asymmetric_keys/public_key.c index f5d85b47fcc6..77e0ae7840ff 100644 --- a/crypto/asymmetric_keys/public_key.c +++ b/crypto/asymmetric_keys/public_key.c @@ -45,6 +45,7 @@ void public_key_free(struct public_key *key) { if (key) { kfree(key->key); + kfree(key->params); kfree(key); } } @@ -94,6 +95,12 @@ int software_key_determine_akcipher(const char *encoding, return -ENOPKG; } +static u8 *pkey_pack_u32(u8 *dst, u32 val) +{ + memcpy(dst, &val, sizeof(val)); + return dst + sizeof(val); +} + /* * Query information about a key. */ @@ -103,6 +110,7 @@ static int software_key_query(const struct kernel_pkey_params *params, struct crypto_akcipher *tfm; struct public_key *pkey = params->key->payload.data[asym_crypto]; char alg_name[CRYPTO_MAX_ALG_NAME]; + u8 *key, *ptr; int ret, len; ret = software_key_determine_akcipher(params->encoding, @@ -115,14 +123,22 @@ static int software_key_query(const struct kernel_pkey_params *params, if (IS_ERR(tfm)) return PTR_ERR(tfm); + key = kmalloc(pkey->keylen + sizeof(u32) * 2 + pkey->paramlen, + GFP_KERNEL); + if (!key) + goto error_free_tfm; + memcpy(key, pkey->key, pkey->keylen); + ptr = key + pkey->keylen; + ptr = pkey_pack_u32(ptr, pkey->algo); + ptr = pkey_pack_u32(ptr, pkey->paramlen); + memcpy(ptr, pkey->params, pkey->paramlen); + if (pkey->key_is_private) - ret = crypto_akcipher_set_priv_key(tfm, - pkey->key, pkey->keylen); + ret = crypto_akcipher_set_priv_key(tfm, key, pkey->keylen); else - ret = crypto_akcipher_set_pub_key(tfm, - pkey->key, pkey->keylen); + ret = crypto_akcipher_set_pub_key(tfm, key, pkey->keylen); if (ret < 0) - goto error_free_tfm; + goto error_free_key; len = crypto_akcipher_maxsize(tfm); info->key_size = len * 8; @@ -137,6 +153,8 @@ static int software_key_query(const struct kernel_pkey_params *params, KEYCTL_SUPPORTS_SIGN); ret = 0; +error_free_key: + kfree(key); error_free_tfm: crypto_free_akcipher(tfm); pr_devel("<==%s() = %d\n", __func__, ret); @@ -155,6 +173,7 @@ static int software_key_eds_op(struct kernel_pkey_params *params, struct crypto_wait cwait; struct scatterlist in_sg, out_sg; char alg_name[CRYPTO_MAX_ALG_NAME]; + char *key, *ptr; int ret; pr_devel("==>%s()\n", __func__); @@ -173,14 +192,23 @@ static int software_key_eds_op(struct kernel_pkey_params *params, if (!req) goto error_free_tfm; + key = kmalloc(pkey->keylen + sizeof(u32) * 2 + pkey->paramlen, + GFP_KERNEL); + if (!key) + goto error_free_req; + + memcpy(key, pkey->key, pkey->keylen); + ptr = key + pkey->keylen; + ptr = pkey_pack_u32(ptr, pkey->algo); + ptr = pkey_pack_u32(ptr, pkey->paramlen); + memcpy(ptr, pkey->params, pkey->paramlen); + if (pkey->key_is_private) - ret = crypto_akcipher_set_priv_key(tfm, - pkey->key, pkey->keylen); + ret = crypto_akcipher_set_priv_key(tfm, key, pkey->keylen); else - ret = crypto_akcipher_set_pub_key(tfm, - pkey->key, pkey->keylen); + ret = crypto_akcipher_set_pub_key(tfm, key, pkey->keylen); if (ret) - goto error_free_req; + goto error_free_key; sg_init_one(&in_sg, in, params->in_len); sg_init_one(&out_sg, out, params->out_len); @@ -210,6 +238,8 @@ static int software_key_eds_op(struct kernel_pkey_params *params, if (ret == 0) ret = req->dst_len; +error_free_key: + kfree(key); error_free_req: akcipher_request_free(req); error_free_tfm: @@ -227,10 +257,9 @@ int public_key_verify_signature(const struct public_key *pkey, struct crypto_wait cwait; struct crypto_akcipher *tfm; struct akcipher_request *req; - struct scatterlist sig_sg, digest_sg; + struct scatterlist src_sg[2]; char alg_name[CRYPTO_MAX_ALG_NAME]; - void *output; - unsigned int outlen; + char *key, *ptr; int ret; pr_devel("==>%s()\n", __func__); @@ -254,45 +283,37 @@ int public_key_verify_signature(const struct public_key *pkey, if (!req) goto error_free_tfm; + key = kmalloc(pkey->keylen + sizeof(u32) * 2 + pkey->paramlen, + GFP_KERNEL); + if (!key) + goto error_free_req; + + memcpy(key, pkey->key, pkey->keylen); + ptr = key + pkey->keylen; + ptr = pkey_pack_u32(ptr, pkey->algo); + ptr = pkey_pack_u32(ptr, pkey->paramlen); + memcpy(ptr, pkey->params, pkey->paramlen); + if (pkey->key_is_private) - ret = crypto_akcipher_set_priv_key(tfm, - pkey->key, pkey->keylen); + ret = crypto_akcipher_set_priv_key(tfm, key, pkey->keylen); else - ret = crypto_akcipher_set_pub_key(tfm, - pkey->key, pkey->keylen); + ret = crypto_akcipher_set_pub_key(tfm, key, pkey->keylen); if (ret) - goto error_free_req; - - ret = -ENOMEM; - outlen = crypto_akcipher_maxsize(tfm); - output = kmalloc(outlen, GFP_KERNEL); - if (!output) - goto error_free_req; + goto error_free_key; - sg_init_one(&sig_sg, sig->s, sig->s_size); - sg_init_one(&digest_sg, output, outlen); - akcipher_request_set_crypt(req, &sig_sg, &digest_sg, sig->s_size, - outlen); + sg_init_table(src_sg, 2); + sg_set_buf(&src_sg[0], sig->s, sig->s_size); + sg_set_buf(&src_sg[1], sig->digest, sig->digest_size); + akcipher_request_set_crypt(req, src_sg, NULL, sig->s_size, + sig->digest_size); crypto_init_wait(&cwait); akcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, crypto_req_done, &cwait); - - /* Perform the verification calculation. This doesn't actually do the - * verification, but rather calculates the hash expected by the - * signature and returns that to us. - */ ret = crypto_wait_req(crypto_akcipher_verify(req), &cwait); - if (ret) - goto out_free_output; - - /* Do the actual verification step. */ - if (req->dst_len != sig->digest_size || - memcmp(sig->digest, output, sig->digest_size) != 0) - ret = -EKEYREJECTED; -out_free_output: - kfree(output); +error_free_key: + kfree(key); error_free_req: akcipher_request_free(req); error_free_tfm: diff --git a/crypto/asymmetric_keys/verify_pefile.c b/crypto/asymmetric_keys/verify_pefile.c index d178650fd524..f8e4a932bcfb 100644 --- a/crypto/asymmetric_keys/verify_pefile.c +++ b/crypto/asymmetric_keys/verify_pefile.c @@ -354,7 +354,6 @@ static int pefile_digest_pe(const void *pebuf, unsigned int pelen, goto error_no_desc; desc->tfm = tfm; - desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; ret = crypto_shash_init(desc); if (ret < 0) goto error; diff --git a/crypto/asymmetric_keys/x509.asn1 b/crypto/asymmetric_keys/x509.asn1 index aae0cde414e2..5c9f4e4a5231 100644 --- a/crypto/asymmetric_keys/x509.asn1 +++ b/crypto/asymmetric_keys/x509.asn1 @@ -22,7 +22,7 @@ CertificateSerialNumber ::= INTEGER AlgorithmIdentifier ::= SEQUENCE { algorithm OBJECT IDENTIFIER ({ x509_note_OID }), - parameters ANY OPTIONAL + parameters ANY OPTIONAL ({ x509_note_params }) } Name ::= SEQUENCE OF RelativeDistinguishedName diff --git a/crypto/asymmetric_keys/x509_cert_parser.c b/crypto/asymmetric_keys/x509_cert_parser.c index 991f4d735a4e..5b7bfd95c334 100644 --- a/crypto/asymmetric_keys/x509_cert_parser.c +++ b/crypto/asymmetric_keys/x509_cert_parser.c @@ -26,6 +26,9 @@ struct x509_parse_context { const void *cert_start; /* Start of cert content */ const void *key; /* Key data */ size_t key_size; /* Size of key data */ + const void *params; /* Key parameters */ + size_t params_size; /* Size of key parameters */ + enum OID key_algo; /* Public key algorithm */ enum OID last_oid; /* Last OID encountered */ enum OID algo_oid; /* Algorithm OID */ unsigned char nr_mpi; /* Number of MPIs stored */ @@ -109,6 +112,13 @@ struct x509_certificate *x509_cert_parse(const void *data, size_t datalen) cert->pub->keylen = ctx->key_size; + cert->pub->params = kmemdup(ctx->params, ctx->params_size, GFP_KERNEL); + if (!cert->pub->params) + goto error_decode; + + cert->pub->paramlen = ctx->params_size; + cert->pub->algo = ctx->key_algo; + /* Grab the signature bits */ ret = x509_get_sig_params(cert); if (ret < 0) @@ -220,6 +230,14 @@ int x509_note_pkey_algo(void *context, size_t hdrlen, case OID_sha224WithRSAEncryption: ctx->cert->sig->hash_algo = "sha224"; goto rsa_pkcs1; + + case OID_gost2012Signature256: + ctx->cert->sig->hash_algo = "streebog256"; + goto ecrdsa; + + case OID_gost2012Signature512: + ctx->cert->sig->hash_algo = "streebog512"; + goto ecrdsa; } rsa_pkcs1: @@ -227,6 +245,11 @@ rsa_pkcs1: ctx->cert->sig->encoding = "pkcs1"; ctx->algo_oid = ctx->last_oid; return 0; +ecrdsa: + ctx->cert->sig->pkey_algo = "ecrdsa"; + ctx->cert->sig->encoding = "raw"; + ctx->algo_oid = ctx->last_oid; + return 0; } /* @@ -246,7 +269,8 @@ int x509_note_signature(void *context, size_t hdrlen, return -EINVAL; } - if (strcmp(ctx->cert->sig->pkey_algo, "rsa") == 0) { + if (strcmp(ctx->cert->sig->pkey_algo, "rsa") == 0 || + strcmp(ctx->cert->sig->pkey_algo, "ecrdsa") == 0) { /* Discard the BIT STRING metadata */ if (vlen < 1 || *(const u8 *)value != 0) return -EBADMSG; @@ -401,6 +425,27 @@ int x509_note_subject(void *context, size_t hdrlen, } /* + * Extract the parameters for the public key + */ +int x509_note_params(void *context, size_t hdrlen, + unsigned char tag, + const void *value, size_t vlen) +{ + struct x509_parse_context *ctx = context; + + /* + * AlgorithmIdentifier is used three times in the x509, we should skip + * first and ignore third, using second one which is after subject and + * before subjectPublicKey. + */ + if (!ctx->cert->raw_subject || ctx->key) + return 0; + ctx->params = value - hdrlen; + ctx->params_size = vlen + hdrlen; + return 0; +} + +/* * Extract the data for the public key algorithm */ int x509_extract_key_data(void *context, size_t hdrlen, @@ -409,11 +454,15 @@ int x509_extract_key_data(void *context, size_t hdrlen, { struct x509_parse_context *ctx = context; - if (ctx->last_oid != OID_rsaEncryption) + ctx->key_algo = ctx->last_oid; + if (ctx->last_oid == OID_rsaEncryption) + ctx->cert->pub->pkey_algo = "rsa"; + else if (ctx->last_oid == OID_gost2012PKey256 || + ctx->last_oid == OID_gost2012PKey512) + ctx->cert->pub->pkey_algo = "ecrdsa"; + else return -ENOPKG; - ctx->cert->pub->pkey_algo = "rsa"; - /* Discard the BIT STRING metadata */ if (vlen < 1 || *(const u8 *)value != 0) return -EBADMSG; diff --git a/crypto/asymmetric_keys/x509_public_key.c b/crypto/asymmetric_keys/x509_public_key.c index 9338b4558cdc..bd96683d8cde 100644 --- a/crypto/asymmetric_keys/x509_public_key.c +++ b/crypto/asymmetric_keys/x509_public_key.c @@ -77,7 +77,6 @@ int x509_get_sig_params(struct x509_certificate *cert) goto error; desc->tfm = tfm; - desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; ret = crypto_shash_digest(desc, cert->tbs, cert->tbs_size, sig->digest); if (ret < 0) diff --git a/crypto/authenc.c b/crypto/authenc.c index 4be293a4b5f0..b3eddac7fa3a 100644 --- a/crypto/authenc.c +++ b/crypto/authenc.c @@ -508,7 +508,7 @@ static void __exit crypto_authenc_module_exit(void) crypto_unregister_template(&crypto_authenc_tmpl); } -module_init(crypto_authenc_module_init); +subsys_initcall(crypto_authenc_module_init); module_exit(crypto_authenc_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/authencesn.c b/crypto/authencesn.c index 4741fe89ba2c..58074308e535 100644 --- a/crypto/authencesn.c +++ b/crypto/authencesn.c @@ -523,7 +523,7 @@ static void __exit crypto_authenc_esn_module_exit(void) crypto_unregister_template(&crypto_authenc_esn_tmpl); } -module_init(crypto_authenc_esn_module_init); +subsys_initcall(crypto_authenc_esn_module_init); module_exit(crypto_authenc_esn_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/blowfish_generic.c b/crypto/blowfish_generic.c index 87b392a77a93..8548ced8b074 100644 --- a/crypto/blowfish_generic.c +++ b/crypto/blowfish_generic.c @@ -133,7 +133,7 @@ static void __exit blowfish_mod_fini(void) crypto_unregister_alg(&alg); } -module_init(blowfish_mod_init); +subsys_initcall(blowfish_mod_init); module_exit(blowfish_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/camellia_generic.c b/crypto/camellia_generic.c index 32ddd4836ff5..15ce1281f5d9 100644 --- a/crypto/camellia_generic.c +++ b/crypto/camellia_generic.c @@ -1092,7 +1092,7 @@ static void __exit camellia_fini(void) crypto_unregister_alg(&camellia_alg); } -module_init(camellia_init); +subsys_initcall(camellia_init); module_exit(camellia_fini); MODULE_DESCRIPTION("Camellia Cipher Algorithm"); diff --git a/crypto/cast5_generic.c b/crypto/cast5_generic.c index 66169c178314..24bc7d4e33be 100644 --- a/crypto/cast5_generic.c +++ b/crypto/cast5_generic.c @@ -543,7 +543,7 @@ static void __exit cast5_mod_fini(void) crypto_unregister_alg(&alg); } -module_init(cast5_mod_init); +subsys_initcall(cast5_mod_init); module_exit(cast5_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/cast6_generic.c b/crypto/cast6_generic.c index c8e5ec69790e..edd59cc34991 100644 --- a/crypto/cast6_generic.c +++ b/crypto/cast6_generic.c @@ -285,7 +285,7 @@ static void __exit cast6_mod_fini(void) crypto_unregister_alg(&alg); } -module_init(cast6_mod_init); +subsys_initcall(cast6_mod_init); module_exit(cast6_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/cbc.c b/crypto/cbc.c index d12efaac9230..129f79d03365 100644 --- a/crypto/cbc.c +++ b/crypto/cbc.c @@ -98,7 +98,7 @@ static void __exit crypto_cbc_module_exit(void) crypto_unregister_template(&crypto_cbc_tmpl); } -module_init(crypto_cbc_module_init); +subsys_initcall(crypto_cbc_module_init); module_exit(crypto_cbc_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/ccm.c b/crypto/ccm.c index 50df8f001c1c..c1ef9d0b4271 100644 --- a/crypto/ccm.c +++ b/crypto/ccm.c @@ -458,7 +458,6 @@ static void crypto_ccm_free(struct aead_instance *inst) static int crypto_ccm_create_common(struct crypto_template *tmpl, struct rtattr **tb, - const char *full_name, const char *ctr_name, const char *mac_name) { @@ -486,7 +485,8 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl, mac = __crypto_hash_alg_common(mac_alg); err = -EINVAL; - if (mac->digestsize != 16) + if (strncmp(mac->base.cra_name, "cbcmac(", 7) != 0 || + mac->digestsize != 16) goto out_put_mac; inst = kzalloc(sizeof(*inst) + sizeof(*ictx), GFP_KERNEL); @@ -509,23 +509,27 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl, ctr = crypto_spawn_skcipher_alg(&ictx->ctr); - /* Not a stream cipher? */ + /* The skcipher algorithm must be CTR mode, using 16-byte blocks. */ err = -EINVAL; - if (ctr->base.cra_blocksize != 1) + if (strncmp(ctr->base.cra_name, "ctr(", 4) != 0 || + crypto_skcipher_alg_ivsize(ctr) != 16 || + ctr->base.cra_blocksize != 1) goto err_drop_ctr; - /* We want the real thing! */ - if (crypto_skcipher_alg_ivsize(ctr) != 16) + /* ctr and cbcmac must use the same underlying block cipher. */ + if (strcmp(ctr->base.cra_name + 4, mac->base.cra_name + 7) != 0) goto err_drop_ctr; err = -ENAMETOOLONG; + if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, + "ccm(%s", ctr->base.cra_name + 4) >= CRYPTO_MAX_ALG_NAME) + goto err_drop_ctr; + if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "ccm_base(%s,%s)", ctr->base.cra_driver_name, mac->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto err_drop_ctr; - memcpy(inst->alg.base.cra_name, full_name, CRYPTO_MAX_ALG_NAME); - inst->alg.base.cra_flags = ctr->base.cra_flags & CRYPTO_ALG_ASYNC; inst->alg.base.cra_priority = (mac->base.cra_priority + ctr->base.cra_priority) / 2; @@ -567,7 +571,6 @@ static int crypto_ccm_create(struct crypto_template *tmpl, struct rtattr **tb) const char *cipher_name; char ctr_name[CRYPTO_MAX_ALG_NAME]; char mac_name[CRYPTO_MAX_ALG_NAME]; - char full_name[CRYPTO_MAX_ALG_NAME]; cipher_name = crypto_attr_alg_name(tb[1]); if (IS_ERR(cipher_name)) @@ -581,35 +584,24 @@ static int crypto_ccm_create(struct crypto_template *tmpl, struct rtattr **tb) cipher_name) >= CRYPTO_MAX_ALG_NAME) return -ENAMETOOLONG; - if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "ccm(%s)", cipher_name) >= - CRYPTO_MAX_ALG_NAME) - return -ENAMETOOLONG; - - return crypto_ccm_create_common(tmpl, tb, full_name, ctr_name, - mac_name); + return crypto_ccm_create_common(tmpl, tb, ctr_name, mac_name); } static int crypto_ccm_base_create(struct crypto_template *tmpl, struct rtattr **tb) { const char *ctr_name; - const char *cipher_name; - char full_name[CRYPTO_MAX_ALG_NAME]; + const char *mac_name; ctr_name = crypto_attr_alg_name(tb[1]); if (IS_ERR(ctr_name)) return PTR_ERR(ctr_name); - cipher_name = crypto_attr_alg_name(tb[2]); - if (IS_ERR(cipher_name)) - return PTR_ERR(cipher_name); - - if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "ccm_base(%s,%s)", - ctr_name, cipher_name) >= CRYPTO_MAX_ALG_NAME) - return -ENAMETOOLONG; + mac_name = crypto_attr_alg_name(tb[2]); + if (IS_ERR(mac_name)) + return PTR_ERR(mac_name); - return crypto_ccm_create_common(tmpl, tb, full_name, ctr_name, - cipher_name); + return crypto_ccm_create_common(tmpl, tb, ctr_name, mac_name); } static int crypto_rfc4309_setkey(struct crypto_aead *parent, const u8 *key, @@ -1014,7 +1006,7 @@ static void __exit crypto_ccm_module_exit(void) ARRAY_SIZE(crypto_ccm_tmpls)); } -module_init(crypto_ccm_module_init); +subsys_initcall(crypto_ccm_module_init); module_exit(crypto_ccm_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/cfb.c b/crypto/cfb.c index 03ac847f6d6a..7b68fbb61732 100644 --- a/crypto/cfb.c +++ b/crypto/cfb.c @@ -243,7 +243,7 @@ static void __exit crypto_cfb_module_exit(void) crypto_unregister_template(&crypto_cfb_tmpl); } -module_init(crypto_cfb_module_init); +subsys_initcall(crypto_cfb_module_init); module_exit(crypto_cfb_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/chacha20poly1305.c b/crypto/chacha20poly1305.c index ed2e12e26dd8..e38a2d61819a 100644 --- a/crypto/chacha20poly1305.c +++ b/crypto/chacha20poly1305.c @@ -645,8 +645,8 @@ static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb, err = -ENAMETOOLONG; if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, - "%s(%s,%s)", name, chacha_name, - poly_name) >= CRYPTO_MAX_ALG_NAME) + "%s(%s,%s)", name, chacha->base.cra_name, + poly->cra_name) >= CRYPTO_MAX_ALG_NAME) goto out_drop_chacha; if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s(%s,%s)", name, chacha->base.cra_driver_name, @@ -725,7 +725,7 @@ static void __exit chacha20poly1305_module_exit(void) ARRAY_SIZE(rfc7539_tmpls)); } -module_init(chacha20poly1305_module_init); +subsys_initcall(chacha20poly1305_module_init); module_exit(chacha20poly1305_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/chacha_generic.c b/crypto/chacha_generic.c index 35b583101f4f..d2ec04997832 100644 --- a/crypto/chacha_generic.c +++ b/crypto/chacha_generic.c @@ -22,18 +22,16 @@ static void chacha_docrypt(u32 *state, u8 *dst, const u8 *src, /* aligned to potentially speed up crypto_xor() */ u8 stream[CHACHA_BLOCK_SIZE] __aligned(sizeof(long)); - if (dst != src) - memcpy(dst, src, bytes); - while (bytes >= CHACHA_BLOCK_SIZE) { chacha_block(state, stream, nrounds); - crypto_xor(dst, stream, CHACHA_BLOCK_SIZE); + crypto_xor_cpy(dst, src, stream, CHACHA_BLOCK_SIZE); bytes -= CHACHA_BLOCK_SIZE; dst += CHACHA_BLOCK_SIZE; + src += CHACHA_BLOCK_SIZE; } if (bytes) { chacha_block(state, stream, nrounds); - crypto_xor(dst, stream, bytes); + crypto_xor_cpy(dst, src, stream, bytes); } } @@ -52,7 +50,7 @@ static int chacha_stream_xor(struct skcipher_request *req, unsigned int nbytes = walk.nbytes; if (nbytes < walk.total) - nbytes = round_down(nbytes, walk.stride); + nbytes = round_down(nbytes, CHACHA_BLOCK_SIZE); chacha_docrypt(state, walk.dst.virt.addr, walk.src.virt.addr, nbytes, ctx->nrounds); @@ -203,7 +201,7 @@ static void __exit chacha_generic_mod_fini(void) crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); } -module_init(chacha_generic_mod_init); +subsys_initcall(chacha_generic_mod_init); module_exit(chacha_generic_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/cmac.c b/crypto/cmac.c index 16301f52858c..c60b6c011ec6 100644 --- a/crypto/cmac.c +++ b/crypto/cmac.c @@ -313,7 +313,7 @@ static void __exit crypto_cmac_module_exit(void) crypto_unregister_template(&crypto_cmac_tmpl); } -module_init(crypto_cmac_module_init); +subsys_initcall(crypto_cmac_module_init); module_exit(crypto_cmac_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/crc32_generic.c b/crypto/crc32_generic.c index 00facd27bcc2..9e97912280bd 100644 --- a/crypto/crc32_generic.c +++ b/crypto/crc32_generic.c @@ -146,7 +146,7 @@ static void __exit crc32_mod_fini(void) crypto_unregister_shash(&alg); } -module_init(crc32_mod_init); +subsys_initcall(crc32_mod_init); module_exit(crc32_mod_fini); MODULE_AUTHOR("Alexander Boyko <alexander_boyko@xyratex.com>"); diff --git a/crypto/crc32c_generic.c b/crypto/crc32c_generic.c index 7283066ecc98..ad26f15d4c7b 100644 --- a/crypto/crc32c_generic.c +++ b/crypto/crc32c_generic.c @@ -165,7 +165,7 @@ static void __exit crc32c_mod_fini(void) crypto_unregister_shash(&alg); } -module_init(crc32c_mod_init); +subsys_initcall(crc32c_mod_init); module_exit(crc32c_mod_fini); MODULE_AUTHOR("Clay Haapala <chaapala@cisco.com>"); diff --git a/crypto/crct10dif_generic.c b/crypto/crct10dif_generic.c index 8e94e29dc6fc..d90c0070710e 100644 --- a/crypto/crct10dif_generic.c +++ b/crypto/crct10dif_generic.c @@ -65,10 +65,9 @@ static int chksum_final(struct shash_desc *desc, u8 *out) return 0; } -static int __chksum_finup(__u16 *crcp, const u8 *data, unsigned int len, - u8 *out) +static int __chksum_finup(__u16 crc, const u8 *data, unsigned int len, u8 *out) { - *(__u16 *)out = crc_t10dif_generic(*crcp, data, len); + *(__u16 *)out = crc_t10dif_generic(crc, data, len); return 0; } @@ -77,15 +76,13 @@ static int chksum_finup(struct shash_desc *desc, const u8 *data, { struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); - return __chksum_finup(&ctx->crc, data, len, out); + return __chksum_finup(ctx->crc, data, len, out); } static int chksum_digest(struct shash_desc *desc, const u8 *data, unsigned int length, u8 *out) { - struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); - - return __chksum_finup(&ctx->crc, data, length, out); + return __chksum_finup(0, data, length, out); } static struct shash_alg alg = { @@ -115,7 +112,7 @@ static void __exit crct10dif_mod_fini(void) crypto_unregister_shash(&alg); } -module_init(crct10dif_mod_init); +subsys_initcall(crct10dif_mod_init); module_exit(crct10dif_mod_fini); MODULE_AUTHOR("Tim Chen <tim.c.chen@linux.intel.com>"); diff --git a/crypto/cryptd.c b/crypto/cryptd.c index 5640e5db7bdb..b3bb99390ae7 100644 --- a/crypto/cryptd.c +++ b/crypto/cryptd.c @@ -65,15 +65,6 @@ struct aead_instance_ctx { struct cryptd_queue *queue; }; -struct cryptd_blkcipher_ctx { - atomic_t refcnt; - struct crypto_blkcipher *child; -}; - -struct cryptd_blkcipher_request_ctx { - crypto_completion_t complete; -}; - struct cryptd_skcipher_ctx { atomic_t refcnt; struct crypto_sync_skcipher *child; @@ -216,129 +207,6 @@ static inline void cryptd_check_internal(struct rtattr **tb, u32 *type, *mask |= algt->mask & CRYPTO_ALG_INTERNAL; } -static int cryptd_blkcipher_setkey(struct crypto_ablkcipher *parent, - const u8 *key, unsigned int keylen) -{ - struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(parent); - struct crypto_blkcipher *child = ctx->child; - int err; - - crypto_blkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); - crypto_blkcipher_set_flags(child, crypto_ablkcipher_get_flags(parent) & - CRYPTO_TFM_REQ_MASK); - err = crypto_blkcipher_setkey(child, key, keylen); - crypto_ablkcipher_set_flags(parent, crypto_blkcipher_get_flags(child) & - CRYPTO_TFM_RES_MASK); - return err; -} - -static void cryptd_blkcipher_crypt(struct ablkcipher_request *req, - struct crypto_blkcipher *child, - int err, - int (*crypt)(struct blkcipher_desc *desc, - struct scatterlist *dst, - struct scatterlist *src, - unsigned int len)) -{ - struct cryptd_blkcipher_request_ctx *rctx; - struct cryptd_blkcipher_ctx *ctx; - struct crypto_ablkcipher *tfm; - struct blkcipher_desc desc; - int refcnt; - - rctx = ablkcipher_request_ctx(req); - - if (unlikely(err == -EINPROGRESS)) - goto out; - - desc.tfm = child; - desc.info = req->info; - desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP; - - err = crypt(&desc, req->dst, req->src, req->nbytes); - - req->base.complete = rctx->complete; - -out: - tfm = crypto_ablkcipher_reqtfm(req); - ctx = crypto_ablkcipher_ctx(tfm); - refcnt = atomic_read(&ctx->refcnt); - - local_bh_disable(); - rctx->complete(&req->base, err); - local_bh_enable(); - - if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt)) - crypto_free_ablkcipher(tfm); -} - -static void cryptd_blkcipher_encrypt(struct crypto_async_request *req, int err) -{ - struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm); - struct crypto_blkcipher *child = ctx->child; - - cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err, - crypto_blkcipher_crt(child)->encrypt); -} - -static void cryptd_blkcipher_decrypt(struct crypto_async_request *req, int err) -{ - struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm); - struct crypto_blkcipher *child = ctx->child; - - cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err, - crypto_blkcipher_crt(child)->decrypt); -} - -static int cryptd_blkcipher_enqueue(struct ablkcipher_request *req, - crypto_completion_t compl) -{ - struct cryptd_blkcipher_request_ctx *rctx = ablkcipher_request_ctx(req); - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct cryptd_queue *queue; - - queue = cryptd_get_queue(crypto_ablkcipher_tfm(tfm)); - rctx->complete = req->base.complete; - req->base.complete = compl; - - return cryptd_enqueue_request(queue, &req->base); -} - -static int cryptd_blkcipher_encrypt_enqueue(struct ablkcipher_request *req) -{ - return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_encrypt); -} - -static int cryptd_blkcipher_decrypt_enqueue(struct ablkcipher_request *req) -{ - return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_decrypt); -} - -static int cryptd_blkcipher_init_tfm(struct crypto_tfm *tfm) -{ - struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); - struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst); - struct crypto_spawn *spawn = &ictx->spawn; - struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm); - struct crypto_blkcipher *cipher; - - cipher = crypto_spawn_blkcipher(spawn); - if (IS_ERR(cipher)) - return PTR_ERR(cipher); - - ctx->child = cipher; - tfm->crt_ablkcipher.reqsize = - sizeof(struct cryptd_blkcipher_request_ctx); - return 0; -} - -static void cryptd_blkcipher_exit_tfm(struct crypto_tfm *tfm) -{ - struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm); - - crypto_free_blkcipher(ctx->child); -} - static int cryptd_init_instance(struct crypto_instance *inst, struct crypto_alg *alg) { @@ -382,67 +250,6 @@ out_free_inst: goto out; } -static int cryptd_create_blkcipher(struct crypto_template *tmpl, - struct rtattr **tb, - struct cryptd_queue *queue) -{ - struct cryptd_instance_ctx *ctx; - struct crypto_instance *inst; - struct crypto_alg *alg; - u32 type = CRYPTO_ALG_TYPE_BLKCIPHER; - u32 mask = CRYPTO_ALG_TYPE_MASK; - int err; - - cryptd_check_internal(tb, &type, &mask); - - alg = crypto_get_attr_alg(tb, type, mask); - if (IS_ERR(alg)) - return PTR_ERR(alg); - - inst = cryptd_alloc_instance(alg, 0, sizeof(*ctx)); - err = PTR_ERR(inst); - if (IS_ERR(inst)) - goto out_put_alg; - - ctx = crypto_instance_ctx(inst); - ctx->queue = queue; - - err = crypto_init_spawn(&ctx->spawn, alg, inst, - CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_ASYNC); - if (err) - goto out_free_inst; - - type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC; - if (alg->cra_flags & CRYPTO_ALG_INTERNAL) - type |= CRYPTO_ALG_INTERNAL; - inst->alg.cra_flags = type; - inst->alg.cra_type = &crypto_ablkcipher_type; - - inst->alg.cra_ablkcipher.ivsize = alg->cra_blkcipher.ivsize; - inst->alg.cra_ablkcipher.min_keysize = alg->cra_blkcipher.min_keysize; - inst->alg.cra_ablkcipher.max_keysize = alg->cra_blkcipher.max_keysize; - - inst->alg.cra_ctxsize = sizeof(struct cryptd_blkcipher_ctx); - - inst->alg.cra_init = cryptd_blkcipher_init_tfm; - inst->alg.cra_exit = cryptd_blkcipher_exit_tfm; - - inst->alg.cra_ablkcipher.setkey = cryptd_blkcipher_setkey; - inst->alg.cra_ablkcipher.encrypt = cryptd_blkcipher_encrypt_enqueue; - inst->alg.cra_ablkcipher.decrypt = cryptd_blkcipher_decrypt_enqueue; - - err = crypto_register_instance(tmpl, inst); - if (err) { - crypto_drop_spawn(&ctx->spawn); -out_free_inst: - kfree(inst); - } - -out_put_alg: - crypto_mod_put(alg); - return err; -} - static int cryptd_skcipher_setkey(struct crypto_skcipher *parent, const u8 *key, unsigned int keylen) { @@ -738,7 +545,6 @@ static void cryptd_hash_init(struct crypto_async_request *req_async, int err) goto out; desc->tfm = child; - desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; err = crypto_shash_init(desc); @@ -830,7 +636,6 @@ static void cryptd_hash_digest(struct crypto_async_request *req_async, int err) goto out; desc->tfm = child; - desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; err = shash_ahash_digest(req, desc); @@ -859,7 +664,6 @@ static int cryptd_hash_import(struct ahash_request *req, const void *in) struct shash_desc *desc = cryptd_shash_desc(req); desc->tfm = ctx->child; - desc->flags = req->base.flags; return crypto_shash_import(desc, in); } @@ -1118,10 +922,6 @@ static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb) switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { case CRYPTO_ALG_TYPE_BLKCIPHER: - if ((algt->type & CRYPTO_ALG_TYPE_MASK) == - CRYPTO_ALG_TYPE_BLKCIPHER) - return cryptd_create_blkcipher(tmpl, tb, &queue); - return cryptd_create_skcipher(tmpl, tb, &queue); case CRYPTO_ALG_TYPE_DIGEST: return cryptd_create_hash(tmpl, tb, &queue); @@ -1160,58 +960,6 @@ static struct crypto_template cryptd_tmpl = { .module = THIS_MODULE, }; -struct cryptd_ablkcipher *cryptd_alloc_ablkcipher(const char *alg_name, - u32 type, u32 mask) -{ - char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; - struct cryptd_blkcipher_ctx *ctx; - struct crypto_tfm *tfm; - - if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, - "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) - return ERR_PTR(-EINVAL); - type = crypto_skcipher_type(type); - mask &= ~CRYPTO_ALG_TYPE_MASK; - mask |= CRYPTO_ALG_TYPE_BLKCIPHER_MASK; - tfm = crypto_alloc_base(cryptd_alg_name, type, mask); - if (IS_ERR(tfm)) - return ERR_CAST(tfm); - if (tfm->__crt_alg->cra_module != THIS_MODULE) { - crypto_free_tfm(tfm); - return ERR_PTR(-EINVAL); - } - - ctx = crypto_tfm_ctx(tfm); - atomic_set(&ctx->refcnt, 1); - - return __cryptd_ablkcipher_cast(__crypto_ablkcipher_cast(tfm)); -} -EXPORT_SYMBOL_GPL(cryptd_alloc_ablkcipher); - -struct crypto_blkcipher *cryptd_ablkcipher_child(struct cryptd_ablkcipher *tfm) -{ - struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base); - return ctx->child; -} -EXPORT_SYMBOL_GPL(cryptd_ablkcipher_child); - -bool cryptd_ablkcipher_queued(struct cryptd_ablkcipher *tfm) -{ - struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base); - - return atomic_read(&ctx->refcnt) - 1; -} -EXPORT_SYMBOL_GPL(cryptd_ablkcipher_queued); - -void cryptd_free_ablkcipher(struct cryptd_ablkcipher *tfm) -{ - struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base); - - if (atomic_dec_and_test(&ctx->refcnt)) - crypto_free_ablkcipher(&tfm->base); -} -EXPORT_SYMBOL_GPL(cryptd_free_ablkcipher); - struct cryptd_skcipher *cryptd_alloc_skcipher(const char *alg_name, u32 type, u32 mask) { diff --git a/crypto/crypto_null.c b/crypto/crypto_null.c index 01630a9c7e01..9320d4eaa4a8 100644 --- a/crypto/crypto_null.c +++ b/crypto/crypto_null.c @@ -220,7 +220,7 @@ static void __exit crypto_null_mod_fini(void) crypto_unregister_skcipher(&skcipher_null); } -module_init(crypto_null_mod_init); +subsys_initcall(crypto_null_mod_init); module_exit(crypto_null_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/ctr.c b/crypto/ctr.c index ec8f8b67473a..52cdf2c5605f 100644 --- a/crypto/ctr.c +++ b/crypto/ctr.c @@ -384,7 +384,7 @@ static void __exit crypto_ctr_module_exit(void) ARRAY_SIZE(crypto_ctr_tmpls)); } -module_init(crypto_ctr_module_init); +subsys_initcall(crypto_ctr_module_init); module_exit(crypto_ctr_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/cts.c b/crypto/cts.c index 4e28d83ae37d..6b6087dbb62a 100644 --- a/crypto/cts.c +++ b/crypto/cts.c @@ -152,12 +152,14 @@ static int crypto_cts_encrypt(struct skcipher_request *req) struct skcipher_request *subreq = &rctx->subreq; int bsize = crypto_skcipher_blocksize(tfm); unsigned int nbytes = req->cryptlen; - int cbc_blocks = (nbytes + bsize - 1) / bsize - 1; unsigned int offset; skcipher_request_set_tfm(subreq, ctx->child); - if (cbc_blocks <= 0) { + if (nbytes < bsize) + return -EINVAL; + + if (nbytes == bsize) { skcipher_request_set_callback(subreq, req->base.flags, req->base.complete, req->base.data); @@ -166,7 +168,7 @@ static int crypto_cts_encrypt(struct skcipher_request *req) return crypto_skcipher_encrypt(subreq); } - offset = cbc_blocks * bsize; + offset = rounddown(nbytes - 1, bsize); rctx->offset = offset; skcipher_request_set_callback(subreq, req->base.flags, @@ -244,13 +246,15 @@ static int crypto_cts_decrypt(struct skcipher_request *req) struct skcipher_request *subreq = &rctx->subreq; int bsize = crypto_skcipher_blocksize(tfm); unsigned int nbytes = req->cryptlen; - int cbc_blocks = (nbytes + bsize - 1) / bsize - 1; unsigned int offset; u8 *space; skcipher_request_set_tfm(subreq, ctx->child); - if (cbc_blocks <= 0) { + if (nbytes < bsize) + return -EINVAL; + + if (nbytes == bsize) { skcipher_request_set_callback(subreq, req->base.flags, req->base.complete, req->base.data); @@ -264,10 +268,10 @@ static int crypto_cts_decrypt(struct skcipher_request *req) space = crypto_cts_reqctx_space(req); - offset = cbc_blocks * bsize; + offset = rounddown(nbytes - 1, bsize); rctx->offset = offset; - if (cbc_blocks <= 1) + if (offset <= bsize) memcpy(space, req->iv, bsize); else scatterwalk_map_and_copy(space, req->src, offset - 2 * bsize, @@ -419,7 +423,7 @@ static void __exit crypto_cts_module_exit(void) crypto_unregister_template(&crypto_cts_tmpl); } -module_init(crypto_cts_module_init); +subsys_initcall(crypto_cts_module_init); module_exit(crypto_cts_module_exit); MODULE_LICENSE("Dual BSD/GPL"); diff --git a/crypto/deflate.c b/crypto/deflate.c index 94ec3b36a8e8..aab089cde1bf 100644 --- a/crypto/deflate.c +++ b/crypto/deflate.c @@ -334,7 +334,7 @@ static void __exit deflate_mod_fini(void) crypto_unregister_scomps(scomp, ARRAY_SIZE(scomp)); } -module_init(deflate_mod_init); +subsys_initcall(deflate_mod_init); module_exit(deflate_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/des_generic.c b/crypto/des_generic.c index 1e6621665dd9..d7a88b4fa611 100644 --- a/crypto/des_generic.c +++ b/crypto/des_generic.c @@ -862,14 +862,11 @@ static void des_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) int __des3_ede_setkey(u32 *expkey, u32 *flags, const u8 *key, unsigned int keylen) { - const u32 *K = (const u32 *)key; + int err; - if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) || - !((K[2] ^ K[4]) | (K[3] ^ K[5]))) && - (*flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { - *flags |= CRYPTO_TFM_RES_WEAK_KEY; - return -EINVAL; - } + err = __des3_verify_key(flags, key); + if (unlikely(err)) + return err; des_ekey(expkey, key); expkey += DES_EXPKEY_WORDS; key += DES_KEY_SIZE; dkey(expkey, key); expkey += DES_EXPKEY_WORDS; key += DES_KEY_SIZE; @@ -993,7 +990,7 @@ static void __exit des_generic_mod_fini(void) crypto_unregister_algs(des_algs, ARRAY_SIZE(des_algs)); } -module_init(des_generic_mod_init); +subsys_initcall(des_generic_mod_init); module_exit(des_generic_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/dh.c b/crypto/dh.c index 09a44de4209d..ce77fb4ee8b3 100644 --- a/crypto/dh.c +++ b/crypto/dh.c @@ -236,7 +236,7 @@ static void dh_exit(void) crypto_unregister_kpp(&dh); } -module_init(dh_init); +subsys_initcall(dh_init); module_exit(dh_exit); MODULE_ALIAS_CRYPTO("dh"); MODULE_LICENSE("GPL"); diff --git a/crypto/drbg.c b/crypto/drbg.c index bc52d9562611..2a5b16bb000c 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -1587,7 +1587,6 @@ static int drbg_init_hash_kernel(struct drbg_state *drbg) } sdesc->shash.tfm = tfm; - sdesc->shash.flags = 0; drbg->priv_data = sdesc; return crypto_shash_alignmask(tfm); @@ -2039,7 +2038,7 @@ static void __exit drbg_exit(void) crypto_unregister_rngs(drbg_algs, (ARRAY_SIZE(drbg_cores) * 2)); } -module_init(drbg_init); +subsys_initcall(drbg_init); module_exit(drbg_exit); #ifndef CRYPTO_DRBG_HASH_STRING #define CRYPTO_DRBG_HASH_STRING "" diff --git a/crypto/ecb.c b/crypto/ecb.c index 0732715c8d91..de839129d151 100644 --- a/crypto/ecb.c +++ b/crypto/ecb.c @@ -101,7 +101,7 @@ static void __exit crypto_ecb_module_exit(void) crypto_unregister_template(&crypto_ecb_tmpl); } -module_init(crypto_ecb_module_init); +subsys_initcall(crypto_ecb_module_init); module_exit(crypto_ecb_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/ecc.c b/crypto/ecc.c index ed1237115066..dfe114bc0c4a 100644 --- a/crypto/ecc.c +++ b/crypto/ecc.c @@ -1,6 +1,6 @@ /* - * Copyright (c) 2013, Kenneth MacKay - * All rights reserved. + * Copyright (c) 2013, 2014 Kenneth MacKay. All rights reserved. + * Copyright (c) 2019 Vitaly Chikunov <vt@altlinux.org> * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are @@ -24,12 +24,15 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include <linux/module.h> #include <linux/random.h> #include <linux/slab.h> #include <linux/swab.h> #include <linux/fips.h> #include <crypto/ecdh.h> #include <crypto/rng.h> +#include <asm/unaligned.h> +#include <linux/ratelimit.h> #include "ecc.h" #include "ecc_curve_defs.h" @@ -112,7 +115,7 @@ static void vli_clear(u64 *vli, unsigned int ndigits) } /* Returns true if vli == 0, false otherwise. */ -static bool vli_is_zero(const u64 *vli, unsigned int ndigits) +bool vli_is_zero(const u64 *vli, unsigned int ndigits) { int i; @@ -123,6 +126,7 @@ static bool vli_is_zero(const u64 *vli, unsigned int ndigits) return true; } +EXPORT_SYMBOL(vli_is_zero); /* Returns nonzero if bit bit of vli is set. */ static u64 vli_test_bit(const u64 *vli, unsigned int bit) @@ -130,6 +134,11 @@ static u64 vli_test_bit(const u64 *vli, unsigned int bit) return (vli[bit / 64] & ((u64)1 << (bit % 64))); } +static bool vli_is_negative(const u64 *vli, unsigned int ndigits) +{ + return vli_test_bit(vli, ndigits * 64 - 1); +} + /* Counts the number of 64-bit "digits" in vli. */ static unsigned int vli_num_digits(const u64 *vli, unsigned int ndigits) { @@ -161,6 +170,27 @@ static unsigned int vli_num_bits(const u64 *vli, unsigned int ndigits) return ((num_digits - 1) * 64 + i); } +/* Set dest from unaligned bit string src. */ +void vli_from_be64(u64 *dest, const void *src, unsigned int ndigits) +{ + int i; + const u64 *from = src; + + for (i = 0; i < ndigits; i++) + dest[i] = get_unaligned_be64(&from[ndigits - 1 - i]); +} +EXPORT_SYMBOL(vli_from_be64); + +void vli_from_le64(u64 *dest, const void *src, unsigned int ndigits) +{ + int i; + const u64 *from = src; + + for (i = 0; i < ndigits; i++) + dest[i] = get_unaligned_le64(&from[i]); +} +EXPORT_SYMBOL(vli_from_le64); + /* Sets dest = src. */ static void vli_set(u64 *dest, const u64 *src, unsigned int ndigits) { @@ -171,7 +201,7 @@ static void vli_set(u64 *dest, const u64 *src, unsigned int ndigits) } /* Returns sign of left - right. */ -static int vli_cmp(const u64 *left, const u64 *right, unsigned int ndigits) +int vli_cmp(const u64 *left, const u64 *right, unsigned int ndigits) { int i; @@ -184,6 +214,7 @@ static int vli_cmp(const u64 *left, const u64 *right, unsigned int ndigits) return 0; } +EXPORT_SYMBOL(vli_cmp); /* Computes result = in << c, returning carry. Can modify in place * (if result == in). 0 < shift < 64. @@ -239,8 +270,30 @@ static u64 vli_add(u64 *result, const u64 *left, const u64 *right, return carry; } +/* Computes result = left + right, returning carry. Can modify in place. */ +static u64 vli_uadd(u64 *result, const u64 *left, u64 right, + unsigned int ndigits) +{ + u64 carry = right; + int i; + + for (i = 0; i < ndigits; i++) { + u64 sum; + + sum = left[i] + carry; + if (sum != left[i]) + carry = (sum < left[i]); + else + carry = !!carry; + + result[i] = sum; + } + + return carry; +} + /* Computes result = left - right, returning borrow. Can modify in place. */ -static u64 vli_sub(u64 *result, const u64 *left, const u64 *right, +u64 vli_sub(u64 *result, const u64 *left, const u64 *right, unsigned int ndigits) { u64 borrow = 0; @@ -258,9 +311,37 @@ static u64 vli_sub(u64 *result, const u64 *left, const u64 *right, return borrow; } +EXPORT_SYMBOL(vli_sub); + +/* Computes result = left - right, returning borrow. Can modify in place. */ +static u64 vli_usub(u64 *result, const u64 *left, u64 right, + unsigned int ndigits) +{ + u64 borrow = right; + int i; + + for (i = 0; i < ndigits; i++) { + u64 diff; + + diff = left[i] - borrow; + if (diff != left[i]) + borrow = (diff > left[i]); + + result[i] = diff; + } + + return borrow; +} static uint128_t mul_64_64(u64 left, u64 right) { + uint128_t result; +#if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__) + unsigned __int128 m = (unsigned __int128)left * right; + + result.m_low = m; + result.m_high = m >> 64; +#else u64 a0 = left & 0xffffffffull; u64 a1 = left >> 32; u64 b0 = right & 0xffffffffull; @@ -269,7 +350,6 @@ static uint128_t mul_64_64(u64 left, u64 right) u64 m1 = a0 * b1; u64 m2 = a1 * b0; u64 m3 = a1 * b1; - uint128_t result; m2 += (m0 >> 32); m2 += m1; @@ -280,7 +360,7 @@ static uint128_t mul_64_64(u64 left, u64 right) result.m_low = (m0 & 0xffffffffull) | (m2 << 32); result.m_high = m3 + (m2 >> 32); - +#endif return result; } @@ -330,6 +410,28 @@ static void vli_mult(u64 *result, const u64 *left, const u64 *right, result[ndigits * 2 - 1] = r01.m_low; } +/* Compute product = left * right, for a small right value. */ +static void vli_umult(u64 *result, const u64 *left, u32 right, + unsigned int ndigits) +{ + uint128_t r01 = { 0 }; + unsigned int k; + + for (k = 0; k < ndigits; k++) { + uint128_t product; + + product = mul_64_64(left[k], right); + r01 = add_128_128(r01, product); + /* no carry */ + result[k] = r01.m_low; + r01.m_low = r01.m_high; + r01.m_high = 0; + } + result[k] = r01.m_low; + for (++k; k < ndigits * 2; k++) + result[k] = 0; +} + static void vli_square(u64 *result, const u64 *left, unsigned int ndigits) { uint128_t r01 = { 0, 0 }; @@ -402,6 +504,170 @@ static void vli_mod_sub(u64 *result, const u64 *left, const u64 *right, vli_add(result, result, mod, ndigits); } +/* + * Computes result = product % mod + * for special form moduli: p = 2^k-c, for small c (note the minus sign) + * + * References: + * R. Crandall, C. Pomerance. Prime Numbers: A Computational Perspective. + * 9 Fast Algorithms for Large-Integer Arithmetic. 9.2.3 Moduli of special form + * Algorithm 9.2.13 (Fast mod operation for special-form moduli). + */ +static void vli_mmod_special(u64 *result, const u64 *product, + const u64 *mod, unsigned int ndigits) +{ + u64 c = -mod[0]; + u64 t[ECC_MAX_DIGITS * 2]; + u64 r[ECC_MAX_DIGITS * 2]; + + vli_set(r, product, ndigits * 2); + while (!vli_is_zero(r + ndigits, ndigits)) { + vli_umult(t, r + ndigits, c, ndigits); + vli_clear(r + ndigits, ndigits); + vli_add(r, r, t, ndigits * 2); + } + vli_set(t, mod, ndigits); + vli_clear(t + ndigits, ndigits); + while (vli_cmp(r, t, ndigits * 2) >= 0) + vli_sub(r, r, t, ndigits * 2); + vli_set(result, r, ndigits); +} + +/* + * Computes result = product % mod + * for special form moduli: p = 2^{k-1}+c, for small c (note the plus sign) + * where k-1 does not fit into qword boundary by -1 bit (such as 255). + + * References (loosely based on): + * A. Menezes, P. van Oorschot, S. Vanstone. Handbook of Applied Cryptography. + * 14.3.4 Reduction methods for moduli of special form. Algorithm 14.47. + * URL: http://cacr.uwaterloo.ca/hac/about/chap14.pdf + * + * H. Cohen, G. Frey, R. Avanzi, C. Doche, T. Lange, K. Nguyen, F. Vercauteren. + * Handbook of Elliptic and Hyperelliptic Curve Cryptography. + * Algorithm 10.25 Fast reduction for special form moduli + */ +static void vli_mmod_special2(u64 *result, const u64 *product, + const u64 *mod, unsigned int ndigits) +{ + u64 c2 = mod[0] * 2; + u64 q[ECC_MAX_DIGITS]; + u64 r[ECC_MAX_DIGITS * 2]; + u64 m[ECC_MAX_DIGITS * 2]; /* expanded mod */ + int carry; /* last bit that doesn't fit into q */ + int i; + + vli_set(m, mod, ndigits); + vli_clear(m + ndigits, ndigits); + + vli_set(r, product, ndigits); + /* q and carry are top bits */ + vli_set(q, product + ndigits, ndigits); + vli_clear(r + ndigits, ndigits); + carry = vli_is_negative(r, ndigits); + if (carry) + r[ndigits - 1] &= (1ull << 63) - 1; + for (i = 1; carry || !vli_is_zero(q, ndigits); i++) { + u64 qc[ECC_MAX_DIGITS * 2]; + + vli_umult(qc, q, c2, ndigits); + if (carry) + vli_uadd(qc, qc, mod[0], ndigits * 2); + vli_set(q, qc + ndigits, ndigits); + vli_clear(qc + ndigits, ndigits); + carry = vli_is_negative(qc, ndigits); + if (carry) + qc[ndigits - 1] &= (1ull << 63) - 1; + if (i & 1) + vli_sub(r, r, qc, ndigits * 2); + else + vli_add(r, r, qc, ndigits * 2); + } + while (vli_is_negative(r, ndigits * 2)) + vli_add(r, r, m, ndigits * 2); + while (vli_cmp(r, m, ndigits * 2) >= 0) + vli_sub(r, r, m, ndigits * 2); + + vli_set(result, r, ndigits); +} + +/* + * Computes result = product % mod, where product is 2N words long. + * Reference: Ken MacKay's micro-ecc. + * Currently only designed to work for curve_p or curve_n. + */ +static void vli_mmod_slow(u64 *result, u64 *product, const u64 *mod, + unsigned int ndigits) +{ + u64 mod_m[2 * ECC_MAX_DIGITS]; + u64 tmp[2 * ECC_MAX_DIGITS]; + u64 *v[2] = { tmp, product }; + u64 carry = 0; + unsigned int i; + /* Shift mod so its highest set bit is at the maximum position. */ + int shift = (ndigits * 2 * 64) - vli_num_bits(mod, ndigits); + int word_shift = shift / 64; + int bit_shift = shift % 64; + + vli_clear(mod_m, word_shift); + if (bit_shift > 0) { + for (i = 0; i < ndigits; ++i) { + mod_m[word_shift + i] = (mod[i] << bit_shift) | carry; + carry = mod[i] >> (64 - bit_shift); + } + } else + vli_set(mod_m + word_shift, mod, ndigits); + + for (i = 1; shift >= 0; --shift) { + u64 borrow = 0; + unsigned int j; + + for (j = 0; j < ndigits * 2; ++j) { + u64 diff = v[i][j] - mod_m[j] - borrow; + + if (diff != v[i][j]) + borrow = (diff > v[i][j]); + v[1 - i][j] = diff; + } + i = !(i ^ borrow); /* Swap the index if there was no borrow */ + vli_rshift1(mod_m, ndigits); + mod_m[ndigits - 1] |= mod_m[ndigits] << (64 - 1); + vli_rshift1(mod_m + ndigits, ndigits); + } + vli_set(result, v[i], ndigits); +} + +/* Computes result = product % mod using Barrett's reduction with precomputed + * value mu appended to the mod after ndigits, mu = (2^{2w} / mod) and have + * length ndigits + 1, where mu * (2^w - 1) should not overflow ndigits + * boundary. + * + * Reference: + * R. Brent, P. Zimmermann. Modern Computer Arithmetic. 2010. + * 2.4.1 Barrett's algorithm. Algorithm 2.5. + */ +static void vli_mmod_barrett(u64 *result, u64 *product, const u64 *mod, + unsigned int ndigits) +{ + u64 q[ECC_MAX_DIGITS * 2]; + u64 r[ECC_MAX_DIGITS * 2]; + const u64 *mu = mod + ndigits; + + vli_mult(q, product + ndigits, mu, ndigits); + if (mu[ndigits]) + vli_add(q + ndigits, q + ndigits, product + ndigits, ndigits); + vli_mult(r, mod, q + ndigits, ndigits); + vli_sub(r, product, r, ndigits * 2); + while (!vli_is_zero(r + ndigits, ndigits) || + vli_cmp(r, mod, ndigits) != -1) { + u64 carry; + + carry = vli_sub(r, r, mod, ndigits); + vli_usub(r + ndigits, r + ndigits, carry, ndigits); + } + vli_set(result, r, ndigits); +} + /* Computes p_result = p_product % curve_p. * See algorithm 5 and 6 from * http://www.isys.uni-klu.ac.at/PDF/2001-0126-MT.pdf @@ -509,14 +775,33 @@ static void vli_mmod_fast_256(u64 *result, const u64 *product, } } -/* Computes result = product % curve_prime - * from http://www.nsa.gov/ia/_files/nist-routines.pdf -*/ +/* Computes result = product % curve_prime for different curve_primes. + * + * Note that curve_primes are distinguished just by heuristic check and + * not by complete conformance check. + */ static bool vli_mmod_fast(u64 *result, u64 *product, const u64 *curve_prime, unsigned int ndigits) { u64 tmp[2 * ECC_MAX_DIGITS]; + /* Currently, both NIST primes have -1 in lowest qword. */ + if (curve_prime[0] != -1ull) { + /* Try to handle Pseudo-Marsenne primes. */ + if (curve_prime[ndigits - 1] == -1ull) { + vli_mmod_special(result, product, curve_prime, + ndigits); + return true; + } else if (curve_prime[ndigits - 1] == 1ull << 63 && + curve_prime[ndigits - 2] == 0) { + vli_mmod_special2(result, product, curve_prime, + ndigits); + return true; + } + vli_mmod_barrett(result, product, curve_prime, ndigits); + return true; + } + switch (ndigits) { case 3: vli_mmod_fast_192(result, product, curve_prime, tmp); @@ -525,13 +810,26 @@ static bool vli_mmod_fast(u64 *result, u64 *product, vli_mmod_fast_256(result, product, curve_prime, tmp); break; default: - pr_err("unsupports digits size!\n"); + pr_err_ratelimited("ecc: unsupported digits size!\n"); return false; } return true; } +/* Computes result = (left * right) % mod. + * Assumes that mod is big enough curve order. + */ +void vli_mod_mult_slow(u64 *result, const u64 *left, const u64 *right, + const u64 *mod, unsigned int ndigits) +{ + u64 product[ECC_MAX_DIGITS * 2]; + + vli_mult(product, left, right, ndigits); + vli_mmod_slow(result, product, mod, ndigits); +} +EXPORT_SYMBOL(vli_mod_mult_slow); + /* Computes result = (left * right) % curve_prime. */ static void vli_mod_mult_fast(u64 *result, const u64 *left, const u64 *right, const u64 *curve_prime, unsigned int ndigits) @@ -557,7 +855,7 @@ static void vli_mod_square_fast(u64 *result, const u64 *left, * See "From Euclid's GCD to Montgomery Multiplication to the Great Divide" * https://labs.oracle.com/techrep/2001/smli_tr-2001-95.pdf */ -static void vli_mod_inv(u64 *result, const u64 *input, const u64 *mod, +void vli_mod_inv(u64 *result, const u64 *input, const u64 *mod, unsigned int ndigits) { u64 a[ECC_MAX_DIGITS], b[ECC_MAX_DIGITS]; @@ -630,6 +928,7 @@ static void vli_mod_inv(u64 *result, const u64 *input, const u64 *mod, vli_set(result, u, ndigits); } +EXPORT_SYMBOL(vli_mod_inv); /* ------ Point operations ------ */ @@ -903,6 +1202,85 @@ static void ecc_point_mult(struct ecc_point *result, vli_set(result->y, ry[0], ndigits); } +/* Computes R = P + Q mod p */ +static void ecc_point_add(const struct ecc_point *result, + const struct ecc_point *p, const struct ecc_point *q, + const struct ecc_curve *curve) +{ + u64 z[ECC_MAX_DIGITS]; + u64 px[ECC_MAX_DIGITS]; + u64 py[ECC_MAX_DIGITS]; + unsigned int ndigits = curve->g.ndigits; + + vli_set(result->x, q->x, ndigits); + vli_set(result->y, q->y, ndigits); + vli_mod_sub(z, result->x, p->x, curve->p, ndigits); + vli_set(px, p->x, ndigits); + vli_set(py, p->y, ndigits); + xycz_add(px, py, result->x, result->y, curve->p, ndigits); + vli_mod_inv(z, z, curve->p, ndigits); + apply_z(result->x, result->y, z, curve->p, ndigits); +} + +/* Computes R = u1P + u2Q mod p using Shamir's trick. + * Based on: Kenneth MacKay's micro-ecc (2014). + */ +void ecc_point_mult_shamir(const struct ecc_point *result, + const u64 *u1, const struct ecc_point *p, + const u64 *u2, const struct ecc_point *q, + const struct ecc_curve *curve) +{ + u64 z[ECC_MAX_DIGITS]; + u64 sump[2][ECC_MAX_DIGITS]; + u64 *rx = result->x; + u64 *ry = result->y; + unsigned int ndigits = curve->g.ndigits; + unsigned int num_bits; + struct ecc_point sum = ECC_POINT_INIT(sump[0], sump[1], ndigits); + const struct ecc_point *points[4]; + const struct ecc_point *point; + unsigned int idx; + int i; + + ecc_point_add(&sum, p, q, curve); + points[0] = NULL; + points[1] = p; + points[2] = q; + points[3] = ∑ + + num_bits = max(vli_num_bits(u1, ndigits), + vli_num_bits(u2, ndigits)); + i = num_bits - 1; + idx = (!!vli_test_bit(u1, i)) | ((!!vli_test_bit(u2, i)) << 1); + point = points[idx]; + + vli_set(rx, point->x, ndigits); + vli_set(ry, point->y, ndigits); + vli_clear(z + 1, ndigits - 1); + z[0] = 1; + + for (--i; i >= 0; i--) { + ecc_point_double_jacobian(rx, ry, z, curve->p, ndigits); + idx = (!!vli_test_bit(u1, i)) | ((!!vli_test_bit(u2, i)) << 1); + point = points[idx]; + if (point) { + u64 tx[ECC_MAX_DIGITS]; + u64 ty[ECC_MAX_DIGITS]; + u64 tz[ECC_MAX_DIGITS]; + + vli_set(tx, point->x, ndigits); + vli_set(ty, point->y, ndigits); + apply_z(tx, ty, z, curve->p, ndigits); + vli_mod_sub(tz, rx, tx, curve->p, ndigits); + xycz_add(tx, ty, rx, ry, curve->p, ndigits); + vli_mod_mult_fast(z, z, tz, curve->p, ndigits); + } + } + vli_mod_inv(z, z, curve->p, ndigits); + apply_z(rx, ry, z, curve->p, ndigits); +} +EXPORT_SYMBOL(ecc_point_mult_shamir); + static inline void ecc_swap_digits(const u64 *in, u64 *out, unsigned int ndigits) { @@ -948,6 +1326,7 @@ int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits, return __ecc_is_key_valid(curve, private_key, ndigits); } +EXPORT_SYMBOL(ecc_is_key_valid); /* * ECC private keys are generated using the method of extra random bits, @@ -1000,6 +1379,7 @@ int ecc_gen_privkey(unsigned int curve_id, unsigned int ndigits, u64 *privkey) return 0; } +EXPORT_SYMBOL(ecc_gen_privkey); int ecc_make_pub_key(unsigned int curve_id, unsigned int ndigits, const u64 *private_key, u64 *public_key) @@ -1036,13 +1416,17 @@ err_free_point: out: return ret; } +EXPORT_SYMBOL(ecc_make_pub_key); /* SP800-56A section 5.6.2.3.4 partial verification: ephemeral keys only */ -static int ecc_is_pubkey_valid_partial(const struct ecc_curve *curve, - struct ecc_point *pk) +int ecc_is_pubkey_valid_partial(const struct ecc_curve *curve, + struct ecc_point *pk) { u64 yy[ECC_MAX_DIGITS], xxx[ECC_MAX_DIGITS], w[ECC_MAX_DIGITS]; + if (WARN_ON(pk->ndigits != curve->g.ndigits)) + return -EINVAL; + /* Check 1: Verify key is not the zero point. */ if (ecc_point_is_zero(pk)) return -EINVAL; @@ -1064,8 +1448,8 @@ static int ecc_is_pubkey_valid_partial(const struct ecc_curve *curve, return -EINVAL; return 0; - } +EXPORT_SYMBOL(ecc_is_pubkey_valid_partial); int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits, const u64 *private_key, const u64 *public_key, @@ -1121,3 +1505,6 @@ err_alloc_product: out: return ret; } +EXPORT_SYMBOL(crypto_ecdh_shared_secret); + +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/crypto/ecc.h b/crypto/ecc.h index f75a86baa3bd..ab0eb70b9c09 100644 --- a/crypto/ecc.h +++ b/crypto/ecc.h @@ -26,13 +26,51 @@ #ifndef _CRYPTO_ECC_H #define _CRYPTO_ECC_H +/* One digit is u64 qword. */ #define ECC_CURVE_NIST_P192_DIGITS 3 #define ECC_CURVE_NIST_P256_DIGITS 4 -#define ECC_MAX_DIGITS ECC_CURVE_NIST_P256_DIGITS +#define ECC_MAX_DIGITS (512 / 64) #define ECC_DIGITS_TO_BYTES_SHIFT 3 /** + * struct ecc_point - elliptic curve point in affine coordinates + * + * @x: X coordinate in vli form. + * @y: Y coordinate in vli form. + * @ndigits: Length of vlis in u64 qwords. + */ +struct ecc_point { + u64 *x; + u64 *y; + u8 ndigits; +}; + +#define ECC_POINT_INIT(x, y, ndigits) (struct ecc_point) { x, y, ndigits } + +/** + * struct ecc_curve - definition of elliptic curve + * + * @name: Short name of the curve. + * @g: Generator point of the curve. + * @p: Prime number, if Barrett's reduction is used for this curve + * pre-calculated value 'mu' is appended to the @p after ndigits. + * Use of Barrett's reduction is heuristically determined in + * vli_mmod_fast(). + * @n: Order of the curve group. + * @a: Curve parameter a. + * @b: Curve parameter b. + */ +struct ecc_curve { + char *name; + struct ecc_point g; + u64 *p; + u64 *n; + u64 *a; + u64 *b; +}; + +/** * ecc_is_key_valid() - Validate a given ECDH private key * * @curve_id: id representing the curve to use @@ -91,4 +129,117 @@ int ecc_make_pub_key(const unsigned int curve_id, unsigned int ndigits, int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits, const u64 *private_key, const u64 *public_key, u64 *secret); + +/** + * ecc_is_pubkey_valid_partial() - Partial public key validation + * + * @curve: elliptic curve domain parameters + * @pk: public key as a point + * + * Valdiate public key according to SP800-56A section 5.6.2.3.4 ECC Partial + * Public-Key Validation Routine. + * + * Note: There is no check that the public key is in the correct elliptic curve + * subgroup. + * + * Return: 0 if validation is successful, -EINVAL if validation is failed. + */ +int ecc_is_pubkey_valid_partial(const struct ecc_curve *curve, + struct ecc_point *pk); + +/** + * vli_is_zero() - Determine is vli is zero + * + * @vli: vli to check. + * @ndigits: length of the @vli + */ +bool vli_is_zero(const u64 *vli, unsigned int ndigits); + +/** + * vli_cmp() - compare left and right vlis + * + * @left: vli + * @right: vli + * @ndigits: length of both vlis + * + * Returns sign of @left - @right, i.e. -1 if @left < @right, + * 0 if @left == @right, 1 if @left > @right. + */ +int vli_cmp(const u64 *left, const u64 *right, unsigned int ndigits); + +/** + * vli_sub() - Subtracts right from left + * + * @result: where to write result + * @left: vli + * @right vli + * @ndigits: length of all vlis + * + * Note: can modify in-place. + * + * Return: carry bit. + */ +u64 vli_sub(u64 *result, const u64 *left, const u64 *right, + unsigned int ndigits); + +/** + * vli_from_be64() - Load vli from big-endian u64 array + * + * @dest: destination vli + * @src: source array of u64 BE values + * @ndigits: length of both vli and array + */ +void vli_from_be64(u64 *dest, const void *src, unsigned int ndigits); + +/** + * vli_from_le64() - Load vli from little-endian u64 array + * + * @dest: destination vli + * @src: source array of u64 LE values + * @ndigits: length of both vli and array + */ +void vli_from_le64(u64 *dest, const void *src, unsigned int ndigits); + +/** + * vli_mod_inv() - Modular inversion + * + * @result: where to write vli number + * @input: vli value to operate on + * @mod: modulus + * @ndigits: length of all vlis + */ +void vli_mod_inv(u64 *result, const u64 *input, const u64 *mod, + unsigned int ndigits); + +/** + * vli_mod_mult_slow() - Modular multiplication + * + * @result: where to write result value + * @left: vli number to multiply with @right + * @right: vli number to multiply with @left + * @mod: modulus + * @ndigits: length of all vlis + * + * Note: Assumes that mod is big enough curve order. + */ +void vli_mod_mult_slow(u64 *result, const u64 *left, const u64 *right, + const u64 *mod, unsigned int ndigits); + +/** + * ecc_point_mult_shamir() - Add two points multiplied by scalars + * + * @result: resulting point + * @x: scalar to multiply with @p + * @p: point to multiply with @x + * @y: scalar to multiply with @q + * @q: point to multiply with @y + * @curve: curve + * + * Returns result = x * p + x * q over the curve. + * This works faster than two multiplications and addition. + */ +void ecc_point_mult_shamir(const struct ecc_point *result, + const u64 *x, const struct ecc_point *p, + const u64 *y, const struct ecc_point *q, + const struct ecc_curve *curve); #endif diff --git a/crypto/ecc_curve_defs.h b/crypto/ecc_curve_defs.h index 336ab1805639..69be6c7d228f 100644 --- a/crypto/ecc_curve_defs.h +++ b/crypto/ecc_curve_defs.h @@ -2,21 +2,6 @@ #ifndef _CRYTO_ECC_CURVE_DEFS_H #define _CRYTO_ECC_CURVE_DEFS_H -struct ecc_point { - u64 *x; - u64 *y; - u8 ndigits; -}; - -struct ecc_curve { - char *name; - struct ecc_point g; - u64 *p; - u64 *n; - u64 *a; - u64 *b; -}; - /* NIST P-192: a = p - 3 */ static u64 nist_p192_g_x[] = { 0xF4FF0AFD82FF1012ull, 0x7CBF20EB43A18800ull, 0x188DA80EB03090F6ull }; diff --git a/crypto/ecdh.c b/crypto/ecdh.c index bf6300175b9c..890092bd8989 100644 --- a/crypto/ecdh.c +++ b/crypto/ecdh.c @@ -166,7 +166,7 @@ static void ecdh_exit(void) crypto_unregister_kpp(&ecdh); } -module_init(ecdh_init); +subsys_initcall(ecdh_init); module_exit(ecdh_exit); MODULE_ALIAS_CRYPTO("ecdh"); MODULE_LICENSE("GPL"); diff --git a/crypto/echainiv.c b/crypto/echainiv.c index 77e607fdbfb7..e71d1bc8d850 100644 --- a/crypto/echainiv.c +++ b/crypto/echainiv.c @@ -174,7 +174,7 @@ static void __exit echainiv_module_exit(void) crypto_unregister_template(&echainiv_tmpl); } -module_init(echainiv_module_init); +subsys_initcall(echainiv_module_init); module_exit(echainiv_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/ecrdsa.c b/crypto/ecrdsa.c new file mode 100644 index 000000000000..887ec21aee49 --- /dev/null +++ b/crypto/ecrdsa.c @@ -0,0 +1,296 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Elliptic Curve (Russian) Digital Signature Algorithm for Cryptographic API + * + * Copyright (c) 2019 Vitaly Chikunov <vt@altlinux.org> + * + * References: + * GOST 34.10-2018, GOST R 34.10-2012, RFC 7091, ISO/IEC 14888-3:2018. + * + * Historical references: + * GOST R 34.10-2001, RFC 4357, ISO/IEC 14888-3:2006/Amd 1:2010. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + */ + +#include <linux/module.h> +#include <linux/crypto.h> +#include <crypto/streebog.h> +#include <crypto/internal/akcipher.h> +#include <crypto/akcipher.h> +#include <linux/oid_registry.h> +#include "ecrdsa_params.asn1.h" +#include "ecrdsa_pub_key.asn1.h" +#include "ecc.h" +#include "ecrdsa_defs.h" + +#define ECRDSA_MAX_SIG_SIZE (2 * 512 / 8) +#define ECRDSA_MAX_DIGITS (512 / 64) + +struct ecrdsa_ctx { + enum OID algo_oid; /* overall public key oid */ + enum OID curve_oid; /* parameter */ + enum OID digest_oid; /* parameter */ + const struct ecc_curve *curve; /* curve from oid */ + unsigned int digest_len; /* parameter (bytes) */ + const char *digest; /* digest name from oid */ + unsigned int key_len; /* @key length (bytes) */ + const char *key; /* raw public key */ + struct ecc_point pub_key; + u64 _pubp[2][ECRDSA_MAX_DIGITS]; /* point storage for @pub_key */ +}; + +static const struct ecc_curve *get_curve_by_oid(enum OID oid) +{ + switch (oid) { + case OID_gostCPSignA: + case OID_gostTC26Sign256B: + return &gost_cp256a; + case OID_gostCPSignB: + case OID_gostTC26Sign256C: + return &gost_cp256b; + case OID_gostCPSignC: + case OID_gostTC26Sign256D: + return &gost_cp256c; + case OID_gostTC26Sign512A: + return &gost_tc512a; + case OID_gostTC26Sign512B: + return &gost_tc512b; + /* The following two aren't implemented: */ + case OID_gostTC26Sign256A: + case OID_gostTC26Sign512C: + default: + return NULL; + } +} + +static int ecrdsa_verify(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct ecrdsa_ctx *ctx = akcipher_tfm_ctx(tfm); + unsigned char sig[ECRDSA_MAX_SIG_SIZE]; + unsigned char digest[STREEBOG512_DIGEST_SIZE]; + unsigned int ndigits = req->dst_len / sizeof(u64); + u64 r[ECRDSA_MAX_DIGITS]; /* witness (r) */ + u64 _r[ECRDSA_MAX_DIGITS]; /* -r */ + u64 s[ECRDSA_MAX_DIGITS]; /* second part of sig (s) */ + u64 e[ECRDSA_MAX_DIGITS]; /* h \mod q */ + u64 *v = e; /* e^{-1} \mod q */ + u64 z1[ECRDSA_MAX_DIGITS]; + u64 *z2 = _r; + struct ecc_point cc = ECC_POINT_INIT(s, e, ndigits); /* reuse s, e */ + + /* + * Digest value, digest algorithm, and curve (modulus) should have the + * same length (256 or 512 bits), public key and signature should be + * twice bigger. + */ + if (!ctx->curve || + !ctx->digest || + !req->src || + !ctx->pub_key.x || + req->dst_len != ctx->digest_len || + req->dst_len != ctx->curve->g.ndigits * sizeof(u64) || + ctx->pub_key.ndigits != ctx->curve->g.ndigits || + req->dst_len * 2 != req->src_len || + WARN_ON(req->src_len > sizeof(sig)) || + WARN_ON(req->dst_len > sizeof(digest))) + return -EBADMSG; + + sg_copy_to_buffer(req->src, sg_nents_for_len(req->src, req->src_len), + sig, req->src_len); + sg_pcopy_to_buffer(req->src, + sg_nents_for_len(req->src, + req->src_len + req->dst_len), + digest, req->dst_len, req->src_len); + + vli_from_be64(s, sig, ndigits); + vli_from_be64(r, sig + ndigits * sizeof(u64), ndigits); + + /* Step 1: verify that 0 < r < q, 0 < s < q */ + if (vli_is_zero(r, ndigits) || + vli_cmp(r, ctx->curve->n, ndigits) == 1 || + vli_is_zero(s, ndigits) || + vli_cmp(s, ctx->curve->n, ndigits) == 1) + return -EKEYREJECTED; + + /* Step 2: calculate hash (h) of the message (passed as input) */ + /* Step 3: calculate e = h \mod q */ + vli_from_le64(e, digest, ndigits); + if (vli_cmp(e, ctx->curve->n, ndigits) == 1) + vli_sub(e, e, ctx->curve->n, ndigits); + if (vli_is_zero(e, ndigits)) + e[0] = 1; + + /* Step 4: calculate v = e^{-1} \mod q */ + vli_mod_inv(v, e, ctx->curve->n, ndigits); + + /* Step 5: calculate z_1 = sv \mod q, z_2 = -rv \mod q */ + vli_mod_mult_slow(z1, s, v, ctx->curve->n, ndigits); + vli_sub(_r, ctx->curve->n, r, ndigits); + vli_mod_mult_slow(z2, _r, v, ctx->curve->n, ndigits); + + /* Step 6: calculate point C = z_1P + z_2Q, and R = x_c \mod q */ + ecc_point_mult_shamir(&cc, z1, &ctx->curve->g, z2, &ctx->pub_key, + ctx->curve); + if (vli_cmp(cc.x, ctx->curve->n, ndigits) == 1) + vli_sub(cc.x, cc.x, ctx->curve->n, ndigits); + + /* Step 7: if R == r signature is valid */ + if (!vli_cmp(cc.x, r, ndigits)) + return 0; + else + return -EKEYREJECTED; +} + +int ecrdsa_param_curve(void *context, size_t hdrlen, unsigned char tag, + const void *value, size_t vlen) +{ + struct ecrdsa_ctx *ctx = context; + + ctx->curve_oid = look_up_OID(value, vlen); + if (!ctx->curve_oid) + return -EINVAL; + ctx->curve = get_curve_by_oid(ctx->curve_oid); + return 0; +} + +/* Optional. If present should match expected digest algo OID. */ +int ecrdsa_param_digest(void *context, size_t hdrlen, unsigned char tag, + const void *value, size_t vlen) +{ + struct ecrdsa_ctx *ctx = context; + int digest_oid = look_up_OID(value, vlen); + + if (digest_oid != ctx->digest_oid) + return -EINVAL; + return 0; +} + +int ecrdsa_parse_pub_key(void *context, size_t hdrlen, unsigned char tag, + const void *value, size_t vlen) +{ + struct ecrdsa_ctx *ctx = context; + + ctx->key = value; + ctx->key_len = vlen; + return 0; +} + +static u8 *ecrdsa_unpack_u32(u32 *dst, void *src) +{ + memcpy(dst, src, sizeof(u32)); + return src + sizeof(u32); +} + +/* Parse BER encoded subjectPublicKey. */ +static int ecrdsa_set_pub_key(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + struct ecrdsa_ctx *ctx = akcipher_tfm_ctx(tfm); + unsigned int ndigits; + u32 algo, paramlen; + u8 *params; + int err; + + err = asn1_ber_decoder(&ecrdsa_pub_key_decoder, ctx, key, keylen); + if (err < 0) + return err; + + /* Key parameters is in the key after keylen. */ + params = ecrdsa_unpack_u32(¶mlen, + ecrdsa_unpack_u32(&algo, (u8 *)key + keylen)); + + if (algo == OID_gost2012PKey256) { + ctx->digest = "streebog256"; + ctx->digest_oid = OID_gost2012Digest256; + ctx->digest_len = 256 / 8; + } else if (algo == OID_gost2012PKey512) { + ctx->digest = "streebog512"; + ctx->digest_oid = OID_gost2012Digest512; + ctx->digest_len = 512 / 8; + } else + return -ENOPKG; + ctx->algo_oid = algo; + + /* Parse SubjectPublicKeyInfo.AlgorithmIdentifier.parameters. */ + err = asn1_ber_decoder(&ecrdsa_params_decoder, ctx, params, paramlen); + if (err < 0) + return err; + /* + * Sizes of algo (set in digest_len) and curve should match + * each other. + */ + if (!ctx->curve || + ctx->curve->g.ndigits * sizeof(u64) != ctx->digest_len) + return -ENOPKG; + /* + * Key is two 256- or 512-bit coordinates which should match + * curve size. + */ + if ((ctx->key_len != (2 * 256 / 8) && + ctx->key_len != (2 * 512 / 8)) || + ctx->key_len != ctx->curve->g.ndigits * sizeof(u64) * 2) + return -ENOPKG; + + ndigits = ctx->key_len / sizeof(u64) / 2; + ctx->pub_key = ECC_POINT_INIT(ctx->_pubp[0], ctx->_pubp[1], ndigits); + vli_from_le64(ctx->pub_key.x, ctx->key, ndigits); + vli_from_le64(ctx->pub_key.y, ctx->key + ndigits * sizeof(u64), + ndigits); + + if (ecc_is_pubkey_valid_partial(ctx->curve, &ctx->pub_key)) + return -EKEYREJECTED; + + return 0; +} + +static unsigned int ecrdsa_max_size(struct crypto_akcipher *tfm) +{ + struct ecrdsa_ctx *ctx = akcipher_tfm_ctx(tfm); + + /* + * Verify doesn't need any output, so it's just informational + * for keyctl to determine the key bit size. + */ + return ctx->pub_key.ndigits * sizeof(u64); +} + +static void ecrdsa_exit_tfm(struct crypto_akcipher *tfm) +{ +} + +static struct akcipher_alg ecrdsa_alg = { + .verify = ecrdsa_verify, + .set_pub_key = ecrdsa_set_pub_key, + .max_size = ecrdsa_max_size, + .exit = ecrdsa_exit_tfm, + .base = { + .cra_name = "ecrdsa", + .cra_driver_name = "ecrdsa-generic", + .cra_priority = 100, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct ecrdsa_ctx), + }, +}; + +static int __init ecrdsa_mod_init(void) +{ + return crypto_register_akcipher(&ecrdsa_alg); +} + +static void __exit ecrdsa_mod_fini(void) +{ + crypto_unregister_akcipher(&ecrdsa_alg); +} + +module_init(ecrdsa_mod_init); +module_exit(ecrdsa_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Vitaly Chikunov <vt@altlinux.org>"); +MODULE_DESCRIPTION("EC-RDSA generic algorithm"); +MODULE_ALIAS_CRYPTO("ecrdsa-generic"); diff --git a/crypto/ecrdsa_defs.h b/crypto/ecrdsa_defs.h new file mode 100644 index 000000000000..170baf039007 --- /dev/null +++ b/crypto/ecrdsa_defs.h @@ -0,0 +1,225 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Definitions of EC-RDSA Curve Parameters + * + * Copyright (c) 2019 Vitaly Chikunov <vt@altlinux.org> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + */ + +#ifndef _CRYTO_ECRDSA_DEFS_H +#define _CRYTO_ECRDSA_DEFS_H + +#include "ecc.h" + +#define ECRDSA_MAX_SIG_SIZE (2 * 512 / 8) +#define ECRDSA_MAX_DIGITS (512 / 64) + +/* + * EC-RDSA uses its own set of curves. + * + * cp256{a,b,c} curves first defined for GOST R 34.10-2001 in RFC 4357 (as + * 256-bit {A,B,C}-ParamSet), but inherited for GOST R 34.10-2012 and + * proposed for use in R 50.1.114-2016 and RFC 7836 as the 256-bit curves. + */ +/* OID_gostCPSignA 1.2.643.2.2.35.1 */ +static u64 cp256a_g_x[] = { + 0x0000000000000001ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, }; +static u64 cp256a_g_y[] = { + 0x22ACC99C9E9F1E14ull, 0x35294F2DDF23E3B1ull, + 0x27DF505A453F2B76ull, 0x8D91E471E0989CDAull, }; +static u64 cp256a_p[] = { /* p = 2^256 - 617 */ + 0xFFFFFFFFFFFFFD97ull, 0xFFFFFFFFFFFFFFFFull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull }; +static u64 cp256a_n[] = { + 0x45841B09B761B893ull, 0x6C611070995AD100ull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull }; +static u64 cp256a_a[] = { /* a = p - 3 */ + 0xFFFFFFFFFFFFFD94ull, 0xFFFFFFFFFFFFFFFFull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull }; +static u64 cp256a_b[] = { + 0x00000000000000a6ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull }; + +static struct ecc_curve gost_cp256a = { + .name = "cp256a", + .g = { + .x = cp256a_g_x, + .y = cp256a_g_y, + .ndigits = 256 / 64, + }, + .p = cp256a_p, + .n = cp256a_n, + .a = cp256a_a, + .b = cp256a_b +}; + +/* OID_gostCPSignB 1.2.643.2.2.35.2 */ +static u64 cp256b_g_x[] = { + 0x0000000000000001ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, }; +static u64 cp256b_g_y[] = { + 0x744BF8D717717EFCull, 0xC545C9858D03ECFBull, + 0xB83D1C3EB2C070E5ull, 0x3FA8124359F96680ull, }; +static u64 cp256b_p[] = { /* p = 2^255 + 3225 */ + 0x0000000000000C99ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x8000000000000000ull, }; +static u64 cp256b_n[] = { + 0xE497161BCC8A198Full, 0x5F700CFFF1A624E5ull, + 0x0000000000000001ull, 0x8000000000000000ull, }; +static u64 cp256b_a[] = { /* a = p - 3 */ + 0x0000000000000C96ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x8000000000000000ull, }; +static u64 cp256b_b[] = { + 0x2F49D4CE7E1BBC8Bull, 0xE979259373FF2B18ull, + 0x66A7D3C25C3DF80Aull, 0x3E1AF419A269A5F8ull, }; + +static struct ecc_curve gost_cp256b = { + .name = "cp256b", + .g = { + .x = cp256b_g_x, + .y = cp256b_g_y, + .ndigits = 256 / 64, + }, + .p = cp256b_p, + .n = cp256b_n, + .a = cp256b_a, + .b = cp256b_b +}; + +/* OID_gostCPSignC 1.2.643.2.2.35.3 */ +static u64 cp256c_g_x[] = { + 0x0000000000000000ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, }; +static u64 cp256c_g_y[] = { + 0x366E550DFDB3BB67ull, 0x4D4DC440D4641A8Full, + 0x3CBF3783CD08C0EEull, 0x41ECE55743711A8Cull, }; +static u64 cp256c_p[] = { + 0x7998F7B9022D759Bull, 0xCF846E86789051D3ull, + 0xAB1EC85E6B41C8AAull, 0x9B9F605F5A858107ull, + /* pre-computed value for Barrett's reduction */ + 0xedc283cdd217b5a2ull, 0xbac48fc06398ae59ull, + 0x405384d55f9f3b73ull, 0xa51f176161f1d734ull, + 0x0000000000000001ull, }; +static u64 cp256c_n[] = { + 0xF02F3A6598980BB9ull, 0x582CA3511EDDFB74ull, + 0xAB1EC85E6B41C8AAull, 0x9B9F605F5A858107ull, }; +static u64 cp256c_a[] = { /* a = p - 3 */ + 0x7998F7B9022D7598ull, 0xCF846E86789051D3ull, + 0xAB1EC85E6B41C8AAull, 0x9B9F605F5A858107ull, }; +static u64 cp256c_b[] = { + 0x000000000000805aull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, }; + +static struct ecc_curve gost_cp256c = { + .name = "cp256c", + .g = { + .x = cp256c_g_x, + .y = cp256c_g_y, + .ndigits = 256 / 64, + }, + .p = cp256c_p, + .n = cp256c_n, + .a = cp256c_a, + .b = cp256c_b +}; + +/* tc512{a,b} curves first recommended in 2013 and then standardized in + * R 50.1.114-2016 and RFC 7836 for use with GOST R 34.10-2012 (as TC26 + * 512-bit ParamSet{A,B}). + */ +/* OID_gostTC26Sign512A 1.2.643.7.1.2.1.2.1 */ +static u64 tc512a_g_x[] = { + 0x0000000000000003ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, }; +static u64 tc512a_g_y[] = { + 0x89A589CB5215F2A4ull, 0x8028FE5FC235F5B8ull, + 0x3D75E6A50E3A41E9ull, 0xDF1626BE4FD036E9ull, + 0x778064FDCBEFA921ull, 0xCE5E1C93ACF1ABC1ull, + 0xA61B8816E25450E6ull, 0x7503CFE87A836AE3ull, }; +static u64 tc512a_p[] = { /* p = 2^512 - 569 */ + 0xFFFFFFFFFFFFFDC7ull, 0xFFFFFFFFFFFFFFFFull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, }; +static u64 tc512a_n[] = { + 0xCACDB1411F10B275ull, 0x9B4B38ABFAD2B85Dull, + 0x6FF22B8D4E056060ull, 0x27E69532F48D8911ull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, }; +static u64 tc512a_a[] = { /* a = p - 3 */ + 0xFFFFFFFFFFFFFDC4ull, 0xFFFFFFFFFFFFFFFFull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, }; +static u64 tc512a_b[] = { + 0x503190785A71C760ull, 0x862EF9D4EBEE4761ull, + 0x4CB4574010DA90DDull, 0xEE3CB090F30D2761ull, + 0x79BD081CFD0B6265ull, 0x34B82574761CB0E8ull, + 0xC1BD0B2B6667F1DAull, 0xE8C2505DEDFC86DDull, }; + +static struct ecc_curve gost_tc512a = { + .name = "tc512a", + .g = { + .x = tc512a_g_x, + .y = tc512a_g_y, + .ndigits = 512 / 64, + }, + .p = tc512a_p, + .n = tc512a_n, + .a = tc512a_a, + .b = tc512a_b +}; + +/* OID_gostTC26Sign512B 1.2.643.7.1.2.1.2.2 */ +static u64 tc512b_g_x[] = { + 0x0000000000000002ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, }; +static u64 tc512b_g_y[] = { + 0x7E21340780FE41BDull, 0x28041055F94CEEECull, + 0x152CBCAAF8C03988ull, 0xDCB228FD1EDF4A39ull, + 0xBE6DD9E6C8EC7335ull, 0x3C123B697578C213ull, + 0x2C071E3647A8940Full, 0x1A8F7EDA389B094Cull, }; +static u64 tc512b_p[] = { /* p = 2^511 + 111 */ + 0x000000000000006Full, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x8000000000000000ull, }; +static u64 tc512b_n[] = { + 0xC6346C54374F25BDull, 0x8B996712101BEA0Eull, + 0xACFDB77BD9D40CFAull, 0x49A1EC142565A545ull, + 0x0000000000000001ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x8000000000000000ull, }; +static u64 tc512b_a[] = { /* a = p - 3 */ + 0x000000000000006Cull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x8000000000000000ull, }; +static u64 tc512b_b[] = { + 0xFB8CCBC7C5140116ull, 0x50F78BEE1FA3106Eull, + 0x7F8B276FAD1AB69Cull, 0x3E965D2DB1416D21ull, + 0xBF85DC806C4B289Full, 0xB97C7D614AF138BCull, + 0x7E3E06CF6F5E2517ull, 0x687D1B459DC84145ull, }; + +static struct ecc_curve gost_tc512b = { + .name = "tc512b", + .g = { + .x = tc512b_g_x, + .y = tc512b_g_y, + .ndigits = 512 / 64, + }, + .p = tc512b_p, + .n = tc512b_n, + .a = tc512b_a, + .b = tc512b_b +}; + +#endif diff --git a/crypto/ecrdsa_params.asn1 b/crypto/ecrdsa_params.asn1 new file mode 100644 index 000000000000..aba99c3763cf --- /dev/null +++ b/crypto/ecrdsa_params.asn1 @@ -0,0 +1,4 @@ +EcrdsaParams ::= SEQUENCE { + curve OBJECT IDENTIFIER ({ ecrdsa_param_curve }), + digest OBJECT IDENTIFIER OPTIONAL ({ ecrdsa_param_digest }) +} diff --git a/crypto/ecrdsa_pub_key.asn1 b/crypto/ecrdsa_pub_key.asn1 new file mode 100644 index 000000000000..048cb646bce4 --- /dev/null +++ b/crypto/ecrdsa_pub_key.asn1 @@ -0,0 +1 @@ +EcrdsaPubKey ::= OCTET STRING ({ ecrdsa_parse_pub_key }) diff --git a/crypto/fcrypt.c b/crypto/fcrypt.c index 77286ea28865..4e8704405a3b 100644 --- a/crypto/fcrypt.c +++ b/crypto/fcrypt.c @@ -414,7 +414,7 @@ static void __exit fcrypt_mod_fini(void) crypto_unregister_alg(&fcrypt_alg); } -module_init(fcrypt_mod_init); +subsys_initcall(fcrypt_mod_init); module_exit(fcrypt_mod_fini); MODULE_LICENSE("Dual BSD/GPL"); diff --git a/crypto/fips.c b/crypto/fips.c index 9d627c1cf8bc..9dfed122d6da 100644 --- a/crypto/fips.c +++ b/crypto/fips.c @@ -74,5 +74,5 @@ static void __exit fips_exit(void) crypto_proc_fips_exit(); } -module_init(fips_init); +subsys_initcall(fips_init); module_exit(fips_exit); diff --git a/crypto/gcm.c b/crypto/gcm.c index e1a11f529d25..33f45a980967 100644 --- a/crypto/gcm.c +++ b/crypto/gcm.c @@ -597,7 +597,6 @@ static void crypto_gcm_free(struct aead_instance *inst) static int crypto_gcm_create_common(struct crypto_template *tmpl, struct rtattr **tb, - const char *full_name, const char *ctr_name, const char *ghash_name) { @@ -638,7 +637,8 @@ static int crypto_gcm_create_common(struct crypto_template *tmpl, goto err_free_inst; err = -EINVAL; - if (ghash->digestsize != 16) + if (strcmp(ghash->base.cra_name, "ghash") != 0 || + ghash->digestsize != 16) goto err_drop_ghash; crypto_set_skcipher_spawn(&ctx->ctr, aead_crypto_instance(inst)); @@ -650,24 +650,24 @@ static int crypto_gcm_create_common(struct crypto_template *tmpl, ctr = crypto_spawn_skcipher_alg(&ctx->ctr); - /* We only support 16-byte blocks. */ + /* The skcipher algorithm must be CTR mode, using 16-byte blocks. */ err = -EINVAL; - if (crypto_skcipher_alg_ivsize(ctr) != 16) + if (strncmp(ctr->base.cra_name, "ctr(", 4) != 0 || + crypto_skcipher_alg_ivsize(ctr) != 16 || + ctr->base.cra_blocksize != 1) goto out_put_ctr; - /* Not a stream cipher? */ - if (ctr->base.cra_blocksize != 1) + err = -ENAMETOOLONG; + if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, + "gcm(%s", ctr->base.cra_name + 4) >= CRYPTO_MAX_ALG_NAME) goto out_put_ctr; - err = -ENAMETOOLONG; if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "gcm_base(%s,%s)", ctr->base.cra_driver_name, ghash_alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto out_put_ctr; - memcpy(inst->alg.base.cra_name, full_name, CRYPTO_MAX_ALG_NAME); - inst->alg.base.cra_flags = (ghash->base.cra_flags | ctr->base.cra_flags) & CRYPTO_ALG_ASYNC; inst->alg.base.cra_priority = (ghash->base.cra_priority + @@ -709,7 +709,6 @@ static int crypto_gcm_create(struct crypto_template *tmpl, struct rtattr **tb) { const char *cipher_name; char ctr_name[CRYPTO_MAX_ALG_NAME]; - char full_name[CRYPTO_MAX_ALG_NAME]; cipher_name = crypto_attr_alg_name(tb[1]); if (IS_ERR(cipher_name)) @@ -719,12 +718,7 @@ static int crypto_gcm_create(struct crypto_template *tmpl, struct rtattr **tb) CRYPTO_MAX_ALG_NAME) return -ENAMETOOLONG; - if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm(%s)", cipher_name) >= - CRYPTO_MAX_ALG_NAME) - return -ENAMETOOLONG; - - return crypto_gcm_create_common(tmpl, tb, full_name, - ctr_name, "ghash"); + return crypto_gcm_create_common(tmpl, tb, ctr_name, "ghash"); } static int crypto_gcm_base_create(struct crypto_template *tmpl, @@ -732,7 +726,6 @@ static int crypto_gcm_base_create(struct crypto_template *tmpl, { const char *ctr_name; const char *ghash_name; - char full_name[CRYPTO_MAX_ALG_NAME]; ctr_name = crypto_attr_alg_name(tb[1]); if (IS_ERR(ctr_name)) @@ -742,12 +735,7 @@ static int crypto_gcm_base_create(struct crypto_template *tmpl, if (IS_ERR(ghash_name)) return PTR_ERR(ghash_name); - if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm_base(%s,%s)", - ctr_name, ghash_name) >= CRYPTO_MAX_ALG_NAME) - return -ENAMETOOLONG; - - return crypto_gcm_create_common(tmpl, tb, full_name, - ctr_name, ghash_name); + return crypto_gcm_create_common(tmpl, tb, ctr_name, ghash_name); } static int crypto_rfc4106_setkey(struct crypto_aead *parent, const u8 *key, @@ -1258,7 +1246,7 @@ static void __exit crypto_gcm_module_exit(void) ARRAY_SIZE(crypto_gcm_tmpls)); } -module_init(crypto_gcm_module_init); +subsys_initcall(crypto_gcm_module_init); module_exit(crypto_gcm_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/ghash-generic.c b/crypto/ghash-generic.c index d9f192b953b2..e6307935413c 100644 --- a/crypto/ghash-generic.c +++ b/crypto/ghash-generic.c @@ -149,7 +149,7 @@ static void __exit ghash_mod_exit(void) crypto_unregister_shash(&ghash_alg); } -module_init(ghash_mod_init); +subsys_initcall(ghash_mod_init); module_exit(ghash_mod_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/hmac.c b/crypto/hmac.c index e74730224f0a..a68c1266121f 100644 --- a/crypto/hmac.c +++ b/crypto/hmac.c @@ -57,8 +57,6 @@ static int hmac_setkey(struct crypto_shash *parent, unsigned int i; shash->tfm = hash; - shash->flags = crypto_shash_get_flags(parent) - & CRYPTO_TFM_REQ_MAY_SLEEP; if (keylen > bs) { int err; @@ -91,8 +89,6 @@ static int hmac_export(struct shash_desc *pdesc, void *out) { struct shash_desc *desc = shash_desc_ctx(pdesc); - desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; - return crypto_shash_export(desc, out); } @@ -102,7 +98,6 @@ static int hmac_import(struct shash_desc *pdesc, const void *in) struct hmac_ctx *ctx = hmac_ctx(pdesc->tfm); desc->tfm = ctx->hash; - desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; return crypto_shash_import(desc, in); } @@ -117,8 +112,6 @@ static int hmac_update(struct shash_desc *pdesc, { struct shash_desc *desc = shash_desc_ctx(pdesc); - desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; - return crypto_shash_update(desc, data, nbytes); } @@ -130,8 +123,6 @@ static int hmac_final(struct shash_desc *pdesc, u8 *out) char *opad = crypto_shash_ctx_aligned(parent) + ss; struct shash_desc *desc = shash_desc_ctx(pdesc); - desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; - return crypto_shash_final(desc, out) ?: crypto_shash_import(desc, opad) ?: crypto_shash_finup(desc, out, ds, out); @@ -147,8 +138,6 @@ static int hmac_finup(struct shash_desc *pdesc, const u8 *data, char *opad = crypto_shash_ctx_aligned(parent) + ss; struct shash_desc *desc = shash_desc_ctx(pdesc); - desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; - return crypto_shash_finup(desc, data, nbytes, out) ?: crypto_shash_import(desc, opad) ?: crypto_shash_finup(desc, out, ds, out); @@ -268,7 +257,7 @@ static void __exit hmac_module_exit(void) crypto_unregister_template(&hmac_tmpl); } -module_init(hmac_module_init); +subsys_initcall(hmac_module_init); module_exit(hmac_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/jitterentropy-kcapi.c b/crypto/jitterentropy-kcapi.c index 787dccca3715..6ea1a270b8dc 100644 --- a/crypto/jitterentropy-kcapi.c +++ b/crypto/jitterentropy-kcapi.c @@ -198,7 +198,7 @@ static void __exit jent_mod_exit(void) crypto_unregister_rng(&jent_alg); } -module_init(jent_mod_init); +subsys_initcall(jent_mod_init); module_exit(jent_mod_exit); MODULE_LICENSE("Dual BSD/GPL"); diff --git a/crypto/keywrap.c b/crypto/keywrap.c index a5cfe610d8f4..a155c88105ea 100644 --- a/crypto/keywrap.c +++ b/crypto/keywrap.c @@ -310,7 +310,7 @@ static void __exit crypto_kw_exit(void) crypto_unregister_template(&crypto_kw_tmpl); } -module_init(crypto_kw_init); +subsys_initcall(crypto_kw_init); module_exit(crypto_kw_exit); MODULE_LICENSE("Dual BSD/GPL"); diff --git a/crypto/khazad.c b/crypto/khazad.c index 873eb5ded6d7..b50aa8a3ab4c 100644 --- a/crypto/khazad.c +++ b/crypto/khazad.c @@ -875,7 +875,7 @@ static void __exit khazad_mod_fini(void) } -module_init(khazad_mod_init); +subsys_initcall(khazad_mod_init); module_exit(khazad_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/lrw.c b/crypto/lrw.c index 0430ccd08728..fa302f3f161e 100644 --- a/crypto/lrw.c +++ b/crypto/lrw.c @@ -162,8 +162,10 @@ static int xor_tweak(struct skcipher_request *req, bool second_pass) } err = skcipher_walk_virt(&w, req, false); - iv = (__be32 *)w.iv; + if (err) + return err; + iv = (__be32 *)w.iv; counter[0] = be32_to_cpu(iv[3]); counter[1] = be32_to_cpu(iv[2]); counter[2] = be32_to_cpu(iv[1]); @@ -212,8 +214,12 @@ static void crypt_done(struct crypto_async_request *areq, int err) { struct skcipher_request *req = areq->data; - if (!err) + if (!err) { + struct rctx *rctx = skcipher_request_ctx(req); + + rctx->subreq.base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; err = xor_tweak_post(req); + } skcipher_request_complete(req, err); } @@ -431,7 +437,7 @@ static void __exit crypto_module_exit(void) crypto_unregister_template(&crypto_tmpl); } -module_init(crypto_module_init); +subsys_initcall(crypto_module_init); module_exit(crypto_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/lz4.c b/crypto/lz4.c index c160dfdbf2e0..1e35134d0a98 100644 --- a/crypto/lz4.c +++ b/crypto/lz4.c @@ -164,7 +164,7 @@ static void __exit lz4_mod_fini(void) crypto_unregister_scomp(&scomp); } -module_init(lz4_mod_init); +subsys_initcall(lz4_mod_init); module_exit(lz4_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/lz4hc.c b/crypto/lz4hc.c index 583b5e013d7a..4a220b628fe7 100644 --- a/crypto/lz4hc.c +++ b/crypto/lz4hc.c @@ -165,7 +165,7 @@ static void __exit lz4hc_mod_fini(void) crypto_unregister_scomp(&scomp); } -module_init(lz4hc_mod_init); +subsys_initcall(lz4hc_mod_init); module_exit(lz4hc_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/lzo-rle.c b/crypto/lzo-rle.c index ea9c75b1db49..4c82bf18440f 100644 --- a/crypto/lzo-rle.c +++ b/crypto/lzo-rle.c @@ -167,7 +167,7 @@ static void __exit lzorle_mod_fini(void) crypto_unregister_scomp(&scomp); } -module_init(lzorle_mod_init); +subsys_initcall(lzorle_mod_init); module_exit(lzorle_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/lzo.c b/crypto/lzo.c index 218567d717d6..4a6ac8f247d0 100644 --- a/crypto/lzo.c +++ b/crypto/lzo.c @@ -167,7 +167,7 @@ static void __exit lzo_mod_fini(void) crypto_unregister_scomp(&scomp); } -module_init(lzo_mod_init); +subsys_initcall(lzo_mod_init); module_exit(lzo_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/md4.c b/crypto/md4.c index 9965ec40d9f9..9a1a228a0c69 100644 --- a/crypto/md4.c +++ b/crypto/md4.c @@ -232,7 +232,7 @@ static void __exit md4_mod_fini(void) crypto_unregister_shash(&alg); } -module_init(md4_mod_init); +subsys_initcall(md4_mod_init); module_exit(md4_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/md5.c b/crypto/md5.c index 94dd78144ba3..221c2c0932f8 100644 --- a/crypto/md5.c +++ b/crypto/md5.c @@ -244,7 +244,7 @@ static void __exit md5_mod_fini(void) crypto_unregister_shash(&alg); } -module_init(md5_mod_init); +subsys_initcall(md5_mod_init); module_exit(md5_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/michael_mic.c b/crypto/michael_mic.c index 46195e0d0f4d..538ae7933795 100644 --- a/crypto/michael_mic.c +++ b/crypto/michael_mic.c @@ -178,7 +178,7 @@ static void __exit michael_mic_exit(void) } -module_init(michael_mic_init); +subsys_initcall(michael_mic_init); module_exit(michael_mic_exit); MODULE_LICENSE("GPL v2"); diff --git a/crypto/morus1280.c b/crypto/morus1280.c index 0747732d5b78..f8734c6576af 100644 --- a/crypto/morus1280.c +++ b/crypto/morus1280.c @@ -532,7 +532,7 @@ static void __exit crypto_morus1280_module_exit(void) crypto_unregister_aead(&crypto_morus1280_alg); } -module_init(crypto_morus1280_module_init); +subsys_initcall(crypto_morus1280_module_init); module_exit(crypto_morus1280_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/morus640.c b/crypto/morus640.c index 1617a1eb8be1..ae5aa9482cb4 100644 --- a/crypto/morus640.c +++ b/crypto/morus640.c @@ -523,7 +523,7 @@ static void __exit crypto_morus640_module_exit(void) crypto_unregister_aead(&crypto_morus640_alg); } -module_init(crypto_morus640_module_init); +subsys_initcall(crypto_morus640_module_init); module_exit(crypto_morus640_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/nhpoly1305.c b/crypto/nhpoly1305.c index ec831a5594d8..9ab4e07cde4d 100644 --- a/crypto/nhpoly1305.c +++ b/crypto/nhpoly1305.c @@ -244,7 +244,7 @@ static void __exit nhpoly1305_mod_exit(void) crypto_unregister_shash(&nhpoly1305_alg); } -module_init(nhpoly1305_mod_init); +subsys_initcall(nhpoly1305_mod_init); module_exit(nhpoly1305_mod_exit); MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function"); diff --git a/crypto/ofb.c b/crypto/ofb.c index 34b6e1f426f7..133ff4c7f2c6 100644 --- a/crypto/ofb.c +++ b/crypto/ofb.c @@ -95,7 +95,7 @@ static void __exit crypto_ofb_module_exit(void) crypto_unregister_template(&crypto_ofb_tmpl); } -module_init(crypto_ofb_module_init); +subsys_initcall(crypto_ofb_module_init); module_exit(crypto_ofb_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/pcbc.c b/crypto/pcbc.c index 2fa03fc576fe..31b3ce948474 100644 --- a/crypto/pcbc.c +++ b/crypto/pcbc.c @@ -191,7 +191,7 @@ static void __exit crypto_pcbc_module_exit(void) crypto_unregister_template(&crypto_pcbc_tmpl); } -module_init(crypto_pcbc_module_init); +subsys_initcall(crypto_pcbc_module_init); module_exit(crypto_pcbc_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c index d47cfc47b1b1..0e9ce329fd47 100644 --- a/crypto/pcrypt.c +++ b/crypto/pcrypt.c @@ -512,7 +512,7 @@ static void __exit pcrypt_exit(void) crypto_unregister_template(&pcrypt_tmpl); } -module_init(pcrypt_init); +subsys_initcall(pcrypt_init); module_exit(pcrypt_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/poly1305_generic.c b/crypto/poly1305_generic.c index 2a06874204e8..adc40298c749 100644 --- a/crypto/poly1305_generic.c +++ b/crypto/poly1305_generic.c @@ -318,7 +318,7 @@ static void __exit poly1305_mod_exit(void) crypto_unregister_shash(&poly1305_alg); } -module_init(poly1305_mod_init); +subsys_initcall(poly1305_mod_init); module_exit(poly1305_mod_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/rmd128.c b/crypto/rmd128.c index 5f4472256e27..faf4252c4b85 100644 --- a/crypto/rmd128.c +++ b/crypto/rmd128.c @@ -318,7 +318,7 @@ static void __exit rmd128_mod_fini(void) crypto_unregister_shash(&alg); } -module_init(rmd128_mod_init); +subsys_initcall(rmd128_mod_init); module_exit(rmd128_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/rmd160.c b/crypto/rmd160.c index 737645344d1c..b33309916d4f 100644 --- a/crypto/rmd160.c +++ b/crypto/rmd160.c @@ -362,7 +362,7 @@ static void __exit rmd160_mod_fini(void) crypto_unregister_shash(&alg); } -module_init(rmd160_mod_init); +subsys_initcall(rmd160_mod_init); module_exit(rmd160_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/rmd256.c b/crypto/rmd256.c index 0e9d30676a01..2a643250c9a5 100644 --- a/crypto/rmd256.c +++ b/crypto/rmd256.c @@ -337,7 +337,7 @@ static void __exit rmd256_mod_fini(void) crypto_unregister_shash(&alg); } -module_init(rmd256_mod_init); +subsys_initcall(rmd256_mod_init); module_exit(rmd256_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/rmd320.c b/crypto/rmd320.c index 3ae1df5bb48c..2f062574fc8c 100644 --- a/crypto/rmd320.c +++ b/crypto/rmd320.c @@ -386,7 +386,7 @@ static void __exit rmd320_mod_fini(void) crypto_unregister_shash(&alg); } -module_init(rmd320_mod_init); +subsys_initcall(rmd320_mod_init); module_exit(rmd320_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/rsa-pkcs1pad.c b/crypto/rsa-pkcs1pad.c index 0a6680ca8cb6..29c336068dc0 100644 --- a/crypto/rsa-pkcs1pad.c +++ b/crypto/rsa-pkcs1pad.c @@ -429,7 +429,7 @@ static int pkcs1pad_sign(struct akcipher_request *req) akcipher_request_set_crypt(&req_ctx->child_req, req_ctx->in_sg, req->dst, ctx->key_size - 1, req->dst_len); - err = crypto_akcipher_sign(&req_ctx->child_req); + err = crypto_akcipher_decrypt(&req_ctx->child_req); if (err != -EINPROGRESS && err != -EBUSY) return pkcs1pad_encrypt_sign_complete(req, err); @@ -488,14 +488,21 @@ static int pkcs1pad_verify_complete(struct akcipher_request *req, int err) err = 0; - if (req->dst_len < dst_len - pos) - err = -EOVERFLOW; - req->dst_len = dst_len - pos; - - if (!err) - sg_copy_from_buffer(req->dst, - sg_nents_for_len(req->dst, req->dst_len), - out_buf + pos, req->dst_len); + if (req->dst_len != dst_len - pos) { + err = -EKEYREJECTED; + req->dst_len = dst_len - pos; + goto done; + } + /* Extract appended digest. */ + sg_pcopy_to_buffer(req->src, + sg_nents_for_len(req->src, + req->src_len + req->dst_len), + req_ctx->out_buf + ctx->key_size, + req->dst_len, ctx->key_size); + /* Do the actual verification step. */ + if (memcmp(req_ctx->out_buf + ctx->key_size, out_buf + pos, + req->dst_len) != 0) + err = -EKEYREJECTED; done: kzfree(req_ctx->out_buf); @@ -532,10 +539,12 @@ static int pkcs1pad_verify(struct akcipher_request *req) struct pkcs1pad_request *req_ctx = akcipher_request_ctx(req); int err; - if (!ctx->key_size || req->src_len < ctx->key_size) + if (WARN_ON(req->dst) || + WARN_ON(!req->dst_len) || + !ctx->key_size || req->src_len < ctx->key_size) return -EINVAL; - req_ctx->out_buf = kmalloc(ctx->key_size, GFP_KERNEL); + req_ctx->out_buf = kmalloc(ctx->key_size + req->dst_len, GFP_KERNEL); if (!req_ctx->out_buf) return -ENOMEM; @@ -551,7 +560,7 @@ static int pkcs1pad_verify(struct akcipher_request *req) req_ctx->out_sg, req->src_len, ctx->key_size); - err = crypto_akcipher_verify(&req_ctx->child_req); + err = crypto_akcipher_encrypt(&req_ctx->child_req); if (err != -EINPROGRESS && err != -EBUSY) return pkcs1pad_verify_complete(req, err); diff --git a/crypto/rsa.c b/crypto/rsa.c index 4167980c243d..dcbb03431778 100644 --- a/crypto/rsa.c +++ b/crypto/rsa.c @@ -50,34 +50,6 @@ static int _rsa_dec(const struct rsa_mpi_key *key, MPI m, MPI c) return mpi_powm(m, c, key->d, key->n); } -/* - * RSASP1 function [RFC3447 sec 5.2.1] - * s = m^d mod n - */ -static int _rsa_sign(const struct rsa_mpi_key *key, MPI s, MPI m) -{ - /* (1) Validate 0 <= m < n */ - if (mpi_cmp_ui(m, 0) < 0 || mpi_cmp(m, key->n) >= 0) - return -EINVAL; - - /* (2) s = m^d mod n */ - return mpi_powm(s, m, key->d, key->n); -} - -/* - * RSAVP1 function [RFC3447 sec 5.2.2] - * m = s^e mod n; - */ -static int _rsa_verify(const struct rsa_mpi_key *key, MPI m, MPI s) -{ - /* (1) Validate 0 <= s < n */ - if (mpi_cmp_ui(s, 0) < 0 || mpi_cmp(s, key->n) >= 0) - return -EINVAL; - - /* (2) m = s^e mod n */ - return mpi_powm(m, s, key->e, key->n); -} - static inline struct rsa_mpi_key *rsa_get_key(struct crypto_akcipher *tfm) { return akcipher_tfm_ctx(tfm); @@ -160,85 +132,6 @@ err_free_m: return ret; } -static int rsa_sign(struct akcipher_request *req) -{ - struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); - const struct rsa_mpi_key *pkey = rsa_get_key(tfm); - MPI m, s = mpi_alloc(0); - int ret = 0; - int sign; - - if (!s) - return -ENOMEM; - - if (unlikely(!pkey->n || !pkey->d)) { - ret = -EINVAL; - goto err_free_s; - } - - ret = -ENOMEM; - m = mpi_read_raw_from_sgl(req->src, req->src_len); - if (!m) - goto err_free_s; - - ret = _rsa_sign(pkey, s, m); - if (ret) - goto err_free_m; - - ret = mpi_write_to_sgl(s, req->dst, req->dst_len, &sign); - if (ret) - goto err_free_m; - - if (sign < 0) - ret = -EBADMSG; - -err_free_m: - mpi_free(m); -err_free_s: - mpi_free(s); - return ret; -} - -static int rsa_verify(struct akcipher_request *req) -{ - struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); - const struct rsa_mpi_key *pkey = rsa_get_key(tfm); - MPI s, m = mpi_alloc(0); - int ret = 0; - int sign; - - if (!m) - return -ENOMEM; - - if (unlikely(!pkey->n || !pkey->e)) { - ret = -EINVAL; - goto err_free_m; - } - - s = mpi_read_raw_from_sgl(req->src, req->src_len); - if (!s) { - ret = -ENOMEM; - goto err_free_m; - } - - ret = _rsa_verify(pkey, m, s); - if (ret) - goto err_free_s; - - ret = mpi_write_to_sgl(m, req->dst, req->dst_len, &sign); - if (ret) - goto err_free_s; - - if (sign < 0) - ret = -EBADMSG; - -err_free_s: - mpi_free(s); -err_free_m: - mpi_free(m); - return ret; -} - static void rsa_free_mpi_key(struct rsa_mpi_key *key) { mpi_free(key->d); @@ -353,8 +246,6 @@ static void rsa_exit_tfm(struct crypto_akcipher *tfm) static struct akcipher_alg rsa = { .encrypt = rsa_enc, .decrypt = rsa_dec, - .sign = rsa_sign, - .verify = rsa_verify, .set_priv_key = rsa_set_priv_key, .set_pub_key = rsa_set_pub_key, .max_size = rsa_max_size, @@ -391,7 +282,7 @@ static void rsa_exit(void) crypto_unregister_akcipher(&rsa); } -module_init(rsa_init); +subsys_initcall(rsa_init); module_exit(rsa_exit); MODULE_ALIAS_CRYPTO("rsa"); MODULE_LICENSE("GPL"); diff --git a/crypto/salsa20_generic.c b/crypto/salsa20_generic.c index 00fce32ae17a..c81a44404086 100644 --- a/crypto/salsa20_generic.c +++ b/crypto/salsa20_generic.c @@ -86,18 +86,17 @@ static void salsa20_docrypt(u32 *state, u8 *dst, const u8 *src, { __le32 stream[SALSA20_BLOCK_SIZE / sizeof(__le32)]; - if (dst != src) - memcpy(dst, src, bytes); - while (bytes >= SALSA20_BLOCK_SIZE) { salsa20_block(state, stream); - crypto_xor(dst, (const u8 *)stream, SALSA20_BLOCK_SIZE); + crypto_xor_cpy(dst, src, (const u8 *)stream, + SALSA20_BLOCK_SIZE); bytes -= SALSA20_BLOCK_SIZE; dst += SALSA20_BLOCK_SIZE; + src += SALSA20_BLOCK_SIZE; } if (bytes) { salsa20_block(state, stream); - crypto_xor(dst, (const u8 *)stream, bytes); + crypto_xor_cpy(dst, src, (const u8 *)stream, bytes); } } @@ -161,7 +160,7 @@ static int salsa20_crypt(struct skcipher_request *req) err = skcipher_walk_virt(&walk, req, false); - salsa20_init(state, ctx, walk.iv); + salsa20_init(state, ctx, req->iv); while (walk.nbytes > 0) { unsigned int nbytes = walk.nbytes; @@ -204,7 +203,7 @@ static void __exit salsa20_generic_mod_fini(void) crypto_unregister_skcipher(&alg); } -module_init(salsa20_generic_mod_init); +subsys_initcall(salsa20_generic_mod_init); module_exit(salsa20_generic_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/scompress.c b/crypto/scompress.c index 6f8305f8c300..712b4c2ea021 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -29,9 +29,17 @@ #include <crypto/internal/scompress.h> #include "internal.h" +struct scomp_scratch { + spinlock_t lock; + void *src; + void *dst; +}; + +static DEFINE_PER_CPU(struct scomp_scratch, scomp_scratch) = { + .lock = __SPIN_LOCK_UNLOCKED(scomp_scratch.lock), +}; + static const struct crypto_type crypto_scomp_type; -static void * __percpu *scomp_src_scratches; -static void * __percpu *scomp_dst_scratches; static int scomp_scratch_users; static DEFINE_MUTEX(scomp_lock); @@ -62,76 +70,53 @@ static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg) seq_puts(m, "type : scomp\n"); } -static void crypto_scomp_free_scratches(void * __percpu *scratches) +static void crypto_scomp_free_scratches(void) { + struct scomp_scratch *scratch; int i; - if (!scratches) - return; - - for_each_possible_cpu(i) - vfree(*per_cpu_ptr(scratches, i)); + for_each_possible_cpu(i) { + scratch = per_cpu_ptr(&scomp_scratch, i); - free_percpu(scratches); + vfree(scratch->src); + vfree(scratch->dst); + scratch->src = NULL; + scratch->dst = NULL; + } } -static void * __percpu *crypto_scomp_alloc_scratches(void) +static int crypto_scomp_alloc_scratches(void) { - void * __percpu *scratches; + struct scomp_scratch *scratch; int i; - scratches = alloc_percpu(void *); - if (!scratches) - return NULL; - for_each_possible_cpu(i) { - void *scratch; - - scratch = vmalloc_node(SCOMP_SCRATCH_SIZE, cpu_to_node(i)); - if (!scratch) - goto error; - *per_cpu_ptr(scratches, i) = scratch; - } - - return scratches; - -error: - crypto_scomp_free_scratches(scratches); - return NULL; -} + void *mem; -static void crypto_scomp_free_all_scratches(void) -{ - if (!--scomp_scratch_users) { - crypto_scomp_free_scratches(scomp_src_scratches); - crypto_scomp_free_scratches(scomp_dst_scratches); - scomp_src_scratches = NULL; - scomp_dst_scratches = NULL; - } -} + scratch = per_cpu_ptr(&scomp_scratch, i); -static int crypto_scomp_alloc_all_scratches(void) -{ - if (!scomp_scratch_users++) { - scomp_src_scratches = crypto_scomp_alloc_scratches(); - if (!scomp_src_scratches) - return -ENOMEM; - scomp_dst_scratches = crypto_scomp_alloc_scratches(); - if (!scomp_dst_scratches) { - crypto_scomp_free_scratches(scomp_src_scratches); - scomp_src_scratches = NULL; - return -ENOMEM; - } + mem = vmalloc_node(SCOMP_SCRATCH_SIZE, cpu_to_node(i)); + if (!mem) + goto error; + scratch->src = mem; + mem = vmalloc_node(SCOMP_SCRATCH_SIZE, cpu_to_node(i)); + if (!mem) + goto error; + scratch->dst = mem; } return 0; +error: + crypto_scomp_free_scratches(); + return -ENOMEM; } static int crypto_scomp_init_tfm(struct crypto_tfm *tfm) { - int ret; + int ret = 0; mutex_lock(&scomp_lock); - ret = crypto_scomp_alloc_all_scratches(); + if (!scomp_scratch_users++) + ret = crypto_scomp_alloc_scratches(); mutex_unlock(&scomp_lock); return ret; @@ -143,42 +128,41 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) void **tfm_ctx = acomp_tfm_ctx(tfm); struct crypto_scomp *scomp = *tfm_ctx; void **ctx = acomp_request_ctx(req); - const int cpu = get_cpu(); - u8 *scratch_src = *per_cpu_ptr(scomp_src_scratches, cpu); - u8 *scratch_dst = *per_cpu_ptr(scomp_dst_scratches, cpu); + struct scomp_scratch *scratch; int ret; - if (!req->src || !req->slen || req->slen > SCOMP_SCRATCH_SIZE) { - ret = -EINVAL; - goto out; - } + if (!req->src || !req->slen || req->slen > SCOMP_SCRATCH_SIZE) + return -EINVAL; - if (req->dst && !req->dlen) { - ret = -EINVAL; - goto out; - } + if (req->dst && !req->dlen) + return -EINVAL; if (!req->dlen || req->dlen > SCOMP_SCRATCH_SIZE) req->dlen = SCOMP_SCRATCH_SIZE; - scatterwalk_map_and_copy(scratch_src, req->src, 0, req->slen, 0); + scratch = raw_cpu_ptr(&scomp_scratch); + spin_lock(&scratch->lock); + + scatterwalk_map_and_copy(scratch->src, req->src, 0, req->slen, 0); if (dir) - ret = crypto_scomp_compress(scomp, scratch_src, req->slen, - scratch_dst, &req->dlen, *ctx); + ret = crypto_scomp_compress(scomp, scratch->src, req->slen, + scratch->dst, &req->dlen, *ctx); else - ret = crypto_scomp_decompress(scomp, scratch_src, req->slen, - scratch_dst, &req->dlen, *ctx); + ret = crypto_scomp_decompress(scomp, scratch->src, req->slen, + scratch->dst, &req->dlen, *ctx); if (!ret) { if (!req->dst) { req->dst = sgl_alloc(req->dlen, GFP_ATOMIC, NULL); - if (!req->dst) + if (!req->dst) { + ret = -ENOMEM; goto out; + } } - scatterwalk_map_and_copy(scratch_dst, req->dst, 0, req->dlen, + scatterwalk_map_and_copy(scratch->dst, req->dst, 0, req->dlen, 1); } out: - put_cpu(); + spin_unlock(&scratch->lock); return ret; } @@ -199,7 +183,8 @@ static void crypto_exit_scomp_ops_async(struct crypto_tfm *tfm) crypto_free_scomp(*ctx); mutex_lock(&scomp_lock); - crypto_scomp_free_all_scratches(); + if (!--scomp_scratch_users) + crypto_scomp_free_scratches(); mutex_unlock(&scomp_lock); } diff --git a/crypto/seed.c b/crypto/seed.c index c6ba8438be43..a75ac50fa4fd 100644 --- a/crypto/seed.c +++ b/crypto/seed.c @@ -470,7 +470,7 @@ static void __exit seed_fini(void) crypto_unregister_alg(&seed_alg); } -module_init(seed_init); +subsys_initcall(seed_init); module_exit(seed_fini); MODULE_DESCRIPTION("SEED Cipher Algorithm"); diff --git a/crypto/seqiv.c b/crypto/seqiv.c index ed1b0e9f2436..3f2fad615d26 100644 --- a/crypto/seqiv.c +++ b/crypto/seqiv.c @@ -211,7 +211,7 @@ static void __exit seqiv_module_exit(void) crypto_unregister_template(&seqiv_tmpl); } -module_init(seqiv_module_init); +subsys_initcall(seqiv_module_init); module_exit(seqiv_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/serpent_generic.c b/crypto/serpent_generic.c index 7c3382facc82..ec4ec89ad108 100644 --- a/crypto/serpent_generic.c +++ b/crypto/serpent_generic.c @@ -664,7 +664,7 @@ static void __exit serpent_mod_fini(void) crypto_unregister_algs(srp_algs, ARRAY_SIZE(srp_algs)); } -module_init(serpent_mod_init); +subsys_initcall(serpent_mod_init); module_exit(serpent_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/sha1_generic.c b/crypto/sha1_generic.c index 2af64ef81f40..1b806d4584b2 100644 --- a/crypto/sha1_generic.c +++ b/crypto/sha1_generic.c @@ -92,7 +92,7 @@ static void __exit sha1_generic_mod_fini(void) crypto_unregister_shash(&alg); } -module_init(sha1_generic_mod_init); +subsys_initcall(sha1_generic_mod_init); module_exit(sha1_generic_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/sha256_generic.c b/crypto/sha256_generic.c index 1e5ba6649e8d..5844e9a469e8 100644 --- a/crypto/sha256_generic.c +++ b/crypto/sha256_generic.c @@ -301,7 +301,7 @@ static void __exit sha256_generic_mod_fini(void) crypto_unregister_shashes(sha256_algs, ARRAY_SIZE(sha256_algs)); } -module_init(sha256_generic_mod_init); +subsys_initcall(sha256_generic_mod_init); module_exit(sha256_generic_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/sha3_generic.c b/crypto/sha3_generic.c index 7ed98367d4fb..60fd2be609d8 100644 --- a/crypto/sha3_generic.c +++ b/crypto/sha3_generic.c @@ -294,7 +294,7 @@ static void __exit sha3_generic_mod_fini(void) crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); } -module_init(sha3_generic_mod_init); +subsys_initcall(sha3_generic_mod_init); module_exit(sha3_generic_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/sha512_generic.c b/crypto/sha512_generic.c index 4097cd555eb6..0193ecb8ae10 100644 --- a/crypto/sha512_generic.c +++ b/crypto/sha512_generic.c @@ -223,7 +223,7 @@ static void __exit sha512_generic_mod_fini(void) crypto_unregister_shashes(sha512_algs, ARRAY_SIZE(sha512_algs)); } -module_init(sha512_generic_mod_init); +subsys_initcall(sha512_generic_mod_init); module_exit(sha512_generic_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/shash.c b/crypto/shash.c index 15b369c4745f..e55c1f558bc3 100644 --- a/crypto/shash.c +++ b/crypto/shash.c @@ -238,7 +238,6 @@ static int shash_async_init(struct ahash_request *req) struct shash_desc *desc = ahash_request_ctx(req); desc->tfm = *ctx; - desc->flags = req->base.flags; return crypto_shash_init(desc); } @@ -293,7 +292,6 @@ static int shash_async_finup(struct ahash_request *req) struct shash_desc *desc = ahash_request_ctx(req); desc->tfm = *ctx; - desc->flags = req->base.flags; return shash_ahash_finup(req, desc); } @@ -307,14 +305,13 @@ int shash_ahash_digest(struct ahash_request *req, struct shash_desc *desc) if (nbytes && (sg = req->src, offset = sg->offset, - nbytes < min(sg->length, ((unsigned int)(PAGE_SIZE)) - offset))) { + nbytes <= min(sg->length, ((unsigned int)(PAGE_SIZE)) - offset))) { void *data; data = kmap_atomic(sg_page(sg)); err = crypto_shash_digest(desc, data + offset, nbytes, req->result); kunmap_atomic(data); - crypto_yield(desc->flags); } else err = crypto_shash_init(desc) ?: shash_ahash_finup(req, desc); @@ -329,7 +326,6 @@ static int shash_async_digest(struct ahash_request *req) struct shash_desc *desc = ahash_request_ctx(req); desc->tfm = *ctx; - desc->flags = req->base.flags; return shash_ahash_digest(req, desc); } @@ -345,7 +341,6 @@ static int shash_async_import(struct ahash_request *req, const void *in) struct shash_desc *desc = ahash_request_ctx(req); desc->tfm = *ctx; - desc->flags = req->base.flags; return crypto_shash_import(desc, in); } diff --git a/crypto/simd.c b/crypto/simd.c index 78e8d037ae2b..3e3b1d1a6b1f 100644 --- a/crypto/simd.c +++ b/crypto/simd.c @@ -3,6 +3,7 @@ * * Copyright (c) 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> * Copyright (c) 2016 Herbert Xu <herbert@gondor.apana.org.au> + * Copyright (c) 2019 Google LLC * * Based on aesni-intel_glue.c by: * Copyright (C) 2008, Intel Corp. @@ -20,10 +21,26 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +/* + * Shared crypto SIMD helpers. These functions dynamically create and register + * an skcipher or AEAD algorithm that wraps another, internal algorithm. The + * wrapper ensures that the internal algorithm is only executed in a context + * where SIMD instructions are usable, i.e. where may_use_simd() returns true. + * If SIMD is already usable, the wrapper directly calls the internal algorithm. + * Otherwise it defers execution to a workqueue via cryptd. * + * This is an alternative to the internal algorithm implementing a fallback for + * the !may_use_simd() case itself. + * + * Note that the wrapper algorithm is asynchronous, i.e. it has the + * CRYPTO_ALG_ASYNC flag set. Therefore it won't be found by users who + * explicitly allocate a synchronous algorithm. */ #include <crypto/cryptd.h> +#include <crypto/internal/aead.h> #include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> #include <linux/kernel.h> @@ -31,6 +48,8 @@ #include <linux/preempt.h> #include <asm/simd.h> +/* skcipher support */ + struct simd_skcipher_alg { const char *ialg_name; struct skcipher_alg alg; @@ -66,7 +85,7 @@ static int simd_skcipher_encrypt(struct skcipher_request *req) subreq = skcipher_request_ctx(req); *subreq = *req; - if (!may_use_simd() || + if (!crypto_simd_usable() || (in_atomic() && cryptd_skcipher_queued(ctx->cryptd_tfm))) child = &ctx->cryptd_tfm->base; else @@ -87,7 +106,7 @@ static int simd_skcipher_decrypt(struct skcipher_request *req) subreq = skcipher_request_ctx(req); *subreq = *req; - if (!may_use_simd() || + if (!crypto_simd_usable() || (in_atomic() && cryptd_skcipher_queued(ctx->cryptd_tfm))) child = &ctx->cryptd_tfm->base; else @@ -272,4 +291,254 @@ void simd_unregister_skciphers(struct skcipher_alg *algs, int count, } EXPORT_SYMBOL_GPL(simd_unregister_skciphers); +/* AEAD support */ + +struct simd_aead_alg { + const char *ialg_name; + struct aead_alg alg; +}; + +struct simd_aead_ctx { + struct cryptd_aead *cryptd_tfm; +}; + +static int simd_aead_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int key_len) +{ + struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct crypto_aead *child = &ctx->cryptd_tfm->base; + int err; + + crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(child, crypto_aead_get_flags(tfm) & + CRYPTO_TFM_REQ_MASK); + err = crypto_aead_setkey(child, key, key_len); + crypto_aead_set_flags(tfm, crypto_aead_get_flags(child) & + CRYPTO_TFM_RES_MASK); + return err; +} + +static int simd_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize) +{ + struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct crypto_aead *child = &ctx->cryptd_tfm->base; + + return crypto_aead_setauthsize(child, authsize); +} + +static int simd_aead_encrypt(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct aead_request *subreq; + struct crypto_aead *child; + + subreq = aead_request_ctx(req); + *subreq = *req; + + if (!crypto_simd_usable() || + (in_atomic() && cryptd_aead_queued(ctx->cryptd_tfm))) + child = &ctx->cryptd_tfm->base; + else + child = cryptd_aead_child(ctx->cryptd_tfm); + + aead_request_set_tfm(subreq, child); + + return crypto_aead_encrypt(subreq); +} + +static int simd_aead_decrypt(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct aead_request *subreq; + struct crypto_aead *child; + + subreq = aead_request_ctx(req); + *subreq = *req; + + if (!crypto_simd_usable() || + (in_atomic() && cryptd_aead_queued(ctx->cryptd_tfm))) + child = &ctx->cryptd_tfm->base; + else + child = cryptd_aead_child(ctx->cryptd_tfm); + + aead_request_set_tfm(subreq, child); + + return crypto_aead_decrypt(subreq); +} + +static void simd_aead_exit(struct crypto_aead *tfm) +{ + struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm); + + cryptd_free_aead(ctx->cryptd_tfm); +} + +static int simd_aead_init(struct crypto_aead *tfm) +{ + struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct cryptd_aead *cryptd_tfm; + struct simd_aead_alg *salg; + struct aead_alg *alg; + unsigned reqsize; + + alg = crypto_aead_alg(tfm); + salg = container_of(alg, struct simd_aead_alg, alg); + + cryptd_tfm = cryptd_alloc_aead(salg->ialg_name, CRYPTO_ALG_INTERNAL, + CRYPTO_ALG_INTERNAL); + if (IS_ERR(cryptd_tfm)) + return PTR_ERR(cryptd_tfm); + + ctx->cryptd_tfm = cryptd_tfm; + + reqsize = crypto_aead_reqsize(cryptd_aead_child(cryptd_tfm)); + reqsize = max(reqsize, crypto_aead_reqsize(&cryptd_tfm->base)); + reqsize += sizeof(struct aead_request); + + crypto_aead_set_reqsize(tfm, reqsize); + + return 0; +} + +struct simd_aead_alg *simd_aead_create_compat(const char *algname, + const char *drvname, + const char *basename) +{ + struct simd_aead_alg *salg; + struct crypto_aead *tfm; + struct aead_alg *ialg; + struct aead_alg *alg; + int err; + + tfm = crypto_alloc_aead(basename, CRYPTO_ALG_INTERNAL, + CRYPTO_ALG_INTERNAL | CRYPTO_ALG_ASYNC); + if (IS_ERR(tfm)) + return ERR_CAST(tfm); + + ialg = crypto_aead_alg(tfm); + + salg = kzalloc(sizeof(*salg), GFP_KERNEL); + if (!salg) { + salg = ERR_PTR(-ENOMEM); + goto out_put_tfm; + } + + salg->ialg_name = basename; + alg = &salg->alg; + + err = -ENAMETOOLONG; + if (snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", algname) >= + CRYPTO_MAX_ALG_NAME) + goto out_free_salg; + + if (snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", + drvname) >= CRYPTO_MAX_ALG_NAME) + goto out_free_salg; + + alg->base.cra_flags = CRYPTO_ALG_ASYNC; + alg->base.cra_priority = ialg->base.cra_priority; + alg->base.cra_blocksize = ialg->base.cra_blocksize; + alg->base.cra_alignmask = ialg->base.cra_alignmask; + alg->base.cra_module = ialg->base.cra_module; + alg->base.cra_ctxsize = sizeof(struct simd_aead_ctx); + + alg->ivsize = ialg->ivsize; + alg->maxauthsize = ialg->maxauthsize; + alg->chunksize = ialg->chunksize; + + alg->init = simd_aead_init; + alg->exit = simd_aead_exit; + + alg->setkey = simd_aead_setkey; + alg->setauthsize = simd_aead_setauthsize; + alg->encrypt = simd_aead_encrypt; + alg->decrypt = simd_aead_decrypt; + + err = crypto_register_aead(alg); + if (err) + goto out_free_salg; + +out_put_tfm: + crypto_free_aead(tfm); + return salg; + +out_free_salg: + kfree(salg); + salg = ERR_PTR(err); + goto out_put_tfm; +} +EXPORT_SYMBOL_GPL(simd_aead_create_compat); + +struct simd_aead_alg *simd_aead_create(const char *algname, + const char *basename) +{ + char drvname[CRYPTO_MAX_ALG_NAME]; + + if (snprintf(drvname, CRYPTO_MAX_ALG_NAME, "simd-%s", basename) >= + CRYPTO_MAX_ALG_NAME) + return ERR_PTR(-ENAMETOOLONG); + + return simd_aead_create_compat(algname, drvname, basename); +} +EXPORT_SYMBOL_GPL(simd_aead_create); + +void simd_aead_free(struct simd_aead_alg *salg) +{ + crypto_unregister_aead(&salg->alg); + kfree(salg); +} +EXPORT_SYMBOL_GPL(simd_aead_free); + +int simd_register_aeads_compat(struct aead_alg *algs, int count, + struct simd_aead_alg **simd_algs) +{ + int err; + int i; + const char *algname; + const char *drvname; + const char *basename; + struct simd_aead_alg *simd; + + err = crypto_register_aeads(algs, count); + if (err) + return err; + + for (i = 0; i < count; i++) { + WARN_ON(strncmp(algs[i].base.cra_name, "__", 2)); + WARN_ON(strncmp(algs[i].base.cra_driver_name, "__", 2)); + algname = algs[i].base.cra_name + 2; + drvname = algs[i].base.cra_driver_name + 2; + basename = algs[i].base.cra_driver_name; + simd = simd_aead_create_compat(algname, drvname, basename); + err = PTR_ERR(simd); + if (IS_ERR(simd)) + goto err_unregister; + simd_algs[i] = simd; + } + return 0; + +err_unregister: + simd_unregister_aeads(algs, count, simd_algs); + return err; +} +EXPORT_SYMBOL_GPL(simd_register_aeads_compat); + +void simd_unregister_aeads(struct aead_alg *algs, int count, + struct simd_aead_alg **simd_algs) +{ + int i; + + crypto_unregister_aeads(algs, count); + + for (i = 0; i < count; i++) { + if (simd_algs[i]) { + simd_aead_free(simd_algs[i]); + simd_algs[i] = NULL; + } + } +} +EXPORT_SYMBOL_GPL(simd_unregister_aeads); + MODULE_LICENSE("GPL"); diff --git a/crypto/skcipher.c b/crypto/skcipher.c index bcf13d95f54a..2e66f312e2c4 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -131,8 +131,13 @@ unmap_src: memcpy(walk->dst.virt.addr, walk->page, n); skcipher_unmap_dst(walk); } else if (unlikely(walk->flags & SKCIPHER_WALK_SLOW)) { - if (WARN_ON(err)) { - /* unexpected case; didn't process all bytes */ + if (err) { + /* + * Didn't process all bytes. Either the algorithm is + * broken, or this was the last step and it turned out + * the message wasn't evenly divisible into blocks but + * the algorithm requires it. + */ err = -EINVAL; goto finish; } diff --git a/crypto/sm3_generic.c b/crypto/sm3_generic.c index c0cf87ae7ef6..e227bcada2a2 100644 --- a/crypto/sm3_generic.c +++ b/crypto/sm3_generic.c @@ -199,7 +199,7 @@ static void __exit sm3_generic_mod_fini(void) crypto_unregister_shash(&sm3_alg); } -module_init(sm3_generic_mod_init); +subsys_initcall(sm3_generic_mod_init); module_exit(sm3_generic_mod_fini); MODULE_LICENSE("GPL v2"); diff --git a/crypto/sm4_generic.c b/crypto/sm4_generic.c index c18eebfd5edd..71ffb343709a 100644 --- a/crypto/sm4_generic.c +++ b/crypto/sm4_generic.c @@ -237,7 +237,7 @@ static void __exit sm4_fini(void) crypto_unregister_alg(&sm4_alg); } -module_init(sm4_init); +subsys_initcall(sm4_init); module_exit(sm4_fini); MODULE_DESCRIPTION("SM4 Cipher Algorithm"); diff --git a/crypto/streebog_generic.c b/crypto/streebog_generic.c index 5a2eafed9c29..63663c3bab7e 100644 --- a/crypto/streebog_generic.c +++ b/crypto/streebog_generic.c @@ -996,7 +996,7 @@ static void streebog_add512(const struct streebog_uint512 *x, static void streebog_g(struct streebog_uint512 *h, const struct streebog_uint512 *N, - const u8 *m) + const struct streebog_uint512 *m) { struct streebog_uint512 Ki, data; unsigned int i; @@ -1005,7 +1005,7 @@ static void streebog_g(struct streebog_uint512 *h, /* Starting E() */ Ki = data; - streebog_xlps(&Ki, (const struct streebog_uint512 *)&m[0], &data); + streebog_xlps(&Ki, m, &data); for (i = 0; i < 11; i++) streebog_round(i, &Ki, &data); @@ -1015,16 +1015,19 @@ static void streebog_g(struct streebog_uint512 *h, /* E() done */ streebog_xor(&data, h, &data); - streebog_xor(&data, (const struct streebog_uint512 *)&m[0], h); + streebog_xor(&data, m, h); } static void streebog_stage2(struct streebog_state *ctx, const u8 *data) { - streebog_g(&ctx->h, &ctx->N, data); + struct streebog_uint512 m; + + memcpy(&m, data, sizeof(m)); + + streebog_g(&ctx->h, &ctx->N, &m); streebog_add512(&ctx->N, &buffer512, &ctx->N); - streebog_add512(&ctx->Sigma, (const struct streebog_uint512 *)data, - &ctx->Sigma); + streebog_add512(&ctx->Sigma, &m, &ctx->Sigma); } static void streebog_stage3(struct streebog_state *ctx) @@ -1034,13 +1037,11 @@ static void streebog_stage3(struct streebog_state *ctx) buf.qword[0] = cpu_to_le64(ctx->fillsize << 3); streebog_pad(ctx); - streebog_g(&ctx->h, &ctx->N, (const u8 *)&ctx->buffer); + streebog_g(&ctx->h, &ctx->N, &ctx->m); streebog_add512(&ctx->N, &buf, &ctx->N); - streebog_add512(&ctx->Sigma, - (const struct streebog_uint512 *)&ctx->buffer[0], - &ctx->Sigma); - streebog_g(&ctx->h, &buffer0, (const u8 *)&ctx->N); - streebog_g(&ctx->h, &buffer0, (const u8 *)&ctx->Sigma); + streebog_add512(&ctx->Sigma, &ctx->m, &ctx->Sigma); + streebog_g(&ctx->h, &buffer0, &ctx->N); + streebog_g(&ctx->h, &buffer0, &ctx->Sigma); memcpy(&ctx->hash, &ctx->h, sizeof(struct streebog_uint512)); } @@ -1127,7 +1128,7 @@ static void __exit streebog_mod_fini(void) crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); } -module_init(streebog_mod_init); +subsys_initcall(streebog_mod_init); module_exit(streebog_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 1ea2d5007ff5..798253f05203 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -3053,7 +3053,7 @@ err_free_tv: */ static void __exit tcrypt_mod_fini(void) { } -module_init(tcrypt_mod_init); +subsys_initcall(tcrypt_mod_init); module_exit(tcrypt_mod_fini); module_param(alg, charp, 0); diff --git a/crypto/tea.c b/crypto/tea.c index b70b441c7d1e..786b589e1399 100644 --- a/crypto/tea.c +++ b/crypto/tea.c @@ -274,7 +274,7 @@ MODULE_ALIAS_CRYPTO("tea"); MODULE_ALIAS_CRYPTO("xtea"); MODULE_ALIAS_CRYPTO("xeta"); -module_init(tea_mod_init); +subsys_initcall(tea_mod_init); module_exit(tea_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 8386038d67c7..c9e67c2bd725 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -37,6 +37,7 @@ #include <crypto/akcipher.h> #include <crypto/kpp.h> #include <crypto/acompress.h> +#include <crypto/internal/simd.h> #include "internal.h" @@ -44,6 +45,9 @@ static bool notests; module_param(notests, bool, 0644); MODULE_PARM_DESC(notests, "disable crypto self-tests"); +static bool panic_on_fail; +module_param(panic_on_fail, bool, 0444); + #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS static bool noextratests; module_param(noextratests, bool, 0644); @@ -52,6 +56,9 @@ MODULE_PARM_DESC(noextratests, "disable expensive crypto self-tests"); static unsigned int fuzz_iterations = 100; module_param(fuzz_iterations, uint, 0644); MODULE_PARM_DESC(fuzz_iterations, "number of fuzz test iterations"); + +DEFINE_PER_CPU(bool, crypto_simd_disabled_for_test); +EXPORT_PER_CPU_SYMBOL_GPL(crypto_simd_disabled_for_test); #endif #ifdef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS @@ -121,6 +128,7 @@ struct kpp_test_suite { struct alg_test_desc { const char *alg; + const char *generic_driver; int (*test)(const struct alg_test_desc *desc, const char *driver, u32 type, u32 mask); int fips_allowed; /* set if alg is allowed in fips mode */ @@ -230,12 +238,14 @@ enum finalization_type { * @offset * @flush_type: for hashes, whether an update() should be done now vs. * continuing to accumulate data + * @nosimd: if doing the pending update(), do it with SIMD disabled? */ struct test_sg_division { unsigned int proportion_of_total; unsigned int offset; bool offset_relative_to_alignmask; enum flush_type flush_type; + bool nosimd; }; /** @@ -255,6 +265,7 @@ struct test_sg_division { * @iv_offset_relative_to_alignmask: if true, add the algorithm's alignmask to * the @iv_offset * @finalization_type: what finalization function to use for hashes + * @nosimd: execute with SIMD disabled? Requires !CRYPTO_TFM_REQ_MAY_SLEEP. */ struct testvec_config { const char *name; @@ -265,6 +276,7 @@ struct testvec_config { unsigned int iv_offset; bool iv_offset_relative_to_alignmask; enum finalization_type finalization_type; + bool nosimd; }; #define TESTVEC_CONFIG_NAMELEN 192 @@ -416,8 +428,11 @@ static unsigned int count_test_sg_divisions(const struct test_sg_division *divs) return ndivs; } +#define SGDIVS_HAVE_FLUSHES BIT(0) +#define SGDIVS_HAVE_NOSIMD BIT(1) + static bool valid_sg_divisions(const struct test_sg_division *divs, - unsigned int count, bool *any_flushes_ret) + unsigned int count, int *flags_ret) { unsigned int total = 0; unsigned int i; @@ -428,7 +443,9 @@ static bool valid_sg_divisions(const struct test_sg_division *divs, return false; total += divs[i].proportion_of_total; if (divs[i].flush_type != FLUSH_TYPE_NONE) - *any_flushes_ret = true; + *flags_ret |= SGDIVS_HAVE_FLUSHES; + if (divs[i].nosimd) + *flags_ret |= SGDIVS_HAVE_NOSIMD; } return total == TEST_SG_TOTAL && memchr_inv(&divs[i], 0, (count - i) * sizeof(divs[0])) == NULL; @@ -441,19 +458,18 @@ static bool valid_sg_divisions(const struct test_sg_division *divs, */ static bool valid_testvec_config(const struct testvec_config *cfg) { - bool any_flushes = false; + int flags = 0; if (cfg->name == NULL) return false; if (!valid_sg_divisions(cfg->src_divs, ARRAY_SIZE(cfg->src_divs), - &any_flushes)) + &flags)) return false; if (cfg->dst_divs[0].proportion_of_total) { if (!valid_sg_divisions(cfg->dst_divs, - ARRAY_SIZE(cfg->dst_divs), - &any_flushes)) + ARRAY_SIZE(cfg->dst_divs), &flags)) return false; } else { if (memchr_inv(cfg->dst_divs, 0, sizeof(cfg->dst_divs))) @@ -466,7 +482,12 @@ static bool valid_testvec_config(const struct testvec_config *cfg) MAX_ALGAPI_ALIGNMASK + 1) return false; - if (any_flushes && cfg->finalization_type == FINALIZATION_TYPE_DIGEST) + if ((flags & (SGDIVS_HAVE_FLUSHES | SGDIVS_HAVE_NOSIMD)) && + cfg->finalization_type == FINALIZATION_TYPE_DIGEST) + return false; + + if ((cfg->nosimd || (flags & SGDIVS_HAVE_NOSIMD)) && + (cfg->req_flags & CRYPTO_TFM_REQ_MAY_SLEEP)) return false; return true; @@ -725,15 +746,101 @@ static int build_cipher_test_sglists(struct cipher_test_sglists *tsgls, } #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS + +/* Generate a random length in range [0, max_len], but prefer smaller values */ +static unsigned int generate_random_length(unsigned int max_len) +{ + unsigned int len = prandom_u32() % (max_len + 1); + + switch (prandom_u32() % 4) { + case 0: + return len % 64; + case 1: + return len % 256; + case 2: + return len % 1024; + default: + return len; + } +} + +/* Sometimes make some random changes to the given data buffer */ +static void mutate_buffer(u8 *buf, size_t count) +{ + size_t num_flips; + size_t i; + size_t pos; + + /* Sometimes flip some bits */ + if (prandom_u32() % 4 == 0) { + num_flips = min_t(size_t, 1 << (prandom_u32() % 8), count * 8); + for (i = 0; i < num_flips; i++) { + pos = prandom_u32() % (count * 8); + buf[pos / 8] ^= 1 << (pos % 8); + } + } + + /* Sometimes flip some bytes */ + if (prandom_u32() % 4 == 0) { + num_flips = min_t(size_t, 1 << (prandom_u32() % 8), count); + for (i = 0; i < num_flips; i++) + buf[prandom_u32() % count] ^= 0xff; + } +} + +/* Randomly generate 'count' bytes, but sometimes make them "interesting" */ +static void generate_random_bytes(u8 *buf, size_t count) +{ + u8 b; + u8 increment; + size_t i; + + if (count == 0) + return; + + switch (prandom_u32() % 8) { /* Choose a generation strategy */ + case 0: + case 1: + /* All the same byte, plus optional mutations */ + switch (prandom_u32() % 4) { + case 0: + b = 0x00; + break; + case 1: + b = 0xff; + break; + default: + b = (u8)prandom_u32(); + break; + } + memset(buf, b, count); + mutate_buffer(buf, count); + break; + case 2: + /* Ascending or descending bytes, plus optional mutations */ + increment = (u8)prandom_u32(); + b = (u8)prandom_u32(); + for (i = 0; i < count; i++, b += increment) + buf[i] = b; + mutate_buffer(buf, count); + break; + default: + /* Fully random bytes */ + for (i = 0; i < count; i++) + buf[i] = (u8)prandom_u32(); + } +} + static char *generate_random_sgl_divisions(struct test_sg_division *divs, size_t max_divs, char *p, char *end, - bool gen_flushes) + bool gen_flushes, u32 req_flags) { struct test_sg_division *div = divs; unsigned int remaining = TEST_SG_TOTAL; do { unsigned int this_len; + const char *flushtype_str; if (div == &divs[max_divs - 1] || prandom_u32() % 2 == 0) this_len = remaining; @@ -762,11 +869,31 @@ static char *generate_random_sgl_divisions(struct test_sg_division *divs, } } + if (div->flush_type != FLUSH_TYPE_NONE && + !(req_flags & CRYPTO_TFM_REQ_MAY_SLEEP) && + prandom_u32() % 2 == 0) + div->nosimd = true; + + switch (div->flush_type) { + case FLUSH_TYPE_FLUSH: + if (div->nosimd) + flushtype_str = "<flush,nosimd>"; + else + flushtype_str = "<flush>"; + break; + case FLUSH_TYPE_REIMPORT: + if (div->nosimd) + flushtype_str = "<reimport,nosimd>"; + else + flushtype_str = "<reimport>"; + break; + default: + flushtype_str = ""; + break; + } + BUILD_BUG_ON(TEST_SG_TOTAL != 10000); /* for "%u.%u%%" */ - p += scnprintf(p, end - p, "%s%u.%u%%@%s+%u%s", - div->flush_type == FLUSH_TYPE_NONE ? "" : - div->flush_type == FLUSH_TYPE_FLUSH ? - "<flush> " : "<reimport> ", + p += scnprintf(p, end - p, "%s%u.%u%%@%s+%u%s", flushtype_str, this_len / 100, this_len % 100, div->offset_relative_to_alignmask ? "alignmask" : "", @@ -816,18 +943,26 @@ static void generate_random_testvec_config(struct testvec_config *cfg, break; } + if (!(cfg->req_flags & CRYPTO_TFM_REQ_MAY_SLEEP) && + prandom_u32() % 2 == 0) { + cfg->nosimd = true; + p += scnprintf(p, end - p, " nosimd"); + } + p += scnprintf(p, end - p, " src_divs=["); p = generate_random_sgl_divisions(cfg->src_divs, ARRAY_SIZE(cfg->src_divs), p, end, (cfg->finalization_type != - FINALIZATION_TYPE_DIGEST)); + FINALIZATION_TYPE_DIGEST), + cfg->req_flags); p += scnprintf(p, end - p, "]"); if (!cfg->inplace && prandom_u32() % 2 == 0) { p += scnprintf(p, end - p, " dst_divs=["); p = generate_random_sgl_divisions(cfg->dst_divs, ARRAY_SIZE(cfg->dst_divs), - p, end, false); + p, end, false, + cfg->req_flags); p += scnprintf(p, end - p, "]"); } @@ -838,21 +973,100 @@ static void generate_random_testvec_config(struct testvec_config *cfg, WARN_ON_ONCE(!valid_testvec_config(cfg)); } -#endif /* CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ + +static void crypto_disable_simd_for_test(void) +{ + preempt_disable(); + __this_cpu_write(crypto_simd_disabled_for_test, true); +} + +static void crypto_reenable_simd_for_test(void) +{ + __this_cpu_write(crypto_simd_disabled_for_test, false); + preempt_enable(); +} + +/* + * Given an algorithm name, build the name of the generic implementation of that + * algorithm, assuming the usual naming convention. Specifically, this appends + * "-generic" to every part of the name that is not a template name. Examples: + * + * aes => aes-generic + * cbc(aes) => cbc(aes-generic) + * cts(cbc(aes)) => cts(cbc(aes-generic)) + * rfc7539(chacha20,poly1305) => rfc7539(chacha20-generic,poly1305-generic) + * + * Return: 0 on success, or -ENAMETOOLONG if the generic name would be too long + */ +static int build_generic_driver_name(const char *algname, + char driver_name[CRYPTO_MAX_ALG_NAME]) +{ + const char *in = algname; + char *out = driver_name; + size_t len = strlen(algname); + + if (len >= CRYPTO_MAX_ALG_NAME) + goto too_long; + do { + const char *in_saved = in; + + while (*in && *in != '(' && *in != ')' && *in != ',') + *out++ = *in++; + if (*in != '(' && in > in_saved) { + len += 8; + if (len >= CRYPTO_MAX_ALG_NAME) + goto too_long; + memcpy(out, "-generic", 8); + out += 8; + } + } while ((*out++ = *in++) != '\0'); + return 0; + +too_long: + pr_err("alg: generic driver name for \"%s\" would be too long\n", + algname); + return -ENAMETOOLONG; +} +#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ +static void crypto_disable_simd_for_test(void) +{ +} + +static void crypto_reenable_simd_for_test(void) +{ +} +#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ + +static int do_ahash_op(int (*op)(struct ahash_request *req), + struct ahash_request *req, + struct crypto_wait *wait, bool nosimd) +{ + int err; + + if (nosimd) + crypto_disable_simd_for_test(); + + err = op(req); + + if (nosimd) + crypto_reenable_simd_for_test(); + + return crypto_wait_req(err, wait); +} static int check_nonfinal_hash_op(const char *op, int err, u8 *result, unsigned int digestsize, - const char *driver, unsigned int vec_num, + const char *driver, const char *vec_name, const struct testvec_config *cfg) { if (err) { - pr_err("alg: hash: %s %s() failed with err %d on test vector %u, cfg=\"%s\"\n", - driver, op, err, vec_num, cfg->name); + pr_err("alg: hash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n", + driver, op, err, vec_name, cfg->name); return err; } if (!testmgr_is_poison(result, digestsize)) { - pr_err("alg: hash: %s %s() used result buffer on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: hash: %s %s() used result buffer on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return -EINVAL; } return 0; @@ -860,7 +1074,7 @@ static int check_nonfinal_hash_op(const char *op, int err, static int test_hash_vec_cfg(const char *driver, const struct hash_testvec *vec, - unsigned int vec_num, + const char *vec_name, const struct testvec_config *cfg, struct ahash_request *req, struct test_sglist *tsgl, @@ -885,11 +1099,18 @@ static int test_hash_vec_cfg(const char *driver, if (vec->ksize) { err = crypto_ahash_setkey(tfm, vec->key, vec->ksize); if (err) { - pr_err("alg: hash: %s setkey failed with err %d on test vector %u; flags=%#x\n", - driver, err, vec_num, + if (err == vec->setkey_error) + return 0; + pr_err("alg: hash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n", + driver, vec_name, vec->setkey_error, err, crypto_ahash_get_flags(tfm)); return err; } + if (vec->setkey_error) { + pr_err("alg: hash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n", + driver, vec_name, vec->setkey_error); + return -EINVAL; + } } /* Build the scatterlist for the source data */ @@ -899,8 +1120,8 @@ static int test_hash_vec_cfg(const char *driver, err = build_test_sglist(tsgl, cfg->src_divs, alignmask, vec->psize, &input, divs); if (err) { - pr_err("alg: hash: %s: error preparing scatterlist for test vector %u, cfg=\"%s\"\n", - driver, vec_num, cfg->name); + pr_err("alg: hash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n", + driver, vec_name, cfg->name); return err; } @@ -909,17 +1130,26 @@ static int test_hash_vec_cfg(const char *driver, testmgr_poison(req->__ctx, crypto_ahash_reqsize(tfm)); testmgr_poison(result, digestsize + TESTMGR_POISON_LEN); - if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST) { + if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST || + vec->digest_error) { /* Just using digest() */ ahash_request_set_callback(req, req_flags, crypto_req_done, &wait); ahash_request_set_crypt(req, tsgl->sgl, result, vec->psize); - err = crypto_wait_req(crypto_ahash_digest(req), &wait); + err = do_ahash_op(crypto_ahash_digest, req, &wait, cfg->nosimd); if (err) { - pr_err("alg: hash: %s digest() failed with err %d on test vector %u, cfg=\"%s\"\n", - driver, err, vec_num, cfg->name); + if (err == vec->digest_error) + return 0; + pr_err("alg: hash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n", + driver, vec_name, vec->digest_error, err, + cfg->name); return err; } + if (vec->digest_error) { + pr_err("alg: hash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n", + driver, vec_name, vec->digest_error, cfg->name); + return -EINVAL; + } goto result_ready; } @@ -927,9 +1157,9 @@ static int test_hash_vec_cfg(const char *driver, ahash_request_set_callback(req, req_flags, crypto_req_done, &wait); ahash_request_set_crypt(req, NULL, result, 0); - err = crypto_wait_req(crypto_ahash_init(req), &wait); + err = do_ahash_op(crypto_ahash_init, req, &wait, cfg->nosimd); err = check_nonfinal_hash_op("init", err, result, digestsize, - driver, vec_num, cfg); + driver, vec_name, cfg); if (err) return err; @@ -943,10 +1173,11 @@ static int test_hash_vec_cfg(const char *driver, crypto_req_done, &wait); ahash_request_set_crypt(req, pending_sgl, result, pending_len); - err = crypto_wait_req(crypto_ahash_update(req), &wait); + err = do_ahash_op(crypto_ahash_update, req, &wait, + divs[i]->nosimd); err = check_nonfinal_hash_op("update", err, result, digestsize, - driver, vec_num, cfg); + driver, vec_name, cfg); if (err) return err; pending_sgl = NULL; @@ -959,13 +1190,13 @@ static int test_hash_vec_cfg(const char *driver, err = crypto_ahash_export(req, hashstate); err = check_nonfinal_hash_op("export", err, result, digestsize, - driver, vec_num, cfg); + driver, vec_name, cfg); if (err) return err; if (!testmgr_is_poison(hashstate + statesize, TESTMGR_POISON_LEN)) { - pr_err("alg: hash: %s export() overran state buffer on test vector %u, cfg=\"%s\"\n", - driver, vec_num, cfg->name); + pr_err("alg: hash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n", + driver, vec_name, cfg->name); return -EOVERFLOW; } @@ -973,7 +1204,7 @@ static int test_hash_vec_cfg(const char *driver, err = crypto_ahash_import(req, hashstate); err = check_nonfinal_hash_op("import", err, result, digestsize, - driver, vec_num, cfg); + driver, vec_name, cfg); if (err) return err; } @@ -986,23 +1217,23 @@ static int test_hash_vec_cfg(const char *driver, ahash_request_set_crypt(req, pending_sgl, result, pending_len); if (cfg->finalization_type == FINALIZATION_TYPE_FINAL) { /* finish with update() and final() */ - err = crypto_wait_req(crypto_ahash_update(req), &wait); + err = do_ahash_op(crypto_ahash_update, req, &wait, cfg->nosimd); err = check_nonfinal_hash_op("update", err, result, digestsize, - driver, vec_num, cfg); + driver, vec_name, cfg); if (err) return err; - err = crypto_wait_req(crypto_ahash_final(req), &wait); + err = do_ahash_op(crypto_ahash_final, req, &wait, cfg->nosimd); if (err) { - pr_err("alg: hash: %s final() failed with err %d on test vector %u, cfg=\"%s\"\n", - driver, err, vec_num, cfg->name); + pr_err("alg: hash: %s final() failed with err %d on test vector %s, cfg=\"%s\"\n", + driver, err, vec_name, cfg->name); return err; } } else { /* finish with finup() */ - err = crypto_wait_req(crypto_ahash_finup(req), &wait); + err = do_ahash_op(crypto_ahash_finup, req, &wait, cfg->nosimd); if (err) { - pr_err("alg: hash: %s finup() failed with err %d on test vector %u, cfg=\"%s\"\n", - driver, err, vec_num, cfg->name); + pr_err("alg: hash: %s finup() failed with err %d on test vector %s, cfg=\"%s\"\n", + driver, err, vec_name, cfg->name); return err; } } @@ -1010,13 +1241,13 @@ static int test_hash_vec_cfg(const char *driver, result_ready: /* Check that the algorithm produced the correct digest */ if (memcmp(result, vec->digest, digestsize) != 0) { - pr_err("alg: hash: %s test failed (wrong result) on test vector %u, cfg=\"%s\"\n", - driver, vec_num, cfg->name); + pr_err("alg: hash: %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n", + driver, vec_name, cfg->name); return -EINVAL; } if (!testmgr_is_poison(&result[digestsize], TESTMGR_POISON_LEN)) { - pr_err("alg: hash: %s overran result buffer on test vector %u, cfg=\"%s\"\n", - driver, vec_num, cfg->name); + pr_err("alg: hash: %s overran result buffer on test vector %s, cfg=\"%s\"\n", + driver, vec_name, cfg->name); return -EOVERFLOW; } @@ -1027,11 +1258,14 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec, unsigned int vec_num, struct ahash_request *req, struct test_sglist *tsgl, u8 *hashstate) { + char vec_name[16]; unsigned int i; int err; + sprintf(vec_name, "%u", vec_num); + for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++) { - err = test_hash_vec_cfg(driver, vec, vec_num, + err = test_hash_vec_cfg(driver, vec, vec_name, &default_hash_testvec_configs[i], req, tsgl, hashstate); if (err) @@ -1046,7 +1280,7 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec, for (i = 0; i < fuzz_iterations; i++) { generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); - err = test_hash_vec_cfg(driver, vec, vec_num, &cfg, + err = test_hash_vec_cfg(driver, vec, vec_name, &cfg, req, tsgl, hashstate); if (err) return err; @@ -1056,9 +1290,168 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec, return 0; } +#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS +/* + * Generate a hash test vector from the given implementation. + * Assumes the buffers in 'vec' were already allocated. + */ +static void generate_random_hash_testvec(struct crypto_shash *tfm, + struct hash_testvec *vec, + unsigned int maxkeysize, + unsigned int maxdatasize, + char *name, size_t max_namelen) +{ + SHASH_DESC_ON_STACK(desc, tfm); + + /* Data */ + vec->psize = generate_random_length(maxdatasize); + generate_random_bytes((u8 *)vec->plaintext, vec->psize); + + /* + * Key: length in range [1, maxkeysize], but usually choose maxkeysize. + * If algorithm is unkeyed, then maxkeysize == 0 and set ksize = 0. + */ + vec->setkey_error = 0; + vec->ksize = 0; + if (maxkeysize) { + vec->ksize = maxkeysize; + if (prandom_u32() % 4 == 0) + vec->ksize = 1 + (prandom_u32() % maxkeysize); + generate_random_bytes((u8 *)vec->key, vec->ksize); + + vec->setkey_error = crypto_shash_setkey(tfm, vec->key, + vec->ksize); + /* If the key couldn't be set, no need to continue to digest. */ + if (vec->setkey_error) + goto done; + } + + /* Digest */ + desc->tfm = tfm; + vec->digest_error = crypto_shash_digest(desc, vec->plaintext, + vec->psize, (u8 *)vec->digest); +done: + snprintf(name, max_namelen, "\"random: psize=%u ksize=%u\"", + vec->psize, vec->ksize); +} + +/* + * Test the hash algorithm represented by @req against the corresponding generic + * implementation, if one is available. + */ +static int test_hash_vs_generic_impl(const char *driver, + const char *generic_driver, + unsigned int maxkeysize, + struct ahash_request *req, + struct test_sglist *tsgl, + u8 *hashstate) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + const unsigned int digestsize = crypto_ahash_digestsize(tfm); + const unsigned int blocksize = crypto_ahash_blocksize(tfm); + const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN; + const char *algname = crypto_hash_alg_common(tfm)->base.cra_name; + char _generic_driver[CRYPTO_MAX_ALG_NAME]; + struct crypto_shash *generic_tfm = NULL; + unsigned int i; + struct hash_testvec vec = { 0 }; + char vec_name[64]; + struct testvec_config cfg; + char cfgname[TESTVEC_CONFIG_NAMELEN]; + int err; + + if (noextratests) + return 0; + + if (!generic_driver) { /* Use default naming convention? */ + err = build_generic_driver_name(algname, _generic_driver); + if (err) + return err; + generic_driver = _generic_driver; + } + + if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */ + return 0; + + generic_tfm = crypto_alloc_shash(generic_driver, 0, 0); + if (IS_ERR(generic_tfm)) { + err = PTR_ERR(generic_tfm); + if (err == -ENOENT) { + pr_warn("alg: hash: skipping comparison tests for %s because %s is unavailable\n", + driver, generic_driver); + return 0; + } + pr_err("alg: hash: error allocating %s (generic impl of %s): %d\n", + generic_driver, algname, err); + return err; + } + + /* Check the algorithm properties for consistency. */ + + if (digestsize != crypto_shash_digestsize(generic_tfm)) { + pr_err("alg: hash: digestsize for %s (%u) doesn't match generic impl (%u)\n", + driver, digestsize, + crypto_shash_digestsize(generic_tfm)); + err = -EINVAL; + goto out; + } + + if (blocksize != crypto_shash_blocksize(generic_tfm)) { + pr_err("alg: hash: blocksize for %s (%u) doesn't match generic impl (%u)\n", + driver, blocksize, crypto_shash_blocksize(generic_tfm)); + err = -EINVAL; + goto out; + } + + /* + * Now generate test vectors using the generic implementation, and test + * the other implementation against them. + */ + + vec.key = kmalloc(maxkeysize, GFP_KERNEL); + vec.plaintext = kmalloc(maxdatasize, GFP_KERNEL); + vec.digest = kmalloc(digestsize, GFP_KERNEL); + if (!vec.key || !vec.plaintext || !vec.digest) { + err = -ENOMEM; + goto out; + } + + for (i = 0; i < fuzz_iterations * 8; i++) { + generate_random_hash_testvec(generic_tfm, &vec, + maxkeysize, maxdatasize, + vec_name, sizeof(vec_name)); + generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); + + err = test_hash_vec_cfg(driver, &vec, vec_name, &cfg, + req, tsgl, hashstate); + if (err) + goto out; + cond_resched(); + } + err = 0; +out: + kfree(vec.key); + kfree(vec.plaintext); + kfree(vec.digest); + crypto_free_shash(generic_tfm); + return err; +} +#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ +static int test_hash_vs_generic_impl(const char *driver, + const char *generic_driver, + unsigned int maxkeysize, + struct ahash_request *req, + struct test_sglist *tsgl, + u8 *hashstate) +{ + return 0; +} +#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ + static int __alg_test_hash(const struct hash_testvec *vecs, unsigned int num_vecs, const char *driver, - u32 type, u32 mask) + u32 type, u32 mask, + const char *generic_driver, unsigned int maxkeysize) { struct crypto_ahash *tfm; struct ahash_request *req = NULL; @@ -1106,7 +1499,8 @@ static int __alg_test_hash(const struct hash_testvec *vecs, if (err) goto out; } - err = 0; + err = test_hash_vs_generic_impl(driver, generic_driver, maxkeysize, req, + tsgl, hashstate); out: kfree(hashstate); if (tsgl) { @@ -1124,6 +1518,7 @@ static int alg_test_hash(const struct alg_test_desc *desc, const char *driver, const struct hash_testvec *template = desc->suite.hash.vecs; unsigned int tcount = desc->suite.hash.count; unsigned int nr_unkeyed, nr_keyed; + unsigned int maxkeysize = 0; int err; /* @@ -1142,23 +1537,27 @@ static int alg_test_hash(const struct alg_test_desc *desc, const char *driver, "unkeyed ones must come first\n", desc->alg); return -EINVAL; } + maxkeysize = max_t(unsigned int, maxkeysize, + template[nr_unkeyed + nr_keyed].ksize); } err = 0; if (nr_unkeyed) { - err = __alg_test_hash(template, nr_unkeyed, driver, type, mask); + err = __alg_test_hash(template, nr_unkeyed, driver, type, mask, + desc->generic_driver, maxkeysize); template += nr_unkeyed; } if (!err && nr_keyed) - err = __alg_test_hash(template, nr_keyed, driver, type, mask); + err = __alg_test_hash(template, nr_keyed, driver, type, mask, + desc->generic_driver, maxkeysize); return err; } static int test_aead_vec_cfg(const char *driver, int enc, const struct aead_testvec *vec, - unsigned int vec_num, + const char *vec_name, const struct testvec_config *cfg, struct aead_request *req, struct cipher_test_sglists *tsgls) @@ -1175,6 +1574,7 @@ static int test_aead_vec_cfg(const char *driver, int enc, cfg->iv_offset + (cfg->iv_offset_relative_to_alignmask ? alignmask : 0); struct kvec input[2]; + int expected_error; int err; /* Set the key */ @@ -1183,26 +1583,33 @@ static int test_aead_vec_cfg(const char *driver, int enc, else crypto_aead_clear_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); err = crypto_aead_setkey(tfm, vec->key, vec->klen); - if (err) { - if (vec->fail) /* expectedly failed to set key? */ - return 0; - pr_err("alg: aead: %s setkey failed with err %d on test vector %u; flags=%#x\n", - driver, err, vec_num, crypto_aead_get_flags(tfm)); + if (err && err != vec->setkey_error) { + pr_err("alg: aead: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n", + driver, vec_name, vec->setkey_error, err, + crypto_aead_get_flags(tfm)); return err; } - if (vec->fail) { - pr_err("alg: aead: %s setkey unexpectedly succeeded on test vector %u\n", - driver, vec_num); + if (!err && vec->setkey_error) { + pr_err("alg: aead: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n", + driver, vec_name, vec->setkey_error); return -EINVAL; } /* Set the authentication tag size */ err = crypto_aead_setauthsize(tfm, authsize); - if (err) { - pr_err("alg: aead: %s setauthsize failed with err %d on test vector %u\n", - driver, err, vec_num); + if (err && err != vec->setauthsize_error) { + pr_err("alg: aead: %s setauthsize failed on test vector %s; expected_error=%d, actual_error=%d\n", + driver, vec_name, vec->setauthsize_error, err); return err; } + if (!err && vec->setauthsize_error) { + pr_err("alg: aead: %s setauthsize unexpectedly succeeded on test vector %s; expected_error=%d\n", + driver, vec_name, vec->setauthsize_error); + return -EINVAL; + } + + if (vec->setkey_error || vec->setauthsize_error) + return 0; /* The IV must be copied to a buffer, as the algorithm may modify it */ if (WARN_ON(ivsize > MAX_IVLEN)) @@ -1224,8 +1631,8 @@ static int test_aead_vec_cfg(const char *driver, int enc, vec->plen), input, 2); if (err) { - pr_err("alg: aead: %s %s: error preparing scatterlists for test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: aead: %s %s: error preparing scatterlists for test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return err; } @@ -1235,23 +1642,12 @@ static int test_aead_vec_cfg(const char *driver, int enc, aead_request_set_crypt(req, tsgls->src.sgl_ptr, tsgls->dst.sgl_ptr, enc ? vec->plen : vec->clen, iv); aead_request_set_ad(req, vec->alen); - err = crypto_wait_req(enc ? crypto_aead_encrypt(req) : - crypto_aead_decrypt(req), &wait); - - aead_request_set_tfm(req, tfm); /* TODO: get rid of this */ - - if (err) { - if (err == -EBADMSG && vec->novrfy) - return 0; - pr_err("alg: aead: %s %s failed with err %d on test vector %u, cfg=\"%s\"\n", - driver, op, err, vec_num, cfg->name); - return err; - } - if (vec->novrfy) { - pr_err("alg: aead: %s %s unexpectedly succeeded on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); - return -EINVAL; - } + if (cfg->nosimd) + crypto_disable_simd_for_test(); + err = enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req); + if (cfg->nosimd) + crypto_reenable_simd_for_test(); + err = crypto_wait_req(err, &wait); /* Check that the algorithm didn't overwrite things it shouldn't have */ if (req->cryptlen != (enc ? vec->plen : vec->clen) || @@ -1263,8 +1659,8 @@ static int test_aead_vec_cfg(const char *driver, int enc, req->base.complete != crypto_req_done || req->base.flags != req_flags || req->base.data != &wait) { - pr_err("alg: aead: %s %s corrupted request struct on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: aead: %s %s corrupted request struct on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); if (req->cryptlen != (enc ? vec->plen : vec->clen)) pr_err("alg: aead: changed 'req->cryptlen'\n"); if (req->assoclen != vec->alen) @@ -1286,14 +1682,29 @@ static int test_aead_vec_cfg(const char *driver, int enc, return -EINVAL; } if (is_test_sglist_corrupted(&tsgls->src)) { - pr_err("alg: aead: %s %s corrupted src sgl on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: aead: %s %s corrupted src sgl on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return -EINVAL; } if (tsgls->dst.sgl_ptr != tsgls->src.sgl && is_test_sglist_corrupted(&tsgls->dst)) { - pr_err("alg: aead: %s %s corrupted dst sgl on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: aead: %s %s corrupted dst sgl on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); + return -EINVAL; + } + + /* Check for success or failure */ + expected_error = vec->novrfy ? -EBADMSG : vec->crypt_error; + if (err) { + if (err == expected_error) + return 0; + pr_err("alg: aead: %s %s failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n", + driver, op, vec_name, expected_error, err, cfg->name); + return err; + } + if (expected_error) { + pr_err("alg: aead: %s %s unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n", + driver, op, vec_name, expected_error, cfg->name); return -EINVAL; } @@ -1302,13 +1713,13 @@ static int test_aead_vec_cfg(const char *driver, int enc, enc ? vec->clen : vec->plen, vec->alen, enc || !cfg->inplace); if (err == -EOVERFLOW) { - pr_err("alg: aead: %s %s overran dst buffer on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: aead: %s %s overran dst buffer on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return err; } if (err) { - pr_err("alg: aead: %s %s test failed (wrong result) on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: aead: %s %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return err; } @@ -1320,14 +1731,17 @@ static int test_aead_vec(const char *driver, int enc, struct aead_request *req, struct cipher_test_sglists *tsgls) { + char vec_name[16]; unsigned int i; int err; if (enc && vec->novrfy) return 0; + sprintf(vec_name, "%u", vec_num); + for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++) { - err = test_aead_vec_cfg(driver, enc, vec, vec_num, + err = test_aead_vec_cfg(driver, enc, vec, vec_name, &default_cipher_testvec_configs[i], req, tsgls); if (err) @@ -1342,7 +1756,7 @@ static int test_aead_vec(const char *driver, int enc, for (i = 0; i < fuzz_iterations; i++) { generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); - err = test_aead_vec_cfg(driver, enc, vec, vec_num, + err = test_aead_vec_cfg(driver, enc, vec, vec_name, &cfg, req, tsgls); if (err) return err; @@ -1352,6 +1766,226 @@ static int test_aead_vec(const char *driver, int enc, return 0; } +#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS +/* + * Generate an AEAD test vector from the given implementation. + * Assumes the buffers in 'vec' were already allocated. + */ +static void generate_random_aead_testvec(struct aead_request *req, + struct aead_testvec *vec, + unsigned int maxkeysize, + unsigned int maxdatasize, + char *name, size_t max_namelen) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + const unsigned int ivsize = crypto_aead_ivsize(tfm); + unsigned int maxauthsize = crypto_aead_alg(tfm)->maxauthsize; + unsigned int authsize; + unsigned int total_len; + int i; + struct scatterlist src[2], dst; + u8 iv[MAX_IVLEN]; + DECLARE_CRYPTO_WAIT(wait); + + /* Key: length in [0, maxkeysize], but usually choose maxkeysize */ + vec->klen = maxkeysize; + if (prandom_u32() % 4 == 0) + vec->klen = prandom_u32() % (maxkeysize + 1); + generate_random_bytes((u8 *)vec->key, vec->klen); + vec->setkey_error = crypto_aead_setkey(tfm, vec->key, vec->klen); + + /* IV */ + generate_random_bytes((u8 *)vec->iv, ivsize); + + /* Tag length: in [0, maxauthsize], but usually choose maxauthsize */ + authsize = maxauthsize; + if (prandom_u32() % 4 == 0) + authsize = prandom_u32() % (maxauthsize + 1); + if (WARN_ON(authsize > maxdatasize)) + authsize = maxdatasize; + maxdatasize -= authsize; + vec->setauthsize_error = crypto_aead_setauthsize(tfm, authsize); + + /* Plaintext and associated data */ + total_len = generate_random_length(maxdatasize); + if (prandom_u32() % 4 == 0) + vec->alen = 0; + else + vec->alen = generate_random_length(total_len); + vec->plen = total_len - vec->alen; + generate_random_bytes((u8 *)vec->assoc, vec->alen); + generate_random_bytes((u8 *)vec->ptext, vec->plen); + + vec->clen = vec->plen + authsize; + + /* + * If the key or authentication tag size couldn't be set, no need to + * continue to encrypt. + */ + if (vec->setkey_error || vec->setauthsize_error) + goto done; + + /* Ciphertext */ + sg_init_table(src, 2); + i = 0; + if (vec->alen) + sg_set_buf(&src[i++], vec->assoc, vec->alen); + if (vec->plen) + sg_set_buf(&src[i++], vec->ptext, vec->plen); + sg_init_one(&dst, vec->ctext, vec->alen + vec->clen); + memcpy(iv, vec->iv, ivsize); + aead_request_set_callback(req, 0, crypto_req_done, &wait); + aead_request_set_crypt(req, src, &dst, vec->plen, iv); + aead_request_set_ad(req, vec->alen); + vec->crypt_error = crypto_wait_req(crypto_aead_encrypt(req), &wait); + if (vec->crypt_error == 0) + memmove((u8 *)vec->ctext, vec->ctext + vec->alen, vec->clen); +done: + snprintf(name, max_namelen, + "\"random: alen=%u plen=%u authsize=%u klen=%u\"", + vec->alen, vec->plen, authsize, vec->klen); +} + +/* + * Test the AEAD algorithm represented by @req against the corresponding generic + * implementation, if one is available. + */ +static int test_aead_vs_generic_impl(const char *driver, + const struct alg_test_desc *test_desc, + struct aead_request *req, + struct cipher_test_sglists *tsgls) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + const unsigned int ivsize = crypto_aead_ivsize(tfm); + const unsigned int maxauthsize = crypto_aead_alg(tfm)->maxauthsize; + const unsigned int blocksize = crypto_aead_blocksize(tfm); + const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN; + const char *algname = crypto_aead_alg(tfm)->base.cra_name; + const char *generic_driver = test_desc->generic_driver; + char _generic_driver[CRYPTO_MAX_ALG_NAME]; + struct crypto_aead *generic_tfm = NULL; + struct aead_request *generic_req = NULL; + unsigned int maxkeysize; + unsigned int i; + struct aead_testvec vec = { 0 }; + char vec_name[64]; + struct testvec_config cfg; + char cfgname[TESTVEC_CONFIG_NAMELEN]; + int err; + + if (noextratests) + return 0; + + if (!generic_driver) { /* Use default naming convention? */ + err = build_generic_driver_name(algname, _generic_driver); + if (err) + return err; + generic_driver = _generic_driver; + } + + if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */ + return 0; + + generic_tfm = crypto_alloc_aead(generic_driver, 0, 0); + if (IS_ERR(generic_tfm)) { + err = PTR_ERR(generic_tfm); + if (err == -ENOENT) { + pr_warn("alg: aead: skipping comparison tests for %s because %s is unavailable\n", + driver, generic_driver); + return 0; + } + pr_err("alg: aead: error allocating %s (generic impl of %s): %d\n", + generic_driver, algname, err); + return err; + } + + generic_req = aead_request_alloc(generic_tfm, GFP_KERNEL); + if (!generic_req) { + err = -ENOMEM; + goto out; + } + + /* Check the algorithm properties for consistency. */ + + if (maxauthsize != crypto_aead_alg(generic_tfm)->maxauthsize) { + pr_err("alg: aead: maxauthsize for %s (%u) doesn't match generic impl (%u)\n", + driver, maxauthsize, + crypto_aead_alg(generic_tfm)->maxauthsize); + err = -EINVAL; + goto out; + } + + if (ivsize != crypto_aead_ivsize(generic_tfm)) { + pr_err("alg: aead: ivsize for %s (%u) doesn't match generic impl (%u)\n", + driver, ivsize, crypto_aead_ivsize(generic_tfm)); + err = -EINVAL; + goto out; + } + + if (blocksize != crypto_aead_blocksize(generic_tfm)) { + pr_err("alg: aead: blocksize for %s (%u) doesn't match generic impl (%u)\n", + driver, blocksize, crypto_aead_blocksize(generic_tfm)); + err = -EINVAL; + goto out; + } + + /* + * Now generate test vectors using the generic implementation, and test + * the other implementation against them. + */ + + maxkeysize = 0; + for (i = 0; i < test_desc->suite.aead.count; i++) + maxkeysize = max_t(unsigned int, maxkeysize, + test_desc->suite.aead.vecs[i].klen); + + vec.key = kmalloc(maxkeysize, GFP_KERNEL); + vec.iv = kmalloc(ivsize, GFP_KERNEL); + vec.assoc = kmalloc(maxdatasize, GFP_KERNEL); + vec.ptext = kmalloc(maxdatasize, GFP_KERNEL); + vec.ctext = kmalloc(maxdatasize, GFP_KERNEL); + if (!vec.key || !vec.iv || !vec.assoc || !vec.ptext || !vec.ctext) { + err = -ENOMEM; + goto out; + } + + for (i = 0; i < fuzz_iterations * 8; i++) { + generate_random_aead_testvec(generic_req, &vec, + maxkeysize, maxdatasize, + vec_name, sizeof(vec_name)); + generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); + + err = test_aead_vec_cfg(driver, ENCRYPT, &vec, vec_name, &cfg, + req, tsgls); + if (err) + goto out; + err = test_aead_vec_cfg(driver, DECRYPT, &vec, vec_name, &cfg, + req, tsgls); + if (err) + goto out; + cond_resched(); + } + err = 0; +out: + kfree(vec.key); + kfree(vec.iv); + kfree(vec.assoc); + kfree(vec.ptext); + kfree(vec.ctext); + crypto_free_aead(generic_tfm); + aead_request_free(generic_req); + return err; +} +#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ +static int test_aead_vs_generic_impl(const char *driver, + const struct alg_test_desc *test_desc, + struct aead_request *req, + struct cipher_test_sglists *tsgls) +{ + return 0; +} +#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ + static int test_aead(const char *driver, int enc, const struct aead_test_suite *suite, struct aead_request *req, @@ -1411,6 +2045,10 @@ static int alg_test_aead(const struct alg_test_desc *desc, const char *driver, goto out; err = test_aead(driver, DECRYPT, suite, req, tsgls); + if (err) + goto out; + + err = test_aead_vs_generic_impl(driver, desc, req, tsgls); out: free_cipher_test_sglists(tsgls); aead_request_free(req); @@ -1462,13 +2100,20 @@ static int test_cipher(struct crypto_cipher *tfm, int enc, ret = crypto_cipher_setkey(tfm, template[i].key, template[i].klen); - if (template[i].fail == !ret) { - printk(KERN_ERR "alg: cipher: setkey failed " - "on test %d for %s: flags=%x\n", j, - algo, crypto_cipher_get_flags(tfm)); + if (ret) { + if (ret == template[i].setkey_error) + continue; + pr_err("alg: cipher: %s setkey failed on test vector %u; expected_error=%d, actual_error=%d, flags=%#x\n", + algo, j, template[i].setkey_error, ret, + crypto_cipher_get_flags(tfm)); goto out; - } else if (ret) - continue; + } + if (template[i].setkey_error) { + pr_err("alg: cipher: %s setkey unexpectedly succeeded on test vector %u; expected_error=%d\n", + algo, j, template[i].setkey_error); + ret = -EINVAL; + goto out; + } for (k = 0; k < template[i].len; k += crypto_cipher_blocksize(tfm)) { @@ -1500,7 +2145,7 @@ out_nobuf: static int test_skcipher_vec_cfg(const char *driver, int enc, const struct cipher_testvec *vec, - unsigned int vec_num, + const char *vec_name, const struct testvec_config *cfg, struct skcipher_request *req, struct cipher_test_sglists *tsgls) @@ -1526,15 +2171,16 @@ static int test_skcipher_vec_cfg(const char *driver, int enc, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); err = crypto_skcipher_setkey(tfm, vec->key, vec->klen); if (err) { - if (vec->fail) /* expectedly failed to set key? */ + if (err == vec->setkey_error) return 0; - pr_err("alg: skcipher: %s setkey failed with err %d on test vector %u; flags=%#x\n", - driver, err, vec_num, crypto_skcipher_get_flags(tfm)); + pr_err("alg: skcipher: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n", + driver, vec_name, vec->setkey_error, err, + crypto_skcipher_get_flags(tfm)); return err; } - if (vec->fail) { - pr_err("alg: skcipher: %s setkey unexpectedly succeeded on test vector %u\n", - driver, vec_num); + if (vec->setkey_error) { + pr_err("alg: skcipher: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n", + driver, vec_name, vec->setkey_error); return -EINVAL; } @@ -1550,8 +2196,8 @@ static int test_skcipher_vec_cfg(const char *driver, int enc, memset(iv, 0, ivsize); } else { if (vec->generates_iv) { - pr_err("alg: skcipher: %s has ivsize=0 but test vector %u generates IV!\n", - driver, vec_num); + pr_err("alg: skcipher: %s has ivsize=0 but test vector %s generates IV!\n", + driver, vec_name); return -EINVAL; } iv = NULL; @@ -1563,8 +2209,8 @@ static int test_skcipher_vec_cfg(const char *driver, int enc, err = build_cipher_test_sglists(tsgls, cfg, alignmask, vec->len, vec->len, &input, 1); if (err) { - pr_err("alg: skcipher: %s %s: error preparing scatterlists for test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: skcipher: %s %s: error preparing scatterlists for test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return err; } @@ -1573,13 +2219,12 @@ static int test_skcipher_vec_cfg(const char *driver, int enc, skcipher_request_set_callback(req, req_flags, crypto_req_done, &wait); skcipher_request_set_crypt(req, tsgls->src.sgl_ptr, tsgls->dst.sgl_ptr, vec->len, iv); - err = crypto_wait_req(enc ? crypto_skcipher_encrypt(req) : - crypto_skcipher_decrypt(req), &wait); - if (err) { - pr_err("alg: skcipher: %s %s failed with err %d on test vector %u, cfg=\"%s\"\n", - driver, op, err, vec_num, cfg->name); - return err; - } + if (cfg->nosimd) + crypto_disable_simd_for_test(); + err = enc ? crypto_skcipher_encrypt(req) : crypto_skcipher_decrypt(req); + if (cfg->nosimd) + crypto_reenable_simd_for_test(); + err = crypto_wait_req(err, &wait); /* Check that the algorithm didn't overwrite things it shouldn't have */ if (req->cryptlen != vec->len || @@ -1590,8 +2235,8 @@ static int test_skcipher_vec_cfg(const char *driver, int enc, req->base.complete != crypto_req_done || req->base.flags != req_flags || req->base.data != &wait) { - pr_err("alg: skcipher: %s %s corrupted request struct on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: skcipher: %s %s corrupted request struct on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); if (req->cryptlen != vec->len) pr_err("alg: skcipher: changed 'req->cryptlen'\n"); if (req->iv != iv) @@ -1611,14 +2256,28 @@ static int test_skcipher_vec_cfg(const char *driver, int enc, return -EINVAL; } if (is_test_sglist_corrupted(&tsgls->src)) { - pr_err("alg: skcipher: %s %s corrupted src sgl on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: skcipher: %s %s corrupted src sgl on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return -EINVAL; } if (tsgls->dst.sgl_ptr != tsgls->src.sgl && is_test_sglist_corrupted(&tsgls->dst)) { - pr_err("alg: skcipher: %s %s corrupted dst sgl on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: skcipher: %s %s corrupted dst sgl on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); + return -EINVAL; + } + + /* Check for success or failure */ + if (err) { + if (err == vec->crypt_error) + return 0; + pr_err("alg: skcipher: %s %s failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n", + driver, op, vec_name, vec->crypt_error, err, cfg->name); + return err; + } + if (vec->crypt_error) { + pr_err("alg: skcipher: %s %s unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n", + driver, op, vec_name, vec->crypt_error, cfg->name); return -EINVAL; } @@ -1626,20 +2285,20 @@ static int test_skcipher_vec_cfg(const char *driver, int enc, err = verify_correct_output(&tsgls->dst, enc ? vec->ctext : vec->ptext, vec->len, 0, true); if (err == -EOVERFLOW) { - pr_err("alg: skcipher: %s %s overran dst buffer on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: skcipher: %s %s overran dst buffer on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return err; } if (err) { - pr_err("alg: skcipher: %s %s test failed (wrong result) on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: skcipher: %s %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return err; } /* If applicable, check that the algorithm generated the correct IV */ if (vec->iv_out && memcmp(iv, vec->iv_out, ivsize) != 0) { - pr_err("alg: skcipher: %s %s test failed (wrong output IV) on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: skcipher: %s %s test failed (wrong output IV) on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); hexdump(iv, ivsize); return -EINVAL; } @@ -1653,14 +2312,17 @@ static int test_skcipher_vec(const char *driver, int enc, struct skcipher_request *req, struct cipher_test_sglists *tsgls) { + char vec_name[16]; unsigned int i; int err; if (fips_enabled && vec->fips_skip) return 0; + sprintf(vec_name, "%u", vec_num); + for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++) { - err = test_skcipher_vec_cfg(driver, enc, vec, vec_num, + err = test_skcipher_vec_cfg(driver, enc, vec, vec_name, &default_cipher_testvec_configs[i], req, tsgls); if (err) @@ -1675,7 +2337,7 @@ static int test_skcipher_vec(const char *driver, int enc, for (i = 0; i < fuzz_iterations; i++) { generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); - err = test_skcipher_vec_cfg(driver, enc, vec, vec_num, + err = test_skcipher_vec_cfg(driver, enc, vec, vec_name, &cfg, req, tsgls); if (err) return err; @@ -1685,6 +2347,186 @@ static int test_skcipher_vec(const char *driver, int enc, return 0; } +#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS +/* + * Generate a symmetric cipher test vector from the given implementation. + * Assumes the buffers in 'vec' were already allocated. + */ +static void generate_random_cipher_testvec(struct skcipher_request *req, + struct cipher_testvec *vec, + unsigned int maxdatasize, + char *name, size_t max_namelen) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const unsigned int maxkeysize = tfm->keysize; + const unsigned int ivsize = crypto_skcipher_ivsize(tfm); + struct scatterlist src, dst; + u8 iv[MAX_IVLEN]; + DECLARE_CRYPTO_WAIT(wait); + + /* Key: length in [0, maxkeysize], but usually choose maxkeysize */ + vec->klen = maxkeysize; + if (prandom_u32() % 4 == 0) + vec->klen = prandom_u32() % (maxkeysize + 1); + generate_random_bytes((u8 *)vec->key, vec->klen); + vec->setkey_error = crypto_skcipher_setkey(tfm, vec->key, vec->klen); + + /* IV */ + generate_random_bytes((u8 *)vec->iv, ivsize); + + /* Plaintext */ + vec->len = generate_random_length(maxdatasize); + generate_random_bytes((u8 *)vec->ptext, vec->len); + + /* If the key couldn't be set, no need to continue to encrypt. */ + if (vec->setkey_error) + goto done; + + /* Ciphertext */ + sg_init_one(&src, vec->ptext, vec->len); + sg_init_one(&dst, vec->ctext, vec->len); + memcpy(iv, vec->iv, ivsize); + skcipher_request_set_callback(req, 0, crypto_req_done, &wait); + skcipher_request_set_crypt(req, &src, &dst, vec->len, iv); + vec->crypt_error = crypto_wait_req(crypto_skcipher_encrypt(req), &wait); +done: + snprintf(name, max_namelen, "\"random: len=%u klen=%u\"", + vec->len, vec->klen); +} + +/* + * Test the skcipher algorithm represented by @req against the corresponding + * generic implementation, if one is available. + */ +static int test_skcipher_vs_generic_impl(const char *driver, + const char *generic_driver, + struct skcipher_request *req, + struct cipher_test_sglists *tsgls) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const unsigned int ivsize = crypto_skcipher_ivsize(tfm); + const unsigned int blocksize = crypto_skcipher_blocksize(tfm); + const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN; + const char *algname = crypto_skcipher_alg(tfm)->base.cra_name; + char _generic_driver[CRYPTO_MAX_ALG_NAME]; + struct crypto_skcipher *generic_tfm = NULL; + struct skcipher_request *generic_req = NULL; + unsigned int i; + struct cipher_testvec vec = { 0 }; + char vec_name[64]; + struct testvec_config cfg; + char cfgname[TESTVEC_CONFIG_NAMELEN]; + int err; + + if (noextratests) + return 0; + + /* Keywrap isn't supported here yet as it handles its IV differently. */ + if (strncmp(algname, "kw(", 3) == 0) + return 0; + + if (!generic_driver) { /* Use default naming convention? */ + err = build_generic_driver_name(algname, _generic_driver); + if (err) + return err; + generic_driver = _generic_driver; + } + + if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */ + return 0; + + generic_tfm = crypto_alloc_skcipher(generic_driver, 0, 0); + if (IS_ERR(generic_tfm)) { + err = PTR_ERR(generic_tfm); + if (err == -ENOENT) { + pr_warn("alg: skcipher: skipping comparison tests for %s because %s is unavailable\n", + driver, generic_driver); + return 0; + } + pr_err("alg: skcipher: error allocating %s (generic impl of %s): %d\n", + generic_driver, algname, err); + return err; + } + + generic_req = skcipher_request_alloc(generic_tfm, GFP_KERNEL); + if (!generic_req) { + err = -ENOMEM; + goto out; + } + + /* Check the algorithm properties for consistency. */ + + if (tfm->keysize != generic_tfm->keysize) { + pr_err("alg: skcipher: max keysize for %s (%u) doesn't match generic impl (%u)\n", + driver, tfm->keysize, generic_tfm->keysize); + err = -EINVAL; + goto out; + } + + if (ivsize != crypto_skcipher_ivsize(generic_tfm)) { + pr_err("alg: skcipher: ivsize for %s (%u) doesn't match generic impl (%u)\n", + driver, ivsize, crypto_skcipher_ivsize(generic_tfm)); + err = -EINVAL; + goto out; + } + + if (blocksize != crypto_skcipher_blocksize(generic_tfm)) { + pr_err("alg: skcipher: blocksize for %s (%u) doesn't match generic impl (%u)\n", + driver, blocksize, + crypto_skcipher_blocksize(generic_tfm)); + err = -EINVAL; + goto out; + } + + /* + * Now generate test vectors using the generic implementation, and test + * the other implementation against them. + */ + + vec.key = kmalloc(tfm->keysize, GFP_KERNEL); + vec.iv = kmalloc(ivsize, GFP_KERNEL); + vec.ptext = kmalloc(maxdatasize, GFP_KERNEL); + vec.ctext = kmalloc(maxdatasize, GFP_KERNEL); + if (!vec.key || !vec.iv || !vec.ptext || !vec.ctext) { + err = -ENOMEM; + goto out; + } + + for (i = 0; i < fuzz_iterations * 8; i++) { + generate_random_cipher_testvec(generic_req, &vec, maxdatasize, + vec_name, sizeof(vec_name)); + generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); + + err = test_skcipher_vec_cfg(driver, ENCRYPT, &vec, vec_name, + &cfg, req, tsgls); + if (err) + goto out; + err = test_skcipher_vec_cfg(driver, DECRYPT, &vec, vec_name, + &cfg, req, tsgls); + if (err) + goto out; + cond_resched(); + } + err = 0; +out: + kfree(vec.key); + kfree(vec.iv); + kfree(vec.ptext); + kfree(vec.ctext); + crypto_free_skcipher(generic_tfm); + skcipher_request_free(generic_req); + return err; +} +#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ +static int test_skcipher_vs_generic_impl(const char *driver, + const char *generic_driver, + struct skcipher_request *req, + struct cipher_test_sglists *tsgls) +{ + return 0; +} +#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ + static int test_skcipher(const char *driver, int enc, const struct cipher_test_suite *suite, struct skcipher_request *req, @@ -1744,6 +2586,11 @@ static int alg_test_skcipher(const struct alg_test_desc *desc, goto out; err = test_skcipher(driver, DECRYPT, suite, req, tsgls); + if (err) + goto out; + + err = test_skcipher_vs_generic_impl(driver, desc->generic_driver, req, + tsgls); out: free_cipher_test_sglists(tsgls); skcipher_request_free(req); @@ -2179,7 +3026,6 @@ static int alg_test_crc32c(const struct alg_test_desc *desc, u32 *ctx = (u32 *)shash_desc_ctx(shash); shash->tfm = tfm; - shash->flags = 0; *ctx = 420553207; err = crypto_shash_final(shash, (u8 *)&val); @@ -2493,6 +3339,12 @@ static int alg_test_kpp(const struct alg_test_desc *desc, const char *driver, return err; } +static u8 *test_pack_u32(u8 *dst, u32 val) +{ + memcpy(dst, &val, sizeof(val)); + return dst + sizeof(val); +} + static int test_akcipher_one(struct crypto_akcipher *tfm, const struct akcipher_testvec *vecs) { @@ -2503,10 +3355,11 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, struct crypto_wait wait; unsigned int out_len_max, out_len = 0; int err = -ENOMEM; - struct scatterlist src, dst, src_tab[2]; + struct scatterlist src, dst, src_tab[3]; const char *m, *c; unsigned int m_size, c_size; const char *op; + u8 *key, *ptr; if (testmgr_alloc_buf(xbuf)) return err; @@ -2517,22 +3370,29 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, crypto_init_wait(&wait); + key = kmalloc(vecs->key_len + sizeof(u32) * 2 + vecs->param_len, + GFP_KERNEL); + if (!key) + goto free_xbuf; + memcpy(key, vecs->key, vecs->key_len); + ptr = key + vecs->key_len; + ptr = test_pack_u32(ptr, vecs->algo); + ptr = test_pack_u32(ptr, vecs->param_len); + memcpy(ptr, vecs->params, vecs->param_len); + if (vecs->public_key_vec) - err = crypto_akcipher_set_pub_key(tfm, vecs->key, - vecs->key_len); + err = crypto_akcipher_set_pub_key(tfm, key, vecs->key_len); else - err = crypto_akcipher_set_priv_key(tfm, vecs->key, - vecs->key_len); + err = crypto_akcipher_set_priv_key(tfm, key, vecs->key_len); if (err) goto free_req; - err = -ENOMEM; - out_len_max = crypto_akcipher_maxsize(tfm); - /* * First run test which do not require a private key, such as * encrypt or verify. */ + err = -ENOMEM; + out_len_max = crypto_akcipher_maxsize(tfm); outbuf_enc = kzalloc(out_len_max, GFP_KERNEL); if (!outbuf_enc) goto free_req; @@ -2558,12 +3418,20 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, goto free_all; memcpy(xbuf[0], m, m_size); - sg_init_table(src_tab, 2); + sg_init_table(src_tab, 3); sg_set_buf(&src_tab[0], xbuf[0], 8); sg_set_buf(&src_tab[1], xbuf[0] + 8, m_size - 8); - sg_init_one(&dst, outbuf_enc, out_len_max); - akcipher_request_set_crypt(req, src_tab, &dst, m_size, - out_len_max); + if (vecs->siggen_sigver_test) { + if (WARN_ON(c_size > PAGE_SIZE)) + goto free_all; + memcpy(xbuf[1], c, c_size); + sg_set_buf(&src_tab[2], xbuf[1], c_size); + akcipher_request_set_crypt(req, src_tab, NULL, m_size, c_size); + } else { + sg_init_one(&dst, outbuf_enc, out_len_max); + akcipher_request_set_crypt(req, src_tab, &dst, m_size, + out_len_max); + } akcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, crypto_req_done, &wait); @@ -2576,18 +3444,21 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, pr_err("alg: akcipher: %s test failed. err %d\n", op, err); goto free_all; } - if (req->dst_len != c_size) { - pr_err("alg: akcipher: %s test failed. Invalid output len\n", - op); - err = -EINVAL; - goto free_all; - } - /* verify that encrypted message is equal to expected */ - if (memcmp(c, outbuf_enc, c_size)) { - pr_err("alg: akcipher: %s test failed. Invalid output\n", op); - hexdump(outbuf_enc, c_size); - err = -EINVAL; - goto free_all; + if (!vecs->siggen_sigver_test) { + if (req->dst_len != c_size) { + pr_err("alg: akcipher: %s test failed. Invalid output len\n", + op); + err = -EINVAL; + goto free_all; + } + /* verify that encrypted message is equal to expected */ + if (memcmp(c, outbuf_enc, c_size) != 0) { + pr_err("alg: akcipher: %s test failed. Invalid output\n", + op); + hexdump(outbuf_enc, c_size); + err = -EINVAL; + goto free_all; + } } /* @@ -2642,6 +3513,7 @@ free_all: kfree(outbuf_enc); free_req: akcipher_request_free(req); + kfree(key); free_xbuf: testmgr_free_buf(xbuf); return err; @@ -2699,12 +3571,14 @@ static int alg_test_null(const struct alg_test_desc *desc, static const struct alg_test_desc alg_test_descs[] = { { .alg = "adiantum(xchacha12,aes)", + .generic_driver = "adiantum(xchacha12-generic,aes-generic,nhpoly1305-generic)", .test = alg_test_skcipher, .suite = { .cipher = __VECS(adiantum_xchacha12_aes_tv_template) }, }, { .alg = "adiantum(xchacha20,aes)", + .generic_driver = "adiantum(xchacha20-generic,aes-generic,nhpoly1305-generic)", .test = alg_test_skcipher, .suite = { .cipher = __VECS(adiantum_xchacha20_aes_tv_template) @@ -2921,6 +3795,12 @@ static const struct alg_test_desc alg_test_descs[] = { .test = alg_test_null, .fips_allowed = 1, }, { + /* Same as cbc(sm4) except the key is stored in + * hardware secure memory which we reference by index + */ + .alg = "cbc(psm4)", + .test = alg_test_null, + }, { .alg = "cbc(serpent)", .test = alg_test_skcipher, .suite = { @@ -2947,6 +3827,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "ccm(aes)", + .generic_driver = "ccm_base(ctr(aes-generic),cbcmac(aes-generic))", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -3055,6 +3936,13 @@ static const struct alg_test_desc alg_test_descs[] = { .test = alg_test_null, .fips_allowed = 1, }, { + + /* Same as ctr(sm4) except the key is stored in + * hardware secure memory which we reference by index + */ + .alg = "ctr(psm4)", + .test = alg_test_null, + }, { .alg = "ctr(serpent)", .test = alg_test_skcipher, .suite = { @@ -3080,6 +3968,13 @@ static const struct alg_test_desc alg_test_descs[] = { .cipher = __VECS(cts_mode_tv_template) } }, { + /* Same as cts(cbc((aes)) except the key is stored in + * hardware secure memory which we reference by index + */ + .alg = "cts(cbc(paes))", + .test = alg_test_null, + .fips_allowed = 1, + }, { .alg = "deflate", .test = alg_test_comp, .fips_allowed = 1, @@ -3358,7 +4253,14 @@ static const struct alg_test_desc alg_test_descs[] = { .kpp = __VECS(ecdh_tv_template) } }, { + .alg = "ecrdsa", + .test = alg_test_akcipher, + .suite = { + .akcipher = __VECS(ecrdsa_tv_template) + } + }, { .alg = "gcm(aes)", + .generic_driver = "gcm_base(ctr(aes-generic),ghash-generic)", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -3477,30 +4379,35 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "lrw(aes)", + .generic_driver = "lrw(ecb(aes-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(aes_lrw_tv_template) } }, { .alg = "lrw(camellia)", + .generic_driver = "lrw(ecb(camellia-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(camellia_lrw_tv_template) } }, { .alg = "lrw(cast6)", + .generic_driver = "lrw(ecb(cast6-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(cast6_lrw_tv_template) } }, { .alg = "lrw(serpent)", + .generic_driver = "lrw(ecb(serpent-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(serpent_lrw_tv_template) } }, { .alg = "lrw(twofish)", + .generic_driver = "lrw(ecb(twofish-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(tf_lrw_tv_template) @@ -3625,6 +4532,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "rfc4106(gcm(aes))", + .generic_driver = "rfc4106(gcm_base(ctr(aes-generic),ghash-generic))", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -3632,6 +4540,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "rfc4309(ccm(aes))", + .generic_driver = "rfc4309(ccm_base(ctr(aes-generic),cbcmac(aes-generic)))", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -3639,6 +4548,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "rfc4543(gcm(aes))", + .generic_driver = "rfc4543(gcm_base(ctr(aes-generic),ghash-generic))", .test = alg_test_aead, .suite = { .aead = __VECS(aes_gcm_rfc4543_tv_template) @@ -3835,6 +4745,7 @@ static const struct alg_test_desc alg_test_descs[] = { }, }, { .alg = "xts(aes)", + .generic_driver = "xts(ecb(aes-generic))", .test = alg_test_skcipher, .fips_allowed = 1, .suite = { @@ -3842,12 +4753,14 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "xts(camellia)", + .generic_driver = "xts(ecb(camellia-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(camellia_xts_tv_template) } }, { .alg = "xts(cast6)", + .generic_driver = "xts(ecb(cast6-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(cast6_xts_tv_template) @@ -3861,12 +4774,14 @@ static const struct alg_test_desc alg_test_descs[] = { .fips_allowed = 1, }, { .alg = "xts(serpent)", + .generic_driver = "xts(ecb(serpent-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(serpent_xts_tv_template) } }, { .alg = "xts(twofish)", + .generic_driver = "xts(ecb(twofish-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(tf_xts_tv_template) @@ -4020,8 +4935,9 @@ int alg_test(const char *driver, const char *alg, u32 type, u32 mask) type, mask); test_done: - if (fips_enabled && rc) - panic("%s: %s alg self test failed in fips mode!\n", driver, alg); + if (rc && (fips_enabled || panic_on_fail)) + panic("alg: self-tests for %s (%s) failed in %s mode!\n", + driver, alg, fips_enabled ? "fips" : "panic_on_fail"); if (fips_enabled && !rc) pr_info("alg: self-tests for %s (%s) passed\n", driver, alg); diff --git a/crypto/testmgr.h b/crypto/testmgr.h index f267633cf13a..b6daae1f6a1d 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -25,6 +25,8 @@ #ifndef _CRYPTO_TESTMGR_H #define _CRYPTO_TESTMGR_H +#include <linux/oid_registry.h> + #define MAX_IVLEN 32 /* @@ -34,6 +36,8 @@ * @digest: Pointer to expected digest * @psize: Length of source data in bytes * @ksize: Length of @key in bytes (0 if no key) + * @setkey_error: Expected error from setkey() + * @digest_error: Expected error from digest() */ struct hash_testvec { const char *key; @@ -41,6 +45,8 @@ struct hash_testvec { const char *digest; unsigned short psize; unsigned short ksize; + int setkey_error; + int digest_error; }; /* @@ -52,12 +58,13 @@ struct hash_testvec { * @ptext: Pointer to plaintext * @ctext: Pointer to ciphertext * @len: Length of @ptext and @ctext in bytes - * @fail: If set to one, the test need to fail * @wk: Does the test need CRYPTO_TFM_REQ_FORBID_WEAK_KEYS? * ( e.g. test needs to fail due to a weak key ) * @fips_skip: Skip the test vector in FIPS mode * @generates_iv: Encryption should ignore the given IV, and output @iv_out. * Decryption takes @iv_out. Needed for AES Keywrap ("kw(aes)"). + * @setkey_error: Expected error from setkey() + * @crypt_error: Expected error from encrypt() and decrypt() */ struct cipher_testvec { const char *key; @@ -65,12 +72,13 @@ struct cipher_testvec { const char *iv_out; const char *ptext; const char *ctext; - bool fail; unsigned char wk; /* weak key flag */ - unsigned char klen; + unsigned short klen; unsigned short len; bool fips_skip; bool generates_iv; + int setkey_error; + int crypt_error; }; /* @@ -82,7 +90,6 @@ struct cipher_testvec { * @ctext: Pointer to the full authenticated ciphertext. For AEADs that * produce a separate "ciphertext" and "authentication tag", these * two parts are concatenated: ciphertext || tag. - * @fail: setkey() failure expected? * @novrfy: Decryption verification failure expected? * @wk: Does the test need CRYPTO_TFM_REQ_FORBID_WEAK_KEYS? * (e.g. setkey() needs to fail due to a weak key) @@ -90,6 +97,9 @@ struct cipher_testvec { * @plen: Length of @ptext in bytes * @alen: Length of @assoc in bytes * @clen: Length of @ctext in bytes + * @setkey_error: Expected error from setkey() + * @setauthsize_error: Expected error from setauthsize() + * @crypt_error: Expected error from encrypt() and decrypt() */ struct aead_testvec { const char *key; @@ -97,13 +107,15 @@ struct aead_testvec { const char *ptext; const char *assoc; const char *ctext; - bool fail; unsigned char novrfy; unsigned char wk; unsigned char klen; unsigned short plen; unsigned short clen; unsigned short alen; + int setkey_error; + int setauthsize_error; + int crypt_error; }; struct cprng_testvec { @@ -135,13 +147,16 @@ struct drbg_testvec { struct akcipher_testvec { const unsigned char *key; + const unsigned char *params; const unsigned char *m; const unsigned char *c; unsigned int key_len; + unsigned int param_len; unsigned int m_size; unsigned int c_size; bool public_key_vec; bool siggen_sigver_test; + enum OID algo; }; struct kpp_testvec { @@ -551,6 +566,160 @@ static const struct akcipher_testvec rsa_tv_template[] = { }; /* + * EC-RDSA test vectors are generated by gost-engine. + */ +static const struct akcipher_testvec ecrdsa_tv_template[] = { + { + .key = + "\x04\x40\xd5\xa7\x77\xf9\x26\x2f\x8c\xbd\xcc\xe3\x1f\x01\x94\x05" + "\x3d\x2f\xec\xb5\x00\x34\xf5\x51\x6d\x3b\x90\x4b\x23\x28\x6f\x1d" + "\xc8\x36\x61\x60\x36\xec\xbb\xb4\x0b\x95\x4e\x54\x4f\x15\x21\x05" + "\xd8\x52\x66\x44\x31\x7e\x5d\xc5\xd1\x26\x00\x5f\x60\xd8\xf0\xc7" + "\x27\xfc", + .key_len = 66, + .params = /* OID_gostCPSignA */ + "\x30\x13\x06\x07\x2a\x85\x03\x02\x02\x23\x01\x06\x08\x2a\x85\x03" + "\x07\x01\x01\x02\x02", + .param_len = 21, + .c = + "\x41\x32\x09\x73\xa4\xc1\x38\xd6\x63\x7d\x8b\xf7\x50\x3f\xda\x9f" + "\x68\x48\xc1\x50\xe3\x42\x3a\x9b\x2b\x28\x12\x2a\xa7\xc2\x75\x31" + "\x65\x77\x8c\x3c\x9e\x0d\x56\xb2\xf9\xdc\x04\x33\x3e\xb0\x9e\xf9" + "\x74\x4e\x59\xb3\x83\xf2\x91\x27\xda\x5e\xc7\x33\xc0\xc1\x8f\x41", + .c_size = 64, + .algo = OID_gost2012PKey256, + .m = + "\x75\x1b\x9b\x40\x25\xb9\x96\xd2\x9b\x00\x41\xb3\x58\xbf\x23\x14" + "\x79\xd2\x76\x64\xa3\xbd\x66\x10\x79\x05\x5a\x06\x42\xec\xb9\xc9", + .m_size = 32, + .public_key_vec = true, + .siggen_sigver_test = true, + }, + { + .key = + "\x04\x40\x66\x6f\xd6\xb7\x06\xd0\xf5\xa5\x6f\x69\x5c\xa5\x13\x45" + "\x14\xdd\xcb\x12\x9c\x1b\xf5\x28\x64\x7a\x49\x48\x29\x14\x66\x42" + "\xb8\x1b\x5c\xf9\x56\x6d\x08\x3b\xce\xbb\x62\x2f\xc2\x3c\xc5\x49" + "\x93\x27\x70\x20\xcc\x79\xeb\xdc\x76\x8e\x48\x6e\x04\x96\xc3\x29" + "\xa0\x73", + .key_len = 66, + .params = /* OID_gostCPSignB */ + "\x30\x13\x06\x07\x2a\x85\x03\x02\x02\x23\x02\x06\x08\x2a\x85\x03" + "\x07\x01\x01\x02\x02", + .param_len = 21, + .c = + "\x45\x6d\x4a\x03\x1d\x5c\x0b\x17\x79\xe7\x19\xdb\xbf\x81\x9f\x82" + "\xae\x06\xda\xf5\x47\x00\x05\x80\xc3\x16\x06\x9a\x8e\x7c\xb2\x8e" + "\x7f\x74\xaa\xec\x6b\x7b\x7f\x8b\xc6\x0b\x10\x42\x4e\x91\x2c\xdf" + "\x7b\x8b\x15\xf4\x9e\x59\x0f\xc7\xa4\x68\x2e\xce\x89\xdf\x84\xe9", + .c_size = 64, + .algo = OID_gost2012PKey256, + .m = + "\xd0\x54\x00\x27\x6a\xeb\xce\x6c\xf5\xf6\xfb\x57\x18\x18\x21\x13" + "\x11\x23\x4a\x70\x43\x52\x7a\x68\x11\x65\x45\x37\xbb\x25\xb7\x40", + .m_size = 32, + .public_key_vec = true, + .siggen_sigver_test = true, + }, + { + .key = + "\x04\x40\x05\x91\xa9\x7d\xcb\x87\xdc\x98\xa1\xbf\xff\xdd\x20\x61" + "\xaa\x58\x3b\x2d\x8e\x9c\x41\x9d\x4f\xc6\x23\x17\xf9\xca\x60\x65" + "\xbc\x97\x97\xf6\x6b\x24\xe8\xac\xb1\xa7\x61\x29\x3c\x71\xdc\xad" + "\xcb\x20\xbe\x96\xe8\xf4\x44\x2e\x49\xd5\x2c\xb9\xc9\x3b\x9c\xaa" + "\xba\x15", + .key_len = 66, + .params = /* OID_gostCPSignC */ + "\x30\x13\x06\x07\x2a\x85\x03\x02\x02\x23\x03\x06\x08\x2a\x85\x03" + "\x07\x01\x01\x02\x02", + .param_len = 21, + .c = + "\x3b\x2e\x2e\x74\x74\x47\xda\xea\x93\x90\x6a\xe2\xf5\xf5\xe6\x46" + "\x11\xfc\xab\xdc\x52\xbc\x58\xdb\x45\x44\x12\x4a\xf7\xd0\xab\xc9" + "\x73\xba\x64\xab\x0d\xac\x4e\x72\x10\xa8\x04\xf6\x1e\xe0\x48\x6a" + "\xcd\xe8\xe3\x78\x73\x77\x82\x24\x8d\xf1\xd3\xeb\x4c\x25\x7e\xc0", + .c_size = 64, + .algo = OID_gost2012PKey256, + .m = + "\x52\x33\xf4\x3f\x7b\x5d\xcf\x20\xee\xe4\x5c\xab\x0b\x3f\x14\xd6" + "\x9f\x16\xc6\x1c\xb1\x3f\x84\x41\x69\xec\x34\xfd\xf1\xf9\xa3\x39", + .m_size = 32, + .public_key_vec = true, + .siggen_sigver_test = true, + }, + { + .key = + "\x04\x81\x80\x85\x46\x8f\x16\xf8\x7a\x7e\x4a\xc3\x81\x9e\xf1\x6e" + "\x94\x1e\x5d\x02\x87\xea\xfa\xa0\x0a\x17\x70\x49\x64\xad\x95\x68" + "\x60\x0a\xf0\x57\x29\x41\x79\x30\x3c\x61\x69\xf2\xa6\x94\x87\x17" + "\x54\xfa\x97\x2c\xe6\x1e\x0a\xbb\x55\x10\x57\xbe\xf7\xc1\x77\x2b" + "\x11\x74\x0a\x50\x37\x14\x10\x2a\x45\xfc\x7a\xae\x1c\x4c\xce\x08" + "\x05\xb7\xa4\x50\xc8\x3d\x39\x3d\xdc\x5c\x8f\x96\x6c\xe7\xfc\x21" + "\xc3\x2d\x1e\x9f\x11\xb3\xec\x22\x18\x8a\x8c\x08\x6b\x8b\xed\xf5" + "\xc5\x47\x3c\x7e\x73\x59\x44\x1e\x77\x83\x84\x52\x9e\x3b\x7d\xff" + "\x9d\x86\x1a", + .key_len = 131, + .params = /* OID_gostTC26Sign512A */ + "\x30\x0b\x06\x09\x2a\x85\x03\x07\x01\x02\x01\x02\x01", + .param_len = 13, + .c = + "\x92\x81\x74\x5f\x95\x48\x38\x87\xd9\x8f\x5e\xc8\x8a\xbb\x01\x4e" + "\xb0\x75\x3c\x2f\xc7\x5a\x08\x4c\x68\xab\x75\x01\x32\x75\x75\xb5" + "\x37\xe0\x74\x6d\x94\x84\x31\x2a\x6b\xf4\xf7\xb7\xa7\x39\x7b\x46" + "\x07\xf0\x98\xbd\x33\x18\xa1\x72\xb2\x6d\x54\xe3\xde\x91\xc2\x2e" + "\x4f\x6a\xf8\xb7\xec\xa8\x83\xc9\x8f\xd9\xce\x7c\x45\x06\x02\xf4" + "\x4f\x21\xb5\x24\x3d\xb4\xb5\xd8\x58\x42\xbe\x2d\x29\xae\x93\xc0" + "\x13\x41\x96\x35\x08\x69\xe8\x36\xc7\xd1\x83\x81\xd7\xca\xfb\xc0" + "\xd2\xb7\x78\x32\x3e\x30\x1a\x1e\xce\xdc\x34\x35\xc6\xad\x68\x24", + .c_size = 128, + .algo = OID_gost2012PKey512, + .m = + "\x1f\x70\xb5\xe9\x55\x12\xd6\x88\xcc\x55\xb9\x0c\x7f\xc4\x94\xf2" + "\x04\x77\x41\x12\x02\xd6\xf1\x1f\x83\x56\xe9\xd6\x5a\x6a\x72\xb9" + "\x6e\x8e\x24\x2a\x84\xf1\xba\x67\xe8\xbf\xff\xc1\xd3\xde\xfb\xc6" + "\xa8\xf6\x80\x01\xb9\x27\xac\xd8\x45\x96\x66\xa1\xee\x48\x08\x3f", + .m_size = 64, + .public_key_vec = true, + .siggen_sigver_test = true, + }, + { + .key = + "\x04\x81\x80\x28\xf3\x2b\x92\x04\x32\xea\x66\x20\xde\xa0\x2f\x74" + "\xbf\x2d\xf7\xb5\x30\x76\xb1\xc8\xee\x38\x9f\xea\xe5\xad\xc6\xa3" + "\x28\x1e\x51\x3d\x67\xa3\x41\xcc\x6b\x81\xe2\xe2\x9e\x82\xf3\x78" + "\x56\xd7\x2e\xb2\xb5\xbe\xb4\x50\x21\x05\xe5\x29\x82\xef\x15\x1b" + "\xc0\xd7\x30\xd6\x2f\x96\xe8\xff\x99\x4c\x25\xcf\x9a\xfc\x54\x30" + "\xce\xdf\x59\xe9\xc6\x45\xce\xe4\x22\xe8\x01\xd5\xcd\x2f\xaa\x78" + "\x99\xc6\x04\x1e\x6f\x4c\x25\x6a\x76\xad\xff\x48\xf3\xb3\xb4\xd6" + "\x14\x5c\x2c\x0e\xea\xa2\x4b\xb9\x7e\x89\x77\x02\x3a\x29\xc8\x16" + "\x8e\x78\x48", + .key_len = 131, + .params = /* OID_gostTC26Sign512B */ + "\x30\x0b\x06\x09\x2a\x85\x03\x07\x01\x02\x01\x02\x02", + .param_len = 13, + .c = + "\x0a\xed\xb6\x27\xea\xa7\xa6\x7e\x2f\xc1\x02\x21\x74\xce\x27\xd2" + "\xee\x8a\x92\x4d\xa9\x43\x2d\xa4\x5b\xdc\x23\x02\xfc\x3a\xf3\xb2" + "\x10\x93\x0b\x40\x1b\x75\x95\x3e\x39\x41\x37\xb9\xab\x51\x09\xeb" + "\xf1\xb9\x49\x58\xec\x58\xc7\xf9\x2e\xb9\xc9\x40\xf2\x00\x39\x7e" + "\x3f\xde\x72\xe3\x85\x67\x06\xbe\xd8\xb8\xc1\x81\x1e\xe3\x0a\xfe" + "\xce\xd3\x77\x92\x56\x8c\x58\xf9\x37\x60\x2d\xe6\x8b\x66\xa3\xdd" + "\xd2\xf0\xf8\xda\x1b\x20\xbc\x9c\xec\x29\x5d\xd1\x8f\xcc\x37\xd1" + "\x3b\x8d\xb7\xc1\xe0\xb8\x3b\xef\x14\x1b\x87\xbc\xc1\x03\x9a\x93", + .c_size = 128, + .algo = OID_gost2012PKey512, + .m = + "\x11\x24\x21\x27\xf2\x42\x9f\xce\x5a\xf9\x01\x70\xe0\x07\x2b\x57" + "\xfb\x7d\x77\x5e\x74\x66\xe6\xa5\x40\x4c\x1a\x85\x18\xff\xd0\x63" + "\xe0\x39\xd3\xd6\xe5\x17\xf8\xc3\x4b\xc6\x1c\x33\x1a\xca\xa6\x66" + "\x6d\xf4\xd2\x45\xc2\x83\xa0\x42\x95\x05\x9d\x89\x8e\x0a\xca\xcc", + .m_size = 64, + .public_key_vec = true, + .siggen_sigver_test = true, + }, +}; + +/* * PKCS#1 RSA test vectors. Obtained from CAVS testing. */ static const struct akcipher_testvec pkcs1pad_rsa_tv_template[] = { @@ -5634,7 +5803,49 @@ static const struct hash_testvec poly1305_tv_template[] = { .psize = 80, .digest = "\x13\x00\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00\x00\x00\x00\x00", - }, + }, { /* Regression test for overflow in AVX2 implementation */ + .plaintext = "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff", + .psize = 300, + .digest = "\xfb\x5e\x96\xd8\x61\xd5\xc7\xc8" + "\x78\xe5\x87\xcc\x2d\x5a\x22\xe1", + } }; /* NHPoly1305 test vectors from https://github.com/google/adiantum */ @@ -7042,7 +7253,7 @@ static const struct cipher_testvec des_tv_template[] = { "\xb4\x99\x26\xf7\x1f\xe1\xd4\x90", .len = 24, }, { /* Weak key */ - .fail = true, + .setkey_error = -EINVAL, .wk = 1, .key = "\x01\x01\x01\x01\x01\x01\x01\x01", .klen = 8, diff --git a/crypto/tgr192.c b/crypto/tgr192.c index f8e1d9f9938f..40020f8adc46 100644 --- a/crypto/tgr192.c +++ b/crypto/tgr192.c @@ -677,7 +677,7 @@ MODULE_ALIAS_CRYPTO("tgr192"); MODULE_ALIAS_CRYPTO("tgr160"); MODULE_ALIAS_CRYPTO("tgr128"); -module_init(tgr192_mod_init); +subsys_initcall(tgr192_mod_init); module_exit(tgr192_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/twofish_generic.c b/crypto/twofish_generic.c index 07e62433fbfb..dbac6e233285 100644 --- a/crypto/twofish_generic.c +++ b/crypto/twofish_generic.c @@ -205,7 +205,7 @@ static void __exit twofish_mod_fini(void) crypto_unregister_alg(&alg); } -module_init(twofish_mod_init); +subsys_initcall(twofish_mod_init); module_exit(twofish_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/vmac.c b/crypto/vmac.c index 5f436dfdfc61..f50a85060b39 100644 --- a/crypto/vmac.c +++ b/crypto/vmac.c @@ -690,7 +690,7 @@ static void __exit vmac_module_exit(void) crypto_unregister_template(&vmac64_tmpl); } -module_init(vmac_module_init); +subsys_initcall(vmac_module_init); module_exit(vmac_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/wp512.c b/crypto/wp512.c index 149e577fb772..1b8e502d999f 100644 --- a/crypto/wp512.c +++ b/crypto/wp512.c @@ -1168,7 +1168,7 @@ MODULE_ALIAS_CRYPTO("wp512"); MODULE_ALIAS_CRYPTO("wp384"); MODULE_ALIAS_CRYPTO("wp256"); -module_init(wp512_mod_init); +subsys_initcall(wp512_mod_init); module_exit(wp512_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/xcbc.c b/crypto/xcbc.c index c055f57fab11..94ca694ef091 100644 --- a/crypto/xcbc.c +++ b/crypto/xcbc.c @@ -282,7 +282,7 @@ static void __exit crypto_xcbc_module_exit(void) crypto_unregister_template(&crypto_xcbc_tmpl); } -module_init(crypto_xcbc_module_init); +subsys_initcall(crypto_xcbc_module_init); module_exit(crypto_xcbc_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/xts.c b/crypto/xts.c index 847f54f76789..33cf726df4ac 100644 --- a/crypto/xts.c +++ b/crypto/xts.c @@ -137,8 +137,12 @@ static void crypt_done(struct crypto_async_request *areq, int err) { struct skcipher_request *req = areq->data; - if (!err) + if (!err) { + struct rctx *rctx = skcipher_request_ctx(req); + + rctx->subreq.base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; err = xor_tweak_post(req); + } skcipher_request_complete(req, err); } @@ -359,7 +363,7 @@ static void __exit crypto_module_exit(void) crypto_unregister_template(&crypto_tmpl); } -module_init(crypto_module_init); +subsys_initcall(crypto_module_init); module_exit(crypto_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/zstd.c b/crypto/zstd.c index 9a76b3ed8b8b..2c04055e407f 100644 --- a/crypto/zstd.c +++ b/crypto/zstd.c @@ -257,7 +257,7 @@ static void __exit zstd_mod_fini(void) crypto_unregister_scomp(&scomp); } -module_init(zstd_mod_init); +subsys_initcall(zstd_mod_init); module_exit(zstd_mod_fini); MODULE_LICENSE("GPL"); |