From 793ff5ffc105a5962ab4c978d08988e3f8dc93aa Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 10 Jan 2019 12:17:57 -0800 Subject: crypto: x86/aesni-gcm - make 'struct aesni_gcm_tfm_s' static const Add missing static keywords to fix the following sparse warnings: arch/x86/crypto/aesni-intel_glue.c:197:24: warning: symbol 'aesni_gcm_tfm_sse' was not declared. Should it be static? arch/x86/crypto/aesni-intel_glue.c:246:24: warning: symbol 'aesni_gcm_tfm_avx_gen2' was not declared. Should it be static? arch/x86/crypto/aesni-intel_glue.c:291:24: warning: symbol 'aesni_gcm_tfm_avx_gen4' was not declared. Should it be static? I also made the affected structures 'const', and adjusted the indentation in the struct definition to not be insane. Cc: Dave Watson Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/aesni-intel_glue.c | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) (limited to 'arch') diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index 1321700d6647..9b5ccde3ef31 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -175,26 +175,18 @@ asmlinkage void aesni_gcm_finalize(void *ctx, struct gcm_context_data *gdata, u8 *auth_tag, unsigned long auth_tag_len); -static struct aesni_gcm_tfm_s { -void (*init)(void *ctx, - struct gcm_context_data *gdata, - u8 *iv, - u8 *hash_subkey, const u8 *aad, - unsigned long aad_len); -void (*enc_update)(void *ctx, - struct gcm_context_data *gdata, u8 *out, - const u8 *in, - unsigned long plaintext_len); -void (*dec_update)(void *ctx, - struct gcm_context_data *gdata, u8 *out, - const u8 *in, - unsigned long ciphertext_len); -void (*finalize)(void *ctx, - struct gcm_context_data *gdata, - u8 *auth_tag, unsigned long auth_tag_len); +static const struct aesni_gcm_tfm_s { + void (*init)(void *ctx, struct gcm_context_data *gdata, u8 *iv, + u8 *hash_subkey, const u8 *aad, unsigned long aad_len); + void (*enc_update)(void *ctx, struct gcm_context_data *gdata, u8 *out, + const u8 *in, unsigned long plaintext_len); + void (*dec_update)(void *ctx, struct gcm_context_data *gdata, u8 *out, + const u8 *in, unsigned long ciphertext_len); + void (*finalize)(void *ctx, struct gcm_context_data *gdata, + u8 *auth_tag, unsigned long auth_tag_len); } *aesni_gcm_tfm; -struct aesni_gcm_tfm_s aesni_gcm_tfm_sse = { +static const struct aesni_gcm_tfm_s aesni_gcm_tfm_sse = { .init = &aesni_gcm_init, .enc_update = &aesni_gcm_enc_update, .dec_update = &aesni_gcm_dec_update, @@ -243,7 +235,7 @@ asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx, const u8 *aad, unsigned long aad_len, u8 *auth_tag, unsigned long auth_tag_len); -struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen2 = { +static const struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen2 = { .init = &aesni_gcm_init_avx_gen2, .enc_update = &aesni_gcm_enc_update_avx_gen2, .dec_update = &aesni_gcm_dec_update_avx_gen2, @@ -288,7 +280,7 @@ asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx, const u8 *aad, unsigned long aad_len, u8 *auth_tag, unsigned long auth_tag_len); -struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen4 = { +static const struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen4 = { .init = &aesni_gcm_init_avx_gen4, .enc_update = &aesni_gcm_enc_update_avx_gen4, .dec_update = &aesni_gcm_dec_update_avx_gen4, @@ -778,7 +770,7 @@ static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req, { struct crypto_aead *tfm = crypto_aead_reqtfm(req); unsigned long auth_tag_len = crypto_aead_authsize(tfm); - struct aesni_gcm_tfm_s *gcm_tfm = aesni_gcm_tfm; + const struct aesni_gcm_tfm_s *gcm_tfm = aesni_gcm_tfm; struct gcm_context_data data AESNI_ALIGN_ATTR; struct scatter_walk dst_sg_walk = {}; unsigned long left = req->cryptlen; -- cgit v1.2.3 From 231baecdef7a906579925ccf1bd45aa734f32320 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 18 Jan 2019 22:48:00 -0800 Subject: crypto: clarify name of WEAK_KEY request flag CRYPTO_TFM_REQ_WEAK_KEY confuses newcomers to the crypto API because it sounds like it is requesting a weak key. Actually, it is requesting that weak keys be forbidden (for algorithms that have the notion of "weak keys"; currently only DES and XTS do). Also it is only one letter away from CRYPTO_TFM_RES_WEAK_KEY, with which it can be easily confused. (This in fact happened in the UX500 driver, though just in some debugging messages.) Therefore, make the intent clear by renaming it to CRYPTO_TFM_REQ_FORBID_WEAK_KEYS. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/s390/crypto/des_s390.c | 4 ++-- arch/sparc/crypto/des_glue.c | 4 ++-- crypto/des_generic.c | 4 ++-- crypto/testmgr.c | 14 +++++++------- crypto/testmgr.h | 4 ++-- drivers/crypto/atmel-tdes.c | 2 +- drivers/crypto/bcm/cipher.c | 4 ++-- drivers/crypto/ccp/ccp-crypto-des3.c | 2 +- drivers/crypto/ccree/cc_cipher.c | 3 ++- drivers/crypto/hifn_795x.c | 3 ++- drivers/crypto/inside-secure/safexcel_cipher.c | 2 +- drivers/crypto/ixp4xx_crypto.c | 4 ++-- drivers/crypto/marvell/cipher.c | 2 +- drivers/crypto/n2_core.c | 2 +- drivers/crypto/omap-des.c | 2 +- drivers/crypto/picoxcell_crypto.c | 3 ++- drivers/crypto/qce/ablkcipher.c | 4 ++-- drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c | 2 +- drivers/crypto/sunxi-ss/sun4i-ss-cipher.c | 2 +- drivers/crypto/talitos.c | 2 +- drivers/crypto/ux500/cryp/cryp_core.c | 20 +++++++++++--------- fs/crypto/keyinfo.c | 4 ++-- fs/ecryptfs/crypto.c | 5 +++-- include/crypto/xts.h | 4 ++-- include/linux/crypto.h | 2 +- 25 files changed, 55 insertions(+), 49 deletions(-) (limited to 'arch') diff --git a/arch/s390/crypto/des_s390.c b/arch/s390/crypto/des_s390.c index 5346b5a80bb6..0d15383d0ff1 100644 --- a/arch/s390/crypto/des_s390.c +++ b/arch/s390/crypto/des_s390.c @@ -38,7 +38,7 @@ static int des_setkey(struct crypto_tfm *tfm, const u8 *key, /* check for weak keys */ if (!des_ekey(tmp, key) && - (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + (tfm->crt_flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; return -EINVAL; } @@ -228,7 +228,7 @@ static int des3_setkey(struct crypto_tfm *tfm, const u8 *key, if (!(crypto_memneq(key, &key[DES_KEY_SIZE], DES_KEY_SIZE) && crypto_memneq(&key[DES_KEY_SIZE], &key[DES_KEY_SIZE * 2], DES_KEY_SIZE)) && - (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + (tfm->crt_flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; return -EINVAL; } diff --git a/arch/sparc/crypto/des_glue.c b/arch/sparc/crypto/des_glue.c index 56499ea39fd3..4884315daff4 100644 --- a/arch/sparc/crypto/des_glue.c +++ b/arch/sparc/crypto/des_glue.c @@ -53,7 +53,7 @@ static int des_set_key(struct crypto_tfm *tfm, const u8 *key, * weak key detection code. */ ret = des_ekey(tmp, key); - if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { *flags |= CRYPTO_TFM_RES_WEAK_KEY; return -EINVAL; } @@ -209,7 +209,7 @@ static int des3_ede_set_key(struct crypto_tfm *tfm, const u8 *key, if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) || !((K[2] ^ K[4]) | (K[3] ^ K[5]))) && - (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + (*flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { *flags |= CRYPTO_TFM_RES_WEAK_KEY; return -EINVAL; } diff --git a/crypto/des_generic.c b/crypto/des_generic.c index a71720544d11..1e6621665dd9 100644 --- a/crypto/des_generic.c +++ b/crypto/des_generic.c @@ -789,7 +789,7 @@ static int des_setkey(struct crypto_tfm *tfm, const u8 *key, /* Expand to tmp */ ret = des_ekey(tmp, key); - if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { *flags |= CRYPTO_TFM_RES_WEAK_KEY; return -EINVAL; } @@ -866,7 +866,7 @@ int __des3_ede_setkey(u32 *expkey, u32 *flags, const u8 *key, if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) || !((K[2] ^ K[4]) | (K[3] ^ K[5]))) && - (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + (*flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { *flags |= CRYPTO_TFM_RES_WEAK_KEY; return -EINVAL; } diff --git a/crypto/testmgr.c b/crypto/testmgr.c index e4f3f5f688e7..4ac3d22256c3 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -706,7 +706,8 @@ static int __test_aead(struct crypto_aead *tfm, int enc, crypto_aead_clear_flags(tfm, ~0); if (template[i].wk) - crypto_aead_set_flags(tfm, CRYPTO_TFM_REQ_WEAK_KEY); + crypto_aead_set_flags(tfm, + CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); if (template[i].klen > MAX_KEYLEN) { pr_err("alg: aead%s: setkey failed on test %d for %s: key size %d > %d\n", @@ -820,7 +821,8 @@ static int __test_aead(struct crypto_aead *tfm, int enc, crypto_aead_clear_flags(tfm, ~0); if (template[i].wk) - crypto_aead_set_flags(tfm, CRYPTO_TFM_REQ_WEAK_KEY); + crypto_aead_set_flags(tfm, + CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); if (template[i].klen > MAX_KEYLEN) { pr_err("alg: aead%s: setkey failed on test %d for %s: key size %d > %d\n", d, j, algo, template[i].klen, MAX_KEYLEN); @@ -1078,7 +1080,7 @@ static int test_cipher(struct crypto_cipher *tfm, int enc, crypto_cipher_clear_flags(tfm, ~0); if (template[i].wk) - crypto_cipher_set_flags(tfm, CRYPTO_TFM_REQ_WEAK_KEY); + crypto_cipher_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); ret = crypto_cipher_setkey(tfm, template[i].key, template[i].klen); @@ -1194,8 +1196,7 @@ static int __test_skcipher(struct crypto_skcipher *tfm, int enc, crypto_skcipher_clear_flags(tfm, ~0); if (template[i].wk) - crypto_skcipher_set_flags(tfm, - CRYPTO_TFM_REQ_WEAK_KEY); + crypto_skcipher_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); ret = crypto_skcipher_setkey(tfm, template[i].key, template[i].klen); @@ -1265,8 +1266,7 @@ static int __test_skcipher(struct crypto_skcipher *tfm, int enc, j++; crypto_skcipher_clear_flags(tfm, ~0); if (template[i].wk) - crypto_skcipher_set_flags(tfm, - CRYPTO_TFM_REQ_WEAK_KEY); + crypto_skcipher_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); ret = crypto_skcipher_setkey(tfm, template[i].key, template[i].klen); diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 95297240b0f1..d8f6035c7ff2 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -50,7 +50,7 @@ struct hash_testvec { * @ctext: Pointer to ciphertext * @len: Length of @ptext and @ctext in bytes * @fail: If set to one, the test need to fail - * @wk: Does the test need CRYPTO_TFM_REQ_WEAK_KEY + * @wk: Does the test need CRYPTO_TFM_REQ_FORBID_WEAK_KEYS? * ( e.g. test needs to fail due to a weak key ) * @np: numbers of SG to distribute data in (from 1 to MAX_TAP) * @tap: How to distribute data in @np SGs @@ -91,7 +91,7 @@ struct cipher_testvec { * @anp: Numbers of SG to distribute assoc data in * @fail: setkey() failure expected? * @novrfy: Decryption verification failure expected? - * @wk: Does the test need CRYPTO_TFM_REQ_WEAK_KEY? + * @wk: Does the test need CRYPTO_TFM_REQ_FORBID_WEAK_KEYS? * (e.g. setkey() needs to fail due to a weak key) * @klen: Length of @key in bytes * @plen: Length of @ptext in bytes diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c index 438e1ffb2ec0..65bf1a299562 100644 --- a/drivers/crypto/atmel-tdes.c +++ b/drivers/crypto/atmel-tdes.c @@ -785,7 +785,7 @@ static int atmel_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key, } err = des_ekey(tmp, key); - if (err == 0 && (ctfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + if (err == 0 && (ctfm->crt_flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { ctfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; return -EINVAL; } diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c index 2099d7bcfd44..28f592f7e1b7 100644 --- a/drivers/crypto/bcm/cipher.c +++ b/drivers/crypto/bcm/cipher.c @@ -1818,7 +1818,7 @@ static int des_setkey(struct crypto_ablkcipher *cipher, const u8 *key, if (keylen == DES_KEY_SIZE) { if (des_ekey(tmp, key) == 0) { if (crypto_ablkcipher_get_flags(cipher) & - CRYPTO_TFM_REQ_WEAK_KEY) { + CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) { u32 flags = CRYPTO_TFM_RES_WEAK_KEY; crypto_ablkcipher_set_flags(cipher, flags); @@ -2872,7 +2872,7 @@ static int aead_authenc_setkey(struct crypto_aead *cipher, if (des_ekey(tmp, keys.enckey) == 0) { if (crypto_aead_get_flags(cipher) & - CRYPTO_TFM_REQ_WEAK_KEY) { + CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) { crypto_aead_set_flags(cipher, flags); return -EINVAL; } diff --git a/drivers/crypto/ccp/ccp-crypto-des3.c b/drivers/crypto/ccp/ccp-crypto-des3.c index ae87b741f9d5..c2ff551d215b 100644 --- a/drivers/crypto/ccp/ccp-crypto-des3.c +++ b/drivers/crypto/ccp/ccp-crypto-des3.c @@ -57,7 +57,7 @@ static int ccp_des3_setkey(struct crypto_ablkcipher *tfm, const u8 *key, if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) || !((K[2] ^ K[4]) | (K[3] ^ K[5]))) && - (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + (*flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { *flags |= CRYPTO_TFM_RES_WEAK_KEY; return -EINVAL; } diff --git a/drivers/crypto/ccree/cc_cipher.c b/drivers/crypto/ccree/cc_cipher.c index e202d7c7ea00..5e3361a363b5 100644 --- a/drivers/crypto/ccree/cc_cipher.c +++ b/drivers/crypto/ccree/cc_cipher.c @@ -352,7 +352,8 @@ static int cc_cipher_setkey(struct crypto_skcipher *sktfm, const u8 *key, dev_dbg(dev, "weak 3DES key"); return -EINVAL; } else if (!des_ekey(tmp, key) && - (crypto_tfm_get_flags(tfm) & CRYPTO_TFM_REQ_WEAK_KEY)) { + (crypto_tfm_get_flags(tfm) & + CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; dev_dbg(dev, "weak DES key"); return -EINVAL; diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c index a5a36fe7bf2c..dad212cabe63 100644 --- a/drivers/crypto/hifn_795x.c +++ b/drivers/crypto/hifn_795x.c @@ -1961,7 +1961,8 @@ static int hifn_setkey(struct crypto_ablkcipher *cipher, const u8 *key, u32 tmp[DES_EXPKEY_WORDS]; int ret = des_ekey(tmp, key); - if (unlikely(ret == 0) && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + if (unlikely(ret == 0) && + (tfm->crt_flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; return -EINVAL; } diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index d531c14020dc..7ef30a98cb24 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -940,7 +940,7 @@ static int safexcel_des_setkey(struct crypto_skcipher *ctfm, const u8 *key, } ret = des_ekey(tmp, key); - if (!ret && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + if (!ret && (tfm->crt_flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; return -EINVAL; } diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c index 19fba998b86b..95c1af227bd5 100644 --- a/drivers/crypto/ixp4xx_crypto.c +++ b/drivers/crypto/ixp4xx_crypto.c @@ -847,7 +847,7 @@ static int ablk_setkey(struct crypto_ablkcipher *tfm, const u8 *key, goto out; if (*flags & CRYPTO_TFM_RES_WEAK_KEY) { - if (*flags & CRYPTO_TFM_REQ_WEAK_KEY) { + if (*flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) { ret = -EINVAL; } else { *flags &= ~CRYPTO_TFM_RES_WEAK_KEY; @@ -1125,7 +1125,7 @@ static int aead_setup(struct crypto_aead *tfm, unsigned int authsize) goto out; if (*flags & CRYPTO_TFM_RES_WEAK_KEY) { - if (*flags & CRYPTO_TFM_REQ_WEAK_KEY) { + if (*flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) { ret = -EINVAL; goto out; } else { diff --git a/drivers/crypto/marvell/cipher.c b/drivers/crypto/marvell/cipher.c index 0ae84ec9e21c..066830dcc53e 100644 --- a/drivers/crypto/marvell/cipher.c +++ b/drivers/crypto/marvell/cipher.c @@ -286,7 +286,7 @@ static int mv_cesa_des_setkey(struct crypto_skcipher *cipher, const u8 *key, } ret = des_ekey(tmp, key); - if (!ret && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + if (!ret && (tfm->crt_flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; return -EINVAL; } diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c index 55f34cfc43ff..9450c41211b2 100644 --- a/drivers/crypto/n2_core.c +++ b/drivers/crypto/n2_core.c @@ -772,7 +772,7 @@ static int n2_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key, } err = des_ekey(tmp, key); - if (err == 0 && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + if (err == 0 && (tfm->crt_flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; return -EINVAL; } diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c index 6369019219d4..1ba2633e90d6 100644 --- a/drivers/crypto/omap-des.c +++ b/drivers/crypto/omap-des.c @@ -662,7 +662,7 @@ static int omap_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key, pr_debug("enter, keylen: %d\n", keylen); /* Do we need to test against weak key? */ - if (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY) { + if (tfm->crt_flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) { u32 tmp[DES_EXPKEY_WORDS]; int ret = des_ekey(tmp, key); diff --git a/drivers/crypto/picoxcell_crypto.c b/drivers/crypto/picoxcell_crypto.c index 17068b55fea5..1b3acdeffede 100644 --- a/drivers/crypto/picoxcell_crypto.c +++ b/drivers/crypto/picoxcell_crypto.c @@ -759,7 +759,8 @@ static int spacc_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key, } if (unlikely(!des_ekey(tmp, key)) && - (crypto_ablkcipher_get_flags(cipher) & CRYPTO_TFM_REQ_WEAK_KEY)) { + (crypto_ablkcipher_get_flags(cipher) & + CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; return -EINVAL; } diff --git a/drivers/crypto/qce/ablkcipher.c b/drivers/crypto/qce/ablkcipher.c index 25c13e26d012..154b6baa124e 100644 --- a/drivers/crypto/qce/ablkcipher.c +++ b/drivers/crypto/qce/ablkcipher.c @@ -180,8 +180,8 @@ static int qce_ablkcipher_setkey(struct crypto_ablkcipher *ablk, const u8 *key, u32 tmp[DES_EXPKEY_WORDS]; ret = des_ekey(tmp, key); - if (!ret && crypto_ablkcipher_get_flags(ablk) & - CRYPTO_TFM_REQ_WEAK_KEY) + if (!ret && (crypto_ablkcipher_get_flags(ablk) & + CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) goto weakkey; } diff --git a/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c b/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c index 639c15c5364b..87dd571466c1 100644 --- a/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c +++ b/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c @@ -60,7 +60,7 @@ static int rk_tdes_setkey(struct crypto_ablkcipher *cipher, if (keylen == DES_KEY_SIZE) { if (!des_ekey(tmp, key) && - (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + (tfm->crt_flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; return -EINVAL; } diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c index 5cf64746731a..54fd714d53ca 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c +++ b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c @@ -517,7 +517,7 @@ int sun4i_ss_des_setkey(struct crypto_skcipher *tfm, const u8 *key, flags = crypto_skcipher_get_flags(tfm); ret = des_ekey(tmp, key); - if (unlikely(!ret) && (flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + if (unlikely(!ret) && (flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_WEAK_KEY); dev_dbg(ss->dev, "Weak key %u\n", keylen); return -EINVAL; diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index f8e2c5c3f4eb..de78b54bcfb1 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1535,7 +1535,7 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *cipher, } if (unlikely(crypto_ablkcipher_get_flags(cipher) & - CRYPTO_TFM_REQ_WEAK_KEY) && + CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) && !des_ekey(tmp, key)) { crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_WEAK_KEY); return -EINVAL; diff --git a/drivers/crypto/ux500/cryp/cryp_core.c b/drivers/crypto/ux500/cryp/cryp_core.c index db94f89d8d11..3235611928f2 100644 --- a/drivers/crypto/ux500/cryp/cryp_core.c +++ b/drivers/crypto/ux500/cryp/cryp_core.c @@ -1000,10 +1000,11 @@ static int des_ablkcipher_setkey(struct crypto_ablkcipher *cipher, } ret = des_ekey(tmp, key); - if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + if (unlikely(ret == 0) && + (*flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { *flags |= CRYPTO_TFM_RES_WEAK_KEY; - pr_debug(DEV_DBG_NAME " [%s]: CRYPTO_TFM_REQ_WEAK_KEY", - __func__); + pr_debug(DEV_DBG_NAME " [%s]: CRYPTO_TFM_RES_WEAK_KEY", + __func__); return -EINVAL; } @@ -1034,18 +1035,19 @@ static int des3_ablkcipher_setkey(struct crypto_ablkcipher *cipher, /* Checking key interdependency for weak key detection. */ if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) || !((K[2] ^ K[4]) | (K[3] ^ K[5]))) && - (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + (*flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { *flags |= CRYPTO_TFM_RES_WEAK_KEY; - pr_debug(DEV_DBG_NAME " [%s]: CRYPTO_TFM_REQ_WEAK_KEY", - __func__); + pr_debug(DEV_DBG_NAME " [%s]: CRYPTO_TFM_RES_WEAK_KEY", + __func__); return -EINVAL; } for (i = 0; i < 3; i++) { ret = des_ekey(tmp, key + i*DES_KEY_SIZE); - if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + if (unlikely(ret == 0) && + (*flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { *flags |= CRYPTO_TFM_RES_WEAK_KEY; - pr_debug(DEV_DBG_NAME " [%s]: " - "CRYPTO_TFM_REQ_WEAK_KEY", __func__); + pr_debug(DEV_DBG_NAME " [%s]: CRYPTO_TFM_RES_WEAK_KEY", + __func__); return -EINVAL; } } diff --git a/fs/crypto/keyinfo.c b/fs/crypto/keyinfo.c index 1e11a683f63d..322ce9686bdb 100644 --- a/fs/crypto/keyinfo.c +++ b/fs/crypto/keyinfo.c @@ -47,7 +47,7 @@ static int derive_key_aes(const u8 *master_key, tfm = NULL; goto out; } - crypto_skcipher_set_flags(tfm, CRYPTO_TFM_REQ_WEAK_KEY); + crypto_skcipher_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); req = skcipher_request_alloc(tfm, GFP_NOFS); if (!req) { res = -ENOMEM; @@ -257,7 +257,7 @@ allocate_skcipher_for_mode(struct fscrypt_mode *mode, const u8 *raw_key, mode->friendly_name, crypto_skcipher_alg(tfm)->base.cra_driver_name); } - crypto_skcipher_set_flags(tfm, CRYPTO_TFM_REQ_WEAK_KEY); + crypto_skcipher_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); err = crypto_skcipher_setkey(tfm, raw_key, mode->keysize); if (err) goto err_free_tfm; diff --git a/fs/ecryptfs/crypto.c b/fs/ecryptfs/crypto.c index 4dd842f72846..f664da55234e 100644 --- a/fs/ecryptfs/crypto.c +++ b/fs/ecryptfs/crypto.c @@ -610,7 +610,8 @@ int ecryptfs_init_crypt_ctx(struct ecryptfs_crypt_stat *crypt_stat) full_alg_name); goto out_free; } - crypto_skcipher_set_flags(crypt_stat->tfm, CRYPTO_TFM_REQ_WEAK_KEY); + crypto_skcipher_set_flags(crypt_stat->tfm, + CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); rc = 0; out_free: kfree(full_alg_name); @@ -1590,7 +1591,7 @@ ecryptfs_process_key_cipher(struct crypto_skcipher **key_tfm, "[%s]; rc = [%d]\n", full_alg_name, rc); goto out; } - crypto_skcipher_set_flags(*key_tfm, CRYPTO_TFM_REQ_WEAK_KEY); + crypto_skcipher_set_flags(*key_tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); if (*key_size == 0) *key_size = crypto_skcipher_default_keysize(*key_tfm); get_random_bytes(dummy_key, *key_size); diff --git a/include/crypto/xts.h b/include/crypto/xts.h index 34d94c95445a..75fd96ff976b 100644 --- a/include/crypto/xts.h +++ b/include/crypto/xts.h @@ -47,8 +47,8 @@ static inline int xts_verify_key(struct crypto_skcipher *tfm, } /* ensure that the AES and tweak key are not identical */ - if ((fips_enabled || crypto_skcipher_get_flags(tfm) & - CRYPTO_TFM_REQ_WEAK_KEY) && + if ((fips_enabled || (crypto_skcipher_get_flags(tfm) & + CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) && !crypto_memneq(key, key + (keylen / 2), keylen / 2)) { crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_WEAK_KEY); return -EINVAL; diff --git a/include/linux/crypto.h b/include/linux/crypto.h index c3c98a62e503..f2565a103158 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -118,7 +118,7 @@ #define CRYPTO_TFM_REQ_MASK 0x000fff00 #define CRYPTO_TFM_RES_MASK 0xfff00000 -#define CRYPTO_TFM_REQ_WEAK_KEY 0x00000100 +#define CRYPTO_TFM_REQ_FORBID_WEAK_KEYS 0x00000100 #define CRYPTO_TFM_REQ_MAY_SLEEP 0x00000200 #define CRYPTO_TFM_REQ_MAY_BACKLOG 0x00000400 #define CRYPTO_TFM_RES_WEAK_KEY 0x00100000 -- cgit v1.2.3 From eaf46edf6ea89675bd36245369c8de5063a0272c Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 24 Jan 2019 17:33:45 +0100 Subject: crypto: arm64/aes-ccm - fix logical bug in AAD MAC handling The NEON MAC calculation routine fails to handle the case correctly where there is some data in the buffer, and the input fills it up exactly. In this case, we enter the loop at the end with w8 == 0, while a negative value is assumed, and so the loop carries on until the increment of the 32-bit counter wraps around, which is quite obviously wrong. So omit the loop altogether in this case, and exit right away. Reported-by: Eric Biggers Fixes: a3fd82105b9d1 ("arm64/crypto: AES in CCM mode using ARMv8 Crypto ...") Cc: stable@vger.kernel.org Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/aes-ce-ccm-core.S | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'arch') diff --git a/arch/arm64/crypto/aes-ce-ccm-core.S b/arch/arm64/crypto/aes-ce-ccm-core.S index e3a375c4cb83..1b151442dac1 100644 --- a/arch/arm64/crypto/aes-ce-ccm-core.S +++ b/arch/arm64/crypto/aes-ce-ccm-core.S @@ -74,12 +74,13 @@ ENTRY(ce_aes_ccm_auth_data) beq 10f ext v0.16b, v0.16b, v0.16b, #1 /* rotate out the mac bytes */ b 7b -8: mov w7, w8 +8: cbz w8, 91f + mov w7, w8 add w8, w8, #16 9: ext v1.16b, v1.16b, v1.16b, #1 adds w7, w7, #1 bne 9b - eor v0.16b, v0.16b, v1.16b +91: eor v0.16b, v0.16b, v1.16b st1 {v0.16b}, [x0] 10: str w8, [x3] ret -- cgit v1.2.3 From 969e2f59d589c15f6aaf306e590dde16f12ea4b3 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 24 Jan 2019 17:33:46 +0100 Subject: crypto: arm64/aes-ccm - fix bugs in non-NEON fallback routine Commit 5092fcf34908 ("crypto: arm64/aes-ce-ccm: add non-SIMD generic fallback") introduced C fallback code to replace the NEON routines when invoked from a context where the NEON is not available (i.e., from the context of a softirq taken while the NEON is already being used in kernel process context) Fix two logical flaws in the MAC calculation of the associated data. Reported-by: Eric Biggers Fixes: 5092fcf34908 ("crypto: arm64/aes-ce-ccm: add non-SIMD generic fallback") Cc: stable@vger.kernel.org Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/aes-ce-ccm-glue.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'arch') diff --git a/arch/arm64/crypto/aes-ce-ccm-glue.c b/arch/arm64/crypto/aes-ce-ccm-glue.c index 68b11aa690e4..986191e8c058 100644 --- a/arch/arm64/crypto/aes-ce-ccm-glue.c +++ b/arch/arm64/crypto/aes-ce-ccm-glue.c @@ -125,7 +125,7 @@ static void ccm_update_mac(struct crypto_aes_ctx *key, u8 mac[], u8 const in[], abytes -= added; } - while (abytes > AES_BLOCK_SIZE) { + while (abytes >= AES_BLOCK_SIZE) { __aes_arm64_encrypt(key->key_enc, mac, mac, num_rounds(key)); crypto_xor(mac, in, AES_BLOCK_SIZE); @@ -139,8 +139,6 @@ static void ccm_update_mac(struct crypto_aes_ctx *key, u8 mac[], u8 const in[], num_rounds(key)); crypto_xor(mac, in, abytes); *macp = abytes; - } else { - *macp = 0; } } } -- cgit v1.2.3 From f9352900e064a159942fbf92d3840d6c8eafea56 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 24 Jan 2019 17:33:47 +0100 Subject: crypto: arm64/aes-ccm - don't use an atomic walk needlessly When the AES-CCM code was first added, the NEON register were saved and restored eagerly, and so the code avoided doing so, and executed the scatterwalk in atomic context inside the kernel_neon_begin/end section. This has been changed in the meantime, so switch to non-atomic scatterwalks. Fixes: bd2ad885e30d ("crypto: arm64/aes-ce-ccm - move kernel mode neon ...") Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/aes-ce-ccm-glue.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'arch') diff --git a/arch/arm64/crypto/aes-ce-ccm-glue.c b/arch/arm64/crypto/aes-ce-ccm-glue.c index 986191e8c058..5fc6f51908fd 100644 --- a/arch/arm64/crypto/aes-ce-ccm-glue.c +++ b/arch/arm64/crypto/aes-ce-ccm-glue.c @@ -253,7 +253,7 @@ static int ccm_encrypt(struct aead_request *req) /* preserve the original iv for the final round */ memcpy(buf, req->iv, AES_BLOCK_SIZE); - err = skcipher_walk_aead_encrypt(&walk, req, true); + err = skcipher_walk_aead_encrypt(&walk, req, false); if (may_use_simd()) { while (walk.nbytes) { @@ -311,7 +311,7 @@ static int ccm_decrypt(struct aead_request *req) /* preserve the original iv for the final round */ memcpy(buf, req->iv, AES_BLOCK_SIZE); - err = skcipher_walk_aead_decrypt(&walk, req, true); + err = skcipher_walk_aead_decrypt(&walk, req, false); if (may_use_simd()) { while (walk.nbytes) { -- cgit v1.2.3 From 5a22b198cd5274470d637d45f0e3bcdc0175a349 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 25 Jan 2019 10:36:27 +0100 Subject: crypto: arm64/ghash - register PMULL variants as separate algos The arm64 GHASH implementation either uses 8-bit or 64-bit polynomial multiplication instructions, since the latter are faster but not mandatory in the architecture. Since that prevents us from testing both implementations on the same system, let's expose both implementations to the crypto API, with the priorities reflecting that the P64 version is the preferred one if available. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/ghash-ce-glue.c | 118 +++++++++++++++++++++++++++++--------- 1 file changed, 90 insertions(+), 28 deletions(-) (limited to 'arch') diff --git a/arch/arm64/crypto/ghash-ce-glue.c b/arch/arm64/crypto/ghash-ce-glue.c index 067d8937d5af..791ad422c427 100644 --- a/arch/arm64/crypto/ghash-ce-glue.c +++ b/arch/arm64/crypto/ghash-ce-glue.c @@ -60,10 +60,6 @@ asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src, struct ghash_key const *k, const char *head); -static void (*pmull_ghash_update)(int blocks, u64 dg[], const char *src, - struct ghash_key const *k, - const char *head); - asmlinkage void pmull_gcm_encrypt(int blocks, u64 dg[], u8 dst[], const u8 src[], struct ghash_key const *k, u8 ctr[], u32 const rk[], int rounds, @@ -87,11 +83,15 @@ static int ghash_init(struct shash_desc *desc) } static void ghash_do_update(int blocks, u64 dg[], const char *src, - struct ghash_key *key, const char *head) + struct ghash_key *key, const char *head, + void (*simd_update)(int blocks, u64 dg[], + const char *src, + struct ghash_key const *k, + const char *head)) { if (likely(may_use_simd())) { kernel_neon_begin(); - pmull_ghash_update(blocks, dg, src, key, head); + simd_update(blocks, dg, src, key, head); kernel_neon_end(); } else { be128 dst = { cpu_to_be64(dg[1]), cpu_to_be64(dg[0]) }; @@ -119,8 +119,12 @@ static void ghash_do_update(int blocks, u64 dg[], const char *src, /* avoid hogging the CPU for too long */ #define MAX_BLOCKS (SZ_64K / GHASH_BLOCK_SIZE) -static int ghash_update(struct shash_desc *desc, const u8 *src, - unsigned int len) +static int __ghash_update(struct shash_desc *desc, const u8 *src, + unsigned int len, + void (*simd_update)(int blocks, u64 dg[], + const char *src, + struct ghash_key const *k, + const char *head)) { struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; @@ -146,7 +150,8 @@ static int ghash_update(struct shash_desc *desc, const u8 *src, int chunk = min(blocks, MAX_BLOCKS); ghash_do_update(chunk, ctx->digest, src, key, - partial ? ctx->buf : NULL); + partial ? ctx->buf : NULL, + simd_update); blocks -= chunk; src += chunk * GHASH_BLOCK_SIZE; @@ -158,7 +163,19 @@ static int ghash_update(struct shash_desc *desc, const u8 *src, return 0; } -static int ghash_final(struct shash_desc *desc, u8 *dst) +static int ghash_update_p8(struct shash_desc *desc, const u8 *src, + unsigned int len) +{ + return __ghash_update(desc, src, len, pmull_ghash_update_p8); +} + +static int ghash_update_p64(struct shash_desc *desc, const u8 *src, + unsigned int len) +{ + return __ghash_update(desc, src, len, pmull_ghash_update_p64); +} + +static int ghash_final_p8(struct shash_desc *desc, u8 *dst) { struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; @@ -168,7 +185,28 @@ static int ghash_final(struct shash_desc *desc, u8 *dst) memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial); - ghash_do_update(1, ctx->digest, ctx->buf, key, NULL); + ghash_do_update(1, ctx->digest, ctx->buf, key, NULL, + pmull_ghash_update_p8); + } + put_unaligned_be64(ctx->digest[1], dst); + put_unaligned_be64(ctx->digest[0], dst + 8); + + *ctx = (struct ghash_desc_ctx){}; + return 0; +} + +static int ghash_final_p64(struct shash_desc *desc, u8 *dst) +{ + struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); + unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; + + if (partial) { + struct ghash_key *key = crypto_shash_ctx(desc->tfm); + + memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial); + + ghash_do_update(1, ctx->digest, ctx->buf, key, NULL, + pmull_ghash_update_p64); } put_unaligned_be64(ctx->digest[1], dst); put_unaligned_be64(ctx->digest[0], dst + 8); @@ -224,7 +262,21 @@ static int ghash_setkey(struct crypto_shash *tfm, return __ghash_setkey(key, inkey, keylen); } -static struct shash_alg ghash_alg = { +static struct shash_alg ghash_alg[] = {{ + .base.cra_name = "ghash", + .base.cra_driver_name = "ghash-neon", + .base.cra_priority = 100, + .base.cra_blocksize = GHASH_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ghash_key), + .base.cra_module = THIS_MODULE, + + .digestsize = GHASH_DIGEST_SIZE, + .init = ghash_init, + .update = ghash_update_p8, + .final = ghash_final_p8, + .setkey = ghash_setkey, + .descsize = sizeof(struct ghash_desc_ctx), +}, { .base.cra_name = "ghash", .base.cra_driver_name = "ghash-ce", .base.cra_priority = 200, @@ -234,11 +286,11 @@ static struct shash_alg ghash_alg = { .digestsize = GHASH_DIGEST_SIZE, .init = ghash_init, - .update = ghash_update, - .final = ghash_final, + .update = ghash_update_p64, + .final = ghash_final_p64, .setkey = ghash_setkey, .descsize = sizeof(struct ghash_desc_ctx), -}; +}}; static int num_rounds(struct crypto_aes_ctx *ctx) { @@ -301,7 +353,8 @@ static void gcm_update_mac(u64 dg[], const u8 *src, int count, u8 buf[], int blocks = count / GHASH_BLOCK_SIZE; ghash_do_update(blocks, dg, src, &ctx->ghash_key, - *buf_count ? buf : NULL); + *buf_count ? buf : NULL, + pmull_ghash_update_p64); src += blocks * GHASH_BLOCK_SIZE; count %= GHASH_BLOCK_SIZE; @@ -345,7 +398,8 @@ static void gcm_calculate_auth_mac(struct aead_request *req, u64 dg[]) if (buf_count) { memset(&buf[buf_count], 0, GHASH_BLOCK_SIZE - buf_count); - ghash_do_update(1, dg, buf, &ctx->ghash_key, NULL); + ghash_do_update(1, dg, buf, &ctx->ghash_key, NULL, + pmull_ghash_update_p64); } } @@ -358,7 +412,8 @@ static void gcm_final(struct aead_request *req, struct gcm_aes_ctx *ctx, lengths.a = cpu_to_be64(req->assoclen * 8); lengths.b = cpu_to_be64(cryptlen * 8); - ghash_do_update(1, dg, (void *)&lengths, &ctx->ghash_key, NULL); + ghash_do_update(1, dg, (void *)&lengths, &ctx->ghash_key, NULL, + pmull_ghash_update_p64); put_unaligned_be64(dg[1], mac); put_unaligned_be64(dg[0], mac + 8); @@ -434,7 +489,7 @@ static int gcm_encrypt(struct aead_request *req) ghash_do_update(walk.nbytes / AES_BLOCK_SIZE, dg, walk.dst.virt.addr, &ctx->ghash_key, - NULL); + NULL, pmull_ghash_update_p64); err = skcipher_walk_done(&walk, walk.nbytes % (2 * AES_BLOCK_SIZE)); @@ -469,7 +524,8 @@ static int gcm_encrypt(struct aead_request *req) memcpy(buf, dst, nbytes); memset(buf + nbytes, 0, GHASH_BLOCK_SIZE - nbytes); - ghash_do_update(!!nbytes, dg, buf, &ctx->ghash_key, head); + ghash_do_update(!!nbytes, dg, buf, &ctx->ghash_key, head, + pmull_ghash_update_p64); err = skcipher_walk_done(&walk, 0); } @@ -558,7 +614,8 @@ static int gcm_decrypt(struct aead_request *req) u8 *src = walk.src.virt.addr; ghash_do_update(blocks, dg, walk.src.virt.addr, - &ctx->ghash_key, NULL); + &ctx->ghash_key, NULL, + pmull_ghash_update_p64); do { __aes_arm64_encrypt(ctx->aes_key.key_enc, @@ -602,7 +659,8 @@ static int gcm_decrypt(struct aead_request *req) memcpy(buf, src, nbytes); memset(buf + nbytes, 0, GHASH_BLOCK_SIZE - nbytes); - ghash_do_update(!!nbytes, dg, buf, &ctx->ghash_key, head); + ghash_do_update(!!nbytes, dg, buf, &ctx->ghash_key, head, + pmull_ghash_update_p64); crypto_xor_cpy(walk.dst.virt.addr, walk.src.virt.addr, iv, walk.nbytes); @@ -650,26 +708,30 @@ static int __init ghash_ce_mod_init(void) return -ENODEV; if (elf_hwcap & HWCAP_PMULL) - pmull_ghash_update = pmull_ghash_update_p64; - + ret = crypto_register_shashes(ghash_alg, + ARRAY_SIZE(ghash_alg)); else - pmull_ghash_update = pmull_ghash_update_p8; + /* only register the first array element */ + ret = crypto_register_shash(ghash_alg); - ret = crypto_register_shash(&ghash_alg); if (ret) return ret; if (elf_hwcap & HWCAP_PMULL) { ret = crypto_register_aead(&gcm_aes_alg); if (ret) - crypto_unregister_shash(&ghash_alg); + crypto_unregister_shashes(ghash_alg, + ARRAY_SIZE(ghash_alg)); } return ret; } static void __exit ghash_ce_mod_exit(void) { - crypto_unregister_shash(&ghash_alg); + if (elf_hwcap & HWCAP_PMULL) + crypto_unregister_shashes(ghash_alg, ARRAY_SIZE(ghash_alg)); + else + crypto_unregister_shash(ghash_alg); crypto_unregister_aead(&gcm_aes_alg); } -- cgit v1.2.3 From 62fecf295e3c48be1b5f17c440b93875b9adb4d6 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 27 Jan 2019 10:16:52 +0100 Subject: crypto: arm/crct10dif - revert to C code for short inputs The SIMD routine ported from x86 used to have a special code path for inputs < 16 bytes, which got lost somewhere along the way. Instead, the current glue code aligns the input pointer to permit the NEON routine to use special versions of the vld1 instructions that assume 16 byte alignment, but this could result in inputs of less than 16 bytes to be passed in. This not only fails the new extended tests that Eric has implemented, it also results in the code reading past the end of the input, which could potentially result in crashes when dealing with less than 16 bytes of input at the end of a page which is followed by an unmapped page. So update the glue code to only invoke the NEON routine if the input is at least 16 bytes. Reported-by: Eric Biggers Reviewed-by: Eric Biggers Fixes: 1d481f1cd892 ("crypto: arm/crct10dif - port x86 SSE implementation to ARM") Cc: # v4.10+ Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm/crypto/crct10dif-ce-core.S | 14 +++++++------- arch/arm/crypto/crct10dif-ce-glue.c | 23 ++++++----------------- 2 files changed, 13 insertions(+), 24 deletions(-) (limited to 'arch') diff --git a/arch/arm/crypto/crct10dif-ce-core.S b/arch/arm/crypto/crct10dif-ce-core.S index ce45ba0c0687..16019b5961e7 100644 --- a/arch/arm/crypto/crct10dif-ce-core.S +++ b/arch/arm/crypto/crct10dif-ce-core.S @@ -124,10 +124,10 @@ ENTRY(crc_t10dif_pmull) vext.8 q10, qzr, q0, #4 // receive the initial 64B data, xor the initial crc value - vld1.64 {q0-q1}, [arg2, :128]! - vld1.64 {q2-q3}, [arg2, :128]! - vld1.64 {q4-q5}, [arg2, :128]! - vld1.64 {q6-q7}, [arg2, :128]! + vld1.64 {q0-q1}, [arg2]! + vld1.64 {q2-q3}, [arg2]! + vld1.64 {q4-q5}, [arg2]! + vld1.64 {q6-q7}, [arg2]! CPU_LE( vrev64.8 q0, q0 ) CPU_LE( vrev64.8 q1, q1 ) CPU_LE( vrev64.8 q2, q2 ) @@ -167,7 +167,7 @@ CPU_LE( vrev64.8 q7, q7 ) _fold_64_B_loop: .macro fold64, reg1, reg2 - vld1.64 {q11-q12}, [arg2, :128]! + vld1.64 {q11-q12}, [arg2]! vmull.p64 q8, \reg1\()h, d21 vmull.p64 \reg1, \reg1\()l, d20 @@ -238,7 +238,7 @@ _16B_reduction_loop: vmull.p64 q7, d15, d21 veor.8 q7, q7, q8 - vld1.64 {q0}, [arg2, :128]! + vld1.64 {q0}, [arg2]! CPU_LE( vrev64.8 q0, q0 ) vswp d0, d1 veor.8 q7, q7, q0 @@ -335,7 +335,7 @@ _less_than_128: vmov.i8 q0, #0 vmov s3, arg1_low32 // get the initial crc value - vld1.64 {q7}, [arg2, :128]! + vld1.64 {q7}, [arg2]! CPU_LE( vrev64.8 q7, q7 ) vswp d14, d15 veor.8 q7, q7, q0 diff --git a/arch/arm/crypto/crct10dif-ce-glue.c b/arch/arm/crypto/crct10dif-ce-glue.c index d428355cf38d..14c19c70a841 100644 --- a/arch/arm/crypto/crct10dif-ce-glue.c +++ b/arch/arm/crypto/crct10dif-ce-glue.c @@ -35,26 +35,15 @@ static int crct10dif_update(struct shash_desc *desc, const u8 *data, unsigned int length) { u16 *crc = shash_desc_ctx(desc); - unsigned int l; - if (!may_use_simd()) { - *crc = crc_t10dif_generic(*crc, data, length); + if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && may_use_simd()) { + kernel_neon_begin(); + *crc = crc_t10dif_pmull(*crc, data, length); + kernel_neon_end(); } else { - if (unlikely((u32)data % CRC_T10DIF_PMULL_CHUNK_SIZE)) { - l = min_t(u32, length, CRC_T10DIF_PMULL_CHUNK_SIZE - - ((u32)data % CRC_T10DIF_PMULL_CHUNK_SIZE)); - - *crc = crc_t10dif_generic(*crc, data, l); - - length -= l; - data += l; - } - if (length > 0) { - kernel_neon_begin(); - *crc = crc_t10dif_pmull(*crc, data, length); - kernel_neon_end(); - } + *crc = crc_t10dif_generic(*crc, data, length); } + return 0; } -- cgit v1.2.3 From d72b9d4acd548251f55b16843fc7a05dc5c80de8 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 27 Jan 2019 10:16:53 +0100 Subject: crypto: arm64/crct10dif - revert to C code for short inputs The SIMD routine ported from x86 used to have a special code path for inputs < 16 bytes, which got lost somewhere along the way. Instead, the current glue code aligns the input pointer to 16 bytes, which is not really necessary on this architecture (although it could be beneficial to performance to expose aligned data to the the NEON routine), but this could result in inputs of less than 16 bytes to be passed in. This not only fails the new extended tests that Eric has implemented, it also results in the code reading past the end of the input, which could potentially result in crashes when dealing with less than 16 bytes of input at the end of a page which is followed by an unmapped page. So update the glue code to only invoke the NEON routine if the input is at least 16 bytes. Reported-by: Eric Biggers Reviewed-by: Eric Biggers Fixes: 6ef5737f3931 ("crypto: arm64/crct10dif - port x86 SSE implementation to arm64") Cc: # v4.10+ Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/crct10dif-ce-glue.c | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) (limited to 'arch') diff --git a/arch/arm64/crypto/crct10dif-ce-glue.c b/arch/arm64/crypto/crct10dif-ce-glue.c index b461d62023f2..567c24f3d224 100644 --- a/arch/arm64/crypto/crct10dif-ce-glue.c +++ b/arch/arm64/crypto/crct10dif-ce-glue.c @@ -39,26 +39,13 @@ static int crct10dif_update(struct shash_desc *desc, const u8 *data, unsigned int length) { u16 *crc = shash_desc_ctx(desc); - unsigned int l; - if (unlikely((u64)data % CRC_T10DIF_PMULL_CHUNK_SIZE)) { - l = min_t(u32, length, CRC_T10DIF_PMULL_CHUNK_SIZE - - ((u64)data % CRC_T10DIF_PMULL_CHUNK_SIZE)); - - *crc = crc_t10dif_generic(*crc, data, l); - - length -= l; - data += l; - } - - if (length > 0) { - if (may_use_simd()) { - kernel_neon_begin(); - *crc = crc_t10dif_pmull(*crc, data, length); - kernel_neon_end(); - } else { - *crc = crc_t10dif_generic(*crc, data, length); - } + if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && may_use_simd()) { + kernel_neon_begin(); + *crc = crc_t10dif_pmull(*crc, data, length); + kernel_neon_end(); + } else { + *crc = crc_t10dif_generic(*crc, data, length); } return 0; -- cgit v1.2.3 From c03f3cb40ba93a9f49fc44a71ddaa46f93f90a15 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 27 Jan 2019 10:16:54 +0100 Subject: crypto: arm/crct10dif - remove dead code Remove some code that is no longer called now that we make sure never to invoke the SIMD routine with less that 16 bytes of input. Reviewed-by: Eric Biggers Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm/crypto/crct10dif-ce-core.S | 13 ------------- 1 file changed, 13 deletions(-) (limited to 'arch') diff --git a/arch/arm/crypto/crct10dif-ce-core.S b/arch/arm/crypto/crct10dif-ce-core.S index 16019b5961e7..d058fad423c2 100644 --- a/arch/arm/crypto/crct10dif-ce-core.S +++ b/arch/arm/crypto/crct10dif-ce-core.S @@ -342,7 +342,6 @@ CPU_LE( vrev64.8 q7, q7 ) cmp arg3, #16 beq _128_done // exactly 16 left - blt _less_than_16_left // now if there is, load the constants vldr d20, rk1 @@ -353,18 +352,6 @@ CPU_LE( vrev64.8 q7, q7 ) addlt arg3, arg3, #16 blt _get_last_two_regs b _16B_reduction_loop - -_less_than_16_left: - // shl r9, 4 - adr ip, tbl_shf_table + 16 - sub ip, ip, arg3 - vld1.8 {q0}, [ip] - vmov.i8 q9, #0x80 - veor.8 q0, q0, q9 - vtbl.8 d18, {d14-d15}, d0 - vtbl.8 d15, {d14-d15}, d1 - vmov d14, d18 - b _128_done ENDPROC(crc_t10dif_pmull) // precomputed constants -- cgit v1.2.3 From 1b2ca568ca23a6cfa6aa28e8675b54940807231d Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 27 Jan 2019 10:16:55 +0100 Subject: crypto: arm64/crct10dif - remove dead code Remove some code that is no longer called now that we make sure never to invoke the SIMD routine with less than 16 bytes of input. Reviewed-by: Eric Biggers Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/crct10dif-ce-core.S | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'arch') diff --git a/arch/arm64/crypto/crct10dif-ce-core.S b/arch/arm64/crypto/crct10dif-ce-core.S index 9e82e8e8ed05..f7326259c40d 100644 --- a/arch/arm64/crypto/crct10dif-ce-core.S +++ b/arch/arm64/crypto/crct10dif-ce-core.S @@ -497,7 +497,6 @@ CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 ) cmp arg3, #16 b.eq .L_128_done_\@ // exactly 16 left - b.lt .L_less_than_16_left_\@ ldr_l q10, rk1, x8 // rk1 and rk2 in xmm10 __pmull_pre_\p v10 @@ -509,16 +508,6 @@ CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 ) add arg3, arg3, #16 b .L_get_last_two_regs_\@ - -.L_less_than_16_left_\@: - // shl r9, 4 - adr_l x0, tbl_shf_table + 16 - sub x0, x0, arg3 - ld1 {v0.16b}, [x0] - movi v9.16b, #0x80 - eor v0.16b, v0.16b, v9.16b - tbl v7.16b, {v7.16b}, v0.16b - b .L_128_done_\@ .endm ENTRY(crc_t10dif_pmull_p8) -- cgit v1.2.3 From 8336bdf12a9ea5bb77bb32d215a34485fb66245c Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 25 Jan 2019 10:36:26 +0100 Subject: crypto: arm64/crct10dif - register PMULL variants as separate algos The arm64 CRC-T10DIF implementation either uses 8-bit or 64-bit polynomial multiplication instructions, since the latter are faster but not mandatory in the architecture. Since that prevents us from testing both implementations on the same system, let's expose both implementations to the crypto API, with the priorities reflecting that the P64 version is the preferred one if available. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/crct10dif-ce-glue.c | 54 +++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 12 deletions(-) (limited to 'arch') diff --git a/arch/arm64/crypto/crct10dif-ce-glue.c b/arch/arm64/crypto/crct10dif-ce-glue.c index 567c24f3d224..242757cc6da9 100644 --- a/arch/arm64/crypto/crct10dif-ce-glue.c +++ b/arch/arm64/crypto/crct10dif-ce-glue.c @@ -25,8 +25,6 @@ asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 buf[], u64 len); asmlinkage u16 crc_t10dif_pmull_p8(u16 init_crc, const u8 buf[], u64 len); -static u16 (*crc_t10dif_pmull)(u16 init_crc, const u8 buf[], u64 len); - static int crct10dif_init(struct shash_desc *desc) { u16 *crc = shash_desc_ctx(desc); @@ -35,14 +33,30 @@ static int crct10dif_init(struct shash_desc *desc) return 0; } -static int crct10dif_update(struct shash_desc *desc, const u8 *data, +static int crct10dif_update_pmull_p8(struct shash_desc *desc, const u8 *data, + unsigned int length) +{ + u16 *crc = shash_desc_ctx(desc); + + if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && may_use_simd()) { + kernel_neon_begin(); + *crc = crc_t10dif_pmull_p8(*crc, data, length); + kernel_neon_end(); + } else { + *crc = crc_t10dif_generic(*crc, data, length); + } + + return 0; +} + +static int crct10dif_update_pmull_p64(struct shash_desc *desc, const u8 *data, unsigned int length) { u16 *crc = shash_desc_ctx(desc); if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && may_use_simd()) { kernel_neon_begin(); - *crc = crc_t10dif_pmull(*crc, data, length); + *crc = crc_t10dif_pmull_p64(*crc, data, length); kernel_neon_end(); } else { *crc = crc_t10dif_generic(*crc, data, length); @@ -59,10 +73,22 @@ static int crct10dif_final(struct shash_desc *desc, u8 *out) return 0; } -static struct shash_alg crc_t10dif_alg = { +static struct shash_alg crc_t10dif_alg[] = {{ .digestsize = CRC_T10DIF_DIGEST_SIZE, .init = crct10dif_init, - .update = crct10dif_update, + .update = crct10dif_update_pmull_p8, + .final = crct10dif_final, + .descsize = CRC_T10DIF_DIGEST_SIZE, + + .base.cra_name = "crct10dif", + .base.cra_driver_name = "crct10dif-arm64-neon", + .base.cra_priority = 100, + .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, +}, { + .digestsize = CRC_T10DIF_DIGEST_SIZE, + .init = crct10dif_init, + .update = crct10dif_update_pmull_p64, .final = crct10dif_final, .descsize = CRC_T10DIF_DIGEST_SIZE, @@ -71,21 +97,25 @@ static struct shash_alg crc_t10dif_alg = { .base.cra_priority = 200, .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE, .base.cra_module = THIS_MODULE, -}; +}}; static int __init crc_t10dif_mod_init(void) { if (elf_hwcap & HWCAP_PMULL) - crc_t10dif_pmull = crc_t10dif_pmull_p64; + return crypto_register_shashes(crc_t10dif_alg, + ARRAY_SIZE(crc_t10dif_alg)); else - crc_t10dif_pmull = crc_t10dif_pmull_p8; - - return crypto_register_shash(&crc_t10dif_alg); + /* only register the first array element */ + return crypto_register_shash(crc_t10dif_alg); } static void __exit crc_t10dif_mod_exit(void) { - crypto_unregister_shash(&crc_t10dif_alg); + if (elf_hwcap & HWCAP_PMULL) + crypto_unregister_shashes(crc_t10dif_alg, + ARRAY_SIZE(crc_t10dif_alg)); + else + crypto_unregister_shash(crc_t10dif_alg); } module_cpu_feature_match(ASIMD, crc_t10dif_mod_init); -- cgit v1.2.3 From 0974037fc55cdf27a0297d3487f4fafa57b4a4a9 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 30 Jan 2019 20:42:40 -0800 Subject: crypto: x86/crct10dif-pcl - cleanup and optimizations The x86, arm, and arm64 asm implementations of crct10dif are very difficult to understand partly because many of the comments, labels, and macros are named incorrectly: the lengths mentioned are usually off by a factor of two from the actual code. Many other things are unnecessarily convoluted as well, e.g. there are many more fold constants than actually needed and some aren't fully reduced. This series therefore cleans up all these implementations to be much more maintainable. I also made some small optimizations where I saw opportunities, resulting in slightly better performance. This patch cleans up the x86 version. As part of this, I removed support for len < 16 from the x86 assembly; now the glue code falls back to the generic table-based implementation in this case. Due to the overhead of kernel_fpu_begin(), this actually significantly improves performance on these lengths. (And even if kernel_fpu_begin() were free, the generic code is still faster for about len < 11.) This removal also eliminates error-prone special cases and makes the x86, arm32, and arm64 ports of the code match more closely. Acked-by: Ard Biesheuvel Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/crct10dif-pcl-asm_64.S | 782 ++++++++++---------------------- arch/x86/crypto/crct10dif-pclmul_glue.c | 12 +- 2 files changed, 235 insertions(+), 559 deletions(-) (limited to 'arch') diff --git a/arch/x86/crypto/crct10dif-pcl-asm_64.S b/arch/x86/crypto/crct10dif-pcl-asm_64.S index de04d3e98d8d..3d873e67749d 100644 --- a/arch/x86/crypto/crct10dif-pcl-asm_64.S +++ b/arch/x86/crypto/crct10dif-pcl-asm_64.S @@ -43,609 +43,291 @@ # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -######################################################################## -# Function API: -# UINT16 crc_t10dif_pcl( -# UINT16 init_crc, //initial CRC value, 16 bits -# const unsigned char *buf, //buffer pointer to calculate CRC on -# UINT64 len //buffer length in bytes (64-bit data) -# ); # # Reference paper titled "Fast CRC Computation for Generic # Polynomials Using PCLMULQDQ Instruction" # URL: http://www.intel.com/content/dam/www/public/us/en/documents # /white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf # -# #include .text -#define arg1 %rdi -#define arg2 %rsi -#define arg3 %rdx - -#define arg1_low32 %edi +#define init_crc %edi +#define buf %rsi +#define len %rdx + +#define FOLD_CONSTS %xmm10 +#define BSWAP_MASK %xmm11 + +# Fold reg1, reg2 into the next 32 data bytes, storing the result back into +# reg1, reg2. +.macro fold_32_bytes offset, reg1, reg2 + movdqu \offset(buf), %xmm9 + movdqu \offset+16(buf), %xmm12 + pshufb BSWAP_MASK, %xmm9 + pshufb BSWAP_MASK, %xmm12 + movdqa \reg1, %xmm8 + movdqa \reg2, %xmm13 + pclmulqdq $0x00, FOLD_CONSTS, \reg1 + pclmulqdq $0x11, FOLD_CONSTS, %xmm8 + pclmulqdq $0x00, FOLD_CONSTS, \reg2 + pclmulqdq $0x11, FOLD_CONSTS, %xmm13 + pxor %xmm9 , \reg1 + xorps %xmm8 , \reg1 + pxor %xmm12, \reg2 + xorps %xmm13, \reg2 +.endm + +# Fold src_reg into dst_reg. +.macro fold_16_bytes src_reg, dst_reg + movdqa \src_reg, %xmm8 + pclmulqdq $0x11, FOLD_CONSTS, \src_reg + pclmulqdq $0x00, FOLD_CONSTS, %xmm8 + pxor %xmm8, \dst_reg + xorps \src_reg, \dst_reg +.endm -ENTRY(crc_t10dif_pcl) +# +# u16 crc_t10dif_pcl(u16 init_crc, const *u8 buf, size_t len); +# +# Assumes len >= 16. +# .align 16 +ENTRY(crc_t10dif_pcl) - # adjust the 16-bit initial_crc value, scale it to 32 bits - shl $16, arg1_low32 - - # Allocate Stack Space - mov %rsp, %rcx - sub $16*2, %rsp - # align stack to 16 byte boundary - and $~(0x10 - 1), %rsp - - # check if smaller than 256 - cmp $256, arg3 - - # for sizes less than 128, we can't fold 64B at a time... - jl _less_than_128 - - - # load the initial crc value - movd arg1_low32, %xmm10 # initial crc - - # crc value does not need to be byte-reflected, but it needs - # to be moved to the high part of the register. - # because data will be byte-reflected and will align with - # initial crc at correct place. - pslldq $12, %xmm10 - - movdqa SHUF_MASK(%rip), %xmm11 - # receive the initial 64B data, xor the initial crc value - movdqu 16*0(arg2), %xmm0 - movdqu 16*1(arg2), %xmm1 - movdqu 16*2(arg2), %xmm2 - movdqu 16*3(arg2), %xmm3 - movdqu 16*4(arg2), %xmm4 - movdqu 16*5(arg2), %xmm5 - movdqu 16*6(arg2), %xmm6 - movdqu 16*7(arg2), %xmm7 - - pshufb %xmm11, %xmm0 - # XOR the initial_crc value - pxor %xmm10, %xmm0 - pshufb %xmm11, %xmm1 - pshufb %xmm11, %xmm2 - pshufb %xmm11, %xmm3 - pshufb %xmm11, %xmm4 - pshufb %xmm11, %xmm5 - pshufb %xmm11, %xmm6 - pshufb %xmm11, %xmm7 - - movdqa rk3(%rip), %xmm10 #xmm10 has rk3 and rk4 - #imm value of pclmulqdq instruction - #will determine which constant to use - - ################################################################# - # we subtract 256 instead of 128 to save one instruction from the loop - sub $256, arg3 - - # at this section of the code, there is 64*x+y (0<=y<64) bytes of - # buffer. The _fold_64_B_loop will fold 64B at a time - # until we have 64+y Bytes of buffer - - - # fold 64B at a time. This section of the code folds 4 xmm - # registers in parallel -_fold_64_B_loop: - - # update the buffer pointer - add $128, arg2 # buf += 64# - - movdqu 16*0(arg2), %xmm9 - movdqu 16*1(arg2), %xmm12 - pshufb %xmm11, %xmm9 - pshufb %xmm11, %xmm12 - movdqa %xmm0, %xmm8 - movdqa %xmm1, %xmm13 - pclmulqdq $0x0 , %xmm10, %xmm0 - pclmulqdq $0x11, %xmm10, %xmm8 - pclmulqdq $0x0 , %xmm10, %xmm1 - pclmulqdq $0x11, %xmm10, %xmm13 - pxor %xmm9 , %xmm0 - xorps %xmm8 , %xmm0 - pxor %xmm12, %xmm1 - xorps %xmm13, %xmm1 - - movdqu 16*2(arg2), %xmm9 - movdqu 16*3(arg2), %xmm12 - pshufb %xmm11, %xmm9 - pshufb %xmm11, %xmm12 - movdqa %xmm2, %xmm8 - movdqa %xmm3, %xmm13 - pclmulqdq $0x0, %xmm10, %xmm2 - pclmulqdq $0x11, %xmm10, %xmm8 - pclmulqdq $0x0, %xmm10, %xmm3 - pclmulqdq $0x11, %xmm10, %xmm13 - pxor %xmm9 , %xmm2 - xorps %xmm8 , %xmm2 - pxor %xmm12, %xmm3 - xorps %xmm13, %xmm3 - - movdqu 16*4(arg2), %xmm9 - movdqu 16*5(arg2), %xmm12 - pshufb %xmm11, %xmm9 - pshufb %xmm11, %xmm12 - movdqa %xmm4, %xmm8 - movdqa %xmm5, %xmm13 - pclmulqdq $0x0, %xmm10, %xmm4 - pclmulqdq $0x11, %xmm10, %xmm8 - pclmulqdq $0x0, %xmm10, %xmm5 - pclmulqdq $0x11, %xmm10, %xmm13 - pxor %xmm9 , %xmm4 - xorps %xmm8 , %xmm4 - pxor %xmm12, %xmm5 - xorps %xmm13, %xmm5 - - movdqu 16*6(arg2), %xmm9 - movdqu 16*7(arg2), %xmm12 - pshufb %xmm11, %xmm9 - pshufb %xmm11, %xmm12 - movdqa %xmm6 , %xmm8 - movdqa %xmm7 , %xmm13 - pclmulqdq $0x0 , %xmm10, %xmm6 - pclmulqdq $0x11, %xmm10, %xmm8 - pclmulqdq $0x0 , %xmm10, %xmm7 - pclmulqdq $0x11, %xmm10, %xmm13 - pxor %xmm9 , %xmm6 - xorps %xmm8 , %xmm6 - pxor %xmm12, %xmm7 - xorps %xmm13, %xmm7 - - sub $128, arg3 - - # check if there is another 64B in the buffer to be able to fold - jge _fold_64_B_loop - ################################################################## - - - add $128, arg2 - # at this point, the buffer pointer is pointing at the last y Bytes - # of the buffer the 64B of folded data is in 4 of the xmm - # registers: xmm0, xmm1, xmm2, xmm3 - - - # fold the 8 xmm registers to 1 xmm register with different constants - - movdqa rk9(%rip), %xmm10 - movdqa %xmm0, %xmm8 - pclmulqdq $0x11, %xmm10, %xmm0 - pclmulqdq $0x0 , %xmm10, %xmm8 - pxor %xmm8, %xmm7 - xorps %xmm0, %xmm7 - - movdqa rk11(%rip), %xmm10 - movdqa %xmm1, %xmm8 - pclmulqdq $0x11, %xmm10, %xmm1 - pclmulqdq $0x0 , %xmm10, %xmm8 - pxor %xmm8, %xmm7 - xorps %xmm1, %xmm7 - - movdqa rk13(%rip), %xmm10 - movdqa %xmm2, %xmm8 - pclmulqdq $0x11, %xmm10, %xmm2 - pclmulqdq $0x0 , %xmm10, %xmm8 - pxor %xmm8, %xmm7 - pxor %xmm2, %xmm7 - - movdqa rk15(%rip), %xmm10 - movdqa %xmm3, %xmm8 - pclmulqdq $0x11, %xmm10, %xmm3 - pclmulqdq $0x0 , %xmm10, %xmm8 - pxor %xmm8, %xmm7 - xorps %xmm3, %xmm7 - - movdqa rk17(%rip), %xmm10 - movdqa %xmm4, %xmm8 - pclmulqdq $0x11, %xmm10, %xmm4 - pclmulqdq $0x0 , %xmm10, %xmm8 - pxor %xmm8, %xmm7 - pxor %xmm4, %xmm7 - - movdqa rk19(%rip), %xmm10 - movdqa %xmm5, %xmm8 - pclmulqdq $0x11, %xmm10, %xmm5 - pclmulqdq $0x0 , %xmm10, %xmm8 - pxor %xmm8, %xmm7 - xorps %xmm5, %xmm7 - - movdqa rk1(%rip), %xmm10 #xmm10 has rk1 and rk2 - #imm value of pclmulqdq instruction - #will determine which constant to use - movdqa %xmm6, %xmm8 - pclmulqdq $0x11, %xmm10, %xmm6 - pclmulqdq $0x0 , %xmm10, %xmm8 - pxor %xmm8, %xmm7 - pxor %xmm6, %xmm7 - - - # instead of 64, we add 48 to the loop counter to save 1 instruction - # from the loop instead of a cmp instruction, we use the negative - # flag with the jl instruction - add $128-16, arg3 - jl _final_reduction_for_128 - - # now we have 16+y bytes left to reduce. 16 Bytes is in register xmm7 - # and the rest is in memory. We can fold 16 bytes at a time if y>=16 - # continue folding 16B at a time - -_16B_reduction_loop: + movdqa .Lbswap_mask(%rip), BSWAP_MASK + + # For sizes less than 256 bytes, we can't fold 128 bytes at a time. + cmp $256, len + jl .Lless_than_256_bytes + + # Load the first 128 data bytes. Byte swapping is necessary to make the + # bit order match the polynomial coefficient order. + movdqu 16*0(buf), %xmm0 + movdqu 16*1(buf), %xmm1 + movdqu 16*2(buf), %xmm2 + movdqu 16*3(buf), %xmm3 + movdqu 16*4(buf), %xmm4 + movdqu 16*5(buf), %xmm5 + movdqu 16*6(buf), %xmm6 + movdqu 16*7(buf), %xmm7 + add $128, buf + pshufb BSWAP_MASK, %xmm0 + pshufb BSWAP_MASK, %xmm1 + pshufb BSWAP_MASK, %xmm2 + pshufb BSWAP_MASK, %xmm3 + pshufb BSWAP_MASK, %xmm4 + pshufb BSWAP_MASK, %xmm5 + pshufb BSWAP_MASK, %xmm6 + pshufb BSWAP_MASK, %xmm7 + + # XOR the first 16 data *bits* with the initial CRC value. + pxor %xmm8, %xmm8 + pinsrw $7, init_crc, %xmm8 + pxor %xmm8, %xmm0 + + movdqa .Lfold_across_128_bytes_consts(%rip), FOLD_CONSTS + + # Subtract 128 for the 128 data bytes just consumed. Subtract another + # 128 to simplify the termination condition of the following loop. + sub $256, len + + # While >= 128 data bytes remain (not counting xmm0-7), fold the 128 + # bytes xmm0-7 into them, storing the result back into xmm0-7. +.Lfold_128_bytes_loop: + fold_32_bytes 0, %xmm0, %xmm1 + fold_32_bytes 32, %xmm2, %xmm3 + fold_32_bytes 64, %xmm4, %xmm5 + fold_32_bytes 96, %xmm6, %xmm7 + add $128, buf + sub $128, len + jge .Lfold_128_bytes_loop + + # Now fold the 112 bytes in xmm0-xmm6 into the 16 bytes in xmm7. + + # Fold across 64 bytes. + movdqa .Lfold_across_64_bytes_consts(%rip), FOLD_CONSTS + fold_16_bytes %xmm0, %xmm4 + fold_16_bytes %xmm1, %xmm5 + fold_16_bytes %xmm2, %xmm6 + fold_16_bytes %xmm3, %xmm7 + # Fold across 32 bytes. + movdqa .Lfold_across_32_bytes_consts(%rip), FOLD_CONSTS + fold_16_bytes %xmm4, %xmm6 + fold_16_bytes %xmm5, %xmm7 + # Fold across 16 bytes. + movdqa .Lfold_across_16_bytes_consts(%rip), FOLD_CONSTS + fold_16_bytes %xmm6, %xmm7 + + # Add 128 to get the correct number of data bytes remaining in 0...127 + # (not counting xmm7), following the previous extra subtraction by 128. + # Then subtract 16 to simplify the termination condition of the + # following loop. + add $128-16, len + + # While >= 16 data bytes remain (not counting xmm7), fold the 16 bytes + # xmm7 into them, storing the result back into xmm7. + jl .Lfold_16_bytes_loop_done +.Lfold_16_bytes_loop: movdqa %xmm7, %xmm8 - pclmulqdq $0x11, %xmm10, %xmm7 - pclmulqdq $0x0 , %xmm10, %xmm8 + pclmulqdq $0x11, FOLD_CONSTS, %xmm7 + pclmulqdq $0x00, FOLD_CONSTS, %xmm8 pxor %xmm8, %xmm7 - movdqu (arg2), %xmm0 - pshufb %xmm11, %xmm0 + movdqu (buf), %xmm0 + pshufb BSWAP_MASK, %xmm0 pxor %xmm0 , %xmm7 - add $16, arg2 - sub $16, arg3 - # instead of a cmp instruction, we utilize the flags with the - # jge instruction equivalent of: cmp arg3, 16-16 - # check if there is any more 16B in the buffer to be able to fold - jge _16B_reduction_loop - - #now we have 16+z bytes left to reduce, where 0<= z < 16. - #first, we reduce the data in the xmm7 register - - -_final_reduction_for_128: - # check if any more data to fold. If not, compute the CRC of - # the final 128 bits - add $16, arg3 - je _128_done - - # here we are getting data that is less than 16 bytes. - # since we know that there was data before the pointer, we can - # offset the input pointer before the actual point, to receive - # exactly 16 bytes. after that the registers need to be adjusted. -_get_last_two_xmms: + add $16, buf + sub $16, len + jge .Lfold_16_bytes_loop + +.Lfold_16_bytes_loop_done: + # Add 16 to get the correct number of data bytes remaining in 0...15 + # (not counting xmm7), following the previous extra subtraction by 16. + add $16, len + je .Lreduce_final_16_bytes + +.Lhandle_partial_segment: + # Reduce the last '16 + len' bytes where 1 <= len <= 15 and the first 16 + # bytes are in xmm7 and the rest are the remaining data in 'buf'. To do + # this without needing a fold constant for each possible 'len', redivide + # the bytes into a first chunk of 'len' bytes and a second chunk of 16 + # bytes, then fold the first chunk into the second. + movdqa %xmm7, %xmm2 - movdqu -16(arg2, arg3), %xmm1 - pshufb %xmm11, %xmm1 + # xmm1 = last 16 original data bytes + movdqu -16(buf, len), %xmm1 + pshufb BSWAP_MASK, %xmm1 - # get rid of the extra data that was loaded before - # load the shift constant - lea pshufb_shf_table+16(%rip), %rax - sub arg3, %rax + # xmm2 = high order part of second chunk: xmm7 left-shifted by 'len' bytes. + lea .Lbyteshift_table+16(%rip), %rax + sub len, %rax movdqu (%rax), %xmm0 - - # shift xmm2 to the left by arg3 bytes pshufb %xmm0, %xmm2 - # shift xmm7 to the right by 16-arg3 bytes - pxor mask1(%rip), %xmm0 + # xmm7 = first chunk: xmm7 right-shifted by '16-len' bytes. + pxor .Lmask1(%rip), %xmm0 pshufb %xmm0, %xmm7 + + # xmm1 = second chunk: 'len' bytes from xmm1 (low-order bytes), + # then '16-len' bytes from xmm2 (high-order bytes). pblendvb %xmm2, %xmm1 #xmm0 is implicit - # fold 16 Bytes - movdqa %xmm1, %xmm2 + # Fold the first chunk into the second chunk, storing the result in xmm7. movdqa %xmm7, %xmm8 - pclmulqdq $0x11, %xmm10, %xmm7 - pclmulqdq $0x0 , %xmm10, %xmm8 + pclmulqdq $0x11, FOLD_CONSTS, %xmm7 + pclmulqdq $0x00, FOLD_CONSTS, %xmm8 pxor %xmm8, %xmm7 - pxor %xmm2, %xmm7 + pxor %xmm1, %xmm7 -_128_done: - # compute crc of a 128-bit value - movdqa rk5(%rip), %xmm10 # rk5 and rk6 in xmm10 - movdqa %xmm7, %xmm0 +.Lreduce_final_16_bytes: + # Reduce the 128-bit value M(x), stored in xmm7, to the final 16-bit CRC - #64b fold - pclmulqdq $0x1, %xmm10, %xmm7 - pslldq $8 , %xmm0 - pxor %xmm0, %xmm7 + # Load 'x^48 * (x^48 mod G(x))' and 'x^48 * (x^80 mod G(x))'. + movdqa .Lfinal_fold_consts(%rip), FOLD_CONSTS - #32b fold + # Fold the high 64 bits into the low 64 bits, while also multiplying by + # x^64. This produces a 128-bit value congruent to x^64 * M(x) and + # whose low 48 bits are 0. movdqa %xmm7, %xmm0 + pclmulqdq $0x11, FOLD_CONSTS, %xmm7 # high bits * x^48 * (x^80 mod G(x)) + pslldq $8, %xmm0 + pxor %xmm0, %xmm7 # + low bits * x^64 - pand mask2(%rip), %xmm0 - - psrldq $12, %xmm7 - pclmulqdq $0x10, %xmm10, %xmm7 - pxor %xmm0, %xmm7 - - #barrett reduction -_barrett: - movdqa rk7(%rip), %xmm10 # rk7 and rk8 in xmm10 + # Fold the high 32 bits into the low 96 bits. This produces a 96-bit + # value congruent to x^64 * M(x) and whose low 48 bits are 0. movdqa %xmm7, %xmm0 - pclmulqdq $0x01, %xmm10, %xmm7 - pslldq $4, %xmm7 - pclmulqdq $0x11, %xmm10, %xmm7 + pand .Lmask2(%rip), %xmm0 # zero high 32 bits + psrldq $12, %xmm7 # extract high 32 bits + pclmulqdq $0x00, FOLD_CONSTS, %xmm7 # high 32 bits * x^48 * (x^48 mod G(x)) + pxor %xmm0, %xmm7 # + low bits - pslldq $4, %xmm7 - pxor %xmm0, %xmm7 - pextrd $1, %xmm7, %eax + # Load G(x) and floor(x^48 / G(x)). + movdqa .Lbarrett_reduction_consts(%rip), FOLD_CONSTS -_cleanup: - # scale the result back to 16 bits - shr $16, %eax - mov %rcx, %rsp + # Use Barrett reduction to compute the final CRC value. + movdqa %xmm7, %xmm0 + pclmulqdq $0x11, FOLD_CONSTS, %xmm7 # high 32 bits * floor(x^48 / G(x)) + psrlq $32, %xmm7 # /= x^32 + pclmulqdq $0x00, FOLD_CONSTS, %xmm7 # *= G(x) + psrlq $48, %xmm0 + pxor %xmm7, %xmm0 # + low 16 nonzero bits + # Final CRC value (x^16 * M(x)) mod G(x) is in low 16 bits of xmm0. + + pextrw $0, %xmm0, %eax ret -######################################################################## - .align 16 -_less_than_128: - - # check if there is enough buffer to be able to fold 16B at a time - cmp $32, arg3 - jl _less_than_32 - movdqa SHUF_MASK(%rip), %xmm11 +.Lless_than_256_bytes: + # Checksumming a buffer of length 16...255 bytes - # now if there is, load the constants - movdqa rk1(%rip), %xmm10 # rk1 and rk2 in xmm10 + # Load the first 16 data bytes. + movdqu (buf), %xmm7 + pshufb BSWAP_MASK, %xmm7 + add $16, buf - movd arg1_low32, %xmm0 # get the initial crc value - pslldq $12, %xmm0 # align it to its correct place - movdqu (arg2), %xmm7 # load the plaintext - pshufb %xmm11, %xmm7 # byte-reflect the plaintext + # XOR the first 16 data *bits* with the initial CRC value. + pxor %xmm0, %xmm0 + pinsrw $7, init_crc, %xmm0 pxor %xmm0, %xmm7 - - # update the buffer pointer - add $16, arg2 - - # update the counter. subtract 32 instead of 16 to save one - # instruction from the loop - sub $32, arg3 - - jmp _16B_reduction_loop - - -.align 16 -_less_than_32: - # mov initial crc to the return value. this is necessary for - # zero-length buffers. - mov arg1_low32, %eax - test arg3, arg3 - je _cleanup - - movdqa SHUF_MASK(%rip), %xmm11 - - movd arg1_low32, %xmm0 # get the initial crc value - pslldq $12, %xmm0 # align it to its correct place - - cmp $16, arg3 - je _exact_16_left - jl _less_than_16_left - - movdqu (arg2), %xmm7 # load the plaintext - pshufb %xmm11, %xmm7 # byte-reflect the plaintext - pxor %xmm0 , %xmm7 # xor the initial crc value - add $16, arg2 - sub $16, arg3 - movdqa rk1(%rip), %xmm10 # rk1 and rk2 in xmm10 - jmp _get_last_two_xmms - - -.align 16 -_less_than_16_left: - # use stack space to load data less than 16 bytes, zero-out - # the 16B in memory first. - - pxor %xmm1, %xmm1 - mov %rsp, %r11 - movdqa %xmm1, (%r11) - - cmp $4, arg3 - jl _only_less_than_4 - - # backup the counter value - mov arg3, %r9 - cmp $8, arg3 - jl _less_than_8_left - - # load 8 Bytes - mov (arg2), %rax - mov %rax, (%r11) - add $8, %r11 - sub $8, arg3 - add $8, arg2 -_less_than_8_left: - - cmp $4, arg3 - jl _less_than_4_left - - # load 4 Bytes - mov (arg2), %eax - mov %eax, (%r11) - add $4, %r11 - sub $4, arg3 - add $4, arg2 -_less_than_4_left: - - cmp $2, arg3 - jl _less_than_2_left - - # load 2 Bytes - mov (arg2), %ax - mov %ax, (%r11) - add $2, %r11 - sub $2, arg3 - add $2, arg2 -_less_than_2_left: - cmp $1, arg3 - jl _zero_left - - # load 1 Byte - mov (arg2), %al - mov %al, (%r11) -_zero_left: - movdqa (%rsp), %xmm7 - pshufb %xmm11, %xmm7 - pxor %xmm0 , %xmm7 # xor the initial crc value - - # shl r9, 4 - lea pshufb_shf_table+16(%rip), %rax - sub %r9, %rax - movdqu (%rax), %xmm0 - pxor mask1(%rip), %xmm0 - - pshufb %xmm0, %xmm7 - jmp _128_done - -.align 16 -_exact_16_left: - movdqu (arg2), %xmm7 - pshufb %xmm11, %xmm7 - pxor %xmm0 , %xmm7 # xor the initial crc value - - jmp _128_done - -_only_less_than_4: - cmp $3, arg3 - jl _only_less_than_3 - - # load 3 Bytes - mov (arg2), %al - mov %al, (%r11) - - mov 1(arg2), %al - mov %al, 1(%r11) - - mov 2(arg2), %al - mov %al, 2(%r11) - - movdqa (%rsp), %xmm7 - pshufb %xmm11, %xmm7 - pxor %xmm0 , %xmm7 # xor the initial crc value - - psrldq $5, %xmm7 - - jmp _barrett -_only_less_than_3: - cmp $2, arg3 - jl _only_less_than_2 - - # load 2 Bytes - mov (arg2), %al - mov %al, (%r11) - - mov 1(arg2), %al - mov %al, 1(%r11) - - movdqa (%rsp), %xmm7 - pshufb %xmm11, %xmm7 - pxor %xmm0 , %xmm7 # xor the initial crc value - - psrldq $6, %xmm7 - - jmp _barrett -_only_less_than_2: - - # load 1 Byte - mov (arg2), %al - mov %al, (%r11) - - movdqa (%rsp), %xmm7 - pshufb %xmm11, %xmm7 - pxor %xmm0 , %xmm7 # xor the initial crc value - - psrldq $7, %xmm7 - - jmp _barrett - + movdqa .Lfold_across_16_bytes_consts(%rip), FOLD_CONSTS + cmp $16, len + je .Lreduce_final_16_bytes # len == 16 + sub $32, len + jge .Lfold_16_bytes_loop # 32 <= len <= 255 + add $16, len + jmp .Lhandle_partial_segment # 17 <= len <= 31 ENDPROC(crc_t10dif_pcl) .section .rodata, "a", @progbits .align 16 -# precomputed constants -# these constants are precomputed from the poly: -# 0x8bb70000 (0x8bb7 scaled to 32 bits) -# Q = 0x18BB70000 -# rk1 = 2^(32*3) mod Q << 32 -# rk2 = 2^(32*5) mod Q << 32 -# rk3 = 2^(32*15) mod Q << 32 -# rk4 = 2^(32*17) mod Q << 32 -# rk5 = 2^(32*3) mod Q << 32 -# rk6 = 2^(32*2) mod Q << 32 -# rk7 = floor(2^64/Q) -# rk8 = Q -rk1: -.quad 0x2d56000000000000 -rk2: -.quad 0x06df000000000000 -rk3: -.quad 0x9d9d000000000000 -rk4: -.quad 0x7cf5000000000000 -rk5: -.quad 0x2d56000000000000 -rk6: -.quad 0x1368000000000000 -rk7: -.quad 0x00000001f65a57f8 -rk8: -.quad 0x000000018bb70000 - -rk9: -.quad 0xceae000000000000 -rk10: -.quad 0xbfd6000000000000 -rk11: -.quad 0x1e16000000000000 -rk12: -.quad 0x713c000000000000 -rk13: -.quad 0xf7f9000000000000 -rk14: -.quad 0x80a6000000000000 -rk15: -.quad 0x044c000000000000 -rk16: -.quad 0xe658000000000000 -rk17: -.quad 0xad18000000000000 -rk18: -.quad 0xa497000000000000 -rk19: -.quad 0x6ee3000000000000 -rk20: -.quad 0xe7b5000000000000 - +# Fold constants precomputed from the polynomial 0x18bb7 +# G(x) = x^16 + x^15 + x^11 + x^9 + x^8 + x^7 + x^5 + x^4 + x^2 + x^1 + x^0 +.Lfold_across_128_bytes_consts: + .quad 0x0000000000006123 # x^(8*128) mod G(x) + .quad 0x0000000000002295 # x^(8*128+64) mod G(x) +.Lfold_across_64_bytes_consts: + .quad 0x0000000000001069 # x^(4*128) mod G(x) + .quad 0x000000000000dd31 # x^(4*128+64) mod G(x) +.Lfold_across_32_bytes_consts: + .quad 0x000000000000857d # x^(2*128) mod G(x) + .quad 0x0000000000007acc # x^(2*128+64) mod G(x) +.Lfold_across_16_bytes_consts: + .quad 0x000000000000a010 # x^(1*128) mod G(x) + .quad 0x0000000000001faa # x^(1*128+64) mod G(x) +.Lfinal_fold_consts: + .quad 0x1368000000000000 # x^48 * (x^48 mod G(x)) + .quad 0x2d56000000000000 # x^48 * (x^80 mod G(x)) +.Lbarrett_reduction_consts: + .quad 0x0000000000018bb7 # G(x) + .quad 0x00000001f65a57f8 # floor(x^48 / G(x)) .section .rodata.cst16.mask1, "aM", @progbits, 16 .align 16 -mask1: -.octa 0x80808080808080808080808080808080 +.Lmask1: + .octa 0x80808080808080808080808080808080 .section .rodata.cst16.mask2, "aM", @progbits, 16 .align 16 -mask2: -.octa 0x00000000FFFFFFFFFFFFFFFFFFFFFFFF +.Lmask2: + .octa 0x00000000FFFFFFFFFFFFFFFFFFFFFFFF + +.section .rodata.cst16.bswap_mask, "aM", @progbits, 16 +.align 16 +.Lbswap_mask: + .octa 0x000102030405060708090A0B0C0D0E0F -.section .rodata.cst16.SHUF_MASK, "aM", @progbits, 16 +.section .rodata.cst32.byteshift_table, "aM", @progbits, 32 .align 16 -SHUF_MASK: -.octa 0x000102030405060708090A0B0C0D0E0F - -.section .rodata.cst32.pshufb_shf_table, "aM", @progbits, 32 -.align 32 -pshufb_shf_table: -# use these values for shift constants for the pshufb instruction -# different alignments result in values as shown: -# DDQ 0x008f8e8d8c8b8a898887868584838281 # shl 15 (16-1) / shr1 -# DDQ 0x01008f8e8d8c8b8a8988878685848382 # shl 14 (16-3) / shr2 -# DDQ 0x0201008f8e8d8c8b8a89888786858483 # shl 13 (16-4) / shr3 -# DDQ 0x030201008f8e8d8c8b8a898887868584 # shl 12 (16-4) / shr4 -# DDQ 0x04030201008f8e8d8c8b8a8988878685 # shl 11 (16-5) / shr5 -# DDQ 0x0504030201008f8e8d8c8b8a89888786 # shl 10 (16-6) / shr6 -# DDQ 0x060504030201008f8e8d8c8b8a898887 # shl 9 (16-7) / shr7 -# DDQ 0x07060504030201008f8e8d8c8b8a8988 # shl 8 (16-8) / shr8 -# DDQ 0x0807060504030201008f8e8d8c8b8a89 # shl 7 (16-9) / shr9 -# DDQ 0x090807060504030201008f8e8d8c8b8a # shl 6 (16-10) / shr10 -# DDQ 0x0a090807060504030201008f8e8d8c8b # shl 5 (16-11) / shr11 -# DDQ 0x0b0a090807060504030201008f8e8d8c # shl 4 (16-12) / shr12 -# DDQ 0x0c0b0a090807060504030201008f8e8d # shl 3 (16-13) / shr13 -# DDQ 0x0d0c0b0a090807060504030201008f8e # shl 2 (16-14) / shr14 -# DDQ 0x0e0d0c0b0a090807060504030201008f # shl 1 (16-15) / shr15 -.octa 0x8f8e8d8c8b8a89888786858483828100 -.octa 0x000e0d0c0b0a09080706050403020100 +# For 1 <= len <= 15, the 16-byte vector beginning at &byteshift_table[16 - len] +# is the index vector to shift left by 'len' bytes, and is also {0x80, ..., +# 0x80} XOR the index vector to shift right by '16 - len' bytes. +.Lbyteshift_table: + .byte 0x0, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87 + .byte 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f + .byte 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7 + .byte 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe , 0x0 diff --git a/arch/x86/crypto/crct10dif-pclmul_glue.c b/arch/x86/crypto/crct10dif-pclmul_glue.c index cd4df9322501..0e785c0b2354 100644 --- a/arch/x86/crypto/crct10dif-pclmul_glue.c +++ b/arch/x86/crypto/crct10dif-pclmul_glue.c @@ -33,18 +33,12 @@ #include #include -asmlinkage __u16 crc_t10dif_pcl(__u16 crc, const unsigned char *buf, - size_t len); +asmlinkage u16 crc_t10dif_pcl(u16 init_crc, const u8 *buf, size_t len); struct chksum_desc_ctx { __u16 crc; }; -/* - * Steps through buffer one byte at at time, calculates reflected - * crc using table. - */ - static int chksum_init(struct shash_desc *desc) { struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); @@ -59,7 +53,7 @@ static int chksum_update(struct shash_desc *desc, const u8 *data, { struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); - if (irq_fpu_usable()) { + if (length >= 16 && irq_fpu_usable()) { kernel_fpu_begin(); ctx->crc = crc_t10dif_pcl(ctx->crc, data, length); kernel_fpu_end(); @@ -79,7 +73,7 @@ static int chksum_final(struct shash_desc *desc, u8 *out) static int __chksum_finup(__u16 *crcp, const u8 *data, unsigned int len, u8 *out) { - if (irq_fpu_usable()) { + if (len >= 16 && irq_fpu_usable()) { kernel_fpu_begin(); *(__u16 *)out = crc_t10dif_pcl(*crcp, data, len); kernel_fpu_end(); -- cgit v1.2.3 From e7b3ed338097b1237ca03b05fe36ec1517d328d5 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 30 Jan 2019 20:42:41 -0800 Subject: crypto: arm/crct10dif-ce - cleanup and optimizations The x86, arm, and arm64 asm implementations of crct10dif are very difficult to understand partly because many of the comments, labels, and macros are named incorrectly: the lengths mentioned are usually off by a factor of two from the actual code. Many other things are unnecessarily convoluted as well, e.g. there are many more fold constants than actually needed and some aren't fully reduced. This series therefore cleans up all these implementations to be much more maintainable. I also made some small optimizations where I saw opportunities, resulting in slightly better performance. This patch cleans up the arm version. (Also moved the constants to .rodata as suggested by Ard Biesheuvel.) Acked-by: Ard Biesheuvel Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm/crypto/crct10dif-ce-core.S | 553 +++++++++++++++++------------------- arch/arm/crypto/crct10dif-ce-glue.c | 2 +- 2 files changed, 261 insertions(+), 294 deletions(-) (limited to 'arch') diff --git a/arch/arm/crypto/crct10dif-ce-core.S b/arch/arm/crypto/crct10dif-ce-core.S index d058fad423c2..86be258a803f 100644 --- a/arch/arm/crypto/crct10dif-ce-core.S +++ b/arch/arm/crypto/crct10dif-ce-core.S @@ -2,12 +2,14 @@ // Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions instructions // // Copyright (C) 2016 Linaro Ltd +// Copyright (C) 2019 Google LLC // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License version 2 as // published by the Free Software Foundation. // +// Derived from the x86 version: // // Implement fast CRC-T10DIF computation with SSE and PCLMULQDQ instructions // @@ -54,19 +56,11 @@ // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -// Function API: -// UINT16 crc_t10dif_pcl( -// UINT16 init_crc, //initial CRC value, 16 bits -// const unsigned char *buf, //buffer pointer to calculate CRC on -// UINT64 len //buffer length in bytes (64-bit data) -// ); -// // Reference paper titled "Fast CRC Computation for Generic // Polynomials Using PCLMULQDQ Instruction" // URL: http://www.intel.com/content/dam/www/public/us/en/documents // /white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf // -// #include #include @@ -78,13 +72,14 @@ #endif .text + .arch armv7-a .fpu crypto-neon-fp-armv8 - arg1_low32 .req r0 - arg2 .req r1 - arg3 .req r2 + init_crc .req r0 + buf .req r1 + len .req r2 - qzr .req q13 + fold_consts_ptr .req ip q0l .req d0 q0h .req d1 @@ -102,82 +97,35 @@ q6h .req d13 q7l .req d14 q7h .req d15 - -ENTRY(crc_t10dif_pmull) - vmov.i8 qzr, #0 // init zero register - - // adjust the 16-bit initial_crc value, scale it to 32 bits - lsl arg1_low32, arg1_low32, #16 - - // check if smaller than 256 - cmp arg3, #256 - - // for sizes less than 128, we can't fold 64B at a time... - blt _less_than_128 - - // load the initial crc value - // crc value does not need to be byte-reflected, but it needs - // to be moved to the high part of the register. - // because data will be byte-reflected and will align with - // initial crc at correct place. - vmov s0, arg1_low32 // initial crc - vext.8 q10, qzr, q0, #4 - - // receive the initial 64B data, xor the initial crc value - vld1.64 {q0-q1}, [arg2]! - vld1.64 {q2-q3}, [arg2]! - vld1.64 {q4-q5}, [arg2]! - vld1.64 {q6-q7}, [arg2]! -CPU_LE( vrev64.8 q0, q0 ) -CPU_LE( vrev64.8 q1, q1 ) -CPU_LE( vrev64.8 q2, q2 ) -CPU_LE( vrev64.8 q3, q3 ) -CPU_LE( vrev64.8 q4, q4 ) -CPU_LE( vrev64.8 q5, q5 ) -CPU_LE( vrev64.8 q6, q6 ) -CPU_LE( vrev64.8 q7, q7 ) - - vswp d0, d1 - vswp d2, d3 - vswp d4, d5 - vswp d6, d7 - vswp d8, d9 - vswp d10, d11 - vswp d12, d13 - vswp d14, d15 - - // XOR the initial_crc value - veor.8 q0, q0, q10 - - adr ip, rk3 - vld1.64 {q10}, [ip, :128] // xmm10 has rk3 and rk4 - - // - // we subtract 256 instead of 128 to save one instruction from the loop - // - sub arg3, arg3, #256 - - // at this section of the code, there is 64*x+y (0<=y<64) bytes of - // buffer. The _fold_64_B_loop will fold 64B at a time - // until we have 64+y Bytes of buffer - - - // fold 64B at a time. This section of the code folds 4 vector - // registers in parallel -_fold_64_B_loop: - - .macro fold64, reg1, reg2 - vld1.64 {q11-q12}, [arg2]! - - vmull.p64 q8, \reg1\()h, d21 - vmull.p64 \reg1, \reg1\()l, d20 - vmull.p64 q9, \reg2\()h, d21 - vmull.p64 \reg2, \reg2\()l, d20 - -CPU_LE( vrev64.8 q11, q11 ) -CPU_LE( vrev64.8 q12, q12 ) - vswp d22, d23 - vswp d24, d25 + q8l .req d16 + q8h .req d17 + q9l .req d18 + q9h .req d19 + q10l .req d20 + q10h .req d21 + q11l .req d22 + q11h .req d23 + q12l .req d24 + q12h .req d25 + + FOLD_CONSTS .req q10 + FOLD_CONST_L .req q10l + FOLD_CONST_H .req q10h + + // Fold reg1, reg2 into the next 32 data bytes, storing the result back + // into reg1, reg2. + .macro fold_32_bytes, reg1, reg2 + vld1.64 {q11-q12}, [buf]! + + vmull.p64 q8, \reg1\()h, FOLD_CONST_H + vmull.p64 \reg1, \reg1\()l, FOLD_CONST_L + vmull.p64 q9, \reg2\()h, FOLD_CONST_H + vmull.p64 \reg2, \reg2\()l, FOLD_CONST_L + +CPU_LE( vrev64.8 q11, q11 ) +CPU_LE( vrev64.8 q12, q12 ) + vswp q11l, q11h + vswp q12l, q12h veor.8 \reg1, \reg1, q8 veor.8 \reg2, \reg2, q9 @@ -185,229 +133,248 @@ CPU_LE( vrev64.8 q12, q12 ) veor.8 \reg2, \reg2, q12 .endm - fold64 q0, q1 - fold64 q2, q3 - fold64 q4, q5 - fold64 q6, q7 - - subs arg3, arg3, #128 - - // check if there is another 64B in the buffer to be able to fold - bge _fold_64_B_loop - - // at this point, the buffer pointer is pointing at the last y Bytes - // of the buffer the 64B of folded data is in 4 of the vector - // registers: v0, v1, v2, v3 - - // fold the 8 vector registers to 1 vector register with different - // constants - - adr ip, rk9 - vld1.64 {q10}, [ip, :128]! - - .macro fold16, reg, rk - vmull.p64 q8, \reg\()l, d20 - vmull.p64 \reg, \reg\()h, d21 - .ifnb \rk - vld1.64 {q10}, [ip, :128]! + // Fold src_reg into dst_reg, optionally loading the next fold constants + .macro fold_16_bytes, src_reg, dst_reg, load_next_consts + vmull.p64 q8, \src_reg\()l, FOLD_CONST_L + vmull.p64 \src_reg, \src_reg\()h, FOLD_CONST_H + .ifnb \load_next_consts + vld1.64 {FOLD_CONSTS}, [fold_consts_ptr, :128]! .endif - veor.8 q7, q7, q8 - veor.8 q7, q7, \reg + veor.8 \dst_reg, \dst_reg, q8 + veor.8 \dst_reg, \dst_reg, \src_reg .endm - fold16 q0, rk11 - fold16 q1, rk13 - fold16 q2, rk15 - fold16 q3, rk17 - fold16 q4, rk19 - fold16 q5, rk1 - fold16 q6 - - // instead of 64, we add 48 to the loop counter to save 1 instruction - // from the loop instead of a cmp instruction, we use the negative - // flag with the jl instruction - adds arg3, arg3, #(128-16) - blt _final_reduction_for_128 - - // now we have 16+y bytes left to reduce. 16 Bytes is in register v7 - // and the rest is in memory. We can fold 16 bytes at a time if y>=16 - // continue folding 16B at a time - -_16B_reduction_loop: - vmull.p64 q8, d14, d20 - vmull.p64 q7, d15, d21 - veor.8 q7, q7, q8 - - vld1.64 {q0}, [arg2]! -CPU_LE( vrev64.8 q0, q0 ) - vswp d0, d1 - veor.8 q7, q7, q0 - subs arg3, arg3, #16 - - // instead of a cmp instruction, we utilize the flags with the - // jge instruction equivalent of: cmp arg3, 16-16 - // check if there is any more 16B in the buffer to be able to fold - bge _16B_reduction_loop - - // now we have 16+z bytes left to reduce, where 0<= z < 16. - // first, we reduce the data in the xmm7 register - -_final_reduction_for_128: - // check if any more data to fold. If not, compute the CRC of - // the final 128 bits - adds arg3, arg3, #16 - beq _128_done - - // here we are getting data that is less than 16 bytes. - // since we know that there was data before the pointer, we can - // offset the input pointer before the actual point, to receive - // exactly 16 bytes. after that the registers need to be adjusted. -_get_last_two_regs: - add arg2, arg2, arg3 - sub arg2, arg2, #16 - vld1.64 {q1}, [arg2] -CPU_LE( vrev64.8 q1, q1 ) - vswp d2, d3 - - // get rid of the extra data that was loaded before - // load the shift constant - adr ip, tbl_shf_table + 16 - sub ip, ip, arg3 - vld1.8 {q0}, [ip] - - // shift v2 to the left by arg3 bytes - vtbl.8 d4, {d14-d15}, d0 - vtbl.8 d5, {d14-d15}, d1 - - // shift v7 to the right by 16-arg3 bytes - vmov.i8 q9, #0x80 - veor.8 q0, q0, q9 - vtbl.8 d18, {d14-d15}, d0 - vtbl.8 d19, {d14-d15}, d1 - - // blend - vshr.s8 q0, q0, #7 // convert to 8-bit mask - vbsl.8 q0, q2, q1 - - // fold 16 Bytes - vmull.p64 q8, d18, d20 - vmull.p64 q7, d19, d21 - veor.8 q7, q7, q8 - veor.8 q7, q7, q0 + .macro __adrl, out, sym + movw \out, #:lower16:\sym + movt \out, #:upper16:\sym + .endm -_128_done: - // compute crc of a 128-bit value - vldr d20, rk5 - vldr d21, rk6 // rk5 and rk6 in xmm10 +// +// u16 crc_t10dif_pmull(u16 init_crc, const u8 *buf, size_t len); +// +// Assumes len >= 16. +// +ENTRY(crc_t10dif_pmull) - // 64b fold - vext.8 q0, qzr, q7, #8 - vmull.p64 q7, d15, d20 + // For sizes less than 256 bytes, we can't fold 128 bytes at a time. + cmp len, #256 + blt .Lless_than_256_bytes + + __adrl fold_consts_ptr, .Lfold_across_128_bytes_consts + + // Load the first 128 data bytes. Byte swapping is necessary to make + // the bit order match the polynomial coefficient order. + vld1.64 {q0-q1}, [buf]! + vld1.64 {q2-q3}, [buf]! + vld1.64 {q4-q5}, [buf]! + vld1.64 {q6-q7}, [buf]! +CPU_LE( vrev64.8 q0, q0 ) +CPU_LE( vrev64.8 q1, q1 ) +CPU_LE( vrev64.8 q2, q2 ) +CPU_LE( vrev64.8 q3, q3 ) +CPU_LE( vrev64.8 q4, q4 ) +CPU_LE( vrev64.8 q5, q5 ) +CPU_LE( vrev64.8 q6, q6 ) +CPU_LE( vrev64.8 q7, q7 ) + vswp q0l, q0h + vswp q1l, q1h + vswp q2l, q2h + vswp q3l, q3h + vswp q4l, q4h + vswp q5l, q5h + vswp q6l, q6h + vswp q7l, q7h + + // XOR the first 16 data *bits* with the initial CRC value. + vmov.i8 q8h, #0 + vmov.u16 q8h[3], init_crc + veor q0h, q0h, q8h + + // Load the constants for folding across 128 bytes. + vld1.64 {FOLD_CONSTS}, [fold_consts_ptr, :128]! + + // Subtract 128 for the 128 data bytes just consumed. Subtract another + // 128 to simplify the termination condition of the following loop. + sub len, len, #256 + + // While >= 128 data bytes remain (not counting q0-q7), fold the 128 + // bytes q0-q7 into them, storing the result back into q0-q7. +.Lfold_128_bytes_loop: + fold_32_bytes q0, q1 + fold_32_bytes q2, q3 + fold_32_bytes q4, q5 + fold_32_bytes q6, q7 + subs len, len, #128 + bge .Lfold_128_bytes_loop + + // Now fold the 112 bytes in q0-q6 into the 16 bytes in q7. + + // Fold across 64 bytes. + vld1.64 {FOLD_CONSTS}, [fold_consts_ptr, :128]! + fold_16_bytes q0, q4 + fold_16_bytes q1, q5 + fold_16_bytes q2, q6 + fold_16_bytes q3, q7, 1 + // Fold across 32 bytes. + fold_16_bytes q4, q6 + fold_16_bytes q5, q7, 1 + // Fold across 16 bytes. + fold_16_bytes q6, q7 + + // Add 128 to get the correct number of data bytes remaining in 0...127 + // (not counting q7), following the previous extra subtraction by 128. + // Then subtract 16 to simplify the termination condition of the + // following loop. + adds len, len, #(128-16) + + // While >= 16 data bytes remain (not counting q7), fold the 16 bytes q7 + // into them, storing the result back into q7. + blt .Lfold_16_bytes_loop_done +.Lfold_16_bytes_loop: + vmull.p64 q8, q7l, FOLD_CONST_L + vmull.p64 q7, q7h, FOLD_CONST_H + veor.8 q7, q7, q8 + vld1.64 {q0}, [buf]! +CPU_LE( vrev64.8 q0, q0 ) + vswp q0l, q0h veor.8 q7, q7, q0 - - // 32b fold - vext.8 q0, q7, qzr, #12 - vmov s31, s3 - vmull.p64 q0, d0, d21 - veor.8 q7, q0, q7 - - // barrett reduction -_barrett: - vldr d20, rk7 - vldr d21, rk8 - - vmull.p64 q0, d15, d20 - vext.8 q0, qzr, q0, #12 - vmull.p64 q0, d1, d21 - vext.8 q0, qzr, q0, #12 + subs len, len, #16 + bge .Lfold_16_bytes_loop + +.Lfold_16_bytes_loop_done: + // Add 16 to get the correct number of data bytes remaining in 0...15 + // (not counting q7), following the previous extra subtraction by 16. + adds len, len, #16 + beq .Lreduce_final_16_bytes + +.Lhandle_partial_segment: + // Reduce the last '16 + len' bytes where 1 <= len <= 15 and the first + // 16 bytes are in q7 and the rest are the remaining data in 'buf'. To + // do this without needing a fold constant for each possible 'len', + // redivide the bytes into a first chunk of 'len' bytes and a second + // chunk of 16 bytes, then fold the first chunk into the second. + + // q0 = last 16 original data bytes + add buf, buf, len + sub buf, buf, #16 + vld1.64 {q0}, [buf] +CPU_LE( vrev64.8 q0, q0 ) + vswp q0l, q0h + + // q1 = high order part of second chunk: q7 left-shifted by 'len' bytes. + __adrl r3, .Lbyteshift_table + 16 + sub r3, r3, len + vld1.8 {q2}, [r3] + vtbl.8 q1l, {q7l-q7h}, q2l + vtbl.8 q1h, {q7l-q7h}, q2h + + // q3 = first chunk: q7 right-shifted by '16-len' bytes. + vmov.i8 q3, #0x80 + veor.8 q2, q2, q3 + vtbl.8 q3l, {q7l-q7h}, q2l + vtbl.8 q3h, {q7l-q7h}, q2h + + // Convert to 8-bit masks: 'len' 0x00 bytes, then '16-len' 0xff bytes. + vshr.s8 q2, q2, #7 + + // q2 = second chunk: 'len' bytes from q0 (low-order bytes), + // then '16-len' bytes from q1 (high-order bytes). + vbsl.8 q2, q1, q0 + + // Fold the first chunk into the second chunk, storing the result in q7. + vmull.p64 q0, q3l, FOLD_CONST_L + vmull.p64 q7, q3h, FOLD_CONST_H veor.8 q7, q7, q0 - vmov r0, s29 - -_cleanup: - // scale the result back to 16 bits - lsr r0, r0, #16 + veor.8 q7, q7, q2 + +.Lreduce_final_16_bytes: + // Reduce the 128-bit value M(x), stored in q7, to the final 16-bit CRC. + + // Load 'x^48 * (x^48 mod G(x))' and 'x^48 * (x^80 mod G(x))'. + vld1.64 {FOLD_CONSTS}, [fold_consts_ptr, :128]! + + // Fold the high 64 bits into the low 64 bits, while also multiplying by + // x^64. This produces a 128-bit value congruent to x^64 * M(x) and + // whose low 48 bits are 0. + vmull.p64 q0, q7h, FOLD_CONST_H // high bits * x^48 * (x^80 mod G(x)) + veor.8 q0h, q0h, q7l // + low bits * x^64 + + // Fold the high 32 bits into the low 96 bits. This produces a 96-bit + // value congruent to x^64 * M(x) and whose low 48 bits are 0. + vmov.i8 q1, #0 + vmov s4, s3 // extract high 32 bits + vmov s3, s5 // zero high 32 bits + vmull.p64 q1, q1l, FOLD_CONST_L // high 32 bits * x^48 * (x^48 mod G(x)) + veor.8 q0, q0, q1 // + low bits + + // Load G(x) and floor(x^48 / G(x)). + vld1.64 {FOLD_CONSTS}, [fold_consts_ptr, :128] + + // Use Barrett reduction to compute the final CRC value. + vmull.p64 q1, q0h, FOLD_CONST_H // high 32 bits * floor(x^48 / G(x)) + vshr.u64 q1l, q1l, #32 // /= x^32 + vmull.p64 q1, q1l, FOLD_CONST_L // *= G(x) + vshr.u64 q0l, q0l, #48 + veor.8 q0l, q0l, q1l // + low 16 nonzero bits + // Final CRC value (x^16 * M(x)) mod G(x) is in low 16 bits of q0. + + vmov.u16 r0, q0l[0] bx lr -_less_than_128: - teq arg3, #0 - beq _cleanup +.Lless_than_256_bytes: + // Checksumming a buffer of length 16...255 bytes - vmov.i8 q0, #0 - vmov s3, arg1_low32 // get the initial crc value + __adrl fold_consts_ptr, .Lfold_across_16_bytes_consts - vld1.64 {q7}, [arg2]! -CPU_LE( vrev64.8 q7, q7 ) - vswp d14, d15 - veor.8 q7, q7, q0 + // Load the first 16 data bytes. + vld1.64 {q7}, [buf]! +CPU_LE( vrev64.8 q7, q7 ) + vswp q7l, q7h - cmp arg3, #16 - beq _128_done // exactly 16 left + // XOR the first 16 data *bits* with the initial CRC value. + vmov.i8 q0h, #0 + vmov.u16 q0h[3], init_crc + veor.8 q7h, q7h, q0h - // now if there is, load the constants - vldr d20, rk1 - vldr d21, rk2 // rk1 and rk2 in xmm10 + // Load the fold-across-16-bytes constants. + vld1.64 {FOLD_CONSTS}, [fold_consts_ptr, :128]! - // check if there is enough buffer to be able to fold 16B at a time - subs arg3, arg3, #32 - addlt arg3, arg3, #16 - blt _get_last_two_regs - b _16B_reduction_loop + cmp len, #16 + beq .Lreduce_final_16_bytes // len == 16 + subs len, len, #32 + addlt len, len, #16 + blt .Lhandle_partial_segment // 17 <= len <= 31 + b .Lfold_16_bytes_loop // 32 <= len <= 255 ENDPROC(crc_t10dif_pmull) -// precomputed constants -// these constants are precomputed from the poly: -// 0x8bb70000 (0x8bb7 scaled to 32 bits) + .section ".rodata", "a" .align 4 -// Q = 0x18BB70000 -// rk1 = 2^(32*3) mod Q << 32 -// rk2 = 2^(32*5) mod Q << 32 -// rk3 = 2^(32*15) mod Q << 32 -// rk4 = 2^(32*17) mod Q << 32 -// rk5 = 2^(32*3) mod Q << 32 -// rk6 = 2^(32*2) mod Q << 32 -// rk7 = floor(2^64/Q) -// rk8 = Q - -rk3: .quad 0x9d9d000000000000 -rk4: .quad 0x7cf5000000000000 -rk5: .quad 0x2d56000000000000 -rk6: .quad 0x1368000000000000 -rk7: .quad 0x00000001f65a57f8 -rk8: .quad 0x000000018bb70000 -rk9: .quad 0xceae000000000000 -rk10: .quad 0xbfd6000000000000 -rk11: .quad 0x1e16000000000000 -rk12: .quad 0x713c000000000000 -rk13: .quad 0xf7f9000000000000 -rk14: .quad 0x80a6000000000000 -rk15: .quad 0x044c000000000000 -rk16: .quad 0xe658000000000000 -rk17: .quad 0xad18000000000000 -rk18: .quad 0xa497000000000000 -rk19: .quad 0x6ee3000000000000 -rk20: .quad 0xe7b5000000000000 -rk1: .quad 0x2d56000000000000 -rk2: .quad 0x06df000000000000 - -tbl_shf_table: -// use these values for shift constants for the tbl/tbx instruction -// different alignments result in values as shown: -// DDQ 0x008f8e8d8c8b8a898887868584838281 # shl 15 (16-1) / shr1 -// DDQ 0x01008f8e8d8c8b8a8988878685848382 # shl 14 (16-3) / shr2 -// DDQ 0x0201008f8e8d8c8b8a89888786858483 # shl 13 (16-4) / shr3 -// DDQ 0x030201008f8e8d8c8b8a898887868584 # shl 12 (16-4) / shr4 -// DDQ 0x04030201008f8e8d8c8b8a8988878685 # shl 11 (16-5) / shr5 -// DDQ 0x0504030201008f8e8d8c8b8a89888786 # shl 10 (16-6) / shr6 -// DDQ 0x060504030201008f8e8d8c8b8a898887 # shl 9 (16-7) / shr7 -// DDQ 0x07060504030201008f8e8d8c8b8a8988 # shl 8 (16-8) / shr8 -// DDQ 0x0807060504030201008f8e8d8c8b8a89 # shl 7 (16-9) / shr9 -// DDQ 0x090807060504030201008f8e8d8c8b8a # shl 6 (16-10) / shr10 -// DDQ 0x0a090807060504030201008f8e8d8c8b # shl 5 (16-11) / shr11 -// DDQ 0x0b0a090807060504030201008f8e8d8c # shl 4 (16-12) / shr12 -// DDQ 0x0c0b0a090807060504030201008f8e8d # shl 3 (16-13) / shr13 -// DDQ 0x0d0c0b0a090807060504030201008f8e # shl 2 (16-14) / shr14 -// DDQ 0x0e0d0c0b0a090807060504030201008f # shl 1 (16-15) / shr15 +// Fold constants precomputed from the polynomial 0x18bb7 +// G(x) = x^16 + x^15 + x^11 + x^9 + x^8 + x^7 + x^5 + x^4 + x^2 + x^1 + x^0 +.Lfold_across_128_bytes_consts: + .quad 0x0000000000006123 // x^(8*128) mod G(x) + .quad 0x0000000000002295 // x^(8*128+64) mod G(x) +// .Lfold_across_64_bytes_consts: + .quad 0x0000000000001069 // x^(4*128) mod G(x) + .quad 0x000000000000dd31 // x^(4*128+64) mod G(x) +// .Lfold_across_32_bytes_consts: + .quad 0x000000000000857d // x^(2*128) mod G(x) + .quad 0x0000000000007acc // x^(2*128+64) mod G(x) +.Lfold_across_16_bytes_consts: + .quad 0x000000000000a010 // x^(1*128) mod G(x) + .quad 0x0000000000001faa // x^(1*128+64) mod G(x) +// .Lfinal_fold_consts: + .quad 0x1368000000000000 // x^48 * (x^48 mod G(x)) + .quad 0x2d56000000000000 // x^48 * (x^80 mod G(x)) +// .Lbarrett_reduction_consts: + .quad 0x0000000000018bb7 // G(x) + .quad 0x00000001f65a57f8 // floor(x^48 / G(x)) + +// For 1 <= len <= 15, the 16-byte vector beginning at &byteshift_table[16 - +// len] is the index vector to shift left by 'len' bytes, and is also {0x80, +// ..., 0x80} XOR the index vector to shift right by '16 - len' bytes. +.Lbyteshift_table: .byte 0x0, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87 .byte 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f .byte 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7 diff --git a/arch/arm/crypto/crct10dif-ce-glue.c b/arch/arm/crypto/crct10dif-ce-glue.c index 14c19c70a841..3d6b800b8396 100644 --- a/arch/arm/crypto/crct10dif-ce-glue.c +++ b/arch/arm/crypto/crct10dif-ce-glue.c @@ -21,7 +21,7 @@ #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U -asmlinkage u16 crc_t10dif_pmull(u16 init_crc, const u8 buf[], u32 len); +asmlinkage u16 crc_t10dif_pmull(u16 init_crc, const u8 *buf, size_t len); static int crct10dif_init(struct shash_desc *desc) { -- cgit v1.2.3 From 6227cd12e51c89c9af320e4e429120c8b83c6bd1 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 30 Jan 2019 20:42:42 -0800 Subject: crypto: arm64/crct10dif-ce - cleanup and optimizations The x86, arm, and arm64 asm implementations of crct10dif are very difficult to understand partly because many of the comments, labels, and macros are named incorrectly: the lengths mentioned are usually off by a factor of two from the actual code. Many other things are unnecessarily convoluted as well, e.g. there are many more fold constants than actually needed and some aren't fully reduced. This series therefore cleans up all these implementations to be much more maintainable. I also made some small optimizations where I saw opportunities, resulting in slightly better performance. This patch cleans up the arm64 version. Acked-by: Ard Biesheuvel Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm64/crypto/crct10dif-ce-core.S | 496 ++++++++++++++++------------------ arch/arm64/crypto/crct10dif-ce-glue.c | 4 +- 2 files changed, 233 insertions(+), 267 deletions(-) (limited to 'arch') diff --git a/arch/arm64/crypto/crct10dif-ce-core.S b/arch/arm64/crypto/crct10dif-ce-core.S index f7326259c40d..e545b42e6a46 100644 --- a/arch/arm64/crypto/crct10dif-ce-core.S +++ b/arch/arm64/crypto/crct10dif-ce-core.S @@ -2,12 +2,14 @@ // Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions // // Copyright (C) 2016 Linaro Ltd +// Copyright (C) 2019 Google LLC // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License version 2 as // published by the Free Software Foundation. // +// Derived from the x86 version: // // Implement fast CRC-T10DIF computation with SSE and PCLMULQDQ instructions // @@ -54,19 +56,11 @@ // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -// Function API: -// UINT16 crc_t10dif_pcl( -// UINT16 init_crc, //initial CRC value, 16 bits -// const unsigned char *buf, //buffer pointer to calculate CRC on -// UINT64 len //buffer length in bytes (64-bit data) -// ); -// // Reference paper titled "Fast CRC Computation for Generic // Polynomials Using PCLMULQDQ Instruction" // URL: http://www.intel.com/content/dam/www/public/us/en/documents // /white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf // -// #include #include @@ -74,14 +68,14 @@ .text .cpu generic+crypto - arg1_low32 .req w19 - arg2 .req x20 - arg3 .req x21 + init_crc .req w19 + buf .req x20 + len .req x21 + fold_consts_ptr .req x22 - vzr .req v13 + fold_consts .req v10 ad .req v14 - bd .req v10 k00_16 .req v15 k32_48 .req v16 @@ -143,11 +137,11 @@ __pmull_p8_core: ext t5.8b, ad.8b, ad.8b, #2 // A2 ext t6.8b, ad.8b, ad.8b, #3 // A3 - pmull t4.8h, t4.8b, bd.8b // F = A1*B + pmull t4.8h, t4.8b, fold_consts.8b // F = A1*B pmull t8.8h, ad.8b, bd1.8b // E = A*B1 - pmull t5.8h, t5.8b, bd.8b // H = A2*B + pmull t5.8h, t5.8b, fold_consts.8b // H = A2*B pmull t7.8h, ad.8b, bd2.8b // G = A*B2 - pmull t6.8h, t6.8b, bd.8b // J = A3*B + pmull t6.8h, t6.8b, fold_consts.8b // J = A3*B pmull t9.8h, ad.8b, bd3.8b // I = A*B3 pmull t3.8h, ad.8b, bd4.8b // K = A*B4 b 0f @@ -157,11 +151,11 @@ __pmull_p8_core: tbl t5.16b, {ad.16b}, perm2.16b // A2 tbl t6.16b, {ad.16b}, perm3.16b // A3 - pmull2 t4.8h, t4.16b, bd.16b // F = A1*B + pmull2 t4.8h, t4.16b, fold_consts.16b // F = A1*B pmull2 t8.8h, ad.16b, bd1.16b // E = A*B1 - pmull2 t5.8h, t5.16b, bd.16b // H = A2*B + pmull2 t5.8h, t5.16b, fold_consts.16b // H = A2*B pmull2 t7.8h, ad.16b, bd2.16b // G = A*B2 - pmull2 t6.8h, t6.16b, bd.16b // J = A3*B + pmull2 t6.8h, t6.16b, fold_consts.16b // J = A3*B pmull2 t9.8h, ad.16b, bd3.16b // I = A*B3 pmull2 t3.8h, ad.16b, bd4.16b // K = A*B4 @@ -203,14 +197,14 @@ __pmull_p8_core: ENDPROC(__pmull_p8_core) .macro __pmull_p8, rq, ad, bd, i - .ifnc \bd, v10 + .ifnc \bd, fold_consts .err .endif mov ad.16b, \ad\().16b .ifb \i - pmull \rq\().8h, \ad\().8b, bd.8b // D = A*B + pmull \rq\().8h, \ad\().8b, \bd\().8b // D = A*B .else - pmull2 \rq\().8h, \ad\().16b, bd.16b // D = A*B + pmull2 \rq\().8h, \ad\().16b, \bd\().16b // D = A*B .endif bl .L__pmull_p8_core\i @@ -219,17 +213,19 @@ ENDPROC(__pmull_p8_core) eor \rq\().16b, \rq\().16b, t6.16b .endm - .macro fold64, p, reg1, reg2 - ldp q11, q12, [arg2], #0x20 + // Fold reg1, reg2 into the next 32 data bytes, storing the result back + // into reg1, reg2. + .macro fold_32_bytes, p, reg1, reg2 + ldp q11, q12, [buf], #0x20 - __pmull_\p v8, \reg1, v10, 2 - __pmull_\p \reg1, \reg1, v10 + __pmull_\p v8, \reg1, fold_consts, 2 + __pmull_\p \reg1, \reg1, fold_consts CPU_LE( rev64 v11.16b, v11.16b ) CPU_LE( rev64 v12.16b, v12.16b ) - __pmull_\p v9, \reg2, v10, 2 - __pmull_\p \reg2, \reg2, v10 + __pmull_\p v9, \reg2, fold_consts, 2 + __pmull_\p \reg2, \reg2, fold_consts CPU_LE( ext v11.16b, v11.16b, v11.16b, #8 ) CPU_LE( ext v12.16b, v12.16b, v12.16b, #8 ) @@ -240,15 +236,16 @@ CPU_LE( ext v12.16b, v12.16b, v12.16b, #8 ) eor \reg2\().16b, \reg2\().16b, v12.16b .endm - .macro fold16, p, reg, rk - __pmull_\p v8, \reg, v10 - __pmull_\p \reg, \reg, v10, 2 - .ifnb \rk - ldr_l q10, \rk, x8 - __pmull_pre_\p v10 + // Fold src_reg into dst_reg, optionally loading the next fold constants + .macro fold_16_bytes, p, src_reg, dst_reg, load_next_consts + __pmull_\p v8, \src_reg, fold_consts + __pmull_\p \src_reg, \src_reg, fold_consts, 2 + .ifnb \load_next_consts + ld1 {fold_consts.2d}, [fold_consts_ptr], #16 + __pmull_pre_\p fold_consts .endif - eor v7.16b, v7.16b, v8.16b - eor v7.16b, v7.16b, \reg\().16b + eor \dst_reg\().16b, \dst_reg\().16b, v8.16b + eor \dst_reg\().16b, \dst_reg\().16b, \src_reg\().16b .endm .macro __pmull_p64, rd, rn, rm, n @@ -260,40 +257,27 @@ CPU_LE( ext v12.16b, v12.16b, v12.16b, #8 ) .endm .macro crc_t10dif_pmull, p - frame_push 3, 128 + frame_push 4, 128 - mov arg1_low32, w0 - mov arg2, x1 - mov arg3, x2 - - movi vzr.16b, #0 // init zero register + mov init_crc, w0 + mov buf, x1 + mov len, x2 __pmull_init_\p - // adjust the 16-bit initial_crc value, scale it to 32 bits - lsl arg1_low32, arg1_low32, #16 - - // check if smaller than 256 - cmp arg3, #256 - - // for sizes less than 128, we can't fold 64B at a time... - b.lt .L_less_than_128_\@ + // For sizes less than 256 bytes, we can't fold 128 bytes at a time. + cmp len, #256 + b.lt .Lless_than_256_bytes_\@ - // load the initial crc value - // crc value does not need to be byte-reflected, but it needs - // to be moved to the high part of the register. - // because data will be byte-reflected and will align with - // initial crc at correct place. - movi v10.16b, #0 - mov v10.s[3], arg1_low32 // initial crc - - // receive the initial 64B data, xor the initial crc value - ldp q0, q1, [arg2] - ldp q2, q3, [arg2, #0x20] - ldp q4, q5, [arg2, #0x40] - ldp q6, q7, [arg2, #0x60] - add arg2, arg2, #0x80 + adr_l fold_consts_ptr, .Lfold_across_128_bytes_consts + // Load the first 128 data bytes. Byte swapping is necessary to make + // the bit order match the polynomial coefficient order. + ldp q0, q1, [buf] + ldp q2, q3, [buf, #0x20] + ldp q4, q5, [buf, #0x40] + ldp q6, q7, [buf, #0x60] + add buf, buf, #0x80 CPU_LE( rev64 v0.16b, v0.16b ) CPU_LE( rev64 v1.16b, v1.16b ) CPU_LE( rev64 v2.16b, v2.16b ) @@ -302,7 +286,6 @@ CPU_LE( rev64 v4.16b, v4.16b ) CPU_LE( rev64 v5.16b, v5.16b ) CPU_LE( rev64 v6.16b, v6.16b ) CPU_LE( rev64 v7.16b, v7.16b ) - CPU_LE( ext v0.16b, v0.16b, v0.16b, #8 ) CPU_LE( ext v1.16b, v1.16b, v1.16b, #8 ) CPU_LE( ext v2.16b, v2.16b, v2.16b, #8 ) @@ -312,36 +295,29 @@ CPU_LE( ext v5.16b, v5.16b, v5.16b, #8 ) CPU_LE( ext v6.16b, v6.16b, v6.16b, #8 ) CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 ) - // XOR the initial_crc value - eor v0.16b, v0.16b, v10.16b - - ldr_l q10, rk3, x8 // xmm10 has rk3 and rk4 - // type of pmull instruction - // will determine which constant to use - __pmull_pre_\p v10 - - // - // we subtract 256 instead of 128 to save one instruction from the loop - // - sub arg3, arg3, #256 - - // at this section of the code, there is 64*x+y (0<=y<64) bytes of - // buffer. The _fold_64_B_loop will fold 64B at a time - // until we have 64+y Bytes of buffer + // XOR the first 16 data *bits* with the initial CRC value. + movi v8.16b, #0 + mov v8.h[7], init_crc + eor v0.16b, v0.16b, v8.16b - // fold 64B at a time. This section of the code folds 4 vector - // registers in parallel -.L_fold_64_B_loop_\@: + // Load the constants for folding across 128 bytes. + ld1 {fold_consts.2d}, [fold_consts_ptr] + __pmull_pre_\p fold_consts - fold64 \p, v0, v1 - fold64 \p, v2, v3 - fold64 \p, v4, v5 - fold64 \p, v6, v7 + // Subtract 128 for the 128 data bytes just consumed. Subtract another + // 128 to simplify the termination condition of the following loop. + sub len, len, #256 - subs arg3, arg3, #128 + // While >= 128 data bytes remain (not counting v0-v7), fold the 128 + // bytes v0-v7 into them, storing the result back into v0-v7. +.Lfold_128_bytes_loop_\@: + fold_32_bytes \p, v0, v1 + fold_32_bytes \p, v2, v3 + fold_32_bytes \p, v4, v5 + fold_32_bytes \p, v6, v7 - // check if there is another 64B in the buffer to be able to fold - b.lt .L_fold_64_B_end_\@ + subs len, len, #128 + b.lt .Lfold_128_bytes_loop_done_\@ if_will_cond_yield_neon stp q0, q1, [sp, #.Lframe_local_offset] @@ -353,217 +329,207 @@ CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 ) ldp q2, q3, [sp, #.Lframe_local_offset + 32] ldp q4, q5, [sp, #.Lframe_local_offset + 64] ldp q6, q7, [sp, #.Lframe_local_offset + 96] - ldr_l q10, rk3, x8 - movi vzr.16b, #0 // init zero register + ld1 {fold_consts.2d}, [fold_consts_ptr] __pmull_init_\p - __pmull_pre_\p v10 + __pmull_pre_\p fold_consts endif_yield_neon - b .L_fold_64_B_loop_\@ - -.L_fold_64_B_end_\@: - // at this point, the buffer pointer is pointing at the last y Bytes - // of the buffer the 64B of folded data is in 4 of the vector - // registers: v0, v1, v2, v3 - - // fold the 8 vector registers to 1 vector register with different - // constants - - ldr_l q10, rk9, x8 - __pmull_pre_\p v10 - - fold16 \p, v0, rk11 - fold16 \p, v1, rk13 - fold16 \p, v2, rk15 - fold16 \p, v3, rk17 - fold16 \p, v4, rk19 - fold16 \p, v5, rk1 - fold16 \p, v6 - - // instead of 64, we add 48 to the loop counter to save 1 instruction - // from the loop instead of a cmp instruction, we use the negative - // flag with the jl instruction - adds arg3, arg3, #(128-16) - b.lt .L_final_reduction_for_128_\@ - - // now we have 16+y bytes left to reduce. 16 Bytes is in register v7 - // and the rest is in memory. We can fold 16 bytes at a time if y>=16 - // continue folding 16B at a time - -.L_16B_reduction_loop_\@: - __pmull_\p v8, v7, v10 - __pmull_\p v7, v7, v10, 2 + b .Lfold_128_bytes_loop_\@ + +.Lfold_128_bytes_loop_done_\@: + + // Now fold the 112 bytes in v0-v6 into the 16 bytes in v7. + + // Fold across 64 bytes. + add fold_consts_ptr, fold_consts_ptr, #16 + ld1 {fold_consts.2d}, [fold_consts_ptr], #16 + __pmull_pre_\p fold_consts + fold_16_bytes \p, v0, v4 + fold_16_bytes \p, v1, v5 + fold_16_bytes \p, v2, v6 + fold_16_bytes \p, v3, v7, 1 + // Fold across 32 bytes. + fold_16_bytes \p, v4, v6 + fold_16_bytes \p, v5, v7, 1 + // Fold across 16 bytes. + fold_16_bytes \p, v6, v7 + + // Add 128 to get the correct number of data bytes remaining in 0...127 + // (not counting v7), following the previous extra subtraction by 128. + // Then subtract 16 to simplify the termination condition of the + // following loop. + adds len, len, #(128-16) + + // While >= 16 data bytes remain (not counting v7), fold the 16 bytes v7 + // into them, storing the result back into v7. + b.lt .Lfold_16_bytes_loop_done_\@ +.Lfold_16_bytes_loop_\@: + __pmull_\p v8, v7, fold_consts + __pmull_\p v7, v7, fold_consts, 2 eor v7.16b, v7.16b, v8.16b - - ldr q0, [arg2], #16 + ldr q0, [buf], #16 CPU_LE( rev64 v0.16b, v0.16b ) CPU_LE( ext v0.16b, v0.16b, v0.16b, #8 ) eor v7.16b, v7.16b, v0.16b - subs arg3, arg3, #16 - - // instead of a cmp instruction, we utilize the flags with the - // jge instruction equivalent of: cmp arg3, 16-16 - // check if there is any more 16B in the buffer to be able to fold - b.ge .L_16B_reduction_loop_\@ - - // now we have 16+z bytes left to reduce, where 0<= z < 16. - // first, we reduce the data in the xmm7 register - -.L_final_reduction_for_128_\@: - // check if any more data to fold. If not, compute the CRC of - // the final 128 bits - adds arg3, arg3, #16 - b.eq .L_128_done_\@ - - // here we are getting data that is less than 16 bytes. - // since we know that there was data before the pointer, we can - // offset the input pointer before the actual point, to receive - // exactly 16 bytes. after that the registers need to be adjusted. -.L_get_last_two_regs_\@: - add arg2, arg2, arg3 - ldr q1, [arg2, #-16] -CPU_LE( rev64 v1.16b, v1.16b ) -CPU_LE( ext v1.16b, v1.16b, v1.16b, #8 ) - - // get rid of the extra data that was loaded before - // load the shift constant - adr_l x4, tbl_shf_table + 16 - sub x4, x4, arg3 - ld1 {v0.16b}, [x4] - - // shift v2 to the left by arg3 bytes - tbl v2.16b, {v7.16b}, v0.16b - - // shift v7 to the right by 16-arg3 bytes - movi v9.16b, #0x80 - eor v0.16b, v0.16b, v9.16b - tbl v7.16b, {v7.16b}, v0.16b - - // blend - sshr v0.16b, v0.16b, #7 // convert to 8-bit mask - bsl v0.16b, v2.16b, v1.16b - - // fold 16 Bytes - __pmull_\p v8, v7, v10 - __pmull_\p v7, v7, v10, 2 - eor v7.16b, v7.16b, v8.16b - eor v7.16b, v7.16b, v0.16b + subs len, len, #16 + b.ge .Lfold_16_bytes_loop_\@ + +.Lfold_16_bytes_loop_done_\@: + // Add 16 to get the correct number of data bytes remaining in 0...15 + // (not counting v7), following the previous extra subtraction by 16. + adds len, len, #16 + b.eq .Lreduce_final_16_bytes_\@ + +.Lhandle_partial_segment_\@: + // Reduce the last '16 + len' bytes where 1 <= len <= 15 and the first + // 16 bytes are in v7 and the rest are the remaining data in 'buf'. To + // do this without needing a fold constant for each possible 'len', + // redivide the bytes into a first chunk of 'len' bytes and a second + // chunk of 16 bytes, then fold the first chunk into the second. + + // v0 = last 16 original data bytes + add buf, buf, len + ldr q0, [buf, #-16] +CPU_LE( rev64 v0.16b, v0.16b ) +CPU_LE( ext v0.16b, v0.16b, v0.16b, #8 ) -.L_128_done_\@: - // compute crc of a 128-bit value - ldr_l q10, rk5, x8 // rk5 and rk6 in xmm10 - __pmull_pre_\p v10 + // v1 = high order part of second chunk: v7 left-shifted by 'len' bytes. + adr_l x4, .Lbyteshift_table + 16 + sub x4, x4, len + ld1 {v2.16b}, [x4] + tbl v1.16b, {v7.16b}, v2.16b - // 64b fold - ext v0.16b, vzr.16b, v7.16b, #8 - mov v7.d[0], v7.d[1] - __pmull_\p v7, v7, v10 - eor v7.16b, v7.16b, v0.16b + // v3 = first chunk: v7 right-shifted by '16-len' bytes. + movi v3.16b, #0x80 + eor v2.16b, v2.16b, v3.16b + tbl v3.16b, {v7.16b}, v2.16b - // 32b fold - ext v0.16b, v7.16b, vzr.16b, #4 - mov v7.s[3], vzr.s[0] - __pmull_\p v0, v0, v10, 2 - eor v7.16b, v7.16b, v0.16b + // Convert to 8-bit masks: 'len' 0x00 bytes, then '16-len' 0xff bytes. + sshr v2.16b, v2.16b, #7 - // barrett reduction - ldr_l q10, rk7, x8 - __pmull_pre_\p v10 - mov v0.d[0], v7.d[1] + // v2 = second chunk: 'len' bytes from v0 (low-order bytes), + // then '16-len' bytes from v1 (high-order bytes). + bsl v2.16b, v1.16b, v0.16b - __pmull_\p v0, v0, v10 - ext v0.16b, vzr.16b, v0.16b, #12 - __pmull_\p v0, v0, v10, 2 - ext v0.16b, vzr.16b, v0.16b, #12 + // Fold the first chunk into the second chunk, storing the result in v7. + __pmull_\p v0, v3, fold_consts + __pmull_\p v7, v3, fold_consts, 2 eor v7.16b, v7.16b, v0.16b - mov w0, v7.s[1] - -.L_cleanup_\@: - // scale the result back to 16 bits - lsr x0, x0, #16 + eor v7.16b, v7.16b, v2.16b + +.Lreduce_final_16_bytes_\@: + // Reduce the 128-bit value M(x), stored in v7, to the final 16-bit CRC. + + movi v2.16b, #0 // init zero register + + // Load 'x^48 * (x^48 mod G(x))' and 'x^48 * (x^80 mod G(x))'. + ld1 {fold_consts.2d}, [fold_consts_ptr], #16 + __pmull_pre_\p fold_consts + + // Fold the high 64 bits into the low 64 bits, while also multiplying by + // x^64. This produces a 128-bit value congruent to x^64 * M(x) and + // whose low 48 bits are 0. + ext v0.16b, v2.16b, v7.16b, #8 + __pmull_\p v7, v7, fold_consts, 2 // high bits * x^48 * (x^80 mod G(x)) + eor v0.16b, v0.16b, v7.16b // + low bits * x^64 + + // Fold the high 32 bits into the low 96 bits. This produces a 96-bit + // value congruent to x^64 * M(x) and whose low 48 bits are 0. + ext v1.16b, v0.16b, v2.16b, #12 // extract high 32 bits + mov v0.s[3], v2.s[0] // zero high 32 bits + __pmull_\p v1, v1, fold_consts // high 32 bits * x^48 * (x^48 mod G(x)) + eor v0.16b, v0.16b, v1.16b // + low bits + + // Load G(x) and floor(x^48 / G(x)). + ld1 {fold_consts.2d}, [fold_consts_ptr] + __pmull_pre_\p fold_consts + + // Use Barrett reduction to compute the final CRC value. + __pmull_\p v1, v0, fold_consts, 2 // high 32 bits * floor(x^48 / G(x)) + ushr v1.2d, v1.2d, #32 // /= x^32 + __pmull_\p v1, v1, fold_consts // *= G(x) + ushr v0.2d, v0.2d, #48 + eor v0.16b, v0.16b, v1.16b // + low 16 nonzero bits + // Final CRC value (x^16 * M(x)) mod G(x) is in low 16 bits of v0. + + umov w0, v0.h[0] frame_pop ret -.L_less_than_128_\@: - cbz arg3, .L_cleanup_\@ +.Lless_than_256_bytes_\@: + // Checksumming a buffer of length 16...255 bytes - movi v0.16b, #0 - mov v0.s[3], arg1_low32 // get the initial crc value + adr_l fold_consts_ptr, .Lfold_across_16_bytes_consts - ldr q7, [arg2], #0x10 + // Load the first 16 data bytes. + ldr q7, [buf], #0x10 CPU_LE( rev64 v7.16b, v7.16b ) CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 ) - eor v7.16b, v7.16b, v0.16b // xor the initial crc value - - cmp arg3, #16 - b.eq .L_128_done_\@ // exactly 16 left - ldr_l q10, rk1, x8 // rk1 and rk2 in xmm10 - __pmull_pre_\p v10 + // XOR the first 16 data *bits* with the initial CRC value. + movi v0.16b, #0 + mov v0.h[7], init_crc + eor v7.16b, v7.16b, v0.16b - // update the counter. subtract 32 instead of 16 to save one - // instruction from the loop - subs arg3, arg3, #32 - b.ge .L_16B_reduction_loop_\@ + // Load the fold-across-16-bytes constants. + ld1 {fold_consts.2d}, [fold_consts_ptr], #16 + __pmull_pre_\p fold_consts - add arg3, arg3, #16 - b .L_get_last_two_regs_\@ + cmp len, #16 + b.eq .Lreduce_final_16_bytes_\@ // len == 16 + subs len, len, #32 + b.ge .Lfold_16_bytes_loop_\@ // 32 <= len <= 255 + add len, len, #16 + b .Lhandle_partial_segment_\@ // 17 <= len <= 31 .endm +// +// u16 crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len); +// +// Assumes len >= 16. +// ENTRY(crc_t10dif_pmull_p8) crc_t10dif_pmull p8 ENDPROC(crc_t10dif_pmull_p8) .align 5 +// +// u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len); +// +// Assumes len >= 16. +// ENTRY(crc_t10dif_pmull_p64) crc_t10dif_pmull p64 ENDPROC(crc_t10dif_pmull_p64) -// precomputed constants -// these constants are precomputed from the poly: -// 0x8bb70000 (0x8bb7 scaled to 32 bits) .section ".rodata", "a" .align 4 -// Q = 0x18BB70000 -// rk1 = 2^(32*3) mod Q << 32 -// rk2 = 2^(32*5) mod Q << 32 -// rk3 = 2^(32*15) mod Q << 32 -// rk4 = 2^(32*17) mod Q << 32 -// rk5 = 2^(32*3) mod Q << 32 -// rk6 = 2^(32*2) mod Q << 32 -// rk7 = floor(2^64/Q) -// rk8 = Q - -rk1: .octa 0x06df0000000000002d56000000000000 -rk3: .octa 0x7cf50000000000009d9d000000000000 -rk5: .octa 0x13680000000000002d56000000000000 -rk7: .octa 0x000000018bb7000000000001f65a57f8 -rk9: .octa 0xbfd6000000000000ceae000000000000 -rk11: .octa 0x713c0000000000001e16000000000000 -rk13: .octa 0x80a6000000000000f7f9000000000000 -rk15: .octa 0xe658000000000000044c000000000000 -rk17: .octa 0xa497000000000000ad18000000000000 -rk19: .octa 0xe7b50000000000006ee3000000000000 - -tbl_shf_table: -// use these values for shift constants for the tbl/tbx instruction -// different alignments result in values as shown: -// DDQ 0x008f8e8d8c8b8a898887868584838281 # shl 15 (16-1) / shr1 -// DDQ 0x01008f8e8d8c8b8a8988878685848382 # shl 14 (16-3) / shr2 -// DDQ 0x0201008f8e8d8c8b8a89888786858483 # shl 13 (16-4) / shr3 -// DDQ 0x030201008f8e8d8c8b8a898887868584 # shl 12 (16-4) / shr4 -// DDQ 0x04030201008f8e8d8c8b8a8988878685 # shl 11 (16-5) / shr5 -// DDQ 0x0504030201008f8e8d8c8b8a89888786 # shl 10 (16-6) / shr6 -// DDQ 0x060504030201008f8e8d8c8b8a898887 # shl 9 (16-7) / shr7 -// DDQ 0x07060504030201008f8e8d8c8b8a8988 # shl 8 (16-8) / shr8 -// DDQ 0x0807060504030201008f8e8d8c8b8a89 # shl 7 (16-9) / shr9 -// DDQ 0x090807060504030201008f8e8d8c8b8a # shl 6 (16-10) / shr10 -// DDQ 0x0a090807060504030201008f8e8d8c8b # shl 5 (16-11) / shr11 -// DDQ 0x0b0a090807060504030201008f8e8d8c # shl 4 (16-12) / shr12 -// DDQ 0x0c0b0a090807060504030201008f8e8d # shl 3 (16-13) / shr13 -// DDQ 0x0d0c0b0a090807060504030201008f8e # shl 2 (16-14) / shr14 -// DDQ 0x0e0d0c0b0a090807060504030201008f # shl 1 (16-15) / shr15 +// Fold constants precomputed from the polynomial 0x18bb7 +// G(x) = x^16 + x^15 + x^11 + x^9 + x^8 + x^7 + x^5 + x^4 + x^2 + x^1 + x^0 +.Lfold_across_128_bytes_consts: + .quad 0x0000000000006123 // x^(8*128) mod G(x) + .quad 0x0000000000002295 // x^(8*128+64) mod G(x) +// .Lfold_across_64_bytes_consts: + .quad 0x0000000000001069 // x^(4*128) mod G(x) + .quad 0x000000000000dd31 // x^(4*128+64) mod G(x) +// .Lfold_across_32_bytes_consts: + .quad 0x000000000000857d // x^(2*128) mod G(x) + .quad 0x0000000000007acc // x^(2*128+64) mod G(x) +.Lfold_across_16_bytes_consts: + .quad 0x000000000000a010 // x^(1*128) mod G(x) + .quad 0x0000000000001faa // x^(1*128+64) mod G(x) +// .Lfinal_fold_consts: + .quad 0x1368000000000000 // x^48 * (x^48 mod G(x)) + .quad 0x2d56000000000000 // x^48 * (x^80 mod G(x)) +// .Lbarrett_reduction_consts: + .quad 0x0000000000018bb7 // G(x) + .quad 0x00000001f65a57f8 // floor(x^48 / G(x)) + +// For 1 <= len <= 15, the 16-byte vector beginning at &byteshift_table[16 - +// len] is the index vector to shift left by 'len' bytes, and is also {0x80, +// ..., 0x80} XOR the index vector to shift right by '16 - len' bytes. +.Lbyteshift_table: .byte 0x0, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87 .byte 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f .byte 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7 diff --git a/arch/arm64/crypto/crct10dif-ce-glue.c b/arch/arm64/crypto/crct10dif-ce-glue.c index 242757cc6da9..dd325829ee44 100644 --- a/arch/arm64/crypto/crct10dif-ce-glue.c +++ b/arch/arm64/crypto/crct10dif-ce-glue.c @@ -22,8 +22,8 @@ #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U -asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 buf[], u64 len); -asmlinkage u16 crc_t10dif_pmull_p8(u16 init_crc, const u8 buf[], u64 len); +asmlinkage u16 crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len); +asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len); static int crct10dif_init(struct shash_desc *desc) { -- cgit v1.2.3 From ba6771c0a0bc2fac9d6a8759bab8493bd1cffe3b Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 31 Jan 2019 23:51:38 -0800 Subject: crypto: x86/aegis - fix handling chunked inputs and MAY_SLEEP The x86 AEGIS implementations all fail the improved AEAD tests because they produce the wrong result with some data layouts. The issue is that they assume that if the skcipher_walk API gives 'nbytes' not aligned to the walksize (a.k.a. walk.stride), then it is the end of the data. In fact, this can happen before the end. Also, when the CRYPTO_TFM_REQ_MAY_SLEEP flag is given, they can incorrectly sleep in the skcipher_walk_*() functions while preemption has been disabled by kernel_fpu_begin(). Fix these bugs. Fixes: 1d373d4e8e15 ("crypto: x86 - Add optimized AEGIS implementations") Cc: # v4.18+ Cc: Ondrej Mosnacek Signed-off-by: Eric Biggers Reviewed-by: Ondrej Mosnacek Signed-off-by: Herbert Xu --- arch/x86/crypto/aegis128-aesni-glue.c | 38 ++++++++++++++-------------------- arch/x86/crypto/aegis128l-aesni-glue.c | 38 ++++++++++++++-------------------- arch/x86/crypto/aegis256-aesni-glue.c | 38 ++++++++++++++-------------------- 3 files changed, 45 insertions(+), 69 deletions(-) (limited to 'arch') diff --git a/arch/x86/crypto/aegis128-aesni-glue.c b/arch/x86/crypto/aegis128-aesni-glue.c index 2a356b948720..3ea71b871813 100644 --- a/arch/x86/crypto/aegis128-aesni-glue.c +++ b/arch/x86/crypto/aegis128-aesni-glue.c @@ -119,31 +119,20 @@ static void crypto_aegis128_aesni_process_ad( } static void crypto_aegis128_aesni_process_crypt( - struct aegis_state *state, struct aead_request *req, + struct aegis_state *state, struct skcipher_walk *walk, const struct aegis_crypt_ops *ops) { - struct skcipher_walk walk; - u8 *src, *dst; - unsigned int chunksize, base; - - ops->skcipher_walk_init(&walk, req, false); - - while (walk.nbytes) { - src = walk.src.virt.addr; - dst = walk.dst.virt.addr; - chunksize = walk.nbytes; - - ops->crypt_blocks(state, chunksize, src, dst); - - base = chunksize & ~(AEGIS128_BLOCK_SIZE - 1); - src += base; - dst += base; - chunksize &= AEGIS128_BLOCK_SIZE - 1; - - if (chunksize > 0) - ops->crypt_tail(state, chunksize, src, dst); + while (walk->nbytes >= AEGIS128_BLOCK_SIZE) { + ops->crypt_blocks(state, + round_down(walk->nbytes, AEGIS128_BLOCK_SIZE), + walk->src.virt.addr, walk->dst.virt.addr); + skcipher_walk_done(walk, walk->nbytes % AEGIS128_BLOCK_SIZE); + } - skcipher_walk_done(&walk, 0); + if (walk->nbytes) { + ops->crypt_tail(state, walk->nbytes, walk->src.virt.addr, + walk->dst.virt.addr); + skcipher_walk_done(walk, 0); } } @@ -186,13 +175,16 @@ static void crypto_aegis128_aesni_crypt(struct aead_request *req, { struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct aegis_ctx *ctx = crypto_aegis128_aesni_ctx(tfm); + struct skcipher_walk walk; struct aegis_state state; + ops->skcipher_walk_init(&walk, req, true); + kernel_fpu_begin(); crypto_aegis128_aesni_init(&state, ctx->key.bytes, req->iv); crypto_aegis128_aesni_process_ad(&state, req->src, req->assoclen); - crypto_aegis128_aesni_process_crypt(&state, req, ops); + crypto_aegis128_aesni_process_crypt(&state, &walk, ops); crypto_aegis128_aesni_final(&state, tag_xor, req->assoclen, cryptlen); kernel_fpu_end(); diff --git a/arch/x86/crypto/aegis128l-aesni-glue.c b/arch/x86/crypto/aegis128l-aesni-glue.c index dbe8bb980da1..1b1b39c66c5e 100644 --- a/arch/x86/crypto/aegis128l-aesni-glue.c +++ b/arch/x86/crypto/aegis128l-aesni-glue.c @@ -119,31 +119,20 @@ static void crypto_aegis128l_aesni_process_ad( } static void crypto_aegis128l_aesni_process_crypt( - struct aegis_state *state, struct aead_request *req, + struct aegis_state *state, struct skcipher_walk *walk, const struct aegis_crypt_ops *ops) { - struct skcipher_walk walk; - u8 *src, *dst; - unsigned int chunksize, base; - - ops->skcipher_walk_init(&walk, req, false); - - while (walk.nbytes) { - src = walk.src.virt.addr; - dst = walk.dst.virt.addr; - chunksize = walk.nbytes; - - ops->crypt_blocks(state, chunksize, src, dst); - - base = chunksize & ~(AEGIS128L_BLOCK_SIZE - 1); - src += base; - dst += base; - chunksize &= AEGIS128L_BLOCK_SIZE - 1; - - if (chunksize > 0) - ops->crypt_tail(state, chunksize, src, dst); + while (walk->nbytes >= AEGIS128L_BLOCK_SIZE) { + ops->crypt_blocks(state, round_down(walk->nbytes, + AEGIS128L_BLOCK_SIZE), + walk->src.virt.addr, walk->dst.virt.addr); + skcipher_walk_done(walk, walk->nbytes % AEGIS128L_BLOCK_SIZE); + } - skcipher_walk_done(&walk, 0); + if (walk->nbytes) { + ops->crypt_tail(state, walk->nbytes, walk->src.virt.addr, + walk->dst.virt.addr); + skcipher_walk_done(walk, 0); } } @@ -186,13 +175,16 @@ static void crypto_aegis128l_aesni_crypt(struct aead_request *req, { struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct aegis_ctx *ctx = crypto_aegis128l_aesni_ctx(tfm); + struct skcipher_walk walk; struct aegis_state state; + ops->skcipher_walk_init(&walk, req, true); + kernel_fpu_begin(); crypto_aegis128l_aesni_init(&state, ctx->key.bytes, req->iv); crypto_aegis128l_aesni_process_ad(&state, req->src, req->assoclen); - crypto_aegis128l_aesni_process_crypt(&state, req, ops); + crypto_aegis128l_aesni_process_crypt(&state, &walk, ops); crypto_aegis128l_aesni_final(&state, tag_xor, req->assoclen, cryptlen); kernel_fpu_end(); diff --git a/arch/x86/crypto/aegis256-aesni-glue.c b/arch/x86/crypto/aegis256-aesni-glue.c index 8bebda2de92f..6227ca3220a0 100644 --- a/arch/x86/crypto/aegis256-aesni-glue.c +++ b/arch/x86/crypto/aegis256-aesni-glue.c @@ -119,31 +119,20 @@ static void crypto_aegis256_aesni_process_ad( } static void crypto_aegis256_aesni_process_crypt( - struct aegis_state *state, struct aead_request *req, + struct aegis_state *state, struct skcipher_walk *walk, const struct aegis_crypt_ops *ops) { - struct skcipher_walk walk; - u8 *src, *dst; - unsigned int chunksize, base; - - ops->skcipher_walk_init(&walk, req, false); - - while (walk.nbytes) { - src = walk.src.virt.addr; - dst = walk.dst.virt.addr; - chunksize = walk.nbytes; - - ops->crypt_blocks(state, chunksize, src, dst); - - base = chunksize & ~(AEGIS256_BLOCK_SIZE - 1); - src += base; - dst += base; - chunksize &= AEGIS256_BLOCK_SIZE - 1; - - if (chunksize > 0) - ops->crypt_tail(state, chunksize, src, dst); + while (walk->nbytes >= AEGIS256_BLOCK_SIZE) { + ops->crypt_blocks(state, + round_down(walk->nbytes, AEGIS256_BLOCK_SIZE), + walk->src.virt.addr, walk->dst.virt.addr); + skcipher_walk_done(walk, walk->nbytes % AEGIS256_BLOCK_SIZE); + } - skcipher_walk_done(&walk, 0); + if (walk->nbytes) { + ops->crypt_tail(state, walk->nbytes, walk->src.virt.addr, + walk->dst.virt.addr); + skcipher_walk_done(walk, 0); } } @@ -186,13 +175,16 @@ static void crypto_aegis256_aesni_crypt(struct aead_request *req, { struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct aegis_ctx *ctx = crypto_aegis256_aesni_ctx(tfm); + struct skcipher_walk walk; struct aegis_state state; + ops->skcipher_walk_init(&walk, req, true); + kernel_fpu_begin(); crypto_aegis256_aesni_init(&state, ctx->key, req->iv); crypto_aegis256_aesni_process_ad(&state, req->src, req->assoclen); - crypto_aegis256_aesni_process_crypt(&state, req, ops); + crypto_aegis256_aesni_process_crypt(&state, &walk, ops); crypto_aegis256_aesni_final(&state, tag_xor, req->assoclen, cryptlen); kernel_fpu_end(); -- cgit v1.2.3 From 2060e284e9595fc3baed6e035903c05b93266555 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 31 Jan 2019 23:51:39 -0800 Subject: crypto: x86/morus - fix handling chunked inputs and MAY_SLEEP The x86 MORUS implementations all fail the improved AEAD tests because they produce the wrong result with some data layouts. The issue is that they assume that if the skcipher_walk API gives 'nbytes' not aligned to the walksize (a.k.a. walk.stride), then it is the end of the data. In fact, this can happen before the end. Also, when the CRYPTO_TFM_REQ_MAY_SLEEP flag is given, they can incorrectly sleep in the skcipher_walk_*() functions while preemption has been disabled by kernel_fpu_begin(). Fix these bugs. Fixes: 56e8e57fc3a7 ("crypto: morus - Add common SIMD glue code for MORUS") Cc: # v4.18+ Cc: Ondrej Mosnacek Signed-off-by: Eric Biggers Reviewed-by: Ondrej Mosnacek Signed-off-by: Herbert Xu --- arch/x86/crypto/morus1280_glue.c | 40 ++++++++++++++++------------------------ arch/x86/crypto/morus640_glue.c | 39 +++++++++++++++------------------------ 2 files changed, 31 insertions(+), 48 deletions(-) (limited to 'arch') diff --git a/arch/x86/crypto/morus1280_glue.c b/arch/x86/crypto/morus1280_glue.c index 0dccdda1eb3a..7e600f8bcdad 100644 --- a/arch/x86/crypto/morus1280_glue.c +++ b/arch/x86/crypto/morus1280_glue.c @@ -85,31 +85,20 @@ static void crypto_morus1280_glue_process_ad( static void crypto_morus1280_glue_process_crypt(struct morus1280_state *state, struct morus1280_ops ops, - struct aead_request *req) + struct skcipher_walk *walk) { - struct skcipher_walk walk; - u8 *cursor_src, *cursor_dst; - unsigned int chunksize, base; - - ops.skcipher_walk_init(&walk, req, false); - - while (walk.nbytes) { - cursor_src = walk.src.virt.addr; - cursor_dst = walk.dst.virt.addr; - chunksize = walk.nbytes; - - ops.crypt_blocks(state, cursor_src, cursor_dst, chunksize); - - base = chunksize & ~(MORUS1280_BLOCK_SIZE - 1); - cursor_src += base; - cursor_dst += base; - chunksize &= MORUS1280_BLOCK_SIZE - 1; - - if (chunksize > 0) - ops.crypt_tail(state, cursor_src, cursor_dst, - chunksize); + while (walk->nbytes >= MORUS1280_BLOCK_SIZE) { + ops.crypt_blocks(state, walk->src.virt.addr, + walk->dst.virt.addr, + round_down(walk->nbytes, + MORUS1280_BLOCK_SIZE)); + skcipher_walk_done(walk, walk->nbytes % MORUS1280_BLOCK_SIZE); + } - skcipher_walk_done(&walk, 0); + if (walk->nbytes) { + ops.crypt_tail(state, walk->src.virt.addr, walk->dst.virt.addr, + walk->nbytes); + skcipher_walk_done(walk, 0); } } @@ -147,12 +136,15 @@ static void crypto_morus1280_glue_crypt(struct aead_request *req, struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct morus1280_ctx *ctx = crypto_aead_ctx(tfm); struct morus1280_state state; + struct skcipher_walk walk; + + ops.skcipher_walk_init(&walk, req, true); kernel_fpu_begin(); ctx->ops->init(&state, &ctx->key, req->iv); crypto_morus1280_glue_process_ad(&state, ctx->ops, req->src, req->assoclen); - crypto_morus1280_glue_process_crypt(&state, ops, req); + crypto_morus1280_glue_process_crypt(&state, ops, &walk); ctx->ops->final(&state, tag_xor, req->assoclen, cryptlen); kernel_fpu_end(); diff --git a/arch/x86/crypto/morus640_glue.c b/arch/x86/crypto/morus640_glue.c index 7b58fe4d9bd1..cb3a81732016 100644 --- a/arch/x86/crypto/morus640_glue.c +++ b/arch/x86/crypto/morus640_glue.c @@ -85,31 +85,19 @@ static void crypto_morus640_glue_process_ad( static void crypto_morus640_glue_process_crypt(struct morus640_state *state, struct morus640_ops ops, - struct aead_request *req) + struct skcipher_walk *walk) { - struct skcipher_walk walk; - u8 *cursor_src, *cursor_dst; - unsigned int chunksize, base; - - ops.skcipher_walk_init(&walk, req, false); - - while (walk.nbytes) { - cursor_src = walk.src.virt.addr; - cursor_dst = walk.dst.virt.addr; - chunksize = walk.nbytes; - - ops.crypt_blocks(state, cursor_src, cursor_dst, chunksize); - - base = chunksize & ~(MORUS640_BLOCK_SIZE - 1); - cursor_src += base; - cursor_dst += base; - chunksize &= MORUS640_BLOCK_SIZE - 1; - - if (chunksize > 0) - ops.crypt_tail(state, cursor_src, cursor_dst, - chunksize); + while (walk->nbytes >= MORUS640_BLOCK_SIZE) { + ops.crypt_blocks(state, walk->src.virt.addr, + walk->dst.virt.addr, + round_down(walk->nbytes, MORUS640_BLOCK_SIZE)); + skcipher_walk_done(walk, walk->nbytes % MORUS640_BLOCK_SIZE); + } - skcipher_walk_done(&walk, 0); + if (walk->nbytes) { + ops.crypt_tail(state, walk->src.virt.addr, walk->dst.virt.addr, + walk->nbytes); + skcipher_walk_done(walk, 0); } } @@ -143,12 +131,15 @@ static void crypto_morus640_glue_crypt(struct aead_request *req, struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct morus640_ctx *ctx = crypto_aead_ctx(tfm); struct morus640_state state; + struct skcipher_walk walk; + + ops.skcipher_walk_init(&walk, req, true); kernel_fpu_begin(); ctx->ops->init(&state, &ctx->key, req->iv); crypto_morus640_glue_process_ad(&state, ctx->ops, req->src, req->assoclen); - crypto_morus640_glue_process_crypt(&state, ops, req); + crypto_morus640_glue_process_crypt(&state, ops, &walk); ctx->ops->final(&state, tag_xor, req->assoclen, cryptlen); kernel_fpu_end(); -- cgit v1.2.3 From 3af349639597fea582a93604734d717e59a0e223 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 31 Jan 2019 23:51:40 -0800 Subject: crypto: x86/aesni-gcm - fix crash on empty plaintext gcmaes_crypt_by_sg() dereferences the NULL pointer returned by scatterwalk_ffwd() when encrypting an empty plaintext and the source scatterlist ends immediately after the associated data. Fix it by only fast-forwarding to the src/dst data scatterlists if the data length is nonzero. This bug is reproduced by the "rfc4543(gcm(aes))" test vectors when run with the new AEAD test manager. Fixes: e845520707f8 ("crypto: aesni - Update aesni-intel_glue to use scatter/gather") Cc: # v4.17+ Cc: Dave Watson Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/aesni-intel_glue.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'arch') diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index 9b5ccde3ef31..1e3d2102033a 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -813,11 +813,14 @@ static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req, scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0); } - src_sg = scatterwalk_ffwd(src_start, req->src, req->assoclen); - scatterwalk_start(&src_sg_walk, src_sg); - if (req->src != req->dst) { - dst_sg = scatterwalk_ffwd(dst_start, req->dst, req->assoclen); - scatterwalk_start(&dst_sg_walk, dst_sg); + if (left) { + src_sg = scatterwalk_ffwd(src_start, req->src, req->assoclen); + scatterwalk_start(&src_sg_walk, src_sg); + if (req->src != req->dst) { + dst_sg = scatterwalk_ffwd(dst_start, req->dst, + req->assoclen); + scatterwalk_start(&dst_sg_walk, dst_sg); + } } kernel_fpu_begin(); -- cgit v1.2.3 From 12455e320e19e9cc7ad97f4ab89c280fe297387c Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 31 Jan 2019 23:51:42 -0800 Subject: crypto: arm64/aes-neonbs - fix returning final keystream block The arm64 NEON bit-sliced implementation of AES-CTR fails the improved skcipher tests because it sometimes produces the wrong ciphertext. The bug is that the final keystream block isn't returned from the assembly code when the number of non-final blocks is zero. This can happen if the input data ends a few bytes after a page boundary. In this case the last bytes get "encrypted" by XOR'ing them with uninitialized memory. Fix the assembly code to return the final keystream block when needed. Fixes: 88a3f582bea9 ("crypto: arm64/aes - don't use IV buffer to return final keystream block") Cc: # v4.11+ Reviewed-by: Ard Biesheuvel Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm64/crypto/aes-neonbs-core.S | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'arch') diff --git a/arch/arm64/crypto/aes-neonbs-core.S b/arch/arm64/crypto/aes-neonbs-core.S index e613a87f8b53..8432c8d0dea6 100644 --- a/arch/arm64/crypto/aes-neonbs-core.S +++ b/arch/arm64/crypto/aes-neonbs-core.S @@ -971,18 +971,22 @@ CPU_LE( rev x8, x8 ) 8: next_ctr v0 st1 {v0.16b}, [x24] - cbz x23, 0f + cbz x23, .Lctr_done cond_yield_neon 98b b 99b -0: frame_pop +.Lctr_done: + frame_pop ret /* * If we are handling the tail of the input (x6 != NULL), return the * final keystream block back to the caller. */ +0: cbz x25, 8b + st1 {v0.16b}, [x25] + b 8b 1: cbz x25, 8b st1 {v1.16b}, [x25] b 8b -- cgit v1.2.3 From fa5fd3afc7e64473bd9d417726c046b322ec0696 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 14 Feb 2019 00:03:54 -0800 Subject: crypto: arm64/aes-blk - update IV after partial final CTR block Make the arm64 ctr-aes-neon and ctr-aes-ce algorithms update the IV buffer to contain the next counter after processing a partial final block, rather than leave it as the last counter. This makes these algorithms pass the updated AES-CTR tests. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm64/crypto/aes-modes.S | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'arch') diff --git a/arch/arm64/crypto/aes-modes.S b/arch/arm64/crypto/aes-modes.S index 67700045a0e0..4c7ce231963c 100644 --- a/arch/arm64/crypto/aes-modes.S +++ b/arch/arm64/crypto/aes-modes.S @@ -320,8 +320,7 @@ AES_ENTRY(aes_ctr_encrypt) .Lctrtailblock: st1 {v0.16b}, [x0] - ldp x29, x30, [sp], #16 - ret + b .Lctrout .Lctrcarry: umov x7, v4.d[0] /* load upper word of ctr */ -- cgit v1.2.3 From 511306b2d075e3cb1e5dd805bffcf4041535f2b5 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 14 Feb 2019 00:03:55 -0800 Subject: crypto: arm/aes-ce - update IV after partial final CTR block Make the arm ctr-aes-ce algorithm update the IV buffer to contain the next counter after processing a partial final block, rather than leave it as the last counter. This makes ctr-aes-ce pass the updated AES-CTR tests. This change also makes the code match the arm64 version in arch/arm64/crypto/aes-modes.S more closely. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm/crypto/aes-ce-core.S | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'arch') diff --git a/arch/arm/crypto/aes-ce-core.S b/arch/arm/crypto/aes-ce-core.S index ba8e6a32fdc9..bc53bcaa772e 100644 --- a/arch/arm/crypto/aes-ce-core.S +++ b/arch/arm/crypto/aes-ce-core.S @@ -317,25 +317,27 @@ ENTRY(ce_aes_ctr_encrypt) .Lctrloop: vmov q0, q6 bl aes_encrypt - subs r4, r4, #1 - bmi .Lctrtailblock @ blocks < 0 means tail block - vld1.8 {q3}, [r1]! - veor q3, q0, q3 - vst1.8 {q3}, [r0]! adds r6, r6, #1 @ increment BE ctr rev ip, r6 vmov s27, ip bcs .Lctrcarry - teq r4, #0 + +.Lctrcarrydone: + subs r4, r4, #1 + bmi .Lctrtailblock @ blocks < 0 means tail block + vld1.8 {q3}, [r1]! + veor q3, q0, q3 + vst1.8 {q3}, [r0]! bne .Lctrloop + .Lctrout: - vst1.8 {q6}, [r5] + vst1.8 {q6}, [r5] @ return next CTR value pop {r4-r6, pc} .Lctrtailblock: - vst1.8 {q0}, [r0, :64] @ return just the key stream - pop {r4-r6, pc} + vst1.8 {q0}, [r0, :64] @ return the key stream + b .Lctrout .Lctrcarry: .irp sreg, s26, s25, s24 @@ -344,11 +346,9 @@ ENTRY(ce_aes_ctr_encrypt) adds ip, ip, #1 rev ip, ip vmov \sreg, ip - bcc 0f + bcc .Lctrcarrydone .endr -0: teq r4, #0 - beq .Lctrout - b .Lctrloop + b .Lctrcarrydone ENDPROC(ce_aes_ctr_encrypt) /* -- cgit v1.2.3 From 7748168c66404a3ee732972a3a55b5332245eb25 Mon Sep 17 00:00:00 2001 From: Tommi Hirvola Date: Tue, 19 Feb 2019 17:45:54 +0200 Subject: crypto: x86/poly1305 - Clear key material from stack in SSE2 variant 1-block SSE2 variant of poly1305 stores variables s1..s4 containing key material on the stack. This commit adds missing zeroing of the stack memory. Benchmarks show negligible performance hit (tested on i7-3770). Signed-off-by: Tommi Hirvola Signed-off-by: Herbert Xu --- arch/x86/crypto/poly1305-sse2-x86_64.S | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'arch') diff --git a/arch/x86/crypto/poly1305-sse2-x86_64.S b/arch/x86/crypto/poly1305-sse2-x86_64.S index c88c670cb5fc..e6add74d78a5 100644 --- a/arch/x86/crypto/poly1305-sse2-x86_64.S +++ b/arch/x86/crypto/poly1305-sse2-x86_64.S @@ -272,6 +272,10 @@ ENTRY(poly1305_block_sse2) dec %rcx jnz .Ldoblock + # Zeroing of key material + mov %rcx,0x00(%rsp) + mov %rcx,0x08(%rsp) + add $0x10,%rsp pop %r12 pop %rbx -- cgit v1.2.3