From bc83b8299cb4ac2a9f64215a04854e4c934d1510 Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Tue, 18 Oct 2011 00:03:08 +0300 Subject: crypto: serpent - export common functions for x86_64/i386-sse2 assembler implementations Serpent SSE2 assembler implementations only provide 4-way/8-way parallel functions and need setkey and one-block encrypt/decrypt functions. CC: Dag Arne Osvik Signed-off-by: Jussi Kivilinna Signed-off-by: Herbert Xu --- include/crypto/serpent.h | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 include/crypto/serpent.h (limited to 'include') diff --git a/include/crypto/serpent.h b/include/crypto/serpent.h new file mode 100644 index 000000000000..40df885f9d1f --- /dev/null +++ b/include/crypto/serpent.h @@ -0,0 +1,25 @@ +/* + * Common values for serpent algorithms + */ + +#ifndef _CRYPTO_SERPENT_H +#define _CRYPTO_SERPENT_H + +#include +#include + +#define SERPENT_MIN_KEY_SIZE 0 +#define SERPENT_MAX_KEY_SIZE 32 +#define SERPENT_EXPKEY_WORDS 132 +#define SERPENT_BLOCK_SIZE 16 + +struct serpent_ctx { + u32 expkey[SERPENT_EXPKEY_WORDS]; +}; + +int serpent_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen); + +void __serpent_encrypt(struct serpent_ctx *ctx, u8 *dst, const u8 *src); +void __serpent_decrypt(struct serpent_ctx *ctx, u8 *dst, const u8 *src); + +#endif -- cgit v1.2.3 From 6c2205b8ffec035f4925b8ee84b7758afeee58b5 Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Wed, 9 Nov 2011 11:50:31 +0800 Subject: crypto: lrw - add interface for parallelized cipher implementions Export gf128mul table initialization routines and add lrw_crypt() function that can be used by cipher implementations that can benefit from parallelized cipher operations. Signed-off-by: Jussi Kivilinna Signed-off-by: Herbert Xu --- crypto/lrw.c | 106 +++++++++++++++++++++++++++++++++++++++++---------- include/crypto/lrw.h | 43 +++++++++++++++++++++ 2 files changed, 129 insertions(+), 20 deletions(-) create mode 100644 include/crypto/lrw.h (limited to 'include') diff --git a/crypto/lrw.c b/crypto/lrw.c index 91c17fa18374..ba42acc4deba 100644 --- a/crypto/lrw.c +++ b/crypto/lrw.c @@ -3,7 +3,7 @@ * * Copyright (c) 2006 Rik Snel * - * Based om ecb.c + * Based on ecb.c * Copyright (c) 2006 Herbert Xu * * This program is free software; you can redistribute it and/or modify it @@ -16,6 +16,7 @@ * http://www.mail-archive.com/stds-p1619@listserv.ieee.org/msg00173.html * * The test vectors are included in the testing module tcrypt.[ch] */ + #include #include #include @@ -26,23 +27,7 @@ #include #include - -#define LRW_BLOCK_SIZE 16 - -struct lrw_table_ctx { - /* optimizes multiplying a random (non incrementing, as at the - * start of a new sector) value with key2, we could also have - * used 4k optimization tables or no optimization at all. In the - * latter case we would have to store key2 here */ - struct gf128mul_64k *table; - /* stores: - * key2*{ 0,0,...0,0,0,0,1 }, key2*{ 0,0,...0,0,0,1,1 }, - * key2*{ 0,0,...0,0,1,1,1 }, key2*{ 0,0,...0,1,1,1,1 } - * key2*{ 0,0,...1,1,1,1,1 }, etc - * needed for optimized multiplication of incrementing values - * with key2 */ - be128 mulinc[128]; -}; +#include struct priv { struct crypto_cipher *child; @@ -60,7 +45,7 @@ static inline void setbit128_bbe(void *b, int bit) ), b); } -static int lrw_init_table(struct lrw_table_ctx *ctx, const u8 *tweak) +int lrw_init_table(struct lrw_table_ctx *ctx, const u8 *tweak) { be128 tmp = { 0 }; int i; @@ -82,12 +67,14 @@ static int lrw_init_table(struct lrw_table_ctx *ctx, const u8 *tweak) return 0; } +EXPORT_SYMBOL_GPL(lrw_init_table); -static void lrw_free_table(struct lrw_table_ctx *ctx) +void lrw_free_table(struct lrw_table_ctx *ctx) { if (ctx->table) gf128mul_free_64k(ctx->table); } +EXPORT_SYMBOL_GPL(lrw_free_table); static int setkey(struct crypto_tfm *parent, const u8 *key, unsigned int keylen) @@ -227,6 +214,85 @@ static int decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, crypto_cipher_alg(ctx->child)->cia_decrypt); } +int lrw_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst, + struct scatterlist *ssrc, unsigned int nbytes, + struct lrw_crypt_req *req) +{ + const unsigned int bsize = LRW_BLOCK_SIZE; + const unsigned int max_blks = req->tbuflen / bsize; + struct lrw_table_ctx *ctx = req->table_ctx; + struct blkcipher_walk walk; + unsigned int nblocks; + be128 *iv, *src, *dst, *t; + be128 *t_buf = req->tbuf; + int err, i; + + BUG_ON(max_blks < 1); + + blkcipher_walk_init(&walk, sdst, ssrc, nbytes); + + err = blkcipher_walk_virt(desc, &walk); + nbytes = walk.nbytes; + if (!nbytes) + return err; + + nblocks = min(walk.nbytes / bsize, max_blks); + src = (be128 *)walk.src.virt.addr; + dst = (be128 *)walk.dst.virt.addr; + + /* calculate first value of T */ + iv = (be128 *)walk.iv; + t_buf[0] = *iv; + + /* T <- I*Key2 */ + gf128mul_64k_bbe(&t_buf[0], ctx->table); + + i = 0; + goto first; + + for (;;) { + do { + for (i = 0; i < nblocks; i++) { + /* T <- I*Key2, using the optimization + * discussed in the specification */ + be128_xor(&t_buf[i], t, + &ctx->mulinc[get_index128(iv)]); + inc(iv); +first: + t = &t_buf[i]; + + /* PP <- T xor P */ + be128_xor(dst + i, t, src + i); + } + + /* CC <- E(Key2,PP) */ + req->crypt_fn(req->crypt_ctx, (u8 *)dst, + nblocks * bsize); + + /* C <- T xor CC */ + for (i = 0; i < nblocks; i++) + be128_xor(dst + i, dst + i, &t_buf[i]); + + src += nblocks; + dst += nblocks; + nbytes -= nblocks * bsize; + nblocks = min(nbytes / bsize, max_blks); + } while (nblocks > 0); + + err = blkcipher_walk_done(desc, &walk, nbytes); + nbytes = walk.nbytes; + if (!nbytes) + break; + + nblocks = min(nbytes / bsize, max_blks); + src = (be128 *)walk.src.virt.addr; + dst = (be128 *)walk.dst.virt.addr; + } + + return err; +} +EXPORT_SYMBOL_GPL(lrw_crypt); + static int init_tfm(struct crypto_tfm *tfm) { struct crypto_cipher *cipher; diff --git a/include/crypto/lrw.h b/include/crypto/lrw.h new file mode 100644 index 000000000000..25a2c8716375 --- /dev/null +++ b/include/crypto/lrw.h @@ -0,0 +1,43 @@ +#ifndef _CRYPTO_LRW_H +#define _CRYPTO_LRW_H + +#include + +struct scatterlist; +struct gf128mul_64k; +struct blkcipher_desc; + +#define LRW_BLOCK_SIZE 16 + +struct lrw_table_ctx { + /* optimizes multiplying a random (non incrementing, as at the + * start of a new sector) value with key2, we could also have + * used 4k optimization tables or no optimization at all. In the + * latter case we would have to store key2 here */ + struct gf128mul_64k *table; + /* stores: + * key2*{ 0,0,...0,0,0,0,1 }, key2*{ 0,0,...0,0,0,1,1 }, + * key2*{ 0,0,...0,0,1,1,1 }, key2*{ 0,0,...0,1,1,1,1 } + * key2*{ 0,0,...1,1,1,1,1 }, etc + * needed for optimized multiplication of incrementing values + * with key2 */ + be128 mulinc[128]; +}; + +int lrw_init_table(struct lrw_table_ctx *ctx, const u8 *tweak); +void lrw_free_table(struct lrw_table_ctx *ctx); + +struct lrw_crypt_req { + be128 *tbuf; + unsigned int tbuflen; + + struct lrw_table_ctx *table_ctx; + void *crypt_ctx; + void (*crypt_fn)(void *ctx, u8 *blks, unsigned int nbytes); +}; + +int lrw_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes, + struct lrw_crypt_req *req); + +#endif /* _CRYPTO_LRW_H */ -- cgit v1.2.3 From 81559f9ad3d88c033e4ec3b6468012dbfda3b31d Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Tue, 18 Oct 2011 13:33:02 +0300 Subject: crypto: twofish-x86_64-3way - add lrw support Patch adds LRW support for twofish-x86_64-3way by using lrw_crypt(). Patch has been tested with tcrypt and automated filesystem tests. Tcrypt benchmarks results (twofish-3way/twofish-asm speed ratios): Intel Celeron T1600 (fam:6, model:15, step:13): size lrw-enc lrw-dec 16B 0.99x 1.00x 64B 1.17x 1.17x 256B 1.26x 1.27x 1024B 1.30x 1.31x 8192B 1.31x 1.32x AMD Phenom II 1055T (fam:16, model:10): size lrw-enc lrw-dec 16B 1.06x 1.01x 64B 1.08x 1.14x 256B 1.19x 1.20x 1024B 1.21x 1.22x 8192B 1.23x 1.24x Signed-off-by: Jussi Kivilinna Signed-off-by: Herbert Xu --- arch/x86/crypto/twofish_glue_3way.c | 135 ++++++++++++++++++++++++++++++++++++ crypto/twofish_common.c | 13 ++-- include/crypto/twofish.h | 2 + 3 files changed, 145 insertions(+), 5 deletions(-) (limited to 'include') diff --git a/arch/x86/crypto/twofish_glue_3way.c b/arch/x86/crypto/twofish_glue_3way.c index 5ede9c444c3e..fa9151df3637 100644 --- a/arch/x86/crypto/twofish_glue_3way.c +++ b/arch/x86/crypto/twofish_glue_3way.c @@ -32,6 +32,11 @@ #include #include #include +#include + +#if defined(CONFIG_CRYPTO_LRW) || defined(CONFIG_CRYPTO_LRW_MODULE) +#define HAS_LRW +#endif /* regular block cipher functions from twofish_x86_64 module */ asmlinkage void twofish_enc_blk(struct twofish_ctx *ctx, u8 *dst, @@ -432,6 +437,124 @@ static struct crypto_alg blk_ctr_alg = { }, }; +#ifdef HAS_LRW + +static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) +{ + const unsigned int bsize = TF_BLOCK_SIZE; + struct twofish_ctx *ctx = priv; + int i; + + if (nbytes == 3 * bsize) { + twofish_enc_blk_3way(ctx, srcdst, srcdst); + return; + } + + for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) + twofish_enc_blk(ctx, srcdst, srcdst); +} + +static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) +{ + const unsigned int bsize = TF_BLOCK_SIZE; + struct twofish_ctx *ctx = priv; + int i; + + if (nbytes == 3 * bsize) { + twofish_dec_blk_3way(ctx, srcdst, srcdst); + return; + } + + for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) + twofish_dec_blk(ctx, srcdst, srcdst); +} + +struct twofish_lrw_ctx { + struct lrw_table_ctx lrw_table; + struct twofish_ctx twofish_ctx; +}; + +static int lrw_twofish_setkey(struct crypto_tfm *tfm, const u8 *key, + unsigned int keylen) +{ + struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm); + int err; + + err = __twofish_setkey(&ctx->twofish_ctx, key, keylen - TF_BLOCK_SIZE, + &tfm->crt_flags); + if (err) + return err; + + return lrw_init_table(&ctx->lrw_table, key + keylen - TF_BLOCK_SIZE); +} + +static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + be128 buf[3]; + struct lrw_crypt_req req = { + .tbuf = buf, + .tbuflen = sizeof(buf), + + .table_ctx = &ctx->lrw_table, + .crypt_ctx = &ctx->twofish_ctx, + .crypt_fn = encrypt_callback, + }; + + return lrw_crypt(desc, dst, src, nbytes, &req); +} + +static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + be128 buf[3]; + struct lrw_crypt_req req = { + .tbuf = buf, + .tbuflen = sizeof(buf), + + .table_ctx = &ctx->lrw_table, + .crypt_ctx = &ctx->twofish_ctx, + .crypt_fn = decrypt_callback, + }; + + return lrw_crypt(desc, dst, src, nbytes, &req); +} + +static void lrw_exit_tfm(struct crypto_tfm *tfm) +{ + struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm); + + lrw_free_table(&ctx->lrw_table); +} + +static struct crypto_alg blk_lrw_alg = { + .cra_name = "lrw(twofish)", + .cra_driver_name = "lrw-twofish-3way", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = TF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct twofish_lrw_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(blk_lrw_alg.cra_list), + .cra_exit = lrw_exit_tfm, + .cra_u = { + .blkcipher = { + .min_keysize = TF_MIN_KEY_SIZE + TF_BLOCK_SIZE, + .max_keysize = TF_MAX_KEY_SIZE + TF_BLOCK_SIZE, + .ivsize = TF_BLOCK_SIZE, + .setkey = lrw_twofish_setkey, + .encrypt = lrw_encrypt, + .decrypt = lrw_decrypt, + }, + }, +}; + +#endif + int __init init(void) { int err; @@ -445,9 +568,18 @@ int __init init(void) err = crypto_register_alg(&blk_ctr_alg); if (err) goto ctr_err; +#ifdef HAS_LRW + err = crypto_register_alg(&blk_lrw_alg); + if (err) + goto blk_lrw_err; +#endif return 0; +#ifdef HAS_LRW +blk_lrw_err: + crypto_unregister_alg(&blk_ctr_alg); +#endif ctr_err: crypto_unregister_alg(&blk_cbc_alg); cbc_err: @@ -458,6 +590,9 @@ ecb_err: void __exit fini(void) { +#ifdef HAS_LRW + crypto_unregister_alg(&blk_lrw_alg); +#endif crypto_unregister_alg(&blk_ctr_alg); crypto_unregister_alg(&blk_cbc_alg); crypto_unregister_alg(&blk_ecb_alg); diff --git a/crypto/twofish_common.c b/crypto/twofish_common.c index 0af216c75d7e..5f62c4f9f6e0 100644 --- a/crypto/twofish_common.c +++ b/crypto/twofish_common.c @@ -580,12 +580,9 @@ static const u8 calc_sb_tbl[512] = { ctx->a[(j) + 1] = rol32(y, 9) /* Perform the key setup. */ -int twofish_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int key_len) +int __twofish_setkey(struct twofish_ctx *ctx, const u8 *key, + unsigned int key_len, u32 *flags) { - - struct twofish_ctx *ctx = crypto_tfm_ctx(tfm); - u32 *flags = &tfm->crt_flags; - int i, j, k; /* Temporaries for CALC_K. */ @@ -701,7 +698,13 @@ int twofish_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int key_len) return 0; } +EXPORT_SYMBOL_GPL(__twofish_setkey); +int twofish_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int key_len) +{ + return __twofish_setkey(crypto_tfm_ctx(tfm), key, key_len, + &tfm->crt_flags); +} EXPORT_SYMBOL_GPL(twofish_setkey); MODULE_LICENSE("GPL"); diff --git a/include/crypto/twofish.h b/include/crypto/twofish.h index c408522595c6..095c901a8af3 100644 --- a/include/crypto/twofish.h +++ b/include/crypto/twofish.h @@ -17,6 +17,8 @@ struct twofish_ctx { u32 s[4][256], w[8], k[32]; }; +int __twofish_setkey(struct twofish_ctx *ctx, const u8 *key, + unsigned int key_len, u32 *flags); int twofish_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int key_len); #endif -- cgit v1.2.3 From ce0045561e1edb92e4a509eb433ff52d3afaa258 Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Wed, 9 Nov 2011 11:56:06 +0800 Subject: crypto: xts: add interface for parallelized cipher implementations Add xts_crypt() function that can be used by cipher implementations that can benefit from parallelized cipher operations. Signed-off-by: Jussi Kivilinna Signed-off-by: Herbert Xu --- crypto/xts.c | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++-- include/crypto/xts.h | 27 +++++++++++++++++++ 2 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 include/crypto/xts.h (limited to 'include') diff --git a/crypto/xts.c b/crypto/xts.c index 96f3f88d576e..ca1608f44cb5 100644 --- a/crypto/xts.c +++ b/crypto/xts.c @@ -21,11 +21,10 @@ #include #include +#include #include #include -#define XTS_BLOCK_SIZE 16 - struct priv { struct crypto_cipher *child; struct crypto_cipher *tweak; @@ -167,6 +166,78 @@ static int decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, crypto_cipher_alg(ctx->child)->cia_decrypt); } +int xts_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst, + struct scatterlist *ssrc, unsigned int nbytes, + struct xts_crypt_req *req) +{ + const unsigned int bsize = XTS_BLOCK_SIZE; + const unsigned int max_blks = req->tbuflen / bsize; + struct blkcipher_walk walk; + unsigned int nblocks; + be128 *src, *dst, *t; + be128 *t_buf = req->tbuf; + int err, i; + + BUG_ON(max_blks < 1); + + blkcipher_walk_init(&walk, sdst, ssrc, nbytes); + + err = blkcipher_walk_virt(desc, &walk); + nbytes = walk.nbytes; + if (!nbytes) + return err; + + nblocks = min(nbytes / bsize, max_blks); + src = (be128 *)walk.src.virt.addr; + dst = (be128 *)walk.dst.virt.addr; + + /* calculate first value of T */ + req->tweak_fn(req->tweak_ctx, (u8 *)&t_buf[0], walk.iv); + + i = 0; + goto first; + + for (;;) { + do { + for (i = 0; i < nblocks; i++) { + gf128mul_x_ble(&t_buf[i], t); +first: + t = &t_buf[i]; + + /* PP <- T xor P */ + be128_xor(dst + i, t, src + i); + } + + /* CC <- E(Key2,PP) */ + req->crypt_fn(req->crypt_ctx, (u8 *)dst, + nblocks * bsize); + + /* C <- T xor CC */ + for (i = 0; i < nblocks; i++) + be128_xor(dst + i, dst + i, &t_buf[i]); + + src += nblocks; + dst += nblocks; + nbytes -= nblocks * bsize; + nblocks = min(nbytes / bsize, max_blks); + } while (nblocks > 0); + + *(be128 *)walk.iv = *t; + + err = blkcipher_walk_done(desc, &walk, nbytes); + nbytes = walk.nbytes; + if (!nbytes) + break; + + nblocks = min(nbytes / bsize, max_blks); + src = (be128 *)walk.src.virt.addr; + dst = (be128 *)walk.dst.virt.addr; + } + + return err; +} +EXPORT_SYMBOL_GPL(xts_crypt); + static int init_tfm(struct crypto_tfm *tfm) { struct crypto_cipher *cipher; diff --git a/include/crypto/xts.h b/include/crypto/xts.h new file mode 100644 index 000000000000..72c09eb56437 --- /dev/null +++ b/include/crypto/xts.h @@ -0,0 +1,27 @@ +#ifndef _CRYPTO_XTS_H +#define _CRYPTO_XTS_H + +#include + +struct scatterlist; +struct blkcipher_desc; + +#define XTS_BLOCK_SIZE 16 + +struct xts_crypt_req { + be128 *tbuf; + unsigned int tbuflen; + + void *tweak_ctx; + void (*tweak_fn)(void *ctx, u8* dst, const u8* src); + void *crypt_ctx; + void (*crypt_fn)(void *ctx, u8 *blks, unsigned int nbytes); +}; + +#define XTS_TWEAK_CAST(x) ((void (*)(void *, u8*, const u8*))(x)) + +int xts_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes, + struct xts_crypt_req *req); + +#endif /* _CRYPTO_XTS_H */ -- cgit v1.2.3 From ce3fd840f588d85a8c1be651cf90fa1ba1f029e9 Mon Sep 17 00:00:00 2001 From: Steffen Klassert Date: Tue, 8 Nov 2011 10:09:17 +0100 Subject: crypto: Unlink and free instances when deleted We leak the crypto instance when we unregister an instance with crypto_del_alg(). Therefore we introduce crypto_unregister_instance() to unlink the crypto instance from the template's instances list and to free the recources of the instance properly. Signed-off-by: Steffen Klassert Signed-off-by: Herbert Xu --- crypto/algapi.c | 29 +++++++++++++++++++++++++++++ crypto/crypto_user.c | 2 +- include/crypto/algapi.h | 1 + 3 files changed, 31 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/crypto/algapi.c b/crypto/algapi.c index 54dd4e33b5d6..9d4a9fe913f8 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -518,6 +518,35 @@ err: } EXPORT_SYMBOL_GPL(crypto_register_instance); +int crypto_unregister_instance(struct crypto_alg *alg) +{ + int err; + struct crypto_instance *inst = (void *)alg; + struct crypto_template *tmpl = inst->tmpl; + LIST_HEAD(users); + + if (!(alg->cra_flags & CRYPTO_ALG_INSTANCE)) + return -EINVAL; + + BUG_ON(atomic_read(&alg->cra_refcnt) != 1); + + down_write(&crypto_alg_sem); + + hlist_del_init(&inst->list); + err = crypto_remove_alg(alg, &users); + + up_write(&crypto_alg_sem); + + if (err) + return err; + + tmpl->free(inst); + crypto_remove_final(&users); + + return 0; +} +EXPORT_SYMBOL_GPL(crypto_unregister_instance); + int crypto_init_spawn(struct crypto_spawn *spawn, struct crypto_alg *alg, struct crypto_instance *inst, u32 mask) { diff --git a/crypto/crypto_user.c b/crypto/crypto_user.c index 2abca780312d..edeebe1a84bb 100644 --- a/crypto/crypto_user.c +++ b/crypto/crypto_user.c @@ -301,7 +301,7 @@ static int crypto_del_alg(struct sk_buff *skb, struct nlmsghdr *nlh, if (atomic_read(&alg->cra_refcnt) != 1) return -EBUSY; - return crypto_unregister_alg(alg); + return crypto_unregister_instance(alg); } static int crypto_add_alg(struct sk_buff *skb, struct nlmsghdr *nlh, diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index ecc721def10c..418d270e1806 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -134,6 +134,7 @@ struct crypto_template *crypto_lookup_template(const char *name); int crypto_register_instance(struct crypto_template *tmpl, struct crypto_instance *inst); +int crypto_unregister_instance(struct crypto_alg *alg); int crypto_init_spawn(struct crypto_spawn *spawn, struct crypto_alg *alg, struct crypto_instance *inst, u32 mask); -- cgit v1.2.3 From 18482053f92b099663bd36a10e8f6bd2c8544669 Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Wed, 9 Nov 2011 16:26:36 +0200 Subject: crypto: serpent-sse2 - add lrw support Patch adds LRW support for serpent-sse2 by using lrw_crypt(). Patch has been tested with tcrypt and automated filesystem tests. Tcrypt benchmarks results (serpent-sse2/serpent_generic speed ratios): Benchmark results with tcrypt: Intel Celeron T1600 (x86_64) (fam:6, model:15, step:13): size lrw-enc lrw-dec 16B 1.00x 0.96x 64B 1.01x 1.01x 256B 3.01x 2.97x 1024B 3.39x 3.33x 8192B 3.35x 3.33x AMD Phenom II 1055T (x86_64) (fam:16, model:10): size lrw-enc lrw-dec 16B 0.98x 1.03x 64B 1.01x 1.04x 256B 2.10x 2.14x 1024B 2.28x 2.33x 8192B 2.30x 2.33x Intel Atom N270 (i586): size lrw-enc lrw-dec 16B 0.97x 0.97x 64B 1.47x 1.50x 256B 1.72x 1.69x 1024B 1.88x 1.81x 8192B 1.84x 1.79x Signed-off-by: Jussi Kivilinna Signed-off-by: Herbert Xu --- arch/x86/crypto/serpent_sse2_glue.c | 211 ++++++++++++++++++++++++++++++++++++ crypto/serpent.c | 10 +- include/crypto/serpent.h | 2 + 3 files changed, 221 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/arch/x86/crypto/serpent_sse2_glue.c b/arch/x86/crypto/serpent_sse2_glue.c index 947cf570f6a7..db318e5cb240 100644 --- a/arch/x86/crypto/serpent_sse2_glue.c +++ b/arch/x86/crypto/serpent_sse2_glue.c @@ -39,12 +39,17 @@ #include #include #include +#include #include #include #include #include #include +#if defined(CONFIG_CRYPTO_LRW) || defined(CONFIG_CRYPTO_LRW_MODULE) +#define HAS_LRW +#endif + struct async_serpent_ctx { struct cryptd_ablkcipher *cryptd_tfm; }; @@ -460,6 +465,152 @@ static struct crypto_alg blk_ctr_alg = { }, }; +#ifdef HAS_LRW + +struct crypt_priv { + struct serpent_ctx *ctx; + bool fpu_enabled; +}; + +static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) +{ + const unsigned int bsize = SERPENT_BLOCK_SIZE; + struct crypt_priv *ctx = priv; + int i; + + ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes); + + if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) { + serpent_enc_blk_xway(ctx->ctx, srcdst, srcdst); + return; + } + + for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) + __serpent_encrypt(ctx->ctx, srcdst, srcdst); +} + +static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) +{ + const unsigned int bsize = SERPENT_BLOCK_SIZE; + struct crypt_priv *ctx = priv; + int i; + + ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes); + + if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) { + serpent_dec_blk_xway(ctx->ctx, srcdst, srcdst); + return; + } + + for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) + __serpent_decrypt(ctx->ctx, srcdst, srcdst); +} + +struct serpent_lrw_ctx { + struct lrw_table_ctx lrw_table; + struct serpent_ctx serpent_ctx; +}; + +static int lrw_serpent_setkey(struct crypto_tfm *tfm, const u8 *key, + unsigned int keylen) +{ + struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm); + int err; + + err = __serpent_setkey(&ctx->serpent_ctx, key, keylen - + SERPENT_BLOCK_SIZE); + if (err) + return err; + + return lrw_init_table(&ctx->lrw_table, key + keylen - + SERPENT_BLOCK_SIZE); +} + +static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + be128 buf[SERPENT_PARALLEL_BLOCKS]; + struct crypt_priv crypt_ctx = { + .ctx = &ctx->serpent_ctx, + .fpu_enabled = false, + }; + struct lrw_crypt_req req = { + .tbuf = buf, + .tbuflen = sizeof(buf), + + .table_ctx = &ctx->lrw_table, + .crypt_ctx = &crypt_ctx, + .crypt_fn = encrypt_callback, + }; + int ret; + + ret = lrw_crypt(desc, dst, src, nbytes, &req); + serpent_fpu_end(crypt_ctx.fpu_enabled); + + return ret; +} + +static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + be128 buf[SERPENT_PARALLEL_BLOCKS]; + struct crypt_priv crypt_ctx = { + .ctx = &ctx->serpent_ctx, + .fpu_enabled = false, + }; + struct lrw_crypt_req req = { + .tbuf = buf, + .tbuflen = sizeof(buf), + + .table_ctx = &ctx->lrw_table, + .crypt_ctx = &crypt_ctx, + .crypt_fn = decrypt_callback, + }; + int ret; + + ret = lrw_crypt(desc, dst, src, nbytes, &req); + serpent_fpu_end(crypt_ctx.fpu_enabled); + + return ret; +} + +static void lrw_exit_tfm(struct crypto_tfm *tfm) +{ + struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm); + + lrw_free_table(&ctx->lrw_table); +} + +static struct crypto_alg blk_lrw_alg = { + .cra_name = "__lrw-serpent-sse2", + .cra_driver_name = "__driver-lrw-serpent-sse2", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = SERPENT_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct serpent_lrw_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(blk_lrw_alg.cra_list), + .cra_exit = lrw_exit_tfm, + .cra_u = { + .blkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE + + SERPENT_BLOCK_SIZE, + .max_keysize = SERPENT_MAX_KEY_SIZE + + SERPENT_BLOCK_SIZE, + .ivsize = SERPENT_BLOCK_SIZE, + .setkey = lrw_serpent_setkey, + .encrypt = lrw_encrypt, + .decrypt = lrw_decrypt, + }, + }, +}; + +#endif + static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key, unsigned int key_len) { @@ -658,6 +809,48 @@ static struct crypto_alg ablk_ctr_alg = { }, }; +#ifdef HAS_LRW + +static int ablk_lrw_init(struct crypto_tfm *tfm) +{ + struct cryptd_ablkcipher *cryptd_tfm; + + cryptd_tfm = cryptd_alloc_ablkcipher("__driver-lrw-serpent-sse2", 0, 0); + if (IS_ERR(cryptd_tfm)) + return PTR_ERR(cryptd_tfm); + ablk_init_common(tfm, cryptd_tfm); + return 0; +} + +static struct crypto_alg ablk_lrw_alg = { + .cra_name = "lrw(serpent)", + .cra_driver_name = "lrw-serpent-sse2", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = SERPENT_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_serpent_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(ablk_lrw_alg.cra_list), + .cra_init = ablk_lrw_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE + + SERPENT_BLOCK_SIZE, + .max_keysize = SERPENT_MAX_KEY_SIZE + + SERPENT_BLOCK_SIZE, + .ivsize = SERPENT_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + }, + }, +}; + +#endif + static int __init serpent_sse2_init(void) { int err; @@ -685,8 +878,22 @@ static int __init serpent_sse2_init(void) err = crypto_register_alg(&ablk_ctr_alg); if (err) goto ablk_ctr_err; +#ifdef HAS_LRW + err = crypto_register_alg(&blk_lrw_alg); + if (err) + goto blk_lrw_err; + err = crypto_register_alg(&ablk_lrw_alg); + if (err) + goto ablk_lrw_err; +#endif return err; +#ifdef HAS_LRW +ablk_lrw_err: + crypto_unregister_alg(&blk_lrw_alg); +blk_lrw_err: + crypto_unregister_alg(&ablk_ctr_alg); +#endif ablk_ctr_err: crypto_unregister_alg(&ablk_cbc_alg); ablk_cbc_err: @@ -703,6 +910,10 @@ blk_ecb_err: static void __exit serpent_sse2_exit(void) { +#ifdef HAS_LRW + crypto_unregister_alg(&ablk_lrw_alg); + crypto_unregister_alg(&blk_lrw_alg); +#endif crypto_unregister_alg(&ablk_ctr_alg); crypto_unregister_alg(&ablk_cbc_alg); crypto_unregister_alg(&ablk_ecb_alg); diff --git a/crypto/serpent.c b/crypto/serpent.c index eb6163041b08..7cc2caeb88a5 100644 --- a/crypto/serpent.c +++ b/crypto/serpent.c @@ -206,9 +206,9 @@ x1 ^= x4; x3 ^= x4; x4 &= x0; \ x4 ^= x2; -int serpent_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen) +int __serpent_setkey(struct serpent_ctx *ctx, const u8 *key, + unsigned int keylen) { - struct serpent_ctx *ctx = crypto_tfm_ctx(tfm); u32 *k = ctx->expkey; u8 *k8 = (u8 *)k; u32 r0,r1,r2,r3,r4; @@ -349,6 +349,12 @@ int serpent_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen) return 0; } +EXPORT_SYMBOL_GPL(__serpent_setkey); + +int serpent_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen) +{ + return __serpent_setkey(crypto_tfm_ctx(tfm), key, keylen); +} EXPORT_SYMBOL_GPL(serpent_setkey); void __serpent_encrypt(struct serpent_ctx *ctx, u8 *dst, const u8 *src) diff --git a/include/crypto/serpent.h b/include/crypto/serpent.h index 40df885f9d1f..b7e0941eb6fc 100644 --- a/include/crypto/serpent.h +++ b/include/crypto/serpent.h @@ -17,6 +17,8 @@ struct serpent_ctx { u32 expkey[SERPENT_EXPKEY_WORDS]; }; +int __serpent_setkey(struct serpent_ctx *ctx, const u8 *key, + unsigned int keylen); int serpent_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen); void __serpent_encrypt(struct serpent_ctx *ctx, u8 *dst, const u8 *src); -- cgit v1.2.3