diff options
Diffstat (limited to 'drivers/crypto/amcc')
-rw-r--r-- | drivers/crypto/amcc/Makefile | 2 | ||||
-rw-r--r-- | drivers/crypto/amcc/crypto4xx_alg.c | 512 | ||||
-rw-r--r-- | drivers/crypto/amcc/crypto4xx_core.c | 831 | ||||
-rw-r--r-- | drivers/crypto/amcc/crypto4xx_core.h | 199 | ||||
-rw-r--r-- | drivers/crypto/amcc/crypto4xx_reg_def.h | 3 | ||||
-rw-r--r-- | drivers/crypto/amcc/crypto4xx_sa.c | 85 | ||||
-rw-r--r-- | drivers/crypto/amcc/crypto4xx_sa.h | 99 |
7 files changed, 1123 insertions, 608 deletions
diff --git a/drivers/crypto/amcc/Makefile b/drivers/crypto/amcc/Makefile index b95539928fdf..e33c185fc163 100644 --- a/drivers/crypto/amcc/Makefile +++ b/drivers/crypto/amcc/Makefile @@ -1,3 +1,3 @@ obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += crypto4xx.o -crypto4xx-y := crypto4xx_core.o crypto4xx_alg.o crypto4xx_sa.o +crypto4xx-y := crypto4xx_core.o crypto4xx_alg.o crypto4xx-$(CONFIG_HW_RANDOM_PPC4XX) += crypto4xx_trng.o diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index 4afca3968773..eeaf27859d80 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -26,11 +26,14 @@ #include <crypto/internal/hash.h> #include <linux/dma-mapping.h> #include <crypto/algapi.h> +#include <crypto/aead.h> #include <crypto/aes.h> +#include <crypto/gcm.h> #include <crypto/sha.h> +#include <crypto/ctr.h> #include "crypto4xx_reg_def.h" -#include "crypto4xx_sa.h" #include "crypto4xx_core.h" +#include "crypto4xx_sa.h" static void set_dynamic_sa_command_0(struct dynamic_sa_ctl *sa, u32 save_h, u32 save_iv, u32 ld_h, u32 ld_iv, @@ -62,6 +65,7 @@ static void set_dynamic_sa_command_1(struct dynamic_sa_ctl *sa, u32 cm, sa->sa_command_1.bf.crypto_mode9_8 = cm & 3; sa->sa_command_1.bf.feedback_mode = cfb, sa->sa_command_1.bf.sa_rev = 1; + sa->sa_command_1.bf.hmac_muting = hmac_mc; sa->sa_command_1.bf.extended_seq_num = esn; sa->sa_command_1.bf.seq_num_mask = sn_mask; sa->sa_command_1.bf.mutable_bit_proc = mute; @@ -73,29 +77,29 @@ static void set_dynamic_sa_command_1(struct dynamic_sa_ctl *sa, u32 cm, int crypto4xx_encrypt(struct ablkcipher_request *req) { struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + unsigned int ivlen = crypto_ablkcipher_ivsize( + crypto_ablkcipher_reqtfm(req)); + __le32 iv[ivlen]; - ctx->direction = DIR_OUTBOUND; - ctx->hash_final = 0; - ctx->is_hash = 0; - ctx->pd_ctl = 0x1; + if (ivlen) + crypto4xx_memcpy_to_le32(iv, req->info, ivlen); return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, - req->nbytes, req->info, - get_dynamic_sa_iv_size(ctx)); + req->nbytes, iv, ivlen, ctx->sa_out, ctx->sa_len, 0); } int crypto4xx_decrypt(struct ablkcipher_request *req) { struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + unsigned int ivlen = crypto_ablkcipher_ivsize( + crypto_ablkcipher_reqtfm(req)); + __le32 iv[ivlen]; - ctx->direction = DIR_INBOUND; - ctx->hash_final = 0; - ctx->is_hash = 0; - ctx->pd_ctl = 1; + if (ivlen) + crypto4xx_memcpy_to_le32(iv, req->info, ivlen); return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, - req->nbytes, req->info, - get_dynamic_sa_iv_size(ctx)); + req->nbytes, iv, ivlen, ctx->sa_in, ctx->sa_len, 0); } /** @@ -120,23 +124,15 @@ static int crypto4xx_setkey_aes(struct crypto_ablkcipher *cipher, } /* Create SA */ - if (ctx->sa_in_dma_addr || ctx->sa_out_dma_addr) + if (ctx->sa_in || ctx->sa_out) crypto4xx_free_sa(ctx); rc = crypto4xx_alloc_sa(ctx, SA_AES128_LEN + (keylen-16) / 4); if (rc) return rc; - if (ctx->state_record_dma_addr == 0) { - rc = crypto4xx_alloc_state_record(ctx); - if (rc) { - crypto4xx_free_sa(ctx); - return rc; - } - } /* Setup SA */ - sa = (struct dynamic_sa_ctl *) ctx->sa_in; - ctx->hash_final = 0; + sa = ctx->sa_in; set_dynamic_sa_command_0(sa, SA_NOT_SAVE_HASH, SA_NOT_SAVE_IV, SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE, @@ -150,18 +146,13 @@ static int crypto4xx_setkey_aes(struct crypto_ablkcipher *cipher, SA_SEQ_MASK_OFF, SA_MC_ENABLE, SA_NOT_COPY_PAD, SA_NOT_COPY_PAYLOAD, SA_NOT_COPY_HDR); - crypto4xx_memcpy_le(ctx->sa_in + get_dynamic_sa_offset_key_field(ctx), - key, keylen); - sa->sa_contents = SA_AES_CONTENTS | (keylen << 2); + crypto4xx_memcpy_to_le32(get_dynamic_sa_key_field(sa), + key, keylen); + sa->sa_contents.w = SA_AES_CONTENTS | (keylen << 2); sa->sa_command_1.bf.key_len = keylen >> 3; - ctx->is_hash = 0; - ctx->direction = DIR_INBOUND; - memcpy(ctx->sa_in + get_dynamic_sa_offset_state_ptr_field(ctx), - (void *)&ctx->state_record_dma_addr, 4); - ctx->offset_to_sr_ptr = get_dynamic_sa_offset_state_ptr_field(ctx); memcpy(ctx->sa_out, ctx->sa_in, ctx->sa_len * 4); - sa = (struct dynamic_sa_ctl *) ctx->sa_out; + sa = ctx->sa_out; sa->sa_command_0.bf.dir = DIR_OUTBOUND; return 0; @@ -174,6 +165,396 @@ int crypto4xx_setkey_aes_cbc(struct crypto_ablkcipher *cipher, CRYPTO_FEEDBACK_MODE_NO_FB); } +int crypto4xx_setkey_aes_cfb(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen) +{ + return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_CFB, + CRYPTO_FEEDBACK_MODE_128BIT_CFB); +} + +int crypto4xx_setkey_aes_ecb(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen) +{ + return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_ECB, + CRYPTO_FEEDBACK_MODE_NO_FB); +} + +int crypto4xx_setkey_aes_ofb(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen) +{ + return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_OFB, + CRYPTO_FEEDBACK_MODE_64BIT_OFB); +} + +int crypto4xx_setkey_rfc3686(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); + int rc; + + rc = crypto4xx_setkey_aes(cipher, key, keylen - CTR_RFC3686_NONCE_SIZE, + CRYPTO_MODE_CTR, CRYPTO_FEEDBACK_MODE_NO_FB); + if (rc) + return rc; + + ctx->iv_nonce = cpu_to_le32p((u32 *)&key[keylen - + CTR_RFC3686_NONCE_SIZE]); + + return 0; +} + +int crypto4xx_rfc3686_encrypt(struct ablkcipher_request *req) +{ + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + __le32 iv[AES_IV_SIZE / 4] = { + ctx->iv_nonce, + cpu_to_le32p((u32 *) req->info), + cpu_to_le32p((u32 *) (req->info + 4)), + cpu_to_le32(1) }; + + return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, + req->nbytes, iv, AES_IV_SIZE, + ctx->sa_out, ctx->sa_len, 0); +} + +int crypto4xx_rfc3686_decrypt(struct ablkcipher_request *req) +{ + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + __le32 iv[AES_IV_SIZE / 4] = { + ctx->iv_nonce, + cpu_to_le32p((u32 *) req->info), + cpu_to_le32p((u32 *) (req->info + 4)), + cpu_to_le32(1) }; + + return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, + req->nbytes, iv, AES_IV_SIZE, + ctx->sa_out, ctx->sa_len, 0); +} + +static inline bool crypto4xx_aead_need_fallback(struct aead_request *req, + bool is_ccm, bool decrypt) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + + /* authsize has to be a multiple of 4 */ + if (aead->authsize & 3) + return true; + + /* + * hardware does not handle cases where cryptlen + * is less than a block + */ + if (req->cryptlen < AES_BLOCK_SIZE) + return true; + + /* assoc len needs to be a multiple of 4 */ + if (req->assoclen & 0x3) + return true; + + /* CCM supports only counter field length of 2 and 4 bytes */ + if (is_ccm && !(req->iv[0] == 1 || req->iv[0] == 3)) + return true; + + /* CCM - fix CBC MAC mismatch in special case */ + if (is_ccm && decrypt && !req->assoclen) + return true; + + return false; +} + +static int crypto4xx_aead_fallback(struct aead_request *req, + struct crypto4xx_ctx *ctx, bool do_decrypt) +{ + char aead_req_data[sizeof(struct aead_request) + + crypto_aead_reqsize(ctx->sw_cipher.aead)] + __aligned(__alignof__(struct aead_request)); + + struct aead_request *subreq = (void *) aead_req_data; + + memset(subreq, 0, sizeof(aead_req_data)); + + aead_request_set_tfm(subreq, ctx->sw_cipher.aead); + aead_request_set_callback(subreq, req->base.flags, + req->base.complete, req->base.data); + aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, + req->iv); + aead_request_set_ad(subreq, req->assoclen); + return do_decrypt ? crypto_aead_decrypt(subreq) : + crypto_aead_encrypt(subreq); +} + +static int crypto4xx_setup_fallback(struct crypto4xx_ctx *ctx, + struct crypto_aead *cipher, + const u8 *key, + unsigned int keylen) +{ + int rc; + + crypto_aead_clear_flags(ctx->sw_cipher.aead, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(ctx->sw_cipher.aead, + crypto_aead_get_flags(cipher) & CRYPTO_TFM_REQ_MASK); + rc = crypto_aead_setkey(ctx->sw_cipher.aead, key, keylen); + crypto_aead_clear_flags(cipher, CRYPTO_TFM_RES_MASK); + crypto_aead_set_flags(cipher, + crypto_aead_get_flags(ctx->sw_cipher.aead) & + CRYPTO_TFM_RES_MASK); + + return rc; +} + +/** + * AES-CCM Functions + */ + +int crypto4xx_setkey_aes_ccm(struct crypto_aead *cipher, const u8 *key, + unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(cipher); + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); + struct dynamic_sa_ctl *sa; + int rc = 0; + + rc = crypto4xx_setup_fallback(ctx, cipher, key, keylen); + if (rc) + return rc; + + if (ctx->sa_in || ctx->sa_out) + crypto4xx_free_sa(ctx); + + rc = crypto4xx_alloc_sa(ctx, SA_AES128_CCM_LEN + (keylen - 16) / 4); + if (rc) + return rc; + + /* Setup SA */ + sa = (struct dynamic_sa_ctl *) ctx->sa_in; + sa->sa_contents.w = SA_AES_CCM_CONTENTS | (keylen << 2); + + set_dynamic_sa_command_0(sa, SA_NOT_SAVE_HASH, SA_NOT_SAVE_IV, + SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE, + SA_NO_HEADER_PROC, SA_HASH_ALG_CBC_MAC, + SA_CIPHER_ALG_AES, + SA_PAD_TYPE_ZERO, SA_OP_GROUP_BASIC, + SA_OPCODE_HASH_DECRYPT, DIR_INBOUND); + + set_dynamic_sa_command_1(sa, CRYPTO_MODE_CTR, SA_HASH_MODE_HASH, + CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF, + SA_SEQ_MASK_OFF, SA_MC_ENABLE, + SA_NOT_COPY_PAD, SA_COPY_PAYLOAD, + SA_NOT_COPY_HDR); + + sa->sa_command_1.bf.key_len = keylen >> 3; + + crypto4xx_memcpy_to_le32(get_dynamic_sa_key_field(sa), key, keylen); + + memcpy(ctx->sa_out, ctx->sa_in, ctx->sa_len * 4); + sa = (struct dynamic_sa_ctl *) ctx->sa_out; + + set_dynamic_sa_command_0(sa, SA_SAVE_HASH, SA_NOT_SAVE_IV, + SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE, + SA_NO_HEADER_PROC, SA_HASH_ALG_CBC_MAC, + SA_CIPHER_ALG_AES, + SA_PAD_TYPE_ZERO, SA_OP_GROUP_BASIC, + SA_OPCODE_ENCRYPT_HASH, DIR_OUTBOUND); + + set_dynamic_sa_command_1(sa, CRYPTO_MODE_CTR, SA_HASH_MODE_HASH, + CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF, + SA_SEQ_MASK_OFF, SA_MC_ENABLE, + SA_COPY_PAD, SA_COPY_PAYLOAD, + SA_NOT_COPY_HDR); + + sa->sa_command_1.bf.key_len = keylen >> 3; + return 0; +} + +static int crypto4xx_crypt_aes_ccm(struct aead_request *req, bool decrypt) +{ + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + unsigned int len = req->cryptlen; + __le32 iv[16]; + u32 tmp_sa[ctx->sa_len * 4]; + struct dynamic_sa_ctl *sa = (struct dynamic_sa_ctl *)tmp_sa; + + if (crypto4xx_aead_need_fallback(req, true, decrypt)) + return crypto4xx_aead_fallback(req, ctx, decrypt); + + if (decrypt) + len -= crypto_aead_authsize(aead); + + memcpy(tmp_sa, decrypt ? ctx->sa_in : ctx->sa_out, sizeof(tmp_sa)); + sa->sa_command_0.bf.digest_len = crypto_aead_authsize(aead) >> 2; + + if (req->iv[0] == 1) { + /* CRYPTO_MODE_AES_ICM */ + sa->sa_command_1.bf.crypto_mode9_8 = 1; + } + + iv[3] = cpu_to_le32(0); + crypto4xx_memcpy_to_le32(iv, req->iv, 16 - (req->iv[0] + 1)); + + return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, + len, iv, sizeof(iv), + sa, ctx->sa_len, req->assoclen); +} + +int crypto4xx_encrypt_aes_ccm(struct aead_request *req) +{ + return crypto4xx_crypt_aes_ccm(req, false); +} + +int crypto4xx_decrypt_aes_ccm(struct aead_request *req) +{ + return crypto4xx_crypt_aes_ccm(req, true); +} + +int crypto4xx_setauthsize_aead(struct crypto_aead *cipher, + unsigned int authsize) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(cipher); + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); + + return crypto_aead_setauthsize(ctx->sw_cipher.aead, authsize); +} + +/** + * AES-GCM Functions + */ + +static int crypto4xx_aes_gcm_validate_keylen(unsigned int keylen) +{ + switch (keylen) { + case 16: + case 24: + case 32: + return 0; + default: + return -EINVAL; + } +} + +static int crypto4xx_compute_gcm_hash_key_sw(__le32 *hash_start, const u8 *key, + unsigned int keylen) +{ + struct crypto_cipher *aes_tfm = NULL; + uint8_t src[16] = { 0 }; + int rc = 0; + + aes_tfm = crypto_alloc_cipher("aes", 0, CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(aes_tfm)) { + rc = PTR_ERR(aes_tfm); + pr_warn("could not load aes cipher driver: %d\n", rc); + return rc; + } + + rc = crypto_cipher_setkey(aes_tfm, key, keylen); + if (rc) { + pr_err("setkey() failed: %d\n", rc); + goto out; + } + + crypto_cipher_encrypt_one(aes_tfm, src, src); + crypto4xx_memcpy_to_le32(hash_start, src, 16); +out: + crypto_free_cipher(aes_tfm); + return rc; +} + +int crypto4xx_setkey_aes_gcm(struct crypto_aead *cipher, + const u8 *key, unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(cipher); + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); + struct dynamic_sa_ctl *sa; + int rc = 0; + + if (crypto4xx_aes_gcm_validate_keylen(keylen) != 0) { + crypto_aead_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + rc = crypto4xx_setup_fallback(ctx, cipher, key, keylen); + if (rc) + return rc; + + if (ctx->sa_in || ctx->sa_out) + crypto4xx_free_sa(ctx); + + rc = crypto4xx_alloc_sa(ctx, SA_AES128_GCM_LEN + (keylen - 16) / 4); + if (rc) + return rc; + + sa = (struct dynamic_sa_ctl *) ctx->sa_in; + + sa->sa_contents.w = SA_AES_GCM_CONTENTS | (keylen << 2); + set_dynamic_sa_command_0(sa, SA_SAVE_HASH, SA_NOT_SAVE_IV, + SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE, + SA_NO_HEADER_PROC, SA_HASH_ALG_GHASH, + SA_CIPHER_ALG_AES, SA_PAD_TYPE_ZERO, + SA_OP_GROUP_BASIC, SA_OPCODE_HASH_DECRYPT, + DIR_INBOUND); + set_dynamic_sa_command_1(sa, CRYPTO_MODE_CTR, SA_HASH_MODE_HASH, + CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF, + SA_SEQ_MASK_ON, SA_MC_DISABLE, + SA_NOT_COPY_PAD, SA_COPY_PAYLOAD, + SA_NOT_COPY_HDR); + + sa->sa_command_1.bf.key_len = keylen >> 3; + + crypto4xx_memcpy_to_le32(get_dynamic_sa_key_field(sa), + key, keylen); + + rc = crypto4xx_compute_gcm_hash_key_sw(get_dynamic_sa_inner_digest(sa), + key, keylen); + if (rc) { + pr_err("GCM hash key setting failed = %d\n", rc); + goto err; + } + + memcpy(ctx->sa_out, ctx->sa_in, ctx->sa_len * 4); + sa = (struct dynamic_sa_ctl *) ctx->sa_out; + sa->sa_command_0.bf.dir = DIR_OUTBOUND; + sa->sa_command_0.bf.opcode = SA_OPCODE_ENCRYPT_HASH; + + return 0; +err: + crypto4xx_free_sa(ctx); + return rc; +} + +static inline int crypto4xx_crypt_aes_gcm(struct aead_request *req, + bool decrypt) +{ + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + unsigned int len = req->cryptlen; + __le32 iv[4]; + + if (crypto4xx_aead_need_fallback(req, false, decrypt)) + return crypto4xx_aead_fallback(req, ctx, decrypt); + + crypto4xx_memcpy_to_le32(iv, req->iv, GCM_AES_IV_SIZE); + iv[3] = cpu_to_le32(1); + + if (decrypt) + len -= crypto_aead_authsize(crypto_aead_reqtfm(req)); + + return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, + len, iv, sizeof(iv), + decrypt ? ctx->sa_in : ctx->sa_out, + ctx->sa_len, req->assoclen); +} + +int crypto4xx_encrypt_aes_gcm(struct aead_request *req) +{ + return crypto4xx_crypt_aes_gcm(req, false); +} + +int crypto4xx_decrypt_aes_gcm(struct aead_request *req) +{ + return crypto4xx_crypt_aes_gcm(req, true); +} + /** * HASH SHA1 Functions */ @@ -183,53 +564,39 @@ static int crypto4xx_hash_alg_init(struct crypto_tfm *tfm, unsigned char hm) { struct crypto_alg *alg = tfm->__crt_alg; - struct crypto4xx_alg *my_alg = crypto_alg_to_crypto4xx_alg(alg); + struct crypto4xx_alg *my_alg; struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); - struct dynamic_sa_ctl *sa; - struct dynamic_sa_hash160 *sa_in; + struct dynamic_sa_hash160 *sa; int rc; + my_alg = container_of(__crypto_ahash_alg(alg), struct crypto4xx_alg, + alg.u.hash); ctx->dev = my_alg->dev; - ctx->is_hash = 1; - ctx->hash_final = 0; /* Create SA */ - if (ctx->sa_in_dma_addr || ctx->sa_out_dma_addr) + if (ctx->sa_in || ctx->sa_out) crypto4xx_free_sa(ctx); rc = crypto4xx_alloc_sa(ctx, sa_len); if (rc) return rc; - if (ctx->state_record_dma_addr == 0) { - crypto4xx_alloc_state_record(ctx); - if (!ctx->state_record_dma_addr) { - crypto4xx_free_sa(ctx); - return -ENOMEM; - } - } - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), sizeof(struct crypto4xx_ctx)); - sa = (struct dynamic_sa_ctl *) ctx->sa_in; - set_dynamic_sa_command_0(sa, SA_SAVE_HASH, SA_NOT_SAVE_IV, + sa = (struct dynamic_sa_hash160 *)ctx->sa_in; + set_dynamic_sa_command_0(&sa->ctrl, SA_SAVE_HASH, SA_NOT_SAVE_IV, SA_NOT_LOAD_HASH, SA_LOAD_IV_FROM_SA, SA_NO_HEADER_PROC, ha, SA_CIPHER_ALG_NULL, SA_PAD_TYPE_ZERO, SA_OP_GROUP_BASIC, SA_OPCODE_HASH, DIR_INBOUND); - set_dynamic_sa_command_1(sa, 0, SA_HASH_MODE_HASH, + set_dynamic_sa_command_1(&sa->ctrl, 0, SA_HASH_MODE_HASH, CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF, SA_SEQ_MASK_OFF, SA_MC_ENABLE, SA_NOT_COPY_PAD, SA_NOT_COPY_PAYLOAD, SA_NOT_COPY_HDR); - ctx->direction = DIR_INBOUND; - sa->sa_contents = SA_HASH160_CONTENTS; - sa_in = (struct dynamic_sa_hash160 *) ctx->sa_in; /* Need to zero hash digest in SA */ - memset(sa_in->inner_digest, 0, sizeof(sa_in->inner_digest)); - memset(sa_in->outer_digest, 0, sizeof(sa_in->outer_digest)); - sa_in->state_ptr = ctx->state_record_dma_addr; - ctx->offset_to_sr_ptr = get_dynamic_sa_offset_state_ptr_field(ctx); + memset(sa->inner_digest, 0, sizeof(sa->inner_digest)); + memset(sa->outer_digest, 0, sizeof(sa->outer_digest)); return 0; } @@ -240,29 +607,27 @@ int crypto4xx_hash_init(struct ahash_request *req) int ds; struct dynamic_sa_ctl *sa; - sa = (struct dynamic_sa_ctl *) ctx->sa_in; + sa = ctx->sa_in; ds = crypto_ahash_digestsize( __crypto_ahash_cast(req->base.tfm)); sa->sa_command_0.bf.digest_len = ds >> 2; sa->sa_command_0.bf.load_hash_state = SA_LOAD_HASH_FROM_SA; - ctx->is_hash = 1; - ctx->direction = DIR_INBOUND; return 0; } int crypto4xx_hash_update(struct ahash_request *req) { + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct scatterlist dst; + unsigned int ds = crypto_ahash_digestsize(ahash); - ctx->is_hash = 1; - ctx->hash_final = 0; - ctx->pd_ctl = 0x11; - ctx->direction = DIR_INBOUND; + sg_init_one(&dst, req->result, ds); - return crypto4xx_build_pd(&req->base, ctx, req->src, - (struct scatterlist *) req->result, - req->nbytes, NULL, 0); + return crypto4xx_build_pd(&req->base, ctx, req->src, &dst, + req->nbytes, NULL, 0, ctx->sa_in, + ctx->sa_len, 0); } int crypto4xx_hash_final(struct ahash_request *req) @@ -272,15 +637,16 @@ int crypto4xx_hash_final(struct ahash_request *req) int crypto4xx_hash_digest(struct ahash_request *req) { + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct scatterlist dst; + unsigned int ds = crypto_ahash_digestsize(ahash); - ctx->hash_final = 1; - ctx->pd_ctl = 0x11; - ctx->direction = DIR_INBOUND; + sg_init_one(&dst, req->result, ds); - return crypto4xx_build_pd(&req->base, ctx, req->src, - (struct scatterlist *) req->result, - req->nbytes, NULL, 0); + return crypto4xx_build_pd(&req->base, ctx, req->src, &dst, + req->nbytes, NULL, 0, ctx->sa_in, + ctx->sa_len, 0); } /** @@ -291,5 +657,3 @@ int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm) return crypto4xx_hash_alg_init(tfm, SA_HASH160_LEN, SA_HASH_ALG_SHA1, SA_HASH_MODE_HASH); } - - diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 65dc78b91dea..c44954e274bc 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -35,8 +35,14 @@ #include <asm/dcr.h> #include <asm/dcr-regs.h> #include <asm/cacheflush.h> +#include <crypto/aead.h> #include <crypto/aes.h> +#include <crypto/ctr.h> +#include <crypto/gcm.h> #include <crypto/sha.h> +#include <crypto/scatterwalk.h> +#include <crypto/internal/aead.h> +#include <crypto/internal/skcipher.h> #include "crypto4xx_reg_def.h" #include "crypto4xx_core.h" #include "crypto4xx_sa.h" @@ -127,21 +133,17 @@ static void crypto4xx_hw_init(struct crypto4xx_device *dev) int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size) { - ctx->sa_in = dma_alloc_coherent(ctx->dev->core_dev->device, size * 4, - &ctx->sa_in_dma_addr, GFP_ATOMIC); + ctx->sa_in = kzalloc(size * 4, GFP_ATOMIC); if (ctx->sa_in == NULL) return -ENOMEM; - ctx->sa_out = dma_alloc_coherent(ctx->dev->core_dev->device, size * 4, - &ctx->sa_out_dma_addr, GFP_ATOMIC); + ctx->sa_out = kzalloc(size * 4, GFP_ATOMIC); if (ctx->sa_out == NULL) { - dma_free_coherent(ctx->dev->core_dev->device, size * 4, - ctx->sa_in, ctx->sa_in_dma_addr); + kfree(ctx->sa_in); + ctx->sa_in = NULL; return -ENOMEM; } - memset(ctx->sa_in, 0, size * 4); - memset(ctx->sa_out, 0, size * 4); ctx->sa_len = size; return 0; @@ -149,40 +151,13 @@ int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size) void crypto4xx_free_sa(struct crypto4xx_ctx *ctx) { - if (ctx->sa_in != NULL) - dma_free_coherent(ctx->dev->core_dev->device, ctx->sa_len * 4, - ctx->sa_in, ctx->sa_in_dma_addr); - if (ctx->sa_out != NULL) - dma_free_coherent(ctx->dev->core_dev->device, ctx->sa_len * 4, - ctx->sa_out, ctx->sa_out_dma_addr); - - ctx->sa_in_dma_addr = 0; - ctx->sa_out_dma_addr = 0; + kfree(ctx->sa_in); + ctx->sa_in = NULL; + kfree(ctx->sa_out); + ctx->sa_out = NULL; ctx->sa_len = 0; } -u32 crypto4xx_alloc_state_record(struct crypto4xx_ctx *ctx) -{ - ctx->state_record = dma_alloc_coherent(ctx->dev->core_dev->device, - sizeof(struct sa_state_record), - &ctx->state_record_dma_addr, GFP_ATOMIC); - if (!ctx->state_record_dma_addr) - return -ENOMEM; - memset(ctx->state_record, 0, sizeof(struct sa_state_record)); - - return 0; -} - -void crypto4xx_free_state_record(struct crypto4xx_ctx *ctx) -{ - if (ctx->state_record != NULL) - dma_free_coherent(ctx->dev->core_dev->device, - sizeof(struct sa_state_record), - ctx->state_record, - ctx->state_record_dma_addr); - ctx->state_record_dma_addr = 0; -} - /** * alloc memory for the gather ring * no need to alloc buf for the ring @@ -191,7 +166,6 @@ void crypto4xx_free_state_record(struct crypto4xx_ctx *ctx) static u32 crypto4xx_build_pdr(struct crypto4xx_device *dev) { int i; - struct pd_uinfo *pd_uinfo; dev->pdr = dma_alloc_coherent(dev->core_dev->device, sizeof(struct ce_pd) * PPC4XX_NUM_PD, &dev->pdr_pa, GFP_ATOMIC); @@ -207,9 +181,9 @@ static u32 crypto4xx_build_pdr(struct crypto4xx_device *dev) dev->pdr_pa); return -ENOMEM; } - memset(dev->pdr, 0, sizeof(struct ce_pd) * PPC4XX_NUM_PD); + memset(dev->pdr, 0, sizeof(struct ce_pd) * PPC4XX_NUM_PD); dev->shadow_sa_pool = dma_alloc_coherent(dev->core_dev->device, - 256 * PPC4XX_NUM_PD, + sizeof(union shadow_sa_buf) * PPC4XX_NUM_PD, &dev->shadow_sa_pool_pa, GFP_ATOMIC); if (!dev->shadow_sa_pool) @@ -221,16 +195,17 @@ static u32 crypto4xx_build_pdr(struct crypto4xx_device *dev) if (!dev->shadow_sr_pool) return -ENOMEM; for (i = 0; i < PPC4XX_NUM_PD; i++) { - pd_uinfo = (struct pd_uinfo *) (dev->pdr_uinfo + - sizeof(struct pd_uinfo) * i); + struct ce_pd *pd = &dev->pdr[i]; + struct pd_uinfo *pd_uinfo = &dev->pdr_uinfo[i]; + + pd->sa = dev->shadow_sa_pool_pa + + sizeof(union shadow_sa_buf) * i; /* alloc 256 bytes which is enough for any kind of dynamic sa */ - pd_uinfo->sa_va = dev->shadow_sa_pool + 256 * i; - pd_uinfo->sa_pa = dev->shadow_sa_pool_pa + 256 * i; + pd_uinfo->sa_va = &dev->shadow_sa_pool[i].sa; /* alloc state record */ - pd_uinfo->sr_va = dev->shadow_sr_pool + - sizeof(struct sa_state_record) * i; + pd_uinfo->sr_va = &dev->shadow_sr_pool[i]; pd_uinfo->sr_pa = dev->shadow_sr_pool_pa + sizeof(struct sa_state_record) * i; } @@ -240,13 +215,16 @@ static u32 crypto4xx_build_pdr(struct crypto4xx_device *dev) static void crypto4xx_destroy_pdr(struct crypto4xx_device *dev) { - if (dev->pdr != NULL) + if (dev->pdr) dma_free_coherent(dev->core_dev->device, sizeof(struct ce_pd) * PPC4XX_NUM_PD, dev->pdr, dev->pdr_pa); + if (dev->shadow_sa_pool) - dma_free_coherent(dev->core_dev->device, 256 * PPC4XX_NUM_PD, - dev->shadow_sa_pool, dev->shadow_sa_pool_pa); + dma_free_coherent(dev->core_dev->device, + sizeof(union shadow_sa_buf) * PPC4XX_NUM_PD, + dev->shadow_sa_pool, dev->shadow_sa_pool_pa); + if (dev->shadow_sr_pool) dma_free_coherent(dev->core_dev->device, sizeof(struct sa_state_record) * PPC4XX_NUM_PD, @@ -273,28 +251,21 @@ static u32 crypto4xx_get_pd_from_pdr_nolock(struct crypto4xx_device *dev) static u32 crypto4xx_put_pd_to_pdr(struct crypto4xx_device *dev, u32 idx) { - struct pd_uinfo *pd_uinfo; + struct pd_uinfo *pd_uinfo = &dev->pdr_uinfo[idx]; + u32 tail; unsigned long flags; - pd_uinfo = (struct pd_uinfo *)(dev->pdr_uinfo + - sizeof(struct pd_uinfo) * idx); spin_lock_irqsave(&dev->core_dev->lock, flags); + pd_uinfo->state = PD_ENTRY_FREE; + if (dev->pdr_tail != PPC4XX_LAST_PD) dev->pdr_tail++; else dev->pdr_tail = 0; - pd_uinfo->state = PD_ENTRY_FREE; + tail = dev->pdr_tail; spin_unlock_irqrestore(&dev->core_dev->lock, flags); - return 0; -} - -static struct ce_pd *crypto4xx_get_pdp(struct crypto4xx_device *dev, - dma_addr_t *pd_dma, u32 idx) -{ - *pd_dma = dev->pdr_pa + sizeof(struct ce_pd) * idx; - - return dev->pdr + sizeof(struct ce_pd) * idx; + return tail; } /** @@ -326,10 +297,11 @@ static inline void crypto4xx_destroy_gdr(struct crypto4xx_device *dev) * when this function is called. * preemption or interrupt must be disabled */ -u32 crypto4xx_get_n_gd(struct crypto4xx_device *dev, int n) +static u32 crypto4xx_get_n_gd(struct crypto4xx_device *dev, int n) { u32 retval; u32 tmp; + if (n >= PPC4XX_NUM_GD) return ERING_WAS_FULL; @@ -372,7 +344,7 @@ static inline struct ce_gd *crypto4xx_get_gdp(struct crypto4xx_device *dev, { *gd_dma = dev->gdr_pa + sizeof(struct ce_gd) * idx; - return (struct ce_gd *) (dev->gdr + sizeof(struct ce_gd) * idx); + return &dev->gdr[idx]; } /** @@ -383,7 +355,6 @@ static inline struct ce_gd *crypto4xx_get_gdp(struct crypto4xx_device *dev, static u32 crypto4xx_build_sdr(struct crypto4xx_device *dev) { int i; - struct ce_sd *sd_array; /* alloc memory for scatter descriptor ring */ dev->sdr = dma_alloc_coherent(dev->core_dev->device, @@ -392,10 +363,9 @@ static u32 crypto4xx_build_sdr(struct crypto4xx_device *dev) if (!dev->sdr) return -ENOMEM; - dev->scatter_buffer_size = PPC4XX_SD_BUFFER_SIZE; dev->scatter_buffer_va = dma_alloc_coherent(dev->core_dev->device, - dev->scatter_buffer_size * PPC4XX_NUM_SD, + PPC4XX_SD_BUFFER_SIZE * PPC4XX_NUM_SD, &dev->scatter_buffer_pa, GFP_ATOMIC); if (!dev->scatter_buffer_va) { dma_free_coherent(dev->core_dev->device, @@ -404,11 +374,9 @@ static u32 crypto4xx_build_sdr(struct crypto4xx_device *dev) return -ENOMEM; } - sd_array = dev->sdr; - for (i = 0; i < PPC4XX_NUM_SD; i++) { - sd_array[i].ptr = dev->scatter_buffer_pa + - dev->scatter_buffer_size * i; + dev->sdr[i].ptr = dev->scatter_buffer_pa + + PPC4XX_SD_BUFFER_SIZE * i; } return 0; @@ -416,14 +384,14 @@ static u32 crypto4xx_build_sdr(struct crypto4xx_device *dev) static void crypto4xx_destroy_sdr(struct crypto4xx_device *dev) { - if (dev->sdr != NULL) + if (dev->sdr) dma_free_coherent(dev->core_dev->device, sizeof(struct ce_sd) * PPC4XX_NUM_SD, dev->sdr, dev->sdr_pa); - if (dev->scatter_buffer_va != NULL) + if (dev->scatter_buffer_va) dma_free_coherent(dev->core_dev->device, - dev->scatter_buffer_size * PPC4XX_NUM_SD, + PPC4XX_SD_BUFFER_SIZE * PPC4XX_NUM_SD, dev->scatter_buffer_va, dev->scatter_buffer_pa); } @@ -477,63 +445,7 @@ static inline struct ce_sd *crypto4xx_get_sdp(struct crypto4xx_device *dev, { *sd_dma = dev->sdr_pa + sizeof(struct ce_sd) * idx; - return (struct ce_sd *)(dev->sdr + sizeof(struct ce_sd) * idx); -} - -static u32 crypto4xx_fill_one_page(struct crypto4xx_device *dev, - dma_addr_t *addr, u32 *length, - u32 *idx, u32 *offset, u32 *nbytes) -{ - u32 len; - - if (*length > dev->scatter_buffer_size) { - memcpy(phys_to_virt(*addr), - dev->scatter_buffer_va + - *idx * dev->scatter_buffer_size + *offset, - dev->scatter_buffer_size); - *offset = 0; - *length -= dev->scatter_buffer_size; - *nbytes -= dev->scatter_buffer_size; - if (*idx == PPC4XX_LAST_SD) - *idx = 0; - else - (*idx)++; - *addr = *addr + dev->scatter_buffer_size; - return 1; - } else if (*length < dev->scatter_buffer_size) { - memcpy(phys_to_virt(*addr), - dev->scatter_buffer_va + - *idx * dev->scatter_buffer_size + *offset, *length); - if ((*offset + *length) == dev->scatter_buffer_size) { - if (*idx == PPC4XX_LAST_SD) - *idx = 0; - else - (*idx)++; - *nbytes -= *length; - *offset = 0; - } else { - *nbytes -= *length; - *offset += *length; - } - - return 0; - } else { - len = (*nbytes <= dev->scatter_buffer_size) ? - (*nbytes) : dev->scatter_buffer_size; - memcpy(phys_to_virt(*addr), - dev->scatter_buffer_va + - *idx * dev->scatter_buffer_size + *offset, - len); - *offset = 0; - *nbytes -= len; - - if (*idx == PPC4XX_LAST_SD) - *idx = 0; - else - (*idx)++; - - return 0; - } + return &dev->sdr[idx]; } static void crypto4xx_copy_pkt_to_dst(struct crypto4xx_device *dev, @@ -542,66 +454,52 @@ static void crypto4xx_copy_pkt_to_dst(struct crypto4xx_device *dev, u32 nbytes, struct scatterlist *dst) { - dma_addr_t addr; - u32 this_sd; - u32 offset; - u32 len; - u32 i; - u32 sg_len; - struct scatterlist *sg; + unsigned int first_sd = pd_uinfo->first_sd; + unsigned int last_sd; + unsigned int overflow = 0; + unsigned int to_copy; + unsigned int dst_start = 0; - this_sd = pd_uinfo->first_sd; - offset = 0; - i = 0; + /* + * Because the scatter buffers are all neatly organized in one + * big continuous ringbuffer; scatterwalk_map_and_copy() can + * be instructed to copy a range of buffers in one go. + */ + + last_sd = (first_sd + pd_uinfo->num_sd); + if (last_sd > PPC4XX_LAST_SD) { + last_sd = PPC4XX_LAST_SD; + overflow = last_sd % PPC4XX_NUM_SD; + } while (nbytes) { - sg = &dst[i]; - sg_len = sg->length; - addr = dma_map_page(dev->core_dev->device, sg_page(sg), - sg->offset, sg->length, DMA_TO_DEVICE); - - if (offset == 0) { - len = (nbytes <= sg->length) ? nbytes : sg->length; - while (crypto4xx_fill_one_page(dev, &addr, &len, - &this_sd, &offset, &nbytes)) - ; - if (!nbytes) - return; - i++; - } else { - len = (nbytes <= (dev->scatter_buffer_size - offset)) ? - nbytes : (dev->scatter_buffer_size - offset); - len = (sg->length < len) ? sg->length : len; - while (crypto4xx_fill_one_page(dev, &addr, &len, - &this_sd, &offset, &nbytes)) - ; - if (!nbytes) - return; - sg_len -= len; - if (sg_len) { - addr += len; - while (crypto4xx_fill_one_page(dev, &addr, - &sg_len, &this_sd, &offset, &nbytes)) - ; - } - i++; + void *buf = dev->scatter_buffer_va + + first_sd * PPC4XX_SD_BUFFER_SIZE; + + to_copy = min(nbytes, PPC4XX_SD_BUFFER_SIZE * + (1 + last_sd - first_sd)); + scatterwalk_map_and_copy(buf, dst, dst_start, to_copy, 1); + nbytes -= to_copy; + + if (overflow) { + first_sd = 0; + last_sd = overflow; + dst_start += to_copy; + overflow = 0; } } } -static u32 crypto4xx_copy_digest_to_dst(struct pd_uinfo *pd_uinfo, +static void crypto4xx_copy_digest_to_dst(void *dst, + struct pd_uinfo *pd_uinfo, struct crypto4xx_ctx *ctx) { struct dynamic_sa_ctl *sa = (struct dynamic_sa_ctl *) ctx->sa_in; - struct sa_state_record *state_record = - (struct sa_state_record *) pd_uinfo->sr_va; if (sa->sa_command_0.bf.hash_alg == SA_HASH_ALG_SHA1) { - memcpy((void *) pd_uinfo->dest_va, state_record->save_digest, + memcpy(dst, pd_uinfo->sr_va->save_digest, SA_HASH_ALG_SHA1_DIGEST_SIZE); } - - return 0; } static void crypto4xx_ret_sg_desc(struct crypto4xx_device *dev, @@ -623,7 +521,7 @@ static void crypto4xx_ret_sg_desc(struct crypto4xx_device *dev, } } -static u32 crypto4xx_ablkcipher_done(struct crypto4xx_device *dev, +static void crypto4xx_ablkcipher_done(struct crypto4xx_device *dev, struct pd_uinfo *pd_uinfo, struct ce_pd *pd) { @@ -644,13 +542,13 @@ static u32 crypto4xx_ablkcipher_done(struct crypto4xx_device *dev, dst->offset, dst->length, DMA_FROM_DEVICE); } crypto4xx_ret_sg_desc(dev, pd_uinfo); - if (ablk_req->base.complete != NULL) - ablk_req->base.complete(&ablk_req->base, 0); - return 0; + if (pd_uinfo->state & PD_ENTRY_BUSY) + ablkcipher_request_complete(ablk_req, -EINPROGRESS); + ablkcipher_request_complete(ablk_req, 0); } -static u32 crypto4xx_ahash_done(struct crypto4xx_device *dev, +static void crypto4xx_ahash_done(struct crypto4xx_device *dev, struct pd_uinfo *pd_uinfo) { struct crypto4xx_ctx *ctx; @@ -659,62 +557,93 @@ static u32 crypto4xx_ahash_done(struct crypto4xx_device *dev, ahash_req = ahash_request_cast(pd_uinfo->async_req); ctx = crypto_tfm_ctx(ahash_req->base.tfm); - crypto4xx_copy_digest_to_dst(pd_uinfo, + crypto4xx_copy_digest_to_dst(ahash_req->result, pd_uinfo, crypto_tfm_ctx(ahash_req->base.tfm)); crypto4xx_ret_sg_desc(dev, pd_uinfo); - /* call user provided callback function x */ - if (ahash_req->base.complete != NULL) - ahash_req->base.complete(&ahash_req->base, 0); - return 0; + if (pd_uinfo->state & PD_ENTRY_BUSY) + ahash_request_complete(ahash_req, -EINPROGRESS); + ahash_request_complete(ahash_req, 0); } -static u32 crypto4xx_pd_done(struct crypto4xx_device *dev, u32 idx) +static void crypto4xx_aead_done(struct crypto4xx_device *dev, + struct pd_uinfo *pd_uinfo, + struct ce_pd *pd) { - struct ce_pd *pd; - struct pd_uinfo *pd_uinfo; + struct aead_request *aead_req; + struct crypto4xx_ctx *ctx; + struct scatterlist *dst = pd_uinfo->dest_va; + int err = 0; - pd = dev->pdr + sizeof(struct ce_pd)*idx; - pd_uinfo = dev->pdr_uinfo + sizeof(struct pd_uinfo)*idx; - if (crypto_tfm_alg_type(pd_uinfo->async_req->tfm) == - CRYPTO_ALG_TYPE_ABLKCIPHER) - return crypto4xx_ablkcipher_done(dev, pd_uinfo, pd); - else - return crypto4xx_ahash_done(dev, pd_uinfo); + aead_req = container_of(pd_uinfo->async_req, struct aead_request, + base); + ctx = crypto_tfm_ctx(aead_req->base.tfm); + + if (pd_uinfo->using_sd) { + crypto4xx_copy_pkt_to_dst(dev, pd, pd_uinfo, + pd->pd_ctl_len.bf.pkt_len, + dst); + } else { + __dma_sync_page(sg_page(dst), dst->offset, dst->length, + DMA_FROM_DEVICE); + } + + if (pd_uinfo->sa_va->sa_command_0.bf.dir == DIR_OUTBOUND) { + /* append icv at the end */ + size_t cp_len = crypto_aead_authsize( + crypto_aead_reqtfm(aead_req)); + u32 icv[cp_len]; + + crypto4xx_memcpy_from_le32(icv, pd_uinfo->sr_va->save_digest, + cp_len); + + scatterwalk_map_and_copy(icv, dst, aead_req->cryptlen, + cp_len, 1); + } + + crypto4xx_ret_sg_desc(dev, pd_uinfo); + + if (pd->pd_ctl.bf.status & 0xff) { + if (pd->pd_ctl.bf.status & 0x1) { + /* authentication error */ + err = -EBADMSG; + } else { + if (!__ratelimit(&dev->aead_ratelimit)) { + if (pd->pd_ctl.bf.status & 2) + pr_err("pad fail error\n"); + if (pd->pd_ctl.bf.status & 4) + pr_err("seqnum fail\n"); + if (pd->pd_ctl.bf.status & 8) + pr_err("error _notify\n"); + pr_err("aead return err status = 0x%02x\n", + pd->pd_ctl.bf.status & 0xff); + pr_err("pd pad_ctl = 0x%08x\n", + pd->pd_ctl.bf.pd_pad_ctl); + } + err = -EINVAL; + } + } + + if (pd_uinfo->state & PD_ENTRY_BUSY) + aead_request_complete(aead_req, -EINPROGRESS); + + aead_request_complete(aead_req, err); } -/** - * Note: Only use this function to copy items that is word aligned. - */ -void crypto4xx_memcpy_le(unsigned int *dst, - const unsigned char *buf, - int len) +static void crypto4xx_pd_done(struct crypto4xx_device *dev, u32 idx) { - u8 *tmp; - for (; len >= 4; buf += 4, len -= 4) - *dst++ = cpu_to_le32(*(unsigned int *) buf); - - tmp = (u8 *)dst; - switch (len) { - case 3: - *tmp++ = 0; - *tmp++ = *(buf+2); - *tmp++ = *(buf+1); - *tmp++ = *buf; - break; - case 2: - *tmp++ = 0; - *tmp++ = 0; - *tmp++ = *(buf+1); - *tmp++ = *buf; + struct ce_pd *pd = &dev->pdr[idx]; + struct pd_uinfo *pd_uinfo = &dev->pdr_uinfo[idx]; + + switch (crypto_tfm_alg_type(pd_uinfo->async_req->tfm)) { + case CRYPTO_ALG_TYPE_ABLKCIPHER: + crypto4xx_ablkcipher_done(dev, pd_uinfo, pd); break; - case 1: - *tmp++ = 0; - *tmp++ = 0; - *tmp++ = 0; - *tmp++ = *buf; + case CRYPTO_ALG_TYPE_AEAD: + crypto4xx_aead_done(dev, pd_uinfo, pd); break; - default: + case CRYPTO_ALG_TYPE_AHASH: + crypto4xx_ahash_done(dev, pd_uinfo); break; } } @@ -729,17 +658,6 @@ static void crypto4xx_stop_all(struct crypto4xx_core_device *core_dev) kfree(core_dev); } -void crypto4xx_return_pd(struct crypto4xx_device *dev, - u32 pd_entry, struct ce_pd *pd, - struct pd_uinfo *pd_uinfo) -{ - /* irq should be already disabled */ - dev->pdr_head = pd_entry; - pd->pd_ctl.w = 0; - pd->pd_ctl_len.w = 0; - pd_uinfo->state = PD_ENTRY_FREE; -} - static u32 get_next_gd(u32 current) { if (current != PPC4XX_LAST_GD) @@ -756,17 +674,19 @@ static u32 get_next_sd(u32 current) return 0; } -u32 crypto4xx_build_pd(struct crypto_async_request *req, +int crypto4xx_build_pd(struct crypto_async_request *req, struct crypto4xx_ctx *ctx, struct scatterlist *src, struct scatterlist *dst, - unsigned int datalen, - void *iv, u32 iv_len) + const unsigned int datalen, + const __le32 *iv, const u32 iv_len, + const struct dynamic_sa_ctl *req_sa, + const unsigned int sa_len, + const unsigned int assoclen) { + struct scatterlist _dst[2]; struct crypto4xx_device *dev = ctx->dev; - dma_addr_t addr, pd_dma, sd_dma, gd_dma; struct dynamic_sa_ctl *sa; - struct scatterlist *sg; struct ce_gd *gd; struct ce_pd *pd; u32 num_gd, num_sd; @@ -774,22 +694,30 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, u32 fst_sd = 0xffffffff; u32 pd_entry; unsigned long flags; - struct pd_uinfo *pd_uinfo = NULL; - unsigned int nbytes = datalen, idx; - unsigned int ivlen = 0; + struct pd_uinfo *pd_uinfo; + unsigned int nbytes = datalen; + size_t offset_to_sr_ptr; u32 gd_idx = 0; + int tmp; + bool is_busy; - /* figure how many gd is needed */ - num_gd = sg_nents_for_len(src, datalen); - if ((int)num_gd < 0) { + /* figure how many gd are needed */ + tmp = sg_nents_for_len(src, assoclen + datalen); + if (tmp < 0) { dev_err(dev->core_dev->device, "Invalid number of src SG.\n"); - return -EINVAL; + return tmp; } - if (num_gd == 1) - num_gd = 0; + if (tmp == 1) + tmp = 0; + num_gd = tmp; - /* figure how many sd is needed */ - if (sg_is_last(dst) || ctx->is_hash) { + if (assoclen) { + nbytes += assoclen; + dst = scatterwalk_ffwd(_dst, dst, assoclen); + } + + /* figure how many sd are needed */ + if (sg_is_last(dst)) { num_sd = 0; } else { if (datalen > PPC4XX_SD_BUFFER_SIZE) { @@ -808,6 +736,31 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, * already got must be return the original place. */ spin_lock_irqsave(&dev->core_dev->lock, flags); + /* + * Let the caller know to slow down, once more than 13/16ths = 81% + * of the available data contexts are being used simultaneously. + * + * With PPC4XX_NUM_PD = 256, this will leave a "backlog queue" for + * 31 more contexts. Before new requests have to be rejected. + */ + if (req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG) { + is_busy = ((dev->pdr_head - dev->pdr_tail) % PPC4XX_NUM_PD) >= + ((PPC4XX_NUM_PD * 13) / 16); + } else { + /* + * To fix contention issues between ipsec (no blacklog) and + * dm-crypto (backlog) reserve 32 entries for "no backlog" + * data contexts. + */ + is_busy = ((dev->pdr_head - dev->pdr_tail) % PPC4XX_NUM_PD) >= + ((PPC4XX_NUM_PD * 15) / 16); + + if (is_busy) { + spin_unlock_irqrestore(&dev->core_dev->lock, flags); + return -EBUSY; + } + } + if (num_gd) { fst_gd = crypto4xx_get_n_gd(dev, num_gd); if (fst_gd == ERING_WAS_FULL) { @@ -835,38 +788,28 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, } spin_unlock_irqrestore(&dev->core_dev->lock, flags); - pd_uinfo = (struct pd_uinfo *)(dev->pdr_uinfo + - sizeof(struct pd_uinfo) * pd_entry); - pd = crypto4xx_get_pdp(dev, &pd_dma, pd_entry); + pd = &dev->pdr[pd_entry]; + pd->sa_len = sa_len; + + pd_uinfo = &dev->pdr_uinfo[pd_entry]; pd_uinfo->async_req = req; pd_uinfo->num_gd = num_gd; pd_uinfo->num_sd = num_sd; - if (iv_len || ctx->is_hash) { - ivlen = iv_len; - pd->sa = pd_uinfo->sa_pa; - sa = (struct dynamic_sa_ctl *) pd_uinfo->sa_va; - if (ctx->direction == DIR_INBOUND) - memcpy(sa, ctx->sa_in, ctx->sa_len * 4); - else - memcpy(sa, ctx->sa_out, ctx->sa_len * 4); + if (iv_len) + memcpy(pd_uinfo->sr_va->save_iv, iv, iv_len); - memcpy((void *) sa + ctx->offset_to_sr_ptr, - &pd_uinfo->sr_pa, 4); + sa = pd_uinfo->sa_va; + memcpy(sa, req_sa, sa_len * 4); + + sa->sa_command_1.bf.hash_crypto_offset = (assoclen >> 2); + offset_to_sr_ptr = get_dynamic_sa_offset_state_ptr_field(sa); + *(u32 *)((unsigned long)sa + offset_to_sr_ptr) = pd_uinfo->sr_pa; - if (iv_len) - crypto4xx_memcpy_le(pd_uinfo->sr_va, iv, iv_len); - } else { - if (ctx->direction == DIR_INBOUND) { - pd->sa = ctx->sa_in_dma_addr; - sa = (struct dynamic_sa_ctl *) ctx->sa_in; - } else { - pd->sa = ctx->sa_out_dma_addr; - sa = (struct dynamic_sa_ctl *) ctx->sa_out; - } - } - pd->sa_len = ctx->sa_len; if (num_gd) { + dma_addr_t gd_dma; + struct scatterlist *sg; + /* get first gd we are going to use */ gd_idx = fst_gd; pd_uinfo->first_gd = fst_gd; @@ -875,27 +818,30 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, pd->src = gd_dma; /* enable gather */ sa->sa_command_0.bf.gather = 1; - idx = 0; - src = &src[0]; /* walk the sg, and setup gather array */ + + sg = src; while (nbytes) { - sg = &src[idx]; - addr = dma_map_page(dev->core_dev->device, sg_page(sg), - sg->offset, sg->length, DMA_TO_DEVICE); - gd->ptr = addr; - gd->ctl_len.len = sg->length; + size_t len; + + len = min(sg->length, nbytes); + gd->ptr = dma_map_page(dev->core_dev->device, + sg_page(sg), sg->offset, len, DMA_TO_DEVICE); + gd->ctl_len.len = len; gd->ctl_len.done = 0; gd->ctl_len.ready = 1; - if (sg->length >= nbytes) + if (len >= nbytes) break; + nbytes -= sg->length; gd_idx = get_next_gd(gd_idx); gd = crypto4xx_get_gdp(dev, &gd_dma, gd_idx); - idx++; + sg = sg_next(sg); } } else { pd->src = (u32)dma_map_page(dev->core_dev->device, sg_page(src), - src->offset, src->length, DMA_TO_DEVICE); + src->offset, min(nbytes, src->length), + DMA_TO_DEVICE); /* * Disable gather in sa command */ @@ -906,25 +852,24 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, pd_uinfo->first_gd = 0xffffffff; pd_uinfo->num_gd = 0; } - if (ctx->is_hash || sg_is_last(dst)) { + if (sg_is_last(dst)) { /* * we know application give us dst a whole piece of memory * no need to use scatter ring. - * In case of is_hash, the icv is always at end of src data. */ pd_uinfo->using_sd = 0; pd_uinfo->first_sd = 0xffffffff; pd_uinfo->num_sd = 0; pd_uinfo->dest_va = dst; sa->sa_command_0.bf.scatter = 0; - if (ctx->is_hash) - pd->dest = virt_to_phys((void *)dst); - else - pd->dest = (u32)dma_map_page(dev->core_dev->device, - sg_page(dst), dst->offset, - dst->length, DMA_TO_DEVICE); + pd->dest = (u32)dma_map_page(dev->core_dev->device, + sg_page(dst), dst->offset, + min(datalen, dst->length), + DMA_TO_DEVICE); } else { + dma_addr_t sd_dma; struct ce_sd *sd = NULL; + u32 sd_idx = fst_sd; nbytes = datalen; sa->sa_command_0.bf.scatter = 1; @@ -938,7 +883,6 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, sd->ctl.done = 0; sd->ctl.rdy = 1; /* sd->ptr should be setup by sd_init routine*/ - idx = 0; if (nbytes >= PPC4XX_SD_BUFFER_SIZE) nbytes -= PPC4XX_SD_BUFFER_SIZE; else @@ -949,67 +893,97 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, /* setup scatter descriptor */ sd->ctl.done = 0; sd->ctl.rdy = 1; - if (nbytes >= PPC4XX_SD_BUFFER_SIZE) + if (nbytes >= PPC4XX_SD_BUFFER_SIZE) { nbytes -= PPC4XX_SD_BUFFER_SIZE; - else + } else { /* * SD entry can hold PPC4XX_SD_BUFFER_SIZE, * which is more than nbytes, so done. */ nbytes = 0; + } } } - sa->sa_command_1.bf.hash_crypto_offset = 0; - pd->pd_ctl.w = ctx->pd_ctl; - pd->pd_ctl_len.w = 0x00400000 | (ctx->bypass << 24) | datalen; - pd_uinfo->state = PD_ENTRY_INUSE; + pd->pd_ctl.w = PD_CTL_HOST_READY | + ((crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AHASH) | + (crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AEAD) ? + PD_CTL_HASH_FINAL : 0); + pd->pd_ctl_len.w = 0x00400000 | (assoclen + datalen); + pd_uinfo->state = PD_ENTRY_INUSE | (is_busy ? PD_ENTRY_BUSY : 0); + wmb(); /* write any value to push engine to read a pd */ + writel(0, dev->ce_base + CRYPTO4XX_INT_DESCR_RD); writel(1, dev->ce_base + CRYPTO4XX_INT_DESCR_RD); - return -EINPROGRESS; + return is_busy ? -EBUSY : -EINPROGRESS; } /** * Algorithm Registration Functions */ -static int crypto4xx_alg_init(struct crypto_tfm *tfm) +static void crypto4xx_ctx_init(struct crypto4xx_alg *amcc_alg, + struct crypto4xx_ctx *ctx) { - struct crypto_alg *alg = tfm->__crt_alg; - struct crypto4xx_alg *amcc_alg = crypto_alg_to_crypto4xx_alg(alg); - struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); - ctx->dev = amcc_alg->dev; ctx->sa_in = NULL; ctx->sa_out = NULL; - ctx->sa_in_dma_addr = 0; - ctx->sa_out_dma_addr = 0; ctx->sa_len = 0; +} - switch (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) { - default: - tfm->crt_ablkcipher.reqsize = sizeof(struct crypto4xx_ctx); - break; - case CRYPTO_ALG_TYPE_AHASH: - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), - sizeof(struct crypto4xx_ctx)); - break; - } +static int crypto4xx_ablk_init(struct crypto_tfm *tfm) +{ + struct crypto_alg *alg = tfm->__crt_alg; + struct crypto4xx_alg *amcc_alg; + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); + amcc_alg = container_of(alg, struct crypto4xx_alg, alg.u.cipher); + crypto4xx_ctx_init(amcc_alg, ctx); + tfm->crt_ablkcipher.reqsize = sizeof(struct crypto4xx_ctx); return 0; } -static void crypto4xx_alg_exit(struct crypto_tfm *tfm) +static void crypto4xx_common_exit(struct crypto4xx_ctx *ctx) { - struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); - crypto4xx_free_sa(ctx); - crypto4xx_free_state_record(ctx); } -int crypto4xx_register_alg(struct crypto4xx_device *sec_dev, - struct crypto4xx_alg_common *crypto_alg, - int array_size) +static void crypto4xx_ablk_exit(struct crypto_tfm *tfm) +{ + crypto4xx_common_exit(crypto_tfm_ctx(tfm)); +} + +static int crypto4xx_aead_init(struct crypto_aead *tfm) +{ + struct aead_alg *alg = crypto_aead_alg(tfm); + struct crypto4xx_ctx *ctx = crypto_aead_ctx(tfm); + struct crypto4xx_alg *amcc_alg; + + ctx->sw_cipher.aead = crypto_alloc_aead(alg->base.cra_name, 0, + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC); + if (IS_ERR(ctx->sw_cipher.aead)) + return PTR_ERR(ctx->sw_cipher.aead); + + amcc_alg = container_of(alg, struct crypto4xx_alg, alg.u.aead); + crypto4xx_ctx_init(amcc_alg, ctx); + crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) + + max(sizeof(struct crypto4xx_ctx), 32 + + crypto_aead_reqsize(ctx->sw_cipher.aead))); + return 0; +} + +static void crypto4xx_aead_exit(struct crypto_aead *tfm) +{ + struct crypto4xx_ctx *ctx = crypto_aead_ctx(tfm); + + crypto4xx_common_exit(ctx); + crypto_free_aead(ctx->sw_cipher.aead); +} + +static int crypto4xx_register_alg(struct crypto4xx_device *sec_dev, + struct crypto4xx_alg_common *crypto_alg, + int array_size) { struct crypto4xx_alg *alg; int i; @@ -1024,6 +998,10 @@ int crypto4xx_register_alg(struct crypto4xx_device *sec_dev, alg->dev = sec_dev; switch (alg->alg.type) { + case CRYPTO_ALG_TYPE_AEAD: + rc = crypto_register_aead(&alg->alg.u.aead); + break; + case CRYPTO_ALG_TYPE_AHASH: rc = crypto_register_ahash(&alg->alg.u.hash); break; @@ -1033,12 +1011,10 @@ int crypto4xx_register_alg(struct crypto4xx_device *sec_dev, break; } - if (rc) { - list_del(&alg->entry); + if (rc) kfree(alg); - } else { + else list_add_tail(&alg->entry, &sec_dev->alg_list); - } } return 0; @@ -1055,6 +1031,10 @@ static void crypto4xx_unregister_alg(struct crypto4xx_device *sec_dev) crypto_unregister_ahash(&alg->alg.u.hash); break; + case CRYPTO_ALG_TYPE_AEAD: + crypto_unregister_aead(&alg->alg.u.aead); + break; + default: crypto_unregister_alg(&alg->alg.u.cipher); } @@ -1068,25 +1048,23 @@ static void crypto4xx_bh_tasklet_cb(unsigned long data) struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev); struct pd_uinfo *pd_uinfo; struct ce_pd *pd; - u32 tail; - - while (core_dev->dev->pdr_head != core_dev->dev->pdr_tail) { - tail = core_dev->dev->pdr_tail; - pd_uinfo = core_dev->dev->pdr_uinfo + - sizeof(struct pd_uinfo)*tail; - pd = core_dev->dev->pdr + sizeof(struct ce_pd) * tail; - if ((pd_uinfo->state == PD_ENTRY_INUSE) && - pd->pd_ctl.bf.pe_done && - !pd->pd_ctl.bf.host_ready) { - pd->pd_ctl.bf.pe_done = 0; + u32 tail = core_dev->dev->pdr_tail; + u32 head = core_dev->dev->pdr_head; + + do { + pd_uinfo = &core_dev->dev->pdr_uinfo[tail]; + pd = &core_dev->dev->pdr[tail]; + if ((pd_uinfo->state & PD_ENTRY_INUSE) && + ((READ_ONCE(pd->pd_ctl.w) & + (PD_CTL_PE_DONE | PD_CTL_HOST_READY)) == + PD_CTL_PE_DONE)) { crypto4xx_pd_done(core_dev->dev, tail); - crypto4xx_put_pd_to_pdr(core_dev->dev, tail); - pd_uinfo->state = PD_ENTRY_FREE; + tail = crypto4xx_put_pd_to_pdr(core_dev->dev, tail); } else { /* if tail not done, break */ break; } - } + } while (head != tail); } /** @@ -1110,18 +1088,20 @@ static irqreturn_t crypto4xx_ce_interrupt_handler(int irq, void *data) /** * Supported Crypto Algorithms */ -struct crypto4xx_alg_common crypto4xx_alg[] = { +static struct crypto4xx_alg_common crypto4xx_alg[] = { /* Crypto AES modes */ { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .u.cipher = { .cra_name = "cbc(aes)", .cra_driver_name = "cbc-aes-ppc4xx", .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto4xx_ctx), .cra_type = &crypto_ablkcipher_type, - .cra_init = crypto4xx_alg_init, - .cra_exit = crypto4xx_alg_exit, + .cra_init = crypto4xx_ablk_init, + .cra_exit = crypto4xx_ablk_exit, .cra_module = THIS_MODULE, .cra_u = { .ablkcipher = { @@ -1134,6 +1114,147 @@ struct crypto4xx_alg_common crypto4xx_alg[] = { } } }}, + { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .u.cipher = { + .cra_name = "cfb(aes)", + .cra_driver_name = "cfb-aes-ppc4xx", + .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct crypto4xx_ctx), + .cra_type = &crypto_ablkcipher_type, + .cra_init = crypto4xx_ablk_init, + .cra_exit = crypto4xx_ablk_exit, + .cra_module = THIS_MODULE, + .cra_u = { + .ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_IV_SIZE, + .setkey = crypto4xx_setkey_aes_cfb, + .encrypt = crypto4xx_encrypt, + .decrypt = crypto4xx_decrypt, + } + } + } }, + { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .u.cipher = { + .cra_name = "rfc3686(ctr(aes))", + .cra_driver_name = "rfc3686-ctr-aes-ppc4xx", + .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct crypto4xx_ctx), + .cra_type = &crypto_ablkcipher_type, + .cra_init = crypto4xx_ablk_init, + .cra_exit = crypto4xx_ablk_exit, + .cra_module = THIS_MODULE, + .cra_u = { + .ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE + + CTR_RFC3686_NONCE_SIZE, + .max_keysize = AES_MAX_KEY_SIZE + + CTR_RFC3686_NONCE_SIZE, + .ivsize = CTR_RFC3686_IV_SIZE, + .setkey = crypto4xx_setkey_rfc3686, + .encrypt = crypto4xx_rfc3686_encrypt, + .decrypt = crypto4xx_rfc3686_decrypt, + } + } + } }, + { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .u.cipher = { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-ppc4xx", + .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct crypto4xx_ctx), + .cra_type = &crypto_ablkcipher_type, + .cra_init = crypto4xx_ablk_init, + .cra_exit = crypto4xx_ablk_exit, + .cra_module = THIS_MODULE, + .cra_u = { + .ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = crypto4xx_setkey_aes_ecb, + .encrypt = crypto4xx_encrypt, + .decrypt = crypto4xx_decrypt, + } + } + } }, + { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .u.cipher = { + .cra_name = "ofb(aes)", + .cra_driver_name = "ofb-aes-ppc4xx", + .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct crypto4xx_ctx), + .cra_type = &crypto_ablkcipher_type, + .cra_init = crypto4xx_ablk_init, + .cra_exit = crypto4xx_ablk_exit, + .cra_module = THIS_MODULE, + .cra_u = { + .ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_IV_SIZE, + .setkey = crypto4xx_setkey_aes_ofb, + .encrypt = crypto4xx_encrypt, + .decrypt = crypto4xx_decrypt, + } + } + } }, + + /* AEAD */ + { .type = CRYPTO_ALG_TYPE_AEAD, .u.aead = { + .setkey = crypto4xx_setkey_aes_ccm, + .setauthsize = crypto4xx_setauthsize_aead, + .encrypt = crypto4xx_encrypt_aes_ccm, + .decrypt = crypto4xx_decrypt_aes_ccm, + .init = crypto4xx_aead_init, + .exit = crypto4xx_aead_exit, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = 16, + .base = { + .cra_name = "ccm(aes)", + .cra_driver_name = "ccm-aes-ppc4xx", + .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct crypto4xx_ctx), + .cra_module = THIS_MODULE, + }, + } }, + { .type = CRYPTO_ALG_TYPE_AEAD, .u.aead = { + .setkey = crypto4xx_setkey_aes_gcm, + .setauthsize = crypto4xx_setauthsize_aead, + .encrypt = crypto4xx_encrypt_aes_gcm, + .decrypt = crypto4xx_decrypt_aes_gcm, + .init = crypto4xx_aead_init, + .exit = crypto4xx_aead_exit, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = 16, + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "gcm-aes-ppc4xx", + .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct crypto4xx_ctx), + .cra_module = THIS_MODULE, + }, + } }, }; /** @@ -1187,13 +1308,14 @@ static int crypto4xx_probe(struct platform_device *ofdev) core_dev->device = dev; spin_lock_init(&core_dev->lock); INIT_LIST_HEAD(&core_dev->dev->alg_list); + ratelimit_default_init(&core_dev->dev->aead_ratelimit); rc = crypto4xx_build_pdr(core_dev->dev); if (rc) goto err_build_pdr; rc = crypto4xx_build_gdr(core_dev->dev); if (rc) - goto err_build_gdr; + goto err_build_pdr; rc = crypto4xx_build_sdr(core_dev->dev); if (rc) @@ -1236,12 +1358,11 @@ err_iomap: err_request_irq: irq_dispose_mapping(core_dev->irq); tasklet_kill(&core_dev->tasklet); - crypto4xx_destroy_sdr(core_dev->dev); err_build_sdr: + crypto4xx_destroy_sdr(core_dev->dev); crypto4xx_destroy_gdr(core_dev->dev); -err_build_gdr: - crypto4xx_destroy_pdr(core_dev->dev); err_build_pdr: + crypto4xx_destroy_pdr(core_dev->dev); kfree(core_dev->dev); err_alloc_dev: kfree(core_dev); diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index ecfdcfe3698d..8ac3bd37203b 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -22,7 +22,11 @@ #ifndef __CRYPTO4XX_CORE_H__ #define __CRYPTO4XX_CORE_H__ +#include <linux/ratelimit.h> #include <crypto/internal/hash.h> +#include <crypto/internal/aead.h> +#include "crypto4xx_reg_def.h" +#include "crypto4xx_sa.h" #define MODULE_NAME "crypto4xx" @@ -34,20 +38,28 @@ #define PPC405EX_CE_RESET 0x00000008 #define CRYPTO4XX_CRYPTO_PRIORITY 300 -#define PPC4XX_LAST_PD 63 -#define PPC4XX_NUM_PD 64 -#define PPC4XX_LAST_GD 1023 +#define PPC4XX_NUM_PD 256 +#define PPC4XX_LAST_PD (PPC4XX_NUM_PD - 1) #define PPC4XX_NUM_GD 1024 -#define PPC4XX_LAST_SD 63 -#define PPC4XX_NUM_SD 64 +#define PPC4XX_LAST_GD (PPC4XX_NUM_GD - 1) +#define PPC4XX_NUM_SD 256 +#define PPC4XX_LAST_SD (PPC4XX_NUM_SD - 1) #define PPC4XX_SD_BUFFER_SIZE 2048 -#define PD_ENTRY_INUSE 1 +#define PD_ENTRY_BUSY BIT(1) +#define PD_ENTRY_INUSE BIT(0) #define PD_ENTRY_FREE 0 #define ERING_WAS_FULL 0xffffffff struct crypto4xx_device; +union shadow_sa_buf { + struct dynamic_sa_ctl sa; + + /* alloc 256 bytes which is enough for any kind of dynamic sa */ + u8 buf[256]; +} __packed; + struct pd_uinfo { struct crypto4xx_device *dev; u32 state; @@ -60,9 +72,8 @@ struct pd_uinfo { used by this packet */ u32 num_sd; /* number of scatter discriptors used by this packet */ - void *sa_va; /* shadow sa, when using cp from ctx->sa */ - u32 sa_pa; - void *sr_va; /* state record for shadow sa */ + struct dynamic_sa_ctl *sa_va; /* shadow sa */ + struct sa_state_record *sr_va; /* state record for shadow sa */ u32 sr_pa; struct scatterlist *dest_va; struct crypto_async_request *async_req; /* base crypto request @@ -72,27 +83,21 @@ struct pd_uinfo { struct crypto4xx_device { struct crypto4xx_core_device *core_dev; char *name; - u64 ce_phy_address; void __iomem *ce_base; void __iomem *trng_base; - void *pdr; /* base address of packet - descriptor ring */ - dma_addr_t pdr_pa; /* physical address used to - program ce pdr_base_register */ - void *gdr; /* gather descriptor ring */ - dma_addr_t gdr_pa; /* physical address used to - program ce gdr_base_register */ - void *sdr; /* scatter descriptor ring */ - dma_addr_t sdr_pa; /* physical address used to - program ce sdr_base_register */ + struct ce_pd *pdr; /* base address of packet descriptor ring */ + dma_addr_t pdr_pa; /* physical address of pdr_base_register */ + struct ce_gd *gdr; /* gather descriptor ring */ + dma_addr_t gdr_pa; /* physical address of gdr_base_register */ + struct ce_sd *sdr; /* scatter descriptor ring */ + dma_addr_t sdr_pa; /* physical address of sdr_base_register */ void *scatter_buffer_va; dma_addr_t scatter_buffer_pa; - u32 scatter_buffer_size; - void *shadow_sa_pool; /* pool of memory for sa in pd_uinfo */ + union shadow_sa_buf *shadow_sa_pool; dma_addr_t shadow_sa_pool_pa; - void *shadow_sr_pool; /* pool of memory for sr in pd_uinfo */ + struct sa_state_record *shadow_sr_pool; dma_addr_t shadow_sr_pool_pa; u32 pdr_tail; u32 pdr_head; @@ -100,9 +105,10 @@ struct crypto4xx_device { u32 gdr_head; u32 sdr_tail; u32 sdr_head; - void *pdr_uinfo; + struct pd_uinfo *pdr_uinfo; struct list_head alg_list; /* List of algorithm supported by this device */ + struct ratelimit_state aead_ratelimit; }; struct crypto4xx_core_device { @@ -118,30 +124,13 @@ struct crypto4xx_core_device { struct crypto4xx_ctx { struct crypto4xx_device *dev; - void *sa_in; - dma_addr_t sa_in_dma_addr; - void *sa_out; - dma_addr_t sa_out_dma_addr; - void *state_record; - dma_addr_t state_record_dma_addr; + struct dynamic_sa_ctl *sa_in; + struct dynamic_sa_ctl *sa_out; + __le32 iv_nonce; u32 sa_len; - u32 offset_to_sr_ptr; /* offset to state ptr, in dynamic sa */ - u32 direction; - u32 next_hdr; - u32 save_iv; - u32 pd_ctl_len; - u32 pd_ctl; - u32 bypass; - u32 is_hash; - u32 hash_final; -}; - -struct crypto4xx_req_ctx { - struct crypto4xx_device *dev; /* Device in which - operation to send to */ - void *sa; - u32 sa_dma_addr; - u16 sa_len; + union { + struct crypto_aead *aead; + } sw_cipher; }; struct crypto4xx_alg_common { @@ -149,6 +138,7 @@ struct crypto4xx_alg_common { union { struct crypto_alg cipher; struct ahash_alg hash; + struct aead_alg aead; } u; }; @@ -158,43 +148,90 @@ struct crypto4xx_alg { struct crypto4xx_device *dev; }; -static inline struct crypto4xx_alg *crypto_alg_to_crypto4xx_alg( - struct crypto_alg *x) +int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size); +void crypto4xx_free_sa(struct crypto4xx_ctx *ctx); +void crypto4xx_free_ctx(struct crypto4xx_ctx *ctx); +int crypto4xx_build_pd(struct crypto_async_request *req, + struct crypto4xx_ctx *ctx, + struct scatterlist *src, + struct scatterlist *dst, + const unsigned int datalen, + const __le32 *iv, const u32 iv_len, + const struct dynamic_sa_ctl *sa, + const unsigned int sa_len, + const unsigned int assoclen); +int crypto4xx_setkey_aes_cbc(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen); +int crypto4xx_setkey_aes_cfb(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen); +int crypto4xx_setkey_aes_ecb(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen); +int crypto4xx_setkey_aes_ofb(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen); +int crypto4xx_setkey_rfc3686(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen); +int crypto4xx_encrypt(struct ablkcipher_request *req); +int crypto4xx_decrypt(struct ablkcipher_request *req); +int crypto4xx_rfc3686_encrypt(struct ablkcipher_request *req); +int crypto4xx_rfc3686_decrypt(struct ablkcipher_request *req); +int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm); +int crypto4xx_hash_digest(struct ahash_request *req); +int crypto4xx_hash_final(struct ahash_request *req); +int crypto4xx_hash_update(struct ahash_request *req); +int crypto4xx_hash_init(struct ahash_request *req); + +/** + * Note: Only use this function to copy items that is word aligned. + */ +static inline void crypto4xx_memcpy_swab32(u32 *dst, const void *buf, + size_t len) { - switch (x->cra_flags & CRYPTO_ALG_TYPE_MASK) { - case CRYPTO_ALG_TYPE_AHASH: - return container_of(__crypto_ahash_alg(x), - struct crypto4xx_alg, alg.u.hash); + for (; len >= 4; buf += 4, len -= 4) + *dst++ = __swab32p((u32 *) buf); + + if (len) { + const u8 *tmp = (u8 *)buf; + + switch (len) { + case 3: + *dst = (tmp[2] << 16) | + (tmp[1] << 8) | + tmp[0]; + break; + case 2: + *dst = (tmp[1] << 8) | + tmp[0]; + break; + case 1: + *dst = tmp[0]; + break; + default: + break; + } } +} - return container_of(x, struct crypto4xx_alg, alg.u.cipher); +static inline void crypto4xx_memcpy_from_le32(u32 *dst, const void *buf, + size_t len) +{ + crypto4xx_memcpy_swab32(dst, buf, len); } -extern int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size); -extern void crypto4xx_free_sa(struct crypto4xx_ctx *ctx); -extern u32 crypto4xx_alloc_sa_rctx(struct crypto4xx_ctx *ctx, - struct crypto4xx_ctx *rctx); -extern void crypto4xx_free_sa_rctx(struct crypto4xx_ctx *rctx); -extern void crypto4xx_free_ctx(struct crypto4xx_ctx *ctx); -extern u32 crypto4xx_alloc_state_record(struct crypto4xx_ctx *ctx); -extern u32 get_dynamic_sa_offset_state_ptr_field(struct crypto4xx_ctx *ctx); -extern u32 get_dynamic_sa_offset_key_field(struct crypto4xx_ctx *ctx); -extern u32 get_dynamic_sa_iv_size(struct crypto4xx_ctx *ctx); -extern void crypto4xx_memcpy_le(unsigned int *dst, - const unsigned char *buf, int len); -extern u32 crypto4xx_build_pd(struct crypto_async_request *req, - struct crypto4xx_ctx *ctx, - struct scatterlist *src, - struct scatterlist *dst, - unsigned int datalen, - void *iv, u32 iv_len); -extern int crypto4xx_setkey_aes_cbc(struct crypto_ablkcipher *cipher, - const u8 *key, unsigned int keylen); -extern int crypto4xx_encrypt(struct ablkcipher_request *req); -extern int crypto4xx_decrypt(struct ablkcipher_request *req); -extern int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm); -extern int crypto4xx_hash_digest(struct ahash_request *req); -extern int crypto4xx_hash_final(struct ahash_request *req); -extern int crypto4xx_hash_update(struct ahash_request *req); -extern int crypto4xx_hash_init(struct ahash_request *req); +static inline void crypto4xx_memcpy_to_le32(__le32 *dst, const void *buf, + size_t len) +{ + crypto4xx_memcpy_swab32((u32 *)dst, buf, len); +} + +int crypto4xx_setauthsize_aead(struct crypto_aead *ciper, + unsigned int authsize); +int crypto4xx_setkey_aes_ccm(struct crypto_aead *cipher, + const u8 *key, unsigned int keylen); +int crypto4xx_encrypt_aes_ccm(struct aead_request *req); +int crypto4xx_decrypt_aes_ccm(struct aead_request *req); +int crypto4xx_setkey_aes_gcm(struct crypto_aead *cipher, + const u8 *key, unsigned int keylen); +int crypto4xx_encrypt_aes_gcm(struct aead_request *req); +int crypto4xx_decrypt_aes_gcm(struct aead_request *req); + #endif diff --git a/drivers/crypto/amcc/crypto4xx_reg_def.h b/drivers/crypto/amcc/crypto4xx_reg_def.h index 279b8725559f..0a22ec5d1a96 100644 --- a/drivers/crypto/amcc/crypto4xx_reg_def.h +++ b/drivers/crypto/amcc/crypto4xx_reg_def.h @@ -261,6 +261,9 @@ union ce_pd_ctl { } bf; u32 w; } __attribute__((packed)); +#define PD_CTL_HASH_FINAL BIT(4) +#define PD_CTL_PE_DONE BIT(1) +#define PD_CTL_HOST_READY BIT(0) union ce_pd_ctl_len { struct { diff --git a/drivers/crypto/amcc/crypto4xx_sa.c b/drivers/crypto/amcc/crypto4xx_sa.c deleted file mode 100644 index 69182e2cc3ea..000000000000 --- a/drivers/crypto/amcc/crypto4xx_sa.c +++ /dev/null @@ -1,85 +0,0 @@ -/** - * AMCC SoC PPC4xx Crypto Driver - * - * Copyright (c) 2008 Applied Micro Circuits Corporation. - * All rights reserved. James Hsiao <jhsiao@amcc.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * @file crypto4xx_sa.c - * - * This file implements the security context - * associate format. - */ -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/moduleparam.h> -#include <linux/mod_devicetable.h> -#include <linux/interrupt.h> -#include <linux/spinlock_types.h> -#include <linux/highmem.h> -#include <linux/scatterlist.h> -#include <linux/crypto.h> -#include <crypto/algapi.h> -#include <crypto/des.h> -#include "crypto4xx_reg_def.h" -#include "crypto4xx_sa.h" -#include "crypto4xx_core.h" - -u32 get_dynamic_sa_offset_state_ptr_field(struct crypto4xx_ctx *ctx) -{ - u32 offset; - union dynamic_sa_contents cts; - - if (ctx->direction == DIR_INBOUND) - cts.w = ((struct dynamic_sa_ctl *) ctx->sa_in)->sa_contents; - else - cts.w = ((struct dynamic_sa_ctl *) ctx->sa_out)->sa_contents; - offset = cts.bf.key_size - + cts.bf.inner_size - + cts.bf.outer_size - + cts.bf.spi - + cts.bf.seq_num0 - + cts.bf.seq_num1 - + cts.bf.seq_num_mask0 - + cts.bf.seq_num_mask1 - + cts.bf.seq_num_mask2 - + cts.bf.seq_num_mask3 - + cts.bf.iv0 - + cts.bf.iv1 - + cts.bf.iv2 - + cts.bf.iv3; - - return sizeof(struct dynamic_sa_ctl) + offset * 4; -} - -u32 get_dynamic_sa_iv_size(struct crypto4xx_ctx *ctx) -{ - union dynamic_sa_contents cts; - - if (ctx->direction == DIR_INBOUND) - cts.w = ((struct dynamic_sa_ctl *) ctx->sa_in)->sa_contents; - else - cts.w = ((struct dynamic_sa_ctl *) ctx->sa_out)->sa_contents; - return (cts.bf.iv0 + cts.bf.iv1 + cts.bf.iv2 + cts.bf.iv3) * 4; -} - -u32 get_dynamic_sa_offset_key_field(struct crypto4xx_ctx *ctx) -{ - union dynamic_sa_contents cts; - - if (ctx->direction == DIR_INBOUND) - cts.w = ((struct dynamic_sa_ctl *) ctx->sa_in)->sa_contents; - else - cts.w = ((struct dynamic_sa_ctl *) ctx->sa_out)->sa_contents; - - return sizeof(struct dynamic_sa_ctl); -} diff --git a/drivers/crypto/amcc/crypto4xx_sa.h b/drivers/crypto/amcc/crypto4xx_sa.h index 1352d58d4e34..a4d403528db5 100644 --- a/drivers/crypto/amcc/crypto4xx_sa.h +++ b/drivers/crypto/amcc/crypto4xx_sa.h @@ -55,6 +55,8 @@ union dynamic_sa_contents { #define SA_OP_GROUP_BASIC 0 #define SA_OPCODE_ENCRYPT 0 #define SA_OPCODE_DECRYPT 0 +#define SA_OPCODE_ENCRYPT_HASH 1 +#define SA_OPCODE_HASH_DECRYPT 1 #define SA_OPCODE_HASH 3 #define SA_CIPHER_ALG_DES 0 #define SA_CIPHER_ALG_3DES 1 @@ -65,6 +67,8 @@ union dynamic_sa_contents { #define SA_HASH_ALG_MD5 0 #define SA_HASH_ALG_SHA1 1 +#define SA_HASH_ALG_GHASH 12 +#define SA_HASH_ALG_CBC_MAC 14 #define SA_HASH_ALG_NULL 15 #define SA_HASH_ALG_SHA1_DIGEST_SIZE 20 @@ -112,6 +116,9 @@ union sa_command_0 { #define CRYPTO_MODE_ECB 0 #define CRYPTO_MODE_CBC 1 +#define CRYPTO_MODE_OFB 2 +#define CRYPTO_MODE_CFB 3 +#define CRYPTO_MODE_CTR 4 #define CRYPTO_FEEDBACK_MODE_NO_FB 0 #define CRYPTO_FEEDBACK_MODE_64BIT_OFB 0 @@ -169,7 +176,7 @@ union sa_command_1 { } __attribute__((packed)); struct dynamic_sa_ctl { - u32 sa_contents; + union dynamic_sa_contents sa_contents; union sa_command_0 sa_command_0; union sa_command_1 sa_command_1; } __attribute__((packed)); @@ -178,9 +185,12 @@ struct dynamic_sa_ctl { * State Record for Security Association (SA) */ struct sa_state_record { - u32 save_iv[4]; - u32 save_hash_byte_cnt[2]; - u32 save_digest[16]; + __le32 save_iv[4]; + __le32 save_hash_byte_cnt[2]; + union { + u32 save_digest[16]; /* for MD5/SHA */ + __le32 save_digest_le32[16]; /* GHASH / CBC */ + }; } __attribute__((packed)); /** @@ -189,8 +199,8 @@ struct sa_state_record { */ struct dynamic_sa_aes128 { struct dynamic_sa_ctl ctrl; - u32 key[4]; - u32 iv[4]; /* for CBC, OFC, and CFB mode */ + __le32 key[4]; + __le32 iv[4]; /* for CBC, OFC, and CFB mode */ u32 state_ptr; u32 reserved; } __attribute__((packed)); @@ -203,8 +213,8 @@ struct dynamic_sa_aes128 { */ struct dynamic_sa_aes192 { struct dynamic_sa_ctl ctrl; - u32 key[6]; - u32 iv[4]; /* for CBC, OFC, and CFB mode */ + __le32 key[6]; + __le32 iv[4]; /* for CBC, OFC, and CFB mode */ u32 state_ptr; u32 reserved; } __attribute__((packed)); @@ -217,8 +227,8 @@ struct dynamic_sa_aes192 { */ struct dynamic_sa_aes256 { struct dynamic_sa_ctl ctrl; - u32 key[8]; - u32 iv[4]; /* for CBC, OFC, and CFB mode */ + __le32 key[8]; + __le32 iv[4]; /* for CBC, OFC, and CFB mode */ u32 state_ptr; u32 reserved; } __attribute__((packed)); @@ -228,16 +238,81 @@ struct dynamic_sa_aes256 { #define SA_AES_CONTENTS 0x3e000002 /** + * Security Association (SA) for AES128 CCM + */ +struct dynamic_sa_aes128_ccm { + struct dynamic_sa_ctl ctrl; + __le32 key[4]; + __le32 iv[4]; + u32 state_ptr; + u32 reserved; +} __packed; +#define SA_AES128_CCM_LEN (sizeof(struct dynamic_sa_aes128_ccm)/4) +#define SA_AES128_CCM_CONTENTS 0x3e000042 +#define SA_AES_CCM_CONTENTS 0x3e000002 + +/** + * Security Association (SA) for AES128_GCM + */ +struct dynamic_sa_aes128_gcm { + struct dynamic_sa_ctl ctrl; + __le32 key[4]; + __le32 inner_digest[4]; + __le32 iv[4]; + u32 state_ptr; + u32 reserved; +} __packed; + +#define SA_AES128_GCM_LEN (sizeof(struct dynamic_sa_aes128_gcm)/4) +#define SA_AES128_GCM_CONTENTS 0x3e000442 +#define SA_AES_GCM_CONTENTS 0x3e000402 + +/** * Security Association (SA) for HASH160: HMAC-SHA1 */ struct dynamic_sa_hash160 { struct dynamic_sa_ctl ctrl; - u32 inner_digest[5]; - u32 outer_digest[5]; + __le32 inner_digest[5]; + __le32 outer_digest[5]; u32 state_ptr; u32 reserved; } __attribute__((packed)); #define SA_HASH160_LEN (sizeof(struct dynamic_sa_hash160)/4) #define SA_HASH160_CONTENTS 0x2000a502 +static inline u32 +get_dynamic_sa_offset_state_ptr_field(struct dynamic_sa_ctl *cts) +{ + u32 offset; + + offset = cts->sa_contents.bf.key_size + + cts->sa_contents.bf.inner_size + + cts->sa_contents.bf.outer_size + + cts->sa_contents.bf.spi + + cts->sa_contents.bf.seq_num0 + + cts->sa_contents.bf.seq_num1 + + cts->sa_contents.bf.seq_num_mask0 + + cts->sa_contents.bf.seq_num_mask1 + + cts->sa_contents.bf.seq_num_mask2 + + cts->sa_contents.bf.seq_num_mask3 + + cts->sa_contents.bf.iv0 + + cts->sa_contents.bf.iv1 + + cts->sa_contents.bf.iv2 + + cts->sa_contents.bf.iv3; + + return sizeof(struct dynamic_sa_ctl) + offset * 4; +} + +static inline __le32 *get_dynamic_sa_key_field(struct dynamic_sa_ctl *cts) +{ + return (__le32 *) ((unsigned long)cts + sizeof(struct dynamic_sa_ctl)); +} + +static inline __le32 *get_dynamic_sa_inner_digest(struct dynamic_sa_ctl *cts) +{ + return (__le32 *) ((unsigned long)cts + + sizeof(struct dynamic_sa_ctl) + + cts->sa_contents.bf.key_size * 4); +} + #endif |