summaryrefslogtreecommitdiffstats
path: root/drivers/crypto/sa2ul.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/crypto/sa2ul.c')
-rw-r--r--drivers/crypto/sa2ul.c235
1 files changed, 124 insertions, 111 deletions
diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c
index 5bc099052bd2..eda93fab95fe 100644
--- a/drivers/crypto/sa2ul.c
+++ b/drivers/crypto/sa2ul.c
@@ -143,33 +143,38 @@ struct sa_alg_tmpl {
};
/**
+ * struct sa_mapped_sg: scatterlist information for tx and rx
+ * @mapped: Set to true if the @sgt is mapped
+ * @dir: mapping direction used for @sgt
+ * @split_sg: Set if the sg is split and needs to be freed up
+ * @static_sg: Static scatterlist entry for overriding data
+ * @sgt: scatterlist table for DMA API use
+ */
+struct sa_mapped_sg {
+ bool mapped;
+ enum dma_data_direction dir;
+ struct scatterlist static_sg;
+ struct scatterlist *split_sg;
+ struct sg_table sgt;
+};
+/**
* struct sa_rx_data: RX Packet miscellaneous data place holder
* @req: crypto request data pointer
* @ddev: pointer to the DMA device
* @tx_in: dma_async_tx_descriptor pointer for rx channel
- * @split_src_sg: Set if the src sg is split and needs to be freed up
- * @split_dst_sg: Set if the dst sg is split and needs to be freed up
+ * @mapped_sg: Information on tx (0) and rx (1) scatterlist DMA mapping
* @enc: Flag indicating either encryption or decryption
* @enc_iv_size: Initialisation vector size
* @iv_idx: Initialisation vector index
- * @rx_sg: Static scatterlist entry for overriding RX data
- * @tx_sg: Static scatterlist entry for overriding TX data
- * @src: Source data pointer
- * @dst: Destination data pointer
*/
struct sa_rx_data {
void *req;
struct device *ddev;
struct dma_async_tx_descriptor *tx_in;
- struct scatterlist *split_src_sg;
- struct scatterlist *split_dst_sg;
+ struct sa_mapped_sg mapped_sg[2];
u8 enc;
u8 enc_iv_size;
u8 iv_idx;
- struct scatterlist rx_sg;
- struct scatterlist tx_sg;
- struct scatterlist *src;
- struct scatterlist *dst;
};
/**
@@ -976,23 +981,46 @@ static int sa_3des_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key,
return sa_cipher_setkey(tfm, key, keylen, &ad);
}
+static void sa_sync_from_device(struct sa_rx_data *rxd)
+{
+ struct sg_table *sgt;
+
+ if (rxd->mapped_sg[0].dir == DMA_BIDIRECTIONAL)
+ sgt = &rxd->mapped_sg[0].sgt;
+ else
+ sgt = &rxd->mapped_sg[1].sgt;
+
+ dma_sync_sgtable_for_cpu(rxd->ddev, sgt, DMA_FROM_DEVICE);
+}
+
+static void sa_free_sa_rx_data(struct sa_rx_data *rxd)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(rxd->mapped_sg); i++) {
+ struct sa_mapped_sg *mapped_sg = &rxd->mapped_sg[i];
+
+ if (mapped_sg->mapped) {
+ dma_unmap_sgtable(rxd->ddev, &mapped_sg->sgt,
+ mapped_sg->dir, 0);
+ kfree(mapped_sg->split_sg);
+ }
+ }
+
+ kfree(rxd);
+}
+
static void sa_aes_dma_in_callback(void *data)
{
struct sa_rx_data *rxd = (struct sa_rx_data *)data;
struct skcipher_request *req;
- int sglen;
u32 *result;
__be32 *mdptr;
size_t ml, pl;
int i;
- enum dma_data_direction dir_src;
- bool diff_dst;
+ sa_sync_from_device(rxd);
req = container_of(rxd->req, struct skcipher_request, base);
- sglen = sg_nents_for_len(req->src, req->cryptlen);
-
- diff_dst = (req->src != req->dst) ? true : false;
- dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL;
if (req->iv) {
mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl,
@@ -1003,18 +1031,7 @@ static void sa_aes_dma_in_callback(void *data)
result[i] = be32_to_cpu(mdptr[i + rxd->iv_idx]);
}
- dma_unmap_sg(rxd->ddev, req->src, sglen, dir_src);
- kfree(rxd->split_src_sg);
-
- if (diff_dst) {
- sglen = sg_nents_for_len(req->dst, req->cryptlen);
-
- dma_unmap_sg(rxd->ddev, req->dst, sglen,
- DMA_FROM_DEVICE);
- kfree(rxd->split_dst_sg);
- }
-
- kfree(rxd);
+ sa_free_sa_rx_data(rxd);
skcipher_request_complete(req, 0);
}
@@ -1043,7 +1060,6 @@ static int sa_run(struct sa_req *req)
struct device *ddev;
struct dma_chan *dma_rx;
int sg_nents, src_nents, dst_nents;
- int mapped_src_nents, mapped_dst_nents;
struct scatterlist *src, *dst;
size_t pl, ml, split_size;
struct sa_ctx_info *sa_ctx = req->enc ? &req->ctx->enc : &req->ctx->dec;
@@ -1052,6 +1068,7 @@ static int sa_run(struct sa_req *req)
u32 *mdptr;
bool diff_dst;
enum dma_data_direction dir_src;
+ struct sa_mapped_sg *mapped_sg;
gfp_flags = req->base->flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
GFP_KERNEL : GFP_ATOMIC;
@@ -1082,6 +1099,7 @@ static int sa_run(struct sa_req *req)
dma_rx = pdata->dma_rx1;
ddev = dma_rx->device->dev;
+ rxd->ddev = ddev;
memcpy(cmdl, sa_ctx->cmdl, sa_ctx->cmdl_size);
@@ -1109,60 +1127,90 @@ static int sa_run(struct sa_req *req)
split_size = req->size;
+ mapped_sg = &rxd->mapped_sg[0];
if (sg_nents == 1 && split_size <= req->src->length) {
- src = &rxd->rx_sg;
+ src = &mapped_sg->static_sg;
+ src_nents = 1;
sg_init_table(src, 1);
sg_set_page(src, sg_page(req->src), split_size,
req->src->offset);
- src_nents = 1;
- dma_map_sg(ddev, src, sg_nents, dir_src);
+
+ mapped_sg->sgt.sgl = src;
+ mapped_sg->sgt.orig_nents = src_nents;
+ ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0);
+ if (ret)
+ return ret;
+
+ mapped_sg->dir = dir_src;
+ mapped_sg->mapped = true;
} else {
- mapped_src_nents = dma_map_sg(ddev, req->src, sg_nents,
- dir_src);
- ret = sg_split(req->src, mapped_src_nents, 0, 1, &split_size,
- &src, &src_nents, gfp_flags);
+ mapped_sg->sgt.sgl = req->src;
+ mapped_sg->sgt.orig_nents = sg_nents;
+ ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0);
+ if (ret)
+ return ret;
+
+ mapped_sg->dir = dir_src;
+ mapped_sg->mapped = true;
+
+ ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents, 0, 1,
+ &split_size, &src, &src_nents, gfp_flags);
if (ret) {
- src_nents = sg_nents;
- src = req->src;
+ src_nents = mapped_sg->sgt.nents;
+ src = mapped_sg->sgt.sgl;
} else {
- rxd->split_src_sg = src;
+ mapped_sg->split_sg = src;
}
}
+ dma_sync_sgtable_for_device(ddev, &mapped_sg->sgt, DMA_TO_DEVICE);
+
if (!diff_dst) {
dst_nents = src_nents;
dst = src;
} else {
dst_nents = sg_nents_for_len(req->dst, req->size);
+ mapped_sg = &rxd->mapped_sg[1];
if (dst_nents == 1 && split_size <= req->dst->length) {
- dst = &rxd->tx_sg;
+ dst = &mapped_sg->static_sg;
+ dst_nents = 1;
sg_init_table(dst, 1);
sg_set_page(dst, sg_page(req->dst), split_size,
req->dst->offset);
- dst_nents = 1;
- dma_map_sg(ddev, dst, dst_nents, DMA_FROM_DEVICE);
+
+ mapped_sg->sgt.sgl = dst;
+ mapped_sg->sgt.orig_nents = dst_nents;
+ ret = dma_map_sgtable(ddev, &mapped_sg->sgt,
+ DMA_FROM_DEVICE, 0);
+ if (ret)
+ goto err_cleanup;
+
+ mapped_sg->dir = DMA_FROM_DEVICE;
+ mapped_sg->mapped = true;
} else {
- mapped_dst_nents = dma_map_sg(ddev, req->dst, dst_nents,
- DMA_FROM_DEVICE);
- ret = sg_split(req->dst, mapped_dst_nents, 0, 1,
- &split_size, &dst, &dst_nents,
+ mapped_sg->sgt.sgl = req->dst;
+ mapped_sg->sgt.orig_nents = dst_nents;
+ ret = dma_map_sgtable(ddev, &mapped_sg->sgt,
+ DMA_FROM_DEVICE, 0);
+ if (ret)
+ goto err_cleanup;
+
+ mapped_sg->dir = DMA_FROM_DEVICE;
+ mapped_sg->mapped = true;
+
+ ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents,
+ 0, 1, &split_size, &dst, &dst_nents,
gfp_flags);
if (ret) {
- dst_nents = dst_nents;
- dst = req->dst;
+ dst_nents = mapped_sg->sgt.nents;
+ dst = mapped_sg->sgt.sgl;
} else {
- rxd->split_dst_sg = dst;
+ mapped_sg->split_sg = dst;
}
}
}
- if (unlikely(src_nents != sg_nents)) {
- dev_warn_ratelimited(sa_k3_dev, "failed to map tx pkt\n");
- ret = -EIO;
- goto err_cleanup;
- }
-
rxd->tx_in = dmaengine_prep_slave_sg(dma_rx, dst, dst_nents,
DMA_DEV_TO_MEM,
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
@@ -1174,9 +1222,6 @@ static int sa_run(struct sa_req *req)
rxd->req = (void *)req->base;
rxd->enc = req->enc;
- rxd->ddev = ddev;
- rxd->src = src;
- rxd->dst = dst;
rxd->iv_idx = req->ctx->iv_idx;
rxd->enc_iv_size = sa_ctx->cmdl_upd_info.enc_iv.size;
rxd->tx_in->callback = req->callback;
@@ -1214,16 +1259,7 @@ static int sa_run(struct sa_req *req)
return -EINPROGRESS;
err_cleanup:
- dma_unmap_sg(ddev, req->src, sg_nents, DMA_TO_DEVICE);
- kfree(rxd->split_src_sg);
-
- if (req->src != req->dst) {
- dst_nents = sg_nents_for_len(req->dst, req->size);
- dma_unmap_sg(ddev, req->dst, dst_nents, DMA_FROM_DEVICE);
- kfree(rxd->split_dst_sg);
- }
-
- kfree(rxd);
+ sa_free_sa_rx_data(rxd);
return ret;
}
@@ -1293,11 +1329,12 @@ static void sa_sha_dma_in_callback(void *data)
struct ahash_request *req;
struct crypto_ahash *tfm;
unsigned int authsize;
- int i, sg_nents;
+ int i;
size_t ml, pl;
u32 *result;
__be32 *mdptr;
+ sa_sync_from_device(rxd);
req = container_of(rxd->req, struct ahash_request, base);
tfm = crypto_ahash_reqtfm(req);
authsize = crypto_ahash_digestsize(tfm);
@@ -1308,12 +1345,7 @@ static void sa_sha_dma_in_callback(void *data)
for (i = 0; i < (authsize / 4); i++)
result[i] = be32_to_cpu(mdptr[i + 4]);
- sg_nents = sg_nents_for_len(req->src, req->nbytes);
- dma_unmap_sg(rxd->ddev, req->src, sg_nents, DMA_FROM_DEVICE);
-
- kfree(rxd->split_src_sg);
-
- kfree(rxd);
+ sa_free_sa_rx_data(rxd);
ahash_request_complete(req, 0);
}
@@ -1482,8 +1514,8 @@ static int sa_sha_init(struct ahash_request *req)
struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
- dev_dbg(sa_k3_dev, "init: digest size: %d, rctx=%llx\n",
- crypto_ahash_digestsize(tfm), (u64)rctx);
+ dev_dbg(sa_k3_dev, "init: digest size: %u, rctx=%p\n",
+ crypto_ahash_digestsize(tfm), rctx);
ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
rctx->fallback_req.base.flags =
@@ -1637,43 +1669,28 @@ static void sa_aead_dma_in_callback(void *data)
unsigned int authsize;
u8 auth_tag[SA_MAX_AUTH_TAG_SZ];
size_t pl, ml;
- int i, sglen;
+ int i;
int err = 0;
u16 auth_len;
u32 *mdptr;
- bool diff_dst;
- enum dma_data_direction dir_src;
+ sa_sync_from_device(rxd);
req = container_of(rxd->req, struct aead_request, base);
tfm = crypto_aead_reqtfm(req);
start = req->assoclen + req->cryptlen;
authsize = crypto_aead_authsize(tfm);
- diff_dst = (req->src != req->dst) ? true : false;
- dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL;
-
mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml);
for (i = 0; i < (authsize / 4); i++)
mdptr[i + 4] = swab32(mdptr[i + 4]);
auth_len = req->assoclen + req->cryptlen;
- if (!rxd->enc)
- auth_len -= authsize;
-
- sglen = sg_nents_for_len(rxd->src, auth_len);
- dma_unmap_sg(rxd->ddev, rxd->src, sglen, dir_src);
- kfree(rxd->split_src_sg);
-
- if (diff_dst) {
- sglen = sg_nents_for_len(rxd->dst, auth_len);
- dma_unmap_sg(rxd->ddev, rxd->dst, sglen, DMA_FROM_DEVICE);
- kfree(rxd->split_dst_sg);
- }
if (rxd->enc) {
scatterwalk_map_and_copy(&mdptr[4], req->dst, start, authsize,
1);
} else {
+ auth_len -= authsize;
start -= authsize;
scatterwalk_map_and_copy(auth_tag, req->src, start, authsize,
0);
@@ -1681,7 +1698,7 @@ static void sa_aead_dma_in_callback(void *data)
err = memcmp(&mdptr[4], auth_tag, authsize) ? -EBADMSG : 0;
}
- kfree(rxd);
+ sa_free_sa_rx_data(rxd);
aead_request_complete(req, err);
}
@@ -2243,25 +2260,21 @@ static int sa_dma_init(struct sa_crypto_data *dd)
return ret;
dd->dma_rx1 = dma_request_chan(dd->dev, "rx1");
- if (IS_ERR(dd->dma_rx1)) {
- if (PTR_ERR(dd->dma_rx1) != -EPROBE_DEFER)
- dev_err(dd->dev, "Unable to request rx1 DMA channel\n");
- return PTR_ERR(dd->dma_rx1);
- }
+ if (IS_ERR(dd->dma_rx1))
+ return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx1),
+ "Unable to request rx1 DMA channel\n");
dd->dma_rx2 = dma_request_chan(dd->dev, "rx2");
if (IS_ERR(dd->dma_rx2)) {
dma_release_channel(dd->dma_rx1);
- if (PTR_ERR(dd->dma_rx2) != -EPROBE_DEFER)
- dev_err(dd->dev, "Unable to request rx2 DMA channel\n");
- return PTR_ERR(dd->dma_rx2);
+ return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx2),
+ "Unable to request rx2 DMA channel\n");
}
dd->dma_tx = dma_request_chan(dd->dev, "tx");
if (IS_ERR(dd->dma_tx)) {
- if (PTR_ERR(dd->dma_tx) != -EPROBE_DEFER)
- dev_err(dd->dev, "Unable to request tx DMA channel\n");
- ret = PTR_ERR(dd->dma_tx);
+ ret = dev_err_probe(dd->dev, PTR_ERR(dd->dma_tx),
+ "Unable to request tx DMA channel\n");
goto err_dma_tx;
}
@@ -2333,7 +2346,7 @@ static int sa_ul_probe(struct platform_device *pdev)
pm_runtime_enable(dev);
ret = pm_runtime_get_sync(dev);
- if (ret) {
+ if (ret < 0) {
dev_err(&pdev->dev, "%s: failed to get sync: %d\n", __func__,
ret);
return ret;